Deploying to gh-pages from @ GreptimeTeam/greptimedb@97913b4b3a 🚀

This commit is contained in:
github-merge-queue[bot]
2026-10-07 08:03:01 +00:00
commit ff419bc025
25679 changed files with 2969188 additions and 0 deletions
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
+731
View File
@@ -0,0 +1,731 @@
<!DOCTYPE html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1.0"><meta name="generator" content="rustdoc"><meta name="description" content="Source of the Rust file `src/flow/src/adapter/stateless.rs`."><title>stateless.rs - source</title><script>if(window.location.protocol!=="file:")document.head.insertAdjacentHTML("beforeend","SourceSerif4-Regular-6b053e98.ttf.woff2,FiraSans-Italic-81dc35de.woff2,FiraSans-Regular-0fe48ade.woff2,FiraSans-MediumItalic-ccf7e434.woff2,FiraSans-Medium-e1aa3f0a.woff2,SourceCodePro-Regular-8badfe75.ttf.woff2,SourceCodePro-Semibold-aa29a496.ttf.woff2".split(",").map(f=>`<link rel="preload" as="font" type="font/woff2"href="../../../static.files/${f}">`).join(""))</script><link rel="stylesheet" href="../../../static.files/normalize-9960930a.css"><link rel="stylesheet" href="../../../static.files/rustdoc-17e0aaed.css"><meta name="rustdoc-vars" data-root-path="../../../" data-static-root-path="../../../static.files/" data-current-crate="flow" data-themes="" data-resource-suffix="" data-rustdoc-version="1.96.1 (31fca3adb 2026-06-26)" data-channel="1.96.1" data-search-js="search-b5634cc7.js" data-stringdex-js="stringdex-2da4960a.js" data-settings-js="settings-170eb4bf.js" ><script src="../../../static.files/storage-41dd4d93.js"></script><script defer src="../../../static.files/src-script-813739b1.js"></script><script defer src="../../../src-files.js"></script><script defer src="../../../static.files/main-5013f961.js"></script><noscript><link rel="stylesheet" href="../../../static.files/noscript-f7c3ffd8.css"></noscript><link rel="alternate icon" type="image/png" href="../../../static.files/favicon-32x32-eab170b8.png"><link rel="icon" type="image/svg+xml" href="../../../static.files/favicon-044be391.svg"></head><body class="rustdoc src"><a class="skip-main-content" href="#main-content">Skip to main content</a><!--[if lte IE 11]><div class="warning">This old browser is unsupported and will most likely display funky things.</div><![endif]--><nav class="sidebar"><div class="src-sidebar-title"><h2>Files</h2></div></nav><div class="sidebar-resizer" title="Drag to resize sidebar"></div><main><section id="main-content" class="content" tabindex="-1"><div class="main-heading"><h1><div class="sub-heading">flow/adapter/</div>stateless.rs</h1><rustdoc-toolbar></rustdoc-toolbar></div><div class="example-wrap digits-3"><pre class="rust"><code><a href=#1 id=1 data-nosnippet>1</a><span class="comment">// Copyright 2023 Greptime Team
<a href=#2 id=2 data-nosnippet>2</a>//
<a href=#3 id=3 data-nosnippet>3</a>// Licensed under the Apache License, Version 2.0 (the "License");
<a href=#4 id=4 data-nosnippet>4</a>// you may not use this file except in compliance with the License.
<a href=#5 id=5 data-nosnippet>5</a>// You may obtain a copy of the License at
<a href=#6 id=6 data-nosnippet>6</a>//
<a href=#7 id=7 data-nosnippet>7</a>// http://www.apache.org/licenses/LICENSE-2.0
<a href=#8 id=8 data-nosnippet>8</a>//
<a href=#9 id=9 data-nosnippet>9</a>// Unless required by applicable law or agreed to in writing, software
<a href=#10 id=10 data-nosnippet>10</a>// distributed under the License is distributed on an "AS IS" BASIS,
<a href=#11 id=11 data-nosnippet>11</a>// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
<a href=#12 id=12 data-nosnippet>12</a>// See the License for the specific language governing permissions and
<a href=#13 id=13 data-nosnippet>13</a>// limitations under the License.
<a href=#14 id=14 data-nosnippet>14</a>
<a href=#15 id=15 data-nosnippet>15</a></span><span class="doccomment">//! Stateless DataFusion execution for streaming flows.
<a href=#16 id=16 data-nosnippet>16</a>
<a href=#17 id=17 data-nosnippet>17</a></span><span class="kw">use </span>std::collections::HashSet;
<a href=#18 id=18 data-nosnippet>18</a><span class="kw">use </span>std::sync::Arc;
<a href=#19 id=19 data-nosnippet>19</a>
<a href=#20 id=20 data-nosnippet>20</a><span class="kw">use </span>api::helper::{to_grpc_value, vectors_to_rows};
<a href=#21 id=21 data-nosnippet>21</a><span class="kw">use </span>api::v1::greptime_request::Request;
<a href=#22 id=22 data-nosnippet>22</a><span class="kw">use </span>api::v1::{RowInsertRequest, RowInsertRequests, Rows};
<a href=#23 id=23 data-nosnippet>23</a><span class="kw">use </span>common_error::ext::BoxedError;
<a href=#24 id=24 data-nosnippet>24</a><span class="kw">use </span>common_query::OutputData;
<a href=#25 id=25 data-nosnippet>25</a><span class="kw">use </span>common_recordbatch::{RecordBatch, RecordBatches, map_dictionary_to_values_data_type};
<a href=#26 id=26 data-nosnippet>26</a><span class="kw">use </span>common_time::Timestamp;
<a href=#27 id=27 data-nosnippet>27</a><span class="kw">use </span>datafusion::catalog::MemTable;
<a href=#28 id=28 data-nosnippet>28</a><span class="kw">use </span>datafusion::datasource::{TableProvider, provider_as_source, source_as_provider};
<a href=#29 id=29 data-nosnippet>29</a><span class="kw">use </span>datafusion_common::tree_node::{Transformed, TreeNode};
<a href=#30 id=30 data-nosnippet>30</a><span class="kw">use </span>datafusion_common::{Column, DFSchema, TableReference};
<a href=#31 id=31 data-nosnippet>31</a><span class="kw">use </span>datafusion_expr::logical_plan::{Distinct, Projection};
<a href=#32 id=32 data-nosnippet>32</a><span class="kw">use </span>datafusion_expr::{Expr, LogicalPlan};
<a href=#33 id=33 data-nosnippet>33</a><span class="kw">use </span>datatypes::schema::{ColumnSchema, SchemaRef};
<a href=#34 id=34 data-nosnippet>34</a><span class="kw">use </span>datatypes::value::Value;
<a href=#35 id=35 data-nosnippet>35</a><span class="kw">use </span>query::QueryEngine;
<a href=#36 id=36 data-nosnippet>36</a><span class="kw">use </span>session::context::QueryContextRef;
<a href=#37 id=37 data-nosnippet>37</a><span class="kw">use </span>snafu::{OptionExt, ResultExt, ensure};
<a href=#38 id=38 data-nosnippet>38</a><span class="kw">use </span>table::metadata::TableId;
<a href=#39 id=39 data-nosnippet>39</a><span class="kw">use </span>table::table::adapter::DfTableProviderAdapter;
<a href=#40 id=40 data-nosnippet>40</a>
<a href=#41 id=41 data-nosnippet>41</a><span class="kw">use </span><span class="kw">crate</span>::TableName;
<a href=#42 id=42 data-nosnippet>42</a><span class="kw">use </span><span class="kw">crate</span>::adapter::util::column_schemas_to_proto;
<a href=#43 id=43 data-nosnippet>43</a><span class="kw">use </span><span class="kw">crate</span>::batching_mode::frontend_client::FrontendClient;
<a href=#44 id=44 data-nosnippet>44</a><span class="kw">use </span><span class="kw">crate</span>::error::{DatafusionSnafu, Error, ExternalSnafu, InvalidQuerySnafu, UnexpectedSnafu};
<a href=#45 id=45 data-nosnippet>45</a><span class="kw">use </span><span class="kw">crate</span>::repr::DiffRow;
<a href=#46 id=46 data-nosnippet>46</a>
<a href=#47 id=47 data-nosnippet>47</a><span class="doccomment">/// The validated, immutable part of one streaming flow.
<a href=#48 id=48 data-nosnippet>48</a></span><span class="attr">#[derive(Clone)]
<a href=#49 id=49 data-nosnippet>49</a></span><span class="kw">pub</span>(<span class="kw">crate</span>) <span class="kw">struct </span>StatelessFlow {
<a href=#50 id=50 data-nosnippet>50</a> <span class="kw">pub</span>(<span class="kw">crate</span>) source_table_id: TableId,
<a href=#51 id=51 data-nosnippet>51</a> <span class="kw">pub</span>(<span class="kw">crate</span>) source_table_name: TableName,
<a href=#52 id=52 data-nosnippet>52</a> <span class="kw">pub</span>(<span class="kw">crate</span>) source_schema: SchemaRef,
<a href=#53 id=53 data-nosnippet>53</a> <span class="kw">pub</span>(<span class="kw">crate</span>) source_schema_version: u32,
<a href=#54 id=54 data-nosnippet>54</a> <span class="kw">pub</span>(<span class="kw">crate</span>) sink_table_name: TableName,
<a href=#55 id=55 data-nosnippet>55</a> <span class="kw">pub</span>(<span class="kw">crate</span>) sink_schema: Vec&lt;ColumnSchema&gt;,
<a href=#56 id=56 data-nosnippet>56</a> <span class="kw">pub</span>(<span class="kw">crate</span>) sink_primary_keys: Vec&lt;String&gt;,
<a href=#57 id=57 data-nosnippet>57</a> <span class="doccomment">/// The exact trailing columns resolved when the flow was created.
<a href=#58 id=58 data-nosnippet>58</a> </span><span class="kw">pub</span>(<span class="kw">crate</span>) auto_columns: Vec&lt;ColumnSchema&gt;,
<a href=#59 id=59 data-nosnippet>59</a> <span class="kw">pub</span>(<span class="kw">crate</span>) plan: LogicalPlan,
<a href=#60 id=60 data-nosnippet>60</a> <span class="kw">pub</span>(<span class="kw">crate</span>) query_ctx: QueryContextRef,
<a href=#61 id=61 data-nosnippet>61</a> <span class="kw">pub</span>(<span class="kw">crate</span>) create_args: <span class="kw">crate</span>::CreateFlowArgs,
<a href=#62 id=62 data-nosnippet>62</a>}
<a href=#63 id=63 data-nosnippet>63</a>
<a href=#64 id=64 data-nosnippet>64</a><span class="doccomment">/// Per-request input provider. It owns no catalog or storage state.
<a href=#65 id=65 data-nosnippet>65</a></span><span class="attr">#[cfg(test)]
<a href=#66 id=66 data-nosnippet>66</a></span><span class="kw">fn </span>test_source_plan(table_name: TableReference, provider: Arc&lt;<span class="kw">dyn </span>TableProvider&gt;) -&gt; LogicalPlan {
<a href=#67 id=67 data-nosnippet>67</a> datafusion_expr::LogicalPlanBuilder::scan(table_name, provider_as_source(provider), <span class="prelude-val">None</span>)
<a href=#68 id=68 data-nosnippet>68</a> .unwrap()
<a href=#69 id=69 data-nosnippet>69</a> .filter(datafusion_expr::col(<span class="string">"number"</span>).gt(datafusion_expr::lit(<span class="number">1</span>)))
<a href=#70 id=70 data-nosnippet>70</a> .unwrap()
<a href=#71 id=71 data-nosnippet>71</a> .project(<span class="macro">vec!</span>[datafusion_expr::col(<span class="string">"number"</span>)])
<a href=#72 id=72 data-nosnippet>72</a> .unwrap()
<a href=#73 id=73 data-nosnippet>73</a> .build()
<a href=#74 id=74 data-nosnippet>74</a> .unwrap()
<a href=#75 id=75 data-nosnippet>75</a>}
<a href=#76 id=76 data-nosnippet>76</a>
<a href=#77 id=77 data-nosnippet>77</a><span class="kw">fn </span>input_provider(batch: <span class="kw-2">&amp;</span>RecordBatch) -&gt; <span class="prelude-ty">Result</span>&lt;Arc&lt;<span class="kw">dyn </span>TableProvider&gt;, Error&gt; {
<a href=#78 id=78 data-nosnippet>78</a> <span class="kw">let </span>arrow_batch = batch.df_record_batch().clone();
<a href=#79 id=79 data-nosnippet>79</a> <span class="kw">let </span>provider = MemTable::try_new(arrow_batch.schema(), <span class="macro">vec!</span>[<span class="macro">vec!</span>[arrow_batch]]).context(
<a href=#80 id=80 data-nosnippet>80</a> DatafusionSnafu {
<a href=#81 id=81 data-nosnippet>81</a> context: <span class="string">"Failed to create transient flow input provider"</span>,
<a href=#82 id=82 data-nosnippet>82</a> },
<a href=#83 id=83 data-nosnippet>83</a> )<span class="question-mark">?</span>;
<a href=#84 id=84 data-nosnippet>84</a> <span class="prelude-val">Ok</span>(Arc::new(provider))
<a href=#85 id=85 data-nosnippet>85</a>}
<a href=#86 id=86 data-nosnippet>86</a>
<a href=#87 id=87 data-nosnippet>87</a><span class="doccomment">/// Adds the source timestamp to every supported plan node that has to carry it through a
<a href=#88 id=88 data-nosnippet>88</a>/// filter or projection. The expression is appended only after the visible expressions, so the
<a href=#89 id=89 data-nosnippet>89</a>/// sink contract remains positional.
<a href=#90 id=90 data-nosnippet>90</a></span><span class="kw">pub</span>(<span class="kw">crate</span>) <span class="kw">fn </span>rewrite_source_timestamp(
<a href=#91 id=91 data-nosnippet>91</a> plan: LogicalPlan,
<a href=#92 id=92 data-nosnippet>92</a> source_name: <span class="kw-2">&amp;</span>TableReference,
<a href=#93 id=93 data-nosnippet>93</a> source_timestamp_name: <span class="kw-2">&amp;</span>str,
<a href=#94 id=94 data-nosnippet>94</a>) -&gt; <span class="prelude-ty">Result</span>&lt;LogicalPlan, Error&gt; {
<a href=#95 id=95 data-nosnippet>95</a> <span class="kw">let </span><span class="kw-2">mut </span>names = HashSet::new();
<a href=#96 id=96 data-nosnippet>96</a> plan.apply(|node| {
<a href=#97 id=97 data-nosnippet>97</a> names.extend(
<a href=#98 id=98 data-nosnippet>98</a> node.schema()
<a href=#99 id=99 data-nosnippet>99</a> .fields()
<a href=#100 id=100 data-nosnippet>100</a> .iter()
<a href=#101 id=101 data-nosnippet>101</a> .map(|field| field.name().clone()),
<a href=#102 id=102 data-nosnippet>102</a> );
<a href=#103 id=103 data-nosnippet>103</a> <span class="prelude-val">Ok</span>(datafusion_common::tree_node::TreeNodeRecursion::Continue)
<a href=#104 id=104 data-nosnippet>104</a> })
<a href=#105 id=105 data-nosnippet>105</a> .context(DatafusionSnafu {
<a href=#106 id=106 data-nosnippet>106</a> context: <span class="string">"Failed to inspect streaming flow plan schema"</span>,
<a href=#107 id=107 data-nosnippet>107</a> })<span class="question-mark">?</span>;
<a href=#108 id=108 data-nosnippet>108</a> <span class="kw">let </span><span class="kw-2">mut </span>hidden_name = <span class="string">"__flow_source_timestamp"</span>.to_string();
<a href=#109 id=109 data-nosnippet>109</a> <span class="kw">let </span><span class="kw-2">mut </span>suffix = <span class="number">0</span>;
<a href=#110 id=110 data-nosnippet>110</a> <span class="kw">while </span>!names.insert(hidden_name.clone()) {
<a href=#111 id=111 data-nosnippet>111</a> suffix += <span class="number">1</span>;
<a href=#112 id=112 data-nosnippet>112</a> hidden_name = <span class="macro">format!</span>(<span class="string">"__flow_source_timestamp_{suffix}"</span>);
<a href=#113 id=113 data-nosnippet>113</a> }
<a href=#114 id=114 data-nosnippet>114</a> <span class="kw">let </span>visible_count = plan.schema().fields().len();
<a href=#115 id=115 data-nosnippet>115</a> <span class="kw">let </span>source_timestamp = Column::from_name(source_timestamp_name);
<a href=#116 id=116 data-nosnippet>116</a> <span class="kw">let </span><span class="kw-2">mut </span>plan = plan
<a href=#117 id=117 data-nosnippet>117</a> .transform_up_with_subqueries(|node| <span class="kw">match </span>node {
<a href=#118 id=118 data-nosnippet>118</a> LogicalPlan::TableScan(<span class="kw-2">mut </span>scan) =&gt; {
<a href=#119 id=119 data-nosnippet>119</a> <span class="kw">if </span>scan.table_name.resolved_eq(source_name)
<a href=#120 id=120 data-nosnippet>120</a> &amp;&amp; <span class="kw">let </span><span class="prelude-val">Some</span>(projection) = <span class="kw-2">&amp;mut </span>scan.projection
<a href=#121 id=121 data-nosnippet>121</a> {
<a href=#122 id=122 data-nosnippet>122</a> <span class="kw">let </span>timestamp_index = scan
<a href=#123 id=123 data-nosnippet>123</a> .source
<a href=#124 id=124 data-nosnippet>124</a> .schema()
<a href=#125 id=125 data-nosnippet>125</a> .index_of(source_timestamp_name)
<a href=#126 id=126 data-nosnippet>126</a> .map_err(|<span class="kw">_</span>| {
<a href=#127 id=127 data-nosnippet>127</a> datafusion::error::DataFusionError::Plan(
<a href=#128 id=128 data-nosnippet>128</a> <span class="string">"Source timestamp is absent from source scan"</span>.into(),
<a href=#129 id=129 data-nosnippet>129</a> )
<a href=#130 id=130 data-nosnippet>130</a> })<span class="question-mark">?</span>;
<a href=#131 id=131 data-nosnippet>131</a> <span class="kw">if </span>!projection.contains(<span class="kw-2">&amp;</span>timestamp_index) {
<a href=#132 id=132 data-nosnippet>132</a> projection.push(timestamp_index);
<a href=#133 id=133 data-nosnippet>133</a> <span class="kw">let </span>schema = scan.source.schema();
<a href=#134 id=134 data-nosnippet>134</a> scan.projected_schema = Arc::new(DFSchema::new_with_metadata(
<a href=#135 id=135 data-nosnippet>135</a> projection
<a href=#136 id=136 data-nosnippet>136</a> .iter()
<a href=#137 id=137 data-nosnippet>137</a> .map(|index| {
<a href=#138 id=138 data-nosnippet>138</a> (
<a href=#139 id=139 data-nosnippet>139</a> <span class="prelude-val">Some</span>(scan.table_name.clone()),
<a href=#140 id=140 data-nosnippet>140</a> Arc::new(schema.field(<span class="kw-2">*</span>index).clone()),
<a href=#141 id=141 data-nosnippet>141</a> )
<a href=#142 id=142 data-nosnippet>142</a> })
<a href=#143 id=143 data-nosnippet>143</a> .collect(),
<a href=#144 id=144 data-nosnippet>144</a> schema.metadata().clone(),
<a href=#145 id=145 data-nosnippet>145</a> )<span class="question-mark">?</span>);
<a href=#146 id=146 data-nosnippet>146</a> }
<a href=#147 id=147 data-nosnippet>147</a> }
<a href=#148 id=148 data-nosnippet>148</a> <span class="prelude-val">Ok</span>(Transformed::yes(LogicalPlan::TableScan(scan)))
<a href=#149 id=149 data-nosnippet>149</a> }
<a href=#150 id=150 data-nosnippet>150</a> LogicalPlan::Projection(<span class="kw-2">mut </span>projection) =&gt; {
<a href=#151 id=151 data-nosnippet>151</a> <span class="kw">let </span>hidden_expr = <span class="kw">if </span>projection
<a href=#152 id=152 data-nosnippet>152</a> .input
<a href=#153 id=153 data-nosnippet>153</a> .schema()
<a href=#154 id=154 data-nosnippet>154</a> .fields()
<a href=#155 id=155 data-nosnippet>155</a> .iter()
<a href=#156 id=156 data-nosnippet>156</a> .any(|field| field.name() == <span class="kw-2">&amp;</span>hidden_name)
<a href=#157 id=157 data-nosnippet>157</a> {
<a href=#158 id=158 data-nosnippet>158</a> Expr::Column(Column::from_name(hidden_name.clone()))
<a href=#159 id=159 data-nosnippet>159</a> } <span class="kw">else </span>{
<a href=#160 id=160 data-nosnippet>160</a> Expr::Column(source_timestamp.clone())
<a href=#161 id=161 data-nosnippet>161</a> };
<a href=#162 id=162 data-nosnippet>162</a> projection.expr.push(hidden_expr.alias(hidden_name.clone()));
<a href=#163 id=163 data-nosnippet>163</a> <span class="kw">let </span>projection = Projection::try_new(projection.expr, projection.input)<span class="question-mark">?</span>;
<a href=#164 id=164 data-nosnippet>164</a> <span class="prelude-val">Ok</span>(Transformed::yes(LogicalPlan::Projection(projection)))
<a href=#165 id=165 data-nosnippet>165</a> }
<a href=#166 id=166 data-nosnippet>166</a> <span class="kw">_ </span>=&gt; <span class="prelude-val">Ok</span>(Transformed::no(node)),
<a href=#167 id=167 data-nosnippet>167</a> })
<a href=#168 id=168 data-nosnippet>168</a> .context(DatafusionSnafu {
<a href=#169 id=169 data-nosnippet>169</a> context: <span class="string">"Failed to add source timestamp to streaming flow plan"</span>,
<a href=#170 id=170 data-nosnippet>170</a> })<span class="question-mark">?
<a href=#171 id=171 data-nosnippet>171</a> </span>.data;
<a href=#172 id=172 data-nosnippet>172</a>
<a href=#173 id=173 data-nosnippet>173</a> <span class="comment">// A plan ending at a scan or filter has no projection at which to give the carried value its
<a href=#174 id=174 data-nosnippet>174</a> // hidden name. Add one only in that case; a projection below a filter already carries it.
<a href=#175 id=175 data-nosnippet>175</a> </span><span class="kw">if </span>!plan
<a href=#176 id=176 data-nosnippet>176</a> .schema()
<a href=#177 id=177 data-nosnippet>177</a> .fields()
<a href=#178 id=178 data-nosnippet>178</a> .iter()
<a href=#179 id=179 data-nosnippet>179</a> .any(|field| field.name() == <span class="kw-2">&amp;</span>hidden_name)
<a href=#180 id=180 data-nosnippet>180</a> {
<a href=#181 id=181 data-nosnippet>181</a> <span class="kw">let </span>expressions = plan
<a href=#182 id=182 data-nosnippet>182</a> .schema()
<a href=#183 id=183 data-nosnippet>183</a> .fields()
<a href=#184 id=184 data-nosnippet>184</a> .iter()
<a href=#185 id=185 data-nosnippet>185</a> .take(visible_count)
<a href=#186 id=186 data-nosnippet>186</a> .map(|field| Expr::Column(Column::from_name(field.name())))
<a href=#187 id=187 data-nosnippet>187</a> .chain(std::iter::once(
<a href=#188 id=188 data-nosnippet>188</a> Expr::Column(source_timestamp).alias(hidden_name),
<a href=#189 id=189 data-nosnippet>189</a> ))
<a href=#190 id=190 data-nosnippet>190</a> .collect::&lt;Vec&lt;<span class="kw">_</span>&gt;&gt;();
<a href=#191 id=191 data-nosnippet>191</a> plan = Projection::try_new(expressions, Arc::new(plan))
<a href=#192 id=192 data-nosnippet>192</a> .map(LogicalPlan::Projection)
<a href=#193 id=193 data-nosnippet>193</a> .context(DatafusionSnafu {
<a href=#194 id=194 data-nosnippet>194</a> context: <span class="string">"Failed to finalize source timestamp in streaming flow plan"</span>,
<a href=#195 id=195 data-nosnippet>195</a> })<span class="question-mark">?</span>;
<a href=#196 id=196 data-nosnippet>196</a> }
<a href=#197 id=197 data-nosnippet>197</a> <span class="prelude-val">Ok</span>(plan)
<a href=#198 id=198 data-nosnippet>198</a>}
<a href=#199 id=199 data-nosnippet>199</a>
<a href=#200 id=200 data-nosnippet>200</a><span class="kw">fn </span>replace_source(
<a href=#201 id=201 data-nosnippet>201</a> plan: LogicalPlan,
<a href=#202 id=202 data-nosnippet>202</a> source_name: <span class="kw-2">&amp;</span>TableReference,
<a href=#203 id=203 data-nosnippet>203</a> provider: Arc&lt;<span class="kw">dyn </span>TableProvider&gt;,
<a href=#204 id=204 data-nosnippet>204</a>) -&gt; <span class="prelude-ty">Result</span>&lt;LogicalPlan, Error&gt; {
<a href=#205 id=205 data-nosnippet>205</a> <span class="kw">let </span><span class="kw-2">mut </span>scan_count = <span class="number">0</span>;
<a href=#206 id=206 data-nosnippet>206</a> <span class="kw">let </span><span class="kw-2">mut </span>replaced_scan_count = <span class="number">0</span>;
<a href=#207 id=207 data-nosnippet>207</a> <span class="kw">let </span>plan = plan
<a href=#208 id=208 data-nosnippet>208</a> .transform_up_with_subqueries(|node| <span class="kw">match </span>node {
<a href=#209 id=209 data-nosnippet>209</a> LogicalPlan::TableScan(<span class="kw-2">mut </span>scan) =&gt; {
<a href=#210 id=210 data-nosnippet>210</a> scan_count += <span class="number">1</span>;
<a href=#211 id=211 data-nosnippet>211</a> <span class="kw">if </span>scan.table_name.resolved_eq(source_name) {
<a href=#212 id=212 data-nosnippet>212</a> replaced_scan_count += <span class="number">1</span>;
<a href=#213 id=213 data-nosnippet>213</a> scan.source = provider_as_source(provider.clone());
<a href=#214 id=214 data-nosnippet>214</a> <span class="kw">let </span>schema = scan.source.schema();
<a href=#215 id=215 data-nosnippet>215</a> scan.projected_schema = <span class="kw">if let </span><span class="prelude-val">Some</span>(projection) = <span class="kw-2">&amp;</span>scan.projection {
<a href=#216 id=216 data-nosnippet>216</a> Arc::new(DFSchema::new_with_metadata(
<a href=#217 id=217 data-nosnippet>217</a> projection
<a href=#218 id=218 data-nosnippet>218</a> .iter()
<a href=#219 id=219 data-nosnippet>219</a> .map(|index| {
<a href=#220 id=220 data-nosnippet>220</a> (
<a href=#221 id=221 data-nosnippet>221</a> <span class="prelude-val">Some</span>(scan.table_name.clone()),
<a href=#222 id=222 data-nosnippet>222</a> Arc::new(schema.field(<span class="kw-2">*</span>index).clone()),
<a href=#223 id=223 data-nosnippet>223</a> )
<a href=#224 id=224 data-nosnippet>224</a> })
<a href=#225 id=225 data-nosnippet>225</a> .collect(),
<a href=#226 id=226 data-nosnippet>226</a> schema.metadata().clone(),
<a href=#227 id=227 data-nosnippet>227</a> )<span class="question-mark">?</span>)
<a href=#228 id=228 data-nosnippet>228</a> } <span class="kw">else </span>{
<a href=#229 id=229 data-nosnippet>229</a> Arc::new(DFSchema::try_from_qualified_schema(
<a href=#230 id=230 data-nosnippet>230</a> scan.table_name.clone(),
<a href=#231 id=231 data-nosnippet>231</a> <span class="kw-2">&amp;</span>schema,
<a href=#232 id=232 data-nosnippet>232</a> )<span class="question-mark">?</span>)
<a href=#233 id=233 data-nosnippet>233</a> };
<a href=#234 id=234 data-nosnippet>234</a> <span class="prelude-val">Ok</span>(Transformed::yes(LogicalPlan::TableScan(scan)))
<a href=#235 id=235 data-nosnippet>235</a> } <span class="kw">else </span>{
<a href=#236 id=236 data-nosnippet>236</a> <span class="prelude-val">Ok</span>(Transformed::no(LogicalPlan::TableScan(scan)))
<a href=#237 id=237 data-nosnippet>237</a> }
<a href=#238 id=238 data-nosnippet>238</a> }
<a href=#239 id=239 data-nosnippet>239</a> LogicalPlan::Join(<span class="kw">_</span>) | LogicalPlan::Aggregate(<span class="kw">_</span>) =&gt; {
<a href=#240 id=240 data-nosnippet>240</a> <span class="prelude-val">Err</span>(datafusion::error::DataFusionError::Plan(
<a href=#241 id=241 data-nosnippet>241</a> <span class="string">"Streaming flow supports only a single projection/filter source"</span>.into(),
<a href=#242 id=242 data-nosnippet>242</a> ))
<a href=#243 id=243 data-nosnippet>243</a> }
<a href=#244 id=244 data-nosnippet>244</a> <span class="kw">_ </span>=&gt; <span class="prelude-val">Ok</span>(Transformed::no(node)),
<a href=#245 id=245 data-nosnippet>245</a> })
<a href=#246 id=246 data-nosnippet>246</a> .context(DatafusionSnafu {
<a href=#247 id=247 data-nosnippet>247</a> context: <span class="string">"Failed to substitute transient flow input provider"</span>,
<a href=#248 id=248 data-nosnippet>248</a> })<span class="question-mark">?
<a href=#249 id=249 data-nosnippet>249</a> </span>.data;
<a href=#250 id=250 data-nosnippet>250</a> <span class="macro">ensure!</span>(
<a href=#251 id=251 data-nosnippet>251</a> scan_count == <span class="number">1 </span>&amp;&amp; replaced_scan_count == <span class="number">1</span>,
<a href=#252 id=252 data-nosnippet>252</a> InvalidQuerySnafu {
<a href=#253 id=253 data-nosnippet>253</a> reason: <span class="macro">format!</span>(
<a href=#254 id=254 data-nosnippet>254</a> <span class="string">"Expected one source scan matching {:?}, found {scan_count} scans and {replaced_scan_count} matches"</span>,
<a href=#255 id=255 data-nosnippet>255</a> source_name
<a href=#256 id=256 data-nosnippet>256</a> )
<a href=#257 id=257 data-nosnippet>257</a> }
<a href=#258 id=258 data-nosnippet>258</a> );
<a href=#259 id=259 data-nosnippet>259</a> <span class="prelude-val">Ok</span>(plan)
<a href=#260 id=260 data-nosnippet>260</a>}
<a href=#261 id=261 data-nosnippet>261</a>
<a href=#262 id=262 data-nosnippet>262</a><span class="doccomment">/// Validates the deliberately small stateless streaming SQL subset.
<a href=#263 id=263 data-nosnippet>263</a></span><span class="kw">pub</span>(<span class="kw">crate</span>) <span class="kw">fn </span>validate_plan(plan: <span class="kw-2">&amp;</span>LogicalPlan) -&gt; <span class="prelude-ty">Result</span>&lt;(), Error&gt; {
<a href=#264 id=264 data-nosnippet>264</a> <span class="kw">let </span><span class="kw-2">mut </span>scans = <span class="number">0</span>;
<a href=#265 id=265 data-nosnippet>265</a> plan.apply(|node| {
<a href=#266 id=266 data-nosnippet>266</a> <span class="kw">match </span>node {
<a href=#267 id=267 data-nosnippet>267</a> LogicalPlan::TableScan(<span class="kw">_</span>) =&gt; scans += <span class="number">1</span>,
<a href=#268 id=268 data-nosnippet>268</a> LogicalPlan::Projection(<span class="kw">_</span>) | LogicalPlan::Filter(<span class="kw">_</span>) =&gt; {}
<a href=#269 id=269 data-nosnippet>269</a> LogicalPlan::Aggregate(<span class="kw">_</span>) =&gt; {
<a href=#270 id=270 data-nosnippet>270</a> <span class="kw">return </span><span class="prelude-val">Err</span>(datafusion::error::DataFusionError::Plan(
<a href=#271 id=271 data-nosnippet>271</a> <span class="string">"Aggregation is unsupported in streaming flows. Recreate the flow to select batching mode. A source table with TTL=instant must use persisted retention first. Aggregation SQL without a time window requires EVAL INTERVAL."</span>.into(),
<a href=#272 id=272 data-nosnippet>272</a> ));
<a href=#273 id=273 data-nosnippet>273</a> }
<a href=#274 id=274 data-nosnippet>274</a> <span class="comment">// DISTINCT is evaluated against this request's transient input only. DISTINCT ON
<a href=#275 id=275 data-nosnippet>275</a> // has ordering/selection semantics beyond the supported stateless subset.
<a href=#276 id=276 data-nosnippet>276</a> </span>LogicalPlan::Distinct(Distinct::All(<span class="kw">_</span>)) =&gt; {}
<a href=#277 id=277 data-nosnippet>277</a> <span class="kw">_ </span>=&gt; {
<a href=#278 id=278 data-nosnippet>278</a> <span class="kw">return </span><span class="prelude-val">Err</span>(datafusion::error::DataFusionError::Plan(
<a href=#279 id=279 data-nosnippet>279</a> <span class="string">"Streaming flow supports only projection and filter over one source scan"
<a href=#280 id=280 data-nosnippet>280</a> </span>.into(),
<a href=#281 id=281 data-nosnippet>281</a> ));
<a href=#282 id=282 data-nosnippet>282</a> }
<a href=#283 id=283 data-nosnippet>283</a> }
<a href=#284 id=284 data-nosnippet>284</a> <span class="prelude-val">Ok</span>(datafusion_common::tree_node::TreeNodeRecursion::Continue)
<a href=#285 id=285 data-nosnippet>285</a> })
<a href=#286 id=286 data-nosnippet>286</a> .context(DatafusionSnafu {
<a href=#287 id=287 data-nosnippet>287</a> context: <span class="string">"Failed to validate streaming flow plan"</span>,
<a href=#288 id=288 data-nosnippet>288</a> })<span class="question-mark">?</span>;
<a href=#289 id=289 data-nosnippet>289</a> <span class="macro">ensure!</span>(
<a href=#290 id=290 data-nosnippet>290</a> scans == <span class="number">1</span>,
<a href=#291 id=291 data-nosnippet>291</a> InvalidQuerySnafu {
<a href=#292 id=292 data-nosnippet>292</a> reason: <span class="macro">format!</span>(<span class="string">"Expected one source scan, found {scans}"</span>)
<a href=#293 id=293 data-nosnippet>293</a> }
<a href=#294 id=294 data-nosnippet>294</a> );
<a href=#295 id=295 data-nosnippet>295</a> <span class="prelude-val">Ok</span>(())
<a href=#296 id=296 data-nosnippet>296</a>}
<a href=#297 id=297 data-nosnippet>297</a>
<a href=#298 id=298 data-nosnippet>298</a><span class="doccomment">/// Ensures the retained scan was planned against the source metadata captured for this flow.
<a href=#299 id=299 data-nosnippet>299</a></span><span class="kw">pub</span>(<span class="kw">crate</span>) <span class="kw">fn </span>validate_source_scan(
<a href=#300 id=300 data-nosnippet>300</a> plan: <span class="kw-2">&amp;</span>LogicalPlan,
<a href=#301 id=301 data-nosnippet>301</a> source_table_id: TableId,
<a href=#302 id=302 data-nosnippet>302</a> source_schema: <span class="kw-2">&amp;</span>SchemaRef,
<a href=#303 id=303 data-nosnippet>303</a>) -&gt; <span class="prelude-ty">Result</span>&lt;(), Error&gt; {
<a href=#304 id=304 data-nosnippet>304</a> plan.apply(|node| {
<a href=#305 id=305 data-nosnippet>305</a> <span class="kw">if let </span>LogicalPlan::TableScan(scan) = node {
<a href=#306 id=306 data-nosnippet>306</a> <span class="kw">let </span>provider = source_as_provider(<span class="kw-2">&amp;</span>scan.source)<span class="question-mark">?</span>;
<a href=#307 id=307 data-nosnippet>307</a> <span class="kw">let </span>provider = provider
<a href=#308 id=308 data-nosnippet>308</a> .downcast_ref::&lt;DfTableProviderAdapter&gt;()
<a href=#309 id=309 data-nosnippet>309</a> .ok_or_else(|| {
<a href=#310 id=310 data-nosnippet>310</a> datafusion::error::DataFusionError::Plan(
<a href=#311 id=311 data-nosnippet>311</a> <span class="string">"Streaming flow source scan does not use a table provider"</span>.into(),
<a href=#312 id=312 data-nosnippet>312</a> )
<a href=#313 id=313 data-nosnippet>313</a> })<span class="question-mark">?</span>;
<a href=#314 id=314 data-nosnippet>314</a> <span class="kw">let </span>table_info = provider.table().table_info();
<a href=#315 id=315 data-nosnippet>315</a> <span class="kw">if </span>table_info.ident.table_id != source_table_id
<a href=#316 id=316 data-nosnippet>316</a> || table_info.meta.schema.as_ref() != source_schema.as_ref()
<a href=#317 id=317 data-nosnippet>317</a> {
<a href=#318 id=318 data-nosnippet>318</a> <span class="kw">return </span><span class="prelude-val">Err</span>(datafusion::error::DataFusionError::Plan(<span class="macro">format!</span>(
<a href=#319 id=319 data-nosnippet>319</a> <span class="string">"Streaming flow source scan does not match source table {source_table_id} schema version {}"</span>,
<a href=#320 id=320 data-nosnippet>320</a> source_schema.version()
<a href=#321 id=321 data-nosnippet>321</a> )));
<a href=#322 id=322 data-nosnippet>322</a> }
<a href=#323 id=323 data-nosnippet>323</a> }
<a href=#324 id=324 data-nosnippet>324</a> <span class="prelude-val">Ok</span>(datafusion_common::tree_node::TreeNodeRecursion::Continue)
<a href=#325 id=325 data-nosnippet>325</a> })
<a href=#326 id=326 data-nosnippet>326</a> .context(DatafusionSnafu {
<a href=#327 id=327 data-nosnippet>327</a> context: <span class="string">"Failed to validate streaming flow source scan"</span>,
<a href=#328 id=328 data-nosnippet>328</a> })<span class="question-mark">?</span>;
<a href=#329 id=329 data-nosnippet>329</a> <span class="prelude-val">Ok</span>(())
<a href=#330 id=330 data-nosnippet>330</a>}
<a href=#331 id=331 data-nosnippet>331</a>
<a href=#332 id=332 data-nosnippet>332</a><span class="doccomment">/// Rejects execution when the source metadata changed after the flow plan was retained.
<a href=#333 id=333 data-nosnippet>333</a>/// The outer adapter replans a flow when it observes a new source schema version; this guard
<a href=#334 id=334 data-nosnippet>334</a>/// rejects a request if the version changes again before execution.
<a href=#335 id=335 data-nosnippet>335</a></span><span class="kw">fn </span>validate_source_schema_version(
<a href=#336 id=336 data-nosnippet>336</a> retained_version: u32,
<a href=#337 id=337 data-nosnippet>337</a> current_version: u32,
<a href=#338 id=338 data-nosnippet>338</a>) -&gt; <span class="prelude-ty">Result</span>&lt;(), Error&gt; {
<a href=#339 id=339 data-nosnippet>339</a> <span class="macro">ensure!</span>(
<a href=#340 id=340 data-nosnippet>340</a> retained_version == current_version,
<a href=#341 id=341 data-nosnippet>341</a> InvalidQuerySnafu {
<a href=#342 id=342 data-nosnippet>342</a> reason: <span class="macro">format!</span>(
<a href=#343 id=343 data-nosnippet>343</a> <span class="string">"Source schema version changed from {retained_version} to {current_version}; recreate or recover the flow before writing"
<a href=#344 id=344 data-nosnippet>344</a> </span>)
<a href=#345 id=345 data-nosnippet>345</a> }
<a href=#346 id=346 data-nosnippet>346</a> );
<a href=#347 id=347 data-nosnippet>347</a> <span class="prelude-val">Ok</span>(())
<a href=#348 id=348 data-nosnippet>348</a>}
<a href=#349 id=349 data-nosnippet>349</a>
<a href=#350 id=350 data-nosnippet>350</a><span class="kw">fn </span>synthesize_auto_values(columns: <span class="kw-2">&amp;</span>[ColumnSchema], now: Timestamp) -&gt; <span class="prelude-ty">Result</span>&lt;Vec&lt;Value&gt;, Error&gt; {
<a href=#351 id=351 data-nosnippet>351</a> columns
<a href=#352 id=352 data-nosnippet>352</a> .iter()
<a href=#353 id=353 data-nosnippet>353</a> .map(|column| {
<a href=#354 id=354 data-nosnippet>354</a> <span class="kw">let </span>timestamp_type = column.data_type.as_timestamp().context(InvalidQuerySnafu {
<a href=#355 id=355 data-nosnippet>355</a> reason: <span class="macro">format!</span>(<span class="string">"Auto sink column {} is not a timestamp"</span>, column.name),
<a href=#356 id=356 data-nosnippet>356</a> })<span class="question-mark">?</span>;
<a href=#357 id=357 data-nosnippet>357</a> <span class="kw">let </span>value = <span class="kw">if </span>column.name == <span class="kw">crate</span>::adapter::AUTO_CREATED_UPDATE_AT_TS_COL {
<a href=#358 id=358 data-nosnippet>358</a> now.convert_to(timestamp_type.unit())
<a href=#359 id=359 data-nosnippet>359</a> .context(InvalidQuerySnafu {
<a href=#360 id=360 data-nosnippet>360</a> reason: <span class="string">"Current timestamp cannot be represented in sink timestamp unit"</span>,
<a href=#361 id=361 data-nosnippet>361</a> })<span class="question-mark">?
<a href=#362 id=362 data-nosnippet>362</a> </span>} <span class="kw">else if </span>column.name == <span class="kw">crate</span>::adapter::AUTO_CREATED_PLACEHOLDER_TS_COL {
<a href=#363 id=363 data-nosnippet>363</a> Timestamp::new(<span class="number">0</span>, timestamp_type.unit())
<a href=#364 id=364 data-nosnippet>364</a> } <span class="kw">else </span>{
<a href=#365 id=365 data-nosnippet>365</a> <span class="kw">return </span>InvalidQuerySnafu {
<a href=#366 id=366 data-nosnippet>366</a> reason: <span class="macro">format!</span>(<span class="string">"Unsupported auto sink column {}"</span>, column.name),
<a href=#367 id=367 data-nosnippet>367</a> }
<a href=#368 id=368 data-nosnippet>368</a> .fail();
<a href=#369 id=369 data-nosnippet>369</a> };
<a href=#370 id=370 data-nosnippet>370</a> <span class="prelude-val">Ok</span>(Value::Timestamp(value))
<a href=#371 id=371 data-nosnippet>371</a> })
<a href=#372 id=372 data-nosnippet>372</a> .collect()
<a href=#373 id=373 data-nosnippet>373</a>}
<a href=#374 id=374 data-nosnippet>374</a>
<a href=#375 id=375 data-nosnippet>375</a><span class="doccomment">/// Executes one mirror write using only the supplied batch and writes its output.
<a href=#376 id=376 data-nosnippet>376</a></span><span class="kw">pub</span>(<span class="kw">crate</span>) <span class="kw">async fn </span>execute(
<a href=#377 id=377 data-nosnippet>377</a> flow: <span class="kw-2">&amp;</span>StatelessFlow,
<a href=#378 id=378 data-nosnippet>378</a> rows: <span class="kw-2">&amp;</span>[DiffRow],
<a href=#379 id=379 data-nosnippet>379</a> batch_datatypes: <span class="kw-2">&amp;</span>[datatypes::data_type::ConcreteDataType],
<a href=#380 id=380 data-nosnippet>380</a> query_engine: <span class="kw-2">&amp;</span>Arc&lt;<span class="kw">dyn </span>QueryEngine&gt;,
<a href=#381 id=381 data-nosnippet>381</a> frontend_client: <span class="kw-2">&amp;</span>Arc&lt;FrontendClient&gt;,
<a href=#382 id=382 data-nosnippet>382</a> current_source_schema_version: u32,
<a href=#383 id=383 data-nosnippet>383</a>) -&gt; <span class="prelude-ty">Result</span>&lt;usize, Error&gt; {
<a href=#384 id=384 data-nosnippet>384</a> validate_source_schema_version(flow.source_schema_version, current_source_schema_version)<span class="question-mark">?</span>;
<a href=#385 id=385 data-nosnippet>385</a> <span class="kw">let </span>values = rows.iter().map(|(row, <span class="kw">_</span>, <span class="kw">_</span>)| row.clone()).collect();
<a href=#386 id=386 data-nosnippet>386</a> <span class="kw">let </span>batch = <span class="kw">crate</span>::expr::Batch::try_from_rows_with_types(values, batch_datatypes)
<a href=#387 id=387 data-nosnippet>387</a> .map_err(BoxedError::new)
<a href=#388 id=388 data-nosnippet>388</a> .context(ExternalSnafu)<span class="question-mark">?</span>;
<a href=#389 id=389 data-nosnippet>389</a> <span class="kw">let </span>batch = RecordBatch::new(flow.source_schema.clone(), batch.batch().to_vec())
<a href=#390 id=390 data-nosnippet>390</a> .map_err(BoxedError::new)
<a href=#391 id=391 data-nosnippet>391</a> .context(ExternalSnafu)<span class="question-mark">?</span>;
<a href=#392 id=392 data-nosnippet>392</a> <span class="kw">let </span>provider = input_provider(<span class="kw-2">&amp;</span>batch)<span class="question-mark">?</span>;
<a href=#393 id=393 data-nosnippet>393</a> <span class="kw">let </span>source_ref = TableReference::full(
<a href=#394 id=394 data-nosnippet>394</a> flow.source_table_name[<span class="number">0</span>].clone(),
<a href=#395 id=395 data-nosnippet>395</a> flow.source_table_name[<span class="number">1</span>].clone(),
<a href=#396 id=396 data-nosnippet>396</a> flow.source_table_name[<span class="number">2</span>].clone(),
<a href=#397 id=397 data-nosnippet>397</a> );
<a href=#398 id=398 data-nosnippet>398</a> <span class="kw">let </span>plan = replace_source(flow.plan.clone(), <span class="kw-2">&amp;</span>source_ref, provider)<span class="question-mark">?</span>;
<a href=#399 id=399 data-nosnippet>399</a> <span class="kw">let </span>output = query_engine
<a href=#400 id=400 data-nosnippet>400</a> .execute(plan, flow.query_ctx.clone())
<a href=#401 id=401 data-nosnippet>401</a> .<span class="kw">await
<a href=#402 id=402 data-nosnippet>402</a> </span>.map_err(BoxedError::new)
<a href=#403 id=403 data-nosnippet>403</a> .context(ExternalSnafu)<span class="question-mark">?</span>;
<a href=#404 id=404 data-nosnippet>404</a> <span class="kw">let </span>batches = <span class="kw">match </span>output.data {
<a href=#405 id=405 data-nosnippet>405</a> OutputData::RecordBatches(batches) =&gt; batches,
<a href=#406 id=406 data-nosnippet>406</a> OutputData::Stream(stream) =&gt; RecordBatches::try_collect(stream)
<a href=#407 id=407 data-nosnippet>407</a> .<span class="kw">await
<a href=#408 id=408 data-nosnippet>408</a> </span>.map_err(BoxedError::new)
<a href=#409 id=409 data-nosnippet>409</a> .context(ExternalSnafu)<span class="question-mark">?</span>,
<a href=#410 id=410 data-nosnippet>410</a> OutputData::AffectedRows(<span class="kw">_</span>) =&gt; {
<a href=#411 id=411 data-nosnippet>411</a> <span class="kw">return </span>UnexpectedSnafu {
<a href=#412 id=412 data-nosnippet>412</a> reason: <span class="string">"Streaming flow query returned affected rows"</span>,
<a href=#413 id=413 data-nosnippet>413</a> }
<a href=#414 id=414 data-nosnippet>414</a> .fail();
<a href=#415 id=415 data-nosnippet>415</a> }
<a href=#416 id=416 data-nosnippet>416</a> };
<a href=#417 id=417 data-nosnippet>417</a>
<a href=#418 id=418 data-nosnippet>418</a> <span class="kw">let </span>output_schema = batches
<a href=#419 id=419 data-nosnippet>419</a> .schema()
<a href=#420 id=420 data-nosnippet>420</a> .column_schemas()
<a href=#421 id=421 data-nosnippet>421</a> .iter()
<a href=#422 id=422 data-nosnippet>422</a> .cloned()
<a href=#423 id=423 data-nosnippet>423</a> .map(|<span class="kw-2">mut </span>column| {
<a href=#424 id=424 data-nosnippet>424</a> column.data_type = map_dictionary_to_values_data_type(<span class="kw-2">&amp;</span>column.data_type);
<a href=#425 id=425 data-nosnippet>425</a> column
<a href=#426 id=426 data-nosnippet>426</a> })
<a href=#427 id=427 data-nosnippet>427</a> .collect::&lt;Vec&lt;<span class="kw">_</span>&gt;&gt;();
<a href=#428 id=428 data-nosnippet>428</a> <span class="kw">crate</span>::adapter::validate_sink_layout_with_suffix(
<a href=#429 id=429 data-nosnippet>429</a> <span class="kw-2">&amp;</span>output_schema,
<a href=#430 id=430 data-nosnippet>430</a> <span class="kw-2">&amp;</span>flow.sink_schema,
<a href=#431 id=431 data-nosnippet>431</a> <span class="kw-2">&amp;</span>flow.auto_columns,
<a href=#432 id=432 data-nosnippet>432</a> )<span class="question-mark">?</span>;
<a href=#433 id=433 data-nosnippet>433</a> <span class="kw">let </span><span class="kw-2">mut </span>output_rows = Vec::new();
<a href=#434 id=434 data-nosnippet>434</a> <span class="kw">for </span>batch <span class="kw">in </span>batches {
<a href=#435 id=435 data-nosnippet>435</a> <span class="kw">let </span>vectors = datatypes::vectors::Helper::try_into_vectors(batch.columns())
<a href=#436 id=436 data-nosnippet>436</a> .map_err(BoxedError::new)
<a href=#437 id=437 data-nosnippet>437</a> .context(ExternalSnafu)<span class="question-mark">?</span>;
<a href=#438 id=438 data-nosnippet>438</a> output_rows.extend(vectors_to_rows(vectors.iter(), batch.num_rows()));
<a href=#439 id=439 data-nosnippet>439</a> }
<a href=#440 id=440 data-nosnippet>440</a> <span class="kw">if </span>output_rows.is_empty() {
<a href=#441 id=441 data-nosnippet>441</a> <span class="kw">return </span><span class="prelude-val">Ok</span>(<span class="number">0</span>);
<a href=#442 id=442 data-nosnippet>442</a> }
<a href=#443 id=443 data-nosnippet>443</a>
<a href=#444 id=444 data-nosnippet>444</a> <span class="comment">// Auto columns are deliberately synthesized here rather than in the query plan. This keeps
<a href=#445 id=445 data-nosnippet>445</a> // one current timestamp for the whole request and preserves the sink's timestamp precision.
<a href=#446 id=446 data-nosnippet>446</a> </span><span class="kw">let </span>auto_values = synthesize_auto_values(<span class="kw-2">&amp;</span>flow.auto_columns, Timestamp::current_millis())<span class="question-mark">?
<a href=#447 id=447 data-nosnippet>447</a> </span>.into_iter()
<a href=#448 id=448 data-nosnippet>448</a> .map(to_grpc_value)
<a href=#449 id=449 data-nosnippet>449</a> .collect::&lt;Vec&lt;<span class="kw">_</span>&gt;&gt;();
<a href=#450 id=450 data-nosnippet>450</a> <span class="kw">for </span>row <span class="kw">in </span><span class="kw-2">&amp;mut </span>output_rows {
<a href=#451 id=451 data-nosnippet>451</a> row.values.extend(auto_values.iter().cloned());
<a href=#452 id=452 data-nosnippet>452</a> }
<a href=#453 id=453 data-nosnippet>453</a>
<a href=#454 id=454 data-nosnippet>454</a> <span class="kw">let </span>output_row_count = output_rows.len();
<a href=#455 id=455 data-nosnippet>455</a> <span class="kw">let </span>proto_schema = column_schemas_to_proto(flow.sink_schema.clone(), <span class="kw-2">&amp;</span>flow.sink_primary_keys)<span class="question-mark">?</span>;
<a href=#456 id=456 data-nosnippet>456</a> <span class="kw">let </span>request = Request::RowInserts(RowInsertRequests {
<a href=#457 id=457 data-nosnippet>457</a> inserts: <span class="macro">vec!</span>[RowInsertRequest {
<a href=#458 id=458 data-nosnippet>458</a> table_name: flow.sink_table_name[<span class="number">2</span>].clone(),
<a href=#459 id=459 data-nosnippet>459</a> rows: <span class="prelude-val">Some</span>(Rows {
<a href=#460 id=460 data-nosnippet>460</a> schema: proto_schema,
<a href=#461 id=461 data-nosnippet>461</a> rows: output_rows,
<a href=#462 id=462 data-nosnippet>462</a> }),
<a href=#463 id=463 data-nosnippet>463</a> }],
<a href=#464 id=464 data-nosnippet>464</a> });
<a href=#465 id=465 data-nosnippet>465</a> <span class="kw">let </span><span class="kw-2">mut </span>peer = <span class="prelude-val">None</span>;
<a href=#466 id=466 data-nosnippet>466</a> frontend_client
<a href=#467 id=467 data-nosnippet>467</a> .handle_insert_once(
<a href=#468 id=468 data-nosnippet>468</a> request,
<a href=#469 id=469 data-nosnippet>469</a> <span class="kw-2">&amp;</span>flow.sink_table_name[<span class="number">0</span>],
<a href=#470 id=470 data-nosnippet>470</a> <span class="kw-2">&amp;</span>flow.sink_table_name[<span class="number">1</span>],
<a href=#471 id=471 data-nosnippet>471</a> <span class="kw-2">&amp;mut </span>peer,
<a href=#472 id=472 data-nosnippet>472</a> )
<a href=#473 id=473 data-nosnippet>473</a> .<span class="kw">await
<a href=#474 id=474 data-nosnippet>474</a> </span>.map_err(BoxedError::new)
<a href=#475 id=475 data-nosnippet>475</a> .context(ExternalSnafu)<span class="question-mark">?</span>;
<a href=#476 id=476 data-nosnippet>476</a> <span class="prelude-val">Ok</span>(output_row_count)
<a href=#477 id=477 data-nosnippet>477</a>}
<a href=#478 id=478 data-nosnippet>478</a>
<a href=#479 id=479 data-nosnippet>479</a><span class="attr">#[cfg(test)]
<a href=#480 id=480 data-nosnippet>480</a></span><span class="kw">mod </span>tests {
<a href=#481 id=481 data-nosnippet>481</a> <span class="kw">use </span>datafusion::catalog::MemTable;
<a href=#482 id=482 data-nosnippet>482</a> <span class="kw">use </span>datafusion::logical_expr::LogicalPlanBuilder;
<a href=#483 id=483 data-nosnippet>483</a> <span class="kw">use </span>datatypes::data_type::ConcreteDataType;
<a href=#484 id=484 data-nosnippet>484</a> <span class="kw">use </span>datatypes::schema::{ColumnSchema, Schema};
<a href=#485 id=485 data-nosnippet>485</a> <span class="kw">use </span>datatypes::vectors::{Int32Vector, TimestampMillisecondVector};
<a href=#486 id=486 data-nosnippet>486</a> <span class="kw">use </span>session::context::QueryContext;
<a href=#487 id=487 data-nosnippet>487</a>
<a href=#488 id=488 data-nosnippet>488</a> <span class="kw">use </span>super::<span class="kw-2">*</span>;
<a href=#489 id=489 data-nosnippet>489</a>
<a href=#490 id=490 data-nosnippet>490</a> <span class="attr">#[test]
<a href=#491 id=491 data-nosnippet>491</a> </span><span class="kw">fn </span>validation_accepts_distinct_over_one_source() {
<a href=#492 id=492 data-nosnippet>492</a> <span class="kw">let </span>schema = Arc::new(Schema::new(<span class="macro">vec!</span>[ColumnSchema::new(
<a href=#493 id=493 data-nosnippet>493</a> <span class="string">"number"</span>,
<a href=#494 id=494 data-nosnippet>494</a> ConcreteDataType::int32_datatype(),
<a href=#495 id=495 data-nosnippet>495</a> <span class="bool-val">false</span>,
<a href=#496 id=496 data-nosnippet>496</a> )]));
<a href=#497 id=497 data-nosnippet>497</a> <span class="kw">let </span>plan = LogicalPlanBuilder::scan(
<a href=#498 id=498 data-nosnippet>498</a> TableReference::bare(<span class="string">"source"</span>),
<a href=#499 id=499 data-nosnippet>499</a> provider_as_source(provider(<span class="kw-2">&amp;</span>schema, <span class="number">1</span>)),
<a href=#500 id=500 data-nosnippet>500</a> <span class="prelude-val">None</span>,
<a href=#501 id=501 data-nosnippet>501</a> )
<a href=#502 id=502 data-nosnippet>502</a> .unwrap()
<a href=#503 id=503 data-nosnippet>503</a> .project(<span class="macro">vec!</span>[datafusion_expr::col(<span class="string">"number"</span>)])
<a href=#504 id=504 data-nosnippet>504</a> .unwrap()
<a href=#505 id=505 data-nosnippet>505</a> .distinct()
<a href=#506 id=506 data-nosnippet>506</a> .unwrap()
<a href=#507 id=507 data-nosnippet>507</a> .build()
<a href=#508 id=508 data-nosnippet>508</a> .unwrap();
<a href=#509 id=509 data-nosnippet>509</a> <span class="macro">assert!</span>(validate_plan(<span class="kw-2">&amp;</span>plan).is_ok());
<a href=#510 id=510 data-nosnippet>510</a> }
<a href=#511 id=511 data-nosnippet>511</a>
<a href=#512 id=512 data-nosnippet>512</a> <span class="attr">#[test]
<a href=#513 id=513 data-nosnippet>513</a> </span><span class="kw">fn </span>validation_rejects_plan_without_source_scan() {
<a href=#514 id=514 data-nosnippet>514</a> <span class="kw">let </span>plan = LogicalPlan::EmptyRelation(datafusion_expr::logical_plan::EmptyRelation {
<a href=#515 id=515 data-nosnippet>515</a> produce_one_row: <span class="bool-val">false</span>,
<a href=#516 id=516 data-nosnippet>516</a> schema: Arc::new(DFSchema::empty()),
<a href=#517 id=517 data-nosnippet>517</a> });
<a href=#518 id=518 data-nosnippet>518</a> <span class="macro">assert!</span>(validate_plan(<span class="kw-2">&amp;</span>plan).is_err());
<a href=#519 id=519 data-nosnippet>519</a> }
<a href=#520 id=520 data-nosnippet>520</a>
<a href=#521 id=521 data-nosnippet>521</a> <span class="kw">fn </span>provider(schema: <span class="kw-2">&amp;</span>SchemaRef, value: i32) -&gt; Arc&lt;<span class="kw">dyn </span>TableProvider&gt; {
<a href=#522 id=522 data-nosnippet>522</a> <span class="kw">let </span>batch = RecordBatch::new(
<a href=#523 id=523 data-nosnippet>523</a> schema.clone(),
<a href=#524 id=524 data-nosnippet>524</a> <span class="macro">vec!</span>[Arc::new(Int32Vector::from_slice([value])) <span class="kw">as </span>datatypes::prelude::VectorRef],
<a href=#525 id=525 data-nosnippet>525</a> )
<a href=#526 id=526 data-nosnippet>526</a> .unwrap();
<a href=#527 id=527 data-nosnippet>527</a> <span class="kw">let </span>arrow = batch.df_record_batch().clone();
<a href=#528 id=528 data-nosnippet>528</a> Arc::new(MemTable::try_new(arrow.schema(), <span class="macro">vec!</span>[<span class="macro">vec!</span>[arrow]]).unwrap())
<a href=#529 id=529 data-nosnippet>529</a> }
<a href=#530 id=530 data-nosnippet>530</a>
<a href=#531 id=531 data-nosnippet>531</a> <span class="attr">#[test]
<a href=#532 id=532 data-nosnippet>532</a> </span><span class="kw">fn </span>source_schema_version_must_match_retained_plan() {
<a href=#533 id=533 data-nosnippet>533</a> <span class="macro">assert!</span>(validate_source_schema_version(<span class="number">7</span>, <span class="number">7</span>).is_ok());
<a href=#534 id=534 data-nosnippet>534</a>
<a href=#535 id=535 data-nosnippet>535</a> <span class="kw">let </span>error = validate_source_schema_version(<span class="number">7</span>, <span class="number">8</span>).unwrap_err();
<a href=#536 id=536 data-nosnippet>536</a> <span class="macro">assert!</span>(<span class="macro">matches!</span>(error, Error::InvalidQuery { reason, .. } <span class="kw">if
<a href=#537 id=537 data-nosnippet>537</a> </span>reason.contains(<span class="string">"Source schema version changed from 7 to 8"</span>)
<a href=#538 id=538 data-nosnippet>538</a> &amp;&amp; reason.contains(<span class="string">"recreate or recover"</span>)
<a href=#539 id=539 data-nosnippet>539</a> ));
<a href=#540 id=540 data-nosnippet>540</a> }
<a href=#541 id=541 data-nosnippet>541</a>
<a href=#542 id=542 data-nosnippet>542</a> <span class="attr">#[test]
<a href=#543 id=543 data-nosnippet>543</a> </span><span class="kw">fn </span>auto_values_use_the_sink_timestamp_units() {
<a href=#544 id=544 data-nosnippet>544</a> <span class="kw">let </span>update_at = ColumnSchema::new(
<a href=#545 id=545 data-nosnippet>545</a> <span class="kw">crate</span>::adapter::AUTO_CREATED_UPDATE_AT_TS_COL,
<a href=#546 id=546 data-nosnippet>546</a> datatypes::data_type::ConcreteDataType::timestamp_second_datatype(),
<a href=#547 id=547 data-nosnippet>547</a> <span class="bool-val">true</span>,
<a href=#548 id=548 data-nosnippet>548</a> );
<a href=#549 id=549 data-nosnippet>549</a> <span class="kw">let </span>placeholder = ColumnSchema::new(
<a href=#550 id=550 data-nosnippet>550</a> <span class="kw">crate</span>::adapter::AUTO_CREATED_PLACEHOLDER_TS_COL,
<a href=#551 id=551 data-nosnippet>551</a> datatypes::data_type::ConcreteDataType::timestamp_nanosecond_datatype(),
<a href=#552 id=552 data-nosnippet>552</a> <span class="bool-val">true</span>,
<a href=#553 id=553 data-nosnippet>553</a> );
<a href=#554 id=554 data-nosnippet>554</a> <span class="kw">let </span>values = synthesize_auto_values(
<a href=#555 id=555 data-nosnippet>555</a> <span class="kw-2">&amp;</span>[update_at],
<a href=#556 id=556 data-nosnippet>556</a> Timestamp::new(<span class="number">1_234</span>, common_time::timestamp::TimeUnit::Millisecond),
<a href=#557 id=557 data-nosnippet>557</a> )
<a href=#558 id=558 data-nosnippet>558</a> .unwrap();
<a href=#559 id=559 data-nosnippet>559</a> <span class="macro">assert_eq!</span>(values.len(), <span class="number">1</span>);
<a href=#560 id=560 data-nosnippet>560</a> <span class="macro">assert_eq!</span>(
<a href=#561 id=561 data-nosnippet>561</a> values[<span class="number">0</span>].as_timestamp().unwrap().unit(),
<a href=#562 id=562 data-nosnippet>562</a> common_time::timestamp::TimeUnit::Second
<a href=#563 id=563 data-nosnippet>563</a> );
<a href=#564 id=564 data-nosnippet>564</a> <span class="macro">assert!</span>(values[<span class="number">0</span>].as_timestamp().unwrap().value() &gt; <span class="number">0</span>);
<a href=#565 id=565 data-nosnippet>565</a>
<a href=#566 id=566 data-nosnippet>566</a> <span class="kw">let </span>values = synthesize_auto_values(
<a href=#567 id=567 data-nosnippet>567</a> <span class="kw-2">&amp;</span>[
<a href=#568 id=568 data-nosnippet>568</a> ColumnSchema::new(
<a href=#569 id=569 data-nosnippet>569</a> <span class="kw">crate</span>::adapter::AUTO_CREATED_UPDATE_AT_TS_COL,
<a href=#570 id=570 data-nosnippet>570</a> datatypes::data_type::ConcreteDataType::timestamp_microsecond_datatype(),
<a href=#571 id=571 data-nosnippet>571</a> <span class="bool-val">true</span>,
<a href=#572 id=572 data-nosnippet>572</a> ),
<a href=#573 id=573 data-nosnippet>573</a> placeholder,
<a href=#574 id=574 data-nosnippet>574</a> ],
<a href=#575 id=575 data-nosnippet>575</a> Timestamp::new(<span class="number">1_234</span>, common_time::timestamp::TimeUnit::Millisecond),
<a href=#576 id=576 data-nosnippet>576</a> )
<a href=#577 id=577 data-nosnippet>577</a> .unwrap();
<a href=#578 id=578 data-nosnippet>578</a> <span class="macro">assert_eq!</span>(values[<span class="number">1</span>].as_timestamp().unwrap().value(), <span class="number">0</span>);
<a href=#579 id=579 data-nosnippet>579</a> <span class="macro">assert_eq!</span>(
<a href=#580 id=580 data-nosnippet>580</a> values[<span class="number">1</span>].as_timestamp().unwrap().unit(),
<a href=#581 id=581 data-nosnippet>581</a> common_time::timestamp::TimeUnit::Nanosecond
<a href=#582 id=582 data-nosnippet>582</a> );
<a href=#583 id=583 data-nosnippet>583</a> }
<a href=#584 id=584 data-nosnippet>584</a>
<a href=#585 id=585 data-nosnippet>585</a> <span class="attr">#[test]
<a href=#586 id=586 data-nosnippet>586</a> </span><span class="kw">fn </span>auto_values_reject_arbitrary_columns() {
<a href=#587 id=587 data-nosnippet>587</a> <span class="kw">let </span>column = ColumnSchema::new(
<a href=#588 id=588 data-nosnippet>588</a> <span class="string">"other"</span>,
<a href=#589 id=589 data-nosnippet>589</a> datatypes::data_type::ConcreteDataType::timestamp_millisecond_datatype(),
<a href=#590 id=590 data-nosnippet>590</a> <span class="bool-val">true</span>,
<a href=#591 id=591 data-nosnippet>591</a> );
<a href=#592 id=592 data-nosnippet>592</a> <span class="macro">assert!</span>(synthesize_auto_values(<span class="kw-2">&amp;</span>[column], Timestamp::current_millis()).is_err());
<a href=#593 id=593 data-nosnippet>593</a> }
<a href=#594 id=594 data-nosnippet>594</a>
<a href=#595 id=595 data-nosnippet>595</a> <span class="attr">#[test]
<a href=#596 id=596 data-nosnippet>596</a> </span><span class="kw">fn </span>rewrite_source_timestamp_appends_collision_free_hidden_output() {
<a href=#597 id=597 data-nosnippet>597</a> <span class="kw">let </span>schema = Arc::new(Schema::new(<span class="macro">vec!</span>[
<a href=#598 id=598 data-nosnippet>598</a> ColumnSchema::new(<span class="string">"number"</span>, ConcreteDataType::int32_datatype(), <span class="bool-val">false</span>),
<a href=#599 id=599 data-nosnippet>599</a> ColumnSchema::new(
<a href=#600 id=600 data-nosnippet>600</a> <span class="string">"ts"</span>,
<a href=#601 id=601 data-nosnippet>601</a> ConcreteDataType::timestamp_millisecond_datatype(),
<a href=#602 id=602 data-nosnippet>602</a> <span class="bool-val">false</span>,
<a href=#603 id=603 data-nosnippet>603</a> )
<a href=#604 id=604 data-nosnippet>604</a> .with_time_index(<span class="bool-val">true</span>),
<a href=#605 id=605 data-nosnippet>605</a> ColumnSchema::new(
<a href=#606 id=606 data-nosnippet>606</a> <span class="string">"__flow_source_timestamp"</span>,
<a href=#607 id=607 data-nosnippet>607</a> ConcreteDataType::int32_datatype(),
<a href=#608 id=608 data-nosnippet>608</a> <span class="bool-val">false</span>,
<a href=#609 id=609 data-nosnippet>609</a> ),
<a href=#610 id=610 data-nosnippet>610</a> ]));
<a href=#611 id=611 data-nosnippet>611</a> <span class="kw">let </span>batch = RecordBatch::new(
<a href=#612 id=612 data-nosnippet>612</a> schema.clone(),
<a href=#613 id=613 data-nosnippet>613</a> <span class="macro">vec!</span>[
<a href=#614 id=614 data-nosnippet>614</a> Arc::new(Int32Vector::from_slice([<span class="number">2</span>])) <span class="kw">as </span>datatypes::prelude::VectorRef,
<a href=#615 id=615 data-nosnippet>615</a> Arc::new(TimestampMillisecondVector::from_slice([<span class="number">42</span>]))
<a href=#616 id=616 data-nosnippet>616</a> <span class="kw">as </span>datatypes::prelude::VectorRef,
<a href=#617 id=617 data-nosnippet>617</a> Arc::new(Int32Vector::from_slice([<span class="number">7</span>])) <span class="kw">as </span>datatypes::prelude::VectorRef,
<a href=#618 id=618 data-nosnippet>618</a> ],
<a href=#619 id=619 data-nosnippet>619</a> )
<a href=#620 id=620 data-nosnippet>620</a> .unwrap();
<a href=#621 id=621 data-nosnippet>621</a> <span class="kw">let </span>plan = datafusion_expr::LogicalPlanBuilder::scan(
<a href=#622 id=622 data-nosnippet>622</a> TableReference::bare(<span class="string">"source"</span>),
<a href=#623 id=623 data-nosnippet>623</a> provider_as_source(input_provider(<span class="kw-2">&amp;</span>batch).unwrap()),
<a href=#624 id=624 data-nosnippet>624</a> <span class="prelude-val">None</span>,
<a href=#625 id=625 data-nosnippet>625</a> )
<a href=#626 id=626 data-nosnippet>626</a> .unwrap()
<a href=#627 id=627 data-nosnippet>627</a> .filter(datafusion_expr::col(<span class="string">"number"</span>).gt(datafusion_expr::lit(<span class="number">1</span>)))
<a href=#628 id=628 data-nosnippet>628</a> .unwrap()
<a href=#629 id=629 data-nosnippet>629</a> .project(<span class="macro">vec!</span>[datafusion_expr::col(<span class="string">"number"</span>)])
<a href=#630 id=630 data-nosnippet>630</a> .unwrap()
<a href=#631 id=631 data-nosnippet>631</a> .build()
<a href=#632 id=632 data-nosnippet>632</a> .unwrap();
<a href=#633 id=633 data-nosnippet>633</a> <span class="kw">let </span>rewritten =
<a href=#634 id=634 data-nosnippet>634</a> rewrite_source_timestamp(plan, <span class="kw-2">&amp;</span>TableReference::bare(<span class="string">"source"</span>), <span class="string">"ts"</span>).unwrap();
<a href=#635 id=635 data-nosnippet>635</a> <span class="macro">assert_eq!</span>(rewritten.schema().fields().len(), <span class="number">2</span>);
<a href=#636 id=636 data-nosnippet>636</a> <span class="macro">assert!</span>(
<a href=#637 id=637 data-nosnippet>637</a> rewritten
<a href=#638 id=638 data-nosnippet>638</a> .schema()
<a href=#639 id=639 data-nosnippet>639</a> .field(<span class="number">1</span>)
<a href=#640 id=640 data-nosnippet>640</a> .name()
<a href=#641 id=641 data-nosnippet>641</a> .starts_with(<span class="string">"__flow_source_timestamp"</span>)
<a href=#642 id=642 data-nosnippet>642</a> );
<a href=#643 id=643 data-nosnippet>643</a> }
<a href=#644 id=644 data-nosnippet>644</a>
<a href=#645 id=645 data-nosnippet>645</a> <span class="attr">#[tokio::test]
<a href=#646 id=646 data-nosnippet>646</a> </span><span class="kw">async fn </span>finite_projection_filter_does_not_retain_previous_batch() {
<a href=#647 id=647 data-nosnippet>647</a> <span class="kw">let </span>schema = Arc::new(Schema::new(<span class="macro">vec!</span>[ColumnSchema::new(
<a href=#648 id=648 data-nosnippet>648</a> <span class="string">"number"</span>,
<a href=#649 id=649 data-nosnippet>649</a> ConcreteDataType::int32_datatype(),
<a href=#650 id=650 data-nosnippet>650</a> <span class="bool-val">false</span>,
<a href=#651 id=651 data-nosnippet>651</a> )]));
<a href=#652 id=652 data-nosnippet>652</a> <span class="kw">let </span>initial_provider = provider(<span class="kw-2">&amp;</span>schema, <span class="number">0</span>);
<a href=#653 id=653 data-nosnippet>653</a> <span class="kw">let </span>plan = test_source_plan(TableReference::bare(<span class="string">"source"</span>), initial_provider);
<a href=#654 id=654 data-nosnippet>654</a> <span class="macro">assert!</span>(validate_plan(<span class="kw-2">&amp;</span>plan).is_ok());
<a href=#655 id=655 data-nosnippet>655</a>
<a href=#656 id=656 data-nosnippet>656</a> <span class="kw">let </span>engine = <span class="kw">crate</span>::test_utils::create_test_query_engine();
<a href=#657 id=657 data-nosnippet>657</a> <span class="kw">let </span>run = |input: Arc&lt;<span class="kw">dyn </span>TableProvider&gt;| {
<a href=#658 id=658 data-nosnippet>658</a> <span class="kw">let </span>engine = engine.clone();
<a href=#659 id=659 data-nosnippet>659</a> <span class="kw">let </span>plan = plan.clone();
<a href=#660 id=660 data-nosnippet>660</a> <span class="kw">async move </span>{
<a href=#661 id=661 data-nosnippet>661</a> <span class="kw">let </span>plan = replace_source(plan, <span class="kw-2">&amp;</span>TableReference::bare(<span class="string">"source"</span>), input).unwrap();
<a href=#662 id=662 data-nosnippet>662</a> <span class="kw">let </span>output = engine.execute(plan, QueryContext::arc()).<span class="kw">await</span>.unwrap();
<a href=#663 id=663 data-nosnippet>663</a> <span class="kw">match </span>output.data {
<a href=#664 id=664 data-nosnippet>664</a> OutputData::Stream(stream) =&gt; RecordBatches::try_collect(stream)
<a href=#665 id=665 data-nosnippet>665</a> .<span class="kw">await
<a href=#666 id=666 data-nosnippet>666</a> </span>.unwrap()
<a href=#667 id=667 data-nosnippet>667</a> .iter()
<a href=#668 id=668 data-nosnippet>668</a> .map(|batch| batch.num_rows())
<a href=#669 id=669 data-nosnippet>669</a> .sum(),
<a href=#670 id=670 data-nosnippet>670</a> OutputData::RecordBatches(batches) =&gt; {
<a href=#671 id=671 data-nosnippet>671</a> batches.iter().map(|batch| batch.num_rows()).sum()
<a href=#672 id=672 data-nosnippet>672</a> }
<a href=#673 id=673 data-nosnippet>673</a> OutputData::AffectedRows(<span class="kw">_</span>) =&gt; <span class="number">0</span>,
<a href=#674 id=674 data-nosnippet>674</a> }
<a href=#675 id=675 data-nosnippet>675</a> }
<a href=#676 id=676 data-nosnippet>676</a> };
<a href=#677 id=677 data-nosnippet>677</a>
<a href=#678 id=678 data-nosnippet>678</a> <span class="macro">assert_eq!</span>(run(provider(<span class="kw-2">&amp;</span>schema, <span class="number">1</span>)).<span class="kw">await</span>, <span class="number">0</span>);
<a href=#679 id=679 data-nosnippet>679</a> <span class="macro">assert_eq!</span>(run(provider(<span class="kw-2">&amp;</span>schema, <span class="number">2</span>)).<span class="kw">await</span>, <span class="number">1</span>);
<a href=#680 id=680 data-nosnippet>680</a> }
<a href=#681 id=681 data-nosnippet>681</a>
<a href=#682 id=682 data-nosnippet>682</a> <span class="attr">#[tokio::test]
<a href=#683 id=683 data-nosnippet>683</a> </span><span class="kw">async fn </span>finite_distinct_collapses_duplicates_per_request() {
<a href=#684 id=684 data-nosnippet>684</a> <span class="kw">let </span>schema = Arc::new(Schema::new(<span class="macro">vec!</span>[ColumnSchema::new(
<a href=#685 id=685 data-nosnippet>685</a> <span class="string">"number"</span>,
<a href=#686 id=686 data-nosnippet>686</a> ConcreteDataType::int32_datatype(),
<a href=#687 id=687 data-nosnippet>687</a> <span class="bool-val">false</span>,
<a href=#688 id=688 data-nosnippet>688</a> )]));
<a href=#689 id=689 data-nosnippet>689</a> <span class="kw">let </span>input = |values: <span class="kw-2">&amp;</span>[i32]| {
<a href=#690 id=690 data-nosnippet>690</a> <span class="kw">let </span>batch = RecordBatch::new(
<a href=#691 id=691 data-nosnippet>691</a> schema.clone(),
<a href=#692 id=692 data-nosnippet>692</a> <span class="macro">vec!</span>[Arc::new(Int32Vector::from_slice(values)) <span class="kw">as </span>datatypes::prelude::VectorRef],
<a href=#693 id=693 data-nosnippet>693</a> )
<a href=#694 id=694 data-nosnippet>694</a> .unwrap();
<a href=#695 id=695 data-nosnippet>695</a> input_provider(<span class="kw-2">&amp;</span>batch).unwrap()
<a href=#696 id=696 data-nosnippet>696</a> };
<a href=#697 id=697 data-nosnippet>697</a> <span class="kw">let </span>plan = LogicalPlanBuilder::scan(
<a href=#698 id=698 data-nosnippet>698</a> TableReference::bare(<span class="string">"source"</span>),
<a href=#699 id=699 data-nosnippet>699</a> provider_as_source(input(<span class="kw-2">&amp;</span>[<span class="number">1</span>])),
<a href=#700 id=700 data-nosnippet>700</a> <span class="prelude-val">None</span>,
<a href=#701 id=701 data-nosnippet>701</a> )
<a href=#702 id=702 data-nosnippet>702</a> .unwrap()
<a href=#703 id=703 data-nosnippet>703</a> .project(<span class="macro">vec!</span>[datafusion_expr::col(<span class="string">"number"</span>)])
<a href=#704 id=704 data-nosnippet>704</a> .unwrap()
<a href=#705 id=705 data-nosnippet>705</a> .distinct()
<a href=#706 id=706 data-nosnippet>706</a> .unwrap()
<a href=#707 id=707 data-nosnippet>707</a> .build()
<a href=#708 id=708 data-nosnippet>708</a> .unwrap();
<a href=#709 id=709 data-nosnippet>709</a> <span class="macro">assert!</span>(validate_plan(<span class="kw-2">&amp;</span>plan).is_ok());
<a href=#710 id=710 data-nosnippet>710</a>
<a href=#711 id=711 data-nosnippet>711</a> <span class="kw">let </span>engine = <span class="kw">crate</span>::test_utils::create_test_query_engine();
<a href=#712 id=712 data-nosnippet>712</a> <span class="kw">let </span>run = |provider: Arc&lt;<span class="kw">dyn </span>TableProvider&gt;| {
<a href=#713 id=713 data-nosnippet>713</a> <span class="kw">let </span>engine = engine.clone();
<a href=#714 id=714 data-nosnippet>714</a> <span class="kw">let </span>plan = plan.clone();
<a href=#715 id=715 data-nosnippet>715</a> <span class="kw">async move </span>{
<a href=#716 id=716 data-nosnippet>716</a> <span class="kw">let </span>plan = replace_source(plan, <span class="kw-2">&amp;</span>TableReference::bare(<span class="string">"source"</span>), provider).unwrap();
<a href=#717 id=717 data-nosnippet>717</a> <span class="kw">let </span>output = engine.execute(plan, QueryContext::arc()).<span class="kw">await</span>.unwrap();
<a href=#718 id=718 data-nosnippet>718</a> <span class="kw">let </span>batches = <span class="kw">match </span>output.data {
<a href=#719 id=719 data-nosnippet>719</a> OutputData::Stream(stream) =&gt; RecordBatches::try_collect(stream).<span class="kw">await</span>.unwrap(),
<a href=#720 id=720 data-nosnippet>720</a> OutputData::RecordBatches(batches) =&gt; batches,
<a href=#721 id=721 data-nosnippet>721</a> OutputData::AffectedRows(<span class="kw">_</span>) =&gt; <span class="macro">panic!</span>(<span class="string">"unexpected affected rows"</span>),
<a href=#722 id=722 data-nosnippet>722</a> };
<a href=#723 id=723 data-nosnippet>723</a> batches.iter().map(|batch| batch.num_rows()).sum::&lt;usize&gt;()
<a href=#724 id=724 data-nosnippet>724</a> }
<a href=#725 id=725 data-nosnippet>725</a> };
<a href=#726 id=726 data-nosnippet>726</a>
<a href=#727 id=727 data-nosnippet>727</a> <span class="macro">assert_eq!</span>(run(input(<span class="kw-2">&amp;</span>[<span class="number">1</span>, <span class="number">1</span>, <span class="number">2</span>])).<span class="kw">await</span>, <span class="number">2</span>);
<a href=#728 id=728 data-nosnippet>728</a> <span class="macro">assert_eq!</span>(run(input(<span class="kw-2">&amp;</span>[<span class="number">1</span>, <span class="number">1</span>])).<span class="kw">await</span>, <span class="number">1</span>);
<a href=#729 id=729 data-nosnippet>729</a> }
<a href=#730 id=730 data-nosnippet>730</a>}
</code></pre></div></section></main></body></html>
+351
View File
@@ -0,0 +1,351 @@
<!DOCTYPE html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1.0"><meta name="generator" content="rustdoc"><meta name="description" content="Source of the Rust file `src/flow/src/adapter/table_source.rs`."><title>table_source.rs - source</title><script>if(window.location.protocol!=="file:")document.head.insertAdjacentHTML("beforeend","SourceSerif4-Regular-6b053e98.ttf.woff2,FiraSans-Italic-81dc35de.woff2,FiraSans-Regular-0fe48ade.woff2,FiraSans-MediumItalic-ccf7e434.woff2,FiraSans-Medium-e1aa3f0a.woff2,SourceCodePro-Regular-8badfe75.ttf.woff2,SourceCodePro-Semibold-aa29a496.ttf.woff2".split(",").map(f=>`<link rel="preload" as="font" type="font/woff2"href="../../../static.files/${f}">`).join(""))</script><link rel="stylesheet" href="../../../static.files/normalize-9960930a.css"><link rel="stylesheet" href="../../../static.files/rustdoc-17e0aaed.css"><meta name="rustdoc-vars" data-root-path="../../../" data-static-root-path="../../../static.files/" data-current-crate="flow" data-themes="" data-resource-suffix="" data-rustdoc-version="1.96.1 (31fca3adb 2026-06-26)" data-channel="1.96.1" data-search-js="search-b5634cc7.js" data-stringdex-js="stringdex-2da4960a.js" data-settings-js="settings-170eb4bf.js" ><script src="../../../static.files/storage-41dd4d93.js"></script><script defer src="../../../static.files/src-script-813739b1.js"></script><script defer src="../../../src-files.js"></script><script defer src="../../../static.files/main-5013f961.js"></script><noscript><link rel="stylesheet" href="../../../static.files/noscript-f7c3ffd8.css"></noscript><link rel="alternate icon" type="image/png" href="../../../static.files/favicon-32x32-eab170b8.png"><link rel="icon" type="image/svg+xml" href="../../../static.files/favicon-044be391.svg"></head><body class="rustdoc src"><a class="skip-main-content" href="#main-content">Skip to main content</a><!--[if lte IE 11]><div class="warning">This old browser is unsupported and will most likely display funky things.</div><![endif]--><nav class="sidebar"><div class="src-sidebar-title"><h2>Files</h2></div></nav><div class="sidebar-resizer" title="Drag to resize sidebar"></div><main><section id="main-content" class="content" tabindex="-1"><div class="main-heading"><h1><div class="sub-heading">flow/adapter/</div>table_source.rs</h1><rustdoc-toolbar></rustdoc-toolbar></div><div class="example-wrap digits-3"><pre class="rust"><code><a href=#1 id=1 data-nosnippet>1</a><span class="comment">// Copyright 2023 Greptime Team
<a href=#2 id=2 data-nosnippet>2</a>//
<a href=#3 id=3 data-nosnippet>3</a>// Licensed under the Apache License, Version 2.0 (the "License");
<a href=#4 id=4 data-nosnippet>4</a>// you may not use this file except in compliance with the License.
<a href=#5 id=5 data-nosnippet>5</a>// You may obtain a copy of the License at
<a href=#6 id=6 data-nosnippet>6</a>//
<a href=#7 id=7 data-nosnippet>7</a>// http://www.apache.org/licenses/LICENSE-2.0
<a href=#8 id=8 data-nosnippet>8</a>//
<a href=#9 id=9 data-nosnippet>9</a>// Unless required by applicable law or agreed to in writing, software
<a href=#10 id=10 data-nosnippet>10</a>// distributed under the License is distributed on an "AS IS" BASIS,
<a href=#11 id=11 data-nosnippet>11</a>// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
<a href=#12 id=12 data-nosnippet>12</a>// See the License for the specific language governing permissions and
<a href=#13 id=13 data-nosnippet>13</a>// limitations under the License.
<a href=#14 id=14 data-nosnippet>14</a>
<a href=#15 id=15 data-nosnippet>15</a></span><span class="doccomment">//! How to query table information from database
<a href=#16 id=16 data-nosnippet>16</a>
<a href=#17 id=17 data-nosnippet>17</a></span><span class="kw">use </span>common_error::ext::BoxedError;
<a href=#18 id=18 data-nosnippet>18</a><span class="kw">use </span>common_meta::key::table_info::{TableInfoManager, TableInfoValue};
<a href=#19 id=19 data-nosnippet>19</a><span class="kw">use </span>common_meta::key::table_name::{TableNameKey, TableNameManager};
<a href=#20 id=20 data-nosnippet>20</a><span class="kw">use </span>datatypes::schema::ColumnDefaultConstraint;
<a href=#21 id=21 data-nosnippet>21</a><span class="kw">use </span>serde::{Deserialize, Serialize};
<a href=#22 id=22 data-nosnippet>22</a><span class="kw">use </span>snafu::{OptionExt, ResultExt};
<a href=#23 id=23 data-nosnippet>23</a><span class="kw">use </span>table::metadata::TableId;
<a href=#24 id=24 data-nosnippet>24</a>
<a href=#25 id=25 data-nosnippet>25</a><span class="kw">use </span><span class="kw">crate</span>::adapter::TableName;
<a href=#26 id=26 data-nosnippet>26</a><span class="kw">use </span><span class="kw">crate</span>::adapter::util::table_info_value_to_relation_desc;
<a href=#27 id=27 data-nosnippet>27</a><span class="kw">use </span><span class="kw">crate</span>::error::{
<a href=#28 id=28 data-nosnippet>28</a> Error, ExternalSnafu, TableNotFoundMetaSnafu, TableNotFoundSnafu, UnexpectedSnafu,
<a href=#29 id=29 data-nosnippet>29</a>};
<a href=#30 id=30 data-nosnippet>30</a><span class="kw">use </span><span class="kw">crate</span>::repr::RelationDesc;
<a href=#31 id=31 data-nosnippet>31</a>
<a href=#32 id=32 data-nosnippet>32</a><span class="doccomment">/// Table description, include relation desc and default values, which is the minimal information flow needed for table
<a href=#33 id=33 data-nosnippet>33</a></span><span class="attr">#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
<a href=#34 id=34 data-nosnippet>34</a></span><span class="kw">pub struct </span>TableDesc {
<a href=#35 id=35 data-nosnippet>35</a> <span class="kw">pub </span>relation_desc: RelationDesc,
<a href=#36 id=36 data-nosnippet>36</a> <span class="kw">pub </span>default_values: Vec&lt;<span class="prelude-ty">Option</span>&lt;ColumnDefaultConstraint&gt;&gt;,
<a href=#37 id=37 data-nosnippet>37</a>}
<a href=#38 id=38 data-nosnippet>38</a>
<a href=#39 id=39 data-nosnippet>39</a><span class="kw">impl </span>TableDesc {
<a href=#40 id=40 data-nosnippet>40</a> <span class="kw">pub fn </span>new(
<a href=#41 id=41 data-nosnippet>41</a> relation_desc: RelationDesc,
<a href=#42 id=42 data-nosnippet>42</a> default_values: Vec&lt;<span class="prelude-ty">Option</span>&lt;ColumnDefaultConstraint&gt;&gt;,
<a href=#43 id=43 data-nosnippet>43</a> ) -&gt; <span class="self">Self </span>{
<a href=#44 id=44 data-nosnippet>44</a> <span class="self">Self </span>{
<a href=#45 id=45 data-nosnippet>45</a> relation_desc,
<a href=#46 id=46 data-nosnippet>46</a> default_values,
<a href=#47 id=47 data-nosnippet>47</a> }
<a href=#48 id=48 data-nosnippet>48</a> }
<a href=#49 id=49 data-nosnippet>49</a>
<a href=#50 id=50 data-nosnippet>50</a> <span class="kw">pub fn </span>new_no_default(relation_desc: RelationDesc) -&gt; <span class="self">Self </span>{
<a href=#51 id=51 data-nosnippet>51</a> <span class="self">Self </span>{
<a href=#52 id=52 data-nosnippet>52</a> relation_desc,
<a href=#53 id=53 data-nosnippet>53</a> default_values: <span class="macro">vec!</span>[],
<a href=#54 id=54 data-nosnippet>54</a> }
<a href=#55 id=55 data-nosnippet>55</a> }
<a href=#56 id=56 data-nosnippet>56</a>}
<a href=#57 id=57 data-nosnippet>57</a>
<a href=#58 id=58 data-nosnippet>58</a><span class="doccomment">/// Table source but for flow, provide table schema by table name/id
<a href=#59 id=59 data-nosnippet>59</a></span><span class="attr">#[async_trait::async_trait]
<a href=#60 id=60 data-nosnippet>60</a></span><span class="kw">pub trait </span>FlowTableSource: Send + Sync + std::fmt::Debug {
<a href=#61 id=61 data-nosnippet>61</a> <span class="kw">async fn </span>table_name_from_id(<span class="kw-2">&amp;</span><span class="self">self</span>, table_id: <span class="kw-2">&amp;</span>TableId) -&gt; <span class="prelude-ty">Result</span>&lt;TableName, Error&gt;;
<a href=#62 id=62 data-nosnippet>62</a> <span class="kw">async fn </span>table_id_from_name(<span class="kw-2">&amp;</span><span class="self">self</span>, name: <span class="kw-2">&amp;</span>TableName) -&gt; <span class="prelude-ty">Result</span>&lt;TableId, Error&gt;;
<a href=#63 id=63 data-nosnippet>63</a>
<a href=#64 id=64 data-nosnippet>64</a> <span class="doccomment">/// Get the table schema by table name
<a href=#65 id=65 data-nosnippet>65</a> </span><span class="kw">async fn </span>table(<span class="kw-2">&amp;</span><span class="self">self</span>, name: <span class="kw-2">&amp;</span>TableName) -&gt; <span class="prelude-ty">Result</span>&lt;TableDesc, Error&gt; {
<a href=#66 id=66 data-nosnippet>66</a> <span class="kw">let </span>id = <span class="self">self</span>.table_id_from_name(name).<span class="kw">await</span><span class="question-mark">?</span>;
<a href=#67 id=67 data-nosnippet>67</a> <span class="self">self</span>.table_from_id(<span class="kw-2">&amp;</span>id).<span class="kw">await
<a href=#68 id=68 data-nosnippet>68</a> </span>}
<a href=#69 id=69 data-nosnippet>69</a> <span class="kw">async fn </span>table_from_id(<span class="kw-2">&amp;</span><span class="self">self</span>, table_id: <span class="kw-2">&amp;</span>TableId) -&gt; <span class="prelude-ty">Result</span>&lt;TableDesc, Error&gt;;
<a href=#70 id=70 data-nosnippet>70</a>}
<a href=#71 id=71 data-nosnippet>71</a>
<a href=#72 id=72 data-nosnippet>72</a><span class="doccomment">/// managed table source information, query from table info manager and table name manager
<a href=#73 id=73 data-nosnippet>73</a></span><span class="attr">#[derive(Clone)]
<a href=#74 id=74 data-nosnippet>74</a></span><span class="kw">pub struct </span>ManagedTableSource {
<a href=#75 id=75 data-nosnippet>75</a> <span class="doccomment">/// for query `TableId -&gt; TableName` mapping
<a href=#76 id=76 data-nosnippet>76</a> </span>table_info_manager: TableInfoManager,
<a href=#77 id=77 data-nosnippet>77</a> table_name_manager: TableNameManager,
<a href=#78 id=78 data-nosnippet>78</a>}
<a href=#79 id=79 data-nosnippet>79</a>
<a href=#80 id=80 data-nosnippet>80</a><span class="attr">#[async_trait::async_trait]
<a href=#81 id=81 data-nosnippet>81</a></span><span class="kw">impl </span>FlowTableSource <span class="kw">for </span>ManagedTableSource {
<a href=#82 id=82 data-nosnippet>82</a> <span class="kw">async fn </span>table_from_id(<span class="kw-2">&amp;</span><span class="self">self</span>, table_id: <span class="kw-2">&amp;</span>TableId) -&gt; <span class="prelude-ty">Result</span>&lt;TableDesc, Error&gt; {
<a href=#83 id=83 data-nosnippet>83</a> <span class="kw">let </span>table_info_value = <span class="self">self
<a href=#84 id=84 data-nosnippet>84</a> </span>.get_table_info_value(table_id)
<a href=#85 id=85 data-nosnippet>85</a> .<span class="kw">await</span><span class="question-mark">?
<a href=#86 id=86 data-nosnippet>86</a> </span>.with_context(|| TableNotFoundSnafu {
<a href=#87 id=87 data-nosnippet>87</a> name: <span class="macro">format!</span>(<span class="string">"TableId = {:?}, Can't found table info"</span>, table_id),
<a href=#88 id=88 data-nosnippet>88</a> })<span class="question-mark">?</span>;
<a href=#89 id=89 data-nosnippet>89</a> <span class="kw">let </span>desc = table_info_value_to_relation_desc(table_info_value)<span class="question-mark">?</span>;
<a href=#90 id=90 data-nosnippet>90</a>
<a href=#91 id=91 data-nosnippet>91</a> <span class="prelude-val">Ok</span>(desc)
<a href=#92 id=92 data-nosnippet>92</a> }
<a href=#93 id=93 data-nosnippet>93</a> <span class="kw">async fn </span>table_name_from_id(<span class="kw-2">&amp;</span><span class="self">self</span>, table_id: <span class="kw-2">&amp;</span>TableId) -&gt; <span class="prelude-ty">Result</span>&lt;TableName, Error&gt; {
<a href=#94 id=94 data-nosnippet>94</a> <span class="self">self</span>.get_table_name(table_id).<span class="kw">await
<a href=#95 id=95 data-nosnippet>95</a> </span>}
<a href=#96 id=96 data-nosnippet>96</a> <span class="kw">async fn </span>table_id_from_name(<span class="kw-2">&amp;</span><span class="self">self</span>, name: <span class="kw-2">&amp;</span>TableName) -&gt; <span class="prelude-ty">Result</span>&lt;TableId, Error&gt; {
<a href=#97 id=97 data-nosnippet>97</a> <span class="self">self</span>.get_opt_table_id_from_name(name)
<a href=#98 id=98 data-nosnippet>98</a> .<span class="kw">await</span><span class="question-mark">?
<a href=#99 id=99 data-nosnippet>99</a> </span>.with_context(|| TableNotFoundSnafu {
<a href=#100 id=100 data-nosnippet>100</a> name: name.join(<span class="string">"."</span>),
<a href=#101 id=101 data-nosnippet>101</a> })
<a href=#102 id=102 data-nosnippet>102</a> }
<a href=#103 id=103 data-nosnippet>103</a>}
<a href=#104 id=104 data-nosnippet>104</a>
<a href=#105 id=105 data-nosnippet>105</a><span class="kw">impl </span>ManagedTableSource {
<a href=#106 id=106 data-nosnippet>106</a> <span class="kw">pub fn </span>new(table_info_manager: TableInfoManager, table_name_manager: TableNameManager) -&gt; <span class="self">Self </span>{
<a href=#107 id=107 data-nosnippet>107</a> ManagedTableSource {
<a href=#108 id=108 data-nosnippet>108</a> table_info_manager,
<a href=#109 id=109 data-nosnippet>109</a> table_name_manager,
<a href=#110 id=110 data-nosnippet>110</a> }
<a href=#111 id=111 data-nosnippet>111</a> }
<a href=#112 id=112 data-nosnippet>112</a>
<a href=#113 id=113 data-nosnippet>113</a> <span class="doccomment">/// Get the time index column from table id
<a href=#114 id=114 data-nosnippet>114</a> </span><span class="kw">pub async fn </span>get_time_index_column_from_table_id(
<a href=#115 id=115 data-nosnippet>115</a> <span class="kw-2">&amp;</span><span class="self">self</span>,
<a href=#116 id=116 data-nosnippet>116</a> table_id: TableId,
<a href=#117 id=117 data-nosnippet>117</a> ) -&gt; <span class="prelude-ty">Result</span>&lt;(usize, datatypes::schema::ColumnSchema), Error&gt; {
<a href=#118 id=118 data-nosnippet>118</a> <span class="kw">let </span>info = <span class="self">self
<a href=#119 id=119 data-nosnippet>119</a> </span>.table_info_manager
<a href=#120 id=120 data-nosnippet>120</a> .get(table_id)
<a href=#121 id=121 data-nosnippet>121</a> .<span class="kw">await
<a href=#122 id=122 data-nosnippet>122</a> </span>.map_err(BoxedError::new)
<a href=#123 id=123 data-nosnippet>123</a> .context(ExternalSnafu)<span class="question-mark">?
<a href=#124 id=124 data-nosnippet>124</a> </span>.context(UnexpectedSnafu {
<a href=#125 id=125 data-nosnippet>125</a> reason: <span class="macro">format!</span>(<span class="string">"Table id = {:?}, couldn't found table info"</span>, table_id),
<a href=#126 id=126 data-nosnippet>126</a> })<span class="question-mark">?</span>;
<a href=#127 id=127 data-nosnippet>127</a> <span class="kw">let </span>schema = <span class="kw-2">&amp;</span>info.table_info.meta.schema;
<a href=#128 id=128 data-nosnippet>128</a> <span class="kw">let </span><span class="prelude-val">Some</span>(ts_index) = schema.timestamp_index() <span class="kw">else </span>{
<a href=#129 id=129 data-nosnippet>129</a> UnexpectedSnafu {
<a href=#130 id=130 data-nosnippet>130</a> reason: <span class="macro">format!</span>(<span class="string">"Table id = {:?}, couldn't found timestamp index"</span>, table_id),
<a href=#131 id=131 data-nosnippet>131</a> }
<a href=#132 id=132 data-nosnippet>132</a> .fail()<span class="question-mark">?
<a href=#133 id=133 data-nosnippet>133</a> </span>};
<a href=#134 id=134 data-nosnippet>134</a> <span class="kw">let </span>col_schema = schema.column_schemas()[ts_index].clone();
<a href=#135 id=135 data-nosnippet>135</a> <span class="prelude-val">Ok</span>((ts_index, col_schema))
<a href=#136 id=136 data-nosnippet>136</a> }
<a href=#137 id=137 data-nosnippet>137</a>
<a href=#138 id=138 data-nosnippet>138</a> <span class="kw">pub async fn </span>get_table_id_from_proto_name(
<a href=#139 id=139 data-nosnippet>139</a> <span class="kw-2">&amp;</span><span class="self">self</span>,
<a href=#140 id=140 data-nosnippet>140</a> name: <span class="kw-2">&amp;</span>greptime_proto::v1::TableName,
<a href=#141 id=141 data-nosnippet>141</a> ) -&gt; <span class="prelude-ty">Result</span>&lt;TableId, Error&gt; {
<a href=#142 id=142 data-nosnippet>142</a> <span class="self">self</span>.table_name_manager
<a href=#143 id=143 data-nosnippet>143</a> .get(TableNameKey::new(
<a href=#144 id=144 data-nosnippet>144</a> <span class="kw-2">&amp;</span>name.catalog_name,
<a href=#145 id=145 data-nosnippet>145</a> <span class="kw-2">&amp;</span>name.schema_name,
<a href=#146 id=146 data-nosnippet>146</a> <span class="kw-2">&amp;</span>name.table_name,
<a href=#147 id=147 data-nosnippet>147</a> ))
<a href=#148 id=148 data-nosnippet>148</a> .<span class="kw">await
<a href=#149 id=149 data-nosnippet>149</a> </span>.with_context(|<span class="kw">_</span>| TableNotFoundMetaSnafu {
<a href=#150 id=150 data-nosnippet>150</a> msg: <span class="macro">format!</span>(<span class="string">"Table name = {:?}, couldn't found table id"</span>, name),
<a href=#151 id=151 data-nosnippet>151</a> })<span class="question-mark">?
<a href=#152 id=152 data-nosnippet>152</a> </span>.with_context(|| UnexpectedSnafu {
<a href=#153 id=153 data-nosnippet>153</a> reason: <span class="macro">format!</span>(<span class="string">"Table name = {:?}, couldn't found table id"</span>, name),
<a href=#154 id=154 data-nosnippet>154</a> })
<a href=#155 id=155 data-nosnippet>155</a> .map(|id| id.table_id())
<a href=#156 id=156 data-nosnippet>156</a> }
<a href=#157 id=157 data-nosnippet>157</a>
<a href=#158 id=158 data-nosnippet>158</a> <span class="doccomment">/// If the table haven't been created in database, the tableId returned would be null
<a href=#159 id=159 data-nosnippet>159</a> </span><span class="kw">pub async fn </span>get_opt_table_id_from_name(
<a href=#160 id=160 data-nosnippet>160</a> <span class="kw-2">&amp;</span><span class="self">self</span>,
<a href=#161 id=161 data-nosnippet>161</a> name: <span class="kw-2">&amp;</span>TableName,
<a href=#162 id=162 data-nosnippet>162</a> ) -&gt; <span class="prelude-ty">Result</span>&lt;<span class="prelude-ty">Option</span>&lt;TableId&gt;, Error&gt; {
<a href=#163 id=163 data-nosnippet>163</a> <span class="kw">let </span>ret = <span class="self">self
<a href=#164 id=164 data-nosnippet>164</a> </span>.table_name_manager
<a href=#165 id=165 data-nosnippet>165</a> .get(TableNameKey::new(<span class="kw-2">&amp;</span>name[<span class="number">0</span>], <span class="kw-2">&amp;</span>name[<span class="number">1</span>], <span class="kw-2">&amp;</span>name[<span class="number">2</span>]))
<a href=#166 id=166 data-nosnippet>166</a> .<span class="kw">await
<a href=#167 id=167 data-nosnippet>167</a> </span>.with_context(|<span class="kw">_</span>| TableNotFoundMetaSnafu {
<a href=#168 id=168 data-nosnippet>168</a> msg: <span class="macro">format!</span>(<span class="string">"Table name = {:?}, couldn't found table id"</span>, name),
<a href=#169 id=169 data-nosnippet>169</a> })<span class="question-mark">?
<a href=#170 id=170 data-nosnippet>170</a> </span>.map(|id| id.table_id());
<a href=#171 id=171 data-nosnippet>171</a> <span class="prelude-val">Ok</span>(ret)
<a href=#172 id=172 data-nosnippet>172</a> }
<a href=#173 id=173 data-nosnippet>173</a>
<a href=#174 id=174 data-nosnippet>174</a> <span class="doccomment">/// query metasrv about the table name and table id
<a href=#175 id=175 data-nosnippet>175</a> </span><span class="kw">pub async fn </span>get_table_name(<span class="kw-2">&amp;</span><span class="self">self</span>, table_id: <span class="kw-2">&amp;</span>TableId) -&gt; <span class="prelude-ty">Result</span>&lt;TableName, Error&gt; {
<a href=#176 id=176 data-nosnippet>176</a> <span class="self">self</span>.table_info_manager
<a href=#177 id=177 data-nosnippet>177</a> .get(<span class="kw-2">*</span>table_id)
<a href=#178 id=178 data-nosnippet>178</a> .<span class="kw">await
<a href=#179 id=179 data-nosnippet>179</a> </span>.map_err(BoxedError::new)
<a href=#180 id=180 data-nosnippet>180</a> .context(ExternalSnafu)<span class="question-mark">?
<a href=#181 id=181 data-nosnippet>181</a> </span>.with_context(|| UnexpectedSnafu {
<a href=#182 id=182 data-nosnippet>182</a> reason: <span class="macro">format!</span>(<span class="string">"Table id = {:?}, couldn't found table name"</span>, table_id),
<a href=#183 id=183 data-nosnippet>183</a> })
<a href=#184 id=184 data-nosnippet>184</a> .map(|name| name.table_name())
<a href=#185 id=185 data-nosnippet>185</a> .map(|name| [name.catalog_name, name.schema_name, name.table_name])
<a href=#186 id=186 data-nosnippet>186</a> }
<a href=#187 id=187 data-nosnippet>187</a>
<a href=#188 id=188 data-nosnippet>188</a> <span class="doccomment">/// query metasrv about the `TableInfoValue` and table id
<a href=#189 id=189 data-nosnippet>189</a> </span><span class="kw">pub async fn </span>get_table_info_value(
<a href=#190 id=190 data-nosnippet>190</a> <span class="kw-2">&amp;</span><span class="self">self</span>,
<a href=#191 id=191 data-nosnippet>191</a> table_id: <span class="kw-2">&amp;</span>TableId,
<a href=#192 id=192 data-nosnippet>192</a> ) -&gt; <span class="prelude-ty">Result</span>&lt;<span class="prelude-ty">Option</span>&lt;TableInfoValue&gt;, Error&gt; {
<a href=#193 id=193 data-nosnippet>193</a> <span class="prelude-val">Ok</span>(<span class="self">self
<a href=#194 id=194 data-nosnippet>194</a> </span>.table_info_manager
<a href=#195 id=195 data-nosnippet>195</a> .get(<span class="kw-2">*</span>table_id)
<a href=#196 id=196 data-nosnippet>196</a> .<span class="kw">await
<a href=#197 id=197 data-nosnippet>197</a> </span>.with_context(|<span class="kw">_</span>| TableNotFoundMetaSnafu {
<a href=#198 id=198 data-nosnippet>198</a> msg: <span class="macro">format!</span>(<span class="string">"TableId = {:?}, couldn't found table name"</span>, table_id),
<a href=#199 id=199 data-nosnippet>199</a> })<span class="question-mark">?
<a href=#200 id=200 data-nosnippet>200</a> </span>.map(|v| v.into_inner()))
<a href=#201 id=201 data-nosnippet>201</a> }
<a href=#202 id=202 data-nosnippet>202</a>
<a href=#203 id=203 data-nosnippet>203</a> <span class="kw">pub async fn </span>get_table_name_schema(
<a href=#204 id=204 data-nosnippet>204</a> <span class="kw-2">&amp;</span><span class="self">self</span>,
<a href=#205 id=205 data-nosnippet>205</a> table_id: <span class="kw-2">&amp;</span>TableId,
<a href=#206 id=206 data-nosnippet>206</a> ) -&gt; <span class="prelude-ty">Result</span>&lt;(TableName, TableDesc), Error&gt; {
<a href=#207 id=207 data-nosnippet>207</a> <span class="kw">let </span>table_info_value = <span class="self">self
<a href=#208 id=208 data-nosnippet>208</a> </span>.get_table_info_value(table_id)
<a href=#209 id=209 data-nosnippet>209</a> .<span class="kw">await</span><span class="question-mark">?
<a href=#210 id=210 data-nosnippet>210</a> </span>.with_context(|| TableNotFoundSnafu {
<a href=#211 id=211 data-nosnippet>211</a> name: <span class="macro">format!</span>(<span class="string">"TableId = {:?}, Can't found table info"</span>, table_id),
<a href=#212 id=212 data-nosnippet>212</a> })<span class="question-mark">?</span>;
<a href=#213 id=213 data-nosnippet>213</a>
<a href=#214 id=214 data-nosnippet>214</a> <span class="kw">let </span>table_name = table_info_value.table_name();
<a href=#215 id=215 data-nosnippet>215</a> <span class="kw">let </span>table_name = [
<a href=#216 id=216 data-nosnippet>216</a> table_name.catalog_name,
<a href=#217 id=217 data-nosnippet>217</a> table_name.schema_name,
<a href=#218 id=218 data-nosnippet>218</a> table_name.table_name,
<a href=#219 id=219 data-nosnippet>219</a> ];
<a href=#220 id=220 data-nosnippet>220</a>
<a href=#221 id=221 data-nosnippet>221</a> <span class="kw">let </span>desc = table_info_value_to_relation_desc(table_info_value)<span class="question-mark">?</span>;
<a href=#222 id=222 data-nosnippet>222</a> <span class="prelude-val">Ok</span>((table_name, desc))
<a href=#223 id=223 data-nosnippet>223</a> }
<a href=#224 id=224 data-nosnippet>224</a>
<a href=#225 id=225 data-nosnippet>225</a> <span class="kw">pub async fn </span>check_table_exist(<span class="kw-2">&amp;</span><span class="self">self</span>, table_id: <span class="kw-2">&amp;</span>TableId) -&gt; <span class="prelude-ty">Result</span>&lt;bool, Error&gt; {
<a href=#226 id=226 data-nosnippet>226</a> <span class="self">self</span>.table_info_manager
<a href=#227 id=227 data-nosnippet>227</a> .exists(<span class="kw-2">*</span>table_id)
<a href=#228 id=228 data-nosnippet>228</a> .<span class="kw">await
<a href=#229 id=229 data-nosnippet>229</a> </span>.map_err(BoxedError::new)
<a href=#230 id=230 data-nosnippet>230</a> .context(ExternalSnafu)
<a href=#231 id=231 data-nosnippet>231</a> }
<a href=#232 id=232 data-nosnippet>232</a>}
<a href=#233 id=233 data-nosnippet>233</a>
<a href=#234 id=234 data-nosnippet>234</a><span class="kw">impl </span>std::fmt::Debug <span class="kw">for </span>ManagedTableSource {
<a href=#235 id=235 data-nosnippet>235</a> <span class="kw">fn </span>fmt(<span class="kw-2">&amp;</span><span class="self">self</span>, f: <span class="kw-2">&amp;mut </span>std::fmt::Formatter&lt;<span class="lifetime">'_</span>&gt;) -&gt; std::fmt::Result {
<a href=#236 id=236 data-nosnippet>236</a> f.debug_struct(<span class="string">"KvBackendTableSource"</span>).finish()
<a href=#237 id=237 data-nosnippet>237</a> }
<a href=#238 id=238 data-nosnippet>238</a>}
<a href=#239 id=239 data-nosnippet>239</a>
<a href=#240 id=240 data-nosnippet>240</a><span class="attr">#[cfg(test)]
<a href=#241 id=241 data-nosnippet>241</a></span><span class="kw">pub</span>(<span class="kw">crate</span>) <span class="kw">mod </span>test {
<a href=#242 id=242 data-nosnippet>242</a> <span class="kw">use </span>std::collections::HashMap;
<a href=#243 id=243 data-nosnippet>243</a>
<a href=#244 id=244 data-nosnippet>244</a> <span class="kw">use </span>datatypes::data_type::ConcreteDataType <span class="kw">as </span>CDT;
<a href=#245 id=245 data-nosnippet>245</a>
<a href=#246 id=246 data-nosnippet>246</a> <span class="kw">use </span>super::<span class="kw-2">*</span>;
<a href=#247 id=247 data-nosnippet>247</a> <span class="kw">use </span><span class="kw">crate</span>::repr::{ColumnType, RelationType};
<a href=#248 id=248 data-nosnippet>248</a>
<a href=#249 id=249 data-nosnippet>249</a> <span class="kw">pub struct </span>FlowDummyTableSource {
<a href=#250 id=250 data-nosnippet>250</a> <span class="kw">pub </span>id_names_to_desc: Vec&lt;(TableId, TableName, TableDesc)&gt;,
<a href=#251 id=251 data-nosnippet>251</a> id_to_idx: HashMap&lt;TableId, usize&gt;,
<a href=#252 id=252 data-nosnippet>252</a> name_to_idx: HashMap&lt;TableName, usize&gt;,
<a href=#253 id=253 data-nosnippet>253</a> }
<a href=#254 id=254 data-nosnippet>254</a>
<a href=#255 id=255 data-nosnippet>255</a> <span class="kw">impl </span>Default <span class="kw">for </span>FlowDummyTableSource {
<a href=#256 id=256 data-nosnippet>256</a> <span class="kw">fn </span>default() -&gt; <span class="self">Self </span>{
<a href=#257 id=257 data-nosnippet>257</a> <span class="kw">let </span>id_names_to_desc = <span class="macro">vec!</span>[
<a href=#258 id=258 data-nosnippet>258</a> (
<a href=#259 id=259 data-nosnippet>259</a> <span class="number">1024</span>,
<a href=#260 id=260 data-nosnippet>260</a> [
<a href=#261 id=261 data-nosnippet>261</a> <span class="string">"greptime"</span>.to_string(),
<a href=#262 id=262 data-nosnippet>262</a> <span class="string">"public"</span>.to_string(),
<a href=#263 id=263 data-nosnippet>263</a> <span class="string">"numbers"</span>.to_string(),
<a href=#264 id=264 data-nosnippet>264</a> ],
<a href=#265 id=265 data-nosnippet>265</a> TableDesc::new_no_default(
<a href=#266 id=266 data-nosnippet>266</a> RelationType::new(<span class="macro">vec!</span>[ColumnType::new(CDT::uint32_datatype(), <span class="bool-val">false</span>)])
<a href=#267 id=267 data-nosnippet>267</a> .into_named(<span class="macro">vec!</span>[<span class="prelude-val">Some</span>(<span class="string">"number"</span>.to_string())]),
<a href=#268 id=268 data-nosnippet>268</a> ),
<a href=#269 id=269 data-nosnippet>269</a> ),
<a href=#270 id=270 data-nosnippet>270</a> (
<a href=#271 id=271 data-nosnippet>271</a> <span class="number">1025</span>,
<a href=#272 id=272 data-nosnippet>272</a> [
<a href=#273 id=273 data-nosnippet>273</a> <span class="string">"greptime"</span>.to_string(),
<a href=#274 id=274 data-nosnippet>274</a> <span class="string">"public"</span>.to_string(),
<a href=#275 id=275 data-nosnippet>275</a> <span class="string">"numbers_with_ts"</span>.to_string(),
<a href=#276 id=276 data-nosnippet>276</a> ],
<a href=#277 id=277 data-nosnippet>277</a> TableDesc::new_no_default(
<a href=#278 id=278 data-nosnippet>278</a> RelationType::new(<span class="macro">vec!</span>[
<a href=#279 id=279 data-nosnippet>279</a> ColumnType::new(CDT::uint32_datatype(), <span class="bool-val">false</span>),
<a href=#280 id=280 data-nosnippet>280</a> ColumnType::new(CDT::timestamp_millisecond_datatype(), <span class="bool-val">false</span>),
<a href=#281 id=281 data-nosnippet>281</a> ])
<a href=#282 id=282 data-nosnippet>282</a> .into_named(<span class="macro">vec!</span>[<span class="prelude-val">Some</span>(<span class="string">"number"</span>.to_string()), <span class="prelude-val">Some</span>(<span class="string">"ts"</span>.to_string())]),
<a href=#283 id=283 data-nosnippet>283</a> ),
<a href=#284 id=284 data-nosnippet>284</a> ),
<a href=#285 id=285 data-nosnippet>285</a> ];
<a href=#286 id=286 data-nosnippet>286</a> <span class="kw">let </span>id_to_idx = id_names_to_desc
<a href=#287 id=287 data-nosnippet>287</a> .iter()
<a href=#288 id=288 data-nosnippet>288</a> .enumerate()
<a href=#289 id=289 data-nosnippet>289</a> .map(|(idx, (id, _name, _desc))| (<span class="kw-2">*</span>id, idx))
<a href=#290 id=290 data-nosnippet>290</a> .collect();
<a href=#291 id=291 data-nosnippet>291</a> <span class="kw">let </span>name_to_idx = id_names_to_desc
<a href=#292 id=292 data-nosnippet>292</a> .iter()
<a href=#293 id=293 data-nosnippet>293</a> .enumerate()
<a href=#294 id=294 data-nosnippet>294</a> .map(|(idx, (_id, name, _desc))| (name.clone(), idx))
<a href=#295 id=295 data-nosnippet>295</a> .collect();
<a href=#296 id=296 data-nosnippet>296</a> <span class="self">Self </span>{
<a href=#297 id=297 data-nosnippet>297</a> id_names_to_desc,
<a href=#298 id=298 data-nosnippet>298</a> id_to_idx,
<a href=#299 id=299 data-nosnippet>299</a> name_to_idx,
<a href=#300 id=300 data-nosnippet>300</a> }
<a href=#301 id=301 data-nosnippet>301</a> }
<a href=#302 id=302 data-nosnippet>302</a> }
<a href=#303 id=303 data-nosnippet>303</a>
<a href=#304 id=304 data-nosnippet>304</a> <span class="attr">#[async_trait::async_trait]
<a href=#305 id=305 data-nosnippet>305</a> </span><span class="kw">impl </span>FlowTableSource <span class="kw">for </span>FlowDummyTableSource {
<a href=#306 id=306 data-nosnippet>306</a> <span class="kw">async fn </span>table_from_id(<span class="kw-2">&amp;</span><span class="self">self</span>, table_id: <span class="kw-2">&amp;</span>TableId) -&gt; <span class="prelude-ty">Result</span>&lt;TableDesc, Error&gt; {
<a href=#307 id=307 data-nosnippet>307</a> <span class="kw">let </span>idx = <span class="self">self</span>.id_to_idx.get(table_id).context(TableNotFoundSnafu {
<a href=#308 id=308 data-nosnippet>308</a> name: <span class="macro">format!</span>(<span class="string">"Table id = {:?}, couldn't found table desc"</span>, table_id),
<a href=#309 id=309 data-nosnippet>309</a> })<span class="question-mark">?</span>;
<a href=#310 id=310 data-nosnippet>310</a> <span class="kw">let </span>desc = <span class="self">self
<a href=#311 id=311 data-nosnippet>311</a> </span>.id_names_to_desc
<a href=#312 id=312 data-nosnippet>312</a> .get(<span class="kw-2">*</span>idx)
<a href=#313 id=313 data-nosnippet>313</a> .map(|x| x.<span class="number">2</span>.clone())
<a href=#314 id=314 data-nosnippet>314</a> .context(TableNotFoundSnafu {
<a href=#315 id=315 data-nosnippet>315</a> name: <span class="macro">format!</span>(<span class="string">"Table id = {:?}, couldn't found table desc"</span>, table_id),
<a href=#316 id=316 data-nosnippet>316</a> })<span class="question-mark">?</span>;
<a href=#317 id=317 data-nosnippet>317</a> <span class="prelude-val">Ok</span>(desc)
<a href=#318 id=318 data-nosnippet>318</a> }
<a href=#319 id=319 data-nosnippet>319</a>
<a href=#320 id=320 data-nosnippet>320</a> <span class="kw">async fn </span>table_name_from_id(<span class="kw-2">&amp;</span><span class="self">self</span>, table_id: <span class="kw-2">&amp;</span>TableId) -&gt; <span class="prelude-ty">Result</span>&lt;TableName, Error&gt; {
<a href=#321 id=321 data-nosnippet>321</a> <span class="kw">let </span>idx = <span class="self">self</span>.id_to_idx.get(table_id).context(TableNotFoundSnafu {
<a href=#322 id=322 data-nosnippet>322</a> name: <span class="macro">format!</span>(<span class="string">"Table id = {:?}, couldn't found table desc"</span>, table_id),
<a href=#323 id=323 data-nosnippet>323</a> })<span class="question-mark">?</span>;
<a href=#324 id=324 data-nosnippet>324</a> <span class="self">self</span>.id_names_to_desc
<a href=#325 id=325 data-nosnippet>325</a> .get(<span class="kw-2">*</span>idx)
<a href=#326 id=326 data-nosnippet>326</a> .map(|x| x.<span class="number">1</span>.clone())
<a href=#327 id=327 data-nosnippet>327</a> .context(TableNotFoundSnafu {
<a href=#328 id=328 data-nosnippet>328</a> name: <span class="macro">format!</span>(<span class="string">"Table id = {:?}, couldn't found table desc"</span>, table_id),
<a href=#329 id=329 data-nosnippet>329</a> })
<a href=#330 id=330 data-nosnippet>330</a> }
<a href=#331 id=331 data-nosnippet>331</a>
<a href=#332 id=332 data-nosnippet>332</a> <span class="kw">async fn </span>table_id_from_name(<span class="kw-2">&amp;</span><span class="self">self</span>, name: <span class="kw-2">&amp;</span>TableName) -&gt; <span class="prelude-ty">Result</span>&lt;TableId, Error&gt; {
<a href=#333 id=333 data-nosnippet>333</a> <span class="kw">for </span>(id, table_name, _desc) <span class="kw">in </span><span class="kw-2">&amp;</span><span class="self">self</span>.id_names_to_desc {
<a href=#334 id=334 data-nosnippet>334</a> <span class="kw">if </span>name == table_name {
<a href=#335 id=335 data-nosnippet>335</a> <span class="kw">return </span><span class="prelude-val">Ok</span>(<span class="kw-2">*</span>id);
<a href=#336 id=336 data-nosnippet>336</a> }
<a href=#337 id=337 data-nosnippet>337</a> }
<a href=#338 id=338 data-nosnippet>338</a> TableNotFoundSnafu {
<a href=#339 id=339 data-nosnippet>339</a> name: <span class="macro">format!</span>(<span class="string">"Table name = {:?}, couldn't found table id"</span>, name),
<a href=#340 id=340 data-nosnippet>340</a> }
<a href=#341 id=341 data-nosnippet>341</a> .fail()<span class="question-mark">?
<a href=#342 id=342 data-nosnippet>342</a> </span>}
<a href=#343 id=343 data-nosnippet>343</a> }
<a href=#344 id=344 data-nosnippet>344</a>
<a href=#345 id=345 data-nosnippet>345</a> <span class="kw">impl </span>std::fmt::Debug <span class="kw">for </span>FlowDummyTableSource {
<a href=#346 id=346 data-nosnippet>346</a> <span class="kw">fn </span>fmt(<span class="kw-2">&amp;</span><span class="self">self</span>, f: <span class="kw-2">&amp;mut </span>std::fmt::Formatter&lt;<span class="lifetime">'_</span>&gt;) -&gt; std::fmt::Result {
<a href=#347 id=347 data-nosnippet>347</a> f.debug_struct(<span class="string">"DummyTableSource"</span>).finish()
<a href=#348 id=348 data-nosnippet>348</a> }
<a href=#349 id=349 data-nosnippet>349</a> }
<a href=#350 id=350 data-nosnippet>350</a>}
</code></pre></div></section></main></body></html>
+225
View File
@@ -0,0 +1,225 @@
<!DOCTYPE html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1.0"><meta name="generator" content="rustdoc"><meta name="description" content="Source of the Rust file `src/flow/src/adapter/util.rs`."><title>util.rs - source</title><script>if(window.location.protocol!=="file:")document.head.insertAdjacentHTML("beforeend","SourceSerif4-Regular-6b053e98.ttf.woff2,FiraSans-Italic-81dc35de.woff2,FiraSans-Regular-0fe48ade.woff2,FiraSans-MediumItalic-ccf7e434.woff2,FiraSans-Medium-e1aa3f0a.woff2,SourceCodePro-Regular-8badfe75.ttf.woff2,SourceCodePro-Semibold-aa29a496.ttf.woff2".split(",").map(f=>`<link rel="preload" as="font" type="font/woff2"href="../../../static.files/${f}">`).join(""))</script><link rel="stylesheet" href="../../../static.files/normalize-9960930a.css"><link rel="stylesheet" href="../../../static.files/rustdoc-17e0aaed.css"><meta name="rustdoc-vars" data-root-path="../../../" data-static-root-path="../../../static.files/" data-current-crate="flow" data-themes="" data-resource-suffix="" data-rustdoc-version="1.96.1 (31fca3adb 2026-06-26)" data-channel="1.96.1" data-search-js="search-b5634cc7.js" data-stringdex-js="stringdex-2da4960a.js" data-settings-js="settings-170eb4bf.js" ><script src="../../../static.files/storage-41dd4d93.js"></script><script defer src="../../../static.files/src-script-813739b1.js"></script><script defer src="../../../src-files.js"></script><script defer src="../../../static.files/main-5013f961.js"></script><noscript><link rel="stylesheet" href="../../../static.files/noscript-f7c3ffd8.css"></noscript><link rel="alternate icon" type="image/png" href="../../../static.files/favicon-32x32-eab170b8.png"><link rel="icon" type="image/svg+xml" href="../../../static.files/favicon-044be391.svg"></head><body class="rustdoc src"><a class="skip-main-content" href="#main-content">Skip to main content</a><!--[if lte IE 11]><div class="warning">This old browser is unsupported and will most likely display funky things.</div><![endif]--><nav class="sidebar"><div class="src-sidebar-title"><h2>Files</h2></div></nav><div class="sidebar-resizer" title="Drag to resize sidebar"></div><main><section id="main-content" class="content" tabindex="-1"><div class="main-heading"><h1><div class="sub-heading">flow/adapter/</div>util.rs</h1><rustdoc-toolbar></rustdoc-toolbar></div><div class="example-wrap digits-3"><pre class="rust"><code><a href=#1 id=1 data-nosnippet>1</a><span class="comment">// Copyright 2023 Greptime Team
<a href=#2 id=2 data-nosnippet>2</a>//
<a href=#3 id=3 data-nosnippet>3</a>// Licensed under the Apache License, Version 2.0 (the "License");
<a href=#4 id=4 data-nosnippet>4</a>// you may not use this file except in compliance with the License.
<a href=#5 id=5 data-nosnippet>5</a>// You may obtain a copy of the License at
<a href=#6 id=6 data-nosnippet>6</a>//
<a href=#7 id=7 data-nosnippet>7</a>// http://www.apache.org/licenses/LICENSE-2.0
<a href=#8 id=8 data-nosnippet>8</a>//
<a href=#9 id=9 data-nosnippet>9</a>// Unless required by applicable law or agreed to in writing, software
<a href=#10 id=10 data-nosnippet>10</a>// distributed under the License is distributed on an "AS IS" BASIS,
<a href=#11 id=11 data-nosnippet>11</a>// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
<a href=#12 id=12 data-nosnippet>12</a>// See the License for the specific language governing permissions and
<a href=#13 id=13 data-nosnippet>13</a>// limitations under the License.
<a href=#14 id=14 data-nosnippet>14</a>
<a href=#15 id=15 data-nosnippet>15</a></span><span class="doccomment">//! Util functions for adapter
<a href=#16 id=16 data-nosnippet>16</a>
<a href=#17 id=17 data-nosnippet>17</a></span><span class="kw">use </span>api::helper::ColumnDataTypeWrapper;
<a href=#18 id=18 data-nosnippet>18</a><span class="kw">use </span>api::v1::column_def::options_from_column_schema;
<a href=#19 id=19 data-nosnippet>19</a><span class="kw">use </span>api::v1::{ColumnDataType, ColumnDataTypeExtension, SemanticType};
<a href=#20 id=20 data-nosnippet>20</a><span class="kw">use </span>common_error::ext::BoxedError;
<a href=#21 id=21 data-nosnippet>21</a><span class="kw">use </span>common_meta::key::table_info::TableInfoValue;
<a href=#22 id=22 data-nosnippet>22</a><span class="kw">use </span>datatypes::prelude::ConcreteDataType;
<a href=#23 id=23 data-nosnippet>23</a><span class="kw">use </span>datatypes::schema::{ColumnDefaultConstraint, ColumnSchema};
<a href=#24 id=24 data-nosnippet>24</a><span class="kw">use </span>itertools::Itertools;
<a href=#25 id=25 data-nosnippet>25</a><span class="kw">use </span>operator::expr_helper;
<a href=#26 id=26 data-nosnippet>26</a><span class="kw">use </span>snafu::ResultExt;
<a href=#27 id=27 data-nosnippet>27</a><span class="kw">use </span>table::table_reference::TableReference;
<a href=#28 id=28 data-nosnippet>28</a>
<a href=#29 id=29 data-nosnippet>29</a><span class="kw">use </span><span class="kw">crate</span>::StreamingEngine;
<a href=#30 id=30 data-nosnippet>30</a><span class="kw">use </span><span class="kw">crate</span>::adapter::table_source::TableDesc;
<a href=#31 id=31 data-nosnippet>31</a><span class="kw">use </span><span class="kw">crate</span>::adapter::{AUTO_CREATED_PLACEHOLDER_TS_COL, TableName};
<a href=#32 id=32 data-nosnippet>32</a><span class="kw">use </span><span class="kw">crate</span>::error::{Error, ExternalSnafu};
<a href=#33 id=33 data-nosnippet>33</a><span class="kw">use </span><span class="kw">crate</span>::repr::{ColumnType, RelationDesc, RelationType};
<a href=#34 id=34 data-nosnippet>34</a><span class="kw">impl </span>StreamingEngine {
<a href=#35 id=35 data-nosnippet>35</a> <span class="doccomment">/// Create table from given schema(will adjust to add auto column if needed), return true if table is created
<a href=#36 id=36 data-nosnippet>36</a> </span><span class="kw">pub</span>(<span class="kw">crate</span>) <span class="kw">async fn </span>create_table_from_relation(
<a href=#37 id=37 data-nosnippet>37</a> <span class="kw-2">&amp;</span><span class="self">self</span>,
<a href=#38 id=38 data-nosnippet>38</a> flow_name: <span class="kw-2">&amp;</span>str,
<a href=#39 id=39 data-nosnippet>39</a> table_name: <span class="kw-2">&amp;</span>TableName,
<a href=#40 id=40 data-nosnippet>40</a> relation_desc: <span class="kw-2">&amp;</span>RelationDesc,
<a href=#41 id=41 data-nosnippet>41</a> ) -&gt; <span class="prelude-ty">Result</span>&lt;bool, Error&gt; {
<a href=#42 id=42 data-nosnippet>42</a> <span class="kw">if </span><span class="self">self</span>.fetch_table_pk_schema(table_name).<span class="kw">await</span><span class="question-mark">?</span>.is_some() {
<a href=#43 id=43 data-nosnippet>43</a> <span class="kw">return </span><span class="prelude-val">Ok</span>(<span class="bool-val">false</span>);
<a href=#44 id=44 data-nosnippet>44</a> }
<a href=#45 id=45 data-nosnippet>45</a> <span class="kw">let </span>(pks, tys, <span class="kw">_</span>) = <span class="self">self</span>.adjust_auto_created_table_schema(relation_desc).<span class="kw">await</span><span class="question-mark">?</span>;
<a href=#46 id=46 data-nosnippet>46</a>
<a href=#47 id=47 data-nosnippet>47</a> <span class="comment">//create sink table using pks, column types and is_ts_auto
<a href=#48 id=48 data-nosnippet>48</a>
<a href=#49 id=49 data-nosnippet>49</a> </span><span class="kw">let </span>proto_schema = column_schemas_to_proto(tys.clone(), <span class="kw-2">&amp;</span>pks)<span class="question-mark">?</span>;
<a href=#50 id=50 data-nosnippet>50</a>
<a href=#51 id=51 data-nosnippet>51</a> <span class="comment">// create sink table
<a href=#52 id=52 data-nosnippet>52</a> </span><span class="kw">let </span>create_expr = expr_helper::create_table_expr_by_column_schemas(
<a href=#53 id=53 data-nosnippet>53</a> <span class="kw-2">&amp;</span>TableReference {
<a href=#54 id=54 data-nosnippet>54</a> catalog: <span class="kw-2">&amp;</span>table_name[<span class="number">0</span>],
<a href=#55 id=55 data-nosnippet>55</a> schema: <span class="kw-2">&amp;</span>table_name[<span class="number">1</span>],
<a href=#56 id=56 data-nosnippet>56</a> table: <span class="kw-2">&amp;</span>table_name[<span class="number">2</span>],
<a href=#57 id=57 data-nosnippet>57</a> },
<a href=#58 id=58 data-nosnippet>58</a> <span class="kw-2">&amp;</span>proto_schema,
<a href=#59 id=59 data-nosnippet>59</a> <span class="string">"mito"</span>,
<a href=#60 id=60 data-nosnippet>60</a> <span class="prelude-val">Some</span>(<span class="kw-2">&amp;</span><span class="macro">format!</span>(<span class="string">"Sink table for flow {}"</span>, flow_name)),
<a href=#61 id=61 data-nosnippet>61</a> )
<a href=#62 id=62 data-nosnippet>62</a> .map_err(BoxedError::new)
<a href=#63 id=63 data-nosnippet>63</a> .context(ExternalSnafu)<span class="question-mark">?</span>;
<a href=#64 id=64 data-nosnippet>64</a>
<a href=#65 id=65 data-nosnippet>65</a> <span class="self">self</span>.frontend_client
<a href=#66 id=66 data-nosnippet>66</a> .create(create_expr, <span class="kw-2">&amp;</span>table_name[<span class="number">0</span>], <span class="kw-2">&amp;</span>table_name[<span class="number">1</span>])
<a href=#67 id=67 data-nosnippet>67</a> .<span class="kw">await</span><span class="question-mark">?</span>;
<a href=#68 id=68 data-nosnippet>68</a> <span class="prelude-val">Ok</span>(<span class="bool-val">true</span>)
<a href=#69 id=69 data-nosnippet>69</a> }
<a href=#70 id=70 data-nosnippet>70</a>
<a href=#71 id=71 data-nosnippet>71</a> <span class="doccomment">/// Try fetch table with adjusted schema(added auto column if needed)
<a href=#72 id=72 data-nosnippet>72</a> </span><span class="kw">pub</span>(<span class="kw">crate</span>) <span class="kw">async fn </span>try_fetch_existing_table(
<a href=#73 id=73 data-nosnippet>73</a> <span class="kw-2">&amp;</span><span class="self">self</span>,
<a href=#74 id=74 data-nosnippet>74</a> table_name: <span class="kw-2">&amp;</span>TableName,
<a href=#75 id=75 data-nosnippet>75</a> ) -&gt; <span class="prelude-ty">Result</span>&lt;<span class="prelude-ty">Option</span>&lt;(bool, Vec&lt;api::v1::ColumnSchema&gt;)&gt;, Error&gt; {
<a href=#76 id=76 data-nosnippet>76</a> <span class="kw">if let </span><span class="prelude-val">Some</span>((primary_keys, time_index, schema)) =
<a href=#77 id=77 data-nosnippet>77</a> <span class="self">self</span>.fetch_table_pk_schema(table_name).<span class="kw">await</span><span class="question-mark">?
<a href=#78 id=78 data-nosnippet>78</a> </span>{
<a href=#79 id=79 data-nosnippet>79</a> <span class="comment">// check if the last column is the auto created timestamp column, hence the table is auto created from
<a href=#80 id=80 data-nosnippet>80</a> // flow's plan type
<a href=#81 id=81 data-nosnippet>81</a> </span><span class="kw">let </span>is_auto_create = {
<a href=#82 id=82 data-nosnippet>82</a> <span class="kw">let </span>correct_name = schema
<a href=#83 id=83 data-nosnippet>83</a> .last()
<a href=#84 id=84 data-nosnippet>84</a> .map(|s| s.name == AUTO_CREATED_PLACEHOLDER_TS_COL)
<a href=#85 id=85 data-nosnippet>85</a> .unwrap_or(<span class="bool-val">false</span>);
<a href=#86 id=86 data-nosnippet>86</a> <span class="kw">let </span>correct_time_index = time_index == <span class="prelude-val">Some</span>(schema.len() - <span class="number">1</span>);
<a href=#87 id=87 data-nosnippet>87</a> correct_name &amp;&amp; correct_time_index
<a href=#88 id=88 data-nosnippet>88</a> };
<a href=#89 id=89 data-nosnippet>89</a> <span class="kw">let </span>proto_schema = column_schemas_to_proto(schema, <span class="kw-2">&amp;</span>primary_keys)<span class="question-mark">?</span>;
<a href=#90 id=90 data-nosnippet>90</a> <span class="prelude-val">Ok</span>(<span class="prelude-val">Some</span>((is_auto_create, proto_schema)))
<a href=#91 id=91 data-nosnippet>91</a> } <span class="kw">else </span>{
<a href=#92 id=92 data-nosnippet>92</a> <span class="prelude-val">Ok</span>(<span class="prelude-val">None</span>)
<a href=#93 id=93 data-nosnippet>93</a> }
<a href=#94 id=94 data-nosnippet>94</a> }
<a href=#95 id=95 data-nosnippet>95</a>}
<a href=#96 id=96 data-nosnippet>96</a>
<a href=#97 id=97 data-nosnippet>97</a><span class="kw">pub fn </span>table_info_value_to_relation_desc(
<a href=#98 id=98 data-nosnippet>98</a> table_info_value: TableInfoValue,
<a href=#99 id=99 data-nosnippet>99</a>) -&gt; <span class="prelude-ty">Result</span>&lt;TableDesc, Error&gt; {
<a href=#100 id=100 data-nosnippet>100</a> <span class="kw">let </span>raw_schema = table_info_value.table_info.meta.schema;
<a href=#101 id=101 data-nosnippet>101</a> <span class="kw">let </span>(column_types, col_names): (Vec&lt;<span class="kw">_</span>&gt;, Vec&lt;<span class="kw">_</span>&gt;) = raw_schema
<a href=#102 id=102 data-nosnippet>102</a> .column_schemas()
<a href=#103 id=103 data-nosnippet>103</a> .to_vec()
<a href=#104 id=104 data-nosnippet>104</a> .into_iter()
<a href=#105 id=105 data-nosnippet>105</a> .map(|col| {
<a href=#106 id=106 data-nosnippet>106</a> (
<a href=#107 id=107 data-nosnippet>107</a> ColumnType {
<a href=#108 id=108 data-nosnippet>108</a> nullable: col.is_nullable(),
<a href=#109 id=109 data-nosnippet>109</a> scalar_type: col.data_type,
<a href=#110 id=110 data-nosnippet>110</a> },
<a href=#111 id=111 data-nosnippet>111</a> <span class="prelude-val">Some</span>(col.name),
<a href=#112 id=112 data-nosnippet>112</a> )
<a href=#113 id=113 data-nosnippet>113</a> })
<a href=#114 id=114 data-nosnippet>114</a> .unzip();
<a href=#115 id=115 data-nosnippet>115</a>
<a href=#116 id=116 data-nosnippet>116</a> <span class="kw">let </span>key = table_info_value.table_info.meta.primary_key_indices;
<a href=#117 id=117 data-nosnippet>117</a> <span class="kw">let </span>keys = <span class="macro">vec!</span>[<span class="kw">crate</span>::repr::Key::from(key)];
<a href=#118 id=118 data-nosnippet>118</a>
<a href=#119 id=119 data-nosnippet>119</a> <span class="kw">let </span>time_index = raw_schema.timestamp_index();
<a href=#120 id=120 data-nosnippet>120</a> <span class="kw">let </span>relation_desc = RelationDesc {
<a href=#121 id=121 data-nosnippet>121</a> typ: RelationType {
<a href=#122 id=122 data-nosnippet>122</a> column_types,
<a href=#123 id=123 data-nosnippet>123</a> keys,
<a href=#124 id=124 data-nosnippet>124</a> time_index,
<a href=#125 id=125 data-nosnippet>125</a> <span class="comment">// by default table schema's column are all non-auto
<a href=#126 id=126 data-nosnippet>126</a> </span>auto_columns: <span class="macro">vec!</span>[],
<a href=#127 id=127 data-nosnippet>127</a> },
<a href=#128 id=128 data-nosnippet>128</a> names: col_names,
<a href=#129 id=129 data-nosnippet>129</a> };
<a href=#130 id=130 data-nosnippet>130</a> <span class="kw">let </span>default_values = raw_schema
<a href=#131 id=131 data-nosnippet>131</a> .column_schemas()
<a href=#132 id=132 data-nosnippet>132</a> .iter()
<a href=#133 id=133 data-nosnippet>133</a> .map(|c| {
<a href=#134 id=134 data-nosnippet>134</a> c.default_constraint().cloned().or_else(|| {
<a href=#135 id=135 data-nosnippet>135</a> <span class="kw">if </span>c.is_nullable() {
<a href=#136 id=136 data-nosnippet>136</a> <span class="prelude-val">Some</span>(ColumnDefaultConstraint::null_value())
<a href=#137 id=137 data-nosnippet>137</a> } <span class="kw">else </span>{
<a href=#138 id=138 data-nosnippet>138</a> <span class="prelude-val">None
<a href=#139 id=139 data-nosnippet>139</a> </span>}
<a href=#140 id=140 data-nosnippet>140</a> })
<a href=#141 id=141 data-nosnippet>141</a> })
<a href=#142 id=142 data-nosnippet>142</a> .collect_vec();
<a href=#143 id=143 data-nosnippet>143</a>
<a href=#144 id=144 data-nosnippet>144</a> <span class="prelude-val">Ok</span>(TableDesc::new(relation_desc, default_values))
<a href=#145 id=145 data-nosnippet>145</a>}
<a href=#146 id=146 data-nosnippet>146</a>
<a href=#147 id=147 data-nosnippet>147</a><span class="kw">pub fn </span>from_proto_to_data_type(
<a href=#148 id=148 data-nosnippet>148</a> column_schema: <span class="kw-2">&amp;</span>api::v1::ColumnSchema,
<a href=#149 id=149 data-nosnippet>149</a>) -&gt; <span class="prelude-ty">Result</span>&lt;ConcreteDataType, Error&gt; {
<a href=#150 id=150 data-nosnippet>150</a> <span class="kw">let </span>wrapper = ColumnDataTypeWrapper::try_new(
<a href=#151 id=151 data-nosnippet>151</a> column_schema.datatype,
<a href=#152 id=152 data-nosnippet>152</a> column_schema.datatype_extension.clone(),
<a href=#153 id=153 data-nosnippet>153</a> )
<a href=#154 id=154 data-nosnippet>154</a> .map_err(BoxedError::new)
<a href=#155 id=155 data-nosnippet>155</a> .context(ExternalSnafu)<span class="question-mark">?</span>;
<a href=#156 id=156 data-nosnippet>156</a> <span class="kw">let </span>cdt = ConcreteDataType::from(wrapper);
<a href=#157 id=157 data-nosnippet>157</a>
<a href=#158 id=158 data-nosnippet>158</a> <span class="prelude-val">Ok</span>(cdt)
<a href=#159 id=159 data-nosnippet>159</a>}
<a href=#160 id=160 data-nosnippet>160</a>
<a href=#161 id=161 data-nosnippet>161</a><span class="doccomment">/// convert `ColumnSchema` lists to it's corresponding proto type
<a href=#162 id=162 data-nosnippet>162</a></span><span class="kw">pub fn </span>column_schemas_to_proto(
<a href=#163 id=163 data-nosnippet>163</a> column_schemas: Vec&lt;ColumnSchema&gt;,
<a href=#164 id=164 data-nosnippet>164</a> primary_keys: <span class="kw-2">&amp;</span>[String],
<a href=#165 id=165 data-nosnippet>165</a>) -&gt; <span class="prelude-ty">Result</span>&lt;Vec&lt;api::v1::ColumnSchema&gt;, Error&gt; {
<a href=#166 id=166 data-nosnippet>166</a> <span class="kw">let </span>column_datatypes: Vec&lt;(ColumnDataType, <span class="prelude-ty">Option</span>&lt;ColumnDataTypeExtension&gt;)&gt; = column_schemas
<a href=#167 id=167 data-nosnippet>167</a> .iter()
<a href=#168 id=168 data-nosnippet>168</a> .map(|c| {
<a href=#169 id=169 data-nosnippet>169</a> ColumnDataTypeWrapper::try_from(c.data_type.clone())
<a href=#170 id=170 data-nosnippet>170</a> .map(|w| w.to_parts())
<a href=#171 id=171 data-nosnippet>171</a> .map_err(BoxedError::new)
<a href=#172 id=172 data-nosnippet>172</a> .context(ExternalSnafu)
<a href=#173 id=173 data-nosnippet>173</a> })
<a href=#174 id=174 data-nosnippet>174</a> .try_collect()<span class="question-mark">?</span>;
<a href=#175 id=175 data-nosnippet>175</a>
<a href=#176 id=176 data-nosnippet>176</a> <span class="kw">let </span>ret = column_schemas
<a href=#177 id=177 data-nosnippet>177</a> .iter()
<a href=#178 id=178 data-nosnippet>178</a> .zip(column_datatypes)
<a href=#179 id=179 data-nosnippet>179</a> .map(|(schema, datatype)| {
<a href=#180 id=180 data-nosnippet>180</a> <span class="kw">let </span>semantic_type = <span class="kw">if </span>schema.is_time_index() {
<a href=#181 id=181 data-nosnippet>181</a> SemanticType::Timestamp
<a href=#182 id=182 data-nosnippet>182</a> } <span class="kw">else if </span>primary_keys.contains(<span class="kw-2">&amp;</span>schema.name) {
<a href=#183 id=183 data-nosnippet>183</a> SemanticType::Tag
<a href=#184 id=184 data-nosnippet>184</a> } <span class="kw">else </span>{
<a href=#185 id=185 data-nosnippet>185</a> SemanticType::Field
<a href=#186 id=186 data-nosnippet>186</a> } <span class="kw">as </span>i32;
<a href=#187 id=187 data-nosnippet>187</a>
<a href=#188 id=188 data-nosnippet>188</a> api::v1::ColumnSchema {
<a href=#189 id=189 data-nosnippet>189</a> column_name: schema.name.clone(),
<a href=#190 id=190 data-nosnippet>190</a> datatype: datatype.<span class="number">0 </span><span class="kw">as </span>i32,
<a href=#191 id=191 data-nosnippet>191</a> semantic_type,
<a href=#192 id=192 data-nosnippet>192</a> datatype_extension: datatype.<span class="number">1</span>,
<a href=#193 id=193 data-nosnippet>193</a> options: options_from_column_schema(schema),
<a href=#194 id=194 data-nosnippet>194</a> }
<a href=#195 id=195 data-nosnippet>195</a> })
<a href=#196 id=196 data-nosnippet>196</a> .collect();
<a href=#197 id=197 data-nosnippet>197</a> <span class="prelude-val">Ok</span>(ret)
<a href=#198 id=198 data-nosnippet>198</a>}
<a href=#199 id=199 data-nosnippet>199</a>
<a href=#200 id=200 data-nosnippet>200</a><span class="doccomment">/// Convert `RelationDesc` to `ColumnSchema` list,
<a href=#201 id=201 data-nosnippet>201</a>/// if the column name is not present, use `col_{idx}` as the column name
<a href=#202 id=202 data-nosnippet>202</a></span><span class="kw">pub fn </span>relation_desc_to_column_schemas_with_fallback(schema: <span class="kw-2">&amp;</span>RelationDesc) -&gt; Vec&lt;ColumnSchema&gt; {
<a href=#203 id=203 data-nosnippet>203</a> schema
<a href=#204 id=204 data-nosnippet>204</a> .typ()
<a href=#205 id=205 data-nosnippet>205</a> .column_types
<a href=#206 id=206 data-nosnippet>206</a> .clone()
<a href=#207 id=207 data-nosnippet>207</a> .into_iter()
<a href=#208 id=208 data-nosnippet>208</a> .enumerate()
<a href=#209 id=209 data-nosnippet>209</a> .map(|(idx, typ)| {
<a href=#210 id=210 data-nosnippet>210</a> <span class="kw">let </span>name = schema
<a href=#211 id=211 data-nosnippet>211</a> .names
<a href=#212 id=212 data-nosnippet>212</a> .get(idx)
<a href=#213 id=213 data-nosnippet>213</a> .cloned()
<a href=#214 id=214 data-nosnippet>214</a> .flatten()
<a href=#215 id=215 data-nosnippet>215</a> .unwrap_or(<span class="macro">format!</span>(<span class="string">"col_{}"</span>, idx));
<a href=#216 id=216 data-nosnippet>216</a> <span class="kw">let </span>ret = ColumnSchema::new(name, typ.scalar_type, typ.nullable);
<a href=#217 id=217 data-nosnippet>217</a> <span class="kw">if </span>schema.typ().time_index == <span class="prelude-val">Some</span>(idx) {
<a href=#218 id=218 data-nosnippet>218</a> ret.with_time_index(<span class="bool-val">true</span>)
<a href=#219 id=219 data-nosnippet>219</a> } <span class="kw">else </span>{
<a href=#220 id=220 data-nosnippet>220</a> ret
<a href=#221 id=221 data-nosnippet>221</a> }
<a href=#222 id=222 data-nosnippet>222</a> })
<a href=#223 id=223 data-nosnippet>223</a> .collect_vec()
<a href=#224 id=224 data-nosnippet>224</a>}
</code></pre></div></section></main></body></html>
+89
View File
@@ -0,0 +1,89 @@
<!DOCTYPE html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1.0"><meta name="generator" content="rustdoc"><meta name="description" content="Source of the Rust file `src/flow/src/batching_mode.rs`."><title>batching_mode.rs - source</title><script>if(window.location.protocol!=="file:")document.head.insertAdjacentHTML("beforeend","SourceSerif4-Regular-6b053e98.ttf.woff2,FiraSans-Italic-81dc35de.woff2,FiraSans-Regular-0fe48ade.woff2,FiraSans-MediumItalic-ccf7e434.woff2,FiraSans-Medium-e1aa3f0a.woff2,SourceCodePro-Regular-8badfe75.ttf.woff2,SourceCodePro-Semibold-aa29a496.ttf.woff2".split(",").map(f=>`<link rel="preload" as="font" type="font/woff2"href="../../static.files/${f}">`).join(""))</script><link rel="stylesheet" href="../../static.files/normalize-9960930a.css"><link rel="stylesheet" href="../../static.files/rustdoc-17e0aaed.css"><meta name="rustdoc-vars" data-root-path="../../" data-static-root-path="../../static.files/" data-current-crate="flow" data-themes="" data-resource-suffix="" data-rustdoc-version="1.96.1 (31fca3adb 2026-06-26)" data-channel="1.96.1" data-search-js="search-b5634cc7.js" data-stringdex-js="stringdex-2da4960a.js" data-settings-js="settings-170eb4bf.js" ><script src="../../static.files/storage-41dd4d93.js"></script><script defer src="../../static.files/src-script-813739b1.js"></script><script defer src="../../src-files.js"></script><script defer src="../../static.files/main-5013f961.js"></script><noscript><link rel="stylesheet" href="../../static.files/noscript-f7c3ffd8.css"></noscript><link rel="alternate icon" type="image/png" href="../../static.files/favicon-32x32-eab170b8.png"><link rel="icon" type="image/svg+xml" href="../../static.files/favicon-044be391.svg"></head><body class="rustdoc src"><a class="skip-main-content" href="#main-content">Skip to main content</a><!--[if lte IE 11]><div class="warning">This old browser is unsupported and will most likely display funky things.</div><![endif]--><nav class="sidebar"><div class="src-sidebar-title"><h2>Files</h2></div></nav><div class="sidebar-resizer" title="Drag to resize sidebar"></div><main><section id="main-content" class="content" tabindex="-1"><div class="main-heading"><h1><div class="sub-heading">flow/</div>batching_mode.rs</h1><rustdoc-toolbar></rustdoc-toolbar></div><div class="example-wrap digits-2"><pre class="rust"><code><a href=#1 id=1 data-nosnippet>1</a><span class="comment">// Copyright 2023 Greptime Team
<a href=#2 id=2 data-nosnippet>2</a>//
<a href=#3 id=3 data-nosnippet>3</a>// Licensed under the Apache License, Version 2.0 (the "License");
<a href=#4 id=4 data-nosnippet>4</a>// you may not use this file except in compliance with the License.
<a href=#5 id=5 data-nosnippet>5</a>// You may obtain a copy of the License at
<a href=#6 id=6 data-nosnippet>6</a>//
<a href=#7 id=7 data-nosnippet>7</a>// http://www.apache.org/licenses/LICENSE-2.0
<a href=#8 id=8 data-nosnippet>8</a>//
<a href=#9 id=9 data-nosnippet>9</a>// Unless required by applicable law or agreed to in writing, software
<a href=#10 id=10 data-nosnippet>10</a>// distributed under the License is distributed on an "AS IS" BASIS,
<a href=#11 id=11 data-nosnippet>11</a>// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
<a href=#12 id=12 data-nosnippet>12</a>// See the License for the specific language governing permissions and
<a href=#13 id=13 data-nosnippet>13</a>// limitations under the License.
<a href=#14 id=14 data-nosnippet>14</a>
<a href=#15 id=15 data-nosnippet>15</a></span><span class="doccomment">//! Run flow as batching mode which is time-window-aware normal query triggered when new data arrives
<a href=#16 id=16 data-nosnippet>16</a>
<a href=#17 id=17 data-nosnippet>17</a></span><span class="kw">use </span>std::time::Duration;
<a href=#18 id=18 data-nosnippet>18</a>
<a href=#19 id=19 data-nosnippet>19</a><span class="kw">use </span>common_grpc::channel_manager::ClientTlsOption;
<a href=#20 id=20 data-nosnippet>20</a><span class="kw">use </span>serde::{Deserialize, Serialize};
<a href=#21 id=21 data-nosnippet>21</a><span class="kw">use </span>session::ReadPreference;
<a href=#22 id=22 data-nosnippet>22</a>
<a href=#23 id=23 data-nosnippet>23</a><span class="kw">pub</span>(<span class="kw">crate</span>) <span class="kw">mod </span>batching_execution;
<a href=#24 id=24 data-nosnippet>24</a><span class="kw">pub</span>(<span class="kw">crate</span>) <span class="kw">mod </span>checkpoint;
<a href=#25 id=25 data-nosnippet>25</a><span class="kw">pub</span>(<span class="kw">crate</span>) <span class="kw">mod </span>engine;
<a href=#26 id=26 data-nosnippet>26</a><span class="kw">mod </span>eval_schedule;
<a href=#27 id=27 data-nosnippet>27</a><span class="kw">pub</span>(<span class="kw">crate</span>) <span class="kw">mod </span>frontend_client;
<a href=#28 id=28 data-nosnippet>28</a><span class="kw">pub</span>(<span class="kw">crate</span>) <span class="kw">mod </span>state;
<a href=#29 id=29 data-nosnippet>29</a><span class="kw">mod </span>table_creator;
<a href=#30 id=30 data-nosnippet>30</a><span class="kw">pub</span>(<span class="kw">crate</span>) <span class="kw">mod </span>task;
<a href=#31 id=31 data-nosnippet>31</a><span class="kw">pub</span>(<span class="kw">crate</span>) <span class="kw">mod </span>time_window;
<a href=#32 id=32 data-nosnippet>32</a><span class="kw">pub</span>(<span class="kw">crate</span>) <span class="kw">mod </span>utils;
<a href=#33 id=33 data-nosnippet>33</a>
<a href=#34 id=34 data-nosnippet>34</a><span class="attr">#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
<a href=#35 id=35 data-nosnippet>35</a></span><span class="kw">pub struct </span>BatchingModeOptions {
<a href=#36 id=36 data-nosnippet>36</a> <span class="doccomment">/// The default batching engine query timeout is 10 minutes
<a href=#37 id=37 data-nosnippet>37</a> </span><span class="attr">#[serde(with = <span class="string">"humantime_serde"</span>)]
<a href=#38 id=38 data-nosnippet>38</a> </span><span class="kw">pub </span>query_timeout: Duration,
<a href=#39 id=39 data-nosnippet>39</a> <span class="doccomment">/// will output a warn log for any query that runs for more that this threshold
<a href=#40 id=40 data-nosnippet>40</a> </span><span class="attr">#[serde(with = <span class="string">"humantime_serde"</span>)]
<a href=#41 id=41 data-nosnippet>41</a> </span><span class="kw">pub </span>slow_query_threshold: Duration,
<a href=#42 id=42 data-nosnippet>42</a> <span class="doccomment">/// The minimum duration between two queries execution by batching mode task
<a href=#43 id=43 data-nosnippet>43</a> </span><span class="attr">#[serde(with = <span class="string">"humantime_serde"</span>)]
<a href=#44 id=44 data-nosnippet>44</a> </span><span class="kw">pub </span>experimental_min_refresh_duration: Duration,
<a href=#45 id=45 data-nosnippet>45</a> <span class="doccomment">/// The gRPC connection timeout
<a href=#46 id=46 data-nosnippet>46</a> </span><span class="attr">#[serde(with = <span class="string">"humantime_serde"</span>)]
<a href=#47 id=47 data-nosnippet>47</a> </span><span class="kw">pub </span>grpc_conn_timeout: Duration,
<a href=#48 id=48 data-nosnippet>48</a> <span class="attr">#[serde(with = <span class="string">"humantime_serde"</span>)]
<a href=#49 id=49 data-nosnippet>49</a> </span><span class="kw">pub </span>experimental_flight_do_get_timeout: Duration,
<a href=#50 id=50 data-nosnippet>50</a> <span class="doccomment">/// The gRPC max retry number
<a href=#51 id=51 data-nosnippet>51</a> </span><span class="kw">pub </span>experimental_grpc_max_retries: u32,
<a href=#52 id=52 data-nosnippet>52</a> <span class="doccomment">/// Flow wait for available frontend timeout,
<a href=#53 id=53 data-nosnippet>53</a> /// if failed to find available frontend after frontend_scan_timeout elapsed, return error
<a href=#54 id=54 data-nosnippet>54</a> /// which prevent flownode from starting
<a href=#55 id=55 data-nosnippet>55</a> </span><span class="attr">#[serde(with = <span class="string">"humantime_serde"</span>)]
<a href=#56 id=56 data-nosnippet>56</a> </span><span class="kw">pub </span>experimental_frontend_scan_timeout: Duration,
<a href=#57 id=57 data-nosnippet>57</a> <span class="doccomment">/// Maximum number of filters allowed in a single query
<a href=#58 id=58 data-nosnippet>58</a> </span><span class="kw">pub </span>experimental_max_filter_num_per_query: usize,
<a href=#59 id=59 data-nosnippet>59</a> <span class="doccomment">/// Time window merge distance
<a href=#60 id=60 data-nosnippet>60</a> </span><span class="kw">pub </span>experimental_time_window_merge_threshold: usize,
<a href=#61 id=61 data-nosnippet>61</a> <span class="doccomment">/// Whether to enable experimental flow incremental source reads.
<a href=#62 id=62 data-nosnippet>62</a> ///
<a href=#63 id=63 data-nosnippet>63</a> /// When disabled, batching flows always execute full-snapshot queries.
<a href=#64 id=64 data-nosnippet>64</a> </span><span class="kw">pub </span>experimental_enable_incremental_read: bool,
<a href=#65 id=65 data-nosnippet>65</a> <span class="doccomment">/// Read preference of the Frontend client.
<a href=#66 id=66 data-nosnippet>66</a> </span><span class="kw">pub </span>read_preference: ReadPreference,
<a href=#67 id=67 data-nosnippet>67</a> <span class="doccomment">/// TLS option for client connections to frontends.
<a href=#68 id=68 data-nosnippet>68</a> </span><span class="kw">pub </span>frontend_tls: <span class="prelude-ty">Option</span>&lt;ClientTlsOption&gt;,
<a href=#69 id=69 data-nosnippet>69</a>}
<a href=#70 id=70 data-nosnippet>70</a>
<a href=#71 id=71 data-nosnippet>71</a><span class="kw">impl </span>Default <span class="kw">for </span>BatchingModeOptions {
<a href=#72 id=72 data-nosnippet>72</a> <span class="kw">fn </span>default() -&gt; <span class="self">Self </span>{
<a href=#73 id=73 data-nosnippet>73</a> <span class="self">Self </span>{
<a href=#74 id=74 data-nosnippet>74</a> query_timeout: Duration::from_secs(<span class="number">10 </span>* <span class="number">60</span>),
<a href=#75 id=75 data-nosnippet>75</a> slow_query_threshold: Duration::from_secs(<span class="number">60</span>),
<a href=#76 id=76 data-nosnippet>76</a> experimental_min_refresh_duration: Duration::new(<span class="number">5</span>, <span class="number">0</span>),
<a href=#77 id=77 data-nosnippet>77</a> grpc_conn_timeout: Duration::from_secs(<span class="number">5</span>),
<a href=#78 id=78 data-nosnippet>78</a> experimental_flight_do_get_timeout: Duration::from_secs(<span class="number">10</span>),
<a href=#79 id=79 data-nosnippet>79</a> experimental_grpc_max_retries: <span class="number">3</span>,
<a href=#80 id=80 data-nosnippet>80</a> experimental_frontend_scan_timeout: Duration::from_secs(<span class="number">30</span>),
<a href=#81 id=81 data-nosnippet>81</a> experimental_max_filter_num_per_query: <span class="number">20</span>,
<a href=#82 id=82 data-nosnippet>82</a> experimental_time_window_merge_threshold: <span class="number">3</span>,
<a href=#83 id=83 data-nosnippet>83</a> experimental_enable_incremental_read: <span class="bool-val">false</span>,
<a href=#84 id=84 data-nosnippet>84</a> read_preference: Default::default(),
<a href=#85 id=85 data-nosnippet>85</a> frontend_tls: <span class="prelude-val">None</span>,
<a href=#86 id=86 data-nosnippet>86</a> }
<a href=#87 id=87 data-nosnippet>87</a> }
<a href=#88 id=88 data-nosnippet>88</a>}
</code></pre></div></section></main></body></html>
@@ -0,0 +1,61 @@
<!DOCTYPE html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1.0"><meta name="generator" content="rustdoc"><meta name="description" content="Source of the Rust file `src/flow/src/batching_mode/batching_execution.rs`."><title>batching_execution.rs - source</title><script>if(window.location.protocol!=="file:")document.head.insertAdjacentHTML("beforeend","SourceSerif4-Regular-6b053e98.ttf.woff2,FiraSans-Italic-81dc35de.woff2,FiraSans-Regular-0fe48ade.woff2,FiraSans-MediumItalic-ccf7e434.woff2,FiraSans-Medium-e1aa3f0a.woff2,SourceCodePro-Regular-8badfe75.ttf.woff2,SourceCodePro-Semibold-aa29a496.ttf.woff2".split(",").map(f=>`<link rel="preload" as="font" type="font/woff2"href="../../../static.files/${f}">`).join(""))</script><link rel="stylesheet" href="../../../static.files/normalize-9960930a.css"><link rel="stylesheet" href="../../../static.files/rustdoc-17e0aaed.css"><meta name="rustdoc-vars" data-root-path="../../../" data-static-root-path="../../../static.files/" data-current-crate="flow" data-themes="" data-resource-suffix="" data-rustdoc-version="1.96.1 (31fca3adb 2026-06-26)" data-channel="1.96.1" data-search-js="search-b5634cc7.js" data-stringdex-js="stringdex-2da4960a.js" data-settings-js="settings-170eb4bf.js" ><script src="../../../static.files/storage-41dd4d93.js"></script><script defer src="../../../static.files/src-script-813739b1.js"></script><script defer src="../../../src-files.js"></script><script defer src="../../../static.files/main-5013f961.js"></script><noscript><link rel="stylesheet" href="../../../static.files/noscript-f7c3ffd8.css"></noscript><link rel="alternate icon" type="image/png" href="../../../static.files/favicon-32x32-eab170b8.png"><link rel="icon" type="image/svg+xml" href="../../../static.files/favicon-044be391.svg"></head><body class="rustdoc src"><a class="skip-main-content" href="#main-content">Skip to main content</a><!--[if lte IE 11]><div class="warning">This old browser is unsupported and will most likely display funky things.</div><![endif]--><nav class="sidebar"><div class="src-sidebar-title"><h2>Files</h2></div></nav><div class="sidebar-resizer" title="Drag to resize sidebar"></div><main><section id="main-content" class="content" tabindex="-1"><div class="main-heading"><h1><div class="sub-heading">flow/batching_mode/</div>batching_execution.rs</h1><rustdoc-toolbar></rustdoc-toolbar></div><div class="example-wrap digits-2"><pre class="rust"><code><a href=#1 id=1 data-nosnippet>1</a><span class="comment">// Copyright 2023 Greptime Team
<a href=#2 id=2 data-nosnippet>2</a>//
<a href=#3 id=3 data-nosnippet>3</a>// Licensed under the Apache License, Version 2.0 (the "License");
<a href=#4 id=4 data-nosnippet>4</a>// you may not use this file except in compliance with the License.
<a href=#5 id=5 data-nosnippet>5</a>// You may obtain a copy of the License at
<a href=#6 id=6 data-nosnippet>6</a>//
<a href=#7 id=7 data-nosnippet>7</a>// http://www.apache.org/licenses/LICENSE-2.0
<a href=#8 id=8 data-nosnippet>8</a>//
<a href=#9 id=9 data-nosnippet>9</a>// Unless required by applicable law or agreed to in writing, software
<a href=#10 id=10 data-nosnippet>10</a>// distributed under the License is distributed on an "AS IS" BASIS,
<a href=#11 id=11 data-nosnippet>11</a>// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
<a href=#12 id=12 data-nosnippet>12</a>// See the License for the specific language governing permissions and
<a href=#13 id=13 data-nosnippet>13</a>// limitations under the License.
<a href=#14 id=14 data-nosnippet>14</a>
<a href=#15 id=15 data-nosnippet>15</a></span><span class="doccomment">//! Optional execution collaborator for batching tasks.
<a href=#16 id=16 data-nosnippet>16</a>
<a href=#17 id=17 data-nosnippet>17</a></span><span class="kw">use </span>std::sync::Arc;
<a href=#18 id=18 data-nosnippet>18</a>
<a href=#19 id=19 data-nosnippet>19</a><span class="kw">use </span>datafusion_expr::LogicalPlan;
<a href=#20 id=20 data-nosnippet>20</a><span class="kw">use </span>query::QueryEngineRef;
<a href=#21 id=21 data-nosnippet>21</a><span class="kw">use </span>table::TableRef;
<a href=#22 id=22 data-nosnippet>22</a>
<a href=#23 id=23 data-nosnippet>23</a><span class="kw">use </span><span class="kw">crate</span>::Result;
<a href=#24 id=24 data-nosnippet>24</a><span class="kw">use </span><span class="kw">crate</span>::batching_mode::frontend_client::FrontendClient;
<a href=#25 id=25 data-nosnippet>25</a><span class="kw">use </span><span class="kw">crate</span>::batching_mode::task::{BatchingExecutionGuard, BatchingTask, ExecuteOnceOutcome};
<a href=#26 id=26 data-nosnippet>26</a>
<a href=#27 id=27 data-nosnippet>27</a><span class="attr">#[async_trait::async_trait]
<a href=#28 id=28 data-nosnippet>28</a></span><span class="kw">pub trait </span>BatchingExecution: Send + Sync + <span class="lifetime">'static </span>{
<a href=#29 id=29 data-nosnippet>29</a> <span class="doccomment">/// Execute one round while retaining the guard through all task-state updates.
<a href=#30 id=30 data-nosnippet>30</a> /// An implementation that continues after caller cancellation must retain the
<a href=#31 id=31 data-nosnippet>31</a> /// guard with that work and make it stoppable through [`Self::stop`].
<a href=#32 id=32 data-nosnippet>32</a> </span><span class="kw">async fn </span>execute_once(
<a href=#33 id=33 data-nosnippet>33</a> <span class="self">self</span>: Arc&lt;<span class="self">Self</span>&gt;,
<a href=#34 id=34 data-nosnippet>34</a> guard: BatchingExecutionGuard,
<a href=#35 id=35 data-nosnippet>35</a> task: <span class="kw-2">&amp;</span>BatchingTask,
<a href=#36 id=36 data-nosnippet>36</a> engine: <span class="kw-2">&amp;</span>QueryEngineRef,
<a href=#37 id=37 data-nosnippet>37</a> frontend: <span class="kw-2">&amp;</span>Arc&lt;FrontendClient&gt;,
<a href=#38 id=38 data-nosnippet>38</a> max_window_cnt: <span class="prelude-ty">Option</span>&lt;usize&gt;,
<a href=#39 id=39 data-nosnippet>39</a> ) -&gt; ExecuteOnceOutcome;
<a href=#40 id=40 data-nosnippet>40</a>
<a href=#41 id=41 data-nosnippet>41</a> <span class="doccomment">/// Rewrite the completed query plan after incremental merging and before execution.
<a href=#42 id=42 data-nosnippet>42</a> </span><span class="kw">fn </span>rewrite_plan(<span class="kw-2">&amp;</span><span class="self">self</span>, _task: <span class="kw-2">&amp;</span>BatchingTask, plan: LogicalPlan) -&gt; <span class="prelude-ty">Result</span>&lt;LogicalPlan&gt; {
<a href=#43 id=43 data-nosnippet>43</a> <span class="prelude-val">Ok</span>(plan)
<a href=#44 id=44 data-nosnippet>44</a> }
<a href=#45 id=45 data-nosnippet>45</a>
<a href=#46 id=46 data-nosnippet>46</a> <span class="doccomment">/// Retire this execution instance, rejecting new work and requesting that any
<a href=#47 id=47 data-nosnippet>47</a> /// retained local work stop. This is not an acknowledgement of remote quiescence.
<a href=#48 id=48 data-nosnippet>48</a> </span><span class="kw">fn </span>stop(<span class="kw-2">&amp;</span><span class="self">self</span>) {}
<a href=#49 id=49 data-nosnippet>49</a>}
<a href=#50 id=50 data-nosnippet>50</a>
<a href=#51 id=51 data-nosnippet>51</a><span class="attr">#[async_trait::async_trait]
<a href=#52 id=52 data-nosnippet>52</a></span><span class="kw">pub trait </span>BatchingExecutionFactory: Send + Sync + <span class="lifetime">'static </span>{
<a href=#53 id=53 data-nosnippet>53</a> <span class="kw">async fn </span>create(
<a href=#54 id=54 data-nosnippet>54</a> <span class="kw-2">&amp;</span><span class="self">self</span>,
<a href=#55 id=55 data-nosnippet>55</a> task: <span class="kw-2">&amp;</span>BatchingTask,
<a href=#56 id=56 data-nosnippet>56</a> sink: TableRef,
<a href=#57 id=57 data-nosnippet>57</a> engine: <span class="kw-2">&amp;</span>QueryEngineRef,
<a href=#58 id=58 data-nosnippet>58</a> frontend: <span class="kw-2">&amp;</span>Arc&lt;FrontendClient&gt;,
<a href=#59 id=59 data-nosnippet>59</a> ) -&gt; <span class="prelude-ty">Result</span>&lt;<span class="prelude-ty">Option</span>&lt;Arc&lt;<span class="kw">dyn </span>BatchingExecution&gt;&gt;&gt;;
<a href=#60 id=60 data-nosnippet>60</a>}
</code></pre></div></section></main></body></html>
+154
View File
@@ -0,0 +1,154 @@
<!DOCTYPE html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1.0"><meta name="generator" content="rustdoc"><meta name="description" content="Source of the Rust file `src/flow/src/batching_mode/checkpoint.rs`."><title>checkpoint.rs - source</title><script>if(window.location.protocol!=="file:")document.head.insertAdjacentHTML("beforeend","SourceSerif4-Regular-6b053e98.ttf.woff2,FiraSans-Italic-81dc35de.woff2,FiraSans-Regular-0fe48ade.woff2,FiraSans-MediumItalic-ccf7e434.woff2,FiraSans-Medium-e1aa3f0a.woff2,SourceCodePro-Regular-8badfe75.ttf.woff2,SourceCodePro-Semibold-aa29a496.ttf.woff2".split(",").map(f=>`<link rel="preload" as="font" type="font/woff2"href="../../../static.files/${f}">`).join(""))</script><link rel="stylesheet" href="../../../static.files/normalize-9960930a.css"><link rel="stylesheet" href="../../../static.files/rustdoc-17e0aaed.css"><meta name="rustdoc-vars" data-root-path="../../../" data-static-root-path="../../../static.files/" data-current-crate="flow" data-themes="" data-resource-suffix="" data-rustdoc-version="1.96.1 (31fca3adb 2026-06-26)" data-channel="1.96.1" data-search-js="search-b5634cc7.js" data-stringdex-js="stringdex-2da4960a.js" data-settings-js="settings-170eb4bf.js" ><script src="../../../static.files/storage-41dd4d93.js"></script><script defer src="../../../static.files/src-script-813739b1.js"></script><script defer src="../../../src-files.js"></script><script defer src="../../../static.files/main-5013f961.js"></script><noscript><link rel="stylesheet" href="../../../static.files/noscript-f7c3ffd8.css"></noscript><link rel="alternate icon" type="image/png" href="../../../static.files/favicon-32x32-eab170b8.png"><link rel="icon" type="image/svg+xml" href="../../../static.files/favicon-044be391.svg"></head><body class="rustdoc src"><a class="skip-main-content" href="#main-content">Skip to main content</a><!--[if lte IE 11]><div class="warning">This old browser is unsupported and will most likely display funky things.</div><![endif]--><nav class="sidebar"><div class="src-sidebar-title"><h2>Files</h2></div></nav><div class="sidebar-resizer" title="Drag to resize sidebar"></div><main><section id="main-content" class="content" tabindex="-1"><div class="main-heading"><h1><div class="sub-heading">flow/batching_mode/</div>checkpoint.rs</h1><rustdoc-toolbar></rustdoc-toolbar></div><div class="example-wrap digits-3"><pre class="rust"><code><a href=#1 id=1 data-nosnippet>1</a><span class="comment">// Copyright 2023 Greptime Team
<a href=#2 id=2 data-nosnippet>2</a>//
<a href=#3 id=3 data-nosnippet>3</a>// Licensed under the Apache License, Version 2.0 (the "License");
<a href=#4 id=4 data-nosnippet>4</a>// you may not use this file except in compliance with the License.
<a href=#5 id=5 data-nosnippet>5</a>// You may obtain a copy of the License at
<a href=#6 id=6 data-nosnippet>6</a>//
<a href=#7 id=7 data-nosnippet>7</a>// http://www.apache.org/licenses/LICENSE-2.0
<a href=#8 id=8 data-nosnippet>8</a>//
<a href=#9 id=9 data-nosnippet>9</a>// Unless required by applicable law or agreed to in writing, software
<a href=#10 id=10 data-nosnippet>10</a>// distributed under the License is distributed on an "AS IS" BASIS,
<a href=#11 id=11 data-nosnippet>11</a>// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
<a href=#12 id=12 data-nosnippet>12</a>// See the License for the specific language governing permissions and
<a href=#13 id=13 data-nosnippet>13</a>// limitations under the License.
<a href=#14 id=14 data-nosnippet>14</a>
<a href=#15 id=15 data-nosnippet>15</a></span><span class="kw">use </span><span class="kw">crate</span>::batching_mode::state::CheckpointMode;
<a href=#16 id=16 data-nosnippet>16</a>
<a href=#17 id=17 data-nosnippet>17</a><span class="kw">pub</span>(<span class="kw">super</span>) <span class="kw">const </span>CHECKPOINT_DECISION_ADVANCE: <span class="kw-2">&amp;</span>str = <span class="string">"advance"</span>;
<a href=#18 id=18 data-nosnippet>18</a><span class="kw">pub</span>(<span class="kw">super</span>) <span class="kw">const </span>CHECKPOINT_DECISION_FALLBACK: <span class="kw-2">&amp;</span>str = <span class="string">"fallback"</span>;
<a href=#19 id=19 data-nosnippet>19</a><span class="kw">pub</span>(<span class="kw">super</span>) <span class="kw">const </span>CHECKPOINT_DECISION_CONTINUE_REPAIR: <span class="kw-2">&amp;</span>str = <span class="string">"continue_repair"</span>;
<a href=#20 id=20 data-nosnippet>20</a><span class="kw">pub</span>(<span class="kw">super</span>) <span class="kw">const </span>CHECKPOINT_REASON_NONE: <span class="kw-2">&amp;</span>str = <span class="string">"none"</span>;
<a href=#21 id=21 data-nosnippet>21</a>
<a href=#22 id=22 data-nosnippet>22</a><span class="doccomment">/// Why the task fell back to full snapshot mode.
<a href=#23 id=23 data-nosnippet>23</a></span><span class="attr">#[derive(Debug, Clone, Copy, PartialEq, Eq)]
<a href=#24 id=24 data-nosnippet>24</a></span><span class="kw">pub</span>(<span class="kw">super</span>) <span class="kw">enum </span>FlowQueryFallbackReason {
<a href=#25 id=25 data-nosnippet>25</a> <span class="doccomment">/// The query result did not include a region-watermark map at all.
<a href=#26 id=26 data-nosnippet>26</a> </span>MissingRegionWatermark,
<a href=#27 id=27 data-nosnippet>27</a> <span class="doccomment">/// Some participating regions could not prove safe advancement against
<a href=#28 id=28 data-nosnippet>28</a> /// both the returned watermarks and the checkpoint map.
<a href=#29 id=29 data-nosnippet>29</a> </span>IncompleteRegionWatermark,
<a href=#30 id=30 data-nosnippet>30</a> <span class="doccomment">/// The query only covered part of the dirty backlog, so global checkpoints
<a href=#31 id=31 data-nosnippet>31</a> /// cannot advance yet. Incremental SQL drains all dirty windows before
<a href=#32 id=32 data-nosnippet>32</a> /// checkpoint advancement; this primarily protects scoped full-snapshot
<a href=#33 id=33 data-nosnippet>33</a> /// runs capped by the per-query dirty-window limit.
<a href=#34 id=34 data-nosnippet>34</a> </span>DirtyBacklogPending,
<a href=#35 id=35 data-nosnippet>35</a> <span class="doccomment">/// The datanode detected a stale incremental cursor and the Flow
<a href=#36 id=36 data-nosnippet>36</a> /// must recompute from scratch.
<a href=#37 id=37 data-nosnippet>37</a> </span>StaleCursor,
<a href=#38 id=38 data-nosnippet>38</a> <span class="doccomment">/// A fenced repair chunk tried to use a snapshot upper bound that the
<a href=#39 id=39 data-nosnippet>39</a> /// storage engine can no longer enforce, so the current repair high must
<a href=#40 id=40 data-nosnippet>40</a> /// be abandoned and rebound by a fresh scoped full snapshot repair.
<a href=#41 id=41 data-nosnippet>41</a> </span>SnapshotFenceExpired,
<a href=#42 id=42 data-nosnippet>42</a> <span class="doccomment">/// A non-stale-cursor query failure; the Flow resets to full snapshot
<a href=#43 id=43 data-nosnippet>43</a> /// to avoid cascading errors.
<a href=#44 id=44 data-nosnippet>44</a> </span>IncrementalQueryFailure,
<a href=#45 id=45 data-nosnippet>45</a> <span class="doccomment">/// A non-incremental query failed while the task was already in full
<a href=#46 id=46 data-nosnippet>46</a> /// snapshot or scoped repair mode.
<a href=#47 id=47 data-nosnippet>47</a> </span>QueryFailure,
<a href=#48 id=48 data-nosnippet>48</a> <span class="doccomment">/// Incremental mode has been permanently disabled for this Flow
<a href=#49 id=49 data-nosnippet>49</a> /// (e.g. because the query shape is not incrementally safe).
<a href=#50 id=50 data-nosnippet>50</a> </span>IncrementalDisabled,
<a href=#51 id=51 data-nosnippet>51</a>}
<a href=#52 id=52 data-nosnippet>52</a>
<a href=#53 id=53 data-nosnippet>53</a><span class="kw">impl </span>FlowQueryFallbackReason {
<a href=#54 id=54 data-nosnippet>54</a> <span class="kw">pub</span>(<span class="kw">super</span>) <span class="kw">fn </span>as_label(<span class="self">self</span>) -&gt; <span class="kw-2">&amp;</span><span class="lifetime">'static </span>str {
<a href=#55 id=55 data-nosnippet>55</a> <span class="kw">match </span><span class="self">self </span>{
<a href=#56 id=56 data-nosnippet>56</a> <span class="self">Self</span>::MissingRegionWatermark =&gt; <span class="string">"missing_region_watermark"</span>,
<a href=#57 id=57 data-nosnippet>57</a> <span class="self">Self</span>::IncompleteRegionWatermark =&gt; <span class="string">"incomplete_region_watermark"</span>,
<a href=#58 id=58 data-nosnippet>58</a> <span class="self">Self</span>::DirtyBacklogPending =&gt; <span class="string">"dirty_backlog_pending"</span>,
<a href=#59 id=59 data-nosnippet>59</a> <span class="self">Self</span>::StaleCursor =&gt; <span class="string">"stale_cursor"</span>,
<a href=#60 id=60 data-nosnippet>60</a> <span class="self">Self</span>::SnapshotFenceExpired =&gt; <span class="string">"snapshot_fence_expired"</span>,
<a href=#61 id=61 data-nosnippet>61</a> <span class="self">Self</span>::IncrementalQueryFailure =&gt; <span class="string">"incremental_query_failure"</span>,
<a href=#62 id=62 data-nosnippet>62</a> <span class="self">Self</span>::QueryFailure =&gt; <span class="string">"query_failure"</span>,
<a href=#63 id=63 data-nosnippet>63</a> <span class="self">Self</span>::IncrementalDisabled =&gt; <span class="string">"incremental_disabled"</span>,
<a href=#64 id=64 data-nosnippet>64</a> }
<a href=#65 id=65 data-nosnippet>65</a> }
<a href=#66 id=66 data-nosnippet>66</a>}
<a href=#67 id=67 data-nosnippet>67</a>
<a href=#68 id=68 data-nosnippet>68</a><span class="doccomment">/// Decision produced by `BatchingTask::apply_query_result_to_state` after
<a href=#69 id=69 data-nosnippet>69</a>/// each Flow query execution. Describes whether the task advanced its
<a href=#70 id=70 data-nosnippet>70</a>/// checkpoint state or fell back to full snapshot, and why.
<a href=#71 id=71 data-nosnippet>71</a></span><span class="attr">#[derive(Debug, Clone, Copy, PartialEq, Eq)]
<a href=#72 id=72 data-nosnippet>72</a></span><span class="kw">pub</span>(<span class="kw">super</span>) <span class="kw">enum </span>FlowCheckpointDecision {
<a href=#73 id=73 data-nosnippet>73</a> <span class="doccomment">/// FullSnapshot → Incremental transition.
<a href=#74 id=74 data-nosnippet>74</a> ///
<a href=#75 id=75 data-nosnippet>75</a> /// The query exercised every participating region, all returned valid
<a href=#76 id=76 data-nosnippet>76</a> /// watermarks, and the checkpoint map was populated from scratch.
<a href=#77 id=77 data-nosnippet>77</a> /// Subsequent executions will use incremental after-seqs.
<a href=#78 id=78 data-nosnippet>78</a> </span>AdvancedFromFullSnapshot {
<a href=#79 id=79 data-nosnippet>79</a> participating_regions: usize,
<a href=#80 id=80 data-nosnippet>80</a> watermarks: usize,
<a href=#81 id=81 data-nosnippet>81</a> },
<a href=#82 id=82 data-nosnippet>82</a> <span class="doccomment">/// Existing Incremental → Incremental (in-place advancement).
<a href=#83 id=83 data-nosnippet>83</a> ///
<a href=#84 id=84 data-nosnippet>84</a> /// A subset of participating regions advanced their watermarks. The
<a href=#85 id=85 data-nosnippet>85</a> /// task stays in incremental mode with an updated checkpoint map.
<a href=#86 id=86 data-nosnippet>86</a> </span>AdvancedIncremental {
<a href=#87 id=87 data-nosnippet>87</a> participating_regions: usize,
<a href=#88 id=88 data-nosnippet>88</a> watermarks: usize,
<a href=#89 id=89 data-nosnippet>89</a> },
<a href=#90 id=90 data-nosnippet>90</a> <span class="doccomment">/// FullSnapshot stayed in full snapshot mode because a scoped base repair
<a href=#91 id=91 data-nosnippet>91</a> /// found additional dirty windows that may be concurrent with the returned
<a href=#92 id=92 data-nosnippet>92</a> /// high watermark. These windows must be repaired under the fixed high
<a href=#93 id=93 data-nosnippet>93</a> /// before checkpoints can advance.
<a href=#94 id=94 data-nosnippet>94</a> </span>ContinuedFencedRepair {
<a href=#95 id=95 data-nosnippet>95</a> pending_windows: usize,
<a href=#96 id=96 data-nosnippet>96</a> watermarks: usize,
<a href=#97 id=97 data-nosnippet>97</a> },
<a href=#98 id=98 data-nosnippet>98</a> <span class="doccomment">/// Any mode → FullSnapshot.
<a href=#99 id=99 data-nosnippet>99</a> ///
<a href=#100 id=100 data-nosnippet>100</a> /// Watermark information was incomplete, a participating region was
<a href=#101 id=101 data-nosnippet>101</a> /// absent from the existing checkpoint map, the task has permanently
<a href=#102 id=102 data-nosnippet>102</a> /// disabled incremental mode, or the query itself failed. The task
<a href=#103 id=103 data-nosnippet>103</a> /// resets to full snapshot semantics for the next execution.
<a href=#104 id=104 data-nosnippet>104</a> </span>FallbackToFullSnapshot {
<a href=#105 id=105 data-nosnippet>105</a> previous_mode: CheckpointMode,
<a href=#106 id=106 data-nosnippet>106</a> reason: FlowQueryFallbackReason,
<a href=#107 id=107 data-nosnippet>107</a> },
<a href=#108 id=108 data-nosnippet>108</a>}
<a href=#109 id=109 data-nosnippet>109</a>
<a href=#110 id=110 data-nosnippet>110</a><span class="kw">impl </span>FlowCheckpointDecision {
<a href=#111 id=111 data-nosnippet>111</a> <span class="kw">pub</span>(<span class="kw">super</span>) <span class="kw">fn </span>mode_label(<span class="self">self</span>) -&gt; <span class="kw-2">&amp;</span><span class="lifetime">'static </span>str {
<a href=#112 id=112 data-nosnippet>112</a> <span class="kw">match </span><span class="self">self </span>{
<a href=#113 id=113 data-nosnippet>113</a> <span class="self">Self</span>::AdvancedFromFullSnapshot { .. } =&gt; {
<a href=#114 id=114 data-nosnippet>114</a> checkpoint_mode_label(CheckpointMode::FullSnapshot)
<a href=#115 id=115 data-nosnippet>115</a> }
<a href=#116 id=116 data-nosnippet>116</a> <span class="self">Self</span>::AdvancedIncremental { .. } =&gt; checkpoint_mode_label(CheckpointMode::Incremental),
<a href=#117 id=117 data-nosnippet>117</a> <span class="comment">// Fenced repair is intentionally a FullSnapshot sub-state, not a
<a href=#118 id=118 data-nosnippet>118</a> // third top-level checkpoint mode, so metrics keep the
<a href=#119 id=119 data-nosnippet>119</a> // `full_snapshot` mode label while the decision label carries
<a href=#120 id=120 data-nosnippet>120</a> // `continue_repair`.
<a href=#121 id=121 data-nosnippet>121</a> </span><span class="self">Self</span>::ContinuedFencedRepair { .. } =&gt; {
<a href=#122 id=122 data-nosnippet>122</a> checkpoint_mode_label(CheckpointMode::FullSnapshot)
<a href=#123 id=123 data-nosnippet>123</a> }
<a href=#124 id=124 data-nosnippet>124</a> <span class="self">Self</span>::FallbackToFullSnapshot { previous_mode, .. } =&gt; {
<a href=#125 id=125 data-nosnippet>125</a> checkpoint_mode_label(previous_mode)
<a href=#126 id=126 data-nosnippet>126</a> }
<a href=#127 id=127 data-nosnippet>127</a> }
<a href=#128 id=128 data-nosnippet>128</a> }
<a href=#129 id=129 data-nosnippet>129</a>
<a href=#130 id=130 data-nosnippet>130</a> <span class="kw">pub</span>(<span class="kw">super</span>) <span class="kw">fn </span>decision_label(<span class="self">self</span>) -&gt; <span class="kw-2">&amp;</span><span class="lifetime">'static </span>str {
<a href=#131 id=131 data-nosnippet>131</a> <span class="kw">match </span><span class="self">self </span>{
<a href=#132 id=132 data-nosnippet>132</a> <span class="self">Self</span>::AdvancedFromFullSnapshot { .. } | <span class="self">Self</span>::AdvancedIncremental { .. } =&gt; {
<a href=#133 id=133 data-nosnippet>133</a> CHECKPOINT_DECISION_ADVANCE
<a href=#134 id=134 data-nosnippet>134</a> }
<a href=#135 id=135 data-nosnippet>135</a> <span class="self">Self</span>::ContinuedFencedRepair { .. } =&gt; CHECKPOINT_DECISION_CONTINUE_REPAIR,
<a href=#136 id=136 data-nosnippet>136</a> <span class="self">Self</span>::FallbackToFullSnapshot { .. } =&gt; CHECKPOINT_DECISION_FALLBACK,
<a href=#137 id=137 data-nosnippet>137</a> }
<a href=#138 id=138 data-nosnippet>138</a> }
<a href=#139 id=139 data-nosnippet>139</a>
<a href=#140 id=140 data-nosnippet>140</a> <span class="kw">pub</span>(<span class="kw">super</span>) <span class="kw">fn </span>reason_label(<span class="self">self</span>) -&gt; <span class="kw-2">&amp;</span><span class="lifetime">'static </span>str {
<a href=#141 id=141 data-nosnippet>141</a> <span class="kw">match </span><span class="self">self </span>{
<a href=#142 id=142 data-nosnippet>142</a> <span class="self">Self</span>::FallbackToFullSnapshot { reason, .. } =&gt; reason.as_label(),
<a href=#143 id=143 data-nosnippet>143</a> <span class="kw">_ </span>=&gt; CHECKPOINT_REASON_NONE,
<a href=#144 id=144 data-nosnippet>144</a> }
<a href=#145 id=145 data-nosnippet>145</a> }
<a href=#146 id=146 data-nosnippet>146</a>}
<a href=#147 id=147 data-nosnippet>147</a>
<a href=#148 id=148 data-nosnippet>148</a><span class="kw">pub</span>(<span class="kw">super</span>) <span class="kw">fn </span>checkpoint_mode_label(mode: CheckpointMode) -&gt; <span class="kw-2">&amp;</span><span class="lifetime">'static </span>str {
<a href=#149 id=149 data-nosnippet>149</a> <span class="kw">match </span>mode {
<a href=#150 id=150 data-nosnippet>150</a> CheckpointMode::FullSnapshot =&gt; <span class="string">"full_snapshot"</span>,
<a href=#151 id=151 data-nosnippet>151</a> CheckpointMode::Incremental =&gt; <span class="string">"incremental"</span>,
<a href=#152 id=152 data-nosnippet>152</a> }
<a href=#153 id=153 data-nosnippet>153</a>}
</code></pre></div></section></main></body></html>
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,525 @@
<!DOCTYPE html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1.0"><meta name="generator" content="rustdoc"><meta name="description" content="Source of the Rust file `src/flow/src/batching_mode/eval_schedule.rs`."><title>eval_schedule.rs - source</title><script>if(window.location.protocol!=="file:")document.head.insertAdjacentHTML("beforeend","SourceSerif4-Regular-6b053e98.ttf.woff2,FiraSans-Italic-81dc35de.woff2,FiraSans-Regular-0fe48ade.woff2,FiraSans-MediumItalic-ccf7e434.woff2,FiraSans-Medium-e1aa3f0a.woff2,SourceCodePro-Regular-8badfe75.ttf.woff2,SourceCodePro-Semibold-aa29a496.ttf.woff2".split(",").map(f=>`<link rel="preload" as="font" type="font/woff2"href="../../../static.files/${f}">`).join(""))</script><link rel="stylesheet" href="../../../static.files/normalize-9960930a.css"><link rel="stylesheet" href="../../../static.files/rustdoc-17e0aaed.css"><meta name="rustdoc-vars" data-root-path="../../../" data-static-root-path="../../../static.files/" data-current-crate="flow" data-themes="" data-resource-suffix="" data-rustdoc-version="1.96.1 (31fca3adb 2026-06-26)" data-channel="1.96.1" data-search-js="search-b5634cc7.js" data-stringdex-js="stringdex-2da4960a.js" data-settings-js="settings-170eb4bf.js" ><script src="../../../static.files/storage-41dd4d93.js"></script><script defer src="../../../static.files/src-script-813739b1.js"></script><script defer src="../../../src-files.js"></script><script defer src="../../../static.files/main-5013f961.js"></script><noscript><link rel="stylesheet" href="../../../static.files/noscript-f7c3ffd8.css"></noscript><link rel="alternate icon" type="image/png" href="../../../static.files/favicon-32x32-eab170b8.png"><link rel="icon" type="image/svg+xml" href="../../../static.files/favicon-044be391.svg"></head><body class="rustdoc src"><a class="skip-main-content" href="#main-content">Skip to main content</a><!--[if lte IE 11]><div class="warning">This old browser is unsupported and will most likely display funky things.</div><![endif]--><nav class="sidebar"><div class="src-sidebar-title"><h2>Files</h2></div></nav><div class="sidebar-resizer" title="Drag to resize sidebar"></div><main><section id="main-content" class="content" tabindex="-1"><div class="main-heading"><h1><div class="sub-heading">flow/batching_mode/</div>eval_schedule.rs</h1><rustdoc-toolbar></rustdoc-toolbar></div><div class="example-wrap digits-3"><pre class="rust"><code><a href=#1 id=1 data-nosnippet>1</a><span class="comment">// Copyright 2023 Greptime Team
<a href=#2 id=2 data-nosnippet>2</a>//
<a href=#3 id=3 data-nosnippet>3</a>// Licensed under the Apache License, Version 2.0 (the "License");
<a href=#4 id=4 data-nosnippet>4</a>// you may not use this file except in compliance with the License.
<a href=#5 id=5 data-nosnippet>5</a>// You may obtain a copy of the License at
<a href=#6 id=6 data-nosnippet>6</a>//
<a href=#7 id=7 data-nosnippet>7</a>// http://www.apache.org/licenses/LICENSE-2.0
<a href=#8 id=8 data-nosnippet>8</a>//
<a href=#9 id=9 data-nosnippet>9</a>// Unless required by applicable law or agreed to in writing, software
<a href=#10 id=10 data-nosnippet>10</a>// distributed under the License is distributed on an "AS IS" BASIS,
<a href=#11 id=11 data-nosnippet>11</a>// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
<a href=#12 id=12 data-nosnippet>12</a>// See the License for the specific language governing permissions and
<a href=#13 id=13 data-nosnippet>13</a>// limitations under the License.
<a href=#14 id=14 data-nosnippet>14</a>
<a href=#15 id=15 data-nosnippet>15</a></span><span class="doccomment">//! Helpers for stable `EVAL INTERVAL` scheduled times.
<a href=#16 id=16 data-nosnippet>16</a>
<a href=#17 id=17 data-nosnippet>17</a></span><span class="kw">pub use </span>common_meta::key::flow::flow_info::{FlowMissedTickPolicy, FlowScheduleConfig};
<a href=#18 id=18 data-nosnippet>18</a><span class="kw">use </span>snafu::ensure;
<a href=#19 id=19 data-nosnippet>19</a>
<a href=#20 id=20 data-nosnippet>20</a><span class="kw">use </span><span class="kw">crate</span>::error::{InvalidQuerySnafu, <span class="prelude-ty">Result</span>, UnexpectedSnafu};
<a href=#21 id=21 data-nosnippet>21</a>
<a href=#22 id=22 data-nosnippet>22</a><span class="doccomment">/// Schedule for an `EVAL INTERVAL` flow.
<a href=#23 id=23 data-nosnippet>23</a></span><span class="attr">#[derive(Debug, Clone, PartialEq)]
<a href=#24 id=24 data-nosnippet>24</a></span><span class="kw">pub struct </span>EvalSchedule {
<a href=#25 id=25 data-nosnippet>25</a> <span class="doccomment">/// Interval between scheduled times in seconds.
<a href=#26 id=26 data-nosnippet>26</a> </span><span class="kw">pub </span>interval_secs: i64,
<a href=#27 id=27 data-nosnippet>27</a> <span class="doccomment">/// Anchor timestamp as seconds since Unix epoch.
<a href=#28 id=28 data-nosnippet>28</a> </span><span class="kw">pub </span>anchor_secs: i64,
<a href=#29 id=29 data-nosnippet>29</a> <span class="doccomment">/// First scheduled time as seconds since Unix epoch.
<a href=#30 id=30 data-nosnippet>30</a> </span><span class="kw">pub </span>start_secs: i64,
<a href=#31 id=31 data-nosnippet>31</a> <span class="doccomment">/// Policy for handling missed scheduled times.
<a href=#32 id=32 data-nosnippet>32</a> </span><span class="kw">pub </span>missed_tick_policy: FlowMissedTickPolicy,
<a href=#33 id=33 data-nosnippet>33</a> <span class="doccomment">/// Maximum number of due scheduled times to catch up.
<a href=#34 id=34 data-nosnippet>34</a> </span><span class="kw">pub </span>max_runs: u32,
<a href=#35 id=35 data-nosnippet>35</a> <span class="doccomment">/// Maximum age of a due scheduled time to keep for catch-up.
<a href=#36 id=36 data-nosnippet>36</a> </span><span class="kw">pub </span>max_lag_secs: i64,
<a href=#37 id=37 data-nosnippet>37</a>}
<a href=#38 id=38 data-nosnippet>38</a>
<a href=#39 id=39 data-nosnippet>39</a><span class="kw">impl </span>EvalSchedule {
<a href=#40 id=40 data-nosnippet>40</a> <span class="kw">pub fn </span>from_config(
<a href=#41 id=41 data-nosnippet>41</a> eval_interval_secs: <span class="prelude-ty">Option</span>&lt;i64&gt;,
<a href=#42 id=42 data-nosnippet>42</a> config: <span class="prelude-ty">Option</span>&lt;<span class="kw-2">&amp;</span>FlowScheduleConfig&gt;,
<a href=#43 id=43 data-nosnippet>43</a> ) -&gt; <span class="prelude-ty">Result</span>&lt;<span class="prelude-ty">Option</span>&lt;<span class="self">Self</span>&gt;&gt; {
<a href=#44 id=44 data-nosnippet>44</a> <span class="kw">let </span><span class="prelude-val">Some</span>(interval_secs) = eval_interval_secs <span class="kw">else </span>{
<a href=#45 id=45 data-nosnippet>45</a> <span class="kw">return </span><span class="prelude-val">Ok</span>(<span class="prelude-val">None</span>);
<a href=#46 id=46 data-nosnippet>46</a> };
<a href=#47 id=47 data-nosnippet>47</a> <span class="macro">ensure!</span>(
<a href=#48 id=48 data-nosnippet>48</a> interval_secs &gt; <span class="number">0</span>,
<a href=#49 id=49 data-nosnippet>49</a> InvalidQuerySnafu {
<a href=#50 id=50 data-nosnippet>50</a> reason: <span class="macro">format!</span>(
<a href=#51 id=51 data-nosnippet>51</a> <span class="string">"Invalid eval_interval_secs: must be positive, got {interval_secs}"
<a href=#52 id=52 data-nosnippet>52</a> </span>)
<a href=#53 id=53 data-nosnippet>53</a> }
<a href=#54 id=54 data-nosnippet>54</a> );
<a href=#55 id=55 data-nosnippet>55</a>
<a href=#56 id=56 data-nosnippet>56</a> <span class="prelude-val">Ok</span>(<span class="prelude-val">Some</span>(<span class="kw">match </span>config {
<a href=#57 id=57 data-nosnippet>57</a> <span class="prelude-val">Some</span>(c) =&gt; {
<a href=#58 id=58 data-nosnippet>58</a> <span class="macro">ensure!</span>(
<a href=#59 id=59 data-nosnippet>59</a> c.catchup_max_runs &gt; <span class="number">0</span>,
<a href=#60 id=60 data-nosnippet>60</a> InvalidQuerySnafu {
<a href=#61 id=61 data-nosnippet>61</a> reason:
<a href=#62 id=62 data-nosnippet>62</a> <span class="string">"Invalid FlowScheduleConfig.catchup_max_runs: must be positive, got 0"
<a href=#63 id=63 data-nosnippet>63</a> </span>.to_string()
<a href=#64 id=64 data-nosnippet>64</a> }
<a href=#65 id=65 data-nosnippet>65</a> );
<a href=#66 id=66 data-nosnippet>66</a> <span class="macro">ensure!</span>(
<a href=#67 id=67 data-nosnippet>67</a> c.catchup_max_lag_secs &gt; <span class="number">0</span>,
<a href=#68 id=68 data-nosnippet>68</a> InvalidQuerySnafu {
<a href=#69 id=69 data-nosnippet>69</a> reason: <span class="macro">format!</span>(
<a href=#70 id=70 data-nosnippet>70</a> <span class="string">"Invalid FlowScheduleConfig.catchup_max_lag_secs: must be positive, got {}"</span>,
<a href=#71 id=71 data-nosnippet>71</a> c.catchup_max_lag_secs
<a href=#72 id=72 data-nosnippet>72</a> )
<a href=#73 id=73 data-nosnippet>73</a> }
<a href=#74 id=74 data-nosnippet>74</a> );
<a href=#75 id=75 data-nosnippet>75</a> <span class="comment">// The anchor defines the epoch phase `anchor + k * interval`; it
<a href=#76 id=76 data-nosnippet>76</a> // must be a valid offset within one interval.
<a href=#77 id=77 data-nosnippet>77</a> </span><span class="macro">ensure!</span>(
<a href=#78 id=78 data-nosnippet>78</a> c.anchor_secs &gt;= <span class="number">0 </span>&amp;&amp; c.anchor_secs &lt; interval_secs,
<a href=#79 id=79 data-nosnippet>79</a> InvalidQuerySnafu {
<a href=#80 id=80 data-nosnippet>80</a> reason: <span class="macro">format!</span>(
<a href=#81 id=81 data-nosnippet>81</a> <span class="string">"Invalid FlowScheduleConfig.anchor_secs: must be in [0, {interval_secs}), got {}"</span>,
<a href=#82 id=82 data-nosnippet>82</a> c.anchor_secs
<a href=#83 id=83 data-nosnippet>83</a> )
<a href=#84 id=84 data-nosnippet>84</a> }
<a href=#85 id=85 data-nosnippet>85</a> );
<a href=#86 id=86 data-nosnippet>86</a> <span class="comment">// The start must be phase-consistent with the anchor (on an
<a href=#87 id=87 data-nosnippet>87</a> // `anchor + k * interval` boundary) and not before the anchor.
<a href=#88 id=88 data-nosnippet>88</a> </span><span class="macro">ensure!</span>(
<a href=#89 id=89 data-nosnippet>89</a> c.start_secs &gt;= c.anchor_secs
<a href=#90 id=90 data-nosnippet>90</a> &amp;&amp; (c.start_secs - c.anchor_secs) % interval_secs == <span class="number">0</span>,
<a href=#91 id=91 data-nosnippet>91</a> InvalidQuerySnafu {
<a href=#92 id=92 data-nosnippet>92</a> reason: <span class="macro">format!</span>(
<a href=#93 id=93 data-nosnippet>93</a> <span class="string">"Invalid FlowScheduleConfig.start_secs: must be on an anchor + k * interval boundary and &gt;= anchor, got start={}, anchor={}, interval={}"</span>,
<a href=#94 id=94 data-nosnippet>94</a> c.start_secs, c.anchor_secs, interval_secs
<a href=#95 id=95 data-nosnippet>95</a> )
<a href=#96 id=96 data-nosnippet>96</a> }
<a href=#97 id=97 data-nosnippet>97</a> );
<a href=#98 id=98 data-nosnippet>98</a>
<a href=#99 id=99 data-nosnippet>99</a> <span class="self">Self </span>{
<a href=#100 id=100 data-nosnippet>100</a> interval_secs,
<a href=#101 id=101 data-nosnippet>101</a> anchor_secs: c.anchor_secs,
<a href=#102 id=102 data-nosnippet>102</a> start_secs: c.start_secs,
<a href=#103 id=103 data-nosnippet>103</a> missed_tick_policy: c.missed_tick_policy,
<a href=#104 id=104 data-nosnippet>104</a> max_runs: c.catchup_max_runs,
<a href=#105 id=105 data-nosnippet>105</a> max_lag_secs: c.catchup_max_lag_secs,
<a href=#106 id=106 data-nosnippet>106</a> }
<a href=#107 id=107 data-nosnippet>107</a> }
<a href=#108 id=108 data-nosnippet>108</a> <span class="prelude-val">None </span>=&gt; {
<a href=#109 id=109 data-nosnippet>109</a> <span class="kw">let </span>c = FlowScheduleConfig::default_with_start(<span class="number">0</span>, interval_secs);
<a href=#110 id=110 data-nosnippet>110</a> <span class="self">Self </span>{
<a href=#111 id=111 data-nosnippet>111</a> interval_secs,
<a href=#112 id=112 data-nosnippet>112</a> anchor_secs: c.anchor_secs,
<a href=#113 id=113 data-nosnippet>113</a> start_secs: c.start_secs,
<a href=#114 id=114 data-nosnippet>114</a> missed_tick_policy: c.missed_tick_policy,
<a href=#115 id=115 data-nosnippet>115</a> max_runs: c.catchup_max_runs,
<a href=#116 id=116 data-nosnippet>116</a> max_lag_secs: c.catchup_max_lag_secs,
<a href=#117 id=117 data-nosnippet>117</a> }
<a href=#118 id=118 data-nosnippet>118</a> }
<a href=#119 id=119 data-nosnippet>119</a> }))
<a href=#120 id=120 data-nosnippet>120</a> }
<a href=#121 id=121 data-nosnippet>121</a>
<a href=#122 id=122 data-nosnippet>122</a> <span class="doccomment">/// Returns the next scheduled time strictly after `cursor_secs`, on the
<a href=#123 id=123 data-nosnippet>123</a> /// `anchor + k * interval` lattice.
<a href=#124 id=124 data-nosnippet>124</a> ///
<a href=#125 id=125 data-nosnippet>125</a> /// Fallible: a non-positive interval or a next boundary that does not fit
<a href=#126 id=126 data-nosnippet>126</a> /// in `i64` yields an explicit error instead of a saturated non-phase
<a href=#127 id=127 data-nosnippet>127</a> /// value such as `i64::MAX`.
<a href=#128 id=128 data-nosnippet>128</a> </span><span class="kw">pub fn </span>next_scheduled_time_after(<span class="kw-2">&amp;</span><span class="self">self</span>, cursor_secs: i64) -&gt; <span class="prelude-ty">Result</span>&lt;i64&gt; {
<a href=#129 id=129 data-nosnippet>129</a> next_in_sequence(cursor_secs, <span class="self">self</span>.start_secs, <span class="self">self</span>.interval_secs)
<a href=#130 id=130 data-nosnippet>130</a> }
<a href=#131 id=131 data-nosnippet>131</a>}
<a href=#132 id=132 data-nosnippet>132</a>
<a href=#133 id=133 data-nosnippet>133</a><span class="doccomment">/// The smallest `start + k * interval` value that is strictly after `cursor`
<a href=#134 id=134 data-nosnippet>134</a>/// (`start` itself lies on the `anchor + k * interval` lattice, so every
<a href=#135 id=135 data-nosnippet>135</a>/// result is phase-consistent with the anchor). All arithmetic happens in
<a href=#136 id=136 data-nosnippet>136</a>/// `i128`: `cursor - start` cannot overflow and the result is either exactly
<a href=#137 id=137 data-nosnippet>137</a>/// on the lattice or an explicit error.
<a href=#138 id=138 data-nosnippet>138</a></span><span class="kw">fn </span>next_in_sequence(cursor: i64, start: i64, interval: i64) -&gt; <span class="prelude-ty">Result</span>&lt;i64&gt; {
<a href=#139 id=139 data-nosnippet>139</a> <span class="macro">ensure!</span>(
<a href=#140 id=140 data-nosnippet>140</a> interval &gt; <span class="number">0</span>,
<a href=#141 id=141 data-nosnippet>141</a> InvalidQuerySnafu {
<a href=#142 id=142 data-nosnippet>142</a> reason: <span class="macro">format!</span>(<span class="string">"Invalid eval interval: must be positive, got {interval}"</span>)
<a href=#143 id=143 data-nosnippet>143</a> }
<a href=#144 id=144 data-nosnippet>144</a> );
<a href=#145 id=145 data-nosnippet>145</a> <span class="kw">let </span>interval = i128::from(interval);
<a href=#146 id=146 data-nosnippet>146</a> <span class="kw">let </span>start = i128::from(start);
<a href=#147 id=147 data-nosnippet>147</a> <span class="kw">let </span>cursor = i128::from(cursor);
<a href=#148 id=148 data-nosnippet>148</a>
<a href=#149 id=149 data-nosnippet>149</a> <span class="kw">let </span>next = <span class="kw">if </span>cursor &lt; start {
<a href=#150 id=150 data-nosnippet>150</a> start
<a href=#151 id=151 data-nosnippet>151</a> } <span class="kw">else </span>{
<a href=#152 id=152 data-nosnippet>152</a> <span class="kw">let </span>k = (cursor - start) / interval;
<a href=#153 id=153 data-nosnippet>153</a> start + (k + <span class="number">1</span>) * interval
<a href=#154 id=154 data-nosnippet>154</a> };
<a href=#155 id=155 data-nosnippet>155</a>
<a href=#156 id=156 data-nosnippet>156</a> i64::try_from(next).map_err(|<span class="kw">_</span>| {
<a href=#157 id=157 data-nosnippet>157</a> UnexpectedSnafu {
<a href=#158 id=158 data-nosnippet>158</a> reason: <span class="macro">format!</span>(
<a href=#159 id=159 data-nosnippet>159</a> <span class="string">"Cannot advance the eval schedule past cursor {cursor}: the next scheduled time {next} does not fit in i64 (start={start}, interval={interval})"
<a href=#160 id=160 data-nosnippet>160</a> </span>),
<a href=#161 id=161 data-nosnippet>161</a> }
<a href=#162 id=162 data-nosnippet>162</a> .build()
<a href=#163 id=163 data-nosnippet>163</a> })
<a href=#164 id=164 data-nosnippet>164</a>}
<a href=#165 id=165 data-nosnippet>165</a>
<a href=#166 id=166 data-nosnippet>166</a><span class="kw">fn </span>first_due_in_sequence(cursor: i64, start: i64, interval: i64) -&gt; <span class="prelude-ty">Result</span>&lt;i64&gt; {
<a href=#167 id=167 data-nosnippet>167</a> <span class="kw">if </span>cursor &lt; start {
<a href=#168 id=168 data-nosnippet>168</a> <span class="prelude-val">Ok</span>(start)
<a href=#169 id=169 data-nosnippet>169</a> } <span class="kw">else </span>{
<a href=#170 id=170 data-nosnippet>170</a> next_in_sequence(cursor, start, interval)
<a href=#171 id=171 data-nosnippet>171</a> }
<a href=#172 id=172 data-nosnippet>172</a>}
<a href=#173 id=173 data-nosnippet>173</a>
<a href=#174 id=174 data-nosnippet>174</a><span class="doccomment">/// Scheduled times selected for execution in one scheduler pass.
<a href=#175 id=175 data-nosnippet>175</a>///
<a href=#176 id=176 data-nosnippet>176</a>/// A scheduled time is the logical evaluation timestamp for one flow run. When
<a href=#177 id=177 data-nosnippet>177</a>/// executing a timestamp from `scheduled_times_secs`, SQL/TQL `now()` is bound
<a href=#178 id=178 data-nosnippet>178</a>/// to that timestamp instead of the wall-clock execution time.
<a href=#179 id=179 data-nosnippet>179</a></span><span class="attr">#[derive(Debug, Clone, PartialEq)]
<a href=#180 id=180 data-nosnippet>180</a></span><span class="kw">pub struct </span>DueScheduledTimes {
<a href=#181 id=181 data-nosnippet>181</a> <span class="doccomment">/// Scheduled times to execute, ordered oldest to newest.
<a href=#182 id=182 data-nosnippet>182</a> </span><span class="kw">pub </span>scheduled_times_secs: Vec&lt;i64&gt;,
<a href=#183 id=183 data-nosnippet>183</a> <span class="doccomment">/// Number of due scheduled times skipped by lag or max-runs limits.
<a href=#184 id=184 data-nosnippet>184</a> </span><span class="kw">pub </span>skipped: u64,
<a href=#185 id=185 data-nosnippet>185</a>}
<a href=#186 id=186 data-nosnippet>186</a>
<a href=#187 id=187 data-nosnippet>187</a><span class="doccomment">/// Select due scheduled times `&lt;= wall_now_secs` without materializing all missed ticks.
<a href=#188 id=188 data-nosnippet>188</a>///
<a href=#189 id=189 data-nosnippet>189</a>/// Fallible: a non-positive interval or a scheduled time that does not fit in
<a href=#190 id=190 data-nosnippet>190</a>/// `i64` yields an explicit error instead of silently producing saturated
<a href=#191 id=191 data-nosnippet>191</a>/// non-phase timestamps.
<a href=#192 id=192 data-nosnippet>192</a></span><span class="kw">pub fn </span>select_due_scheduled_times(
<a href=#193 id=193 data-nosnippet>193</a> schedule: <span class="kw-2">&amp;</span>EvalSchedule,
<a href=#194 id=194 data-nosnippet>194</a> cursor_secs: i64,
<a href=#195 id=195 data-nosnippet>195</a> wall_now_secs: i64,
<a href=#196 id=196 data-nosnippet>196</a>) -&gt; <span class="prelude-ty">Result</span>&lt;DueScheduledTimes&gt; {
<a href=#197 id=197 data-nosnippet>197</a> <span class="kw">let </span>interval = schedule.interval_secs;
<a href=#198 id=198 data-nosnippet>198</a> <span class="macro">ensure!</span>(
<a href=#199 id=199 data-nosnippet>199</a> interval &gt; <span class="number">0</span>,
<a href=#200 id=200 data-nosnippet>200</a> InvalidQuerySnafu {
<a href=#201 id=201 data-nosnippet>201</a> reason: <span class="macro">format!</span>(<span class="string">"Invalid eval interval: must be positive, got {interval}"</span>)
<a href=#202 id=202 data-nosnippet>202</a> }
<a href=#203 id=203 data-nosnippet>203</a> );
<a href=#204 id=204 data-nosnippet>204</a>
<a href=#205 id=205 data-nosnippet>205</a> <span class="kw">let </span>first_due = first_due_in_sequence(cursor_secs, schedule.start_secs, interval)<span class="question-mark">?</span>;
<a href=#206 id=206 data-nosnippet>206</a> <span class="kw">if </span>first_due &gt; wall_now_secs {
<a href=#207 id=207 data-nosnippet>207</a> <span class="kw">return </span><span class="prelude-val">Ok</span>(DueScheduledTimes {
<a href=#208 id=208 data-nosnippet>208</a> scheduled_times_secs: <span class="macro">vec!</span>[],
<a href=#209 id=209 data-nosnippet>209</a> skipped: <span class="number">0</span>,
<a href=#210 id=210 data-nosnippet>210</a> });
<a href=#211 id=211 data-nosnippet>211</a> }
<a href=#212 id=212 data-nosnippet>212</a>
<a href=#213 id=213 data-nosnippet>213</a> <span class="comment">// Count and select due scheduled times in i128 so every value stays
<a href=#214 id=214 data-nosnippet>214</a> // exactly on the `anchor + k * interval` lattice; a value beyond `i64` is
<a href=#215 id=215 data-nosnippet>215</a> // an explicit error, never a saturated non-phase timestamp.
<a href=#216 id=216 data-nosnippet>216</a> </span><span class="kw">let </span>first_due = i128::from(first_due);
<a href=#217 id=217 data-nosnippet>217</a> <span class="kw">let </span>wall_now = i128::from(wall_now_secs);
<a href=#218 id=218 data-nosnippet>218</a> <span class="kw">let </span>interval = i128::from(interval);
<a href=#219 id=219 data-nosnippet>219</a>
<a href=#220 id=220 data-nosnippet>220</a> <span class="kw">let </span>total_count = (wall_now - first_due) / interval + <span class="number">1</span>;
<a href=#221 id=221 data-nosnippet>221</a> <span class="comment">// `first_due &gt;= 0` and `wall_now &lt;= i64::MAX`, so this always fits in u64.
<a href=#222 id=222 data-nosnippet>222</a> </span><span class="kw">let </span>total_count = u64::try_from(total_count).map_err(|<span class="kw">_</span>| {
<a href=#223 id=223 data-nosnippet>223</a> UnexpectedSnafu {
<a href=#224 id=224 data-nosnippet>224</a> reason: <span class="macro">format!</span>(
<a href=#225 id=225 data-nosnippet>225</a> <span class="string">"Cannot count due eval scheduled times up to {wall_now}: {total_count} does not fit in u64"
<a href=#226 id=226 data-nosnippet>226</a> </span>),
<a href=#227 id=227 data-nosnippet>227</a> }
<a href=#228 id=228 data-nosnippet>228</a> .build()
<a href=#229 id=229 data-nosnippet>229</a> })<span class="question-mark">?</span>;
<a href=#230 id=230 data-nosnippet>230</a>
<a href=#231 id=231 data-nosnippet>231</a> <span class="kw">match </span>schedule.missed_tick_policy {
<a href=#232 id=232 data-nosnippet>232</a> FlowMissedTickPolicy::Skip =&gt; {
<a href=#233 id=233 data-nosnippet>233</a> <span class="comment">// Keep only the latest due scheduled time; it is still on-lattice
<a href=#234 id=234 data-nosnippet>234</a> // and `&lt;= wall_now`.
<a href=#235 id=235 data-nosnippet>235</a> </span><span class="kw">let </span>last = i64::try_from(first_due + i128::from(total_count - <span class="number">1</span>) * interval)
<a href=#236 id=236 data-nosnippet>236</a> .map_err(|<span class="kw">_</span>| {
<a href=#237 id=237 data-nosnippet>237</a> UnexpectedSnafu {
<a href=#238 id=238 data-nosnippet>238</a> reason: <span class="macro">format!</span>(
<a href=#239 id=239 data-nosnippet>239</a> <span class="string">"Cannot compute the latest due eval scheduled time (first_due={first_due}, interval={interval}, count={total_count}): result does not fit in i64"
<a href=#240 id=240 data-nosnippet>240</a> </span>),
<a href=#241 id=241 data-nosnippet>241</a> }
<a href=#242 id=242 data-nosnippet>242</a> .build()
<a href=#243 id=243 data-nosnippet>243</a> })<span class="question-mark">?</span>;
<a href=#244 id=244 data-nosnippet>244</a> <span class="prelude-val">Ok</span>(DueScheduledTimes {
<a href=#245 id=245 data-nosnippet>245</a> scheduled_times_secs: <span class="macro">vec!</span>[last],
<a href=#246 id=246 data-nosnippet>246</a> skipped: total_count - <span class="number">1</span>,
<a href=#247 id=247 data-nosnippet>247</a> })
<a href=#248 id=248 data-nosnippet>248</a> }
<a href=#249 id=249 data-nosnippet>249</a> FlowMissedTickPolicy::BoundedCatchUp =&gt; {
<a href=#250 id=250 data-nosnippet>250</a> <span class="comment">// The cutoff is computed in i128: `wall_now - max_lag` may
<a href=#251 id=251 data-nosnippet>251</a> // legitimately underflow i64 (a cutoff before the Unix epoch) and
<a href=#252 id=252 data-nosnippet>252</a> // must not saturate to a wrong value.
<a href=#253 id=253 data-nosnippet>253</a> </span><span class="kw">let </span>cutoff = wall_now - i128::from(schedule.max_lag_secs);
<a href=#254 id=254 data-nosnippet>254</a> <span class="kw">let </span>skipped_by_cutoff = <span class="kw">if </span>first_due &gt;= cutoff {
<a href=#255 id=255 data-nosnippet>255</a> <span class="number">0
<a href=#256 id=256 data-nosnippet>256</a> </span>} <span class="kw">else </span>{
<a href=#257 id=257 data-nosnippet>257</a> <span class="comment">// ceil((cutoff - first_due) / interval), capped at u64::MAX
<a href=#258 id=258 data-nosnippet>258</a> // before the `.min(total_count)` below.
<a href=#259 id=259 data-nosnippet>259</a> </span><span class="kw">let </span>skipped = (cutoff - first_due + interval - <span class="number">1</span>) / interval;
<a href=#260 id=260 data-nosnippet>260</a> u64::try_from(skipped).unwrap_or(u64::MAX)
<a href=#261 id=261 data-nosnippet>261</a> }
<a href=#262 id=262 data-nosnippet>262</a> .min(total_count);
<a href=#263 id=263 data-nosnippet>263</a>
<a href=#264 id=264 data-nosnippet>264</a> <span class="kw">let </span>remaining = total_count - skipped_by_cutoff;
<a href=#265 id=265 data-nosnippet>265</a> <span class="kw">if </span>remaining == <span class="number">0 </span>{
<a href=#266 id=266 data-nosnippet>266</a> <span class="kw">return </span><span class="prelude-val">Ok</span>(DueScheduledTimes {
<a href=#267 id=267 data-nosnippet>267</a> scheduled_times_secs: <span class="macro">vec!</span>[],
<a href=#268 id=268 data-nosnippet>268</a> skipped: total_count,
<a href=#269 id=269 data-nosnippet>269</a> });
<a href=#270 id=270 data-nosnippet>270</a> }
<a href=#271 id=271 data-nosnippet>271</a>
<a href=#272 id=272 data-nosnippet>272</a> <span class="comment">// max_lag decides which missed scheduled times are recent enough to
<a href=#273 id=273 data-nosnippet>273</a> // run; max_runs caps how many of those times execute back-to-back
<a href=#274 id=274 data-nosnippet>274</a> // in one scheduler pass.
<a href=#275 id=275 data-nosnippet>275</a> </span><span class="kw">let </span>keep_count = remaining.min(u64::from(schedule.max_runs));
<a href=#276 id=276 data-nosnippet>276</a> <span class="kw">let </span>keep_start = skipped_by_cutoff + remaining - keep_count;
<a href=#277 id=277 data-nosnippet>277</a> <span class="kw">let </span><span class="kw-2">mut </span>scheduled_times_secs = Vec::with_capacity(keep_count <span class="kw">as </span>usize);
<a href=#278 id=278 data-nosnippet>278</a> <span class="kw">for </span>i <span class="kw">in </span><span class="number">0</span>..keep_count {
<a href=#279 id=279 data-nosnippet>279</a> <span class="kw">let </span>t = i64::try_from(
<a href=#280 id=280 data-nosnippet>280</a> first_due + (i128::from(keep_start) + i128::from(i)) * interval,
<a href=#281 id=281 data-nosnippet>281</a> )
<a href=#282 id=282 data-nosnippet>282</a> .map_err(|<span class="kw">_</span>| {
<a href=#283 id=283 data-nosnippet>283</a> UnexpectedSnafu {
<a href=#284 id=284 data-nosnippet>284</a> reason: <span class="macro">format!</span>(
<a href=#285 id=285 data-nosnippet>285</a> <span class="string">"Cannot compute a due eval scheduled time (first_due={first_due}, interval={interval}, index={i}): result does not fit in i64"
<a href=#286 id=286 data-nosnippet>286</a> </span>),
<a href=#287 id=287 data-nosnippet>287</a> }
<a href=#288 id=288 data-nosnippet>288</a> .build()
<a href=#289 id=289 data-nosnippet>289</a> })<span class="question-mark">?</span>;
<a href=#290 id=290 data-nosnippet>290</a> scheduled_times_secs.push(t);
<a href=#291 id=291 data-nosnippet>291</a> }
<a href=#292 id=292 data-nosnippet>292</a>
<a href=#293 id=293 data-nosnippet>293</a> <span class="prelude-val">Ok</span>(DueScheduledTimes {
<a href=#294 id=294 data-nosnippet>294</a> scheduled_times_secs,
<a href=#295 id=295 data-nosnippet>295</a> skipped: total_count - keep_count,
<a href=#296 id=296 data-nosnippet>296</a> })
<a href=#297 id=297 data-nosnippet>297</a> }
<a href=#298 id=298 data-nosnippet>298</a> }
<a href=#299 id=299 data-nosnippet>299</a>}
<a href=#300 id=300 data-nosnippet>300</a>
<a href=#301 id=301 data-nosnippet>301</a><span class="doccomment">/// Ceils `time` to the next `anchor + k * interval` boundary.
<a href=#302 id=302 data-nosnippet>302</a>///
<a href=#303 id=303 data-nosnippet>303</a>/// Fallible: if the next boundary does not fit in `i64`, an explicit error is
<a href=#304 id=304 data-nosnippet>304</a>/// returned instead of clamping to a non-phase value such as `i64::MAX`.
<a href=#305 id=305 data-nosnippet>305</a></span><span class="kw">pub fn </span>ceil_to_boundary(time: i64, anchor: i64, interval: i64) -&gt; <span class="prelude-ty">Result</span>&lt;i64&gt; {
<a href=#306 id=306 data-nosnippet>306</a> <span class="kw">if </span>interval &lt;= <span class="number">0 </span>{
<a href=#307 id=307 data-nosnippet>307</a> <span class="kw">return </span><span class="prelude-val">Ok</span>(time);
<a href=#308 id=308 data-nosnippet>308</a> }
<a href=#309 id=309 data-nosnippet>309</a> <span class="kw">if </span>time &lt;= anchor {
<a href=#310 id=310 data-nosnippet>310</a> <span class="kw">return </span><span class="prelude-val">Ok</span>(anchor);
<a href=#311 id=311 data-nosnippet>311</a> }
<a href=#312 id=312 data-nosnippet>312</a>
<a href=#313 id=313 data-nosnippet>313</a> <span class="kw">let </span>diff = i128::from(time) - i128::from(anchor);
<a href=#314 id=314 data-nosnippet>314</a> <span class="kw">let </span>interval = i128::from(interval);
<a href=#315 id=315 data-nosnippet>315</a> <span class="kw">let </span>k = (diff + interval - <span class="number">1</span>) / interval;
<a href=#316 id=316 data-nosnippet>316</a> <span class="kw">let </span>boundary = i128::from(anchor) + k * interval;
<a href=#317 id=317 data-nosnippet>317</a>
<a href=#318 id=318 data-nosnippet>318</a> i64::try_from(boundary).map_err(|<span class="kw">_</span>| {
<a href=#319 id=319 data-nosnippet>319</a> <span class="kw">crate</span>::error::UnexpectedSnafu {
<a href=#320 id=320 data-nosnippet>320</a> reason: <span class="macro">format!</span>(
<a href=#321 id=321 data-nosnippet>321</a> <span class="string">"Cannot align time {time} to the next `anchor + k * interval` boundary (anchor={anchor}, interval={interval}): result {boundary} does not fit in i64"
<a href=#322 id=322 data-nosnippet>322</a> </span>),
<a href=#323 id=323 data-nosnippet>323</a> }
<a href=#324 id=324 data-nosnippet>324</a> .build()
<a href=#325 id=325 data-nosnippet>325</a> })
<a href=#326 id=326 data-nosnippet>326</a>}
<a href=#327 id=327 data-nosnippet>327</a>
<a href=#328 id=328 data-nosnippet>328</a><span class="attr">#[cfg(test)]
<a href=#329 id=329 data-nosnippet>329</a></span><span class="kw">mod </span>test {
<a href=#330 id=330 data-nosnippet>330</a> <span class="kw">use </span>super::<span class="kw-2">*</span>;
<a href=#331 id=331 data-nosnippet>331</a>
<a href=#332 id=332 data-nosnippet>332</a> <span class="kw">fn </span>schedule(
<a href=#333 id=333 data-nosnippet>333</a> start: i64,
<a href=#334 id=334 data-nosnippet>334</a> policy: FlowMissedTickPolicy,
<a href=#335 id=335 data-nosnippet>335</a> max_runs: u32,
<a href=#336 id=336 data-nosnippet>336</a> max_lag_secs: i64,
<a href=#337 id=337 data-nosnippet>337</a> ) -&gt; EvalSchedule {
<a href=#338 id=338 data-nosnippet>338</a> EvalSchedule {
<a href=#339 id=339 data-nosnippet>339</a> interval_secs: <span class="number">60</span>,
<a href=#340 id=340 data-nosnippet>340</a> anchor_secs: <span class="number">0</span>,
<a href=#341 id=341 data-nosnippet>341</a> start_secs: start,
<a href=#342 id=342 data-nosnippet>342</a> missed_tick_policy: policy,
<a href=#343 id=343 data-nosnippet>343</a> max_runs,
<a href=#344 id=344 data-nosnippet>344</a> max_lag_secs,
<a href=#345 id=345 data-nosnippet>345</a> }
<a href=#346 id=346 data-nosnippet>346</a> }
<a href=#347 id=347 data-nosnippet>347</a>
<a href=#348 id=348 data-nosnippet>348</a> <span class="kw">fn </span>config(policy: FlowMissedTickPolicy) -&gt; FlowScheduleConfig {
<a href=#349 id=349 data-nosnippet>349</a> FlowScheduleConfig {
<a href=#350 id=350 data-nosnippet>350</a> anchor_secs: <span class="number">10</span>,
<a href=#351 id=351 data-nosnippet>351</a> <span class="comment">// phase-consistent: 310 = anchor(10) + 1 * interval(300)
<a href=#352 id=352 data-nosnippet>352</a> </span>start_secs: <span class="number">310</span>,
<a href=#353 id=353 data-nosnippet>353</a> missed_tick_policy: policy,
<a href=#354 id=354 data-nosnippet>354</a> catchup_max_runs: <span class="number">4</span>,
<a href=#355 id=355 data-nosnippet>355</a> catchup_max_lag_secs: <span class="number">600</span>,
<a href=#356 id=356 data-nosnippet>356</a> }
<a href=#357 id=357 data-nosnippet>357</a> }
<a href=#358 id=358 data-nosnippet>358</a>
<a href=#359 id=359 data-nosnippet>359</a> <span class="attr">#[test]
<a href=#360 id=360 data-nosnippet>360</a> </span><span class="kw">fn </span>ceil_to_boundary_handles_anchor_and_interval_edges() {
<a href=#361 id=361 data-nosnippet>361</a> <span class="macro">assert_eq!</span>(ceil_to_boundary(-<span class="number">10</span>, <span class="number">0</span>, <span class="number">60</span>).unwrap(), <span class="number">0</span>);
<a href=#362 id=362 data-nosnippet>362</a> <span class="macro">assert_eq!</span>(ceil_to_boundary(<span class="number">0</span>, <span class="number">0</span>, <span class="number">60</span>).unwrap(), <span class="number">0</span>);
<a href=#363 id=363 data-nosnippet>363</a> <span class="macro">assert_eq!</span>(ceil_to_boundary(<span class="number">1</span>, <span class="number">0</span>, <span class="number">60</span>).unwrap(), <span class="number">60</span>);
<a href=#364 id=364 data-nosnippet>364</a> <span class="macro">assert_eq!</span>(ceil_to_boundary(<span class="number">60</span>, <span class="number">0</span>, <span class="number">60</span>).unwrap(), <span class="number">60</span>);
<a href=#365 id=365 data-nosnippet>365</a> <span class="macro">assert_eq!</span>(ceil_to_boundary(<span class="number">101</span>, <span class="number">100</span>, <span class="number">60</span>).unwrap(), <span class="number">160</span>);
<a href=#366 id=366 data-nosnippet>366</a> <span class="macro">assert_eq!</span>(ceil_to_boundary(<span class="number">50</span>, <span class="number">0</span>, <span class="number">0</span>).unwrap(), <span class="number">50</span>);
<a href=#367 id=367 data-nosnippet>367</a> <span class="comment">// Never clamp to the non-phase i64::MAX: the next boundary does not fit.
<a href=#368 id=368 data-nosnippet>368</a> </span><span class="macro">assert!</span>(ceil_to_boundary(i64::MAX, <span class="number">0</span>, <span class="number">60</span>).is_err());
<a href=#369 id=369 data-nosnippet>369</a> <span class="macro">assert!</span>(ceil_to_boundary(i64::MAX - <span class="number">1</span>, i64::MIN, <span class="number">60</span>).is_err());
<a href=#370 id=370 data-nosnippet>370</a> }
<a href=#371 id=371 data-nosnippet>371</a>
<a href=#372 id=372 data-nosnippet>372</a> <span class="attr">#[test]
<a href=#373 id=373 data-nosnippet>373</a> </span><span class="kw">fn </span>from_config_maps_typed_config_and_defaults() {
<a href=#374 id=374 data-nosnippet>374</a> <span class="macro">assert!</span>(EvalSchedule::from_config(<span class="prelude-val">None</span>, <span class="prelude-val">None</span>).unwrap().is_none());
<a href=#375 id=375 data-nosnippet>375</a> <span class="macro">assert!</span>(EvalSchedule::from_config(<span class="prelude-val">Some</span>(<span class="number">0</span>), <span class="prelude-val">None</span>).is_err());
<a href=#376 id=376 data-nosnippet>376</a>
<a href=#377 id=377 data-nosnippet>377</a> <span class="kw">let </span>from_typed =
<a href=#378 id=378 data-nosnippet>378</a> EvalSchedule::from_config(<span class="prelude-val">Some</span>(<span class="number">300</span>), <span class="prelude-val">Some</span>(<span class="kw-2">&amp;</span>config(FlowMissedTickPolicy::Skip)))
<a href=#379 id=379 data-nosnippet>379</a> .unwrap()
<a href=#380 id=380 data-nosnippet>380</a> .unwrap();
<a href=#381 id=381 data-nosnippet>381</a> <span class="macro">assert_eq!</span>(from_typed.interval_secs, <span class="number">300</span>);
<a href=#382 id=382 data-nosnippet>382</a> <span class="macro">assert_eq!</span>(from_typed.anchor_secs, <span class="number">10</span>);
<a href=#383 id=383 data-nosnippet>383</a> <span class="macro">assert_eq!</span>(from_typed.start_secs, <span class="number">310</span>);
<a href=#384 id=384 data-nosnippet>384</a> <span class="macro">assert_eq!</span>(from_typed.missed_tick_policy, FlowMissedTickPolicy::Skip);
<a href=#385 id=385 data-nosnippet>385</a> <span class="macro">assert_eq!</span>(from_typed.max_runs, <span class="number">4</span>);
<a href=#386 id=386 data-nosnippet>386</a> <span class="macro">assert_eq!</span>(from_typed.max_lag_secs, <span class="number">600</span>);
<a href=#387 id=387 data-nosnippet>387</a>
<a href=#388 id=388 data-nosnippet>388</a> <span class="kw">let </span>defaulted = EvalSchedule::from_config(<span class="prelude-val">Some</span>(<span class="number">300</span>), <span class="prelude-val">None</span>).unwrap().unwrap();
<a href=#389 id=389 data-nosnippet>389</a> <span class="macro">assert_eq!</span>(defaulted.start_secs, <span class="number">0</span>);
<a href=#390 id=390 data-nosnippet>390</a> <span class="macro">assert_eq!</span>(defaulted.max_runs, <span class="number">3</span>);
<a href=#391 id=391 data-nosnippet>391</a> <span class="macro">assert_eq!</span>(defaulted.max_lag_secs, <span class="number">900</span>);
<a href=#392 id=392 data-nosnippet>392</a> }
<a href=#393 id=393 data-nosnippet>393</a>
<a href=#394 id=394 data-nosnippet>394</a> <span class="attr">#[test]
<a href=#395 id=395 data-nosnippet>395</a> </span><span class="kw">fn </span>from_config_rejects_invalid_catchup_limits() {
<a href=#396 id=396 data-nosnippet>396</a> <span class="kw">let </span><span class="kw-2">mut </span>c = config(FlowMissedTickPolicy::BoundedCatchUp);
<a href=#397 id=397 data-nosnippet>397</a> c.catchup_max_runs = <span class="number">0</span>;
<a href=#398 id=398 data-nosnippet>398</a> <span class="macro">assert!</span>(EvalSchedule::from_config(<span class="prelude-val">Some</span>(<span class="number">300</span>), <span class="prelude-val">Some</span>(<span class="kw-2">&amp;</span>c)).is_err());
<a href=#399 id=399 data-nosnippet>399</a>
<a href=#400 id=400 data-nosnippet>400</a> <span class="kw">let </span><span class="kw-2">mut </span>c = config(FlowMissedTickPolicy::BoundedCatchUp);
<a href=#401 id=401 data-nosnippet>401</a> c.catchup_max_lag_secs = <span class="number">0</span>;
<a href=#402 id=402 data-nosnippet>402</a> <span class="macro">assert!</span>(EvalSchedule::from_config(<span class="prelude-val">Some</span>(<span class="number">300</span>), <span class="prelude-val">Some</span>(<span class="kw-2">&amp;</span>c)).is_err());
<a href=#403 id=403 data-nosnippet>403</a> }
<a href=#404 id=404 data-nosnippet>404</a>
<a href=#405 id=405 data-nosnippet>405</a> <span class="attr">#[test]
<a href=#406 id=406 data-nosnippet>406</a> </span><span class="kw">fn </span>nonzero_anchor_due_selection_follows_phase() {
<a href=#407 id=407 data-nosnippet>407</a> <span class="comment">// anchor=120 (i.e. `EVAL OFFSET '2 minutes'`), interval=3600:
<a href=#408 id=408 data-nosnippet>408</a> // boundaries at :02 every hour. start=3720 (120 + 3600).
<a href=#409 id=409 data-nosnippet>409</a> </span><span class="kw">let </span>s = EvalSchedule {
<a href=#410 id=410 data-nosnippet>410</a> interval_secs: <span class="number">3600</span>,
<a href=#411 id=411 data-nosnippet>411</a> anchor_secs: <span class="number">120</span>,
<a href=#412 id=412 data-nosnippet>412</a> start_secs: <span class="number">3720</span>,
<a href=#413 id=413 data-nosnippet>413</a> missed_tick_policy: FlowMissedTickPolicy::BoundedCatchUp,
<a href=#414 id=414 data-nosnippet>414</a> max_runs: <span class="number">3</span>,
<a href=#415 id=415 data-nosnippet>415</a> max_lag_secs: <span class="number">3600</span>,
<a href=#416 id=416 data-nosnippet>416</a> };
<a href=#417 id=417 data-nosnippet>417</a> <span class="macro">assert_eq!</span>(
<a href=#418 id=418 data-nosnippet>418</a> select_due_scheduled_times(<span class="kw-2">&amp;</span>s, <span class="number">0</span>, <span class="number">100</span>)
<a href=#419 id=419 data-nosnippet>419</a> .unwrap()
<a href=#420 id=420 data-nosnippet>420</a> .scheduled_times_secs,
<a href=#421 id=421 data-nosnippet>421</a> Vec::&lt;i64&gt;::new()
<a href=#422 id=422 data-nosnippet>422</a> );
<a href=#423 id=423 data-nosnippet>423</a> <span class="comment">// From 3720 on, every selected time must be on the :02 phase.
<a href=#424 id=424 data-nosnippet>424</a> </span><span class="kw">let </span>due = select_due_scheduled_times(<span class="kw-2">&amp;</span>s, <span class="number">0</span>, <span class="number">3720</span>).unwrap();
<a href=#425 id=425 data-nosnippet>425</a> <span class="macro">assert_eq!</span>(due.scheduled_times_secs, <span class="macro">vec!</span>[<span class="number">3720</span>]);
<a href=#426 id=426 data-nosnippet>426</a> <span class="kw">let </span>due = select_due_scheduled_times(<span class="kw-2">&amp;</span>s, <span class="number">3720</span>, <span class="number">7320</span>).unwrap();
<a href=#427 id=427 data-nosnippet>427</a> <span class="macro">assert_eq!</span>(due.scheduled_times_secs, <span class="macro">vec!</span>[<span class="number">7320</span>]);
<a href=#428 id=428 data-nosnippet>428</a> <span class="kw">for </span>t <span class="kw">in </span><span class="kw-2">&amp;</span>due.scheduled_times_secs {
<a href=#429 id=429 data-nosnippet>429</a> <span class="macro">assert_eq!</span>((t - <span class="number">120</span>) % <span class="number">3600</span>, <span class="number">0</span>);
<a href=#430 id=430 data-nosnippet>430</a> }
<a href=#431 id=431 data-nosnippet>431</a> <span class="macro">assert_eq!</span>(s.next_scheduled_time_after(<span class="number">3720</span>).unwrap(), <span class="number">7320</span>);
<a href=#432 id=432 data-nosnippet>432</a> <span class="macro">assert_eq!</span>(s.next_scheduled_time_after(<span class="number">7300</span>).unwrap(), <span class="number">7320</span>);
<a href=#433 id=433 data-nosnippet>433</a> }
<a href=#434 id=434 data-nosnippet>434</a>
<a href=#435 id=435 data-nosnippet>435</a> <span class="attr">#[test]
<a href=#436 id=436 data-nosnippet>436</a> </span><span class="kw">fn </span>next_scheduled_time_after_respects_start_sequence() {
<a href=#437 id=437 data-nosnippet>437</a> <span class="kw">let </span>s = schedule(<span class="number">50</span>, FlowMissedTickPolicy::BoundedCatchUp, <span class="number">3</span>, <span class="number">300</span>);
<a href=#438 id=438 data-nosnippet>438</a> <span class="macro">assert_eq!</span>(s.next_scheduled_time_after(<span class="number">0</span>).unwrap(), <span class="number">50</span>);
<a href=#439 id=439 data-nosnippet>439</a> <span class="macro">assert_eq!</span>(s.next_scheduled_time_after(<span class="number">50</span>).unwrap(), <span class="number">110</span>);
<a href=#440 id=440 data-nosnippet>440</a> <span class="macro">assert_eq!</span>(s.next_scheduled_time_after(<span class="number">100</span>).unwrap(), <span class="number">110</span>);
<a href=#441 id=441 data-nosnippet>441</a> }
<a href=#442 id=442 data-nosnippet>442</a>
<a href=#443 id=443 data-nosnippet>443</a> <span class="attr">#[test]
<a href=#444 id=444 data-nosnippet>444</a> </span><span class="kw">fn </span>near_i64_max_advancement_is_exact_or_explicit_error() {
<a href=#445 id=445 data-nosnippet>445</a> <span class="comment">// anchor=0, interval=60: the next boundary after i64::MAX - 60 is
<a href=#446 id=446 data-nosnippet>446</a> // 9223372036854775800, still in range and exactly on the lattice.
<a href=#447 id=447 data-nosnippet>447</a> </span><span class="kw">let </span>s = schedule(<span class="number">0</span>, FlowMissedTickPolicy::Skip, <span class="number">5</span>, <span class="number">3600</span>);
<a href=#448 id=448 data-nosnippet>448</a> <span class="kw">let </span>cursor = i64::MAX - <span class="number">60</span>;
<a href=#449 id=449 data-nosnippet>449</a> <span class="kw">let </span>next = s.next_scheduled_time_after(cursor).unwrap();
<a href=#450 id=450 data-nosnippet>450</a> <span class="macro">assert_eq!</span>(next, <span class="number">9223372036854775800</span>);
<a href=#451 id=451 data-nosnippet>451</a> <span class="macro">assert_eq!</span>(next % <span class="number">60</span>, <span class="number">0</span>);
<a href=#452 id=452 data-nosnippet>452</a>
<a href=#453 id=453 data-nosnippet>453</a> <span class="comment">// Advancing past the last representable boundary is an explicit error,
<a href=#454 id=454 data-nosnippet>454</a> // never a saturated non-phase value like i64::MAX.
<a href=#455 id=455 data-nosnippet>455</a> </span><span class="kw">let </span>err = s
<a href=#456 id=456 data-nosnippet>456</a> .next_scheduled_time_after(<span class="number">9223372036854775800</span>)
<a href=#457 id=457 data-nosnippet>457</a> .unwrap_err();
<a href=#458 id=458 data-nosnippet>458</a> <span class="macro">assert!</span>(err.to_string().contains(<span class="string">"does not fit in i64"</span>));
<a href=#459 id=459 data-nosnippet>459</a>
<a href=#460 id=460 data-nosnippet>460</a> <span class="comment">// A non-positive interval is an explicit error, not a saturating
<a href=#461 id=461 data-nosnippet>461</a> // `cursor + 1` result.
<a href=#462 id=462 data-nosnippet>462</a> </span><span class="kw">let </span>invalid = EvalSchedule {
<a href=#463 id=463 data-nosnippet>463</a> interval_secs: <span class="number">0</span>,
<a href=#464 id=464 data-nosnippet>464</a> anchor_secs: <span class="number">0</span>,
<a href=#465 id=465 data-nosnippet>465</a> start_secs: <span class="number">0</span>,
<a href=#466 id=466 data-nosnippet>466</a> missed_tick_policy: FlowMissedTickPolicy::Skip,
<a href=#467 id=467 data-nosnippet>467</a> max_runs: <span class="number">3</span>,
<a href=#468 id=468 data-nosnippet>468</a> max_lag_secs: <span class="number">900</span>,
<a href=#469 id=469 data-nosnippet>469</a> };
<a href=#470 id=470 data-nosnippet>470</a> <span class="macro">assert!</span>(invalid.next_scheduled_time_after(<span class="number">0</span>).is_err());
<a href=#471 id=471 data-nosnippet>471</a> }
<a href=#472 id=472 data-nosnippet>472</a>
<a href=#473 id=473 data-nosnippet>473</a> <span class="attr">#[test]
<a href=#474 id=474 data-nosnippet>474</a> </span><span class="kw">fn </span>due_scheduled_time_selection_handles_empty_and_start_boundary() {
<a href=#475 id=475 data-nosnippet>475</a> <span class="kw">let </span>s = schedule(<span class="number">120</span>, FlowMissedTickPolicy::BoundedCatchUp, <span class="number">10</span>, <span class="number">3600</span>);
<a href=#476 id=476 data-nosnippet>476</a> <span class="macro">assert_eq!</span>(
<a href=#477 id=477 data-nosnippet>477</a> select_due_scheduled_times(<span class="kw-2">&amp;</span>s, <span class="number">0</span>, <span class="number">100</span>)
<a href=#478 id=478 data-nosnippet>478</a> .unwrap()
<a href=#479 id=479 data-nosnippet>479</a> .scheduled_times_secs,
<a href=#480 id=480 data-nosnippet>480</a> Vec::&lt;i64&gt;::new()
<a href=#481 id=481 data-nosnippet>481</a> );
<a href=#482 id=482 data-nosnippet>482</a> <span class="macro">assert_eq!</span>(
<a href=#483 id=483 data-nosnippet>483</a> select_due_scheduled_times(<span class="kw-2">&amp;</span>s, <span class="number">0</span>, <span class="number">300</span>)
<a href=#484 id=484 data-nosnippet>484</a> .unwrap()
<a href=#485 id=485 data-nosnippet>485</a> .scheduled_times_secs,
<a href=#486 id=486 data-nosnippet>486</a> <span class="macro">vec!</span>[<span class="number">120</span>, <span class="number">180</span>, <span class="number">240</span>, <span class="number">300</span>]
<a href=#487 id=487 data-nosnippet>487</a> );
<a href=#488 id=488 data-nosnippet>488</a> }
<a href=#489 id=489 data-nosnippet>489</a>
<a href=#490 id=490 data-nosnippet>490</a> <span class="attr">#[test]
<a href=#491 id=491 data-nosnippet>491</a> </span><span class="kw">fn </span>bounded_catch_up_applies_lag_and_max_runs() {
<a href=#492 id=492 data-nosnippet>492</a> <span class="kw">let </span>s = schedule(<span class="number">0</span>, FlowMissedTickPolicy::BoundedCatchUp, <span class="number">2</span>, <span class="number">180</span>);
<a href=#493 id=493 data-nosnippet>493</a> <span class="kw">let </span>due = select_due_scheduled_times(<span class="kw-2">&amp;</span>s, <span class="number">0</span>, <span class="number">600</span>).unwrap();
<a href=#494 id=494 data-nosnippet>494</a> <span class="macro">assert_eq!</span>(due.scheduled_times_secs, <span class="macro">vec!</span>[<span class="number">540</span>, <span class="number">600</span>]);
<a href=#495 id=495 data-nosnippet>495</a> <span class="macro">assert_eq!</span>(due.skipped, <span class="number">8</span>);
<a href=#496 id=496 data-nosnippet>496</a> }
<a href=#497 id=497 data-nosnippet>497</a>
<a href=#498 id=498 data-nosnippet>498</a> <span class="attr">#[test]
<a href=#499 id=499 data-nosnippet>499</a> </span><span class="kw">fn </span>bounded_catch_up_can_skip_all_due_scheduled_times() {
<a href=#500 id=500 data-nosnippet>500</a> <span class="kw">let </span>s = schedule(<span class="number">0</span>, FlowMissedTickPolicy::BoundedCatchUp, <span class="number">3</span>, <span class="number">30</span>);
<a href=#501 id=501 data-nosnippet>501</a> <span class="kw">let </span>due = select_due_scheduled_times(<span class="kw-2">&amp;</span>s, <span class="number">0</span>, <span class="number">100</span>).unwrap();
<a href=#502 id=502 data-nosnippet>502</a> <span class="macro">assert!</span>(due.scheduled_times_secs.is_empty());
<a href=#503 id=503 data-nosnippet>503</a> <span class="macro">assert_eq!</span>(due.skipped, <span class="number">1</span>);
<a href=#504 id=504 data-nosnippet>504</a> }
<a href=#505 id=505 data-nosnippet>505</a>
<a href=#506 id=506 data-nosnippet>506</a> <span class="attr">#[test]
<a href=#507 id=507 data-nosnippet>507</a> </span><span class="kw">fn </span>skip_policy_keeps_only_latest_due_scheduled_time() {
<a href=#508 id=508 data-nosnippet>508</a> <span class="kw">let </span>s = schedule(<span class="number">0</span>, FlowMissedTickPolicy::Skip, <span class="number">5</span>, <span class="number">3600</span>);
<a href=#509 id=509 data-nosnippet>509</a> <span class="kw">let </span>due = select_due_scheduled_times(<span class="kw-2">&amp;</span>s, <span class="number">0</span>, <span class="number">300</span>).unwrap();
<a href=#510 id=510 data-nosnippet>510</a> <span class="macro">assert_eq!</span>(due.scheduled_times_secs, <span class="macro">vec!</span>[<span class="number">300</span>]);
<a href=#511 id=511 data-nosnippet>511</a> <span class="macro">assert_eq!</span>(due.skipped, <span class="number">4</span>);
<a href=#512 id=512 data-nosnippet>512</a> }
<a href=#513 id=513 data-nosnippet>513</a>
<a href=#514 id=514 data-nosnippet>514</a> <span class="attr">#[test]
<a href=#515 id=515 data-nosnippet>515</a> </span><span class="kw">fn </span>huge_missed_gap_allocates_only_kept_scheduled_times() {
<a href=#516 id=516 data-nosnippet>516</a> <span class="kw">let </span>s = schedule(<span class="number">0</span>, FlowMissedTickPolicy::BoundedCatchUp, <span class="number">5</span>, <span class="number">3600</span>);
<a href=#517 id=517 data-nosnippet>517</a> <span class="kw">let </span>due = select_due_scheduled_times(<span class="kw-2">&amp;</span>s, <span class="number">0</span>, <span class="number">86400</span>).unwrap();
<a href=#518 id=518 data-nosnippet>518</a> <span class="macro">assert_eq!</span>(
<a href=#519 id=519 data-nosnippet>519</a> due.scheduled_times_secs,
<a href=#520 id=520 data-nosnippet>520</a> <span class="macro">vec!</span>[<span class="number">86160</span>, <span class="number">86220</span>, <span class="number">86280</span>, <span class="number">86340</span>, <span class="number">86400</span>]
<a href=#521 id=521 data-nosnippet>521</a> );
<a href=#522 id=522 data-nosnippet>522</a> <span class="macro">assert_eq!</span>(due.skipped, <span class="number">1435</span>);
<a href=#523 id=523 data-nosnippet>523</a> }
<a href=#524 id=524 data-nosnippet>524</a>}
</code></pre></div></section></main></body></html>
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,554 @@
<!DOCTYPE html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1.0"><meta name="generator" content="rustdoc"><meta name="description" content="Source of the Rust file `src/flow/src/batching_mode/table_creator.rs`."><title>table_creator.rs - source</title><script>if(window.location.protocol!=="file:")document.head.insertAdjacentHTML("beforeend","SourceSerif4-Regular-6b053e98.ttf.woff2,FiraSans-Italic-81dc35de.woff2,FiraSans-Regular-0fe48ade.woff2,FiraSans-MediumItalic-ccf7e434.woff2,FiraSans-Medium-e1aa3f0a.woff2,SourceCodePro-Regular-8badfe75.ttf.woff2,SourceCodePro-Semibold-aa29a496.ttf.woff2".split(",").map(f=>`<link rel="preload" as="font" type="font/woff2"href="../../../static.files/${f}">`).join(""))</script><link rel="stylesheet" href="../../../static.files/normalize-9960930a.css"><link rel="stylesheet" href="../../../static.files/rustdoc-17e0aaed.css"><meta name="rustdoc-vars" data-root-path="../../../" data-static-root-path="../../../static.files/" data-current-crate="flow" data-themes="" data-resource-suffix="" data-rustdoc-version="1.96.1 (31fca3adb 2026-06-26)" data-channel="1.96.1" data-search-js="search-b5634cc7.js" data-stringdex-js="stringdex-2da4960a.js" data-settings-js="settings-170eb4bf.js" ><script src="../../../static.files/storage-41dd4d93.js"></script><script defer src="../../../static.files/src-script-813739b1.js"></script><script defer src="../../../src-files.js"></script><script defer src="../../../static.files/main-5013f961.js"></script><noscript><link rel="stylesheet" href="../../../static.files/noscript-f7c3ffd8.css"></noscript><link rel="alternate icon" type="image/png" href="../../../static.files/favicon-32x32-eab170b8.png"><link rel="icon" type="image/svg+xml" href="../../../static.files/favicon-044be391.svg"></head><body class="rustdoc src"><a class="skip-main-content" href="#main-content">Skip to main content</a><!--[if lte IE 11]><div class="warning">This old browser is unsupported and will most likely display funky things.</div><![endif]--><nav class="sidebar"><div class="src-sidebar-title"><h2>Files</h2></div></nav><div class="sidebar-resizer" title="Drag to resize sidebar"></div><main><section id="main-content" class="content" tabindex="-1"><div class="main-heading"><h1><div class="sub-heading">flow/batching_mode/</div>table_creator.rs</h1><rustdoc-toolbar></rustdoc-toolbar></div><div class="example-wrap digits-3"><pre class="rust"><code><a href=#1 id=1 data-nosnippet>1</a><span class="comment">// Copyright 2023 Greptime Team
<a href=#2 id=2 data-nosnippet>2</a>//
<a href=#3 id=3 data-nosnippet>3</a>// Licensed under the Apache License, Version 2.0 (the "License");
<a href=#4 id=4 data-nosnippet>4</a>// you may not use this file except in compliance with the License.
<a href=#5 id=5 data-nosnippet>5</a>// You may obtain a copy of the License at
<a href=#6 id=6 data-nosnippet>6</a>//
<a href=#7 id=7 data-nosnippet>7</a>// http://www.apache.org/licenses/LICENSE-2.0
<a href=#8 id=8 data-nosnippet>8</a>//
<a href=#9 id=9 data-nosnippet>9</a>// Unless required by applicable law or agreed to in writing, software
<a href=#10 id=10 data-nosnippet>10</a>// distributed under the License is distributed on an "AS IS" BASIS,
<a href=#11 id=11 data-nosnippet>11</a>// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
<a href=#12 id=12 data-nosnippet>12</a>// See the License for the specific language governing permissions and
<a href=#13 id=13 data-nosnippet>13</a>// limitations under the License.
<a href=#14 id=14 data-nosnippet>14</a>
<a href=#15 id=15 data-nosnippet>15</a></span><span class="kw">use </span>api::v1::CreateTableExpr;
<a href=#16 id=16 data-nosnippet>16</a><span class="kw">use </span>common_recordbatch::map_dictionary_to_values_data_type;
<a href=#17 id=17 data-nosnippet>17</a><span class="kw">use </span>datafusion_common::tree_node::TreeNode;
<a href=#18 id=18 data-nosnippet>18</a><span class="kw">use </span>datafusion_expr::LogicalPlan;
<a href=#19 id=19 data-nosnippet>19</a><span class="kw">use </span>datatypes::prelude::ConcreteDataType;
<a href=#20 id=20 data-nosnippet>20</a><span class="kw">use </span>datatypes::schema::ColumnSchema;
<a href=#21 id=21 data-nosnippet>21</a><span class="kw">use </span>operator::expr_helper::column_schemas_to_defs;
<a href=#22 id=22 data-nosnippet>22</a><span class="kw">use </span>snafu::ResultExt;
<a href=#23 id=23 data-nosnippet>23</a>
<a href=#24 id=24 data-nosnippet>24</a><span class="kw">use </span><span class="kw">crate</span>::Error;
<a href=#25 id=25 data-nosnippet>25</a><span class="kw">use </span><span class="kw">crate</span>::adapter::{AUTO_CREATED_PLACEHOLDER_TS_COL, AUTO_CREATED_UPDATE_AT_TS_COL};
<a href=#26 id=26 data-nosnippet>26</a><span class="kw">use </span><span class="kw">crate</span>::batching_mode::utils::FindGroupByFinalName;
<a href=#27 id=27 data-nosnippet>27</a><span class="kw">use </span><span class="kw">crate</span>::error::{ConvertColumnSchemaSnafu, DatafusionSnafu};
<a href=#28 id=28 data-nosnippet>28</a>
<a href=#29 id=29 data-nosnippet>29</a><span class="attr">#[derive(Debug, Clone, PartialEq, Eq)]
<a href=#30 id=30 data-nosnippet>30</a></span><span class="kw">pub enum </span>QueryType {
<a href=#31 id=31 data-nosnippet>31</a> <span class="doccomment">/// query is a tql query
<a href=#32 id=32 data-nosnippet>32</a> </span>Tql,
<a href=#33 id=33 data-nosnippet>33</a> <span class="doccomment">/// query is a sql query
<a href=#34 id=34 data-nosnippet>34</a> </span>Sql,
<a href=#35 id=35 data-nosnippet>35</a>}
<a href=#36 id=36 data-nosnippet>36</a>
<a href=#37 id=37 data-nosnippet>37</a><span class="comment">// auto created table have a auto added column `update_at`, and optional have a `AUTO_CREATED_PLACEHOLDER_TS_COL` column for time index placeholder if no timestamp column is specified
<a href=#38 id=38 data-nosnippet>38</a>// TODO(discord9): for now no default value is set for auto added column for compatibility reason with streaming mode, but this might change in favor of simpler code?
<a href=#39 id=39 data-nosnippet>39</a></span><span class="kw">pub</span>(<span class="kw">super</span>) <span class="kw">fn </span>create_table_with_expr(
<a href=#40 id=40 data-nosnippet>40</a> plan: <span class="kw-2">&amp;</span>LogicalPlan,
<a href=#41 id=41 data-nosnippet>41</a> sink_table_name: <span class="kw-2">&amp;</span>[String; <span class="number">3</span>],
<a href=#42 id=42 data-nosnippet>42</a> query_type: <span class="kw-2">&amp;</span>QueryType,
<a href=#43 id=43 data-nosnippet>43</a>) -&gt; <span class="prelude-ty">Result</span>&lt;CreateTableExpr, Error&gt; {
<a href=#44 id=44 data-nosnippet>44</a> <span class="kw">let </span>table_def = <span class="kw">match </span>query_type {
<a href=#45 id=45 data-nosnippet>45</a> <span class="kw-2">&amp;</span>QueryType::Sql =&gt; {
<a href=#46 id=46 data-nosnippet>46</a> <span class="kw">if let </span><span class="prelude-val">Some</span>(def) = build_pk_from_aggr(plan)<span class="question-mark">? </span>{
<a href=#47 id=47 data-nosnippet>47</a> def
<a href=#48 id=48 data-nosnippet>48</a> } <span class="kw">else </span>{
<a href=#49 id=49 data-nosnippet>49</a> build_by_sql_schema(plan)<span class="question-mark">?
<a href=#50 id=50 data-nosnippet>50</a> </span>}
<a href=#51 id=51 data-nosnippet>51</a> }
<a href=#52 id=52 data-nosnippet>52</a> QueryType::Tql =&gt; {
<a href=#53 id=53 data-nosnippet>53</a> <span class="comment">// first try build from aggr, then from tql schema because tql query might not have aggr node
<a href=#54 id=54 data-nosnippet>54</a> </span><span class="kw">if let </span><span class="prelude-val">Some</span>(table_def) = build_pk_from_aggr(plan)<span class="question-mark">? </span>{
<a href=#55 id=55 data-nosnippet>55</a> table_def
<a href=#56 id=56 data-nosnippet>56</a> } <span class="kw">else </span>{
<a href=#57 id=57 data-nosnippet>57</a> build_by_tql_schema(plan)<span class="question-mark">?
<a href=#58 id=58 data-nosnippet>58</a> </span>}
<a href=#59 id=59 data-nosnippet>59</a> }
<a href=#60 id=60 data-nosnippet>60</a> };
<a href=#61 id=61 data-nosnippet>61</a> <span class="kw">let </span>first_time_stamp = table_def.ts_col;
<a href=#62 id=62 data-nosnippet>62</a> <span class="kw">let </span>primary_keys = table_def.pks;
<a href=#63 id=63 data-nosnippet>63</a>
<a href=#64 id=64 data-nosnippet>64</a> <span class="kw">let </span><span class="kw-2">mut </span>column_schemas = Vec::new();
<a href=#65 id=65 data-nosnippet>65</a> <span class="kw">for </span>field <span class="kw">in </span>plan.schema().fields() {
<a href=#66 id=66 data-nosnippet>66</a> <span class="kw">let </span>name = field.name();
<a href=#67 id=67 data-nosnippet>67</a> <span class="kw">let </span>ty = map_dictionary_to_values_data_type(<span class="kw-2">&amp;</span>ConcreteDataType::from_arrow_type(
<a href=#68 id=68 data-nosnippet>68</a> field.data_type(),
<a href=#69 id=69 data-nosnippet>69</a> ));
<a href=#70 id=70 data-nosnippet>70</a> <span class="kw">let </span>col_schema = <span class="kw">if </span>first_time_stamp == <span class="prelude-val">Some</span>(name.clone()) {
<a href=#71 id=71 data-nosnippet>71</a> ColumnSchema::new(name, ty, <span class="bool-val">false</span>).with_time_index(<span class="bool-val">true</span>)
<a href=#72 id=72 data-nosnippet>72</a> } <span class="kw">else </span>{
<a href=#73 id=73 data-nosnippet>73</a> ColumnSchema::new(name, ty, <span class="bool-val">true</span>)
<a href=#74 id=74 data-nosnippet>74</a> };
<a href=#75 id=75 data-nosnippet>75</a>
<a href=#76 id=76 data-nosnippet>76</a> <span class="kw">match </span>query_type {
<a href=#77 id=77 data-nosnippet>77</a> QueryType::Sql =&gt; {
<a href=#78 id=78 data-nosnippet>78</a> column_schemas.push(col_schema);
<a href=#79 id=79 data-nosnippet>79</a> }
<a href=#80 id=80 data-nosnippet>80</a> QueryType::Tql =&gt; {
<a href=#81 id=81 data-nosnippet>81</a> <span class="comment">// if is val column, need to rename as val DOUBLE NULL
<a href=#82 id=82 data-nosnippet>82</a> // if is tag column, need to cast type as STRING NULL
<a href=#83 id=83 data-nosnippet>83</a> </span><span class="kw">let </span>is_tag_column = primary_keys.contains(name);
<a href=#84 id=84 data-nosnippet>84</a> <span class="kw">let </span>is_val_column = !is_tag_column &amp;&amp; first_time_stamp.as_ref() != <span class="prelude-val">Some</span>(name);
<a href=#85 id=85 data-nosnippet>85</a> <span class="kw">if </span>is_val_column {
<a href=#86 id=86 data-nosnippet>86</a> <span class="kw">let </span>col_schema =
<a href=#87 id=87 data-nosnippet>87</a> ColumnSchema::new(name, ConcreteDataType::float64_datatype(), <span class="bool-val">true</span>);
<a href=#88 id=88 data-nosnippet>88</a> column_schemas.push(col_schema);
<a href=#89 id=89 data-nosnippet>89</a> } <span class="kw">else if </span>is_tag_column {
<a href=#90 id=90 data-nosnippet>90</a> <span class="kw">let </span>col_schema =
<a href=#91 id=91 data-nosnippet>91</a> ColumnSchema::new(name, ConcreteDataType::string_datatype(), <span class="bool-val">true</span>);
<a href=#92 id=92 data-nosnippet>92</a> column_schemas.push(col_schema);
<a href=#93 id=93 data-nosnippet>93</a> } <span class="kw">else </span>{
<a href=#94 id=94 data-nosnippet>94</a> <span class="comment">// time index column
<a href=#95 id=95 data-nosnippet>95</a> </span>column_schemas.push(col_schema);
<a href=#96 id=96 data-nosnippet>96</a> }
<a href=#97 id=97 data-nosnippet>97</a> }
<a href=#98 id=98 data-nosnippet>98</a> }
<a href=#99 id=99 data-nosnippet>99</a> }
<a href=#100 id=100 data-nosnippet>100</a>
<a href=#101 id=101 data-nosnippet>101</a> <span class="kw">if </span>query_type == <span class="kw-2">&amp;</span>QueryType::Sql {
<a href=#102 id=102 data-nosnippet>102</a> <span class="kw">let </span>update_at_schema = ColumnSchema::new(
<a href=#103 id=103 data-nosnippet>103</a> AUTO_CREATED_UPDATE_AT_TS_COL,
<a href=#104 id=104 data-nosnippet>104</a> ConcreteDataType::timestamp_millisecond_datatype(),
<a href=#105 id=105 data-nosnippet>105</a> <span class="bool-val">true</span>,
<a href=#106 id=106 data-nosnippet>106</a> );
<a href=#107 id=107 data-nosnippet>107</a> column_schemas.push(update_at_schema);
<a href=#108 id=108 data-nosnippet>108</a> }
<a href=#109 id=109 data-nosnippet>109</a>
<a href=#110 id=110 data-nosnippet>110</a> <span class="kw">let </span>time_index = <span class="kw">if let </span><span class="prelude-val">Some</span>(time_index) = first_time_stamp {
<a href=#111 id=111 data-nosnippet>111</a> time_index
<a href=#112 id=112 data-nosnippet>112</a> } <span class="kw">else </span>{
<a href=#113 id=113 data-nosnippet>113</a> column_schemas.push(
<a href=#114 id=114 data-nosnippet>114</a> ColumnSchema::new(
<a href=#115 id=115 data-nosnippet>115</a> AUTO_CREATED_PLACEHOLDER_TS_COL,
<a href=#116 id=116 data-nosnippet>116</a> ConcreteDataType::timestamp_millisecond_datatype(),
<a href=#117 id=117 data-nosnippet>117</a> <span class="bool-val">false</span>,
<a href=#118 id=118 data-nosnippet>118</a> )
<a href=#119 id=119 data-nosnippet>119</a> .with_time_index(<span class="bool-val">true</span>),
<a href=#120 id=120 data-nosnippet>120</a> );
<a href=#121 id=121 data-nosnippet>121</a> AUTO_CREATED_PLACEHOLDER_TS_COL.to_string()
<a href=#122 id=122 data-nosnippet>122</a> };
<a href=#123 id=123 data-nosnippet>123</a>
<a href=#124 id=124 data-nosnippet>124</a> <span class="kw">let </span>column_defs =
<a href=#125 id=125 data-nosnippet>125</a> column_schemas_to_defs(column_schemas, <span class="kw-2">&amp;</span>primary_keys).context(ConvertColumnSchemaSnafu)<span class="question-mark">?</span>;
<a href=#126 id=126 data-nosnippet>126</a> <span class="prelude-val">Ok</span>(CreateTableExpr {
<a href=#127 id=127 data-nosnippet>127</a> catalog_name: sink_table_name[<span class="number">0</span>].clone(),
<a href=#128 id=128 data-nosnippet>128</a> schema_name: sink_table_name[<span class="number">1</span>].clone(),
<a href=#129 id=129 data-nosnippet>129</a> table_name: sink_table_name[<span class="number">2</span>].clone(),
<a href=#130 id=130 data-nosnippet>130</a> desc: <span class="string">"Auto created table by flow engine"</span>.to_string(),
<a href=#131 id=131 data-nosnippet>131</a> column_defs,
<a href=#132 id=132 data-nosnippet>132</a> time_index,
<a href=#133 id=133 data-nosnippet>133</a> primary_keys,
<a href=#134 id=134 data-nosnippet>134</a> create_if_not_exists: <span class="bool-val">true</span>,
<a href=#135 id=135 data-nosnippet>135</a> table_options: Default::default(),
<a href=#136 id=136 data-nosnippet>136</a> table_id: <span class="prelude-val">None</span>,
<a href=#137 id=137 data-nosnippet>137</a> engine: <span class="string">"mito"</span>.to_string(),
<a href=#138 id=138 data-nosnippet>138</a> })
<a href=#139 id=139 data-nosnippet>139</a>}
<a href=#140 id=140 data-nosnippet>140</a>
<a href=#141 id=141 data-nosnippet>141</a><span class="doccomment">/// simply build by schema, return first timestamp column and no primary key
<a href=#142 id=142 data-nosnippet>142</a></span><span class="kw">fn </span>build_by_sql_schema(plan: <span class="kw-2">&amp;</span>LogicalPlan) -&gt; <span class="prelude-ty">Result</span>&lt;TableDef, Error&gt; {
<a href=#143 id=143 data-nosnippet>143</a> <span class="kw">let </span>first_time_stamp = plan.schema().fields().iter().find_map(|f| {
<a href=#144 id=144 data-nosnippet>144</a> <span class="kw">if </span>ConcreteDataType::from_arrow_type(f.data_type()).is_timestamp() {
<a href=#145 id=145 data-nosnippet>145</a> <span class="prelude-val">Some</span>(f.name().clone())
<a href=#146 id=146 data-nosnippet>146</a> } <span class="kw">else </span>{
<a href=#147 id=147 data-nosnippet>147</a> <span class="prelude-val">None
<a href=#148 id=148 data-nosnippet>148</a> </span>}
<a href=#149 id=149 data-nosnippet>149</a> });
<a href=#150 id=150 data-nosnippet>150</a> <span class="prelude-val">Ok</span>(TableDef {
<a href=#151 id=151 data-nosnippet>151</a> ts_col: first_time_stamp,
<a href=#152 id=152 data-nosnippet>152</a> pks: <span class="macro">vec!</span>[],
<a href=#153 id=153 data-nosnippet>153</a> })
<a href=#154 id=154 data-nosnippet>154</a>}
<a href=#155 id=155 data-nosnippet>155</a>
<a href=#156 id=156 data-nosnippet>156</a><span class="doccomment">/// Return first timestamp column found in output schema and all string columns
<a href=#157 id=157 data-nosnippet>157</a></span><span class="kw">fn </span>build_by_tql_schema(plan: <span class="kw-2">&amp;</span>LogicalPlan) -&gt; <span class="prelude-ty">Result</span>&lt;TableDef, Error&gt; {
<a href=#158 id=158 data-nosnippet>158</a> <span class="kw">let </span>first_time_stamp = plan.schema().fields().iter().find_map(|f| {
<a href=#159 id=159 data-nosnippet>159</a> <span class="kw">if </span>ConcreteDataType::from_arrow_type(f.data_type()).is_timestamp() {
<a href=#160 id=160 data-nosnippet>160</a> <span class="prelude-val">Some</span>(f.name().clone())
<a href=#161 id=161 data-nosnippet>161</a> } <span class="kw">else </span>{
<a href=#162 id=162 data-nosnippet>162</a> <span class="prelude-val">None
<a href=#163 id=163 data-nosnippet>163</a> </span>}
<a href=#164 id=164 data-nosnippet>164</a> });
<a href=#165 id=165 data-nosnippet>165</a> <span class="kw">let </span>string_columns = plan
<a href=#166 id=166 data-nosnippet>166</a> .schema()
<a href=#167 id=167 data-nosnippet>167</a> .fields()
<a href=#168 id=168 data-nosnippet>168</a> .iter()
<a href=#169 id=169 data-nosnippet>169</a> .filter_map(|f| {
<a href=#170 id=170 data-nosnippet>170</a> <span class="kw">if </span>map_dictionary_to_values_data_type(<span class="kw-2">&amp;</span>ConcreteDataType::from_arrow_type(f.data_type()))
<a href=#171 id=171 data-nosnippet>171</a> .is_string()
<a href=#172 id=172 data-nosnippet>172</a> {
<a href=#173 id=173 data-nosnippet>173</a> <span class="prelude-val">Some</span>(f.name().clone())
<a href=#174 id=174 data-nosnippet>174</a> } <span class="kw">else </span>{
<a href=#175 id=175 data-nosnippet>175</a> <span class="prelude-val">None
<a href=#176 id=176 data-nosnippet>176</a> </span>}
<a href=#177 id=177 data-nosnippet>177</a> })
<a href=#178 id=178 data-nosnippet>178</a> .collect::&lt;Vec&lt;<span class="kw">_</span>&gt;&gt;();
<a href=#179 id=179 data-nosnippet>179</a>
<a href=#180 id=180 data-nosnippet>180</a> <span class="prelude-val">Ok</span>(TableDef {
<a href=#181 id=181 data-nosnippet>181</a> ts_col: first_time_stamp,
<a href=#182 id=182 data-nosnippet>182</a> pks: string_columns,
<a href=#183 id=183 data-nosnippet>183</a> })
<a href=#184 id=184 data-nosnippet>184</a>}
<a href=#185 id=185 data-nosnippet>185</a>
<a href=#186 id=186 data-nosnippet>186</a><span class="kw">struct </span>TableDef {
<a href=#187 id=187 data-nosnippet>187</a> ts_col: <span class="prelude-ty">Option</span>&lt;String&gt;,
<a href=#188 id=188 data-nosnippet>188</a> pks: Vec&lt;String&gt;,
<a href=#189 id=189 data-nosnippet>189</a>}
<a href=#190 id=190 data-nosnippet>190</a>
<a href=#191 id=191 data-nosnippet>191</a><span class="doccomment">/// Return first timestamp column which is in group by clause and other columns which are also in group by clause
<a href=#192 id=192 data-nosnippet>192</a>///
<a href=#193 id=193 data-nosnippet>193</a>/// # Returns
<a href=#194 id=194 data-nosnippet>194</a>///
<a href=#195 id=195 data-nosnippet>195</a>/// * `Option&lt;String&gt;` - first timestamp column which is in group by clause
<a href=#196 id=196 data-nosnippet>196</a>/// * `Vec&lt;String&gt;` - other columns which are also in group by clause
<a href=#197 id=197 data-nosnippet>197</a>///
<a href=#198 id=198 data-nosnippet>198</a>/// if no aggregation found, return None
<a href=#199 id=199 data-nosnippet>199</a></span><span class="kw">fn </span>build_pk_from_aggr(plan: <span class="kw-2">&amp;</span>LogicalPlan) -&gt; <span class="prelude-ty">Result</span>&lt;<span class="prelude-ty">Option</span>&lt;TableDef&gt;, Error&gt; {
<a href=#200 id=200 data-nosnippet>200</a> <span class="kw">let </span>fields = plan.schema().fields();
<a href=#201 id=201 data-nosnippet>201</a> <span class="kw">let </span><span class="kw-2">mut </span>pk_names = FindGroupByFinalName::default();
<a href=#202 id=202 data-nosnippet>202</a>
<a href=#203 id=203 data-nosnippet>203</a> plan.visit(<span class="kw-2">&amp;mut </span>pk_names)
<a href=#204 id=204 data-nosnippet>204</a> .with_context(|<span class="kw">_</span>| DatafusionSnafu {
<a href=#205 id=205 data-nosnippet>205</a> context: <span class="macro">format!</span>(<span class="string">"Can't find aggr expr in plan {plan:?}"</span>),
<a href=#206 id=206 data-nosnippet>206</a> })<span class="question-mark">?</span>;
<a href=#207 id=207 data-nosnippet>207</a>
<a href=#208 id=208 data-nosnippet>208</a> <span class="comment">// if no group by clause, return empty with first timestamp column found in output schema
<a href=#209 id=209 data-nosnippet>209</a> </span><span class="kw">let </span><span class="prelude-val">Some</span>(pk_final_names) = pk_names.get_group_expr_names() <span class="kw">else </span>{
<a href=#210 id=210 data-nosnippet>210</a> <span class="kw">return </span><span class="prelude-val">Ok</span>(<span class="prelude-val">None</span>);
<a href=#211 id=211 data-nosnippet>211</a> };
<a href=#212 id=212 data-nosnippet>212</a> <span class="kw">if </span>pk_final_names.is_empty() {
<a href=#213 id=213 data-nosnippet>213</a> <span class="kw">let </span>first_ts_col = fields
<a href=#214 id=214 data-nosnippet>214</a> .iter()
<a href=#215 id=215 data-nosnippet>215</a> .find(|f| ConcreteDataType::from_arrow_type(f.data_type()).is_timestamp())
<a href=#216 id=216 data-nosnippet>216</a> .map(|f| f.name().clone());
<a href=#217 id=217 data-nosnippet>217</a> <span class="kw">return </span><span class="prelude-val">Ok</span>(<span class="prelude-val">Some</span>(TableDef {
<a href=#218 id=218 data-nosnippet>218</a> ts_col: first_ts_col,
<a href=#219 id=219 data-nosnippet>219</a> pks: <span class="macro">vec!</span>[],
<a href=#220 id=220 data-nosnippet>220</a> }));
<a href=#221 id=221 data-nosnippet>221</a> }
<a href=#222 id=222 data-nosnippet>222</a>
<a href=#223 id=223 data-nosnippet>223</a> <span class="kw">let </span>all_pk_cols: Vec&lt;<span class="kw">_</span>&gt; = fields
<a href=#224 id=224 data-nosnippet>224</a> .iter()
<a href=#225 id=225 data-nosnippet>225</a> .filter(|f| pk_final_names.contains(f.name()))
<a href=#226 id=226 data-nosnippet>226</a> .map(|f| f.name().clone())
<a href=#227 id=227 data-nosnippet>227</a> .collect();
<a href=#228 id=228 data-nosnippet>228</a> <span class="comment">// Auto-created tables use the first timestamp column in the group-by keys
<a href=#229 id=229 data-nosnippet>229</a> // as the time index. It is possible that timestamp columns appear only as
<a href=#230 id=230 data-nosnippet>230</a> // aggregate outputs (for example `max(ts)`) and are not group-by keys; in
<a href=#231 id=231 data-nosnippet>231</a> // that case `first_time_stamp` stays `None` and the caller falls back to a
<a href=#232 id=232 data-nosnippet>232</a> // placeholder time index column.
<a href=#233 id=233 data-nosnippet>233</a> </span><span class="kw">let </span>first_time_stamp = fields
<a href=#234 id=234 data-nosnippet>234</a> .iter()
<a href=#235 id=235 data-nosnippet>235</a> .find(|f| {
<a href=#236 id=236 data-nosnippet>236</a> all_pk_cols.contains(<span class="kw-2">&amp;</span>f.name().clone())
<a href=#237 id=237 data-nosnippet>237</a> &amp;&amp; ConcreteDataType::from_arrow_type(f.data_type()).is_timestamp()
<a href=#238 id=238 data-nosnippet>238</a> })
<a href=#239 id=239 data-nosnippet>239</a> .map(|f| f.name().clone());
<a href=#240 id=240 data-nosnippet>240</a>
<a href=#241 id=241 data-nosnippet>241</a> <span class="kw">let </span>all_pk_cols: Vec&lt;<span class="kw">_</span>&gt; = all_pk_cols
<a href=#242 id=242 data-nosnippet>242</a> .into_iter()
<a href=#243 id=243 data-nosnippet>243</a> .filter(|col| first_time_stamp.as_ref() != <span class="prelude-val">Some</span>(col))
<a href=#244 id=244 data-nosnippet>244</a> .collect();
<a href=#245 id=245 data-nosnippet>245</a>
<a href=#246 id=246 data-nosnippet>246</a> <span class="prelude-val">Ok</span>(<span class="prelude-val">Some</span>(TableDef {
<a href=#247 id=247 data-nosnippet>247</a> ts_col: first_time_stamp,
<a href=#248 id=248 data-nosnippet>248</a> pks: all_pk_cols,
<a href=#249 id=249 data-nosnippet>249</a> }))
<a href=#250 id=250 data-nosnippet>250</a>}
<a href=#251 id=251 data-nosnippet>251</a>
<a href=#252 id=252 data-nosnippet>252</a><span class="attr">#[cfg(test)]
<a href=#253 id=253 data-nosnippet>253</a></span><span class="kw">mod </span>test {
<a href=#254 id=254 data-nosnippet>254</a> <span class="kw">use </span>std::sync::Arc;
<a href=#255 id=255 data-nosnippet>255</a>
<a href=#256 id=256 data-nosnippet>256</a> <span class="kw">use </span>api::v1::column_def::try_as_column_schema;
<a href=#257 id=257 data-nosnippet>257</a> <span class="kw">use </span>catalog::RegisterTableRequest;
<a href=#258 id=258 data-nosnippet>258</a> <span class="kw">use </span>catalog::memory::new_memory_catalog_manager;
<a href=#259 id=259 data-nosnippet>259</a> <span class="kw">use </span>common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME};
<a href=#260 id=260 data-nosnippet>260</a> <span class="kw">use </span>datafusion::arrow::datatypes::{
<a href=#261 id=261 data-nosnippet>261</a> DataType <span class="kw">as </span>ArrowDataType, Field, Schema <span class="kw">as </span>ArrowSchema, TimeUnit,
<a href=#262 id=262 data-nosnippet>262</a> };
<a href=#263 id=263 data-nosnippet>263</a> <span class="kw">use </span>datafusion_common::DFSchema;
<a href=#264 id=264 data-nosnippet>264</a> <span class="kw">use </span>datafusion_expr::logical_plan::EmptyRelation;
<a href=#265 id=265 data-nosnippet>265</a> <span class="kw">use </span>datatypes::prelude::ConcreteDataType;
<a href=#266 id=266 data-nosnippet>266</a> <span class="kw">use </span>datatypes::schema::{ColumnSchema, Schema};
<a href=#267 id=267 data-nosnippet>267</a> <span class="kw">use </span>pretty_assertions::assert_eq;
<a href=#268 id=268 data-nosnippet>268</a> <span class="kw">use </span>query::options::QueryOptions;
<a href=#269 id=269 data-nosnippet>269</a> <span class="kw">use </span>query::{QueryEngineFactory, QueryEngineRef};
<a href=#270 id=270 data-nosnippet>270</a> <span class="kw">use </span>session::context::QueryContext;
<a href=#271 id=271 data-nosnippet>271</a> <span class="kw">use </span>table::metadata::{TableInfoBuilder, TableMetaBuilder};
<a href=#272 id=272 data-nosnippet>272</a> <span class="kw">use </span>table::test_util::EmptyTable;
<a href=#273 id=273 data-nosnippet>273</a>
<a href=#274 id=274 data-nosnippet>274</a> <span class="kw">use </span>super::<span class="kw-2">*</span>;
<a href=#275 id=275 data-nosnippet>275</a> <span class="kw">use </span><span class="kw">crate</span>::adapter::{AUTO_CREATED_PLACEHOLDER_TS_COL, AUTO_CREATED_UPDATE_AT_TS_COL};
<a href=#276 id=276 data-nosnippet>276</a> <span class="kw">use </span><span class="kw">crate</span>::batching_mode::utils::sql_to_df_plan;
<a href=#277 id=277 data-nosnippet>277</a> <span class="kw">use </span><span class="kw">crate</span>::test_utils::create_test_query_engine;
<a href=#278 id=278 data-nosnippet>278</a>
<a href=#279 id=279 data-nosnippet>279</a> <span class="attr">#[test]
<a href=#280 id=280 data-nosnippet>280</a> </span><span class="kw">fn </span>test_tql_dictionary_string_is_label() {
<a href=#281 id=281 data-nosnippet>281</a> <span class="kw">let </span>arrow_schema = Arc::new(ArrowSchema::new(<span class="macro">vec!</span>[
<a href=#282 id=282 data-nosnippet>282</a> Field::new_dictionary(<span class="string">"host"</span>, ArrowDataType::UInt32, ArrowDataType::Utf8, <span class="bool-val">true</span>),
<a href=#283 id=283 data-nosnippet>283</a> Field::new(<span class="string">"value"</span>, ArrowDataType::Float64, <span class="bool-val">true</span>),
<a href=#284 id=284 data-nosnippet>284</a> Field::new(
<a href=#285 id=285 data-nosnippet>285</a> <span class="string">"ts"</span>,
<a href=#286 id=286 data-nosnippet>286</a> ArrowDataType::Timestamp(TimeUnit::Millisecond, <span class="prelude-val">None</span>),
<a href=#287 id=287 data-nosnippet>287</a> <span class="bool-val">false</span>,
<a href=#288 id=288 data-nosnippet>288</a> ),
<a href=#289 id=289 data-nosnippet>289</a> ]));
<a href=#290 id=290 data-nosnippet>290</a> <span class="kw">let </span>plan = LogicalPlan::EmptyRelation(EmptyRelation {
<a href=#291 id=291 data-nosnippet>291</a> produce_one_row: <span class="bool-val">false</span>,
<a href=#292 id=292 data-nosnippet>292</a> schema: Arc::new(DFSchema::try_from(arrow_schema).unwrap()),
<a href=#293 id=293 data-nosnippet>293</a> });
<a href=#294 id=294 data-nosnippet>294</a>
<a href=#295 id=295 data-nosnippet>295</a> <span class="kw">let </span>expr = create_table_with_expr(
<a href=#296 id=296 data-nosnippet>296</a> <span class="kw-2">&amp;</span>plan,
<a href=#297 id=297 data-nosnippet>297</a> <span class="kw-2">&amp;</span>[
<a href=#298 id=298 data-nosnippet>298</a> <span class="string">"greptime"</span>.to_string(),
<a href=#299 id=299 data-nosnippet>299</a> <span class="string">"public"</span>.to_string(),
<a href=#300 id=300 data-nosnippet>300</a> <span class="string">"sink"</span>.to_string(),
<a href=#301 id=301 data-nosnippet>301</a> ],
<a href=#302 id=302 data-nosnippet>302</a> <span class="kw-2">&amp;</span>QueryType::Tql,
<a href=#303 id=303 data-nosnippet>303</a> )
<a href=#304 id=304 data-nosnippet>304</a> .unwrap();
<a href=#305 id=305 data-nosnippet>305</a> <span class="kw">let </span>columns = expr
<a href=#306 id=306 data-nosnippet>306</a> .column_defs
<a href=#307 id=307 data-nosnippet>307</a> .iter()
<a href=#308 id=308 data-nosnippet>308</a> .map(|column| try_as_column_schema(column).unwrap())
<a href=#309 id=309 data-nosnippet>309</a> .collect::&lt;Vec&lt;<span class="kw">_</span>&gt;&gt;();
<a href=#310 id=310 data-nosnippet>310</a>
<a href=#311 id=311 data-nosnippet>311</a> <span class="macro">assert_eq!</span>(<span class="macro">vec!</span>[<span class="string">"host"</span>.to_string()], expr.primary_keys);
<a href=#312 id=312 data-nosnippet>312</a> <span class="macro">assert_eq!</span>(<span class="string">"ts"</span>, expr.time_index);
<a href=#313 id=313 data-nosnippet>313</a> <span class="macro">assert_eq!</span>(ConcreteDataType::string_datatype(), columns[<span class="number">0</span>].data_type);
<a href=#314 id=314 data-nosnippet>314</a> <span class="macro">assert_eq!</span>(ConcreteDataType::float64_datatype(), columns[<span class="number">1</span>].data_type);
<a href=#315 id=315 data-nosnippet>315</a> <span class="macro">assert!</span>(columns[<span class="number">2</span>].is_time_index());
<a href=#316 id=316 data-nosnippet>316</a> }
<a href=#317 id=317 data-nosnippet>317</a>
<a href=#318 id=318 data-nosnippet>318</a> <span class="doccomment">/// Creates a query engine holding a Prometheus shaped table `http_requests`: tags
<a href=#319 id=319 data-nosnippet>319</a> /// (`host`, `idc`), a single f64 value column (`val`) and a time index (`ts`), so that
<a href=#320 id=320 data-nosnippet>320</a> /// TQL queries can be planned against it.
<a href=#321 id=321 data-nosnippet>321</a> </span><span class="kw">fn </span>create_tql_test_query_engine() -&gt; QueryEngineRef {
<a href=#322 id=322 data-nosnippet>322</a> <span class="kw">let </span>catalog_list = new_memory_catalog_manager().unwrap();
<a href=#323 id=323 data-nosnippet>323</a> <span class="kw">let </span>table_meta = TableMetaBuilder::empty()
<a href=#324 id=324 data-nosnippet>324</a> .schema(Arc::new(Schema::new(<span class="macro">vec!</span>[
<a href=#325 id=325 data-nosnippet>325</a> ColumnSchema::new(
<a href=#326 id=326 data-nosnippet>326</a> <span class="string">"ts"</span>,
<a href=#327 id=327 data-nosnippet>327</a> ConcreteDataType::timestamp_millisecond_datatype(),
<a href=#328 id=328 data-nosnippet>328</a> <span class="bool-val">false</span>,
<a href=#329 id=329 data-nosnippet>329</a> )
<a href=#330 id=330 data-nosnippet>330</a> .with_time_index(<span class="bool-val">true</span>),
<a href=#331 id=331 data-nosnippet>331</a> ColumnSchema::new(<span class="string">"host"</span>, ConcreteDataType::string_datatype(), <span class="bool-val">true</span>),
<a href=#332 id=332 data-nosnippet>332</a> ColumnSchema::new(<span class="string">"idc"</span>, ConcreteDataType::string_datatype(), <span class="bool-val">true</span>),
<a href=#333 id=333 data-nosnippet>333</a> ColumnSchema::new(<span class="string">"val"</span>, ConcreteDataType::float64_datatype(), <span class="bool-val">true</span>),
<a href=#334 id=334 data-nosnippet>334</a> ])))
<a href=#335 id=335 data-nosnippet>335</a> <span class="comment">// `host` and `idc` are tags, `val` is the value column
<a href=#336 id=336 data-nosnippet>336</a> </span>.primary_key_indices(<span class="macro">vec!</span>[<span class="number">1</span>, <span class="number">2</span>])
<a href=#337 id=337 data-nosnippet>337</a> .value_indices(<span class="macro">vec!</span>[<span class="number">3</span>])
<a href=#338 id=338 data-nosnippet>338</a> .engine(<span class="string">"mito"</span>.to_string())
<a href=#339 id=339 data-nosnippet>339</a> .next_column_id(<span class="number">1026</span>)
<a href=#340 id=340 data-nosnippet>340</a> .build()
<a href=#341 id=341 data-nosnippet>341</a> .unwrap();
<a href=#342 id=342 data-nosnippet>342</a> <span class="kw">let </span>table_info = TableInfoBuilder::default()
<a href=#343 id=343 data-nosnippet>343</a> .name(<span class="string">"http_requests"</span>.to_string())
<a href=#344 id=344 data-nosnippet>344</a> .meta(table_meta)
<a href=#345 id=345 data-nosnippet>345</a> .build()
<a href=#346 id=346 data-nosnippet>346</a> .unwrap();
<a href=#347 id=347 data-nosnippet>347</a> <span class="macro">assert!</span>(
<a href=#348 id=348 data-nosnippet>348</a> catalog_list
<a href=#349 id=349 data-nosnippet>349</a> .register_table_sync(RegisterTableRequest {
<a href=#350 id=350 data-nosnippet>350</a> catalog: DEFAULT_CATALOG_NAME.to_string(),
<a href=#351 id=351 data-nosnippet>351</a> schema: DEFAULT_SCHEMA_NAME.to_string(),
<a href=#352 id=352 data-nosnippet>352</a> table_name: <span class="string">"http_requests"</span>.to_string(),
<a href=#353 id=353 data-nosnippet>353</a> table_id: <span class="number">1026</span>,
<a href=#354 id=354 data-nosnippet>354</a> table: EmptyTable::from_table_info(<span class="kw-2">&amp;</span>table_info),
<a href=#355 id=355 data-nosnippet>355</a> })
<a href=#356 id=356 data-nosnippet>356</a> .is_ok()
<a href=#357 id=357 data-nosnippet>357</a> );
<a href=#358 id=358 data-nosnippet>358</a>
<a href=#359 id=359 data-nosnippet>359</a> QueryEngineFactory::new(
<a href=#360 id=360 data-nosnippet>360</a> catalog_list,
<a href=#361 id=361 data-nosnippet>361</a> <span class="prelude-val">None</span>,
<a href=#362 id=362 data-nosnippet>362</a> <span class="prelude-val">None</span>,
<a href=#363 id=363 data-nosnippet>363</a> <span class="prelude-val">None</span>,
<a href=#364 id=364 data-nosnippet>364</a> <span class="prelude-val">None</span>,
<a href=#365 id=365 data-nosnippet>365</a> <span class="bool-val">false</span>,
<a href=#366 id=366 data-nosnippet>366</a> QueryOptions::default(),
<a href=#367 id=367 data-nosnippet>367</a> )
<a href=#368 id=368 data-nosnippet>368</a> .query_engine()
<a href=#369 id=369 data-nosnippet>369</a> }
<a href=#370 id=370 data-nosnippet>370</a>
<a href=#371 id=371 data-nosnippet>371</a> <span class="doccomment">/// A TQL `count_values` flow must keep the generated label as a primary key of the auto
<a href=#372 id=372 data-nosnippet>372</a> /// created sink table. `count_values("status_code", http_requests)` groups by the sample
<a href=#373 id=373 data-nosnippet>373</a> /// value column and projects the label as a unary scalar expression of that column
<a href=#374 id=374 data-nosnippet>374</a> /// (`prom_float_to_string(val) AS status_code`), so the label only derives from a group by
<a href=#375 id=375 data-nosnippet>375</a> /// column instead of referencing it directly.
<a href=#376 id=376 data-nosnippet>376</a> ///
<a href=#377 id=377 data-nosnippet>377</a> /// The plan is built with the same path a flow task uses (including the DataFusion
<a href=#378 id=378 data-nosnippet>378</a> /// optimizers, which may rewrite the shape of the alias), and the assertion covers both.
<a href=#379 id=379 data-nosnippet>379</a> </span><span class="attr">#[tokio::test]
<a href=#380 id=380 data-nosnippet>380</a> </span><span class="kw">async fn </span>test_tql_count_values_generated_label_is_primary_key() {
<a href=#381 id=381 data-nosnippet>381</a> <span class="kw">let </span>query_engine = create_tql_test_query_engine();
<a href=#382 id=382 data-nosnippet>382</a> <span class="kw">let </span>ctx = QueryContext::arc();
<a href=#383 id=383 data-nosnippet>383</a>
<a href=#384 id=384 data-nosnippet>384</a> <span class="kw">for </span>optimize <span class="kw">in </span>[<span class="bool-val">false</span>, <span class="bool-val">true</span>] {
<a href=#385 id=385 data-nosnippet>385</a> <span class="kw">let </span>plan = sql_to_df_plan(
<a href=#386 id=386 data-nosnippet>386</a> ctx.clone(),
<a href=#387 id=387 data-nosnippet>387</a> query_engine.clone(),
<a href=#388 id=388 data-nosnippet>388</a> <span class="string">r#"TQL EVAL (0, 15, '5s') count_values("status_code", http_requests)"#</span>,
<a href=#389 id=389 data-nosnippet>389</a> optimize,
<a href=#390 id=390 data-nosnippet>390</a> )
<a href=#391 id=391 data-nosnippet>391</a> .<span class="kw">await
<a href=#392 id=392 data-nosnippet>392</a> </span>.unwrap();
<a href=#393 id=393 data-nosnippet>393</a> <span class="kw">let </span>plan_display = plan.display_indent_schema().to_string();
<a href=#394 id=394 data-nosnippet>394</a> <span class="kw">let </span>expr = create_table_with_expr(
<a href=#395 id=395 data-nosnippet>395</a> <span class="kw-2">&amp;</span>plan,
<a href=#396 id=396 data-nosnippet>396</a> <span class="kw-2">&amp;</span>[
<a href=#397 id=397 data-nosnippet>397</a> <span class="string">"greptime"</span>.to_string(),
<a href=#398 id=398 data-nosnippet>398</a> <span class="string">"public"</span>.to_string(),
<a href=#399 id=399 data-nosnippet>399</a> <span class="string">"sink"</span>.to_string(),
<a href=#400 id=400 data-nosnippet>400</a> ],
<a href=#401 id=401 data-nosnippet>401</a> <span class="kw-2">&amp;</span>QueryType::Tql,
<a href=#402 id=402 data-nosnippet>402</a> )
<a href=#403 id=403 data-nosnippet>403</a> .unwrap();
<a href=#404 id=404 data-nosnippet>404</a> <span class="kw">let </span>columns = expr
<a href=#405 id=405 data-nosnippet>405</a> .column_defs
<a href=#406 id=406 data-nosnippet>406</a> .iter()
<a href=#407 id=407 data-nosnippet>407</a> .map(|column| try_as_column_schema(column).unwrap())
<a href=#408 id=408 data-nosnippet>408</a> .collect::&lt;Vec&lt;<span class="kw">_</span>&gt;&gt;();
<a href=#409 id=409 data-nosnippet>409</a>
<a href=#410 id=410 data-nosnippet>410</a> <span class="macro">assert_eq!</span>(
<a href=#411 id=411 data-nosnippet>411</a> <span class="macro">vec!</span>[<span class="string">"status_code"</span>.to_string()],
<a href=#412 id=412 data-nosnippet>412</a> expr.primary_keys,
<a href=#413 id=413 data-nosnippet>413</a> <span class="string">"optimize={optimize}, plan:\n{plan_display}"
<a href=#414 id=414 data-nosnippet>414</a> </span>);
<a href=#415 id=415 data-nosnippet>415</a> <span class="macro">assert_eq!</span>(
<a href=#416 id=416 data-nosnippet>416</a> <span class="string">"ts"</span>, expr.time_index,
<a href=#417 id=417 data-nosnippet>417</a> <span class="string">"optimize={optimize}, plan:\n{plan_display}"
<a href=#418 id=418 data-nosnippet>418</a> </span>);
<a href=#419 id=419 data-nosnippet>419</a> <span class="comment">// the aggregation output is a value column, the generated label is a tag column
<a href=#420 id=420 data-nosnippet>420</a> </span><span class="macro">assert_eq!</span>(
<a href=#421 id=421 data-nosnippet>421</a> <span class="string">"count(http_requests.val)"</span>, columns[<span class="number">0</span>].name,
<a href=#422 id=422 data-nosnippet>422</a> <span class="string">"optimize={optimize}, plan:\n{plan_display}"
<a href=#423 id=423 data-nosnippet>423</a> </span>);
<a href=#424 id=424 data-nosnippet>424</a> <span class="macro">assert_eq!</span>(ConcreteDataType::float64_datatype(), columns[<span class="number">0</span>].data_type);
<a href=#425 id=425 data-nosnippet>425</a> <span class="macro">assert_eq!</span>(<span class="string">"ts"</span>, columns[<span class="number">1</span>].name);
<a href=#426 id=426 data-nosnippet>426</a> <span class="macro">assert!</span>(columns[<span class="number">1</span>].is_time_index());
<a href=#427 id=427 data-nosnippet>427</a> <span class="macro">assert_eq!</span>(<span class="string">"status_code"</span>, columns[<span class="number">2</span>].name);
<a href=#428 id=428 data-nosnippet>428</a> <span class="macro">assert_eq!</span>(ConcreteDataType::string_datatype(), columns[<span class="number">2</span>].data_type);
<a href=#429 id=429 data-nosnippet>429</a> }
<a href=#430 id=430 data-nosnippet>430</a> }
<a href=#431 id=431 data-nosnippet>431</a>
<a href=#432 id=432 data-nosnippet>432</a> <span class="attr">#[tokio::test]
<a href=#433 id=433 data-nosnippet>433</a> </span><span class="kw">async fn </span>test_gen_create_table_sql() {
<a href=#434 id=434 data-nosnippet>434</a> <span class="kw">let </span>query_engine = create_test_query_engine();
<a href=#435 id=435 data-nosnippet>435</a> <span class="kw">let </span>ctx = QueryContext::arc();
<a href=#436 id=436 data-nosnippet>436</a> <span class="kw">struct </span>TestCase {
<a href=#437 id=437 data-nosnippet>437</a> sql: String,
<a href=#438 id=438 data-nosnippet>438</a> sink_table_name: String,
<a href=#439 id=439 data-nosnippet>439</a> column_schemas: Vec&lt;ColumnSchema&gt;,
<a href=#440 id=440 data-nosnippet>440</a> primary_keys: Vec&lt;String&gt;,
<a href=#441 id=441 data-nosnippet>441</a> time_index: String,
<a href=#442 id=442 data-nosnippet>442</a> }
<a href=#443 id=443 data-nosnippet>443</a>
<a href=#444 id=444 data-nosnippet>444</a> <span class="kw">let </span>update_at_schema = ColumnSchema::new(
<a href=#445 id=445 data-nosnippet>445</a> AUTO_CREATED_UPDATE_AT_TS_COL,
<a href=#446 id=446 data-nosnippet>446</a> ConcreteDataType::timestamp_millisecond_datatype(),
<a href=#447 id=447 data-nosnippet>447</a> <span class="bool-val">true</span>,
<a href=#448 id=448 data-nosnippet>448</a> );
<a href=#449 id=449 data-nosnippet>449</a>
<a href=#450 id=450 data-nosnippet>450</a> <span class="kw">let </span>ts_placeholder_schema = ColumnSchema::new(
<a href=#451 id=451 data-nosnippet>451</a> AUTO_CREATED_PLACEHOLDER_TS_COL,
<a href=#452 id=452 data-nosnippet>452</a> ConcreteDataType::timestamp_millisecond_datatype(),
<a href=#453 id=453 data-nosnippet>453</a> <span class="bool-val">false</span>,
<a href=#454 id=454 data-nosnippet>454</a> )
<a href=#455 id=455 data-nosnippet>455</a> .with_time_index(<span class="bool-val">true</span>);
<a href=#456 id=456 data-nosnippet>456</a>
<a href=#457 id=457 data-nosnippet>457</a> <span class="kw">let </span>testcases = <span class="macro">vec!</span>[
<a href=#458 id=458 data-nosnippet>458</a> TestCase {
<a href=#459 id=459 data-nosnippet>459</a> sql: <span class="string">"SELECT number, ts FROM numbers_with_ts"</span>.to_string(),
<a href=#460 id=460 data-nosnippet>460</a> sink_table_name: <span class="string">"new_table"</span>.to_string(),
<a href=#461 id=461 data-nosnippet>461</a> column_schemas: <span class="macro">vec!</span>[
<a href=#462 id=462 data-nosnippet>462</a> ColumnSchema::new(<span class="string">"number"</span>, ConcreteDataType::uint32_datatype(), <span class="bool-val">true</span>),
<a href=#463 id=463 data-nosnippet>463</a> ColumnSchema::new(
<a href=#464 id=464 data-nosnippet>464</a> <span class="string">"ts"</span>,
<a href=#465 id=465 data-nosnippet>465</a> ConcreteDataType::timestamp_millisecond_datatype(),
<a href=#466 id=466 data-nosnippet>466</a> <span class="bool-val">false</span>,
<a href=#467 id=467 data-nosnippet>467</a> )
<a href=#468 id=468 data-nosnippet>468</a> .with_time_index(<span class="bool-val">true</span>),
<a href=#469 id=469 data-nosnippet>469</a> update_at_schema.clone(),
<a href=#470 id=470 data-nosnippet>470</a> ],
<a href=#471 id=471 data-nosnippet>471</a> primary_keys: <span class="macro">vec!</span>[],
<a href=#472 id=472 data-nosnippet>472</a> time_index: <span class="string">"ts"</span>.to_string(),
<a href=#473 id=473 data-nosnippet>473</a> },
<a href=#474 id=474 data-nosnippet>474</a> TestCase {
<a href=#475 id=475 data-nosnippet>475</a> sql: <span class="string">"SELECT number, max(ts) FROM numbers_with_ts GROUP BY number"</span>.to_string(),
<a href=#476 id=476 data-nosnippet>476</a> sink_table_name: <span class="string">"new_table"</span>.to_string(),
<a href=#477 id=477 data-nosnippet>477</a> column_schemas: <span class="macro">vec!</span>[
<a href=#478 id=478 data-nosnippet>478</a> ColumnSchema::new(<span class="string">"number"</span>, ConcreteDataType::uint32_datatype(), <span class="bool-val">true</span>),
<a href=#479 id=479 data-nosnippet>479</a> ColumnSchema::new(
<a href=#480 id=480 data-nosnippet>480</a> <span class="string">"max(numbers_with_ts.ts)"</span>,
<a href=#481 id=481 data-nosnippet>481</a> ConcreteDataType::timestamp_millisecond_datatype(),
<a href=#482 id=482 data-nosnippet>482</a> <span class="bool-val">true</span>,
<a href=#483 id=483 data-nosnippet>483</a> ),
<a href=#484 id=484 data-nosnippet>484</a> update_at_schema.clone(),
<a href=#485 id=485 data-nosnippet>485</a> ts_placeholder_schema.clone(),
<a href=#486 id=486 data-nosnippet>486</a> ],
<a href=#487 id=487 data-nosnippet>487</a> primary_keys: <span class="macro">vec!</span>[<span class="string">"number"</span>.to_string()],
<a href=#488 id=488 data-nosnippet>488</a> time_index: AUTO_CREATED_PLACEHOLDER_TS_COL.to_string(),
<a href=#489 id=489 data-nosnippet>489</a> },
<a href=#490 id=490 data-nosnippet>490</a> TestCase {
<a href=#491 id=491 data-nosnippet>491</a> sql: <span class="string">"SELECT max(number), ts FROM numbers_with_ts GROUP BY ts"</span>.to_string(),
<a href=#492 id=492 data-nosnippet>492</a> sink_table_name: <span class="string">"new_table"</span>.to_string(),
<a href=#493 id=493 data-nosnippet>493</a> column_schemas: <span class="macro">vec!</span>[
<a href=#494 id=494 data-nosnippet>494</a> ColumnSchema::new(
<a href=#495 id=495 data-nosnippet>495</a> <span class="string">"max(numbers_with_ts.number)"</span>,
<a href=#496 id=496 data-nosnippet>496</a> ConcreteDataType::uint32_datatype(),
<a href=#497 id=497 data-nosnippet>497</a> <span class="bool-val">true</span>,
<a href=#498 id=498 data-nosnippet>498</a> ),
<a href=#499 id=499 data-nosnippet>499</a> ColumnSchema::new(
<a href=#500 id=500 data-nosnippet>500</a> <span class="string">"ts"</span>,
<a href=#501 id=501 data-nosnippet>501</a> ConcreteDataType::timestamp_millisecond_datatype(),
<a href=#502 id=502 data-nosnippet>502</a> <span class="bool-val">false</span>,
<a href=#503 id=503 data-nosnippet>503</a> )
<a href=#504 id=504 data-nosnippet>504</a> .with_time_index(<span class="bool-val">true</span>),
<a href=#505 id=505 data-nosnippet>505</a> update_at_schema.clone(),
<a href=#506 id=506 data-nosnippet>506</a> ],
<a href=#507 id=507 data-nosnippet>507</a> primary_keys: <span class="macro">vec!</span>[],
<a href=#508 id=508 data-nosnippet>508</a> time_index: <span class="string">"ts"</span>.to_string(),
<a href=#509 id=509 data-nosnippet>509</a> },
<a href=#510 id=510 data-nosnippet>510</a> TestCase {
<a href=#511 id=511 data-nosnippet>511</a> sql: <span class="string">"SELECT number, ts FROM numbers_with_ts GROUP BY ts, number"</span>.to_string(),
<a href=#512 id=512 data-nosnippet>512</a> sink_table_name: <span class="string">"new_table"</span>.to_string(),
<a href=#513 id=513 data-nosnippet>513</a> column_schemas: <span class="macro">vec!</span>[
<a href=#514 id=514 data-nosnippet>514</a> ColumnSchema::new(<span class="string">"number"</span>, ConcreteDataType::uint32_datatype(), <span class="bool-val">true</span>),
<a href=#515 id=515 data-nosnippet>515</a> ColumnSchema::new(
<a href=#516 id=516 data-nosnippet>516</a> <span class="string">"ts"</span>,
<a href=#517 id=517 data-nosnippet>517</a> ConcreteDataType::timestamp_millisecond_datatype(),
<a href=#518 id=518 data-nosnippet>518</a> <span class="bool-val">false</span>,
<a href=#519 id=519 data-nosnippet>519</a> )
<a href=#520 id=520 data-nosnippet>520</a> .with_time_index(<span class="bool-val">true</span>),
<a href=#521 id=521 data-nosnippet>521</a> update_at_schema.clone(),
<a href=#522 id=522 data-nosnippet>522</a> ],
<a href=#523 id=523 data-nosnippet>523</a> primary_keys: <span class="macro">vec!</span>[<span class="string">"number"</span>.to_string()],
<a href=#524 id=524 data-nosnippet>524</a> time_index: <span class="string">"ts"</span>.to_string(),
<a href=#525 id=525 data-nosnippet>525</a> },
<a href=#526 id=526 data-nosnippet>526</a> ];
<a href=#527 id=527 data-nosnippet>527</a>
<a href=#528 id=528 data-nosnippet>528</a> <span class="kw">for </span>tc <span class="kw">in </span>testcases {
<a href=#529 id=529 data-nosnippet>529</a> <span class="kw">let </span>plan = sql_to_df_plan(ctx.clone(), query_engine.clone(), <span class="kw-2">&amp;</span>tc.sql, <span class="bool-val">true</span>)
<a href=#530 id=530 data-nosnippet>530</a> .<span class="kw">await
<a href=#531 id=531 data-nosnippet>531</a> </span>.unwrap();
<a href=#532 id=532 data-nosnippet>532</a> <span class="kw">let </span>expr = create_table_with_expr(
<a href=#533 id=533 data-nosnippet>533</a> <span class="kw-2">&amp;</span>plan,
<a href=#534 id=534 data-nosnippet>534</a> <span class="kw-2">&amp;</span>[
<a href=#535 id=535 data-nosnippet>535</a> <span class="string">"greptime"</span>.to_string(),
<a href=#536 id=536 data-nosnippet>536</a> <span class="string">"public"</span>.to_string(),
<a href=#537 id=537 data-nosnippet>537</a> tc.sink_table_name.clone(),
<a href=#538 id=538 data-nosnippet>538</a> ],
<a href=#539 id=539 data-nosnippet>539</a> <span class="kw-2">&amp;</span>QueryType::Sql,
<a href=#540 id=540 data-nosnippet>540</a> )
<a href=#541 id=541 data-nosnippet>541</a> .unwrap();
<a href=#542 id=542 data-nosnippet>542</a> <span class="comment">// TODO(discord9): assert expr
<a href=#543 id=543 data-nosnippet>543</a> </span><span class="kw">let </span>column_schemas = expr
<a href=#544 id=544 data-nosnippet>544</a> .column_defs
<a href=#545 id=545 data-nosnippet>545</a> .iter()
<a href=#546 id=546 data-nosnippet>546</a> .map(|c| try_as_column_schema(c).unwrap())
<a href=#547 id=547 data-nosnippet>547</a> .collect::&lt;Vec&lt;<span class="kw">_</span>&gt;&gt;();
<a href=#548 id=548 data-nosnippet>548</a> <span class="macro">assert_eq!</span>(tc.column_schemas, column_schemas, <span class="string">"{:?}"</span>, tc.sql);
<a href=#549 id=549 data-nosnippet>549</a> <span class="macro">assert_eq!</span>(tc.primary_keys, expr.primary_keys, <span class="string">"{:?}"</span>, tc.sql);
<a href=#550 id=550 data-nosnippet>550</a> <span class="macro">assert_eq!</span>(tc.time_index, expr.time_index, <span class="string">"{:?}"</span>, tc.sql);
<a href=#551 id=551 data-nosnippet>551</a> }
<a href=#552 id=552 data-nosnippet>552</a> }
<a href=#553 id=553 data-nosnippet>553</a>}
</code></pre></div></section></main></body></html>
File diff suppressed because it is too large Load Diff
+311
View File
@@ -0,0 +1,311 @@
<!DOCTYPE html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1.0"><meta name="generator" content="rustdoc"><meta name="description" content="Source of the Rust file `src/flow/src/batching_mode/task/ckpt.rs`."><title>ckpt.rs - source</title><script>if(window.location.protocol!=="file:")document.head.insertAdjacentHTML("beforeend","SourceSerif4-Regular-6b053e98.ttf.woff2,FiraSans-Italic-81dc35de.woff2,FiraSans-Regular-0fe48ade.woff2,FiraSans-MediumItalic-ccf7e434.woff2,FiraSans-Medium-e1aa3f0a.woff2,SourceCodePro-Regular-8badfe75.ttf.woff2,SourceCodePro-Semibold-aa29a496.ttf.woff2".split(",").map(f=>`<link rel="preload" as="font" type="font/woff2"href="../../../../static.files/${f}">`).join(""))</script><link rel="stylesheet" href="../../../../static.files/normalize-9960930a.css"><link rel="stylesheet" href="../../../../static.files/rustdoc-17e0aaed.css"><meta name="rustdoc-vars" data-root-path="../../../../" data-static-root-path="../../../../static.files/" data-current-crate="flow" data-themes="" data-resource-suffix="" data-rustdoc-version="1.96.1 (31fca3adb 2026-06-26)" data-channel="1.96.1" data-search-js="search-b5634cc7.js" data-stringdex-js="stringdex-2da4960a.js" data-settings-js="settings-170eb4bf.js" ><script src="../../../../static.files/storage-41dd4d93.js"></script><script defer src="../../../../static.files/src-script-813739b1.js"></script><script defer src="../../../../src-files.js"></script><script defer src="../../../../static.files/main-5013f961.js"></script><noscript><link rel="stylesheet" href="../../../../static.files/noscript-f7c3ffd8.css"></noscript><link rel="alternate icon" type="image/png" href="../../../../static.files/favicon-32x32-eab170b8.png"><link rel="icon" type="image/svg+xml" href="../../../../static.files/favicon-044be391.svg"></head><body class="rustdoc src"><a class="skip-main-content" href="#main-content">Skip to main content</a><!--[if lte IE 11]><div class="warning">This old browser is unsupported and will most likely display funky things.</div><![endif]--><nav class="sidebar"><div class="src-sidebar-title"><h2>Files</h2></div></nav><div class="sidebar-resizer" title="Drag to resize sidebar"></div><main><section id="main-content" class="content" tabindex="-1"><div class="main-heading"><h1><div class="sub-heading">flow/batching_mode/task/</div>ckpt.rs</h1><rustdoc-toolbar></rustdoc-toolbar></div><div class="example-wrap digits-3"><pre class="rust"><code><a href=#1 id=1 data-nosnippet>1</a><span class="comment">// Copyright 2023 Greptime Team
<a href=#2 id=2 data-nosnippet>2</a>//
<a href=#3 id=3 data-nosnippet>3</a>// Licensed under the Apache License, Version 2.0 (the "License");
<a href=#4 id=4 data-nosnippet>4</a>// you may not use this file except in compliance with the License.
<a href=#5 id=5 data-nosnippet>5</a>// You may obtain a copy of the License at
<a href=#6 id=6 data-nosnippet>6</a>//
<a href=#7 id=7 data-nosnippet>7</a>// http://www.apache.org/licenses/LICENSE-2.0
<a href=#8 id=8 data-nosnippet>8</a>//
<a href=#9 id=9 data-nosnippet>9</a>// Unless required by applicable law or agreed to in writing, software
<a href=#10 id=10 data-nosnippet>10</a>// distributed under the License is distributed on an "AS IS" BASIS,
<a href=#11 id=11 data-nosnippet>11</a>// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
<a href=#12 id=12 data-nosnippet>12</a>// See the License for the specific language governing permissions and
<a href=#13 id=13 data-nosnippet>13</a>// limitations under the License.
<a href=#14 id=14 data-nosnippet>14</a>
<a href=#15 id=15 data-nosnippet>15</a></span><span class="kw">use </span>std::time::Duration;
<a href=#16 id=16 data-nosnippet>16</a>
<a href=#17 id=17 data-nosnippet>17</a><span class="kw">use </span>client::OutputWithMetrics;
<a href=#18 id=18 data-nosnippet>18</a><span class="kw">use </span>common_error::ext::ErrorExt;
<a href=#19 id=19 data-nosnippet>19</a><span class="kw">use </span>common_error::status_code::StatusCode;
<a href=#20 id=20 data-nosnippet>20</a><span class="kw">use </span>common_telemetry::tracing::warn;
<a href=#21 id=21 data-nosnippet>21</a><span class="kw">use </span>common_telemetry::{debug, info};
<a href=#22 id=22 data-nosnippet>22</a>
<a href=#23 id=23 data-nosnippet>23</a><span class="kw">use </span><span class="kw">crate</span>::batching_mode::checkpoint::{
<a href=#24 id=24 data-nosnippet>24</a> FlowCheckpointDecision, FlowQueryFallbackReason, checkpoint_mode_label,
<a href=#25 id=25 data-nosnippet>25</a>};
<a href=#26 id=26 data-nosnippet>26</a><span class="kw">use </span><span class="kw">crate</span>::batching_mode::state::{CheckpointMode, TaskState};
<a href=#27 id=27 data-nosnippet>27</a><span class="kw">use </span><span class="kw">crate</span>::batching_mode::task::{BatchingTask, QueryCoverage};
<a href=#28 id=28 data-nosnippet>28</a><span class="kw">use </span><span class="kw">crate</span>::metrics::{
<a href=#29 id=29 data-nosnippet>29</a> METRIC_FLOW_BATCHING_ENGINE_CHECKPOINT_DECISION_CNT, METRIC_FLOW_BATCHING_ENGINE_QUERY_MODE_CNT,
<a href=#30 id=30 data-nosnippet>30</a>};
<a href=#31 id=31 data-nosnippet>31</a><span class="kw">use </span>crate::{Error, FlowId};
<a href=#32 id=32 data-nosnippet>32</a>
<a href=#33 id=33 data-nosnippet>33</a><span class="kw">impl </span>BatchingTask {
<a href=#34 id=34 data-nosnippet>34</a> <span class="doccomment">/// Classify execution errors into checkpoint fallback reasons. A stale
<a href=#35 id=35 data-nosnippet>35</a> /// snapshot fence is special only for fenced repair chunks.
<a href=#36 id=36 data-nosnippet>36</a> </span><span class="kw">pub</span>(<span class="kw">super</span>) <span class="kw">fn </span>query_failure_reason(
<a href=#37 id=37 data-nosnippet>37</a> err: <span class="kw-2">&amp;</span>Error,
<a href=#38 id=38 data-nosnippet>38</a> coverage: <span class="kw-2">&amp;</span>QueryCoverage,
<a href=#39 id=39 data-nosnippet>39</a> ) -&gt; FlowQueryFallbackReason {
<a href=#40 id=40 data-nosnippet>40</a> <span class="kw">if </span>err.status_code() == StatusCode::RequestOutdated {
<a href=#41 id=41 data-nosnippet>41</a> <span class="kw">if </span><span class="macro">matches!</span>(coverage, QueryCoverage::FencedRepairChunk { .. }) {
<a href=#42 id=42 data-nosnippet>42</a> FlowQueryFallbackReason::SnapshotFenceExpired
<a href=#43 id=43 data-nosnippet>43</a> } <span class="kw">else </span>{
<a href=#44 id=44 data-nosnippet>44</a> FlowQueryFallbackReason::StaleCursor
<a href=#45 id=45 data-nosnippet>45</a> }
<a href=#46 id=46 data-nosnippet>46</a> } <span class="kw">else if </span><span class="macro">matches!</span>(coverage, QueryCoverage::IncrementalDelta) {
<a href=#47 id=47 data-nosnippet>47</a> FlowQueryFallbackReason::IncrementalQueryFailure
<a href=#48 id=48 data-nosnippet>48</a> } <span class="kw">else </span>{
<a href=#49 id=49 data-nosnippet>49</a> FlowQueryFallbackReason::QueryFailure
<a href=#50 id=50 data-nosnippet>50</a> }
<a href=#51 id=51 data-nosnippet>51</a> }
<a href=#52 id=52 data-nosnippet>52</a>
<a href=#53 id=53 data-nosnippet>53</a> <span class="doccomment">/// Apply the conservative state transition for a failed query. This never
<a href=#54 id=54 data-nosnippet>54</a> /// advances checkpoints; it only decides whether to abandon repair state or
<a href=#55 id=55 data-nosnippet>55</a> /// fall back from incremental mode.
<a href=#56 id=56 data-nosnippet>56</a> </span><span class="kw">pub</span>(<span class="kw">super</span>) <span class="kw">fn </span>apply_query_failure_to_state(
<a href=#57 id=57 data-nosnippet>57</a> state: <span class="kw-2">&amp;mut </span>TaskState,
<a href=#58 id=58 data-nosnippet>58</a> elapsed: Duration,
<a href=#59 id=59 data-nosnippet>59</a> coverage: <span class="kw-2">&amp;</span>QueryCoverage,
<a href=#60 id=60 data-nosnippet>60</a> reason: FlowQueryFallbackReason,
<a href=#61 id=61 data-nosnippet>61</a> ) -&gt; <span class="prelude-ty">Option</span>&lt;FlowCheckpointDecision&gt; {
<a href=#62 id=62 data-nosnippet>62</a> state.after_query_exec(elapsed, <span class="bool-val">false</span>);
<a href=#63 id=63 data-nosnippet>63</a> <span class="kw">let </span>checkpoint_mode = state.checkpoint_mode();
<a href=#64 id=64 data-nosnippet>64</a> <span class="kw">if </span><span class="macro">matches!</span>(coverage, QueryCoverage::FencedRepairChunk { .. })
<a href=#65 id=65 data-nosnippet>65</a> &amp;&amp; <span class="macro">matches!</span>(reason, FlowQueryFallbackReason::SnapshotFenceExpired)
<a href=#66 id=66 data-nosnippet>66</a> {
<a href=#67 id=67 data-nosnippet>67</a> <span class="comment">// `abandon_fenced_repair()` restores the not-yet-in-flight pending
<a href=#68 id=68 data-nosnippet>68</a> // windows from the stale repair back to live dirty windows. The
<a href=#69 id=69 data-nosnippet>69</a> // currently executing chunk is restored separately by the outer
<a href=#70 id=70 data-nosnippet>70</a> // failure path (`handle_executed_query_failure`), after this state
<a href=#71 id=71 data-nosnippet>71</a> // transition has removed `pending_fenced_repair`, so it also lands
<a href=#72 id=72 data-nosnippet>72</a> // in live dirty windows for the next scoped base repair.
<a href=#73 id=73 data-nosnippet>73</a> </span>state.abandon_fenced_repair();
<a href=#74 id=74 data-nosnippet>74</a> <span class="kw">return </span><span class="prelude-val">Some</span>(FlowCheckpointDecision::FallbackToFullSnapshot {
<a href=#75 id=75 data-nosnippet>75</a> previous_mode: checkpoint_mode,
<a href=#76 id=76 data-nosnippet>76</a> reason,
<a href=#77 id=77 data-nosnippet>77</a> });
<a href=#78 id=78 data-nosnippet>78</a> }
<a href=#79 id=79 data-nosnippet>79</a>
<a href=#80 id=80 data-nosnippet>80</a> <span class="kw">if </span>checkpoint_mode == CheckpointMode::Incremental {
<a href=#81 id=81 data-nosnippet>81</a> state.mark_full_snapshot();
<a href=#82 id=82 data-nosnippet>82</a> }
<a href=#83 id=83 data-nosnippet>83</a> <span class="prelude-val">Some</span>(FlowCheckpointDecision::FallbackToFullSnapshot {
<a href=#84 id=84 data-nosnippet>84</a> previous_mode: checkpoint_mode,
<a href=#85 id=85 data-nosnippet>85</a> reason,
<a href=#86 id=86 data-nosnippet>86</a> })
<a href=#87 id=87 data-nosnippet>87</a> }
<a href=#88 id=88 data-nosnippet>88</a>
<a href=#89 id=89 data-nosnippet>89</a> <span class="doccomment">/// Apply checkpoint transitions for a successfully executed query using its
<a href=#90 id=90 data-nosnippet>90</a> /// terminal watermark proof and declared coverage.
<a href=#91 id=91 data-nosnippet>91</a> </span><span class="kw">pub</span>(<span class="kw">super</span>) <span class="kw">fn </span>apply_query_result_to_state(
<a href=#92 id=92 data-nosnippet>92</a> state: <span class="kw-2">&amp;mut </span>TaskState,
<a href=#93 id=93 data-nosnippet>93</a> res: <span class="kw-2">&amp;</span>OutputWithMetrics,
<a href=#94 id=94 data-nosnippet>94</a> elapsed: Duration,
<a href=#95 id=95 data-nosnippet>95</a> coverage: <span class="kw-2">&amp;</span>QueryCoverage,
<a href=#96 id=96 data-nosnippet>96</a> ) -&gt; FlowCheckpointDecision {
<a href=#97 id=97 data-nosnippet>97</a> state.after_query_exec(elapsed, <span class="bool-val">true</span>);
<a href=#98 id=98 data-nosnippet>98</a> <span class="kw">let </span>checkpoint_mode = state.checkpoint_mode();
<a href=#99 id=99 data-nosnippet>99</a> <span class="kw">if let </span>(<span class="prelude-val">Some</span>(participating_regions), <span class="prelude-val">Some</span>(watermark_map)) =
<a href=#100 id=100 data-nosnippet>100</a> (res.participating_regions(), res.region_watermark_map())
<a href=#101 id=101 data-nosnippet>101</a> {
<a href=#102 id=102 data-nosnippet>102</a> <span class="kw">let </span>participating_region_count = participating_regions.len();
<a href=#103 id=103 data-nosnippet>103</a> <span class="kw">let </span>watermark_count = watermark_map.len();
<a href=#104 id=104 data-nosnippet>104</a> <span class="kw">match </span>coverage {
<a href=#105 id=105 data-nosnippet>105</a> QueryCoverage::ScopedBaseRepair =&gt; {
<a href=#106 id=106 data-nosnippet>106</a> <span class="kw">if </span>!state.can_advance_full_snapshot_checkpoints(
<a href=#107 id=107 data-nosnippet>107</a> <span class="kw-2">&amp;</span>participating_regions,
<a href=#108 id=108 data-nosnippet>108</a> <span class="kw-2">&amp;</span>watermark_map,
<a href=#109 id=109 data-nosnippet>109</a> ) {
<a href=#110 id=110 data-nosnippet>110</a> <span class="kw">return </span>FlowCheckpointDecision::FallbackToFullSnapshot {
<a href=#111 id=111 data-nosnippet>111</a> previous_mode: checkpoint_mode,
<a href=#112 id=112 data-nosnippet>112</a> reason: FlowQueryFallbackReason::IncompleteRegionWatermark,
<a href=#113 id=113 data-nosnippet>113</a> };
<a href=#114 id=114 data-nosnippet>114</a> }
<a href=#115 id=115 data-nosnippet>115</a>
<a href=#116 id=116 data-nosnippet>116</a> <span class="kw">if </span>state.is_incremental_disabled() {
<a href=#117 id=117 data-nosnippet>117</a> <span class="kw">return </span>FlowCheckpointDecision::FallbackToFullSnapshot {
<a href=#118 id=118 data-nosnippet>118</a> previous_mode: CheckpointMode::FullSnapshot,
<a href=#119 id=119 data-nosnippet>119</a> reason: FlowQueryFallbackReason::IncrementalDisabled,
<a href=#120 id=120 data-nosnippet>120</a> };
<a href=#121 id=121 data-nosnippet>121</a> }
<a href=#122 id=122 data-nosnippet>122</a>
<a href=#123 id=123 data-nosnippet>123</a> <span class="kw">if </span>state.dirty_time_windows.is_empty() {
<a href=#124 id=124 data-nosnippet>124</a> state.advance_checkpoints(watermark_map);
<a href=#125 id=125 data-nosnippet>125</a> FlowCheckpointDecision::AdvancedFromFullSnapshot {
<a href=#126 id=126 data-nosnippet>126</a> participating_regions: participating_region_count,
<a href=#127 id=127 data-nosnippet>127</a> watermarks: watermark_count,
<a href=#128 id=128 data-nosnippet>128</a> }
<a href=#129 id=129 data-nosnippet>129</a> } <span class="kw">else if let </span><span class="prelude-val">Some</span>(repair) =
<a href=#130 id=130 data-nosnippet>130</a> state.start_fenced_repair(watermark_map.into_iter().collect())
<a href=#131 id=131 data-nosnippet>131</a> {
<a href=#132 id=132 data-nosnippet>132</a> FlowCheckpointDecision::ContinuedFencedRepair {
<a href=#133 id=133 data-nosnippet>133</a> pending_windows: repair.pending_windows().len(),
<a href=#134 id=134 data-nosnippet>134</a> watermarks: repair.high().len(),
<a href=#135 id=135 data-nosnippet>135</a> }
<a href=#136 id=136 data-nosnippet>136</a> } <span class="kw">else </span>{
<a href=#137 id=137 data-nosnippet>137</a> FlowCheckpointDecision::FallbackToFullSnapshot {
<a href=#138 id=138 data-nosnippet>138</a> previous_mode: checkpoint_mode,
<a href=#139 id=139 data-nosnippet>139</a> reason: FlowQueryFallbackReason::DirtyBacklogPending,
<a href=#140 id=140 data-nosnippet>140</a> }
<a href=#141 id=141 data-nosnippet>141</a> }
<a href=#142 id=142 data-nosnippet>142</a> }
<a href=#143 id=143 data-nosnippet>143</a> QueryCoverage::FencedRepairChunk { .. } =&gt; {
<a href=#144 id=144 data-nosnippet>144</a> <span class="kw">if </span>!state
<a href=#145 id=145 data-nosnippet>145</a> .fenced_repair_watermarks_match_high(<span class="kw-2">&amp;</span>participating_regions, <span class="kw-2">&amp;</span>watermark_map)
<a href=#146 id=146 data-nosnippet>146</a> {
<a href=#147 id=147 data-nosnippet>147</a> <span class="comment">// A successful repair chunk whose terminal watermark
<a href=#148 id=148 data-nosnippet>148</a> // differs from the frozen fence `H` cannot prove that
<a href=#149 id=149 data-nosnippet>149</a> // continuing the same repair is safe. The pre-`H`
<a href=#150 id=150 data-nosnippet>150</a> // repair work item already executed under the snapshot
<a href=#151 id=151 data-nosnippet>151</a> // fence, so do not requeue it; restore only
<a href=#152 id=152 data-nosnippet>152</a> // not-yet-in-flight pending windows, then start over
<a href=#153 id=153 data-nosnippet>153</a> // with a fresh H. Later/post-`H` writes still rely on
<a href=#154 id=154 data-nosnippet>154</a> // live dirty notifications.
<a href=#155 id=155 data-nosnippet>155</a> </span>state.abandon_fenced_repair();
<a href=#156 id=156 data-nosnippet>156</a> <span class="kw">return </span>FlowCheckpointDecision::FallbackToFullSnapshot {
<a href=#157 id=157 data-nosnippet>157</a> previous_mode: checkpoint_mode,
<a href=#158 id=158 data-nosnippet>158</a> reason: FlowQueryFallbackReason::IncompleteRegionWatermark,
<a href=#159 id=159 data-nosnippet>159</a> };
<a href=#160 id=160 data-nosnippet>160</a> }
<a href=#161 id=161 data-nosnippet>161</a>
<a href=#162 id=162 data-nosnippet>162</a> <span class="kw">if </span>state.fenced_repair_pending_is_empty() {
<a href=#163 id=163 data-nosnippet>163</a> state.finish_fenced_repair();
<a href=#164 id=164 data-nosnippet>164</a> <span class="kw">if </span>state.is_incremental_disabled() {
<a href=#165 id=165 data-nosnippet>165</a> FlowCheckpointDecision::FallbackToFullSnapshot {
<a href=#166 id=166 data-nosnippet>166</a> previous_mode: CheckpointMode::FullSnapshot,
<a href=#167 id=167 data-nosnippet>167</a> reason: FlowQueryFallbackReason::IncrementalDisabled,
<a href=#168 id=168 data-nosnippet>168</a> }
<a href=#169 id=169 data-nosnippet>169</a> } <span class="kw">else </span>{
<a href=#170 id=170 data-nosnippet>170</a> FlowCheckpointDecision::AdvancedFromFullSnapshot {
<a href=#171 id=171 data-nosnippet>171</a> participating_regions: participating_region_count,
<a href=#172 id=172 data-nosnippet>172</a> watermarks: watermark_count,
<a href=#173 id=173 data-nosnippet>173</a> }
<a href=#174 id=174 data-nosnippet>174</a> }
<a href=#175 id=175 data-nosnippet>175</a> } <span class="kw">else </span>{
<a href=#176 id=176 data-nosnippet>176</a> <span class="kw">let </span>repair = state
<a href=#177 id=177 data-nosnippet>177</a> .pending_fenced_repair()
<a href=#178 id=178 data-nosnippet>178</a> .expect(<span class="string">"fenced repair exists after matching repair chunk watermark"</span>);
<a href=#179 id=179 data-nosnippet>179</a> FlowCheckpointDecision::ContinuedFencedRepair {
<a href=#180 id=180 data-nosnippet>180</a> pending_windows: repair.pending_windows().len(),
<a href=#181 id=181 data-nosnippet>181</a> watermarks: repair.high().len(),
<a href=#182 id=182 data-nosnippet>182</a> }
<a href=#183 id=183 data-nosnippet>183</a> }
<a href=#184 id=184 data-nosnippet>184</a> }
<a href=#185 id=185 data-nosnippet>185</a> QueryCoverage::UnfilteredFull =&gt; {
<a href=#186 id=186 data-nosnippet>186</a> <span class="kw">if </span>state.can_advance_full_snapshot_checkpoints(
<a href=#187 id=187 data-nosnippet>187</a> <span class="kw-2">&amp;</span>participating_regions,
<a href=#188 id=188 data-nosnippet>188</a> <span class="kw-2">&amp;</span>watermark_map,
<a href=#189 id=189 data-nosnippet>189</a> ) {
<a href=#190 id=190 data-nosnippet>190</a> state.advance_checkpoints(watermark_map);
<a href=#191 id=191 data-nosnippet>191</a> <span class="kw">if </span>state.is_incremental_disabled() {
<a href=#192 id=192 data-nosnippet>192</a> FlowCheckpointDecision::FallbackToFullSnapshot {
<a href=#193 id=193 data-nosnippet>193</a> previous_mode: CheckpointMode::FullSnapshot,
<a href=#194 id=194 data-nosnippet>194</a> reason: FlowQueryFallbackReason::IncrementalDisabled,
<a href=#195 id=195 data-nosnippet>195</a> }
<a href=#196 id=196 data-nosnippet>196</a> } <span class="kw">else </span>{
<a href=#197 id=197 data-nosnippet>197</a> FlowCheckpointDecision::AdvancedFromFullSnapshot {
<a href=#198 id=198 data-nosnippet>198</a> participating_regions: participating_region_count,
<a href=#199 id=199 data-nosnippet>199</a> watermarks: watermark_count,
<a href=#200 id=200 data-nosnippet>200</a> }
<a href=#201 id=201 data-nosnippet>201</a> }
<a href=#202 id=202 data-nosnippet>202</a> } <span class="kw">else </span>{
<a href=#203 id=203 data-nosnippet>203</a> <span class="macro">debug_assert_ne!</span>(checkpoint_mode, CheckpointMode::Incremental);
<a href=#204 id=204 data-nosnippet>204</a> FlowCheckpointDecision::FallbackToFullSnapshot {
<a href=#205 id=205 data-nosnippet>205</a> previous_mode: checkpoint_mode,
<a href=#206 id=206 data-nosnippet>206</a> reason: FlowQueryFallbackReason::IncompleteRegionWatermark,
<a href=#207 id=207 data-nosnippet>207</a> }
<a href=#208 id=208 data-nosnippet>208</a> }
<a href=#209 id=209 data-nosnippet>209</a> }
<a href=#210 id=210 data-nosnippet>210</a> QueryCoverage::IncrementalDelta =&gt; {
<a href=#211 id=211 data-nosnippet>211</a> <span class="kw">if </span>state.can_advance_incremental_checkpoints_with_participation(
<a href=#212 id=212 data-nosnippet>212</a> <span class="kw-2">&amp;</span>participating_regions,
<a href=#213 id=213 data-nosnippet>213</a> <span class="kw-2">&amp;</span>watermark_map,
<a href=#214 id=214 data-nosnippet>214</a> ) {
<a href=#215 id=215 data-nosnippet>215</a> state.advance_incremental_checkpoints_with_participation(
<a href=#216 id=216 data-nosnippet>216</a> <span class="kw-2">&amp;</span>participating_regions,
<a href=#217 id=217 data-nosnippet>217</a> watermark_map,
<a href=#218 id=218 data-nosnippet>218</a> );
<a href=#219 id=219 data-nosnippet>219</a> FlowCheckpointDecision::AdvancedIncremental {
<a href=#220 id=220 data-nosnippet>220</a> participating_regions: participating_region_count,
<a href=#221 id=221 data-nosnippet>221</a> watermarks: watermark_count,
<a href=#222 id=222 data-nosnippet>222</a> }
<a href=#223 id=223 data-nosnippet>223</a> } <span class="kw">else </span>{
<a href=#224 id=224 data-nosnippet>224</a> state.mark_full_snapshot();
<a href=#225 id=225 data-nosnippet>225</a> FlowCheckpointDecision::FallbackToFullSnapshot {
<a href=#226 id=226 data-nosnippet>226</a> previous_mode: checkpoint_mode,
<a href=#227 id=227 data-nosnippet>227</a> reason: FlowQueryFallbackReason::IncompleteRegionWatermark,
<a href=#228 id=228 data-nosnippet>228</a> }
<a href=#229 id=229 data-nosnippet>229</a> }
<a href=#230 id=230 data-nosnippet>230</a> }
<a href=#231 id=231 data-nosnippet>231</a> }
<a href=#232 id=232 data-nosnippet>232</a> } <span class="kw">else </span>{
<a href=#233 id=233 data-nosnippet>233</a> <span class="kw">if </span><span class="macro">matches!</span>(coverage, QueryCoverage::FencedRepairChunk { .. }) {
<a href=#234 id=234 data-nosnippet>234</a> state.abandon_fenced_repair();
<a href=#235 id=235 data-nosnippet>235</a> }
<a href=#236 id=236 data-nosnippet>236</a> <span class="kw">if </span><span class="macro">matches!</span>(checkpoint_mode, CheckpointMode::Incremental) {
<a href=#237 id=237 data-nosnippet>237</a> state.mark_full_snapshot();
<a href=#238 id=238 data-nosnippet>238</a> }
<a href=#239 id=239 data-nosnippet>239</a> FlowCheckpointDecision::FallbackToFullSnapshot {
<a href=#240 id=240 data-nosnippet>240</a> previous_mode: checkpoint_mode,
<a href=#241 id=241 data-nosnippet>241</a> reason: FlowQueryFallbackReason::MissingRegionWatermark,
<a href=#242 id=242 data-nosnippet>242</a> }
<a href=#243 id=243 data-nosnippet>243</a> }
<a href=#244 id=244 data-nosnippet>244</a> }
<a href=#245 id=245 data-nosnippet>245</a>
<a href=#246 id=246 data-nosnippet>246</a> <span class="kw">pub</span>(<span class="kw">super</span>) <span class="kw">fn </span>record_checkpoint_decision(flow_id: FlowId, decision: FlowCheckpointDecision) {
<a href=#247 id=247 data-nosnippet>247</a> <span class="kw">let </span>flow_id = flow_id.to_string();
<a href=#248 id=248 data-nosnippet>248</a> METRIC_FLOW_BATCHING_ENGINE_CHECKPOINT_DECISION_CNT
<a href=#249 id=249 data-nosnippet>249</a> .with_label_values(<span class="kw-2">&amp;</span>[
<a href=#250 id=250 data-nosnippet>250</a> flow_id.as_str(),
<a href=#251 id=251 data-nosnippet>251</a> decision.mode_label(),
<a href=#252 id=252 data-nosnippet>252</a> decision.decision_label(),
<a href=#253 id=253 data-nosnippet>253</a> decision.reason_label(),
<a href=#254 id=254 data-nosnippet>254</a> ])
<a href=#255 id=255 data-nosnippet>255</a> .inc();
<a href=#256 id=256 data-nosnippet>256</a>
<a href=#257 id=257 data-nosnippet>257</a> <span class="kw">match </span>decision {
<a href=#258 id=258 data-nosnippet>258</a> FlowCheckpointDecision::AdvancedFromFullSnapshot {
<a href=#259 id=259 data-nosnippet>259</a> participating_regions,
<a href=#260 id=260 data-nosnippet>260</a> watermarks,
<a href=#261 id=261 data-nosnippet>261</a> } =&gt; {
<a href=#262 id=262 data-nosnippet>262</a> <span class="macro">info!</span>(
<a href=#263 id=263 data-nosnippet>263</a> <span class="string">"Flow {flow_id} switched to incremental mode after full snapshot, participating_regions={participating_regions}, watermarks={watermarks}"
<a href=#264 id=264 data-nosnippet>264</a> </span>);
<a href=#265 id=265 data-nosnippet>265</a> }
<a href=#266 id=266 data-nosnippet>266</a> FlowCheckpointDecision::AdvancedIncremental {
<a href=#267 id=267 data-nosnippet>267</a> participating_regions,
<a href=#268 id=268 data-nosnippet>268</a> watermarks,
<a href=#269 id=269 data-nosnippet>269</a> } =&gt; {
<a href=#270 id=270 data-nosnippet>270</a> <span class="macro">debug!</span>(
<a href=#271 id=271 data-nosnippet>271</a> <span class="string">"Flow {flow_id} advanced incremental checkpoints, participating_regions={participating_regions}, watermarks={watermarks}"
<a href=#272 id=272 data-nosnippet>272</a> </span>);
<a href=#273 id=273 data-nosnippet>273</a> }
<a href=#274 id=274 data-nosnippet>274</a> FlowCheckpointDecision::ContinuedFencedRepair {
<a href=#275 id=275 data-nosnippet>275</a> pending_windows,
<a href=#276 id=276 data-nosnippet>276</a> watermarks,
<a href=#277 id=277 data-nosnippet>277</a> } =&gt; {
<a href=#278 id=278 data-nosnippet>278</a> <span class="macro">debug!</span>(
<a href=#279 id=279 data-nosnippet>279</a> <span class="string">"Flow {flow_id} continued fenced repair, pending_windows={pending_windows}, watermarks={watermarks}"
<a href=#280 id=280 data-nosnippet>280</a> </span>);
<a href=#281 id=281 data-nosnippet>281</a> }
<a href=#282 id=282 data-nosnippet>282</a> FlowCheckpointDecision::FallbackToFullSnapshot {
<a href=#283 id=283 data-nosnippet>283</a> previous_mode: CheckpointMode::FullSnapshot,
<a href=#284 id=284 data-nosnippet>284</a> reason: FlowQueryFallbackReason::IncrementalDisabled,
<a href=#285 id=285 data-nosnippet>285</a> } =&gt; {
<a href=#286 id=286 data-nosnippet>286</a> <span class="macro">debug!</span>(
<a href=#287 id=287 data-nosnippet>287</a> <span class="string">"Flow {flow_id} remains in full snapshot mode, reason={}"</span>,
<a href=#288 id=288 data-nosnippet>288</a> FlowQueryFallbackReason::IncrementalDisabled.as_label()
<a href=#289 id=289 data-nosnippet>289</a> );
<a href=#290 id=290 data-nosnippet>290</a> }
<a href=#291 id=291 data-nosnippet>291</a> FlowCheckpointDecision::FallbackToFullSnapshot {
<a href=#292 id=292 data-nosnippet>292</a> previous_mode,
<a href=#293 id=293 data-nosnippet>293</a> reason,
<a href=#294 id=294 data-nosnippet>294</a> } =&gt; {
<a href=#295 id=295 data-nosnippet>295</a> <span class="macro">warn!</span>(
<a href=#296 id=296 data-nosnippet>296</a> <span class="string">"Flow {flow_id} switched to full snapshot mode, previous_mode={}, reason={}"</span>,
<a href=#297 id=297 data-nosnippet>297</a> checkpoint_mode_label(previous_mode),
<a href=#298 id=298 data-nosnippet>298</a> reason.as_label()
<a href=#299 id=299 data-nosnippet>299</a> );
<a href=#300 id=300 data-nosnippet>300</a> }
<a href=#301 id=301 data-nosnippet>301</a> }
<a href=#302 id=302 data-nosnippet>302</a> }
<a href=#303 id=303 data-nosnippet>303</a>
<a href=#304 id=304 data-nosnippet>304</a> <span class="kw">pub</span>(<span class="kw">super</span>) <span class="kw">fn </span>record_query_mode(flow_id: FlowId, mode: CheckpointMode) {
<a href=#305 id=305 data-nosnippet>305</a> <span class="kw">let </span>flow_id = flow_id.to_string();
<a href=#306 id=306 data-nosnippet>306</a> METRIC_FLOW_BATCHING_ENGINE_QUERY_MODE_CNT
<a href=#307 id=307 data-nosnippet>307</a> .with_label_values(<span class="kw-2">&amp;</span>[flow_id.as_str(), checkpoint_mode_label(mode)])
<a href=#308 id=308 data-nosnippet>308</a> .inc();
<a href=#309 id=309 data-nosnippet>309</a> }
<a href=#310 id=310 data-nosnippet>310</a>}
</code></pre></div></section></main></body></html>
+330
View File
@@ -0,0 +1,330 @@
<!DOCTYPE html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1.0"><meta name="generator" content="rustdoc"><meta name="description" content="Source of the Rust file `src/flow/src/batching_mode/task/inc.rs`."><title>inc.rs - source</title><script>if(window.location.protocol!=="file:")document.head.insertAdjacentHTML("beforeend","SourceSerif4-Regular-6b053e98.ttf.woff2,FiraSans-Italic-81dc35de.woff2,FiraSans-Regular-0fe48ade.woff2,FiraSans-MediumItalic-ccf7e434.woff2,FiraSans-Medium-e1aa3f0a.woff2,SourceCodePro-Regular-8badfe75.ttf.woff2,SourceCodePro-Semibold-aa29a496.ttf.woff2".split(",").map(f=>`<link rel="preload" as="font" type="font/woff2"href="../../../../static.files/${f}">`).join(""))</script><link rel="stylesheet" href="../../../../static.files/normalize-9960930a.css"><link rel="stylesheet" href="../../../../static.files/rustdoc-17e0aaed.css"><meta name="rustdoc-vars" data-root-path="../../../../" data-static-root-path="../../../../static.files/" data-current-crate="flow" data-themes="" data-resource-suffix="" data-rustdoc-version="1.96.1 (31fca3adb 2026-06-26)" data-channel="1.96.1" data-search-js="search-b5634cc7.js" data-stringdex-js="stringdex-2da4960a.js" data-settings-js="settings-170eb4bf.js" ><script src="../../../../static.files/storage-41dd4d93.js"></script><script defer src="../../../../static.files/src-script-813739b1.js"></script><script defer src="../../../../src-files.js"></script><script defer src="../../../../static.files/main-5013f961.js"></script><noscript><link rel="stylesheet" href="../../../../static.files/noscript-f7c3ffd8.css"></noscript><link rel="alternate icon" type="image/png" href="../../../../static.files/favicon-32x32-eab170b8.png"><link rel="icon" type="image/svg+xml" href="../../../../static.files/favicon-044be391.svg"></head><body class="rustdoc src"><a class="skip-main-content" href="#main-content">Skip to main content</a><!--[if lte IE 11]><div class="warning">This old browser is unsupported and will most likely display funky things.</div><![endif]--><nav class="sidebar"><div class="src-sidebar-title"><h2>Files</h2></div></nav><div class="sidebar-resizer" title="Drag to resize sidebar"></div><main><section id="main-content" class="content" tabindex="-1"><div class="main-heading"><h1><div class="sub-heading">flow/batching_mode/task/</div>inc.rs</h1><rustdoc-toolbar></rustdoc-toolbar></div><div class="example-wrap digits-3"><pre class="rust"><code><a href=#1 id=1 data-nosnippet>1</a><span class="comment">// Copyright 2023 Greptime Team
<a href=#2 id=2 data-nosnippet>2</a>//
<a href=#3 id=3 data-nosnippet>3</a>// Licensed under the Apache License, Version 2.0 (the "License");
<a href=#4 id=4 data-nosnippet>4</a>// you may not use this file except in compliance with the License.
<a href=#5 id=5 data-nosnippet>5</a>// You may obtain a copy of the License at
<a href=#6 id=6 data-nosnippet>6</a>//
<a href=#7 id=7 data-nosnippet>7</a>// http://www.apache.org/licenses/LICENSE-2.0
<a href=#8 id=8 data-nosnippet>8</a>//
<a href=#9 id=9 data-nosnippet>9</a>// Unless required by applicable law or agreed to in writing, software
<a href=#10 id=10 data-nosnippet>10</a>// distributed under the License is distributed on an "AS IS" BASIS,
<a href=#11 id=11 data-nosnippet>11</a>// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
<a href=#12 id=12 data-nosnippet>12</a>// See the License for the specific language governing permissions and
<a href=#13 id=13 data-nosnippet>13</a>// limitations under the License.
<a href=#14 id=14 data-nosnippet>14</a>
<a href=#15 id=15 data-nosnippet>15</a></span><span class="kw">use </span>std::sync::Arc;
<a href=#16 id=16 data-nosnippet>16</a>
<a href=#17 id=17 data-nosnippet>17</a><span class="kw">use </span>common_error::ext::BoxedError;
<a href=#18 id=18 data-nosnippet>18</a><span class="kw">use </span>common_telemetry::debug;
<a href=#19 id=19 data-nosnippet>19</a><span class="kw">use </span>common_telemetry::tracing::warn;
<a href=#20 id=20 data-nosnippet>20</a><span class="kw">use </span>datafusion_expr::{DmlStatement, LogicalPlan};
<a href=#21 id=21 data-nosnippet>21</a><span class="kw">use </span>query::QueryEngineRef;
<a href=#22 id=22 data-nosnippet>22</a><span class="kw">use </span>query::options::{
<a href=#23 id=23 data-nosnippet>23</a> FLOW_INCREMENTAL_AFTER_SEQS, FLOW_INCREMENTAL_MODE, FLOW_INCREMENTAL_MODE_MEMTABLE_ONLY,
<a href=#24 id=24 data-nosnippet>24</a> FLOW_INCREMENTAL_MODE_SEQUENCE_RANGE, FLOW_SINK_TABLE_ID,
<a href=#25 id=25 data-nosnippet>25</a>};
<a href=#26 id=26 data-nosnippet>26</a><span class="kw">use </span>snafu::ResultExt;
<a href=#27 id=27 data-nosnippet>27</a><span class="kw">use </span>store_api::mito_engine_options::PRESERVE_ROW_SEQUENCE;
<a href=#28 id=28 data-nosnippet>28</a><span class="kw">use </span>table::metadata::TableId;
<a href=#29 id=29 data-nosnippet>29</a>
<a href=#30 id=30 data-nosnippet>30</a><span class="kw">use </span><span class="kw">crate</span>::Error;
<a href=#31 id=31 data-nosnippet>31</a><span class="kw">use </span><span class="kw">crate</span>::batching_mode::state::CheckpointMode;
<a href=#32 id=32 data-nosnippet>32</a><span class="kw">use </span><span class="kw">crate</span>::batching_mode::table_creator::QueryType;
<a href=#33 id=33 data-nosnippet>33</a><span class="kw">use </span><span class="kw">crate</span>::batching_mode::task::BatchingTask;
<a href=#34 id=34 data-nosnippet>34</a><span class="kw">use </span><span class="kw">crate</span>::batching_mode::utils::{
<a href=#35 id=35 data-nosnippet>35</a> analyze_incremental_aggregate_plan, get_table_info_df_schema,
<a href=#36 id=36 data-nosnippet>36</a> rewrite_incremental_aggregate_with_sink_merge,
<a href=#37 id=37 data-nosnippet>37</a>};
<a href=#38 id=38 data-nosnippet>38</a><span class="kw">use </span><span class="kw">crate</span>::error::{ExternalSnafu, UnexpectedSnafu};
<a href=#39 id=39 data-nosnippet>39</a>
<a href=#40 id=40 data-nosnippet>40</a><span class="kw">impl </span>BatchingTask {
<a href=#41 id=41 data-nosnippet>41</a> <span class="kw">async fn </span>sink_table_id(<span class="kw-2">&amp;</span><span class="self">self</span>) -&gt; <span class="prelude-ty">Result</span>&lt;TableId, Error&gt; {
<a href=#42 id=42 data-nosnippet>42</a> <span class="kw">let </span>table = <span class="self">self
<a href=#43 id=43 data-nosnippet>43</a> </span>.config
<a href=#44 id=44 data-nosnippet>44</a> .catalog_manager
<a href=#45 id=45 data-nosnippet>45</a> .table(
<a href=#46 id=46 data-nosnippet>46</a> <span class="kw-2">&amp;</span><span class="self">self</span>.config.sink_table_name[<span class="number">0</span>],
<a href=#47 id=47 data-nosnippet>47</a> <span class="kw-2">&amp;</span><span class="self">self</span>.config.sink_table_name[<span class="number">1</span>],
<a href=#48 id=48 data-nosnippet>48</a> <span class="kw-2">&amp;</span><span class="self">self</span>.config.sink_table_name[<span class="number">2</span>],
<a href=#49 id=49 data-nosnippet>49</a> <span class="prelude-val">None</span>,
<a href=#50 id=50 data-nosnippet>50</a> )
<a href=#51 id=51 data-nosnippet>51</a> .<span class="kw">await
<a href=#52 id=52 data-nosnippet>52</a> </span>.map_err(BoxedError::new)
<a href=#53 id=53 data-nosnippet>53</a> .context(ExternalSnafu)<span class="question-mark">?
<a href=#54 id=54 data-nosnippet>54</a> </span>.ok_or_else(|| {
<a href=#55 id=55 data-nosnippet>55</a> UnexpectedSnafu {
<a href=#56 id=56 data-nosnippet>56</a> reason: <span class="macro">format!</span>(
<a href=#57 id=57 data-nosnippet>57</a> <span class="string">"Flow {} cannot build incremental extensions because sink table {:?} was not found"</span>,
<a href=#58 id=58 data-nosnippet>58</a> <span class="self">self</span>.config.flow_id, <span class="self">self</span>.config.sink_table_name
<a href=#59 id=59 data-nosnippet>59</a> ),
<a href=#60 id=60 data-nosnippet>60</a> }
<a href=#61 id=61 data-nosnippet>61</a> .build()
<a href=#62 id=62 data-nosnippet>62</a> })<span class="question-mark">?</span>;
<a href=#63 id=63 data-nosnippet>63</a> <span class="prelude-val">Ok</span>(table.table_info().table_id())
<a href=#64 id=64 data-nosnippet>64</a> }
<a href=#65 id=65 data-nosnippet>65</a>
<a href=#66 id=66 data-nosnippet>66</a> <span class="doccomment">/// Whether every source table provably supports exact sequence-range
<a href=#67 id=67 data-nosnippet>67</a> /// scans: it must be the canonical mito engine and declare the
<a href=#68 id=68 data-nosnippet>68</a> /// `preserve_row_sequence` capability (a mito region option enforced to
<a href=#69 id=69 data-nosnippet>69</a> /// require append-only mode). When the capability is unknown — a source
<a href=#70 id=70 data-nosnippet>70</a> /// table that cannot be resolved, is not the mito engine, or lacks the
<a href=#71 id=71 data-nosnippet>71</a> /// option — this returns `false` so the caller keeps the historical
<a href=#72 id=72 data-nosnippet>72</a> /// `memtable_only` mode instead of upgrading.
<a href=#73 id=73 data-nosnippet>73</a> </span><span class="kw">pub async fn </span>sequence_range_capable(<span class="kw-2">&amp;</span><span class="self">self</span>) -&gt; <span class="prelude-ty">Result</span>&lt;bool, Error&gt; {
<a href=#74 id=74 data-nosnippet>74</a> <span class="kw">for </span>name <span class="kw">in </span><span class="kw-2">&amp;</span><span class="self">self</span>.config.source_table_names {
<a href=#75 id=75 data-nosnippet>75</a> <span class="kw">let </span>table = <span class="kw">match </span><span class="self">self
<a href=#76 id=76 data-nosnippet>76</a> </span>.config
<a href=#77 id=77 data-nosnippet>77</a> .catalog_manager
<a href=#78 id=78 data-nosnippet>78</a> .table(<span class="kw-2">&amp;</span>name[<span class="number">0</span>], <span class="kw-2">&amp;</span>name[<span class="number">1</span>], <span class="kw-2">&amp;</span>name[<span class="number">2</span>], <span class="prelude-val">None</span>)
<a href=#79 id=79 data-nosnippet>79</a> .<span class="kw">await
<a href=#80 id=80 data-nosnippet>80</a> </span>{
<a href=#81 id=81 data-nosnippet>81</a> <span class="prelude-val">Ok</span>(<span class="prelude-val">Some</span>(table)) =&gt; table,
<a href=#82 id=82 data-nosnippet>82</a> <span class="prelude-val">Ok</span>(<span class="prelude-val">None</span>) =&gt; {
<a href=#83 id=83 data-nosnippet>83</a> <span class="kw">return </span><span class="prelude-val">Err</span>(UnexpectedSnafu {
<a href=#84 id=84 data-nosnippet>84</a> reason: <span class="macro">format!</span>(
<a href=#85 id=85 data-nosnippet>85</a> <span class="string">"Flow {} source table {} not found for sequence_range capability check"</span>,
<a href=#86 id=86 data-nosnippet>86</a> <span class="self">self</span>.config.flow_id,
<a href=#87 id=87 data-nosnippet>87</a> name.join(<span class="string">"."</span>)
<a href=#88 id=88 data-nosnippet>88</a> ),
<a href=#89 id=89 data-nosnippet>89</a> }
<a href=#90 id=90 data-nosnippet>90</a> .build());
<a href=#91 id=91 data-nosnippet>91</a> }
<a href=#92 id=92 data-nosnippet>92</a> <span class="prelude-val">Err</span>(err) =&gt; <span class="prelude-val">Err</span>(BoxedError::new(err)).context(ExternalSnafu)<span class="question-mark">?</span>,
<a href=#93 id=93 data-nosnippet>93</a> };
<a href=#94 id=94 data-nosnippet>94</a>
<a href=#95 id=95 data-nosnippet>95</a> <span class="kw">let </span>info = table.table_info();
<a href=#96 id=96 data-nosnippet>96</a> <span class="kw">let </span>preserves = info.meta.engine == <span class="string">"mito"
<a href=#97 id=97 data-nosnippet>97</a> </span>&amp;&amp; info
<a href=#98 id=98 data-nosnippet>98</a> .meta
<a href=#99 id=99 data-nosnippet>99</a> .options
<a href=#100 id=100 data-nosnippet>100</a> .extra_options
<a href=#101 id=101 data-nosnippet>101</a> .get(PRESERVE_ROW_SEQUENCE)
<a href=#102 id=102 data-nosnippet>102</a> .is_some_and(|value| value.eq_ignore_ascii_case(<span class="string">"true"</span>));
<a href=#103 id=103 data-nosnippet>103</a> <span class="kw">if </span>!preserves {
<a href=#104 id=104 data-nosnippet>104</a> <span class="kw">return </span><span class="prelude-val">Ok</span>(<span class="bool-val">false</span>);
<a href=#105 id=105 data-nosnippet>105</a> }
<a href=#106 id=106 data-nosnippet>106</a> }
<a href=#107 id=107 data-nosnippet>107</a> <span class="prelude-val">Ok</span>(!<span class="self">self</span>.config.source_table_names.is_empty())
<a href=#108 id=108 data-nosnippet>108</a> }
<a href=#109 id=109 data-nosnippet>109</a>
<a href=#110 id=110 data-nosnippet>110</a> <span class="doccomment">/// For incremental-mode SQL queries, attempt to prepare an executable plan
<a href=#111 id=111 data-nosnippet>111</a> /// that is safe for incremental scan extensions.
<a href=#112 id=112 data-nosnippet>112</a> ///
<a href=#113 id=113 data-nosnippet>113</a> /// Returns `Some(plan)` when incremental extensions are safe. For an
<a href=#114 id=114 data-nosnippet>114</a> /// incremental-delta query, `None` means the caller must not execute the
<a href=#115 id=115 data-nosnippet>115</a> /// original plan without incremental extensions, because that would become
<a href=#116 id=116 data-nosnippet>116</a> /// an unfiltered full snapshot; the caller should restore the dirty signal
<a href=#117 id=117 data-nosnippet>117</a> /// and skip the current round instead. The returned plan may be either a
<a href=#118 id=118 data-nosnippet>118</a> /// rewritten delta-LEFT-JOIN-sink merge plan or the original plan. In
<a href=#119 id=119 data-nosnippet>119</a> /// particular, plain GROUP BY queries with no aggregate merge columns are
<a href=#120 id=120 data-nosnippet>120</a> /// incremental safe without a rewrite, so they return `Some(original_plan)`.
<a href=#121 id=121 data-nosnippet>121</a> </span><span class="kw">pub</span>(<span class="kw">super</span>) <span class="kw">async fn </span>prepare_plan_for_incremental(
<a href=#122 id=122 data-nosnippet>122</a> <span class="kw-2">&amp;</span><span class="self">self</span>,
<a href=#123 id=123 data-nosnippet>123</a> engine: <span class="kw-2">&amp;</span>QueryEngineRef,
<a href=#124 id=124 data-nosnippet>124</a> plan: <span class="kw-2">&amp;</span>LogicalPlan,
<a href=#125 id=125 data-nosnippet>125</a> ) -&gt; <span class="prelude-ty">Result</span>&lt;<span class="prelude-ty">Option</span>&lt;LogicalPlan&gt;, Error&gt; {
<a href=#126 id=126 data-nosnippet>126</a> <span class="kw">let </span>is_incremental_sql = {
<a href=#127 id=127 data-nosnippet>127</a> <span class="kw">let </span>state = <span class="self">self</span>.state.read().unwrap();
<a href=#128 id=128 data-nosnippet>128</a> <span class="kw">if </span>state.is_incremental_disabled() {
<a href=#129 id=129 data-nosnippet>129</a> <span class="kw">return </span><span class="prelude-val">Ok</span>(<span class="prelude-val">None</span>);
<a href=#130 id=130 data-nosnippet>130</a> }
<a href=#131 id=131 data-nosnippet>131</a> state.checkpoint_mode() == CheckpointMode::Incremental
<a href=#132 id=132 data-nosnippet>132</a> &amp;&amp; <span class="macro">matches!</span>(<span class="self">self</span>.config.query_type, QueryType::Sql)
<a href=#133 id=133 data-nosnippet>133</a> };
<a href=#134 id=134 data-nosnippet>134</a>
<a href=#135 id=135 data-nosnippet>135</a> <span class="kw">if </span>!is_incremental_sql {
<a href=#136 id=136 data-nosnippet>136</a> <span class="kw">return </span><span class="prelude-val">Ok</span>(<span class="prelude-val">None</span>);
<a href=#137 id=137 data-nosnippet>137</a> }
<a href=#138 id=138 data-nosnippet>138</a>
<a href=#139 id=139 data-nosnippet>139</a> <span class="comment">// Extract inner query plan from the DML wrapper.
<a href=#140 id=140 data-nosnippet>140</a> // Non-DML or non-SQL plans bypass the rewrite and keep checkpoint mode;
<a href=#141 id=141 data-nosnippet>141</a> // non-aggregate TQL or non-INSERT plans do not need incremental scan extensions.
<a href=#142 id=142 data-nosnippet>142</a> </span><span class="kw">let </span>inner_plan = <span class="kw">match </span>plan {
<a href=#143 id=143 data-nosnippet>143</a> LogicalPlan::Dml(dml) =&gt; dml.input.as_ref().clone(),
<a href=#144 id=144 data-nosnippet>144</a> <span class="kw">_ </span>=&gt; <span class="kw">return </span><span class="prelude-val">Ok</span>(<span class="prelude-val">None</span>),
<a href=#145 id=145 data-nosnippet>145</a> };
<a href=#146 id=146 data-nosnippet>146</a>
<a href=#147 id=147 data-nosnippet>147</a> <span class="comment">// Analyze the plan for incremental rewritability.
<a href=#148 id=148 data-nosnippet>148</a> // Incremental reads currently require aggregate / group-by plans that
<a href=#149 id=149 data-nosnippet>149</a> // can be rewritten into a delta-left-join-sink merge. Non-aggregate SQL
<a href=#150 id=150 data-nosnippet>150</a> // (projection, filter, or other non-aggregate shapes) stays full-snapshot
<a href=#151 id=151 data-nosnippet>151</a> // until separately supported, and incremental mode is permanently
<a href=#152 id=152 data-nosnippet>152</a> // disabled for this flow.
<a href=#153 id=153 data-nosnippet>153</a> </span><span class="kw">let </span><span class="prelude-val">Some</span>(analysis) = analyze_incremental_aggregate_plan(<span class="kw-2">&amp;</span>inner_plan)<span class="question-mark">? </span><span class="kw">else </span>{
<a href=#154 id=154 data-nosnippet>154</a> <span class="macro">warn!</span>(
<a href=#155 id=155 data-nosnippet>155</a> <span class="string">"Flow {} incremental mode but plan is not an aggregate query; \
<a href=#156 id=156 data-nosnippet>156</a> permanently disabling incremental for this flow"</span>,
<a href=#157 id=157 data-nosnippet>157</a> <span class="self">self</span>.config.flow_id
<a href=#158 id=158 data-nosnippet>158</a> );
<a href=#159 id=159 data-nosnippet>159</a> <span class="kw">if </span><span class="self">self</span>.config.exact_sequence_range_required {
<a href=#160 id=160 data-nosnippet>160</a> <span class="kw">return </span><span class="prelude-val">Err</span>(UnexpectedSnafu {
<a href=#161 id=161 data-nosnippet>161</a> reason: <span class="macro">format!</span>(
<a href=#162 id=162 data-nosnippet>162</a> <span class="string">"Flow {} requires exact sequence-range reads, but its incremental plan is unsupported"</span>,
<a href=#163 id=163 data-nosnippet>163</a> <span class="self">self</span>.config.flow_id
<a href=#164 id=164 data-nosnippet>164</a> ),
<a href=#165 id=165 data-nosnippet>165</a> }
<a href=#166 id=166 data-nosnippet>166</a> .build());
<a href=#167 id=167 data-nosnippet>167</a> }
<a href=#168 id=168 data-nosnippet>168</a> <span class="self">self</span>.state.write().unwrap().disable_incremental();
<a href=#169 id=169 data-nosnippet>169</a> <span class="kw">return </span><span class="prelude-val">Ok</span>(<span class="prelude-val">None</span>);
<a href=#170 id=170 data-nosnippet>170</a> };
<a href=#171 id=171 data-nosnippet>171</a>
<a href=#172 id=172 data-nosnippet>172</a> <span class="kw">if </span>!analysis.unsupported_exprs.is_empty() {
<a href=#173 id=173 data-nosnippet>173</a> <span class="macro">warn!</span>(
<a href=#174 id=174 data-nosnippet>174</a> <span class="string">"Flow {} incremental aggregate contains unsupported expressions {:?}; \
<a href=#175 id=175 data-nosnippet>175</a> permanently disabling incremental for this flow"</span>,
<a href=#176 id=176 data-nosnippet>176</a> <span class="self">self</span>.config.flow_id, analysis.unsupported_exprs
<a href=#177 id=177 data-nosnippet>177</a> );
<a href=#178 id=178 data-nosnippet>178</a> <span class="kw">if </span><span class="self">self</span>.config.exact_sequence_range_required {
<a href=#179 id=179 data-nosnippet>179</a> <span class="kw">return </span><span class="prelude-val">Err</span>(UnexpectedSnafu {
<a href=#180 id=180 data-nosnippet>180</a> reason: <span class="macro">format!</span>(
<a href=#181 id=181 data-nosnippet>181</a> <span class="string">"Flow {} requires exact sequence-range reads, but its incremental aggregate is unsupported: {:?}"</span>,
<a href=#182 id=182 data-nosnippet>182</a> <span class="self">self</span>.config.flow_id, analysis.unsupported_exprs
<a href=#183 id=183 data-nosnippet>183</a> ),
<a href=#184 id=184 data-nosnippet>184</a> }
<a href=#185 id=185 data-nosnippet>185</a> .build());
<a href=#186 id=186 data-nosnippet>186</a> }
<a href=#187 id=187 data-nosnippet>187</a> <span class="self">self</span>.state.write().unwrap().disable_incremental();
<a href=#188 id=188 data-nosnippet>188</a> <span class="kw">return </span><span class="prelude-val">Ok</span>(<span class="prelude-val">None</span>);
<a href=#189 id=189 data-nosnippet>189</a> }
<a href=#190 id=190 data-nosnippet>190</a>
<a href=#191 id=191 data-nosnippet>191</a> <span class="comment">// Plain GROUP BY without aggregate expressions has no values to
<a href=#192 id=192 data-nosnippet>192</a> // merge between delta and sink. The incremental delta scan emits
<a href=#193 id=193 data-nosnippet>193</a> // changed groups, and sink primary-key write semantics make this
<a href=#194 id=194 data-nosnippet>194</a> // idempotent; no explicit left-join rewrite is needed.
<a href=#195 id=195 data-nosnippet>195</a> </span><span class="kw">if </span>analysis.merge_columns.is_empty() {
<a href=#196 id=196 data-nosnippet>196</a> <span class="kw">return </span><span class="prelude-val">Ok</span>(<span class="prelude-val">Some</span>(plan.clone()));
<a href=#197 id=197 data-nosnippet>197</a> }
<a href=#198 id=198 data-nosnippet>198</a>
<a href=#199 id=199 data-nosnippet>199</a> <span class="comment">// Fetch sink table for the merge rewrite.
<a href=#200 id=200 data-nosnippet>200</a> // Transient errors (catalog, schema, filter, or rewrite) should not
<a href=#201 id=201 data-nosnippet>201</a> // permanently disable incremental mode. They also must not execute the
<a href=#202 id=202 data-nosnippet>202</a> // original plan without incremental extensions, because that would be an
<a href=#203 id=203 data-nosnippet>203</a> // unfiltered full snapshot. The caller will restore the dirty signal and
<a href=#204 id=204 data-nosnippet>204</a> // skip this round while keeping incremental retryable.
<a href=#205 id=205 data-nosnippet>205</a> </span><span class="kw">let </span>sink_table = <span class="kw">match </span>get_table_info_df_schema(
<a href=#206 id=206 data-nosnippet>206</a> <span class="self">self</span>.config.catalog_manager.clone(),
<a href=#207 id=207 data-nosnippet>207</a> <span class="self">self</span>.config.sink_table_name.clone(),
<a href=#208 id=208 data-nosnippet>208</a> )
<a href=#209 id=209 data-nosnippet>209</a> .<span class="kw">await
<a href=#210 id=210 data-nosnippet>210</a> </span>{
<a href=#211 id=211 data-nosnippet>211</a> <span class="prelude-val">Ok</span>((table, <span class="kw">_</span>)) =&gt; table,
<a href=#212 id=212 data-nosnippet>212</a> <span class="prelude-val">Err</span>(err) =&gt; {
<a href=#213 id=213 data-nosnippet>213</a> <span class="macro">warn!</span>(
<a href=#214 id=214 data-nosnippet>214</a> <span class="string">"Flow {} failed to fetch sink table for incremental rewrite; \
<a href=#215 id=215 data-nosnippet>215</a> skipping this round to avoid unfiltered full snapshot: {:?}"</span>,
<a href=#216 id=216 data-nosnippet>216</a> <span class="self">self</span>.config.flow_id, err
<a href=#217 id=217 data-nosnippet>217</a> );
<a href=#218 id=218 data-nosnippet>218</a> <span class="kw">return </span><span class="prelude-val">Ok</span>(<span class="prelude-val">None</span>);
<a href=#219 id=219 data-nosnippet>219</a> }
<a href=#220 id=220 data-nosnippet>220</a> };
<a href=#221 id=221 data-nosnippet>221</a> <span class="kw">let </span>rewritten_inner = <span class="kw">match </span>rewrite_incremental_aggregate_with_sink_merge(
<a href=#222 id=222 data-nosnippet>222</a> <span class="kw-2">&amp;</span>inner_plan,
<a href=#223 id=223 data-nosnippet>223</a> <span class="kw-2">&amp;</span>analysis,
<a href=#224 id=224 data-nosnippet>224</a> engine,
<a href=#225 id=225 data-nosnippet>225</a> sink_table,
<a href=#226 id=226 data-nosnippet>226</a> <span class="kw-2">&amp;</span><span class="self">self</span>.config.sink_table_name,
<a href=#227 id=227 data-nosnippet>227</a> <span class="prelude-val">None</span>,
<a href=#228 id=228 data-nosnippet>228</a> )
<a href=#229 id=229 data-nosnippet>229</a> .<span class="kw">await
<a href=#230 id=230 data-nosnippet>230</a> </span>{
<a href=#231 id=231 data-nosnippet>231</a> <span class="prelude-val">Ok</span>(plan) =&gt; plan,
<a href=#232 id=232 data-nosnippet>232</a> <span class="prelude-val">Err</span>(err) =&gt; {
<a href=#233 id=233 data-nosnippet>233</a> <span class="macro">warn!</span>(
<a href=#234 id=234 data-nosnippet>234</a> <span class="string">"Flow {} failed to rewrite incremental aggregate with sink merge; \
<a href=#235 id=235 data-nosnippet>235</a> skipping this round to avoid unfiltered full snapshot: {:?}"</span>,
<a href=#236 id=236 data-nosnippet>236</a> <span class="self">self</span>.config.flow_id, err
<a href=#237 id=237 data-nosnippet>237</a> );
<a href=#238 id=238 data-nosnippet>238</a> <span class="kw">return </span><span class="prelude-val">Ok</span>(<span class="prelude-val">None</span>);
<a href=#239 id=239 data-nosnippet>239</a> }
<a href=#240 id=240 data-nosnippet>240</a> };
<a href=#241 id=241 data-nosnippet>241</a>
<a href=#242 id=242 data-nosnippet>242</a> <span class="comment">// Reconstruct DML plan with the rewritten inner plan
<a href=#243 id=243 data-nosnippet>243</a> </span><span class="kw">let </span>rewritten = <span class="kw">match </span>plan {
<a href=#244 id=244 data-nosnippet>244</a> LogicalPlan::Dml(dml) =&gt; LogicalPlan::Dml(DmlStatement::new(
<a href=#245 id=245 data-nosnippet>245</a> dml.table_name.clone(),
<a href=#246 id=246 data-nosnippet>246</a> dml.target.clone(),
<a href=#247 id=247 data-nosnippet>247</a> dml.op.clone(),
<a href=#248 id=248 data-nosnippet>248</a> Arc::new(rewritten_inner),
<a href=#249 id=249 data-nosnippet>249</a> )),
<a href=#250 id=250 data-nosnippet>250</a> <span class="kw">_ </span>=&gt; <span class="macro">unreachable!</span>(<span class="string">"already matched Dml above"</span>),
<a href=#251 id=251 data-nosnippet>251</a> };
<a href=#252 id=252 data-nosnippet>252</a>
<a href=#253 id=253 data-nosnippet>253</a> <span class="macro">debug!</span>(
<a href=#254 id=254 data-nosnippet>254</a> <span class="string">"Flow {} rewrote incremental SQL aggregate query with sink merge"</span>,
<a href=#255 id=255 data-nosnippet>255</a> <span class="self">self</span>.config.flow_id
<a href=#256 id=256 data-nosnippet>256</a> );
<a href=#257 id=257 data-nosnippet>257</a>
<a href=#258 id=258 data-nosnippet>258</a> <span class="prelude-val">Ok</span>(<span class="prelude-val">Some</span>(rewritten))
<a href=#259 id=259 data-nosnippet>259</a> }
<a href=#260 id=260 data-nosnippet>260</a>
<a href=#261 id=261 data-nosnippet>261</a> <span class="kw">pub</span>(<span class="kw">super</span>) <span class="kw">async fn </span>build_flow_query_extensions(
<a href=#262 id=262 data-nosnippet>262</a> <span class="kw-2">&amp;</span><span class="self">self</span>,
<a href=#263 id=263 data-nosnippet>263</a> incremental_safe: bool,
<a href=#264 id=264 data-nosnippet>264</a> can_advance_checkpoints: bool,
<a href=#265 id=265 data-nosnippet>265</a> ) -&gt; <span class="prelude-ty">Result</span>&lt;Vec&lt;(<span class="kw-2">&amp;</span><span class="lifetime">'static </span>str, String)&gt;, Error&gt; {
<a href=#266 id=266 data-nosnippet>266</a> <span class="kw">let </span><span class="kw-2">mut </span>extensions = <span class="macro">vec!</span>[(<span class="string">"flow.return_region_seq"</span>, <span class="string">"true"</span>.to_string())];
<a href=#267 id=267 data-nosnippet>267</a>
<a href=#268 id=268 data-nosnippet>268</a> <span class="kw">let </span>incremental_checkpoints_json = {
<a href=#269 id=269 data-nosnippet>269</a> <span class="kw">let </span>state = <span class="self">self</span>.state.read().unwrap();
<a href=#270 id=270 data-nosnippet>270</a> <span class="kw">if </span>incremental_safe
<a href=#271 id=271 data-nosnippet>271</a> &amp;&amp; can_advance_checkpoints
<a href=#272 id=272 data-nosnippet>272</a> &amp;&amp; !state.is_incremental_disabled()
<a href=#273 id=273 data-nosnippet>273</a> &amp;&amp; state.checkpoint_mode() == CheckpointMode::Incremental
<a href=#274 id=274 data-nosnippet>274</a> &amp;&amp; !state.checkpoints().is_empty()
<a href=#275 id=275 data-nosnippet>275</a> {
<a href=#276 id=276 data-nosnippet>276</a> <span class="prelude-val">Some</span>(serde_json::to_string(state.checkpoints()).map_err(|err| {
<a href=#277 id=277 data-nosnippet>277</a> UnexpectedSnafu {
<a href=#278 id=278 data-nosnippet>278</a> reason: <span class="macro">format!</span>(<span class="string">"Failed to serialize checkpoint map: {err}"</span>),
<a href=#279 id=279 data-nosnippet>279</a> }
<a href=#280 id=280 data-nosnippet>280</a> .build()
<a href=#281 id=281 data-nosnippet>281</a> })<span class="question-mark">?</span>)
<a href=#282 id=282 data-nosnippet>282</a> } <span class="kw">else </span>{
<a href=#283 id=283 data-nosnippet>283</a> <span class="prelude-val">None
<a href=#284 id=284 data-nosnippet>284</a> </span>}
<a href=#285 id=285 data-nosnippet>285</a> };
<a href=#286 id=286 data-nosnippet>286</a>
<a href=#287 id=287 data-nosnippet>287</a> <span class="kw">if let </span><span class="prelude-val">Some</span>(checkpoints_json) = incremental_checkpoints_json {
<a href=#288 id=288 data-nosnippet>288</a> <span class="comment">// Select `sequence_range` only when every append-only source table
<a href=#289 id=289 data-nosnippet>289</a> // proves the `preserve_row_sequence` capability; otherwise retain
<a href=#290 id=290 data-nosnippet>290</a> // the historical `memtable_only` mode. The `sequence_range` scan
<a href=#291 id=291 data-nosnippet>291</a> // keeps SSTs and reads the exact (checkpoint, scan-open snapshot]
<a href=#292 id=292 data-nosnippet>292</a> // row-level delta; the engine fails closed when the capability
<a href=#293 id=293 data-nosnippet>293</a> // does not hold at scan time.
<a href=#294 id=294 data-nosnippet>294</a> </span><span class="kw">let </span>capable = <span class="kw">match </span><span class="self">self</span>.sequence_range_capable().<span class="kw">await </span>{
<a href=#295 id=295 data-nosnippet>295</a> <span class="prelude-val">Ok</span>(capable) =&gt; capable,
<a href=#296 id=296 data-nosnippet>296</a> <span class="prelude-val">Err</span>(err) =&gt; {
<a href=#297 id=297 data-nosnippet>297</a> <span class="kw">if </span><span class="self">self</span>.config.exact_sequence_range_required {
<a href=#298 id=298 data-nosnippet>298</a> <span class="kw">return </span><span class="prelude-val">Err</span>(err);
<a href=#299 id=299 data-nosnippet>299</a> }
<a href=#300 id=300 data-nosnippet>300</a> <span class="bool-val">false
<a href=#301 id=301 data-nosnippet>301</a> </span>}
<a href=#302 id=302 data-nosnippet>302</a> };
<a href=#303 id=303 data-nosnippet>303</a> <span class="kw">if </span><span class="self">self</span>.config.exact_sequence_range_required &amp;&amp; !capable {
<a href=#304 id=304 data-nosnippet>304</a> <span class="kw">return </span><span class="prelude-val">Err</span>(UnexpectedSnafu {
<a href=#305 id=305 data-nosnippet>305</a> reason: <span class="macro">format!</span>(
<a href=#306 id=306 data-nosnippet>306</a> <span class="string">"Flow {} requires exact sequence-range reads, but source capability was revoked"</span>,
<a href=#307 id=307 data-nosnippet>307</a> <span class="self">self</span>.config.flow_id
<a href=#308 id=308 data-nosnippet>308</a> ),
<a href=#309 id=309 data-nosnippet>309</a> }
<a href=#310 id=310 data-nosnippet>310</a> .build());
<a href=#311 id=311 data-nosnippet>311</a> }
<a href=#312 id=312 data-nosnippet>312</a> <span class="kw">let </span>sink_table_id = <span class="self">self</span>.sink_table_id().<span class="kw">await</span><span class="question-mark">?</span>;
<a href=#313 id=313 data-nosnippet>313</a> <span class="kw">let </span>incremental_mode = <span class="kw">if </span>capable {
<a href=#314 id=314 data-nosnippet>314</a> <span class="macro">debug!</span>(
<a href=#315 id=315 data-nosnippet>315</a> <span class="string">"Flow {} selected sequence_range incremental mode"</span>,
<a href=#316 id=316 data-nosnippet>316</a> <span class="self">self</span>.config.flow_id
<a href=#317 id=317 data-nosnippet>317</a> );
<a href=#318 id=318 data-nosnippet>318</a> FLOW_INCREMENTAL_MODE_SEQUENCE_RANGE
<a href=#319 id=319 data-nosnippet>319</a> } <span class="kw">else </span>{
<a href=#320 id=320 data-nosnippet>320</a> FLOW_INCREMENTAL_MODE_MEMTABLE_ONLY
<a href=#321 id=321 data-nosnippet>321</a> };
<a href=#322 id=322 data-nosnippet>322</a> extensions.push((FLOW_SINK_TABLE_ID, sink_table_id.to_string()));
<a href=#323 id=323 data-nosnippet>323</a> extensions.push((FLOW_INCREMENTAL_MODE, incremental_mode.to_string()));
<a href=#324 id=324 data-nosnippet>324</a> extensions.push((FLOW_INCREMENTAL_AFTER_SEQS, checkpoints_json));
<a href=#325 id=325 data-nosnippet>325</a> }
<a href=#326 id=326 data-nosnippet>326</a>
<a href=#327 id=327 data-nosnippet>327</a> <span class="prelude-val">Ok</span>(extensions)
<a href=#328 id=328 data-nosnippet>328</a> }
<a href=#329 id=329 data-nosnippet>329</a>}
</code></pre></div></section></main></body></html>
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
+160
View File
@@ -0,0 +1,160 @@
<!DOCTYPE html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1.0"><meta name="generator" content="rustdoc"><meta name="description" content="Source of the Rust file `src/flow/src/df_optimizer.rs`."><title>df_optimizer.rs - source</title><script>if(window.location.protocol!=="file:")document.head.insertAdjacentHTML("beforeend","SourceSerif4-Regular-6b053e98.ttf.woff2,FiraSans-Italic-81dc35de.woff2,FiraSans-Regular-0fe48ade.woff2,FiraSans-MediumItalic-ccf7e434.woff2,FiraSans-Medium-e1aa3f0a.woff2,SourceCodePro-Regular-8badfe75.ttf.woff2,SourceCodePro-Semibold-aa29a496.ttf.woff2".split(",").map(f=>`<link rel="preload" as="font" type="font/woff2"href="../../static.files/${f}">`).join(""))</script><link rel="stylesheet" href="../../static.files/normalize-9960930a.css"><link rel="stylesheet" href="../../static.files/rustdoc-17e0aaed.css"><meta name="rustdoc-vars" data-root-path="../../" data-static-root-path="../../static.files/" data-current-crate="flow" data-themes="" data-resource-suffix="" data-rustdoc-version="1.96.1 (31fca3adb 2026-06-26)" data-channel="1.96.1" data-search-js="search-b5634cc7.js" data-stringdex-js="stringdex-2da4960a.js" data-settings-js="settings-170eb4bf.js" ><script src="../../static.files/storage-41dd4d93.js"></script><script defer src="../../static.files/src-script-813739b1.js"></script><script defer src="../../src-files.js"></script><script defer src="../../static.files/main-5013f961.js"></script><noscript><link rel="stylesheet" href="../../static.files/noscript-f7c3ffd8.css"></noscript><link rel="alternate icon" type="image/png" href="../../static.files/favicon-32x32-eab170b8.png"><link rel="icon" type="image/svg+xml" href="../../static.files/favicon-044be391.svg"></head><body class="rustdoc src"><a class="skip-main-content" href="#main-content">Skip to main content</a><!--[if lte IE 11]><div class="warning">This old browser is unsupported and will most likely display funky things.</div><![endif]--><nav class="sidebar"><div class="src-sidebar-title"><h2>Files</h2></div></nav><div class="sidebar-resizer" title="Drag to resize sidebar"></div><main><section id="main-content" class="content" tabindex="-1"><div class="main-heading"><h1><div class="sub-heading">flow/</div>df_optimizer.rs</h1><rustdoc-toolbar></rustdoc-toolbar></div><div class="example-wrap digits-3"><pre class="rust"><code><a href=#1 id=1 data-nosnippet>1</a><span class="comment">// Copyright 2023 Greptime Team
<a href=#2 id=2 data-nosnippet>2</a>//
<a href=#3 id=3 data-nosnippet>3</a>// Licensed under the Apache License, Version 2.0 (the "License");
<a href=#4 id=4 data-nosnippet>4</a>// you may not use this file except in compliance with the License.
<a href=#5 id=5 data-nosnippet>5</a>// You may obtain a copy of the License at
<a href=#6 id=6 data-nosnippet>6</a>//
<a href=#7 id=7 data-nosnippet>7</a>// http://www.apache.org/licenses/LICENSE-2.0
<a href=#8 id=8 data-nosnippet>8</a>//
<a href=#9 id=9 data-nosnippet>9</a>// Unless required by applicable law or agreed to in writing, software
<a href=#10 id=10 data-nosnippet>10</a>// distributed under the License is distributed on an "AS IS" BASIS,
<a href=#11 id=11 data-nosnippet>11</a>// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
<a href=#12 id=12 data-nosnippet>12</a>// See the License for the specific language governing permissions and
<a href=#13 id=13 data-nosnippet>13</a>// limitations under the License.
<a href=#14 id=14 data-nosnippet>14</a>
<a href=#15 id=15 data-nosnippet>15</a></span><span class="doccomment">//! Datafusion optimizer for flow plan
<a href=#16 id=16 data-nosnippet>16</a>
<a href=#17 id=17 data-nosnippet>17</a></span><span class="attr">#![warn(unused)]
<a href=#18 id=18 data-nosnippet>18</a>
<a href=#19 id=19 data-nosnippet>19</a></span><span class="kw">use </span>std::collections::HashSet;
<a href=#20 id=20 data-nosnippet>20</a><span class="kw">use </span>std::sync::Arc;
<a href=#21 id=21 data-nosnippet>21</a>
<a href=#22 id=22 data-nosnippet>22</a><span class="kw">use </span>common_error::ext::BoxedError;
<a href=#23 id=23 data-nosnippet>23</a><span class="kw">use </span>common_telemetry::debug;
<a href=#24 id=24 data-nosnippet>24</a><span class="kw">use </span>datafusion::config::ConfigOptions;
<a href=#25 id=25 data-nosnippet>25</a><span class="kw">use </span>datafusion::error::DataFusionError;
<a href=#26 id=26 data-nosnippet>26</a><span class="kw">use </span>datafusion::optimizer::analyzer::type_coercion::TypeCoercion;
<a href=#27 id=27 data-nosnippet>27</a><span class="kw">use </span>datafusion::optimizer::common_subexpr_eliminate::CommonSubexprEliminate;
<a href=#28 id=28 data-nosnippet>28</a><span class="kw">use </span>datafusion::optimizer::optimize_projections::OptimizeProjections;
<a href=#29 id=29 data-nosnippet>29</a><span class="kw">use </span>datafusion::optimizer::simplify_expressions::SimplifyExpressions;
<a href=#30 id=30 data-nosnippet>30</a><span class="kw">use </span>datafusion::optimizer::{Analyzer, AnalyzerRule, Optimizer, OptimizerContext};
<a href=#31 id=31 data-nosnippet>31</a><span class="kw">use </span>datafusion_common::tree_node::{Transformed, TreeNode, TreeNodeRecursion, TreeNodeVisitor};
<a href=#32 id=32 data-nosnippet>32</a><span class="kw">use </span>query::optimizer::count_wildcard::CountWildcardToTimeIndexRule;
<a href=#33 id=33 data-nosnippet>33</a><span class="kw">use </span>session::context::QueryContextRef;
<a href=#34 id=34 data-nosnippet>34</a><span class="kw">use </span>snafu::ResultExt;
<a href=#35 id=35 data-nosnippet>35</a>
<a href=#36 id=36 data-nosnippet>36</a><span class="doccomment">/// note here we are using the `substrait_proto_df` crate from the `substrait` module and
<a href=#37 id=37 data-nosnippet>37</a>/// rename it to `substrait_proto`
<a href=#38 id=38 data-nosnippet>38</a></span><span class="kw">use </span><span class="kw">crate</span>::error::{DatafusionSnafu, Error, ExternalSnafu};
<a href=#39 id=39 data-nosnippet>39</a>
<a href=#40 id=40 data-nosnippet>40</a><span class="comment">// TODO(discord9): use `Analyzer` to manage rules if more `AnalyzerRule` is needed
<a href=#41 id=41 data-nosnippet>41</a></span><span class="kw">pub async fn </span>apply_df_optimizer(
<a href=#42 id=42 data-nosnippet>42</a> plan: datafusion_expr::LogicalPlan,
<a href=#43 id=43 data-nosnippet>43</a> query_ctx: <span class="kw-2">&amp;</span>QueryContextRef,
<a href=#44 id=44 data-nosnippet>44</a>) -&gt; <span class="prelude-ty">Result</span>&lt;datafusion_expr::LogicalPlan, Error&gt; {
<a href=#45 id=45 data-nosnippet>45</a> <span class="kw">let </span>cfg = query_ctx.create_config_options();
<a href=#46 id=46 data-nosnippet>46</a> <span class="kw">let </span>analyzer = Analyzer::with_rules(<span class="macro">vec!</span>[
<a href=#47 id=47 data-nosnippet>47</a> Arc::new(CountWildcardToTimeIndexRule),
<a href=#48 id=48 data-nosnippet>48</a> Arc::new(CheckGroupByRule::new()),
<a href=#49 id=49 data-nosnippet>49</a> Arc::new(TypeCoercion::new()),
<a href=#50 id=50 data-nosnippet>50</a> ]);
<a href=#51 id=51 data-nosnippet>51</a> <span class="kw">let </span>plan = analyzer
<a href=#52 id=52 data-nosnippet>52</a> .execute_and_check(plan, <span class="kw-2">&amp;</span>cfg, |p, r| {
<a href=#53 id=53 data-nosnippet>53</a> <span class="macro">debug!</span>(<span class="string">"After apply rule {}, get plan: \n{:?}"</span>, r.name(), p);
<a href=#54 id=54 data-nosnippet>54</a> })
<a href=#55 id=55 data-nosnippet>55</a> .context(DatafusionSnafu {
<a href=#56 id=56 data-nosnippet>56</a> context: <span class="string">"Fail to apply analyzer"</span>,
<a href=#57 id=57 data-nosnippet>57</a> })<span class="question-mark">?</span>;
<a href=#58 id=58 data-nosnippet>58</a>
<a href=#59 id=59 data-nosnippet>59</a> <span class="kw">let </span><span class="kw-2">mut </span>ctx = OptimizerContext::new();
<a href=#60 id=60 data-nosnippet>60</a> <span class="kw">let </span>scheduled_time = query::options::parse_scheduled_time_datetime(<span class="kw-2">&amp;</span>query_ctx.extensions())
<a href=#61 id=61 data-nosnippet>61</a> .map_err(BoxedError::new)
<a href=#62 id=62 data-nosnippet>62</a> .context(ExternalSnafu)<span class="question-mark">?</span>;
<a href=#63 id=63 data-nosnippet>63</a> <span class="kw">if let </span><span class="prelude-val">Some</span>(dt) = scheduled_time {
<a href=#64 id=64 data-nosnippet>64</a> ctx = ctx.with_query_execution_start_time(dt);
<a href=#65 id=65 data-nosnippet>65</a> }
<a href=#66 id=66 data-nosnippet>66</a> <span class="kw">let </span>optimizer = Optimizer::with_rules(<span class="macro">vec!</span>[
<a href=#67 id=67 data-nosnippet>67</a> Arc::new(OptimizeProjections::new()),
<a href=#68 id=68 data-nosnippet>68</a> Arc::new(CommonSubexprEliminate::new()),
<a href=#69 id=69 data-nosnippet>69</a> Arc::new(SimplifyExpressions::new()),
<a href=#70 id=70 data-nosnippet>70</a> ]);
<a href=#71 id=71 data-nosnippet>71</a> <span class="kw">let </span>plan = optimizer
<a href=#72 id=72 data-nosnippet>72</a> .optimize(plan, <span class="kw-2">&amp;</span>ctx, |<span class="kw">_</span>, <span class="kw">_</span>| {})
<a href=#73 id=73 data-nosnippet>73</a> .context(DatafusionSnafu {
<a href=#74 id=74 data-nosnippet>74</a> context: <span class="string">"Fail to apply optimizer"</span>,
<a href=#75 id=75 data-nosnippet>75</a> })<span class="question-mark">?</span>;
<a href=#76 id=76 data-nosnippet>76</a>
<a href=#77 id=77 data-nosnippet>77</a> <span class="prelude-val">Ok</span>(plan)
<a href=#78 id=78 data-nosnippet>78</a>}
<a href=#79 id=79 data-nosnippet>79</a>
<a href=#80 id=80 data-nosnippet>80</a><span class="doccomment">/// This rule check all group by exprs, and make sure they are also in select clause in a aggr query
<a href=#81 id=81 data-nosnippet>81</a></span><span class="attr">#[derive(Debug)]
<a href=#82 id=82 data-nosnippet>82</a></span><span class="kw">struct </span>CheckGroupByRule {}
<a href=#83 id=83 data-nosnippet>83</a>
<a href=#84 id=84 data-nosnippet>84</a><span class="kw">impl </span>CheckGroupByRule {
<a href=#85 id=85 data-nosnippet>85</a> <span class="kw">pub fn </span>new() -&gt; <span class="self">Self </span>{
<a href=#86 id=86 data-nosnippet>86</a> <span class="self">Self </span>{}
<a href=#87 id=87 data-nosnippet>87</a> }
<a href=#88 id=88 data-nosnippet>88</a>}
<a href=#89 id=89 data-nosnippet>89</a>
<a href=#90 id=90 data-nosnippet>90</a><span class="kw">impl </span>AnalyzerRule <span class="kw">for </span>CheckGroupByRule {
<a href=#91 id=91 data-nosnippet>91</a> <span class="kw">fn </span>analyze(
<a href=#92 id=92 data-nosnippet>92</a> <span class="kw-2">&amp;</span><span class="self">self</span>,
<a href=#93 id=93 data-nosnippet>93</a> plan: datafusion_expr::LogicalPlan,
<a href=#94 id=94 data-nosnippet>94</a> _config: <span class="kw-2">&amp;</span>ConfigOptions,
<a href=#95 id=95 data-nosnippet>95</a> ) -&gt; datafusion_common::Result&lt;datafusion_expr::LogicalPlan&gt; {
<a href=#96 id=96 data-nosnippet>96</a> <span class="kw">let </span>transformed = plan
<a href=#97 id=97 data-nosnippet>97</a> .transform_up_with_subqueries(check_group_by_analyzer)<span class="question-mark">?
<a href=#98 id=98 data-nosnippet>98</a> </span>.data;
<a href=#99 id=99 data-nosnippet>99</a> <span class="prelude-val">Ok</span>(transformed)
<a href=#100 id=100 data-nosnippet>100</a> }
<a href=#101 id=101 data-nosnippet>101</a>
<a href=#102 id=102 data-nosnippet>102</a> <span class="kw">fn </span>name(<span class="kw-2">&amp;</span><span class="self">self</span>) -&gt; <span class="kw-2">&amp;</span>str {
<a href=#103 id=103 data-nosnippet>103</a> <span class="string">"check_groupby"
<a href=#104 id=104 data-nosnippet>104</a> </span>}
<a href=#105 id=105 data-nosnippet>105</a>}
<a href=#106 id=106 data-nosnippet>106</a>
<a href=#107 id=107 data-nosnippet>107</a><span class="doccomment">/// make sure everything in group by's expr is in select
<a href=#108 id=108 data-nosnippet>108</a></span><span class="kw">fn </span>check_group_by_analyzer(
<a href=#109 id=109 data-nosnippet>109</a> plan: datafusion_expr::LogicalPlan,
<a href=#110 id=110 data-nosnippet>110</a>) -&gt; <span class="prelude-ty">Result</span>&lt;Transformed&lt;datafusion_expr::LogicalPlan&gt;, DataFusionError&gt; {
<a href=#111 id=111 data-nosnippet>111</a> <span class="kw">if let </span>datafusion_expr::LogicalPlan::Projection(proj) = <span class="kw-2">&amp;</span>plan
<a href=#112 id=112 data-nosnippet>112</a> &amp;&amp; <span class="kw">let </span>datafusion_expr::LogicalPlan::Aggregate(aggr) = proj.input.as_ref()
<a href=#113 id=113 data-nosnippet>113</a> {
<a href=#114 id=114 data-nosnippet>114</a> <span class="kw">let </span><span class="kw-2">mut </span>found_column_used = FindColumn::new();
<a href=#115 id=115 data-nosnippet>115</a> proj.expr
<a href=#116 id=116 data-nosnippet>116</a> .iter()
<a href=#117 id=117 data-nosnippet>117</a> .map(|i| i.visit(<span class="kw-2">&amp;mut </span>found_column_used))
<a href=#118 id=118 data-nosnippet>118</a> .count();
<a href=#119 id=119 data-nosnippet>119</a> <span class="kw">for </span>expr <span class="kw">in </span>aggr.group_expr.iter() {
<a href=#120 id=120 data-nosnippet>120</a> <span class="kw">if </span>!found_column_used
<a href=#121 id=121 data-nosnippet>121</a> .names_for_alias
<a href=#122 id=122 data-nosnippet>122</a> .contains(<span class="kw-2">&amp;</span>expr.name_for_alias()<span class="question-mark">?</span>)
<a href=#123 id=123 data-nosnippet>123</a> {
<a href=#124 id=124 data-nosnippet>124</a> <span class="kw">return </span><span class="prelude-val">Err</span>(DataFusionError::Plan(<span class="macro">format!</span>(
<a href=#125 id=125 data-nosnippet>125</a> <span class="string">"Expect {} expr in group by also exist in select list, but select list only contain {:?}"</span>,
<a href=#126 id=126 data-nosnippet>126</a> expr.name_for_alias()<span class="question-mark">?</span>,
<a href=#127 id=127 data-nosnippet>127</a> found_column_used.names_for_alias
<a href=#128 id=128 data-nosnippet>128</a> )));
<a href=#129 id=129 data-nosnippet>129</a> }
<a href=#130 id=130 data-nosnippet>130</a> }
<a href=#131 id=131 data-nosnippet>131</a> }
<a href=#132 id=132 data-nosnippet>132</a>
<a href=#133 id=133 data-nosnippet>133</a> <span class="prelude-val">Ok</span>(Transformed::no(plan))
<a href=#134 id=134 data-nosnippet>134</a>}
<a href=#135 id=135 data-nosnippet>135</a>
<a href=#136 id=136 data-nosnippet>136</a><span class="doccomment">/// Find all column names in a plan
<a href=#137 id=137 data-nosnippet>137</a></span><span class="attr">#[derive(Debug, Default)]
<a href=#138 id=138 data-nosnippet>138</a></span><span class="kw">struct </span>FindColumn {
<a href=#139 id=139 data-nosnippet>139</a> names_for_alias: HashSet&lt;String&gt;,
<a href=#140 id=140 data-nosnippet>140</a>}
<a href=#141 id=141 data-nosnippet>141</a>
<a href=#142 id=142 data-nosnippet>142</a><span class="kw">impl </span>FindColumn {
<a href=#143 id=143 data-nosnippet>143</a> <span class="kw">fn </span>new() -&gt; <span class="self">Self </span>{
<a href=#144 id=144 data-nosnippet>144</a> Default::default()
<a href=#145 id=145 data-nosnippet>145</a> }
<a href=#146 id=146 data-nosnippet>146</a>}
<a href=#147 id=147 data-nosnippet>147</a>
<a href=#148 id=148 data-nosnippet>148</a><span class="kw">impl </span>TreeNodeVisitor&lt;<span class="lifetime">'_</span>&gt; <span class="kw">for </span>FindColumn {
<a href=#149 id=149 data-nosnippet>149</a> <span class="kw">type </span>Node = datafusion_expr::Expr;
<a href=#150 id=150 data-nosnippet>150</a> <span class="kw">fn </span>f_down(
<a href=#151 id=151 data-nosnippet>151</a> <span class="kw-2">&amp;mut </span><span class="self">self</span>,
<a href=#152 id=152 data-nosnippet>152</a> node: <span class="kw-2">&amp;</span>datafusion_expr::Expr,
<a href=#153 id=153 data-nosnippet>153</a> ) -&gt; <span class="prelude-ty">Result</span>&lt;TreeNodeRecursion, DataFusionError&gt; {
<a href=#154 id=154 data-nosnippet>154</a> <span class="kw">if let </span>datafusion_expr::Expr::Column(<span class="kw">_</span>) = node {
<a href=#155 id=155 data-nosnippet>155</a> <span class="self">self</span>.names_for_alias.insert(node.name_for_alias()<span class="question-mark">?</span>);
<a href=#156 id=156 data-nosnippet>156</a> }
<a href=#157 id=157 data-nosnippet>157</a> <span class="prelude-val">Ok</span>(TreeNodeRecursion::Continue)
<a href=#158 id=158 data-nosnippet>158</a> }
<a href=#159 id=159 data-nosnippet>159</a>}
</code></pre></div></section></main></body></html>
+76
View File
@@ -0,0 +1,76 @@
<!DOCTYPE html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1.0"><meta name="generator" content="rustdoc"><meta name="description" content="Source of the Rust file `src/flow/src/engine.rs`."><title>engine.rs - source</title><script>if(window.location.protocol!=="file:")document.head.insertAdjacentHTML("beforeend","SourceSerif4-Regular-6b053e98.ttf.woff2,FiraSans-Italic-81dc35de.woff2,FiraSans-Regular-0fe48ade.woff2,FiraSans-MediumItalic-ccf7e434.woff2,FiraSans-Medium-e1aa3f0a.woff2,SourceCodePro-Regular-8badfe75.ttf.woff2,SourceCodePro-Semibold-aa29a496.ttf.woff2".split(",").map(f=>`<link rel="preload" as="font" type="font/woff2"href="../../static.files/${f}">`).join(""))</script><link rel="stylesheet" href="../../static.files/normalize-9960930a.css"><link rel="stylesheet" href="../../static.files/rustdoc-17e0aaed.css"><meta name="rustdoc-vars" data-root-path="../../" data-static-root-path="../../static.files/" data-current-crate="flow" data-themes="" data-resource-suffix="" data-rustdoc-version="1.96.1 (31fca3adb 2026-06-26)" data-channel="1.96.1" data-search-js="search-b5634cc7.js" data-stringdex-js="stringdex-2da4960a.js" data-settings-js="settings-170eb4bf.js" ><script src="../../static.files/storage-41dd4d93.js"></script><script defer src="../../static.files/src-script-813739b1.js"></script><script defer src="../../src-files.js"></script><script defer src="../../static.files/main-5013f961.js"></script><noscript><link rel="stylesheet" href="../../static.files/noscript-f7c3ffd8.css"></noscript><link rel="alternate icon" type="image/png" href="../../static.files/favicon-32x32-eab170b8.png"><link rel="icon" type="image/svg+xml" href="../../static.files/favicon-044be391.svg"></head><body class="rustdoc src"><a class="skip-main-content" href="#main-content">Skip to main content</a><!--[if lte IE 11]><div class="warning">This old browser is unsupported and will most likely display funky things.</div><![endif]--><nav class="sidebar"><div class="src-sidebar-title"><h2>Files</h2></div></nav><div class="sidebar-resizer" title="Drag to resize sidebar"></div><main><section id="main-content" class="content" tabindex="-1"><div class="main-heading"><h1><div class="sub-heading">flow/</div>engine.rs</h1><rustdoc-toolbar></rustdoc-toolbar></div><div class="example-wrap digits-2"><pre class="rust"><code><a href=#1 id=1 data-nosnippet>1</a><span class="comment">// Copyright 2023 Greptime Team
<a href=#2 id=2 data-nosnippet>2</a>//
<a href=#3 id=3 data-nosnippet>3</a>// Licensed under the Apache License, Version 2.0 (the "License");
<a href=#4 id=4 data-nosnippet>4</a>// you may not use this file except in compliance with the License.
<a href=#5 id=5 data-nosnippet>5</a>// You may obtain a copy of the License at
<a href=#6 id=6 data-nosnippet>6</a>//
<a href=#7 id=7 data-nosnippet>7</a>// http://www.apache.org/licenses/LICENSE-2.0
<a href=#8 id=8 data-nosnippet>8</a>//
<a href=#9 id=9 data-nosnippet>9</a>// Unless required by applicable law or agreed to in writing, software
<a href=#10 id=10 data-nosnippet>10</a>// distributed under the License is distributed on an "AS IS" BASIS,
<a href=#11 id=11 data-nosnippet>11</a>// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
<a href=#12 id=12 data-nosnippet>12</a>// See the License for the specific language governing permissions and
<a href=#13 id=13 data-nosnippet>13</a>// limitations under the License.
<a href=#14 id=14 data-nosnippet>14</a>
<a href=#15 id=15 data-nosnippet>15</a></span><span class="doccomment">//! Define a trait for flow engine, which is used by both streaming engine and batch engine
<a href=#16 id=16 data-nosnippet>16</a>
<a href=#17 id=17 data-nosnippet>17</a></span><span class="kw">use </span>std::collections::HashMap;
<a href=#18 id=18 data-nosnippet>18</a>
<a href=#19 id=19 data-nosnippet>19</a><span class="kw">use </span>common_meta::key::flow::flow_state::FlowStat;
<a href=#20 id=20 data-nosnippet>20</a><span class="kw">use </span>session::context::QueryContext;
<a href=#21 id=21 data-nosnippet>21</a><span class="kw">use </span>table::metadata::TableId;
<a href=#22 id=22 data-nosnippet>22</a>
<a href=#23 id=23 data-nosnippet>23</a><span class="kw">use </span><span class="kw">crate</span>::Error;
<a href=#24 id=24 data-nosnippet>24</a><span class="comment">// TODO(discord9): refactor common types for flow to a separate module
<a href=#25 id=25 data-nosnippet>25</a></span><span class="doccomment">/// FlowId is a unique identifier for a flow task
<a href=#26 id=26 data-nosnippet>26</a></span><span class="kw">pub type </span>FlowId = u64;
<a href=#27 id=27 data-nosnippet>27</a><span class="kw">pub type </span>TableName = [String; <span class="number">3</span>];
<a href=#28 id=28 data-nosnippet>28</a>
<a href=#29 id=29 data-nosnippet>29</a><span class="doccomment">/// The arguments to create a flow
<a href=#30 id=30 data-nosnippet>30</a></span><span class="attr">#[derive(Debug, Clone)]
<a href=#31 id=31 data-nosnippet>31</a></span><span class="kw">pub struct </span>CreateFlowArgs {
<a href=#32 id=32 data-nosnippet>32</a> <span class="kw">pub </span>flow_id: FlowId,
<a href=#33 id=33 data-nosnippet>33</a> <span class="kw">pub </span>sink_table_name: TableName,
<a href=#34 id=34 data-nosnippet>34</a> <span class="kw">pub </span>source_table_ids: Vec&lt;TableId&gt;,
<a href=#35 id=35 data-nosnippet>35</a> <span class="kw">pub </span>create_if_not_exists: bool,
<a href=#36 id=36 data-nosnippet>36</a> <span class="kw">pub </span>or_replace: bool,
<a href=#37 id=37 data-nosnippet>37</a> <span class="doccomment">/// Duration in seconds.
<a href=#38 id=38 data-nosnippet>38</a> </span><span class="kw">pub </span>expire_after: <span class="prelude-ty">Option</span>&lt;i64&gt;,
<a href=#39 id=39 data-nosnippet>39</a> <span class="kw">pub </span>eval_interval: <span class="prelude-ty">Option</span>&lt;i64&gt;,
<a href=#40 id=40 data-nosnippet>40</a> <span class="kw">pub </span>comment: <span class="prelude-ty">Option</span>&lt;String&gt;,
<a href=#41 id=41 data-nosnippet>41</a> <span class="kw">pub </span>sql: String,
<a href=#42 id=42 data-nosnippet>42</a> <span class="kw">pub </span>flow_options: HashMap&lt;String, String&gt;,
<a href=#43 id=43 data-nosnippet>43</a> <span class="kw">pub </span>query_ctx: <span class="prelude-ty">Option</span>&lt;QueryContext&gt;,
<a href=#44 id=44 data-nosnippet>44</a> <span class="doccomment">/// Typed schedule configuration for `EVAL INTERVAL` flows.
<a href=#45 id=45 data-nosnippet>45</a> </span><span class="kw">pub </span>eval_schedule: <span class="prelude-ty">Option</span>&lt;common_meta::key::flow::flow_info::FlowScheduleConfig&gt;,
<a href=#46 id=46 data-nosnippet>46</a>}
<a href=#47 id=47 data-nosnippet>47</a>
<a href=#48 id=48 data-nosnippet>48</a><span class="kw">pub trait </span>FlowEngine {
<a href=#49 id=49 data-nosnippet>49</a> <span class="doccomment">/// Create a flow using the provided arguments, return previous flow id if exists and is replaced
<a href=#50 id=50 data-nosnippet>50</a> </span><span class="kw">async fn </span>create_flow(<span class="kw-2">&amp;</span><span class="self">self</span>, args: CreateFlowArgs) -&gt; <span class="prelude-ty">Result</span>&lt;<span class="prelude-ty">Option</span>&lt;FlowId&gt;, Error&gt;;
<a href=#51 id=51 data-nosnippet>51</a> <span class="doccomment">/// Remove a flow by its ID
<a href=#52 id=52 data-nosnippet>52</a> </span><span class="kw">async fn </span>remove_flow(<span class="kw-2">&amp;</span><span class="self">self</span>, flow_id: FlowId) -&gt; <span class="prelude-ty">Result</span>&lt;(), Error&gt;;
<a href=#53 id=53 data-nosnippet>53</a> <span class="doccomment">/// Flush the flow, return the number of rows flushed
<a href=#54 id=54 data-nosnippet>54</a> </span><span class="kw">async fn </span>flush_flow(<span class="kw-2">&amp;</span><span class="self">self</span>, flow_id: FlowId) -&gt; <span class="prelude-ty">Result</span>&lt;usize, Error&gt;;
<a href=#55 id=55 data-nosnippet>55</a> <span class="doccomment">/// Check if the flow exists
<a href=#56 id=56 data-nosnippet>56</a> </span><span class="kw">async fn </span>flow_exist(<span class="kw-2">&amp;</span><span class="self">self</span>, flow_id: FlowId) -&gt; <span class="prelude-ty">Result</span>&lt;bool, Error&gt;;
<a href=#57 id=57 data-nosnippet>57</a> <span class="doccomment">/// List all flows
<a href=#58 id=58 data-nosnippet>58</a> </span><span class="kw">async fn </span>list_flows(<span class="kw-2">&amp;</span><span class="self">self</span>) -&gt; <span class="prelude-ty">Result</span>&lt;<span class="kw">impl </span>IntoIterator&lt;Item = FlowId&gt;, Error&gt;;
<a href=#59 id=59 data-nosnippet>59</a> <span class="doccomment">/// Handle the insert requests for the flow
<a href=#60 id=60 data-nosnippet>60</a> </span><span class="kw">async fn </span>handle_flow_inserts(
<a href=#61 id=61 data-nosnippet>61</a> <span class="kw-2">&amp;</span><span class="self">self</span>,
<a href=#62 id=62 data-nosnippet>62</a> request: api::v1::region::InsertRequests,
<a href=#63 id=63 data-nosnippet>63</a> ) -&gt; <span class="prelude-ty">Result</span>&lt;(), Error&gt;;
<a href=#64 id=64 data-nosnippet>64</a>
<a href=#65 id=65 data-nosnippet>65</a> <span class="kw">async fn </span>handle_mark_window_dirty(
<a href=#66 id=66 data-nosnippet>66</a> <span class="kw-2">&amp;</span><span class="self">self</span>,
<a href=#67 id=67 data-nosnippet>67</a> req: api::v1::flow::DirtyWindowRequests,
<a href=#68 id=68 data-nosnippet>68</a> ) -&gt; <span class="prelude-ty">Result</span>&lt;(), Error&gt;;
<a href=#69 id=69 data-nosnippet>69</a>}
<a href=#70 id=70 data-nosnippet>70</a>
<a href=#71 id=71 data-nosnippet>71</a><span class="doccomment">/// Provides flow runtime statistics for information schema and heartbeat reporting.
<a href=#72 id=72 data-nosnippet>72</a></span><span class="kw">pub trait </span>FlowStatProvider {
<a href=#73 id=73 data-nosnippet>73</a> <span class="doccomment">/// Returns current runtime stats of an engine.
<a href=#74 id=74 data-nosnippet>74</a> </span><span class="kw">async fn </span>flow_stat(<span class="kw-2">&amp;</span><span class="self">self</span>) -&gt; FlowStat;
<a href=#75 id=75 data-nosnippet>75</a>}
</code></pre></div></section></main></body></html>
+382
View File
@@ -0,0 +1,382 @@
<!DOCTYPE html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1.0"><meta name="generator" content="rustdoc"><meta name="description" content="Source of the Rust file `src/flow/src/error.rs`."><title>error.rs - source</title><script>if(window.location.protocol!=="file:")document.head.insertAdjacentHTML("beforeend","SourceSerif4-Regular-6b053e98.ttf.woff2,FiraSans-Italic-81dc35de.woff2,FiraSans-Regular-0fe48ade.woff2,FiraSans-MediumItalic-ccf7e434.woff2,FiraSans-Medium-e1aa3f0a.woff2,SourceCodePro-Regular-8badfe75.ttf.woff2,SourceCodePro-Semibold-aa29a496.ttf.woff2".split(",").map(f=>`<link rel="preload" as="font" type="font/woff2"href="../../static.files/${f}">`).join(""))</script><link rel="stylesheet" href="../../static.files/normalize-9960930a.css"><link rel="stylesheet" href="../../static.files/rustdoc-17e0aaed.css"><meta name="rustdoc-vars" data-root-path="../../" data-static-root-path="../../static.files/" data-current-crate="flow" data-themes="" data-resource-suffix="" data-rustdoc-version="1.96.1 (31fca3adb 2026-06-26)" data-channel="1.96.1" data-search-js="search-b5634cc7.js" data-stringdex-js="stringdex-2da4960a.js" data-settings-js="settings-170eb4bf.js" ><script src="../../static.files/storage-41dd4d93.js"></script><script defer src="../../static.files/src-script-813739b1.js"></script><script defer src="../../src-files.js"></script><script defer src="../../static.files/main-5013f961.js"></script><noscript><link rel="stylesheet" href="../../static.files/noscript-f7c3ffd8.css"></noscript><link rel="alternate icon" type="image/png" href="../../static.files/favicon-32x32-eab170b8.png"><link rel="icon" type="image/svg+xml" href="../../static.files/favicon-044be391.svg"></head><body class="rustdoc src"><a class="skip-main-content" href="#main-content">Skip to main content</a><!--[if lte IE 11]><div class="warning">This old browser is unsupported and will most likely display funky things.</div><![endif]--><nav class="sidebar"><div class="src-sidebar-title"><h2>Files</h2></div></nav><div class="sidebar-resizer" title="Drag to resize sidebar"></div><main><section id="main-content" class="content" tabindex="-1"><div class="main-heading"><h1><div class="sub-heading">flow/</div>error.rs</h1><rustdoc-toolbar></rustdoc-toolbar></div><div class="example-wrap digits-3"><pre class="rust"><code><a href=#1 id=1 data-nosnippet>1</a><span class="comment">// Copyright 2023 Greptime Team
<a href=#2 id=2 data-nosnippet>2</a>//
<a href=#3 id=3 data-nosnippet>3</a>// Licensed under the Apache License, Version 2.0 (the "License");
<a href=#4 id=4 data-nosnippet>4</a>// you may not use this file except in compliance with the License.
<a href=#5 id=5 data-nosnippet>5</a>// You may obtain a copy of the License at
<a href=#6 id=6 data-nosnippet>6</a>//
<a href=#7 id=7 data-nosnippet>7</a>// http://www.apache.org/licenses/LICENSE-2.0
<a href=#8 id=8 data-nosnippet>8</a>//
<a href=#9 id=9 data-nosnippet>9</a>// Unless required by applicable law or agreed to in writing, software
<a href=#10 id=10 data-nosnippet>10</a>// distributed under the License is distributed on an "AS IS" BASIS,
<a href=#11 id=11 data-nosnippet>11</a>// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
<a href=#12 id=12 data-nosnippet>12</a>// See the License for the specific language governing permissions and
<a href=#13 id=13 data-nosnippet>13</a>// limitations under the License.
<a href=#14 id=14 data-nosnippet>14</a>
<a href=#15 id=15 data-nosnippet>15</a></span><span class="doccomment">//! Error definition for flow module
<a href=#16 id=16 data-nosnippet>16</a>
<a href=#17 id=17 data-nosnippet>17</a></span><span class="kw">use </span>std::any::Any;
<a href=#18 id=18 data-nosnippet>18</a>
<a href=#19 id=19 data-nosnippet>19</a><span class="kw">use </span>api::v1::CreateTableExpr;
<a href=#20 id=20 data-nosnippet>20</a><span class="kw">use </span>arrow_schema::ArrowError;
<a href=#21 id=21 data-nosnippet>21</a><span class="kw">use </span>common_error::ext::{BoxedError, RetryHint};
<a href=#22 id=22 data-nosnippet>22</a><span class="kw">use </span>common_error::{
<a href=#23 id=23 data-nosnippet>23</a> GREPTIME_DB_HEADER_ERROR_RETRY_HINT, define_into_tonic_status, from_err_code_msg_to_header,
<a href=#24 id=24 data-nosnippet>24</a>};
<a href=#25 id=25 data-nosnippet>25</a><span class="kw">use </span>common_macro::stack_trace_debug;
<a href=#26 id=26 data-nosnippet>26</a><span class="kw">use </span>common_telemetry::common_error::ext::ErrorExt;
<a href=#27 id=27 data-nosnippet>27</a><span class="kw">use </span>common_telemetry::common_error::status_code::StatusCode;
<a href=#28 id=28 data-nosnippet>28</a><span class="kw">use </span>snafu::{Location, Snafu};
<a href=#29 id=29 data-nosnippet>29</a><span class="kw">use </span>tonic::codegen::http::HeaderValue;
<a href=#30 id=30 data-nosnippet>30</a><span class="kw">use </span>tonic::metadata::MetadataMap;
<a href=#31 id=31 data-nosnippet>31</a>
<a href=#32 id=32 data-nosnippet>32</a><span class="kw">use </span><span class="kw">crate</span>::FlowId;
<a href=#33 id=33 data-nosnippet>33</a>
<a href=#34 id=34 data-nosnippet>34</a><span class="doccomment">/// This error is used to represent all possible errors that can occur in the flow module.
<a href=#35 id=35 data-nosnippet>35</a></span><span class="attr">#[derive(Snafu)]
<a href=#36 id=36 data-nosnippet>36</a>#[snafu(visibility(<span class="kw">pub</span>))]
<a href=#37 id=37 data-nosnippet>37</a>#[stack_trace_debug]
<a href=#38 id=38 data-nosnippet>38</a></span><span class="kw">pub enum </span>Error {
<a href=#39 id=39 data-nosnippet>39</a> <span class="attr">#[snafu(display(
<a href=#40 id=40 data-nosnippet>40</a> <span class="string">"Failed to insert into flow: region_id={}, flow_ids={:?}"</span>,
<a href=#41 id=41 data-nosnippet>41</a> region_id,
<a href=#42 id=42 data-nosnippet>42</a> flow_ids
<a href=#43 id=43 data-nosnippet>43</a> ))]
<a href=#44 id=44 data-nosnippet>44</a> </span>InsertIntoFlow {
<a href=#45 id=45 data-nosnippet>45</a> region_id: u64,
<a href=#46 id=46 data-nosnippet>46</a> flow_ids: Vec&lt;u64&gt;,
<a href=#47 id=47 data-nosnippet>47</a> source: BoxedError,
<a href=#48 id=48 data-nosnippet>48</a> <span class="attr">#[snafu(implicit)]
<a href=#49 id=49 data-nosnippet>49</a> </span>location: Location,
<a href=#50 id=50 data-nosnippet>50</a> },
<a href=#51 id=51 data-nosnippet>51</a>
<a href=#52 id=52 data-nosnippet>52</a> <span class="attr">#[snafu(display(<span class="string">"Flow engine is still recovering"</span>))]
<a href=#53 id=53 data-nosnippet>53</a> </span>FlowNotRecovered {
<a href=#54 id=54 data-nosnippet>54</a> <span class="attr">#[snafu(implicit)]
<a href=#55 id=55 data-nosnippet>55</a> </span>location: Location,
<a href=#56 id=56 data-nosnippet>56</a> },
<a href=#57 id=57 data-nosnippet>57</a>
<a href=#58 id=58 data-nosnippet>58</a> <span class="attr">#[snafu(display(<span class="string">"Error encountered while creating flow: {sql}"</span>))]
<a href=#59 id=59 data-nosnippet>59</a> </span>CreateFlow {
<a href=#60 id=60 data-nosnippet>60</a> sql: String,
<a href=#61 id=61 data-nosnippet>61</a> source: BoxedError,
<a href=#62 id=62 data-nosnippet>62</a> <span class="attr">#[snafu(implicit)]
<a href=#63 id=63 data-nosnippet>63</a> </span>location: Location,
<a href=#64 id=64 data-nosnippet>64</a> },
<a href=#65 id=65 data-nosnippet>65</a>
<a href=#66 id=66 data-nosnippet>66</a> <span class="attr">#[snafu(display(<span class="string">"Error encountered while creating sink table for flow: {create:?}"</span>))]
<a href=#67 id=67 data-nosnippet>67</a> </span>CreateSinkTable {
<a href=#68 id=68 data-nosnippet>68</a> create: CreateTableExpr,
<a href=#69 id=69 data-nosnippet>69</a> source: BoxedError,
<a href=#70 id=70 data-nosnippet>70</a> <span class="attr">#[snafu(implicit)]
<a href=#71 id=71 data-nosnippet>71</a> </span>location: Location,
<a href=#72 id=72 data-nosnippet>72</a> },
<a href=#73 id=73 data-nosnippet>73</a>
<a href=#74 id=74 data-nosnippet>74</a> <span class="attr">#[snafu(display(<span class="string">"Time error"</span>))]
<a href=#75 id=75 data-nosnippet>75</a> </span>Time {
<a href=#76 id=76 data-nosnippet>76</a> source: common_time::error::Error,
<a href=#77 id=77 data-nosnippet>77</a> <span class="attr">#[snafu(implicit)]
<a href=#78 id=78 data-nosnippet>78</a> </span>location: Location,
<a href=#79 id=79 data-nosnippet>79</a> },
<a href=#80 id=80 data-nosnippet>80</a>
<a href=#81 id=81 data-nosnippet>81</a> <span class="attr">#[snafu(display(<span class="string">"No available frontend found after timeout: {timeout:?}, context: {context}"</span>))]
<a href=#82 id=82 data-nosnippet>82</a> </span>NoAvailableFrontend {
<a href=#83 id=83 data-nosnippet>83</a> timeout: std::time::Duration,
<a href=#84 id=84 data-nosnippet>84</a> context: String,
<a href=#85 id=85 data-nosnippet>85</a> <span class="attr">#[snafu(implicit)]
<a href=#86 id=86 data-nosnippet>86</a> </span>location: Location,
<a href=#87 id=87 data-nosnippet>87</a> },
<a href=#88 id=88 data-nosnippet>88</a>
<a href=#89 id=89 data-nosnippet>89</a> <span class="attr">#[snafu(display(<span class="string">"External error"</span>))]
<a href=#90 id=90 data-nosnippet>90</a> </span>External {
<a href=#91 id=91 data-nosnippet>91</a> source: BoxedError,
<a href=#92 id=92 data-nosnippet>92</a> <span class="attr">#[snafu(implicit)]
<a href=#93 id=93 data-nosnippet>93</a> </span>location: Location,
<a href=#94 id=94 data-nosnippet>94</a> },
<a href=#95 id=95 data-nosnippet>95</a>
<a href=#96 id=96 data-nosnippet>96</a> <span class="attr">#[snafu(display(<span class="string">"Internal error"</span>))]
<a href=#97 id=97 data-nosnippet>97</a> </span>Internal {
<a href=#98 id=98 data-nosnippet>98</a> reason: String,
<a href=#99 id=99 data-nosnippet>99</a> <span class="attr">#[snafu(implicit)]
<a href=#100 id=100 data-nosnippet>100</a> </span>location: Location,
<a href=#101 id=101 data-nosnippet>101</a> },
<a href=#102 id=102 data-nosnippet>102</a>
<a href=#103 id=103 data-nosnippet>103</a> <span class="attr">#[snafu(display(<span class="string">"Table not found: {name}"</span>))]
<a href=#104 id=104 data-nosnippet>104</a> </span>TableNotFound {
<a href=#105 id=105 data-nosnippet>105</a> name: String,
<a href=#106 id=106 data-nosnippet>106</a> <span class="attr">#[snafu(implicit)]
<a href=#107 id=107 data-nosnippet>107</a> </span>location: Location,
<a href=#108 id=108 data-nosnippet>108</a> },
<a href=#109 id=109 data-nosnippet>109</a>
<a href=#110 id=110 data-nosnippet>110</a> <span class="attr">#[snafu(display(<span class="string">"Table not found: {msg}, meta error: {source}"</span>))]
<a href=#111 id=111 data-nosnippet>111</a> </span>TableNotFoundMeta {
<a href=#112 id=112 data-nosnippet>112</a> source: common_meta::error::Error,
<a href=#113 id=113 data-nosnippet>113</a> msg: String,
<a href=#114 id=114 data-nosnippet>114</a> <span class="attr">#[snafu(implicit)]
<a href=#115 id=115 data-nosnippet>115</a> </span>location: Location,
<a href=#116 id=116 data-nosnippet>116</a> },
<a href=#117 id=117 data-nosnippet>117</a>
<a href=#118 id=118 data-nosnippet>118</a> <span class="attr">#[snafu(display(<span class="string">"Flow not found, id={id}"</span>))]
<a href=#119 id=119 data-nosnippet>119</a> </span>FlowNotFound {
<a href=#120 id=120 data-nosnippet>120</a> id: FlowId,
<a href=#121 id=121 data-nosnippet>121</a> <span class="attr">#[snafu(implicit)]
<a href=#122 id=122 data-nosnippet>122</a> </span>location: Location,
<a href=#123 id=123 data-nosnippet>123</a> },
<a href=#124 id=124 data-nosnippet>124</a>
<a href=#125 id=125 data-nosnippet>125</a> <span class="attr">#[snafu(display(<span class="string">"Failed to list flows in flownode={id:?}"</span>))]
<a href=#126 id=126 data-nosnippet>126</a> </span>ListFlows {
<a href=#127 id=127 data-nosnippet>127</a> id: <span class="prelude-ty">Option</span>&lt;common_meta::FlownodeId&gt;,
<a href=#128 id=128 data-nosnippet>128</a> source: common_meta::error::Error,
<a href=#129 id=129 data-nosnippet>129</a> <span class="attr">#[snafu(implicit)]
<a href=#130 id=130 data-nosnippet>130</a> </span>location: Location,
<a href=#131 id=131 data-nosnippet>131</a> },
<a href=#132 id=132 data-nosnippet>132</a>
<a href=#133 id=133 data-nosnippet>133</a> <span class="attr">#[snafu(display(<span class="string">"Flow already exist, id={id}"</span>))]
<a href=#134 id=134 data-nosnippet>134</a> </span>FlowAlreadyExist {
<a href=#135 id=135 data-nosnippet>135</a> id: FlowId,
<a href=#136 id=136 data-nosnippet>136</a> <span class="attr">#[snafu(implicit)]
<a href=#137 id=137 data-nosnippet>137</a> </span>location: Location,
<a href=#138 id=138 data-nosnippet>138</a> },
<a href=#139 id=139 data-nosnippet>139</a>
<a href=#140 id=140 data-nosnippet>140</a> <span class="attr">#[snafu(display(<span class="string">"Failed to join task"</span>))]
<a href=#141 id=141 data-nosnippet>141</a> </span>JoinTask {
<a href=#142 id=142 data-nosnippet>142</a> <span class="attr">#[snafu(source)]
<a href=#143 id=143 data-nosnippet>143</a> </span>error: tokio::task::JoinError,
<a href=#144 id=144 data-nosnippet>144</a> <span class="attr">#[snafu(implicit)]
<a href=#145 id=145 data-nosnippet>145</a> </span>location: Location,
<a href=#146 id=146 data-nosnippet>146</a> },
<a href=#147 id=147 data-nosnippet>147</a>
<a href=#148 id=148 data-nosnippet>148</a> <span class="attr">#[snafu(display(<span class="string">"Invalid query: {reason}"</span>))]
<a href=#149 id=149 data-nosnippet>149</a> </span>InvalidQuery {
<a href=#150 id=150 data-nosnippet>150</a> reason: String,
<a href=#151 id=151 data-nosnippet>151</a> <span class="attr">#[snafu(implicit)]
<a href=#152 id=152 data-nosnippet>152</a> </span>location: Location,
<a href=#153 id=153 data-nosnippet>153</a> },
<a href=#154 id=154 data-nosnippet>154</a>
<a href=#155 id=155 data-nosnippet>155</a> <span class="attr">#[snafu(display(<span class="string">"Flow plan error: {reason}"</span>))]
<a href=#156 id=156 data-nosnippet>156</a> </span>Plan {
<a href=#157 id=157 data-nosnippet>157</a> reason: String,
<a href=#158 id=158 data-nosnippet>158</a> <span class="attr">#[snafu(implicit)]
<a href=#159 id=159 data-nosnippet>159</a> </span>location: Location,
<a href=#160 id=160 data-nosnippet>160</a> },
<a href=#161 id=161 data-nosnippet>161</a>
<a href=#162 id=162 data-nosnippet>162</a> <span class="attr">#[snafu(display(<span class="string">"Unsupported: {reason}"</span>))]
<a href=#163 id=163 data-nosnippet>163</a> </span>Unsupported {
<a href=#164 id=164 data-nosnippet>164</a> reason: String,
<a href=#165 id=165 data-nosnippet>165</a> <span class="attr">#[snafu(implicit)]
<a href=#166 id=166 data-nosnippet>166</a> </span>location: Location,
<a href=#167 id=167 data-nosnippet>167</a> },
<a href=#168 id=168 data-nosnippet>168</a>
<a href=#169 id=169 data-nosnippet>169</a> <span class="attr">#[snafu(display(<span class="string">"Datatypes error: {source} with extra message: {extra}"</span>))]
<a href=#170 id=170 data-nosnippet>170</a> </span>Datatypes {
<a href=#171 id=171 data-nosnippet>171</a> source: datatypes::Error,
<a href=#172 id=172 data-nosnippet>172</a> extra: String,
<a href=#173 id=173 data-nosnippet>173</a> <span class="attr">#[snafu(implicit)]
<a href=#174 id=174 data-nosnippet>174</a> </span>location: Location,
<a href=#175 id=175 data-nosnippet>175</a> },
<a href=#176 id=176 data-nosnippet>176</a>
<a href=#177 id=177 data-nosnippet>177</a> <span class="attr">#[snafu(display(<span class="string">"Arrow error: {raw:?} in context: {context}"</span>))]
<a href=#178 id=178 data-nosnippet>178</a> </span>Arrow {
<a href=#179 id=179 data-nosnippet>179</a> <span class="attr">#[snafu(source)]
<a href=#180 id=180 data-nosnippet>180</a> </span>raw: ArrowError,
<a href=#181 id=181 data-nosnippet>181</a> context: String,
<a href=#182 id=182 data-nosnippet>182</a> <span class="attr">#[snafu(implicit)]
<a href=#183 id=183 data-nosnippet>183</a> </span>location: Location,
<a href=#184 id=184 data-nosnippet>184</a> },
<a href=#185 id=185 data-nosnippet>185</a>
<a href=#186 id=186 data-nosnippet>186</a> <span class="attr">#[snafu(display(<span class="string">"Datafusion error: {raw:?} in context: {context}"</span>))]
<a href=#187 id=187 data-nosnippet>187</a> </span>Datafusion {
<a href=#188 id=188 data-nosnippet>188</a> <span class="attr">#[snafu(source)]
<a href=#189 id=189 data-nosnippet>189</a> </span>raw: datafusion_common::DataFusionError,
<a href=#190 id=190 data-nosnippet>190</a> context: String,
<a href=#191 id=191 data-nosnippet>191</a> <span class="attr">#[snafu(implicit)]
<a href=#192 id=192 data-nosnippet>192</a> </span>location: Location,
<a href=#193 id=193 data-nosnippet>193</a> },
<a href=#194 id=194 data-nosnippet>194</a>
<a href=#195 id=195 data-nosnippet>195</a> <span class="attr">#[snafu(display(<span class="string">"Unexpected: {reason}"</span>))]
<a href=#196 id=196 data-nosnippet>196</a> </span>Unexpected {
<a href=#197 id=197 data-nosnippet>197</a> reason: String,
<a href=#198 id=198 data-nosnippet>198</a> <span class="attr">#[snafu(implicit)]
<a href=#199 id=199 data-nosnippet>199</a> </span>location: Location,
<a href=#200 id=200 data-nosnippet>200</a> },
<a href=#201 id=201 data-nosnippet>201</a>
<a href=#202 id=202 data-nosnippet>202</a> <span class="attr">#[snafu(display(<span class="string">"Illegal check task state: {reason}"</span>))]
<a href=#203 id=203 data-nosnippet>203</a> </span>IllegalCheckTaskState {
<a href=#204 id=204 data-nosnippet>204</a> reason: String,
<a href=#205 id=205 data-nosnippet>205</a> <span class="attr">#[snafu(implicit)]
<a href=#206 id=206 data-nosnippet>206</a> </span>location: Location,
<a href=#207 id=207 data-nosnippet>207</a> },
<a href=#208 id=208 data-nosnippet>208</a>
<a href=#209 id=209 data-nosnippet>209</a> <span class="attr">#[snafu(display(
<a href=#210 id=210 data-nosnippet>210</a> <span class="string">"Failed to sync with check task for flow {} with allow_drop={}"</span>,
<a href=#211 id=211 data-nosnippet>211</a> flow_id,
<a href=#212 id=212 data-nosnippet>212</a> allow_drop
<a href=#213 id=213 data-nosnippet>213</a> ))]
<a href=#214 id=214 data-nosnippet>214</a> </span>SyncCheckTask {
<a href=#215 id=215 data-nosnippet>215</a> flow_id: FlowId,
<a href=#216 id=216 data-nosnippet>216</a> allow_drop: bool,
<a href=#217 id=217 data-nosnippet>217</a> <span class="attr">#[snafu(implicit)]
<a href=#218 id=218 data-nosnippet>218</a> </span>location: Location,
<a href=#219 id=219 data-nosnippet>219</a> },
<a href=#220 id=220 data-nosnippet>220</a>
<a href=#221 id=221 data-nosnippet>221</a> <span class="attr">#[snafu(display(<span class="string">"Failed to start server"</span>))]
<a href=#222 id=222 data-nosnippet>222</a> </span>StartServer {
<a href=#223 id=223 data-nosnippet>223</a> <span class="attr">#[snafu(implicit)]
<a href=#224 id=224 data-nosnippet>224</a> </span>location: Location,
<a href=#225 id=225 data-nosnippet>225</a> source: servers::error::Error,
<a href=#226 id=226 data-nosnippet>226</a> },
<a href=#227 id=227 data-nosnippet>227</a>
<a href=#228 id=228 data-nosnippet>228</a> <span class="attr">#[snafu(display(<span class="string">"Failed to shutdown server"</span>))]
<a href=#229 id=229 data-nosnippet>229</a> </span>ShutdownServer {
<a href=#230 id=230 data-nosnippet>230</a> <span class="attr">#[snafu(implicit)]
<a href=#231 id=231 data-nosnippet>231</a> </span>location: Location,
<a href=#232 id=232 data-nosnippet>232</a> source: servers::error::Error,
<a href=#233 id=233 data-nosnippet>233</a> },
<a href=#234 id=234 data-nosnippet>234</a>
<a href=#235 id=235 data-nosnippet>235</a> <span class="attr">#[snafu(display(<span class="string">"Failed to initialize meta client"</span>))]
<a href=#236 id=236 data-nosnippet>236</a> </span>MetaClientInit {
<a href=#237 id=237 data-nosnippet>237</a> <span class="attr">#[snafu(implicit)]
<a href=#238 id=238 data-nosnippet>238</a> </span>location: Location,
<a href=#239 id=239 data-nosnippet>239</a> source: meta_client::error::Error,
<a href=#240 id=240 data-nosnippet>240</a> },
<a href=#241 id=241 data-nosnippet>241</a>
<a href=#242 id=242 data-nosnippet>242</a> <span class="attr">#[snafu(display(<span class="string">"Failed to parse address {}"</span>, addr))]
<a href=#243 id=243 data-nosnippet>243</a> </span>ParseAddr {
<a href=#244 id=244 data-nosnippet>244</a> addr: String,
<a href=#245 id=245 data-nosnippet>245</a> <span class="attr">#[snafu(source)]
<a href=#246 id=246 data-nosnippet>246</a> </span>error: std::net::AddrParseError,
<a href=#247 id=247 data-nosnippet>247</a> },
<a href=#248 id=248 data-nosnippet>248</a>
<a href=#249 id=249 data-nosnippet>249</a> <span class="attr">#[snafu(display(<span class="string">"Failed to get cache from cache registry: {}"</span>, name))]
<a href=#250 id=250 data-nosnippet>250</a> </span>CacheRequired {
<a href=#251 id=251 data-nosnippet>251</a> <span class="attr">#[snafu(implicit)]
<a href=#252 id=252 data-nosnippet>252</a> </span>location: Location,
<a href=#253 id=253 data-nosnippet>253</a> name: String,
<a href=#254 id=254 data-nosnippet>254</a> },
<a href=#255 id=255 data-nosnippet>255</a>
<a href=#256 id=256 data-nosnippet>256</a> <span class="attr">#[snafu(display(<span class="string">"Invalid request: {context}"</span>))]
<a href=#257 id=257 data-nosnippet>257</a> </span>InvalidRequest {
<a href=#258 id=258 data-nosnippet>258</a> context: String,
<a href=#259 id=259 data-nosnippet>259</a> source: client::Error,
<a href=#260 id=260 data-nosnippet>260</a> <span class="attr">#[snafu(implicit)]
<a href=#261 id=261 data-nosnippet>261</a> </span>location: Location,
<a href=#262 id=262 data-nosnippet>262</a> },
<a href=#263 id=263 data-nosnippet>263</a>
<a href=#264 id=264 data-nosnippet>264</a> <span class="attr">#[snafu(display(<span class="string">"Failed to encode logical plan in substrait"</span>))]
<a href=#265 id=265 data-nosnippet>265</a> </span>SubstraitEncodeLogicalPlan {
<a href=#266 id=266 data-nosnippet>266</a> <span class="attr">#[snafu(implicit)]
<a href=#267 id=267 data-nosnippet>267</a> </span>location: Location,
<a href=#268 id=268 data-nosnippet>268</a> source: substrait::error::Error,
<a href=#269 id=269 data-nosnippet>269</a> },
<a href=#270 id=270 data-nosnippet>270</a>
<a href=#271 id=271 data-nosnippet>271</a> <span class="attr">#[snafu(display(<span class="string">"Failed to convert column schema to proto column def"</span>))]
<a href=#272 id=272 data-nosnippet>272</a> </span>ConvertColumnSchema {
<a href=#273 id=273 data-nosnippet>273</a> <span class="attr">#[snafu(implicit)]
<a href=#274 id=274 data-nosnippet>274</a> </span>location: Location,
<a href=#275 id=275 data-nosnippet>275</a> source: operator::error::Error,
<a href=#276 id=276 data-nosnippet>276</a> },
<a href=#277 id=277 data-nosnippet>277</a>
<a href=#278 id=278 data-nosnippet>278</a> <span class="attr">#[snafu(display(<span class="string">"Failed to create channel manager for gRPC client"</span>))]
<a href=#279 id=279 data-nosnippet>279</a> </span>InvalidClientConfig {
<a href=#280 id=280 data-nosnippet>280</a> <span class="attr">#[snafu(implicit)]
<a href=#281 id=281 data-nosnippet>281</a> </span>location: Location,
<a href=#282 id=282 data-nosnippet>282</a> source: common_grpc::error::Error,
<a href=#283 id=283 data-nosnippet>283</a> },
<a href=#284 id=284 data-nosnippet>284</a>}
<a href=#285 id=285 data-nosnippet>285</a>
<a href=#286 id=286 data-nosnippet>286</a><span class="doccomment">/// the outer message is the full error stack, and inner message in header is the last error message that can be show directly to user
<a href=#287 id=287 data-nosnippet>287</a></span><span class="kw">pub fn </span>to_status_with_last_err(err: <span class="kw">impl </span>ErrorExt) -&gt; tonic::Status {
<a href=#288 id=288 data-nosnippet>288</a> <span class="kw">let </span>msg = err.to_string();
<a href=#289 id=289 data-nosnippet>289</a> <span class="kw">let </span>last_err_msg = common_error::ext::StackError::last(<span class="kw-2">&amp;</span>err).to_string();
<a href=#290 id=290 data-nosnippet>290</a> <span class="kw">let </span>code = err.status_code() <span class="kw">as </span>u32;
<a href=#291 id=291 data-nosnippet>291</a> <span class="kw">let </span><span class="kw-2">mut </span>header = from_err_code_msg_to_header(code, <span class="kw-2">&amp;</span>last_err_msg);
<a href=#292 id=292 data-nosnippet>292</a> header.insert(
<a href=#293 id=293 data-nosnippet>293</a> GREPTIME_DB_HEADER_ERROR_RETRY_HINT,
<a href=#294 id=294 data-nosnippet>294</a> HeaderValue::from_static(err.retry_hint().as_str()),
<a href=#295 id=295 data-nosnippet>295</a> );
<a href=#296 id=296 data-nosnippet>296</a>
<a href=#297 id=297 data-nosnippet>297</a> tonic::Status::with_metadata(
<a href=#298 id=298 data-nosnippet>298</a> tonic::Code::InvalidArgument,
<a href=#299 id=299 data-nosnippet>299</a> msg,
<a href=#300 id=300 data-nosnippet>300</a> MetadataMap::from_headers(header),
<a href=#301 id=301 data-nosnippet>301</a> )
<a href=#302 id=302 data-nosnippet>302</a>}
<a href=#303 id=303 data-nosnippet>303</a>
<a href=#304 id=304 data-nosnippet>304</a><span class="doccomment">/// Result type for flow module
<a href=#305 id=305 data-nosnippet>305</a></span><span class="kw">pub type </span><span class="prelude-ty">Result</span>&lt;T&gt; = std::result::Result&lt;T, Error&gt;;
<a href=#306 id=306 data-nosnippet>306</a>
<a href=#307 id=307 data-nosnippet>307</a><span class="kw">impl </span>ErrorExt <span class="kw">for </span>Error {
<a href=#308 id=308 data-nosnippet>308</a> <span class="kw">fn </span>status_code(<span class="kw-2">&amp;</span><span class="self">self</span>) -&gt; StatusCode {
<a href=#309 id=309 data-nosnippet>309</a> <span class="kw">match </span><span class="self">self </span>{
<a href=#310 id=310 data-nosnippet>310</a> <span class="self">Self</span>::JoinTask { .. }
<a href=#311 id=311 data-nosnippet>311</a> | <span class="self">Self</span>::Datafusion { .. }
<a href=#312 id=312 data-nosnippet>312</a> | <span class="self">Self</span>::InsertIntoFlow { .. }
<a href=#313 id=313 data-nosnippet>313</a> | <span class="self">Self</span>::NoAvailableFrontend { .. }
<a href=#314 id=314 data-nosnippet>314</a> | <span class="self">Self</span>::FlowNotRecovered { .. } =&gt; StatusCode::Internal,
<a href=#315 id=315 data-nosnippet>315</a> <span class="self">Self</span>::FlowAlreadyExist { .. } =&gt; StatusCode::TableAlreadyExists,
<a href=#316 id=316 data-nosnippet>316</a> <span class="self">Self</span>::TableNotFound { .. }
<a href=#317 id=317 data-nosnippet>317</a> | <span class="self">Self</span>::TableNotFoundMeta { .. }
<a href=#318 id=318 data-nosnippet>318</a> | <span class="self">Self</span>::ListFlows { .. } =&gt; StatusCode::TableNotFound,
<a href=#319 id=319 data-nosnippet>319</a> <span class="self">Self</span>::FlowNotFound { .. } =&gt; StatusCode::FlowNotFound,
<a href=#320 id=320 data-nosnippet>320</a> <span class="self">Self</span>::Plan { .. } | <span class="self">Self</span>::Datatypes { .. } =&gt; StatusCode::PlanQuery,
<a href=#321 id=321 data-nosnippet>321</a> <span class="self">Self</span>::CreateFlow { .. }
<a href=#322 id=322 data-nosnippet>322</a> | <span class="self">Self</span>::CreateSinkTable { .. }
<a href=#323 id=323 data-nosnippet>323</a> | <span class="self">Self</span>::Arrow { .. }
<a href=#324 id=324 data-nosnippet>324</a> | <span class="self">Self</span>::Time { .. } =&gt; StatusCode::EngineExecuteQuery,
<a href=#325 id=325 data-nosnippet>325</a> <span class="self">Self</span>::Unexpected { .. }
<a href=#326 id=326 data-nosnippet>326</a> | <span class="self">Self</span>::SyncCheckTask { .. }
<a href=#327 id=327 data-nosnippet>327</a> | <span class="self">Self</span>::IllegalCheckTaskState { .. } =&gt; StatusCode::Unexpected,
<a href=#328 id=328 data-nosnippet>328</a> <span class="self">Self</span>::Unsupported { .. } =&gt; StatusCode::Unsupported,
<a href=#329 id=329 data-nosnippet>329</a> <span class="self">Self</span>::External { source, .. } =&gt; source.status_code(),
<a href=#330 id=330 data-nosnippet>330</a> <span class="self">Self</span>::Internal { .. } | <span class="self">Self</span>::CacheRequired { .. } =&gt; StatusCode::Internal,
<a href=#331 id=331 data-nosnippet>331</a> <span class="self">Self</span>::StartServer { source, .. } | <span class="self">Self</span>::ShutdownServer { source, .. } =&gt; {
<a href=#332 id=332 data-nosnippet>332</a> source.status_code()
<a href=#333 id=333 data-nosnippet>333</a> }
<a href=#334 id=334 data-nosnippet>334</a> <span class="self">Self</span>::MetaClientInit { source, .. } =&gt; source.status_code(),
<a href=#335 id=335 data-nosnippet>335</a>
<a href=#336 id=336 data-nosnippet>336</a> <span class="self">Self</span>::InvalidQuery { .. }
<a href=#337 id=337 data-nosnippet>337</a> | <span class="self">Self</span>::InvalidRequest { .. }
<a href=#338 id=338 data-nosnippet>338</a> | <span class="self">Self</span>::ParseAddr { .. }
<a href=#339 id=339 data-nosnippet>339</a> | <span class="self">Self</span>::InvalidClientConfig { .. } =&gt; StatusCode::InvalidArguments,
<a href=#340 id=340 data-nosnippet>340</a>
<a href=#341 id=341 data-nosnippet>341</a> Error::SubstraitEncodeLogicalPlan { source, .. } =&gt; source.status_code(),
<a href=#342 id=342 data-nosnippet>342</a>
<a href=#343 id=343 data-nosnippet>343</a> Error::ConvertColumnSchema { source, .. } =&gt; source.status_code(),
<a href=#344 id=344 data-nosnippet>344</a> }
<a href=#345 id=345 data-nosnippet>345</a> }
<a href=#346 id=346 data-nosnippet>346</a>
<a href=#347 id=347 data-nosnippet>347</a> <span class="kw">fn </span>as_any(<span class="kw-2">&amp;</span><span class="self">self</span>) -&gt; <span class="kw-2">&amp;</span><span class="kw">dyn </span>Any {
<a href=#348 id=348 data-nosnippet>348</a> <span class="self">self
<a href=#349 id=349 data-nosnippet>349</a> </span>}
<a href=#350 id=350 data-nosnippet>350</a>
<a href=#351 id=351 data-nosnippet>351</a> <span class="kw">fn </span>retry_hint(<span class="kw-2">&amp;</span><span class="self">self</span>) -&gt; RetryHint {
<a href=#352 id=352 data-nosnippet>352</a> <span class="kw">match </span><span class="self">self </span>{
<a href=#353 id=353 data-nosnippet>353</a> <span class="self">Self</span>::FlowNotRecovered { .. } | <span class="self">Self</span>::NoAvailableFrontend { .. } =&gt; {
<a href=#354 id=354 data-nosnippet>354</a> RetryHint::Retryable
<a href=#355 id=355 data-nosnippet>355</a> }
<a href=#356 id=356 data-nosnippet>356</a>
<a href=#357 id=357 data-nosnippet>357</a> <span class="self">Self</span>::InsertIntoFlow { source, .. }
<a href=#358 id=358 data-nosnippet>358</a> | <span class="self">Self</span>::CreateFlow { source, .. }
<a href=#359 id=359 data-nosnippet>359</a> | <span class="self">Self</span>::CreateSinkTable { source, .. }
<a href=#360 id=360 data-nosnippet>360</a> | <span class="self">Self</span>::External { source, .. } =&gt; source.retry_hint(),
<a href=#361 id=361 data-nosnippet>361</a>
<a href=#362 id=362 data-nosnippet>362</a> <span class="self">Self</span>::Time { source, .. } =&gt; source.retry_hint(),
<a href=#363 id=363 data-nosnippet>363</a> <span class="self">Self</span>::TableNotFoundMeta { source, .. } | <span class="self">Self</span>::ListFlows { source, .. } =&gt; {
<a href=#364 id=364 data-nosnippet>364</a> source.retry_hint()
<a href=#365 id=365 data-nosnippet>365</a> }
<a href=#366 id=366 data-nosnippet>366</a> <span class="self">Self</span>::Datatypes { source, .. } =&gt; source.retry_hint(),
<a href=#367 id=367 data-nosnippet>367</a> <span class="self">Self</span>::StartServer { source, .. } | <span class="self">Self</span>::ShutdownServer { source, .. } =&gt; {
<a href=#368 id=368 data-nosnippet>368</a> source.retry_hint()
<a href=#369 id=369 data-nosnippet>369</a> }
<a href=#370 id=370 data-nosnippet>370</a> <span class="self">Self</span>::MetaClientInit { source, .. } =&gt; source.retry_hint(),
<a href=#371 id=371 data-nosnippet>371</a> <span class="self">Self</span>::InvalidRequest { source, .. } =&gt; source.retry_hint(),
<a href=#372 id=372 data-nosnippet>372</a> <span class="self">Self</span>::SubstraitEncodeLogicalPlan { source, .. } =&gt; source.retry_hint(),
<a href=#373 id=373 data-nosnippet>373</a> <span class="self">Self</span>::ConvertColumnSchema { source, .. } =&gt; source.retry_hint(),
<a href=#374 id=374 data-nosnippet>374</a> <span class="self">Self</span>::InvalidClientConfig { source, .. } =&gt; source.retry_hint(),
<a href=#375 id=375 data-nosnippet>375</a>
<a href=#376 id=376 data-nosnippet>376</a> <span class="kw">_ </span>=&gt; RetryHint::NonRetryable,
<a href=#377 id=377 data-nosnippet>377</a> }
<a href=#378 id=378 data-nosnippet>378</a> }
<a href=#379 id=379 data-nosnippet>379</a>}
<a href=#380 id=380 data-nosnippet>380</a>
<a href=#381 id=381 data-nosnippet>381</a><span class="macro">define_into_tonic_status!</span>(Error);
</code></pre></div></section></main></body></html>
+90
View File
@@ -0,0 +1,90 @@
<!DOCTYPE html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1.0"><meta name="generator" content="rustdoc"><meta name="description" content="Source of the Rust file `src/flow/src/expr.rs`."><title>expr.rs - source</title><script>if(window.location.protocol!=="file:")document.head.insertAdjacentHTML("beforeend","SourceSerif4-Regular-6b053e98.ttf.woff2,FiraSans-Italic-81dc35de.woff2,FiraSans-Regular-0fe48ade.woff2,FiraSans-MediumItalic-ccf7e434.woff2,FiraSans-Medium-e1aa3f0a.woff2,SourceCodePro-Regular-8badfe75.ttf.woff2,SourceCodePro-Semibold-aa29a496.ttf.woff2".split(",").map(f=>`<link rel="preload" as="font" type="font/woff2"href="../../static.files/${f}">`).join(""))</script><link rel="stylesheet" href="../../static.files/normalize-9960930a.css"><link rel="stylesheet" href="../../static.files/rustdoc-17e0aaed.css"><meta name="rustdoc-vars" data-root-path="../../" data-static-root-path="../../static.files/" data-current-crate="flow" data-themes="" data-resource-suffix="" data-rustdoc-version="1.96.1 (31fca3adb 2026-06-26)" data-channel="1.96.1" data-search-js="search-b5634cc7.js" data-stringdex-js="stringdex-2da4960a.js" data-settings-js="settings-170eb4bf.js" ><script src="../../static.files/storage-41dd4d93.js"></script><script defer src="../../static.files/src-script-813739b1.js"></script><script defer src="../../src-files.js"></script><script defer src="../../static.files/main-5013f961.js"></script><noscript><link rel="stylesheet" href="../../static.files/noscript-f7c3ffd8.css"></noscript><link rel="alternate icon" type="image/png" href="../../static.files/favicon-32x32-eab170b8.png"><link rel="icon" type="image/svg+xml" href="../../static.files/favicon-044be391.svg"></head><body class="rustdoc src"><a class="skip-main-content" href="#main-content">Skip to main content</a><!--[if lte IE 11]><div class="warning">This old browser is unsupported and will most likely display funky things.</div><![endif]--><nav class="sidebar"><div class="src-sidebar-title"><h2>Files</h2></div></nav><div class="sidebar-resizer" title="Drag to resize sidebar"></div><main><section id="main-content" class="content" tabindex="-1"><div class="main-heading"><h1><div class="sub-heading">flow/</div>expr.rs</h1><rustdoc-toolbar></rustdoc-toolbar></div><div class="example-wrap digits-2"><pre class="rust"><code><a href=#1 id=1 data-nosnippet>1</a><span class="comment">// Copyright 2023 Greptime Team
<a href=#2 id=2 data-nosnippet>2</a>//
<a href=#3 id=3 data-nosnippet>3</a>// Licensed under the Apache License, Version 2.0 (the "License");
<a href=#4 id=4 data-nosnippet>4</a>// you may not use this file except in compliance with the License.
<a href=#5 id=5 data-nosnippet>5</a>// You may obtain a copy of the License at
<a href=#6 id=6 data-nosnippet>6</a>//
<a href=#7 id=7 data-nosnippet>7</a>// http://www.apache.org/licenses/LICENSE-2.0
<a href=#8 id=8 data-nosnippet>8</a>//
<a href=#9 id=9 data-nosnippet>9</a>// Unless required by applicable law or agreed to in writing, software
<a href=#10 id=10 data-nosnippet>10</a>// distributed under the License is distributed on an "AS IS" BASIS,
<a href=#11 id=11 data-nosnippet>11</a>// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
<a href=#12 id=12 data-nosnippet>12</a>// See the License for the specific language governing permissions and
<a href=#13 id=13 data-nosnippet>13</a>// limitations under the License.
<a href=#14 id=14 data-nosnippet>14</a>
<a href=#15 id=15 data-nosnippet>15</a></span><span class="doccomment">//! Small row-batch conversion helper used by stateless streaming.
<a href=#16 id=16 data-nosnippet>16</a>
<a href=#17 id=17 data-nosnippet>17</a></span><span class="kw">pub</span>(<span class="kw">crate</span>) <span class="kw">mod </span>error;
<a href=#18 id=18 data-nosnippet>18</a><span class="kw">use </span>common_recordbatch::RecordBatch;
<a href=#19 id=19 data-nosnippet>19</a><span class="kw">use </span>datatypes::data_type::DataType;
<a href=#20 id=20 data-nosnippet>20</a><span class="kw">use </span>datatypes::prelude::ConcreteDataType;
<a href=#21 id=21 data-nosnippet>21</a><span class="kw">use </span>datatypes::vectors::{Helper, VectorRef};
<a href=#22 id=22 data-nosnippet>22</a><span class="kw">use </span>itertools::Itertools;
<a href=#23 id=23 data-nosnippet>23</a><span class="kw">use </span>snafu::{ResultExt, ensure};
<a href=#24 id=24 data-nosnippet>24</a>
<a href=#25 id=25 data-nosnippet>25</a><span class="kw">use </span><span class="kw">crate</span>::Error;
<a href=#26 id=26 data-nosnippet>26</a><span class="kw">use </span><span class="kw">crate</span>::error::DatatypesSnafu;
<a href=#27 id=27 data-nosnippet>27</a><span class="kw">use </span><span class="kw">crate</span>::repr::Row;
<a href=#28 id=28 data-nosnippet>28</a>
<a href=#29 id=29 data-nosnippet>29</a><span class="attr">#[derive(Debug, Clone)]
<a href=#30 id=30 data-nosnippet>30</a></span><span class="kw">pub struct </span>Batch {
<a href=#31 id=31 data-nosnippet>31</a> batch: Vec&lt;VectorRef&gt;,
<a href=#32 id=32 data-nosnippet>32</a> row_count: usize,
<a href=#33 id=33 data-nosnippet>33</a>}
<a href=#34 id=34 data-nosnippet>34</a>
<a href=#35 id=35 data-nosnippet>35</a><span class="kw">impl </span>TryFrom&lt;RecordBatch&gt; <span class="kw">for </span>Batch {
<a href=#36 id=36 data-nosnippet>36</a> <span class="kw">type </span>Error = Error;
<a href=#37 id=37 data-nosnippet>37</a> <span class="kw">fn </span>try_from(value: RecordBatch) -&gt; <span class="prelude-ty">Result</span>&lt;<span class="self">Self</span>, <span class="self">Self</span>::Error&gt; {
<a href=#38 id=38 data-nosnippet>38</a> <span class="prelude-val">Ok</span>(<span class="self">Self </span>{
<a href=#39 id=39 data-nosnippet>39</a> batch: Helper::try_into_vectors(value.columns()).context(DatatypesSnafu {
<a href=#40 id=40 data-nosnippet>40</a> extra: <span class="string">"failed to convert Arrow array to vector"</span>,
<a href=#41 id=41 data-nosnippet>41</a> })<span class="question-mark">?</span>,
<a href=#42 id=42 data-nosnippet>42</a> row_count: value.num_rows(),
<a href=#43 id=43 data-nosnippet>43</a> })
<a href=#44 id=44 data-nosnippet>44</a> }
<a href=#45 id=45 data-nosnippet>45</a>}
<a href=#46 id=46 data-nosnippet>46</a>
<a href=#47 id=47 data-nosnippet>47</a><span class="kw">impl </span>Batch {
<a href=#48 id=48 data-nosnippet>48</a> <span class="kw">pub fn </span>try_from_rows_with_types(
<a href=#49 id=49 data-nosnippet>49</a> rows: Vec&lt;Row&gt;,
<a href=#50 id=50 data-nosnippet>50</a> types: <span class="kw-2">&amp;</span>[ConcreteDataType],
<a href=#51 id=51 data-nosnippet>51</a> ) -&gt; <span class="prelude-ty">Result</span>&lt;<span class="self">Self</span>, error::EvalError&gt; {
<a href=#52 id=52 data-nosnippet>52</a> <span class="kw">if </span>rows.is_empty() {
<a href=#53 id=53 data-nosnippet>53</a> <span class="kw">return </span><span class="prelude-val">Ok</span>(<span class="self">Self </span>{
<a href=#54 id=54 data-nosnippet>54</a> batch: <span class="macro">vec!</span>[],
<a href=#55 id=55 data-nosnippet>55</a> row_count: <span class="number">0</span>,
<a href=#56 id=56 data-nosnippet>56</a> });
<a href=#57 id=57 data-nosnippet>57</a> }
<a href=#58 id=58 data-nosnippet>58</a> <span class="kw">let </span>len = rows.len();
<a href=#59 id=59 data-nosnippet>59</a> <span class="kw">let </span><span class="kw-2">mut </span>builders = types
<a href=#60 id=60 data-nosnippet>60</a> .iter()
<a href=#61 id=61 data-nosnippet>61</a> .map(|ty| ty.create_mutable_vector(len))
<a href=#62 id=62 data-nosnippet>62</a> .collect_vec();
<a href=#63 id=63 data-nosnippet>63</a> <span class="macro">ensure!</span>(
<a href=#64 id=64 data-nosnippet>64</a> rows.iter().all(|row| row.len() == builders.len()),
<a href=#65 id=65 data-nosnippet>65</a> error::InvalidArgumentSnafu {
<a href=#66 id=66 data-nosnippet>66</a> reason: <span class="string">"row length does not match schema"</span>.to_string()
<a href=#67 id=67 data-nosnippet>67</a> }
<a href=#68 id=68 data-nosnippet>68</a> );
<a href=#69 id=69 data-nosnippet>69</a> <span class="kw">for </span>row <span class="kw">in </span>rows {
<a href=#70 id=70 data-nosnippet>70</a> <span class="kw">for </span>(idx, value) <span class="kw">in </span>row.iter().enumerate() {
<a href=#71 id=71 data-nosnippet>71</a> builders[idx]
<a href=#72 id=72 data-nosnippet>72</a> .try_push_value_ref(<span class="kw-2">&amp;</span>value.as_value_ref())
<a href=#73 id=73 data-nosnippet>73</a> .context(error::DataTypeSnafu {
<a href=#74 id=74 data-nosnippet>74</a> msg: <span class="string">"failed to convert rows to columns"</span>,
<a href=#75 id=75 data-nosnippet>75</a> })<span class="question-mark">?</span>;
<a href=#76 id=76 data-nosnippet>76</a> }
<a href=#77 id=77 data-nosnippet>77</a> }
<a href=#78 id=78 data-nosnippet>78</a> <span class="prelude-val">Ok</span>(<span class="self">Self </span>{
<a href=#79 id=79 data-nosnippet>79</a> batch: builders.into_iter().map(|<span class="kw-2">mut </span>b| b.to_vector()).collect(),
<a href=#80 id=80 data-nosnippet>80</a> row_count: len,
<a href=#81 id=81 data-nosnippet>81</a> })
<a href=#82 id=82 data-nosnippet>82</a> }
<a href=#83 id=83 data-nosnippet>83</a> <span class="kw">pub fn </span>batch(<span class="kw-2">&amp;</span><span class="self">self</span>) -&gt; <span class="kw-2">&amp;</span>[VectorRef] {
<a href=#84 id=84 data-nosnippet>84</a> <span class="kw-2">&amp;</span><span class="self">self</span>.batch
<a href=#85 id=85 data-nosnippet>85</a> }
<a href=#86 id=86 data-nosnippet>86</a> <span class="kw">pub fn </span>row_count(<span class="kw-2">&amp;</span><span class="self">self</span>) -&gt; usize {
<a href=#87 id=87 data-nosnippet>87</a> <span class="self">self</span>.row_count
<a href=#88 id=88 data-nosnippet>88</a> }
<a href=#89 id=89 data-nosnippet>89</a>}
</code></pre></div></section></main></body></html>
+134
View File
@@ -0,0 +1,134 @@
<!DOCTYPE html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1.0"><meta name="generator" content="rustdoc"><meta name="description" content="Source of the Rust file `src/flow/src/expr/error.rs`."><title>error.rs - source</title><script>if(window.location.protocol!=="file:")document.head.insertAdjacentHTML("beforeend","SourceSerif4-Regular-6b053e98.ttf.woff2,FiraSans-Italic-81dc35de.woff2,FiraSans-Regular-0fe48ade.woff2,FiraSans-MediumItalic-ccf7e434.woff2,FiraSans-Medium-e1aa3f0a.woff2,SourceCodePro-Regular-8badfe75.ttf.woff2,SourceCodePro-Semibold-aa29a496.ttf.woff2".split(",").map(f=>`<link rel="preload" as="font" type="font/woff2"href="../../../static.files/${f}">`).join(""))</script><link rel="stylesheet" href="../../../static.files/normalize-9960930a.css"><link rel="stylesheet" href="../../../static.files/rustdoc-17e0aaed.css"><meta name="rustdoc-vars" data-root-path="../../../" data-static-root-path="../../../static.files/" data-current-crate="flow" data-themes="" data-resource-suffix="" data-rustdoc-version="1.96.1 (31fca3adb 2026-06-26)" data-channel="1.96.1" data-search-js="search-b5634cc7.js" data-stringdex-js="stringdex-2da4960a.js" data-settings-js="settings-170eb4bf.js" ><script src="../../../static.files/storage-41dd4d93.js"></script><script defer src="../../../static.files/src-script-813739b1.js"></script><script defer src="../../../src-files.js"></script><script defer src="../../../static.files/main-5013f961.js"></script><noscript><link rel="stylesheet" href="../../../static.files/noscript-f7c3ffd8.css"></noscript><link rel="alternate icon" type="image/png" href="../../../static.files/favicon-32x32-eab170b8.png"><link rel="icon" type="image/svg+xml" href="../../../static.files/favicon-044be391.svg"></head><body class="rustdoc src"><a class="skip-main-content" href="#main-content">Skip to main content</a><!--[if lte IE 11]><div class="warning">This old browser is unsupported and will most likely display funky things.</div><![endif]--><nav class="sidebar"><div class="src-sidebar-title"><h2>Files</h2></div></nav><div class="sidebar-resizer" title="Drag to resize sidebar"></div><main><section id="main-content" class="content" tabindex="-1"><div class="main-heading"><h1><div class="sub-heading">flow/expr/</div>error.rs</h1><rustdoc-toolbar></rustdoc-toolbar></div><div class="example-wrap digits-3"><pre class="rust"><code><a href=#1 id=1 data-nosnippet>1</a><span class="comment">// Copyright 2023 Greptime Team
<a href=#2 id=2 data-nosnippet>2</a>//
<a href=#3 id=3 data-nosnippet>3</a>// Licensed under the Apache License, Version 2.0 (the "License");
<a href=#4 id=4 data-nosnippet>4</a>// you may not use this file except in compliance with the License.
<a href=#5 id=5 data-nosnippet>5</a>// You may obtain a copy of the License at
<a href=#6 id=6 data-nosnippet>6</a>//
<a href=#7 id=7 data-nosnippet>7</a>// http://www.apache.org/licenses/LICENSE-2.0
<a href=#8 id=8 data-nosnippet>8</a>//
<a href=#9 id=9 data-nosnippet>9</a>// Unless required by applicable law or agreed to in writing, software
<a href=#10 id=10 data-nosnippet>10</a>// distributed under the License is distributed on an "AS IS" BASIS,
<a href=#11 id=11 data-nosnippet>11</a>// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
<a href=#12 id=12 data-nosnippet>12</a>// See the License for the specific language governing permissions and
<a href=#13 id=13 data-nosnippet>13</a>// limitations under the License.
<a href=#14 id=14 data-nosnippet>14</a>
<a href=#15 id=15 data-nosnippet>15</a></span><span class="doccomment">//! Error handling for expression evaluation.
<a href=#16 id=16 data-nosnippet>16</a>
<a href=#17 id=17 data-nosnippet>17</a></span><span class="kw">use </span>std::any::Any;
<a href=#18 id=18 data-nosnippet>18</a>
<a href=#19 id=19 data-nosnippet>19</a><span class="kw">use </span>arrow_schema::ArrowError;
<a href=#20 id=20 data-nosnippet>20</a><span class="kw">use </span>common_error::ext::{BoxedError, ErrorExt};
<a href=#21 id=21 data-nosnippet>21</a><span class="kw">use </span>common_error::status_code::StatusCode;
<a href=#22 id=22 data-nosnippet>22</a><span class="kw">use </span>common_macro::stack_trace_debug;
<a href=#23 id=23 data-nosnippet>23</a><span class="kw">use </span>datafusion_common::DataFusionError;
<a href=#24 id=24 data-nosnippet>24</a><span class="kw">use </span>datatypes::data_type::ConcreteDataType;
<a href=#25 id=25 data-nosnippet>25</a><span class="kw">use </span>snafu::{Location, Snafu};
<a href=#26 id=26 data-nosnippet>26</a>
<a href=#27 id=27 data-nosnippet>27</a><span class="doccomment">/// EvalError is about errors happen on columnar evaluation
<a href=#28 id=28 data-nosnippet>28</a>///
<a href=#29 id=29 data-nosnippet>29</a>/// TODO(discord9): add detailed location of column/operator(instead of code) to errors tp help identify related column
<a href=#30 id=30 data-nosnippet>30</a></span><span class="attr">#[derive(Snafu)]
<a href=#31 id=31 data-nosnippet>31</a>#[snafu(visibility(<span class="kw">pub</span>))]
<a href=#32 id=32 data-nosnippet>32</a>#[stack_trace_debug]
<a href=#33 id=33 data-nosnippet>33</a></span><span class="kw">pub enum </span>EvalError {
<a href=#34 id=34 data-nosnippet>34</a> <span class="attr">#[snafu(display(<span class="string">"Type mismatch: expected {expected}, actual {actual}"</span>))]
<a href=#35 id=35 data-nosnippet>35</a> </span>TypeMismatch {
<a href=#36 id=36 data-nosnippet>36</a> expected: ConcreteDataType,
<a href=#37 id=37 data-nosnippet>37</a> actual: ConcreteDataType,
<a href=#38 id=38 data-nosnippet>38</a> <span class="attr">#[snafu(implicit)]
<a href=#39 id=39 data-nosnippet>39</a> </span>location: Location,
<a href=#40 id=40 data-nosnippet>40</a> },
<a href=#41 id=41 data-nosnippet>41</a>
<a href=#42 id=42 data-nosnippet>42</a> <span class="doccomment">/// can't nest datatypes error because EvalError need to be store in map and serialization
<a href=#43 id=43 data-nosnippet>43</a> </span><span class="attr">#[snafu(display(<span class="string">"Fail to unpack from value to given type: {msg}"</span>))]
<a href=#44 id=44 data-nosnippet>44</a> </span>TryFromValue {
<a href=#45 id=45 data-nosnippet>45</a> msg: String,
<a href=#46 id=46 data-nosnippet>46</a> <span class="attr">#[snafu(implicit)]
<a href=#47 id=47 data-nosnippet>47</a> </span>location: Location,
<a href=#48 id=48 data-nosnippet>48</a> },
<a href=#49 id=49 data-nosnippet>49</a>
<a href=#50 id=50 data-nosnippet>50</a> <span class="attr">#[snafu(display(<span class="string">"Fail to cast value of type {from} to given type {to}"</span>))]
<a href=#51 id=51 data-nosnippet>51</a> </span>CastValue {
<a href=#52 id=52 data-nosnippet>52</a> from: ConcreteDataType,
<a href=#53 id=53 data-nosnippet>53</a> to: ConcreteDataType,
<a href=#54 id=54 data-nosnippet>54</a> source: datatypes::Error,
<a href=#55 id=55 data-nosnippet>55</a> <span class="attr">#[snafu(implicit)]
<a href=#56 id=56 data-nosnippet>56</a> </span>location: Location,
<a href=#57 id=57 data-nosnippet>57</a> },
<a href=#58 id=58 data-nosnippet>58</a>
<a href=#59 id=59 data-nosnippet>59</a> <span class="attr">#[snafu(display(<span class="string">"{msg}"</span>))]
<a href=#60 id=60 data-nosnippet>60</a> </span>DataType {
<a href=#61 id=61 data-nosnippet>61</a> msg: String,
<a href=#62 id=62 data-nosnippet>62</a> source: datatypes::Error,
<a href=#63 id=63 data-nosnippet>63</a> <span class="attr">#[snafu(implicit)]
<a href=#64 id=64 data-nosnippet>64</a> </span>location: Location,
<a href=#65 id=65 data-nosnippet>65</a> },
<a href=#66 id=66 data-nosnippet>66</a>
<a href=#67 id=67 data-nosnippet>67</a> <span class="attr">#[snafu(display(<span class="string">"Invalid argument: {reason}"</span>))]
<a href=#68 id=68 data-nosnippet>68</a> </span>InvalidArgument {
<a href=#69 id=69 data-nosnippet>69</a> reason: String,
<a href=#70 id=70 data-nosnippet>70</a> <span class="attr">#[snafu(implicit)]
<a href=#71 id=71 data-nosnippet>71</a> </span>location: Location,
<a href=#72 id=72 data-nosnippet>72</a> },
<a href=#73 id=73 data-nosnippet>73</a>
<a href=#74 id=74 data-nosnippet>74</a> <span class="attr">#[snafu(display(<span class="string">"Internal error: {reason}"</span>))]
<a href=#75 id=75 data-nosnippet>75</a> </span>Internal {
<a href=#76 id=76 data-nosnippet>76</a> reason: String,
<a href=#77 id=77 data-nosnippet>77</a> <span class="attr">#[snafu(implicit)]
<a href=#78 id=78 data-nosnippet>78</a> </span>location: Location,
<a href=#79 id=79 data-nosnippet>79</a> },
<a href=#80 id=80 data-nosnippet>80</a>
<a href=#81 id=81 data-nosnippet>81</a> <span class="attr">#[snafu(display(<span class="string">"Overflowed during evaluation"</span>))]
<a href=#82 id=82 data-nosnippet>82</a> </span>Overflow {
<a href=#83 id=83 data-nosnippet>83</a> <span class="attr">#[snafu(implicit)]
<a href=#84 id=84 data-nosnippet>84</a> </span>location: Location,
<a href=#85 id=85 data-nosnippet>85</a> },
<a href=#86 id=86 data-nosnippet>86</a>
<a href=#87 id=87 data-nosnippet>87</a> <span class="attr">#[snafu(display(<span class="string">"Arrow error: {error:?}, context: {context}"</span>))]
<a href=#88 id=88 data-nosnippet>88</a> </span>Arrow {
<a href=#89 id=89 data-nosnippet>89</a> <span class="attr">#[snafu(source)]
<a href=#90 id=90 data-nosnippet>90</a> </span>error: ArrowError,
<a href=#91 id=91 data-nosnippet>91</a> context: String,
<a href=#92 id=92 data-nosnippet>92</a> <span class="attr">#[snafu(implicit)]
<a href=#93 id=93 data-nosnippet>93</a> </span>location: Location,
<a href=#94 id=94 data-nosnippet>94</a> },
<a href=#95 id=95 data-nosnippet>95</a>
<a href=#96 id=96 data-nosnippet>96</a> <span class="attr">#[snafu(display(<span class="string">"DataFusion error: {error:?}, context: {context}"</span>))]
<a href=#97 id=97 data-nosnippet>97</a> </span>Datafusion {
<a href=#98 id=98 data-nosnippet>98</a> <span class="attr">#[snafu(source)]
<a href=#99 id=99 data-nosnippet>99</a> </span>error: DataFusionError,
<a href=#100 id=100 data-nosnippet>100</a> context: String,
<a href=#101 id=101 data-nosnippet>101</a> <span class="attr">#[snafu(implicit)]
<a href=#102 id=102 data-nosnippet>102</a> </span>location: Location,
<a href=#103 id=103 data-nosnippet>103</a> },
<a href=#104 id=104 data-nosnippet>104</a>
<a href=#105 id=105 data-nosnippet>105</a> <span class="attr">#[snafu(display(<span class="string">"External error"</span>))]
<a href=#106 id=106 data-nosnippet>106</a> </span>External {
<a href=#107 id=107 data-nosnippet>107</a> <span class="attr">#[snafu(implicit)]
<a href=#108 id=108 data-nosnippet>108</a> </span>location: Location,
<a href=#109 id=109 data-nosnippet>109</a> source: BoxedError,
<a href=#110 id=110 data-nosnippet>110</a> },
<a href=#111 id=111 data-nosnippet>111</a>}
<a href=#112 id=112 data-nosnippet>112</a>
<a href=#113 id=113 data-nosnippet>113</a><span class="kw">impl </span>ErrorExt <span class="kw">for </span>EvalError {
<a href=#114 id=114 data-nosnippet>114</a> <span class="kw">fn </span>status_code(<span class="kw-2">&amp;</span><span class="self">self</span>) -&gt; StatusCode {
<a href=#115 id=115 data-nosnippet>115</a> <span class="kw">use </span>EvalError::<span class="kw-2">*</span>;
<a href=#116 id=116 data-nosnippet>116</a> <span class="kw">match </span><span class="self">self </span>{
<a href=#117 id=117 data-nosnippet>117</a> TypeMismatch { .. }
<a href=#118 id=118 data-nosnippet>118</a> | TryFromValue { .. }
<a href=#119 id=119 data-nosnippet>119</a> | InvalidArgument { .. }
<a href=#120 id=120 data-nosnippet>120</a> | Overflow { .. } =&gt; StatusCode::InvalidArguments,
<a href=#121 id=121 data-nosnippet>121</a>
<a href=#122 id=122 data-nosnippet>122</a> CastValue { source, .. } | DataType { source, .. } =&gt; source.status_code(),
<a href=#123 id=123 data-nosnippet>123</a>
<a href=#124 id=124 data-nosnippet>124</a> Internal { .. } | Arrow { .. } | Datafusion { .. } | External { .. } =&gt; {
<a href=#125 id=125 data-nosnippet>125</a> StatusCode::Internal
<a href=#126 id=126 data-nosnippet>126</a> }
<a href=#127 id=127 data-nosnippet>127</a> }
<a href=#128 id=128 data-nosnippet>128</a> }
<a href=#129 id=129 data-nosnippet>129</a>
<a href=#130 id=130 data-nosnippet>130</a> <span class="kw">fn </span>as_any(<span class="kw-2">&amp;</span><span class="self">self</span>) -&gt; <span class="kw-2">&amp;</span><span class="kw">dyn </span>Any {
<a href=#131 id=131 data-nosnippet>131</a> <span class="self">self
<a href=#132 id=132 data-nosnippet>132</a> </span>}
<a href=#133 id=133 data-nosnippet>133</a>}
</code></pre></div></section></main></body></html>
+332
View File
@@ -0,0 +1,332 @@
<!DOCTYPE html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1.0"><meta name="generator" content="rustdoc"><meta name="description" content="Source of the Rust file `src/flow/src/heartbeat.rs`."><title>heartbeat.rs - source</title><script>if(window.location.protocol!=="file:")document.head.insertAdjacentHTML("beforeend","SourceSerif4-Regular-6b053e98.ttf.woff2,FiraSans-Italic-81dc35de.woff2,FiraSans-Regular-0fe48ade.woff2,FiraSans-MediumItalic-ccf7e434.woff2,FiraSans-Medium-e1aa3f0a.woff2,SourceCodePro-Regular-8badfe75.ttf.woff2,SourceCodePro-Semibold-aa29a496.ttf.woff2".split(",").map(f=>`<link rel="preload" as="font" type="font/woff2"href="../../static.files/${f}">`).join(""))</script><link rel="stylesheet" href="../../static.files/normalize-9960930a.css"><link rel="stylesheet" href="../../static.files/rustdoc-17e0aaed.css"><meta name="rustdoc-vars" data-root-path="../../" data-static-root-path="../../static.files/" data-current-crate="flow" data-themes="" data-resource-suffix="" data-rustdoc-version="1.96.1 (31fca3adb 2026-06-26)" data-channel="1.96.1" data-search-js="search-b5634cc7.js" data-stringdex-js="stringdex-2da4960a.js" data-settings-js="settings-170eb4bf.js" ><script src="../../static.files/storage-41dd4d93.js"></script><script defer src="../../static.files/src-script-813739b1.js"></script><script defer src="../../src-files.js"></script><script defer src="../../static.files/main-5013f961.js"></script><noscript><link rel="stylesheet" href="../../static.files/noscript-f7c3ffd8.css"></noscript><link rel="alternate icon" type="image/png" href="../../static.files/favicon-32x32-eab170b8.png"><link rel="icon" type="image/svg+xml" href="../../static.files/favicon-044be391.svg"></head><body class="rustdoc src"><a class="skip-main-content" href="#main-content">Skip to main content</a><!--[if lte IE 11]><div class="warning">This old browser is unsupported and will most likely display funky things.</div><![endif]--><nav class="sidebar"><div class="src-sidebar-title"><h2>Files</h2></div></nav><div class="sidebar-resizer" title="Drag to resize sidebar"></div><main><section id="main-content" class="content" tabindex="-1"><div class="main-heading"><h1><div class="sub-heading">flow/</div>heartbeat.rs</h1><rustdoc-toolbar></rustdoc-toolbar></div><div class="example-wrap digits-3"><pre class="rust"><code><a href=#1 id=1 data-nosnippet>1</a><span class="comment">// Copyright 2023 Greptime Team
<a href=#2 id=2 data-nosnippet>2</a>//
<a href=#3 id=3 data-nosnippet>3</a>// Licensed under the Apache License, Version 2.0 (the "License");
<a href=#4 id=4 data-nosnippet>4</a>// you may not use this file except in compliance with the License.
<a href=#5 id=5 data-nosnippet>5</a>// You may obtain a copy of the License at
<a href=#6 id=6 data-nosnippet>6</a>//
<a href=#7 id=7 data-nosnippet>7</a>// http://www.apache.org/licenses/LICENSE-2.0
<a href=#8 id=8 data-nosnippet>8</a>//
<a href=#9 id=9 data-nosnippet>9</a>// Unless required by applicable law or agreed to in writing, software
<a href=#10 id=10 data-nosnippet>10</a>// distributed under the License is distributed on an "AS IS" BASIS,
<a href=#11 id=11 data-nosnippet>11</a>// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
<a href=#12 id=12 data-nosnippet>12</a>// See the License for the specific language governing permissions and
<a href=#13 id=13 data-nosnippet>13</a>// limitations under the License.
<a href=#14 id=14 data-nosnippet>14</a>
<a href=#15 id=15 data-nosnippet>15</a></span><span class="doccomment">//! Send heartbeat from flownode to metasrv
<a href=#16 id=16 data-nosnippet>16</a>
<a href=#17 id=17 data-nosnippet>17</a></span><span class="kw">use </span>std::sync::Arc;
<a href=#18 id=18 data-nosnippet>18</a><span class="kw">use </span>std::sync::atomic::{AtomicBool, Ordering};
<a href=#19 id=19 data-nosnippet>19</a>
<a href=#20 id=20 data-nosnippet>20</a><span class="kw">use </span>api::v1::meta::{HeartbeatRequest, Peer};
<a href=#21 id=21 data-nosnippet>21</a><span class="kw">use </span>common_error::ext::BoxedError;
<a href=#22 id=22 data-nosnippet>22</a><span class="kw">use </span>common_meta::heartbeat::handler::{
<a href=#23 id=23 data-nosnippet>23</a> HeartbeatResponseHandlerContext, HeartbeatResponseHandlerExecutorRef,
<a href=#24 id=24 data-nosnippet>24</a>};
<a href=#25 id=25 data-nosnippet>25</a><span class="kw">use </span>common_meta::heartbeat::mailbox::{HeartbeatMailbox, MailboxRef, OutgoingMessage};
<a href=#26 id=26 data-nosnippet>26</a><span class="kw">use </span>common_meta::heartbeat::utils::outgoing_message_to_mailbox_message;
<a href=#27 id=27 data-nosnippet>27</a><span class="kw">use </span>common_meta::key::flow::flow_state::FlowStat;
<a href=#28 id=28 data-nosnippet>28</a><span class="kw">use </span>common_stat::ResourceStatRef;
<a href=#29 id=29 data-nosnippet>29</a><span class="kw">use </span>common_telemetry::{debug, error, info, warn};
<a href=#30 id=30 data-nosnippet>30</a><span class="kw">use </span>greptime_proto::v1::meta::NodeInfo;
<a href=#31 id=31 data-nosnippet>31</a><span class="kw">use </span>meta_client::client::{HeartbeatSender, HeartbeatStream, MetaClient};
<a href=#32 id=32 data-nosnippet>32</a><span class="kw">use </span>servers::addrs;
<a href=#33 id=33 data-nosnippet>33</a><span class="kw">use </span>snafu::ResultExt;
<a href=#34 id=34 data-nosnippet>34</a><span class="kw">use </span>tokio::sync::mpsc;
<a href=#35 id=35 data-nosnippet>35</a><span class="kw">use </span>tokio::time::Duration;
<a href=#36 id=36 data-nosnippet>36</a>
<a href=#37 id=37 data-nosnippet>37</a><span class="kw">use </span><span class="kw">crate</span>::error::ExternalSnafu;
<a href=#38 id=38 data-nosnippet>38</a><span class="kw">use </span><span class="kw">crate</span>::utils::SizeReportSender;
<a href=#39 id=39 data-nosnippet>39</a><span class="kw">use </span>crate::{Error, FlownodeOptions};
<a href=#40 id=40 data-nosnippet>40</a>
<a href=#41 id=41 data-nosnippet>41</a><span class="kw">async fn </span>query_flow_state(
<a href=#42 id=42 data-nosnippet>42</a> query_stat_size: <span class="kw-2">&amp;</span><span class="prelude-ty">Option</span>&lt;SizeReportSender&gt;,
<a href=#43 id=43 data-nosnippet>43</a> timeout: Duration,
<a href=#44 id=44 data-nosnippet>44</a>) -&gt; <span class="prelude-ty">Option</span>&lt;FlowStat&gt; {
<a href=#45 id=45 data-nosnippet>45</a> <span class="kw">if let </span><span class="prelude-val">Some</span>(report_requester) = query_stat_size.as_ref() {
<a href=#46 id=46 data-nosnippet>46</a> <span class="kw">let </span>ret = report_requester.query(timeout).<span class="kw">await</span>;
<a href=#47 id=47 data-nosnippet>47</a> <span class="kw">match </span>ret {
<a href=#48 id=48 data-nosnippet>48</a> <span class="prelude-val">Ok</span>(latest) =&gt; <span class="prelude-val">Some</span>(latest),
<a href=#49 id=49 data-nosnippet>49</a> <span class="prelude-val">Err</span>(err) =&gt; {
<a href=#50 id=50 data-nosnippet>50</a> <span class="macro">error!</span>(err; <span class="string">"Failed to get query stat size"</span>);
<a href=#51 id=51 data-nosnippet>51</a> <span class="prelude-val">None
<a href=#52 id=52 data-nosnippet>52</a> </span>}
<a href=#53 id=53 data-nosnippet>53</a> }
<a href=#54 id=54 data-nosnippet>54</a> } <span class="kw">else </span>{
<a href=#55 id=55 data-nosnippet>55</a> <span class="prelude-val">None
<a href=#56 id=56 data-nosnippet>56</a> </span>}
<a href=#57 id=57 data-nosnippet>57</a>}
<a href=#58 id=58 data-nosnippet>58</a>
<a href=#59 id=59 data-nosnippet>59</a><span class="doccomment">/// The flownode heartbeat task which sending `[HeartbeatRequest]` to Metasrv periodically in background.
<a href=#60 id=60 data-nosnippet>60</a></span><span class="attr">#[derive(Clone)]
<a href=#61 id=61 data-nosnippet>61</a></span><span class="kw">pub struct </span>HeartbeatTask {
<a href=#62 id=62 data-nosnippet>62</a> node_id: u64,
<a href=#63 id=63 data-nosnippet>63</a> node_epoch: u64,
<a href=#64 id=64 data-nosnippet>64</a> peer_addr: String,
<a href=#65 id=65 data-nosnippet>65</a> meta_client: Arc&lt;MetaClient&gt;,
<a href=#66 id=66 data-nosnippet>66</a> resp_handler_executor: HeartbeatResponseHandlerExecutorRef,
<a href=#67 id=67 data-nosnippet>67</a> running: Arc&lt;AtomicBool&gt;,
<a href=#68 id=68 data-nosnippet>68</a> query_stat_size: <span class="prelude-ty">Option</span>&lt;SizeReportSender&gt;,
<a href=#69 id=69 data-nosnippet>69</a> resource_stat: ResourceStatRef,
<a href=#70 id=70 data-nosnippet>70</a>}
<a href=#71 id=71 data-nosnippet>71</a>
<a href=#72 id=72 data-nosnippet>72</a><span class="kw">impl </span>HeartbeatTask {
<a href=#73 id=73 data-nosnippet>73</a> <span class="kw">pub fn </span>with_query_stat_size(<span class="kw-2">mut </span><span class="self">self</span>, query_stat_size: SizeReportSender) -&gt; <span class="self">Self </span>{
<a href=#74 id=74 data-nosnippet>74</a> <span class="self">self</span>.query_stat_size = <span class="prelude-val">Some</span>(query_stat_size);
<a href=#75 id=75 data-nosnippet>75</a> <span class="self">self
<a href=#76 id=76 data-nosnippet>76</a> </span>}
<a href=#77 id=77 data-nosnippet>77</a>
<a href=#78 id=78 data-nosnippet>78</a> <span class="kw">pub fn </span>new(
<a href=#79 id=79 data-nosnippet>79</a> opts: <span class="kw-2">&amp;</span>FlownodeOptions,
<a href=#80 id=80 data-nosnippet>80</a> meta_client: Arc&lt;MetaClient&gt;,
<a href=#81 id=81 data-nosnippet>81</a> resp_handler_executor: HeartbeatResponseHandlerExecutorRef,
<a href=#82 id=82 data-nosnippet>82</a> resource_stat: ResourceStatRef,
<a href=#83 id=83 data-nosnippet>83</a> ) -&gt; <span class="self">Self </span>{
<a href=#84 id=84 data-nosnippet>84</a> <span class="self">Self </span>{
<a href=#85 id=85 data-nosnippet>85</a> node_id: opts.node_id.unwrap_or(<span class="number">0</span>),
<a href=#86 id=86 data-nosnippet>86</a> node_epoch: common_time::util::current_time_millis() <span class="kw">as </span>u64,
<a href=#87 id=87 data-nosnippet>87</a> peer_addr: addrs::resolve_addr(<span class="kw-2">&amp;</span>opts.grpc.bind_addr, <span class="prelude-val">Some</span>(<span class="kw-2">&amp;</span>opts.grpc.server_addr)),
<a href=#88 id=88 data-nosnippet>88</a> meta_client,
<a href=#89 id=89 data-nosnippet>89</a> resp_handler_executor,
<a href=#90 id=90 data-nosnippet>90</a> running: Arc::new(AtomicBool::new(<span class="bool-val">false</span>)),
<a href=#91 id=91 data-nosnippet>91</a> query_stat_size: <span class="prelude-val">None</span>,
<a href=#92 id=92 data-nosnippet>92</a> resource_stat,
<a href=#93 id=93 data-nosnippet>93</a> }
<a href=#94 id=94 data-nosnippet>94</a> }
<a href=#95 id=95 data-nosnippet>95</a>
<a href=#96 id=96 data-nosnippet>96</a> <span class="kw">pub async fn </span>start(<span class="kw-2">&amp;</span><span class="self">self</span>) -&gt; <span class="prelude-ty">Result</span>&lt;(), Error&gt; {
<a href=#97 id=97 data-nosnippet>97</a> <span class="kw">if </span><span class="self">self
<a href=#98 id=98 data-nosnippet>98</a> </span>.running
<a href=#99 id=99 data-nosnippet>99</a> .compare_exchange(<span class="bool-val">false</span>, <span class="bool-val">true</span>, Ordering::AcqRel, Ordering::Acquire)
<a href=#100 id=100 data-nosnippet>100</a> .is_err()
<a href=#101 id=101 data-nosnippet>101</a> {
<a href=#102 id=102 data-nosnippet>102</a> <span class="macro">warn!</span>(<span class="string">"Heartbeat task started multiple times"</span>);
<a href=#103 id=103 data-nosnippet>103</a> <span class="kw">return </span><span class="prelude-val">Ok</span>(());
<a href=#104 id=104 data-nosnippet>104</a> }
<a href=#105 id=105 data-nosnippet>105</a>
<a href=#106 id=106 data-nosnippet>106</a> <span class="self">self</span>.create_streams().<span class="kw">await
<a href=#107 id=107 data-nosnippet>107</a> </span>}
<a href=#108 id=108 data-nosnippet>108</a>
<a href=#109 id=109 data-nosnippet>109</a> <span class="kw">async fn </span>create_streams(<span class="kw-2">&amp;</span><span class="self">self</span>) -&gt; <span class="prelude-ty">Result</span>&lt;(), Error&gt; {
<a href=#110 id=110 data-nosnippet>110</a> <span class="macro">info!</span>(<span class="string">"Establishing heartbeat connection to Metasrv..."</span>);
<a href=#111 id=111 data-nosnippet>111</a>
<a href=#112 id=112 data-nosnippet>112</a> <span class="kw">let </span>(req_sender, resp_stream, config) = <span class="self">self
<a href=#113 id=113 data-nosnippet>113</a> </span>.meta_client
<a href=#114 id=114 data-nosnippet>114</a> .heartbeat()
<a href=#115 id=115 data-nosnippet>115</a> .<span class="kw">await
<a href=#116 id=116 data-nosnippet>116</a> </span>.map_err(BoxedError::new)
<a href=#117 id=117 data-nosnippet>117</a> .context(ExternalSnafu)<span class="question-mark">?</span>;
<a href=#118 id=118 data-nosnippet>118</a>
<a href=#119 id=119 data-nosnippet>119</a> <span class="macro">info!</span>(
<a href=#120 id=120 data-nosnippet>120</a> <span class="string">"Heartbeat started for flownode {}, Metasrv config: {}"</span>,
<a href=#121 id=121 data-nosnippet>121</a> <span class="self">self</span>.node_id, config
<a href=#122 id=122 data-nosnippet>122</a> );
<a href=#123 id=123 data-nosnippet>123</a>
<a href=#124 id=124 data-nosnippet>124</a> <span class="kw">let </span>(outgoing_tx, outgoing_rx) = mpsc::channel(<span class="number">16</span>);
<a href=#125 id=125 data-nosnippet>125</a> <span class="kw">let </span>mailbox = Arc::new(HeartbeatMailbox::new(outgoing_tx));
<a href=#126 id=126 data-nosnippet>126</a>
<a href=#127 id=127 data-nosnippet>127</a> <span class="self">self</span>.start_handle_resp_stream(resp_stream, mailbox, config.retry_interval);
<a href=#128 id=128 data-nosnippet>128</a>
<a href=#129 id=129 data-nosnippet>129</a> <span class="self">self</span>.start_heartbeat_report(req_sender, outgoing_rx, config.interval);
<a href=#130 id=130 data-nosnippet>130</a>
<a href=#131 id=131 data-nosnippet>131</a> <span class="prelude-val">Ok</span>(())
<a href=#132 id=132 data-nosnippet>132</a> }
<a href=#133 id=133 data-nosnippet>133</a>
<a href=#134 id=134 data-nosnippet>134</a> <span class="kw">pub fn </span>shutdown(<span class="kw-2">&amp;</span><span class="self">self</span>) {
<a href=#135 id=135 data-nosnippet>135</a> <span class="macro">info!</span>(<span class="string">"Close heartbeat task for flownode"</span>);
<a href=#136 id=136 data-nosnippet>136</a> <span class="kw">if </span><span class="self">self
<a href=#137 id=137 data-nosnippet>137</a> </span>.running
<a href=#138 id=138 data-nosnippet>138</a> .compare_exchange(<span class="bool-val">true</span>, <span class="bool-val">false</span>, Ordering::AcqRel, Ordering::Acquire)
<a href=#139 id=139 data-nosnippet>139</a> .is_err()
<a href=#140 id=140 data-nosnippet>140</a> {
<a href=#141 id=141 data-nosnippet>141</a> <span class="macro">warn!</span>(<span class="string">"Call close heartbeat task multiple times"</span>);
<a href=#142 id=142 data-nosnippet>142</a> }
<a href=#143 id=143 data-nosnippet>143</a> }
<a href=#144 id=144 data-nosnippet>144</a>
<a href=#145 id=145 data-nosnippet>145</a> <span class="kw">fn </span>new_heartbeat_request(
<a href=#146 id=146 data-nosnippet>146</a> heartbeat_request: <span class="kw-2">&amp;</span>HeartbeatRequest,
<a href=#147 id=147 data-nosnippet>147</a> message: <span class="prelude-ty">Option</span>&lt;OutgoingMessage&gt;,
<a href=#148 id=148 data-nosnippet>148</a> latest_report: <span class="kw-2">&amp;</span><span class="prelude-ty">Option</span>&lt;FlowStat&gt;,
<a href=#149 id=149 data-nosnippet>149</a> cpu_usage: i64,
<a href=#150 id=150 data-nosnippet>150</a> memory_usage: i64,
<a href=#151 id=151 data-nosnippet>151</a> ) -&gt; <span class="prelude-ty">Option</span>&lt;HeartbeatRequest&gt; {
<a href=#152 id=152 data-nosnippet>152</a> <span class="kw">let </span>mailbox_message = <span class="kw">match </span>message.map(outgoing_message_to_mailbox_message) {
<a href=#153 id=153 data-nosnippet>153</a> <span class="prelude-val">Some</span>(<span class="prelude-val">Ok</span>(message)) =&gt; <span class="prelude-val">Some</span>(message),
<a href=#154 id=154 data-nosnippet>154</a> <span class="prelude-val">Some</span>(<span class="prelude-val">Err</span>(e)) =&gt; {
<a href=#155 id=155 data-nosnippet>155</a> <span class="macro">error!</span>(e; <span class="string">"Failed to encode mailbox messages"</span>);
<a href=#156 id=156 data-nosnippet>156</a> <span class="kw">return </span><span class="prelude-val">None</span>;
<a href=#157 id=157 data-nosnippet>157</a> }
<a href=#158 id=158 data-nosnippet>158</a> <span class="prelude-val">None </span>=&gt; <span class="prelude-val">None</span>,
<a href=#159 id=159 data-nosnippet>159</a> };
<a href=#160 id=160 data-nosnippet>160</a> <span class="kw">let </span>flow_stat = latest_report
<a href=#161 id=161 data-nosnippet>161</a> .as_ref()
<a href=#162 id=162 data-nosnippet>162</a> .map(|report| api::v1::meta::FlowStat {
<a href=#163 id=163 data-nosnippet>163</a> flow_stat_size: report
<a href=#164 id=164 data-nosnippet>164</a> .state_size
<a href=#165 id=165 data-nosnippet>165</a> .iter()
<a href=#166 id=166 data-nosnippet>166</a> .map(|(k, v)| (<span class="kw-2">*</span>k, <span class="kw-2">*</span>v <span class="kw">as </span>u64))
<a href=#167 id=167 data-nosnippet>167</a> .collect(),
<a href=#168 id=168 data-nosnippet>168</a> flow_last_exec_time_map: report
<a href=#169 id=169 data-nosnippet>169</a> .last_exec_time_map
<a href=#170 id=170 data-nosnippet>170</a> .iter()
<a href=#171 id=171 data-nosnippet>171</a> .map(|(k, v)| (<span class="kw-2">*</span>k, <span class="kw-2">*</span>v))
<a href=#172 id=172 data-nosnippet>172</a> .collect(),
<a href=#173 id=173 data-nosnippet>173</a> });
<a href=#174 id=174 data-nosnippet>174</a>
<a href=#175 id=175 data-nosnippet>175</a> <span class="kw">let </span><span class="kw-2">mut </span>heartbeat_request = HeartbeatRequest {
<a href=#176 id=176 data-nosnippet>176</a> mailbox_message,
<a href=#177 id=177 data-nosnippet>177</a> flow_stat,
<a href=#178 id=178 data-nosnippet>178</a> ..heartbeat_request.clone()
<a href=#179 id=179 data-nosnippet>179</a> };
<a href=#180 id=180 data-nosnippet>180</a>
<a href=#181 id=181 data-nosnippet>181</a> <span class="kw">if let </span><span class="prelude-val">Some</span>(info) = heartbeat_request.info.as_mut() {
<a href=#182 id=182 data-nosnippet>182</a> info.cpu_usage_millicores = cpu_usage;
<a href=#183 id=183 data-nosnippet>183</a> info.memory_usage_bytes = memory_usage;
<a href=#184 id=184 data-nosnippet>184</a> }
<a href=#185 id=185 data-nosnippet>185</a>
<a href=#186 id=186 data-nosnippet>186</a> <span class="prelude-val">Some</span>(heartbeat_request)
<a href=#187 id=187 data-nosnippet>187</a> }
<a href=#188 id=188 data-nosnippet>188</a>
<a href=#189 id=189 data-nosnippet>189</a> <span class="attr">#[allow(deprecated)]
<a href=#190 id=190 data-nosnippet>190</a> </span><span class="kw">fn </span>build_node_info(
<a href=#191 id=191 data-nosnippet>191</a> start_time_ms: u64,
<a href=#192 id=192 data-nosnippet>192</a> total_cpu_millicores: i64,
<a href=#193 id=193 data-nosnippet>193</a> total_memory_bytes: i64,
<a href=#194 id=194 data-nosnippet>194</a> ) -&gt; <span class="prelude-ty">Option</span>&lt;NodeInfo&gt; {
<a href=#195 id=195 data-nosnippet>195</a> <span class="kw">let </span>build_info = common_version::build_info();
<a href=#196 id=196 data-nosnippet>196</a> <span class="prelude-val">Some</span>(NodeInfo {
<a href=#197 id=197 data-nosnippet>197</a> version: build_info.version.to_string(),
<a href=#198 id=198 data-nosnippet>198</a> git_commit: build_info.commit_short.to_string(),
<a href=#199 id=199 data-nosnippet>199</a> start_time_ms,
<a href=#200 id=200 data-nosnippet>200</a> total_cpu_millicores,
<a href=#201 id=201 data-nosnippet>201</a> total_memory_bytes,
<a href=#202 id=202 data-nosnippet>202</a> cpu_usage_millicores: <span class="number">0</span>,
<a href=#203 id=203 data-nosnippet>203</a> memory_usage_bytes: <span class="number">0</span>,
<a href=#204 id=204 data-nosnippet>204</a> <span class="comment">// TODO(zyy17): Remove these deprecated fields when the deprecated fields are removed from the proto.
<a href=#205 id=205 data-nosnippet>205</a> </span>cpus: total_cpu_millicores <span class="kw">as </span>u32,
<a href=#206 id=206 data-nosnippet>206</a> memory_bytes: total_memory_bytes <span class="kw">as </span>u64,
<a href=#207 id=207 data-nosnippet>207</a> hostname: hostname::get()
<a href=#208 id=208 data-nosnippet>208</a> .unwrap_or_default()
<a href=#209 id=209 data-nosnippet>209</a> .to_string_lossy()
<a href=#210 id=210 data-nosnippet>210</a> .to_string(),
<a href=#211 id=211 data-nosnippet>211</a> })
<a href=#212 id=212 data-nosnippet>212</a> }
<a href=#213 id=213 data-nosnippet>213</a>
<a href=#214 id=214 data-nosnippet>214</a> <span class="kw">fn </span>start_heartbeat_report(
<a href=#215 id=215 data-nosnippet>215</a> <span class="kw-2">&amp;</span><span class="self">self</span>,
<a href=#216 id=216 data-nosnippet>216</a> req_sender: HeartbeatSender,
<a href=#217 id=217 data-nosnippet>217</a> <span class="kw-2">mut </span>outgoing_rx: mpsc::Receiver&lt;OutgoingMessage&gt;,
<a href=#218 id=218 data-nosnippet>218</a> report_interval: Duration,
<a href=#219 id=219 data-nosnippet>219</a> ) {
<a href=#220 id=220 data-nosnippet>220</a> <span class="kw">let </span>node_epoch = <span class="self">self</span>.node_epoch;
<a href=#221 id=221 data-nosnippet>221</a> <span class="kw">let </span>self_peer = <span class="prelude-val">Some</span>(Peer {
<a href=#222 id=222 data-nosnippet>222</a> id: <span class="self">self</span>.node_id,
<a href=#223 id=223 data-nosnippet>223</a> addr: <span class="self">self</span>.peer_addr.clone(),
<a href=#224 id=224 data-nosnippet>224</a> });
<a href=#225 id=225 data-nosnippet>225</a> <span class="kw">let </span>total_cpu_millicores = <span class="self">self</span>.resource_stat.get_total_cpu_millicores();
<a href=#226 id=226 data-nosnippet>226</a> <span class="kw">let </span>total_memory_bytes = <span class="self">self</span>.resource_stat.get_total_memory_bytes();
<a href=#227 id=227 data-nosnippet>227</a> <span class="kw">let </span>resource_stat = <span class="self">self</span>.resource_stat.clone();
<a href=#228 id=228 data-nosnippet>228</a> <span class="kw">let </span>query_stat_size = <span class="self">self</span>.query_stat_size.clone();
<a href=#229 id=229 data-nosnippet>229</a>
<a href=#230 id=230 data-nosnippet>230</a> common_runtime::spawn_hb(<span class="kw">async move </span>{
<a href=#231 id=231 data-nosnippet>231</a> <span class="comment">// note that using interval will cause it to first immediately send
<a href=#232 id=232 data-nosnippet>232</a> // a heartbeat
<a href=#233 id=233 data-nosnippet>233</a> </span><span class="kw">let </span><span class="kw-2">mut </span>interval = tokio::time::interval(report_interval);
<a href=#234 id=234 data-nosnippet>234</a> interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
<a href=#235 id=235 data-nosnippet>235</a> <span class="kw">let </span><span class="kw-2">mut </span>latest_report = <span class="prelude-val">None</span>;
<a href=#236 id=236 data-nosnippet>236</a>
<a href=#237 id=237 data-nosnippet>237</a> <span class="kw">let </span>heartbeat_request = HeartbeatRequest {
<a href=#238 id=238 data-nosnippet>238</a> peer: self_peer,
<a href=#239 id=239 data-nosnippet>239</a> node_epoch,
<a href=#240 id=240 data-nosnippet>240</a> info: <span class="self">Self</span>::build_node_info(node_epoch, total_cpu_millicores, total_memory_bytes),
<a href=#241 id=241 data-nosnippet>241</a> ..Default::default()
<a href=#242 id=242 data-nosnippet>242</a> };
<a href=#243 id=243 data-nosnippet>243</a>
<a href=#244 id=244 data-nosnippet>244</a> <span class="kw">loop </span>{
<a href=#245 id=245 data-nosnippet>245</a> <span class="kw">let </span>req = <span class="macro">tokio::select!</span> {
<a href=#246 id=246 data-nosnippet>246</a> message = outgoing_rx.recv() =&gt; {
<a href=#247 id=247 data-nosnippet>247</a> <span class="kw">if let </span><span class="prelude-val">Some</span>(message) = message {
<a href=#248 id=248 data-nosnippet>248</a> <span class="self">Self</span>::new_heartbeat_request(<span class="kw-2">&amp;</span>heartbeat_request, <span class="prelude-val">Some</span>(message), <span class="kw-2">&amp;</span>latest_report, <span class="number">0</span>, <span class="number">0</span>)
<a href=#249 id=249 data-nosnippet>249</a> } <span class="kw">else </span>{
<a href=#250 id=250 data-nosnippet>250</a> <span class="macro">warn!</span>(<span class="string">"Sender has been dropped, exiting the heartbeat loop"</span>);
<a href=#251 id=251 data-nosnippet>251</a> <span class="comment">// Receives None that means Sender was dropped, we need to break the current loop
<a href=#252 id=252 data-nosnippet>252</a> </span><span class="kw">break
<a href=#253 id=253 data-nosnippet>253</a> </span>}
<a href=#254 id=254 data-nosnippet>254</a> }
<a href=#255 id=255 data-nosnippet>255</a> <span class="kw">_ </span>= interval.tick() =&gt; {
<a href=#256 id=256 data-nosnippet>256</a> <span class="self">Self</span>::new_heartbeat_request(<span class="kw-2">&amp;</span>heartbeat_request, <span class="prelude-val">None</span>, <span class="kw-2">&amp;</span>latest_report, resource_stat.get_cpu_usage_millicores(), resource_stat.get_memory_usage_bytes())
<a href=#257 id=257 data-nosnippet>257</a> }
<a href=#258 id=258 data-nosnippet>258</a> };
<a href=#259 id=259 data-nosnippet>259</a>
<a href=#260 id=260 data-nosnippet>260</a> <span class="kw">if let </span><span class="prelude-val">Some</span>(req) = req {
<a href=#261 id=261 data-nosnippet>261</a> <span class="kw">if let </span><span class="prelude-val">Err</span>(e) = req_sender.send(req.clone()).<span class="kw">await </span>{
<a href=#262 id=262 data-nosnippet>262</a> <span class="macro">error!</span>(e; <span class="string">"Failed to send heartbeat to metasrv"</span>);
<a href=#263 id=263 data-nosnippet>263</a> <span class="kw">break</span>;
<a href=#264 id=264 data-nosnippet>264</a> } <span class="kw">else </span>{
<a href=#265 id=265 data-nosnippet>265</a> <span class="macro">debug!</span>(<span class="string">"Send a heartbeat request to metasrv, content: {:?}"</span>, req);
<a href=#266 id=266 data-nosnippet>266</a> }
<a href=#267 id=267 data-nosnippet>267</a> }
<a href=#268 id=268 data-nosnippet>268</a> <span class="comment">// after sending heartbeat, try to get the latest report
<a href=#269 id=269 data-nosnippet>269</a> // TODO(discord9): consider a better place to update the size report
<a href=#270 id=270 data-nosnippet>270</a> // set the timeout to half of the report interval so that it wouldn't delay heartbeat if something went horribly wrong
<a href=#271 id=271 data-nosnippet>271</a> </span>latest_report = query_flow_state(<span class="kw-2">&amp;</span>query_stat_size, report_interval / <span class="number">2</span>).<span class="kw">await</span>;
<a href=#272 id=272 data-nosnippet>272</a> }
<a href=#273 id=273 data-nosnippet>273</a>
<a href=#274 id=274 data-nosnippet>274</a> <span class="macro">info!</span>(<span class="string">"flownode heartbeat task stopped."</span>);
<a href=#275 id=275 data-nosnippet>275</a> });
<a href=#276 id=276 data-nosnippet>276</a> }
<a href=#277 id=277 data-nosnippet>277</a>
<a href=#278 id=278 data-nosnippet>278</a> <span class="kw">fn </span>start_handle_resp_stream(
<a href=#279 id=279 data-nosnippet>279</a> <span class="kw-2">&amp;</span><span class="self">self</span>,
<a href=#280 id=280 data-nosnippet>280</a> <span class="kw-2">mut </span>resp_stream: HeartbeatStream,
<a href=#281 id=281 data-nosnippet>281</a> mailbox: MailboxRef,
<a href=#282 id=282 data-nosnippet>282</a> retry_interval: Duration,
<a href=#283 id=283 data-nosnippet>283</a> ) {
<a href=#284 id=284 data-nosnippet>284</a> <span class="kw">let </span>capture_self = <span class="self">self</span>.clone();
<a href=#285 id=285 data-nosnippet>285</a>
<a href=#286 id=286 data-nosnippet>286</a> <span class="kw">let </span>_handle = common_runtime::spawn_hb(<span class="kw">async move </span>{
<a href=#287 id=287 data-nosnippet>287</a> <span class="kw">loop </span>{
<a href=#288 id=288 data-nosnippet>288</a> <span class="kw">match </span>resp_stream.message().<span class="kw">await </span>{
<a href=#289 id=289 data-nosnippet>289</a> <span class="prelude-val">Ok</span>(<span class="prelude-val">Some</span>(resp)) =&gt; {
<a href=#290 id=290 data-nosnippet>290</a> <span class="macro">debug!</span>(<span class="string">"Receiving heartbeat response: {:?}"</span>, resp);
<a href=#291 id=291 data-nosnippet>291</a> <span class="kw">let </span>ctx = HeartbeatResponseHandlerContext::new(mailbox.clone(), resp);
<a href=#292 id=292 data-nosnippet>292</a> <span class="kw">if let </span><span class="prelude-val">Err</span>(e) = capture_self.handle_response(ctx).<span class="kw">await </span>{
<a href=#293 id=293 data-nosnippet>293</a> <span class="macro">error!</span>(e; <span class="string">"Error while handling heartbeat response"</span>);
<a href=#294 id=294 data-nosnippet>294</a> }
<a href=#295 id=295 data-nosnippet>295</a> }
<a href=#296 id=296 data-nosnippet>296</a> <span class="prelude-val">Ok</span>(<span class="prelude-val">None</span>) =&gt; {
<a href=#297 id=297 data-nosnippet>297</a> <span class="macro">warn!</span>(<span class="string">"Heartbeat response stream closed"</span>);
<a href=#298 id=298 data-nosnippet>298</a> capture_self.start_with_retry(retry_interval).<span class="kw">await</span>;
<a href=#299 id=299 data-nosnippet>299</a> <span class="kw">break</span>;
<a href=#300 id=300 data-nosnippet>300</a> }
<a href=#301 id=301 data-nosnippet>301</a> <span class="prelude-val">Err</span>(e) =&gt; {
<a href=#302 id=302 data-nosnippet>302</a> <span class="macro">error!</span>(e; <span class="string">"Occur error while reading heartbeat response"</span>);
<a href=#303 id=303 data-nosnippet>303</a> capture_self.start_with_retry(retry_interval).<span class="kw">await</span>;
<a href=#304 id=304 data-nosnippet>304</a>
<a href=#305 id=305 data-nosnippet>305</a> <span class="kw">break</span>;
<a href=#306 id=306 data-nosnippet>306</a> }
<a href=#307 id=307 data-nosnippet>307</a> }
<a href=#308 id=308 data-nosnippet>308</a> }
<a href=#309 id=309 data-nosnippet>309</a> });
<a href=#310 id=310 data-nosnippet>310</a> }
<a href=#311 id=311 data-nosnippet>311</a>
<a href=#312 id=312 data-nosnippet>312</a> <span class="kw">async fn </span>handle_response(<span class="kw-2">&amp;</span><span class="self">self</span>, ctx: HeartbeatResponseHandlerContext) -&gt; <span class="prelude-ty">Result</span>&lt;(), Error&gt; {
<a href=#313 id=313 data-nosnippet>313</a> <span class="self">self</span>.resp_handler_executor
<a href=#314 id=314 data-nosnippet>314</a> .handle(ctx)
<a href=#315 id=315 data-nosnippet>315</a> .<span class="kw">await
<a href=#316 id=316 data-nosnippet>316</a> </span>.map_err(BoxedError::new)
<a href=#317 id=317 data-nosnippet>317</a> .context(ExternalSnafu)
<a href=#318 id=318 data-nosnippet>318</a> }
<a href=#319 id=319 data-nosnippet>319</a>
<a href=#320 id=320 data-nosnippet>320</a> <span class="kw">async fn </span>start_with_retry(<span class="kw-2">&amp;</span><span class="self">self</span>, retry_interval: Duration) {
<a href=#321 id=321 data-nosnippet>321</a> <span class="kw">loop </span>{
<a href=#322 id=322 data-nosnippet>322</a> tokio::time::sleep(retry_interval).<span class="kw">await</span>;
<a href=#323 id=323 data-nosnippet>323</a>
<a href=#324 id=324 data-nosnippet>324</a> <span class="macro">info!</span>(<span class="string">"Try to re-establish the heartbeat connection to metasrv."</span>);
<a href=#325 id=325 data-nosnippet>325</a>
<a href=#326 id=326 data-nosnippet>326</a> <span class="kw">if </span><span class="self">self</span>.create_streams().<span class="kw">await</span>.is_ok() {
<a href=#327 id=327 data-nosnippet>327</a> <span class="kw">break</span>;
<a href=#328 id=328 data-nosnippet>328</a> }
<a href=#329 id=329 data-nosnippet>329</a> }
<a href=#330 id=330 data-nosnippet>330</a> }
<a href=#331 id=331 data-nosnippet>331</a>}
</code></pre></div></section></main></body></html>
+60
View File
@@ -0,0 +1,60 @@
<!DOCTYPE html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1.0"><meta name="generator" content="rustdoc"><meta name="description" content="Source of the Rust file `src/flow/src/lib.rs`."><title>lib.rs - source</title><script>if(window.location.protocol!=="file:")document.head.insertAdjacentHTML("beforeend","SourceSerif4-Regular-6b053e98.ttf.woff2,FiraSans-Italic-81dc35de.woff2,FiraSans-Regular-0fe48ade.woff2,FiraSans-MediumItalic-ccf7e434.woff2,FiraSans-Medium-e1aa3f0a.woff2,SourceCodePro-Regular-8badfe75.ttf.woff2,SourceCodePro-Semibold-aa29a496.ttf.woff2".split(",").map(f=>`<link rel="preload" as="font" type="font/woff2"href="../../static.files/${f}">`).join(""))</script><link rel="stylesheet" href="../../static.files/normalize-9960930a.css"><link rel="stylesheet" href="../../static.files/rustdoc-17e0aaed.css"><meta name="rustdoc-vars" data-root-path="../../" data-static-root-path="../../static.files/" data-current-crate="flow" data-themes="" data-resource-suffix="" data-rustdoc-version="1.96.1 (31fca3adb 2026-06-26)" data-channel="1.96.1" data-search-js="search-b5634cc7.js" data-stringdex-js="stringdex-2da4960a.js" data-settings-js="settings-170eb4bf.js" ><script src="../../static.files/storage-41dd4d93.js"></script><script defer src="../../static.files/src-script-813739b1.js"></script><script defer src="../../src-files.js"></script><script defer src="../../static.files/main-5013f961.js"></script><noscript><link rel="stylesheet" href="../../static.files/noscript-f7c3ffd8.css"></noscript><link rel="alternate icon" type="image/png" href="../../static.files/favicon-32x32-eab170b8.png"><link rel="icon" type="image/svg+xml" href="../../static.files/favicon-044be391.svg"></head><body class="rustdoc src"><a class="skip-main-content" href="#main-content">Skip to main content</a><!--[if lte IE 11]><div class="warning">This old browser is unsupported and will most likely display funky things.</div><![endif]--><nav class="sidebar"><div class="src-sidebar-title"><h2>Files</h2></div></nav><div class="sidebar-resizer" title="Drag to resize sidebar"></div><main><section id="main-content" class="content" tabindex="-1"><div class="main-heading"><h1><div class="sub-heading">flow/</div>lib.rs</h1><rustdoc-toolbar></rustdoc-toolbar></div><div class="example-wrap digits-2"><pre class="rust"><code><a href=#1 id=1 data-nosnippet>1</a><span class="comment">// Copyright 2023 Greptime Team
<a href=#2 id=2 data-nosnippet>2</a>//
<a href=#3 id=3 data-nosnippet>3</a>// Licensed under the Apache License, Version 2.0 (the "License");
<a href=#4 id=4 data-nosnippet>4</a>// you may not use this file except in compliance with the License.
<a href=#5 id=5 data-nosnippet>5</a>// You may obtain a copy of the License at
<a href=#6 id=6 data-nosnippet>6</a>//
<a href=#7 id=7 data-nosnippet>7</a>// http://www.apache.org/licenses/LICENSE-2.0
<a href=#8 id=8 data-nosnippet>8</a>//
<a href=#9 id=9 data-nosnippet>9</a>// Unless required by applicable law or agreed to in writing, software
<a href=#10 id=10 data-nosnippet>10</a>// distributed under the License is distributed on an "AS IS" BASIS,
<a href=#11 id=11 data-nosnippet>11</a>// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
<a href=#12 id=12 data-nosnippet>12</a>// See the License for the specific language governing permissions and
<a href=#13 id=13 data-nosnippet>13</a>// limitations under the License.
<a href=#14 id=14 data-nosnippet>14</a>
<a href=#15 id=15 data-nosnippet>15</a></span><span class="doccomment">//! Flow execution and flownode services.
<a href=#16 id=16 data-nosnippet>16</a>
<a href=#17 id=17 data-nosnippet>17</a></span><span class="attr">#![allow(dead_code)]
<a href=#18 id=18 data-nosnippet>18</a>#![warn(clippy::too_many_lines)]
<a href=#19 id=19 data-nosnippet>19</a>
<a href=#20 id=20 data-nosnippet>20</a></span><span class="comment">// TODO(discord9): enable this lint to handle out of bound access
<a href=#21 id=21 data-nosnippet>21</a>// #![cfg_attr(not(test), warn(clippy::indexing_slicing))]
<a href=#22 id=22 data-nosnippet>22</a>
<a href=#23 id=23 data-nosnippet>23</a>// allow unused for now because it should be use later
<a href=#24 id=24 data-nosnippet>24</a></span><span class="kw">mod </span>adapter;
<a href=#25 id=25 data-nosnippet>25</a><span class="kw">pub</span>(<span class="kw">crate</span>) <span class="kw">mod </span>batching_mode;
<a href=#26 id=26 data-nosnippet>26</a><span class="kw">mod </span>df_optimizer;
<a href=#27 id=27 data-nosnippet>27</a><span class="kw">pub</span>(<span class="kw">crate</span>) <span class="kw">mod </span>engine;
<a href=#28 id=28 data-nosnippet>28</a><span class="kw">pub mod </span>error;
<a href=#29 id=29 data-nosnippet>29</a><span class="kw">mod </span>expr;
<a href=#30 id=30 data-nosnippet>30</a><span class="kw">pub mod </span>heartbeat;
<a href=#31 id=31 data-nosnippet>31</a><span class="kw">mod </span>metrics;
<a href=#32 id=32 data-nosnippet>32</a><span class="kw">mod </span>repr;
<a href=#33 id=33 data-nosnippet>33</a><span class="kw">mod </span>server;
<a href=#34 id=34 data-nosnippet>34</a><span class="kw">mod </span>utils;
<a href=#35 id=35 data-nosnippet>35</a>
<a href=#36 id=36 data-nosnippet>36</a><span class="attr">#[cfg(test)]
<a href=#37 id=37 data-nosnippet>37</a></span><span class="kw">mod </span>test_utils;
<a href=#38 id=38 data-nosnippet>38</a>
<a href=#39 id=39 data-nosnippet>39</a><span class="kw">pub use </span>adapter::flownode_impl::FlowDualEngineRef;
<a href=#40 id=40 data-nosnippet>40</a><span class="kw">pub use </span>adapter::{FlowConfig, FlowStreamingEngineRef, StreamingEngine};
<a href=#41 id=41 data-nosnippet>41</a><span class="kw">pub use </span>batching_mode::BatchingModeOptions;
<a href=#42 id=42 data-nosnippet>42</a><span class="kw">pub use </span>batching_mode::batching_execution::{BatchingExecution, BatchingExecutionFactory};
<a href=#43 id=43 data-nosnippet>43</a><span class="kw">pub use </span>batching_mode::frontend_client::{
<a href=#44 id=44 data-nosnippet>44</a> FrontendClient, GrpcQueryHandlerWithBoxedError, PeerDesc,
<a href=#45 id=45 data-nosnippet>45</a>};
<a href=#46 id=46 data-nosnippet>46</a><span class="kw">pub use </span>batching_mode::task::{
<a href=#47 id=47 data-nosnippet>47</a> BatchingExecutionGuard, BatchingTask, DirtyRestore, ExecuteOnceOutcome, PlanInfo,
<a href=#48 id=48 data-nosnippet>48</a> QueryCoverage, TaskArgs,
<a href=#49 id=49 data-nosnippet>49</a>};
<a href=#50 id=50 data-nosnippet>50</a><span class="kw">pub use </span>batching_mode::time_window::{TimeWindowExpr, find_time_window_expr};
<a href=#51 id=51 data-nosnippet>51</a><span class="kw">pub use </span>batching_mode::utils::{
<a href=#52 id=52 data-nosnippet>52</a> IncrementalAggregateAnalysis, IncrementalAggregateMergeColumn, IncrementalAggregateMergeOp,
<a href=#53 id=53 data-nosnippet>53</a> analyze_incremental_aggregate_plan, sql_to_df_plan,
<a href=#54 id=54 data-nosnippet>54</a>};
<a href=#55 id=55 data-nosnippet>55</a><span class="kw">pub</span>(<span class="kw">crate</span>) <span class="kw">use </span>engine::{CreateFlowArgs, FlowId, TableName};
<a href=#56 id=56 data-nosnippet>56</a><span class="kw">pub use </span>error::{Error, <span class="prelude-ty">Result</span>};
<a href=#57 id=57 data-nosnippet>57</a><span class="kw">pub use </span>server::{FlownodeBuilder, FlownodeInstance, FlownodeServer, FlownodeServiceBuilder};
<a href=#58 id=58 data-nosnippet>58</a>
<a href=#59 id=59 data-nosnippet>59</a><span class="kw">pub use </span><span class="kw">crate</span>::adapter::FlownodeOptions;
</code></pre></div></section></main></body></html>
+109
View File
@@ -0,0 +1,109 @@
<!DOCTYPE html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1.0"><meta name="generator" content="rustdoc"><meta name="description" content="Source of the Rust file `src/flow/src/metrics.rs`."><title>metrics.rs - source</title><script>if(window.location.protocol!=="file:")document.head.insertAdjacentHTML("beforeend","SourceSerif4-Regular-6b053e98.ttf.woff2,FiraSans-Italic-81dc35de.woff2,FiraSans-Regular-0fe48ade.woff2,FiraSans-MediumItalic-ccf7e434.woff2,FiraSans-Medium-e1aa3f0a.woff2,SourceCodePro-Regular-8badfe75.ttf.woff2,SourceCodePro-Semibold-aa29a496.ttf.woff2".split(",").map(f=>`<link rel="preload" as="font" type="font/woff2"href="../../static.files/${f}">`).join(""))</script><link rel="stylesheet" href="../../static.files/normalize-9960930a.css"><link rel="stylesheet" href="../../static.files/rustdoc-17e0aaed.css"><meta name="rustdoc-vars" data-root-path="../../" data-static-root-path="../../static.files/" data-current-crate="flow" data-themes="" data-resource-suffix="" data-rustdoc-version="1.96.1 (31fca3adb 2026-06-26)" data-channel="1.96.1" data-search-js="search-b5634cc7.js" data-stringdex-js="stringdex-2da4960a.js" data-settings-js="settings-170eb4bf.js" ><script src="../../static.files/storage-41dd4d93.js"></script><script defer src="../../static.files/src-script-813739b1.js"></script><script defer src="../../src-files.js"></script><script defer src="../../static.files/main-5013f961.js"></script><noscript><link rel="stylesheet" href="../../static.files/noscript-f7c3ffd8.css"></noscript><link rel="alternate icon" type="image/png" href="../../static.files/favicon-32x32-eab170b8.png"><link rel="icon" type="image/svg+xml" href="../../static.files/favicon-044be391.svg"></head><body class="rustdoc src"><a class="skip-main-content" href="#main-content">Skip to main content</a><!--[if lte IE 11]><div class="warning">This old browser is unsupported and will most likely display funky things.</div><![endif]--><nav class="sidebar"><div class="src-sidebar-title"><h2>Files</h2></div></nav><div class="sidebar-resizer" title="Drag to resize sidebar"></div><main><section id="main-content" class="content" tabindex="-1"><div class="main-heading"><h1><div class="sub-heading">flow/</div>metrics.rs</h1><rustdoc-toolbar></rustdoc-toolbar></div><div class="example-wrap digits-3"><pre class="rust"><code><a href=#1 id=1 data-nosnippet>1</a><span class="comment">// Copyright 2023 Greptime Team
<a href=#2 id=2 data-nosnippet>2</a>//
<a href=#3 id=3 data-nosnippet>3</a>// Licensed under the Apache License, Version 2.0 (the "License");
<a href=#4 id=4 data-nosnippet>4</a>// you may not use this file except in compliance with the License.
<a href=#5 id=5 data-nosnippet>5</a>// You may obtain a copy of the License at
<a href=#6 id=6 data-nosnippet>6</a>//
<a href=#7 id=7 data-nosnippet>7</a>// http://www.apache.org/licenses/LICENSE-2.0
<a href=#8 id=8 data-nosnippet>8</a>//
<a href=#9 id=9 data-nosnippet>9</a>// Unless required by applicable law or agreed to in writing, software
<a href=#10 id=10 data-nosnippet>10</a>// distributed under the License is distributed on an "AS IS" BASIS,
<a href=#11 id=11 data-nosnippet>11</a>// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
<a href=#12 id=12 data-nosnippet>12</a>// See the License for the specific language governing permissions and
<a href=#13 id=13 data-nosnippet>13</a>// limitations under the License.
<a href=#14 id=14 data-nosnippet>14</a>
<a href=#15 id=15 data-nosnippet>15</a></span><span class="doccomment">//! Some of the metrics used in the flow module.
<a href=#16 id=16 data-nosnippet>16</a>
<a href=#17 id=17 data-nosnippet>17</a></span><span class="kw">use </span>lazy_static::lazy_static;
<a href=#18 id=18 data-nosnippet>18</a><span class="kw">use </span>prometheus::<span class="kw-2">*</span>;
<a href=#19 id=19 data-nosnippet>19</a>
<a href=#20 id=20 data-nosnippet>20</a><span class="macro">lazy_static!</span> {
<a href=#21 id=21 data-nosnippet>21</a> <span class="kw">pub static </span><span class="kw-2">ref </span>METRIC_FLOW_TASK_COUNT: IntGauge =
<a href=#22 id=22 data-nosnippet>22</a> <span class="macro">register_int_gauge!</span>(<span class="string">"greptime_flow_task_count"</span>, <span class="string">"flow task count"</span>).unwrap();
<a href=#23 id=23 data-nosnippet>23</a> <span class="kw">pub static </span><span class="kw-2">ref </span>METRIC_FLOW_BATCHING_ENGINE_QUERY_TIME: HistogramVec = <span class="macro">register_histogram_vec!</span>(
<a href=#24 id=24 data-nosnippet>24</a> <span class="string">"greptime_flow_batching_engine_query_time_secs"</span>,
<a href=#25 id=25 data-nosnippet>25</a> <span class="string">"flow batching engine query time(seconds)"</span>,
<a href=#26 id=26 data-nosnippet>26</a> <span class="kw-2">&amp;</span>[<span class="string">"flow_id"</span>],
<a href=#27 id=27 data-nosnippet>27</a> <span class="macro">vec!</span>[<span class="number">0.0</span>, <span class="number">5.</span>, <span class="number">10.</span>, <span class="number">20.</span>, <span class="number">40.</span>, <span class="number">80.</span>, <span class="number">160.</span>, <span class="number">320.</span>, <span class="number">640.</span>,]
<a href=#28 id=28 data-nosnippet>28</a> )
<a href=#29 id=29 data-nosnippet>29</a> .unwrap();
<a href=#30 id=30 data-nosnippet>30</a> <span class="kw">pub static </span><span class="kw-2">ref </span>METRIC_FLOW_BATCHING_ENGINE_SLOW_QUERY: HistogramVec = <span class="macro">register_histogram_vec!</span>(
<a href=#31 id=31 data-nosnippet>31</a> <span class="string">"greptime_flow_batching_engine_slow_query_secs"</span>,
<a href=#32 id=32 data-nosnippet>32</a> <span class="string">"flow batching engine slow query(seconds)"</span>,
<a href=#33 id=33 data-nosnippet>33</a> <span class="kw-2">&amp;</span>[<span class="string">"flow_id"</span>, <span class="string">"peer"</span>],
<a href=#34 id=34 data-nosnippet>34</a> <span class="macro">vec!</span>[<span class="number">60.</span>, <span class="number">2. </span>* <span class="number">60.</span>, <span class="number">3. </span>* <span class="number">60.</span>, <span class="number">5. </span>* <span class="number">60.</span>, <span class="number">10. </span>* <span class="number">60.</span>]
<a href=#35 id=35 data-nosnippet>35</a> )
<a href=#36 id=36 data-nosnippet>36</a> .unwrap();
<a href=#37 id=37 data-nosnippet>37</a> <span class="kw">pub static </span><span class="kw-2">ref </span>METRIC_FLOW_BATCHING_ENGINE_QUERY_WINDOW_CNT: HistogramVec =
<a href=#38 id=38 data-nosnippet>38</a> <span class="macro">register_histogram_vec!</span>(
<a href=#39 id=39 data-nosnippet>39</a> <span class="string">"greptime_flow_batching_engine_query_window_cnt"</span>,
<a href=#40 id=40 data-nosnippet>40</a> <span class="string">"flow batching engine query time window count"</span>,
<a href=#41 id=41 data-nosnippet>41</a> <span class="kw-2">&amp;</span>[<span class="string">"flow_id"</span>],
<a href=#42 id=42 data-nosnippet>42</a> <span class="macro">vec!</span>[<span class="number">0.0</span>, <span class="number">5.</span>, <span class="number">10.</span>, <span class="number">20.</span>, <span class="number">40.</span>]
<a href=#43 id=43 data-nosnippet>43</a> )
<a href=#44 id=44 data-nosnippet>44</a> .unwrap();
<a href=#45 id=45 data-nosnippet>45</a> <span class="kw">pub static </span><span class="kw-2">ref </span>METRIC_FLOW_BATCHING_ENGINE_QUERY_WINDOW_SIZE: HistogramVec =
<a href=#46 id=46 data-nosnippet>46</a> <span class="macro">register_histogram_vec!</span>(
<a href=#47 id=47 data-nosnippet>47</a> <span class="string">"greptime_flow_batching_engine_query_window_size_secs"</span>,
<a href=#48 id=48 data-nosnippet>48</a> <span class="string">"flow batching engine query window size(seconds)"</span>,
<a href=#49 id=49 data-nosnippet>49</a> <span class="kw-2">&amp;</span>[<span class="string">"flow_id"</span>],
<a href=#50 id=50 data-nosnippet>50</a> <span class="macro">vec!</span>[<span class="number">60.</span>, <span class="number">4. </span>* <span class="number">60.</span>, <span class="number">16. </span>* <span class="number">60.</span>, <span class="number">64. </span>* <span class="number">60.</span>, <span class="number">256. </span>* <span class="number">60.</span>]
<a href=#51 id=51 data-nosnippet>51</a> )
<a href=#52 id=52 data-nosnippet>52</a> .unwrap();
<a href=#53 id=53 data-nosnippet>53</a> <span class="kw">pub static </span><span class="kw-2">ref </span>METRIC_FLOW_BATCHING_ENGINE_STALLED_WINDOW_SIZE: HistogramVec =
<a href=#54 id=54 data-nosnippet>54</a> <span class="macro">register_histogram_vec!</span>(
<a href=#55 id=55 data-nosnippet>55</a> <span class="string">"greptime_flow_batching_engine_stalled_window_size_secs"</span>,
<a href=#56 id=56 data-nosnippet>56</a> <span class="string">"flow batching engine stalled window size(seconds)"</span>,
<a href=#57 id=57 data-nosnippet>57</a> <span class="kw-2">&amp;</span>[<span class="string">"flow_id"</span>],
<a href=#58 id=58 data-nosnippet>58</a> <span class="macro">vec!</span>[<span class="number">60.</span>, <span class="number">4. </span>* <span class="number">60.</span>, <span class="number">16. </span>* <span class="number">60.</span>, <span class="number">64. </span>* <span class="number">60.</span>, <span class="number">256. </span>* <span class="number">60.</span>]
<a href=#59 id=59 data-nosnippet>59</a> )
<a href=#60 id=60 data-nosnippet>60</a> .unwrap();
<a href=#61 id=61 data-nosnippet>61</a> <span class="kw">pub static </span><span class="kw-2">ref </span>METRIC_FLOW_BATCHING_ENGINE_BULK_MARK_TIME_WINDOW: GaugeVec =
<a href=#62 id=62 data-nosnippet>62</a> <span class="macro">register_gauge_vec!</span>(
<a href=#63 id=63 data-nosnippet>63</a> <span class="string">"greptime_flow_batching_engine_bulk_mark_time_window"</span>,
<a href=#64 id=64 data-nosnippet>64</a> <span class="string">"flow batching engine query time window count marked by bulk inserts"</span>,
<a href=#65 id=65 data-nosnippet>65</a> <span class="kw-2">&amp;</span>[<span class="string">"flow_id"</span>],
<a href=#66 id=66 data-nosnippet>66</a> )
<a href=#67 id=67 data-nosnippet>67</a> .unwrap();
<a href=#68 id=68 data-nosnippet>68</a> <span class="kw">pub static </span><span class="kw-2">ref </span>METRIC_FLOW_BATCHING_ENGINE_START_QUERY_CNT: IntCounterVec =
<a href=#69 id=69 data-nosnippet>69</a> <span class="macro">register_int_counter_vec!</span>(
<a href=#70 id=70 data-nosnippet>70</a> <span class="string">"greptime_flow_batching_start_query_count"</span>,
<a href=#71 id=71 data-nosnippet>71</a> <span class="string">"flow batching engine started query count"</span>,
<a href=#72 id=72 data-nosnippet>72</a> <span class="kw-2">&amp;</span>[<span class="string">"flow_id"</span>],
<a href=#73 id=73 data-nosnippet>73</a> )
<a href=#74 id=74 data-nosnippet>74</a> .unwrap();
<a href=#75 id=75 data-nosnippet>75</a> <span class="kw">pub static </span><span class="kw-2">ref </span>METRIC_FLOW_BATCHING_ENGINE_ERROR_CNT: IntCounterVec =
<a href=#76 id=76 data-nosnippet>76</a> <span class="macro">register_int_counter_vec!</span>(
<a href=#77 id=77 data-nosnippet>77</a> <span class="string">"greptime_flow_batching_error_count"</span>,
<a href=#78 id=78 data-nosnippet>78</a> <span class="string">"flow batching engine error count per flow id"</span>,
<a href=#79 id=79 data-nosnippet>79</a> <span class="kw-2">&amp;</span>[<span class="string">"flow_id"</span>],
<a href=#80 id=80 data-nosnippet>80</a> )
<a href=#81 id=81 data-nosnippet>81</a> .unwrap();
<a href=#82 id=82 data-nosnippet>82</a> <span class="kw">pub static </span><span class="kw-2">ref </span>METRIC_FLOW_BATCHING_ENGINE_CHECKPOINT_DECISION_CNT: IntCounterVec =
<a href=#83 id=83 data-nosnippet>83</a> <span class="macro">register_int_counter_vec!</span>(
<a href=#84 id=84 data-nosnippet>84</a> <span class="string">"greptime_flow_batching_checkpoint_decision_count"</span>,
<a href=#85 id=85 data-nosnippet>85</a> <span class="string">"flow batching checkpoint state-machine decisions"</span>,
<a href=#86 id=86 data-nosnippet>86</a> <span class="kw-2">&amp;</span>[<span class="string">"flow_id"</span>, <span class="string">"mode"</span>, <span class="string">"decision"</span>, <span class="string">"reason"</span>],
<a href=#87 id=87 data-nosnippet>87</a> )
<a href=#88 id=88 data-nosnippet>88</a> .unwrap();
<a href=#89 id=89 data-nosnippet>89</a> <span class="kw">pub static </span><span class="kw-2">ref </span>METRIC_FLOW_BATCHING_ENGINE_QUERY_MODE_CNT: IntCounterVec =
<a href=#90 id=90 data-nosnippet>90</a> <span class="macro">register_int_counter_vec!</span>(
<a href=#91 id=91 data-nosnippet>91</a> <span class="string">"greptime_flow_batching_query_mode_count"</span>,
<a href=#92 id=92 data-nosnippet>92</a> <span class="string">"flow batching query attempts by checkpoint mode"</span>,
<a href=#93 id=93 data-nosnippet>93</a> <span class="kw-2">&amp;</span>[<span class="string">"flow_id"</span>, <span class="string">"mode"</span>],
<a href=#94 id=94 data-nosnippet>94</a> )
<a href=#95 id=95 data-nosnippet>95</a> .unwrap();
<a href=#96 id=96 data-nosnippet>96</a> <span class="kw">pub static </span><span class="kw-2">ref </span>METRIC_FLOW_ROWS: IntCounterVec = <span class="macro">register_int_counter_vec!</span>(
<a href=#97 id=97 data-nosnippet>97</a> <span class="string">"greptime_flow_processed_rows"</span>,
<a href=#98 id=98 data-nosnippet>98</a> <span class="string">"Count of rows flowing through the system."</span>,
<a href=#99 id=99 data-nosnippet>99</a> <span class="kw-2">&amp;</span>[<span class="string">"direction"</span>]
<a href=#100 id=100 data-nosnippet>100</a> )
<a href=#101 id=101 data-nosnippet>101</a> .unwrap();
<a href=#102 id=102 data-nosnippet>102</a> <span class="kw">pub static </span><span class="kw-2">ref </span>METRIC_FLOW_PROCESSING_TIME: HistogramVec = <span class="macro">register_histogram_vec!</span>(
<a href=#103 id=103 data-nosnippet>103</a> <span class="string">"greptime_flow_processing_time"</span>,
<a href=#104 id=104 data-nosnippet>104</a> <span class="string">"Time spent processing requests"</span>,
<a href=#105 id=105 data-nosnippet>105</a> <span class="kw-2">&amp;</span>[<span class="string">"type"</span>]
<a href=#106 id=106 data-nosnippet>106</a> )
<a href=#107 id=107 data-nosnippet>107</a> .unwrap();
<a href=#108 id=108 data-nosnippet>108</a>}
</code></pre></div></section></main></body></html>
+263
View File
@@ -0,0 +1,263 @@
<!DOCTYPE html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1.0"><meta name="generator" content="rustdoc"><meta name="description" content="Source of the Rust file `src/flow/src/repr.rs`."><title>repr.rs - source</title><script>if(window.location.protocol!=="file:")document.head.insertAdjacentHTML("beforeend","SourceSerif4-Regular-6b053e98.ttf.woff2,FiraSans-Italic-81dc35de.woff2,FiraSans-Regular-0fe48ade.woff2,FiraSans-MediumItalic-ccf7e434.woff2,FiraSans-Medium-e1aa3f0a.woff2,SourceCodePro-Regular-8badfe75.ttf.woff2,SourceCodePro-Semibold-aa29a496.ttf.woff2".split(",").map(f=>`<link rel="preload" as="font" type="font/woff2"href="../../static.files/${f}">`).join(""))</script><link rel="stylesheet" href="../../static.files/normalize-9960930a.css"><link rel="stylesheet" href="../../static.files/rustdoc-17e0aaed.css"><meta name="rustdoc-vars" data-root-path="../../" data-static-root-path="../../static.files/" data-current-crate="flow" data-themes="" data-resource-suffix="" data-rustdoc-version="1.96.1 (31fca3adb 2026-06-26)" data-channel="1.96.1" data-search-js="search-b5634cc7.js" data-stringdex-js="stringdex-2da4960a.js" data-settings-js="settings-170eb4bf.js" ><script src="../../static.files/storage-41dd4d93.js"></script><script defer src="../../static.files/src-script-813739b1.js"></script><script defer src="../../src-files.js"></script><script defer src="../../static.files/main-5013f961.js"></script><noscript><link rel="stylesheet" href="../../static.files/noscript-f7c3ffd8.css"></noscript><link rel="alternate icon" type="image/png" href="../../static.files/favicon-32x32-eab170b8.png"><link rel="icon" type="image/svg+xml" href="../../static.files/favicon-044be391.svg"></head><body class="rustdoc src"><a class="skip-main-content" href="#main-content">Skip to main content</a><!--[if lte IE 11]><div class="warning">This old browser is unsupported and will most likely display funky things.</div><![endif]--><nav class="sidebar"><div class="src-sidebar-title"><h2>Files</h2></div></nav><div class="sidebar-resizer" title="Drag to resize sidebar"></div><main><section id="main-content" class="content" tabindex="-1"><div class="main-heading"><h1><div class="sub-heading">flow/</div>repr.rs</h1><rustdoc-toolbar></rustdoc-toolbar></div><div class="example-wrap digits-3"><pre class="rust"><code><a href=#1 id=1 data-nosnippet>1</a><span class="comment">// Copyright 2023 Greptime Team
<a href=#2 id=2 data-nosnippet>2</a>//
<a href=#3 id=3 data-nosnippet>3</a>// Licensed under the Apache License, Version 2.0 (the "License");
<a href=#4 id=4 data-nosnippet>4</a>// you may not use this file except in compliance with the License.
<a href=#5 id=5 data-nosnippet>5</a>// You may obtain a copy of the License at
<a href=#6 id=6 data-nosnippet>6</a>//
<a href=#7 id=7 data-nosnippet>7</a>// http://www.apache.org/licenses/LICENSE-2.0
<a href=#8 id=8 data-nosnippet>8</a>//
<a href=#9 id=9 data-nosnippet>9</a>// Unless required by applicable law or agreed to in writing, software
<a href=#10 id=10 data-nosnippet>10</a>// distributed under the License is distributed on an "AS IS" BASIS,
<a href=#11 id=11 data-nosnippet>11</a>// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
<a href=#12 id=12 data-nosnippet>12</a>// See the License for the specific language governing permissions and
<a href=#13 id=13 data-nosnippet>13</a>// limitations under the License.
<a href=#14 id=14 data-nosnippet>14</a>
<a href=#15 id=15 data-nosnippet>15</a></span><span class="doccomment">//! basically a wrapper around the `datatype` crate
<a href=#16 id=16 data-nosnippet>16</a>//! for basic Data Representation
<a href=#17 id=17 data-nosnippet>17</a>
<a href=#18 id=18 data-nosnippet>18</a></span><span class="kw">mod </span>relation;
<a href=#19 id=19 data-nosnippet>19</a>
<a href=#20 id=20 data-nosnippet>20</a><span class="kw">use </span>api::helper::{pb_value_to_value_ref, to_grpc_value};
<a href=#21 id=21 data-nosnippet>21</a><span class="kw">use </span>api::v1::Row <span class="kw">as </span>ProtoRow;
<a href=#22 id=22 data-nosnippet>22</a><span class="kw">use </span>datatypes::data_type::ConcreteDataType;
<a href=#23 id=23 data-nosnippet>23</a><span class="kw">use </span>datatypes::types::cast;
<a href=#24 id=24 data-nosnippet>24</a><span class="kw">use </span>datatypes::value::Value;
<a href=#25 id=25 data-nosnippet>25</a><span class="kw">use </span>get_size2::GetSize;
<a href=#26 id=26 data-nosnippet>26</a><span class="kw">use </span>itertools::Itertools;
<a href=#27 id=27 data-nosnippet>27</a><span class="kw">pub</span>(<span class="kw">crate</span>) <span class="kw">use </span>relation::{ColumnType, Key, RelationDesc, RelationType};
<a href=#28 id=28 data-nosnippet>28</a><span class="kw">use </span>serde::{Deserialize, Serialize};
<a href=#29 id=29 data-nosnippet>29</a><span class="kw">use </span>snafu::ResultExt;
<a href=#30 id=30 data-nosnippet>30</a>
<a href=#31 id=31 data-nosnippet>31</a><span class="kw">use </span><span class="kw">crate</span>::expr::error::{CastValueSnafu, EvalError, InvalidArgumentSnafu};
<a href=#32 id=32 data-nosnippet>32</a><span class="kw">use </span><span class="kw">crate</span>::utils::get_value_heap_size;
<a href=#33 id=33 data-nosnippet>33</a>
<a href=#34 id=34 data-nosnippet>34</a><span class="doccomment">/// System-wide Record count difference type. Useful for capture data change
<a href=#35 id=35 data-nosnippet>35</a>///
<a href=#36 id=36 data-nosnippet>36</a>/// i.e. +1 means insert one record, -1 means remove,
<a href=#37 id=37 data-nosnippet>37</a>/// and +/-n means insert/remove multiple duplicate records.
<a href=#38 id=38 data-nosnippet>38</a></span><span class="kw">pub type </span>Diff = i64;
<a href=#39 id=39 data-nosnippet>39</a>
<a href=#40 id=40 data-nosnippet>40</a><span class="doccomment">/// System-wide default timestamp type, in milliseconds
<a href=#41 id=41 data-nosnippet>41</a></span><span class="kw">pub type </span>Timestamp = i64;
<a href=#42 id=42 data-nosnippet>42</a>
<a href=#43 id=43 data-nosnippet>43</a><span class="doccomment">/// System-wide default duration type, in milliseconds
<a href=#44 id=44 data-nosnippet>44</a></span><span class="kw">pub type </span>Duration = i64;
<a href=#45 id=45 data-nosnippet>45</a>
<a href=#46 id=46 data-nosnippet>46</a><span class="doccomment">/// Default type for a repr of changes to a collection.
<a href=#47 id=47 data-nosnippet>47</a></span><span class="kw">pub type </span>DiffRow = (Row, Timestamp, Diff);
<a href=#48 id=48 data-nosnippet>48</a>
<a href=#49 id=49 data-nosnippet>49</a><span class="doccomment">/// Row with key-value pair, timestamp and diff
<a href=#50 id=50 data-nosnippet>50</a></span><span class="kw">pub type </span>KeyValDiffRow = ((Row, Row), Timestamp, Diff);
<a href=#51 id=51 data-nosnippet>51</a>
<a href=#52 id=52 data-nosnippet>52</a><span class="doccomment">/// broadcast channel capacity, can be important to memory consumption, since this influence how many
<a href=#53 id=53 data-nosnippet>53</a>/// updates can be buffered in memory in the entire dataflow
<a href=#54 id=54 data-nosnippet>54</a>/// TODO(discord9): add config for this, so cpu&amp;mem usage can be balanced and configured by this
<a href=#55 id=55 data-nosnippet>55</a></span><span class="kw">pub const </span>BROADCAST_CAP: usize = <span class="number">1024</span>;
<a href=#56 id=56 data-nosnippet>56</a>
<a href=#57 id=57 data-nosnippet>57</a><span class="doccomment">/// The maximum capacity of the send buffer, to prevent the buffer from growing too large
<a href=#58 id=58 data-nosnippet>58</a></span><span class="kw">pub const </span>SEND_BUF_CAP: usize = BROADCAST_CAP * <span class="number">2</span>;
<a href=#59 id=59 data-nosnippet>59</a>
<a href=#60 id=60 data-nosnippet>60</a><span class="doccomment">/// Batch size used by the batching execution path.
<a href=#61 id=61 data-nosnippet>61</a></span><span class="kw">pub const </span>BATCH_SIZE: usize = <span class="number">32 </span>* <span class="number">16384</span>;
<a href=#62 id=62 data-nosnippet>62</a>
<a href=#63 id=63 data-nosnippet>63</a><span class="doccomment">/// Convert a value that is or can be converted to Datetime to internal timestamp
<a href=#64 id=64 data-nosnippet>64</a>///
<a href=#65 id=65 data-nosnippet>65</a>/// support types are: `Date`, `DateTime`, `TimeStamp`, `i64`
<a href=#66 id=66 data-nosnippet>66</a></span><span class="kw">pub fn </span>value_to_internal_ts(value: Value) -&gt; <span class="prelude-ty">Result</span>&lt;i64, EvalError&gt; {
<a href=#67 id=67 data-nosnippet>67</a> <span class="kw">let </span>is_supported_time_type = |arg: <span class="kw-2">&amp;</span>Value| {
<a href=#68 id=68 data-nosnippet>68</a> <span class="kw">let </span>ty = arg.data_type();
<a href=#69 id=69 data-nosnippet>69</a> <span class="macro">matches!</span>(
<a href=#70 id=70 data-nosnippet>70</a> ty,
<a href=#71 id=71 data-nosnippet>71</a> ConcreteDataType::Date(..) | ConcreteDataType::Timestamp(..)
<a href=#72 id=72 data-nosnippet>72</a> )
<a href=#73 id=73 data-nosnippet>73</a> };
<a href=#74 id=74 data-nosnippet>74</a> <span class="kw">match </span>value {
<a href=#75 id=75 data-nosnippet>75</a> Value::Int64(ts) =&gt; <span class="prelude-val">Ok</span>(ts),
<a href=#76 id=76 data-nosnippet>76</a> arg <span class="kw">if </span>is_supported_time_type(<span class="kw-2">&amp;</span>arg) =&gt; {
<a href=#77 id=77 data-nosnippet>77</a> <span class="kw">let </span>arg_ty = arg.data_type();
<a href=#78 id=78 data-nosnippet>78</a> <span class="kw">let </span>res = cast(arg, <span class="kw-2">&amp;</span>ConcreteDataType::timestamp_millisecond_datatype()).context({
<a href=#79 id=79 data-nosnippet>79</a> CastValueSnafu {
<a href=#80 id=80 data-nosnippet>80</a> from: arg_ty,
<a href=#81 id=81 data-nosnippet>81</a> to: ConcreteDataType::timestamp_millisecond_datatype(),
<a href=#82 id=82 data-nosnippet>82</a> }
<a href=#83 id=83 data-nosnippet>83</a> })<span class="question-mark">?</span>;
<a href=#84 id=84 data-nosnippet>84</a> <span class="kw">if let </span>Value::Timestamp(ts) = res {
<a href=#85 id=85 data-nosnippet>85</a> <span class="prelude-val">Ok</span>(ts.value())
<a href=#86 id=86 data-nosnippet>86</a> } <span class="kw">else </span>{
<a href=#87 id=87 data-nosnippet>87</a> <span class="macro">unreachable!</span>()
<a href=#88 id=88 data-nosnippet>88</a> }
<a href=#89 id=89 data-nosnippet>89</a> }
<a href=#90 id=90 data-nosnippet>90</a> <span class="kw">_ </span>=&gt; InvalidArgumentSnafu {
<a href=#91 id=91 data-nosnippet>91</a> reason: <span class="macro">format!</span>(<span class="string">"Expect a time type or i64, got {:?}"</span>, value.data_type()),
<a href=#92 id=92 data-nosnippet>92</a> }
<a href=#93 id=93 data-nosnippet>93</a> .fail(),
<a href=#94 id=94 data-nosnippet>94</a> }
<a href=#95 id=95 data-nosnippet>95</a>}
<a href=#96 id=96 data-nosnippet>96</a>
<a href=#97 id=97 data-nosnippet>97</a><span class="doccomment">/// A row is a vector of values.
<a href=#98 id=98 data-nosnippet>98</a>///
<a href=#99 id=99 data-nosnippet>99</a>/// TODO(discord9): use a more efficient representation
<a href=#100 id=100 data-nosnippet>100</a>/// i.e. more compact like raw u8 of \[tag0, value0, tag1, value1, ...\]
<a href=#101 id=101 data-nosnippet>101</a></span><span class="attr">#[derive(Clone, Debug, Hash, PartialEq, Eq, PartialOrd, Ord, Default, Serialize, Deserialize)]
<a href=#102 id=102 data-nosnippet>102</a></span><span class="kw">pub struct </span>Row {
<a href=#103 id=103 data-nosnippet>103</a> <span class="doccomment">/// The inner vector of values
<a href=#104 id=104 data-nosnippet>104</a> </span><span class="kw">pub </span>inner: Vec&lt;Value&gt;,
<a href=#105 id=105 data-nosnippet>105</a>}
<a href=#106 id=106 data-nosnippet>106</a>
<a href=#107 id=107 data-nosnippet>107</a><span class="kw">impl </span>GetSize <span class="kw">for </span>Row {
<a href=#108 id=108 data-nosnippet>108</a> <span class="kw">fn </span>get_heap_size(<span class="kw-2">&amp;</span><span class="self">self</span>) -&gt; usize {
<a href=#109 id=109 data-nosnippet>109</a> <span class="self">self</span>.inner.iter().map(get_value_heap_size).sum()
<a href=#110 id=110 data-nosnippet>110</a> }
<a href=#111 id=111 data-nosnippet>111</a>}
<a href=#112 id=112 data-nosnippet>112</a>
<a href=#113 id=113 data-nosnippet>113</a><span class="kw">impl </span>Row {
<a href=#114 id=114 data-nosnippet>114</a> <span class="doccomment">/// Create an empty row
<a href=#115 id=115 data-nosnippet>115</a> </span><span class="kw">pub fn </span>empty() -&gt; <span class="self">Self </span>{
<a href=#116 id=116 data-nosnippet>116</a> <span class="self">Self </span>{ inner: <span class="macro">vec!</span>[] }
<a href=#117 id=117 data-nosnippet>117</a> }
<a href=#118 id=118 data-nosnippet>118</a>
<a href=#119 id=119 data-nosnippet>119</a> <span class="doccomment">/// Returns true if the Row contains no elements.
<a href=#120 id=120 data-nosnippet>120</a> </span><span class="kw">pub fn </span>is_empty(<span class="kw-2">&amp;</span><span class="self">self</span>) -&gt; bool {
<a href=#121 id=121 data-nosnippet>121</a> <span class="self">self</span>.inner.is_empty()
<a href=#122 id=122 data-nosnippet>122</a> }
<a href=#123 id=123 data-nosnippet>123</a>
<a href=#124 id=124 data-nosnippet>124</a> <span class="doccomment">/// Create a row from a vector of values
<a href=#125 id=125 data-nosnippet>125</a> </span><span class="kw">pub fn </span>new(row: Vec&lt;Value&gt;) -&gt; <span class="self">Self </span>{
<a href=#126 id=126 data-nosnippet>126</a> <span class="self">Self </span>{ inner: row }
<a href=#127 id=127 data-nosnippet>127</a> }
<a href=#128 id=128 data-nosnippet>128</a>
<a href=#129 id=129 data-nosnippet>129</a> <span class="doccomment">/// Get the value at the given index
<a href=#130 id=130 data-nosnippet>130</a> </span><span class="kw">pub fn </span>get(<span class="kw-2">&amp;</span><span class="self">self</span>, idx: usize) -&gt; <span class="prelude-ty">Option</span>&lt;<span class="kw-2">&amp;</span>Value&gt; {
<a href=#131 id=131 data-nosnippet>131</a> <span class="self">self</span>.inner.get(idx)
<a href=#132 id=132 data-nosnippet>132</a> }
<a href=#133 id=133 data-nosnippet>133</a>
<a href=#134 id=134 data-nosnippet>134</a> <span class="doccomment">/// Clear the row
<a href=#135 id=135 data-nosnippet>135</a> </span><span class="kw">pub fn </span>clear(<span class="kw-2">&amp;mut </span><span class="self">self</span>) {
<a href=#136 id=136 data-nosnippet>136</a> <span class="self">self</span>.inner.clear();
<a href=#137 id=137 data-nosnippet>137</a> }
<a href=#138 id=138 data-nosnippet>138</a>
<a href=#139 id=139 data-nosnippet>139</a> <span class="doccomment">/// clear and return the inner vector
<a href=#140 id=140 data-nosnippet>140</a> ///
<a href=#141 id=141 data-nosnippet>141</a> /// useful if you want to reuse the vector as a buffer
<a href=#142 id=142 data-nosnippet>142</a> </span><span class="kw">pub fn </span>packer(<span class="kw-2">&amp;mut </span><span class="self">self</span>) -&gt; <span class="kw-2">&amp;mut </span>Vec&lt;Value&gt; {
<a href=#143 id=143 data-nosnippet>143</a> <span class="self">self</span>.inner.clear();
<a href=#144 id=144 data-nosnippet>144</a> <span class="kw-2">&amp;mut </span><span class="self">self</span>.inner
<a href=#145 id=145 data-nosnippet>145</a> }
<a href=#146 id=146 data-nosnippet>146</a>
<a href=#147 id=147 data-nosnippet>147</a> <span class="doccomment">/// pack a iterator of values into a row
<a href=#148 id=148 data-nosnippet>148</a> </span><span class="kw">pub fn </span>pack&lt;I&gt;(iter: I) -&gt; Row
<a href=#149 id=149 data-nosnippet>149</a> <span class="kw">where
<a href=#150 id=150 data-nosnippet>150</a> </span>I: IntoIterator&lt;Item = Value&gt;,
<a href=#151 id=151 data-nosnippet>151</a> {
<a href=#152 id=152 data-nosnippet>152</a> <span class="self">Self </span>{
<a href=#153 id=153 data-nosnippet>153</a> inner: iter.into_iter().collect(),
<a href=#154 id=154 data-nosnippet>154</a> }
<a href=#155 id=155 data-nosnippet>155</a> }
<a href=#156 id=156 data-nosnippet>156</a>
<a href=#157 id=157 data-nosnippet>157</a> <span class="doccomment">/// unpack a row into a vector of values
<a href=#158 id=158 data-nosnippet>158</a> </span><span class="kw">pub fn </span>unpack(<span class="self">self</span>) -&gt; Vec&lt;Value&gt; {
<a href=#159 id=159 data-nosnippet>159</a> <span class="self">self</span>.inner
<a href=#160 id=160 data-nosnippet>160</a> }
<a href=#161 id=161 data-nosnippet>161</a>
<a href=#162 id=162 data-nosnippet>162</a> <span class="doccomment">/// extend the row with values from an iterator
<a href=#163 id=163 data-nosnippet>163</a> </span><span class="kw">pub fn </span>extend&lt;I&gt;(<span class="kw-2">&amp;mut </span><span class="self">self</span>, iter: I)
<a href=#164 id=164 data-nosnippet>164</a> <span class="kw">where
<a href=#165 id=165 data-nosnippet>165</a> </span>I: IntoIterator&lt;Item = Value&gt;,
<a href=#166 id=166 data-nosnippet>166</a> {
<a href=#167 id=167 data-nosnippet>167</a> <span class="self">self</span>.inner.extend(iter);
<a href=#168 id=168 data-nosnippet>168</a> }
<a href=#169 id=169 data-nosnippet>169</a>
<a href=#170 id=170 data-nosnippet>170</a> <span class="doccomment">/// Creates a consuming iterator, that is, one that moves each value out of the `Row` (from start to end). The `Row` cannot be used after calling this
<a href=#171 id=171 data-nosnippet>171</a> </span><span class="kw">pub fn </span>into_iter(<span class="self">self</span>) -&gt; <span class="kw">impl </span>Iterator&lt;Item = Value&gt; {
<a href=#172 id=172 data-nosnippet>172</a> <span class="self">self</span>.inner.into_iter()
<a href=#173 id=173 data-nosnippet>173</a> }
<a href=#174 id=174 data-nosnippet>174</a>
<a href=#175 id=175 data-nosnippet>175</a> <span class="doccomment">/// Returns an iterator over the slice.
<a href=#176 id=176 data-nosnippet>176</a> </span><span class="kw">pub fn </span>iter(<span class="kw-2">&amp;</span><span class="self">self</span>) -&gt; <span class="kw">impl </span>Iterator&lt;Item = <span class="kw-2">&amp;</span>Value&gt; {
<a href=#177 id=177 data-nosnippet>177</a> <span class="self">self</span>.inner.iter()
<a href=#178 id=178 data-nosnippet>178</a> }
<a href=#179 id=179 data-nosnippet>179</a>
<a href=#180 id=180 data-nosnippet>180</a> <span class="doccomment">/// Returns the number of elements in the row, also known as its 'length'.
<a href=#181 id=181 data-nosnippet>181</a> </span><span class="kw">pub fn </span>len(<span class="kw-2">&amp;</span><span class="self">self</span>) -&gt; usize {
<a href=#182 id=182 data-nosnippet>182</a> <span class="self">self</span>.inner.len()
<a href=#183 id=183 data-nosnippet>183</a> }
<a href=#184 id=184 data-nosnippet>184</a>}
<a href=#185 id=185 data-nosnippet>185</a>
<a href=#186 id=186 data-nosnippet>186</a><span class="kw">impl </span>From&lt;Vec&lt;Value&gt;&gt; <span class="kw">for </span>Row {
<a href=#187 id=187 data-nosnippet>187</a> <span class="kw">fn </span>from(row: Vec&lt;Value&gt;) -&gt; <span class="self">Self </span>{
<a href=#188 id=188 data-nosnippet>188</a> Row::new(row)
<a href=#189 id=189 data-nosnippet>189</a> }
<a href=#190 id=190 data-nosnippet>190</a>}
<a href=#191 id=191 data-nosnippet>191</a>
<a href=#192 id=192 data-nosnippet>192</a><span class="kw">impl </span>From&lt;ProtoRow&gt; <span class="kw">for </span>Row {
<a href=#193 id=193 data-nosnippet>193</a> <span class="kw">fn </span>from(row: ProtoRow) -&gt; <span class="self">Self </span>{
<a href=#194 id=194 data-nosnippet>194</a> Row::pack(
<a href=#195 id=195 data-nosnippet>195</a> row.values
<a href=#196 id=196 data-nosnippet>196</a> .iter()
<a href=#197 id=197 data-nosnippet>197</a> .map(|pb_val| -&gt; Value { pb_value_to_value_ref(pb_val, <span class="prelude-val">None</span>).into() }),
<a href=#198 id=198 data-nosnippet>198</a> )
<a href=#199 id=199 data-nosnippet>199</a> }
<a href=#200 id=200 data-nosnippet>200</a>}
<a href=#201 id=201 data-nosnippet>201</a>
<a href=#202 id=202 data-nosnippet>202</a><span class="kw">impl </span>From&lt;Row&gt; <span class="kw">for </span>ProtoRow {
<a href=#203 id=203 data-nosnippet>203</a> <span class="kw">fn </span>from(row: Row) -&gt; <span class="self">Self </span>{
<a href=#204 id=204 data-nosnippet>204</a> <span class="kw">let </span>values = row.unpack().into_iter().map(to_grpc_value).collect_vec();
<a href=#205 id=205 data-nosnippet>205</a> ProtoRow { values }
<a href=#206 id=206 data-nosnippet>206</a> }
<a href=#207 id=207 data-nosnippet>207</a>}
<a href=#208 id=208 data-nosnippet>208</a><span class="attr">#[cfg(test)]
<a href=#209 id=209 data-nosnippet>209</a></span><span class="kw">mod </span>test {
<a href=#210 id=210 data-nosnippet>210</a> <span class="kw">use </span>common_time::{Date, Timestamp};
<a href=#211 id=211 data-nosnippet>211</a>
<a href=#212 id=212 data-nosnippet>212</a> <span class="kw">use </span>super::<span class="kw-2">*</span>;
<a href=#213 id=213 data-nosnippet>213</a>
<a href=#214 id=214 data-nosnippet>214</a> <span class="attr">#[test]
<a href=#215 id=215 data-nosnippet>215</a> </span><span class="kw">fn </span>test_row() {
<a href=#216 id=216 data-nosnippet>216</a> <span class="kw">let </span>row = Row::empty();
<a href=#217 id=217 data-nosnippet>217</a> <span class="kw">let </span>row_1 = Row::new(<span class="macro">vec!</span>[]);
<a href=#218 id=218 data-nosnippet>218</a> <span class="macro">assert_eq!</span>(row, row_1);
<a href=#219 id=219 data-nosnippet>219</a> <span class="kw">let </span><span class="kw-2">mut </span>row_2 = Row::new(<span class="macro">vec!</span>[Value::Int32(<span class="number">1</span>), Value::Int32(<span class="number">2</span>)]);
<a href=#220 id=220 data-nosnippet>220</a> <span class="macro">assert_eq!</span>(row_2.get(<span class="number">0</span>), <span class="prelude-val">Some</span>(<span class="kw-2">&amp;</span>Value::Int32(<span class="number">1</span>)));
<a href=#221 id=221 data-nosnippet>221</a> row_2.clear();
<a href=#222 id=222 data-nosnippet>222</a> <span class="macro">assert_eq!</span>(row_2.get(<span class="number">0</span>), <span class="prelude-val">None</span>);
<a href=#223 id=223 data-nosnippet>223</a> row_2
<a href=#224 id=224 data-nosnippet>224</a> .packer()
<a href=#225 id=225 data-nosnippet>225</a> .extend(<span class="macro">vec!</span>[Value::Int32(<span class="number">1</span>), Value::Int32(<span class="number">2</span>)]);
<a href=#226 id=226 data-nosnippet>226</a> <span class="macro">assert_eq!</span>(row_2.get(<span class="number">0</span>), <span class="prelude-val">Some</span>(<span class="kw-2">&amp;</span>Value::Int32(<span class="number">1</span>)));
<a href=#227 id=227 data-nosnippet>227</a> row_2.extend(<span class="macro">vec!</span>[Value::Int32(<span class="number">1</span>), Value::Int32(<span class="number">2</span>)]);
<a href=#228 id=228 data-nosnippet>228</a> <span class="macro">assert_eq!</span>(row_2.len(), <span class="number">4</span>);
<a href=#229 id=229 data-nosnippet>229</a> <span class="kw">let </span>row_3 = Row::pack(row_2.into_iter());
<a href=#230 id=230 data-nosnippet>230</a> <span class="macro">assert_eq!</span>(row_3.len(), <span class="number">4</span>);
<a href=#231 id=231 data-nosnippet>231</a> <span class="kw">let </span>row_4 = Row::pack(row_3.iter().cloned());
<a href=#232 id=232 data-nosnippet>232</a> <span class="macro">assert_eq!</span>(row_3, row_4);
<a href=#233 id=233 data-nosnippet>233</a> }
<a href=#234 id=234 data-nosnippet>234</a>
<a href=#235 id=235 data-nosnippet>235</a> <span class="attr">#[test]
<a href=#236 id=236 data-nosnippet>236</a> </span><span class="kw">fn </span>test_cast_to_internal_ts() {
<a href=#237 id=237 data-nosnippet>237</a> {
<a href=#238 id=238 data-nosnippet>238</a> <span class="kw">let </span>a = Value::from(<span class="number">1i32</span>);
<a href=#239 id=239 data-nosnippet>239</a> <span class="kw">let </span>b = Value::from(<span class="number">1i64</span>);
<a href=#240 id=240 data-nosnippet>240</a> <span class="kw">let </span>c = Value::Timestamp(Timestamp::new_millisecond(<span class="number">1i64</span>));
<a href=#241 id=241 data-nosnippet>241</a> <span class="kw">let </span>d = Value::from(<span class="number">1.0</span>);
<a href=#242 id=242 data-nosnippet>242</a>
<a href=#243 id=243 data-nosnippet>243</a> <span class="macro">assert!</span>(value_to_internal_ts(a).is_err());
<a href=#244 id=244 data-nosnippet>244</a> <span class="macro">assert_eq!</span>(value_to_internal_ts(b).unwrap(), <span class="number">1i64</span>);
<a href=#245 id=245 data-nosnippet>245</a> <span class="macro">assert_eq!</span>(value_to_internal_ts(c).unwrap(), <span class="number">1i64</span>);
<a href=#246 id=246 data-nosnippet>246</a> <span class="macro">assert!</span>(value_to_internal_ts(d).is_err());
<a href=#247 id=247 data-nosnippet>247</a> }
<a href=#248 id=248 data-nosnippet>248</a>
<a href=#249 id=249 data-nosnippet>249</a> {
<a href=#250 id=250 data-nosnippet>250</a> <span class="comment">// time related type
<a href=#251 id=251 data-nosnippet>251</a> </span><span class="kw">let </span>a = Value::Date(Date::new(<span class="number">1</span>));
<a href=#252 id=252 data-nosnippet>252</a> <span class="macro">assert_eq!</span>(value_to_internal_ts(a).unwrap(), <span class="number">86400 </span>* <span class="number">1000i64</span>);
<a href=#253 id=253 data-nosnippet>253</a> <span class="kw">let </span>b = Value::Timestamp(common_time::Timestamp::new_second(<span class="number">1</span>));
<a href=#254 id=254 data-nosnippet>254</a> <span class="macro">assert_eq!</span>(value_to_internal_ts(b).unwrap(), <span class="number">1000i64</span>);
<a href=#255 id=255 data-nosnippet>255</a> <span class="kw">let </span>c = Value::Time(common_time::time::Time::new_second(<span class="number">1</span>));
<a href=#256 id=256 data-nosnippet>256</a> <span class="macro">assert!</span>(<span class="macro">matches!</span>(
<a href=#257 id=257 data-nosnippet>257</a> value_to_internal_ts(c),
<a href=#258 id=258 data-nosnippet>258</a> <span class="prelude-val">Err</span>(EvalError::InvalidArgument { .. })
<a href=#259 id=259 data-nosnippet>259</a> ));
<a href=#260 id=260 data-nosnippet>260</a> }
<a href=#261 id=261 data-nosnippet>261</a> }
<a href=#262 id=262 data-nosnippet>262</a>}
</code></pre></div></section></main></body></html>
+458
View File
@@ -0,0 +1,458 @@
<!DOCTYPE html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1.0"><meta name="generator" content="rustdoc"><meta name="description" content="Source of the Rust file `src/flow/src/repr/relation.rs`."><title>relation.rs - source</title><script>if(window.location.protocol!=="file:")document.head.insertAdjacentHTML("beforeend","SourceSerif4-Regular-6b053e98.ttf.woff2,FiraSans-Italic-81dc35de.woff2,FiraSans-Regular-0fe48ade.woff2,FiraSans-MediumItalic-ccf7e434.woff2,FiraSans-Medium-e1aa3f0a.woff2,SourceCodePro-Regular-8badfe75.ttf.woff2,SourceCodePro-Semibold-aa29a496.ttf.woff2".split(",").map(f=>`<link rel="preload" as="font" type="font/woff2"href="../../../static.files/${f}">`).join(""))</script><link rel="stylesheet" href="../../../static.files/normalize-9960930a.css"><link rel="stylesheet" href="../../../static.files/rustdoc-17e0aaed.css"><meta name="rustdoc-vars" data-root-path="../../../" data-static-root-path="../../../static.files/" data-current-crate="flow" data-themes="" data-resource-suffix="" data-rustdoc-version="1.96.1 (31fca3adb 2026-06-26)" data-channel="1.96.1" data-search-js="search-b5634cc7.js" data-stringdex-js="stringdex-2da4960a.js" data-settings-js="settings-170eb4bf.js" ><script src="../../../static.files/storage-41dd4d93.js"></script><script defer src="../../../static.files/src-script-813739b1.js"></script><script defer src="../../../src-files.js"></script><script defer src="../../../static.files/main-5013f961.js"></script><noscript><link rel="stylesheet" href="../../../static.files/noscript-f7c3ffd8.css"></noscript><link rel="alternate icon" type="image/png" href="../../../static.files/favicon-32x32-eab170b8.png"><link rel="icon" type="image/svg+xml" href="../../../static.files/favicon-044be391.svg"></head><body class="rustdoc src"><a class="skip-main-content" href="#main-content">Skip to main content</a><!--[if lte IE 11]><div class="warning">This old browser is unsupported and will most likely display funky things.</div><![endif]--><nav class="sidebar"><div class="src-sidebar-title"><h2>Files</h2></div></nav><div class="sidebar-resizer" title="Drag to resize sidebar"></div><main><section id="main-content" class="content" tabindex="-1"><div class="main-heading"><h1><div class="sub-heading">flow/repr/</div>relation.rs</h1><rustdoc-toolbar></rustdoc-toolbar></div><div class="example-wrap digits-3"><pre class="rust"><code><a href=#1 id=1 data-nosnippet>1</a><span class="comment">// Copyright 2023 Greptime Team
<a href=#2 id=2 data-nosnippet>2</a>//
<a href=#3 id=3 data-nosnippet>3</a>// Licensed under the Apache License, Version 2.0 (the "License");
<a href=#4 id=4 data-nosnippet>4</a>// you may not use this file except in compliance with the License.
<a href=#5 id=5 data-nosnippet>5</a>// You may obtain a copy of the License at
<a href=#6 id=6 data-nosnippet>6</a>//
<a href=#7 id=7 data-nosnippet>7</a>// http://www.apache.org/licenses/LICENSE-2.0
<a href=#8 id=8 data-nosnippet>8</a>//
<a href=#9 id=9 data-nosnippet>9</a>// Unless required by applicable law or agreed to in writing, software
<a href=#10 id=10 data-nosnippet>10</a>// distributed under the License is distributed on an "AS IS" BASIS,
<a href=#11 id=11 data-nosnippet>11</a>// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
<a href=#12 id=12 data-nosnippet>12</a>// See the License for the specific language governing permissions and
<a href=#13 id=13 data-nosnippet>13</a>// limitations under the License.
<a href=#14 id=14 data-nosnippet>14</a>
<a href=#15 id=15 data-nosnippet>15</a></span><span class="kw">use </span>datafusion_common::DFSchema;
<a href=#16 id=16 data-nosnippet>16</a><span class="kw">use </span>datatypes::data_type::DataType;
<a href=#17 id=17 data-nosnippet>17</a><span class="kw">use </span>datatypes::prelude::ConcreteDataType;
<a href=#18 id=18 data-nosnippet>18</a><span class="kw">use </span>serde::{Deserialize, Serialize};
<a href=#19 id=19 data-nosnippet>19</a><span class="kw">use </span>snafu::{ResultExt, ensure};
<a href=#20 id=20 data-nosnippet>20</a>
<a href=#21 id=21 data-nosnippet>21</a><span class="kw">use </span><span class="kw">crate</span>::error::{DatafusionSnafu, InternalSnafu, InvalidQuerySnafu, <span class="prelude-ty">Result</span>};
<a href=#22 id=22 data-nosnippet>22</a>
<a href=#23 id=23 data-nosnippet>23</a><span class="doccomment">/// a set of column indices that are "keys" for the collection.
<a href=#24 id=24 data-nosnippet>24</a></span><span class="attr">#[derive(Default, Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Serialize, Deserialize, Hash)]
<a href=#25 id=25 data-nosnippet>25</a></span><span class="kw">pub struct </span>Key {
<a href=#26 id=26 data-nosnippet>26</a> <span class="doccomment">/// indicate whose column form key
<a href=#27 id=27 data-nosnippet>27</a> </span><span class="kw">pub </span>column_indices: Vec&lt;usize&gt;,
<a href=#28 id=28 data-nosnippet>28</a>}
<a href=#29 id=29 data-nosnippet>29</a>
<a href=#30 id=30 data-nosnippet>30</a><span class="kw">impl </span>Key {
<a href=#31 id=31 data-nosnippet>31</a> <span class="doccomment">/// create a new Key
<a href=#32 id=32 data-nosnippet>32</a> </span><span class="kw">pub fn </span>new() -&gt; <span class="self">Self </span>{
<a href=#33 id=33 data-nosnippet>33</a> Default::default()
<a href=#34 id=34 data-nosnippet>34</a> }
<a href=#35 id=35 data-nosnippet>35</a>
<a href=#36 id=36 data-nosnippet>36</a> <span class="doccomment">/// create a new Key from a vector of column indices
<a href=#37 id=37 data-nosnippet>37</a> </span><span class="kw">pub fn </span>from(<span class="kw-2">mut </span>column_indices: Vec&lt;usize&gt;) -&gt; <span class="self">Self </span>{
<a href=#38 id=38 data-nosnippet>38</a> column_indices.sort_unstable();
<a href=#39 id=39 data-nosnippet>39</a> <span class="self">Self </span>{ column_indices }
<a href=#40 id=40 data-nosnippet>40</a> }
<a href=#41 id=41 data-nosnippet>41</a>
<a href=#42 id=42 data-nosnippet>42</a> <span class="doccomment">/// Add a column to Key
<a href=#43 id=43 data-nosnippet>43</a> </span><span class="kw">pub fn </span>add_col(<span class="kw-2">&amp;mut </span><span class="self">self</span>, col: usize) {
<a href=#44 id=44 data-nosnippet>44</a> <span class="self">self</span>.column_indices.push(col);
<a href=#45 id=45 data-nosnippet>45</a> }
<a href=#46 id=46 data-nosnippet>46</a>
<a href=#47 id=47 data-nosnippet>47</a> <span class="doccomment">/// Remove a column from Key
<a href=#48 id=48 data-nosnippet>48</a> </span><span class="kw">pub fn </span>remove_col(<span class="kw-2">&amp;mut </span><span class="self">self</span>, col: usize) {
<a href=#49 id=49 data-nosnippet>49</a> <span class="self">self</span>.column_indices.retain(|<span class="kw-2">&amp;</span>r| r != col);
<a href=#50 id=50 data-nosnippet>50</a> }
<a href=#51 id=51 data-nosnippet>51</a>
<a href=#52 id=52 data-nosnippet>52</a> <span class="doccomment">/// get all columns in Key
<a href=#53 id=53 data-nosnippet>53</a> </span><span class="kw">pub fn </span>get(<span class="kw-2">&amp;</span><span class="self">self</span>) -&gt; <span class="kw-2">&amp;</span>Vec&lt;usize&gt; {
<a href=#54 id=54 data-nosnippet>54</a> <span class="kw-2">&amp;</span><span class="self">self</span>.column_indices
<a href=#55 id=55 data-nosnippet>55</a> }
<a href=#56 id=56 data-nosnippet>56</a>
<a href=#57 id=57 data-nosnippet>57</a> <span class="doccomment">/// True if Key is empty
<a href=#58 id=58 data-nosnippet>58</a> </span><span class="kw">pub fn </span>is_empty(<span class="kw-2">&amp;</span><span class="self">self</span>) -&gt; bool {
<a href=#59 id=59 data-nosnippet>59</a> <span class="self">self</span>.column_indices.is_empty()
<a href=#60 id=60 data-nosnippet>60</a> }
<a href=#61 id=61 data-nosnippet>61</a>
<a href=#62 id=62 data-nosnippet>62</a> <span class="doccomment">/// True if all columns in self are also in other
<a href=#63 id=63 data-nosnippet>63</a> </span><span class="kw">pub fn </span>subset_of(<span class="kw-2">&amp;</span><span class="self">self</span>, other: <span class="kw-2">&amp;</span>Key) -&gt; bool {
<a href=#64 id=64 data-nosnippet>64</a> <span class="self">self</span>.column_indices
<a href=#65 id=65 data-nosnippet>65</a> .iter()
<a href=#66 id=66 data-nosnippet>66</a> .all(|c| other.column_indices.contains(c))
<a href=#67 id=67 data-nosnippet>67</a> }
<a href=#68 id=68 data-nosnippet>68</a>}
<a href=#69 id=69 data-nosnippet>69</a>
<a href=#70 id=70 data-nosnippet>70</a><span class="doccomment">/// The type of a relation.
<a href=#71 id=71 data-nosnippet>71</a></span><span class="attr">#[derive(Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Serialize, Deserialize, Hash)]
<a href=#72 id=72 data-nosnippet>72</a></span><span class="kw">pub struct </span>RelationType {
<a href=#73 id=73 data-nosnippet>73</a> <span class="doccomment">/// The type for each column, in order.
<a href=#74 id=74 data-nosnippet>74</a> </span><span class="kw">pub </span>column_types: Vec&lt;ColumnType&gt;,
<a href=#75 id=75 data-nosnippet>75</a> <span class="doccomment">/// Sets of indices that are "keys" for the collection.
<a href=#76 id=76 data-nosnippet>76</a> ///
<a href=#77 id=77 data-nosnippet>77</a> /// Each element in this list is a set of column indices, each with the
<a href=#78 id=78 data-nosnippet>78</a> /// property that the collection contains at most one record with each
<a href=#79 id=79 data-nosnippet>79</a> /// distinct set of values for each column. Alternately, for a specific set
<a href=#80 id=80 data-nosnippet>80</a> /// of values assigned to the these columns there is at most one record.
<a href=#81 id=81 data-nosnippet>81</a> ///
<a href=#82 id=82 data-nosnippet>82</a> /// A collection can contain multiple sets of keys, although it is common to
<a href=#83 id=83 data-nosnippet>83</a> /// have either zero or one sets of key indices.
<a href=#84 id=84 data-nosnippet>84</a> </span><span class="kw">pub </span>keys: Vec&lt;Key&gt;,
<a href=#85 id=85 data-nosnippet>85</a> <span class="doccomment">/// optionally indicate the column that is TIME INDEX
<a href=#86 id=86 data-nosnippet>86</a> </span><span class="kw">pub </span>time_index: <span class="prelude-ty">Option</span>&lt;usize&gt;,
<a href=#87 id=87 data-nosnippet>87</a> <span class="doccomment">/// mark all the columns that are added automatically by flow, but are not present in original sql
<a href=#88 id=88 data-nosnippet>88</a> </span><span class="kw">pub </span>auto_columns: Vec&lt;usize&gt;,
<a href=#89 id=89 data-nosnippet>89</a>}
<a href=#90 id=90 data-nosnippet>90</a>
<a href=#91 id=91 data-nosnippet>91</a><span class="kw">impl </span>RelationType {
<a href=#92 id=92 data-nosnippet>92</a> <span class="kw">pub fn </span>with_autos(<span class="kw-2">mut </span><span class="self">self</span>, auto_cols: <span class="kw-2">&amp;</span>[usize]) -&gt; <span class="self">Self </span>{
<a href=#93 id=93 data-nosnippet>93</a> <span class="self">self</span>.auto_columns = auto_cols.to_vec();
<a href=#94 id=94 data-nosnippet>94</a> <span class="self">self
<a href=#95 id=95 data-nosnippet>95</a> </span>}
<a href=#96 id=96 data-nosnippet>96</a>
<a href=#97 id=97 data-nosnippet>97</a> <span class="doccomment">/// Constructs a `RelationType` representing the relation with no columns and
<a href=#98 id=98 data-nosnippet>98</a> /// no keys.
<a href=#99 id=99 data-nosnippet>99</a> </span><span class="kw">pub fn </span>empty() -&gt; <span class="self">Self </span>{
<a href=#100 id=100 data-nosnippet>100</a> RelationType::new(<span class="macro">vec!</span>[])
<a href=#101 id=101 data-nosnippet>101</a> }
<a href=#102 id=102 data-nosnippet>102</a>
<a href=#103 id=103 data-nosnippet>103</a> <span class="doccomment">/// Constructs a new `RelationType` from specified column types.
<a href=#104 id=104 data-nosnippet>104</a> ///
<a href=#105 id=105 data-nosnippet>105</a> /// The `RelationType` will have no keys.
<a href=#106 id=106 data-nosnippet>106</a> </span><span class="kw">pub fn </span>new(column_types: Vec&lt;ColumnType&gt;) -&gt; <span class="self">Self </span>{
<a href=#107 id=107 data-nosnippet>107</a> RelationType {
<a href=#108 id=108 data-nosnippet>108</a> column_types,
<a href=#109 id=109 data-nosnippet>109</a> keys: Vec::new(),
<a href=#110 id=110 data-nosnippet>110</a> time_index: <span class="prelude-val">None</span>,
<a href=#111 id=111 data-nosnippet>111</a> auto_columns: <span class="macro">vec!</span>[],
<a href=#112 id=112 data-nosnippet>112</a> }
<a href=#113 id=113 data-nosnippet>113</a> }
<a href=#114 id=114 data-nosnippet>114</a>
<a href=#115 id=115 data-nosnippet>115</a> <span class="doccomment">/// Adds a new key for the relation. Also sorts the key indices.
<a href=#116 id=116 data-nosnippet>116</a> ///
<a href=#117 id=117 data-nosnippet>117</a> /// will ignore empty key
<a href=#118 id=118 data-nosnippet>118</a> </span><span class="kw">pub fn </span>with_key(<span class="kw-2">mut </span><span class="self">self</span>, <span class="kw-2">mut </span>indices: Vec&lt;usize&gt;) -&gt; <span class="self">Self </span>{
<a href=#119 id=119 data-nosnippet>119</a> <span class="kw">if </span>indices.is_empty() {
<a href=#120 id=120 data-nosnippet>120</a> <span class="kw">return </span><span class="self">self</span>;
<a href=#121 id=121 data-nosnippet>121</a> }
<a href=#122 id=122 data-nosnippet>122</a> indices.sort_unstable();
<a href=#123 id=123 data-nosnippet>123</a> <span class="kw">let </span>key = Key::from(indices);
<a href=#124 id=124 data-nosnippet>124</a> <span class="kw">if </span>!<span class="self">self</span>.keys.contains(<span class="kw-2">&amp;</span>key) {
<a href=#125 id=125 data-nosnippet>125</a> <span class="self">self</span>.keys.push(key);
<a href=#126 id=126 data-nosnippet>126</a> }
<a href=#127 id=127 data-nosnippet>127</a> <span class="self">self
<a href=#128 id=128 data-nosnippet>128</a> </span>}
<a href=#129 id=129 data-nosnippet>129</a>
<a href=#130 id=130 data-nosnippet>130</a> <span class="doccomment">/// Adds new keys for the relation. Also sorts the key indices.
<a href=#131 id=131 data-nosnippet>131</a> ///
<a href=#132 id=132 data-nosnippet>132</a> /// will ignore empty keys
<a href=#133 id=133 data-nosnippet>133</a> </span><span class="kw">pub fn </span>with_keys(<span class="kw-2">mut </span><span class="self">self</span>, keys: Vec&lt;Vec&lt;usize&gt;&gt;) -&gt; <span class="self">Self </span>{
<a href=#134 id=134 data-nosnippet>134</a> <span class="kw">for </span>key <span class="kw">in </span>keys {
<a href=#135 id=135 data-nosnippet>135</a> <span class="self">self </span>= <span class="self">self</span>.with_key(key)
<a href=#136 id=136 data-nosnippet>136</a> }
<a href=#137 id=137 data-nosnippet>137</a> <span class="self">self
<a href=#138 id=138 data-nosnippet>138</a> </span>}
<a href=#139 id=139 data-nosnippet>139</a>
<a href=#140 id=140 data-nosnippet>140</a> <span class="doccomment">/// will also remove time index from keys if it's in keys
<a href=#141 id=141 data-nosnippet>141</a> </span><span class="kw">pub fn </span>with_time_index(<span class="kw-2">mut </span><span class="self">self</span>, time_index: <span class="prelude-ty">Option</span>&lt;usize&gt;) -&gt; <span class="self">Self </span>{
<a href=#142 id=142 data-nosnippet>142</a> <span class="self">self</span>.time_index = time_index;
<a href=#143 id=143 data-nosnippet>143</a> <span class="kw">for </span>key <span class="kw">in </span><span class="kw-2">&amp;mut </span><span class="self">self</span>.keys {
<a href=#144 id=144 data-nosnippet>144</a> key.remove_col(time_index.unwrap_or(usize::MAX));
<a href=#145 id=145 data-nosnippet>145</a> }
<a href=#146 id=146 data-nosnippet>146</a> <span class="comment">// remove empty keys
<a href=#147 id=147 data-nosnippet>147</a> </span><span class="self">self</span>.keys.retain(|key| !key.is_empty());
<a href=#148 id=148 data-nosnippet>148</a> <span class="self">self
<a href=#149 id=149 data-nosnippet>149</a> </span>}
<a href=#150 id=150 data-nosnippet>150</a>
<a href=#151 id=151 data-nosnippet>151</a> <span class="doccomment">/// Computes the number of columns in the relation.
<a href=#152 id=152 data-nosnippet>152</a> </span><span class="kw">pub fn </span>arity(<span class="kw-2">&amp;</span><span class="self">self</span>) -&gt; usize {
<a href=#153 id=153 data-nosnippet>153</a> <span class="self">self</span>.column_types.len()
<a href=#154 id=154 data-nosnippet>154</a> }
<a href=#155 id=155 data-nosnippet>155</a>
<a href=#156 id=156 data-nosnippet>156</a> <span class="doccomment">/// Gets the index of the columns used when creating a default index.
<a href=#157 id=157 data-nosnippet>157</a> </span><span class="kw">pub fn </span>default_key(<span class="kw-2">&amp;</span><span class="self">self</span>) -&gt; Vec&lt;usize&gt; {
<a href=#158 id=158 data-nosnippet>158</a> <span class="kw">if let </span><span class="prelude-val">Some</span>(key) = <span class="self">self</span>.keys.first() {
<a href=#159 id=159 data-nosnippet>159</a> <span class="kw">if </span>key.is_empty() {
<a href=#160 id=160 data-nosnippet>160</a> (<span class="number">0</span>..<span class="self">self</span>.column_types.len()).collect()
<a href=#161 id=161 data-nosnippet>161</a> } <span class="kw">else </span>{
<a href=#162 id=162 data-nosnippet>162</a> key.get().clone()
<a href=#163 id=163 data-nosnippet>163</a> }
<a href=#164 id=164 data-nosnippet>164</a> } <span class="kw">else </span>{
<a href=#165 id=165 data-nosnippet>165</a> (<span class="number">0</span>..<span class="self">self</span>.column_types.len()).collect()
<a href=#166 id=166 data-nosnippet>166</a> }
<a href=#167 id=167 data-nosnippet>167</a> }
<a href=#168 id=168 data-nosnippet>168</a>
<a href=#169 id=169 data-nosnippet>169</a> <span class="doccomment">/// True if any collection described by `self` could safely be described by `other`.
<a href=#170 id=170 data-nosnippet>170</a> ///
<a href=#171 id=171 data-nosnippet>171</a> /// In practice this means checking that the scalar types match exactly, and that the
<a href=#172 id=172 data-nosnippet>172</a> /// nullability of `self` is at least as strict as `other`, and that all keys of `other`
<a href=#173 id=173 data-nosnippet>173</a> /// contain some key of `self` (as a set of key columns is less strict than any subset).
<a href=#174 id=174 data-nosnippet>174</a> </span><span class="kw">pub fn </span>subtypes(<span class="kw-2">&amp;</span><span class="self">self</span>, other: <span class="kw-2">&amp;</span>RelationType) -&gt; bool {
<a href=#175 id=175 data-nosnippet>175</a> <span class="kw">if </span><span class="self">self</span>.column_types.len() != other.column_types.len() {
<a href=#176 id=176 data-nosnippet>176</a> <span class="kw">return </span><span class="bool-val">false</span>;
<a href=#177 id=177 data-nosnippet>177</a> }
<a href=#178 id=178 data-nosnippet>178</a>
<a href=#179 id=179 data-nosnippet>179</a> <span class="kw">for </span>(col1, col2) <span class="kw">in </span><span class="self">self</span>.column_types.iter().zip(other.column_types.iter()) {
<a href=#180 id=180 data-nosnippet>180</a> <span class="kw">if </span>col1.nullable &amp;&amp; !col2.nullable {
<a href=#181 id=181 data-nosnippet>181</a> <span class="kw">return </span><span class="bool-val">false</span>;
<a href=#182 id=182 data-nosnippet>182</a> }
<a href=#183 id=183 data-nosnippet>183</a> <span class="kw">if </span>col1.scalar_type != col2.scalar_type {
<a href=#184 id=184 data-nosnippet>184</a> <span class="kw">return </span><span class="bool-val">false</span>;
<a href=#185 id=185 data-nosnippet>185</a> }
<a href=#186 id=186 data-nosnippet>186</a> }
<a href=#187 id=187 data-nosnippet>187</a>
<a href=#188 id=188 data-nosnippet>188</a> <span class="kw">let </span>all_keys = other
<a href=#189 id=189 data-nosnippet>189</a> .keys
<a href=#190 id=190 data-nosnippet>190</a> .iter()
<a href=#191 id=191 data-nosnippet>191</a> .all(|key1| <span class="self">self</span>.keys.iter().any(|key2| key1.subset_of(key2)));
<a href=#192 id=192 data-nosnippet>192</a> <span class="kw">if </span>!all_keys {
<a href=#193 id=193 data-nosnippet>193</a> <span class="kw">return </span><span class="bool-val">false</span>;
<a href=#194 id=194 data-nosnippet>194</a> }
<a href=#195 id=195 data-nosnippet>195</a>
<a href=#196 id=196 data-nosnippet>196</a> <span class="bool-val">true
<a href=#197 id=197 data-nosnippet>197</a> </span>}
<a href=#198 id=198 data-nosnippet>198</a>
<a href=#199 id=199 data-nosnippet>199</a> <span class="doccomment">/// Return relation describe with column names
<a href=#200 id=200 data-nosnippet>200</a> </span><span class="kw">pub fn </span>into_named(<span class="self">self</span>, names: Vec&lt;<span class="prelude-ty">Option</span>&lt;ColumnName&gt;&gt;) -&gt; RelationDesc {
<a href=#201 id=201 data-nosnippet>201</a> RelationDesc { typ: <span class="self">self</span>, names }
<a href=#202 id=202 data-nosnippet>202</a> }
<a href=#203 id=203 data-nosnippet>203</a>
<a href=#204 id=204 data-nosnippet>204</a> <span class="doccomment">/// Return relation describe without column names
<a href=#205 id=205 data-nosnippet>205</a> </span><span class="kw">pub fn </span>into_unnamed(<span class="self">self</span>) -&gt; RelationDesc {
<a href=#206 id=206 data-nosnippet>206</a> RelationDesc {
<a href=#207 id=207 data-nosnippet>207</a> names: <span class="macro">vec!</span>[<span class="prelude-val">None</span>; <span class="self">self</span>.column_types.len()],
<a href=#208 id=208 data-nosnippet>208</a> typ: <span class="self">self</span>,
<a href=#209 id=209 data-nosnippet>209</a> }
<a href=#210 id=210 data-nosnippet>210</a> }
<a href=#211 id=211 data-nosnippet>211</a>}
<a href=#212 id=212 data-nosnippet>212</a>
<a href=#213 id=213 data-nosnippet>213</a><span class="doccomment">/// The type of a `Value`
<a href=#214 id=214 data-nosnippet>214</a>///
<a href=#215 id=215 data-nosnippet>215</a>/// [`ColumnType`] bundles information about the scalar type of a datum (e.g.,
<a href=#216 id=216 data-nosnippet>216</a>/// Int32 or String) with its nullability.
<a href=#217 id=217 data-nosnippet>217</a>///
<a href=#218 id=218 data-nosnippet>218</a>/// To construct a column type, either initialize the struct directly, or
<a href=#219 id=219 data-nosnippet>219</a>/// use the [`ScalarType::nullable`] method.
<a href=#220 id=220 data-nosnippet>220</a></span><span class="attr">#[derive(Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Serialize, Deserialize, Hash)]
<a href=#221 id=221 data-nosnippet>221</a></span><span class="kw">pub struct </span>ColumnType {
<a href=#222 id=222 data-nosnippet>222</a> <span class="doccomment">/// The underlying scalar type (e.g., Int32 or String) of this column.
<a href=#223 id=223 data-nosnippet>223</a> </span><span class="kw">pub </span>scalar_type: ConcreteDataType,
<a href=#224 id=224 data-nosnippet>224</a> <span class="doccomment">/// Whether this datum can be null.
<a href=#225 id=225 data-nosnippet>225</a> </span><span class="attr">#[serde(default = <span class="string">"return_true"</span>)]
<a href=#226 id=226 data-nosnippet>226</a> </span><span class="kw">pub </span>nullable: bool,
<a href=#227 id=227 data-nosnippet>227</a>}
<a href=#228 id=228 data-nosnippet>228</a>
<a href=#229 id=229 data-nosnippet>229</a><span class="kw">impl </span>ColumnType {
<a href=#230 id=230 data-nosnippet>230</a> <span class="doccomment">/// Constructs a new `ColumnType` from a scalar type and a nullability flag.
<a href=#231 id=231 data-nosnippet>231</a> </span><span class="kw">pub fn </span>new(scalar_type: ConcreteDataType, nullable: bool) -&gt; <span class="self">Self </span>{
<a href=#232 id=232 data-nosnippet>232</a> ColumnType {
<a href=#233 id=233 data-nosnippet>233</a> scalar_type,
<a href=#234 id=234 data-nosnippet>234</a> nullable,
<a href=#235 id=235 data-nosnippet>235</a> }
<a href=#236 id=236 data-nosnippet>236</a> }
<a href=#237 id=237 data-nosnippet>237</a>
<a href=#238 id=238 data-nosnippet>238</a> <span class="doccomment">/// Constructs a new `ColumnType` from a scalar type, with nullability set to
<a href=#239 id=239 data-nosnippet>239</a> /// ***true***
<a href=#240 id=240 data-nosnippet>240</a> </span><span class="kw">pub fn </span>new_nullable(scalar_type: ConcreteDataType) -&gt; <span class="self">Self </span>{
<a href=#241 id=241 data-nosnippet>241</a> ColumnType {
<a href=#242 id=242 data-nosnippet>242</a> scalar_type,
<a href=#243 id=243 data-nosnippet>243</a> nullable: <span class="bool-val">true</span>,
<a href=#244 id=244 data-nosnippet>244</a> }
<a href=#245 id=245 data-nosnippet>245</a> }
<a href=#246 id=246 data-nosnippet>246</a>
<a href=#247 id=247 data-nosnippet>247</a> <span class="doccomment">/// Returns the scalar type of this column.
<a href=#248 id=248 data-nosnippet>248</a> </span><span class="kw">pub fn </span>scalar_type(<span class="kw-2">&amp;</span><span class="self">self</span>) -&gt; <span class="kw-2">&amp;</span>ConcreteDataType {
<a href=#249 id=249 data-nosnippet>249</a> <span class="kw-2">&amp;</span><span class="self">self</span>.scalar_type
<a href=#250 id=250 data-nosnippet>250</a> }
<a href=#251 id=251 data-nosnippet>251</a>
<a href=#252 id=252 data-nosnippet>252</a> <span class="doccomment">/// Returns true if this column can be null.
<a href=#253 id=253 data-nosnippet>253</a> </span><span class="kw">pub fn </span>nullable(<span class="kw-2">&amp;</span><span class="self">self</span>) -&gt; bool {
<a href=#254 id=254 data-nosnippet>254</a> <span class="self">self</span>.nullable
<a href=#255 id=255 data-nosnippet>255</a> }
<a href=#256 id=256 data-nosnippet>256</a>}
<a href=#257 id=257 data-nosnippet>257</a>
<a href=#258 id=258 data-nosnippet>258</a><span class="doccomment">/// This method exists solely for the purpose of making ColumnType nullable by
<a href=#259 id=259 data-nosnippet>259</a>/// default in unit tests. The default value of a bool is false, and the only
<a href=#260 id=260 data-nosnippet>260</a>/// way to make an object take on any other value by default is to pass it a
<a href=#261 id=261 data-nosnippet>261</a>/// function that returns the desired default value. See
<a href=#262 id=262 data-nosnippet>262</a>/// &lt;https://github.com/serde-rs/serde/issues/1030&gt;
<a href=#263 id=263 data-nosnippet>263</a></span><span class="attr">#[inline(always)]
<a href=#264 id=264 data-nosnippet>264</a></span><span class="kw">fn </span>return_true() -&gt; bool {
<a href=#265 id=265 data-nosnippet>265</a> <span class="bool-val">true
<a href=#266 id=266 data-nosnippet>266</a></span>}
<a href=#267 id=267 data-nosnippet>267</a>
<a href=#268 id=268 data-nosnippet>268</a><span class="doccomment">/// A description of the shape of a relation.
<a href=#269 id=269 data-nosnippet>269</a>///
<a href=#270 id=270 data-nosnippet>270</a>/// It bundles a [`RelationType`] with the name of each column in the relation.
<a href=#271 id=271 data-nosnippet>271</a>/// Individual column names are optional.
<a href=#272 id=272 data-nosnippet>272</a></span><span class="attr">#[derive(Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Serialize, Deserialize, Hash)]
<a href=#273 id=273 data-nosnippet>273</a></span><span class="kw">pub struct </span>RelationDesc {
<a href=#274 id=274 data-nosnippet>274</a> <span class="kw">pub </span>typ: RelationType,
<a href=#275 id=275 data-nosnippet>275</a> <span class="kw">pub </span>names: Vec&lt;<span class="prelude-ty">Option</span>&lt;ColumnName&gt;&gt;,
<a href=#276 id=276 data-nosnippet>276</a>}
<a href=#277 id=277 data-nosnippet>277</a>
<a href=#278 id=278 data-nosnippet>278</a><span class="kw">impl </span>RelationDesc {
<a href=#279 id=279 data-nosnippet>279</a> <span class="kw">pub fn </span>len(<span class="kw-2">&amp;</span><span class="self">self</span>) -&gt; <span class="prelude-ty">Result</span>&lt;usize&gt; {
<a href=#280 id=280 data-nosnippet>280</a> <span class="macro">ensure!</span>(
<a href=#281 id=281 data-nosnippet>281</a> <span class="self">self</span>.typ.column_types.len() == <span class="self">self</span>.names.len(),
<a href=#282 id=282 data-nosnippet>282</a> InternalSnafu {
<a href=#283 id=283 data-nosnippet>283</a> reason: <span class="string">"Expect typ and names field to be of same length"
<a href=#284 id=284 data-nosnippet>284</a> </span>}
<a href=#285 id=285 data-nosnippet>285</a> );
<a href=#286 id=286 data-nosnippet>286</a> <span class="prelude-val">Ok</span>(<span class="self">self</span>.names.len())
<a href=#287 id=287 data-nosnippet>287</a> }
<a href=#288 id=288 data-nosnippet>288</a>
<a href=#289 id=289 data-nosnippet>289</a> <span class="kw">pub fn </span>to_df_schema(<span class="kw-2">&amp;</span><span class="self">self</span>) -&gt; <span class="prelude-ty">Result</span>&lt;DFSchema&gt; {
<a href=#290 id=290 data-nosnippet>290</a> <span class="kw">let </span>fields: Vec&lt;<span class="kw">_</span>&gt; = <span class="self">self
<a href=#291 id=291 data-nosnippet>291</a> </span>.iter()
<a href=#292 id=292 data-nosnippet>292</a> .enumerate()
<a href=#293 id=293 data-nosnippet>293</a> .map(|(i, (name, typ))| {
<a href=#294 id=294 data-nosnippet>294</a> <span class="kw">let </span>name = name.clone().unwrap_or(<span class="macro">format!</span>(<span class="string">"Col_{i}"</span>));
<a href=#295 id=295 data-nosnippet>295</a> <span class="kw">let </span>nullable = typ.nullable;
<a href=#296 id=296 data-nosnippet>296</a> <span class="kw">let </span>data_type = typ.scalar_type.clone().as_arrow_type();
<a href=#297 id=297 data-nosnippet>297</a> arrow_schema::Field::new(name, data_type, nullable)
<a href=#298 id=298 data-nosnippet>298</a> })
<a href=#299 id=299 data-nosnippet>299</a> .collect();
<a href=#300 id=300 data-nosnippet>300</a> <span class="kw">let </span>arrow_schema = arrow_schema::Schema::new(fields);
<a href=#301 id=301 data-nosnippet>301</a>
<a href=#302 id=302 data-nosnippet>302</a> DFSchema::try_from(arrow_schema.clone()).with_context(|_e| DatafusionSnafu {
<a href=#303 id=303 data-nosnippet>303</a> context: <span class="macro">format!</span>(<span class="string">"Error when converting to DFSchema: {:?}"</span>, arrow_schema),
<a href=#304 id=304 data-nosnippet>304</a> })
<a href=#305 id=305 data-nosnippet>305</a> }
<a href=#306 id=306 data-nosnippet>306</a>}
<a href=#307 id=307 data-nosnippet>307</a>
<a href=#308 id=308 data-nosnippet>308</a><span class="kw">impl </span>RelationDesc {
<a href=#309 id=309 data-nosnippet>309</a> <span class="doccomment">/// Constructs a new `RelationDesc` that represents the empty relation
<a href=#310 id=310 data-nosnippet>310</a> /// with no columns and no keys.
<a href=#311 id=311 data-nosnippet>311</a> </span><span class="kw">pub fn </span>empty() -&gt; <span class="self">Self </span>{
<a href=#312 id=312 data-nosnippet>312</a> RelationDesc {
<a href=#313 id=313 data-nosnippet>313</a> typ: RelationType::empty(),
<a href=#314 id=314 data-nosnippet>314</a> names: <span class="macro">vec!</span>[],
<a href=#315 id=315 data-nosnippet>315</a> }
<a href=#316 id=316 data-nosnippet>316</a> }
<a href=#317 id=317 data-nosnippet>317</a>
<a href=#318 id=318 data-nosnippet>318</a> <span class="doccomment">/// Constructs a new `RelationDesc` from a `RelationType` and an iterator
<a href=#319 id=319 data-nosnippet>319</a> /// over column names.
<a href=#320 id=320 data-nosnippet>320</a> ///
<a href=#321 id=321 data-nosnippet>321</a> </span><span class="kw">pub fn </span>try_new&lt;I, N&gt;(typ: RelationType, names: I) -&gt; <span class="prelude-ty">Result</span>&lt;<span class="self">Self</span>&gt;
<a href=#322 id=322 data-nosnippet>322</a> <span class="kw">where
<a href=#323 id=323 data-nosnippet>323</a> </span>I: IntoIterator&lt;Item = N&gt;,
<a href=#324 id=324 data-nosnippet>324</a> N: Into&lt;<span class="prelude-ty">Option</span>&lt;ColumnName&gt;&gt;,
<a href=#325 id=325 data-nosnippet>325</a> {
<a href=#326 id=326 data-nosnippet>326</a> <span class="kw">let </span>names: Vec&lt;<span class="kw">_</span>&gt; = names.into_iter().map(|name| name.into()).collect();
<a href=#327 id=327 data-nosnippet>327</a> <span class="macro">ensure!</span>(
<a href=#328 id=328 data-nosnippet>328</a> typ.arity() == names.len(),
<a href=#329 id=329 data-nosnippet>329</a> InvalidQuerySnafu {
<a href=#330 id=330 data-nosnippet>330</a> reason: <span class="macro">format!</span>(
<a href=#331 id=331 data-nosnippet>331</a> <span class="string">"Length mismatch between RelationType {:?} and column names {:?}"</span>,
<a href=#332 id=332 data-nosnippet>332</a> typ.column_types, names
<a href=#333 id=333 data-nosnippet>333</a> )
<a href=#334 id=334 data-nosnippet>334</a> }
<a href=#335 id=335 data-nosnippet>335</a> );
<a href=#336 id=336 data-nosnippet>336</a> <span class="prelude-val">Ok</span>(RelationDesc { typ, names })
<a href=#337 id=337 data-nosnippet>337</a> }
<a href=#338 id=338 data-nosnippet>338</a>
<a href=#339 id=339 data-nosnippet>339</a> <span class="doccomment">/// Constructs a new `RelationDesc` from a `RelationType` and an iterator
<a href=#340 id=340 data-nosnippet>340</a> /// over column names.
<a href=#341 id=341 data-nosnippet>341</a> ///
<a href=#342 id=342 data-nosnippet>342</a> /// # Panics
<a href=#343 id=343 data-nosnippet>343</a> ///
<a href=#344 id=344 data-nosnippet>344</a> /// Panics if the arity of the `RelationType` is not equal to the number of
<a href=#345 id=345 data-nosnippet>345</a> /// items in `names`.
<a href=#346 id=346 data-nosnippet>346</a> </span><span class="kw">pub fn </span>new_unchecked&lt;I, N&gt;(typ: RelationType, names: I) -&gt; <span class="self">Self
<a href=#347 id=347 data-nosnippet>347</a> </span><span class="kw">where
<a href=#348 id=348 data-nosnippet>348</a> </span>I: IntoIterator&lt;Item = N&gt;,
<a href=#349 id=349 data-nosnippet>349</a> N: Into&lt;<span class="prelude-ty">Option</span>&lt;ColumnName&gt;&gt;,
<a href=#350 id=350 data-nosnippet>350</a> {
<a href=#351 id=351 data-nosnippet>351</a> <span class="kw">let </span>names: Vec&lt;<span class="kw">_</span>&gt; = names.into_iter().map(|name| name.into()).collect();
<a href=#352 id=352 data-nosnippet>352</a> <span class="macro">assert_eq!</span>(typ.arity(), names.len());
<a href=#353 id=353 data-nosnippet>353</a> RelationDesc { typ, names }
<a href=#354 id=354 data-nosnippet>354</a> }
<a href=#355 id=355 data-nosnippet>355</a>
<a href=#356 id=356 data-nosnippet>356</a> <span class="kw">pub fn </span>from_names_and_types&lt;I, T, N&gt;(iter: I) -&gt; <span class="self">Self
<a href=#357 id=357 data-nosnippet>357</a> </span><span class="kw">where
<a href=#358 id=358 data-nosnippet>358</a> </span>I: IntoIterator&lt;Item = (N, T)&gt;,
<a href=#359 id=359 data-nosnippet>359</a> T: Into&lt;ColumnType&gt;,
<a href=#360 id=360 data-nosnippet>360</a> N: Into&lt;<span class="prelude-ty">Option</span>&lt;ColumnName&gt;&gt;,
<a href=#361 id=361 data-nosnippet>361</a> {
<a href=#362 id=362 data-nosnippet>362</a> <span class="kw">let </span>(names, types): (Vec&lt;<span class="kw">_</span>&gt;, Vec&lt;<span class="kw">_</span>&gt;) = iter.into_iter().unzip();
<a href=#363 id=363 data-nosnippet>363</a> <span class="kw">let </span>types = types.into_iter().map(Into::into).collect();
<a href=#364 id=364 data-nosnippet>364</a> <span class="kw">let </span>typ = RelationType::new(types);
<a href=#365 id=365 data-nosnippet>365</a> <span class="self">Self</span>::new_unchecked(typ, names)
<a href=#366 id=366 data-nosnippet>366</a> }
<a href=#367 id=367 data-nosnippet>367</a> <span class="doccomment">/// Concatenates a `RelationDesc` onto the end of this `RelationDesc`.
<a href=#368 id=368 data-nosnippet>368</a> </span><span class="kw">pub fn </span>concat(<span class="kw-2">mut </span><span class="self">self</span>, other: <span class="self">Self</span>) -&gt; <span class="self">Self </span>{
<a href=#369 id=369 data-nosnippet>369</a> <span class="kw">let </span>self_len = <span class="self">self</span>.typ.column_types.len();
<a href=#370 id=370 data-nosnippet>370</a> <span class="self">self</span>.names.extend(other.names);
<a href=#371 id=371 data-nosnippet>371</a> <span class="self">self</span>.typ.column_types.extend(other.typ.column_types);
<a href=#372 id=372 data-nosnippet>372</a> <span class="kw">for </span>k <span class="kw">in </span>other.typ.keys {
<a href=#373 id=373 data-nosnippet>373</a> <span class="kw">let </span>k = k
<a href=#374 id=374 data-nosnippet>374</a> .column_indices
<a href=#375 id=375 data-nosnippet>375</a> .into_iter()
<a href=#376 id=376 data-nosnippet>376</a> .map(|idx| idx + self_len)
<a href=#377 id=377 data-nosnippet>377</a> .collect();
<a href=#378 id=378 data-nosnippet>378</a> <span class="self">self </span>= <span class="self">self</span>.with_key(k);
<a href=#379 id=379 data-nosnippet>379</a> }
<a href=#380 id=380 data-nosnippet>380</a> <span class="self">self
<a href=#381 id=381 data-nosnippet>381</a> </span>}
<a href=#382 id=382 data-nosnippet>382</a>
<a href=#383 id=383 data-nosnippet>383</a> <span class="doccomment">/// Appends a column with the specified name and type.
<a href=#384 id=384 data-nosnippet>384</a> </span><span class="kw">pub fn </span>with_column&lt;N&gt;(<span class="kw-2">mut </span><span class="self">self</span>, name: N, column_type: ColumnType) -&gt; <span class="self">Self
<a href=#385 id=385 data-nosnippet>385</a> </span><span class="kw">where
<a href=#386 id=386 data-nosnippet>386</a> </span>N: Into&lt;<span class="prelude-ty">Option</span>&lt;ColumnName&gt;&gt;,
<a href=#387 id=387 data-nosnippet>387</a> {
<a href=#388 id=388 data-nosnippet>388</a> <span class="self">self</span>.typ.column_types.push(column_type);
<a href=#389 id=389 data-nosnippet>389</a> <span class="self">self</span>.names.push(name.into());
<a href=#390 id=390 data-nosnippet>390</a> <span class="self">self
<a href=#391 id=391 data-nosnippet>391</a> </span>}
<a href=#392 id=392 data-nosnippet>392</a>
<a href=#393 id=393 data-nosnippet>393</a> <span class="doccomment">/// Adds a new key for the relation.
<a href=#394 id=394 data-nosnippet>394</a> </span><span class="kw">pub fn </span>with_key(<span class="kw-2">mut </span><span class="self">self</span>, indices: Vec&lt;usize&gt;) -&gt; <span class="self">Self </span>{
<a href=#395 id=395 data-nosnippet>395</a> <span class="self">self</span>.typ = <span class="self">self</span>.typ.with_key(indices);
<a href=#396 id=396 data-nosnippet>396</a> <span class="self">self
<a href=#397 id=397 data-nosnippet>397</a> </span>}
<a href=#398 id=398 data-nosnippet>398</a>
<a href=#399 id=399 data-nosnippet>399</a> <span class="doccomment">/// Builds a new relation description with the column names replaced with
<a href=#400 id=400 data-nosnippet>400</a> /// new names.
<a href=#401 id=401 data-nosnippet>401</a> ///
<a href=#402 id=402 data-nosnippet>402</a> </span><span class="kw">pub fn </span>try_with_names&lt;I, N&gt;(<span class="self">self</span>, names: I) -&gt; <span class="prelude-ty">Result</span>&lt;<span class="self">Self</span>&gt;
<a href=#403 id=403 data-nosnippet>403</a> <span class="kw">where
<a href=#404 id=404 data-nosnippet>404</a> </span>I: IntoIterator&lt;Item = N&gt;,
<a href=#405 id=405 data-nosnippet>405</a> N: Into&lt;<span class="prelude-ty">Option</span>&lt;ColumnName&gt;&gt;,
<a href=#406 id=406 data-nosnippet>406</a> {
<a href=#407 id=407 data-nosnippet>407</a> <span class="self">Self</span>::try_new(<span class="self">self</span>.typ, names)
<a href=#408 id=408 data-nosnippet>408</a> }
<a href=#409 id=409 data-nosnippet>409</a>
<a href=#410 id=410 data-nosnippet>410</a> <span class="doccomment">/// Computes the number of columns in the relation.
<a href=#411 id=411 data-nosnippet>411</a> </span><span class="kw">pub fn </span>arity(<span class="kw-2">&amp;</span><span class="self">self</span>) -&gt; usize {
<a href=#412 id=412 data-nosnippet>412</a> <span class="self">self</span>.typ.arity()
<a href=#413 id=413 data-nosnippet>413</a> }
<a href=#414 id=414 data-nosnippet>414</a>
<a href=#415 id=415 data-nosnippet>415</a> <span class="doccomment">/// Returns the relation type underlying this relation description.
<a href=#416 id=416 data-nosnippet>416</a> </span><span class="kw">pub fn </span>typ(<span class="kw-2">&amp;</span><span class="self">self</span>) -&gt; <span class="kw-2">&amp;</span>RelationType {
<a href=#417 id=417 data-nosnippet>417</a> <span class="kw-2">&amp;</span><span class="self">self</span>.typ
<a href=#418 id=418 data-nosnippet>418</a> }
<a href=#419 id=419 data-nosnippet>419</a>
<a href=#420 id=420 data-nosnippet>420</a> <span class="doccomment">/// Returns an iterator over the columns in this relation.
<a href=#421 id=421 data-nosnippet>421</a> </span><span class="kw">pub fn </span>iter(<span class="kw-2">&amp;</span><span class="self">self</span>) -&gt; <span class="kw">impl </span>Iterator&lt;Item = (<span class="kw-2">&amp;</span><span class="prelude-ty">Option</span>&lt;ColumnName&gt;, <span class="kw-2">&amp;</span>ColumnType)&gt; {
<a href=#422 id=422 data-nosnippet>422</a> <span class="self">self</span>.iter_names().zip(<span class="self">self</span>.iter_types())
<a href=#423 id=423 data-nosnippet>423</a> }
<a href=#424 id=424 data-nosnippet>424</a>
<a href=#425 id=425 data-nosnippet>425</a> <span class="doccomment">/// Returns an iterator over the types of the columns in this relation.
<a href=#426 id=426 data-nosnippet>426</a> </span><span class="kw">pub fn </span>iter_types(<span class="kw-2">&amp;</span><span class="self">self</span>) -&gt; <span class="kw">impl </span>Iterator&lt;Item = <span class="kw-2">&amp;</span>ColumnType&gt; {
<a href=#427 id=427 data-nosnippet>427</a> <span class="self">self</span>.typ.column_types.iter()
<a href=#428 id=428 data-nosnippet>428</a> }
<a href=#429 id=429 data-nosnippet>429</a>
<a href=#430 id=430 data-nosnippet>430</a> <span class="doccomment">/// Returns an iterator over the names of the columns in this relation.
<a href=#431 id=431 data-nosnippet>431</a> </span><span class="kw">pub fn </span>iter_names(<span class="kw-2">&amp;</span><span class="self">self</span>) -&gt; <span class="kw">impl </span>Iterator&lt;Item = <span class="kw-2">&amp;</span><span class="prelude-ty">Option</span>&lt;ColumnName&gt;&gt; {
<a href=#432 id=432 data-nosnippet>432</a> <span class="self">self</span>.names.iter()
<a href=#433 id=433 data-nosnippet>433</a> }
<a href=#434 id=434 data-nosnippet>434</a>
<a href=#435 id=435 data-nosnippet>435</a> <span class="doccomment">/// Finds a column by name.
<a href=#436 id=436 data-nosnippet>436</a> ///
<a href=#437 id=437 data-nosnippet>437</a> /// Returns the index and type of the column named `name`. If no column with
<a href=#438 id=438 data-nosnippet>438</a> /// the specified name exists, returns `None`. If multiple columns have the
<a href=#439 id=439 data-nosnippet>439</a> /// specified name, the leftmost column is returned.
<a href=#440 id=440 data-nosnippet>440</a> </span><span class="kw">pub fn </span>get_by_name(<span class="kw-2">&amp;</span><span class="self">self</span>, name: <span class="kw-2">&amp;</span>ColumnName) -&gt; <span class="prelude-ty">Option</span>&lt;(usize, <span class="kw-2">&amp;</span>ColumnType)&gt; {
<a href=#441 id=441 data-nosnippet>441</a> <span class="self">self</span>.iter_names()
<a href=#442 id=442 data-nosnippet>442</a> .position(|n| n.as_ref() == <span class="prelude-val">Some</span>(name))
<a href=#443 id=443 data-nosnippet>443</a> .map(|i| (i, <span class="kw-2">&amp;</span><span class="self">self</span>.typ.column_types[i]))
<a href=#444 id=444 data-nosnippet>444</a> }
<a href=#445 id=445 data-nosnippet>445</a>
<a href=#446 id=446 data-nosnippet>446</a> <span class="doccomment">/// Gets the name of the `i`th column.
<a href=#447 id=447 data-nosnippet>447</a> ///
<a href=#448 id=448 data-nosnippet>448</a> /// # Panics
<a href=#449 id=449 data-nosnippet>449</a> ///
<a href=#450 id=450 data-nosnippet>450</a> /// Panics if `i` is not a valid column index.
<a href=#451 id=451 data-nosnippet>451</a> </span><span class="kw">pub fn </span>get_name(<span class="kw-2">&amp;</span><span class="self">self</span>, i: usize) -&gt; <span class="kw-2">&amp;</span><span class="prelude-ty">Option</span>&lt;ColumnName&gt; {
<a href=#452 id=452 data-nosnippet>452</a> <span class="kw-2">&amp;</span><span class="self">self</span>.names[i]
<a href=#453 id=453 data-nosnippet>453</a> }
<a href=#454 id=454 data-nosnippet>454</a>}
<a href=#455 id=455 data-nosnippet>455</a>
<a href=#456 id=456 data-nosnippet>456</a><span class="doccomment">/// The name of a column in a [`RelationDesc`].
<a href=#457 id=457 data-nosnippet>457</a></span><span class="kw">pub type </span>ColumnName = String;
</code></pre></div></section></main></body></html>
+564
View File
@@ -0,0 +1,564 @@
<!DOCTYPE html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1.0"><meta name="generator" content="rustdoc"><meta name="description" content="Source of the Rust file `src/flow/src/server.rs`."><title>server.rs - source</title><script>if(window.location.protocol!=="file:")document.head.insertAdjacentHTML("beforeend","SourceSerif4-Regular-6b053e98.ttf.woff2,FiraSans-Italic-81dc35de.woff2,FiraSans-Regular-0fe48ade.woff2,FiraSans-MediumItalic-ccf7e434.woff2,FiraSans-Medium-e1aa3f0a.woff2,SourceCodePro-Regular-8badfe75.ttf.woff2,SourceCodePro-Semibold-aa29a496.ttf.woff2".split(",").map(f=>`<link rel="preload" as="font" type="font/woff2"href="../../static.files/${f}">`).join(""))</script><link rel="stylesheet" href="../../static.files/normalize-9960930a.css"><link rel="stylesheet" href="../../static.files/rustdoc-17e0aaed.css"><meta name="rustdoc-vars" data-root-path="../../" data-static-root-path="../../static.files/" data-current-crate="flow" data-themes="" data-resource-suffix="" data-rustdoc-version="1.96.1 (31fca3adb 2026-06-26)" data-channel="1.96.1" data-search-js="search-b5634cc7.js" data-stringdex-js="stringdex-2da4960a.js" data-settings-js="settings-170eb4bf.js" ><script src="../../static.files/storage-41dd4d93.js"></script><script defer src="../../static.files/src-script-813739b1.js"></script><script defer src="../../src-files.js"></script><script defer src="../../static.files/main-5013f961.js"></script><noscript><link rel="stylesheet" href="../../static.files/noscript-f7c3ffd8.css"></noscript><link rel="alternate icon" type="image/png" href="../../static.files/favicon-32x32-eab170b8.png"><link rel="icon" type="image/svg+xml" href="../../static.files/favicon-044be391.svg"></head><body class="rustdoc src"><a class="skip-main-content" href="#main-content">Skip to main content</a><!--[if lte IE 11]><div class="warning">This old browser is unsupported and will most likely display funky things.</div><![endif]--><nav class="sidebar"><div class="src-sidebar-title"><h2>Files</h2></div></nav><div class="sidebar-resizer" title="Drag to resize sidebar"></div><main><section id="main-content" class="content" tabindex="-1"><div class="main-heading"><h1><div class="sub-heading">flow/</div>server.rs</h1><rustdoc-toolbar></rustdoc-toolbar></div><div class="example-wrap digits-3"><pre class="rust"><code><a href=#1 id=1 data-nosnippet>1</a><span class="comment">// Copyright 2023 Greptime Team
<a href=#2 id=2 data-nosnippet>2</a>//
<a href=#3 id=3 data-nosnippet>3</a>// Licensed under the Apache License, Version 2.0 (the "License");
<a href=#4 id=4 data-nosnippet>4</a>// you may not use this file except in compliance with the License.
<a href=#5 id=5 data-nosnippet>5</a>// You may obtain a copy of the License at
<a href=#6 id=6 data-nosnippet>6</a>//
<a href=#7 id=7 data-nosnippet>7</a>// http://www.apache.org/licenses/LICENSE-2.0
<a href=#8 id=8 data-nosnippet>8</a>//
<a href=#9 id=9 data-nosnippet>9</a>// Unless required by applicable law or agreed to in writing, software
<a href=#10 id=10 data-nosnippet>10</a>// distributed under the License is distributed on an "AS IS" BASIS,
<a href=#11 id=11 data-nosnippet>11</a>// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
<a href=#12 id=12 data-nosnippet>12</a>// See the License for the specific language governing permissions and
<a href=#13 id=13 data-nosnippet>13</a>// limitations under the License.
<a href=#14 id=14 data-nosnippet>14</a>
<a href=#15 id=15 data-nosnippet>15</a></span><span class="doccomment">//! Implementation of grpc service for flow node
<a href=#16 id=16 data-nosnippet>16</a>
<a href=#17 id=17 data-nosnippet>17</a></span><span class="kw">use </span>std::net::SocketAddr;
<a href=#18 id=18 data-nosnippet>18</a><span class="kw">use </span>std::sync::Arc;
<a href=#19 id=19 data-nosnippet>19</a>
<a href=#20 id=20 data-nosnippet>20</a><span class="kw">use </span>api::v1::flow::DirtyWindowRequests;
<a href=#21 id=21 data-nosnippet>21</a><span class="kw">use </span>catalog::CatalogManagerRef;
<a href=#22 id=22 data-nosnippet>22</a><span class="kw">use </span>common_base::Plugins;
<a href=#23 id=23 data-nosnippet>23</a><span class="kw">use </span>common_error::ext::BoxedError;
<a href=#24 id=24 data-nosnippet>24</a><span class="kw">use </span>common_meta::key::TableMetadataManagerRef;
<a href=#25 id=25 data-nosnippet>25</a><span class="kw">use </span>common_meta::key::flow::FlowMetadataManagerRef;
<a href=#26 id=26 data-nosnippet>26</a><span class="kw">use </span>common_meta::node_manager::Flownode;
<a href=#27 id=27 data-nosnippet>27</a><span class="kw">use </span>futures::TryStreamExt;
<a href=#28 id=28 data-nosnippet>28</a><span class="kw">use </span>greptime_proto::v1::flow::{FlowRequest, FlowResponse, InsertRequests, flow_server};
<a href=#29 id=29 data-nosnippet>29</a><span class="kw">use </span>itertools::Itertools;
<a href=#30 id=30 data-nosnippet>30</a><span class="kw">use </span>query::QueryEngineFactory;
<a href=#31 id=31 data-nosnippet>31</a><span class="kw">use </span>servers::add_service;
<a href=#32 id=32 data-nosnippet>32</a><span class="kw">use </span>servers::grpc::builder::GrpcServerBuilder;
<a href=#33 id=33 data-nosnippet>33</a><span class="kw">use </span>servers::grpc::{GrpcServer, GrpcServerConfig};
<a href=#34 id=34 data-nosnippet>34</a><span class="kw">use </span>servers::http::HttpServerBuilder;
<a href=#35 id=35 data-nosnippet>35</a><span class="kw">use </span>servers::metrics_handler::MetricsHandler;
<a href=#36 id=36 data-nosnippet>36</a><span class="kw">use </span>servers::server::{ServerHandler, ServerHandlers};
<a href=#37 id=37 data-nosnippet>37</a><span class="kw">use </span>snafu::ResultExt;
<a href=#38 id=38 data-nosnippet>38</a><span class="kw">use </span>tonic::codec::CompressionEncoding;
<a href=#39 id=39 data-nosnippet>39</a><span class="kw">use </span>tonic::{Request, Response, Status};
<a href=#40 id=40 data-nosnippet>40</a>
<a href=#41 id=41 data-nosnippet>41</a><span class="kw">use </span><span class="kw">crate</span>::adapter::flownode_impl::{FlowDualEngine, FlowDualEngineRef};
<a href=#42 id=42 data-nosnippet>42</a><span class="kw">use </span><span class="kw">crate</span>::batching_mode::engine::BatchingEngine;
<a href=#43 id=43 data-nosnippet>43</a><span class="kw">use </span><span class="kw">crate</span>::error::{
<a href=#44 id=44 data-nosnippet>44</a> DatafusionSnafu, ExternalSnafu, ListFlowsSnafu, ParseAddrSnafu, ShutdownServerSnafu,
<a href=#45 id=45 data-nosnippet>45</a> StartServerSnafu, to_status_with_last_err,
<a href=#46 id=46 data-nosnippet>46</a>};
<a href=#47 id=47 data-nosnippet>47</a><span class="kw">use </span><span class="kw">crate</span>::heartbeat::HeartbeatTask;
<a href=#48 id=48 data-nosnippet>48</a><span class="kw">use </span><span class="kw">crate</span>::metrics::{METRIC_FLOW_PROCESSING_TIME, METRIC_FLOW_ROWS};
<a href=#49 id=49 data-nosnippet>49</a><span class="kw">use </span><span class="kw">crate</span>::utils::{SizeReportSender, StateReportHandler};
<a href=#50 id=50 data-nosnippet>50</a><span class="kw">use </span>crate::{BatchingExecutionFactory, Error, FlownodeOptions, FrontendClient, StreamingEngine};
<a href=#51 id=51 data-nosnippet>51</a>
<a href=#52 id=52 data-nosnippet>52</a><span class="kw">pub const </span>FLOW_NODE_SERVER_NAME: <span class="kw-2">&amp;</span>str = <span class="string">"FLOW_NODE_SERVER"</span>;
<a href=#53 id=53 data-nosnippet>53</a><span class="doccomment">/// wrapping flow node manager to avoid orphan rule with Arc&lt;...&gt;
<a href=#54 id=54 data-nosnippet>54</a></span><span class="attr">#[derive(Clone)]
<a href=#55 id=55 data-nosnippet>55</a></span><span class="kw">pub struct </span>FlowService {
<a href=#56 id=56 data-nosnippet>56</a> <span class="kw">pub </span>dual_engine: FlowDualEngineRef,
<a href=#57 id=57 data-nosnippet>57</a>}
<a href=#58 id=58 data-nosnippet>58</a>
<a href=#59 id=59 data-nosnippet>59</a><span class="kw">impl </span>FlowService {
<a href=#60 id=60 data-nosnippet>60</a> <span class="kw">pub fn </span>new(manager: FlowDualEngineRef) -&gt; <span class="self">Self </span>{
<a href=#61 id=61 data-nosnippet>61</a> <span class="self">Self </span>{
<a href=#62 id=62 data-nosnippet>62</a> dual_engine: manager,
<a href=#63 id=63 data-nosnippet>63</a> }
<a href=#64 id=64 data-nosnippet>64</a> }
<a href=#65 id=65 data-nosnippet>65</a>}
<a href=#66 id=66 data-nosnippet>66</a>
<a href=#67 id=67 data-nosnippet>67</a><span class="attr">#[async_trait::async_trait]
<a href=#68 id=68 data-nosnippet>68</a></span><span class="kw">impl </span>flow_server::Flow <span class="kw">for </span>FlowService {
<a href=#69 id=69 data-nosnippet>69</a> <span class="kw">async fn </span>handle_create_remove(
<a href=#70 id=70 data-nosnippet>70</a> <span class="kw-2">&amp;</span><span class="self">self</span>,
<a href=#71 id=71 data-nosnippet>71</a> request: Request&lt;FlowRequest&gt;,
<a href=#72 id=72 data-nosnippet>72</a> ) -&gt; <span class="prelude-ty">Result</span>&lt;Response&lt;FlowResponse&gt;, Status&gt; {
<a href=#73 id=73 data-nosnippet>73</a> <span class="kw">let </span>_timer = METRIC_FLOW_PROCESSING_TIME
<a href=#74 id=74 data-nosnippet>74</a> .with_label_values(<span class="kw-2">&amp;</span>[<span class="string">"ddl"</span>])
<a href=#75 id=75 data-nosnippet>75</a> .start_timer();
<a href=#76 id=76 data-nosnippet>76</a>
<a href=#77 id=77 data-nosnippet>77</a> <span class="kw">let </span>request = request.into_inner();
<a href=#78 id=78 data-nosnippet>78</a> <span class="self">self</span>.dual_engine
<a href=#79 id=79 data-nosnippet>79</a> .handle(request)
<a href=#80 id=80 data-nosnippet>80</a> .<span class="kw">await
<a href=#81 id=81 data-nosnippet>81</a> </span>.map_err(|err| {
<a href=#82 id=82 data-nosnippet>82</a> <span class="macro">common_telemetry::error!</span>(err; <span class="string">"Failed to handle flow request"</span>);
<a href=#83 id=83 data-nosnippet>83</a> err
<a href=#84 id=84 data-nosnippet>84</a> })
<a href=#85 id=85 data-nosnippet>85</a> .map(Response::new)
<a href=#86 id=86 data-nosnippet>86</a> .map_err(to_status_with_last_err)
<a href=#87 id=87 data-nosnippet>87</a> }
<a href=#88 id=88 data-nosnippet>88</a>
<a href=#89 id=89 data-nosnippet>89</a> <span class="kw">async fn </span>handle_mirror_request(
<a href=#90 id=90 data-nosnippet>90</a> <span class="kw-2">&amp;</span><span class="self">self</span>,
<a href=#91 id=91 data-nosnippet>91</a> request: Request&lt;InsertRequests&gt;,
<a href=#92 id=92 data-nosnippet>92</a> ) -&gt; <span class="prelude-ty">Result</span>&lt;Response&lt;FlowResponse&gt;, Status&gt; {
<a href=#93 id=93 data-nosnippet>93</a> <span class="kw">let </span>_timer = METRIC_FLOW_PROCESSING_TIME
<a href=#94 id=94 data-nosnippet>94</a> .with_label_values(<span class="kw-2">&amp;</span>[<span class="string">"insert"</span>])
<a href=#95 id=95 data-nosnippet>95</a> .start_timer();
<a href=#96 id=96 data-nosnippet>96</a>
<a href=#97 id=97 data-nosnippet>97</a> <span class="kw">let </span>request = request.into_inner();
<a href=#98 id=98 data-nosnippet>98</a> <span class="comment">// TODO(discord9): fix protobuf import order shenanigans to remove this duplicated define
<a href=#99 id=99 data-nosnippet>99</a> </span><span class="kw">let </span><span class="kw-2">mut </span>row_count = <span class="number">0</span>;
<a href=#100 id=100 data-nosnippet>100</a> <span class="kw">let </span>request = api::v1::region::InsertRequests {
<a href=#101 id=101 data-nosnippet>101</a> requests: request
<a href=#102 id=102 data-nosnippet>102</a> .requests
<a href=#103 id=103 data-nosnippet>103</a> .into_iter()
<a href=#104 id=104 data-nosnippet>104</a> .map(|insert| {
<a href=#105 id=105 data-nosnippet>105</a> insert.rows.as_ref().inspect(|x| row_count += x.rows.len());
<a href=#106 id=106 data-nosnippet>106</a> api::v1::region::InsertRequest {
<a href=#107 id=107 data-nosnippet>107</a> region_id: insert.region_id,
<a href=#108 id=108 data-nosnippet>108</a> rows: insert.rows,
<a href=#109 id=109 data-nosnippet>109</a> skip_wal: <span class="bool-val">false</span>,
<a href=#110 id=110 data-nosnippet>110</a> partition_expr_version: insert.partition_expr_version,
<a href=#111 id=111 data-nosnippet>111</a> }
<a href=#112 id=112 data-nosnippet>112</a> })
<a href=#113 id=113 data-nosnippet>113</a> .collect_vec(),
<a href=#114 id=114 data-nosnippet>114</a> };
<a href=#115 id=115 data-nosnippet>115</a>
<a href=#116 id=116 data-nosnippet>116</a> METRIC_FLOW_ROWS
<a href=#117 id=117 data-nosnippet>117</a> .with_label_values(<span class="kw-2">&amp;</span>[<span class="string">"in"</span>])
<a href=#118 id=118 data-nosnippet>118</a> .inc_by(row_count <span class="kw">as </span>u64);
<a href=#119 id=119 data-nosnippet>119</a>
<a href=#120 id=120 data-nosnippet>120</a> <span class="self">self</span>.dual_engine
<a href=#121 id=121 data-nosnippet>121</a> .handle_inserts(request)
<a href=#122 id=122 data-nosnippet>122</a> .<span class="kw">await
<a href=#123 id=123 data-nosnippet>123</a> </span>.map(Response::new)
<a href=#124 id=124 data-nosnippet>124</a> .map_err(to_status_with_last_err)
<a href=#125 id=125 data-nosnippet>125</a> }
<a href=#126 id=126 data-nosnippet>126</a>
<a href=#127 id=127 data-nosnippet>127</a> <span class="kw">async fn </span>handle_mark_dirty_time_window(
<a href=#128 id=128 data-nosnippet>128</a> <span class="kw-2">&amp;</span><span class="self">self</span>,
<a href=#129 id=129 data-nosnippet>129</a> reqs: Request&lt;DirtyWindowRequests&gt;,
<a href=#130 id=130 data-nosnippet>130</a> ) -&gt; <span class="prelude-ty">Result</span>&lt;Response&lt;FlowResponse&gt;, Status&gt; {
<a href=#131 id=131 data-nosnippet>131</a> <span class="self">self</span>.dual_engine
<a href=#132 id=132 data-nosnippet>132</a> .handle_mark_window_dirty(reqs.into_inner())
<a href=#133 id=133 data-nosnippet>133</a> .<span class="kw">await
<a href=#134 id=134 data-nosnippet>134</a> </span>.map(Response::new)
<a href=#135 id=135 data-nosnippet>135</a> .map_err(to_status_with_last_err)
<a href=#136 id=136 data-nosnippet>136</a> }
<a href=#137 id=137 data-nosnippet>137</a>}
<a href=#138 id=138 data-nosnippet>138</a>
<a href=#139 id=139 data-nosnippet>139</a><span class="attr">#[derive(Clone)]
<a href=#140 id=140 data-nosnippet>140</a></span><span class="kw">pub struct </span>FlownodeServer {
<a href=#141 id=141 data-nosnippet>141</a> inner: Arc&lt;FlownodeServerInner&gt;,
<a href=#142 id=142 data-nosnippet>142</a>}
<a href=#143 id=143 data-nosnippet>143</a>
<a href=#144 id=144 data-nosnippet>144</a><span class="doccomment">/// FlownodeServerInner is the inner state of FlownodeServer,
<a href=#145 id=145 data-nosnippet>145</a>/// this struct mostly useful for construct/start and stop the
<a href=#146 id=146 data-nosnippet>146</a>/// flow node server
<a href=#147 id=147 data-nosnippet>147</a></span><span class="kw">struct </span>FlownodeServerInner {
<a href=#148 id=148 data-nosnippet>148</a> flow_service: FlowService,
<a href=#149 id=149 data-nosnippet>149</a>}
<a href=#150 id=150 data-nosnippet>150</a>
<a href=#151 id=151 data-nosnippet>151</a><span class="kw">impl </span>FlownodeServer {
<a href=#152 id=152 data-nosnippet>152</a> <span class="kw">pub fn </span>new(flow_service: FlowService) -&gt; <span class="self">Self </span>{
<a href=#153 id=153 data-nosnippet>153</a> <span class="self">Self </span>{
<a href=#154 id=154 data-nosnippet>154</a> inner: Arc::new(FlownodeServerInner { flow_service }),
<a href=#155 id=155 data-nosnippet>155</a> }
<a href=#156 id=156 data-nosnippet>156</a> }
<a href=#157 id=157 data-nosnippet>157</a>}
<a href=#158 id=158 data-nosnippet>158</a>
<a href=#159 id=159 data-nosnippet>159</a><span class="kw">impl </span>FlownodeServer {
<a href=#160 id=160 data-nosnippet>160</a> <span class="kw">pub fn </span>create_flow_service(<span class="kw-2">&amp;</span><span class="self">self</span>) -&gt; flow_server::FlowServer&lt;<span class="kw">impl </span>flow_server::Flow&gt; {
<a href=#161 id=161 data-nosnippet>161</a> flow_server::FlowServer::new(<span class="self">self</span>.inner.flow_service.clone())
<a href=#162 id=162 data-nosnippet>162</a> .accept_compressed(CompressionEncoding::Gzip)
<a href=#163 id=163 data-nosnippet>163</a> .send_compressed(CompressionEncoding::Gzip)
<a href=#164 id=164 data-nosnippet>164</a> .accept_compressed(CompressionEncoding::Zstd)
<a href=#165 id=165 data-nosnippet>165</a> .send_compressed(CompressionEncoding::Zstd)
<a href=#166 id=166 data-nosnippet>166</a> }
<a href=#167 id=167 data-nosnippet>167</a>}
<a href=#168 id=168 data-nosnippet>168</a>
<a href=#169 id=169 data-nosnippet>169</a><span class="doccomment">/// The flownode server instance.
<a href=#170 id=170 data-nosnippet>170</a></span><span class="kw">pub struct </span>FlownodeInstance {
<a href=#171 id=171 data-nosnippet>171</a> flownode_server: FlownodeServer,
<a href=#172 id=172 data-nosnippet>172</a> services: ServerHandlers,
<a href=#173 id=173 data-nosnippet>173</a> heartbeat_task: <span class="prelude-ty">Option</span>&lt;HeartbeatTask&gt;,
<a href=#174 id=174 data-nosnippet>174</a> state_report_task: <span class="prelude-ty">Option</span>&lt;common_runtime::JoinHandle&lt;()&gt;&gt;,
<a href=#175 id=175 data-nosnippet>175</a> consistent_check_task_started: bool,
<a href=#176 id=176 data-nosnippet>176</a>}
<a href=#177 id=177 data-nosnippet>177</a>
<a href=#178 id=178 data-nosnippet>178</a><span class="kw">impl </span>FlownodeInstance {
<a href=#179 id=179 data-nosnippet>179</a> <span class="kw">pub async fn </span>start(<span class="kw-2">&amp;mut </span><span class="self">self</span>) -&gt; <span class="prelude-ty">Result</span>&lt;(), <span class="kw">crate</span>::Error&gt; {
<a href=#180 id=180 data-nosnippet>180</a> <span class="kw">if let </span><span class="prelude-val">Some</span>(task) = <span class="kw-2">&amp;</span><span class="self">self</span>.heartbeat_task {
<a href=#181 id=181 data-nosnippet>181</a> task.start().<span class="kw">await</span><span class="question-mark">?</span>;
<a href=#182 id=182 data-nosnippet>182</a> }
<a href=#183 id=183 data-nosnippet>183</a>
<a href=#184 id=184 data-nosnippet>184</a> <span class="kw">let </span>engine = <span class="self">self</span>.flow_engine();
<a href=#185 id=185 data-nosnippet>185</a> <span class="comment">// The state-report task owns the only report receiver, so keep it alive
<a href=#186 id=186 data-nosnippet>186</a> // across an ordinary stop/start cycle of this instance.
<a href=#187 id=187 data-nosnippet>187</a> </span><span class="kw">if </span><span class="self">self</span>.state_report_task.is_none() {
<a href=#188 id=188 data-nosnippet>188</a> <span class="self">self</span>.state_report_task = engine.clone().start_state_report_task().<span class="kw">await</span>;
<a href=#189 id=189 data-nosnippet>189</a> }
<a href=#190 id=190 data-nosnippet>190</a> <span class="kw">if let </span><span class="prelude-val">Err</span>(err) = engine.start_flow_consistent_check_task().<span class="kw">await </span>{
<a href=#191 id=191 data-nosnippet>191</a> <span class="self">self</span>.rollback_background_tasks().<span class="kw">await</span>;
<a href=#192 id=192 data-nosnippet>192</a> <span class="kw">return </span><span class="prelude-val">Err</span>(err);
<a href=#193 id=193 data-nosnippet>193</a> }
<a href=#194 id=194 data-nosnippet>194</a> <span class="self">self</span>.consistent_check_task_started = <span class="bool-val">true</span>;
<a href=#195 id=195 data-nosnippet>195</a>
<a href=#196 id=196 data-nosnippet>196</a> <span class="kw">if let </span><span class="prelude-val">Err</span>(err) = <span class="self">self</span>.services.start_all().<span class="kw">await</span>.context(StartServerSnafu) {
<a href=#197 id=197 data-nosnippet>197</a> <span class="self">self</span>.rollback_background_tasks().<span class="kw">await</span>;
<a href=#198 id=198 data-nosnippet>198</a> <span class="kw">return </span><span class="prelude-val">Err</span>(err);
<a href=#199 id=199 data-nosnippet>199</a> }
<a href=#200 id=200 data-nosnippet>200</a>
<a href=#201 id=201 data-nosnippet>201</a> <span class="prelude-val">Ok</span>(())
<a href=#202 id=202 data-nosnippet>202</a> }
<a href=#203 id=203 data-nosnippet>203</a> <span class="kw">pub async fn </span>shutdown(<span class="kw-2">&amp;mut </span><span class="self">self</span>) -&gt; <span class="prelude-ty">Result</span>&lt;(), Error&gt; {
<a href=#204 id=204 data-nosnippet>204</a> <span class="kw">let </span>services_result = <span class="self">self
<a href=#205 id=205 data-nosnippet>205</a> </span>.services
<a href=#206 id=206 data-nosnippet>206</a> .shutdown_all()
<a href=#207 id=207 data-nosnippet>207</a> .<span class="kw">await
<a href=#208 id=208 data-nosnippet>208</a> </span>.context(ShutdownServerSnafu);
<a href=#209 id=209 data-nosnippet>209</a> <span class="kw">let </span>tasks_result = <span class="self">self</span>.stop_background_tasks().<span class="kw">await</span>;
<a href=#210 id=210 data-nosnippet>210</a>
<a href=#211 id=211 data-nosnippet>211</a> services_result<span class="question-mark">?</span>;
<a href=#212 id=212 data-nosnippet>212</a> tasks_result<span class="question-mark">?</span>;
<a href=#213 id=213 data-nosnippet>213</a> <span class="prelude-val">Ok</span>(())
<a href=#214 id=214 data-nosnippet>214</a> }
<a href=#215 id=215 data-nosnippet>215</a>
<a href=#216 id=216 data-nosnippet>216</a> <span class="kw">async fn </span>stop_background_tasks(<span class="kw-2">&amp;mut </span><span class="self">self</span>) -&gt; <span class="prelude-ty">Result</span>&lt;(), Error&gt; {
<a href=#217 id=217 data-nosnippet>217</a> <span class="kw">let </span>check_result = <span class="kw">if </span><span class="self">self</span>.consistent_check_task_started {
<a href=#218 id=218 data-nosnippet>218</a> <span class="self">self</span>.consistent_check_task_started = <span class="bool-val">false</span>;
<a href=#219 id=219 data-nosnippet>219</a> <span class="self">self</span>.flow_engine().stop_flow_consistent_check_task().<span class="kw">await
<a href=#220 id=220 data-nosnippet>220</a> </span>} <span class="kw">else </span>{
<a href=#221 id=221 data-nosnippet>221</a> <span class="prelude-val">Ok</span>(())
<a href=#222 id=222 data-nosnippet>222</a> };
<a href=#223 id=223 data-nosnippet>223</a>
<a href=#224 id=224 data-nosnippet>224</a> <span class="kw">if let </span><span class="prelude-val">Some</span>(task) = <span class="kw-2">&amp;</span><span class="self">self</span>.heartbeat_task {
<a href=#225 id=225 data-nosnippet>225</a> task.shutdown();
<a href=#226 id=226 data-nosnippet>226</a> }
<a href=#227 id=227 data-nosnippet>227</a>
<a href=#228 id=228 data-nosnippet>228</a> check_result
<a href=#229 id=229 data-nosnippet>229</a> }
<a href=#230 id=230 data-nosnippet>230</a>
<a href=#231 id=231 data-nosnippet>231</a> <span class="kw">async fn </span>rollback_background_tasks(<span class="kw-2">&amp;mut </span><span class="self">self</span>) {
<a href=#232 id=232 data-nosnippet>232</a> <span class="kw">if let </span><span class="prelude-val">Err</span>(err) = <span class="self">self</span>.stop_background_tasks().<span class="kw">await </span>{
<a href=#233 id=233 data-nosnippet>233</a> <span class="macro">common_telemetry::error!</span>(err; <span class="string">"Failed to roll back flownode background tasks"</span>);
<a href=#234 id=234 data-nosnippet>234</a> }
<a href=#235 id=235 data-nosnippet>235</a> }
<a href=#236 id=236 data-nosnippet>236</a>
<a href=#237 id=237 data-nosnippet>237</a> <span class="kw">pub fn </span>flownode_server(<span class="kw-2">&amp;</span><span class="self">self</span>) -&gt; <span class="kw-2">&amp;</span>FlownodeServer {
<a href=#238 id=238 data-nosnippet>238</a> <span class="kw-2">&amp;</span><span class="self">self</span>.flownode_server
<a href=#239 id=239 data-nosnippet>239</a> }
<a href=#240 id=240 data-nosnippet>240</a>
<a href=#241 id=241 data-nosnippet>241</a> <span class="kw">pub fn </span>flow_engine(<span class="kw-2">&amp;</span><span class="self">self</span>) -&gt; FlowDualEngineRef {
<a href=#242 id=242 data-nosnippet>242</a> <span class="self">self</span>.flownode_server.inner.flow_service.dual_engine.clone()
<a href=#243 id=243 data-nosnippet>243</a> }
<a href=#244 id=244 data-nosnippet>244</a>
<a href=#245 id=245 data-nosnippet>245</a> <span class="kw">pub fn </span>setup_services(<span class="kw-2">&amp;mut </span><span class="self">self</span>, services: ServerHandlers) {
<a href=#246 id=246 data-nosnippet>246</a> <span class="self">self</span>.services = services;
<a href=#247 id=247 data-nosnippet>247</a> }
<a href=#248 id=248 data-nosnippet>248</a>}
<a href=#249 id=249 data-nosnippet>249</a>
<a href=#250 id=250 data-nosnippet>250</a><span class="doccomment">/// [`FlownodeInstance`] Builder
<a href=#251 id=251 data-nosnippet>251</a></span><span class="kw">pub struct </span>FlownodeBuilder {
<a href=#252 id=252 data-nosnippet>252</a> opts: FlownodeOptions,
<a href=#253 id=253 data-nosnippet>253</a> plugins: Plugins,
<a href=#254 id=254 data-nosnippet>254</a> table_meta: TableMetadataManagerRef,
<a href=#255 id=255 data-nosnippet>255</a> catalog_manager: CatalogManagerRef,
<a href=#256 id=256 data-nosnippet>256</a> flow_metadata_manager: FlowMetadataManagerRef,
<a href=#257 id=257 data-nosnippet>257</a> heartbeat_task: <span class="prelude-ty">Option</span>&lt;HeartbeatTask&gt;,
<a href=#258 id=258 data-nosnippet>258</a> <span class="doccomment">/// receive a oneshot sender to send state size report
<a href=#259 id=259 data-nosnippet>259</a> </span>state_report_handler: <span class="prelude-ty">Option</span>&lt;StateReportHandler&gt;,
<a href=#260 id=260 data-nosnippet>260</a> frontend_client: Arc&lt;FrontendClient&gt;,
<a href=#261 id=261 data-nosnippet>261</a>}
<a href=#262 id=262 data-nosnippet>262</a>
<a href=#263 id=263 data-nosnippet>263</a><span class="kw">impl </span>FlownodeBuilder {
<a href=#264 id=264 data-nosnippet>264</a> <span class="doccomment">/// init flownode builder
<a href=#265 id=265 data-nosnippet>265</a> </span><span class="kw">pub fn </span>new(
<a href=#266 id=266 data-nosnippet>266</a> opts: FlownodeOptions,
<a href=#267 id=267 data-nosnippet>267</a> plugins: Plugins,
<a href=#268 id=268 data-nosnippet>268</a> table_meta: TableMetadataManagerRef,
<a href=#269 id=269 data-nosnippet>269</a> catalog_manager: CatalogManagerRef,
<a href=#270 id=270 data-nosnippet>270</a> flow_metadata_manager: FlowMetadataManagerRef,
<a href=#271 id=271 data-nosnippet>271</a> frontend_client: Arc&lt;FrontendClient&gt;,
<a href=#272 id=272 data-nosnippet>272</a> ) -&gt; <span class="self">Self </span>{
<a href=#273 id=273 data-nosnippet>273</a> <span class="self">Self </span>{
<a href=#274 id=274 data-nosnippet>274</a> opts,
<a href=#275 id=275 data-nosnippet>275</a> plugins,
<a href=#276 id=276 data-nosnippet>276</a> table_meta,
<a href=#277 id=277 data-nosnippet>277</a> catalog_manager,
<a href=#278 id=278 data-nosnippet>278</a> flow_metadata_manager,
<a href=#279 id=279 data-nosnippet>279</a> heartbeat_task: <span class="prelude-val">None</span>,
<a href=#280 id=280 data-nosnippet>280</a> state_report_handler: <span class="prelude-val">None</span>,
<a href=#281 id=281 data-nosnippet>281</a> frontend_client,
<a href=#282 id=282 data-nosnippet>282</a> }
<a href=#283 id=283 data-nosnippet>283</a> }
<a href=#284 id=284 data-nosnippet>284</a>
<a href=#285 id=285 data-nosnippet>285</a> <span class="kw">pub fn </span>with_heartbeat_task(<span class="self">self</span>, heartbeat_task: HeartbeatTask) -&gt; <span class="self">Self </span>{
<a href=#286 id=286 data-nosnippet>286</a> <span class="kw">let </span>(sender, receiver) = SizeReportSender::new();
<a href=#287 id=287 data-nosnippet>287</a> <span class="self">Self </span>{
<a href=#288 id=288 data-nosnippet>288</a> heartbeat_task: <span class="prelude-val">Some</span>(heartbeat_task.with_query_stat_size(sender)),
<a href=#289 id=289 data-nosnippet>289</a> state_report_handler: <span class="prelude-val">Some</span>(receiver),
<a href=#290 id=290 data-nosnippet>290</a> ..<span class="self">self
<a href=#291 id=291 data-nosnippet>291</a> </span>}
<a href=#292 id=292 data-nosnippet>292</a> }
<a href=#293 id=293 data-nosnippet>293</a>
<a href=#294 id=294 data-nosnippet>294</a> <span class="kw">pub fn </span>opts(<span class="kw-2">&amp;</span><span class="self">self</span>) -&gt; <span class="kw-2">&amp;</span>FlownodeOptions {
<a href=#295 id=295 data-nosnippet>295</a> <span class="kw-2">&amp;</span><span class="self">self</span>.opts
<a href=#296 id=296 data-nosnippet>296</a> }
<a href=#297 id=297 data-nosnippet>297</a>
<a href=#298 id=298 data-nosnippet>298</a> <span class="kw">pub fn </span>table_meta(<span class="kw-2">&amp;</span><span class="self">self</span>) -&gt; <span class="kw-2">&amp;</span>TableMetadataManagerRef {
<a href=#299 id=299 data-nosnippet>299</a> <span class="kw-2">&amp;</span><span class="self">self</span>.table_meta
<a href=#300 id=300 data-nosnippet>300</a> }
<a href=#301 id=301 data-nosnippet>301</a>
<a href=#302 id=302 data-nosnippet>302</a> <span class="kw">pub fn </span>catalog_manager(<span class="kw-2">&amp;</span><span class="self">self</span>) -&gt; <span class="kw-2">&amp;</span>CatalogManagerRef {
<a href=#303 id=303 data-nosnippet>303</a> <span class="kw-2">&amp;</span><span class="self">self</span>.catalog_manager
<a href=#304 id=304 data-nosnippet>304</a> }
<a href=#305 id=305 data-nosnippet>305</a>
<a href=#306 id=306 data-nosnippet>306</a> <span class="kw">pub fn </span>flow_metadata_manager(<span class="kw-2">&amp;</span><span class="self">self</span>) -&gt; <span class="kw-2">&amp;</span>FlowMetadataManagerRef {
<a href=#307 id=307 data-nosnippet>307</a> <span class="kw-2">&amp;</span><span class="self">self</span>.flow_metadata_manager
<a href=#308 id=308 data-nosnippet>308</a> }
<a href=#309 id=309 data-nosnippet>309</a>
<a href=#310 id=310 data-nosnippet>310</a> <span class="kw">pub fn </span>frontend_client(<span class="kw-2">&amp;</span><span class="self">self</span>) -&gt; <span class="kw-2">&amp;</span>Arc&lt;FrontendClient&gt; {
<a href=#311 id=311 data-nosnippet>311</a> <span class="kw-2">&amp;</span><span class="self">self</span>.frontend_client
<a href=#312 id=312 data-nosnippet>312</a> }
<a href=#313 id=313 data-nosnippet>313</a>
<a href=#314 id=314 data-nosnippet>314</a> <span class="kw">pub fn </span>set_plugins(<span class="kw-2">&amp;mut </span><span class="self">self</span>, plugins: Plugins) {
<a href=#315 id=315 data-nosnippet>315</a> <span class="self">self</span>.plugins = plugins;
<a href=#316 id=316 data-nosnippet>316</a> }
<a href=#317 id=317 data-nosnippet>317</a>
<a href=#318 id=318 data-nosnippet>318</a> <span class="kw">pub async fn </span>build(<span class="kw-2">mut </span><span class="self">self</span>) -&gt; <span class="prelude-ty">Result</span>&lt;FlownodeInstance, Error&gt; {
<a href=#319 id=319 data-nosnippet>319</a> <span class="comment">// TODO(discord9): does this query engine need those?
<a href=#320 id=320 data-nosnippet>320</a> </span><span class="kw">let </span>query_engine_factory = QueryEngineFactory::try_new_with_plugins(
<a href=#321 id=321 data-nosnippet>321</a> <span class="comment">// query engine in flownode is only used for translate plan with resolved table source.
<a href=#322 id=322 data-nosnippet>322</a> </span><span class="self">self</span>.catalog_manager.clone(),
<a href=#323 id=323 data-nosnippet>323</a> <span class="prelude-val">None</span>,
<a href=#324 id=324 data-nosnippet>324</a> <span class="prelude-val">None</span>,
<a href=#325 id=325 data-nosnippet>325</a> <span class="prelude-val">None</span>,
<a href=#326 id=326 data-nosnippet>326</a> <span class="prelude-val">None</span>,
<a href=#327 id=327 data-nosnippet>327</a> <span class="prelude-val">None</span>,
<a href=#328 id=328 data-nosnippet>328</a> <span class="bool-val">false</span>,
<a href=#329 id=329 data-nosnippet>329</a> Default::default(),
<a href=#330 id=330 data-nosnippet>330</a> <span class="self">self</span>.opts.query.clone(),
<a href=#331 id=331 data-nosnippet>331</a> )
<a href=#332 id=332 data-nosnippet>332</a> .context(DatafusionSnafu {
<a href=#333 id=333 data-nosnippet>333</a> context: <span class="string">"Failed to build query engine"</span>,
<a href=#334 id=334 data-nosnippet>334</a> })<span class="question-mark">?</span>;
<a href=#335 id=335 data-nosnippet>335</a> <span class="kw">let </span>manager = Arc::new(<span class="self">self</span>.build_manager(query_engine_factory.query_engine()));
<a href=#336 id=336 data-nosnippet>336</a> <span class="kw">let </span>batching = Arc::new(BatchingEngine::new_with_execution(
<a href=#337 id=337 data-nosnippet>337</a> <span class="self">self</span>.frontend_client.clone(),
<a href=#338 id=338 data-nosnippet>338</a> query_engine_factory.query_engine(),
<a href=#339 id=339 data-nosnippet>339</a> <span class="self">self</span>.flow_metadata_manager.clone(),
<a href=#340 id=340 data-nosnippet>340</a> <span class="self">self</span>.table_meta.clone(),
<a href=#341 id=341 data-nosnippet>341</a> <span class="self">self</span>.catalog_manager.clone(),
<a href=#342 id=342 data-nosnippet>342</a> <span class="self">self</span>.opts.flow.batching_mode.clone(),
<a href=#343 id=343 data-nosnippet>343</a> <span class="self">self</span>.plugins.get::&lt;Arc&lt;<span class="kw">dyn </span>BatchingExecutionFactory&gt;&gt;(),
<a href=#344 id=344 data-nosnippet>344</a> ));
<a href=#345 id=345 data-nosnippet>345</a> <span class="kw">let </span>dual = Arc::new(FlowDualEngine::new(
<a href=#346 id=346 data-nosnippet>346</a> manager.clone(),
<a href=#347 id=347 data-nosnippet>347</a> batching,
<a href=#348 id=348 data-nosnippet>348</a> <span class="self">self</span>.flow_metadata_manager.clone(),
<a href=#349 id=349 data-nosnippet>349</a> <span class="self">self</span>.catalog_manager.clone(),
<a href=#350 id=350 data-nosnippet>350</a> <span class="self">self</span>.plugins.clone(),
<a href=#351 id=351 data-nosnippet>351</a> ));
<a href=#352 id=352 data-nosnippet>352</a> <span class="kw">if let </span><span class="prelude-val">Some</span>(handler) = <span class="self">self</span>.state_report_handler.take() {
<a href=#353 id=353 data-nosnippet>353</a> dual.set_state_report_handler(handler).<span class="kw">await</span>;
<a href=#354 id=354 data-nosnippet>354</a> }
<a href=#355 id=355 data-nosnippet>355</a>
<a href=#356 id=356 data-nosnippet>356</a> <span class="kw">let </span>server = FlownodeServer::new(FlowService::new(dual));
<a href=#357 id=357 data-nosnippet>357</a>
<a href=#358 id=358 data-nosnippet>358</a> <span class="kw">let </span>heartbeat_task = <span class="self">self</span>.heartbeat_task;
<a href=#359 id=359 data-nosnippet>359</a>
<a href=#360 id=360 data-nosnippet>360</a> <span class="kw">let </span>instance = FlownodeInstance {
<a href=#361 id=361 data-nosnippet>361</a> flownode_server: server,
<a href=#362 id=362 data-nosnippet>362</a> services: ServerHandlers::default(),
<a href=#363 id=363 data-nosnippet>363</a> heartbeat_task,
<a href=#364 id=364 data-nosnippet>364</a> state_report_task: <span class="prelude-val">None</span>,
<a href=#365 id=365 data-nosnippet>365</a> consistent_check_task_started: <span class="bool-val">false</span>,
<a href=#366 id=366 data-nosnippet>366</a> };
<a href=#367 id=367 data-nosnippet>367</a> <span class="prelude-val">Ok</span>(instance)
<a href=#368 id=368 data-nosnippet>368</a> }
<a href=#369 id=369 data-nosnippet>369</a>
<a href=#370 id=370 data-nosnippet>370</a> <span class="kw">fn </span>build_manager(<span class="kw-2">&amp;</span><span class="self">self</span>, query_engine: Arc&lt;<span class="kw">dyn </span>query::QueryEngine&gt;) -&gt; StreamingEngine {
<a href=#371 id=371 data-nosnippet>371</a> StreamingEngine::new(
<a href=#372 id=372 data-nosnippet>372</a> <span class="self">self</span>.opts.node_id.map(|id| id <span class="kw">as </span>u32),
<a href=#373 id=373 data-nosnippet>373</a> query_engine,
<a href=#374 id=374 data-nosnippet>374</a> <span class="self">self</span>.table_meta.clone(),
<a href=#375 id=375 data-nosnippet>375</a> <span class="self">self</span>.frontend_client.clone(),
<a href=#376 id=376 data-nosnippet>376</a> )
<a href=#377 id=377 data-nosnippet>377</a> }
<a href=#378 id=378 data-nosnippet>378</a>}
<a href=#379 id=379 data-nosnippet>379</a>
<a href=#380 id=380 data-nosnippet>380</a><span class="doccomment">/// Useful in distributed mode
<a href=#381 id=381 data-nosnippet>381</a></span><span class="kw">pub struct </span>FlownodeServiceBuilder&lt;<span class="lifetime">'a</span>&gt; {
<a href=#382 id=382 data-nosnippet>382</a> opts: <span class="kw-2">&amp;</span><span class="lifetime">'a </span>FlownodeOptions,
<a href=#383 id=383 data-nosnippet>383</a> grpc_server: <span class="prelude-ty">Option</span>&lt;GrpcServer&gt;,
<a href=#384 id=384 data-nosnippet>384</a> enable_http_service: bool,
<a href=#385 id=385 data-nosnippet>385</a>}
<a href=#386 id=386 data-nosnippet>386</a>
<a href=#387 id=387 data-nosnippet>387</a><span class="kw">impl</span>&lt;<span class="lifetime">'a</span>&gt; FlownodeServiceBuilder&lt;<span class="lifetime">'a</span>&gt; {
<a href=#388 id=388 data-nosnippet>388</a> <span class="kw">pub fn </span>new(opts: <span class="kw-2">&amp;</span><span class="lifetime">'a </span>FlownodeOptions) -&gt; <span class="self">Self </span>{
<a href=#389 id=389 data-nosnippet>389</a> <span class="self">Self </span>{
<a href=#390 id=390 data-nosnippet>390</a> opts,
<a href=#391 id=391 data-nosnippet>391</a> grpc_server: <span class="prelude-val">None</span>,
<a href=#392 id=392 data-nosnippet>392</a> enable_http_service: <span class="bool-val">false</span>,
<a href=#393 id=393 data-nosnippet>393</a> }
<a href=#394 id=394 data-nosnippet>394</a> }
<a href=#395 id=395 data-nosnippet>395</a>
<a href=#396 id=396 data-nosnippet>396</a> <span class="kw">pub fn </span>enable_http_service(<span class="self">self</span>) -&gt; <span class="self">Self </span>{
<a href=#397 id=397 data-nosnippet>397</a> <span class="self">Self </span>{
<a href=#398 id=398 data-nosnippet>398</a> enable_http_service: <span class="bool-val">true</span>,
<a href=#399 id=399 data-nosnippet>399</a> ..<span class="self">self
<a href=#400 id=400 data-nosnippet>400</a> </span>}
<a href=#401 id=401 data-nosnippet>401</a> }
<a href=#402 id=402 data-nosnippet>402</a>
<a href=#403 id=403 data-nosnippet>403</a> <span class="kw">pub fn </span>with_grpc_server(<span class="self">self</span>, grpc_server: GrpcServer) -&gt; <span class="self">Self </span>{
<a href=#404 id=404 data-nosnippet>404</a> <span class="self">Self </span>{
<a href=#405 id=405 data-nosnippet>405</a> grpc_server: <span class="prelude-val">Some</span>(grpc_server),
<a href=#406 id=406 data-nosnippet>406</a> ..<span class="self">self
<a href=#407 id=407 data-nosnippet>407</a> </span>}
<a href=#408 id=408 data-nosnippet>408</a> }
<a href=#409 id=409 data-nosnippet>409</a>
<a href=#410 id=410 data-nosnippet>410</a> <span class="kw">pub fn </span>with_default_grpc_server(<span class="kw-2">mut </span><span class="self">self</span>, flownode_server: <span class="kw-2">&amp;</span>FlownodeServer) -&gt; <span class="self">Self </span>{
<a href=#411 id=411 data-nosnippet>411</a> <span class="kw">let </span>grpc_server = <span class="self">Self</span>::grpc_server_builder(<span class="self">self</span>.opts, flownode_server).build();
<a href=#412 id=412 data-nosnippet>412</a> <span class="self">self</span>.grpc_server = <span class="prelude-val">Some</span>(grpc_server);
<a href=#413 id=413 data-nosnippet>413</a> <span class="self">self
<a href=#414 id=414 data-nosnippet>414</a> </span>}
<a href=#415 id=415 data-nosnippet>415</a>
<a href=#416 id=416 data-nosnippet>416</a> <span class="kw">pub fn </span>build(<span class="kw-2">mut </span><span class="self">self</span>) -&gt; <span class="prelude-ty">Result</span>&lt;ServerHandlers, Error&gt; {
<a href=#417 id=417 data-nosnippet>417</a> <span class="kw">let </span>handlers = ServerHandlers::default();
<a href=#418 id=418 data-nosnippet>418</a> <span class="kw">if let </span><span class="prelude-val">Some</span>(grpc_server) = <span class="self">self</span>.grpc_server.take() {
<a href=#419 id=419 data-nosnippet>419</a> <span class="kw">let </span>addr: SocketAddr = <span class="self">self</span>.opts.grpc.bind_addr.parse().context(ParseAddrSnafu {
<a href=#420 id=420 data-nosnippet>420</a> addr: <span class="kw-2">&amp;</span><span class="self">self</span>.opts.grpc.bind_addr,
<a href=#421 id=421 data-nosnippet>421</a> })<span class="question-mark">?</span>;
<a href=#422 id=422 data-nosnippet>422</a> <span class="kw">let </span>handler: ServerHandler = (Box::new(grpc_server), addr);
<a href=#423 id=423 data-nosnippet>423</a> handlers.insert(handler);
<a href=#424 id=424 data-nosnippet>424</a> }
<a href=#425 id=425 data-nosnippet>425</a>
<a href=#426 id=426 data-nosnippet>426</a> <span class="kw">if </span><span class="self">self</span>.enable_http_service {
<a href=#427 id=427 data-nosnippet>427</a> <span class="kw">let </span>http_server = HttpServerBuilder::new(<span class="self">self</span>.opts.http.clone())
<a href=#428 id=428 data-nosnippet>428</a> .with_metrics_handler(MetricsHandler)
<a href=#429 id=429 data-nosnippet>429</a> .build();
<a href=#430 id=430 data-nosnippet>430</a> <span class="kw">let </span>addr: SocketAddr = <span class="self">self</span>.opts.http.addr.parse().context(ParseAddrSnafu {
<a href=#431 id=431 data-nosnippet>431</a> addr: <span class="kw-2">&amp;</span><span class="self">self</span>.opts.http.addr,
<a href=#432 id=432 data-nosnippet>432</a> })<span class="question-mark">?</span>;
<a href=#433 id=433 data-nosnippet>433</a> <span class="kw">let </span>handler: ServerHandler = (Box::new(http_server), addr);
<a href=#434 id=434 data-nosnippet>434</a> handlers.insert(handler);
<a href=#435 id=435 data-nosnippet>435</a> }
<a href=#436 id=436 data-nosnippet>436</a> <span class="prelude-val">Ok</span>(handlers)
<a href=#437 id=437 data-nosnippet>437</a> }
<a href=#438 id=438 data-nosnippet>438</a>
<a href=#439 id=439 data-nosnippet>439</a> <span class="kw">pub fn </span>grpc_server_builder(
<a href=#440 id=440 data-nosnippet>440</a> opts: <span class="kw-2">&amp;</span>FlownodeOptions,
<a href=#441 id=441 data-nosnippet>441</a> flownode_server: <span class="kw-2">&amp;</span>FlownodeServer,
<a href=#442 id=442 data-nosnippet>442</a> ) -&gt; GrpcServerBuilder {
<a href=#443 id=443 data-nosnippet>443</a> <span class="kw">let </span>config = GrpcServerConfig {
<a href=#444 id=444 data-nosnippet>444</a> max_recv_message_size: opts.grpc.max_recv_message_size.as_bytes() <span class="kw">as </span>usize,
<a href=#445 id=445 data-nosnippet>445</a> max_send_message_size: opts.grpc.max_send_message_size.as_bytes() <span class="kw">as </span>usize,
<a href=#446 id=446 data-nosnippet>446</a> tls: opts.grpc.tls.clone(),
<a href=#447 id=447 data-nosnippet>447</a> max_connection_age: opts.grpc.max_connection_age,
<a href=#448 id=448 data-nosnippet>448</a> };
<a href=#449 id=449 data-nosnippet>449</a> <span class="kw">let </span>service = flownode_server.create_flow_service();
<a href=#450 id=450 data-nosnippet>450</a> <span class="kw">let </span>runtime = common_runtime::global_runtime();
<a href=#451 id=451 data-nosnippet>451</a> <span class="kw">let </span><span class="kw-2">mut </span>builder = GrpcServerBuilder::new(config, runtime);
<a href=#452 id=452 data-nosnippet>452</a> <span class="macro">add_service!</span>(builder, service);
<a href=#453 id=453 data-nosnippet>453</a> builder
<a href=#454 id=454 data-nosnippet>454</a> }
<a href=#455 id=455 data-nosnippet>455</a>}
<a href=#456 id=456 data-nosnippet>456</a>
<a href=#457 id=457 data-nosnippet>457</a><span class="doccomment">/// get all flow ids in this flownode
<a href=#458 id=458 data-nosnippet>458</a></span><span class="kw">pub</span>(<span class="kw">crate</span>) <span class="kw">async fn </span>get_all_flow_ids(
<a href=#459 id=459 data-nosnippet>459</a> flow_metadata_manager: <span class="kw-2">&amp;</span>FlowMetadataManagerRef,
<a href=#460 id=460 data-nosnippet>460</a> catalog_manager: <span class="kw-2">&amp;</span>CatalogManagerRef,
<a href=#461 id=461 data-nosnippet>461</a> nodeid: <span class="prelude-ty">Option</span>&lt;u64&gt;,
<a href=#462 id=462 data-nosnippet>462</a>) -&gt; <span class="prelude-ty">Result</span>&lt;Vec&lt;u32&gt;, Error&gt; {
<a href=#463 id=463 data-nosnippet>463</a> <span class="kw">let </span>ret = <span class="kw">if let </span><span class="prelude-val">Some</span>(nodeid) = nodeid {
<a href=#464 id=464 data-nosnippet>464</a> <span class="kw">let </span>flow_ids_one_node = flow_metadata_manager
<a href=#465 id=465 data-nosnippet>465</a> .flownode_flow_manager()
<a href=#466 id=466 data-nosnippet>466</a> .flows(nodeid)
<a href=#467 id=467 data-nosnippet>467</a> .try_collect::&lt;Vec&lt;<span class="kw">_</span>&gt;&gt;()
<a href=#468 id=468 data-nosnippet>468</a> .<span class="kw">await
<a href=#469 id=469 data-nosnippet>469</a> </span>.context(ListFlowsSnafu { id: <span class="prelude-val">Some</span>(nodeid) })<span class="question-mark">?</span>;
<a href=#470 id=470 data-nosnippet>470</a> flow_ids_one_node.into_iter().map(|(id, <span class="kw">_</span>)| id).collect()
<a href=#471 id=471 data-nosnippet>471</a> } <span class="kw">else </span>{
<a href=#472 id=472 data-nosnippet>472</a> <span class="kw">let </span>all_catalogs = catalog_manager
<a href=#473 id=473 data-nosnippet>473</a> .catalog_names()
<a href=#474 id=474 data-nosnippet>474</a> .<span class="kw">await
<a href=#475 id=475 data-nosnippet>475</a> </span>.map_err(BoxedError::new)
<a href=#476 id=476 data-nosnippet>476</a> .context(ExternalSnafu)<span class="question-mark">?</span>;
<a href=#477 id=477 data-nosnippet>477</a> <span class="kw">let </span><span class="kw-2">mut </span>all_flow_ids = <span class="macro">vec!</span>[];
<a href=#478 id=478 data-nosnippet>478</a> <span class="kw">for </span>catalog <span class="kw">in </span>all_catalogs {
<a href=#479 id=479 data-nosnippet>479</a> <span class="kw">let </span>flows = flow_metadata_manager
<a href=#480 id=480 data-nosnippet>480</a> .flow_name_manager()
<a href=#481 id=481 data-nosnippet>481</a> .flow_names(<span class="kw-2">&amp;</span>catalog)
<a href=#482 id=482 data-nosnippet>482</a> .<span class="kw">await
<a href=#483 id=483 data-nosnippet>483</a> </span>.try_collect::&lt;Vec&lt;<span class="kw">_</span>&gt;&gt;()
<a href=#484 id=484 data-nosnippet>484</a> .<span class="kw">await
<a href=#485 id=485 data-nosnippet>485</a> </span>.map_err(BoxedError::new)
<a href=#486 id=486 data-nosnippet>486</a> .context(ExternalSnafu)<span class="question-mark">?</span>;
<a href=#487 id=487 data-nosnippet>487</a>
<a href=#488 id=488 data-nosnippet>488</a> all_flow_ids.extend(flows.into_iter().map(|(<span class="kw">_</span>, id)| id.flow_id()));
<a href=#489 id=489 data-nosnippet>489</a> }
<a href=#490 id=490 data-nosnippet>490</a> all_flow_ids
<a href=#491 id=491 data-nosnippet>491</a> };
<a href=#492 id=492 data-nosnippet>492</a>
<a href=#493 id=493 data-nosnippet>493</a> <span class="prelude-val">Ok</span>(ret)
<a href=#494 id=494 data-nosnippet>494</a>}
<a href=#495 id=495 data-nosnippet>495</a>
<a href=#496 id=496 data-nosnippet>496</a><span class="attr">#[cfg(test)]
<a href=#497 id=497 data-nosnippet>497</a></span><span class="kw">mod </span>tests {
<a href=#498 id=498 data-nosnippet>498</a> <span class="kw">use </span>std::sync::Arc;
<a href=#499 id=499 data-nosnippet>499</a>
<a href=#500 id=500 data-nosnippet>500</a> <span class="kw">use </span>catalog::memory::new_memory_catalog_manager;
<a href=#501 id=501 data-nosnippet>501</a> <span class="kw">use </span>common_base::Plugins;
<a href=#502 id=502 data-nosnippet>502</a> <span class="kw">use </span>common_meta::key::TableMetadataManager;
<a href=#503 id=503 data-nosnippet>503</a> <span class="kw">use </span>common_meta::key::flow::FlowMetadataManager;
<a href=#504 id=504 data-nosnippet>504</a> <span class="kw">use </span>common_meta::kv_backend::memory::MemoryKvBackend;
<a href=#505 id=505 data-nosnippet>505</a> <span class="kw">use </span>query::options::QueryOptions;
<a href=#506 id=506 data-nosnippet>506</a>
<a href=#507 id=507 data-nosnippet>507</a> <span class="kw">use </span>super::<span class="kw-2">*</span>;
<a href=#508 id=508 data-nosnippet>508</a> <span class="kw">use </span><span class="kw">crate</span>::batching_mode::BatchingModeOptions;
<a href=#509 id=509 data-nosnippet>509</a>
<a href=#510 id=510 data-nosnippet>510</a> <span class="kw">async fn </span>new_test_flownode_server() -&gt; (FlownodeServer, SizeReportSender) {
<a href=#511 id=511 data-nosnippet>511</a> <span class="kw">let </span>(frontend_client, _handler) =
<a href=#512 id=512 data-nosnippet>512</a> FrontendClient::from_empty_grpc_handler(QueryOptions::default());
<a href=#513 id=513 data-nosnippet>513</a>
<a href=#514 id=514 data-nosnippet>514</a> new_test_flownode_server_with_frontend_client(
<a href=#515 id=515 data-nosnippet>515</a> frontend_client,
<a href=#516 id=516 data-nosnippet>516</a> BatchingModeOptions::default(),
<a href=#517 id=517 data-nosnippet>517</a> <span class="prelude-val">None</span>,
<a href=#518 id=518 data-nosnippet>518</a> )
<a href=#519 id=519 data-nosnippet>519</a> .<span class="kw">await
<a href=#520 id=520 data-nosnippet>520</a> </span>}
<a href=#521 id=521 data-nosnippet>521</a>
<a href=#522 id=522 data-nosnippet>522</a> <span class="kw">async fn </span>new_test_flownode_server_with_frontend_client(
<a href=#523 id=523 data-nosnippet>523</a> frontend_client: FrontendClient,
<a href=#524 id=524 data-nosnippet>524</a> batching_opts: BatchingModeOptions,
<a href=#525 id=525 data-nosnippet>525</a> node_id: <span class="prelude-ty">Option</span>&lt;u32&gt;,
<a href=#526 id=526 data-nosnippet>526</a> ) -&gt; (FlownodeServer, SizeReportSender) {
<a href=#527 id=527 data-nosnippet>527</a> <span class="kw">let </span>kv_backend = Arc::new(MemoryKvBackend::new());
<a href=#528 id=528 data-nosnippet>528</a> <span class="kw">let </span>table_meta = Arc::new(TableMetadataManager::new(kv_backend.clone()));
<a href=#529 id=529 data-nosnippet>529</a> table_meta.init().<span class="kw">await</span>.unwrap();
<a href=#530 id=530 data-nosnippet>530</a> <span class="kw">let </span>flow_meta = Arc::new(FlowMetadataManager::new(kv_backend.clone()));
<a href=#531 id=531 data-nosnippet>531</a> <span class="kw">let </span>catalog_manager = new_memory_catalog_manager().unwrap();
<a href=#532 id=532 data-nosnippet>532</a> <span class="kw">let </span>query_engine = <span class="kw">crate</span>::test_utils::create_test_query_engine();
<a href=#533 id=533 data-nosnippet>533</a>
<a href=#534 id=534 data-nosnippet>534</a> <span class="kw">let </span>frontend_client = Arc::new(frontend_client);
<a href=#535 id=535 data-nosnippet>535</a> <span class="kw">let </span>streaming_engine = Arc::new(StreamingEngine::new(
<a href=#536 id=536 data-nosnippet>536</a> node_id,
<a href=#537 id=537 data-nosnippet>537</a> query_engine.clone(),
<a href=#538 id=538 data-nosnippet>538</a> table_meta.clone(),
<a href=#539 id=539 data-nosnippet>539</a> frontend_client.clone(),
<a href=#540 id=540 data-nosnippet>540</a> ));
<a href=#541 id=541 data-nosnippet>541</a> <span class="kw">let </span>batching_engine = Arc::new(BatchingEngine::new(
<a href=#542 id=542 data-nosnippet>542</a> frontend_client,
<a href=#543 id=543 data-nosnippet>543</a> query_engine,
<a href=#544 id=544 data-nosnippet>544</a> flow_meta.clone(),
<a href=#545 id=545 data-nosnippet>545</a> table_meta,
<a href=#546 id=546 data-nosnippet>546</a> catalog_manager.clone(),
<a href=#547 id=547 data-nosnippet>547</a> batching_opts,
<a href=#548 id=548 data-nosnippet>548</a> ));
<a href=#549 id=549 data-nosnippet>549</a> <span class="kw">let </span>dual_engine = Arc::new(FlowDualEngine::new(
<a href=#550 id=550 data-nosnippet>550</a> streaming_engine,
<a href=#551 id=551 data-nosnippet>551</a> batching_engine,
<a href=#552 id=552 data-nosnippet>552</a> flow_meta,
<a href=#553 id=553 data-nosnippet>553</a> catalog_manager,
<a href=#554 id=554 data-nosnippet>554</a> Plugins::new(),
<a href=#555 id=555 data-nosnippet>555</a> ));
<a href=#556 id=556 data-nosnippet>556</a>
<a href=#557 id=557 data-nosnippet>557</a> <span class="kw">let </span>(report_sender, report_handler) = SizeReportSender::new();
<a href=#558 id=558 data-nosnippet>558</a> dual_engine.set_state_report_handler(report_handler).<span class="kw">await</span>;
<a href=#559 id=559 data-nosnippet>559</a>
<a href=#560 id=560 data-nosnippet>560</a> <span class="kw">let </span>server = FlownodeServer::new(FlowService::new(dual_engine));
<a href=#561 id=561 data-nosnippet>561</a> (server, report_sender)
<a href=#562 id=562 data-nosnippet>562</a> }
<a href=#563 id=563 data-nosnippet>563</a>}
</code></pre></div></section></main></body></html>
+64
View File
@@ -0,0 +1,64 @@
<!DOCTYPE html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1.0"><meta name="generator" content="rustdoc"><meta name="description" content="Source of the Rust file `src/flow/src/utils.rs`."><title>utils.rs - source</title><script>if(window.location.protocol!=="file:")document.head.insertAdjacentHTML("beforeend","SourceSerif4-Regular-6b053e98.ttf.woff2,FiraSans-Italic-81dc35de.woff2,FiraSans-Regular-0fe48ade.woff2,FiraSans-MediumItalic-ccf7e434.woff2,FiraSans-Medium-e1aa3f0a.woff2,SourceCodePro-Regular-8badfe75.ttf.woff2,SourceCodePro-Semibold-aa29a496.ttf.woff2".split(",").map(f=>`<link rel="preload" as="font" type="font/woff2"href="../../static.files/${f}">`).join(""))</script><link rel="stylesheet" href="../../static.files/normalize-9960930a.css"><link rel="stylesheet" href="../../static.files/rustdoc-17e0aaed.css"><meta name="rustdoc-vars" data-root-path="../../" data-static-root-path="../../static.files/" data-current-crate="flow" data-themes="" data-resource-suffix="" data-rustdoc-version="1.96.1 (31fca3adb 2026-06-26)" data-channel="1.96.1" data-search-js="search-b5634cc7.js" data-stringdex-js="stringdex-2da4960a.js" data-settings-js="settings-170eb4bf.js" ><script src="../../static.files/storage-41dd4d93.js"></script><script defer src="../../static.files/src-script-813739b1.js"></script><script defer src="../../src-files.js"></script><script defer src="../../static.files/main-5013f961.js"></script><noscript><link rel="stylesheet" href="../../static.files/noscript-f7c3ffd8.css"></noscript><link rel="alternate icon" type="image/png" href="../../static.files/favicon-32x32-eab170b8.png"><link rel="icon" type="image/svg+xml" href="../../static.files/favicon-044be391.svg"></head><body class="rustdoc src"><a class="skip-main-content" href="#main-content">Skip to main content</a><!--[if lte IE 11]><div class="warning">This old browser is unsupported and will most likely display funky things.</div><![endif]--><nav class="sidebar"><div class="src-sidebar-title"><h2>Files</h2></div></nav><div class="sidebar-resizer" title="Drag to resize sidebar"></div><main><section id="main-content" class="content" tabindex="-1"><div class="main-heading"><h1><div class="sub-heading">flow/</div>utils.rs</h1><rustdoc-toolbar></rustdoc-toolbar></div><div class="example-wrap digits-2"><pre class="rust"><code><a href=#1 id=1 data-nosnippet>1</a><span class="comment">// Copyright 2023 Greptime Team
<a href=#2 id=2 data-nosnippet>2</a>//
<a href=#3 id=3 data-nosnippet>3</a>// Licensed under the Apache License, Version 2.0 (the "License");
<a href=#4 id=4 data-nosnippet>4</a>// you may not use this file except in compliance with the License.
<a href=#5 id=5 data-nosnippet>5</a>// You may obtain a copy of the License at
<a href=#6 id=6 data-nosnippet>6</a>//
<a href=#7 id=7 data-nosnippet>7</a>// http://www.apache.org/licenses/LICENSE-2.0
<a href=#8 id=8 data-nosnippet>8</a>//
<a href=#9 id=9 data-nosnippet>9</a>// Unless required by applicable law or agreed to in writing, software
<a href=#10 id=10 data-nosnippet>10</a>// distributed under the License is distributed on an "AS IS" BASIS,
<a href=#11 id=11 data-nosnippet>11</a>// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
<a href=#12 id=12 data-nosnippet>12</a>// See the License for the specific language governing permissions and
<a href=#13 id=13 data-nosnippet>13</a>// limitations under the License.
<a href=#14 id=14 data-nosnippet>14</a>
<a href=#15 id=15 data-nosnippet>15</a></span><span class="kw">use </span>common_meta::key::flow::flow_state::FlowStat;
<a href=#16 id=16 data-nosnippet>16</a><span class="kw">use </span>datatypes::value::Value;
<a href=#17 id=17 data-nosnippet>17</a><span class="kw">use </span>tokio::sync::{mpsc, oneshot};
<a href=#18 id=18 data-nosnippet>18</a>
<a href=#19 id=19 data-nosnippet>19</a><span class="kw">use </span><span class="kw">crate</span>::error::InternalSnafu;
<a href=#20 id=20 data-nosnippet>20</a>
<a href=#21 id=21 data-nosnippet>21</a><span class="kw">pub fn </span>get_value_heap_size(value: <span class="kw-2">&amp;</span>Value) -&gt; usize {
<a href=#22 id=22 data-nosnippet>22</a> <span class="kw">match </span>value {
<a href=#23 id=23 data-nosnippet>23</a> Value::Binary(v) =&gt; v.len(),
<a href=#24 id=24 data-nosnippet>24</a> Value::String(v) =&gt; v.len(),
<a href=#25 id=25 data-nosnippet>25</a> Value::List(v) =&gt; v.items().iter().map(get_value_heap_size).sum(),
<a href=#26 id=26 data-nosnippet>26</a> <span class="kw">_ </span>=&gt; <span class="number">0</span>,
<a href=#27 id=27 data-nosnippet>27</a> }
<a href=#28 id=28 data-nosnippet>28</a>}
<a href=#29 id=29 data-nosnippet>29</a>
<a href=#30 id=30 data-nosnippet>30</a><span class="attr">#[derive(Clone)]
<a href=#31 id=31 data-nosnippet>31</a></span><span class="kw">pub struct </span>SizeReportSender {
<a href=#32 id=32 data-nosnippet>32</a> inner: mpsc::Sender&lt;oneshot::Sender&lt;FlowStat&gt;&gt;,
<a href=#33 id=33 data-nosnippet>33</a>}
<a href=#34 id=34 data-nosnippet>34</a><span class="kw">impl </span>SizeReportSender {
<a href=#35 id=35 data-nosnippet>35</a> <span class="kw">pub fn </span>new() -&gt; (<span class="self">Self</span>, StateReportHandler) {
<a href=#36 id=36 data-nosnippet>36</a> <span class="kw">let </span>(tx, rx) = mpsc::channel(<span class="number">1</span>);
<a href=#37 id=37 data-nosnippet>37</a> (<span class="self">Self </span>{ inner: tx }, rx)
<a href=#38 id=38 data-nosnippet>38</a> }
<a href=#39 id=39 data-nosnippet>39</a> <span class="kw">pub async fn </span>query(<span class="kw-2">&amp;</span><span class="self">self</span>, timeout: std::time::Duration) -&gt; <span class="kw">crate</span>::Result&lt;FlowStat&gt; {
<a href=#40 id=40 data-nosnippet>40</a> <span class="kw">let </span>(tx, rx) = oneshot::channel();
<a href=#41 id=41 data-nosnippet>41</a> <span class="self">self</span>.inner.send(tx).<span class="kw">await</span>.map_err(|<span class="kw">_</span>| {
<a href=#42 id=42 data-nosnippet>42</a> InternalSnafu {
<a href=#43 id=43 data-nosnippet>43</a> reason: <span class="string">"failed to send size report request"</span>,
<a href=#44 id=44 data-nosnippet>44</a> }
<a href=#45 id=45 data-nosnippet>45</a> .build()
<a href=#46 id=46 data-nosnippet>46</a> })<span class="question-mark">?</span>;
<a href=#47 id=47 data-nosnippet>47</a> tokio::time::timeout(timeout, rx)
<a href=#48 id=48 data-nosnippet>48</a> .<span class="kw">await
<a href=#49 id=49 data-nosnippet>49</a> </span>.map_err(|<span class="kw">_</span>| {
<a href=#50 id=50 data-nosnippet>50</a> InternalSnafu {
<a href=#51 id=51 data-nosnippet>51</a> reason: <span class="string">"failed to receive size report after timeout"</span>,
<a href=#52 id=52 data-nosnippet>52</a> }
<a href=#53 id=53 data-nosnippet>53</a> .build()
<a href=#54 id=54 data-nosnippet>54</a> })<span class="question-mark">?
<a href=#55 id=55 data-nosnippet>55</a> </span>.map_err(|<span class="kw">_</span>| {
<a href=#56 id=56 data-nosnippet>56</a> InternalSnafu {
<a href=#57 id=57 data-nosnippet>57</a> reason: <span class="string">"size report sender dropped"</span>,
<a href=#58 id=58 data-nosnippet>58</a> }
<a href=#59 id=59 data-nosnippet>59</a> .build()
<a href=#60 id=60 data-nosnippet>60</a> })
<a href=#61 id=61 data-nosnippet>61</a> }
<a href=#62 id=62 data-nosnippet>62</a>}
<a href=#63 id=63 data-nosnippet>63</a><span class="kw">pub type </span>StateReportHandler = mpsc::Receiver&lt;oneshot::Sender&lt;FlowStat&gt;&gt;;
</code></pre></div></section></main></body></html>