rust/greptimedb
1
0
Fork 0
mirror of https://github.com/GreptimeTeam/greptimedb.git synced 2026-05-23 08:20:36 +00:00
Code Issues Packages Projects Releases Wiki Activity

Deploying to gh-pages from @ GreptimeTeam/greptimedb@28f97191a0 🚀

Browse Source
This commit is contained in:
github-merge-queue[bot]
2026-03-12 03:02:27 +00:00
commit dbdbafa5c3
20974 changed files with 2441584 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

882
src/common_recordbatch/adapter.rs.html Normal file
View File

@@ -0,0 +1,882 @@
<!DOCTYPE html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1.0"><meta name="generator" content="rustdoc"><meta name="description" content="Source of the Rust file `src/common/recordbatch/src/adapter.rs`."><title>adapter.rs - source</title><script>if(window.location.protocol!=="file:")document.head.insertAdjacentHTML("beforeend","SourceSerif4-Regular-6b053e98.ttf.woff2,FiraSans-Italic-81dc35de.woff2,FiraSans-Regular-0fe48ade.woff2,FiraSans-MediumItalic-ccf7e434.woff2,FiraSans-Medium-e1aa3f0a.woff2,SourceCodePro-Regular-8badfe75.ttf.woff2,SourceCodePro-Semibold-aa29a496.ttf.woff2".split(",").map(f=>`<link rel="preload" as="font" type="font/woff2"href="../../static.files/${f}">`).join(""))</script><link rel="stylesheet" href="../../static.files/normalize-9960930a.css"><link rel="stylesheet" href="../../static.files/rustdoc-e56847b5.css"><meta name="rustdoc-vars" data-root-path="../../" data-static-root-path="../../static.files/" data-current-crate="common_recordbatch" data-themes="" data-resource-suffix="" data-rustdoc-version="1.92.0-nightly (fa3155a64 2025-09-30)" data-channel="nightly" data-search-js="search-e256b49e.js" data-stringdex-js="stringdex-828709d0.js" data-settings-js="settings-c38705f0.js" ><script src="../../static.files/storage-e2aeef58.js"></script><script defer src="../../static.files/src-script-813739b1.js"></script><script defer src="../../src-files.js"></script><script defer src="../../static.files/main-ce535bd0.js"></script><noscript><link rel="stylesheet" href="../../static.files/noscript-263c88ec.css"></noscript><link rel="alternate icon" type="image/png" href="../../static.files/favicon-32x32-eab170b8.png"><link rel="icon" type="image/svg+xml" href="../../static.files/favicon-044be391.svg"></head><body class="rustdoc src"><!--[if lte IE 11]><div class="warning">This old browser is unsupported and will most likely display funky things.</div><![endif]--><nav class="sidebar"><div class="src-sidebar-title"><h2>Files</h2></div></nav><div class="sidebar-resizer" title="Drag to resize sidebar"></div><main><section id="main-content" class="content"><div class="main-heading"><h1><div class="sub-heading">common_recordbatch/</div>adapter.rs</h1><rustdoc-toolbar></rustdoc-toolbar></div><div class="example-wrap digits-3"><pre class="rust"><code><a href=#1 id=1 data-nosnippet>1</a><span class="comment">// Copyright 2023 Greptime Team
<a href=#2 id=2 data-nosnippet>2</a>//
<a href=#3 id=3 data-nosnippet>3</a>// Licensed under the Apache License, Version 2.0 (the "License");
<a href=#4 id=4 data-nosnippet>4</a>// you may not use this file except in compliance with the License.
<a href=#5 id=5 data-nosnippet>5</a>// You may obtain a copy of the License at
<a href=#6 id=6 data-nosnippet>6</a>//
<a href=#7 id=7 data-nosnippet>7</a>// http://www.apache.org/licenses/LICENSE-2.0
<a href=#8 id=8 data-nosnippet>8</a>//
<a href=#9 id=9 data-nosnippet>9</a>// Unless required by applicable law or agreed to in writing, software
<a href=#10 id=10 data-nosnippet>10</a>// distributed under the License is distributed on an "AS IS" BASIS,
<a href=#11 id=11 data-nosnippet>11</a>// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
<a href=#12 id=12 data-nosnippet>12</a>// See the License for the specific language governing permissions and
<a href=#13 id=13 data-nosnippet>13</a>// limitations under the License.
<a href=#14 id=14 data-nosnippet>14</a>
<a href=#15 id=15 data-nosnippet>15</a></span><span class="kw">use </span>std::fmt::{<span class="self">self</span>, Display};
<a href=#16 id=16 data-nosnippet>16</a><span class="kw">use </span>std::future::Future;
<a href=#17 id=17 data-nosnippet>17</a><span class="kw">use </span>std::marker::PhantomData;
<a href=#18 id=18 data-nosnippet>18</a><span class="kw">use </span>std::pin::Pin;
<a href=#19 id=19 data-nosnippet>19</a><span class="kw">use </span>std::str::FromStr;
<a href=#20 id=20 data-nosnippet>20</a><span class="kw">use </span>std::sync::Arc;
<a href=#21 id=21 data-nosnippet>21</a><span class="kw">use </span>std::task::{Context, Poll};
<a href=#22 id=22 data-nosnippet>22</a>
<a href=#23 id=23 data-nosnippet>23</a><span class="kw">use </span>common_base::readable_size::ReadableSize;
<a href=#24 id=24 data-nosnippet>24</a><span class="kw">use </span>common_telemetry::tracing::{Span, info_span};
<a href=#25 id=25 data-nosnippet>25</a><span class="kw">use </span>common_time::util::format_nanoseconds_human_readable;
<a href=#26 id=26 data-nosnippet>26</a><span class="kw">use </span>datafusion::arrow::compute::cast;
<a href=#27 id=27 data-nosnippet>27</a><span class="kw">use </span>datafusion::arrow::datatypes::SchemaRef <span class="kw">as </span>DfSchemaRef;
<a href=#28 id=28 data-nosnippet>28</a><span class="kw">use </span>datafusion::error::Result <span class="kw">as </span>DfResult;
<a href=#29 id=29 data-nosnippet>29</a><span class="kw">use </span>datafusion::execution::context::ExecutionProps;
<a href=#30 id=30 data-nosnippet>30</a><span class="kw">use </span>datafusion::logical_expr::Expr;
<a href=#31 id=31 data-nosnippet>31</a><span class="kw">use </span>datafusion::logical_expr::utils::conjunction;
<a href=#32 id=32 data-nosnippet>32</a><span class="kw">use </span>datafusion::physical_expr::create_physical_expr;
<a href=#33 id=33 data-nosnippet>33</a><span class="kw">use </span>datafusion::physical_plan::metrics::{BaselineMetrics, MetricValue};
<a href=#34 id=34 data-nosnippet>34</a><span class="kw">use </span>datafusion::physical_plan::{
<a href=#35 id=35 data-nosnippet>35</a> DisplayFormatType, ExecutionPlan, ExecutionPlanVisitor, PhysicalExpr,
<a href=#36 id=36 data-nosnippet>36</a> RecordBatchStream <span class="kw">as </span>DfRecordBatchStream, accept,
<a href=#37 id=37 data-nosnippet>37</a>};
<a href=#38 id=38 data-nosnippet>38</a><span class="kw">use </span>datafusion_common::arrow::error::ArrowError;
<a href=#39 id=39 data-nosnippet>39</a><span class="kw">use </span>datafusion_common::{DataFusionError, ToDFSchema};
<a href=#40 id=40 data-nosnippet>40</a><span class="kw">use </span>datatypes::arrow::array::Array;
<a href=#41 id=41 data-nosnippet>41</a><span class="kw">use </span>datatypes::arrow::datatypes::DataType <span class="kw">as </span>ArrowDataType;
<a href=#42 id=42 data-nosnippet>42</a><span class="kw">use </span>datatypes::schema::{ColumnExtType, Schema, SchemaRef};
<a href=#43 id=43 data-nosnippet>43</a><span class="kw">use </span>futures::ready;
<a href=#44 id=44 data-nosnippet>44</a><span class="kw">use </span>jsonb;
<a href=#45 id=45 data-nosnippet>45</a><span class="kw">use </span>pin_project::pin_project;
<a href=#46 id=46 data-nosnippet>46</a><span class="kw">use </span>snafu::ResultExt;
<a href=#47 id=47 data-nosnippet>47</a>
<a href=#48 id=48 data-nosnippet>48</a><span class="kw">use </span><span class="kw">crate</span>::error::{<span class="self">self</span>, <span class="prelude-ty">Result</span>};
<a href=#49 id=49 data-nosnippet>49</a><span class="kw">use </span><span class="kw">crate</span>::filter::batch_filter;
<a href=#50 id=50 data-nosnippet>50</a><span class="kw">use crate</span>::{
<a href=#51 id=51 data-nosnippet>51</a> DfRecordBatch, DfSendableRecordBatchStream, OrderOption, RecordBatch, RecordBatchStream,
<a href=#52 id=52 data-nosnippet>52</a> SendableRecordBatchStream, Stream,
<a href=#53 id=53 data-nosnippet>53</a>};
<a href=#54 id=54 data-nosnippet>54</a>
<a href=#55 id=55 data-nosnippet>55</a><span class="kw">type </span>FutureStream =
<a href=#56 id=56 data-nosnippet>56</a> Pin&lt;Box&lt;<span class="kw">dyn </span>std::future::Future&lt;Output = <span class="prelude-ty">Result</span>&lt;SendableRecordBatchStream&gt;&gt; + Send&gt;&gt;;
<a href=#57 id=57 data-nosnippet>57</a>
<a href=#58 id=58 data-nosnippet>58</a><span class="doccomment">/// Casts the `RecordBatch`es of `stream` against the `output_schema`.
<a href=#59 id=59 data-nosnippet>59</a></span><span class="attr">#[pin_project]
<a href=#60 id=60 data-nosnippet>60</a></span><span class="kw">pub struct </span>RecordBatchStreamTypeAdapter&lt;T, E&gt; {
<a href=#61 id=61 data-nosnippet>61</a> <span class="attr">#[pin]
<a href=#62 id=62 data-nosnippet>62</a> </span>stream: T,
<a href=#63 id=63 data-nosnippet>63</a> projected_schema: DfSchemaRef,
<a href=#64 id=64 data-nosnippet>64</a> projection: Vec&lt;usize&gt;,
<a href=#65 id=65 data-nosnippet>65</a> predicate: <span class="prelude-ty">Option</span>&lt;Arc&lt;<span class="kw">dyn </span>PhysicalExpr&gt;&gt;,
<a href=#66 id=66 data-nosnippet>66</a> phantom: PhantomData&lt;E&gt;,
<a href=#67 id=67 data-nosnippet>67</a>}
<a href=#68 id=68 data-nosnippet>68</a>
<a href=#69 id=69 data-nosnippet>69</a><span class="kw">impl</span>&lt;T, E&gt; RecordBatchStreamTypeAdapter&lt;T, E&gt;
<a href=#70 id=70 data-nosnippet>70</a><span class="kw">where
<a href=#71 id=71 data-nosnippet>71</a> </span>T: Stream&lt;Item = std::result::Result&lt;DfRecordBatch, E&gt;&gt;,
<a href=#72 id=72 data-nosnippet>72</a> E: std::error::Error + Send + Sync + <span class="lifetime">'static</span>,
<a href=#73 id=73 data-nosnippet>73</a>{
<a href=#74 id=74 data-nosnippet>74</a> <span class="kw">pub fn </span>new(projected_schema: DfSchemaRef, stream: T, projection: <span class="prelude-ty">Option</span>&lt;Vec&lt;usize&gt;&gt;) -&gt; <span class="self">Self </span>{
<a href=#75 id=75 data-nosnippet>75</a> <span class="kw">let </span>projection = <span class="kw">if let </span><span class="prelude-val">Some</span>(projection) = projection {
<a href=#76 id=76 data-nosnippet>76</a> projection
<a href=#77 id=77 data-nosnippet>77</a> } <span class="kw">else </span>{
<a href=#78 id=78 data-nosnippet>78</a> (<span class="number">0</span>..projected_schema.fields().len()).collect()
<a href=#79 id=79 data-nosnippet>79</a> };
<a href=#80 id=80 data-nosnippet>80</a>
<a href=#81 id=81 data-nosnippet>81</a> <span class="self">Self </span>{
<a href=#82 id=82 data-nosnippet>82</a> stream,
<a href=#83 id=83 data-nosnippet>83</a> projected_schema,
<a href=#84 id=84 data-nosnippet>84</a> projection,
<a href=#85 id=85 data-nosnippet>85</a> predicate: <span class="prelude-val">None</span>,
<a href=#86 id=86 data-nosnippet>86</a> phantom: Default::default(),
<a href=#87 id=87 data-nosnippet>87</a> }
<a href=#88 id=88 data-nosnippet>88</a> }
<a href=#89 id=89 data-nosnippet>89</a>
<a href=#90 id=90 data-nosnippet>90</a> <span class="kw">pub fn </span>with_filter(<span class="kw-2">mut </span><span class="self">self</span>, filters: Vec&lt;Expr&gt;) -&gt; <span class="prelude-ty">Result</span>&lt;<span class="self">Self</span>&gt; {
<a href=#91 id=91 data-nosnippet>91</a> <span class="kw">let </span>filters = <span class="kw">if let </span><span class="prelude-val">Some</span>(expr) = conjunction(filters) {
<a href=#92 id=92 data-nosnippet>92</a> <span class="kw">let </span>df_schema = <span class="self">self
<a href=#93 id=93 data-nosnippet>93</a> </span>.projected_schema
<a href=#94 id=94 data-nosnippet>94</a> .clone()
<a href=#95 id=95 data-nosnippet>95</a> .to_dfschema_ref()
<a href=#96 id=96 data-nosnippet>96</a> .context(error::PhysicalExprSnafu)<span class="question-mark">?</span>;
<a href=#97 id=97 data-nosnippet>97</a>
<a href=#98 id=98 data-nosnippet>98</a> <span class="kw">let </span>filters = create_physical_expr(<span class="kw-2">&amp;</span>expr, <span class="kw-2">&amp;</span>df_schema, <span class="kw-2">&amp;</span>ExecutionProps::new())
<a href=#99 id=99 data-nosnippet>99</a> .context(error::PhysicalExprSnafu)<span class="question-mark">?</span>;
<a href=#100 id=100 data-nosnippet>100</a> <span class="prelude-val">Some</span>(filters)
<a href=#101 id=101 data-nosnippet>101</a> } <span class="kw">else </span>{
<a href=#102 id=102 data-nosnippet>102</a> <span class="prelude-val">None
<a href=#103 id=103 data-nosnippet>103</a> </span>};
<a href=#104 id=104 data-nosnippet>104</a> <span class="self">self</span>.predicate = filters;
<a href=#105 id=105 data-nosnippet>105</a> <span class="prelude-val">Ok</span>(<span class="self">self</span>)
<a href=#106 id=106 data-nosnippet>106</a> }
<a href=#107 id=107 data-nosnippet>107</a>}
<a href=#108 id=108 data-nosnippet>108</a>
<a href=#109 id=109 data-nosnippet>109</a><span class="kw">impl</span>&lt;T, E&gt; DfRecordBatchStream <span class="kw">for </span>RecordBatchStreamTypeAdapter&lt;T, E&gt;
<a href=#110 id=110 data-nosnippet>110</a><span class="kw">where
<a href=#111 id=111 data-nosnippet>111</a> </span>T: Stream&lt;Item = std::result::Result&lt;DfRecordBatch, E&gt;&gt;,
<a href=#112 id=112 data-nosnippet>112</a> E: std::error::Error + Send + Sync + <span class="lifetime">'static</span>,
<a href=#113 id=113 data-nosnippet>113</a>{
<a href=#114 id=114 data-nosnippet>114</a> <span class="kw">fn </span>schema(<span class="kw-2">&amp;</span><span class="self">self</span>) -&gt; DfSchemaRef {
<a href=#115 id=115 data-nosnippet>115</a> <span class="self">self</span>.projected_schema.clone()
<a href=#116 id=116 data-nosnippet>116</a> }
<a href=#117 id=117 data-nosnippet>117</a>}
<a href=#118 id=118 data-nosnippet>118</a>
<a href=#119 id=119 data-nosnippet>119</a><span class="kw">impl</span>&lt;T, E&gt; Stream <span class="kw">for </span>RecordBatchStreamTypeAdapter&lt;T, E&gt;
<a href=#120 id=120 data-nosnippet>120</a><span class="kw">where
<a href=#121 id=121 data-nosnippet>121</a> </span>T: Stream&lt;Item = std::result::Result&lt;DfRecordBatch, E&gt;&gt;,
<a href=#122 id=122 data-nosnippet>122</a> E: std::error::Error + Send + Sync + <span class="lifetime">'static</span>,
<a href=#123 id=123 data-nosnippet>123</a>{
<a href=#124 id=124 data-nosnippet>124</a> <span class="kw">type </span>Item = DfResult&lt;DfRecordBatch&gt;;
<a href=#125 id=125 data-nosnippet>125</a>
<a href=#126 id=126 data-nosnippet>126</a> <span class="kw">fn </span>poll_next(<span class="self">self</span>: Pin&lt;<span class="kw-2">&amp;mut </span><span class="self">Self</span>&gt;, cx: <span class="kw-2">&amp;mut </span>Context&lt;<span class="lifetime">'_</span>&gt;) -&gt; Poll&lt;<span class="prelude-ty">Option</span>&lt;<span class="self">Self</span>::Item&gt;&gt; {
<a href=#127 id=127 data-nosnippet>127</a> <span class="kw">let </span>this = <span class="self">self</span>.project();
<a href=#128 id=128 data-nosnippet>128</a>
<a href=#129 id=129 data-nosnippet>129</a> <span class="kw">let </span>batch = <span class="macro">futures::ready!</span>(this.stream.poll_next(cx))
<a href=#130 id=130 data-nosnippet>130</a> .map(|r| r.map_err(|e| DataFusionError::External(Box::new(e))));
<a href=#131 id=131 data-nosnippet>131</a>
<a href=#132 id=132 data-nosnippet>132</a> <span class="kw">let </span>projected_schema = this.projected_schema.clone();
<a href=#133 id=133 data-nosnippet>133</a> <span class="kw">let </span>projection = this.projection.clone();
<a href=#134 id=134 data-nosnippet>134</a> <span class="kw">let </span>predicate = this.predicate.clone();
<a href=#135 id=135 data-nosnippet>135</a>
<a href=#136 id=136 data-nosnippet>136</a> <span class="kw">let </span>batch = batch.map(|b| {
<a href=#137 id=137 data-nosnippet>137</a> b.and_then(|b| {
<a href=#138 id=138 data-nosnippet>138</a> <span class="kw">let </span>projected_column = b.project(<span class="kw-2">&amp;</span>projection)<span class="question-mark">?</span>;
<a href=#139 id=139 data-nosnippet>139</a> <span class="kw">if </span>projected_column.schema().fields.len() != projected_schema.fields.len() {
<a href=#140 id=140 data-nosnippet>140</a> <span class="kw">return </span><span class="prelude-val">Err</span>(DataFusionError::ArrowError(Box::new(ArrowError::SchemaError(<span class="macro">format!</span>(
<a href=#141 id=141 data-nosnippet>141</a> <span class="string">"Trying to cast a RecordBatch into an incompatible schema. RecordBatch: {}, Target: {}"</span>,
<a href=#142 id=142 data-nosnippet>142</a> projected_column.schema(),
<a href=#143 id=143 data-nosnippet>143</a> projected_schema,
<a href=#144 id=144 data-nosnippet>144</a> ))), <span class="prelude-val">None</span>));
<a href=#145 id=145 data-nosnippet>145</a> }
<a href=#146 id=146 data-nosnippet>146</a>
<a href=#147 id=147 data-nosnippet>147</a> <span class="kw">let </span><span class="kw-2">mut </span>columns = Vec::with_capacity(projected_schema.fields.len());
<a href=#148 id=148 data-nosnippet>148</a> <span class="kw">for </span>(idx,field) <span class="kw">in </span>projected_schema.fields.iter().enumerate() {
<a href=#149 id=149 data-nosnippet>149</a> <span class="kw">let </span>column = projected_column.column(idx);
<a href=#150 id=150 data-nosnippet>150</a> <span class="kw">let </span>extype = field.metadata().get(<span class="string">"greptime:type"</span>).and_then(|s| ColumnExtType::from_str(s).ok());
<a href=#151 id=151 data-nosnippet>151</a> <span class="kw">let </span>output = custom_cast(<span class="kw-2">&amp;</span>column, field.data_type(), extype)<span class="question-mark">?</span>;
<a href=#152 id=152 data-nosnippet>152</a> columns.push(output)
<a href=#153 id=153 data-nosnippet>153</a> }
<a href=#154 id=154 data-nosnippet>154</a> <span class="kw">let </span>record_batch = DfRecordBatch::try_new(projected_schema, columns)<span class="question-mark">?</span>;
<a href=#155 id=155 data-nosnippet>155</a> <span class="kw">let </span>record_batch = <span class="kw">if let </span><span class="prelude-val">Some</span>(predicate) = predicate {
<a href=#156 id=156 data-nosnippet>156</a> batch_filter(<span class="kw-2">&amp;</span>record_batch, <span class="kw-2">&amp;</span>predicate)<span class="question-mark">?
<a href=#157 id=157 data-nosnippet>157</a> </span>} <span class="kw">else </span>{
<a href=#158 id=158 data-nosnippet>158</a> record_batch
<a href=#159 id=159 data-nosnippet>159</a> };
<a href=#160 id=160 data-nosnippet>160</a> <span class="prelude-val">Ok</span>(record_batch)
<a href=#161 id=161 data-nosnippet>161</a> })
<a href=#162 id=162 data-nosnippet>162</a> });
<a href=#163 id=163 data-nosnippet>163</a>
<a href=#164 id=164 data-nosnippet>164</a> Poll::Ready(batch)
<a href=#165 id=165 data-nosnippet>165</a> }
<a href=#166 id=166 data-nosnippet>166</a>
<a href=#167 id=167 data-nosnippet>167</a> <span class="attr">#[inline]
<a href=#168 id=168 data-nosnippet>168</a> </span><span class="kw">fn </span>size_hint(<span class="kw-2">&amp;</span><span class="self">self</span>) -&gt; (usize, <span class="prelude-ty">Option</span>&lt;usize&gt;) {
<a href=#169 id=169 data-nosnippet>169</a> <span class="self">self</span>.stream.size_hint()
<a href=#170 id=170 data-nosnippet>170</a> }
<a href=#171 id=171 data-nosnippet>171</a>}
<a href=#172 id=172 data-nosnippet>172</a>
<a href=#173 id=173 data-nosnippet>173</a><span class="doccomment">/// Greptime SendableRecordBatchStream -&gt; DataFusion RecordBatchStream.
<a href=#174 id=174 data-nosnippet>174</a>/// The reverse one is [RecordBatchStreamAdapter].
<a href=#175 id=175 data-nosnippet>175</a></span><span class="kw">pub struct </span>DfRecordBatchStreamAdapter {
<a href=#176 id=176 data-nosnippet>176</a> stream: SendableRecordBatchStream,
<a href=#177 id=177 data-nosnippet>177</a>}
<a href=#178 id=178 data-nosnippet>178</a>
<a href=#179 id=179 data-nosnippet>179</a><span class="kw">impl </span>DfRecordBatchStreamAdapter {
<a href=#180 id=180 data-nosnippet>180</a> <span class="kw">pub fn </span>new(stream: SendableRecordBatchStream) -&gt; <span class="self">Self </span>{
<a href=#181 id=181 data-nosnippet>181</a> <span class="self">Self </span>{ stream }
<a href=#182 id=182 data-nosnippet>182</a> }
<a href=#183 id=183 data-nosnippet>183</a>}
<a href=#184 id=184 data-nosnippet>184</a>
<a href=#185 id=185 data-nosnippet>185</a><span class="kw">impl </span>DfRecordBatchStream <span class="kw">for </span>DfRecordBatchStreamAdapter {
<a href=#186 id=186 data-nosnippet>186</a> <span class="kw">fn </span>schema(<span class="kw-2">&amp;</span><span class="self">self</span>) -&gt; DfSchemaRef {
<a href=#187 id=187 data-nosnippet>187</a> <span class="self">self</span>.stream.schema().arrow_schema().clone()
<a href=#188 id=188 data-nosnippet>188</a> }
<a href=#189 id=189 data-nosnippet>189</a>}
<a href=#190 id=190 data-nosnippet>190</a>
<a href=#191 id=191 data-nosnippet>191</a><span class="kw">impl </span>Stream <span class="kw">for </span>DfRecordBatchStreamAdapter {
<a href=#192 id=192 data-nosnippet>192</a> <span class="kw">type </span>Item = DfResult&lt;DfRecordBatch&gt;;
<a href=#193 id=193 data-nosnippet>193</a>
<a href=#194 id=194 data-nosnippet>194</a> <span class="kw">fn </span>poll_next(<span class="kw-2">mut </span><span class="self">self</span>: Pin&lt;<span class="kw-2">&amp;mut </span><span class="self">Self</span>&gt;, cx: <span class="kw-2">&amp;mut </span>Context&lt;<span class="lifetime">'_</span>&gt;) -&gt; Poll&lt;<span class="prelude-ty">Option</span>&lt;<span class="self">Self</span>::Item&gt;&gt; {
<a href=#195 id=195 data-nosnippet>195</a> <span class="kw">match </span>Pin::new(<span class="kw-2">&amp;mut </span><span class="self">self</span>.stream).poll_next(cx) {
<a href=#196 id=196 data-nosnippet>196</a> Poll::Pending =&gt; Poll::Pending,
<a href=#197 id=197 data-nosnippet>197</a> Poll::Ready(<span class="prelude-val">Some</span>(recordbatch)) =&gt; <span class="kw">match </span>recordbatch {
<a href=#198 id=198 data-nosnippet>198</a> <span class="prelude-val">Ok</span>(recordbatch) =&gt; Poll::Ready(<span class="prelude-val">Some</span>(<span class="prelude-val">Ok</span>(recordbatch.into_df_record_batch()))),
<a href=#199 id=199 data-nosnippet>199</a> <span class="prelude-val">Err</span>(e) =&gt; Poll::Ready(<span class="prelude-val">Some</span>(<span class="prelude-val">Err</span>(DataFusionError::External(Box::new(e))))),
<a href=#200 id=200 data-nosnippet>200</a> },
<a href=#201 id=201 data-nosnippet>201</a> Poll::Ready(<span class="prelude-val">None</span>) =&gt; Poll::Ready(<span class="prelude-val">None</span>),
<a href=#202 id=202 data-nosnippet>202</a> }
<a href=#203 id=203 data-nosnippet>203</a> }
<a href=#204 id=204 data-nosnippet>204</a>
<a href=#205 id=205 data-nosnippet>205</a> <span class="attr">#[inline]
<a href=#206 id=206 data-nosnippet>206</a> </span><span class="kw">fn </span>size_hint(<span class="kw-2">&amp;</span><span class="self">self</span>) -&gt; (usize, <span class="prelude-ty">Option</span>&lt;usize&gt;) {
<a href=#207 id=207 data-nosnippet>207</a> <span class="self">self</span>.stream.size_hint()
<a href=#208 id=208 data-nosnippet>208</a> }
<a href=#209 id=209 data-nosnippet>209</a>}
<a href=#210 id=210 data-nosnippet>210</a>
<a href=#211 id=211 data-nosnippet>211</a><span class="doccomment">/// DataFusion [SendableRecordBatchStream](DfSendableRecordBatchStream) -&gt; Greptime [RecordBatchStream].
<a href=#212 id=212 data-nosnippet>212</a>/// The reverse one is [DfRecordBatchStreamAdapter].
<a href=#213 id=213 data-nosnippet>213</a>/// It can collect metrics from DataFusion execution plan.
<a href=#214 id=214 data-nosnippet>214</a></span><span class="kw">pub struct </span>RecordBatchStreamAdapter {
<a href=#215 id=215 data-nosnippet>215</a> schema: SchemaRef,
<a href=#216 id=216 data-nosnippet>216</a> stream: DfSendableRecordBatchStream,
<a href=#217 id=217 data-nosnippet>217</a> metrics: <span class="prelude-ty">Option</span>&lt;BaselineMetrics&gt;,
<a href=#218 id=218 data-nosnippet>218</a> <span class="doccomment">/// Aggregated plan-level metrics. Resolved after an [ExecutionPlan] is finished.
<a href=#219 id=219 data-nosnippet>219</a> </span>metrics_2: Metrics,
<a href=#220 id=220 data-nosnippet>220</a> <span class="doccomment">/// Display plan and metrics in verbose mode.
<a href=#221 id=221 data-nosnippet>221</a> </span>explain_verbose: bool,
<a href=#222 id=222 data-nosnippet>222</a> span: Span,
<a href=#223 id=223 data-nosnippet>223</a>}
<a href=#224 id=224 data-nosnippet>224</a>
<a href=#225 id=225 data-nosnippet>225</a><span class="doccomment">/// Json encoded metrics. Contains metric from a whole plan tree.
<a href=#226 id=226 data-nosnippet>226</a></span><span class="kw">enum </span>Metrics {
<a href=#227 id=227 data-nosnippet>227</a> Unavailable,
<a href=#228 id=228 data-nosnippet>228</a> Unresolved(Arc&lt;<span class="kw">dyn </span>ExecutionPlan&gt;),
<a href=#229 id=229 data-nosnippet>229</a> PartialResolved(Arc&lt;<span class="kw">dyn </span>ExecutionPlan&gt;, RecordBatchMetrics),
<a href=#230 id=230 data-nosnippet>230</a> Resolved(RecordBatchMetrics),
<a href=#231 id=231 data-nosnippet>231</a>}
<a href=#232 id=232 data-nosnippet>232</a>
<a href=#233 id=233 data-nosnippet>233</a><span class="kw">impl </span>RecordBatchStreamAdapter {
<a href=#234 id=234 data-nosnippet>234</a> <span class="kw">pub fn </span>try_new(stream: DfSendableRecordBatchStream) -&gt; <span class="prelude-ty">Result</span>&lt;<span class="self">Self</span>&gt; {
<a href=#235 id=235 data-nosnippet>235</a> <span class="kw">let </span>schema =
<a href=#236 id=236 data-nosnippet>236</a> Arc::new(Schema::try_from(stream.schema()).context(error::SchemaConversionSnafu)<span class="question-mark">?</span>);
<a href=#237 id=237 data-nosnippet>237</a> <span class="prelude-val">Ok</span>(<span class="self">Self </span>{
<a href=#238 id=238 data-nosnippet>238</a> schema,
<a href=#239 id=239 data-nosnippet>239</a> stream,
<a href=#240 id=240 data-nosnippet>240</a> metrics: <span class="prelude-val">None</span>,
<a href=#241 id=241 data-nosnippet>241</a> metrics_2: Metrics::Unavailable,
<a href=#242 id=242 data-nosnippet>242</a> explain_verbose: <span class="bool-val">false</span>,
<a href=#243 id=243 data-nosnippet>243</a> span: Span::current(),
<a href=#244 id=244 data-nosnippet>244</a> })
<a href=#245 id=245 data-nosnippet>245</a> }
<a href=#246 id=246 data-nosnippet>246</a>
<a href=#247 id=247 data-nosnippet>247</a> <span class="kw">pub fn </span>try_new_with_span(stream: DfSendableRecordBatchStream, span: Span) -&gt; <span class="prelude-ty">Result</span>&lt;<span class="self">Self</span>&gt; {
<a href=#248 id=248 data-nosnippet>248</a> <span class="kw">let </span>schema =
<a href=#249 id=249 data-nosnippet>249</a> Arc::new(Schema::try_from(stream.schema()).context(error::SchemaConversionSnafu)<span class="question-mark">?</span>);
<a href=#250 id=250 data-nosnippet>250</a> <span class="kw">let </span>subspan = <span class="macro">info_span!</span>(parent: <span class="kw-2">&amp;</span>span, <span class="string">"RecordBatchStreamAdapter"</span>);
<a href=#251 id=251 data-nosnippet>251</a> <span class="prelude-val">Ok</span>(<span class="self">Self </span>{
<a href=#252 id=252 data-nosnippet>252</a> schema,
<a href=#253 id=253 data-nosnippet>253</a> stream,
<a href=#254 id=254 data-nosnippet>254</a> metrics: <span class="prelude-val">None</span>,
<a href=#255 id=255 data-nosnippet>255</a> metrics_2: Metrics::Unavailable,
<a href=#256 id=256 data-nosnippet>256</a> explain_verbose: <span class="bool-val">false</span>,
<a href=#257 id=257 data-nosnippet>257</a> span: subspan,
<a href=#258 id=258 data-nosnippet>258</a> })
<a href=#259 id=259 data-nosnippet>259</a> }
<a href=#260 id=260 data-nosnippet>260</a>
<a href=#261 id=261 data-nosnippet>261</a> <span class="kw">pub fn </span>set_metrics2(<span class="kw-2">&amp;mut </span><span class="self">self</span>, plan: Arc&lt;<span class="kw">dyn </span>ExecutionPlan&gt;) {
<a href=#262 id=262 data-nosnippet>262</a> <span class="self">self</span>.metrics_2 = Metrics::Unresolved(plan)
<a href=#263 id=263 data-nosnippet>263</a> }
<a href=#264 id=264 data-nosnippet>264</a>
<a href=#265 id=265 data-nosnippet>265</a> <span class="doccomment">/// Set the verbose mode for displaying plan and metrics.
<a href=#266 id=266 data-nosnippet>266</a> </span><span class="kw">pub fn </span>set_explain_verbose(<span class="kw-2">&amp;mut </span><span class="self">self</span>, verbose: bool) {
<a href=#267 id=267 data-nosnippet>267</a> <span class="self">self</span>.explain_verbose = verbose;
<a href=#268 id=268 data-nosnippet>268</a> }
<a href=#269 id=269 data-nosnippet>269</a>}
<a href=#270 id=270 data-nosnippet>270</a>
<a href=#271 id=271 data-nosnippet>271</a><span class="kw">impl </span>RecordBatchStream <span class="kw">for </span>RecordBatchStreamAdapter {
<a href=#272 id=272 data-nosnippet>272</a> <span class="kw">fn </span>name(<span class="kw-2">&amp;</span><span class="self">self</span>) -&gt; <span class="kw-2">&amp;</span>str {
<a href=#273 id=273 data-nosnippet>273</a> <span class="string">"RecordBatchStreamAdapter"
<a href=#274 id=274 data-nosnippet>274</a> </span>}
<a href=#275 id=275 data-nosnippet>275</a>
<a href=#276 id=276 data-nosnippet>276</a> <span class="kw">fn </span>schema(<span class="kw-2">&amp;</span><span class="self">self</span>) -&gt; SchemaRef {
<a href=#277 id=277 data-nosnippet>277</a> <span class="self">self</span>.schema.clone()
<a href=#278 id=278 data-nosnippet>278</a> }
<a href=#279 id=279 data-nosnippet>279</a>
<a href=#280 id=280 data-nosnippet>280</a> <span class="kw">fn </span>metrics(<span class="kw-2">&amp;</span><span class="self">self</span>) -&gt; <span class="prelude-ty">Option</span>&lt;RecordBatchMetrics&gt; {
<a href=#281 id=281 data-nosnippet>281</a> <span class="kw">match </span><span class="kw-2">&amp;</span><span class="self">self</span>.metrics_2 {
<a href=#282 id=282 data-nosnippet>282</a> Metrics::Resolved(metrics) | Metrics::PartialResolved(<span class="kw">_</span>, metrics) =&gt; {
<a href=#283 id=283 data-nosnippet>283</a> <span class="prelude-val">Some</span>(metrics.clone())
<a href=#284 id=284 data-nosnippet>284</a> }
<a href=#285 id=285 data-nosnippet>285</a> Metrics::Unavailable | Metrics::Unresolved(<span class="kw">_</span>) =&gt; <span class="prelude-val">None</span>,
<a href=#286 id=286 data-nosnippet>286</a> }
<a href=#287 id=287 data-nosnippet>287</a> }
<a href=#288 id=288 data-nosnippet>288</a>
<a href=#289 id=289 data-nosnippet>289</a> <span class="kw">fn </span>output_ordering(<span class="kw-2">&amp;</span><span class="self">self</span>) -&gt; <span class="prelude-ty">Option</span>&lt;<span class="kw-2">&amp;</span>[OrderOption]&gt; {
<a href=#290 id=290 data-nosnippet>290</a> <span class="prelude-val">None
<a href=#291 id=291 data-nosnippet>291</a> </span>}
<a href=#292 id=292 data-nosnippet>292</a>}
<a href=#293 id=293 data-nosnippet>293</a>
<a href=#294 id=294 data-nosnippet>294</a><span class="kw">impl </span>Stream <span class="kw">for </span>RecordBatchStreamAdapter {
<a href=#295 id=295 data-nosnippet>295</a> <span class="kw">type </span>Item = <span class="prelude-ty">Result</span>&lt;RecordBatch&gt;;
<a href=#296 id=296 data-nosnippet>296</a>
<a href=#297 id=297 data-nosnippet>297</a> <span class="kw">fn </span>poll_next(<span class="kw-2">mut </span><span class="self">self</span>: Pin&lt;<span class="kw-2">&amp;mut </span><span class="self">Self</span>&gt;, cx: <span class="kw-2">&amp;mut </span>Context&lt;<span class="lifetime">'_</span>&gt;) -&gt; Poll&lt;<span class="prelude-ty">Option</span>&lt;<span class="self">Self</span>::Item&gt;&gt; {
<a href=#298 id=298 data-nosnippet>298</a> <span class="kw">let </span>timer = <span class="self">self
<a href=#299 id=299 data-nosnippet>299</a> </span>.metrics
<a href=#300 id=300 data-nosnippet>300</a> .as_ref()
<a href=#301 id=301 data-nosnippet>301</a> .map(|m| m.elapsed_compute().clone())
<a href=#302 id=302 data-nosnippet>302</a> .unwrap_or_default();
<a href=#303 id=303 data-nosnippet>303</a> <span class="kw">let </span>_guard = timer.timer();
<a href=#304 id=304 data-nosnippet>304</a> <span class="kw">let </span>poll_span = <span class="macro">info_span!</span>(parent: <span class="kw-2">&amp;</span><span class="self">self</span>.span, <span class="string">"poll_next"</span>);
<a href=#305 id=305 data-nosnippet>305</a> <span class="kw">let </span>_entered = poll_span.enter();
<a href=#306 id=306 data-nosnippet>306</a> <span class="kw">match </span>Pin::new(<span class="kw-2">&amp;mut </span><span class="self">self</span>.stream).poll_next(cx) {
<a href=#307 id=307 data-nosnippet>307</a> Poll::Pending =&gt; Poll::Pending,
<a href=#308 id=308 data-nosnippet>308</a> Poll::Ready(<span class="prelude-val">Some</span>(df_record_batch)) =&gt; {
<a href=#309 id=309 data-nosnippet>309</a> <span class="kw">let </span>df_record_batch = df_record_batch<span class="question-mark">?</span>;
<a href=#310 id=310 data-nosnippet>310</a> <span class="kw">if let </span>Metrics::Unresolved(df_plan) | Metrics::PartialResolved(df_plan, <span class="kw">_</span>) =
<a href=#311 id=311 data-nosnippet>311</a> <span class="kw-2">&amp;</span><span class="self">self</span>.metrics_2
<a href=#312 id=312 data-nosnippet>312</a> {
<a href=#313 id=313 data-nosnippet>313</a> <span class="kw">let </span><span class="kw-2">mut </span>metric_collector = MetricCollector::new(<span class="self">self</span>.explain_verbose);
<a href=#314 id=314 data-nosnippet>314</a> accept(df_plan.as_ref(), <span class="kw-2">&amp;mut </span>metric_collector).unwrap();
<a href=#315 id=315 data-nosnippet>315</a> <span class="self">self</span>.metrics_2 = Metrics::PartialResolved(
<a href=#316 id=316 data-nosnippet>316</a> df_plan.clone(),
<a href=#317 id=317 data-nosnippet>317</a> metric_collector.record_batch_metrics,
<a href=#318 id=318 data-nosnippet>318</a> );
<a href=#319 id=319 data-nosnippet>319</a> }
<a href=#320 id=320 data-nosnippet>320</a> Poll::Ready(<span class="prelude-val">Some</span>(<span class="prelude-val">Ok</span>(RecordBatch::from_df_record_batch(
<a href=#321 id=321 data-nosnippet>321</a> <span class="self">self</span>.schema(),
<a href=#322 id=322 data-nosnippet>322</a> df_record_batch,
<a href=#323 id=323 data-nosnippet>323</a> ))))
<a href=#324 id=324 data-nosnippet>324</a> }
<a href=#325 id=325 data-nosnippet>325</a> Poll::Ready(<span class="prelude-val">None</span>) =&gt; {
<a href=#326 id=326 data-nosnippet>326</a> <span class="kw">if let </span>Metrics::Unresolved(df_plan) | Metrics::PartialResolved(df_plan, <span class="kw">_</span>) =
<a href=#327 id=327 data-nosnippet>327</a> <span class="kw-2">&amp;</span><span class="self">self</span>.metrics_2
<a href=#328 id=328 data-nosnippet>328</a> {
<a href=#329 id=329 data-nosnippet>329</a> <span class="kw">let </span><span class="kw-2">mut </span>metric_collector = MetricCollector::new(<span class="self">self</span>.explain_verbose);
<a href=#330 id=330 data-nosnippet>330</a> accept(df_plan.as_ref(), <span class="kw-2">&amp;mut </span>metric_collector).unwrap();
<a href=#331 id=331 data-nosnippet>331</a> <span class="self">self</span>.metrics_2 = Metrics::Resolved(metric_collector.record_batch_metrics);
<a href=#332 id=332 data-nosnippet>332</a> }
<a href=#333 id=333 data-nosnippet>333</a> Poll::Ready(<span class="prelude-val">None</span>)
<a href=#334 id=334 data-nosnippet>334</a> }
<a href=#335 id=335 data-nosnippet>335</a> }
<a href=#336 id=336 data-nosnippet>336</a> }
<a href=#337 id=337 data-nosnippet>337</a>
<a href=#338 id=338 data-nosnippet>338</a> <span class="attr">#[inline]
<a href=#339 id=339 data-nosnippet>339</a> </span><span class="kw">fn </span>size_hint(<span class="kw-2">&amp;</span><span class="self">self</span>) -&gt; (usize, <span class="prelude-ty">Option</span>&lt;usize&gt;) {
<a href=#340 id=340 data-nosnippet>340</a> <span class="self">self</span>.stream.size_hint()
<a href=#341 id=341 data-nosnippet>341</a> }
<a href=#342 id=342 data-nosnippet>342</a>}
<a href=#343 id=343 data-nosnippet>343</a>
<a href=#344 id=344 data-nosnippet>344</a><span class="doccomment">/// An [ExecutionPlanVisitor] to collect metrics from a [ExecutionPlan].
<a href=#345 id=345 data-nosnippet>345</a></span><span class="kw">pub struct </span>MetricCollector {
<a href=#346 id=346 data-nosnippet>346</a> current_level: usize,
<a href=#347 id=347 data-nosnippet>347</a> <span class="kw">pub </span>record_batch_metrics: RecordBatchMetrics,
<a href=#348 id=348 data-nosnippet>348</a> verbose: bool,
<a href=#349 id=349 data-nosnippet>349</a>}
<a href=#350 id=350 data-nosnippet>350</a>
<a href=#351 id=351 data-nosnippet>351</a><span class="kw">impl </span>MetricCollector {
<a href=#352 id=352 data-nosnippet>352</a> <span class="kw">pub fn </span>new(verbose: bool) -&gt; <span class="self">Self </span>{
<a href=#353 id=353 data-nosnippet>353</a> <span class="self">Self </span>{
<a href=#354 id=354 data-nosnippet>354</a> current_level: <span class="number">0</span>,
<a href=#355 id=355 data-nosnippet>355</a> record_batch_metrics: RecordBatchMetrics::default(),
<a href=#356 id=356 data-nosnippet>356</a> verbose,
<a href=#357 id=357 data-nosnippet>357</a> }
<a href=#358 id=358 data-nosnippet>358</a> }
<a href=#359 id=359 data-nosnippet>359</a>}
<a href=#360 id=360 data-nosnippet>360</a>
<a href=#361 id=361 data-nosnippet>361</a><span class="kw">impl </span>ExecutionPlanVisitor <span class="kw">for </span>MetricCollector {
<a href=#362 id=362 data-nosnippet>362</a> <span class="kw">type </span>Error = !;
<a href=#363 id=363 data-nosnippet>363</a>
<a href=#364 id=364 data-nosnippet>364</a> <span class="kw">fn </span>pre_visit(<span class="kw-2">&amp;mut </span><span class="self">self</span>, plan: <span class="kw-2">&amp;</span><span class="kw">dyn </span>ExecutionPlan) -&gt; std::result::Result&lt;bool, <span class="self">Self</span>::Error&gt; {
<a href=#365 id=365 data-nosnippet>365</a> <span class="comment">// skip if no metric available
<a href=#366 id=366 data-nosnippet>366</a> </span><span class="kw">let </span><span class="prelude-val">Some</span>(metric) = plan.metrics() <span class="kw">else </span>{
<a href=#367 id=367 data-nosnippet>367</a> <span class="self">self</span>.record_batch_metrics.plan_metrics.push(PlanMetrics {
<a href=#368 id=368 data-nosnippet>368</a> plan: plan.name().to_string(),
<a href=#369 id=369 data-nosnippet>369</a> level: <span class="self">self</span>.current_level,
<a href=#370 id=370 data-nosnippet>370</a> metrics: <span class="macro">vec!</span>[],
<a href=#371 id=371 data-nosnippet>371</a> });
<a href=#372 id=372 data-nosnippet>372</a> <span class="self">self</span>.current_level += <span class="number">1</span>;
<a href=#373 id=373 data-nosnippet>373</a> <span class="kw">return </span><span class="prelude-val">Ok</span>(<span class="bool-val">true</span>);
<a href=#374 id=374 data-nosnippet>374</a> };
<a href=#375 id=375 data-nosnippet>375</a>
<a href=#376 id=376 data-nosnippet>376</a> <span class="comment">// scrape plan metrics
<a href=#377 id=377 data-nosnippet>377</a> </span><span class="kw">let </span>metric = metric
<a href=#378 id=378 data-nosnippet>378</a> .aggregate_by_name()
<a href=#379 id=379 data-nosnippet>379</a> .sorted_for_display()
<a href=#380 id=380 data-nosnippet>380</a> .timestamps_removed();
<a href=#381 id=381 data-nosnippet>381</a> <span class="kw">let </span><span class="kw-2">mut </span>plan_metric = PlanMetrics {
<a href=#382 id=382 data-nosnippet>382</a> plan: one_line(plan, <span class="self">self</span>.verbose).to_string(),
<a href=#383 id=383 data-nosnippet>383</a> level: <span class="self">self</span>.current_level,
<a href=#384 id=384 data-nosnippet>384</a> metrics: Vec::with_capacity(metric.iter().size_hint().<span class="number">0</span>),
<a href=#385 id=385 data-nosnippet>385</a> };
<a href=#386 id=386 data-nosnippet>386</a> <span class="kw">for </span>m <span class="kw">in </span>metric.iter() {
<a href=#387 id=387 data-nosnippet>387</a> plan_metric
<a href=#388 id=388 data-nosnippet>388</a> .metrics
<a href=#389 id=389 data-nosnippet>389</a> .push((m.value().name().to_string(), m.value().as_usize()));
<a href=#390 id=390 data-nosnippet>390</a>
<a href=#391 id=391 data-nosnippet>391</a> <span class="comment">// aggregate high-level metrics
<a href=#392 id=392 data-nosnippet>392</a> </span><span class="kw">match </span>m.value() {
<a href=#393 id=393 data-nosnippet>393</a> MetricValue::ElapsedCompute(ec) =&gt; {
<a href=#394 id=394 data-nosnippet>394</a> <span class="self">self</span>.record_batch_metrics.elapsed_compute += ec.value()
<a href=#395 id=395 data-nosnippet>395</a> }
<a href=#396 id=396 data-nosnippet>396</a> MetricValue::CurrentMemoryUsage(m) =&gt; {
<a href=#397 id=397 data-nosnippet>397</a> <span class="self">self</span>.record_batch_metrics.memory_usage += m.value()
<a href=#398 id=398 data-nosnippet>398</a> }
<a href=#399 id=399 data-nosnippet>399</a> <span class="kw">_ </span>=&gt; {}
<a href=#400 id=400 data-nosnippet>400</a> }
<a href=#401 id=401 data-nosnippet>401</a> }
<a href=#402 id=402 data-nosnippet>402</a> <span class="self">self</span>.record_batch_metrics.plan_metrics.push(plan_metric);
<a href=#403 id=403 data-nosnippet>403</a>
<a href=#404 id=404 data-nosnippet>404</a> <span class="self">self</span>.current_level += <span class="number">1</span>;
<a href=#405 id=405 data-nosnippet>405</a> <span class="prelude-val">Ok</span>(<span class="bool-val">true</span>)
<a href=#406 id=406 data-nosnippet>406</a> }
<a href=#407 id=407 data-nosnippet>407</a>
<a href=#408 id=408 data-nosnippet>408</a> <span class="kw">fn </span>post_visit(<span class="kw-2">&amp;mut </span><span class="self">self</span>, _plan: <span class="kw-2">&amp;</span><span class="kw">dyn </span>ExecutionPlan) -&gt; std::result::Result&lt;bool, <span class="self">Self</span>::Error&gt; {
<a href=#409 id=409 data-nosnippet>409</a> <span class="self">self</span>.current_level -= <span class="number">1</span>;
<a href=#410 id=410 data-nosnippet>410</a> <span class="prelude-val">Ok</span>(<span class="bool-val">true</span>)
<a href=#411 id=411 data-nosnippet>411</a> }
<a href=#412 id=412 data-nosnippet>412</a>}
<a href=#413 id=413 data-nosnippet>413</a>
<a href=#414 id=414 data-nosnippet>414</a><span class="doccomment">/// Returns a single-line summary of the root of the plan.
<a href=#415 id=415 data-nosnippet>415</a>/// If the `verbose` flag is set, it will display detailed information about the plan.
<a href=#416 id=416 data-nosnippet>416</a></span><span class="kw">fn </span>one_line(plan: <span class="kw-2">&amp;</span><span class="kw">dyn </span>ExecutionPlan, verbose: bool) -&gt; <span class="kw">impl </span>fmt::Display + <span class="lifetime">'_ </span>{
<a href=#417 id=417 data-nosnippet>417</a> <span class="kw">struct </span>Wrapper&lt;<span class="lifetime">'a</span>&gt; {
<a href=#418 id=418 data-nosnippet>418</a> plan: <span class="kw-2">&amp;</span><span class="lifetime">'a </span><span class="kw">dyn </span>ExecutionPlan,
<a href=#419 id=419 data-nosnippet>419</a> format_type: DisplayFormatType,
<a href=#420 id=420 data-nosnippet>420</a> }
<a href=#421 id=421 data-nosnippet>421</a>
<a href=#422 id=422 data-nosnippet>422</a> <span class="kw">impl </span>fmt::Display <span class="kw">for </span>Wrapper&lt;<span class="lifetime">'_</span>&gt; {
<a href=#423 id=423 data-nosnippet>423</a> <span class="kw">fn </span>fmt(<span class="kw-2">&amp;</span><span class="self">self</span>, f: <span class="kw-2">&amp;mut </span>fmt::Formatter&lt;<span class="lifetime">'_</span>&gt;) -&gt; fmt::Result {
<a href=#424 id=424 data-nosnippet>424</a> <span class="self">self</span>.plan.fmt_as(<span class="self">self</span>.format_type, f)<span class="question-mark">?</span>;
<a href=#425 id=425 data-nosnippet>425</a> <span class="macro">writeln!</span>(f)
<a href=#426 id=426 data-nosnippet>426</a> }
<a href=#427 id=427 data-nosnippet>427</a> }
<a href=#428 id=428 data-nosnippet>428</a>
<a href=#429 id=429 data-nosnippet>429</a> <span class="kw">let </span>format_type = <span class="kw">if </span>verbose {
<a href=#430 id=430 data-nosnippet>430</a> DisplayFormatType::Verbose
<a href=#431 id=431 data-nosnippet>431</a> } <span class="kw">else </span>{
<a href=#432 id=432 data-nosnippet>432</a> DisplayFormatType::Default
<a href=#433 id=433 data-nosnippet>433</a> };
<a href=#434 id=434 data-nosnippet>434</a> Wrapper { plan, format_type }
<a href=#435 id=435 data-nosnippet>435</a>}
<a href=#436 id=436 data-nosnippet>436</a>
<a href=#437 id=437 data-nosnippet>437</a><span class="doccomment">/// [`RecordBatchMetrics`] carrys metrics value
<a href=#438 id=438 data-nosnippet>438</a>/// from datanode to frontend through gRPC
<a href=#439 id=439 data-nosnippet>439</a></span><span class="attr">#[derive(serde::Serialize, serde::Deserialize, Default, Debug, Clone)]
<a href=#440 id=440 data-nosnippet>440</a></span><span class="kw">pub struct </span>RecordBatchMetrics {
<a href=#441 id=441 data-nosnippet>441</a> <span class="comment">// High-level aggregated metrics
<a href=#442 id=442 data-nosnippet>442</a> </span><span class="doccomment">/// CPU consumption in nanoseconds
<a href=#443 id=443 data-nosnippet>443</a> </span><span class="kw">pub </span>elapsed_compute: usize,
<a href=#444 id=444 data-nosnippet>444</a> <span class="doccomment">/// Memory used by the plan in bytes
<a href=#445 id=445 data-nosnippet>445</a> </span><span class="kw">pub </span>memory_usage: usize,
<a href=#446 id=446 data-nosnippet>446</a> <span class="comment">// Detailed per-plan metrics
<a href=#447 id=447 data-nosnippet>447</a> </span><span class="doccomment">/// An ordered list of plan metrics, from top to bottom in post-order.
<a href=#448 id=448 data-nosnippet>448</a> </span><span class="kw">pub </span>plan_metrics: Vec&lt;PlanMetrics&gt;,
<a href=#449 id=449 data-nosnippet>449</a>}
<a href=#450 id=450 data-nosnippet>450</a>
<a href=#451 id=451 data-nosnippet>451</a><span class="doccomment">/// Determines if a metric name represents a time measurement that should be formatted.
<a href=#452 id=452 data-nosnippet>452</a></span><span class="kw">fn </span>is_time_metric(metric_name: <span class="kw-2">&amp;</span>str) -&gt; bool {
<a href=#453 id=453 data-nosnippet>453</a> metric_name.contains(<span class="string">"elapsed"</span>) || metric_name.contains(<span class="string">"time"</span>) || metric_name.contains(<span class="string">"cost"</span>)
<a href=#454 id=454 data-nosnippet>454</a>}
<a href=#455 id=455 data-nosnippet>455</a>
<a href=#456 id=456 data-nosnippet>456</a><span class="doccomment">/// Determines if a metric name represents a bytes measurement that should be formatted.
<a href=#457 id=457 data-nosnippet>457</a></span><span class="kw">fn </span>is_bytes_metric(metric_name: <span class="kw-2">&amp;</span>str) -&gt; bool {
<a href=#458 id=458 data-nosnippet>458</a> metric_name.contains(<span class="string">"bytes"</span>) || metric_name.contains(<span class="string">"mem"</span>)
<a href=#459 id=459 data-nosnippet>459</a>}
<a href=#460 id=460 data-nosnippet>460</a>
<a href=#461 id=461 data-nosnippet>461</a><span class="kw">fn </span>format_bytes_human_readable(bytes: usize) -&gt; String {
<a href=#462 id=462 data-nosnippet>462</a> <span class="macro">format!</span>(<span class="string">"{}"</span>, ReadableSize(bytes <span class="kw">as </span>u64))
<a href=#463 id=463 data-nosnippet>463</a>}
<a href=#464 id=464 data-nosnippet>464</a>
<a href=#465 id=465 data-nosnippet>465</a><span class="doccomment">/// Only display `plan_metrics` with indent ` ` (2 spaces).
<a href=#466 id=466 data-nosnippet>466</a></span><span class="kw">impl </span>Display <span class="kw">for </span>RecordBatchMetrics {
<a href=#467 id=467 data-nosnippet>467</a> <span class="kw">fn </span>fmt(<span class="kw-2">&amp;</span><span class="self">self</span>, f: <span class="kw-2">&amp;mut </span>std::fmt::Formatter&lt;<span class="lifetime">'_</span>&gt;) -&gt; std::fmt::Result {
<a href=#468 id=468 data-nosnippet>468</a> <span class="kw">for </span>metric <span class="kw">in </span><span class="kw-2">&amp;</span><span class="self">self</span>.plan_metrics {
<a href=#469 id=469 data-nosnippet>469</a> <span class="macro">write!</span>(
<a href=#470 id=470 data-nosnippet>470</a> f,
<a href=#471 id=471 data-nosnippet>471</a> <span class="string">"{:indent$}{} metrics=["</span>,
<a href=#472 id=472 data-nosnippet>472</a> <span class="string">" "</span>,
<a href=#473 id=473 data-nosnippet>473</a> metric.plan.trim_end(),
<a href=#474 id=474 data-nosnippet>474</a> indent = metric.level * <span class="number">2</span>,
<a href=#475 id=475 data-nosnippet>475</a> )<span class="question-mark">?</span>;
<a href=#476 id=476 data-nosnippet>476</a> <span class="kw">for </span>(label, value) <span class="kw">in </span><span class="kw-2">&amp;</span>metric.metrics {
<a href=#477 id=477 data-nosnippet>477</a> <span class="kw">if </span>is_time_metric(label) {
<a href=#478 id=478 data-nosnippet>478</a> <span class="macro">write!</span>(
<a href=#479 id=479 data-nosnippet>479</a> f,
<a href=#480 id=480 data-nosnippet>480</a> <span class="string">"{}: {}, "</span>,
<a href=#481 id=481 data-nosnippet>481</a> label,
<a href=#482 id=482 data-nosnippet>482</a> format_nanoseconds_human_readable(<span class="kw-2">*</span>value),
<a href=#483 id=483 data-nosnippet>483</a> )<span class="question-mark">?</span>;
<a href=#484 id=484 data-nosnippet>484</a> } <span class="kw">else if </span>is_bytes_metric(label) {
<a href=#485 id=485 data-nosnippet>485</a> <span class="macro">write!</span>(f, <span class="string">"{}: {}, "</span>, label, format_bytes_human_readable(<span class="kw-2">*</span>value),)<span class="question-mark">?</span>;
<a href=#486 id=486 data-nosnippet>486</a> } <span class="kw">else </span>{
<a href=#487 id=487 data-nosnippet>487</a> <span class="macro">write!</span>(f, <span class="string">"{}: {}, "</span>, label, value)<span class="question-mark">?</span>;
<a href=#488 id=488 data-nosnippet>488</a> }
<a href=#489 id=489 data-nosnippet>489</a> }
<a href=#490 id=490 data-nosnippet>490</a> <span class="macro">writeln!</span>(f, <span class="string">"]"</span>)<span class="question-mark">?</span>;
<a href=#491 id=491 data-nosnippet>491</a> }
<a href=#492 id=492 data-nosnippet>492</a>
<a href=#493 id=493 data-nosnippet>493</a> <span class="prelude-val">Ok</span>(())
<a href=#494 id=494 data-nosnippet>494</a> }
<a href=#495 id=495 data-nosnippet>495</a>}
<a href=#496 id=496 data-nosnippet>496</a>
<a href=#497 id=497 data-nosnippet>497</a><span class="attr">#[derive(serde::Serialize, serde::Deserialize, Default, Debug, Clone)]
<a href=#498 id=498 data-nosnippet>498</a></span><span class="kw">pub struct </span>PlanMetrics {
<a href=#499 id=499 data-nosnippet>499</a> <span class="doccomment">/// The plan name
<a href=#500 id=500 data-nosnippet>500</a> </span><span class="kw">pub </span>plan: String,
<a href=#501 id=501 data-nosnippet>501</a> <span class="doccomment">/// The level of the plan, starts from 0
<a href=#502 id=502 data-nosnippet>502</a> </span><span class="kw">pub </span>level: usize,
<a href=#503 id=503 data-nosnippet>503</a> <span class="doccomment">/// An ordered key-value list of metrics.
<a href=#504 id=504 data-nosnippet>504</a> /// Key is metric label and value is metric value.
<a href=#505 id=505 data-nosnippet>505</a> </span><span class="kw">pub </span>metrics: Vec&lt;(String, usize)&gt;,
<a href=#506 id=506 data-nosnippet>506</a>}
<a href=#507 id=507 data-nosnippet>507</a>
<a href=#508 id=508 data-nosnippet>508</a><span class="kw">enum </span>AsyncRecordBatchStreamAdapterState {
<a href=#509 id=509 data-nosnippet>509</a> Uninit(FutureStream),
<a href=#510 id=510 data-nosnippet>510</a> Ready(SendableRecordBatchStream),
<a href=#511 id=511 data-nosnippet>511</a> Failed,
<a href=#512 id=512 data-nosnippet>512</a>}
<a href=#513 id=513 data-nosnippet>513</a>
<a href=#514 id=514 data-nosnippet>514</a><span class="kw">pub struct </span>AsyncRecordBatchStreamAdapter {
<a href=#515 id=515 data-nosnippet>515</a> schema: SchemaRef,
<a href=#516 id=516 data-nosnippet>516</a> state: AsyncRecordBatchStreamAdapterState,
<a href=#517 id=517 data-nosnippet>517</a>}
<a href=#518 id=518 data-nosnippet>518</a>
<a href=#519 id=519 data-nosnippet>519</a><span class="kw">impl </span>AsyncRecordBatchStreamAdapter {
<a href=#520 id=520 data-nosnippet>520</a> <span class="kw">pub fn </span>new(schema: SchemaRef, stream: FutureStream) -&gt; <span class="self">Self </span>{
<a href=#521 id=521 data-nosnippet>521</a> <span class="self">Self </span>{
<a href=#522 id=522 data-nosnippet>522</a> schema,
<a href=#523 id=523 data-nosnippet>523</a> state: AsyncRecordBatchStreamAdapterState::Uninit(stream),
<a href=#524 id=524 data-nosnippet>524</a> }
<a href=#525 id=525 data-nosnippet>525</a> }
<a href=#526 id=526 data-nosnippet>526</a>}
<a href=#527 id=527 data-nosnippet>527</a>
<a href=#528 id=528 data-nosnippet>528</a><span class="kw">impl </span>RecordBatchStream <span class="kw">for </span>AsyncRecordBatchStreamAdapter {
<a href=#529 id=529 data-nosnippet>529</a> <span class="kw">fn </span>schema(<span class="kw-2">&amp;</span><span class="self">self</span>) -&gt; SchemaRef {
<a href=#530 id=530 data-nosnippet>530</a> <span class="self">self</span>.schema.clone()
<a href=#531 id=531 data-nosnippet>531</a> }
<a href=#532 id=532 data-nosnippet>532</a>
<a href=#533 id=533 data-nosnippet>533</a> <span class="kw">fn </span>output_ordering(<span class="kw-2">&amp;</span><span class="self">self</span>) -&gt; <span class="prelude-ty">Option</span>&lt;<span class="kw-2">&amp;</span>[OrderOption]&gt; {
<a href=#534 id=534 data-nosnippet>534</a> <span class="prelude-val">None
<a href=#535 id=535 data-nosnippet>535</a> </span>}
<a href=#536 id=536 data-nosnippet>536</a>
<a href=#537 id=537 data-nosnippet>537</a> <span class="kw">fn </span>metrics(<span class="kw-2">&amp;</span><span class="self">self</span>) -&gt; <span class="prelude-ty">Option</span>&lt;RecordBatchMetrics&gt; {
<a href=#538 id=538 data-nosnippet>538</a> <span class="prelude-val">None
<a href=#539 id=539 data-nosnippet>539</a> </span>}
<a href=#540 id=540 data-nosnippet>540</a>}
<a href=#541 id=541 data-nosnippet>541</a>
<a href=#542 id=542 data-nosnippet>542</a><span class="kw">impl </span>Stream <span class="kw">for </span>AsyncRecordBatchStreamAdapter {
<a href=#543 id=543 data-nosnippet>543</a> <span class="kw">type </span>Item = <span class="prelude-ty">Result</span>&lt;RecordBatch&gt;;
<a href=#544 id=544 data-nosnippet>544</a>
<a href=#545 id=545 data-nosnippet>545</a> <span class="kw">fn </span>poll_next(<span class="kw-2">mut </span><span class="self">self</span>: Pin&lt;<span class="kw-2">&amp;mut </span><span class="self">Self</span>&gt;, cx: <span class="kw-2">&amp;mut </span>Context&lt;<span class="lifetime">'_</span>&gt;) -&gt; Poll&lt;<span class="prelude-ty">Option</span>&lt;<span class="self">Self</span>::Item&gt;&gt; {
<a href=#546 id=546 data-nosnippet>546</a> <span class="kw">loop </span>{
<a href=#547 id=547 data-nosnippet>547</a> <span class="kw">match </span><span class="kw-2">&amp;mut </span><span class="self">self</span>.state {
<a href=#548 id=548 data-nosnippet>548</a> AsyncRecordBatchStreamAdapterState::Uninit(stream_future) =&gt; {
<a href=#549 id=549 data-nosnippet>549</a> <span class="kw">match </span><span class="macro">ready!</span>(Pin::new(stream_future).poll(cx)) {
<a href=#550 id=550 data-nosnippet>550</a> <span class="prelude-val">Ok</span>(stream) =&gt; {
<a href=#551 id=551 data-nosnippet>551</a> <span class="self">self</span>.state = AsyncRecordBatchStreamAdapterState::Ready(stream);
<a href=#552 id=552 data-nosnippet>552</a> <span class="kw">continue</span>;
<a href=#553 id=553 data-nosnippet>553</a> }
<a href=#554 id=554 data-nosnippet>554</a> <span class="prelude-val">Err</span>(e) =&gt; {
<a href=#555 id=555 data-nosnippet>555</a> <span class="self">self</span>.state = AsyncRecordBatchStreamAdapterState::Failed;
<a href=#556 id=556 data-nosnippet>556</a> <span class="kw">return </span>Poll::Ready(<span class="prelude-val">Some</span>(<span class="prelude-val">Err</span>(e)));
<a href=#557 id=557 data-nosnippet>557</a> }
<a href=#558 id=558 data-nosnippet>558</a> };
<a href=#559 id=559 data-nosnippet>559</a> }
<a href=#560 id=560 data-nosnippet>560</a> AsyncRecordBatchStreamAdapterState::Ready(stream) =&gt; {
<a href=#561 id=561 data-nosnippet>561</a> <span class="kw">return </span>Poll::Ready(<span class="macro">ready!</span>(Pin::new(stream).poll_next(cx)));
<a href=#562 id=562 data-nosnippet>562</a> }
<a href=#563 id=563 data-nosnippet>563</a> AsyncRecordBatchStreamAdapterState::Failed =&gt; <span class="kw">return </span>Poll::Ready(<span class="prelude-val">None</span>),
<a href=#564 id=564 data-nosnippet>564</a> }
<a href=#565 id=565 data-nosnippet>565</a> }
<a href=#566 id=566 data-nosnippet>566</a> }
<a href=#567 id=567 data-nosnippet>567</a>
<a href=#568 id=568 data-nosnippet>568</a> <span class="comment">// This is not supported for lazy stream.
<a href=#569 id=569 data-nosnippet>569</a> </span><span class="attr">#[inline]
<a href=#570 id=570 data-nosnippet>570</a> </span><span class="kw">fn </span>size_hint(<span class="kw-2">&amp;</span><span class="self">self</span>) -&gt; (usize, <span class="prelude-ty">Option</span>&lt;usize&gt;) {
<a href=#571 id=571 data-nosnippet>571</a> (<span class="number">0</span>, <span class="prelude-val">None</span>)
<a href=#572 id=572 data-nosnippet>572</a> }
<a href=#573 id=573 data-nosnippet>573</a>}
<a href=#574 id=574 data-nosnippet>574</a>
<a href=#575 id=575 data-nosnippet>575</a><span class="doccomment">/// Custom cast function that handles Map -&gt; Binary (JSON) conversion
<a href=#576 id=576 data-nosnippet>576</a></span><span class="kw">fn </span>custom_cast(
<a href=#577 id=577 data-nosnippet>577</a> array: <span class="kw-2">&amp;</span><span class="kw">dyn </span>Array,
<a href=#578 id=578 data-nosnippet>578</a> target_type: <span class="kw-2">&amp;</span>ArrowDataType,
<a href=#579 id=579 data-nosnippet>579</a> extype: <span class="prelude-ty">Option</span>&lt;ColumnExtType&gt;,
<a href=#580 id=580 data-nosnippet>580</a>) -&gt; std::result::Result&lt;Arc&lt;<span class="kw">dyn </span>Array&gt;, ArrowError&gt; {
<a href=#581 id=581 data-nosnippet>581</a> <span class="kw">if let </span>ArrowDataType::Map(<span class="kw">_</span>, <span class="kw">_</span>) = array.data_type()
<a href=#582 id=582 data-nosnippet>582</a> &amp;&amp; <span class="kw">let </span>ArrowDataType::Binary = target_type
<a href=#583 id=583 data-nosnippet>583</a> {
<a href=#584 id=584 data-nosnippet>584</a> <span class="kw">return </span>convert_map_to_json_binary(array, extype);
<a href=#585 id=585 data-nosnippet>585</a> }
<a href=#586 id=586 data-nosnippet>586</a>
<a href=#587 id=587 data-nosnippet>587</a> cast(array, target_type)
<a href=#588 id=588 data-nosnippet>588</a>}
<a href=#589 id=589 data-nosnippet>589</a>
<a href=#590 id=590 data-nosnippet>590</a><span class="doccomment">/// Convert a Map array to a Binary array containing JSON data
<a href=#591 id=591 data-nosnippet>591</a></span><span class="kw">fn </span>convert_map_to_json_binary(
<a href=#592 id=592 data-nosnippet>592</a> array: <span class="kw-2">&amp;</span><span class="kw">dyn </span>Array,
<a href=#593 id=593 data-nosnippet>593</a> extype: <span class="prelude-ty">Option</span>&lt;ColumnExtType&gt;,
<a href=#594 id=594 data-nosnippet>594</a>) -&gt; std::result::Result&lt;Arc&lt;<span class="kw">dyn </span>Array&gt;, ArrowError&gt; {
<a href=#595 id=595 data-nosnippet>595</a> <span class="kw">use </span>datatypes::arrow::array::{BinaryArray, MapArray};
<a href=#596 id=596 data-nosnippet>596</a> <span class="kw">use </span>serde_json::Value;
<a href=#597 id=597 data-nosnippet>597</a>
<a href=#598 id=598 data-nosnippet>598</a> <span class="kw">let </span>map_array = array
<a href=#599 id=599 data-nosnippet>599</a> .as_any()
<a href=#600 id=600 data-nosnippet>600</a> .downcast_ref::&lt;MapArray&gt;()
<a href=#601 id=601 data-nosnippet>601</a> .ok_or_else(|| ArrowError::CastError(<span class="string">"Failed to downcast to MapArray"</span>.to_string()))<span class="question-mark">?</span>;
<a href=#602 id=602 data-nosnippet>602</a>
<a href=#603 id=603 data-nosnippet>603</a> <span class="kw">let </span><span class="kw-2">mut </span>json_values = Vec::with_capacity(map_array.len());
<a href=#604 id=604 data-nosnippet>604</a>
<a href=#605 id=605 data-nosnippet>605</a> <span class="kw">for </span>i <span class="kw">in </span><span class="number">0</span>..map_array.len() {
<a href=#606 id=606 data-nosnippet>606</a> <span class="kw">if </span>map_array.is_null(i) {
<a href=#607 id=607 data-nosnippet>607</a> json_values.push(<span class="prelude-val">None</span>);
<a href=#608 id=608 data-nosnippet>608</a> } <span class="kw">else </span>{
<a href=#609 id=609 data-nosnippet>609</a> <span class="comment">// Extract the map entry at index i
<a href=#610 id=610 data-nosnippet>610</a> </span><span class="kw">let </span>map_entry = map_array.value(i);
<a href=#611 id=611 data-nosnippet>611</a> <span class="kw">let </span>key_value_array = map_entry
<a href=#612 id=612 data-nosnippet>612</a> .as_any()
<a href=#613 id=613 data-nosnippet>613</a> .downcast_ref::&lt;datatypes::arrow::array::StructArray&gt;()
<a href=#614 id=614 data-nosnippet>614</a> .ok_or_else(|| {
<a href=#615 id=615 data-nosnippet>615</a> ArrowError::CastError(<span class="string">"Failed to downcast to StructArray"</span>.to_string())
<a href=#616 id=616 data-nosnippet>616</a> })<span class="question-mark">?</span>;
<a href=#617 id=617 data-nosnippet>617</a>
<a href=#618 id=618 data-nosnippet>618</a> <span class="comment">// Convert to JSON object
<a href=#619 id=619 data-nosnippet>619</a> </span><span class="kw">let </span><span class="kw-2">mut </span>json_obj = serde_json::Map::with_capacity(key_value_array.len());
<a href=#620 id=620 data-nosnippet>620</a>
<a href=#621 id=621 data-nosnippet>621</a> <span class="kw">for </span>j <span class="kw">in </span><span class="number">0</span>..key_value_array.len() {
<a href=#622 id=622 data-nosnippet>622</a> <span class="kw">if </span>key_value_array.is_null(j) {
<a href=#623 id=623 data-nosnippet>623</a> <span class="kw">continue</span>;
<a href=#624 id=624 data-nosnippet>624</a> }
<a href=#625 id=625 data-nosnippet>625</a> <span class="kw">let </span>key_field = key_value_array.column(<span class="number">0</span>);
<a href=#626 id=626 data-nosnippet>626</a> <span class="kw">let </span>value_field = key_value_array.column(<span class="number">1</span>);
<a href=#627 id=627 data-nosnippet>627</a>
<a href=#628 id=628 data-nosnippet>628</a> <span class="kw">if </span>key_field.is_null(j) {
<a href=#629 id=629 data-nosnippet>629</a> <span class="kw">continue</span>;
<a href=#630 id=630 data-nosnippet>630</a> }
<a href=#631 id=631 data-nosnippet>631</a>
<a href=#632 id=632 data-nosnippet>632</a> <span class="kw">let </span>key = key_field
<a href=#633 id=633 data-nosnippet>633</a> .as_any()
<a href=#634 id=634 data-nosnippet>634</a> .downcast_ref::&lt;datatypes::arrow::array::StringArray&gt;()
<a href=#635 id=635 data-nosnippet>635</a> .ok_or_else(|| {
<a href=#636 id=636 data-nosnippet>636</a> ArrowError::CastError(<span class="string">"Failed to downcast key to StringArray"</span>.to_string())
<a href=#637 id=637 data-nosnippet>637</a> })<span class="question-mark">?
<a href=#638 id=638 data-nosnippet>638</a> </span>.value(j);
<a href=#639 id=639 data-nosnippet>639</a>
<a href=#640 id=640 data-nosnippet>640</a> <span class="kw">let </span>value = <span class="kw">if </span>value_field.is_null(j) {
<a href=#641 id=641 data-nosnippet>641</a> Value::Null
<a href=#642 id=642 data-nosnippet>642</a> } <span class="kw">else </span>{
<a href=#643 id=643 data-nosnippet>643</a> <span class="kw">let </span>value_str = value_field
<a href=#644 id=644 data-nosnippet>644</a> .as_any()
<a href=#645 id=645 data-nosnippet>645</a> .downcast_ref::&lt;datatypes::arrow::array::StringArray&gt;()
<a href=#646 id=646 data-nosnippet>646</a> .ok_or_else(|| {
<a href=#647 id=647 data-nosnippet>647</a> ArrowError::CastError(
<a href=#648 id=648 data-nosnippet>648</a> <span class="string">"Failed to downcast value to StringArray"</span>.to_string(),
<a href=#649 id=649 data-nosnippet>649</a> )
<a href=#650 id=650 data-nosnippet>650</a> })<span class="question-mark">?
<a href=#651 id=651 data-nosnippet>651</a> </span>.value(j);
<a href=#652 id=652 data-nosnippet>652</a> Value::String(value_str.to_string())
<a href=#653 id=653 data-nosnippet>653</a> };
<a href=#654 id=654 data-nosnippet>654</a>
<a href=#655 id=655 data-nosnippet>655</a> json_obj.insert(key.to_string(), value);
<a href=#656 id=656 data-nosnippet>656</a> }
<a href=#657 id=657 data-nosnippet>657</a>
<a href=#658 id=658 data-nosnippet>658</a> <span class="kw">let </span>json_value = Value::Object(json_obj);
<a href=#659 id=659 data-nosnippet>659</a> <span class="kw">let </span>json_bytes = <span class="kw">match </span>extype {
<a href=#660 id=660 data-nosnippet>660</a> <span class="prelude-val">Some</span>(ColumnExtType::Json) =&gt; {
<a href=#661 id=661 data-nosnippet>661</a> <span class="kw">let </span>json_string = <span class="kw">match </span>serde_json::to_string(<span class="kw-2">&amp;</span>json_value) {
<a href=#662 id=662 data-nosnippet>662</a> <span class="prelude-val">Ok</span>(s) =&gt; s,
<a href=#663 id=663 data-nosnippet>663</a> <span class="prelude-val">Err</span>(e) =&gt; {
<a href=#664 id=664 data-nosnippet>664</a> <span class="kw">return </span><span class="prelude-val">Err</span>(ArrowError::CastError(<span class="macro">format!</span>(
<a href=#665 id=665 data-nosnippet>665</a> <span class="string">"Failed to serialize JSON: {}"</span>,
<a href=#666 id=666 data-nosnippet>666</a> e
<a href=#667 id=667 data-nosnippet>667</a> )));
<a href=#668 id=668 data-nosnippet>668</a> }
<a href=#669 id=669 data-nosnippet>669</a> };
<a href=#670 id=670 data-nosnippet>670</a> <span class="kw">match </span>jsonb::parse_value(json_string.as_bytes()) {
<a href=#671 id=671 data-nosnippet>671</a> <span class="prelude-val">Ok</span>(jsonb_value) =&gt; jsonb_value.to_vec(),
<a href=#672 id=672 data-nosnippet>672</a> <span class="prelude-val">Err</span>(e) =&gt; {
<a href=#673 id=673 data-nosnippet>673</a> <span class="kw">return </span><span class="prelude-val">Err</span>(ArrowError::CastError(<span class="macro">format!</span>(
<a href=#674 id=674 data-nosnippet>674</a> <span class="string">"Failed to serialize JSONB: {}"</span>,
<a href=#675 id=675 data-nosnippet>675</a> e
<a href=#676 id=676 data-nosnippet>676</a> )));
<a href=#677 id=677 data-nosnippet>677</a> }
<a href=#678 id=678 data-nosnippet>678</a> }
<a href=#679 id=679 data-nosnippet>679</a> }
<a href=#680 id=680 data-nosnippet>680</a> <span class="kw">_ </span>=&gt; <span class="kw">match </span>serde_json::to_vec(<span class="kw-2">&amp;</span>json_value) {
<a href=#681 id=681 data-nosnippet>681</a> <span class="prelude-val">Ok</span>(b) =&gt; b,
<a href=#682 id=682 data-nosnippet>682</a> <span class="prelude-val">Err</span>(e) =&gt; {
<a href=#683 id=683 data-nosnippet>683</a> <span class="kw">return </span><span class="prelude-val">Err</span>(ArrowError::CastError(<span class="macro">format!</span>(
<a href=#684 id=684 data-nosnippet>684</a> <span class="string">"Failed to serialize JSON: {}"</span>,
<a href=#685 id=685 data-nosnippet>685</a> e
<a href=#686 id=686 data-nosnippet>686</a> )));
<a href=#687 id=687 data-nosnippet>687</a> }
<a href=#688 id=688 data-nosnippet>688</a> },
<a href=#689 id=689 data-nosnippet>689</a> };
<a href=#690 id=690 data-nosnippet>690</a> json_values.push(<span class="prelude-val">Some</span>(json_bytes));
<a href=#691 id=691 data-nosnippet>691</a> }
<a href=#692 id=692 data-nosnippet>692</a> }
<a href=#693 id=693 data-nosnippet>693</a>
<a href=#694 id=694 data-nosnippet>694</a> <span class="kw">let </span>binary_array = BinaryArray::from_iter(json_values);
<a href=#695 id=695 data-nosnippet>695</a> <span class="prelude-val">Ok</span>(Arc::new(binary_array))
<a href=#696 id=696 data-nosnippet>696</a>}
<a href=#697 id=697 data-nosnippet>697</a>
<a href=#698 id=698 data-nosnippet>698</a><span class="attr">#[cfg(test)]
<a href=#699 id=699 data-nosnippet>699</a></span><span class="kw">mod </span>test {
<a href=#700 id=700 data-nosnippet>700</a> <span class="kw">use </span>common_error::ext::BoxedError;
<a href=#701 id=701 data-nosnippet>701</a> <span class="kw">use </span>common_error::mock::MockError;
<a href=#702 id=702 data-nosnippet>702</a> <span class="kw">use </span>common_error::status_code::StatusCode;
<a href=#703 id=703 data-nosnippet>703</a> <span class="kw">use </span>datatypes::arrow::array::{ArrayRef, MapArray, StringArray, StructArray};
<a href=#704 id=704 data-nosnippet>704</a> <span class="kw">use </span>datatypes::arrow::buffer::OffsetBuffer;
<a href=#705 id=705 data-nosnippet>705</a> <span class="kw">use </span>datatypes::arrow::datatypes::Field;
<a href=#706 id=706 data-nosnippet>706</a> <span class="kw">use </span>datatypes::prelude::ConcreteDataType;
<a href=#707 id=707 data-nosnippet>707</a> <span class="kw">use </span>datatypes::schema::ColumnSchema;
<a href=#708 id=708 data-nosnippet>708</a> <span class="kw">use </span>datatypes::vectors::Int32Vector;
<a href=#709 id=709 data-nosnippet>709</a> <span class="kw">use </span>snafu::IntoError;
<a href=#710 id=710 data-nosnippet>710</a>
<a href=#711 id=711 data-nosnippet>711</a> <span class="kw">use super</span>::<span class="kw-2">*</span>;
<a href=#712 id=712 data-nosnippet>712</a> <span class="kw">use </span><span class="kw">crate</span>::RecordBatches;
<a href=#713 id=713 data-nosnippet>713</a> <span class="kw">use </span><span class="kw">crate</span>::error::Error;
<a href=#714 id=714 data-nosnippet>714</a>
<a href=#715 id=715 data-nosnippet>715</a> <span class="attr">#[tokio::test]
<a href=#716 id=716 data-nosnippet>716</a> </span><span class="kw">async fn </span>test_async_recordbatch_stream_adaptor() {
<a href=#717 id=717 data-nosnippet>717</a> <span class="kw">struct </span>MaybeErrorRecordBatchStream {
<a href=#718 id=718 data-nosnippet>718</a> items: Vec&lt;<span class="prelude-ty">Result</span>&lt;RecordBatch&gt;&gt;,
<a href=#719 id=719 data-nosnippet>719</a> }
<a href=#720 id=720 data-nosnippet>720</a>
<a href=#721 id=721 data-nosnippet>721</a> <span class="kw">impl </span>RecordBatchStream <span class="kw">for </span>MaybeErrorRecordBatchStream {
<a href=#722 id=722 data-nosnippet>722</a> <span class="kw">fn </span>schema(<span class="kw-2">&amp;</span><span class="self">self</span>) -&gt; SchemaRef {
<a href=#723 id=723 data-nosnippet>723</a> <span class="macro">unimplemented!</span>()
<a href=#724 id=724 data-nosnippet>724</a> }
<a href=#725 id=725 data-nosnippet>725</a>
<a href=#726 id=726 data-nosnippet>726</a> <span class="kw">fn </span>output_ordering(<span class="kw-2">&amp;</span><span class="self">self</span>) -&gt; <span class="prelude-ty">Option</span>&lt;<span class="kw-2">&amp;</span>[OrderOption]&gt; {
<a href=#727 id=727 data-nosnippet>727</a> <span class="prelude-val">None
<a href=#728 id=728 data-nosnippet>728</a> </span>}
<a href=#729 id=729 data-nosnippet>729</a>
<a href=#730 id=730 data-nosnippet>730</a> <span class="kw">fn </span>metrics(<span class="kw-2">&amp;</span><span class="self">self</span>) -&gt; <span class="prelude-ty">Option</span>&lt;RecordBatchMetrics&gt; {
<a href=#731 id=731 data-nosnippet>731</a> <span class="prelude-val">None
<a href=#732 id=732 data-nosnippet>732</a> </span>}
<a href=#733 id=733 data-nosnippet>733</a> }
<a href=#734 id=734 data-nosnippet>734</a>
<a href=#735 id=735 data-nosnippet>735</a> <span class="kw">impl </span>Stream <span class="kw">for </span>MaybeErrorRecordBatchStream {
<a href=#736 id=736 data-nosnippet>736</a> <span class="kw">type </span>Item = <span class="prelude-ty">Result</span>&lt;RecordBatch&gt;;
<a href=#737 id=737 data-nosnippet>737</a>
<a href=#738 id=738 data-nosnippet>738</a> <span class="kw">fn </span>poll_next(
<a href=#739 id=739 data-nosnippet>739</a> <span class="kw-2">mut </span><span class="self">self</span>: Pin&lt;<span class="kw-2">&amp;mut </span><span class="self">Self</span>&gt;,
<a href=#740 id=740 data-nosnippet>740</a> <span class="kw">_</span>: <span class="kw-2">&amp;mut </span>Context&lt;<span class="lifetime">'_</span>&gt;,
<a href=#741 id=741 data-nosnippet>741</a> ) -&gt; Poll&lt;<span class="prelude-ty">Option</span>&lt;<span class="self">Self</span>::Item&gt;&gt; {
<a href=#742 id=742 data-nosnippet>742</a> <span class="kw">if let </span><span class="prelude-val">Some</span>(batch) = <span class="self">self</span>.items.pop() {
<a href=#743 id=743 data-nosnippet>743</a> Poll::Ready(<span class="prelude-val">Some</span>(<span class="prelude-val">Ok</span>(batch<span class="question-mark">?</span>)))
<a href=#744 id=744 data-nosnippet>744</a> } <span class="kw">else </span>{
<a href=#745 id=745 data-nosnippet>745</a> Poll::Ready(<span class="prelude-val">None</span>)
<a href=#746 id=746 data-nosnippet>746</a> }
<a href=#747 id=747 data-nosnippet>747</a> }
<a href=#748 id=748 data-nosnippet>748</a> }
<a href=#749 id=749 data-nosnippet>749</a>
<a href=#750 id=750 data-nosnippet>750</a> <span class="kw">fn </span>new_future_stream(
<a href=#751 id=751 data-nosnippet>751</a> maybe_recordbatches: <span class="prelude-ty">Result</span>&lt;Vec&lt;<span class="prelude-ty">Result</span>&lt;RecordBatch&gt;&gt;&gt;,
<a href=#752 id=752 data-nosnippet>752</a> ) -&gt; FutureStream {
<a href=#753 id=753 data-nosnippet>753</a> Box::pin(<span class="kw">async move </span>{
<a href=#754 id=754 data-nosnippet>754</a> maybe_recordbatches
<a href=#755 id=755 data-nosnippet>755</a> .map(|items| Box::pin(MaybeErrorRecordBatchStream { items }) <span class="kw">as _</span>)
<a href=#756 id=756 data-nosnippet>756</a> })
<a href=#757 id=757 data-nosnippet>757</a> }
<a href=#758 id=758 data-nosnippet>758</a>
<a href=#759 id=759 data-nosnippet>759</a> <span class="kw">let </span>schema = Arc::new(Schema::new(<span class="macro">vec!</span>[ColumnSchema::new(
<a href=#760 id=760 data-nosnippet>760</a> <span class="string">"a"</span>,
<a href=#761 id=761 data-nosnippet>761</a> ConcreteDataType::int32_datatype(),
<a href=#762 id=762 data-nosnippet>762</a> <span class="bool-val">false</span>,
<a href=#763 id=763 data-nosnippet>763</a> )]));
<a href=#764 id=764 data-nosnippet>764</a> <span class="kw">let </span>batch1 = RecordBatch::new(
<a href=#765 id=765 data-nosnippet>765</a> schema.clone(),
<a href=#766 id=766 data-nosnippet>766</a> <span class="macro">vec!</span>[Arc::new(Int32Vector::from_slice([<span class="number">1</span>])) <span class="kw">as _</span>],
<a href=#767 id=767 data-nosnippet>767</a> )
<a href=#768 id=768 data-nosnippet>768</a> .unwrap();
<a href=#769 id=769 data-nosnippet>769</a> <span class="kw">let </span>batch2 = RecordBatch::new(
<a href=#770 id=770 data-nosnippet>770</a> schema.clone(),
<a href=#771 id=771 data-nosnippet>771</a> <span class="macro">vec!</span>[Arc::new(Int32Vector::from_slice([<span class="number">2</span>])) <span class="kw">as _</span>],
<a href=#772 id=772 data-nosnippet>772</a> )
<a href=#773 id=773 data-nosnippet>773</a> .unwrap();
<a href=#774 id=774 data-nosnippet>774</a>
<a href=#775 id=775 data-nosnippet>775</a> <span class="kw">let </span>success_stream = new_future_stream(<span class="prelude-val">Ok</span>(<span class="macro">vec!</span>[<span class="prelude-val">Ok</span>(batch1.clone()), <span class="prelude-val">Ok</span>(batch2.clone())]));
<a href=#776 id=776 data-nosnippet>776</a> <span class="kw">let </span>adapter = AsyncRecordBatchStreamAdapter::new(schema.clone(), success_stream);
<a href=#777 id=777 data-nosnippet>777</a> <span class="kw">let </span>collected = RecordBatches::try_collect(Box::pin(adapter)).<span class="kw">await</span>.unwrap();
<a href=#778 id=778 data-nosnippet>778</a> <span class="macro">assert_eq!</span>(
<a href=#779 id=779 data-nosnippet>779</a> collected,
<a href=#780 id=780 data-nosnippet>780</a> RecordBatches::try_new(schema.clone(), <span class="macro">vec!</span>[batch2.clone(), batch1.clone()]).unwrap()
<a href=#781 id=781 data-nosnippet>781</a> );
<a href=#782 id=782 data-nosnippet>782</a>
<a href=#783 id=783 data-nosnippet>783</a> <span class="kw">let </span>poll_err_stream = new_future_stream(<span class="prelude-val">Ok</span>(<span class="macro">vec!</span>[
<a href=#784 id=784 data-nosnippet>784</a> <span class="prelude-val">Ok</span>(batch1.clone()),
<a href=#785 id=785 data-nosnippet>785</a> <span class="prelude-val">Err</span>(error::ExternalSnafu
<a href=#786 id=786 data-nosnippet>786</a> .into_error(BoxedError::new(MockError::new(StatusCode::Unknown)))),
<a href=#787 id=787 data-nosnippet>787</a> ]));
<a href=#788 id=788 data-nosnippet>788</a> <span class="kw">let </span>adapter = AsyncRecordBatchStreamAdapter::new(schema.clone(), poll_err_stream);
<a href=#789 id=789 data-nosnippet>789</a> <span class="kw">let </span>err = RecordBatches::try_collect(Box::pin(adapter))
<a href=#790 id=790 data-nosnippet>790</a> .<span class="kw">await
<a href=#791 id=791 data-nosnippet>791</a> </span>.unwrap_err();
<a href=#792 id=792 data-nosnippet>792</a> <span class="macro">assert!</span>(
<a href=#793 id=793 data-nosnippet>793</a> <span class="macro">matches!</span>(err, Error::External { .. }),
<a href=#794 id=794 data-nosnippet>794</a> <span class="string">"unexpected err {err}"
<a href=#795 id=795 data-nosnippet>795</a> </span>);
<a href=#796 id=796 data-nosnippet>796</a>
<a href=#797 id=797 data-nosnippet>797</a> <span class="kw">let </span>failed_to_init_stream =
<a href=#798 id=798 data-nosnippet>798</a> new_future_stream(<span class="prelude-val">Err</span>(error::ExternalSnafu
<a href=#799 id=799 data-nosnippet>799</a> .into_error(BoxedError::new(MockError::new(StatusCode::Internal)))));
<a href=#800 id=800 data-nosnippet>800</a> <span class="kw">let </span>adapter = AsyncRecordBatchStreamAdapter::new(schema.clone(), failed_to_init_stream);
<a href=#801 id=801 data-nosnippet>801</a> <span class="kw">let </span>err = RecordBatches::try_collect(Box::pin(adapter))
<a href=#802 id=802 data-nosnippet>802</a> .<span class="kw">await
<a href=#803 id=803 data-nosnippet>803</a> </span>.unwrap_err();
<a href=#804 id=804 data-nosnippet>804</a> <span class="macro">assert!</span>(
<a href=#805 id=805 data-nosnippet>805</a> <span class="macro">matches!</span>(err, Error::External { .. }),
<a href=#806 id=806 data-nosnippet>806</a> <span class="string">"unexpected err {err}"
<a href=#807 id=807 data-nosnippet>807</a> </span>);
<a href=#808 id=808 data-nosnippet>808</a> }
<a href=#809 id=809 data-nosnippet>809</a>
<a href=#810 id=810 data-nosnippet>810</a> <span class="attr">#[test]
<a href=#811 id=811 data-nosnippet>811</a> </span><span class="kw">fn </span>test_convert_map_to_json_binary() {
<a href=#812 id=812 data-nosnippet>812</a> <span class="kw">let </span>keys = StringArray::from(<span class="macro">vec!</span>[<span class="prelude-val">Some</span>(<span class="string">"a"</span>), <span class="prelude-val">Some</span>(<span class="string">"b"</span>), <span class="prelude-val">Some</span>(<span class="string">"c"</span>), <span class="prelude-val">Some</span>(<span class="string">"x"</span>)]);
<a href=#813 id=813 data-nosnippet>813</a> <span class="kw">let </span>values = StringArray::from(<span class="macro">vec!</span>[<span class="prelude-val">Some</span>(<span class="string">"1"</span>), <span class="prelude-val">None</span>, <span class="prelude-val">Some</span>(<span class="string">"3"</span>), <span class="prelude-val">Some</span>(<span class="string">"42"</span>)]);
<a href=#814 id=814 data-nosnippet>814</a> <span class="kw">let </span>key_field = Arc::new(Field::new(<span class="string">"key"</span>, ArrowDataType::Utf8, <span class="bool-val">false</span>));
<a href=#815 id=815 data-nosnippet>815</a> <span class="kw">let </span>value_field = Arc::new(Field::new(<span class="string">"value"</span>, ArrowDataType::Utf8, <span class="bool-val">true</span>));
<a href=#816 id=816 data-nosnippet>816</a> <span class="kw">let </span>struct_type = ArrowDataType::Struct(<span class="macro">vec!</span>[key_field, value_field].into());
<a href=#817 id=817 data-nosnippet>817</a>
<a href=#818 id=818 data-nosnippet>818</a> <span class="kw">let </span>entries_field = Arc::new(Field::new(<span class="string">"entries"</span>, struct_type, <span class="bool-val">false</span>));
<a href=#819 id=819 data-nosnippet>819</a>
<a href=#820 id=820 data-nosnippet>820</a> <span class="kw">let </span>struct_array = StructArray::from(<span class="macro">vec!</span>[
<a href=#821 id=821 data-nosnippet>821</a> (
<a href=#822 id=822 data-nosnippet>822</a> Arc::new(Field::new(<span class="string">"key"</span>, ArrowDataType::Utf8, <span class="bool-val">false</span>)),
<a href=#823 id=823 data-nosnippet>823</a> Arc::new(keys) <span class="kw">as </span>ArrayRef,
<a href=#824 id=824 data-nosnippet>824</a> ),
<a href=#825 id=825 data-nosnippet>825</a> (
<a href=#826 id=826 data-nosnippet>826</a> Arc::new(Field::new(<span class="string">"value"</span>, ArrowDataType::Utf8, <span class="bool-val">true</span>)),
<a href=#827 id=827 data-nosnippet>827</a> Arc::new(values) <span class="kw">as </span>ArrayRef,
<a href=#828 id=828 data-nosnippet>828</a> ),
<a href=#829 id=829 data-nosnippet>829</a> ]);
<a href=#830 id=830 data-nosnippet>830</a>
<a href=#831 id=831 data-nosnippet>831</a> <span class="kw">let </span>offsets = OffsetBuffer::from_lengths([<span class="number">3</span>, <span class="number">0</span>, <span class="number">1</span>]);
<a href=#832 id=832 data-nosnippet>832</a> <span class="kw">let </span>nulls = datatypes::arrow::buffer::NullBuffer::from(<span class="macro">vec!</span>[<span class="bool-val">true</span>, <span class="bool-val">false</span>, <span class="bool-val">true</span>]);
<a href=#833 id=833 data-nosnippet>833</a>
<a href=#834 id=834 data-nosnippet>834</a> <span class="kw">let </span>map_array = MapArray::new(
<a href=#835 id=835 data-nosnippet>835</a> entries_field,
<a href=#836 id=836 data-nosnippet>836</a> offsets,
<a href=#837 id=837 data-nosnippet>837</a> struct_array,
<a href=#838 id=838 data-nosnippet>838</a> <span class="prelude-val">Some</span>(nulls), <span class="comment">// nulls
<a href=#839 id=839 data-nosnippet>839</a> </span><span class="bool-val">false</span>,
<a href=#840 id=840 data-nosnippet>840</a> );
<a href=#841 id=841 data-nosnippet>841</a>
<a href=#842 id=842 data-nosnippet>842</a> <span class="kw">let </span>result = convert_map_to_json_binary(<span class="kw-2">&amp;</span>map_array, <span class="prelude-val">None</span>).unwrap();
<a href=#843 id=843 data-nosnippet>843</a> <span class="kw">let </span>binary_array = result
<a href=#844 id=844 data-nosnippet>844</a> .as_any()
<a href=#845 id=845 data-nosnippet>845</a> .downcast_ref::&lt;datatypes::arrow::array::BinaryArray&gt;()
<a href=#846 id=846 data-nosnippet>846</a> .unwrap();
<a href=#847 id=847 data-nosnippet>847</a>
<a href=#848 id=848 data-nosnippet>848</a> <span class="kw">let </span>expected_jsons = [
<a href=#849 id=849 data-nosnippet>849</a> <span class="prelude-val">Some</span>(<span class="string">r#"{"a":"1","b":null,"c":"3"}"#</span>),
<a href=#850 id=850 data-nosnippet>850</a> <span class="prelude-val">None</span>,
<a href=#851 id=851 data-nosnippet>851</a> <span class="prelude-val">Some</span>(<span class="string">r#"{"x":"42"}"#</span>),
<a href=#852 id=852 data-nosnippet>852</a> ];
<a href=#853 id=853 data-nosnippet>853</a>
<a href=#854 id=854 data-nosnippet>854</a> <span class="kw">for </span>(i, <span class="kw">_</span>) <span class="kw">in </span>expected_jsons.iter().enumerate() {
<a href=#855 id=855 data-nosnippet>855</a> <span class="kw">if let </span><span class="prelude-val">Some</span>(expected) = <span class="kw-2">&amp;</span>expected_jsons[i] {
<a href=#856 id=856 data-nosnippet>856</a> <span class="macro">assert!</span>(!binary_array.is_null(i));
<a href=#857 id=857 data-nosnippet>857</a> <span class="kw">let </span>actual_bytes = binary_array.value(i);
<a href=#858 id=858 data-nosnippet>858</a> <span class="kw">let </span>actual_str = std::str::from_utf8(actual_bytes).unwrap();
<a href=#859 id=859 data-nosnippet>859</a> <span class="macro">assert_eq!</span>(actual_str, <span class="kw-2">*</span>expected);
<a href=#860 id=860 data-nosnippet>860</a> } <span class="kw">else </span>{
<a href=#861 id=861 data-nosnippet>861</a> <span class="macro">assert!</span>(binary_array.is_null(i));
<a href=#862 id=862 data-nosnippet>862</a> }
<a href=#863 id=863 data-nosnippet>863</a> }
<a href=#864 id=864 data-nosnippet>864</a>
<a href=#865 id=865 data-nosnippet>865</a> <span class="kw">let </span>result_json =
<a href=#866 id=866 data-nosnippet>866</a> convert_map_to_json_binary(<span class="kw-2">&amp;</span>map_array, <span class="prelude-val">Some</span>(ColumnExtType::Json)).unwrap();
<a href=#867 id=867 data-nosnippet>867</a> <span class="kw">let </span>binary_array_json = result_json
<a href=#868 id=868 data-nosnippet>868</a> .as_any()
<a href=#869 id=869 data-nosnippet>869</a> .downcast_ref::&lt;datatypes::arrow::array::BinaryArray&gt;()
<a href=#870 id=870 data-nosnippet>870</a> .unwrap();
<a href=#871 id=871 data-nosnippet>871</a>
<a href=#872 id=872 data-nosnippet>872</a> <span class="kw">for </span>(i, <span class="kw">_</span>) <span class="kw">in </span>expected_jsons.iter().enumerate() {
<a href=#873 id=873 data-nosnippet>873</a> <span class="kw">if </span>expected_jsons[i].is_some() {
<a href=#874 id=874 data-nosnippet>874</a> <span class="macro">assert!</span>(!binary_array_json.is_null(i));
<a href=#875 id=875 data-nosnippet>875</a> <span class="kw">let </span>actual_bytes = binary_array_json.value(i);
<a href=#876 id=876 data-nosnippet>876</a> <span class="macro">assert_ne!</span>(actual_bytes, expected_jsons[i].unwrap().as_bytes());
<a href=#877 id=877 data-nosnippet>877</a> } <span class="kw">else </span>{
<a href=#878 id=878 data-nosnippet>878</a> <span class="macro">assert!</span>(binary_array_json.is_null(i));
<a href=#879 id=879 data-nosnippet>879</a> }
<a href=#880 id=880 data-nosnippet>880</a> }
<a href=#881 id=881 data-nosnippet>881</a> }
<a href=#882 id=882 data-nosnippet>882</a>}</code></pre></div></section></main></body></html>

173
src/common_recordbatch/cursor.rs.html Normal file
View File

@@ -0,0 +1,173 @@
<!DOCTYPE html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1.0"><meta name="generator" content="rustdoc"><meta name="description" content="Source of the Rust file `src/common/recordbatch/src/cursor.rs`."><title>cursor.rs - source</title><script>if(window.location.protocol!=="file:")document.head.insertAdjacentHTML("beforeend","SourceSerif4-Regular-6b053e98.ttf.woff2,FiraSans-Italic-81dc35de.woff2,FiraSans-Regular-0fe48ade.woff2,FiraSans-MediumItalic-ccf7e434.woff2,FiraSans-Medium-e1aa3f0a.woff2,SourceCodePro-Regular-8badfe75.ttf.woff2,SourceCodePro-Semibold-aa29a496.ttf.woff2".split(",").map(f=>`<link rel="preload" as="font" type="font/woff2"href="../../static.files/${f}">`).join(""))</script><link rel="stylesheet" href="../../static.files/normalize-9960930a.css"><link rel="stylesheet" href="../../static.files/rustdoc-e56847b5.css"><meta name="rustdoc-vars" data-root-path="../../" data-static-root-path="../../static.files/" data-current-crate="common_recordbatch" data-themes="" data-resource-suffix="" data-rustdoc-version="1.92.0-nightly (fa3155a64 2025-09-30)" data-channel="nightly" data-search-js="search-e256b49e.js" data-stringdex-js="stringdex-828709d0.js" data-settings-js="settings-c38705f0.js" ><script src="../../static.files/storage-e2aeef58.js"></script><script defer src="../../static.files/src-script-813739b1.js"></script><script defer src="../../src-files.js"></script><script defer src="../../static.files/main-ce535bd0.js"></script><noscript><link rel="stylesheet" href="../../static.files/noscript-263c88ec.css"></noscript><link rel="alternate icon" type="image/png" href="../../static.files/favicon-32x32-eab170b8.png"><link rel="icon" type="image/svg+xml" href="../../static.files/favicon-044be391.svg"></head><body class="rustdoc src"><!--[if lte IE 11]><div class="warning">This old browser is unsupported and will most likely display funky things.</div><![endif]--><nav class="sidebar"><div class="src-sidebar-title"><h2>Files</h2></div></nav><div class="sidebar-resizer" title="Drag to resize sidebar"></div><main><section id="main-content" class="content"><div class="main-heading"><h1><div class="sub-heading">common_recordbatch/</div>cursor.rs</h1><rustdoc-toolbar></rustdoc-toolbar></div><div class="example-wrap digits-3"><pre class="rust"><code><a href=#1 id=1 data-nosnippet>1</a><span class="comment">// Copyright 2023 Greptime Team
<a href=#2 id=2 data-nosnippet>2</a>//
<a href=#3 id=3 data-nosnippet>3</a>// Licensed under the Apache License, Version 2.0 (the "License");
<a href=#4 id=4 data-nosnippet>4</a>// you may not use this file except in compliance with the License.
<a href=#5 id=5 data-nosnippet>5</a>// You may obtain a copy of the License at
<a href=#6 id=6 data-nosnippet>6</a>//
<a href=#7 id=7 data-nosnippet>7</a>// http://www.apache.org/licenses/LICENSE-2.0
<a href=#8 id=8 data-nosnippet>8</a>//
<a href=#9 id=9 data-nosnippet>9</a>// Unless required by applicable law or agreed to in writing, software
<a href=#10 id=10 data-nosnippet>10</a>// distributed under the License is distributed on an "AS IS" BASIS,
<a href=#11 id=11 data-nosnippet>11</a>// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
<a href=#12 id=12 data-nosnippet>12</a>// See the License for the specific language governing permissions and
<a href=#13 id=13 data-nosnippet>13</a>// limitations under the License.
<a href=#14 id=14 data-nosnippet>14</a>
<a href=#15 id=15 data-nosnippet>15</a></span><span class="kw">use </span>futures::StreamExt;
<a href=#16 id=16 data-nosnippet>16</a><span class="kw">use </span>tokio::sync::Mutex;
<a href=#17 id=17 data-nosnippet>17</a>
<a href=#18 id=18 data-nosnippet>18</a><span class="kw">use </span><span class="kw">crate</span>::error::Result;
<a href=#19 id=19 data-nosnippet>19</a><span class="kw">use </span><span class="kw">crate</span>::recordbatch::merge_record_batches;
<a href=#20 id=20 data-nosnippet>20</a><span class="kw">use crate</span>::{RecordBatch, SendableRecordBatchStream};
<a href=#21 id=21 data-nosnippet>21</a>
<a href=#22 id=22 data-nosnippet>22</a><span class="kw">struct </span>Inner {
<a href=#23 id=23 data-nosnippet>23</a> stream: SendableRecordBatchStream,
<a href=#24 id=24 data-nosnippet>24</a> current_row_index: usize,
<a href=#25 id=25 data-nosnippet>25</a> current_batch: <span class="prelude-ty">Option</span>&lt;RecordBatch&gt;,
<a href=#26 id=26 data-nosnippet>26</a> total_rows_in_current_batch: usize,
<a href=#27 id=27 data-nosnippet>27</a>}
<a href=#28 id=28 data-nosnippet>28</a>
<a href=#29 id=29 data-nosnippet>29</a><span class="doccomment">/// A cursor on RecordBatchStream that fetches data batch by batch
<a href=#30 id=30 data-nosnippet>30</a></span><span class="kw">pub struct </span>RecordBatchStreamCursor {
<a href=#31 id=31 data-nosnippet>31</a> inner: Mutex&lt;Inner&gt;,
<a href=#32 id=32 data-nosnippet>32</a>}
<a href=#33 id=33 data-nosnippet>33</a>
<a href=#34 id=34 data-nosnippet>34</a><span class="kw">impl </span>RecordBatchStreamCursor {
<a href=#35 id=35 data-nosnippet>35</a> <span class="kw">pub fn </span>new(stream: SendableRecordBatchStream) -&gt; RecordBatchStreamCursor {
<a href=#36 id=36 data-nosnippet>36</a> <span class="self">Self </span>{
<a href=#37 id=37 data-nosnippet>37</a> inner: Mutex::new(Inner {
<a href=#38 id=38 data-nosnippet>38</a> stream,
<a href=#39 id=39 data-nosnippet>39</a> current_row_index: <span class="number">0</span>,
<a href=#40 id=40 data-nosnippet>40</a> current_batch: <span class="prelude-val">None</span>,
<a href=#41 id=41 data-nosnippet>41</a> total_rows_in_current_batch: <span class="number">0</span>,
<a href=#42 id=42 data-nosnippet>42</a> }),
<a href=#43 id=43 data-nosnippet>43</a> }
<a href=#44 id=44 data-nosnippet>44</a> }
<a href=#45 id=45 data-nosnippet>45</a>
<a href=#46 id=46 data-nosnippet>46</a> <span class="doccomment">/// Take `size` of row from the `RecordBatchStream` and create a new
<a href=#47 id=47 data-nosnippet>47</a> /// `RecordBatch` for these rows.
<a href=#48 id=48 data-nosnippet>48</a> </span><span class="kw">pub async fn </span>take(<span class="kw-2">&amp;</span><span class="self">self</span>, size: usize) -&gt; <span class="prelude-ty">Result</span>&lt;RecordBatch&gt; {
<a href=#49 id=49 data-nosnippet>49</a> <span class="kw">let </span><span class="kw-2">mut </span>remaining_rows_to_take = size;
<a href=#50 id=50 data-nosnippet>50</a> <span class="kw">let </span><span class="kw-2">mut </span>accumulated_rows = Vec::new();
<a href=#51 id=51 data-nosnippet>51</a>
<a href=#52 id=52 data-nosnippet>52</a> <span class="kw">let </span><span class="kw-2">mut </span>inner = <span class="self">self</span>.inner.lock().<span class="kw">await</span>;
<a href=#53 id=53 data-nosnippet>53</a>
<a href=#54 id=54 data-nosnippet>54</a> <span class="kw">while </span>remaining_rows_to_take &gt; <span class="number">0 </span>{
<a href=#55 id=55 data-nosnippet>55</a> <span class="comment">// Ensure we have a current batch or fetch the next one
<a href=#56 id=56 data-nosnippet>56</a> </span><span class="kw">if </span>inner.current_batch.is_none()
<a href=#57 id=57 data-nosnippet>57</a> || inner.current_row_index &gt;= inner.total_rows_in_current_batch
<a href=#58 id=58 data-nosnippet>58</a> {
<a href=#59 id=59 data-nosnippet>59</a> <span class="kw">match </span>inner.stream.next().<span class="kw">await </span>{
<a href=#60 id=60 data-nosnippet>60</a> <span class="prelude-val">Some</span>(<span class="prelude-val">Ok</span>(batch)) =&gt; {
<a href=#61 id=61 data-nosnippet>61</a> inner.total_rows_in_current_batch = batch.num_rows();
<a href=#62 id=62 data-nosnippet>62</a> inner.current_batch = <span class="prelude-val">Some</span>(batch);
<a href=#63 id=63 data-nosnippet>63</a> inner.current_row_index = <span class="number">0</span>;
<a href=#64 id=64 data-nosnippet>64</a> }
<a href=#65 id=65 data-nosnippet>65</a> <span class="prelude-val">Some</span>(<span class="prelude-val">Err</span>(e)) =&gt; <span class="kw">return </span><span class="prelude-val">Err</span>(e),
<a href=#66 id=66 data-nosnippet>66</a> <span class="prelude-val">None </span>=&gt; {
<a href=#67 id=67 data-nosnippet>67</a> <span class="comment">// Stream is exhausted
<a href=#68 id=68 data-nosnippet>68</a> </span><span class="kw">break</span>;
<a href=#69 id=69 data-nosnippet>69</a> }
<a href=#70 id=70 data-nosnippet>70</a> }
<a href=#71 id=71 data-nosnippet>71</a> }
<a href=#72 id=72 data-nosnippet>72</a>
<a href=#73 id=73 data-nosnippet>73</a> <span class="comment">// If we still have no batch after attempting to fetch
<a href=#74 id=74 data-nosnippet>74</a> </span><span class="kw">let </span>current_batch = <span class="kw">match </span><span class="kw-2">&amp;</span>inner.current_batch {
<a href=#75 id=75 data-nosnippet>75</a> <span class="prelude-val">Some</span>(batch) =&gt; batch,
<a href=#76 id=76 data-nosnippet>76</a> <span class="prelude-val">None </span>=&gt; <span class="kw">break</span>,
<a href=#77 id=77 data-nosnippet>77</a> };
<a href=#78 id=78 data-nosnippet>78</a>
<a href=#79 id=79 data-nosnippet>79</a> <span class="comment">// Calculate how many rows we can take from this batch
<a href=#80 id=80 data-nosnippet>80</a> </span><span class="kw">let </span>rows_to_take_from_batch = remaining_rows_to_take
<a href=#81 id=81 data-nosnippet>81</a> .min(inner.total_rows_in_current_batch - inner.current_row_index);
<a href=#82 id=82 data-nosnippet>82</a>
<a href=#83 id=83 data-nosnippet>83</a> <span class="comment">// Slice the current batch to get the desired rows
<a href=#84 id=84 data-nosnippet>84</a> </span><span class="kw">let </span>taken_batch =
<a href=#85 id=85 data-nosnippet>85</a> current_batch.slice(inner.current_row_index, rows_to_take_from_batch)<span class="question-mark">?</span>;
<a href=#86 id=86 data-nosnippet>86</a>
<a href=#87 id=87 data-nosnippet>87</a> <span class="comment">// Add the taken batch to accumulated rows
<a href=#88 id=88 data-nosnippet>88</a> </span>accumulated_rows.push(taken_batch);
<a href=#89 id=89 data-nosnippet>89</a>
<a href=#90 id=90 data-nosnippet>90</a> <span class="comment">// Update cursor and remaining rows
<a href=#91 id=91 data-nosnippet>91</a> </span>inner.current_row_index += rows_to_take_from_batch;
<a href=#92 id=92 data-nosnippet>92</a> remaining_rows_to_take -= rows_to_take_from_batch;
<a href=#93 id=93 data-nosnippet>93</a> }
<a href=#94 id=94 data-nosnippet>94</a>
<a href=#95 id=95 data-nosnippet>95</a> <span class="comment">// If no rows were accumulated, return empty
<a href=#96 id=96 data-nosnippet>96</a> </span><span class="kw">if </span>accumulated_rows.is_empty() {
<a href=#97 id=97 data-nosnippet>97</a> <span class="kw">return </span><span class="prelude-val">Ok</span>(RecordBatch::new_empty(inner.stream.schema()));
<a href=#98 id=98 data-nosnippet>98</a> }
<a href=#99 id=99 data-nosnippet>99</a>
<a href=#100 id=100 data-nosnippet>100</a> <span class="comment">// If only one batch was accumulated, return it directly
<a href=#101 id=101 data-nosnippet>101</a> </span><span class="kw">if </span>accumulated_rows.len() == <span class="number">1 </span>{
<a href=#102 id=102 data-nosnippet>102</a> <span class="kw">return </span><span class="prelude-val">Ok</span>(accumulated_rows.remove(<span class="number">0</span>));
<a href=#103 id=103 data-nosnippet>103</a> }
<a href=#104 id=104 data-nosnippet>104</a>
<a href=#105 id=105 data-nosnippet>105</a> <span class="comment">// Merge multiple batches
<a href=#106 id=106 data-nosnippet>106</a> </span>merge_record_batches(inner.stream.schema(), <span class="kw-2">&amp;</span>accumulated_rows)
<a href=#107 id=107 data-nosnippet>107</a> }
<a href=#108 id=108 data-nosnippet>108</a>}
<a href=#109 id=109 data-nosnippet>109</a>
<a href=#110 id=110 data-nosnippet>110</a><span class="attr">#[cfg(test)]
<a href=#111 id=111 data-nosnippet>111</a></span><span class="kw">mod </span>tests {
<a href=#112 id=112 data-nosnippet>112</a> <span class="kw">use </span>std::sync::Arc;
<a href=#113 id=113 data-nosnippet>113</a>
<a href=#114 id=114 data-nosnippet>114</a> <span class="kw">use </span>datatypes::prelude::ConcreteDataType;
<a href=#115 id=115 data-nosnippet>115</a> <span class="kw">use </span>datatypes::schema::{ColumnSchema, Schema};
<a href=#116 id=116 data-nosnippet>116</a> <span class="kw">use </span>datatypes::vectors::StringVector;
<a href=#117 id=117 data-nosnippet>117</a>
<a href=#118 id=118 data-nosnippet>118</a> <span class="kw">use super</span>::<span class="kw-2">*</span>;
<a href=#119 id=119 data-nosnippet>119</a> <span class="kw">use </span><span class="kw">crate</span>::RecordBatches;
<a href=#120 id=120 data-nosnippet>120</a>
<a href=#121 id=121 data-nosnippet>121</a> <span class="attr">#[tokio::test]
<a href=#122 id=122 data-nosnippet>122</a> </span><span class="kw">async fn </span>test_cursor() {
<a href=#123 id=123 data-nosnippet>123</a> <span class="kw">let </span>schema = Arc::new(Schema::new(<span class="macro">vec!</span>[ColumnSchema::new(
<a href=#124 id=124 data-nosnippet>124</a> <span class="string">"a"</span>,
<a href=#125 id=125 data-nosnippet>125</a> ConcreteDataType::string_datatype(),
<a href=#126 id=126 data-nosnippet>126</a> <span class="bool-val">false</span>,
<a href=#127 id=127 data-nosnippet>127</a> )]));
<a href=#128 id=128 data-nosnippet>128</a>
<a href=#129 id=129 data-nosnippet>129</a> <span class="kw">let </span>rbs = RecordBatches::try_from_columns(
<a href=#130 id=130 data-nosnippet>130</a> schema.clone(),
<a href=#131 id=131 data-nosnippet>131</a> <span class="macro">vec!</span>[Arc::new(StringVector::from(<span class="macro">vec!</span>[<span class="string">"hello"</span>, <span class="string">"world"</span>])) <span class="kw">as _</span>],
<a href=#132 id=132 data-nosnippet>132</a> )
<a href=#133 id=133 data-nosnippet>133</a> .unwrap();
<a href=#134 id=134 data-nosnippet>134</a>
<a href=#135 id=135 data-nosnippet>135</a> <span class="kw">let </span>cursor = RecordBatchStreamCursor::new(rbs.as_stream());
<a href=#136 id=136 data-nosnippet>136</a> <span class="kw">let </span>result_rb = cursor.take(<span class="number">1</span>).<span class="kw">await</span>.expect(<span class="string">"take from cursor failed"</span>);
<a href=#137 id=137 data-nosnippet>137</a> <span class="macro">assert_eq!</span>(result_rb.num_rows(), <span class="number">1</span>);
<a href=#138 id=138 data-nosnippet>138</a>
<a href=#139 id=139 data-nosnippet>139</a> <span class="kw">let </span>result_rb = cursor.take(<span class="number">1</span>).<span class="kw">await</span>.expect(<span class="string">"take from cursor failed"</span>);
<a href=#140 id=140 data-nosnippet>140</a> <span class="macro">assert_eq!</span>(result_rb.num_rows(), <span class="number">1</span>);
<a href=#141 id=141 data-nosnippet>141</a>
<a href=#142 id=142 data-nosnippet>142</a> <span class="kw">let </span>result_rb = cursor.take(<span class="number">1</span>).<span class="kw">await</span>.expect(<span class="string">"take from cursor failed"</span>);
<a href=#143 id=143 data-nosnippet>143</a> <span class="macro">assert_eq!</span>(result_rb.num_rows(), <span class="number">0</span>);
<a href=#144 id=144 data-nosnippet>144</a>
<a href=#145 id=145 data-nosnippet>145</a> <span class="kw">let </span>rb = RecordBatch::new(
<a href=#146 id=146 data-nosnippet>146</a> schema.clone(),
<a href=#147 id=147 data-nosnippet>147</a> <span class="macro">vec!</span>[Arc::new(StringVector::from(<span class="macro">vec!</span>[<span class="string">"hello"</span>, <span class="string">"world"</span>])) <span class="kw">as _</span>],
<a href=#148 id=148 data-nosnippet>148</a> )
<a href=#149 id=149 data-nosnippet>149</a> .unwrap();
<a href=#150 id=150 data-nosnippet>150</a> <span class="kw">let </span>rbs2 =
<a href=#151 id=151 data-nosnippet>151</a> RecordBatches::try_new(schema.clone(), <span class="macro">vec!</span>[rb.clone(), rb.clone(), rb]).unwrap();
<a href=#152 id=152 data-nosnippet>152</a> <span class="kw">let </span>cursor = RecordBatchStreamCursor::new(rbs2.as_stream());
<a href=#153 id=153 data-nosnippet>153</a> <span class="kw">let </span>result_rb = cursor.take(<span class="number">3</span>).<span class="kw">await</span>.expect(<span class="string">"take from cursor failed"</span>);
<a href=#154 id=154 data-nosnippet>154</a> <span class="macro">assert_eq!</span>(result_rb.num_rows(), <span class="number">3</span>);
<a href=#155 id=155 data-nosnippet>155</a> <span class="kw">let </span>result_rb = cursor.take(<span class="number">2</span>).<span class="kw">await</span>.expect(<span class="string">"take from cursor failed"</span>);
<a href=#156 id=156 data-nosnippet>156</a> <span class="macro">assert_eq!</span>(result_rb.num_rows(), <span class="number">2</span>);
<a href=#157 id=157 data-nosnippet>157</a> <span class="kw">let </span>result_rb = cursor.take(<span class="number">2</span>).<span class="kw">await</span>.expect(<span class="string">"take from cursor failed"</span>);
<a href=#158 id=158 data-nosnippet>158</a> <span class="macro">assert_eq!</span>(result_rb.num_rows(), <span class="number">1</span>);
<a href=#159 id=159 data-nosnippet>159</a> <span class="kw">let </span>result_rb = cursor.take(<span class="number">2</span>).<span class="kw">await</span>.expect(<span class="string">"take from cursor failed"</span>);
<a href=#160 id=160 data-nosnippet>160</a> <span class="macro">assert_eq!</span>(result_rb.num_rows(), <span class="number">0</span>);
<a href=#161 id=161 data-nosnippet>161</a>
<a href=#162 id=162 data-nosnippet>162</a> <span class="kw">let </span>rb = RecordBatch::new(
<a href=#163 id=163 data-nosnippet>163</a> schema.clone(),
<a href=#164 id=164 data-nosnippet>164</a> <span class="macro">vec!</span>[Arc::new(StringVector::from(<span class="macro">vec!</span>[<span class="string">"hello"</span>, <span class="string">"world"</span>])) <span class="kw">as _</span>],
<a href=#165 id=165 data-nosnippet>165</a> )
<a href=#166 id=166 data-nosnippet>166</a> .unwrap();
<a href=#167 id=167 data-nosnippet>167</a> <span class="kw">let </span>rbs3 =
<a href=#168 id=168 data-nosnippet>168</a> RecordBatches::try_new(schema.clone(), <span class="macro">vec!</span>[rb.clone(), rb.clone(), rb]).unwrap();
<a href=#169 id=169 data-nosnippet>169</a> <span class="kw">let </span>cursor = RecordBatchStreamCursor::new(rbs3.as_stream());
<a href=#170 id=170 data-nosnippet>170</a> <span class="kw">let </span>result_rb = cursor.take(<span class="number">10</span>).<span class="kw">await</span>.expect(<span class="string">"take from cursor failed"</span>);
<a href=#171 id=171 data-nosnippet>171</a> <span class="macro">assert_eq!</span>(result_rb.num_rows(), <span class="number">6</span>);
<a href=#172 id=172 data-nosnippet>172</a> }
<a href=#173 id=173 data-nosnippet>173</a>}</code></pre></div></section></main></body></html>

241
src/common_recordbatch/error.rs.html Normal file
View File

@@ -0,0 +1,241 @@
<!DOCTYPE html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1.0"><meta name="generator" content="rustdoc"><meta name="description" content="Source of the Rust file `src/common/recordbatch/src/error.rs`."><title>error.rs - source</title><script>if(window.location.protocol!=="file:")document.head.insertAdjacentHTML("beforeend","SourceSerif4-Regular-6b053e98.ttf.woff2,FiraSans-Italic-81dc35de.woff2,FiraSans-Regular-0fe48ade.woff2,FiraSans-MediumItalic-ccf7e434.woff2,FiraSans-Medium-e1aa3f0a.woff2,SourceCodePro-Regular-8badfe75.ttf.woff2,SourceCodePro-Semibold-aa29a496.ttf.woff2".split(",").map(f=>`<link rel="preload" as="font" type="font/woff2"href="../../static.files/${f}">`).join(""))</script><link rel="stylesheet" href="../../static.files/normalize-9960930a.css"><link rel="stylesheet" href="../../static.files/rustdoc-e56847b5.css"><meta name="rustdoc-vars" data-root-path="../../" data-static-root-path="../../static.files/" data-current-crate="common_recordbatch" data-themes="" data-resource-suffix="" data-rustdoc-version="1.92.0-nightly (fa3155a64 2025-09-30)" data-channel="nightly" data-search-js="search-e256b49e.js" data-stringdex-js="stringdex-828709d0.js" data-settings-js="settings-c38705f0.js" ><script src="../../static.files/storage-e2aeef58.js"></script><script defer src="../../static.files/src-script-813739b1.js"></script><script defer src="../../src-files.js"></script><script defer src="../../static.files/main-ce535bd0.js"></script><noscript><link rel="stylesheet" href="../../static.files/noscript-263c88ec.css"></noscript><link rel="alternate icon" type="image/png" href="../../static.files/favicon-32x32-eab170b8.png"><link rel="icon" type="image/svg+xml" href="../../static.files/favicon-044be391.svg"></head><body class="rustdoc src"><!--[if lte IE 11]><div class="warning">This old browser is unsupported and will most likely display funky things.</div><![endif]--><nav class="sidebar"><div class="src-sidebar-title"><h2>Files</h2></div></nav><div class="sidebar-resizer" title="Drag to resize sidebar"></div><main><section id="main-content" class="content"><div class="main-heading"><h1><div class="sub-heading">common_recordbatch/</div>error.rs</h1><rustdoc-toolbar></rustdoc-toolbar></div><div class="example-wrap digits-3"><pre class="rust"><code><a href=#1 id=1 data-nosnippet>1</a><span class="comment">// Copyright 2023 Greptime Team
<a href=#2 id=2 data-nosnippet>2</a>//
<a href=#3 id=3 data-nosnippet>3</a>// Licensed under the Apache License, Version 2.0 (the "License");
<a href=#4 id=4 data-nosnippet>4</a>// you may not use this file except in compliance with the License.
<a href=#5 id=5 data-nosnippet>5</a>// You may obtain a copy of the License at
<a href=#6 id=6 data-nosnippet>6</a>//
<a href=#7 id=7 data-nosnippet>7</a>// http://www.apache.org/licenses/LICENSE-2.0
<a href=#8 id=8 data-nosnippet>8</a>//
<a href=#9 id=9 data-nosnippet>9</a>// Unless required by applicable law or agreed to in writing, software
<a href=#10 id=10 data-nosnippet>10</a>// distributed under the License is distributed on an "AS IS" BASIS,
<a href=#11 id=11 data-nosnippet>11</a>// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
<a href=#12 id=12 data-nosnippet>12</a>// See the License for the specific language governing permissions and
<a href=#13 id=13 data-nosnippet>13</a>// limitations under the License.
<a href=#14 id=14 data-nosnippet>14</a>
<a href=#15 id=15 data-nosnippet>15</a></span><span class="doccomment">//! Error of record batch.
<a href=#16 id=16 data-nosnippet>16</a></span><span class="kw">use </span>std::any::Any;
<a href=#17 id=17 data-nosnippet>17</a>
<a href=#18 id=18 data-nosnippet>18</a><span class="kw">use </span>common_error::ext::{BoxedError, ErrorExt};
<a href=#19 id=19 data-nosnippet>19</a><span class="kw">use </span>common_error::status_code::StatusCode;
<a href=#20 id=20 data-nosnippet>20</a><span class="kw">use </span>common_macro::stack_trace_debug;
<a href=#21 id=21 data-nosnippet>21</a><span class="kw">use </span>datafusion_common::ScalarValue;
<a href=#22 id=22 data-nosnippet>22</a><span class="kw">use </span>datatypes::prelude::ConcreteDataType;
<a href=#23 id=23 data-nosnippet>23</a><span class="kw">use </span>datatypes::schema::SchemaRef;
<a href=#24 id=24 data-nosnippet>24</a><span class="kw">use </span>snafu::{Location, Snafu};
<a href=#25 id=25 data-nosnippet>25</a>
<a href=#26 id=26 data-nosnippet>26</a><span class="kw">pub type </span><span class="prelude-ty">Result</span>&lt;T, E = Error&gt; = std::result::Result&lt;T, E&gt;;
<a href=#27 id=27 data-nosnippet>27</a>
<a href=#28 id=28 data-nosnippet>28</a><span class="attr">#[derive(Snafu)]
<a href=#29 id=29 data-nosnippet>29</a>#[snafu(visibility(<span class="kw">pub</span>))]
<a href=#30 id=30 data-nosnippet>30</a>#[stack_trace_debug]
<a href=#31 id=31 data-nosnippet>31</a></span><span class="kw">pub enum </span>Error {
<a href=#32 id=32 data-nosnippet>32</a> <span class="attr">#[snafu(display(<span class="string">"Fail to create datafusion record batch"</span>))]
<a href=#33 id=33 data-nosnippet>33</a> </span>NewDfRecordBatch {
<a href=#34 id=34 data-nosnippet>34</a> <span class="attr">#[snafu(source)]
<a href=#35 id=35 data-nosnippet>35</a> </span>error: datatypes::arrow::error::ArrowError,
<a href=#36 id=36 data-nosnippet>36</a> <span class="attr">#[snafu(implicit)]
<a href=#37 id=37 data-nosnippet>37</a> </span>location: Location,
<a href=#38 id=38 data-nosnippet>38</a> },
<a href=#39 id=39 data-nosnippet>39</a>
<a href=#40 id=40 data-nosnippet>40</a> <span class="attr">#[snafu(display(<span class="string">"Data types error"</span>))]
<a href=#41 id=41 data-nosnippet>41</a> </span>DataTypes {
<a href=#42 id=42 data-nosnippet>42</a> <span class="attr">#[snafu(implicit)]
<a href=#43 id=43 data-nosnippet>43</a> </span>location: Location,
<a href=#44 id=44 data-nosnippet>44</a> source: datatypes::error::Error,
<a href=#45 id=45 data-nosnippet>45</a> },
<a href=#46 id=46 data-nosnippet>46</a>
<a href=#47 id=47 data-nosnippet>47</a> <span class="attr">#[snafu(display(<span class="string">"External error"</span>))]
<a href=#48 id=48 data-nosnippet>48</a> </span>External {
<a href=#49 id=49 data-nosnippet>49</a> <span class="attr">#[snafu(implicit)]
<a href=#50 id=50 data-nosnippet>50</a> </span>location: Location,
<a href=#51 id=51 data-nosnippet>51</a> source: BoxedError,
<a href=#52 id=52 data-nosnippet>52</a> },
<a href=#53 id=53 data-nosnippet>53</a>
<a href=#54 id=54 data-nosnippet>54</a> <span class="attr">#[snafu(display(<span class="string">"Failed to create RecordBatches, reason: {}"</span>, reason))]
<a href=#55 id=55 data-nosnippet>55</a> </span>CreateRecordBatches {
<a href=#56 id=56 data-nosnippet>56</a> reason: String,
<a href=#57 id=57 data-nosnippet>57</a> <span class="attr">#[snafu(implicit)]
<a href=#58 id=58 data-nosnippet>58</a> </span>location: Location,
<a href=#59 id=59 data-nosnippet>59</a> },
<a href=#60 id=60 data-nosnippet>60</a>
<a href=#61 id=61 data-nosnippet>61</a> <span class="attr">#[snafu(display(<span class="string">"Failed to convert Arrow schema"</span>))]
<a href=#62 id=62 data-nosnippet>62</a> </span>SchemaConversion {
<a href=#63 id=63 data-nosnippet>63</a> source: datatypes::error::Error,
<a href=#64 id=64 data-nosnippet>64</a> <span class="attr">#[snafu(implicit)]
<a href=#65 id=65 data-nosnippet>65</a> </span>location: Location,
<a href=#66 id=66 data-nosnippet>66</a> },
<a href=#67 id=67 data-nosnippet>67</a>
<a href=#68 id=68 data-nosnippet>68</a> <span class="attr">#[snafu(transparent)]
<a href=#69 id=69 data-nosnippet>69</a> </span>PollStream {
<a href=#70 id=70 data-nosnippet>70</a> <span class="attr">#[snafu(source)]
<a href=#71 id=71 data-nosnippet>71</a> </span>error: datafusion::error::DataFusionError,
<a href=#72 id=72 data-nosnippet>72</a> <span class="attr">#[snafu(implicit)]
<a href=#73 id=73 data-nosnippet>73</a> </span>location: Location,
<a href=#74 id=74 data-nosnippet>74</a> },
<a href=#75 id=75 data-nosnippet>75</a>
<a href=#76 id=76 data-nosnippet>76</a> <span class="attr">#[snafu(display(<span class="string">"Create physical expr error"</span>))]
<a href=#77 id=77 data-nosnippet>77</a> </span>PhysicalExpr {
<a href=#78 id=78 data-nosnippet>78</a> <span class="attr">#[snafu(source)]
<a href=#79 id=79 data-nosnippet>79</a> </span>error: datafusion::error::DataFusionError,
<a href=#80 id=80 data-nosnippet>80</a> <span class="attr">#[snafu(implicit)]
<a href=#81 id=81 data-nosnippet>81</a> </span>location: Location,
<a href=#82 id=82 data-nosnippet>82</a> },
<a href=#83 id=83 data-nosnippet>83</a>
<a href=#84 id=84 data-nosnippet>84</a> <span class="attr">#[snafu(display(<span class="string">"Fail to format record batch"</span>))]
<a href=#85 id=85 data-nosnippet>85</a> </span>Format {
<a href=#86 id=86 data-nosnippet>86</a> <span class="attr">#[snafu(source)]
<a href=#87 id=87 data-nosnippet>87</a> </span>error: datatypes::arrow::error::ArrowError,
<a href=#88 id=88 data-nosnippet>88</a> <span class="attr">#[snafu(implicit)]
<a href=#89 id=89 data-nosnippet>89</a> </span>location: Location,
<a href=#90 id=90 data-nosnippet>90</a> },
<a href=#91 id=91 data-nosnippet>91</a>
<a href=#92 id=92 data-nosnippet>92</a> <span class="attr">#[snafu(display(<span class="string">"Failed to convert {v:?} to Arrow scalar"</span>))]
<a href=#93 id=93 data-nosnippet>93</a> </span>ToArrowScalar {
<a href=#94 id=94 data-nosnippet>94</a> v: ScalarValue,
<a href=#95 id=95 data-nosnippet>95</a> <span class="attr">#[snafu(source)]
<a href=#96 id=96 data-nosnippet>96</a> </span>error: datafusion_common::DataFusionError,
<a href=#97 id=97 data-nosnippet>97</a> <span class="attr">#[snafu(implicit)]
<a href=#98 id=98 data-nosnippet>98</a> </span>location: Location,
<a href=#99 id=99 data-nosnippet>99</a> },
<a href=#100 id=100 data-nosnippet>100</a>
<a href=#101 id=101 data-nosnippet>101</a> <span class="attr">#[snafu(display(
<a href=#102 id=102 data-nosnippet>102</a> <span class="string">"Failed to project Arrow RecordBatch with schema {:?} and projection {:?}"</span>,
<a href=#103 id=103 data-nosnippet>103</a> schema,
<a href=#104 id=104 data-nosnippet>104</a> projection,
<a href=#105 id=105 data-nosnippet>105</a> ))]
<a href=#106 id=106 data-nosnippet>106</a> </span>ProjectArrowRecordBatch {
<a href=#107 id=107 data-nosnippet>107</a> <span class="attr">#[snafu(source)]
<a href=#108 id=108 data-nosnippet>108</a> </span>error: datatypes::arrow::error::ArrowError,
<a href=#109 id=109 data-nosnippet>109</a> <span class="attr">#[snafu(implicit)]
<a href=#110 id=110 data-nosnippet>110</a> </span>location: Location,
<a href=#111 id=111 data-nosnippet>111</a> schema: datatypes::schema::SchemaRef,
<a href=#112 id=112 data-nosnippet>112</a> projection: Vec&lt;usize&gt;,
<a href=#113 id=113 data-nosnippet>113</a> },
<a href=#114 id=114 data-nosnippet>114</a>
<a href=#115 id=115 data-nosnippet>115</a> <span class="attr">#[snafu(display(<span class="string">"Column {} not exists in table {}"</span>, column_name, table_name))]
<a href=#116 id=116 data-nosnippet>116</a> </span>ColumnNotExists {
<a href=#117 id=117 data-nosnippet>117</a> column_name: String,
<a href=#118 id=118 data-nosnippet>118</a> table_name: String,
<a href=#119 id=119 data-nosnippet>119</a> <span class="attr">#[snafu(implicit)]
<a href=#120 id=120 data-nosnippet>120</a> </span>location: Location,
<a href=#121 id=121 data-nosnippet>121</a> },
<a href=#122 id=122 data-nosnippet>122</a>
<a href=#123 id=123 data-nosnippet>123</a> <span class="attr">#[snafu(display(
<a href=#124 id=124 data-nosnippet>124</a> <span class="string">"Failed to cast vector of type '{:?}' to type '{:?}'"</span>,
<a href=#125 id=125 data-nosnippet>125</a> from_type,
<a href=#126 id=126 data-nosnippet>126</a> to_type,
<a href=#127 id=127 data-nosnippet>127</a> ))]
<a href=#128 id=128 data-nosnippet>128</a> </span>CastVector {
<a href=#129 id=129 data-nosnippet>129</a> from_type: ConcreteDataType,
<a href=#130 id=130 data-nosnippet>130</a> to_type: ConcreteDataType,
<a href=#131 id=131 data-nosnippet>131</a> <span class="attr">#[snafu(implicit)]
<a href=#132 id=132 data-nosnippet>132</a> </span>location: Location,
<a href=#133 id=133 data-nosnippet>133</a> source: datatypes::error::Error,
<a href=#134 id=134 data-nosnippet>134</a> },
<a href=#135 id=135 data-nosnippet>135</a>
<a href=#136 id=136 data-nosnippet>136</a> <span class="attr">#[snafu(display(<span class="string">"Error occurs when performing arrow computation"</span>))]
<a href=#137 id=137 data-nosnippet>137</a> </span>ArrowCompute {
<a href=#138 id=138 data-nosnippet>138</a> <span class="attr">#[snafu(source)]
<a href=#139 id=139 data-nosnippet>139</a> </span>error: datatypes::arrow::error::ArrowError,
<a href=#140 id=140 data-nosnippet>140</a> <span class="attr">#[snafu(implicit)]
<a href=#141 id=141 data-nosnippet>141</a> </span>location: Location,
<a href=#142 id=142 data-nosnippet>142</a> },
<a href=#143 id=143 data-nosnippet>143</a>
<a href=#144 id=144 data-nosnippet>144</a> <span class="attr">#[snafu(display(<span class="string">"Unsupported operation: {}"</span>, reason))]
<a href=#145 id=145 data-nosnippet>145</a> </span>UnsupportedOperation {
<a href=#146 id=146 data-nosnippet>146</a> reason: String,
<a href=#147 id=147 data-nosnippet>147</a> <span class="attr">#[snafu(implicit)]
<a href=#148 id=148 data-nosnippet>148</a> </span>location: Location,
<a href=#149 id=149 data-nosnippet>149</a> },
<a href=#150 id=150 data-nosnippet>150</a>
<a href=#151 id=151 data-nosnippet>151</a> <span class="attr">#[snafu(display(<span class="string">"Cannot construct an empty stream"</span>))]
<a href=#152 id=152 data-nosnippet>152</a> </span>EmptyStream {
<a href=#153 id=153 data-nosnippet>153</a> <span class="attr">#[snafu(implicit)]
<a href=#154 id=154 data-nosnippet>154</a> </span>location: Location,
<a href=#155 id=155 data-nosnippet>155</a> },
<a href=#156 id=156 data-nosnippet>156</a>
<a href=#157 id=157 data-nosnippet>157</a> <span class="attr">#[snafu(display(<span class="string">"Schema not match, left: {:?}, right: {:?}"</span>, left, right))]
<a href=#158 id=158 data-nosnippet>158</a> </span>SchemaNotMatch {
<a href=#159 id=159 data-nosnippet>159</a> left: SchemaRef,
<a href=#160 id=160 data-nosnippet>160</a> right: SchemaRef,
<a href=#161 id=161 data-nosnippet>161</a> <span class="attr">#[snafu(implicit)]
<a href=#162 id=162 data-nosnippet>162</a> </span>location: Location,
<a href=#163 id=163 data-nosnippet>163</a> },
<a href=#164 id=164 data-nosnippet>164</a>
<a href=#165 id=165 data-nosnippet>165</a> <span class="attr">#[snafu(display(<span class="string">"Stream timeout"</span>))]
<a href=#166 id=166 data-nosnippet>166</a> </span>StreamTimeout {
<a href=#167 id=167 data-nosnippet>167</a> <span class="attr">#[snafu(implicit)]
<a href=#168 id=168 data-nosnippet>168</a> </span>location: Location,
<a href=#169 id=169 data-nosnippet>169</a> },
<a href=#170 id=170 data-nosnippet>170</a>
<a href=#171 id=171 data-nosnippet>171</a> <span class="attr">#[snafu(display(<span class="string">"RecordBatch slice index overflow: {visit_index} &gt; {size}"</span>))]
<a href=#172 id=172 data-nosnippet>172</a> </span>RecordBatchSliceIndexOverflow {
<a href=#173 id=173 data-nosnippet>173</a> <span class="attr">#[snafu(implicit)]
<a href=#174 id=174 data-nosnippet>174</a> </span>location: Location,
<a href=#175 id=175 data-nosnippet>175</a> size: usize,
<a href=#176 id=176 data-nosnippet>176</a> visit_index: usize,
<a href=#177 id=177 data-nosnippet>177</a> },
<a href=#178 id=178 data-nosnippet>178</a>
<a href=#179 id=179 data-nosnippet>179</a> <span class="attr">#[snafu(display(<span class="string">"Stream has been cancelled"</span>))]
<a href=#180 id=180 data-nosnippet>180</a> </span>StreamCancelled {
<a href=#181 id=181 data-nosnippet>181</a> <span class="attr">#[snafu(implicit)]
<a href=#182 id=182 data-nosnippet>182</a> </span>location: Location,
<a href=#183 id=183 data-nosnippet>183</a> },
<a href=#184 id=184 data-nosnippet>184</a>
<a href=#185 id=185 data-nosnippet>185</a> <span class="attr">#[snafu(display(<span class="string">"Exceeded memory limit: {}"</span>, msg))]
<a href=#186 id=186 data-nosnippet>186</a> </span>ExceedMemoryLimit {
<a href=#187 id=187 data-nosnippet>187</a> msg: String,
<a href=#188 id=188 data-nosnippet>188</a> <span class="attr">#[snafu(implicit)]
<a href=#189 id=189 data-nosnippet>189</a> </span>location: Location,
<a href=#190 id=190 data-nosnippet>190</a> },
<a href=#191 id=191 data-nosnippet>191</a>
<a href=#192 id=192 data-nosnippet>192</a> <span class="attr">#[snafu(display(<span class="string">"Failed to align JSON array, reason: {reason}"</span>))]
<a href=#193 id=193 data-nosnippet>193</a> </span>AlignJsonArray {
<a href=#194 id=194 data-nosnippet>194</a> reason: String,
<a href=#195 id=195 data-nosnippet>195</a> <span class="attr">#[snafu(implicit)]
<a href=#196 id=196 data-nosnippet>196</a> </span>location: Location,
<a href=#197 id=197 data-nosnippet>197</a> },
<a href=#198 id=198 data-nosnippet>198</a>}
<a href=#199 id=199 data-nosnippet>199</a>
<a href=#200 id=200 data-nosnippet>200</a><span class="kw">impl </span>ErrorExt <span class="kw">for </span>Error {
<a href=#201 id=201 data-nosnippet>201</a> <span class="kw">fn </span>status_code(<span class="kw-2">&amp;</span><span class="self">self</span>) -&gt; StatusCode {
<a href=#202 id=202 data-nosnippet>202</a> <span class="kw">match </span><span class="self">self </span>{
<a href=#203 id=203 data-nosnippet>203</a> Error::NewDfRecordBatch { .. }
<a href=#204 id=204 data-nosnippet>204</a> | Error::EmptyStream { .. }
<a href=#205 id=205 data-nosnippet>205</a> | Error::SchemaNotMatch { .. } =&gt; StatusCode::InvalidArguments,
<a href=#206 id=206 data-nosnippet>206</a>
<a href=#207 id=207 data-nosnippet>207</a> Error::DataTypes { .. }
<a href=#208 id=208 data-nosnippet>208</a> | Error::CreateRecordBatches { .. }
<a href=#209 id=209 data-nosnippet>209</a> | Error::Format { .. }
<a href=#210 id=210 data-nosnippet>210</a> | Error::ToArrowScalar { .. }
<a href=#211 id=211 data-nosnippet>211</a> | Error::ProjectArrowRecordBatch { .. }
<a href=#212 id=212 data-nosnippet>212</a> | Error::PhysicalExpr { .. }
<a href=#213 id=213 data-nosnippet>213</a> | Error::RecordBatchSliceIndexOverflow { .. }
<a href=#214 id=214 data-nosnippet>214</a> | Error::AlignJsonArray { .. } =&gt; StatusCode::Internal,
<a href=#215 id=215 data-nosnippet>215</a>
<a href=#216 id=216 data-nosnippet>216</a> Error::PollStream { .. } =&gt; StatusCode::EngineExecuteQuery,
<a href=#217 id=217 data-nosnippet>217</a>
<a href=#218 id=218 data-nosnippet>218</a> Error::ArrowCompute { .. } =&gt; StatusCode::IllegalState,
<a href=#219 id=219 data-nosnippet>219</a>
<a href=#220 id=220 data-nosnippet>220</a> Error::ColumnNotExists { .. } =&gt; StatusCode::TableColumnNotFound,
<a href=#221 id=221 data-nosnippet>221</a>
<a href=#222 id=222 data-nosnippet>222</a> Error::External { source, .. } =&gt; source.status_code(),
<a href=#223 id=223 data-nosnippet>223</a>
<a href=#224 id=224 data-nosnippet>224</a> Error::UnsupportedOperation { .. } =&gt; StatusCode::Unsupported,
<a href=#225 id=225 data-nosnippet>225</a>
<a href=#226 id=226 data-nosnippet>226</a> Error::SchemaConversion { source, .. } | Error::CastVector { source, .. } =&gt; {
<a href=#227 id=227 data-nosnippet>227</a> source.status_code()
<a href=#228 id=228 data-nosnippet>228</a> }
<a href=#229 id=229 data-nosnippet>229</a>
<a href=#230 id=230 data-nosnippet>230</a> Error::StreamTimeout { .. } =&gt; StatusCode::Cancelled,
<a href=#231 id=231 data-nosnippet>231</a>
<a href=#232 id=232 data-nosnippet>232</a> Error::StreamCancelled { .. } =&gt; StatusCode::Cancelled,
<a href=#233 id=233 data-nosnippet>233</a>
<a href=#234 id=234 data-nosnippet>234</a> Error::ExceedMemoryLimit { .. } =&gt; StatusCode::RuntimeResourcesExhausted,
<a href=#235 id=235 data-nosnippet>235</a> }
<a href=#236 id=236 data-nosnippet>236</a> }
<a href=#237 id=237 data-nosnippet>237</a>
<a href=#238 id=238 data-nosnippet>238</a> <span class="kw">fn </span>as_any(<span class="kw-2">&amp;</span><span class="self">self</span>) -&gt; <span class="kw-2">&amp;</span><span class="kw">dyn </span>Any {
<a href=#239 id=239 data-nosnippet>239</a> <span class="self">self
<a href=#240 id=240 data-nosnippet>240</a> </span>}
<a href=#241 id=241 data-nosnippet>241</a>}</code></pre></div></section></main></body></html>

773
src/common_recordbatch/filter.rs.html Normal file
View File

@@ -0,0 +1,773 @@
<!DOCTYPE html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1.0"><meta name="generator" content="rustdoc"><meta name="description" content="Source of the Rust file `src/common/recordbatch/src/filter.rs`."><title>filter.rs - source</title><script>if(window.location.protocol!=="file:")document.head.insertAdjacentHTML("beforeend","SourceSerif4-Regular-6b053e98.ttf.woff2,FiraSans-Italic-81dc35de.woff2,FiraSans-Regular-0fe48ade.woff2,FiraSans-MediumItalic-ccf7e434.woff2,FiraSans-Medium-e1aa3f0a.woff2,SourceCodePro-Regular-8badfe75.ttf.woff2,SourceCodePro-Semibold-aa29a496.ttf.woff2".split(",").map(f=>`<link rel="preload" as="font" type="font/woff2"href="../../static.files/${f}">`).join(""))</script><link rel="stylesheet" href="../../static.files/normalize-9960930a.css"><link rel="stylesheet" href="../../static.files/rustdoc-e56847b5.css"><meta name="rustdoc-vars" data-root-path="../../" data-static-root-path="../../static.files/" data-current-crate="common_recordbatch" data-themes="" data-resource-suffix="" data-rustdoc-version="1.92.0-nightly (fa3155a64 2025-09-30)" data-channel="nightly" data-search-js="search-e256b49e.js" data-stringdex-js="stringdex-828709d0.js" data-settings-js="settings-c38705f0.js" ><script src="../../static.files/storage-e2aeef58.js"></script><script defer src="../../static.files/src-script-813739b1.js"></script><script defer src="../../src-files.js"></script><script defer src="../../static.files/main-ce535bd0.js"></script><noscript><link rel="stylesheet" href="../../static.files/noscript-263c88ec.css"></noscript><link rel="alternate icon" type="image/png" href="../../static.files/favicon-32x32-eab170b8.png"><link rel="icon" type="image/svg+xml" href="../../static.files/favicon-044be391.svg"></head><body class="rustdoc src"><!--[if lte IE 11]><div class="warning">This old browser is unsupported and will most likely display funky things.</div><![endif]--><nav class="sidebar"><div class="src-sidebar-title"><h2>Files</h2></div></nav><div class="sidebar-resizer" title="Drag to resize sidebar"></div><main><section id="main-content" class="content"><div class="main-heading"><h1><div class="sub-heading">common_recordbatch/</div>filter.rs</h1><rustdoc-toolbar></rustdoc-toolbar></div><div class="example-wrap digits-3"><pre class="rust"><code><a href=#1 id=1 data-nosnippet>1</a><span class="comment">// Copyright 2023 Greptime Team
<a href=#2 id=2 data-nosnippet>2</a>//
<a href=#3 id=3 data-nosnippet>3</a>// Licensed under the Apache License, Version 2.0 (the "License");
<a href=#4 id=4 data-nosnippet>4</a>// you may not use this file except in compliance with the License.
<a href=#5 id=5 data-nosnippet>5</a>// You may obtain a copy of the License at
<a href=#6 id=6 data-nosnippet>6</a>//
<a href=#7 id=7 data-nosnippet>7</a>// http://www.apache.org/licenses/LICENSE-2.0
<a href=#8 id=8 data-nosnippet>8</a>//
<a href=#9 id=9 data-nosnippet>9</a>// Unless required by applicable law or agreed to in writing, software
<a href=#10 id=10 data-nosnippet>10</a>// distributed under the License is distributed on an "AS IS" BASIS,
<a href=#11 id=11 data-nosnippet>11</a>// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
<a href=#12 id=12 data-nosnippet>12</a>// See the License for the specific language governing permissions and
<a href=#13 id=13 data-nosnippet>13</a>// limitations under the License.
<a href=#14 id=14 data-nosnippet>14</a>
<a href=#15 id=15 data-nosnippet>15</a></span><span class="doccomment">//! Util record batch stream wrapper that can perform precise filter.
<a href=#16 id=16 data-nosnippet>16</a>
<a href=#17 id=17 data-nosnippet>17</a></span><span class="kw">use </span>std::sync::Arc;
<a href=#18 id=18 data-nosnippet>18</a>
<a href=#19 id=19 data-nosnippet>19</a><span class="kw">use </span>datafusion::error::Result <span class="kw">as </span>DfResult;
<a href=#20 id=20 data-nosnippet>20</a><span class="kw">use </span>datafusion::logical_expr::{Expr, Literal, Operator};
<a href=#21 id=21 data-nosnippet>21</a><span class="kw">use </span>datafusion::physical_plan::PhysicalExpr;
<a href=#22 id=22 data-nosnippet>22</a><span class="kw">use </span>datafusion_common::arrow::array::{ArrayRef, Datum, Scalar};
<a href=#23 id=23 data-nosnippet>23</a><span class="kw">use </span>datafusion_common::arrow::buffer::BooleanBuffer;
<a href=#24 id=24 data-nosnippet>24</a><span class="kw">use </span>datafusion_common::arrow::compute::kernels::cmp;
<a href=#25 id=25 data-nosnippet>25</a><span class="kw">use </span>datafusion_common::cast::{as_boolean_array, as_null_array, as_string_array};
<a href=#26 id=26 data-nosnippet>26</a><span class="kw">use </span>datafusion_common::{DataFusionError, ScalarValue, internal_err};
<a href=#27 id=27 data-nosnippet>27</a><span class="kw">use </span>datatypes::arrow::array::{
<a href=#28 id=28 data-nosnippet>28</a> Array, ArrayAccessor, ArrayData, BooleanArray, BooleanBufferBuilder, DictionaryArray,
<a href=#29 id=29 data-nosnippet>29</a> RecordBatch, StringArrayType,
<a href=#30 id=30 data-nosnippet>30</a>};
<a href=#31 id=31 data-nosnippet>31</a><span class="kw">use </span>datatypes::arrow::compute::filter_record_batch;
<a href=#32 id=32 data-nosnippet>32</a><span class="kw">use </span>datatypes::arrow::datatypes::{DataType, UInt32Type};
<a href=#33 id=33 data-nosnippet>33</a><span class="kw">use </span>datatypes::arrow::error::ArrowError;
<a href=#34 id=34 data-nosnippet>34</a><span class="kw">use </span>datatypes::compute::or_kleene;
<a href=#35 id=35 data-nosnippet>35</a><span class="kw">use </span>datatypes::value::Value;
<a href=#36 id=36 data-nosnippet>36</a><span class="kw">use </span>datatypes::vectors::VectorRef;
<a href=#37 id=37 data-nosnippet>37</a><span class="kw">use </span>regex::Regex;
<a href=#38 id=38 data-nosnippet>38</a><span class="kw">use </span>snafu::ResultExt;
<a href=#39 id=39 data-nosnippet>39</a>
<a href=#40 id=40 data-nosnippet>40</a><span class="kw">use </span><span class="kw">crate</span>::error::{ArrowComputeSnafu, <span class="prelude-ty">Result</span>, ToArrowScalarSnafu, UnsupportedOperationSnafu};
<a href=#41 id=41 data-nosnippet>41</a>
<a href=#42 id=42 data-nosnippet>42</a><span class="doccomment">/// An inplace expr evaluator for simple filter. Only support
<a href=#43 id=43 data-nosnippet>43</a>/// - `col` `op` `literal`
<a href=#44 id=44 data-nosnippet>44</a>/// - `literal` `op` `col`
<a href=#45 id=45 data-nosnippet>45</a>///
<a href=#46 id=46 data-nosnippet>46</a>/// And the `op` is one of `=`, `!=`, `&gt;`, `&gt;=`, `&lt;`, `&lt;=`,
<a href=#47 id=47 data-nosnippet>47</a>/// or regex operators: `~`, `~*`, `!~`, `!~*`.
<a href=#48 id=48 data-nosnippet>48</a>///
<a href=#49 id=49 data-nosnippet>49</a>/// This struct contains normalized predicate expr. In the form of
<a href=#50 id=50 data-nosnippet>50</a>/// `col` `op` `literal` where the `col` is provided from input.
<a href=#51 id=51 data-nosnippet>51</a></span><span class="attr">#[derive(Debug)]
<a href=#52 id=52 data-nosnippet>52</a></span><span class="kw">pub struct </span>SimpleFilterEvaluator {
<a href=#53 id=53 data-nosnippet>53</a> <span class="doccomment">/// Name of the referenced column.
<a href=#54 id=54 data-nosnippet>54</a> </span>column_name: String,
<a href=#55 id=55 data-nosnippet>55</a> <span class="doccomment">/// The literal value.
<a href=#56 id=56 data-nosnippet>56</a> </span>literal: Scalar&lt;ArrayRef&gt;,
<a href=#57 id=57 data-nosnippet>57</a> <span class="doccomment">/// The operator.
<a href=#58 id=58 data-nosnippet>58</a> </span>op: Operator,
<a href=#59 id=59 data-nosnippet>59</a> <span class="doccomment">/// Only used when the operator is `Or`-chain.
<a href=#60 id=60 data-nosnippet>60</a> </span>literal_list: Vec&lt;Scalar&lt;ArrayRef&gt;&gt;,
<a href=#61 id=61 data-nosnippet>61</a> <span class="doccomment">/// Pre-compiled regex.
<a href=#62 id=62 data-nosnippet>62</a> /// Only used when the operator is regex operators.
<a href=#63 id=63 data-nosnippet>63</a> /// If the regex is empty, it is also `None`.
<a href=#64 id=64 data-nosnippet>64</a> </span>regex: <span class="prelude-ty">Option</span>&lt;Regex&gt;,
<a href=#65 id=65 data-nosnippet>65</a> <span class="doccomment">/// Whether the regex is negative.
<a href=#66 id=66 data-nosnippet>66</a> </span>regex_negative: bool,
<a href=#67 id=67 data-nosnippet>67</a>}
<a href=#68 id=68 data-nosnippet>68</a>
<a href=#69 id=69 data-nosnippet>69</a><span class="kw">impl </span>SimpleFilterEvaluator {
<a href=#70 id=70 data-nosnippet>70</a> <span class="kw">pub fn </span>new&lt;T: Literal&gt;(column_name: String, lit: T, op: Operator) -&gt; <span class="prelude-ty">Option</span>&lt;<span class="self">Self</span>&gt; {
<a href=#71 id=71 data-nosnippet>71</a> <span class="kw">match </span>op {
<a href=#72 id=72 data-nosnippet>72</a> Operator::Eq
<a href=#73 id=73 data-nosnippet>73</a> | Operator::NotEq
<a href=#74 id=74 data-nosnippet>74</a> | Operator::Lt
<a href=#75 id=75 data-nosnippet>75</a> | Operator::LtEq
<a href=#76 id=76 data-nosnippet>76</a> | Operator::Gt
<a href=#77 id=77 data-nosnippet>77</a> | Operator::GtEq =&gt; {}
<a href=#78 id=78 data-nosnippet>78</a> <span class="kw">_ </span>=&gt; <span class="kw">return </span><span class="prelude-val">None</span>,
<a href=#79 id=79 data-nosnippet>79</a> }
<a href=#80 id=80 data-nosnippet>80</a>
<a href=#81 id=81 data-nosnippet>81</a> <span class="kw">let </span>Expr::Literal(val, <span class="kw">_</span>) = lit.lit() <span class="kw">else </span>{
<a href=#82 id=82 data-nosnippet>82</a> <span class="kw">return </span><span class="prelude-val">None</span>;
<a href=#83 id=83 data-nosnippet>83</a> };
<a href=#84 id=84 data-nosnippet>84</a>
<a href=#85 id=85 data-nosnippet>85</a> <span class="prelude-val">Some</span>(<span class="self">Self </span>{
<a href=#86 id=86 data-nosnippet>86</a> column_name,
<a href=#87 id=87 data-nosnippet>87</a> literal: val.to_scalar().ok()<span class="question-mark">?</span>,
<a href=#88 id=88 data-nosnippet>88</a> op,
<a href=#89 id=89 data-nosnippet>89</a> literal_list: <span class="macro">vec!</span>[],
<a href=#90 id=90 data-nosnippet>90</a> regex: <span class="prelude-val">None</span>,
<a href=#91 id=91 data-nosnippet>91</a> regex_negative: <span class="bool-val">false</span>,
<a href=#92 id=92 data-nosnippet>92</a> })
<a href=#93 id=93 data-nosnippet>93</a> }
<a href=#94 id=94 data-nosnippet>94</a>
<a href=#95 id=95 data-nosnippet>95</a> <span class="kw">pub fn </span>try_new(predicate: <span class="kw-2">&amp;</span>Expr) -&gt; <span class="prelude-ty">Option</span>&lt;<span class="self">Self</span>&gt; {
<a href=#96 id=96 data-nosnippet>96</a> <span class="kw">match </span>predicate {
<a href=#97 id=97 data-nosnippet>97</a> Expr::BinaryExpr(binary) =&gt; {
<a href=#98 id=98 data-nosnippet>98</a> <span class="comment">// check if the expr is in the supported form
<a href=#99 id=99 data-nosnippet>99</a> </span><span class="kw">match </span>binary.op {
<a href=#100 id=100 data-nosnippet>100</a> Operator::Eq
<a href=#101 id=101 data-nosnippet>101</a> | Operator::NotEq
<a href=#102 id=102 data-nosnippet>102</a> | Operator::Lt
<a href=#103 id=103 data-nosnippet>103</a> | Operator::LtEq
<a href=#104 id=104 data-nosnippet>104</a> | Operator::Gt
<a href=#105 id=105 data-nosnippet>105</a> | Operator::GtEq
<a href=#106 id=106 data-nosnippet>106</a> | Operator::RegexMatch
<a href=#107 id=107 data-nosnippet>107</a> | Operator::RegexIMatch
<a href=#108 id=108 data-nosnippet>108</a> | Operator::RegexNotMatch
<a href=#109 id=109 data-nosnippet>109</a> | Operator::RegexNotIMatch =&gt; {}
<a href=#110 id=110 data-nosnippet>110</a> Operator::Or =&gt; {
<a href=#111 id=111 data-nosnippet>111</a> <span class="kw">let </span>lhs = <span class="self">Self</span>::try_new(<span class="kw-2">&amp;</span>binary.left)<span class="question-mark">?</span>;
<a href=#112 id=112 data-nosnippet>112</a> <span class="kw">let </span>rhs = <span class="self">Self</span>::try_new(<span class="kw-2">&amp;</span>binary.right)<span class="question-mark">?</span>;
<a href=#113 id=113 data-nosnippet>113</a> <span class="kw">if </span>lhs.column_name != rhs.column_name
<a href=#114 id=114 data-nosnippet>114</a> || !<span class="macro">matches!</span>(lhs.op, Operator::Eq | Operator::Or)
<a href=#115 id=115 data-nosnippet>115</a> || !<span class="macro">matches!</span>(rhs.op, Operator::Eq | Operator::Or)
<a href=#116 id=116 data-nosnippet>116</a> {
<a href=#117 id=117 data-nosnippet>117</a> <span class="kw">return </span><span class="prelude-val">None</span>;
<a href=#118 id=118 data-nosnippet>118</a> }
<a href=#119 id=119 data-nosnippet>119</a> <span class="kw">let </span><span class="kw-2">mut </span>list = <span class="macro">vec!</span>[];
<a href=#120 id=120 data-nosnippet>120</a> <span class="kw">let </span>placeholder_literal = lhs.literal.clone();
<a href=#121 id=121 data-nosnippet>121</a> <span class="comment">// above check guarantees the op is either `Eq` or `Or`
<a href=#122 id=122 data-nosnippet>122</a> </span><span class="kw">if </span><span class="macro">matches!</span>(lhs.op, Operator::Or) {
<a href=#123 id=123 data-nosnippet>123</a> list.extend(lhs.literal_list);
<a href=#124 id=124 data-nosnippet>124</a> } <span class="kw">else </span>{
<a href=#125 id=125 data-nosnippet>125</a> list.push(lhs.literal);
<a href=#126 id=126 data-nosnippet>126</a> }
<a href=#127 id=127 data-nosnippet>127</a> <span class="kw">if </span><span class="macro">matches!</span>(rhs.op, Operator::Or) {
<a href=#128 id=128 data-nosnippet>128</a> list.extend(rhs.literal_list);
<a href=#129 id=129 data-nosnippet>129</a> } <span class="kw">else </span>{
<a href=#130 id=130 data-nosnippet>130</a> list.push(rhs.literal);
<a href=#131 id=131 data-nosnippet>131</a> }
<a href=#132 id=132 data-nosnippet>132</a> <span class="kw">return </span><span class="prelude-val">Some</span>(<span class="self">Self </span>{
<a href=#133 id=133 data-nosnippet>133</a> column_name: lhs.column_name,
<a href=#134 id=134 data-nosnippet>134</a> literal: placeholder_literal,
<a href=#135 id=135 data-nosnippet>135</a> op: Operator::Or,
<a href=#136 id=136 data-nosnippet>136</a> literal_list: list,
<a href=#137 id=137 data-nosnippet>137</a> regex: <span class="prelude-val">None</span>,
<a href=#138 id=138 data-nosnippet>138</a> regex_negative: <span class="bool-val">false</span>,
<a href=#139 id=139 data-nosnippet>139</a> });
<a href=#140 id=140 data-nosnippet>140</a> }
<a href=#141 id=141 data-nosnippet>141</a> <span class="kw">_ </span>=&gt; <span class="kw">return </span><span class="prelude-val">None</span>,
<a href=#142 id=142 data-nosnippet>142</a> }
<a href=#143 id=143 data-nosnippet>143</a>
<a href=#144 id=144 data-nosnippet>144</a> <span class="comment">// swap the expr if it is in the form of `literal` `op` `col`
<a href=#145 id=145 data-nosnippet>145</a> </span><span class="kw">let </span><span class="kw-2">mut </span>op = binary.op;
<a href=#146 id=146 data-nosnippet>146</a> <span class="kw">let </span>(lhs, rhs) = <span class="kw">match </span>(<span class="kw-2">&amp;*</span>binary.left, <span class="kw-2">&amp;*</span>binary.right) {
<a href=#147 id=147 data-nosnippet>147</a> (Expr::Column(col), Expr::Literal(lit, <span class="kw">_</span>)) =&gt; (col, lit),
<a href=#148 id=148 data-nosnippet>148</a> (Expr::Literal(lit, <span class="kw">_</span>), Expr::Column(col)) =&gt; {
<a href=#149 id=149 data-nosnippet>149</a> <span class="comment">// safety: The previous check ensures the operator is able to swap.
<a href=#150 id=150 data-nosnippet>150</a> </span>op = op.swap().unwrap();
<a href=#151 id=151 data-nosnippet>151</a> (col, lit)
<a href=#152 id=152 data-nosnippet>152</a> }
<a href=#153 id=153 data-nosnippet>153</a> <span class="kw">_ </span>=&gt; <span class="kw">return </span><span class="prelude-val">None</span>,
<a href=#154 id=154 data-nosnippet>154</a> };
<a href=#155 id=155 data-nosnippet>155</a>
<a href=#156 id=156 data-nosnippet>156</a> <span class="kw">let </span>(regex, regex_negative) = <span class="self">Self</span>::maybe_build_regex(op, rhs).ok()<span class="question-mark">?</span>;
<a href=#157 id=157 data-nosnippet>157</a> <span class="kw">let </span>literal = rhs.to_scalar().ok()<span class="question-mark">?</span>;
<a href=#158 id=158 data-nosnippet>158</a> <span class="prelude-val">Some</span>(<span class="self">Self </span>{
<a href=#159 id=159 data-nosnippet>159</a> column_name: lhs.name.clone(),
<a href=#160 id=160 data-nosnippet>160</a> literal,
<a href=#161 id=161 data-nosnippet>161</a> op,
<a href=#162 id=162 data-nosnippet>162</a> literal_list: <span class="macro">vec!</span>[],
<a href=#163 id=163 data-nosnippet>163</a> regex,
<a href=#164 id=164 data-nosnippet>164</a> regex_negative,
<a href=#165 id=165 data-nosnippet>165</a> })
<a href=#166 id=166 data-nosnippet>166</a> }
<a href=#167 id=167 data-nosnippet>167</a> <span class="kw">_ </span>=&gt; <span class="prelude-val">None</span>,
<a href=#168 id=168 data-nosnippet>168</a> }
<a href=#169 id=169 data-nosnippet>169</a> }
<a href=#170 id=170 data-nosnippet>170</a>
<a href=#171 id=171 data-nosnippet>171</a> <span class="doccomment">/// Get the name of the referenced column.
<a href=#172 id=172 data-nosnippet>172</a> </span><span class="kw">pub fn </span>column_name(<span class="kw-2">&amp;</span><span class="self">self</span>) -&gt; <span class="kw-2">&amp;</span>str {
<a href=#173 id=173 data-nosnippet>173</a> <span class="kw-2">&amp;</span><span class="self">self</span>.column_name
<a href=#174 id=174 data-nosnippet>174</a> }
<a href=#175 id=175 data-nosnippet>175</a>
<a href=#176 id=176 data-nosnippet>176</a> <span class="kw">pub fn </span>is_eq(<span class="kw-2">&amp;</span><span class="self">self</span>) -&gt; bool {
<a href=#177 id=177 data-nosnippet>177</a> <span class="macro">matches!</span>(<span class="self">self</span>.op, Operator::Eq)
<a href=#178 id=178 data-nosnippet>178</a> }
<a href=#179 id=179 data-nosnippet>179</a>
<a href=#180 id=180 data-nosnippet>180</a> <span class="kw">pub fn </span>is_not_eq(<span class="kw-2">&amp;</span><span class="self">self</span>) -&gt; bool {
<a href=#181 id=181 data-nosnippet>181</a> <span class="macro">matches!</span>(<span class="self">self</span>.op, Operator::NotEq)
<a href=#182 id=182 data-nosnippet>182</a> }
<a href=#183 id=183 data-nosnippet>183</a>
<a href=#184 id=184 data-nosnippet>184</a> <span class="kw">pub fn </span>is_lt(<span class="kw-2">&amp;</span><span class="self">self</span>) -&gt; bool {
<a href=#185 id=185 data-nosnippet>185</a> <span class="macro">matches!</span>(<span class="self">self</span>.op, Operator::Lt)
<a href=#186 id=186 data-nosnippet>186</a> }
<a href=#187 id=187 data-nosnippet>187</a>
<a href=#188 id=188 data-nosnippet>188</a> <span class="kw">pub fn </span>is_lt_eq(<span class="kw-2">&amp;</span><span class="self">self</span>) -&gt; bool {
<a href=#189 id=189 data-nosnippet>189</a> <span class="macro">matches!</span>(<span class="self">self</span>.op, Operator::LtEq)
<a href=#190 id=190 data-nosnippet>190</a> }
<a href=#191 id=191 data-nosnippet>191</a>
<a href=#192 id=192 data-nosnippet>192</a> <span class="kw">pub fn </span>is_gt(<span class="kw-2">&amp;</span><span class="self">self</span>) -&gt; bool {
<a href=#193 id=193 data-nosnippet>193</a> <span class="macro">matches!</span>(<span class="self">self</span>.op, Operator::Gt)
<a href=#194 id=194 data-nosnippet>194</a> }
<a href=#195 id=195 data-nosnippet>195</a>
<a href=#196 id=196 data-nosnippet>196</a> <span class="kw">pub fn </span>is_gt_eq(<span class="kw-2">&amp;</span><span class="self">self</span>) -&gt; bool {
<a href=#197 id=197 data-nosnippet>197</a> <span class="macro">matches!</span>(<span class="self">self</span>.op, Operator::GtEq)
<a href=#198 id=198 data-nosnippet>198</a> }
<a href=#199 id=199 data-nosnippet>199</a>
<a href=#200 id=200 data-nosnippet>200</a> <span class="doccomment">/// Returns true if this filter represents an `OR` chain of equality comparisons, e.g.
<a href=#201 id=201 data-nosnippet>201</a> /// `col = lit1 OR col = lit2 ...`.
<a href=#202 id=202 data-nosnippet>202</a> </span><span class="kw">pub fn </span>is_or_eq_chain(<span class="kw-2">&amp;</span><span class="self">self</span>) -&gt; bool {
<a href=#203 id=203 data-nosnippet>203</a> <span class="macro">matches!</span>(<span class="self">self</span>.op, Operator::Or)
<a href=#204 id=204 data-nosnippet>204</a> }
<a href=#205 id=205 data-nosnippet>205</a>
<a href=#206 id=206 data-nosnippet>206</a> <span class="doccomment">/// Returns the literal as a [`Value`]. It returns `None` if the literal can't be converted.
<a href=#207 id=207 data-nosnippet>207</a> </span><span class="kw">pub fn </span>literal_value(<span class="kw-2">&amp;</span><span class="self">self</span>) -&gt; <span class="prelude-ty">Option</span>&lt;Value&gt; {
<a href=#208 id=208 data-nosnippet>208</a> <span class="kw">let </span>array = <span class="self">self</span>.literal.get().<span class="number">0</span>;
<a href=#209 id=209 data-nosnippet>209</a> <span class="kw">let </span>scalar = ScalarValue::try_from_array(array, <span class="number">0</span>).ok()<span class="question-mark">?</span>;
<a href=#210 id=210 data-nosnippet>210</a> Value::try_from(scalar).ok()
<a href=#211 id=211 data-nosnippet>211</a> }
<a href=#212 id=212 data-nosnippet>212</a>
<a href=#213 id=213 data-nosnippet>213</a> <span class="doccomment">/// Returns the literal list as a list of [`Value`]s. It returns `None` if any literal can't be
<a href=#214 id=214 data-nosnippet>214</a> /// converted.
<a href=#215 id=215 data-nosnippet>215</a> </span><span class="kw">pub fn </span>literal_list_values(<span class="kw-2">&amp;</span><span class="self">self</span>) -&gt; <span class="prelude-ty">Option</span>&lt;Vec&lt;Value&gt;&gt; {
<a href=#216 id=216 data-nosnippet>216</a> <span class="self">self</span>.literal_list
<a href=#217 id=217 data-nosnippet>217</a> .iter()
<a href=#218 id=218 data-nosnippet>218</a> .map(|scalar| {
<a href=#219 id=219 data-nosnippet>219</a> <span class="kw">let </span>array = scalar.get().<span class="number">0</span>;
<a href=#220 id=220 data-nosnippet>220</a> <span class="kw">let </span>scalar = ScalarValue::try_from_array(array, <span class="number">0</span>).ok()<span class="question-mark">?</span>;
<a href=#221 id=221 data-nosnippet>221</a> Value::try_from(scalar).ok()
<a href=#222 id=222 data-nosnippet>222</a> })
<a href=#223 id=223 data-nosnippet>223</a> .collect()
<a href=#224 id=224 data-nosnippet>224</a> }
<a href=#225 id=225 data-nosnippet>225</a>
<a href=#226 id=226 data-nosnippet>226</a> <span class="kw">pub fn </span>evaluate_scalar(<span class="kw-2">&amp;</span><span class="self">self</span>, input: <span class="kw-2">&amp;</span>ScalarValue) -&gt; <span class="prelude-ty">Result</span>&lt;bool&gt; {
<a href=#227 id=227 data-nosnippet>227</a> <span class="kw">let </span>input = input
<a href=#228 id=228 data-nosnippet>228</a> .to_scalar()
<a href=#229 id=229 data-nosnippet>229</a> .with_context(|<span class="kw">_</span>| ToArrowScalarSnafu { v: input.clone() })<span class="question-mark">?</span>;
<a href=#230 id=230 data-nosnippet>230</a> <span class="kw">let </span>result = <span class="self">self</span>.evaluate_datum(<span class="kw-2">&amp;</span>input, <span class="number">1</span>)<span class="question-mark">?</span>;
<a href=#231 id=231 data-nosnippet>231</a> <span class="prelude-val">Ok</span>(result.value(<span class="number">0</span>))
<a href=#232 id=232 data-nosnippet>232</a> }
<a href=#233 id=233 data-nosnippet>233</a>
<a href=#234 id=234 data-nosnippet>234</a> <span class="kw">pub fn </span>evaluate_array(<span class="kw-2">&amp;</span><span class="self">self</span>, input: <span class="kw-2">&amp;</span>ArrayRef) -&gt; <span class="prelude-ty">Result</span>&lt;BooleanBuffer&gt; {
<a href=#235 id=235 data-nosnippet>235</a> <span class="self">self</span>.evaluate_datum(input, input.len())
<a href=#236 id=236 data-nosnippet>236</a> }
<a href=#237 id=237 data-nosnippet>237</a>
<a href=#238 id=238 data-nosnippet>238</a> <span class="kw">pub fn </span>evaluate_vector(<span class="kw-2">&amp;</span><span class="self">self</span>, input: <span class="kw-2">&amp;</span>VectorRef) -&gt; <span class="prelude-ty">Result</span>&lt;BooleanBuffer&gt; {
<a href=#239 id=239 data-nosnippet>239</a> <span class="self">self</span>.evaluate_datum(<span class="kw-2">&amp;</span>input.to_arrow_array(), input.len())
<a href=#240 id=240 data-nosnippet>240</a> }
<a href=#241 id=241 data-nosnippet>241</a>
<a href=#242 id=242 data-nosnippet>242</a> <span class="kw">fn </span>evaluate_datum(<span class="kw-2">&amp;</span><span class="self">self</span>, input: <span class="kw-2">&amp;</span><span class="kw">impl </span>Datum, input_len: usize) -&gt; <span class="prelude-ty">Result</span>&lt;BooleanBuffer&gt; {
<a href=#243 id=243 data-nosnippet>243</a> <span class="kw">let </span>result = <span class="kw">match </span><span class="self">self</span>.op {
<a href=#244 id=244 data-nosnippet>244</a> Operator::Eq =&gt; cmp::eq(input, <span class="kw-2">&amp;</span><span class="self">self</span>.literal),
<a href=#245 id=245 data-nosnippet>245</a> Operator::NotEq =&gt; cmp::neq(input, <span class="kw-2">&amp;</span><span class="self">self</span>.literal),
<a href=#246 id=246 data-nosnippet>246</a> Operator::Lt =&gt; cmp::lt(input, <span class="kw-2">&amp;</span><span class="self">self</span>.literal),
<a href=#247 id=247 data-nosnippet>247</a> Operator::LtEq =&gt; cmp::lt_eq(input, <span class="kw-2">&amp;</span><span class="self">self</span>.literal),
<a href=#248 id=248 data-nosnippet>248</a> Operator::Gt =&gt; cmp::gt(input, <span class="kw-2">&amp;</span><span class="self">self</span>.literal),
<a href=#249 id=249 data-nosnippet>249</a> Operator::GtEq =&gt; cmp::gt_eq(input, <span class="kw-2">&amp;</span><span class="self">self</span>.literal),
<a href=#250 id=250 data-nosnippet>250</a> Operator::RegexMatch =&gt; <span class="self">self</span>.regex_match(input),
<a href=#251 id=251 data-nosnippet>251</a> Operator::RegexIMatch =&gt; <span class="self">self</span>.regex_match(input),
<a href=#252 id=252 data-nosnippet>252</a> Operator::RegexNotMatch =&gt; <span class="self">self</span>.regex_match(input),
<a href=#253 id=253 data-nosnippet>253</a> Operator::RegexNotIMatch =&gt; <span class="self">self</span>.regex_match(input),
<a href=#254 id=254 data-nosnippet>254</a> Operator::Or =&gt; {
<a href=#255 id=255 data-nosnippet>255</a> <span class="comment">// OR operator stands for OR-chained EQs (or INLIST in other words)
<a href=#256 id=256 data-nosnippet>256</a> </span><span class="kw">let </span><span class="kw-2">mut </span>result: BooleanArray = <span class="macro">vec!</span>[<span class="bool-val">false</span>; input_len].into();
<a href=#257 id=257 data-nosnippet>257</a> <span class="kw">for </span>literal <span class="kw">in </span><span class="kw-2">&amp;</span><span class="self">self</span>.literal_list {
<a href=#258 id=258 data-nosnippet>258</a> <span class="kw">let </span>rhs = cmp::eq(input, literal).context(ArrowComputeSnafu)<span class="question-mark">?</span>;
<a href=#259 id=259 data-nosnippet>259</a> result = or_kleene(<span class="kw-2">&amp;</span>result, <span class="kw-2">&amp;</span>rhs).context(ArrowComputeSnafu)<span class="question-mark">?</span>;
<a href=#260 id=260 data-nosnippet>260</a> }
<a href=#261 id=261 data-nosnippet>261</a> <span class="prelude-val">Ok</span>(result)
<a href=#262 id=262 data-nosnippet>262</a> }
<a href=#263 id=263 data-nosnippet>263</a> <span class="kw">_ </span>=&gt; {
<a href=#264 id=264 data-nosnippet>264</a> <span class="kw">return </span>UnsupportedOperationSnafu {
<a href=#265 id=265 data-nosnippet>265</a> reason: <span class="macro">format!</span>(<span class="string">"{:?}"</span>, <span class="self">self</span>.op),
<a href=#266 id=266 data-nosnippet>266</a> }
<a href=#267 id=267 data-nosnippet>267</a> .fail();
<a href=#268 id=268 data-nosnippet>268</a> }
<a href=#269 id=269 data-nosnippet>269</a> };
<a href=#270 id=270 data-nosnippet>270</a> result
<a href=#271 id=271 data-nosnippet>271</a> .context(ArrowComputeSnafu)
<a href=#272 id=272 data-nosnippet>272</a> .map(|array| array.values().clone())
<a href=#273 id=273 data-nosnippet>273</a> }
<a href=#274 id=274 data-nosnippet>274</a>
<a href=#275 id=275 data-nosnippet>275</a> <span class="doccomment">/// Builds a regex pattern from a scalar value and operator.
<a href=#276 id=276 data-nosnippet>276</a> /// Returns the `(regex, negative)` and if successful.
<a href=#277 id=277 data-nosnippet>277</a> ///
<a href=#278 id=278 data-nosnippet>278</a> /// Returns `Err` if
<a href=#279 id=279 data-nosnippet>279</a> /// - the value is not a string
<a href=#280 id=280 data-nosnippet>280</a> /// - the regex pattern is invalid
<a href=#281 id=281 data-nosnippet>281</a> ///
<a href=#282 id=282 data-nosnippet>282</a> /// The regex is `None` if
<a href=#283 id=283 data-nosnippet>283</a> /// - the operator is not a regex operator
<a href=#284 id=284 data-nosnippet>284</a> /// - the pattern is empty
<a href=#285 id=285 data-nosnippet>285</a> </span><span class="kw">fn </span>maybe_build_regex(
<a href=#286 id=286 data-nosnippet>286</a> operator: Operator,
<a href=#287 id=287 data-nosnippet>287</a> value: <span class="kw-2">&amp;</span>ScalarValue,
<a href=#288 id=288 data-nosnippet>288</a> ) -&gt; <span class="prelude-ty">Result</span>&lt;(<span class="prelude-ty">Option</span>&lt;Regex&gt;, bool), ArrowError&gt; {
<a href=#289 id=289 data-nosnippet>289</a> <span class="kw">let </span>(ignore_case, negative) = <span class="kw">match </span>operator {
<a href=#290 id=290 data-nosnippet>290</a> Operator::RegexMatch =&gt; (<span class="bool-val">false</span>, <span class="bool-val">false</span>),
<a href=#291 id=291 data-nosnippet>291</a> Operator::RegexIMatch =&gt; (<span class="bool-val">true</span>, <span class="bool-val">false</span>),
<a href=#292 id=292 data-nosnippet>292</a> Operator::RegexNotMatch =&gt; (<span class="bool-val">false</span>, <span class="bool-val">true</span>),
<a href=#293 id=293 data-nosnippet>293</a> Operator::RegexNotIMatch =&gt; (<span class="bool-val">true</span>, <span class="bool-val">true</span>),
<a href=#294 id=294 data-nosnippet>294</a> <span class="kw">_ </span>=&gt; <span class="kw">return </span><span class="prelude-val">Ok</span>((<span class="prelude-val">None</span>, <span class="bool-val">false</span>)),
<a href=#295 id=295 data-nosnippet>295</a> };
<a href=#296 id=296 data-nosnippet>296</a> <span class="kw">let </span>flag = <span class="kw">if </span>ignore_case { <span class="prelude-val">Some</span>(<span class="string">"i"</span>) } <span class="kw">else </span>{ <span class="prelude-val">None </span>};
<a href=#297 id=297 data-nosnippet>297</a> <span class="kw">let </span>regex = value
<a href=#298 id=298 data-nosnippet>298</a> .try_as_str()
<a href=#299 id=299 data-nosnippet>299</a> .ok_or_else(|| ArrowError::CastError(<span class="macro">format!</span>(<span class="string">"Cannot cast {:?} to str"</span>, value)))<span class="question-mark">?
<a href=#300 id=300 data-nosnippet>300</a> </span>.ok_or_else(|| ArrowError::CastError(<span class="string">"Regex should not be null"</span>.to_string()))<span class="question-mark">?</span>;
<a href=#301 id=301 data-nosnippet>301</a> <span class="kw">let </span>pattern = <span class="kw">match </span>flag {
<a href=#302 id=302 data-nosnippet>302</a> <span class="prelude-val">Some</span>(flag) =&gt; <span class="macro">format!</span>(<span class="string">"(?{flag}){regex}"</span>),
<a href=#303 id=303 data-nosnippet>303</a> <span class="prelude-val">None </span>=&gt; regex.to_string(),
<a href=#304 id=304 data-nosnippet>304</a> };
<a href=#305 id=305 data-nosnippet>305</a> <span class="kw">if </span>pattern.is_empty() {
<a href=#306 id=306 data-nosnippet>306</a> <span class="prelude-val">Ok</span>((<span class="prelude-val">None</span>, negative))
<a href=#307 id=307 data-nosnippet>307</a> } <span class="kw">else </span>{
<a href=#308 id=308 data-nosnippet>308</a> Regex::new(pattern.as_str())
<a href=#309 id=309 data-nosnippet>309</a> .map_err(|e| {
<a href=#310 id=310 data-nosnippet>310</a> ArrowError::ComputeError(<span class="macro">format!</span>(<span class="string">"Regular expression did not compile: {e:?}"</span>))
<a href=#311 id=311 data-nosnippet>311</a> })
<a href=#312 id=312 data-nosnippet>312</a> .map(|regex| (<span class="prelude-val">Some</span>(regex), negative))
<a href=#313 id=313 data-nosnippet>313</a> }
<a href=#314 id=314 data-nosnippet>314</a> }
<a href=#315 id=315 data-nosnippet>315</a>
<a href=#316 id=316 data-nosnippet>316</a> <span class="kw">fn </span>regex_match(<span class="kw-2">&amp;</span><span class="self">self</span>, input: <span class="kw-2">&amp;</span><span class="kw">impl </span>Datum) -&gt; std::result::Result&lt;BooleanArray, ArrowError&gt; {
<a href=#317 id=317 data-nosnippet>317</a> <span class="kw">let </span>array = input.get().<span class="number">0</span>;
<a href=#318 id=318 data-nosnippet>318</a>
<a href=#319 id=319 data-nosnippet>319</a> <span class="comment">// Try to cast to StringArray first
<a href=#320 id=320 data-nosnippet>320</a> </span><span class="kw">if let </span><span class="prelude-val">Ok</span>(string_array) = as_string_array(array) {
<a href=#321 id=321 data-nosnippet>321</a> <span class="kw">let </span><span class="kw-2">mut </span>result = regexp_is_match_scalar(string_array, <span class="self">self</span>.regex.as_ref())<span class="question-mark">?</span>;
<a href=#322 id=322 data-nosnippet>322</a> <span class="kw">if </span><span class="self">self</span>.regex_negative {
<a href=#323 id=323 data-nosnippet>323</a> result = datatypes::compute::not(<span class="kw-2">&amp;</span>result)<span class="question-mark">?</span>;
<a href=#324 id=324 data-nosnippet>324</a> }
<a href=#325 id=325 data-nosnippet>325</a> <span class="kw">return </span><span class="prelude-val">Ok</span>(result);
<a href=#326 id=326 data-nosnippet>326</a> }
<a href=#327 id=327 data-nosnippet>327</a>
<a href=#328 id=328 data-nosnippet>328</a> <span class="comment">// Try to cast to StringDictionaryArray
<a href=#329 id=329 data-nosnippet>329</a> </span><span class="kw">if let </span><span class="prelude-val">Some</span>(dict_array) = array.as_any().downcast_ref::&lt;DictionaryArray&lt;UInt32Type&gt;&gt;() {
<a href=#330 id=330 data-nosnippet>330</a> <span class="kw">let </span><span class="kw-2">mut </span>result = regexp_is_match_dictionary(dict_array, <span class="self">self</span>.regex.as_ref())<span class="question-mark">?</span>;
<a href=#331 id=331 data-nosnippet>331</a> <span class="kw">if </span><span class="self">self</span>.regex_negative {
<a href=#332 id=332 data-nosnippet>332</a> result = datatypes::compute::not(<span class="kw-2">&amp;</span>result)<span class="question-mark">?</span>;
<a href=#333 id=333 data-nosnippet>333</a> }
<a href=#334 id=334 data-nosnippet>334</a> <span class="kw">return </span><span class="prelude-val">Ok</span>(result);
<a href=#335 id=335 data-nosnippet>335</a> }
<a href=#336 id=336 data-nosnippet>336</a>
<a href=#337 id=337 data-nosnippet>337</a> <span class="prelude-val">Err</span>(ArrowError::CastError(<span class="macro">format!</span>(
<a href=#338 id=338 data-nosnippet>338</a> <span class="string">"Cannot cast {:?} to StringArray or StringDictionaryArray"</span>,
<a href=#339 id=339 data-nosnippet>339</a> array.data_type()
<a href=#340 id=340 data-nosnippet>340</a> )))
<a href=#341 id=341 data-nosnippet>341</a> }
<a href=#342 id=342 data-nosnippet>342</a>}
<a href=#343 id=343 data-nosnippet>343</a>
<a href=#344 id=344 data-nosnippet>344</a><span class="doccomment">/// Evaluate the predicate on the input [RecordBatch], and return a new [RecordBatch].
<a href=#345 id=345 data-nosnippet>345</a>/// Copy from datafusion::physical_plan::src::filter.rs
<a href=#346 id=346 data-nosnippet>346</a></span><span class="kw">pub fn </span>batch_filter(
<a href=#347 id=347 data-nosnippet>347</a> batch: <span class="kw-2">&amp;</span>RecordBatch,
<a href=#348 id=348 data-nosnippet>348</a> predicate: <span class="kw-2">&amp;</span>Arc&lt;<span class="kw">dyn </span>PhysicalExpr&gt;,
<a href=#349 id=349 data-nosnippet>349</a>) -&gt; DfResult&lt;RecordBatch&gt; {
<a href=#350 id=350 data-nosnippet>350</a> predicate
<a href=#351 id=351 data-nosnippet>351</a> .evaluate(batch)
<a href=#352 id=352 data-nosnippet>352</a> .and_then(|v| v.into_array(batch.num_rows()))
<a href=#353 id=353 data-nosnippet>353</a> .and_then(|array| {
<a href=#354 id=354 data-nosnippet>354</a> <span class="kw">let </span>filter_array = <span class="kw">match </span>as_boolean_array(<span class="kw-2">&amp;</span>array) {
<a href=#355 id=355 data-nosnippet>355</a> <span class="prelude-val">Ok</span>(boolean_array) =&gt; <span class="prelude-val">Ok</span>(boolean_array.clone()),
<a href=#356 id=356 data-nosnippet>356</a> <span class="prelude-val">Err</span>(<span class="kw">_</span>) =&gt; {
<a href=#357 id=357 data-nosnippet>357</a> <span class="kw">let </span><span class="prelude-val">Ok</span>(null_array) = as_null_array(<span class="kw-2">&amp;</span>array) <span class="kw">else </span>{
<a href=#358 id=358 data-nosnippet>358</a> <span class="kw">return </span><span class="macro">internal_err!</span>(
<a href=#359 id=359 data-nosnippet>359</a> <span class="string">"Cannot create filter_array from non-boolean predicates"
<a href=#360 id=360 data-nosnippet>360</a> </span>);
<a href=#361 id=361 data-nosnippet>361</a> };
<a href=#362 id=362 data-nosnippet>362</a>
<a href=#363 id=363 data-nosnippet>363</a> <span class="comment">// if the predicate is null, then the result is also null
<a href=#364 id=364 data-nosnippet>364</a> </span><span class="prelude-val">Ok</span>::&lt;BooleanArray, DataFusionError&gt;(BooleanArray::new_null(null_array.len()))
<a href=#365 id=365 data-nosnippet>365</a> }
<a href=#366 id=366 data-nosnippet>366</a> }<span class="question-mark">?</span>;
<a href=#367 id=367 data-nosnippet>367</a> <span class="prelude-val">Ok</span>(filter_record_batch(batch, <span class="kw-2">&amp;</span>filter_array)<span class="question-mark">?</span>)
<a href=#368 id=368 data-nosnippet>368</a> })
<a href=#369 id=369 data-nosnippet>369</a>}
<a href=#370 id=370 data-nosnippet>370</a>
<a href=#371 id=371 data-nosnippet>371</a><span class="doccomment">/// The same as arrow [regexp_is_match_scalar()](datatypes::compute::kernels::regexp::regexp_is_match_scalar())
<a href=#372 id=372 data-nosnippet>372</a>/// with pre-compiled regex.
<a href=#373 id=373 data-nosnippet>373</a>/// See &lt;https://github.com/apache/arrow-rs/blob/54.2.0/arrow-string/src/regexp.rs#L204-L246&gt; for the implementation details.
<a href=#374 id=374 data-nosnippet>374</a></span><span class="kw">pub fn </span>regexp_is_match_scalar&lt;<span class="lifetime">'a</span>, S&gt;(
<a href=#375 id=375 data-nosnippet>375</a> array: <span class="kw-2">&amp;</span><span class="lifetime">'a </span>S,
<a href=#376 id=376 data-nosnippet>376</a> regex: <span class="prelude-ty">Option</span>&lt;<span class="kw-2">&amp;</span>Regex&gt;,
<a href=#377 id=377 data-nosnippet>377</a>) -&gt; <span class="prelude-ty">Result</span>&lt;BooleanArray, ArrowError&gt;
<a href=#378 id=378 data-nosnippet>378</a><span class="kw">where
<a href=#379 id=379 data-nosnippet>379</a> </span><span class="kw-2">&amp;</span><span class="lifetime">'a </span>S: StringArrayType&lt;<span class="lifetime">'a</span>&gt;,
<a href=#380 id=380 data-nosnippet>380</a>{
<a href=#381 id=381 data-nosnippet>381</a> <span class="kw">let </span>null_bit_buffer = array.nulls().map(|x| x.inner().sliced());
<a href=#382 id=382 data-nosnippet>382</a> <span class="kw">let </span><span class="kw-2">mut </span>result = BooleanBufferBuilder::new(array.len());
<a href=#383 id=383 data-nosnippet>383</a>
<a href=#384 id=384 data-nosnippet>384</a> <span class="kw">if let </span><span class="prelude-val">Some</span>(re) = regex {
<a href=#385 id=385 data-nosnippet>385</a> <span class="kw">for </span>i <span class="kw">in </span><span class="number">0</span>..array.len() {
<a href=#386 id=386 data-nosnippet>386</a> <span class="kw">let </span>value = array.value(i);
<a href=#387 id=387 data-nosnippet>387</a> result.append(re.is_match(value));
<a href=#388 id=388 data-nosnippet>388</a> }
<a href=#389 id=389 data-nosnippet>389</a> } <span class="kw">else </span>{
<a href=#390 id=390 data-nosnippet>390</a> result.append_n(array.len(), <span class="bool-val">true</span>);
<a href=#391 id=391 data-nosnippet>391</a> }
<a href=#392 id=392 data-nosnippet>392</a>
<a href=#393 id=393 data-nosnippet>393</a> <span class="kw">let </span>buffer = result.into();
<a href=#394 id=394 data-nosnippet>394</a> <span class="kw">let </span>data = <span class="kw">unsafe </span>{
<a href=#395 id=395 data-nosnippet>395</a> ArrayData::new_unchecked(
<a href=#396 id=396 data-nosnippet>396</a> DataType::Boolean,
<a href=#397 id=397 data-nosnippet>397</a> array.len(),
<a href=#398 id=398 data-nosnippet>398</a> <span class="prelude-val">None</span>,
<a href=#399 id=399 data-nosnippet>399</a> null_bit_buffer,
<a href=#400 id=400 data-nosnippet>400</a> <span class="number">0</span>,
<a href=#401 id=401 data-nosnippet>401</a> <span class="macro">vec!</span>[buffer],
<a href=#402 id=402 data-nosnippet>402</a> <span class="macro">vec!</span>[],
<a href=#403 id=403 data-nosnippet>403</a> )
<a href=#404 id=404 data-nosnippet>404</a> };
<a href=#405 id=405 data-nosnippet>405</a>
<a href=#406 id=406 data-nosnippet>406</a> <span class="prelude-val">Ok</span>(BooleanArray::from(data))
<a href=#407 id=407 data-nosnippet>407</a>}
<a href=#408 id=408 data-nosnippet>408</a>
<a href=#409 id=409 data-nosnippet>409</a><span class="doccomment">/// Similar to [regexp_is_match_scalar] but for StringDictionaryArray.
<a href=#410 id=410 data-nosnippet>410</a>/// Iterates through dictionary keys to get string values and applies regex matching.
<a href=#411 id=411 data-nosnippet>411</a></span><span class="kw">pub fn </span>regexp_is_match_dictionary(
<a href=#412 id=412 data-nosnippet>412</a> dict_array: <span class="kw-2">&amp;</span>DictionaryArray&lt;UInt32Type&gt;,
<a href=#413 id=413 data-nosnippet>413</a> regex: <span class="prelude-ty">Option</span>&lt;<span class="kw-2">&amp;</span>Regex&gt;,
<a href=#414 id=414 data-nosnippet>414</a>) -&gt; <span class="prelude-ty">Result</span>&lt;BooleanArray, ArrowError&gt; {
<a href=#415 id=415 data-nosnippet>415</a> <span class="comment">// Get the string values from the dictionary
<a href=#416 id=416 data-nosnippet>416</a> </span><span class="kw">let </span>string_values = dict_array
<a href=#417 id=417 data-nosnippet>417</a> .values()
<a href=#418 id=418 data-nosnippet>418</a> .as_any()
<a href=#419 id=419 data-nosnippet>419</a> .downcast_ref::&lt;datatypes::arrow::array::StringArray&gt;()
<a href=#420 id=420 data-nosnippet>420</a> .ok_or_else(|| {
<a href=#421 id=421 data-nosnippet>421</a> ArrowError::CastError(<span class="string">"Dictionary values must be StringArray"</span>.to_string())
<a href=#422 id=422 data-nosnippet>422</a> })<span class="question-mark">?</span>;
<a href=#423 id=423 data-nosnippet>423</a>
<a href=#424 id=424 data-nosnippet>424</a> <span class="kw">let </span>null_bit_buffer = dict_array.nulls().map(|x| x.inner().sliced());
<a href=#425 id=425 data-nosnippet>425</a> <span class="kw">let </span><span class="kw-2">mut </span>result = BooleanBufferBuilder::new(dict_array.len());
<a href=#426 id=426 data-nosnippet>426</a>
<a href=#427 id=427 data-nosnippet>427</a> <span class="kw">if let </span><span class="prelude-val">Some</span>(re) = regex {
<a href=#428 id=428 data-nosnippet>428</a> <span class="kw">let </span>keys = dict_array.keys().values();
<a href=#429 id=429 data-nosnippet>429</a> <span class="kw">for </span>i <span class="kw">in </span><span class="number">0</span>..dict_array.len() {
<a href=#430 id=430 data-nosnippet>430</a> <span class="kw">if </span>dict_array.is_null(i) {
<a href=#431 id=431 data-nosnippet>431</a> result.append(<span class="bool-val">false</span>);
<a href=#432 id=432 data-nosnippet>432</a> } <span class="kw">else </span>{
<a href=#433 id=433 data-nosnippet>433</a> <span class="kw">let </span>key = keys[i] <span class="kw">as </span>usize;
<a href=#434 id=434 data-nosnippet>434</a> <span class="kw">let </span>string_value = string_values.value(key);
<a href=#435 id=435 data-nosnippet>435</a> result.append(re.is_match(string_value));
<a href=#436 id=436 data-nosnippet>436</a> }
<a href=#437 id=437 data-nosnippet>437</a> }
<a href=#438 id=438 data-nosnippet>438</a> } <span class="kw">else </span>{
<a href=#439 id=439 data-nosnippet>439</a> result.append_n(dict_array.len(), <span class="bool-val">true</span>);
<a href=#440 id=440 data-nosnippet>440</a> }
<a href=#441 id=441 data-nosnippet>441</a>
<a href=#442 id=442 data-nosnippet>442</a> <span class="kw">let </span>buffer = result.into();
<a href=#443 id=443 data-nosnippet>443</a> <span class="kw">let </span>data = <span class="kw">unsafe </span>{
<a href=#444 id=444 data-nosnippet>444</a> ArrayData::new_unchecked(
<a href=#445 id=445 data-nosnippet>445</a> DataType::Boolean,
<a href=#446 id=446 data-nosnippet>446</a> dict_array.len(),
<a href=#447 id=447 data-nosnippet>447</a> <span class="prelude-val">None</span>,
<a href=#448 id=448 data-nosnippet>448</a> null_bit_buffer,
<a href=#449 id=449 data-nosnippet>449</a> <span class="number">0</span>,
<a href=#450 id=450 data-nosnippet>450</a> <span class="macro">vec!</span>[buffer],
<a href=#451 id=451 data-nosnippet>451</a> <span class="macro">vec!</span>[],
<a href=#452 id=452 data-nosnippet>452</a> )
<a href=#453 id=453 data-nosnippet>453</a> };
<a href=#454 id=454 data-nosnippet>454</a>
<a href=#455 id=455 data-nosnippet>455</a> <span class="prelude-val">Ok</span>(BooleanArray::from(data))
<a href=#456 id=456 data-nosnippet>456</a>}
<a href=#457 id=457 data-nosnippet>457</a>
<a href=#458 id=458 data-nosnippet>458</a><span class="attr">#[cfg(test)]
<a href=#459 id=459 data-nosnippet>459</a></span><span class="kw">mod </span>test {
<a href=#460 id=460 data-nosnippet>460</a>
<a href=#461 id=461 data-nosnippet>461</a> <span class="kw">use </span>std::sync::Arc;
<a href=#462 id=462 data-nosnippet>462</a>
<a href=#463 id=463 data-nosnippet>463</a> <span class="kw">use </span>datafusion::execution::context::ExecutionProps;
<a href=#464 id=464 data-nosnippet>464</a> <span class="kw">use </span>datafusion::logical_expr::{BinaryExpr, col, lit};
<a href=#465 id=465 data-nosnippet>465</a> <span class="kw">use </span>datafusion::physical_expr::create_physical_expr;
<a href=#466 id=466 data-nosnippet>466</a> <span class="kw">use </span>datafusion_common::{Column, DFSchema};
<a href=#467 id=467 data-nosnippet>467</a> <span class="kw">use </span>datatypes::arrow::datatypes::{DataType, Field, Schema};
<a href=#468 id=468 data-nosnippet>468</a>
<a href=#469 id=469 data-nosnippet>469</a> <span class="kw">use super</span>::<span class="kw-2">*</span>;
<a href=#470 id=470 data-nosnippet>470</a>
<a href=#471 id=471 data-nosnippet>471</a> <span class="attr">#[test]
<a href=#472 id=472 data-nosnippet>472</a> </span><span class="kw">fn </span>unsupported_filter_op() {
<a href=#473 id=473 data-nosnippet>473</a> <span class="comment">// `+` is not supported
<a href=#474 id=474 data-nosnippet>474</a> </span><span class="kw">let </span>expr = Expr::BinaryExpr(BinaryExpr {
<a href=#475 id=475 data-nosnippet>475</a> left: Box::new(Expr::Column(Column::from_name(<span class="string">"foo"</span>))),
<a href=#476 id=476 data-nosnippet>476</a> op: Operator::Plus,
<a href=#477 id=477 data-nosnippet>477</a> right: Box::new(<span class="number">1</span>.lit()),
<a href=#478 id=478 data-nosnippet>478</a> });
<a href=#479 id=479 data-nosnippet>479</a> <span class="macro">assert!</span>(SimpleFilterEvaluator::try_new(<span class="kw-2">&amp;</span>expr).is_none());
<a href=#480 id=480 data-nosnippet>480</a>
<a href=#481 id=481 data-nosnippet>481</a> <span class="comment">// two literal is not supported
<a href=#482 id=482 data-nosnippet>482</a> </span><span class="kw">let </span>expr = Expr::BinaryExpr(BinaryExpr {
<a href=#483 id=483 data-nosnippet>483</a> left: Box::new(<span class="number">1</span>.lit()),
<a href=#484 id=484 data-nosnippet>484</a> op: Operator::Eq,
<a href=#485 id=485 data-nosnippet>485</a> right: Box::new(<span class="number">1</span>.lit()),
<a href=#486 id=486 data-nosnippet>486</a> });
<a href=#487 id=487 data-nosnippet>487</a> <span class="macro">assert!</span>(SimpleFilterEvaluator::try_new(<span class="kw-2">&amp;</span>expr).is_none());
<a href=#488 id=488 data-nosnippet>488</a>
<a href=#489 id=489 data-nosnippet>489</a> <span class="comment">// two column is not supported
<a href=#490 id=490 data-nosnippet>490</a> </span><span class="kw">let </span>expr = Expr::BinaryExpr(BinaryExpr {
<a href=#491 id=491 data-nosnippet>491</a> left: Box::new(Expr::Column(Column::from_name(<span class="string">"foo"</span>))),
<a href=#492 id=492 data-nosnippet>492</a> op: Operator::Eq,
<a href=#493 id=493 data-nosnippet>493</a> right: Box::new(Expr::Column(Column::from_name(<span class="string">"bar"</span>))),
<a href=#494 id=494 data-nosnippet>494</a> });
<a href=#495 id=495 data-nosnippet>495</a> <span class="macro">assert!</span>(SimpleFilterEvaluator::try_new(<span class="kw-2">&amp;</span>expr).is_none());
<a href=#496 id=496 data-nosnippet>496</a>
<a href=#497 id=497 data-nosnippet>497</a> <span class="comment">// compound expr is not supported
<a href=#498 id=498 data-nosnippet>498</a> </span><span class="kw">let </span>expr = Expr::BinaryExpr(BinaryExpr {
<a href=#499 id=499 data-nosnippet>499</a> left: Box::new(Expr::BinaryExpr(BinaryExpr {
<a href=#500 id=500 data-nosnippet>500</a> left: Box::new(Expr::Column(Column::from_name(<span class="string">"foo"</span>))),
<a href=#501 id=501 data-nosnippet>501</a> op: Operator::Eq,
<a href=#502 id=502 data-nosnippet>502</a> right: Box::new(<span class="number">1</span>.lit()),
<a href=#503 id=503 data-nosnippet>503</a> })),
<a href=#504 id=504 data-nosnippet>504</a> op: Operator::Eq,
<a href=#505 id=505 data-nosnippet>505</a> right: Box::new(<span class="number">1</span>.lit()),
<a href=#506 id=506 data-nosnippet>506</a> });
<a href=#507 id=507 data-nosnippet>507</a> <span class="macro">assert!</span>(SimpleFilterEvaluator::try_new(<span class="kw-2">&amp;</span>expr).is_none());
<a href=#508 id=508 data-nosnippet>508</a> }
<a href=#509 id=509 data-nosnippet>509</a>
<a href=#510 id=510 data-nosnippet>510</a> <span class="attr">#[test]
<a href=#511 id=511 data-nosnippet>511</a> </span><span class="kw">fn </span>supported_filter_op() {
<a href=#512 id=512 data-nosnippet>512</a> <span class="comment">// equal
<a href=#513 id=513 data-nosnippet>513</a> </span><span class="kw">let </span>expr = Expr::BinaryExpr(BinaryExpr {
<a href=#514 id=514 data-nosnippet>514</a> left: Box::new(Expr::Column(Column::from_name(<span class="string">"foo"</span>))),
<a href=#515 id=515 data-nosnippet>515</a> op: Operator::Eq,
<a href=#516 id=516 data-nosnippet>516</a> right: Box::new(<span class="number">1</span>.lit()),
<a href=#517 id=517 data-nosnippet>517</a> });
<a href=#518 id=518 data-nosnippet>518</a> <span class="kw">let _ </span>= SimpleFilterEvaluator::try_new(<span class="kw-2">&amp;</span>expr).unwrap();
<a href=#519 id=519 data-nosnippet>519</a>
<a href=#520 id=520 data-nosnippet>520</a> <span class="comment">// swap operands
<a href=#521 id=521 data-nosnippet>521</a> </span><span class="kw">let </span>expr = Expr::BinaryExpr(BinaryExpr {
<a href=#522 id=522 data-nosnippet>522</a> left: Box::new(<span class="number">1</span>.lit()),
<a href=#523 id=523 data-nosnippet>523</a> op: Operator::Lt,
<a href=#524 id=524 data-nosnippet>524</a> right: Box::new(Expr::Column(Column::from_name(<span class="string">"foo"</span>))),
<a href=#525 id=525 data-nosnippet>525</a> });
<a href=#526 id=526 data-nosnippet>526</a> <span class="kw">let </span>evaluator = SimpleFilterEvaluator::try_new(<span class="kw-2">&amp;</span>expr).unwrap();
<a href=#527 id=527 data-nosnippet>527</a> <span class="macro">assert_eq!</span>(evaluator.op, Operator::Gt);
<a href=#528 id=528 data-nosnippet>528</a> <span class="macro">assert_eq!</span>(evaluator.column_name, <span class="string">"foo"</span>.to_string());
<a href=#529 id=529 data-nosnippet>529</a> }
<a href=#530 id=530 data-nosnippet>530</a>
<a href=#531 id=531 data-nosnippet>531</a> <span class="attr">#[test]
<a href=#532 id=532 data-nosnippet>532</a> </span><span class="kw">fn </span>run_on_array() {
<a href=#533 id=533 data-nosnippet>533</a> <span class="kw">let </span>expr = Expr::BinaryExpr(BinaryExpr {
<a href=#534 id=534 data-nosnippet>534</a> left: Box::new(Expr::Column(Column::from_name(<span class="string">"foo"</span>))),
<a href=#535 id=535 data-nosnippet>535</a> op: Operator::Eq,
<a href=#536 id=536 data-nosnippet>536</a> right: Box::new(<span class="number">1i64</span>.lit()),
<a href=#537 id=537 data-nosnippet>537</a> });
<a href=#538 id=538 data-nosnippet>538</a> <span class="kw">let </span>evaluator = SimpleFilterEvaluator::try_new(<span class="kw-2">&amp;</span>expr).unwrap();
<a href=#539 id=539 data-nosnippet>539</a>
<a href=#540 id=540 data-nosnippet>540</a> <span class="kw">let </span>input_1 = Arc::new(datatypes::arrow::array::Int64Array::from(<span class="macro">vec!</span>[<span class="number">1</span>, <span class="number">2</span>, <span class="number">3</span>])) <span class="kw">as _</span>;
<a href=#541 id=541 data-nosnippet>541</a> <span class="kw">let </span>result = evaluator.evaluate_array(<span class="kw-2">&amp;</span>input_1).unwrap();
<a href=#542 id=542 data-nosnippet>542</a> <span class="macro">assert_eq!</span>(result, BooleanBuffer::from(<span class="macro">vec!</span>[<span class="bool-val">true</span>, <span class="bool-val">false</span>, <span class="bool-val">false</span>]));
<a href=#543 id=543 data-nosnippet>543</a>
<a href=#544 id=544 data-nosnippet>544</a> <span class="kw">let </span>input_2 = Arc::new(datatypes::arrow::array::Int64Array::from(<span class="macro">vec!</span>[<span class="number">1</span>, <span class="number">1</span>, <span class="number">1</span>])) <span class="kw">as _</span>;
<a href=#545 id=545 data-nosnippet>545</a> <span class="kw">let </span>result = evaluator.evaluate_array(<span class="kw-2">&amp;</span>input_2).unwrap();
<a href=#546 id=546 data-nosnippet>546</a> <span class="macro">assert_eq!</span>(result, BooleanBuffer::from(<span class="macro">vec!</span>[<span class="bool-val">true</span>, <span class="bool-val">true</span>, <span class="bool-val">true</span>]));
<a href=#547 id=547 data-nosnippet>547</a>
<a href=#548 id=548 data-nosnippet>548</a> <span class="kw">let </span>input_3 = Arc::new(datatypes::arrow::array::Int64Array::new_null(<span class="number">0</span>)) <span class="kw">as _</span>;
<a href=#549 id=549 data-nosnippet>549</a> <span class="kw">let </span>result = evaluator.evaluate_array(<span class="kw-2">&amp;</span>input_3).unwrap();
<a href=#550 id=550 data-nosnippet>550</a> <span class="macro">assert_eq!</span>(result, BooleanBuffer::from(<span class="macro">vec!</span>[]));
<a href=#551 id=551 data-nosnippet>551</a> }
<a href=#552 id=552 data-nosnippet>552</a>
<a href=#553 id=553 data-nosnippet>553</a> <span class="attr">#[test]
<a href=#554 id=554 data-nosnippet>554</a> </span><span class="kw">fn </span>run_on_scalar() {
<a href=#555 id=555 data-nosnippet>555</a> <span class="kw">let </span>expr = Expr::BinaryExpr(BinaryExpr {
<a href=#556 id=556 data-nosnippet>556</a> left: Box::new(Expr::Column(Column::from_name(<span class="string">"foo"</span>))),
<a href=#557 id=557 data-nosnippet>557</a> op: Operator::Lt,
<a href=#558 id=558 data-nosnippet>558</a> right: Box::new(<span class="number">1i64</span>.lit()),
<a href=#559 id=559 data-nosnippet>559</a> });
<a href=#560 id=560 data-nosnippet>560</a> <span class="kw">let </span>evaluator = SimpleFilterEvaluator::try_new(<span class="kw-2">&amp;</span>expr).unwrap();
<a href=#561 id=561 data-nosnippet>561</a>
<a href=#562 id=562 data-nosnippet>562</a> <span class="kw">let </span>input_1 = ScalarValue::Int64(<span class="prelude-val">Some</span>(<span class="number">1</span>));
<a href=#563 id=563 data-nosnippet>563</a> <span class="kw">let </span>result = evaluator.evaluate_scalar(<span class="kw-2">&amp;</span>input_1).unwrap();
<a href=#564 id=564 data-nosnippet>564</a> <span class="macro">assert!</span>(!result);
<a href=#565 id=565 data-nosnippet>565</a>
<a href=#566 id=566 data-nosnippet>566</a> <span class="kw">let </span>input_2 = ScalarValue::Int64(<span class="prelude-val">Some</span>(<span class="number">0</span>));
<a href=#567 id=567 data-nosnippet>567</a> <span class="kw">let </span>result = evaluator.evaluate_scalar(<span class="kw-2">&amp;</span>input_2).unwrap();
<a href=#568 id=568 data-nosnippet>568</a> <span class="macro">assert!</span>(result);
<a href=#569 id=569 data-nosnippet>569</a>
<a href=#570 id=570 data-nosnippet>570</a> <span class="kw">let </span>input_3 = ScalarValue::Int64(<span class="prelude-val">None</span>);
<a href=#571 id=571 data-nosnippet>571</a> <span class="kw">let </span>result = evaluator.evaluate_scalar(<span class="kw-2">&amp;</span>input_3).unwrap();
<a href=#572 id=572 data-nosnippet>572</a> <span class="macro">assert!</span>(!result);
<a href=#573 id=573 data-nosnippet>573</a> }
<a href=#574 id=574 data-nosnippet>574</a>
<a href=#575 id=575 data-nosnippet>575</a> <span class="attr">#[test]
<a href=#576 id=576 data-nosnippet>576</a> </span><span class="kw">fn </span>batch_filter_test() {
<a href=#577 id=577 data-nosnippet>577</a> <span class="kw">let </span>expr = col(<span class="string">"ts"</span>).gt(lit(<span class="number">123456u64</span>));
<a href=#578 id=578 data-nosnippet>578</a> <span class="kw">let </span>schema = Schema::new(<span class="macro">vec!</span>[
<a href=#579 id=579 data-nosnippet>579</a> Field::new(<span class="string">"a"</span>, DataType::Int32, <span class="bool-val">true</span>),
<a href=#580 id=580 data-nosnippet>580</a> Field::new(<span class="string">"ts"</span>, DataType::UInt64, <span class="bool-val">false</span>),
<a href=#581 id=581 data-nosnippet>581</a> ]);
<a href=#582 id=582 data-nosnippet>582</a> <span class="kw">let </span>df_schema = DFSchema::try_from(schema.clone()).unwrap();
<a href=#583 id=583 data-nosnippet>583</a> <span class="kw">let </span>props = ExecutionProps::new();
<a href=#584 id=584 data-nosnippet>584</a> <span class="kw">let </span>physical_expr = create_physical_expr(<span class="kw-2">&amp;</span>expr, <span class="kw-2">&amp;</span>df_schema, <span class="kw-2">&amp;</span>props).unwrap();
<a href=#585 id=585 data-nosnippet>585</a> <span class="kw">let </span>batch = RecordBatch::try_new(
<a href=#586 id=586 data-nosnippet>586</a> Arc::new(schema),
<a href=#587 id=587 data-nosnippet>587</a> <span class="macro">vec!</span>[
<a href=#588 id=588 data-nosnippet>588</a> Arc::new(datatypes::arrow::array::Int32Array::from(<span class="macro">vec!</span>[<span class="number">4</span>, <span class="number">5</span>, <span class="number">6</span>])),
<a href=#589 id=589 data-nosnippet>589</a> Arc::new(datatypes::arrow::array::UInt64Array::from(<span class="macro">vec!</span>[
<a href=#590 id=590 data-nosnippet>590</a> <span class="number">123456</span>, <span class="number">123457</span>, <span class="number">123458</span>,
<a href=#591 id=591 data-nosnippet>591</a> ])),
<a href=#592 id=592 data-nosnippet>592</a> ],
<a href=#593 id=593 data-nosnippet>593</a> )
<a href=#594 id=594 data-nosnippet>594</a> .unwrap();
<a href=#595 id=595 data-nosnippet>595</a> <span class="kw">let </span>new_batch = batch_filter(<span class="kw-2">&amp;</span>batch, <span class="kw-2">&amp;</span>physical_expr).unwrap();
<a href=#596 id=596 data-nosnippet>596</a> <span class="macro">assert_eq!</span>(new_batch.num_rows(), <span class="number">2</span>);
<a href=#597 id=597 data-nosnippet>597</a> <span class="kw">let </span>first_column_values = new_batch
<a href=#598 id=598 data-nosnippet>598</a> .column(<span class="number">0</span>)
<a href=#599 id=599 data-nosnippet>599</a> .as_any()
<a href=#600 id=600 data-nosnippet>600</a> .downcast_ref::&lt;datatypes::arrow::array::Int32Array&gt;()
<a href=#601 id=601 data-nosnippet>601</a> .unwrap();
<a href=#602 id=602 data-nosnippet>602</a> <span class="kw">let </span>expected = datatypes::arrow::array::Int32Array::from(<span class="macro">vec!</span>[<span class="number">5</span>, <span class="number">6</span>]);
<a href=#603 id=603 data-nosnippet>603</a> <span class="macro">assert_eq!</span>(first_column_values, <span class="kw-2">&amp;</span>expected);
<a href=#604 id=604 data-nosnippet>604</a> }
<a href=#605 id=605 data-nosnippet>605</a>
<a href=#606 id=606 data-nosnippet>606</a> <span class="attr">#[test]
<a href=#607 id=607 data-nosnippet>607</a> </span><span class="kw">fn </span>test_complex_filter_expression() {
<a href=#608 id=608 data-nosnippet>608</a> <span class="comment">// Create an expression tree for: col = 'B' OR col = 'C' OR col = 'D'
<a href=#609 id=609 data-nosnippet>609</a> </span><span class="kw">let </span>col_eq_b = col(<span class="string">"col"</span>).eq(lit(<span class="string">"B"</span>));
<a href=#610 id=610 data-nosnippet>610</a> <span class="kw">let </span>col_eq_c = col(<span class="string">"col"</span>).eq(lit(<span class="string">"C"</span>));
<a href=#611 id=611 data-nosnippet>611</a> <span class="kw">let </span>col_eq_d = col(<span class="string">"col"</span>).eq(lit(<span class="string">"D"</span>));
<a href=#612 id=612 data-nosnippet>612</a>
<a href=#613 id=613 data-nosnippet>613</a> <span class="comment">// Build the OR chain
<a href=#614 id=614 data-nosnippet>614</a> </span><span class="kw">let </span>col_or_expr = col_eq_b.or(col_eq_c).or(col_eq_d);
<a href=#615 id=615 data-nosnippet>615</a>
<a href=#616 id=616 data-nosnippet>616</a> <span class="comment">// Check that SimpleFilterEvaluator can handle OR chain
<a href=#617 id=617 data-nosnippet>617</a> </span><span class="kw">let </span>or_evaluator = SimpleFilterEvaluator::try_new(<span class="kw-2">&amp;</span>col_or_expr).unwrap();
<a href=#618 id=618 data-nosnippet>618</a> <span class="macro">assert_eq!</span>(or_evaluator.column_name, <span class="string">"col"</span>);
<a href=#619 id=619 data-nosnippet>619</a> <span class="macro">assert_eq!</span>(or_evaluator.op, Operator::Or);
<a href=#620 id=620 data-nosnippet>620</a> <span class="macro">assert_eq!</span>(or_evaluator.literal_list.len(), <span class="number">3</span>);
<a href=#621 id=621 data-nosnippet>621</a> <span class="macro">assert_eq!</span>(
<a href=#622 id=622 data-nosnippet>622</a> <span class="macro">format!</span>(<span class="string">"{:?}"</span>, or_evaluator.literal_list),
<a href=#623 id=623 data-nosnippet>623</a> <span class="string">"[Scalar(StringArray\n[\n \"B\",\n]), Scalar(StringArray\n[\n \"C\",\n]), Scalar(StringArray\n[\n \"D\",\n])]"
<a href=#624 id=624 data-nosnippet>624</a> </span>);
<a href=#625 id=625 data-nosnippet>625</a>
<a href=#626 id=626 data-nosnippet>626</a> <span class="comment">// Create a schema and batch for testing
<a href=#627 id=627 data-nosnippet>627</a> </span><span class="kw">let </span>schema = Schema::new(<span class="macro">vec!</span>[Field::new(<span class="string">"col"</span>, DataType::Utf8, <span class="bool-val">false</span>)]);
<a href=#628 id=628 data-nosnippet>628</a> <span class="kw">let </span>df_schema = DFSchema::try_from(schema.clone()).unwrap();
<a href=#629 id=629 data-nosnippet>629</a> <span class="kw">let </span>props = ExecutionProps::new();
<a href=#630 id=630 data-nosnippet>630</a> <span class="kw">let </span>physical_expr = create_physical_expr(<span class="kw-2">&amp;</span>col_or_expr, <span class="kw-2">&amp;</span>df_schema, <span class="kw-2">&amp;</span>props).unwrap();
<a href=#631 id=631 data-nosnippet>631</a>
<a href=#632 id=632 data-nosnippet>632</a> <span class="comment">// Create test data
<a href=#633 id=633 data-nosnippet>633</a> </span><span class="kw">let </span>col_data = Arc::new(datatypes::arrow::array::StringArray::from(<span class="macro">vec!</span>[
<a href=#634 id=634 data-nosnippet>634</a> <span class="string">"B"</span>, <span class="string">"C"</span>, <span class="string">"E"</span>, <span class="string">"B"</span>, <span class="string">"C"</span>, <span class="string">"D"</span>, <span class="string">"F"</span>,
<a href=#635 id=635 data-nosnippet>635</a> ]));
<a href=#636 id=636 data-nosnippet>636</a> <span class="kw">let </span>batch = RecordBatch::try_new(Arc::new(schema), <span class="macro">vec!</span>[col_data]).unwrap();
<a href=#637 id=637 data-nosnippet>637</a> <span class="kw">let </span>expected = datatypes::arrow::array::StringArray::from(<span class="macro">vec!</span>[<span class="string">"B"</span>, <span class="string">"C"</span>, <span class="string">"B"</span>, <span class="string">"C"</span>, <span class="string">"D"</span>]);
<a href=#638 id=638 data-nosnippet>638</a>
<a href=#639 id=639 data-nosnippet>639</a> <span class="comment">// Filter the batch
<a href=#640 id=640 data-nosnippet>640</a> </span><span class="kw">let </span>filtered_batch = batch_filter(<span class="kw-2">&amp;</span>batch, <span class="kw-2">&amp;</span>physical_expr).unwrap();
<a href=#641 id=641 data-nosnippet>641</a>
<a href=#642 id=642 data-nosnippet>642</a> <span class="comment">// Expected: rows with col in ("B", "C", "D")
<a href=#643 id=643 data-nosnippet>643</a> // That would be rows 0, 1, 3, 4, 5
<a href=#644 id=644 data-nosnippet>644</a> </span><span class="macro">assert_eq!</span>(filtered_batch.num_rows(), <span class="number">5</span>);
<a href=#645 id=645 data-nosnippet>645</a>
<a href=#646 id=646 data-nosnippet>646</a> <span class="kw">let </span>col_filtered = filtered_batch
<a href=#647 id=647 data-nosnippet>647</a> .column(<span class="number">0</span>)
<a href=#648 id=648 data-nosnippet>648</a> .as_any()
<a href=#649 id=649 data-nosnippet>649</a> .downcast_ref::&lt;datatypes::arrow::array::StringArray&gt;()
<a href=#650 id=650 data-nosnippet>650</a> .unwrap();
<a href=#651 id=651 data-nosnippet>651</a> <span class="macro">assert_eq!</span>(col_filtered, <span class="kw-2">&amp;</span>expected);
<a href=#652 id=652 data-nosnippet>652</a> }
<a href=#653 id=653 data-nosnippet>653</a>
<a href=#654 id=654 data-nosnippet>654</a> <span class="attr">#[test]
<a href=#655 id=655 data-nosnippet>655</a> </span><span class="kw">fn </span>test_maybe_build_regex() {
<a href=#656 id=656 data-nosnippet>656</a> <span class="comment">// Test case for RegexMatch (case sensitive, non-negative)
<a href=#657 id=657 data-nosnippet>657</a> </span><span class="kw">let </span>(regex, negative) = SimpleFilterEvaluator::maybe_build_regex(
<a href=#658 id=658 data-nosnippet>658</a> Operator::RegexMatch,
<a href=#659 id=659 data-nosnippet>659</a> <span class="kw-2">&amp;</span>ScalarValue::Utf8(<span class="prelude-val">Some</span>(<span class="string">"a.*b"</span>.to_string())),
<a href=#660 id=660 data-nosnippet>660</a> )
<a href=#661 id=661 data-nosnippet>661</a> .unwrap();
<a href=#662 id=662 data-nosnippet>662</a> <span class="macro">assert!</span>(regex.is_some());
<a href=#663 id=663 data-nosnippet>663</a> <span class="macro">assert!</span>(!negative);
<a href=#664 id=664 data-nosnippet>664</a> <span class="macro">assert!</span>(regex.unwrap().is_match(<span class="string">"axxb"</span>));
<a href=#665 id=665 data-nosnippet>665</a>
<a href=#666 id=666 data-nosnippet>666</a> <span class="comment">// Test case for RegexIMatch (case insensitive, non-negative)
<a href=#667 id=667 data-nosnippet>667</a> </span><span class="kw">let </span>(regex, negative) = SimpleFilterEvaluator::maybe_build_regex(
<a href=#668 id=668 data-nosnippet>668</a> Operator::RegexIMatch,
<a href=#669 id=669 data-nosnippet>669</a> <span class="kw-2">&amp;</span>ScalarValue::Utf8(<span class="prelude-val">Some</span>(<span class="string">"a.*b"</span>.to_string())),
<a href=#670 id=670 data-nosnippet>670</a> )
<a href=#671 id=671 data-nosnippet>671</a> .unwrap();
<a href=#672 id=672 data-nosnippet>672</a> <span class="macro">assert!</span>(regex.is_some());
<a href=#673 id=673 data-nosnippet>673</a> <span class="macro">assert!</span>(!negative);
<a href=#674 id=674 data-nosnippet>674</a> <span class="macro">assert!</span>(regex.unwrap().is_match(<span class="string">"AxxB"</span>));
<a href=#675 id=675 data-nosnippet>675</a>
<a href=#676 id=676 data-nosnippet>676</a> <span class="comment">// Test case for RegexNotMatch (case sensitive, negative)
<a href=#677 id=677 data-nosnippet>677</a> </span><span class="kw">let </span>(regex, negative) = SimpleFilterEvaluator::maybe_build_regex(
<a href=#678 id=678 data-nosnippet>678</a> Operator::RegexNotMatch,
<a href=#679 id=679 data-nosnippet>679</a> <span class="kw-2">&amp;</span>ScalarValue::Utf8(<span class="prelude-val">Some</span>(<span class="string">"a.*b"</span>.to_string())),
<a href=#680 id=680 data-nosnippet>680</a> )
<a href=#681 id=681 data-nosnippet>681</a> .unwrap();
<a href=#682 id=682 data-nosnippet>682</a> <span class="macro">assert!</span>(regex.is_some());
<a href=#683 id=683 data-nosnippet>683</a> <span class="macro">assert!</span>(negative);
<a href=#684 id=684 data-nosnippet>684</a>
<a href=#685 id=685 data-nosnippet>685</a> <span class="comment">// Test case for RegexNotIMatch (case insensitive, negative)
<a href=#686 id=686 data-nosnippet>686</a> </span><span class="kw">let </span>(regex, negative) = SimpleFilterEvaluator::maybe_build_regex(
<a href=#687 id=687 data-nosnippet>687</a> Operator::RegexNotIMatch,
<a href=#688 id=688 data-nosnippet>688</a> <span class="kw-2">&amp;</span>ScalarValue::Utf8(<span class="prelude-val">Some</span>(<span class="string">"a.*b"</span>.to_string())),
<a href=#689 id=689 data-nosnippet>689</a> )
<a href=#690 id=690 data-nosnippet>690</a> .unwrap();
<a href=#691 id=691 data-nosnippet>691</a> <span class="macro">assert!</span>(regex.is_some());
<a href=#692 id=692 data-nosnippet>692</a> <span class="macro">assert!</span>(negative);
<a href=#693 id=693 data-nosnippet>693</a>
<a href=#694 id=694 data-nosnippet>694</a> <span class="comment">// Test with empty regex pattern
<a href=#695 id=695 data-nosnippet>695</a> </span><span class="kw">let </span>(regex, negative) = SimpleFilterEvaluator::maybe_build_regex(
<a href=#696 id=696 data-nosnippet>696</a> Operator::RegexMatch,
<a href=#697 id=697 data-nosnippet>697</a> <span class="kw-2">&amp;</span>ScalarValue::Utf8(<span class="prelude-val">Some</span>(<span class="string">""</span>.to_string())),
<a href=#698 id=698 data-nosnippet>698</a> )
<a href=#699 id=699 data-nosnippet>699</a> .unwrap();
<a href=#700 id=700 data-nosnippet>700</a> <span class="macro">assert!</span>(regex.is_none());
<a href=#701 id=701 data-nosnippet>701</a> <span class="macro">assert!</span>(!negative);
<a href=#702 id=702 data-nosnippet>702</a>
<a href=#703 id=703 data-nosnippet>703</a> <span class="comment">// Test with non-regex operator
<a href=#704 id=704 data-nosnippet>704</a> </span><span class="kw">let </span>(regex, negative) = SimpleFilterEvaluator::maybe_build_regex(
<a href=#705 id=705 data-nosnippet>705</a> Operator::Eq,
<a href=#706 id=706 data-nosnippet>706</a> <span class="kw-2">&amp;</span>ScalarValue::Utf8(<span class="prelude-val">Some</span>(<span class="string">"a.*b"</span>.to_string())),
<a href=#707 id=707 data-nosnippet>707</a> )
<a href=#708 id=708 data-nosnippet>708</a> .unwrap();
<a href=#709 id=709 data-nosnippet>709</a> <span class="macro">assert!</span>(regex.is_none());
<a href=#710 id=710 data-nosnippet>710</a> <span class="macro">assert!</span>(!negative);
<a href=#711 id=711 data-nosnippet>711</a>
<a href=#712 id=712 data-nosnippet>712</a> <span class="comment">// Test with invalid regex pattern
<a href=#713 id=713 data-nosnippet>713</a> </span><span class="kw">let </span>result = SimpleFilterEvaluator::maybe_build_regex(
<a href=#714 id=714 data-nosnippet>714</a> Operator::RegexMatch,
<a href=#715 id=715 data-nosnippet>715</a> <span class="kw-2">&amp;</span>ScalarValue::Utf8(<span class="prelude-val">Some</span>(<span class="string">"a(b"</span>.to_string())),
<a href=#716 id=716 data-nosnippet>716</a> );
<a href=#717 id=717 data-nosnippet>717</a> <span class="macro">assert!</span>(result.is_err());
<a href=#718 id=718 data-nosnippet>718</a>
<a href=#719 id=719 data-nosnippet>719</a> <span class="comment">// Test with non-string value
<a href=#720 id=720 data-nosnippet>720</a> </span><span class="kw">let </span>result = SimpleFilterEvaluator::maybe_build_regex(
<a href=#721 id=721 data-nosnippet>721</a> Operator::RegexMatch,
<a href=#722 id=722 data-nosnippet>722</a> <span class="kw-2">&amp;</span>ScalarValue::Int64(<span class="prelude-val">Some</span>(<span class="number">123</span>)),
<a href=#723 id=723 data-nosnippet>723</a> );
<a href=#724 id=724 data-nosnippet>724</a> <span class="macro">assert!</span>(result.is_err());
<a href=#725 id=725 data-nosnippet>725</a>
<a href=#726 id=726 data-nosnippet>726</a> <span class="comment">// Test with null value
<a href=#727 id=727 data-nosnippet>727</a> </span><span class="kw">let </span>result = SimpleFilterEvaluator::maybe_build_regex(
<a href=#728 id=728 data-nosnippet>728</a> Operator::RegexMatch,
<a href=#729 id=729 data-nosnippet>729</a> <span class="kw-2">&amp;</span>ScalarValue::Utf8(<span class="prelude-val">None</span>),
<a href=#730 id=730 data-nosnippet>730</a> );
<a href=#731 id=731 data-nosnippet>731</a> <span class="macro">assert!</span>(result.is_err());
<a href=#732 id=732 data-nosnippet>732</a> }
<a href=#733 id=733 data-nosnippet>733</a>
<a href=#734 id=734 data-nosnippet>734</a> <span class="attr">#[test]
<a href=#735 id=735 data-nosnippet>735</a> </span><span class="kw">fn </span>test_regex_match_dictionary_array() {
<a href=#736 id=736 data-nosnippet>736</a> <span class="kw">use </span>datatypes::arrow::array::StringDictionaryBuilder;
<a href=#737 id=737 data-nosnippet>737</a>
<a href=#738 id=738 data-nosnippet>738</a> <span class="comment">// Create a StringDictionaryArray
<a href=#739 id=739 data-nosnippet>739</a> </span><span class="kw">let </span><span class="kw-2">mut </span>builder = StringDictionaryBuilder::&lt;UInt32Type&gt;::new();
<a href=#740 id=740 data-nosnippet>740</a> builder.append(<span class="string">"apple"</span>).unwrap();
<a href=#741 id=741 data-nosnippet>741</a> builder.append(<span class="string">"banana"</span>).unwrap();
<a href=#742 id=742 data-nosnippet>742</a> builder.append(<span class="string">"apple"</span>).unwrap();
<a href=#743 id=743 data-nosnippet>743</a> builder.append(<span class="string">"cherry"</span>).unwrap();
<a href=#744 id=744 data-nosnippet>744</a> <span class="kw">let </span>dict_array = builder.finish();
<a href=#745 id=745 data-nosnippet>745</a>
<a href=#746 id=746 data-nosnippet>746</a> <span class="comment">// Test regex that matches "apple"
<a href=#747 id=747 data-nosnippet>747</a> </span><span class="kw">let </span>regex = regex::Regex::new(<span class="string">r"app.*"</span>).unwrap();
<a href=#748 id=748 data-nosnippet>748</a> <span class="kw">let </span>result = regexp_is_match_dictionary(<span class="kw-2">&amp;</span>dict_array, <span class="prelude-val">Some</span>(<span class="kw-2">&amp;</span>regex)).unwrap();
<a href=#749 id=749 data-nosnippet>749</a>
<a href=#750 id=750 data-nosnippet>750</a> <span class="comment">// Should match indices 0 and 2 (both "apple")
<a href=#751 id=751 data-nosnippet>751</a> </span><span class="macro">assert_eq!</span>(result.len(), <span class="number">4</span>);
<a href=#752 id=752 data-nosnippet>752</a> <span class="macro">assert!</span>(result.value(<span class="number">0</span>)); <span class="comment">// "apple"
<a href=#753 id=753 data-nosnippet>753</a> </span><span class="macro">assert!</span>(!result.value(<span class="number">1</span>)); <span class="comment">// "banana"
<a href=#754 id=754 data-nosnippet>754</a> </span><span class="macro">assert!</span>(result.value(<span class="number">2</span>)); <span class="comment">// "apple"
<a href=#755 id=755 data-nosnippet>755</a> </span><span class="macro">assert!</span>(!result.value(<span class="number">3</span>)); <span class="comment">// "cherry"
<a href=#756 id=756 data-nosnippet>756</a>
<a href=#757 id=757 data-nosnippet>757</a> // Test regex that matches "banana"
<a href=#758 id=758 data-nosnippet>758</a> </span><span class="kw">let </span>regex2 = regex::Regex::new(<span class="string">r"ban.*"</span>).unwrap();
<a href=#759 id=759 data-nosnippet>759</a> <span class="kw">let </span>result2 = regexp_is_match_dictionary(<span class="kw-2">&amp;</span>dict_array, <span class="prelude-val">Some</span>(<span class="kw-2">&amp;</span>regex2)).unwrap();
<a href=#760 id=760 data-nosnippet>760</a>
<a href=#761 id=761 data-nosnippet>761</a> <span class="macro">assert!</span>(!result2.value(<span class="number">0</span>)); <span class="comment">// "apple"
<a href=#762 id=762 data-nosnippet>762</a> </span><span class="macro">assert!</span>(result2.value(<span class="number">1</span>)); <span class="comment">// "banana"
<a href=#763 id=763 data-nosnippet>763</a> </span><span class="macro">assert!</span>(!result2.value(<span class="number">2</span>)); <span class="comment">// "apple"
<a href=#764 id=764 data-nosnippet>764</a> </span><span class="macro">assert!</span>(!result2.value(<span class="number">3</span>)); <span class="comment">// "cherry"
<a href=#765 id=765 data-nosnippet>765</a>
<a href=#766 id=766 data-nosnippet>766</a> // Test with no regex (should match all)
<a href=#767 id=767 data-nosnippet>767</a> </span><span class="kw">let </span>result3 = regexp_is_match_dictionary(<span class="kw-2">&amp;</span>dict_array, <span class="prelude-val">None</span>).unwrap();
<a href=#768 id=768 data-nosnippet>768</a> <span class="macro">assert!</span>(result3.value(<span class="number">0</span>));
<a href=#769 id=769 data-nosnippet>769</a> <span class="macro">assert!</span>(result3.value(<span class="number">1</span>));
<a href=#770 id=770 data-nosnippet>770</a> <span class="macro">assert!</span>(result3.value(<span class="number">2</span>));
<a href=#771 id=771 data-nosnippet>771</a> <span class="macro">assert!</span>(result3.value(<span class="number">3</span>));
<a href=#772 id=772 data-nosnippet>772</a> }
<a href=#773 id=773 data-nosnippet>773</a>}</code></pre></div></section></main></body></html>

1051
src/common_recordbatch/lib.rs.html Normal file
View File

File diff suppressed because it is too large Load Diff

744
src/common_recordbatch/recordbatch.rs.html Normal file
View File

@@ -0,0 +1,744 @@
<!DOCTYPE html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1.0"><meta name="generator" content="rustdoc"><meta name="description" content="Source of the Rust file `src/common/recordbatch/src/recordbatch.rs`."><title>recordbatch.rs - source</title><script>if(window.location.protocol!=="file:")document.head.insertAdjacentHTML("beforeend","SourceSerif4-Regular-6b053e98.ttf.woff2,FiraSans-Italic-81dc35de.woff2,FiraSans-Regular-0fe48ade.woff2,FiraSans-MediumItalic-ccf7e434.woff2,FiraSans-Medium-e1aa3f0a.woff2,SourceCodePro-Regular-8badfe75.ttf.woff2,SourceCodePro-Semibold-aa29a496.ttf.woff2".split(",").map(f=>`<link rel="preload" as="font" type="font/woff2"href="../../static.files/${f}">`).join(""))</script><link rel="stylesheet" href="../../static.files/normalize-9960930a.css"><link rel="stylesheet" href="../../static.files/rustdoc-e56847b5.css"><meta name="rustdoc-vars" data-root-path="../../" data-static-root-path="../../static.files/" data-current-crate="common_recordbatch" data-themes="" data-resource-suffix="" data-rustdoc-version="1.92.0-nightly (fa3155a64 2025-09-30)" data-channel="nightly" data-search-js="search-e256b49e.js" data-stringdex-js="stringdex-828709d0.js" data-settings-js="settings-c38705f0.js" ><script src="../../static.files/storage-e2aeef58.js"></script><script defer src="../../static.files/src-script-813739b1.js"></script><script defer src="../../src-files.js"></script><script defer src="../../static.files/main-ce535bd0.js"></script><noscript><link rel="stylesheet" href="../../static.files/noscript-263c88ec.css"></noscript><link rel="alternate icon" type="image/png" href="../../static.files/favicon-32x32-eab170b8.png"><link rel="icon" type="image/svg+xml" href="../../static.files/favicon-044be391.svg"></head><body class="rustdoc src"><!--[if lte IE 11]><div class="warning">This old browser is unsupported and will most likely display funky things.</div><![endif]--><nav class="sidebar"><div class="src-sidebar-title"><h2>Files</h2></div></nav><div class="sidebar-resizer" title="Drag to resize sidebar"></div><main><section id="main-content" class="content"><div class="main-heading"><h1><div class="sub-heading">common_recordbatch/</div>recordbatch.rs</h1><rustdoc-toolbar></rustdoc-toolbar></div><div class="example-wrap digits-3"><pre class="rust"><code><a href=#1 id=1 data-nosnippet>1</a><span class="comment">// Copyright 2023 Greptime Team
<a href=#2 id=2 data-nosnippet>2</a>//
<a href=#3 id=3 data-nosnippet>3</a>// Licensed under the Apache License, Version 2.0 (the "License");
<a href=#4 id=4 data-nosnippet>4</a>// you may not use this file except in compliance with the License.
<a href=#5 id=5 data-nosnippet>5</a>// You may obtain a copy of the License at
<a href=#6 id=6 data-nosnippet>6</a>//
<a href=#7 id=7 data-nosnippet>7</a>// http://www.apache.org/licenses/LICENSE-2.0
<a href=#8 id=8 data-nosnippet>8</a>//
<a href=#9 id=9 data-nosnippet>9</a>// Unless required by applicable law or agreed to in writing, software
<a href=#10 id=10 data-nosnippet>10</a>// distributed under the License is distributed on an "AS IS" BASIS,
<a href=#11 id=11 data-nosnippet>11</a>// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
<a href=#12 id=12 data-nosnippet>12</a>// See the License for the specific language governing permissions and
<a href=#13 id=13 data-nosnippet>13</a>// limitations under the License.
<a href=#14 id=14 data-nosnippet>14</a>
<a href=#15 id=15 data-nosnippet>15</a></span><span class="kw">use </span>std::collections::HashMap;
<a href=#16 id=16 data-nosnippet>16</a><span class="kw">use </span>std::slice;
<a href=#17 id=17 data-nosnippet>17</a><span class="kw">use </span>std::sync::Arc;
<a href=#18 id=18 data-nosnippet>18</a>
<a href=#19 id=19 data-nosnippet>19</a><span class="kw">use </span>datafusion::arrow::util::pretty::pretty_format_batches;
<a href=#20 id=20 data-nosnippet>20</a><span class="kw">use </span>datafusion_common::arrow::array::ArrayRef;
<a href=#21 id=21 data-nosnippet>21</a><span class="kw">use </span>datafusion_common::arrow::compute;
<a href=#22 id=22 data-nosnippet>22</a><span class="kw">use </span>datafusion_common::arrow::datatypes::{DataType <span class="kw">as </span>ArrowDataType, SchemaRef <span class="kw">as </span>ArrowSchemaRef};
<a href=#23 id=23 data-nosnippet>23</a><span class="kw">use </span>datatypes::arrow::array::{Array, AsArray, RecordBatchOptions, StructArray, new_null_array};
<a href=#24 id=24 data-nosnippet>24</a><span class="kw">use </span>datatypes::extension::json::is_json_extension_type;
<a href=#25 id=25 data-nosnippet>25</a><span class="kw">use </span>datatypes::prelude::DataType;
<a href=#26 id=26 data-nosnippet>26</a><span class="kw">use </span>datatypes::schema::SchemaRef;
<a href=#27 id=27 data-nosnippet>27</a><span class="kw">use </span>datatypes::vectors::{Helper, VectorRef};
<a href=#28 id=28 data-nosnippet>28</a><span class="kw">use </span>serde::ser::{Error, SerializeStruct};
<a href=#29 id=29 data-nosnippet>29</a><span class="kw">use </span>serde::{Serialize, Serializer};
<a href=#30 id=30 data-nosnippet>30</a><span class="kw">use </span>snafu::{OptionExt, ResultExt, ensure};
<a href=#31 id=31 data-nosnippet>31</a>
<a href=#32 id=32 data-nosnippet>32</a><span class="kw">use </span><span class="kw">crate</span>::DfRecordBatch;
<a href=#33 id=33 data-nosnippet>33</a><span class="kw">use </span><span class="kw">crate</span>::error::{
<a href=#34 id=34 data-nosnippet>34</a> <span class="self">self</span>, AlignJsonArraySnafu, ArrowComputeSnafu, ColumnNotExistsSnafu, DataTypesSnafu,
<a href=#35 id=35 data-nosnippet>35</a> NewDfRecordBatchSnafu, ProjectArrowRecordBatchSnafu, <span class="prelude-ty">Result</span>,
<a href=#36 id=36 data-nosnippet>36</a>};
<a href=#37 id=37 data-nosnippet>37</a>
<a href=#38 id=38 data-nosnippet>38</a><span class="doccomment">/// A two-dimensional batch of column-oriented data with a defined schema.
<a href=#39 id=39 data-nosnippet>39</a></span><span class="attr">#[derive(Clone, Debug, PartialEq)]
<a href=#40 id=40 data-nosnippet>40</a></span><span class="kw">pub struct </span>RecordBatch {
<a href=#41 id=41 data-nosnippet>41</a> <span class="kw">pub </span>schema: SchemaRef,
<a href=#42 id=42 data-nosnippet>42</a> df_record_batch: DfRecordBatch,
<a href=#43 id=43 data-nosnippet>43</a>}
<a href=#44 id=44 data-nosnippet>44</a>
<a href=#45 id=45 data-nosnippet>45</a><span class="kw">impl </span>RecordBatch {
<a href=#46 id=46 data-nosnippet>46</a> <span class="doccomment">/// Create a new [`RecordBatch`] from `schema` and `columns`.
<a href=#47 id=47 data-nosnippet>47</a> </span><span class="kw">pub fn </span>new&lt;I: IntoIterator&lt;Item = VectorRef&gt;&gt;(
<a href=#48 id=48 data-nosnippet>48</a> schema: SchemaRef,
<a href=#49 id=49 data-nosnippet>49</a> columns: I,
<a href=#50 id=50 data-nosnippet>50</a> ) -&gt; <span class="prelude-ty">Result</span>&lt;RecordBatch&gt; {
<a href=#51 id=51 data-nosnippet>51</a> <span class="kw">let </span>columns: Vec&lt;<span class="kw">_</span>&gt; = columns.into_iter().collect();
<a href=#52 id=52 data-nosnippet>52</a> <span class="kw">let </span>arrow_arrays = columns.iter().map(|v| v.to_arrow_array()).collect();
<a href=#53 id=53 data-nosnippet>53</a>
<a href=#54 id=54 data-nosnippet>54</a> <span class="comment">// Casting the arrays here to match the schema, is a temporary solution to support Arrow's
<a href=#55 id=55 data-nosnippet>55</a> // view array types (`StringViewArray` and `BinaryViewArray`).
<a href=#56 id=56 data-nosnippet>56</a> // As to "support": the arrays here are created from vectors, which do not have types
<a href=#57 id=57 data-nosnippet>57</a> // corresponding to view arrays. What we can do is to only cast them.
<a href=#58 id=58 data-nosnippet>58</a> // As to "temporary": we are planing to use Arrow's RecordBatch directly in the read path.
<a href=#59 id=59 data-nosnippet>59</a> // the casting here will be removed in the end.
<a href=#60 id=60 data-nosnippet>60</a> // TODO(LFC): Remove the casting here once `Batch` is no longer used.
<a href=#61 id=61 data-nosnippet>61</a> </span><span class="kw">let </span>arrow_arrays = <span class="self">Self</span>::cast_view_arrays(schema.arrow_schema(), arrow_arrays)<span class="question-mark">?</span>;
<a href=#62 id=62 data-nosnippet>62</a>
<a href=#63 id=63 data-nosnippet>63</a> <span class="kw">let </span>arrow_arrays = maybe_align_json_array_with_schema(schema.arrow_schema(), arrow_arrays)<span class="question-mark">?</span>;
<a href=#64 id=64 data-nosnippet>64</a>
<a href=#65 id=65 data-nosnippet>65</a> <span class="kw">let </span>df_record_batch = DfRecordBatch::try_new(schema.arrow_schema().clone(), arrow_arrays)
<a href=#66 id=66 data-nosnippet>66</a> .context(error::NewDfRecordBatchSnafu)<span class="question-mark">?</span>;
<a href=#67 id=67 data-nosnippet>67</a>
<a href=#68 id=68 data-nosnippet>68</a> <span class="prelude-val">Ok</span>(RecordBatch {
<a href=#69 id=69 data-nosnippet>69</a> schema,
<a href=#70 id=70 data-nosnippet>70</a> df_record_batch,
<a href=#71 id=71 data-nosnippet>71</a> })
<a href=#72 id=72 data-nosnippet>72</a> }
<a href=#73 id=73 data-nosnippet>73</a>
<a href=#74 id=74 data-nosnippet>74</a> <span class="kw">pub fn </span>to_df_record_batch&lt;I: IntoIterator&lt;Item = VectorRef&gt;&gt;(
<a href=#75 id=75 data-nosnippet>75</a> arrow_schema: ArrowSchemaRef,
<a href=#76 id=76 data-nosnippet>76</a> columns: I,
<a href=#77 id=77 data-nosnippet>77</a> ) -&gt; <span class="prelude-ty">Result</span>&lt;DfRecordBatch&gt; {
<a href=#78 id=78 data-nosnippet>78</a> <span class="kw">let </span>columns: Vec&lt;<span class="kw">_</span>&gt; = columns.into_iter().collect();
<a href=#79 id=79 data-nosnippet>79</a> <span class="kw">let </span>arrow_arrays = columns.iter().map(|v| v.to_arrow_array()).collect();
<a href=#80 id=80 data-nosnippet>80</a>
<a href=#81 id=81 data-nosnippet>81</a> <span class="comment">// Casting the arrays here to match the schema, is a temporary solution to support Arrow's
<a href=#82 id=82 data-nosnippet>82</a> // view array types (`StringViewArray` and `BinaryViewArray`).
<a href=#83 id=83 data-nosnippet>83</a> // As to "support": the arrays here are created from vectors, which do not have types
<a href=#84 id=84 data-nosnippet>84</a> // corresponding to view arrays. What we can do is to only cast them.
<a href=#85 id=85 data-nosnippet>85</a> // As to "temporary": we are planing to use Arrow's RecordBatch directly in the read path.
<a href=#86 id=86 data-nosnippet>86</a> // the casting here will be removed in the end.
<a href=#87 id=87 data-nosnippet>87</a> // TODO(LFC): Remove the casting here once `Batch` is no longer used.
<a href=#88 id=88 data-nosnippet>88</a> </span><span class="kw">let </span>arrow_arrays = <span class="self">Self</span>::cast_view_arrays(<span class="kw-2">&amp;</span>arrow_schema, arrow_arrays)<span class="question-mark">?</span>;
<a href=#89 id=89 data-nosnippet>89</a>
<a href=#90 id=90 data-nosnippet>90</a> <span class="kw">let </span>arrow_arrays = maybe_align_json_array_with_schema(<span class="kw-2">&amp;</span>arrow_schema, arrow_arrays)<span class="question-mark">?</span>;
<a href=#91 id=91 data-nosnippet>91</a>
<a href=#92 id=92 data-nosnippet>92</a> <span class="kw">let </span>df_record_batch = DfRecordBatch::try_new(arrow_schema, arrow_arrays)
<a href=#93 id=93 data-nosnippet>93</a> .context(error::NewDfRecordBatchSnafu)<span class="question-mark">?</span>;
<a href=#94 id=94 data-nosnippet>94</a>
<a href=#95 id=95 data-nosnippet>95</a> <span class="prelude-val">Ok</span>(df_record_batch)
<a href=#96 id=96 data-nosnippet>96</a> }
<a href=#97 id=97 data-nosnippet>97</a>
<a href=#98 id=98 data-nosnippet>98</a> <span class="kw">fn </span>cast_view_arrays(
<a href=#99 id=99 data-nosnippet>99</a> schema: <span class="kw-2">&amp;</span>ArrowSchemaRef,
<a href=#100 id=100 data-nosnippet>100</a> <span class="kw-2">mut </span>arrays: Vec&lt;ArrayRef&gt;,
<a href=#101 id=101 data-nosnippet>101</a> ) -&gt; <span class="prelude-ty">Result</span>&lt;Vec&lt;ArrayRef&gt;&gt; {
<a href=#102 id=102 data-nosnippet>102</a> <span class="kw">for </span>(f, a) <span class="kw">in </span>schema.fields().iter().zip(arrays.iter_mut()) {
<a href=#103 id=103 data-nosnippet>103</a> <span class="kw">let </span>expected = f.data_type();
<a href=#104 id=104 data-nosnippet>104</a> <span class="kw">let </span>actual = a.data_type();
<a href=#105 id=105 data-nosnippet>105</a> <span class="kw">if </span><span class="macro">matches!</span>(
<a href=#106 id=106 data-nosnippet>106</a> (expected, actual),
<a href=#107 id=107 data-nosnippet>107</a> (ArrowDataType::Utf8View, ArrowDataType::Utf8)
<a href=#108 id=108 data-nosnippet>108</a> | (ArrowDataType::BinaryView, ArrowDataType::Binary)
<a href=#109 id=109 data-nosnippet>109</a> ) {
<a href=#110 id=110 data-nosnippet>110</a> <span class="kw-2">*</span>a = compute::cast(a, expected).context(ArrowComputeSnafu)<span class="question-mark">?</span>;
<a href=#111 id=111 data-nosnippet>111</a> }
<a href=#112 id=112 data-nosnippet>112</a> }
<a href=#113 id=113 data-nosnippet>113</a> <span class="prelude-val">Ok</span>(arrays)
<a href=#114 id=114 data-nosnippet>114</a> }
<a href=#115 id=115 data-nosnippet>115</a>
<a href=#116 id=116 data-nosnippet>116</a> <span class="doccomment">/// Create an empty [`RecordBatch`] from `schema`.
<a href=#117 id=117 data-nosnippet>117</a> </span><span class="kw">pub fn </span>new_empty(schema: SchemaRef) -&gt; RecordBatch {
<a href=#118 id=118 data-nosnippet>118</a> <span class="kw">let </span>df_record_batch = DfRecordBatch::new_empty(schema.arrow_schema().clone());
<a href=#119 id=119 data-nosnippet>119</a> RecordBatch {
<a href=#120 id=120 data-nosnippet>120</a> schema,
<a href=#121 id=121 data-nosnippet>121</a> df_record_batch,
<a href=#122 id=122 data-nosnippet>122</a> }
<a href=#123 id=123 data-nosnippet>123</a> }
<a href=#124 id=124 data-nosnippet>124</a>
<a href=#125 id=125 data-nosnippet>125</a> <span class="doccomment">/// Create an empty [`RecordBatch`] from `schema` with `num_rows`.
<a href=#126 id=126 data-nosnippet>126</a> </span><span class="kw">pub fn </span>new_with_count(schema: SchemaRef, num_rows: usize) -&gt; <span class="prelude-ty">Result</span>&lt;<span class="self">Self</span>&gt; {
<a href=#127 id=127 data-nosnippet>127</a> <span class="kw">let </span>df_record_batch = DfRecordBatch::try_new_with_options(
<a href=#128 id=128 data-nosnippet>128</a> schema.arrow_schema().clone(),
<a href=#129 id=129 data-nosnippet>129</a> <span class="macro">vec!</span>[],
<a href=#130 id=130 data-nosnippet>130</a> <span class="kw-2">&amp;</span>RecordBatchOptions::new().with_row_count(<span class="prelude-val">Some</span>(num_rows)),
<a href=#131 id=131 data-nosnippet>131</a> )
<a href=#132 id=132 data-nosnippet>132</a> .context(error::NewDfRecordBatchSnafu)<span class="question-mark">?</span>;
<a href=#133 id=133 data-nosnippet>133</a> <span class="prelude-val">Ok</span>(RecordBatch {
<a href=#134 id=134 data-nosnippet>134</a> schema,
<a href=#135 id=135 data-nosnippet>135</a> df_record_batch,
<a href=#136 id=136 data-nosnippet>136</a> })
<a href=#137 id=137 data-nosnippet>137</a> }
<a href=#138 id=138 data-nosnippet>138</a>
<a href=#139 id=139 data-nosnippet>139</a> <span class="kw">pub fn </span>try_project(<span class="kw-2">&amp;</span><span class="self">self</span>, indices: <span class="kw-2">&amp;</span>[usize]) -&gt; <span class="prelude-ty">Result</span>&lt;<span class="self">Self</span>&gt; {
<a href=#140 id=140 data-nosnippet>140</a> <span class="kw">let </span>schema = Arc::new(<span class="self">self</span>.schema.try_project(indices).context(DataTypesSnafu)<span class="question-mark">?</span>);
<a href=#141 id=141 data-nosnippet>141</a> <span class="kw">let </span>df_record_batch = <span class="self">self</span>.df_record_batch.project(indices).with_context(|<span class="kw">_</span>| {
<a href=#142 id=142 data-nosnippet>142</a> ProjectArrowRecordBatchSnafu {
<a href=#143 id=143 data-nosnippet>143</a> schema: <span class="self">self</span>.schema.clone(),
<a href=#144 id=144 data-nosnippet>144</a> projection: indices.to_vec(),
<a href=#145 id=145 data-nosnippet>145</a> }
<a href=#146 id=146 data-nosnippet>146</a> })<span class="question-mark">?</span>;
<a href=#147 id=147 data-nosnippet>147</a>
<a href=#148 id=148 data-nosnippet>148</a> <span class="prelude-val">Ok</span>(<span class="self">Self </span>{
<a href=#149 id=149 data-nosnippet>149</a> schema,
<a href=#150 id=150 data-nosnippet>150</a> df_record_batch,
<a href=#151 id=151 data-nosnippet>151</a> })
<a href=#152 id=152 data-nosnippet>152</a> }
<a href=#153 id=153 data-nosnippet>153</a>
<a href=#154 id=154 data-nosnippet>154</a> <span class="doccomment">/// Create a new [`RecordBatch`] from `schema` and `df_record_batch`.
<a href=#155 id=155 data-nosnippet>155</a> ///
<a href=#156 id=156 data-nosnippet>156</a> /// This method doesn't check the schema.
<a href=#157 id=157 data-nosnippet>157</a> </span><span class="kw">pub fn </span>from_df_record_batch(schema: SchemaRef, df_record_batch: DfRecordBatch) -&gt; RecordBatch {
<a href=#158 id=158 data-nosnippet>158</a> RecordBatch {
<a href=#159 id=159 data-nosnippet>159</a> schema,
<a href=#160 id=160 data-nosnippet>160</a> df_record_batch,
<a href=#161 id=161 data-nosnippet>161</a> }
<a href=#162 id=162 data-nosnippet>162</a> }
<a href=#163 id=163 data-nosnippet>163</a>
<a href=#164 id=164 data-nosnippet>164</a> <span class="attr">#[inline]
<a href=#165 id=165 data-nosnippet>165</a> </span><span class="kw">pub fn </span>df_record_batch(<span class="kw-2">&amp;</span><span class="self">self</span>) -&gt; <span class="kw-2">&amp;</span>DfRecordBatch {
<a href=#166 id=166 data-nosnippet>166</a> <span class="kw-2">&amp;</span><span class="self">self</span>.df_record_batch
<a href=#167 id=167 data-nosnippet>167</a> }
<a href=#168 id=168 data-nosnippet>168</a>
<a href=#169 id=169 data-nosnippet>169</a> <span class="attr">#[inline]
<a href=#170 id=170 data-nosnippet>170</a> </span><span class="kw">pub fn </span>into_df_record_batch(<span class="self">self</span>) -&gt; DfRecordBatch {
<a href=#171 id=171 data-nosnippet>171</a> <span class="self">self</span>.df_record_batch
<a href=#172 id=172 data-nosnippet>172</a> }
<a href=#173 id=173 data-nosnippet>173</a>
<a href=#174 id=174 data-nosnippet>174</a> <span class="attr">#[inline]
<a href=#175 id=175 data-nosnippet>175</a> </span><span class="kw">pub fn </span>columns(<span class="kw-2">&amp;</span><span class="self">self</span>) -&gt; <span class="kw-2">&amp;</span>[ArrayRef] {
<a href=#176 id=176 data-nosnippet>176</a> <span class="self">self</span>.df_record_batch.columns()
<a href=#177 id=177 data-nosnippet>177</a> }
<a href=#178 id=178 data-nosnippet>178</a>
<a href=#179 id=179 data-nosnippet>179</a> <span class="attr">#[inline]
<a href=#180 id=180 data-nosnippet>180</a> </span><span class="kw">pub fn </span>column(<span class="kw-2">&amp;</span><span class="self">self</span>, idx: usize) -&gt; <span class="kw-2">&amp;</span>ArrayRef {
<a href=#181 id=181 data-nosnippet>181</a> <span class="self">self</span>.df_record_batch.column(idx)
<a href=#182 id=182 data-nosnippet>182</a> }
<a href=#183 id=183 data-nosnippet>183</a>
<a href=#184 id=184 data-nosnippet>184</a> <span class="kw">pub fn </span>column_by_name(<span class="kw-2">&amp;</span><span class="self">self</span>, name: <span class="kw-2">&amp;</span>str) -&gt; <span class="prelude-ty">Option</span>&lt;<span class="kw-2">&amp;</span>ArrayRef&gt; {
<a href=#185 id=185 data-nosnippet>185</a> <span class="self">self</span>.df_record_batch.column_by_name(name)
<a href=#186 id=186 data-nosnippet>186</a> }
<a href=#187 id=187 data-nosnippet>187</a>
<a href=#188 id=188 data-nosnippet>188</a> <span class="attr">#[inline]
<a href=#189 id=189 data-nosnippet>189</a> </span><span class="kw">pub fn </span>num_columns(<span class="kw-2">&amp;</span><span class="self">self</span>) -&gt; usize {
<a href=#190 id=190 data-nosnippet>190</a> <span class="self">self</span>.df_record_batch.num_columns()
<a href=#191 id=191 data-nosnippet>191</a> }
<a href=#192 id=192 data-nosnippet>192</a>
<a href=#193 id=193 data-nosnippet>193</a> <span class="attr">#[inline]
<a href=#194 id=194 data-nosnippet>194</a> </span><span class="kw">pub fn </span>num_rows(<span class="kw-2">&amp;</span><span class="self">self</span>) -&gt; usize {
<a href=#195 id=195 data-nosnippet>195</a> <span class="self">self</span>.df_record_batch.num_rows()
<a href=#196 id=196 data-nosnippet>196</a> }
<a href=#197 id=197 data-nosnippet>197</a>
<a href=#198 id=198 data-nosnippet>198</a> <span class="kw">pub fn </span>column_vectors(
<a href=#199 id=199 data-nosnippet>199</a> <span class="kw-2">&amp;</span><span class="self">self</span>,
<a href=#200 id=200 data-nosnippet>200</a> table_name: <span class="kw-2">&amp;</span>str,
<a href=#201 id=201 data-nosnippet>201</a> table_schema: SchemaRef,
<a href=#202 id=202 data-nosnippet>202</a> ) -&gt; <span class="prelude-ty">Result</span>&lt;HashMap&lt;String, VectorRef&gt;&gt; {
<a href=#203 id=203 data-nosnippet>203</a> <span class="kw">let </span><span class="kw-2">mut </span>vectors = HashMap::with_capacity(<span class="self">self</span>.num_columns());
<a href=#204 id=204 data-nosnippet>204</a>
<a href=#205 id=205 data-nosnippet>205</a> <span class="comment">// column schemas in recordbatch must match its vectors, otherwise it's corrupted
<a href=#206 id=206 data-nosnippet>206</a> </span><span class="kw">for </span>(field, array) <span class="kw">in </span><span class="self">self
<a href=#207 id=207 data-nosnippet>207</a> </span>.df_record_batch
<a href=#208 id=208 data-nosnippet>208</a> .schema()
<a href=#209 id=209 data-nosnippet>209</a> .fields()
<a href=#210 id=210 data-nosnippet>210</a> .iter()
<a href=#211 id=211 data-nosnippet>211</a> .zip(<span class="self">self</span>.df_record_batch.columns().iter())
<a href=#212 id=212 data-nosnippet>212</a> {
<a href=#213 id=213 data-nosnippet>213</a> <span class="kw">let </span>column_name = field.name();
<a href=#214 id=214 data-nosnippet>214</a> <span class="kw">let </span>column_schema =
<a href=#215 id=215 data-nosnippet>215</a> table_schema
<a href=#216 id=216 data-nosnippet>216</a> .column_schema_by_name(column_name)
<a href=#217 id=217 data-nosnippet>217</a> .context(ColumnNotExistsSnafu {
<a href=#218 id=218 data-nosnippet>218</a> table_name,
<a href=#219 id=219 data-nosnippet>219</a> column_name,
<a href=#220 id=220 data-nosnippet>220</a> })<span class="question-mark">?</span>;
<a href=#221 id=221 data-nosnippet>221</a> <span class="kw">let </span>vector = <span class="kw">if </span>field.data_type() != <span class="kw-2">&amp;</span>column_schema.data_type.as_arrow_type() {
<a href=#222 id=222 data-nosnippet>222</a> <span class="kw">let </span>array = compute::cast(array, <span class="kw-2">&amp;</span>column_schema.data_type.as_arrow_type())
<a href=#223 id=223 data-nosnippet>223</a> .context(ArrowComputeSnafu)<span class="question-mark">?</span>;
<a href=#224 id=224 data-nosnippet>224</a> Helper::try_into_vector(array).context(DataTypesSnafu)<span class="question-mark">?
<a href=#225 id=225 data-nosnippet>225</a> </span>} <span class="kw">else </span>{
<a href=#226 id=226 data-nosnippet>226</a> Helper::try_into_vector(array).context(DataTypesSnafu)<span class="question-mark">?
<a href=#227 id=227 data-nosnippet>227</a> </span>};
<a href=#228 id=228 data-nosnippet>228</a>
<a href=#229 id=229 data-nosnippet>229</a> <span class="kw">let _ </span>= vectors.insert(column_name.clone(), vector);
<a href=#230 id=230 data-nosnippet>230</a> }
<a href=#231 id=231 data-nosnippet>231</a>
<a href=#232 id=232 data-nosnippet>232</a> <span class="prelude-val">Ok</span>(vectors)
<a href=#233 id=233 data-nosnippet>233</a> }
<a href=#234 id=234 data-nosnippet>234</a>
<a href=#235 id=235 data-nosnippet>235</a> <span class="doccomment">/// Pretty display this record batch like a table
<a href=#236 id=236 data-nosnippet>236</a> </span><span class="kw">pub fn </span>pretty_print(<span class="kw-2">&amp;</span><span class="self">self</span>) -&gt; String {
<a href=#237 id=237 data-nosnippet>237</a> pretty_format_batches(slice::from_ref(<span class="kw-2">&amp;</span><span class="self">self</span>.df_record_batch))
<a href=#238 id=238 data-nosnippet>238</a> .map(|t| t.to_string())
<a href=#239 id=239 data-nosnippet>239</a> .unwrap_or(<span class="string">"failed to pretty display a record batch"</span>.to_string())
<a href=#240 id=240 data-nosnippet>240</a> }
<a href=#241 id=241 data-nosnippet>241</a>
<a href=#242 id=242 data-nosnippet>242</a> <span class="doccomment">/// Return a slice record batch starts from offset, with len rows
<a href=#243 id=243 data-nosnippet>243</a> </span><span class="kw">pub fn </span>slice(<span class="kw-2">&amp;</span><span class="self">self</span>, offset: usize, len: usize) -&gt; <span class="prelude-ty">Result</span>&lt;RecordBatch&gt; {
<a href=#244 id=244 data-nosnippet>244</a> <span class="macro">ensure!</span>(
<a href=#245 id=245 data-nosnippet>245</a> offset + len &lt;= <span class="self">self</span>.num_rows(),
<a href=#246 id=246 data-nosnippet>246</a> error::RecordBatchSliceIndexOverflowSnafu {
<a href=#247 id=247 data-nosnippet>247</a> size: <span class="self">self</span>.num_rows(),
<a href=#248 id=248 data-nosnippet>248</a> visit_index: offset + len
<a href=#249 id=249 data-nosnippet>249</a> }
<a href=#250 id=250 data-nosnippet>250</a> );
<a href=#251 id=251 data-nosnippet>251</a> <span class="kw">let </span>sliced = <span class="self">self</span>.df_record_batch.slice(offset, len);
<a href=#252 id=252 data-nosnippet>252</a> <span class="prelude-val">Ok</span>(RecordBatch::from_df_record_batch(
<a href=#253 id=253 data-nosnippet>253</a> <span class="self">self</span>.schema.clone(),
<a href=#254 id=254 data-nosnippet>254</a> sliced,
<a href=#255 id=255 data-nosnippet>255</a> ))
<a href=#256 id=256 data-nosnippet>256</a> }
<a href=#257 id=257 data-nosnippet>257</a>
<a href=#258 id=258 data-nosnippet>258</a> <span class="doccomment">/// Returns the total number of bytes of memory pointed to by the arrays in this `RecordBatch`.
<a href=#259 id=259 data-nosnippet>259</a> ///
<a href=#260 id=260 data-nosnippet>260</a> /// The buffers store bytes in the Arrow memory format, and include the data as well as the validity map.
<a href=#261 id=261 data-nosnippet>261</a> /// Note that this does not always correspond to the exact memory usage of an array,
<a href=#262 id=262 data-nosnippet>262</a> /// since multiple arrays can share the same buffers or slices thereof.
<a href=#263 id=263 data-nosnippet>263</a> </span><span class="kw">pub fn </span>buffer_memory_size(<span class="kw-2">&amp;</span><span class="self">self</span>) -&gt; usize {
<a href=#264 id=264 data-nosnippet>264</a> <span class="self">self</span>.df_record_batch
<a href=#265 id=265 data-nosnippet>265</a> .columns()
<a href=#266 id=266 data-nosnippet>266</a> .iter()
<a href=#267 id=267 data-nosnippet>267</a> .map(|array| array.get_buffer_memory_size())
<a href=#268 id=268 data-nosnippet>268</a> .sum()
<a href=#269 id=269 data-nosnippet>269</a> }
<a href=#270 id=270 data-nosnippet>270</a>
<a href=#271 id=271 data-nosnippet>271</a> <span class="doccomment">/// Iterate the values as strings in the column at index `i`.
<a href=#272 id=272 data-nosnippet>272</a> ///
<a href=#273 id=273 data-nosnippet>273</a> /// Note that if the underlying array is not a valid GreptimeDB vector, an empty iterator is
<a href=#274 id=274 data-nosnippet>274</a> /// returned.
<a href=#275 id=275 data-nosnippet>275</a> ///
<a href=#276 id=276 data-nosnippet>276</a> /// # Panics
<a href=#277 id=277 data-nosnippet>277</a> /// if index `i` is out of bound.
<a href=#278 id=278 data-nosnippet>278</a> </span><span class="kw">pub fn </span>iter_column_as_string(<span class="kw-2">&amp;</span><span class="self">self</span>, i: usize) -&gt; Box&lt;<span class="kw">dyn </span>Iterator&lt;Item = <span class="prelude-ty">Option</span>&lt;String&gt;&gt; + <span class="lifetime">'_</span>&gt; {
<a href=#279 id=279 data-nosnippet>279</a> <span class="macro">macro_rules!</span> iter {
<a href=#280 id=280 data-nosnippet>280</a> (<span class="macro-nonterminal">$column</span>: ident) =&gt; {
<a href=#281 id=281 data-nosnippet>281</a> Box::new(
<a href=#282 id=282 data-nosnippet>282</a> (<span class="number">0</span>..<span class="macro-nonterminal">$column</span>.len())
<a href=#283 id=283 data-nosnippet>283</a> .map(|i| <span class="macro-nonterminal">$column</span>.is_valid(i).then(|| <span class="macro-nonterminal">$column</span>.value(i).to_string())),
<a href=#284 id=284 data-nosnippet>284</a> )
<a href=#285 id=285 data-nosnippet>285</a> };
<a href=#286 id=286 data-nosnippet>286</a> }
<a href=#287 id=287 data-nosnippet>287</a>
<a href=#288 id=288 data-nosnippet>288</a> <span class="kw">let </span>column = <span class="self">self</span>.df_record_batch.column(i);
<a href=#289 id=289 data-nosnippet>289</a> <span class="kw">match </span>column.data_type() {
<a href=#290 id=290 data-nosnippet>290</a> ArrowDataType::Utf8 =&gt; {
<a href=#291 id=291 data-nosnippet>291</a> <span class="kw">let </span>column = column.as_string::&lt;i32&gt;();
<a href=#292 id=292 data-nosnippet>292</a> <span class="kw">let </span>iter = <span class="macro">iter!</span>(column);
<a href=#293 id=293 data-nosnippet>293</a> iter <span class="kw">as _
<a href=#294 id=294 data-nosnippet>294</a> </span>}
<a href=#295 id=295 data-nosnippet>295</a> ArrowDataType::LargeUtf8 =&gt; {
<a href=#296 id=296 data-nosnippet>296</a> <span class="kw">let </span>column = column.as_string::&lt;i64&gt;();
<a href=#297 id=297 data-nosnippet>297</a> <span class="macro">iter!</span>(column)
<a href=#298 id=298 data-nosnippet>298</a> }
<a href=#299 id=299 data-nosnippet>299</a> ArrowDataType::Utf8View =&gt; {
<a href=#300 id=300 data-nosnippet>300</a> <span class="kw">let </span>column = column.as_string_view();
<a href=#301 id=301 data-nosnippet>301</a> <span class="macro">iter!</span>(column)
<a href=#302 id=302 data-nosnippet>302</a> }
<a href=#303 id=303 data-nosnippet>303</a> <span class="kw">_ </span>=&gt; {
<a href=#304 id=304 data-nosnippet>304</a> <span class="kw">if let </span><span class="prelude-val">Ok</span>(column) = Helper::try_into_vector(column) {
<a href=#305 id=305 data-nosnippet>305</a> Box::new(
<a href=#306 id=306 data-nosnippet>306</a> (<span class="number">0</span>..column.len())
<a href=#307 id=307 data-nosnippet>307</a> .map(<span class="kw">move </span>|i| (!column.is_null(i)).then(|| column.get(i).to_string())),
<a href=#308 id=308 data-nosnippet>308</a> )
<a href=#309 id=309 data-nosnippet>309</a> } <span class="kw">else </span>{
<a href=#310 id=310 data-nosnippet>310</a> Box::new(std::iter::empty())
<a href=#311 id=311 data-nosnippet>311</a> }
<a href=#312 id=312 data-nosnippet>312</a> }
<a href=#313 id=313 data-nosnippet>313</a> }
<a href=#314 id=314 data-nosnippet>314</a> }
<a href=#315 id=315 data-nosnippet>315</a>}
<a href=#316 id=316 data-nosnippet>316</a>
<a href=#317 id=317 data-nosnippet>317</a><span class="kw">impl </span>Serialize <span class="kw">for </span>RecordBatch {
<a href=#318 id=318 data-nosnippet>318</a> <span class="kw">fn </span>serialize&lt;S&gt;(<span class="kw-2">&amp;</span><span class="self">self</span>, serializer: S) -&gt; std::result::Result&lt;S::Ok, S::Error&gt;
<a href=#319 id=319 data-nosnippet>319</a> <span class="kw">where
<a href=#320 id=320 data-nosnippet>320</a> </span>S: Serializer,
<a href=#321 id=321 data-nosnippet>321</a> {
<a href=#322 id=322 data-nosnippet>322</a> <span class="comment">// TODO(yingwen): arrow and arrow2's schemas have different fields, so
<a href=#323 id=323 data-nosnippet>323</a> // it might be better to use our `RawSchema` as serialized field.
<a href=#324 id=324 data-nosnippet>324</a> </span><span class="kw">let </span><span class="kw-2">mut </span>s = serializer.serialize_struct(<span class="string">"record"</span>, <span class="number">2</span>)<span class="question-mark">?</span>;
<a href=#325 id=325 data-nosnippet>325</a> s.serialize_field(<span class="string">"schema"</span>, <span class="kw-2">&amp;**</span><span class="self">self</span>.schema.arrow_schema())<span class="question-mark">?</span>;
<a href=#326 id=326 data-nosnippet>326</a>
<a href=#327 id=327 data-nosnippet>327</a> <span class="kw">let </span>columns = <span class="self">self</span>.df_record_batch.columns();
<a href=#328 id=328 data-nosnippet>328</a> <span class="kw">let </span>columns = Helper::try_into_vectors(columns).map_err(Error::custom)<span class="question-mark">?</span>;
<a href=#329 id=329 data-nosnippet>329</a> <span class="kw">let </span>vec = columns
<a href=#330 id=330 data-nosnippet>330</a> .iter()
<a href=#331 id=331 data-nosnippet>331</a> .map(|c| c.serialize_to_json())
<a href=#332 id=332 data-nosnippet>332</a> .collect::&lt;std::result::Result&lt;Vec&lt;<span class="kw">_</span>&gt;, <span class="kw">_</span>&gt;&gt;()
<a href=#333 id=333 data-nosnippet>333</a> .map_err(S::Error::custom)<span class="question-mark">?</span>;
<a href=#334 id=334 data-nosnippet>334</a>
<a href=#335 id=335 data-nosnippet>335</a> s.serialize_field(<span class="string">"columns"</span>, <span class="kw-2">&amp;</span>vec)<span class="question-mark">?</span>;
<a href=#336 id=336 data-nosnippet>336</a> s.end()
<a href=#337 id=337 data-nosnippet>337</a> }
<a href=#338 id=338 data-nosnippet>338</a>}
<a href=#339 id=339 data-nosnippet>339</a>
<a href=#340 id=340 data-nosnippet>340</a><span class="doccomment">/// merge multiple recordbatch into a single
<a href=#341 id=341 data-nosnippet>341</a></span><span class="kw">pub fn </span>merge_record_batches(schema: SchemaRef, batches: <span class="kw-2">&amp;</span>[RecordBatch]) -&gt; <span class="prelude-ty">Result</span>&lt;RecordBatch&gt; {
<a href=#342 id=342 data-nosnippet>342</a> <span class="kw">let </span>batches_len = batches.len();
<a href=#343 id=343 data-nosnippet>343</a> <span class="kw">if </span>batches_len == <span class="number">0 </span>{
<a href=#344 id=344 data-nosnippet>344</a> <span class="kw">return </span><span class="prelude-val">Ok</span>(RecordBatch::new_empty(schema));
<a href=#345 id=345 data-nosnippet>345</a> }
<a href=#346 id=346 data-nosnippet>346</a>
<a href=#347 id=347 data-nosnippet>347</a> <span class="kw">let </span>record_batch = compute::concat_batches(
<a href=#348 id=348 data-nosnippet>348</a> schema.arrow_schema(),
<a href=#349 id=349 data-nosnippet>349</a> batches.iter().map(|x| x.df_record_batch()),
<a href=#350 id=350 data-nosnippet>350</a> )
<a href=#351 id=351 data-nosnippet>351</a> .context(ArrowComputeSnafu)<span class="question-mark">?</span>;
<a href=#352 id=352 data-nosnippet>352</a>
<a href=#353 id=353 data-nosnippet>353</a> <span class="comment">// Create a new RecordBatch with merged columns
<a href=#354 id=354 data-nosnippet>354</a> </span><span class="prelude-val">Ok</span>(RecordBatch::from_df_record_batch(schema, record_batch))
<a href=#355 id=355 data-nosnippet>355</a>}
<a href=#356 id=356 data-nosnippet>356</a>
<a href=#357 id=357 data-nosnippet>357</a><span class="doccomment">/// Align a json array `json_array` to the json type `schema_type`. The `schema_type` is often the
<a href=#358 id=358 data-nosnippet>358</a>/// "largest" json type after some insertions in the table schema, while the json array previously
<a href=#359 id=359 data-nosnippet>359</a>/// written in the SST could be lagged behind it. So it's important to "amend" the json array's
<a href=#360 id=360 data-nosnippet>360</a>/// missing fields with null arrays, to align the array's data type with the provided one.
<a href=#361 id=361 data-nosnippet>361</a>///
<a href=#362 id=362 data-nosnippet>362</a>/// # Panics
<a href=#363 id=363 data-nosnippet>363</a>///
<a href=#364 id=364 data-nosnippet>364</a>/// - The json array is not an Arrow [StructArray], or the provided data type `schema_type` is not
<a href=#365 id=365 data-nosnippet>365</a>/// of Struct type. Both of which shouldn't happen unless we switch our implementation of how
<a href=#366 id=366 data-nosnippet>366</a>/// json array is physically stored.
<a href=#367 id=367 data-nosnippet>367</a></span><span class="kw">pub fn </span>align_json_array(json_array: <span class="kw-2">&amp;</span>ArrayRef, schema_type: <span class="kw-2">&amp;</span>ArrowDataType) -&gt; <span class="prelude-ty">Result</span>&lt;ArrayRef&gt; {
<a href=#368 id=368 data-nosnippet>368</a> <span class="kw">let </span>json_type = json_array.data_type();
<a href=#369 id=369 data-nosnippet>369</a> <span class="kw">if </span>json_type == schema_type {
<a href=#370 id=370 data-nosnippet>370</a> <span class="kw">return </span><span class="prelude-val">Ok</span>(json_array.clone());
<a href=#371 id=371 data-nosnippet>371</a> }
<a href=#372 id=372 data-nosnippet>372</a>
<a href=#373 id=373 data-nosnippet>373</a> <span class="kw">let </span>json_array = json_array.as_struct();
<a href=#374 id=374 data-nosnippet>374</a> <span class="kw">let </span>array_fields = json_array.fields();
<a href=#375 id=375 data-nosnippet>375</a> <span class="kw">let </span>array_columns = json_array.columns();
<a href=#376 id=376 data-nosnippet>376</a> <span class="kw">let </span>ArrowDataType::Struct(schema_fields) = schema_type <span class="kw">else </span>{
<a href=#377 id=377 data-nosnippet>377</a> <span class="macro">unreachable!</span>()
<a href=#378 id=378 data-nosnippet>378</a> };
<a href=#379 id=379 data-nosnippet>379</a> <span class="kw">let </span><span class="kw-2">mut </span>aligned = Vec::with_capacity(schema_fields.len());
<a href=#380 id=380 data-nosnippet>380</a>
<a href=#381 id=381 data-nosnippet>381</a> <span class="comment">// Compare the fields in the json array and the to-be-aligned schema, amending with null arrays
<a href=#382 id=382 data-nosnippet>382</a> // on the way. It's very important to note that fields in the json array and in the json type
<a href=#383 id=383 data-nosnippet>383</a> // are both SORTED.
<a href=#384 id=384 data-nosnippet>384</a>
<a href=#385 id=385 data-nosnippet>385</a> </span><span class="kw">let </span><span class="kw-2">mut </span>i = <span class="number">0</span>; <span class="comment">// point to the schema fields
<a href=#386 id=386 data-nosnippet>386</a> </span><span class="kw">let </span><span class="kw-2">mut </span>j = <span class="number">0</span>; <span class="comment">// point to the array fields
<a href=#387 id=387 data-nosnippet>387</a> </span><span class="kw">while </span>i &lt; schema_fields.len() &amp;&amp; j &lt; array_fields.len() {
<a href=#388 id=388 data-nosnippet>388</a> <span class="kw">let </span>schema_field = <span class="kw-2">&amp;</span>schema_fields[i];
<a href=#389 id=389 data-nosnippet>389</a> <span class="kw">let </span>array_field = <span class="kw-2">&amp;</span>array_fields[j];
<a href=#390 id=390 data-nosnippet>390</a> <span class="kw">if </span>schema_field.name() == array_field.name() {
<a href=#391 id=391 data-nosnippet>391</a> <span class="kw">if </span><span class="macro">matches!</span>(schema_field.data_type(), ArrowDataType::Struct(<span class="kw">_</span>)) {
<a href=#392 id=392 data-nosnippet>392</a> <span class="comment">// A `StructArray`s in a json array must be another json array. (Like a nested json
<a href=#393 id=393 data-nosnippet>393</a> // object in a json value.)
<a href=#394 id=394 data-nosnippet>394</a> </span>aligned.push(align_json_array(
<a href=#395 id=395 data-nosnippet>395</a> <span class="kw-2">&amp;</span>array_columns[j],
<a href=#396 id=396 data-nosnippet>396</a> schema_field.data_type(),
<a href=#397 id=397 data-nosnippet>397</a> )<span class="question-mark">?</span>);
<a href=#398 id=398 data-nosnippet>398</a> } <span class="kw">else </span>{
<a href=#399 id=399 data-nosnippet>399</a> aligned.push(array_columns[j].clone());
<a href=#400 id=400 data-nosnippet>400</a> }
<a href=#401 id=401 data-nosnippet>401</a> j += <span class="number">1</span>;
<a href=#402 id=402 data-nosnippet>402</a> } <span class="kw">else </span>{
<a href=#403 id=403 data-nosnippet>403</a> aligned.push(new_null_array(schema_field.data_type(), json_array.len()));
<a href=#404 id=404 data-nosnippet>404</a> }
<a href=#405 id=405 data-nosnippet>405</a> i += <span class="number">1</span>;
<a href=#406 id=406 data-nosnippet>406</a> }
<a href=#407 id=407 data-nosnippet>407</a> <span class="kw">if </span>i &lt; schema_fields.len() {
<a href=#408 id=408 data-nosnippet>408</a> <span class="kw">for </span>field <span class="kw">in </span><span class="kw-2">&amp;</span>schema_fields[i..] {
<a href=#409 id=409 data-nosnippet>409</a> aligned.push(new_null_array(field.data_type(), json_array.len()));
<a href=#410 id=410 data-nosnippet>410</a> }
<a href=#411 id=411 data-nosnippet>411</a> }
<a href=#412 id=412 data-nosnippet>412</a> <span class="macro">ensure!</span>(
<a href=#413 id=413 data-nosnippet>413</a> j == array_fields.len(),
<a href=#414 id=414 data-nosnippet>414</a> AlignJsonArraySnafu {
<a href=#415 id=415 data-nosnippet>415</a> reason: <span class="macro">format!</span>(
<a href=#416 id=416 data-nosnippet>416</a> <span class="string">"this json array has more fields {:?}"</span>,
<a href=#417 id=417 data-nosnippet>417</a> array_fields[j..]
<a href=#418 id=418 data-nosnippet>418</a> .iter()
<a href=#419 id=419 data-nosnippet>419</a> .map(|x| x.name())
<a href=#420 id=420 data-nosnippet>420</a> .collect::&lt;Vec&lt;<span class="kw">_</span>&gt;&gt;(),
<a href=#421 id=421 data-nosnippet>421</a> )
<a href=#422 id=422 data-nosnippet>422</a> }
<a href=#423 id=423 data-nosnippet>423</a> );
<a href=#424 id=424 data-nosnippet>424</a>
<a href=#425 id=425 data-nosnippet>425</a> <span class="kw">let </span>json_array =
<a href=#426 id=426 data-nosnippet>426</a> StructArray::try_new(schema_fields.clone(), aligned, json_array.nulls().cloned())
<a href=#427 id=427 data-nosnippet>427</a> .context(NewDfRecordBatchSnafu)<span class="question-mark">?</span>;
<a href=#428 id=428 data-nosnippet>428</a> <span class="prelude-val">Ok</span>(Arc::new(json_array))
<a href=#429 id=429 data-nosnippet>429</a>}
<a href=#430 id=430 data-nosnippet>430</a>
<a href=#431 id=431 data-nosnippet>431</a><span class="kw">fn </span>maybe_align_json_array_with_schema(
<a href=#432 id=432 data-nosnippet>432</a> schema: <span class="kw-2">&amp;</span>ArrowSchemaRef,
<a href=#433 id=433 data-nosnippet>433</a> arrays: Vec&lt;ArrayRef&gt;,
<a href=#434 id=434 data-nosnippet>434</a>) -&gt; <span class="prelude-ty">Result</span>&lt;Vec&lt;ArrayRef&gt;&gt; {
<a href=#435 id=435 data-nosnippet>435</a> <span class="kw">if </span>schema.fields().iter().all(|f| !is_json_extension_type(f)) {
<a href=#436 id=436 data-nosnippet>436</a> <span class="kw">return </span><span class="prelude-val">Ok</span>(arrays);
<a href=#437 id=437 data-nosnippet>437</a> }
<a href=#438 id=438 data-nosnippet>438</a>
<a href=#439 id=439 data-nosnippet>439</a> <span class="kw">let </span><span class="kw-2">mut </span>aligned = Vec::with_capacity(arrays.len());
<a href=#440 id=440 data-nosnippet>440</a> <span class="kw">for </span>(field, array) <span class="kw">in </span>schema.fields().iter().zip(arrays.into_iter()) {
<a href=#441 id=441 data-nosnippet>441</a> <span class="kw">if </span>!is_json_extension_type(field) {
<a href=#442 id=442 data-nosnippet>442</a> aligned.push(array);
<a href=#443 id=443 data-nosnippet>443</a> <span class="kw">continue</span>;
<a href=#444 id=444 data-nosnippet>444</a> }
<a href=#445 id=445 data-nosnippet>445</a>
<a href=#446 id=446 data-nosnippet>446</a> <span class="kw">let </span>json_array = align_json_array(<span class="kw-2">&amp;</span>array, field.data_type())<span class="question-mark">?</span>;
<a href=#447 id=447 data-nosnippet>447</a> aligned.push(json_array);
<a href=#448 id=448 data-nosnippet>448</a> }
<a href=#449 id=449 data-nosnippet>449</a> <span class="prelude-val">Ok</span>(aligned)
<a href=#450 id=450 data-nosnippet>450</a>}
<a href=#451 id=451 data-nosnippet>451</a>
<a href=#452 id=452 data-nosnippet>452</a><span class="attr">#[cfg(test)]
<a href=#453 id=453 data-nosnippet>453</a></span><span class="kw">mod </span>tests {
<a href=#454 id=454 data-nosnippet>454</a> <span class="kw">use </span>std::sync::Arc;
<a href=#455 id=455 data-nosnippet>455</a>
<a href=#456 id=456 data-nosnippet>456</a> <span class="kw">use </span>datatypes::arrow::array::{
<a href=#457 id=457 data-nosnippet>457</a> AsArray, BooleanArray, Float64Array, Int64Array, ListArray, UInt32Array,
<a href=#458 id=458 data-nosnippet>458</a> };
<a href=#459 id=459 data-nosnippet>459</a> <span class="kw">use </span>datatypes::arrow::datatypes::{
<a href=#460 id=460 data-nosnippet>460</a> DataType, Field, Fields, Int64Type, Schema <span class="kw">as </span>ArrowSchema, UInt32Type,
<a href=#461 id=461 data-nosnippet>461</a> };
<a href=#462 id=462 data-nosnippet>462</a> <span class="kw">use </span>datatypes::arrow_array::StringArray;
<a href=#463 id=463 data-nosnippet>463</a> <span class="kw">use </span>datatypes::data_type::ConcreteDataType;
<a href=#464 id=464 data-nosnippet>464</a> <span class="kw">use </span>datatypes::schema::{ColumnSchema, Schema};
<a href=#465 id=465 data-nosnippet>465</a> <span class="kw">use </span>datatypes::vectors::{StringVector, UInt32Vector};
<a href=#466 id=466 data-nosnippet>466</a>
<a href=#467 id=467 data-nosnippet>467</a> <span class="kw">use super</span>::<span class="kw-2">*</span>;
<a href=#468 id=468 data-nosnippet>468</a>
<a href=#469 id=469 data-nosnippet>469</a> <span class="attr">#[test]
<a href=#470 id=470 data-nosnippet>470</a> </span><span class="kw">fn </span>test_align_json_array() -&gt; <span class="prelude-ty">Result</span>&lt;()&gt; {
<a href=#471 id=471 data-nosnippet>471</a> <span class="kw">struct </span>TestCase {
<a href=#472 id=472 data-nosnippet>472</a> json_array: ArrayRef,
<a href=#473 id=473 data-nosnippet>473</a> schema_type: DataType,
<a href=#474 id=474 data-nosnippet>474</a> expected: std::result::Result&lt;ArrayRef, String&gt;,
<a href=#475 id=475 data-nosnippet>475</a> }
<a href=#476 id=476 data-nosnippet>476</a>
<a href=#477 id=477 data-nosnippet>477</a> <span class="kw">impl </span>TestCase {
<a href=#478 id=478 data-nosnippet>478</a> <span class="kw">fn </span>new(
<a href=#479 id=479 data-nosnippet>479</a> json_array: StructArray,
<a href=#480 id=480 data-nosnippet>480</a> schema_type: Fields,
<a href=#481 id=481 data-nosnippet>481</a> expected: std::result::Result&lt;Vec&lt;ArrayRef&gt;, String&gt;,
<a href=#482 id=482 data-nosnippet>482</a> ) -&gt; <span class="self">Self </span>{
<a href=#483 id=483 data-nosnippet>483</a> <span class="self">Self </span>{
<a href=#484 id=484 data-nosnippet>484</a> json_array: Arc::new(json_array),
<a href=#485 id=485 data-nosnippet>485</a> schema_type: DataType::Struct(schema_type.clone()),
<a href=#486 id=486 data-nosnippet>486</a> expected: expected
<a href=#487 id=487 data-nosnippet>487</a> .map(|x| Arc::new(StructArray::new(schema_type, x, <span class="prelude-val">None</span>)) <span class="kw">as </span>ArrayRef),
<a href=#488 id=488 data-nosnippet>488</a> }
<a href=#489 id=489 data-nosnippet>489</a> }
<a href=#490 id=490 data-nosnippet>490</a>
<a href=#491 id=491 data-nosnippet>491</a> <span class="kw">fn </span>test(<span class="self">self</span>) -&gt; <span class="prelude-ty">Result</span>&lt;()&gt; {
<a href=#492 id=492 data-nosnippet>492</a> <span class="kw">let </span>result = align_json_array(<span class="kw-2">&amp;</span><span class="self">self</span>.json_array, <span class="kw-2">&amp;</span><span class="self">self</span>.schema_type);
<a href=#493 id=493 data-nosnippet>493</a> <span class="kw">match </span>(result, <span class="self">self</span>.expected) {
<a href=#494 id=494 data-nosnippet>494</a> (<span class="prelude-val">Ok</span>(json_array), <span class="prelude-val">Ok</span>(expected)) =&gt; <span class="macro">assert_eq!</span>(<span class="kw-2">&amp;</span>json_array, <span class="kw-2">&amp;</span>expected),
<a href=#495 id=495 data-nosnippet>495</a> (<span class="prelude-val">Ok</span>(json_array), <span class="prelude-val">Err</span>(e)) =&gt; {
<a href=#496 id=496 data-nosnippet>496</a> <span class="macro">panic!</span>(<span class="string">"expecting error {e} but actually get: {json_array:?}"</span>)
<a href=#497 id=497 data-nosnippet>497</a> }
<a href=#498 id=498 data-nosnippet>498</a> (<span class="prelude-val">Err</span>(e), <span class="prelude-val">Err</span>(expected)) =&gt; <span class="macro">assert_eq!</span>(e.to_string(), expected),
<a href=#499 id=499 data-nosnippet>499</a> (<span class="prelude-val">Err</span>(e), <span class="prelude-val">Ok</span>(<span class="kw">_</span>)) =&gt; <span class="kw">return </span><span class="prelude-val">Err</span>(e),
<a href=#500 id=500 data-nosnippet>500</a> }
<a href=#501 id=501 data-nosnippet>501</a> <span class="prelude-val">Ok</span>(())
<a href=#502 id=502 data-nosnippet>502</a> }
<a href=#503 id=503 data-nosnippet>503</a> }
<a href=#504 id=504 data-nosnippet>504</a>
<a href=#505 id=505 data-nosnippet>505</a> <span class="comment">// Test empty json array can be aligned with a complex json type.
<a href=#506 id=506 data-nosnippet>506</a> </span>TestCase::new(
<a href=#507 id=507 data-nosnippet>507</a> StructArray::new_empty_fields(<span class="number">2</span>, <span class="prelude-val">None</span>),
<a href=#508 id=508 data-nosnippet>508</a> Fields::from(<span class="macro">vec!</span>[
<a href=#509 id=509 data-nosnippet>509</a> Field::new(<span class="string">"int"</span>, DataType::Int64, <span class="bool-val">true</span>),
<a href=#510 id=510 data-nosnippet>510</a> Field::new_struct(
<a href=#511 id=511 data-nosnippet>511</a> <span class="string">"nested"</span>,
<a href=#512 id=512 data-nosnippet>512</a> <span class="macro">vec!</span>[Field::new(<span class="string">"bool"</span>, DataType::Boolean, <span class="bool-val">true</span>)],
<a href=#513 id=513 data-nosnippet>513</a> <span class="bool-val">true</span>,
<a href=#514 id=514 data-nosnippet>514</a> ),
<a href=#515 id=515 data-nosnippet>515</a> Field::new(<span class="string">"string"</span>, DataType::Utf8, <span class="bool-val">true</span>),
<a href=#516 id=516 data-nosnippet>516</a> ]),
<a href=#517 id=517 data-nosnippet>517</a> <span class="prelude-val">Ok</span>(<span class="macro">vec!</span>[
<a href=#518 id=518 data-nosnippet>518</a> Arc::new(Int64Array::new_null(<span class="number">2</span>)) <span class="kw">as </span>ArrayRef,
<a href=#519 id=519 data-nosnippet>519</a> Arc::new(StructArray::new_null(
<a href=#520 id=520 data-nosnippet>520</a> Fields::from(<span class="macro">vec!</span>[Arc::new(Field::new(<span class="string">"bool"</span>, DataType::Boolean, <span class="bool-val">true</span>))]),
<a href=#521 id=521 data-nosnippet>521</a> <span class="number">2</span>,
<a href=#522 id=522 data-nosnippet>522</a> )),
<a href=#523 id=523 data-nosnippet>523</a> Arc::new(StringArray::new_null(<span class="number">2</span>)),
<a href=#524 id=524 data-nosnippet>524</a> ]),
<a href=#525 id=525 data-nosnippet>525</a> )
<a href=#526 id=526 data-nosnippet>526</a> .test()<span class="question-mark">?</span>;
<a href=#527 id=527 data-nosnippet>527</a>
<a href=#528 id=528 data-nosnippet>528</a> <span class="comment">// Test simple json array alignment.
<a href=#529 id=529 data-nosnippet>529</a> </span>TestCase::new(
<a href=#530 id=530 data-nosnippet>530</a> StructArray::from(<span class="macro">vec!</span>[(
<a href=#531 id=531 data-nosnippet>531</a> Arc::new(Field::new(<span class="string">"float"</span>, DataType::Float64, <span class="bool-val">true</span>)),
<a href=#532 id=532 data-nosnippet>532</a> Arc::new(Float64Array::from(<span class="macro">vec!</span>[<span class="number">1.0</span>, <span class="number">2.0</span>, <span class="number">3.0</span>])) <span class="kw">as </span>ArrayRef,
<a href=#533 id=533 data-nosnippet>533</a> )]),
<a href=#534 id=534 data-nosnippet>534</a> Fields::from(<span class="macro">vec!</span>[
<a href=#535 id=535 data-nosnippet>535</a> Field::new(<span class="string">"float"</span>, DataType::Float64, <span class="bool-val">true</span>),
<a href=#536 id=536 data-nosnippet>536</a> Field::new(<span class="string">"string"</span>, DataType::Utf8, <span class="bool-val">true</span>),
<a href=#537 id=537 data-nosnippet>537</a> ]),
<a href=#538 id=538 data-nosnippet>538</a> <span class="prelude-val">Ok</span>(<span class="macro">vec!</span>[
<a href=#539 id=539 data-nosnippet>539</a> Arc::new(Float64Array::from(<span class="macro">vec!</span>[<span class="number">1.0</span>, <span class="number">2.0</span>, <span class="number">3.0</span>])) <span class="kw">as </span>ArrayRef,
<a href=#540 id=540 data-nosnippet>540</a> Arc::new(StringArray::new_null(<span class="number">3</span>)),
<a href=#541 id=541 data-nosnippet>541</a> ]),
<a href=#542 id=542 data-nosnippet>542</a> )
<a href=#543 id=543 data-nosnippet>543</a> .test()<span class="question-mark">?</span>;
<a href=#544 id=544 data-nosnippet>544</a>
<a href=#545 id=545 data-nosnippet>545</a> <span class="comment">// Test complex json array alignment.
<a href=#546 id=546 data-nosnippet>546</a> </span>TestCase::new(
<a href=#547 id=547 data-nosnippet>547</a> StructArray::from(<span class="macro">vec!</span>[
<a href=#548 id=548 data-nosnippet>548</a> (
<a href=#549 id=549 data-nosnippet>549</a> Arc::new(Field::new_list(
<a href=#550 id=550 data-nosnippet>550</a> <span class="string">"list"</span>,
<a href=#551 id=551 data-nosnippet>551</a> Field::new_list_field(DataType::Int64, <span class="bool-val">true</span>),
<a href=#552 id=552 data-nosnippet>552</a> <span class="bool-val">true</span>,
<a href=#553 id=553 data-nosnippet>553</a> )),
<a href=#554 id=554 data-nosnippet>554</a> Arc::new(ListArray::from_iter_primitive::&lt;Int64Type, <span class="kw">_</span>, <span class="kw">_</span>&gt;(<span class="macro">vec!</span>[
<a href=#555 id=555 data-nosnippet>555</a> <span class="prelude-val">Some</span>(<span class="macro">vec!</span>[<span class="prelude-val">Some</span>(<span class="number">1</span>)]),
<a href=#556 id=556 data-nosnippet>556</a> <span class="prelude-val">None</span>,
<a href=#557 id=557 data-nosnippet>557</a> <span class="prelude-val">Some</span>(<span class="macro">vec!</span>[<span class="prelude-val">Some</span>(<span class="number">2</span>), <span class="prelude-val">Some</span>(<span class="number">3</span>)]),
<a href=#558 id=558 data-nosnippet>558</a> ])) <span class="kw">as </span>ArrayRef,
<a href=#559 id=559 data-nosnippet>559</a> ),
<a href=#560 id=560 data-nosnippet>560</a> (
<a href=#561 id=561 data-nosnippet>561</a> Arc::new(Field::new_struct(
<a href=#562 id=562 data-nosnippet>562</a> <span class="string">"nested"</span>,
<a href=#563 id=563 data-nosnippet>563</a> <span class="macro">vec!</span>[Field::new(<span class="string">"int"</span>, DataType::Int64, <span class="bool-val">true</span>)],
<a href=#564 id=564 data-nosnippet>564</a> <span class="bool-val">true</span>,
<a href=#565 id=565 data-nosnippet>565</a> )),
<a href=#566 id=566 data-nosnippet>566</a> Arc::new(StructArray::from(<span class="macro">vec!</span>[(
<a href=#567 id=567 data-nosnippet>567</a> Arc::new(Field::new(<span class="string">"int"</span>, DataType::Int64, <span class="bool-val">true</span>)),
<a href=#568 id=568 data-nosnippet>568</a> Arc::new(Int64Array::from(<span class="macro">vec!</span>[-<span class="number">1</span>, -<span class="number">2</span>, -<span class="number">3</span>])) <span class="kw">as </span>ArrayRef,
<a href=#569 id=569 data-nosnippet>569</a> )])),
<a href=#570 id=570 data-nosnippet>570</a> ),
<a href=#571 id=571 data-nosnippet>571</a> (
<a href=#572 id=572 data-nosnippet>572</a> Arc::new(Field::new(<span class="string">"string"</span>, DataType::Utf8, <span class="bool-val">true</span>)),
<a href=#573 id=573 data-nosnippet>573</a> Arc::new(StringArray::from(<span class="macro">vec!</span>[<span class="string">"a"</span>, <span class="string">"b"</span>, <span class="string">"c"</span>])),
<a href=#574 id=574 data-nosnippet>574</a> ),
<a href=#575 id=575 data-nosnippet>575</a> ]),
<a href=#576 id=576 data-nosnippet>576</a> Fields::from(<span class="macro">vec!</span>[
<a href=#577 id=577 data-nosnippet>577</a> Field::new(<span class="string">"bool"</span>, DataType::Boolean, <span class="bool-val">true</span>),
<a href=#578 id=578 data-nosnippet>578</a> Field::new_list(<span class="string">"list"</span>, Field::new_list_field(DataType::Int64, <span class="bool-val">true</span>), <span class="bool-val">true</span>),
<a href=#579 id=579 data-nosnippet>579</a> Field::new_struct(
<a href=#580 id=580 data-nosnippet>580</a> <span class="string">"nested"</span>,
<a href=#581 id=581 data-nosnippet>581</a> <span class="macro">vec!</span>[
<a href=#582 id=582 data-nosnippet>582</a> Field::new(<span class="string">"float"</span>, DataType::Float64, <span class="bool-val">true</span>),
<a href=#583 id=583 data-nosnippet>583</a> Field::new(<span class="string">"int"</span>, DataType::Int64, <span class="bool-val">true</span>),
<a href=#584 id=584 data-nosnippet>584</a> ],
<a href=#585 id=585 data-nosnippet>585</a> <span class="bool-val">true</span>,
<a href=#586 id=586 data-nosnippet>586</a> ),
<a href=#587 id=587 data-nosnippet>587</a> Field::new(<span class="string">"string"</span>, DataType::Utf8, <span class="bool-val">true</span>),
<a href=#588 id=588 data-nosnippet>588</a> ]),
<a href=#589 id=589 data-nosnippet>589</a> <span class="prelude-val">Ok</span>(<span class="macro">vec!</span>[
<a href=#590 id=590 data-nosnippet>590</a> Arc::new(BooleanArray::new_null(<span class="number">3</span>)) <span class="kw">as </span>ArrayRef,
<a href=#591 id=591 data-nosnippet>591</a> Arc::new(ListArray::from_iter_primitive::&lt;Int64Type, <span class="kw">_</span>, <span class="kw">_</span>&gt;(<span class="macro">vec!</span>[
<a href=#592 id=592 data-nosnippet>592</a> <span class="prelude-val">Some</span>(<span class="macro">vec!</span>[<span class="prelude-val">Some</span>(<span class="number">1</span>)]),
<a href=#593 id=593 data-nosnippet>593</a> <span class="prelude-val">None</span>,
<a href=#594 id=594 data-nosnippet>594</a> <span class="prelude-val">Some</span>(<span class="macro">vec!</span>[<span class="prelude-val">Some</span>(<span class="number">2</span>), <span class="prelude-val">Some</span>(<span class="number">3</span>)]),
<a href=#595 id=595 data-nosnippet>595</a> ])),
<a href=#596 id=596 data-nosnippet>596</a> Arc::new(StructArray::from(<span class="macro">vec!</span>[
<a href=#597 id=597 data-nosnippet>597</a> (
<a href=#598 id=598 data-nosnippet>598</a> Arc::new(Field::new(<span class="string">"float"</span>, DataType::Float64, <span class="bool-val">true</span>)),
<a href=#599 id=599 data-nosnippet>599</a> Arc::new(Float64Array::new_null(<span class="number">3</span>)) <span class="kw">as </span>ArrayRef,
<a href=#600 id=600 data-nosnippet>600</a> ),
<a href=#601 id=601 data-nosnippet>601</a> (
<a href=#602 id=602 data-nosnippet>602</a> Arc::new(Field::new(<span class="string">"int"</span>, DataType::Int64, <span class="bool-val">true</span>)),
<a href=#603 id=603 data-nosnippet>603</a> Arc::new(Int64Array::from(<span class="macro">vec!</span>[-<span class="number">1</span>, -<span class="number">2</span>, -<span class="number">3</span>])),
<a href=#604 id=604 data-nosnippet>604</a> ),
<a href=#605 id=605 data-nosnippet>605</a> ])),
<a href=#606 id=606 data-nosnippet>606</a> Arc::new(StringArray::from(<span class="macro">vec!</span>[<span class="string">"a"</span>, <span class="string">"b"</span>, <span class="string">"c"</span>])),
<a href=#607 id=607 data-nosnippet>607</a> ]),
<a href=#608 id=608 data-nosnippet>608</a> )
<a href=#609 id=609 data-nosnippet>609</a> .test()<span class="question-mark">?</span>;
<a href=#610 id=610 data-nosnippet>610</a>
<a href=#611 id=611 data-nosnippet>611</a> <span class="comment">// Test align failed.
<a href=#612 id=612 data-nosnippet>612</a> </span>TestCase::new(
<a href=#613 id=613 data-nosnippet>613</a> StructArray::try_from(<span class="macro">vec!</span>[
<a href=#614 id=614 data-nosnippet>614</a> (<span class="string">"i"</span>, Arc::new(Int64Array::from(<span class="macro">vec!</span>[<span class="number">1</span>])) <span class="kw">as </span>ArrayRef),
<a href=#615 id=615 data-nosnippet>615</a> (<span class="string">"j"</span>, Arc::new(Int64Array::from(<span class="macro">vec!</span>[<span class="number">2</span>])) <span class="kw">as </span>ArrayRef),
<a href=#616 id=616 data-nosnippet>616</a> ])
<a href=#617 id=617 data-nosnippet>617</a> .unwrap(),
<a href=#618 id=618 data-nosnippet>618</a> Fields::from(<span class="macro">vec!</span>[Field::new(<span class="string">"i"</span>, DataType::Int64, <span class="bool-val">true</span>)]),
<a href=#619 id=619 data-nosnippet>619</a> <span class="prelude-val">Err</span>(
<a href=#620 id=620 data-nosnippet>620</a> <span class="string">r#"Failed to align JSON array, reason: this json array has more fields ["j"]"#
<a href=#621 id=621 data-nosnippet>621</a> </span>.to_string(),
<a href=#622 id=622 data-nosnippet>622</a> ),
<a href=#623 id=623 data-nosnippet>623</a> )
<a href=#624 id=624 data-nosnippet>624</a> .test()<span class="question-mark">?</span>;
<a href=#625 id=625 data-nosnippet>625</a> <span class="prelude-val">Ok</span>(())
<a href=#626 id=626 data-nosnippet>626</a> }
<a href=#627 id=627 data-nosnippet>627</a>
<a href=#628 id=628 data-nosnippet>628</a> <span class="attr">#[test]
<a href=#629 id=629 data-nosnippet>629</a> </span><span class="kw">fn </span>test_record_batch() {
<a href=#630 id=630 data-nosnippet>630</a> <span class="kw">let </span>arrow_schema = Arc::new(ArrowSchema::new(<span class="macro">vec!</span>[
<a href=#631 id=631 data-nosnippet>631</a> Field::new(<span class="string">"c1"</span>, DataType::UInt32, <span class="bool-val">false</span>),
<a href=#632 id=632 data-nosnippet>632</a> Field::new(<span class="string">"c2"</span>, DataType::UInt32, <span class="bool-val">false</span>),
<a href=#633 id=633 data-nosnippet>633</a> ]));
<a href=#634 id=634 data-nosnippet>634</a> <span class="kw">let </span>schema = Arc::new(Schema::try_from(arrow_schema).unwrap());
<a href=#635 id=635 data-nosnippet>635</a>
<a href=#636 id=636 data-nosnippet>636</a> <span class="kw">let </span>c1 = Arc::new(UInt32Vector::from_slice([<span class="number">1</span>, <span class="number">2</span>, <span class="number">3</span>]));
<a href=#637 id=637 data-nosnippet>637</a> <span class="kw">let </span>c2 = Arc::new(UInt32Vector::from_slice([<span class="number">4</span>, <span class="number">5</span>, <span class="number">6</span>]));
<a href=#638 id=638 data-nosnippet>638</a> <span class="kw">let </span>columns: Vec&lt;VectorRef&gt; = <span class="macro">vec!</span>[c1, c2];
<a href=#639 id=639 data-nosnippet>639</a>
<a href=#640 id=640 data-nosnippet>640</a> <span class="kw">let </span>expected = <span class="macro">vec!</span>[
<a href=#641 id=641 data-nosnippet>641</a> Arc::new(UInt32Array::from_iter_values([<span class="number">1</span>, <span class="number">2</span>, <span class="number">3</span>])) <span class="kw">as </span>ArrayRef,
<a href=#642 id=642 data-nosnippet>642</a> Arc::new(UInt32Array::from_iter_values([<span class="number">4</span>, <span class="number">5</span>, <span class="number">6</span>])),
<a href=#643 id=643 data-nosnippet>643</a> ];
<a href=#644 id=644 data-nosnippet>644</a>
<a href=#645 id=645 data-nosnippet>645</a> <span class="kw">let </span>batch = RecordBatch::new(schema.clone(), columns.clone()).unwrap();
<a href=#646 id=646 data-nosnippet>646</a> <span class="macro">assert_eq!</span>(<span class="number">3</span>, batch.num_rows());
<a href=#647 id=647 data-nosnippet>647</a> <span class="macro">assert_eq!</span>(expected, batch.df_record_batch().columns());
<a href=#648 id=648 data-nosnippet>648</a> <span class="macro">assert_eq!</span>(schema, batch.schema);
<a href=#649 id=649 data-nosnippet>649</a>
<a href=#650 id=650 data-nosnippet>650</a> <span class="macro">assert_eq!</span>(<span class="kw-2">&amp;</span>expected[<span class="number">0</span>], batch.column_by_name(<span class="string">"c1"</span>).unwrap());
<a href=#651 id=651 data-nosnippet>651</a> <span class="macro">assert_eq!</span>(<span class="kw-2">&amp;</span>expected[<span class="number">1</span>], batch.column_by_name(<span class="string">"c2"</span>).unwrap());
<a href=#652 id=652 data-nosnippet>652</a> <span class="macro">assert!</span>(batch.column_by_name(<span class="string">"c3"</span>).is_none());
<a href=#653 id=653 data-nosnippet>653</a>
<a href=#654 id=654 data-nosnippet>654</a> <span class="kw">let </span>converted = RecordBatch::from_df_record_batch(schema, batch.df_record_batch().clone());
<a href=#655 id=655 data-nosnippet>655</a> <span class="macro">assert_eq!</span>(batch, converted);
<a href=#656 id=656 data-nosnippet>656</a> <span class="macro">assert_eq!</span>(<span class="kw-2">*</span>batch.df_record_batch(), converted.into_df_record_batch());
<a href=#657 id=657 data-nosnippet>657</a> }
<a href=#658 id=658 data-nosnippet>658</a>
<a href=#659 id=659 data-nosnippet>659</a> <span class="attr">#[test]
<a href=#660 id=660 data-nosnippet>660</a> </span><span class="kw">pub fn </span>test_serialize_recordbatch() {
<a href=#661 id=661 data-nosnippet>661</a> <span class="kw">let </span>column_schemas = <span class="macro">vec!</span>[ColumnSchema::new(
<a href=#662 id=662 data-nosnippet>662</a> <span class="string">"number"</span>,
<a href=#663 id=663 data-nosnippet>663</a> ConcreteDataType::uint32_datatype(),
<a href=#664 id=664 data-nosnippet>664</a> <span class="bool-val">false</span>,
<a href=#665 id=665 data-nosnippet>665</a> )];
<a href=#666 id=666 data-nosnippet>666</a> <span class="kw">let </span>schema = Arc::new(Schema::try_new(column_schemas).unwrap());
<a href=#667 id=667 data-nosnippet>667</a>
<a href=#668 id=668 data-nosnippet>668</a> <span class="kw">let </span>numbers: Vec&lt;u32&gt; = (<span class="number">0</span>..<span class="number">10</span>).collect();
<a href=#669 id=669 data-nosnippet>669</a> <span class="kw">let </span>columns = <span class="macro">vec!</span>[Arc::new(UInt32Vector::from_slice(numbers)) <span class="kw">as </span>VectorRef];
<a href=#670 id=670 data-nosnippet>670</a> <span class="kw">let </span>batch = RecordBatch::new(schema, columns).unwrap();
<a href=#671 id=671 data-nosnippet>671</a>
<a href=#672 id=672 data-nosnippet>672</a> <span class="kw">let </span>output = serde_json::to_string(<span class="kw-2">&amp;</span>batch).unwrap();
<a href=#673 id=673 data-nosnippet>673</a> <span class="macro">assert_eq!</span>(
<a href=#674 id=674 data-nosnippet>674</a> <span class="string">r#"{"schema":{"fields":[{"name":"number","data_type":"UInt32","nullable":false,"dict_id":0,"dict_is_ordered":false,"metadata":{}}],"metadata":{"greptime:version":"0"}},"columns":[[0,1,2,3,4,5,6,7,8,9]]}"#</span>,
<a href=#675 id=675 data-nosnippet>675</a> output
<a href=#676 id=676 data-nosnippet>676</a> );
<a href=#677 id=677 data-nosnippet>677</a> }
<a href=#678 id=678 data-nosnippet>678</a>
<a href=#679 id=679 data-nosnippet>679</a> <span class="attr">#[test]
<a href=#680 id=680 data-nosnippet>680</a> </span><span class="kw">fn </span>test_record_batch_slice() {
<a href=#681 id=681 data-nosnippet>681</a> <span class="kw">let </span>column_schemas = <span class="macro">vec!</span>[
<a href=#682 id=682 data-nosnippet>682</a> ColumnSchema::new(<span class="string">"numbers"</span>, ConcreteDataType::uint32_datatype(), <span class="bool-val">false</span>),
<a href=#683 id=683 data-nosnippet>683</a> ColumnSchema::new(<span class="string">"strings"</span>, ConcreteDataType::string_datatype(), <span class="bool-val">true</span>),
<a href=#684 id=684 data-nosnippet>684</a> ];
<a href=#685 id=685 data-nosnippet>685</a> <span class="kw">let </span>schema = Arc::new(Schema::new(column_schemas));
<a href=#686 id=686 data-nosnippet>686</a> <span class="kw">let </span>columns: Vec&lt;VectorRef&gt; = <span class="macro">vec!</span>[
<a href=#687 id=687 data-nosnippet>687</a> Arc::new(UInt32Vector::from_slice(<span class="macro">vec!</span>[<span class="number">1</span>, <span class="number">2</span>, <span class="number">3</span>, <span class="number">4</span>])),
<a href=#688 id=688 data-nosnippet>688</a> Arc::new(StringVector::from(<span class="macro">vec!</span>[
<a href=#689 id=689 data-nosnippet>689</a> <span class="prelude-val">None</span>,
<a href=#690 id=690 data-nosnippet>690</a> <span class="prelude-val">Some</span>(<span class="string">"hello"</span>),
<a href=#691 id=691 data-nosnippet>691</a> <span class="prelude-val">Some</span>(<span class="string">"greptime"</span>),
<a href=#692 id=692 data-nosnippet>692</a> <span class="prelude-val">None</span>,
<a href=#693 id=693 data-nosnippet>693</a> ])),
<a href=#694 id=694 data-nosnippet>694</a> ];
<a href=#695 id=695 data-nosnippet>695</a> <span class="kw">let </span>recordbatch = RecordBatch::new(schema, columns).unwrap();
<a href=#696 id=696 data-nosnippet>696</a> <span class="kw">let </span>recordbatch = recordbatch.slice(<span class="number">1</span>, <span class="number">2</span>).expect(<span class="string">"recordbatch slice"</span>);
<a href=#697 id=697 data-nosnippet>697</a>
<a href=#698 id=698 data-nosnippet>698</a> <span class="kw">let </span>expected = <span class="kw-2">&amp;</span>UInt32Array::from_iter_values([<span class="number">2u32</span>, <span class="number">3</span>]);
<a href=#699 id=699 data-nosnippet>699</a> <span class="kw">let </span>array = recordbatch.column(<span class="number">0</span>);
<a href=#700 id=700 data-nosnippet>700</a> <span class="kw">let </span>actual = array.as_primitive::&lt;UInt32Type&gt;();
<a href=#701 id=701 data-nosnippet>701</a> <span class="macro">assert_eq!</span>(expected, actual);
<a href=#702 id=702 data-nosnippet>702</a>
<a href=#703 id=703 data-nosnippet>703</a> <span class="kw">let </span>expected = <span class="kw-2">&amp;</span>StringArray::from(<span class="macro">vec!</span>[<span class="string">"hello"</span>, <span class="string">"greptime"</span>]);
<a href=#704 id=704 data-nosnippet>704</a> <span class="kw">let </span>array = recordbatch.column(<span class="number">1</span>);
<a href=#705 id=705 data-nosnippet>705</a> <span class="kw">let </span>actual = array.as_string::&lt;i32&gt;();
<a href=#706 id=706 data-nosnippet>706</a> <span class="macro">assert_eq!</span>(expected, actual);
<a href=#707 id=707 data-nosnippet>707</a>
<a href=#708 id=708 data-nosnippet>708</a> <span class="macro">assert!</span>(recordbatch.slice(<span class="number">1</span>, <span class="number">5</span>).is_err());
<a href=#709 id=709 data-nosnippet>709</a> }
<a href=#710 id=710 data-nosnippet>710</a>
<a href=#711 id=711 data-nosnippet>711</a> <span class="attr">#[test]
<a href=#712 id=712 data-nosnippet>712</a> </span><span class="kw">fn </span>test_merge_record_batch() {
<a href=#713 id=713 data-nosnippet>713</a> <span class="kw">let </span>column_schemas = <span class="macro">vec!</span>[
<a href=#714 id=714 data-nosnippet>714</a> ColumnSchema::new(<span class="string">"numbers"</span>, ConcreteDataType::uint32_datatype(), <span class="bool-val">false</span>),
<a href=#715 id=715 data-nosnippet>715</a> ColumnSchema::new(<span class="string">"strings"</span>, ConcreteDataType::string_datatype(), <span class="bool-val">true</span>),
<a href=#716 id=716 data-nosnippet>716</a> ];
<a href=#717 id=717 data-nosnippet>717</a> <span class="kw">let </span>schema = Arc::new(Schema::new(column_schemas));
<a href=#718 id=718 data-nosnippet>718</a> <span class="kw">let </span>columns: Vec&lt;VectorRef&gt; = <span class="macro">vec!</span>[
<a href=#719 id=719 data-nosnippet>719</a> Arc::new(UInt32Vector::from_slice(<span class="macro">vec!</span>[<span class="number">1</span>, <span class="number">2</span>, <span class="number">3</span>, <span class="number">4</span>])),
<a href=#720 id=720 data-nosnippet>720</a> Arc::new(StringVector::from(<span class="macro">vec!</span>[
<a href=#721 id=721 data-nosnippet>721</a> <span class="prelude-val">None</span>,
<a href=#722 id=722 data-nosnippet>722</a> <span class="prelude-val">Some</span>(<span class="string">"hello"</span>),
<a href=#723 id=723 data-nosnippet>723</a> <span class="prelude-val">Some</span>(<span class="string">"greptime"</span>),
<a href=#724 id=724 data-nosnippet>724</a> <span class="prelude-val">None</span>,
<a href=#725 id=725 data-nosnippet>725</a> ])),
<a href=#726 id=726 data-nosnippet>726</a> ];
<a href=#727 id=727 data-nosnippet>727</a> <span class="kw">let </span>recordbatch = RecordBatch::new(schema.clone(), columns).unwrap();
<a href=#728 id=728 data-nosnippet>728</a>
<a href=#729 id=729 data-nosnippet>729</a> <span class="kw">let </span>columns: Vec&lt;VectorRef&gt; = <span class="macro">vec!</span>[
<a href=#730 id=730 data-nosnippet>730</a> Arc::new(UInt32Vector::from_slice(<span class="macro">vec!</span>[<span class="number">1</span>, <span class="number">2</span>, <span class="number">3</span>, <span class="number">4</span>])),
<a href=#731 id=731 data-nosnippet>731</a> Arc::new(StringVector::from(<span class="macro">vec!</span>[
<a href=#732 id=732 data-nosnippet>732</a> <span class="prelude-val">None</span>,
<a href=#733 id=733 data-nosnippet>733</a> <span class="prelude-val">Some</span>(<span class="string">"hello"</span>),
<a href=#734 id=734 data-nosnippet>734</a> <span class="prelude-val">Some</span>(<span class="string">"greptime"</span>),
<a href=#735 id=735 data-nosnippet>735</a> <span class="prelude-val">None</span>,
<a href=#736 id=736 data-nosnippet>736</a> ])),
<a href=#737 id=737 data-nosnippet>737</a> ];
<a href=#738 id=738 data-nosnippet>738</a> <span class="kw">let </span>recordbatch2 = RecordBatch::new(schema.clone(), columns).unwrap();
<a href=#739 id=739 data-nosnippet>739</a>
<a href=#740 id=740 data-nosnippet>740</a> <span class="kw">let </span>merged = merge_record_batches(schema.clone(), <span class="kw-2">&amp;</span>[recordbatch, recordbatch2])
<a href=#741 id=741 data-nosnippet>741</a> .expect(<span class="string">"merge recordbatch"</span>);
<a href=#742 id=742 data-nosnippet>742</a> <span class="macro">assert_eq!</span>(merged.num_rows(), <span class="number">8</span>);
<a href=#743 id=743 data-nosnippet>743</a> }
<a href=#744 id=744 data-nosnippet>744</a>}</code></pre></div></section></main></body></html>

213
src/common_recordbatch/util.rs.html Normal file
View File

@@ -0,0 +1,213 @@
<!DOCTYPE html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1.0"><meta name="generator" content="rustdoc"><meta name="description" content="Source of the Rust file `src/common/recordbatch/src/util.rs`."><title>util.rs - source</title><script>if(window.location.protocol!=="file:")document.head.insertAdjacentHTML("beforeend","SourceSerif4-Regular-6b053e98.ttf.woff2,FiraSans-Italic-81dc35de.woff2,FiraSans-Regular-0fe48ade.woff2,FiraSans-MediumItalic-ccf7e434.woff2,FiraSans-Medium-e1aa3f0a.woff2,SourceCodePro-Regular-8badfe75.ttf.woff2,SourceCodePro-Semibold-aa29a496.ttf.woff2".split(",").map(f=>`<link rel="preload" as="font" type="font/woff2"href="../../static.files/${f}">`).join(""))</script><link rel="stylesheet" href="../../static.files/normalize-9960930a.css"><link rel="stylesheet" href="../../static.files/rustdoc-e56847b5.css"><meta name="rustdoc-vars" data-root-path="../../" data-static-root-path="../../static.files/" data-current-crate="common_recordbatch" data-themes="" data-resource-suffix="" data-rustdoc-version="1.92.0-nightly (fa3155a64 2025-09-30)" data-channel="nightly" data-search-js="search-e256b49e.js" data-stringdex-js="stringdex-828709d0.js" data-settings-js="settings-c38705f0.js" ><script src="../../static.files/storage-e2aeef58.js"></script><script defer src="../../static.files/src-script-813739b1.js"></script><script defer src="../../src-files.js"></script><script defer src="../../static.files/main-ce535bd0.js"></script><noscript><link rel="stylesheet" href="../../static.files/noscript-263c88ec.css"></noscript><link rel="alternate icon" type="image/png" href="../../static.files/favicon-32x32-eab170b8.png"><link rel="icon" type="image/svg+xml" href="../../static.files/favicon-044be391.svg"></head><body class="rustdoc src"><!--[if lte IE 11]><div class="warning">This old browser is unsupported and will most likely display funky things.</div><![endif]--><nav class="sidebar"><div class="src-sidebar-title"><h2>Files</h2></div></nav><div class="sidebar-resizer" title="Drag to resize sidebar"></div><main><section id="main-content" class="content"><div class="main-heading"><h1><div class="sub-heading">common_recordbatch/</div>util.rs</h1><rustdoc-toolbar></rustdoc-toolbar></div><div class="example-wrap digits-3"><pre class="rust"><code><a href=#1 id=1 data-nosnippet>1</a><span class="comment">// Copyright 2023 Greptime Team
<a href=#2 id=2 data-nosnippet>2</a>//
<a href=#3 id=3 data-nosnippet>3</a>// Licensed under the Apache License, Version 2.0 (the "License");
<a href=#4 id=4 data-nosnippet>4</a>// you may not use this file except in compliance with the License.
<a href=#5 id=5 data-nosnippet>5</a>// You may obtain a copy of the License at
<a href=#6 id=6 data-nosnippet>6</a>//
<a href=#7 id=7 data-nosnippet>7</a>// http://www.apache.org/licenses/LICENSE-2.0
<a href=#8 id=8 data-nosnippet>8</a>//
<a href=#9 id=9 data-nosnippet>9</a>// Unless required by applicable law or agreed to in writing, software
<a href=#10 id=10 data-nosnippet>10</a>// distributed under the License is distributed on an "AS IS" BASIS,
<a href=#11 id=11 data-nosnippet>11</a>// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
<a href=#12 id=12 data-nosnippet>12</a>// See the License for the specific language governing permissions and
<a href=#13 id=13 data-nosnippet>13</a>// limitations under the License.
<a href=#14 id=14 data-nosnippet>14</a>
<a href=#15 id=15 data-nosnippet>15</a></span><span class="kw">use </span>std::pin::Pin;
<a href=#16 id=16 data-nosnippet>16</a><span class="kw">use </span>std::sync::Arc;
<a href=#17 id=17 data-nosnippet>17</a><span class="kw">use </span>std::task::{Context, Poll};
<a href=#18 id=18 data-nosnippet>18</a>
<a href=#19 id=19 data-nosnippet>19</a><span class="kw">use </span>arc_swap::ArcSwapOption;
<a href=#20 id=20 data-nosnippet>20</a><span class="kw">use </span>datatypes::schema::SchemaRef;
<a href=#21 id=21 data-nosnippet>21</a><span class="kw">use </span>futures::{Stream, StreamExt, TryStreamExt};
<a href=#22 id=22 data-nosnippet>22</a><span class="kw">use </span>snafu::ensure;
<a href=#23 id=23 data-nosnippet>23</a>
<a href=#24 id=24 data-nosnippet>24</a><span class="kw">use </span><span class="kw">crate</span>::adapter::RecordBatchMetrics;
<a href=#25 id=25 data-nosnippet>25</a><span class="kw">use </span><span class="kw">crate</span>::error::{EmptyStreamSnafu, <span class="prelude-ty">Result</span>, SchemaNotMatchSnafu};
<a href=#26 id=26 data-nosnippet>26</a><span class="kw">use crate</span>::{
<a href=#27 id=27 data-nosnippet>27</a> OrderOption, RecordBatch, RecordBatchStream, RecordBatches, SendableRecordBatchStream,
<a href=#28 id=28 data-nosnippet>28</a>};
<a href=#29 id=29 data-nosnippet>29</a>
<a href=#30 id=30 data-nosnippet>30</a><span class="doccomment">/// Collect all the items from the stream into a vector of [`RecordBatch`].
<a href=#31 id=31 data-nosnippet>31</a></span><span class="kw">pub async fn </span>collect(stream: SendableRecordBatchStream) -&gt; <span class="prelude-ty">Result</span>&lt;Vec&lt;RecordBatch&gt;&gt; {
<a href=#32 id=32 data-nosnippet>32</a> stream.try_collect::&lt;Vec&lt;<span class="kw">_</span>&gt;&gt;().<span class="kw">await
<a href=#33 id=33 data-nosnippet>33</a></span>}
<a href=#34 id=34 data-nosnippet>34</a>
<a href=#35 id=35 data-nosnippet>35</a><span class="doccomment">/// Collect all the items from the stream into [RecordBatches].
<a href=#36 id=36 data-nosnippet>36</a></span><span class="kw">pub async fn </span>collect_batches(stream: SendableRecordBatchStream) -&gt; <span class="prelude-ty">Result</span>&lt;RecordBatches&gt; {
<a href=#37 id=37 data-nosnippet>37</a> <span class="kw">let </span>schema = stream.schema();
<a href=#38 id=38 data-nosnippet>38</a> <span class="kw">let </span>batches = stream.try_collect::&lt;Vec&lt;<span class="kw">_</span>&gt;&gt;().<span class="kw">await</span><span class="question-mark">?</span>;
<a href=#39 id=39 data-nosnippet>39</a> RecordBatches::try_new(schema, batches)
<a href=#40 id=40 data-nosnippet>40</a>}
<a href=#41 id=41 data-nosnippet>41</a>
<a href=#42 id=42 data-nosnippet>42</a><span class="doccomment">/// A stream that chains multiple streams into a single stream.
<a href=#43 id=43 data-nosnippet>43</a></span><span class="kw">pub struct </span>ChainedRecordBatchStream {
<a href=#44 id=44 data-nosnippet>44</a> inputs: Vec&lt;SendableRecordBatchStream&gt;,
<a href=#45 id=45 data-nosnippet>45</a> curr_index: usize,
<a href=#46 id=46 data-nosnippet>46</a> schema: SchemaRef,
<a href=#47 id=47 data-nosnippet>47</a> metrics: Arc&lt;ArcSwapOption&lt;RecordBatchMetrics&gt;&gt;,
<a href=#48 id=48 data-nosnippet>48</a>}
<a href=#49 id=49 data-nosnippet>49</a>
<a href=#50 id=50 data-nosnippet>50</a><span class="kw">impl </span>ChainedRecordBatchStream {
<a href=#51 id=51 data-nosnippet>51</a> <span class="kw">pub fn </span>new(inputs: Vec&lt;SendableRecordBatchStream&gt;) -&gt; <span class="prelude-ty">Result</span>&lt;<span class="self">Self</span>&gt; {
<a href=#52 id=52 data-nosnippet>52</a> <span class="comment">// check length
<a href=#53 id=53 data-nosnippet>53</a> </span><span class="macro">ensure!</span>(!inputs.is_empty(), EmptyStreamSnafu);
<a href=#54 id=54 data-nosnippet>54</a>
<a href=#55 id=55 data-nosnippet>55</a> <span class="comment">// check schema
<a href=#56 id=56 data-nosnippet>56</a> </span><span class="kw">let </span>first_schema = inputs[<span class="number">0</span>].schema();
<a href=#57 id=57 data-nosnippet>57</a> <span class="kw">for </span>input <span class="kw">in </span>inputs.iter().skip(<span class="number">1</span>) {
<a href=#58 id=58 data-nosnippet>58</a> <span class="kw">let </span>schema = input.schema();
<a href=#59 id=59 data-nosnippet>59</a> <span class="macro">ensure!</span>(
<a href=#60 id=60 data-nosnippet>60</a> first_schema == schema,
<a href=#61 id=61 data-nosnippet>61</a> SchemaNotMatchSnafu {
<a href=#62 id=62 data-nosnippet>62</a> left: first_schema,
<a href=#63 id=63 data-nosnippet>63</a> right: schema
<a href=#64 id=64 data-nosnippet>64</a> }
<a href=#65 id=65 data-nosnippet>65</a> );
<a href=#66 id=66 data-nosnippet>66</a> }
<a href=#67 id=67 data-nosnippet>67</a>
<a href=#68 id=68 data-nosnippet>68</a> <span class="prelude-val">Ok</span>(<span class="self">Self </span>{
<a href=#69 id=69 data-nosnippet>69</a> inputs,
<a href=#70 id=70 data-nosnippet>70</a> curr_index: <span class="number">0</span>,
<a href=#71 id=71 data-nosnippet>71</a> schema: first_schema,
<a href=#72 id=72 data-nosnippet>72</a> metrics: Default::default(),
<a href=#73 id=73 data-nosnippet>73</a> })
<a href=#74 id=74 data-nosnippet>74</a> }
<a href=#75 id=75 data-nosnippet>75</a>
<a href=#76 id=76 data-nosnippet>76</a> <span class="kw">fn </span>sequence_poll(
<a href=#77 id=77 data-nosnippet>77</a> <span class="kw-2">mut </span><span class="self">self</span>: Pin&lt;<span class="kw-2">&amp;mut </span><span class="self">Self</span>&gt;,
<a href=#78 id=78 data-nosnippet>78</a> ctx: <span class="kw-2">&amp;mut </span>Context&lt;<span class="lifetime">'_</span>&gt;,
<a href=#79 id=79 data-nosnippet>79</a> ) -&gt; Poll&lt;<span class="prelude-ty">Option</span>&lt;<span class="prelude-ty">Result</span>&lt;RecordBatch&gt;&gt;&gt; {
<a href=#80 id=80 data-nosnippet>80</a> <span class="kw">if </span><span class="self">self</span>.curr_index &gt;= <span class="self">self</span>.inputs.len() {
<a href=#81 id=81 data-nosnippet>81</a> <span class="kw">return </span>Poll::Ready(<span class="prelude-val">None</span>);
<a href=#82 id=82 data-nosnippet>82</a> }
<a href=#83 id=83 data-nosnippet>83</a>
<a href=#84 id=84 data-nosnippet>84</a> <span class="kw">let </span>curr_index = <span class="self">self</span>.curr_index;
<a href=#85 id=85 data-nosnippet>85</a> <span class="kw">match </span><span class="self">self</span>.inputs[curr_index].poll_next_unpin(ctx) {
<a href=#86 id=86 data-nosnippet>86</a> Poll::Ready(<span class="prelude-val">Some</span>(<span class="prelude-val">Ok</span>(batch))) =&gt; Poll::Ready(<span class="prelude-val">Some</span>(<span class="prelude-val">Ok</span>(batch))),
<a href=#87 id=87 data-nosnippet>87</a> Poll::Ready(<span class="prelude-val">Some</span>(<span class="prelude-val">Err</span>(e))) =&gt; Poll::Ready(<span class="prelude-val">Some</span>(<span class="prelude-val">Err</span>(e))),
<a href=#88 id=88 data-nosnippet>88</a> Poll::Ready(<span class="prelude-val">None</span>) =&gt; {
<a href=#89 id=89 data-nosnippet>89</a> <span class="self">self</span>.curr_index += <span class="number">1</span>;
<a href=#90 id=90 data-nosnippet>90</a> <span class="kw">if </span><span class="self">self</span>.curr_index &lt; <span class="self">self</span>.inputs.len() {
<a href=#91 id=91 data-nosnippet>91</a> <span class="self">self</span>.sequence_poll(ctx)
<a href=#92 id=92 data-nosnippet>92</a> } <span class="kw">else </span>{
<a href=#93 id=93 data-nosnippet>93</a> Poll::Ready(<span class="prelude-val">None</span>)
<a href=#94 id=94 data-nosnippet>94</a> }
<a href=#95 id=95 data-nosnippet>95</a> }
<a href=#96 id=96 data-nosnippet>96</a> Poll::Pending =&gt; Poll::Pending,
<a href=#97 id=97 data-nosnippet>97</a> }
<a href=#98 id=98 data-nosnippet>98</a> }
<a href=#99 id=99 data-nosnippet>99</a>}
<a href=#100 id=100 data-nosnippet>100</a>
<a href=#101 id=101 data-nosnippet>101</a><span class="kw">impl </span>RecordBatchStream <span class="kw">for </span>ChainedRecordBatchStream {
<a href=#102 id=102 data-nosnippet>102</a> <span class="kw">fn </span>name(<span class="kw-2">&amp;</span><span class="self">self</span>) -&gt; <span class="kw-2">&amp;</span>str {
<a href=#103 id=103 data-nosnippet>103</a> <span class="string">"ChainedRecordBatchStream"
<a href=#104 id=104 data-nosnippet>104</a> </span>}
<a href=#105 id=105 data-nosnippet>105</a>
<a href=#106 id=106 data-nosnippet>106</a> <span class="kw">fn </span>schema(<span class="kw-2">&amp;</span><span class="self">self</span>) -&gt; SchemaRef {
<a href=#107 id=107 data-nosnippet>107</a> <span class="self">self</span>.schema.clone()
<a href=#108 id=108 data-nosnippet>108</a> }
<a href=#109 id=109 data-nosnippet>109</a>
<a href=#110 id=110 data-nosnippet>110</a> <span class="kw">fn </span>output_ordering(<span class="kw-2">&amp;</span><span class="self">self</span>) -&gt; <span class="prelude-ty">Option</span>&lt;<span class="kw-2">&amp;</span>[OrderOption]&gt; {
<a href=#111 id=111 data-nosnippet>111</a> <span class="prelude-val">None
<a href=#112 id=112 data-nosnippet>112</a> </span>}
<a href=#113 id=113 data-nosnippet>113</a>
<a href=#114 id=114 data-nosnippet>114</a> <span class="kw">fn </span>metrics(<span class="kw-2">&amp;</span><span class="self">self</span>) -&gt; <span class="prelude-ty">Option</span>&lt;RecordBatchMetrics&gt; {
<a href=#115 id=115 data-nosnippet>115</a> <span class="self">self</span>.metrics.load().as_ref().map(|m| m.as_ref().clone())
<a href=#116 id=116 data-nosnippet>116</a> }
<a href=#117 id=117 data-nosnippet>117</a>}
<a href=#118 id=118 data-nosnippet>118</a>
<a href=#119 id=119 data-nosnippet>119</a><span class="kw">impl </span>Stream <span class="kw">for </span>ChainedRecordBatchStream {
<a href=#120 id=120 data-nosnippet>120</a> <span class="kw">type </span>Item = <span class="prelude-ty">Result</span>&lt;RecordBatch&gt;;
<a href=#121 id=121 data-nosnippet>121</a>
<a href=#122 id=122 data-nosnippet>122</a> <span class="kw">fn </span>poll_next(<span class="self">self</span>: Pin&lt;<span class="kw-2">&amp;mut </span><span class="self">Self</span>&gt;, ctx: <span class="kw-2">&amp;mut </span>Context&lt;<span class="lifetime">'_</span>&gt;) -&gt; Poll&lt;<span class="prelude-ty">Option</span>&lt;<span class="self">Self</span>::Item&gt;&gt; {
<a href=#123 id=123 data-nosnippet>123</a> <span class="self">self</span>.sequence_poll(ctx)
<a href=#124 id=124 data-nosnippet>124</a> }
<a href=#125 id=125 data-nosnippet>125</a>}
<a href=#126 id=126 data-nosnippet>126</a>
<a href=#127 id=127 data-nosnippet>127</a><span class="attr">#[cfg(test)]
<a href=#128 id=128 data-nosnippet>128</a></span><span class="kw">mod </span>tests {
<a href=#129 id=129 data-nosnippet>129</a> <span class="kw">use </span>std::pin::Pin;
<a href=#130 id=130 data-nosnippet>130</a> <span class="kw">use </span>std::sync::Arc;
<a href=#131 id=131 data-nosnippet>131</a>
<a href=#132 id=132 data-nosnippet>132</a> <span class="kw">use </span>datatypes::prelude::<span class="kw-2">*</span>;
<a href=#133 id=133 data-nosnippet>133</a> <span class="kw">use </span>datatypes::schema::{ColumnSchema, Schema, SchemaRef};
<a href=#134 id=134 data-nosnippet>134</a> <span class="kw">use </span>datatypes::vectors::UInt32Vector;
<a href=#135 id=135 data-nosnippet>135</a> <span class="kw">use </span>futures::Stream;
<a href=#136 id=136 data-nosnippet>136</a> <span class="kw">use </span>futures::task::{Context, Poll};
<a href=#137 id=137 data-nosnippet>137</a>
<a href=#138 id=138 data-nosnippet>138</a> <span class="kw">use super</span>::<span class="kw-2">*</span>;
<a href=#139 id=139 data-nosnippet>139</a> <span class="kw">use </span><span class="kw">crate</span>::adapter::RecordBatchMetrics;
<a href=#140 id=140 data-nosnippet>140</a> <span class="kw">use crate</span>::{OrderOption, RecordBatchStream};
<a href=#141 id=141 data-nosnippet>141</a>
<a href=#142 id=142 data-nosnippet>142</a> <span class="kw">struct </span>MockRecordBatchStream {
<a href=#143 id=143 data-nosnippet>143</a> batch: <span class="prelude-ty">Option</span>&lt;RecordBatch&gt;,
<a href=#144 id=144 data-nosnippet>144</a> schema: SchemaRef,
<a href=#145 id=145 data-nosnippet>145</a> }
<a href=#146 id=146 data-nosnippet>146</a>
<a href=#147 id=147 data-nosnippet>147</a> <span class="kw">impl </span>RecordBatchStream <span class="kw">for </span>MockRecordBatchStream {
<a href=#148 id=148 data-nosnippet>148</a> <span class="kw">fn </span>schema(<span class="kw-2">&amp;</span><span class="self">self</span>) -&gt; SchemaRef {
<a href=#149 id=149 data-nosnippet>149</a> <span class="self">self</span>.schema.clone()
<a href=#150 id=150 data-nosnippet>150</a> }
<a href=#151 id=151 data-nosnippet>151</a>
<a href=#152 id=152 data-nosnippet>152</a> <span class="kw">fn </span>output_ordering(<span class="kw-2">&amp;</span><span class="self">self</span>) -&gt; <span class="prelude-ty">Option</span>&lt;<span class="kw-2">&amp;</span>[OrderOption]&gt; {
<a href=#153 id=153 data-nosnippet>153</a> <span class="prelude-val">None
<a href=#154 id=154 data-nosnippet>154</a> </span>}
<a href=#155 id=155 data-nosnippet>155</a>
<a href=#156 id=156 data-nosnippet>156</a> <span class="kw">fn </span>metrics(<span class="kw-2">&amp;</span><span class="self">self</span>) -&gt; <span class="prelude-ty">Option</span>&lt;RecordBatchMetrics&gt; {
<a href=#157 id=157 data-nosnippet>157</a> <span class="prelude-val">None
<a href=#158 id=158 data-nosnippet>158</a> </span>}
<a href=#159 id=159 data-nosnippet>159</a> }
<a href=#160 id=160 data-nosnippet>160</a>
<a href=#161 id=161 data-nosnippet>161</a> <span class="kw">impl </span>Stream <span class="kw">for </span>MockRecordBatchStream {
<a href=#162 id=162 data-nosnippet>162</a> <span class="kw">type </span>Item = <span class="prelude-ty">Result</span>&lt;RecordBatch&gt;;
<a href=#163 id=163 data-nosnippet>163</a>
<a href=#164 id=164 data-nosnippet>164</a> <span class="kw">fn </span>poll_next(<span class="kw-2">mut </span><span class="self">self</span>: Pin&lt;<span class="kw-2">&amp;mut </span><span class="self">Self</span>&gt;, _cx: <span class="kw-2">&amp;mut </span>Context&lt;<span class="lifetime">'_</span>&gt;) -&gt; Poll&lt;<span class="prelude-ty">Option</span>&lt;<span class="self">Self</span>::Item&gt;&gt; {
<a href=#165 id=165 data-nosnippet>165</a> <span class="kw">let </span>batch = <span class="self">self</span>.batch.take();
<a href=#166 id=166 data-nosnippet>166</a>
<a href=#167 id=167 data-nosnippet>167</a> <span class="kw">if let </span><span class="prelude-val">Some</span>(batch) = batch {
<a href=#168 id=168 data-nosnippet>168</a> Poll::Ready(<span class="prelude-val">Some</span>(<span class="prelude-val">Ok</span>(batch)))
<a href=#169 id=169 data-nosnippet>169</a> } <span class="kw">else </span>{
<a href=#170 id=170 data-nosnippet>170</a> Poll::Ready(<span class="prelude-val">None</span>)
<a href=#171 id=171 data-nosnippet>171</a> }
<a href=#172 id=172 data-nosnippet>172</a> }
<a href=#173 id=173 data-nosnippet>173</a> }
<a href=#174 id=174 data-nosnippet>174</a>
<a href=#175 id=175 data-nosnippet>175</a> <span class="attr">#[tokio::test]
<a href=#176 id=176 data-nosnippet>176</a> </span><span class="kw">async fn </span>test_collect() {
<a href=#177 id=177 data-nosnippet>177</a> <span class="kw">let </span>column_schemas = <span class="macro">vec!</span>[ColumnSchema::new(
<a href=#178 id=178 data-nosnippet>178</a> <span class="string">"number"</span>,
<a href=#179 id=179 data-nosnippet>179</a> ConcreteDataType::uint32_datatype(),
<a href=#180 id=180 data-nosnippet>180</a> <span class="bool-val">false</span>,
<a href=#181 id=181 data-nosnippet>181</a> )];
<a href=#182 id=182 data-nosnippet>182</a>
<a href=#183 id=183 data-nosnippet>183</a> <span class="kw">let </span>schema = Arc::new(Schema::try_new(column_schemas).unwrap());
<a href=#184 id=184 data-nosnippet>184</a>
<a href=#185 id=185 data-nosnippet>185</a> <span class="kw">let </span>stream = MockRecordBatchStream {
<a href=#186 id=186 data-nosnippet>186</a> schema: schema.clone(),
<a href=#187 id=187 data-nosnippet>187</a> batch: <span class="prelude-val">None</span>,
<a href=#188 id=188 data-nosnippet>188</a> };
<a href=#189 id=189 data-nosnippet>189</a>
<a href=#190 id=190 data-nosnippet>190</a> <span class="kw">let </span>batches = collect(Box::pin(stream)).<span class="kw">await</span>.unwrap();
<a href=#191 id=191 data-nosnippet>191</a> <span class="macro">assert_eq!</span>(<span class="number">0</span>, batches.len());
<a href=#192 id=192 data-nosnippet>192</a>
<a href=#193 id=193 data-nosnippet>193</a> <span class="kw">let </span>numbers: Vec&lt;u32&gt; = (<span class="number">0</span>..<span class="number">10</span>).collect();
<a href=#194 id=194 data-nosnippet>194</a> <span class="kw">let </span>columns = [Arc::new(UInt32Vector::from_vec(numbers)) <span class="kw">as _</span>];
<a href=#195 id=195 data-nosnippet>195</a> <span class="kw">let </span>batch = RecordBatch::new(schema.clone(), columns).unwrap();
<a href=#196 id=196 data-nosnippet>196</a>
<a href=#197 id=197 data-nosnippet>197</a> <span class="kw">let </span>stream = MockRecordBatchStream {
<a href=#198 id=198 data-nosnippet>198</a> schema: schema.clone(),
<a href=#199 id=199 data-nosnippet>199</a> batch: <span class="prelude-val">Some</span>(batch.clone()),
<a href=#200 id=200 data-nosnippet>200</a> };
<a href=#201 id=201 data-nosnippet>201</a> <span class="kw">let </span>batches = collect(Box::pin(stream)).<span class="kw">await</span>.unwrap();
<a href=#202 id=202 data-nosnippet>202</a> <span class="macro">assert_eq!</span>(<span class="number">1</span>, batches.len());
<a href=#203 id=203 data-nosnippet>203</a> <span class="macro">assert_eq!</span>(batch, batches[<span class="number">0</span>]);
<a href=#204 id=204 data-nosnippet>204</a>
<a href=#205 id=205 data-nosnippet>205</a> <span class="kw">let </span>stream = MockRecordBatchStream {
<a href=#206 id=206 data-nosnippet>206</a> schema: schema.clone(),
<a href=#207 id=207 data-nosnippet>207</a> batch: <span class="prelude-val">Some</span>(batch.clone()),
<a href=#208 id=208 data-nosnippet>208</a> };
<a href=#209 id=209 data-nosnippet>209</a> <span class="kw">let </span>batches = collect_batches(Box::pin(stream)).<span class="kw">await</span>.unwrap();
<a href=#210 id=210 data-nosnippet>210</a> <span class="kw">let </span>expect_batches = RecordBatches::try_new(schema.clone(), <span class="macro">vec!</span>[batch]).unwrap();
<a href=#211 id=211 data-nosnippet>211</a> <span class="macro">assert_eq!</span>(expect_batches, batches);
<a href=#212 id=212 data-nosnippet>212</a> }
<a href=#213 id=213 data-nosnippet>213</a>}</code></pre></div></section></main></body></html>