Deploying to gh-pages from @ GreptimeTeam/greptimedb@97913b4b3a 🚀

This commit is contained in:
github-merge-queue[bot]
2026-10-07 08:03:01 +00:00
commit ff419bc025
25679 changed files with 2969188 additions and 0 deletions
+19
View File
@@ -0,0 +1,19 @@
<!DOCTYPE html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1.0"><meta name="generator" content="rustdoc"><meta name="description" content="Source of the Rust file `src/file-engine/src/config.rs`."><title>config.rs - source</title><script>if(window.location.protocol!=="file:")document.head.insertAdjacentHTML("beforeend","SourceSerif4-Regular-6b053e98.ttf.woff2,FiraSans-Italic-81dc35de.woff2,FiraSans-Regular-0fe48ade.woff2,FiraSans-MediumItalic-ccf7e434.woff2,FiraSans-Medium-e1aa3f0a.woff2,SourceCodePro-Regular-8badfe75.ttf.woff2,SourceCodePro-Semibold-aa29a496.ttf.woff2".split(",").map(f=>`<link rel="preload" as="font" type="font/woff2"href="../../static.files/${f}">`).join(""))</script><link rel="stylesheet" href="../../static.files/normalize-9960930a.css"><link rel="stylesheet" href="../../static.files/rustdoc-17e0aaed.css"><meta name="rustdoc-vars" data-root-path="../../" data-static-root-path="../../static.files/" data-current-crate="file_engine" data-themes="" data-resource-suffix="" data-rustdoc-version="1.96.1 (31fca3adb 2026-06-26)" data-channel="1.96.1" data-search-js="search-b5634cc7.js" data-stringdex-js="stringdex-2da4960a.js" data-settings-js="settings-170eb4bf.js" ><script src="../../static.files/storage-41dd4d93.js"></script><script defer src="../../static.files/src-script-813739b1.js"></script><script defer src="../../src-files.js"></script><script defer src="../../static.files/main-5013f961.js"></script><noscript><link rel="stylesheet" href="../../static.files/noscript-f7c3ffd8.css"></noscript><link rel="alternate icon" type="image/png" href="../../static.files/favicon-32x32-eab170b8.png"><link rel="icon" type="image/svg+xml" href="../../static.files/favicon-044be391.svg"></head><body class="rustdoc src"><a class="skip-main-content" href="#main-content">Skip to main content</a><!--[if lte IE 11]><div class="warning">This old browser is unsupported and will most likely display funky things.</div><![endif]--><nav class="sidebar"><div class="src-sidebar-title"><h2>Files</h2></div></nav><div class="sidebar-resizer" title="Drag to resize sidebar"></div><main><section id="main-content" class="content" tabindex="-1"><div class="main-heading"><h1><div class="sub-heading">file_engine/</div>config.rs</h1><rustdoc-toolbar></rustdoc-toolbar></div><div class="example-wrap digits-2"><pre class="rust"><code><a href=#1 id=1 data-nosnippet>1</a><span class="comment">// Copyright 2023 Greptime Team
<a href=#2 id=2 data-nosnippet>2</a>//
<a href=#3 id=3 data-nosnippet>3</a>// Licensed under the Apache License, Version 2.0 (the "License");
<a href=#4 id=4 data-nosnippet>4</a>// you may not use this file except in compliance with the License.
<a href=#5 id=5 data-nosnippet>5</a>// You may obtain a copy of the License at
<a href=#6 id=6 data-nosnippet>6</a>//
<a href=#7 id=7 data-nosnippet>7</a>// http://www.apache.org/licenses/LICENSE-2.0
<a href=#8 id=8 data-nosnippet>8</a>//
<a href=#9 id=9 data-nosnippet>9</a>// Unless required by applicable law or agreed to in writing, software
<a href=#10 id=10 data-nosnippet>10</a>// distributed under the License is distributed on an "AS IS" BASIS,
<a href=#11 id=11 data-nosnippet>11</a>// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
<a href=#12 id=12 data-nosnippet>12</a>// See the License for the specific language governing permissions and
<a href=#13 id=13 data-nosnippet>13</a>// limitations under the License.
<a href=#14 id=14 data-nosnippet>14</a>
<a href=#15 id=15 data-nosnippet>15</a></span><span class="kw">use </span>serde::{Deserialize, Serialize};
<a href=#16 id=16 data-nosnippet>16</a>
<a href=#17 id=17 data-nosnippet>17</a><span class="attr">#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)]
<a href=#18 id=18 data-nosnippet>18</a></span><span class="kw">pub struct </span>EngineConfig {}
</code></pre></div></section></main></body></html>
+437
View File
@@ -0,0 +1,437 @@
<!DOCTYPE html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1.0"><meta name="generator" content="rustdoc"><meta name="description" content="Source of the Rust file `src/file-engine/src/engine.rs`."><title>engine.rs - source</title><script>if(window.location.protocol!=="file:")document.head.insertAdjacentHTML("beforeend","SourceSerif4-Regular-6b053e98.ttf.woff2,FiraSans-Italic-81dc35de.woff2,FiraSans-Regular-0fe48ade.woff2,FiraSans-MediumItalic-ccf7e434.woff2,FiraSans-Medium-e1aa3f0a.woff2,SourceCodePro-Regular-8badfe75.ttf.woff2,SourceCodePro-Semibold-aa29a496.ttf.woff2".split(",").map(f=>`<link rel="preload" as="font" type="font/woff2"href="../../static.files/${f}">`).join(""))</script><link rel="stylesheet" href="../../static.files/normalize-9960930a.css"><link rel="stylesheet" href="../../static.files/rustdoc-17e0aaed.css"><meta name="rustdoc-vars" data-root-path="../../" data-static-root-path="../../static.files/" data-current-crate="file_engine" data-themes="" data-resource-suffix="" data-rustdoc-version="1.96.1 (31fca3adb 2026-06-26)" data-channel="1.96.1" data-search-js="search-b5634cc7.js" data-stringdex-js="stringdex-2da4960a.js" data-settings-js="settings-170eb4bf.js" ><script src="../../static.files/storage-41dd4d93.js"></script><script defer src="../../static.files/src-script-813739b1.js"></script><script defer src="../../src-files.js"></script><script defer src="../../static.files/main-5013f961.js"></script><noscript><link rel="stylesheet" href="../../static.files/noscript-f7c3ffd8.css"></noscript><link rel="alternate icon" type="image/png" href="../../static.files/favicon-32x32-eab170b8.png"><link rel="icon" type="image/svg+xml" href="../../static.files/favicon-044be391.svg"></head><body class="rustdoc src"><a class="skip-main-content" href="#main-content">Skip to main content</a><!--[if lte IE 11]><div class="warning">This old browser is unsupported and will most likely display funky things.</div><![endif]--><nav class="sidebar"><div class="src-sidebar-title"><h2>Files</h2></div></nav><div class="sidebar-resizer" title="Drag to resize sidebar"></div><main><section id="main-content" class="content" tabindex="-1"><div class="main-heading"><h1><div class="sub-heading">file_engine/</div>engine.rs</h1><rustdoc-toolbar></rustdoc-toolbar></div><div class="example-wrap digits-3"><pre class="rust"><code><a href=#1 id=1 data-nosnippet>1</a><span class="comment">// Copyright 2023 Greptime Team
<a href=#2 id=2 data-nosnippet>2</a>//
<a href=#3 id=3 data-nosnippet>3</a>// Licensed under the Apache License, Version 2.0 (the "License");
<a href=#4 id=4 data-nosnippet>4</a>// you may not use this file except in compliance with the License.
<a href=#5 id=5 data-nosnippet>5</a>// You may obtain a copy of the License at
<a href=#6 id=6 data-nosnippet>6</a>//
<a href=#7 id=7 data-nosnippet>7</a>// http://www.apache.org/licenses/LICENSE-2.0
<a href=#8 id=8 data-nosnippet>8</a>//
<a href=#9 id=9 data-nosnippet>9</a>// Unless required by applicable law or agreed to in writing, software
<a href=#10 id=10 data-nosnippet>10</a>// distributed under the License is distributed on an "AS IS" BASIS,
<a href=#11 id=11 data-nosnippet>11</a>// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
<a href=#12 id=12 data-nosnippet>12</a>// See the License for the specific language governing permissions and
<a href=#13 id=13 data-nosnippet>13</a>// limitations under the License.
<a href=#14 id=14 data-nosnippet>14</a>
<a href=#15 id=15 data-nosnippet>15</a></span><span class="kw">use </span>std::any::Any;
<a href=#16 id=16 data-nosnippet>16</a><span class="kw">use </span>std::collections::HashMap;
<a href=#17 id=17 data-nosnippet>17</a><span class="kw">use </span>std::sync::{Arc, RwLock};
<a href=#18 id=18 data-nosnippet>18</a>
<a href=#19 id=19 data-nosnippet>19</a><span class="kw">use </span>api::region::RegionResponse;
<a href=#20 id=20 data-nosnippet>20</a><span class="kw">use </span>async_trait::async_trait;
<a href=#21 id=21 data-nosnippet>21</a><span class="kw">use </span>common_catalog::consts::FILE_ENGINE;
<a href=#22 id=22 data-nosnippet>22</a><span class="kw">use </span>common_datasource::object_store::LocalFileAccess;
<a href=#23 id=23 data-nosnippet>23</a><span class="kw">use </span>common_error::ext::BoxedError;
<a href=#24 id=24 data-nosnippet>24</a><span class="kw">use </span>common_recordbatch::SendableRecordBatchStream;
<a href=#25 id=25 data-nosnippet>25</a><span class="kw">use </span>common_telemetry::{error, info};
<a href=#26 id=26 data-nosnippet>26</a><span class="kw">use </span>object_store::ObjectStore;
<a href=#27 id=27 data-nosnippet>27</a><span class="kw">use </span>snafu::{OptionExt, ensure};
<a href=#28 id=28 data-nosnippet>28</a><span class="kw">use </span>store_api::metadata::RegionMetadataRef;
<a href=#29 id=29 data-nosnippet>29</a><span class="kw">use </span>store_api::region_engine::{
<a href=#30 id=30 data-nosnippet>30</a> RegionEngine, RegionRole, RegionScannerRef, RegionStatistic, RemapManifestsRequest,
<a href=#31 id=31 data-nosnippet>31</a> RemapManifestsResponse, SetRegionRoleStateResponse, SetRegionRoleStateSuccess,
<a href=#32 id=32 data-nosnippet>32</a> SettableRegionRoleState, SinglePartitionScanner, SyncRegionFromRequest, SyncRegionFromResponse,
<a href=#33 id=33 data-nosnippet>33</a>};
<a href=#34 id=34 data-nosnippet>34</a><span class="kw">use </span>store_api::region_request::{
<a href=#35 id=35 data-nosnippet>35</a> AffectedRows, RegionCloseRequest, RegionCreateRequest, RegionDropRequest, RegionOpenRequest,
<a href=#36 id=36 data-nosnippet>36</a> RegionRequest, RegionRequirements,
<a href=#37 id=37 data-nosnippet>37</a>};
<a href=#38 id=38 data-nosnippet>38</a><span class="kw">use </span>store_api::storage::{RegionId, ScanRequest, SequenceNumber};
<a href=#39 id=39 data-nosnippet>39</a><span class="kw">use </span>tokio::sync::Mutex;
<a href=#40 id=40 data-nosnippet>40</a>
<a href=#41 id=41 data-nosnippet>41</a><span class="kw">use </span><span class="kw">crate</span>::config::EngineConfig;
<a href=#42 id=42 data-nosnippet>42</a><span class="kw">use </span><span class="kw">crate</span>::error::{
<a href=#43 id=43 data-nosnippet>43</a> RegionNotFoundSnafu, <span class="prelude-ty">Result </span><span class="kw">as </span>EngineResult, UnexpectedEngineSnafu, UnsupportedSnafu,
<a href=#44 id=44 data-nosnippet>44</a>};
<a href=#45 id=45 data-nosnippet>45</a><span class="kw">use </span><span class="kw">crate</span>::region::{FileRegion, FileRegionRef};
<a href=#46 id=46 data-nosnippet>46</a>
<a href=#47 id=47 data-nosnippet>47</a><span class="kw">pub struct </span>FileRegionEngine {
<a href=#48 id=48 data-nosnippet>48</a> inner: EngineInnerRef,
<a href=#49 id=49 data-nosnippet>49</a>}
<a href=#50 id=50 data-nosnippet>50</a>
<a href=#51 id=51 data-nosnippet>51</a><span class="kw">impl </span>FileRegionEngine {
<a href=#52 id=52 data-nosnippet>52</a> <span class="kw">pub fn </span>new(
<a href=#53 id=53 data-nosnippet>53</a> _config: EngineConfig,
<a href=#54 id=54 data-nosnippet>54</a> object_store: ObjectStore,
<a href=#55 id=55 data-nosnippet>55</a> local_file_access: LocalFileAccess,
<a href=#56 id=56 data-nosnippet>56</a> ) -&gt; <span class="self">Self </span>{
<a href=#57 id=57 data-nosnippet>57</a> <span class="self">Self </span>{
<a href=#58 id=58 data-nosnippet>58</a> inner: Arc::new(EngineInner::new(object_store, local_file_access)),
<a href=#59 id=59 data-nosnippet>59</a> }
<a href=#60 id=60 data-nosnippet>60</a> }
<a href=#61 id=61 data-nosnippet>61</a>
<a href=#62 id=62 data-nosnippet>62</a> <span class="kw">async fn </span>handle_query(
<a href=#63 id=63 data-nosnippet>63</a> <span class="kw-2">&amp;</span><span class="self">self</span>,
<a href=#64 id=64 data-nosnippet>64</a> region_id: RegionId,
<a href=#65 id=65 data-nosnippet>65</a> request: ScanRequest,
<a href=#66 id=66 data-nosnippet>66</a> ) -&gt; <span class="prelude-ty">Result</span>&lt;SendableRecordBatchStream, BoxedError&gt; {
<a href=#67 id=67 data-nosnippet>67</a> <span class="self">self</span>.inner
<a href=#68 id=68 data-nosnippet>68</a> .get_region(region_id)
<a href=#69 id=69 data-nosnippet>69</a> .<span class="kw">await
<a href=#70 id=70 data-nosnippet>70</a> </span>.context(RegionNotFoundSnafu { region_id })
<a href=#71 id=71 data-nosnippet>71</a> .map_err(BoxedError::new)<span class="question-mark">?
<a href=#72 id=72 data-nosnippet>72</a> </span>.query(request, <span class="kw-2">&amp;</span><span class="self">self</span>.inner.local_file_access)
<a href=#73 id=73 data-nosnippet>73</a> .<span class="kw">await
<a href=#74 id=74 data-nosnippet>74</a> </span>.map_err(BoxedError::new)
<a href=#75 id=75 data-nosnippet>75</a> }
<a href=#76 id=76 data-nosnippet>76</a>}
<a href=#77 id=77 data-nosnippet>77</a>
<a href=#78 id=78 data-nosnippet>78</a><span class="attr">#[async_trait]
<a href=#79 id=79 data-nosnippet>79</a></span><span class="kw">impl </span>RegionEngine <span class="kw">for </span>FileRegionEngine {
<a href=#80 id=80 data-nosnippet>80</a> <span class="kw">fn </span>name(<span class="kw-2">&amp;</span><span class="self">self</span>) -&gt; <span class="kw-2">&amp;</span>str {
<a href=#81 id=81 data-nosnippet>81</a> FILE_ENGINE
<a href=#82 id=82 data-nosnippet>82</a> }
<a href=#83 id=83 data-nosnippet>83</a>
<a href=#84 id=84 data-nosnippet>84</a> <span class="kw">async fn </span>handle_request(
<a href=#85 id=85 data-nosnippet>85</a> <span class="kw-2">&amp;</span><span class="self">self</span>,
<a href=#86 id=86 data-nosnippet>86</a> region_id: RegionId,
<a href=#87 id=87 data-nosnippet>87</a> request: RegionRequest,
<a href=#88 id=88 data-nosnippet>88</a> ) -&gt; <span class="prelude-ty">Result</span>&lt;RegionResponse, BoxedError&gt; {
<a href=#89 id=89 data-nosnippet>89</a> <span class="self">self</span>.inner
<a href=#90 id=90 data-nosnippet>90</a> .handle_request(region_id, request)
<a href=#91 id=91 data-nosnippet>91</a> .<span class="kw">await
<a href=#92 id=92 data-nosnippet>92</a> </span>.map_err(BoxedError::new)
<a href=#93 id=93 data-nosnippet>93</a> }
<a href=#94 id=94 data-nosnippet>94</a>
<a href=#95 id=95 data-nosnippet>95</a> <span class="kw">async fn </span>handle_query(
<a href=#96 id=96 data-nosnippet>96</a> <span class="kw-2">&amp;</span><span class="self">self</span>,
<a href=#97 id=97 data-nosnippet>97</a> region_id: RegionId,
<a href=#98 id=98 data-nosnippet>98</a> request: ScanRequest,
<a href=#99 id=99 data-nosnippet>99</a> ) -&gt; <span class="prelude-ty">Result</span>&lt;RegionScannerRef, BoxedError&gt; {
<a href=#100 id=100 data-nosnippet>100</a> <span class="kw">let </span>stream = <span class="self">self</span>.handle_query(region_id, request).<span class="kw">await</span><span class="question-mark">?</span>;
<a href=#101 id=101 data-nosnippet>101</a> <span class="kw">let </span>metadata = <span class="self">self</span>.get_metadata(region_id).<span class="kw">await</span><span class="question-mark">?</span>;
<a href=#102 id=102 data-nosnippet>102</a> <span class="comment">// We don't support enabling append mode for file engine.
<a href=#103 id=103 data-nosnippet>103</a> </span><span class="kw">let </span>scanner = Box::new(SinglePartitionScanner::new(stream, <span class="bool-val">false</span>, metadata, <span class="prelude-val">None</span>));
<a href=#104 id=104 data-nosnippet>104</a> <span class="prelude-val">Ok</span>(scanner)
<a href=#105 id=105 data-nosnippet>105</a> }
<a href=#106 id=106 data-nosnippet>106</a>
<a href=#107 id=107 data-nosnippet>107</a> <span class="kw">async fn </span>get_metadata(<span class="kw-2">&amp;</span><span class="self">self</span>, region_id: RegionId) -&gt; <span class="prelude-ty">Result</span>&lt;RegionMetadataRef, BoxedError&gt; {
<a href=#108 id=108 data-nosnippet>108</a> <span class="self">self</span>.inner
<a href=#109 id=109 data-nosnippet>109</a> .get_region(region_id)
<a href=#110 id=110 data-nosnippet>110</a> .<span class="kw">await
<a href=#111 id=111 data-nosnippet>111</a> </span>.map(|r| r.metadata())
<a href=#112 id=112 data-nosnippet>112</a> .context(RegionNotFoundSnafu { region_id })
<a href=#113 id=113 data-nosnippet>113</a> .map_err(BoxedError::new)
<a href=#114 id=114 data-nosnippet>114</a> }
<a href=#115 id=115 data-nosnippet>115</a>
<a href=#116 id=116 data-nosnippet>116</a> <span class="kw">async fn </span>stop(<span class="kw-2">&amp;</span><span class="self">self</span>) -&gt; <span class="prelude-ty">Result</span>&lt;(), BoxedError&gt; {
<a href=#117 id=117 data-nosnippet>117</a> <span class="self">self</span>.inner.stop().<span class="kw">await</span>.map_err(BoxedError::new)
<a href=#118 id=118 data-nosnippet>118</a> }
<a href=#119 id=119 data-nosnippet>119</a>
<a href=#120 id=120 data-nosnippet>120</a> <span class="kw">fn </span>region_statistic(<span class="kw-2">&amp;</span><span class="self">self</span>, <span class="kw">_</span>: RegionId) -&gt; <span class="prelude-ty">Option</span>&lt;RegionStatistic&gt; {
<a href=#121 id=121 data-nosnippet>121</a> <span class="prelude-val">None
<a href=#122 id=122 data-nosnippet>122</a> </span>}
<a href=#123 id=123 data-nosnippet>123</a>
<a href=#124 id=124 data-nosnippet>124</a> <span class="kw">async fn </span>get_committed_sequence(<span class="kw-2">&amp;</span><span class="self">self</span>, <span class="kw">_</span>: RegionId) -&gt; <span class="prelude-ty">Result</span>&lt;SequenceNumber, BoxedError&gt; {
<a href=#125 id=125 data-nosnippet>125</a> <span class="prelude-val">Ok</span>(Default::default())
<a href=#126 id=126 data-nosnippet>126</a> }
<a href=#127 id=127 data-nosnippet>127</a>
<a href=#128 id=128 data-nosnippet>128</a> <span class="kw">fn </span>set_region_role(<span class="kw-2">&amp;</span><span class="self">self</span>, region_id: RegionId, role: RegionRole) -&gt; <span class="prelude-ty">Result</span>&lt;(), BoxedError&gt; {
<a href=#129 id=129 data-nosnippet>129</a> <span class="self">self</span>.inner
<a href=#130 id=130 data-nosnippet>130</a> .set_region_role(region_id, role)
<a href=#131 id=131 data-nosnippet>131</a> .map_err(BoxedError::new)
<a href=#132 id=132 data-nosnippet>132</a> }
<a href=#133 id=133 data-nosnippet>133</a>
<a href=#134 id=134 data-nosnippet>134</a> <span class="kw">async fn </span>set_region_role_state_gracefully(
<a href=#135 id=135 data-nosnippet>135</a> <span class="kw-2">&amp;</span><span class="self">self</span>,
<a href=#136 id=136 data-nosnippet>136</a> region_id: RegionId,
<a href=#137 id=137 data-nosnippet>137</a> _region_role_state: SettableRegionRoleState,
<a href=#138 id=138 data-nosnippet>138</a> ) -&gt; <span class="prelude-ty">Result</span>&lt;SetRegionRoleStateResponse, BoxedError&gt; {
<a href=#139 id=139 data-nosnippet>139</a> <span class="kw">let </span>exists = <span class="self">self</span>.inner.get_region(region_id).<span class="kw">await</span>.is_some();
<a href=#140 id=140 data-nosnippet>140</a>
<a href=#141 id=141 data-nosnippet>141</a> <span class="kw">if </span>exists {
<a href=#142 id=142 data-nosnippet>142</a> <span class="prelude-val">Ok</span>(SetRegionRoleStateResponse::success(
<a href=#143 id=143 data-nosnippet>143</a> SetRegionRoleStateSuccess::file(),
<a href=#144 id=144 data-nosnippet>144</a> ))
<a href=#145 id=145 data-nosnippet>145</a> } <span class="kw">else </span>{
<a href=#146 id=146 data-nosnippet>146</a> <span class="prelude-val">Ok</span>(SetRegionRoleStateResponse::NotFound)
<a href=#147 id=147 data-nosnippet>147</a> }
<a href=#148 id=148 data-nosnippet>148</a> }
<a href=#149 id=149 data-nosnippet>149</a>
<a href=#150 id=150 data-nosnippet>150</a> <span class="kw">async fn </span>sync_region(
<a href=#151 id=151 data-nosnippet>151</a> <span class="kw-2">&amp;</span><span class="self">self</span>,
<a href=#152 id=152 data-nosnippet>152</a> _region_id: RegionId,
<a href=#153 id=153 data-nosnippet>153</a> _request: SyncRegionFromRequest,
<a href=#154 id=154 data-nosnippet>154</a> ) -&gt; <span class="prelude-ty">Result</span>&lt;SyncRegionFromResponse, BoxedError&gt; {
<a href=#155 id=155 data-nosnippet>155</a> <span class="comment">// File engine doesn't need to sync region manifest.
<a href=#156 id=156 data-nosnippet>156</a> </span><span class="prelude-val">Ok</span>(SyncRegionFromResponse::NotSupported)
<a href=#157 id=157 data-nosnippet>157</a> }
<a href=#158 id=158 data-nosnippet>158</a>
<a href=#159 id=159 data-nosnippet>159</a> <span class="kw">async fn </span>remap_manifests(
<a href=#160 id=160 data-nosnippet>160</a> <span class="kw-2">&amp;</span><span class="self">self</span>,
<a href=#161 id=161 data-nosnippet>161</a> _request: RemapManifestsRequest,
<a href=#162 id=162 data-nosnippet>162</a> ) -&gt; <span class="prelude-ty">Result</span>&lt;RemapManifestsResponse, BoxedError&gt; {
<a href=#163 id=163 data-nosnippet>163</a> <span class="prelude-val">Err</span>(BoxedError::new(
<a href=#164 id=164 data-nosnippet>164</a> UnsupportedSnafu {
<a href=#165 id=165 data-nosnippet>165</a> operation: <span class="string">"remap_manifests"</span>,
<a href=#166 id=166 data-nosnippet>166</a> }
<a href=#167 id=167 data-nosnippet>167</a> .build(),
<a href=#168 id=168 data-nosnippet>168</a> ))
<a href=#169 id=169 data-nosnippet>169</a> }
<a href=#170 id=170 data-nosnippet>170</a>
<a href=#171 id=171 data-nosnippet>171</a> <span class="kw">fn </span>role(<span class="kw-2">&amp;</span><span class="self">self</span>, region_id: RegionId) -&gt; <span class="prelude-ty">Option</span>&lt;RegionRole&gt; {
<a href=#172 id=172 data-nosnippet>172</a> <span class="self">self</span>.inner.state(region_id)
<a href=#173 id=173 data-nosnippet>173</a> }
<a href=#174 id=174 data-nosnippet>174</a>
<a href=#175 id=175 data-nosnippet>175</a> <span class="kw">fn </span>as_any(<span class="kw-2">&amp;</span><span class="self">self</span>) -&gt; <span class="kw-2">&amp;</span><span class="kw">dyn </span>Any {
<a href=#176 id=176 data-nosnippet>176</a> <span class="self">self
<a href=#177 id=177 data-nosnippet>177</a> </span>}
<a href=#178 id=178 data-nosnippet>178</a>}
<a href=#179 id=179 data-nosnippet>179</a>
<a href=#180 id=180 data-nosnippet>180</a><span class="kw">struct </span>EngineInner {
<a href=#181 id=181 data-nosnippet>181</a> <span class="doccomment">/// All regions opened by the engine.
<a href=#182 id=182 data-nosnippet>182</a> ///
<a href=#183 id=183 data-nosnippet>183</a> /// Writing to `regions` should also hold the `region_mutex`.
<a href=#184 id=184 data-nosnippet>184</a> </span>regions: RwLock&lt;HashMap&lt;RegionId, FileRegionRef&gt;&gt;,
<a href=#185 id=185 data-nosnippet>185</a>
<a href=#186 id=186 data-nosnippet>186</a> <span class="doccomment">/// Region mutex is used to protect the operations such as creating/opening/closing
<a href=#187 id=187 data-nosnippet>187</a> /// a region, to avoid things like opening the same region simultaneously.
<a href=#188 id=188 data-nosnippet>188</a> </span>region_mutex: Mutex&lt;()&gt;,
<a href=#189 id=189 data-nosnippet>189</a>
<a href=#190 id=190 data-nosnippet>190</a> object_store: ObjectStore,
<a href=#191 id=191 data-nosnippet>191</a>
<a href=#192 id=192 data-nosnippet>192</a> local_file_access: LocalFileAccess,
<a href=#193 id=193 data-nosnippet>193</a>}
<a href=#194 id=194 data-nosnippet>194</a>
<a href=#195 id=195 data-nosnippet>195</a><span class="kw">type </span>EngineInnerRef = Arc&lt;EngineInner&gt;;
<a href=#196 id=196 data-nosnippet>196</a>
<a href=#197 id=197 data-nosnippet>197</a><span class="kw">fn </span>ensure_region_requirements(
<a href=#198 id=198 data-nosnippet>198</a> requirements: RegionRequirements,
<a href=#199 id=199 data-nosnippet>199</a> object_store: <span class="kw-2">&amp;</span>ObjectStore,
<a href=#200 id=200 data-nosnippet>200</a>) -&gt; EngineResult&lt;()&gt; {
<a href=#201 id=201 data-nosnippet>201</a> <span class="kw">if </span>!requirements.object_storage {
<a href=#202 id=202 data-nosnippet>202</a> <span class="kw">return </span><span class="prelude-val">Ok</span>(());
<a href=#203 id=203 data-nosnippet>203</a> }
<a href=#204 id=204 data-nosnippet>204</a>
<a href=#205 id=205 data-nosnippet>205</a> <span class="macro">ensure!</span>(
<a href=#206 id=206 data-nosnippet>206</a> object_store::util::is_object_storage(object_store),
<a href=#207 id=207 data-nosnippet>207</a> UnsupportedSnafu {
<a href=#208 id=208 data-nosnippet>208</a> operation: <span class="string">"open region with object storage requirement on non-object storage"
<a href=#209 id=209 data-nosnippet>209</a> </span>}
<a href=#210 id=210 data-nosnippet>210</a> );
<a href=#211 id=211 data-nosnippet>211</a>
<a href=#212 id=212 data-nosnippet>212</a> <span class="prelude-val">Ok</span>(())
<a href=#213 id=213 data-nosnippet>213</a>}
<a href=#214 id=214 data-nosnippet>214</a>
<a href=#215 id=215 data-nosnippet>215</a><span class="kw">impl </span>EngineInner {
<a href=#216 id=216 data-nosnippet>216</a> <span class="kw">fn </span>new(object_store: ObjectStore, local_file_access: LocalFileAccess) -&gt; <span class="self">Self </span>{
<a href=#217 id=217 data-nosnippet>217</a> <span class="self">Self </span>{
<a href=#218 id=218 data-nosnippet>218</a> regions: RwLock::new(HashMap::new()),
<a href=#219 id=219 data-nosnippet>219</a> region_mutex: Mutex::new(()),
<a href=#220 id=220 data-nosnippet>220</a> object_store,
<a href=#221 id=221 data-nosnippet>221</a> local_file_access,
<a href=#222 id=222 data-nosnippet>222</a> }
<a href=#223 id=223 data-nosnippet>223</a> }
<a href=#224 id=224 data-nosnippet>224</a>
<a href=#225 id=225 data-nosnippet>225</a> <span class="kw">async fn </span>handle_request(
<a href=#226 id=226 data-nosnippet>226</a> <span class="kw-2">&amp;</span><span class="self">self</span>,
<a href=#227 id=227 data-nosnippet>227</a> region_id: RegionId,
<a href=#228 id=228 data-nosnippet>228</a> request: RegionRequest,
<a href=#229 id=229 data-nosnippet>229</a> ) -&gt; EngineResult&lt;RegionResponse&gt; {
<a href=#230 id=230 data-nosnippet>230</a> <span class="kw">let </span>result = <span class="kw">match </span>request {
<a href=#231 id=231 data-nosnippet>231</a> RegionRequest::Create(req) =&gt; <span class="self">self</span>.handle_create(region_id, req).<span class="kw">await</span>,
<a href=#232 id=232 data-nosnippet>232</a> RegionRequest::Drop(req) =&gt; <span class="self">self</span>.handle_drop(region_id, req).<span class="kw">await</span>,
<a href=#233 id=233 data-nosnippet>233</a> RegionRequest::Open(req) =&gt; <span class="self">self</span>.handle_open(region_id, req).<span class="kw">await</span>,
<a href=#234 id=234 data-nosnippet>234</a> RegionRequest::Close(req) =&gt; <span class="self">self</span>.handle_close(region_id, req).<span class="kw">await</span>,
<a href=#235 id=235 data-nosnippet>235</a> <span class="kw">_ </span>=&gt; UnsupportedSnafu {
<a href=#236 id=236 data-nosnippet>236</a> operation: request.to_string(),
<a href=#237 id=237 data-nosnippet>237</a> }
<a href=#238 id=238 data-nosnippet>238</a> .fail(),
<a href=#239 id=239 data-nosnippet>239</a> };
<a href=#240 id=240 data-nosnippet>240</a> result.map(RegionResponse::new)
<a href=#241 id=241 data-nosnippet>241</a> }
<a href=#242 id=242 data-nosnippet>242</a>
<a href=#243 id=243 data-nosnippet>243</a> <span class="kw">async fn </span>stop(<span class="kw-2">&amp;</span><span class="self">self</span>) -&gt; EngineResult&lt;()&gt; {
<a href=#244 id=244 data-nosnippet>244</a> <span class="kw">let </span>_lock = <span class="self">self</span>.region_mutex.lock().<span class="kw">await</span>;
<a href=#245 id=245 data-nosnippet>245</a> <span class="self">self</span>.regions.write().unwrap().clear();
<a href=#246 id=246 data-nosnippet>246</a> <span class="prelude-val">Ok</span>(())
<a href=#247 id=247 data-nosnippet>247</a> }
<a href=#248 id=248 data-nosnippet>248</a>
<a href=#249 id=249 data-nosnippet>249</a> <span class="kw">fn </span>set_region_role(<span class="kw-2">&amp;</span><span class="self">self</span>, _region_id: RegionId, _region_role: RegionRole) -&gt; EngineResult&lt;()&gt; {
<a href=#250 id=250 data-nosnippet>250</a> <span class="comment">// TODO(zhongzc): Improve the semantics and implementation of this API.
<a href=#251 id=251 data-nosnippet>251</a> </span><span class="prelude-val">Ok</span>(())
<a href=#252 id=252 data-nosnippet>252</a> }
<a href=#253 id=253 data-nosnippet>253</a>
<a href=#254 id=254 data-nosnippet>254</a> <span class="kw">fn </span>state(<span class="kw-2">&amp;</span><span class="self">self</span>, region_id: RegionId) -&gt; <span class="prelude-ty">Option</span>&lt;RegionRole&gt; {
<a href=#255 id=255 data-nosnippet>255</a> <span class="kw">if </span><span class="self">self</span>.regions.read().unwrap().get(<span class="kw-2">&amp;</span>region_id).is_some() {
<a href=#256 id=256 data-nosnippet>256</a> <span class="prelude-val">Some</span>(RegionRole::Leader)
<a href=#257 id=257 data-nosnippet>257</a> } <span class="kw">else </span>{
<a href=#258 id=258 data-nosnippet>258</a> <span class="prelude-val">None
<a href=#259 id=259 data-nosnippet>259</a> </span>}
<a href=#260 id=260 data-nosnippet>260</a> }
<a href=#261 id=261 data-nosnippet>261</a>}
<a href=#262 id=262 data-nosnippet>262</a>
<a href=#263 id=263 data-nosnippet>263</a><span class="kw">impl </span>EngineInner {
<a href=#264 id=264 data-nosnippet>264</a> <span class="kw">async fn </span>handle_create(
<a href=#265 id=265 data-nosnippet>265</a> <span class="kw-2">&amp;</span><span class="self">self</span>,
<a href=#266 id=266 data-nosnippet>266</a> region_id: RegionId,
<a href=#267 id=267 data-nosnippet>267</a> request: RegionCreateRequest,
<a href=#268 id=268 data-nosnippet>268</a> ) -&gt; EngineResult&lt;AffectedRows&gt; {
<a href=#269 id=269 data-nosnippet>269</a> <span class="macro">ensure!</span>(
<a href=#270 id=270 data-nosnippet>270</a> request.engine == FILE_ENGINE,
<a href=#271 id=271 data-nosnippet>271</a> UnexpectedEngineSnafu {
<a href=#272 id=272 data-nosnippet>272</a> engine: request.engine
<a href=#273 id=273 data-nosnippet>273</a> }
<a href=#274 id=274 data-nosnippet>274</a> );
<a href=#275 id=275 data-nosnippet>275</a>
<a href=#276 id=276 data-nosnippet>276</a> <span class="kw">if </span><span class="self">self</span>.exists(region_id).<span class="kw">await </span>{
<a href=#277 id=277 data-nosnippet>277</a> <span class="kw">return </span><span class="prelude-val">Ok</span>(<span class="number">0</span>);
<a href=#278 id=278 data-nosnippet>278</a> }
<a href=#279 id=279 data-nosnippet>279</a>
<a href=#280 id=280 data-nosnippet>280</a> <span class="macro">info!</span>(<span class="string">"Try to create region, region_id: {}"</span>, region_id);
<a href=#281 id=281 data-nosnippet>281</a>
<a href=#282 id=282 data-nosnippet>282</a> <span class="kw">let </span>_lock = <span class="self">self</span>.region_mutex.lock().<span class="kw">await</span>;
<a href=#283 id=283 data-nosnippet>283</a> <span class="comment">// Check again after acquiring the lock
<a href=#284 id=284 data-nosnippet>284</a> </span><span class="kw">if </span><span class="self">self</span>.exists(region_id).<span class="kw">await </span>{
<a href=#285 id=285 data-nosnippet>285</a> <span class="kw">return </span><span class="prelude-val">Ok</span>(<span class="number">0</span>);
<a href=#286 id=286 data-nosnippet>286</a> }
<a href=#287 id=287 data-nosnippet>287</a>
<a href=#288 id=288 data-nosnippet>288</a> ensure_region_requirements(request.requirements, <span class="kw-2">&amp;</span><span class="self">self</span>.object_store)<span class="question-mark">?</span>;
<a href=#289 id=289 data-nosnippet>289</a>
<a href=#290 id=290 data-nosnippet>290</a> <span class="kw">let </span>res = FileRegion::create(region_id, request, <span class="kw-2">&amp;</span><span class="self">self</span>.object_store).<span class="kw">await</span>;
<a href=#291 id=291 data-nosnippet>291</a> <span class="kw">let </span>region = res.inspect_err(|err| {
<a href=#292 id=292 data-nosnippet>292</a> <span class="macro">error!</span>(
<a href=#293 id=293 data-nosnippet>293</a> err;
<a href=#294 id=294 data-nosnippet>294</a> <span class="string">"Failed to create region, region_id: {}"</span>,
<a href=#295 id=295 data-nosnippet>295</a> region_id
<a href=#296 id=296 data-nosnippet>296</a> );
<a href=#297 id=297 data-nosnippet>297</a> })<span class="question-mark">?</span>;
<a href=#298 id=298 data-nosnippet>298</a> <span class="self">self</span>.regions.write().unwrap().insert(region_id, region);
<a href=#299 id=299 data-nosnippet>299</a>
<a href=#300 id=300 data-nosnippet>300</a> <span class="macro">info!</span>(<span class="string">"A new region is created, region_id: {}"</span>, region_id);
<a href=#301 id=301 data-nosnippet>301</a> <span class="prelude-val">Ok</span>(<span class="number">0</span>)
<a href=#302 id=302 data-nosnippet>302</a> }
<a href=#303 id=303 data-nosnippet>303</a>
<a href=#304 id=304 data-nosnippet>304</a> <span class="kw">async fn </span>handle_open(
<a href=#305 id=305 data-nosnippet>305</a> <span class="kw-2">&amp;</span><span class="self">self</span>,
<a href=#306 id=306 data-nosnippet>306</a> region_id: RegionId,
<a href=#307 id=307 data-nosnippet>307</a> request: RegionOpenRequest,
<a href=#308 id=308 data-nosnippet>308</a> ) -&gt; EngineResult&lt;AffectedRows&gt; {
<a href=#309 id=309 data-nosnippet>309</a> <span class="kw">if </span><span class="self">self</span>.exists(region_id).<span class="kw">await </span>{
<a href=#310 id=310 data-nosnippet>310</a> <span class="kw">return </span><span class="prelude-val">Ok</span>(<span class="number">0</span>);
<a href=#311 id=311 data-nosnippet>311</a> }
<a href=#312 id=312 data-nosnippet>312</a>
<a href=#313 id=313 data-nosnippet>313</a> <span class="macro">info!</span>(<span class="string">"Try to open region, region_id: {}"</span>, region_id);
<a href=#314 id=314 data-nosnippet>314</a>
<a href=#315 id=315 data-nosnippet>315</a> <span class="kw">let </span>_lock = <span class="self">self</span>.region_mutex.lock().<span class="kw">await</span>;
<a href=#316 id=316 data-nosnippet>316</a> <span class="comment">// Check again after acquiring the lock
<a href=#317 id=317 data-nosnippet>317</a> </span><span class="kw">if </span><span class="self">self</span>.exists(region_id).<span class="kw">await </span>{
<a href=#318 id=318 data-nosnippet>318</a> <span class="kw">return </span><span class="prelude-val">Ok</span>(<span class="number">0</span>);
<a href=#319 id=319 data-nosnippet>319</a> }
<a href=#320 id=320 data-nosnippet>320</a>
<a href=#321 id=321 data-nosnippet>321</a> ensure_region_requirements(request.requirements, <span class="kw-2">&amp;</span><span class="self">self</span>.object_store)<span class="question-mark">?</span>;
<a href=#322 id=322 data-nosnippet>322</a>
<a href=#323 id=323 data-nosnippet>323</a> <span class="kw">let </span>res = FileRegion::open(region_id, request, <span class="kw-2">&amp;</span><span class="self">self</span>.object_store).<span class="kw">await</span>;
<a href=#324 id=324 data-nosnippet>324</a> <span class="kw">let </span>region = res.inspect_err(|err| {
<a href=#325 id=325 data-nosnippet>325</a> <span class="macro">error!</span>(
<a href=#326 id=326 data-nosnippet>326</a> err;
<a href=#327 id=327 data-nosnippet>327</a> <span class="string">"Failed to open region, region_id: {}"</span>,
<a href=#328 id=328 data-nosnippet>328</a> region_id
<a href=#329 id=329 data-nosnippet>329</a> );
<a href=#330 id=330 data-nosnippet>330</a> })<span class="question-mark">?</span>;
<a href=#331 id=331 data-nosnippet>331</a> <span class="self">self</span>.regions.write().unwrap().insert(region_id, region);
<a href=#332 id=332 data-nosnippet>332</a>
<a href=#333 id=333 data-nosnippet>333</a> <span class="macro">info!</span>(<span class="string">"Region opened, region_id: {}"</span>, region_id);
<a href=#334 id=334 data-nosnippet>334</a> <span class="prelude-val">Ok</span>(<span class="number">0</span>)
<a href=#335 id=335 data-nosnippet>335</a> }
<a href=#336 id=336 data-nosnippet>336</a>
<a href=#337 id=337 data-nosnippet>337</a> <span class="kw">async fn </span>handle_close(
<a href=#338 id=338 data-nosnippet>338</a> <span class="kw-2">&amp;</span><span class="self">self</span>,
<a href=#339 id=339 data-nosnippet>339</a> region_id: RegionId,
<a href=#340 id=340 data-nosnippet>340</a> _request: RegionCloseRequest,
<a href=#341 id=341 data-nosnippet>341</a> ) -&gt; EngineResult&lt;AffectedRows&gt; {
<a href=#342 id=342 data-nosnippet>342</a> <span class="kw">let </span>_lock = <span class="self">self</span>.region_mutex.lock().<span class="kw">await</span>;
<a href=#343 id=343 data-nosnippet>343</a>
<a href=#344 id=344 data-nosnippet>344</a> <span class="kw">let </span><span class="kw-2">mut </span>regions = <span class="self">self</span>.regions.write().unwrap();
<a href=#345 id=345 data-nosnippet>345</a> <span class="kw">if </span>regions.remove(<span class="kw-2">&amp;</span>region_id).is_some() {
<a href=#346 id=346 data-nosnippet>346</a> <span class="macro">info!</span>(<span class="string">"Region closed, region_id: {}"</span>, region_id);
<a href=#347 id=347 data-nosnippet>347</a> }
<a href=#348 id=348 data-nosnippet>348</a>
<a href=#349 id=349 data-nosnippet>349</a> <span class="prelude-val">Ok</span>(<span class="number">0</span>)
<a href=#350 id=350 data-nosnippet>350</a> }
<a href=#351 id=351 data-nosnippet>351</a>
<a href=#352 id=352 data-nosnippet>352</a> <span class="kw">async fn </span>handle_drop(
<a href=#353 id=353 data-nosnippet>353</a> <span class="kw-2">&amp;</span><span class="self">self</span>,
<a href=#354 id=354 data-nosnippet>354</a> region_id: RegionId,
<a href=#355 id=355 data-nosnippet>355</a> _request: RegionDropRequest,
<a href=#356 id=356 data-nosnippet>356</a> ) -&gt; EngineResult&lt;AffectedRows&gt; {
<a href=#357 id=357 data-nosnippet>357</a> <span class="kw">if </span>!<span class="self">self</span>.exists(region_id).<span class="kw">await </span>{
<a href=#358 id=358 data-nosnippet>358</a> <span class="kw">return </span>RegionNotFoundSnafu { region_id }.fail();
<a href=#359 id=359 data-nosnippet>359</a> }
<a href=#360 id=360 data-nosnippet>360</a>
<a href=#361 id=361 data-nosnippet>361</a> <span class="macro">info!</span>(<span class="string">"Try to drop region, region_id: {}"</span>, region_id);
<a href=#362 id=362 data-nosnippet>362</a>
<a href=#363 id=363 data-nosnippet>363</a> <span class="kw">let </span>_lock = <span class="self">self</span>.region_mutex.lock().<span class="kw">await</span>;
<a href=#364 id=364 data-nosnippet>364</a>
<a href=#365 id=365 data-nosnippet>365</a> <span class="kw">let </span>region = <span class="self">self</span>.get_region(region_id).<span class="kw">await</span>;
<a href=#366 id=366 data-nosnippet>366</a> <span class="kw">if let </span><span class="prelude-val">Some</span>(region) = region {
<a href=#367 id=367 data-nosnippet>367</a> <span class="kw">let </span>res = FileRegion::drop(<span class="kw-2">&amp;</span>region, <span class="kw-2">&amp;</span><span class="self">self</span>.object_store).<span class="kw">await</span>;
<a href=#368 id=368 data-nosnippet>368</a> res.inspect_err(|err| {
<a href=#369 id=369 data-nosnippet>369</a> <span class="macro">error!</span>(
<a href=#370 id=370 data-nosnippet>370</a> err;
<a href=#371 id=371 data-nosnippet>371</a> <span class="string">"Failed to drop region, region_id: {}"</span>,
<a href=#372 id=372 data-nosnippet>372</a> region_id
<a href=#373 id=373 data-nosnippet>373</a> );
<a href=#374 id=374 data-nosnippet>374</a> })<span class="question-mark">?</span>;
<a href=#375 id=375 data-nosnippet>375</a> }
<a href=#376 id=376 data-nosnippet>376</a> <span class="kw">let _ </span>= <span class="self">self</span>.regions.write().unwrap().remove(<span class="kw-2">&amp;</span>region_id);
<a href=#377 id=377 data-nosnippet>377</a>
<a href=#378 id=378 data-nosnippet>378</a> <span class="macro">info!</span>(<span class="string">"Region dropped, region_id: {}"</span>, region_id);
<a href=#379 id=379 data-nosnippet>379</a> <span class="prelude-val">Ok</span>(<span class="number">0</span>)
<a href=#380 id=380 data-nosnippet>380</a> }
<a href=#381 id=381 data-nosnippet>381</a>
<a href=#382 id=382 data-nosnippet>382</a> <span class="kw">async fn </span>get_region(<span class="kw-2">&amp;</span><span class="self">self</span>, region_id: RegionId) -&gt; <span class="prelude-ty">Option</span>&lt;FileRegionRef&gt; {
<a href=#383 id=383 data-nosnippet>383</a> <span class="self">self</span>.regions.read().unwrap().get(<span class="kw-2">&amp;</span>region_id).cloned()
<a href=#384 id=384 data-nosnippet>384</a> }
<a href=#385 id=385 data-nosnippet>385</a>
<a href=#386 id=386 data-nosnippet>386</a> <span class="kw">async fn </span>exists(<span class="kw-2">&amp;</span><span class="self">self</span>, region_id: RegionId) -&gt; bool {
<a href=#387 id=387 data-nosnippet>387</a> <span class="self">self</span>.regions.read().unwrap().contains_key(<span class="kw-2">&amp;</span>region_id)
<a href=#388 id=388 data-nosnippet>388</a> }
<a href=#389 id=389 data-nosnippet>389</a>}
<a href=#390 id=390 data-nosnippet>390</a>
<a href=#391 id=391 data-nosnippet>391</a><span class="attr">#[cfg(test)]
<a href=#392 id=392 data-nosnippet>392</a></span><span class="kw">mod </span>tests {
<a href=#393 id=393 data-nosnippet>393</a> <span class="kw">use </span>object_store::services::{Fs, S3};
<a href=#394 id=394 data-nosnippet>394</a>
<a href=#395 id=395 data-nosnippet>395</a> <span class="kw">use </span>super::<span class="kw-2">*</span>;
<a href=#396 id=396 data-nosnippet>396</a> <span class="kw">use </span><span class="kw">crate</span>::error::Error;
<a href=#397 id=397 data-nosnippet>397</a>
<a href=#398 id=398 data-nosnippet>398</a> <span class="kw">fn </span>build_fs_object_store() -&gt; ObjectStore {
<a href=#399 id=399 data-nosnippet>399</a> ObjectStore::new(Fs::default().root(<span class="string">"/tmp"</span>)).unwrap()
<a href=#400 id=400 data-nosnippet>400</a> }
<a href=#401 id=401 data-nosnippet>401</a>
<a href=#402 id=402 data-nosnippet>402</a> <span class="kw">fn </span>build_s3_object_store() -&gt; ObjectStore {
<a href=#403 id=403 data-nosnippet>403</a> ObjectStore::new(
<a href=#404 id=404 data-nosnippet>404</a> S3::default()
<a href=#405 id=405 data-nosnippet>405</a> .bucket(<span class="string">"test-bucket"</span>)
<a href=#406 id=406 data-nosnippet>406</a> .region(<span class="string">"us-east-1"</span>)
<a href=#407 id=407 data-nosnippet>407</a> .disable_ec2_metadata(),
<a href=#408 id=408 data-nosnippet>408</a> )
<a href=#409 id=409 data-nosnippet>409</a> .unwrap()
<a href=#410 id=410 data-nosnippet>410</a> }
<a href=#411 id=411 data-nosnippet>411</a>
<a href=#412 id=412 data-nosnippet>412</a> <span class="attr">#[test]
<a href=#413 id=413 data-nosnippet>413</a> </span><span class="kw">fn </span>test_empty_region_requirements_are_supported() {
<a href=#414 id=414 data-nosnippet>414</a> ensure_region_requirements(RegionRequirements::empty(), <span class="kw-2">&amp;</span>build_fs_object_store()).unwrap();
<a href=#415 id=415 data-nosnippet>415</a> }
<a href=#416 id=416 data-nosnippet>416</a>
<a href=#417 id=417 data-nosnippet>417</a> <span class="attr">#[test]
<a href=#418 id=418 data-nosnippet>418</a> </span><span class="kw">fn </span>test_object_storage_region_requirement_rejects_fs_object_store() {
<a href=#419 id=419 data-nosnippet>419</a> <span class="kw">let </span>err = ensure_region_requirements(
<a href=#420 id=420 data-nosnippet>420</a> RegionRequirements::object_storage(),
<a href=#421 id=421 data-nosnippet>421</a> <span class="kw-2">&amp;</span>build_fs_object_store(),
<a href=#422 id=422 data-nosnippet>422</a> )
<a href=#423 id=423 data-nosnippet>423</a> .unwrap_err();
<a href=#424 id=424 data-nosnippet>424</a>
<a href=#425 id=425 data-nosnippet>425</a> <span class="macro">assert!</span>(<span class="macro">matches!</span>(err, Error::Unsupported { .. }));
<a href=#426 id=426 data-nosnippet>426</a> }
<a href=#427 id=427 data-nosnippet>427</a>
<a href=#428 id=428 data-nosnippet>428</a> <span class="attr">#[test]
<a href=#429 id=429 data-nosnippet>429</a> </span><span class="kw">fn </span>test_object_storage_region_requirement_accepts_s3_object_store() {
<a href=#430 id=430 data-nosnippet>430</a> ensure_region_requirements(
<a href=#431 id=431 data-nosnippet>431</a> RegionRequirements::object_storage(),
<a href=#432 id=432 data-nosnippet>432</a> <span class="kw-2">&amp;</span>build_s3_object_store(),
<a href=#433 id=433 data-nosnippet>433</a> )
<a href=#434 id=434 data-nosnippet>434</a> .unwrap();
<a href=#435 id=435 data-nosnippet>435</a> }
<a href=#436 id=436 data-nosnippet>436</a>}
</code></pre></div></section></main></body></html>
+252
View File
@@ -0,0 +1,252 @@
<!DOCTYPE html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1.0"><meta name="generator" content="rustdoc"><meta name="description" content="Source of the Rust file `src/file-engine/src/error.rs`."><title>error.rs - source</title><script>if(window.location.protocol!=="file:")document.head.insertAdjacentHTML("beforeend","SourceSerif4-Regular-6b053e98.ttf.woff2,FiraSans-Italic-81dc35de.woff2,FiraSans-Regular-0fe48ade.woff2,FiraSans-MediumItalic-ccf7e434.woff2,FiraSans-Medium-e1aa3f0a.woff2,SourceCodePro-Regular-8badfe75.ttf.woff2,SourceCodePro-Semibold-aa29a496.ttf.woff2".split(",").map(f=>`<link rel="preload" as="font" type="font/woff2"href="../../static.files/${f}">`).join(""))</script><link rel="stylesheet" href="../../static.files/normalize-9960930a.css"><link rel="stylesheet" href="../../static.files/rustdoc-17e0aaed.css"><meta name="rustdoc-vars" data-root-path="../../" data-static-root-path="../../static.files/" data-current-crate="file_engine" data-themes="" data-resource-suffix="" data-rustdoc-version="1.96.1 (31fca3adb 2026-06-26)" data-channel="1.96.1" data-search-js="search-b5634cc7.js" data-stringdex-js="stringdex-2da4960a.js" data-settings-js="settings-170eb4bf.js" ><script src="../../static.files/storage-41dd4d93.js"></script><script defer src="../../static.files/src-script-813739b1.js"></script><script defer src="../../src-files.js"></script><script defer src="../../static.files/main-5013f961.js"></script><noscript><link rel="stylesheet" href="../../static.files/noscript-f7c3ffd8.css"></noscript><link rel="alternate icon" type="image/png" href="../../static.files/favicon-32x32-eab170b8.png"><link rel="icon" type="image/svg+xml" href="../../static.files/favicon-044be391.svg"></head><body class="rustdoc src"><a class="skip-main-content" href="#main-content">Skip to main content</a><!--[if lte IE 11]><div class="warning">This old browser is unsupported and will most likely display funky things.</div><![endif]--><nav class="sidebar"><div class="src-sidebar-title"><h2>Files</h2></div></nav><div class="sidebar-resizer" title="Drag to resize sidebar"></div><main><section id="main-content" class="content" tabindex="-1"><div class="main-heading"><h1><div class="sub-heading">file_engine/</div>error.rs</h1><rustdoc-toolbar></rustdoc-toolbar></div><div class="example-wrap digits-3"><pre class="rust"><code><a href=#1 id=1 data-nosnippet>1</a><span class="comment">// Copyright 2023 Greptime Team
<a href=#2 id=2 data-nosnippet>2</a>//
<a href=#3 id=3 data-nosnippet>3</a>// Licensed under the Apache License, Version 2.0 (the "License");
<a href=#4 id=4 data-nosnippet>4</a>// you may not use this file except in compliance with the License.
<a href=#5 id=5 data-nosnippet>5</a>// You may obtain a copy of the License at
<a href=#6 id=6 data-nosnippet>6</a>//
<a href=#7 id=7 data-nosnippet>7</a>// http://www.apache.org/licenses/LICENSE-2.0
<a href=#8 id=8 data-nosnippet>8</a>//
<a href=#9 id=9 data-nosnippet>9</a>// Unless required by applicable law or agreed to in writing, software
<a href=#10 id=10 data-nosnippet>10</a>// distributed under the License is distributed on an "AS IS" BASIS,
<a href=#11 id=11 data-nosnippet>11</a>// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
<a href=#12 id=12 data-nosnippet>12</a>// See the License for the specific language governing permissions and
<a href=#13 id=13 data-nosnippet>13</a>// limitations under the License.
<a href=#14 id=14 data-nosnippet>14</a>
<a href=#15 id=15 data-nosnippet>15</a></span><span class="kw">use </span>std::any::Any;
<a href=#16 id=16 data-nosnippet>16</a>
<a href=#17 id=17 data-nosnippet>17</a><span class="kw">use </span>common_error::ext::{ErrorExt, RetryHint};
<a href=#18 id=18 data-nosnippet>18</a><span class="kw">use </span>common_error::status_code::StatusCode;
<a href=#19 id=19 data-nosnippet>19</a><span class="kw">use </span>common_macro::stack_trace_debug;
<a href=#20 id=20 data-nosnippet>20</a><span class="kw">use </span>datafusion::arrow::error::ArrowError;
<a href=#21 id=21 data-nosnippet>21</a><span class="kw">use </span>datafusion::error::DataFusionError;
<a href=#22 id=22 data-nosnippet>22</a><span class="kw">use </span>object_store::error::retry_hint_from_opendal_error;
<a href=#23 id=23 data-nosnippet>23</a><span class="kw">use </span>serde_json::error::Error <span class="kw">as </span>JsonError;
<a href=#24 id=24 data-nosnippet>24</a><span class="kw">use </span>snafu::{Location, Snafu};
<a href=#25 id=25 data-nosnippet>25</a><span class="kw">use </span>store_api::storage::RegionId;
<a href=#26 id=26 data-nosnippet>26</a>
<a href=#27 id=27 data-nosnippet>27</a><span class="attr">#[derive(Snafu)]
<a href=#28 id=28 data-nosnippet>28</a>#[snafu(visibility(<span class="kw">pub</span>))]
<a href=#29 id=29 data-nosnippet>29</a>#[stack_trace_debug]
<a href=#30 id=30 data-nosnippet>30</a></span><span class="kw">pub enum </span>Error {
<a href=#31 id=31 data-nosnippet>31</a> <span class="attr">#[snafu(display(<span class="string">"Unsupported operation: {}"</span>, operation))]
<a href=#32 id=32 data-nosnippet>32</a> </span>Unsupported {
<a href=#33 id=33 data-nosnippet>33</a> operation: String,
<a href=#34 id=34 data-nosnippet>34</a> <span class="attr">#[snafu(implicit)]
<a href=#35 id=35 data-nosnippet>35</a> </span>location: Location,
<a href=#36 id=36 data-nosnippet>36</a> },
<a href=#37 id=37 data-nosnippet>37</a>
<a href=#38 id=38 data-nosnippet>38</a> <span class="attr">#[snafu(display(<span class="string">"Unexpected engine: {}"</span>, engine))]
<a href=#39 id=39 data-nosnippet>39</a> </span>UnexpectedEngine {
<a href=#40 id=40 data-nosnippet>40</a> engine: String,
<a href=#41 id=41 data-nosnippet>41</a> <span class="attr">#[snafu(implicit)]
<a href=#42 id=42 data-nosnippet>42</a> </span>location: Location,
<a href=#43 id=43 data-nosnippet>43</a> },
<a href=#44 id=44 data-nosnippet>44</a>
<a href=#45 id=45 data-nosnippet>45</a> <span class="attr">#[snafu(display(<span class="string">"Invalid region metadata"</span>))]
<a href=#46 id=46 data-nosnippet>46</a> </span>InvalidMetadata {
<a href=#47 id=47 data-nosnippet>47</a> source: store_api::metadata::MetadataError,
<a href=#48 id=48 data-nosnippet>48</a> <span class="attr">#[snafu(implicit)]
<a href=#49 id=49 data-nosnippet>49</a> </span>location: Location,
<a href=#50 id=50 data-nosnippet>50</a> },
<a href=#51 id=51 data-nosnippet>51</a>
<a href=#52 id=52 data-nosnippet>52</a> <span class="attr">#[snafu(display(<span class="string">"Region not found, region_id: {}"</span>, region_id))]
<a href=#53 id=53 data-nosnippet>53</a> </span>RegionNotFound {
<a href=#54 id=54 data-nosnippet>54</a> region_id: RegionId,
<a href=#55 id=55 data-nosnippet>55</a> <span class="attr">#[snafu(implicit)]
<a href=#56 id=56 data-nosnippet>56</a> </span>location: Location,
<a href=#57 id=57 data-nosnippet>57</a> },
<a href=#58 id=58 data-nosnippet>58</a>
<a href=#59 id=59 data-nosnippet>59</a> <span class="attr">#[snafu(display(<span class="string">"Failed to check object from path: {}"</span>, path))]
<a href=#60 id=60 data-nosnippet>60</a> </span>CheckObject {
<a href=#61 id=61 data-nosnippet>61</a> path: String,
<a href=#62 id=62 data-nosnippet>62</a> <span class="attr">#[snafu(implicit)]
<a href=#63 id=63 data-nosnippet>63</a> </span>location: Location,
<a href=#64 id=64 data-nosnippet>64</a> <span class="attr">#[snafu(source)]
<a href=#65 id=65 data-nosnippet>65</a> </span>error: object_store::Error,
<a href=#66 id=66 data-nosnippet>66</a> },
<a href=#67 id=67 data-nosnippet>67</a>
<a href=#68 id=68 data-nosnippet>68</a> <span class="attr">#[snafu(display(<span class="string">"Fail to encode object into json"</span>))]
<a href=#69 id=69 data-nosnippet>69</a> </span>EncodeJson {
<a href=#70 id=70 data-nosnippet>70</a> <span class="attr">#[snafu(implicit)]
<a href=#71 id=71 data-nosnippet>71</a> </span>location: Location,
<a href=#72 id=72 data-nosnippet>72</a> <span class="attr">#[snafu(source)]
<a href=#73 id=73 data-nosnippet>73</a> </span>error: JsonError,
<a href=#74 id=74 data-nosnippet>74</a> },
<a href=#75 id=75 data-nosnippet>75</a>
<a href=#76 id=76 data-nosnippet>76</a> <span class="attr">#[snafu(display(<span class="string">"Fail to decode object from json"</span>))]
<a href=#77 id=77 data-nosnippet>77</a> </span>DecodeJson {
<a href=#78 id=78 data-nosnippet>78</a> <span class="attr">#[snafu(implicit)]
<a href=#79 id=79 data-nosnippet>79</a> </span>location: Location,
<a href=#80 id=80 data-nosnippet>80</a> <span class="attr">#[snafu(source)]
<a href=#81 id=81 data-nosnippet>81</a> </span>error: JsonError,
<a href=#82 id=82 data-nosnippet>82</a> },
<a href=#83 id=83 data-nosnippet>83</a>
<a href=#84 id=84 data-nosnippet>84</a> <span class="attr">#[snafu(display(<span class="string">"Failed to store region manifest, region_id: {}"</span>, region_id))]
<a href=#85 id=85 data-nosnippet>85</a> </span>StoreRegionManifest {
<a href=#86 id=86 data-nosnippet>86</a> <span class="attr">#[snafu(source)]
<a href=#87 id=87 data-nosnippet>87</a> </span>error: object_store::Error,
<a href=#88 id=88 data-nosnippet>88</a> region_id: RegionId,
<a href=#89 id=89 data-nosnippet>89</a> <span class="attr">#[snafu(implicit)]
<a href=#90 id=90 data-nosnippet>90</a> </span>location: Location,
<a href=#91 id=91 data-nosnippet>91</a> },
<a href=#92 id=92 data-nosnippet>92</a>
<a href=#93 id=93 data-nosnippet>93</a> <span class="attr">#[snafu(display(<span class="string">"Failed to load region manifest, region_id: {}"</span>, region_id))]
<a href=#94 id=94 data-nosnippet>94</a> </span>LoadRegionManifest {
<a href=#95 id=95 data-nosnippet>95</a> <span class="attr">#[snafu(source)]
<a href=#96 id=96 data-nosnippet>96</a> </span>error: object_store::Error,
<a href=#97 id=97 data-nosnippet>97</a> region_id: RegionId,
<a href=#98 id=98 data-nosnippet>98</a> <span class="attr">#[snafu(implicit)]
<a href=#99 id=99 data-nosnippet>99</a> </span>location: Location,
<a href=#100 id=100 data-nosnippet>100</a> },
<a href=#101 id=101 data-nosnippet>101</a>
<a href=#102 id=102 data-nosnippet>102</a> <span class="attr">#[snafu(display(<span class="string">"Failed to delete region manifest, region_id: {},"</span>, region_id))]
<a href=#103 id=103 data-nosnippet>103</a> </span>DeleteRegionManifest {
<a href=#104 id=104 data-nosnippet>104</a> <span class="attr">#[snafu(source)]
<a href=#105 id=105 data-nosnippet>105</a> </span>error: object_store::Error,
<a href=#106 id=106 data-nosnippet>106</a> region_id: RegionId,
<a href=#107 id=107 data-nosnippet>107</a> <span class="attr">#[snafu(implicit)]
<a href=#108 id=108 data-nosnippet>108</a> </span>location: Location,
<a href=#109 id=109 data-nosnippet>109</a> },
<a href=#110 id=110 data-nosnippet>110</a>
<a href=#111 id=111 data-nosnippet>111</a> <span class="attr">#[snafu(display(<span class="string">"Manifest already exists: {}"</span>, path))]
<a href=#112 id=112 data-nosnippet>112</a> </span>ManifestExists {
<a href=#113 id=113 data-nosnippet>113</a> path: String,
<a href=#114 id=114 data-nosnippet>114</a> <span class="attr">#[snafu(implicit)]
<a href=#115 id=115 data-nosnippet>115</a> </span>location: Location,
<a href=#116 id=116 data-nosnippet>116</a> },
<a href=#117 id=117 data-nosnippet>117</a>
<a href=#118 id=118 data-nosnippet>118</a> <span class="attr">#[snafu(display(<span class="string">"Missing required field: {}"</span>, name))]
<a href=#119 id=119 data-nosnippet>119</a> </span>MissingRequiredField {
<a href=#120 id=120 data-nosnippet>120</a> name: String,
<a href=#121 id=121 data-nosnippet>121</a> <span class="attr">#[snafu(implicit)]
<a href=#122 id=122 data-nosnippet>122</a> </span>location: Location,
<a href=#123 id=123 data-nosnippet>123</a> },
<a href=#124 id=124 data-nosnippet>124</a>
<a href=#125 id=125 data-nosnippet>125</a> <span class="attr">#[snafu(display(<span class="string">"Failed to build backend"</span>))]
<a href=#126 id=126 data-nosnippet>126</a> </span>BuildBackend {
<a href=#127 id=127 data-nosnippet>127</a> <span class="attr">#[snafu(implicit)]
<a href=#128 id=128 data-nosnippet>128</a> </span>location: Location,
<a href=#129 id=129 data-nosnippet>129</a> source: common_datasource::error::Error,
<a href=#130 id=130 data-nosnippet>130</a> },
<a href=#131 id=131 data-nosnippet>131</a>
<a href=#132 id=132 data-nosnippet>132</a> <span class="attr">#[snafu(display(<span class="string">"Failed to build stream"</span>))]
<a href=#133 id=133 data-nosnippet>133</a> </span>BuildStream {
<a href=#134 id=134 data-nosnippet>134</a> <span class="attr">#[snafu(source)]
<a href=#135 id=135 data-nosnippet>135</a> </span>error: DataFusionError,
<a href=#136 id=136 data-nosnippet>136</a> <span class="attr">#[snafu(implicit)]
<a href=#137 id=137 data-nosnippet>137</a> </span>location: Location,
<a href=#138 id=138 data-nosnippet>138</a> },
<a href=#139 id=139 data-nosnippet>139</a>
<a href=#140 id=140 data-nosnippet>140</a> <span class="attr">#[snafu(display(<span class="string">"Failed to project schema"</span>))]
<a href=#141 id=141 data-nosnippet>141</a> </span>ProjectArrowSchema {
<a href=#142 id=142 data-nosnippet>142</a> <span class="attr">#[snafu(source)]
<a href=#143 id=143 data-nosnippet>143</a> </span>error: ArrowError,
<a href=#144 id=144 data-nosnippet>144</a> <span class="attr">#[snafu(implicit)]
<a href=#145 id=145 data-nosnippet>145</a> </span>location: Location,
<a href=#146 id=146 data-nosnippet>146</a> },
<a href=#147 id=147 data-nosnippet>147</a>
<a href=#148 id=148 data-nosnippet>148</a> <span class="attr">#[snafu(display(<span class="string">"Failed to project schema"</span>))]
<a href=#149 id=149 data-nosnippet>149</a> </span>ProjectSchema {
<a href=#150 id=150 data-nosnippet>150</a> source: datatypes::error::Error,
<a href=#151 id=151 data-nosnippet>151</a> <span class="attr">#[snafu(implicit)]
<a href=#152 id=152 data-nosnippet>152</a> </span>location: Location,
<a href=#153 id=153 data-nosnippet>153</a> },
<a href=#154 id=154 data-nosnippet>154</a>
<a href=#155 id=155 data-nosnippet>155</a> <span class="attr">#[snafu(display(<span class="string">"Failed to parse file format"</span>))]
<a href=#156 id=156 data-nosnippet>156</a> </span>ParseFileFormat {
<a href=#157 id=157 data-nosnippet>157</a> <span class="attr">#[snafu(implicit)]
<a href=#158 id=158 data-nosnippet>158</a> </span>location: Location,
<a href=#159 id=159 data-nosnippet>159</a> source: common_datasource::error::Error,
<a href=#160 id=160 data-nosnippet>160</a> },
<a href=#161 id=161 data-nosnippet>161</a>
<a href=#162 id=162 data-nosnippet>162</a> <span class="attr">#[snafu(display(<span class="string">"Failed to generate parquet scan plan"</span>))]
<a href=#163 id=163 data-nosnippet>163</a> </span>ParquetScanPlan {
<a href=#164 id=164 data-nosnippet>164</a> <span class="attr">#[snafu(source)]
<a href=#165 id=165 data-nosnippet>165</a> </span>error: DataFusionError,
<a href=#166 id=166 data-nosnippet>166</a> <span class="attr">#[snafu(implicit)]
<a href=#167 id=167 data-nosnippet>167</a> </span>location: Location,
<a href=#168 id=168 data-nosnippet>168</a> },
<a href=#169 id=169 data-nosnippet>169</a>
<a href=#170 id=170 data-nosnippet>170</a> <span class="attr">#[snafu(display(
<a href=#171 id=171 data-nosnippet>171</a> <span class="string">"Projection out of bounds, column_index: {}, bounds: {}"</span>,
<a href=#172 id=172 data-nosnippet>172</a> column_index,
<a href=#173 id=173 data-nosnippet>173</a> bounds
<a href=#174 id=174 data-nosnippet>174</a> ))]
<a href=#175 id=175 data-nosnippet>175</a> </span>ProjectionOutOfBounds {
<a href=#176 id=176 data-nosnippet>176</a> column_index: usize,
<a href=#177 id=177 data-nosnippet>177</a> bounds: usize,
<a href=#178 id=178 data-nosnippet>178</a> <span class="attr">#[snafu(implicit)]
<a href=#179 id=179 data-nosnippet>179</a> </span>location: Location,
<a href=#180 id=180 data-nosnippet>180</a> },
<a href=#181 id=181 data-nosnippet>181</a>
<a href=#182 id=182 data-nosnippet>182</a> <span class="attr">#[snafu(transparent)]
<a href=#183 id=183 data-nosnippet>183</a> </span>DataFusion {
<a href=#184 id=184 data-nosnippet>184</a> <span class="attr">#[snafu(source)]
<a href=#185 id=185 data-nosnippet>185</a> </span>error: DataFusionError,
<a href=#186 id=186 data-nosnippet>186</a> <span class="attr">#[snafu(implicit)]
<a href=#187 id=187 data-nosnippet>187</a> </span>location: Location,
<a href=#188 id=188 data-nosnippet>188</a> },
<a href=#189 id=189 data-nosnippet>189</a>
<a href=#190 id=190 data-nosnippet>190</a> <span class="attr">#[snafu(display(<span class="string">"Failed to create default value for column: {}"</span>, column))]
<a href=#191 id=191 data-nosnippet>191</a> </span>CreateDefault {
<a href=#192 id=192 data-nosnippet>192</a> column: String,
<a href=#193 id=193 data-nosnippet>193</a> source: datatypes::error::Error,
<a href=#194 id=194 data-nosnippet>194</a> <span class="attr">#[snafu(implicit)]
<a href=#195 id=195 data-nosnippet>195</a> </span>location: Location,
<a href=#196 id=196 data-nosnippet>196</a> },
<a href=#197 id=197 data-nosnippet>197</a>}
<a href=#198 id=198 data-nosnippet>198</a>
<a href=#199 id=199 data-nosnippet>199</a><span class="kw">pub type </span><span class="prelude-ty">Result</span>&lt;T&gt; = std::result::Result&lt;T, Error&gt;;
<a href=#200 id=200 data-nosnippet>200</a>
<a href=#201 id=201 data-nosnippet>201</a><span class="kw">impl </span>ErrorExt <span class="kw">for </span>Error {
<a href=#202 id=202 data-nosnippet>202</a> <span class="kw">fn </span>status_code(<span class="kw-2">&amp;</span><span class="self">self</span>) -&gt; StatusCode {
<a href=#203 id=203 data-nosnippet>203</a> <span class="kw">use </span>Error::<span class="kw-2">*</span>;
<a href=#204 id=204 data-nosnippet>204</a>
<a href=#205 id=205 data-nosnippet>205</a> <span class="kw">match </span><span class="self">self </span>{
<a href=#206 id=206 data-nosnippet>206</a> ProjectArrowSchema { .. }
<a href=#207 id=207 data-nosnippet>207</a> | ProjectSchema { .. }
<a href=#208 id=208 data-nosnippet>208</a> | MissingRequiredField { .. }
<a href=#209 id=209 data-nosnippet>209</a> | Unsupported { .. }
<a href=#210 id=210 data-nosnippet>210</a> | InvalidMetadata { .. }
<a href=#211 id=211 data-nosnippet>211</a> | ProjectionOutOfBounds { .. }
<a href=#212 id=212 data-nosnippet>212</a> | CreateDefault { .. } =&gt; StatusCode::InvalidArguments,
<a href=#213 id=213 data-nosnippet>213</a>
<a href=#214 id=214 data-nosnippet>214</a> RegionNotFound { .. } =&gt; StatusCode::RegionNotFound,
<a href=#215 id=215 data-nosnippet>215</a>
<a href=#216 id=216 data-nosnippet>216</a> BuildBackend { source, .. } =&gt; source.status_code(),
<a href=#217 id=217 data-nosnippet>217</a> ParseFileFormat { source, .. } =&gt; source.status_code(),
<a href=#218 id=218 data-nosnippet>218</a>
<a href=#219 id=219 data-nosnippet>219</a> CheckObject { .. }
<a href=#220 id=220 data-nosnippet>220</a> | StoreRegionManifest { .. }
<a href=#221 id=221 data-nosnippet>221</a> | LoadRegionManifest { .. }
<a href=#222 id=222 data-nosnippet>222</a> | DeleteRegionManifest { .. } =&gt; StatusCode::StorageUnavailable,
<a href=#223 id=223 data-nosnippet>223</a>
<a href=#224 id=224 data-nosnippet>224</a> EncodeJson { .. }
<a href=#225 id=225 data-nosnippet>225</a> | DecodeJson { .. }
<a href=#226 id=226 data-nosnippet>226</a> | ManifestExists { .. }
<a href=#227 id=227 data-nosnippet>227</a> | BuildStream { .. }
<a href=#228 id=228 data-nosnippet>228</a> | ParquetScanPlan { .. }
<a href=#229 id=229 data-nosnippet>229</a> | UnexpectedEngine { .. } =&gt; StatusCode::Unexpected,
<a href=#230 id=230 data-nosnippet>230</a>
<a href=#231 id=231 data-nosnippet>231</a> DataFusion { .. } =&gt; StatusCode::Internal,
<a href=#232 id=232 data-nosnippet>232</a> }
<a href=#233 id=233 data-nosnippet>233</a> }
<a href=#234 id=234 data-nosnippet>234</a>
<a href=#235 id=235 data-nosnippet>235</a> <span class="kw">fn </span>as_any(<span class="kw-2">&amp;</span><span class="self">self</span>) -&gt; <span class="kw-2">&amp;</span><span class="kw">dyn </span>Any {
<a href=#236 id=236 data-nosnippet>236</a> <span class="self">self
<a href=#237 id=237 data-nosnippet>237</a> </span>}
<a href=#238 id=238 data-nosnippet>238</a>
<a href=#239 id=239 data-nosnippet>239</a> <span class="kw">fn </span>retry_hint(<span class="kw-2">&amp;</span><span class="self">self</span>) -&gt; RetryHint {
<a href=#240 id=240 data-nosnippet>240</a> <span class="kw">use </span>Error::<span class="kw-2">*</span>;
<a href=#241 id=241 data-nosnippet>241</a>
<a href=#242 id=242 data-nosnippet>242</a> <span class="kw">match </span><span class="self">self </span>{
<a href=#243 id=243 data-nosnippet>243</a> BuildBackend { source, .. } | ParseFileFormat { source, .. } =&gt; source.retry_hint(),
<a href=#244 id=244 data-nosnippet>244</a> CheckObject { error, .. }
<a href=#245 id=245 data-nosnippet>245</a> | StoreRegionManifest { error, .. }
<a href=#246 id=246 data-nosnippet>246</a> | LoadRegionManifest { error, .. }
<a href=#247 id=247 data-nosnippet>247</a> | DeleteRegionManifest { error, .. } =&gt; retry_hint_from_opendal_error(error),
<a href=#248 id=248 data-nosnippet>248</a> <span class="kw">_ </span>=&gt; RetryHint::NonRetryable,
<a href=#249 id=249 data-nosnippet>249</a> }
<a href=#250 id=250 data-nosnippet>250</a> }
<a href=#251 id=251 data-nosnippet>251</a>}
</code></pre></div></section></main></body></html>
+33
View File
@@ -0,0 +1,33 @@
<!DOCTYPE html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1.0"><meta name="generator" content="rustdoc"><meta name="description" content="Source of the Rust file `src/file-engine/src/lib.rs`."><title>lib.rs - source</title><script>if(window.location.protocol!=="file:")document.head.insertAdjacentHTML("beforeend","SourceSerif4-Regular-6b053e98.ttf.woff2,FiraSans-Italic-81dc35de.woff2,FiraSans-Regular-0fe48ade.woff2,FiraSans-MediumItalic-ccf7e434.woff2,FiraSans-Medium-e1aa3f0a.woff2,SourceCodePro-Regular-8badfe75.ttf.woff2,SourceCodePro-Semibold-aa29a496.ttf.woff2".split(",").map(f=>`<link rel="preload" as="font" type="font/woff2"href="../../static.files/${f}">`).join(""))</script><link rel="stylesheet" href="../../static.files/normalize-9960930a.css"><link rel="stylesheet" href="../../static.files/rustdoc-17e0aaed.css"><meta name="rustdoc-vars" data-root-path="../../" data-static-root-path="../../static.files/" data-current-crate="file_engine" data-themes="" data-resource-suffix="" data-rustdoc-version="1.96.1 (31fca3adb 2026-06-26)" data-channel="1.96.1" data-search-js="search-b5634cc7.js" data-stringdex-js="stringdex-2da4960a.js" data-settings-js="settings-170eb4bf.js" ><script src="../../static.files/storage-41dd4d93.js"></script><script defer src="../../static.files/src-script-813739b1.js"></script><script defer src="../../src-files.js"></script><script defer src="../../static.files/main-5013f961.js"></script><noscript><link rel="stylesheet" href="../../static.files/noscript-f7c3ffd8.css"></noscript><link rel="alternate icon" type="image/png" href="../../static.files/favicon-32x32-eab170b8.png"><link rel="icon" type="image/svg+xml" href="../../static.files/favicon-044be391.svg"></head><body class="rustdoc src"><a class="skip-main-content" href="#main-content">Skip to main content</a><!--[if lte IE 11]><div class="warning">This old browser is unsupported and will most likely display funky things.</div><![endif]--><nav class="sidebar"><div class="src-sidebar-title"><h2>Files</h2></div></nav><div class="sidebar-resizer" title="Drag to resize sidebar"></div><main><section id="main-content" class="content" tabindex="-1"><div class="main-heading"><h1><div class="sub-heading">file_engine/</div>lib.rs</h1><rustdoc-toolbar></rustdoc-toolbar></div><div class="example-wrap digits-2"><pre class="rust"><code><a href=#1 id=1 data-nosnippet>1</a><span class="comment">// Copyright 2023 Greptime Team
<a href=#2 id=2 data-nosnippet>2</a>//
<a href=#3 id=3 data-nosnippet>3</a>// Licensed under the Apache License, Version 2.0 (the "License");
<a href=#4 id=4 data-nosnippet>4</a>// you may not use this file except in compliance with the License.
<a href=#5 id=5 data-nosnippet>5</a>// You may obtain a copy of the License at
<a href=#6 id=6 data-nosnippet>6</a>//
<a href=#7 id=7 data-nosnippet>7</a>// http://www.apache.org/licenses/LICENSE-2.0
<a href=#8 id=8 data-nosnippet>8</a>//
<a href=#9 id=9 data-nosnippet>9</a>// Unless required by applicable law or agreed to in writing, software
<a href=#10 id=10 data-nosnippet>10</a>// distributed under the License is distributed on an "AS IS" BASIS,
<a href=#11 id=11 data-nosnippet>11</a>// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
<a href=#12 id=12 data-nosnippet>12</a>// See the License for the specific language governing permissions and
<a href=#13 id=13 data-nosnippet>13</a>// limitations under the License.
<a href=#14 id=14 data-nosnippet>14</a>
<a href=#15 id=15 data-nosnippet>15</a></span><span class="kw">pub mod </span>config;
<a href=#16 id=16 data-nosnippet>16</a><span class="kw">pub mod </span>engine;
<a href=#17 id=17 data-nosnippet>17</a><span class="kw">pub mod </span>error;
<a href=#18 id=18 data-nosnippet>18</a><span class="kw">pub</span>(<span class="kw">crate</span>) <span class="kw">mod </span>manifest;
<a href=#19 id=19 data-nosnippet>19</a><span class="kw">pub</span>(<span class="kw">crate</span>) <span class="kw">mod </span>query;
<a href=#20 id=20 data-nosnippet>20</a><span class="kw">pub</span>(<span class="kw">crate</span>) <span class="kw">mod </span>region;
<a href=#21 id=21 data-nosnippet>21</a>
<a href=#22 id=22 data-nosnippet>22</a><span class="attr">#[cfg(any(test, feature = <span class="string">"test"</span>))]
<a href=#23 id=23 data-nosnippet>23</a></span><span class="kw">pub mod </span>test_util;
<a href=#24 id=24 data-nosnippet>24</a>
<a href=#25 id=25 data-nosnippet>25</a><span class="kw">use </span>datatypes::schema::ColumnSchema;
<a href=#26 id=26 data-nosnippet>26</a><span class="kw">use </span>serde::{Deserialize, Serialize};
<a href=#27 id=27 data-nosnippet>27</a>
<a href=#28 id=28 data-nosnippet>28</a><span class="attr">#[derive(Debug, Clone, Serialize, Deserialize)]
<a href=#29 id=29 data-nosnippet>29</a></span><span class="kw">pub struct </span>FileOptions {
<a href=#30 id=30 data-nosnippet>30</a> <span class="kw">pub </span>files: Vec&lt;String&gt;,
<a href=#31 id=31 data-nosnippet>31</a> <span class="kw">pub </span>file_column_schemas: Vec&lt;ColumnSchema&gt;,
<a href=#32 id=32 data-nosnippet>32</a>}
</code></pre></div></section></main></body></html>
+178
View File
@@ -0,0 +1,178 @@
<!DOCTYPE html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1.0"><meta name="generator" content="rustdoc"><meta name="description" content="Source of the Rust file `src/file-engine/src/manifest.rs`."><title>manifest.rs - source</title><script>if(window.location.protocol!=="file:")document.head.insertAdjacentHTML("beforeend","SourceSerif4-Regular-6b053e98.ttf.woff2,FiraSans-Italic-81dc35de.woff2,FiraSans-Regular-0fe48ade.woff2,FiraSans-MediumItalic-ccf7e434.woff2,FiraSans-Medium-e1aa3f0a.woff2,SourceCodePro-Regular-8badfe75.ttf.woff2,SourceCodePro-Semibold-aa29a496.ttf.woff2".split(",").map(f=>`<link rel="preload" as="font" type="font/woff2"href="../../static.files/${f}">`).join(""))</script><link rel="stylesheet" href="../../static.files/normalize-9960930a.css"><link rel="stylesheet" href="../../static.files/rustdoc-17e0aaed.css"><meta name="rustdoc-vars" data-root-path="../../" data-static-root-path="../../static.files/" data-current-crate="file_engine" data-themes="" data-resource-suffix="" data-rustdoc-version="1.96.1 (31fca3adb 2026-06-26)" data-channel="1.96.1" data-search-js="search-b5634cc7.js" data-stringdex-js="stringdex-2da4960a.js" data-settings-js="settings-170eb4bf.js" ><script src="../../static.files/storage-41dd4d93.js"></script><script defer src="../../static.files/src-script-813739b1.js"></script><script defer src="../../src-files.js"></script><script defer src="../../static.files/main-5013f961.js"></script><noscript><link rel="stylesheet" href="../../static.files/noscript-f7c3ffd8.css"></noscript><link rel="alternate icon" type="image/png" href="../../static.files/favicon-32x32-eab170b8.png"><link rel="icon" type="image/svg+xml" href="../../static.files/favicon-044be391.svg"></head><body class="rustdoc src"><a class="skip-main-content" href="#main-content">Skip to main content</a><!--[if lte IE 11]><div class="warning">This old browser is unsupported and will most likely display funky things.</div><![endif]--><nav class="sidebar"><div class="src-sidebar-title"><h2>Files</h2></div></nav><div class="sidebar-resizer" title="Drag to resize sidebar"></div><main><section id="main-content" class="content" tabindex="-1"><div class="main-heading"><h1><div class="sub-heading">file_engine/</div>manifest.rs</h1><rustdoc-toolbar></rustdoc-toolbar></div><div class="example-wrap digits-3"><pre class="rust"><code><a href=#1 id=1 data-nosnippet>1</a><span class="comment">// Copyright 2023 Greptime Team
<a href=#2 id=2 data-nosnippet>2</a>//
<a href=#3 id=3 data-nosnippet>3</a>// Licensed under the Apache License, Version 2.0 (the "License");
<a href=#4 id=4 data-nosnippet>4</a>// you may not use this file except in compliance with the License.
<a href=#5 id=5 data-nosnippet>5</a>// You may obtain a copy of the License at
<a href=#6 id=6 data-nosnippet>6</a>//
<a href=#7 id=7 data-nosnippet>7</a>// http://www.apache.org/licenses/LICENSE-2.0
<a href=#8 id=8 data-nosnippet>8</a>//
<a href=#9 id=9 data-nosnippet>9</a>// Unless required by applicable law or agreed to in writing, software
<a href=#10 id=10 data-nosnippet>10</a>// distributed under the License is distributed on an "AS IS" BASIS,
<a href=#11 id=11 data-nosnippet>11</a>// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
<a href=#12 id=12 data-nosnippet>12</a>// See the License for the specific language governing permissions and
<a href=#13 id=13 data-nosnippet>13</a>// limitations under the License.
<a href=#14 id=14 data-nosnippet>14</a>
<a href=#15 id=15 data-nosnippet>15</a></span><span class="kw">use </span>std::collections::HashMap;
<a href=#16 id=16 data-nosnippet>16</a><span class="kw">use </span>std::sync::Arc;
<a href=#17 id=17 data-nosnippet>17</a>
<a href=#18 id=18 data-nosnippet>18</a><span class="kw">use </span>common_datasource::file_format::Format;
<a href=#19 id=19 data-nosnippet>19</a><span class="kw">use </span>object_store::ObjectStore;
<a href=#20 id=20 data-nosnippet>20</a><span class="kw">use </span>serde::{Deserialize, Serialize};
<a href=#21 id=21 data-nosnippet>21</a><span class="kw">use </span>snafu::{OptionExt, ResultExt, ensure};
<a href=#22 id=22 data-nosnippet>22</a><span class="kw">use </span>store_api::metadata::{ColumnMetadata, RegionMetadataBuilder, RegionMetadataRef};
<a href=#23 id=23 data-nosnippet>23</a><span class="kw">use </span>store_api::storage::{ColumnId, RegionId};
<a href=#24 id=24 data-nosnippet>24</a>
<a href=#25 id=25 data-nosnippet>25</a><span class="kw">use </span><span class="kw">crate</span>::FileOptions;
<a href=#26 id=26 data-nosnippet>26</a><span class="kw">use </span><span class="kw">crate</span>::error::{
<a href=#27 id=27 data-nosnippet>27</a> CheckObjectSnafu, DecodeJsonSnafu, DeleteRegionManifestSnafu, EncodeJsonSnafu,
<a href=#28 id=28 data-nosnippet>28</a> InvalidMetadataSnafu, LoadRegionManifestSnafu, ManifestExistsSnafu, MissingRequiredFieldSnafu,
<a href=#29 id=29 data-nosnippet>29</a> ParseFileFormatSnafu, <span class="prelude-ty">Result</span>, StoreRegionManifestSnafu,
<a href=#30 id=30 data-nosnippet>30</a>};
<a href=#31 id=31 data-nosnippet>31</a>
<a href=#32 id=32 data-nosnippet>32</a><span class="attr">#[inline]
<a href=#33 id=33 data-nosnippet>33</a></span><span class="kw">fn </span>region_manifest_path(region_dir: <span class="kw-2">&amp;</span>str) -&gt; String {
<a href=#34 id=34 data-nosnippet>34</a> <span class="macro">format!</span>(<span class="string">"{region_dir}manifest/_file_manifest"</span>)
<a href=#35 id=35 data-nosnippet>35</a>}
<a href=#36 id=36 data-nosnippet>36</a>
<a href=#37 id=37 data-nosnippet>37</a><span class="attr">#[derive(Debug, Serialize, Deserialize)]
<a href=#38 id=38 data-nosnippet>38</a></span><span class="kw">pub struct </span>FileRegionManifest {
<a href=#39 id=39 data-nosnippet>39</a> <span class="kw">pub </span>region_id: RegionId,
<a href=#40 id=40 data-nosnippet>40</a> <span class="kw">pub </span>column_metadatas: Vec&lt;ColumnMetadata&gt;,
<a href=#41 id=41 data-nosnippet>41</a> <span class="kw">pub </span>primary_key: Vec&lt;ColumnId&gt;,
<a href=#42 id=42 data-nosnippet>42</a> <span class="kw">pub </span>options: HashMap&lt;String, String&gt;,
<a href=#43 id=43 data-nosnippet>43</a>}
<a href=#44 id=44 data-nosnippet>44</a>
<a href=#45 id=45 data-nosnippet>45</a><span class="kw">impl </span>FileRegionManifest {
<a href=#46 id=46 data-nosnippet>46</a> <span class="kw">pub async fn </span>store(<span class="kw-2">&amp;</span><span class="self">self</span>, region_dir: <span class="kw-2">&amp;</span>str, object_store: <span class="kw-2">&amp;</span>ObjectStore) -&gt; <span class="prelude-ty">Result</span>&lt;()&gt; {
<a href=#47 id=47 data-nosnippet>47</a> <span class="kw">let </span>path = <span class="kw-2">&amp;</span>region_manifest_path(region_dir);
<a href=#48 id=48 data-nosnippet>48</a> <span class="kw">let </span>exist = object_store
<a href=#49 id=49 data-nosnippet>49</a> .exists(path)
<a href=#50 id=50 data-nosnippet>50</a> .<span class="kw">await
<a href=#51 id=51 data-nosnippet>51</a> </span>.context(CheckObjectSnafu { path })<span class="question-mark">?</span>;
<a href=#52 id=52 data-nosnippet>52</a> <span class="macro">ensure!</span>(!exist, ManifestExistsSnafu { path });
<a href=#53 id=53 data-nosnippet>53</a>
<a href=#54 id=54 data-nosnippet>54</a> <span class="kw">let </span>bs = <span class="self">self</span>.encode()<span class="question-mark">?</span>;
<a href=#55 id=55 data-nosnippet>55</a> object_store
<a href=#56 id=56 data-nosnippet>56</a> .write(path, bs)
<a href=#57 id=57 data-nosnippet>57</a> .<span class="kw">await
<a href=#58 id=58 data-nosnippet>58</a> </span>.context(StoreRegionManifestSnafu {
<a href=#59 id=59 data-nosnippet>59</a> region_id: <span class="self">self</span>.region_id,
<a href=#60 id=60 data-nosnippet>60</a> })<span class="question-mark">?</span>;
<a href=#61 id=61 data-nosnippet>61</a>
<a href=#62 id=62 data-nosnippet>62</a> <span class="prelude-val">Ok</span>(())
<a href=#63 id=63 data-nosnippet>63</a> }
<a href=#64 id=64 data-nosnippet>64</a>
<a href=#65 id=65 data-nosnippet>65</a> <span class="kw">pub async fn </span>load(
<a href=#66 id=66 data-nosnippet>66</a> region_id: RegionId,
<a href=#67 id=67 data-nosnippet>67</a> region_dir: <span class="kw-2">&amp;</span>str,
<a href=#68 id=68 data-nosnippet>68</a> object_store: <span class="kw-2">&amp;</span>ObjectStore,
<a href=#69 id=69 data-nosnippet>69</a> ) -&gt; <span class="prelude-ty">Result</span>&lt;<span class="self">Self</span>&gt; {
<a href=#70 id=70 data-nosnippet>70</a> <span class="kw">let </span>path = <span class="kw-2">&amp;</span>region_manifest_path(region_dir);
<a href=#71 id=71 data-nosnippet>71</a> <span class="kw">let </span>bs = object_store
<a href=#72 id=72 data-nosnippet>72</a> .read(path)
<a href=#73 id=73 data-nosnippet>73</a> .<span class="kw">await
<a href=#74 id=74 data-nosnippet>74</a> </span>.context(LoadRegionManifestSnafu { region_id })<span class="question-mark">?
<a href=#75 id=75 data-nosnippet>75</a> </span>.to_vec();
<a href=#76 id=76 data-nosnippet>76</a> <span class="self">Self</span>::decode(bs.as_slice())
<a href=#77 id=77 data-nosnippet>77</a> }
<a href=#78 id=78 data-nosnippet>78</a>
<a href=#79 id=79 data-nosnippet>79</a> <span class="kw">pub async fn </span>delete(
<a href=#80 id=80 data-nosnippet>80</a> region_id: RegionId,
<a href=#81 id=81 data-nosnippet>81</a> region_dir: <span class="kw-2">&amp;</span>str,
<a href=#82 id=82 data-nosnippet>82</a> object_store: <span class="kw-2">&amp;</span>ObjectStore,
<a href=#83 id=83 data-nosnippet>83</a> ) -&gt; <span class="prelude-ty">Result</span>&lt;()&gt; {
<a href=#84 id=84 data-nosnippet>84</a> <span class="kw">let </span>path = <span class="kw-2">&amp;</span>region_manifest_path(region_dir);
<a href=#85 id=85 data-nosnippet>85</a> object_store
<a href=#86 id=86 data-nosnippet>86</a> .delete(path)
<a href=#87 id=87 data-nosnippet>87</a> .<span class="kw">await
<a href=#88 id=88 data-nosnippet>88</a> </span>.context(DeleteRegionManifestSnafu { region_id })
<a href=#89 id=89 data-nosnippet>89</a> }
<a href=#90 id=90 data-nosnippet>90</a>
<a href=#91 id=91 data-nosnippet>91</a> <span class="kw">pub fn </span>metadata(<span class="kw-2">&amp;</span><span class="self">self</span>) -&gt; <span class="prelude-ty">Result</span>&lt;RegionMetadataRef&gt; {
<a href=#92 id=92 data-nosnippet>92</a> <span class="kw">let </span><span class="kw-2">mut </span>builder = RegionMetadataBuilder::new(<span class="self">self</span>.region_id);
<a href=#93 id=93 data-nosnippet>93</a> <span class="kw">for </span>column <span class="kw">in </span><span class="kw-2">&amp;</span><span class="self">self</span>.column_metadatas {
<a href=#94 id=94 data-nosnippet>94</a> builder.push_column_metadata(column.clone());
<a href=#95 id=95 data-nosnippet>95</a> }
<a href=#96 id=96 data-nosnippet>96</a> builder.primary_key(<span class="self">self</span>.primary_key.clone());
<a href=#97 id=97 data-nosnippet>97</a> <span class="kw">let </span>metadata = builder
<a href=#98 id=98 data-nosnippet>98</a> .build_without_validation()
<a href=#99 id=99 data-nosnippet>99</a> .context(InvalidMetadataSnafu)<span class="question-mark">?</span>;
<a href=#100 id=100 data-nosnippet>100</a>
<a href=#101 id=101 data-nosnippet>101</a> <span class="prelude-val">Ok</span>(Arc::new(metadata))
<a href=#102 id=102 data-nosnippet>102</a> }
<a href=#103 id=103 data-nosnippet>103</a>
<a href=#104 id=104 data-nosnippet>104</a> <span class="kw">pub fn </span>url(<span class="kw-2">&amp;</span><span class="self">self</span>) -&gt; <span class="prelude-ty">Result</span>&lt;String&gt; {
<a href=#105 id=105 data-nosnippet>105</a> <span class="self">self</span>.get_option(table::requests::FILE_TABLE_LOCATION_KEY)
<a href=#106 id=106 data-nosnippet>106</a> }
<a href=#107 id=107 data-nosnippet>107</a>
<a href=#108 id=108 data-nosnippet>108</a> <span class="kw">pub fn </span>file_options(<span class="kw-2">&amp;</span><span class="self">self</span>) -&gt; <span class="prelude-ty">Result</span>&lt;FileOptions&gt; {
<a href=#109 id=109 data-nosnippet>109</a> <span class="kw">let </span>encoded_opts = <span class="self">self</span>.get_option(table::requests::FILE_TABLE_META_KEY)<span class="question-mark">?</span>;
<a href=#110 id=110 data-nosnippet>110</a> serde_json::from_str(<span class="kw-2">&amp;</span>encoded_opts).context(DecodeJsonSnafu)
<a href=#111 id=111 data-nosnippet>111</a> }
<a href=#112 id=112 data-nosnippet>112</a>
<a href=#113 id=113 data-nosnippet>113</a> <span class="kw">pub fn </span>format(<span class="kw-2">&amp;</span><span class="self">self</span>) -&gt; <span class="prelude-ty">Result</span>&lt;Format&gt; {
<a href=#114 id=114 data-nosnippet>114</a> Format::try_from(<span class="kw-2">&amp;</span><span class="self">self</span>.options).context(ParseFileFormatSnafu)
<a href=#115 id=115 data-nosnippet>115</a> }
<a href=#116 id=116 data-nosnippet>116</a>
<a href=#117 id=117 data-nosnippet>117</a> <span class="kw">fn </span>encode(<span class="kw-2">&amp;</span><span class="self">self</span>) -&gt; <span class="prelude-ty">Result</span>&lt;Vec&lt;u8&gt;&gt; {
<a href=#118 id=118 data-nosnippet>118</a> serde_json::to_vec(<span class="self">self</span>).context(EncodeJsonSnafu)
<a href=#119 id=119 data-nosnippet>119</a> }
<a href=#120 id=120 data-nosnippet>120</a>
<a href=#121 id=121 data-nosnippet>121</a> <span class="kw">fn </span>decode(src: <span class="kw-2">&amp;</span>[u8]) -&gt; <span class="prelude-ty">Result</span>&lt;<span class="self">Self</span>&gt; {
<a href=#122 id=122 data-nosnippet>122</a> serde_json::from_slice(src).context(DecodeJsonSnafu)
<a href=#123 id=123 data-nosnippet>123</a> }
<a href=#124 id=124 data-nosnippet>124</a>
<a href=#125 id=125 data-nosnippet>125</a> <span class="kw">fn </span>get_option(<span class="kw-2">&amp;</span><span class="self">self</span>, name: <span class="kw-2">&amp;</span>str) -&gt; <span class="prelude-ty">Result</span>&lt;String&gt; {
<a href=#126 id=126 data-nosnippet>126</a> <span class="self">self</span>.options
<a href=#127 id=127 data-nosnippet>127</a> .get(name)
<a href=#128 id=128 data-nosnippet>128</a> .cloned()
<a href=#129 id=129 data-nosnippet>129</a> .context(MissingRequiredFieldSnafu { name })
<a href=#130 id=130 data-nosnippet>130</a> }
<a href=#131 id=131 data-nosnippet>131</a>}
<a href=#132 id=132 data-nosnippet>132</a>
<a href=#133 id=133 data-nosnippet>133</a><span class="attr">#[cfg(test)]
<a href=#134 id=134 data-nosnippet>134</a></span><span class="kw">mod </span>tests {
<a href=#135 id=135 data-nosnippet>135</a> <span class="kw">use </span>api::v1::SemanticType;
<a href=#136 id=136 data-nosnippet>136</a> <span class="kw">use </span>datatypes::prelude::ConcreteDataType;
<a href=#137 id=137 data-nosnippet>137</a> <span class="kw">use </span>datatypes::schema::ColumnSchema;
<a href=#138 id=138 data-nosnippet>138</a>
<a href=#139 id=139 data-nosnippet>139</a> <span class="kw">use </span>super::<span class="kw-2">*</span>;
<a href=#140 id=140 data-nosnippet>140</a>
<a href=#141 id=141 data-nosnippet>141</a> <span class="attr">#[test]
<a href=#142 id=142 data-nosnippet>142</a> </span><span class="kw">fn </span>metadata_allows_internal_column_name() {
<a href=#143 id=143 data-nosnippet>143</a> <span class="kw">let </span>manifest = FileRegionManifest {
<a href=#144 id=144 data-nosnippet>144</a> region_id: RegionId::new(<span class="number">1</span>, <span class="number">0</span>),
<a href=#145 id=145 data-nosnippet>145</a> column_metadatas: <span class="macro">vec!</span>[
<a href=#146 id=146 data-nosnippet>146</a> ColumnMetadata {
<a href=#147 id=147 data-nosnippet>147</a> column_schema: ColumnSchema::new(
<a href=#148 id=148 data-nosnippet>148</a> <span class="string">"__primary_key"</span>,
<a href=#149 id=149 data-nosnippet>149</a> ConcreteDataType::string_datatype(),
<a href=#150 id=150 data-nosnippet>150</a> <span class="bool-val">false</span>,
<a href=#151 id=151 data-nosnippet>151</a> ),
<a href=#152 id=152 data-nosnippet>152</a> semantic_type: SemanticType::Tag,
<a href=#153 id=153 data-nosnippet>153</a> column_id: <span class="number">1</span>,
<a href=#154 id=154 data-nosnippet>154</a> },
<a href=#155 id=155 data-nosnippet>155</a> ColumnMetadata {
<a href=#156 id=156 data-nosnippet>156</a> column_schema: ColumnSchema::new(
<a href=#157 id=157 data-nosnippet>157</a> <span class="string">"ts"</span>,
<a href=#158 id=158 data-nosnippet>158</a> ConcreteDataType::timestamp_millisecond_datatype(),
<a href=#159 id=159 data-nosnippet>159</a> <span class="bool-val">false</span>,
<a href=#160 id=160 data-nosnippet>160</a> ),
<a href=#161 id=161 data-nosnippet>161</a> semantic_type: SemanticType::Timestamp,
<a href=#162 id=162 data-nosnippet>162</a> column_id: <span class="number">2</span>,
<a href=#163 id=163 data-nosnippet>163</a> },
<a href=#164 id=164 data-nosnippet>164</a> ],
<a href=#165 id=165 data-nosnippet>165</a> primary_key: <span class="macro">vec!</span>[<span class="number">1</span>],
<a href=#166 id=166 data-nosnippet>166</a> options: HashMap::default(),
<a href=#167 id=167 data-nosnippet>167</a> };
<a href=#168 id=168 data-nosnippet>168</a>
<a href=#169 id=169 data-nosnippet>169</a> <span class="kw">let </span>metadata = manifest.metadata().unwrap();
<a href=#170 id=170 data-nosnippet>170</a> <span class="macro">assert!</span>(
<a href=#171 id=171 data-nosnippet>171</a> metadata
<a href=#172 id=172 data-nosnippet>172</a> .column_metadatas
<a href=#173 id=173 data-nosnippet>173</a> .iter()
<a href=#174 id=174 data-nosnippet>174</a> .any(|c| c.column_schema.name == <span class="string">"__primary_key"</span>)
<a href=#175 id=175 data-nosnippet>175</a> );
<a href=#176 id=176 data-nosnippet>176</a> }
<a href=#177 id=177 data-nosnippet>177</a>}
</code></pre></div></section></main></body></html>
+258
View File
@@ -0,0 +1,258 @@
<!DOCTYPE html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1.0"><meta name="generator" content="rustdoc"><meta name="description" content="Source of the Rust file `src/file-engine/src/query.rs`."><title>query.rs - source</title><script>if(window.location.protocol!=="file:")document.head.insertAdjacentHTML("beforeend","SourceSerif4-Regular-6b053e98.ttf.woff2,FiraSans-Italic-81dc35de.woff2,FiraSans-Regular-0fe48ade.woff2,FiraSans-MediumItalic-ccf7e434.woff2,FiraSans-Medium-e1aa3f0a.woff2,SourceCodePro-Regular-8badfe75.ttf.woff2,SourceCodePro-Semibold-aa29a496.ttf.woff2".split(",").map(f=>`<link rel="preload" as="font" type="font/woff2"href="../../static.files/${f}">`).join(""))</script><link rel="stylesheet" href="../../static.files/normalize-9960930a.css"><link rel="stylesheet" href="../../static.files/rustdoc-17e0aaed.css"><meta name="rustdoc-vars" data-root-path="../../" data-static-root-path="../../static.files/" data-current-crate="file_engine" data-themes="" data-resource-suffix="" data-rustdoc-version="1.96.1 (31fca3adb 2026-06-26)" data-channel="1.96.1" data-search-js="search-b5634cc7.js" data-stringdex-js="stringdex-2da4960a.js" data-settings-js="settings-170eb4bf.js" ><script src="../../static.files/storage-41dd4d93.js"></script><script defer src="../../static.files/src-script-813739b1.js"></script><script defer src="../../src-files.js"></script><script defer src="../../static.files/main-5013f961.js"></script><noscript><link rel="stylesheet" href="../../static.files/noscript-f7c3ffd8.css"></noscript><link rel="alternate icon" type="image/png" href="../../static.files/favicon-32x32-eab170b8.png"><link rel="icon" type="image/svg+xml" href="../../static.files/favicon-044be391.svg"></head><body class="rustdoc src"><a class="skip-main-content" href="#main-content">Skip to main content</a><!--[if lte IE 11]><div class="warning">This old browser is unsupported and will most likely display funky things.</div><![endif]--><nav class="sidebar"><div class="src-sidebar-title"><h2>Files</h2></div></nav><div class="sidebar-resizer" title="Drag to resize sidebar"></div><main><section id="main-content" class="content" tabindex="-1"><div class="main-heading"><h1><div class="sub-heading">file_engine/</div>query.rs</h1><rustdoc-toolbar></rustdoc-toolbar></div><div class="example-wrap digits-3"><pre class="rust"><code><a href=#1 id=1 data-nosnippet>1</a><span class="comment">// Copyright 2023 Greptime Team
<a href=#2 id=2 data-nosnippet>2</a>//
<a href=#3 id=3 data-nosnippet>3</a>// Licensed under the Apache License, Version 2.0 (the "License");
<a href=#4 id=4 data-nosnippet>4</a>// you may not use this file except in compliance with the License.
<a href=#5 id=5 data-nosnippet>5</a>// You may obtain a copy of the License at
<a href=#6 id=6 data-nosnippet>6</a>//
<a href=#7 id=7 data-nosnippet>7</a>// http://www.apache.org/licenses/LICENSE-2.0
<a href=#8 id=8 data-nosnippet>8</a>//
<a href=#9 id=9 data-nosnippet>9</a>// Unless required by applicable law or agreed to in writing, software
<a href=#10 id=10 data-nosnippet>10</a>// distributed under the License is distributed on an "AS IS" BASIS,
<a href=#11 id=11 data-nosnippet>11</a>// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
<a href=#12 id=12 data-nosnippet>12</a>// See the License for the specific language governing permissions and
<a href=#13 id=13 data-nosnippet>13</a>// limitations under the License.
<a href=#14 id=14 data-nosnippet>14</a>
<a href=#15 id=15 data-nosnippet>15</a></span><span class="kw">pub</span>(<span class="kw">crate</span>) <span class="kw">mod </span>file_stream;
<a href=#16 id=16 data-nosnippet>16</a>
<a href=#17 id=17 data-nosnippet>17</a><span class="kw">use </span>std::collections::HashSet;
<a href=#18 id=18 data-nosnippet>18</a><span class="kw">use </span>std::pin::Pin;
<a href=#19 id=19 data-nosnippet>19</a><span class="kw">use </span>std::sync::Arc;
<a href=#20 id=20 data-nosnippet>20</a><span class="kw">use </span>std::task::{Context, Poll};
<a href=#21 id=21 data-nosnippet>21</a>
<a href=#22 id=22 data-nosnippet>22</a><span class="kw">use </span>common_datasource::object_store::{LocalFileAccess, build_backend};
<a href=#23 id=23 data-nosnippet>23</a><span class="kw">use </span>common_recordbatch::adapter::RecordBatchMetrics;
<a href=#24 id=24 data-nosnippet>24</a><span class="kw">use </span>common_recordbatch::error::{<span class="self">self </span><span class="kw">as </span>recordbatch_error, <span class="prelude-ty">Result </span><span class="kw">as </span>RecordBatchResult};
<a href=#25 id=25 data-nosnippet>25</a><span class="kw">use </span>common_recordbatch::{
<a href=#26 id=26 data-nosnippet>26</a> DfSendableRecordBatchStream, OrderOption, RecordBatch, RecordBatchStream,
<a href=#27 id=27 data-nosnippet>27</a> SendableRecordBatchStream,
<a href=#28 id=28 data-nosnippet>28</a>};
<a href=#29 id=29 data-nosnippet>29</a><span class="kw">use </span>datafusion::logical_expr::utils <span class="kw">as </span>df_logical_expr_utils;
<a href=#30 id=30 data-nosnippet>30</a><span class="kw">use </span>datafusion_expr::expr::Expr;
<a href=#31 id=31 data-nosnippet>31</a><span class="kw">use </span>datatypes::arrow::compute <span class="kw">as </span>arrow_compute;
<a href=#32 id=32 data-nosnippet>32</a><span class="kw">use </span>datatypes::data_type::DataType;
<a href=#33 id=33 data-nosnippet>33</a><span class="kw">use </span>datatypes::schema::{Schema, SchemaRef};
<a href=#34 id=34 data-nosnippet>34</a><span class="kw">use </span>datatypes::vectors::Helper;
<a href=#35 id=35 data-nosnippet>35</a><span class="kw">use </span>futures::Stream;
<a href=#36 id=36 data-nosnippet>36</a><span class="kw">use </span>snafu::{GenerateImplicitData, ResultExt, ensure};
<a href=#37 id=37 data-nosnippet>37</a><span class="kw">use </span>store_api::storage::ScanRequest;
<a href=#38 id=38 data-nosnippet>38</a>
<a href=#39 id=39 data-nosnippet>39</a><span class="kw">use </span><span class="self">self</span>::file_stream::ScanPlanConfig;
<a href=#40 id=40 data-nosnippet>40</a><span class="kw">use </span><span class="kw">crate</span>::error::{BuildBackendSnafu, ProjectSchemaSnafu, ProjectionOutOfBoundsSnafu, <span class="prelude-ty">Result</span>};
<a href=#41 id=41 data-nosnippet>41</a><span class="kw">use </span><span class="kw">crate</span>::region::FileRegion;
<a href=#42 id=42 data-nosnippet>42</a>
<a href=#43 id=43 data-nosnippet>43</a><span class="kw">impl </span>FileRegion {
<a href=#44 id=44 data-nosnippet>44</a> <span class="kw">pub async fn </span>query(
<a href=#45 id=45 data-nosnippet>45</a> <span class="kw-2">&amp;</span><span class="self">self</span>,
<a href=#46 id=46 data-nosnippet>46</a> request: ScanRequest,
<a href=#47 id=47 data-nosnippet>47</a> local_file_access: <span class="kw-2">&amp;</span>LocalFileAccess,
<a href=#48 id=48 data-nosnippet>48</a> ) -&gt; <span class="prelude-ty">Result</span>&lt;SendableRecordBatchStream&gt; {
<a href=#49 id=49 data-nosnippet>49</a> <span class="kw">let </span>store = build_backend(<span class="kw-2">&amp;</span><span class="self">self</span>.url, <span class="kw-2">&amp;</span><span class="self">self</span>.options, local_file_access)
<a href=#50 id=50 data-nosnippet>50</a> .<span class="kw">await
<a href=#51 id=51 data-nosnippet>51</a> </span>.context(BuildBackendSnafu)<span class="question-mark">?</span>;
<a href=#52 id=52 data-nosnippet>52</a>
<a href=#53 id=53 data-nosnippet>53</a> <span class="kw">let </span>projection = request.projection.as_deref();
<a href=#54 id=54 data-nosnippet>54</a> <span class="kw">let </span>file_projection = <span class="self">self</span>.projection_pushdown_to_file(projection)<span class="question-mark">?</span>;
<a href=#55 id=55 data-nosnippet>55</a> <span class="kw">let </span>file_filters = <span class="self">self</span>.filters_pushdown_to_file(<span class="kw-2">&amp;</span>request.filters)<span class="question-mark">?</span>;
<a href=#56 id=56 data-nosnippet>56</a> <span class="kw">let </span>file_schema = Arc::new(Schema::new(<span class="self">self</span>.file_options.file_column_schemas.clone()));
<a href=#57 id=57 data-nosnippet>57</a>
<a href=#58 id=58 data-nosnippet>58</a> <span class="kw">let </span>projected_file_schema = <span class="kw">if let </span><span class="prelude-val">Some</span>(projection) = <span class="kw-2">&amp;</span>file_projection {
<a href=#59 id=59 data-nosnippet>59</a> Arc::new(
<a href=#60 id=60 data-nosnippet>60</a> file_schema
<a href=#61 id=61 data-nosnippet>61</a> .try_project(projection)
<a href=#62 id=62 data-nosnippet>62</a> .context(ProjectSchemaSnafu)<span class="question-mark">?</span>,
<a href=#63 id=63 data-nosnippet>63</a> )
<a href=#64 id=64 data-nosnippet>64</a> } <span class="kw">else </span>{
<a href=#65 id=65 data-nosnippet>65</a> file_schema.clone()
<a href=#66 id=66 data-nosnippet>66</a> };
<a href=#67 id=67 data-nosnippet>67</a>
<a href=#68 id=68 data-nosnippet>68</a> <span class="kw">let </span>file_stream = file_stream::create_stream(
<a href=#69 id=69 data-nosnippet>69</a> <span class="kw-2">&amp;</span><span class="self">self</span>.format,
<a href=#70 id=70 data-nosnippet>70</a> <span class="kw-2">&amp;</span>ScanPlanConfig {
<a href=#71 id=71 data-nosnippet>71</a> file_schema,
<a href=#72 id=72 data-nosnippet>72</a> files: <span class="kw-2">&amp;</span><span class="self">self</span>.file_options.files,
<a href=#73 id=73 data-nosnippet>73</a> projection: file_projection.as_ref(),
<a href=#74 id=74 data-nosnippet>74</a> filters: <span class="kw-2">&amp;</span>file_filters,
<a href=#75 id=75 data-nosnippet>75</a> limit: request.limit,
<a href=#76 id=76 data-nosnippet>76</a> store,
<a href=#77 id=77 data-nosnippet>77</a> },
<a href=#78 id=78 data-nosnippet>78</a> )<span class="question-mark">?</span>;
<a href=#79 id=79 data-nosnippet>79</a>
<a href=#80 id=80 data-nosnippet>80</a> <span class="kw">let </span>scan_schema = <span class="self">self</span>.scan_schema(projection)<span class="question-mark">?</span>;
<a href=#81 id=81 data-nosnippet>81</a>
<a href=#82 id=82 data-nosnippet>82</a> <span class="prelude-val">Ok</span>(Box::pin(FileToScanRegionStream::new(
<a href=#83 id=83 data-nosnippet>83</a> scan_schema,
<a href=#84 id=84 data-nosnippet>84</a> projected_file_schema,
<a href=#85 id=85 data-nosnippet>85</a> file_stream,
<a href=#86 id=86 data-nosnippet>86</a> )))
<a href=#87 id=87 data-nosnippet>87</a> }
<a href=#88 id=88 data-nosnippet>88</a>
<a href=#89 id=89 data-nosnippet>89</a> <span class="kw">fn </span>projection_pushdown_to_file(
<a href=#90 id=90 data-nosnippet>90</a> <span class="kw-2">&amp;</span><span class="self">self</span>,
<a href=#91 id=91 data-nosnippet>91</a> req_projection: <span class="prelude-ty">Option</span>&lt;<span class="kw-2">&amp;</span>[usize]&gt;,
<a href=#92 id=92 data-nosnippet>92</a> ) -&gt; <span class="prelude-ty">Result</span>&lt;<span class="prelude-ty">Option</span>&lt;Vec&lt;usize&gt;&gt;&gt; {
<a href=#93 id=93 data-nosnippet>93</a> <span class="kw">let </span><span class="prelude-val">Some</span>(scan_projection) = req_projection <span class="kw">else </span>{
<a href=#94 id=94 data-nosnippet>94</a> <span class="kw">return </span><span class="prelude-val">Ok</span>(<span class="prelude-val">None</span>);
<a href=#95 id=95 data-nosnippet>95</a> };
<a href=#96 id=96 data-nosnippet>96</a>
<a href=#97 id=97 data-nosnippet>97</a> <span class="kw">let </span>file_column_schemas = <span class="kw-2">&amp;</span><span class="self">self</span>.file_options.file_column_schemas;
<a href=#98 id=98 data-nosnippet>98</a> <span class="kw">let </span><span class="kw-2">mut </span>file_projection = Vec::with_capacity(scan_projection.len());
<a href=#99 id=99 data-nosnippet>99</a> <span class="kw">for </span>column_index <span class="kw">in </span>scan_projection {
<a href=#100 id=100 data-nosnippet>100</a> <span class="macro">ensure!</span>(
<a href=#101 id=101 data-nosnippet>101</a> <span class="kw-2">*</span>column_index &lt; <span class="self">self</span>.metadata.schema.num_columns(),
<a href=#102 id=102 data-nosnippet>102</a> ProjectionOutOfBoundsSnafu {
<a href=#103 id=103 data-nosnippet>103</a> column_index: <span class="kw-2">*</span>column_index,
<a href=#104 id=104 data-nosnippet>104</a> bounds: <span class="self">self</span>.metadata.schema.num_columns()
<a href=#105 id=105 data-nosnippet>105</a> }
<a href=#106 id=106 data-nosnippet>106</a> );
<a href=#107 id=107 data-nosnippet>107</a>
<a href=#108 id=108 data-nosnippet>108</a> <span class="kw">let </span>column_name = <span class="self">self</span>.metadata.schema.column_name_by_index(<span class="kw-2">*</span>column_index);
<a href=#109 id=109 data-nosnippet>109</a> <span class="kw">let </span>file_column_index = file_column_schemas
<a href=#110 id=110 data-nosnippet>110</a> .iter()
<a href=#111 id=111 data-nosnippet>111</a> .position(|c| c.name == column_name);
<a href=#112 id=112 data-nosnippet>112</a> <span class="kw">if let </span><span class="prelude-val">Some</span>(file_column_index) = file_column_index {
<a href=#113 id=113 data-nosnippet>113</a> file_projection.push(file_column_index);
<a href=#114 id=114 data-nosnippet>114</a> }
<a href=#115 id=115 data-nosnippet>115</a> }
<a href=#116 id=116 data-nosnippet>116</a> <span class="prelude-val">Ok</span>(<span class="prelude-val">Some</span>(file_projection))
<a href=#117 id=117 data-nosnippet>117</a> }
<a href=#118 id=118 data-nosnippet>118</a>
<a href=#119 id=119 data-nosnippet>119</a> <span class="comment">// Collects filters that can be pushed down to the file, specifically filters where Expr
<a href=#120 id=120 data-nosnippet>120</a> // only contains columns from the file.
<a href=#121 id=121 data-nosnippet>121</a> </span><span class="kw">fn </span>filters_pushdown_to_file(<span class="kw-2">&amp;</span><span class="self">self</span>, scan_filters: <span class="kw-2">&amp;</span>[Expr]) -&gt; <span class="prelude-ty">Result</span>&lt;Vec&lt;Expr&gt;&gt; {
<a href=#122 id=122 data-nosnippet>122</a> <span class="kw">let </span><span class="kw-2">mut </span>file_filters = Vec::with_capacity(scan_filters.len());
<a href=#123 id=123 data-nosnippet>123</a>
<a href=#124 id=124 data-nosnippet>124</a> <span class="kw">let </span>file_column_names = <span class="self">self
<a href=#125 id=125 data-nosnippet>125</a> </span>.file_options
<a href=#126 id=126 data-nosnippet>126</a> .file_column_schemas
<a href=#127 id=127 data-nosnippet>127</a> .iter()
<a href=#128 id=128 data-nosnippet>128</a> .map(|c| <span class="kw-2">&amp;</span>c.name)
<a href=#129 id=129 data-nosnippet>129</a> .collect::&lt;HashSet&lt;<span class="kw">_</span>&gt;&gt;();
<a href=#130 id=130 data-nosnippet>130</a>
<a href=#131 id=131 data-nosnippet>131</a> <span class="kw">let </span><span class="kw-2">mut </span>aux_column_set = HashSet::new();
<a href=#132 id=132 data-nosnippet>132</a> <span class="kw">for </span>scan_filter <span class="kw">in </span>scan_filters {
<a href=#133 id=133 data-nosnippet>133</a> df_logical_expr_utils::expr_to_columns(scan_filter, <span class="kw-2">&amp;mut </span>aux_column_set)<span class="question-mark">?</span>;
<a href=#134 id=134 data-nosnippet>134</a>
<a href=#135 id=135 data-nosnippet>135</a> <span class="kw">let </span>all_file_columns = aux_column_set
<a href=#136 id=136 data-nosnippet>136</a> .iter()
<a href=#137 id=137 data-nosnippet>137</a> .all(|column_in_expr| file_column_names.contains(<span class="kw-2">&amp;</span>column_in_expr.name));
<a href=#138 id=138 data-nosnippet>138</a> <span class="kw">if </span>all_file_columns {
<a href=#139 id=139 data-nosnippet>139</a> file_filters.push(scan_filter.clone());
<a href=#140 id=140 data-nosnippet>140</a> }
<a href=#141 id=141 data-nosnippet>141</a> aux_column_set.clear();
<a href=#142 id=142 data-nosnippet>142</a> }
<a href=#143 id=143 data-nosnippet>143</a> <span class="prelude-val">Ok</span>(file_filters)
<a href=#144 id=144 data-nosnippet>144</a> }
<a href=#145 id=145 data-nosnippet>145</a>
<a href=#146 id=146 data-nosnippet>146</a> <span class="kw">fn </span>scan_schema(<span class="kw-2">&amp;</span><span class="self">self</span>, req_projection: <span class="prelude-ty">Option</span>&lt;<span class="kw-2">&amp;</span>[usize]&gt;) -&gt; <span class="prelude-ty">Result</span>&lt;SchemaRef&gt; {
<a href=#147 id=147 data-nosnippet>147</a> <span class="kw">let </span>schema = <span class="kw">if let </span><span class="prelude-val">Some</span>(indices) = req_projection {
<a href=#148 id=148 data-nosnippet>148</a> Arc::new(
<a href=#149 id=149 data-nosnippet>149</a> <span class="self">self</span>.metadata
<a href=#150 id=150 data-nosnippet>150</a> .schema
<a href=#151 id=151 data-nosnippet>151</a> .try_project(indices)
<a href=#152 id=152 data-nosnippet>152</a> .context(ProjectSchemaSnafu)<span class="question-mark">?</span>,
<a href=#153 id=153 data-nosnippet>153</a> )
<a href=#154 id=154 data-nosnippet>154</a> } <span class="kw">else </span>{
<a href=#155 id=155 data-nosnippet>155</a> <span class="self">self</span>.metadata.schema.clone()
<a href=#156 id=156 data-nosnippet>156</a> };
<a href=#157 id=157 data-nosnippet>157</a>
<a href=#158 id=158 data-nosnippet>158</a> <span class="prelude-val">Ok</span>(schema)
<a href=#159 id=159 data-nosnippet>159</a> }
<a href=#160 id=160 data-nosnippet>160</a>}
<a href=#161 id=161 data-nosnippet>161</a>
<a href=#162 id=162 data-nosnippet>162</a><span class="kw">struct </span>FileToScanRegionStream {
<a href=#163 id=163 data-nosnippet>163</a> scan_schema: SchemaRef,
<a href=#164 id=164 data-nosnippet>164</a> file_stream: DfSendableRecordBatchStream,
<a href=#165 id=165 data-nosnippet>165</a> <span class="doccomment">/// Maps columns in `scan_schema` to their index in the projected file schema.
<a href=#166 id=166 data-nosnippet>166</a> /// `None` means the column doesn't exist in the file and should be filled with default values.
<a href=#167 id=167 data-nosnippet>167</a> </span>scan_to_file_projection: Vec&lt;<span class="prelude-ty">Option</span>&lt;usize&gt;&gt;,
<a href=#168 id=168 data-nosnippet>168</a>}
<a href=#169 id=169 data-nosnippet>169</a>
<a href=#170 id=170 data-nosnippet>170</a><span class="kw">impl </span>RecordBatchStream <span class="kw">for </span>FileToScanRegionStream {
<a href=#171 id=171 data-nosnippet>171</a> <span class="kw">fn </span>schema(<span class="kw-2">&amp;</span><span class="self">self</span>) -&gt; SchemaRef {
<a href=#172 id=172 data-nosnippet>172</a> <span class="self">self</span>.scan_schema.clone()
<a href=#173 id=173 data-nosnippet>173</a> }
<a href=#174 id=174 data-nosnippet>174</a>
<a href=#175 id=175 data-nosnippet>175</a> <span class="kw">fn </span>output_ordering(<span class="kw-2">&amp;</span><span class="self">self</span>) -&gt; <span class="prelude-ty">Option</span>&lt;<span class="kw-2">&amp;</span>[OrderOption]&gt; {
<a href=#176 id=176 data-nosnippet>176</a> <span class="prelude-val">None
<a href=#177 id=177 data-nosnippet>177</a> </span>}
<a href=#178 id=178 data-nosnippet>178</a>
<a href=#179 id=179 data-nosnippet>179</a> <span class="kw">fn </span>metrics(<span class="kw-2">&amp;</span><span class="self">self</span>) -&gt; <span class="prelude-ty">Option</span>&lt;RecordBatchMetrics&gt; {
<a href=#180 id=180 data-nosnippet>180</a> <span class="prelude-val">None
<a href=#181 id=181 data-nosnippet>181</a> </span>}
<a href=#182 id=182 data-nosnippet>182</a>}
<a href=#183 id=183 data-nosnippet>183</a>
<a href=#184 id=184 data-nosnippet>184</a><span class="kw">impl </span>Stream <span class="kw">for </span>FileToScanRegionStream {
<a href=#185 id=185 data-nosnippet>185</a> <span class="kw">type </span>Item = RecordBatchResult&lt;RecordBatch&gt;;
<a href=#186 id=186 data-nosnippet>186</a>
<a href=#187 id=187 data-nosnippet>187</a> <span class="kw">fn </span>poll_next(<span class="kw-2">mut </span><span class="self">self</span>: Pin&lt;<span class="kw-2">&amp;mut </span><span class="self">Self</span>&gt;, ctx: <span class="kw-2">&amp;mut </span>Context&lt;<span class="lifetime">'_</span>&gt;) -&gt; Poll&lt;<span class="prelude-ty">Option</span>&lt;<span class="self">Self</span>::Item&gt;&gt; {
<a href=#188 id=188 data-nosnippet>188</a> <span class="kw">match </span>Pin::new(<span class="kw-2">&amp;mut </span><span class="self">self</span>.file_stream).poll_next(ctx) {
<a href=#189 id=189 data-nosnippet>189</a> Poll::Pending =&gt; Poll::Pending,
<a href=#190 id=190 data-nosnippet>190</a> Poll::Ready(<span class="prelude-val">Some</span>(<span class="prelude-val">Ok</span>(file_record_batch))) =&gt; {
<a href=#191 id=191 data-nosnippet>191</a> <span class="kw">let </span>num_rows = file_record_batch.num_rows();
<a href=#192 id=192 data-nosnippet>192</a> <span class="kw">let </span><span class="kw-2">mut </span>columns = Vec::with_capacity(<span class="self">self</span>.scan_schema.num_columns());
<a href=#193 id=193 data-nosnippet>193</a>
<a href=#194 id=194 data-nosnippet>194</a> <span class="kw">for </span>(idx, column_schema) <span class="kw">in </span><span class="self">self</span>.scan_schema.column_schemas().iter().enumerate() {
<a href=#195 id=195 data-nosnippet>195</a> <span class="kw">if let </span><span class="prelude-val">Some</span>(file_idx) = <span class="self">self</span>.scan_to_file_projection[idx] {
<a href=#196 id=196 data-nosnippet>196</a> <span class="kw">let </span>expected_arrow_type = column_schema.data_type.as_arrow_type();
<a href=#197 id=197 data-nosnippet>197</a> <span class="kw">let </span><span class="kw-2">mut </span>array = file_record_batch.column(file_idx).clone();
<a href=#198 id=198 data-nosnippet>198</a>
<a href=#199 id=199 data-nosnippet>199</a> <span class="kw">if </span>array.data_type() != <span class="kw-2">&amp;</span>expected_arrow_type {
<a href=#200 id=200 data-nosnippet>200</a> array = arrow_compute::cast(array.as_ref(), <span class="kw-2">&amp;</span>expected_arrow_type)
<a href=#201 id=201 data-nosnippet>201</a> .context(recordbatch_error::ArrowComputeSnafu)<span class="question-mark">?</span>;
<a href=#202 id=202 data-nosnippet>202</a> }
<a href=#203 id=203 data-nosnippet>203</a>
<a href=#204 id=204 data-nosnippet>204</a> <span class="kw">let </span>vector = Helper::try_into_vector(array)
<a href=#205 id=205 data-nosnippet>205</a> .context(recordbatch_error::DataTypesSnafu)<span class="question-mark">?</span>;
<a href=#206 id=206 data-nosnippet>206</a> columns.push(vector);
<a href=#207 id=207 data-nosnippet>207</a> } <span class="kw">else </span>{
<a href=#208 id=208 data-nosnippet>208</a> <span class="kw">let </span>vector = column_schema
<a href=#209 id=209 data-nosnippet>209</a> .create_default_vector(num_rows)
<a href=#210 id=210 data-nosnippet>210</a> .context(recordbatch_error::DataTypesSnafu)<span class="question-mark">?
<a href=#211 id=211 data-nosnippet>211</a> </span>.ok_or_else(|| {
<a href=#212 id=212 data-nosnippet>212</a> recordbatch_error::CreateRecordBatchesSnafu {
<a href=#213 id=213 data-nosnippet>213</a> reason: <span class="macro">format!</span>(
<a href=#214 id=214 data-nosnippet>214</a> <span class="string">"column {} is missing from file source and has no default"</span>,
<a href=#215 id=215 data-nosnippet>215</a> column_schema.name
<a href=#216 id=216 data-nosnippet>216</a> ),
<a href=#217 id=217 data-nosnippet>217</a> }
<a href=#218 id=218 data-nosnippet>218</a> .build()
<a href=#219 id=219 data-nosnippet>219</a> })<span class="question-mark">?</span>;
<a href=#220 id=220 data-nosnippet>220</a> columns.push(vector);
<a href=#221 id=221 data-nosnippet>221</a> }
<a href=#222 id=222 data-nosnippet>222</a> }
<a href=#223 id=223 data-nosnippet>223</a>
<a href=#224 id=224 data-nosnippet>224</a> <span class="kw">let </span>record_batch = RecordBatch::new(<span class="self">self</span>.scan_schema.clone(), columns)<span class="question-mark">?</span>;
<a href=#225 id=225 data-nosnippet>225</a>
<a href=#226 id=226 data-nosnippet>226</a> Poll::Ready(<span class="prelude-val">Some</span>(<span class="prelude-val">Ok</span>(record_batch)))
<a href=#227 id=227 data-nosnippet>227</a> }
<a href=#228 id=228 data-nosnippet>228</a> Poll::Ready(<span class="prelude-val">Some</span>(<span class="prelude-val">Err</span>(error))) =&gt; {
<a href=#229 id=229 data-nosnippet>229</a> Poll::Ready(<span class="prelude-val">Some</span>(<span class="prelude-val">Err</span>(recordbatch_error::Error::PollStream {
<a href=#230 id=230 data-nosnippet>230</a> error,
<a href=#231 id=231 data-nosnippet>231</a> location: snafu::Location::generate(),
<a href=#232 id=232 data-nosnippet>232</a> })))
<a href=#233 id=233 data-nosnippet>233</a> }
<a href=#234 id=234 data-nosnippet>234</a> Poll::Ready(<span class="prelude-val">None</span>) =&gt; Poll::Ready(<span class="prelude-val">None</span>),
<a href=#235 id=235 data-nosnippet>235</a> }
<a href=#236 id=236 data-nosnippet>236</a> }
<a href=#237 id=237 data-nosnippet>237</a>}
<a href=#238 id=238 data-nosnippet>238</a>
<a href=#239 id=239 data-nosnippet>239</a><span class="kw">impl </span>FileToScanRegionStream {
<a href=#240 id=240 data-nosnippet>240</a> <span class="kw">fn </span>new(
<a href=#241 id=241 data-nosnippet>241</a> scan_schema: SchemaRef,
<a href=#242 id=242 data-nosnippet>242</a> file_schema: SchemaRef,
<a href=#243 id=243 data-nosnippet>243</a> file_stream: DfSendableRecordBatchStream,
<a href=#244 id=244 data-nosnippet>244</a> ) -&gt; <span class="self">Self </span>{
<a href=#245 id=245 data-nosnippet>245</a> <span class="kw">let </span>scan_to_file_projection = scan_schema
<a href=#246 id=246 data-nosnippet>246</a> .column_schemas()
<a href=#247 id=247 data-nosnippet>247</a> .iter()
<a href=#248 id=248 data-nosnippet>248</a> .map(|column| file_schema.column_index_by_name(<span class="kw-2">&amp;</span>column.name))
<a href=#249 id=249 data-nosnippet>249</a> .collect();
<a href=#250 id=250 data-nosnippet>250</a>
<a href=#251 id=251 data-nosnippet>251</a> <span class="self">Self </span>{
<a href=#252 id=252 data-nosnippet>252</a> scan_schema,
<a href=#253 id=253 data-nosnippet>253</a> file_stream,
<a href=#254 id=254 data-nosnippet>254</a> scan_to_file_projection,
<a href=#255 id=255 data-nosnippet>255</a> }
<a href=#256 id=256 data-nosnippet>256</a> }
<a href=#257 id=257 data-nosnippet>257</a>}
</code></pre></div></section></main></body></html>
+201
View File
@@ -0,0 +1,201 @@
<!DOCTYPE html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1.0"><meta name="generator" content="rustdoc"><meta name="description" content="Source of the Rust file `src/file-engine/src/query/file_stream.rs`."><title>file_stream.rs - source</title><script>if(window.location.protocol!=="file:")document.head.insertAdjacentHTML("beforeend","SourceSerif4-Regular-6b053e98.ttf.woff2,FiraSans-Italic-81dc35de.woff2,FiraSans-Regular-0fe48ade.woff2,FiraSans-MediumItalic-ccf7e434.woff2,FiraSans-Medium-e1aa3f0a.woff2,SourceCodePro-Regular-8badfe75.ttf.woff2,SourceCodePro-Semibold-aa29a496.ttf.woff2".split(",").map(f=>`<link rel="preload" as="font" type="font/woff2"href="../../../static.files/${f}">`).join(""))</script><link rel="stylesheet" href="../../../static.files/normalize-9960930a.css"><link rel="stylesheet" href="../../../static.files/rustdoc-17e0aaed.css"><meta name="rustdoc-vars" data-root-path="../../../" data-static-root-path="../../../static.files/" data-current-crate="file_engine" data-themes="" data-resource-suffix="" data-rustdoc-version="1.96.1 (31fca3adb 2026-06-26)" data-channel="1.96.1" data-search-js="search-b5634cc7.js" data-stringdex-js="stringdex-2da4960a.js" data-settings-js="settings-170eb4bf.js" ><script src="../../../static.files/storage-41dd4d93.js"></script><script defer src="../../../static.files/src-script-813739b1.js"></script><script defer src="../../../src-files.js"></script><script defer src="../../../static.files/main-5013f961.js"></script><noscript><link rel="stylesheet" href="../../../static.files/noscript-f7c3ffd8.css"></noscript><link rel="alternate icon" type="image/png" href="../../../static.files/favicon-32x32-eab170b8.png"><link rel="icon" type="image/svg+xml" href="../../../static.files/favicon-044be391.svg"></head><body class="rustdoc src"><a class="skip-main-content" href="#main-content">Skip to main content</a><!--[if lte IE 11]><div class="warning">This old browser is unsupported and will most likely display funky things.</div><![endif]--><nav class="sidebar"><div class="src-sidebar-title"><h2>Files</h2></div></nav><div class="sidebar-resizer" title="Drag to resize sidebar"></div><main><section id="main-content" class="content" tabindex="-1"><div class="main-heading"><h1><div class="sub-heading">file_engine/query/</div>file_stream.rs</h1><rustdoc-toolbar></rustdoc-toolbar></div><div class="example-wrap digits-3"><pre class="rust"><code><a href=#1 id=1 data-nosnippet>1</a><span class="comment">// Copyright 2023 Greptime Team
<a href=#2 id=2 data-nosnippet>2</a>//
<a href=#3 id=3 data-nosnippet>3</a>// Licensed under the Apache License, Version 2.0 (the "License");
<a href=#4 id=4 data-nosnippet>4</a>// you may not use this file except in compliance with the License.
<a href=#5 id=5 data-nosnippet>5</a>// You may obtain a copy of the License at
<a href=#6 id=6 data-nosnippet>6</a>//
<a href=#7 id=7 data-nosnippet>7</a>// http://www.apache.org/licenses/LICENSE-2.0
<a href=#8 id=8 data-nosnippet>8</a>//
<a href=#9 id=9 data-nosnippet>9</a>// Unless required by applicable law or agreed to in writing, software
<a href=#10 id=10 data-nosnippet>10</a>// distributed under the License is distributed on an "AS IS" BASIS,
<a href=#11 id=11 data-nosnippet>11</a>// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
<a href=#12 id=12 data-nosnippet>12</a>// See the License for the specific language governing permissions and
<a href=#13 id=13 data-nosnippet>13</a>// limitations under the License.
<a href=#14 id=14 data-nosnippet>14</a>
<a href=#15 id=15 data-nosnippet>15</a></span><span class="kw">use </span>std::sync::Arc;
<a href=#16 id=16 data-nosnippet>16</a>
<a href=#17 id=17 data-nosnippet>17</a><span class="kw">use </span>common_datasource::file_format::Format;
<a href=#18 id=18 data-nosnippet>18</a><span class="kw">use </span>common_datasource::file_format::csv::CsvFormat;
<a href=#19 id=19 data-nosnippet>19</a><span class="kw">use </span>common_datasource::file_format::orc::OrcSource;
<a href=#20 id=20 data-nosnippet>20</a><span class="kw">use </span>common_datasource::file_format::parquet::DefaultParquetFileReaderFactory;
<a href=#21 id=21 data-nosnippet>21</a><span class="kw">use </span>datafusion::common::ToDFSchema;
<a href=#22 id=22 data-nosnippet>22</a><span class="kw">use </span>datafusion::config::CsvOptions;
<a href=#23 id=23 data-nosnippet>23</a><span class="kw">use </span>datafusion::datasource::listing::PartitionedFile;
<a href=#24 id=24 data-nosnippet>24</a><span class="kw">use </span>datafusion::datasource::object_store::ObjectStoreUrl;
<a href=#25 id=25 data-nosnippet>25</a><span class="kw">use </span>datafusion::datasource::physical_plan::{
<a href=#26 id=26 data-nosnippet>26</a> CsvSource, FileGroup, FileScanConfigBuilder, FileSource, FileStreamBuilder, JsonSource,
<a href=#27 id=27 data-nosnippet>27</a> ParquetSource,
<a href=#28 id=28 data-nosnippet>28</a>};
<a href=#29 id=29 data-nosnippet>29</a><span class="kw">use </span>datafusion::datasource::source::DataSourceExec;
<a href=#30 id=30 data-nosnippet>30</a><span class="kw">use </span>datafusion::physical_expr::create_physical_expr;
<a href=#31 id=31 data-nosnippet>31</a><span class="kw">use </span>datafusion::physical_expr::execution_props::ExecutionProps;
<a href=#32 id=32 data-nosnippet>32</a><span class="kw">use </span>datafusion::physical_plan::metrics::ExecutionPlanMetricsSet;
<a href=#33 id=33 data-nosnippet>33</a><span class="kw">use </span>datafusion::physical_plan::{
<a href=#34 id=34 data-nosnippet>34</a> ExecutionPlan, SendableRecordBatchStream <span class="kw">as </span>DfSendableRecordBatchStream,
<a href=#35 id=35 data-nosnippet>35</a>};
<a href=#36 id=36 data-nosnippet>36</a><span class="kw">use </span>datafusion::prelude::SessionContext;
<a href=#37 id=37 data-nosnippet>37</a><span class="kw">use </span>datafusion_expr::expr::Expr;
<a href=#38 id=38 data-nosnippet>38</a><span class="kw">use </span>datafusion_expr::physical_planning_context::PhysicalPlanningContext;
<a href=#39 id=39 data-nosnippet>39</a><span class="kw">use </span>datafusion_expr::utils::conjunction;
<a href=#40 id=40 data-nosnippet>40</a><span class="kw">use </span>datatypes::schema::SchemaRef;
<a href=#41 id=41 data-nosnippet>41</a><span class="kw">use </span>object_store::ObjectStore;
<a href=#42 id=42 data-nosnippet>42</a><span class="kw">use </span>snafu::ResultExt;
<a href=#43 id=43 data-nosnippet>43</a>
<a href=#44 id=44 data-nosnippet>44</a><span class="kw">use </span><span class="kw">crate</span>::error::{<span class="self">self</span>, <span class="prelude-ty">Result</span>};
<a href=#45 id=45 data-nosnippet>45</a>
<a href=#46 id=46 data-nosnippet>46</a><span class="kw">const </span>DEFAULT_BATCH_SIZE: usize = <span class="number">8192</span>;
<a href=#47 id=47 data-nosnippet>47</a>
<a href=#48 id=48 data-nosnippet>48</a><span class="kw">fn </span>build_record_batch_stream(
<a href=#49 id=49 data-nosnippet>49</a> scan_plan_config: <span class="kw-2">&amp;</span>ScanPlanConfig,
<a href=#50 id=50 data-nosnippet>50</a> limit: <span class="prelude-ty">Option</span>&lt;usize&gt;,
<a href=#51 id=51 data-nosnippet>51</a> file_source: Arc&lt;<span class="kw">dyn </span>FileSource&gt;,
<a href=#52 id=52 data-nosnippet>52</a>) -&gt; <span class="prelude-ty">Result</span>&lt;DfSendableRecordBatchStream&gt; {
<a href=#53 id=53 data-nosnippet>53</a> <span class="kw">let </span>files = scan_plan_config
<a href=#54 id=54 data-nosnippet>54</a> .files
<a href=#55 id=55 data-nosnippet>55</a> .iter()
<a href=#56 id=56 data-nosnippet>56</a> .map(|filename| PartitionedFile::new(filename.clone(), <span class="number">0</span>))
<a href=#57 id=57 data-nosnippet>57</a> .collect::&lt;Vec&lt;<span class="kw">_</span>&gt;&gt;();
<a href=#58 id=58 data-nosnippet>58</a>
<a href=#59 id=59 data-nosnippet>59</a> <span class="kw">let </span>config =
<a href=#60 id=60 data-nosnippet>60</a> FileScanConfigBuilder::new(ObjectStoreUrl::local_filesystem(), file_source.clone())
<a href=#61 id=61 data-nosnippet>61</a> .with_projection_indices(scan_plan_config.projection.cloned())<span class="question-mark">?
<a href=#62 id=62 data-nosnippet>62</a> </span>.with_limit(limit)
<a href=#63 id=63 data-nosnippet>63</a> .with_file_group(FileGroup::new(files))
<a href=#64 id=64 data-nosnippet>64</a> .build();
<a href=#65 id=65 data-nosnippet>65</a>
<a href=#66 id=66 data-nosnippet>66</a> <span class="kw">let </span>store = Arc::new(object_store_opendal::OpendalStore::new(
<a href=#67 id=67 data-nosnippet>67</a> scan_plan_config.store.clone(),
<a href=#68 id=68 data-nosnippet>68</a> ));
<a href=#69 id=69 data-nosnippet>69</a>
<a href=#70 id=70 data-nosnippet>70</a> <span class="kw">let </span>file_opener = config.file_source().create_file_opener(store, <span class="kw-2">&amp;</span>config, <span class="number">0</span>)<span class="question-mark">?</span>;
<a href=#71 id=71 data-nosnippet>71</a> <span class="kw">let </span>stream = FileStreamBuilder::new(<span class="kw-2">&amp;</span>config)
<a href=#72 id=72 data-nosnippet>72</a> .with_partition(<span class="number">0</span>) <span class="comment">// partition: hard-code
<a href=#73 id=73 data-nosnippet>73</a> </span>.with_file_opener(file_opener)
<a href=#74 id=74 data-nosnippet>74</a> .with_metrics(<span class="kw-2">&amp;</span>ExecutionPlanMetricsSet::new())
<a href=#75 id=75 data-nosnippet>75</a> .build()
<a href=#76 id=76 data-nosnippet>76</a> .context(error::BuildStreamSnafu)<span class="question-mark">?</span>;
<a href=#77 id=77 data-nosnippet>77</a> <span class="prelude-val">Ok</span>(Box::pin(stream))
<a href=#78 id=78 data-nosnippet>78</a>}
<a href=#79 id=79 data-nosnippet>79</a>
<a href=#80 id=80 data-nosnippet>80</a><span class="kw">fn </span>new_csv_stream(
<a href=#81 id=81 data-nosnippet>81</a> config: <span class="kw-2">&amp;</span>ScanPlanConfig,
<a href=#82 id=82 data-nosnippet>82</a> format: <span class="kw-2">&amp;</span>CsvFormat,
<a href=#83 id=83 data-nosnippet>83</a>) -&gt; <span class="prelude-ty">Result</span>&lt;DfSendableRecordBatchStream&gt; {
<a href=#84 id=84 data-nosnippet>84</a> <span class="kw">let </span>file_schema = config.file_schema.arrow_schema().clone();
<a href=#85 id=85 data-nosnippet>85</a>
<a href=#86 id=86 data-nosnippet>86</a> <span class="comment">// push down limit only if there is no filter
<a href=#87 id=87 data-nosnippet>87</a> </span><span class="kw">let </span>limit = config.filters.is_empty().then_some(config.limit).flatten();
<a href=#88 id=88 data-nosnippet>88</a>
<a href=#89 id=89 data-nosnippet>89</a> <span class="kw">let </span>options = CsvOptions::default()
<a href=#90 id=90 data-nosnippet>90</a> .with_has_header(format.has_header)
<a href=#91 id=91 data-nosnippet>91</a> .with_delimiter(format.delimiter);
<a href=#92 id=92 data-nosnippet>92</a> <span class="kw">let </span>csv_source = CsvSource::new(file_schema)
<a href=#93 id=93 data-nosnippet>93</a> .with_csv_options(options)
<a href=#94 id=94 data-nosnippet>94</a> .with_batch_size(DEFAULT_BATCH_SIZE);
<a href=#95 id=95 data-nosnippet>95</a>
<a href=#96 id=96 data-nosnippet>96</a> build_record_batch_stream(config, limit, csv_source)
<a href=#97 id=97 data-nosnippet>97</a>}
<a href=#98 id=98 data-nosnippet>98</a>
<a href=#99 id=99 data-nosnippet>99</a><span class="kw">fn </span>new_json_stream(config: <span class="kw-2">&amp;</span>ScanPlanConfig) -&gt; <span class="prelude-ty">Result</span>&lt;DfSendableRecordBatchStream&gt; {
<a href=#100 id=100 data-nosnippet>100</a> <span class="kw">let </span>file_schema = config.file_schema.arrow_schema().clone();
<a href=#101 id=101 data-nosnippet>101</a>
<a href=#102 id=102 data-nosnippet>102</a> <span class="comment">// push down limit only if there is no filter
<a href=#103 id=103 data-nosnippet>103</a> </span><span class="kw">let </span>limit = config.filters.is_empty().then_some(config.limit).flatten();
<a href=#104 id=104 data-nosnippet>104</a>
<a href=#105 id=105 data-nosnippet>105</a> <span class="kw">let </span>file_source = JsonSource::new(file_schema).with_batch_size(DEFAULT_BATCH_SIZE);
<a href=#106 id=106 data-nosnippet>106</a> build_record_batch_stream(config, limit, file_source)
<a href=#107 id=107 data-nosnippet>107</a>}
<a href=#108 id=108 data-nosnippet>108</a>
<a href=#109 id=109 data-nosnippet>109</a><span class="kw">fn </span>new_parquet_stream_with_exec_plan(
<a href=#110 id=110 data-nosnippet>110</a> config: <span class="kw-2">&amp;</span>ScanPlanConfig,
<a href=#111 id=111 data-nosnippet>111</a>) -&gt; <span class="prelude-ty">Result</span>&lt;DfSendableRecordBatchStream&gt; {
<a href=#112 id=112 data-nosnippet>112</a> <span class="kw">let </span>file_schema = config.file_schema.arrow_schema().clone();
<a href=#113 id=113 data-nosnippet>113</a> <span class="kw">let </span>ScanPlanConfig {
<a href=#114 id=114 data-nosnippet>114</a> files,
<a href=#115 id=115 data-nosnippet>115</a> projection,
<a href=#116 id=116 data-nosnippet>116</a> limit,
<a href=#117 id=117 data-nosnippet>117</a> filters,
<a href=#118 id=118 data-nosnippet>118</a> store,
<a href=#119 id=119 data-nosnippet>119</a> ..
<a href=#120 id=120 data-nosnippet>120</a> } = config;
<a href=#121 id=121 data-nosnippet>121</a>
<a href=#122 id=122 data-nosnippet>122</a> <span class="kw">let </span>file_group = FileGroup::new(
<a href=#123 id=123 data-nosnippet>123</a> files
<a href=#124 id=124 data-nosnippet>124</a> .iter()
<a href=#125 id=125 data-nosnippet>125</a> .map(|filename| PartitionedFile::new(filename.clone(), <span class="number">0</span>))
<a href=#126 id=126 data-nosnippet>126</a> .collect::&lt;Vec&lt;<span class="kw">_</span>&gt;&gt;(),
<a href=#127 id=127 data-nosnippet>127</a> );
<a href=#128 id=128 data-nosnippet>128</a>
<a href=#129 id=129 data-nosnippet>129</a> <span class="kw">let </span><span class="kw-2">mut </span>parquet_source = ParquetSource::new(file_schema.clone())
<a href=#130 id=130 data-nosnippet>130</a> .with_parquet_file_reader_factory(Arc::new(DefaultParquetFileReaderFactory::new(
<a href=#131 id=131 data-nosnippet>131</a> store.clone(),
<a href=#132 id=132 data-nosnippet>132</a> )));
<a href=#133 id=133 data-nosnippet>133</a>
<a href=#134 id=134 data-nosnippet>134</a> <span class="comment">// build predicate filter
<a href=#135 id=135 data-nosnippet>135</a> </span><span class="kw">let </span>filters = filters.to_vec();
<a href=#136 id=136 data-nosnippet>136</a> <span class="kw">if let </span><span class="prelude-val">Some</span>(expr) = conjunction(filters) {
<a href=#137 id=137 data-nosnippet>137</a> <span class="kw">let </span>df_schema = file_schema
<a href=#138 id=138 data-nosnippet>138</a> .clone()
<a href=#139 id=139 data-nosnippet>139</a> .to_dfschema_ref()
<a href=#140 id=140 data-nosnippet>140</a> .context(error::ParquetScanPlanSnafu)<span class="question-mark">?</span>;
<a href=#141 id=141 data-nosnippet>141</a>
<a href=#142 id=142 data-nosnippet>142</a> <span class="kw">let </span>filters = create_physical_expr(
<a href=#143 id=143 data-nosnippet>143</a> <span class="kw-2">&amp;</span>expr,
<a href=#144 id=144 data-nosnippet>144</a> <span class="kw-2">&amp;</span>df_schema,
<a href=#145 id=145 data-nosnippet>145</a> <span class="kw-2">&amp;</span>ExecutionProps::new(),
<a href=#146 id=146 data-nosnippet>146</a> <span class="kw-2">&amp;</span>PhysicalPlanningContext::default(),
<a href=#147 id=147 data-nosnippet>147</a> )
<a href=#148 id=148 data-nosnippet>148</a> .context(error::ParquetScanPlanSnafu)<span class="question-mark">?</span>;
<a href=#149 id=149 data-nosnippet>149</a> parquet_source = parquet_source.with_predicate(filters);
<a href=#150 id=150 data-nosnippet>150</a> };
<a href=#151 id=151 data-nosnippet>151</a>
<a href=#152 id=152 data-nosnippet>152</a> <span class="kw">let </span>file_scan_config =
<a href=#153 id=153 data-nosnippet>153</a> FileScanConfigBuilder::new(ObjectStoreUrl::local_filesystem(), Arc::new(parquet_source))
<a href=#154 id=154 data-nosnippet>154</a> .with_file_group(file_group)
<a href=#155 id=155 data-nosnippet>155</a> .with_projection_indices(projection.cloned())<span class="question-mark">?
<a href=#156 id=156 data-nosnippet>156</a> </span>.with_limit(<span class="kw-2">*</span>limit)
<a href=#157 id=157 data-nosnippet>157</a> .build();
<a href=#158 id=158 data-nosnippet>158</a>
<a href=#159 id=159 data-nosnippet>159</a> <span class="comment">// TODO(ruihang): get this from upper layer
<a href=#160 id=160 data-nosnippet>160</a> </span><span class="kw">let </span>task_ctx = SessionContext::default().task_ctx();
<a href=#161 id=161 data-nosnippet>161</a>
<a href=#162 id=162 data-nosnippet>162</a> <span class="kw">let </span>parquet_exec = DataSourceExec::from_data_source(file_scan_config);
<a href=#163 id=163 data-nosnippet>163</a> <span class="kw">let </span>stream = parquet_exec
<a href=#164 id=164 data-nosnippet>164</a> .execute(<span class="number">0</span>, task_ctx)
<a href=#165 id=165 data-nosnippet>165</a> .context(error::ParquetScanPlanSnafu)<span class="question-mark">?</span>;
<a href=#166 id=166 data-nosnippet>166</a>
<a href=#167 id=167 data-nosnippet>167</a> <span class="prelude-val">Ok</span>(stream)
<a href=#168 id=168 data-nosnippet>168</a>}
<a href=#169 id=169 data-nosnippet>169</a>
<a href=#170 id=170 data-nosnippet>170</a><span class="kw">fn </span>new_orc_stream(config: <span class="kw-2">&amp;</span>ScanPlanConfig) -&gt; <span class="prelude-ty">Result</span>&lt;DfSendableRecordBatchStream&gt; {
<a href=#171 id=171 data-nosnippet>171</a> <span class="kw">let </span>file_schema = config.file_schema.arrow_schema().clone();
<a href=#172 id=172 data-nosnippet>172</a>
<a href=#173 id=173 data-nosnippet>173</a> <span class="comment">// push down limit only if there is no filter
<a href=#174 id=174 data-nosnippet>174</a> </span><span class="kw">let </span>limit = config.filters.is_empty().then_some(config.limit).flatten();
<a href=#175 id=175 data-nosnippet>175</a>
<a href=#176 id=176 data-nosnippet>176</a> <span class="kw">let </span>file_source = OrcSource::new(file_schema.into()).with_batch_size(DEFAULT_BATCH_SIZE);
<a href=#177 id=177 data-nosnippet>177</a> build_record_batch_stream(config, limit, file_source)
<a href=#178 id=178 data-nosnippet>178</a>}
<a href=#179 id=179 data-nosnippet>179</a>
<a href=#180 id=180 data-nosnippet>180</a><span class="attr">#[derive(Debug, Clone)]
<a href=#181 id=181 data-nosnippet>181</a></span><span class="kw">pub struct </span>ScanPlanConfig&lt;<span class="lifetime">'a</span>&gt; {
<a href=#182 id=182 data-nosnippet>182</a> <span class="kw">pub </span>file_schema: SchemaRef,
<a href=#183 id=183 data-nosnippet>183</a> <span class="kw">pub </span>files: <span class="kw-2">&amp;</span><span class="lifetime">'a </span>Vec&lt;String&gt;,
<a href=#184 id=184 data-nosnippet>184</a> <span class="kw">pub </span>projection: <span class="prelude-ty">Option</span>&lt;<span class="kw-2">&amp;</span><span class="lifetime">'a </span>Vec&lt;usize&gt;&gt;,
<a href=#185 id=185 data-nosnippet>185</a> <span class="kw">pub </span>filters: <span class="kw-2">&amp;</span><span class="lifetime">'a </span>[Expr],
<a href=#186 id=186 data-nosnippet>186</a> <span class="kw">pub </span>limit: <span class="prelude-ty">Option</span>&lt;usize&gt;,
<a href=#187 id=187 data-nosnippet>187</a> <span class="kw">pub </span>store: ObjectStore,
<a href=#188 id=188 data-nosnippet>188</a>}
<a href=#189 id=189 data-nosnippet>189</a>
<a href=#190 id=190 data-nosnippet>190</a><span class="kw">pub fn </span>create_stream(
<a href=#191 id=191 data-nosnippet>191</a> format: <span class="kw-2">&amp;</span>Format,
<a href=#192 id=192 data-nosnippet>192</a> config: <span class="kw-2">&amp;</span>ScanPlanConfig,
<a href=#193 id=193 data-nosnippet>193</a>) -&gt; <span class="prelude-ty">Result</span>&lt;DfSendableRecordBatchStream&gt; {
<a href=#194 id=194 data-nosnippet>194</a> <span class="kw">match </span>format {
<a href=#195 id=195 data-nosnippet>195</a> Format::Csv(format) =&gt; new_csv_stream(config, format),
<a href=#196 id=196 data-nosnippet>196</a> Format::Json(<span class="kw">_</span>) =&gt; new_json_stream(config),
<a href=#197 id=197 data-nosnippet>197</a> Format::Parquet(<span class="kw">_</span>) =&gt; new_parquet_stream_with_exec_plan(config),
<a href=#198 id=198 data-nosnippet>198</a> Format::Orc(<span class="kw">_</span>) =&gt; new_orc_stream(config),
<a href=#199 id=199 data-nosnippet>199</a> }
<a href=#200 id=200 data-nosnippet>200</a>}
</code></pre></div></section></main></body></html>
+292
View File
@@ -0,0 +1,292 @@
<!DOCTYPE html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1.0"><meta name="generator" content="rustdoc"><meta name="description" content="Source of the Rust file `src/file-engine/src/region.rs`."><title>region.rs - source</title><script>if(window.location.protocol!=="file:")document.head.insertAdjacentHTML("beforeend","SourceSerif4-Regular-6b053e98.ttf.woff2,FiraSans-Italic-81dc35de.woff2,FiraSans-Regular-0fe48ade.woff2,FiraSans-MediumItalic-ccf7e434.woff2,FiraSans-Medium-e1aa3f0a.woff2,SourceCodePro-Regular-8badfe75.ttf.woff2,SourceCodePro-Semibold-aa29a496.ttf.woff2".split(",").map(f=>`<link rel="preload" as="font" type="font/woff2"href="../../static.files/${f}">`).join(""))</script><link rel="stylesheet" href="../../static.files/normalize-9960930a.css"><link rel="stylesheet" href="../../static.files/rustdoc-17e0aaed.css"><meta name="rustdoc-vars" data-root-path="../../" data-static-root-path="../../static.files/" data-current-crate="file_engine" data-themes="" data-resource-suffix="" data-rustdoc-version="1.96.1 (31fca3adb 2026-06-26)" data-channel="1.96.1" data-search-js="search-b5634cc7.js" data-stringdex-js="stringdex-2da4960a.js" data-settings-js="settings-170eb4bf.js" ><script src="../../static.files/storage-41dd4d93.js"></script><script defer src="../../static.files/src-script-813739b1.js"></script><script defer src="../../src-files.js"></script><script defer src="../../static.files/main-5013f961.js"></script><noscript><link rel="stylesheet" href="../../static.files/noscript-f7c3ffd8.css"></noscript><link rel="alternate icon" type="image/png" href="../../static.files/favicon-32x32-eab170b8.png"><link rel="icon" type="image/svg+xml" href="../../static.files/favicon-044be391.svg"></head><body class="rustdoc src"><a class="skip-main-content" href="#main-content">Skip to main content</a><!--[if lte IE 11]><div class="warning">This old browser is unsupported and will most likely display funky things.</div><![endif]--><nav class="sidebar"><div class="src-sidebar-title"><h2>Files</h2></div></nav><div class="sidebar-resizer" title="Drag to resize sidebar"></div><main><section id="main-content" class="content" tabindex="-1"><div class="main-heading"><h1><div class="sub-heading">file_engine/</div>region.rs</h1><rustdoc-toolbar></rustdoc-toolbar></div><div class="example-wrap digits-3"><pre class="rust"><code><a href=#1 id=1 data-nosnippet>1</a><span class="comment">// Copyright 2023 Greptime Team
<a href=#2 id=2 data-nosnippet>2</a>//
<a href=#3 id=3 data-nosnippet>3</a>// Licensed under the Apache License, Version 2.0 (the "License");
<a href=#4 id=4 data-nosnippet>4</a>// you may not use this file except in compliance with the License.
<a href=#5 id=5 data-nosnippet>5</a>// You may obtain a copy of the License at
<a href=#6 id=6 data-nosnippet>6</a>//
<a href=#7 id=7 data-nosnippet>7</a>// http://www.apache.org/licenses/LICENSE-2.0
<a href=#8 id=8 data-nosnippet>8</a>//
<a href=#9 id=9 data-nosnippet>9</a>// Unless required by applicable law or agreed to in writing, software
<a href=#10 id=10 data-nosnippet>10</a>// distributed under the License is distributed on an "AS IS" BASIS,
<a href=#11 id=11 data-nosnippet>11</a>// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
<a href=#12 id=12 data-nosnippet>12</a>// See the License for the specific language governing permissions and
<a href=#13 id=13 data-nosnippet>13</a>// limitations under the License.
<a href=#14 id=14 data-nosnippet>14</a>
<a href=#15 id=15 data-nosnippet>15</a></span><span class="kw">use </span>std::collections::HashMap;
<a href=#16 id=16 data-nosnippet>16</a><span class="kw">use </span>std::sync::Arc;
<a href=#17 id=17 data-nosnippet>17</a>
<a href=#18 id=18 data-nosnippet>18</a><span class="kw">use </span>common_datasource::file_format::Format;
<a href=#19 id=19 data-nosnippet>19</a><span class="kw">use </span>object_store::ObjectStore;
<a href=#20 id=20 data-nosnippet>20</a><span class="kw">use </span>store_api::metadata::RegionMetadataRef;
<a href=#21 id=21 data-nosnippet>21</a><span class="kw">use </span>store_api::path_utils::region_name;
<a href=#22 id=22 data-nosnippet>22</a><span class="kw">use </span>store_api::region_request::{RegionCreateRequest, RegionOpenRequest};
<a href=#23 id=23 data-nosnippet>23</a><span class="kw">use </span>store_api::storage::RegionId;
<a href=#24 id=24 data-nosnippet>24</a>
<a href=#25 id=25 data-nosnippet>25</a><span class="kw">use </span><span class="kw">crate</span>::FileOptions;
<a href=#26 id=26 data-nosnippet>26</a><span class="kw">use </span><span class="kw">crate</span>::error::Result;
<a href=#27 id=27 data-nosnippet>27</a><span class="kw">use </span><span class="kw">crate</span>::manifest::FileRegionManifest;
<a href=#28 id=28 data-nosnippet>28</a>
<a href=#29 id=29 data-nosnippet>29</a><span class="attr">#[derive(Debug)]
<a href=#30 id=30 data-nosnippet>30</a></span><span class="kw">pub struct </span>FileRegion {
<a href=#31 id=31 data-nosnippet>31</a> <span class="kw">pub</span>(<span class="kw">crate</span>) region_dir: String,
<a href=#32 id=32 data-nosnippet>32</a> <span class="kw">pub</span>(<span class="kw">crate</span>) file_options: FileOptions,
<a href=#33 id=33 data-nosnippet>33</a> <span class="kw">pub</span>(<span class="kw">crate</span>) url: String,
<a href=#34 id=34 data-nosnippet>34</a> <span class="kw">pub</span>(<span class="kw">crate</span>) format: Format,
<a href=#35 id=35 data-nosnippet>35</a> <span class="kw">pub</span>(<span class="kw">crate</span>) options: HashMap&lt;String, String&gt;,
<a href=#36 id=36 data-nosnippet>36</a> <span class="kw">pub</span>(<span class="kw">crate</span>) metadata: RegionMetadataRef,
<a href=#37 id=37 data-nosnippet>37</a>}
<a href=#38 id=38 data-nosnippet>38</a>
<a href=#39 id=39 data-nosnippet>39</a><span class="kw">pub type </span>FileRegionRef = Arc&lt;FileRegion&gt;;
<a href=#40 id=40 data-nosnippet>40</a>
<a href=#41 id=41 data-nosnippet>41</a><span class="kw">impl </span>FileRegion {
<a href=#42 id=42 data-nosnippet>42</a> <span class="kw">pub async fn </span>create(
<a href=#43 id=43 data-nosnippet>43</a> region_id: RegionId,
<a href=#44 id=44 data-nosnippet>44</a> request: RegionCreateRequest,
<a href=#45 id=45 data-nosnippet>45</a> object_store: <span class="kw-2">&amp;</span>ObjectStore,
<a href=#46 id=46 data-nosnippet>46</a> ) -&gt; <span class="prelude-ty">Result</span>&lt;FileRegionRef&gt; {
<a href=#47 id=47 data-nosnippet>47</a> <span class="kw">let </span>manifest = FileRegionManifest {
<a href=#48 id=48 data-nosnippet>48</a> region_id,
<a href=#49 id=49 data-nosnippet>49</a> column_metadatas: request.column_metadatas.clone(),
<a href=#50 id=50 data-nosnippet>50</a> primary_key: request.primary_key.clone(),
<a href=#51 id=51 data-nosnippet>51</a> options: request.options,
<a href=#52 id=52 data-nosnippet>52</a> };
<a href=#53 id=53 data-nosnippet>53</a>
<a href=#54 id=54 data-nosnippet>54</a> <span class="kw">let </span>region_dir = object_store::util::join_dir(
<a href=#55 id=55 data-nosnippet>55</a> <span class="kw-2">&amp;</span>request.table_dir,
<a href=#56 id=56 data-nosnippet>56</a> <span class="kw-2">&amp;</span>region_name(region_id.table_id(), region_id.region_sequence()),
<a href=#57 id=57 data-nosnippet>57</a> );
<a href=#58 id=58 data-nosnippet>58</a> <span class="kw">let </span>url = manifest.url()<span class="question-mark">?</span>;
<a href=#59 id=59 data-nosnippet>59</a> <span class="kw">let </span>file_options = manifest.file_options()<span class="question-mark">?</span>;
<a href=#60 id=60 data-nosnippet>60</a> <span class="kw">let </span>format = manifest.format()<span class="question-mark">?</span>;
<a href=#61 id=61 data-nosnippet>61</a> <span class="kw">let </span>options = manifest.options.clone();
<a href=#62 id=62 data-nosnippet>62</a> <span class="kw">let </span>metadata = manifest.metadata()<span class="question-mark">?</span>;
<a href=#63 id=63 data-nosnippet>63</a>
<a href=#64 id=64 data-nosnippet>64</a> manifest.store(<span class="kw-2">&amp;</span>region_dir, object_store).<span class="kw">await</span><span class="question-mark">?</span>;
<a href=#65 id=65 data-nosnippet>65</a>
<a href=#66 id=66 data-nosnippet>66</a> <span class="prelude-val">Ok</span>(Arc::new(<span class="self">Self </span>{
<a href=#67 id=67 data-nosnippet>67</a> region_dir,
<a href=#68 id=68 data-nosnippet>68</a> url,
<a href=#69 id=69 data-nosnippet>69</a> file_options,
<a href=#70 id=70 data-nosnippet>70</a> format,
<a href=#71 id=71 data-nosnippet>71</a> options,
<a href=#72 id=72 data-nosnippet>72</a> metadata,
<a href=#73 id=73 data-nosnippet>73</a> }))
<a href=#74 id=74 data-nosnippet>74</a> }
<a href=#75 id=75 data-nosnippet>75</a>
<a href=#76 id=76 data-nosnippet>76</a> <span class="kw">pub async fn </span>open(
<a href=#77 id=77 data-nosnippet>77</a> region_id: RegionId,
<a href=#78 id=78 data-nosnippet>78</a> request: RegionOpenRequest,
<a href=#79 id=79 data-nosnippet>79</a> object_store: <span class="kw-2">&amp;</span>ObjectStore,
<a href=#80 id=80 data-nosnippet>80</a> ) -&gt; <span class="prelude-ty">Result</span>&lt;FileRegionRef&gt; {
<a href=#81 id=81 data-nosnippet>81</a> <span class="kw">let </span>region_dir = object_store::util::join_dir(
<a href=#82 id=82 data-nosnippet>82</a> <span class="kw-2">&amp;</span>request.table_dir,
<a href=#83 id=83 data-nosnippet>83</a> <span class="kw-2">&amp;</span>region_name(region_id.table_id(), region_id.region_sequence()),
<a href=#84 id=84 data-nosnippet>84</a> );
<a href=#85 id=85 data-nosnippet>85</a> <span class="kw">let </span>manifest = FileRegionManifest::load(region_id, <span class="kw-2">&amp;</span>region_dir, object_store).<span class="kw">await</span><span class="question-mark">?</span>;
<a href=#86 id=86 data-nosnippet>86</a>
<a href=#87 id=87 data-nosnippet>87</a> <span class="prelude-val">Ok</span>(Arc::new(<span class="self">Self </span>{
<a href=#88 id=88 data-nosnippet>88</a> region_dir,
<a href=#89 id=89 data-nosnippet>89</a> url: manifest.url()<span class="question-mark">?</span>,
<a href=#90 id=90 data-nosnippet>90</a> file_options: manifest.file_options()<span class="question-mark">?</span>,
<a href=#91 id=91 data-nosnippet>91</a> format: manifest.format()<span class="question-mark">?</span>,
<a href=#92 id=92 data-nosnippet>92</a> metadata: manifest.metadata()<span class="question-mark">?</span>,
<a href=#93 id=93 data-nosnippet>93</a> options: manifest.options,
<a href=#94 id=94 data-nosnippet>94</a> }))
<a href=#95 id=95 data-nosnippet>95</a> }
<a href=#96 id=96 data-nosnippet>96</a>
<a href=#97 id=97 data-nosnippet>97</a> <span class="kw">pub async fn </span>drop(<span class="kw-2">&amp;</span><span class="self">self</span>, object_store: <span class="kw-2">&amp;</span>ObjectStore) -&gt; <span class="prelude-ty">Result</span>&lt;()&gt; {
<a href=#98 id=98 data-nosnippet>98</a> FileRegionManifest::delete(<span class="self">self</span>.metadata.region_id, <span class="kw-2">&amp;</span><span class="self">self</span>.region_dir, object_store).<span class="kw">await
<a href=#99 id=99 data-nosnippet>99</a> </span>}
<a href=#100 id=100 data-nosnippet>100</a>
<a href=#101 id=101 data-nosnippet>101</a> <span class="kw">pub fn </span>metadata(<span class="kw-2">&amp;</span><span class="self">self</span>) -&gt; RegionMetadataRef {
<a href=#102 id=102 data-nosnippet>102</a> <span class="self">self</span>.metadata.clone()
<a href=#103 id=103 data-nosnippet>103</a> }
<a href=#104 id=104 data-nosnippet>104</a>}
<a href=#105 id=105 data-nosnippet>105</a>
<a href=#106 id=106 data-nosnippet>106</a><span class="attr">#[cfg(test)]
<a href=#107 id=107 data-nosnippet>107</a></span><span class="kw">mod </span>tests {
<a href=#108 id=108 data-nosnippet>108</a> <span class="kw">use </span>std::assert_matches;
<a href=#109 id=109 data-nosnippet>109</a>
<a href=#110 id=110 data-nosnippet>110</a> <span class="kw">use </span>common_datasource::object_store::LocalFileAccess;
<a href=#111 id=111 data-nosnippet>111</a> <span class="kw">use </span>common_error::ext::{ErrorExt, RetryHint};
<a href=#112 id=112 data-nosnippet>112</a> <span class="kw">use </span>common_error::status_code::StatusCode;
<a href=#113 id=113 data-nosnippet>113</a> <span class="kw">use </span>store_api::region_request::PathType;
<a href=#114 id=114 data-nosnippet>114</a> <span class="kw">use </span>store_api::storage::ScanRequest;
<a href=#115 id=115 data-nosnippet>115</a>
<a href=#116 id=116 data-nosnippet>116</a> <span class="kw">use </span>super::<span class="kw-2">*</span>;
<a href=#117 id=117 data-nosnippet>117</a> <span class="kw">use </span><span class="kw">crate</span>::error::Error;
<a href=#118 id=118 data-nosnippet>118</a> <span class="kw">use </span><span class="kw">crate</span>::test_util::{new_test_column_metadata, new_test_object_store, new_test_options};
<a href=#119 id=119 data-nosnippet>119</a>
<a href=#120 id=120 data-nosnippet>120</a> <span class="attr">#[tokio::test]
<a href=#121 id=121 data-nosnippet>121</a> </span><span class="kw">async fn </span>test_create_region() {
<a href=#122 id=122 data-nosnippet>122</a> <span class="kw">let </span>(_dir, object_store) = new_test_object_store(<span class="string">"test_create_region"</span>);
<a href=#123 id=123 data-nosnippet>123</a>
<a href=#124 id=124 data-nosnippet>124</a> <span class="kw">let </span>request = RegionCreateRequest {
<a href=#125 id=125 data-nosnippet>125</a> engine: <span class="string">"file"</span>.to_string(),
<a href=#126 id=126 data-nosnippet>126</a> column_metadatas: new_test_column_metadata(),
<a href=#127 id=127 data-nosnippet>127</a> primary_key: <span class="macro">vec!</span>[<span class="number">1</span>],
<a href=#128 id=128 data-nosnippet>128</a> options: new_test_options(),
<a href=#129 id=129 data-nosnippet>129</a> table_dir: <span class="string">"create_region_dir/"</span>.to_string(),
<a href=#130 id=130 data-nosnippet>130</a> path_type: PathType::Bare,
<a href=#131 id=131 data-nosnippet>131</a> partition_expr_json: <span class="prelude-val">Some</span>(<span class="string">""</span>.to_string()),
<a href=#132 id=132 data-nosnippet>132</a> requirements: Default::default(),
<a href=#133 id=133 data-nosnippet>133</a> };
<a href=#134 id=134 data-nosnippet>134</a> <span class="kw">let </span>region_id = RegionId::new(<span class="number">1</span>, <span class="number">0</span>);
<a href=#135 id=135 data-nosnippet>135</a>
<a href=#136 id=136 data-nosnippet>136</a> <span class="kw">let </span>region = FileRegion::create(region_id, request.clone(), <span class="kw-2">&amp;</span>object_store)
<a href=#137 id=137 data-nosnippet>137</a> .<span class="kw">await
<a href=#138 id=138 data-nosnippet>138</a> </span>.unwrap();
<a href=#139 id=139 data-nosnippet>139</a>
<a href=#140 id=140 data-nosnippet>140</a> <span class="macro">assert_eq!</span>(region.region_dir, <span class="string">"create_region_dir/1_0000000000/"</span>);
<a href=#141 id=141 data-nosnippet>141</a> <span class="macro">assert_eq!</span>(region.url, <span class="string">"test"</span>);
<a href=#142 id=142 data-nosnippet>142</a> <span class="macro">assert_eq!</span>(region.file_options.files, <span class="macro">vec!</span>[<span class="string">"1.csv"</span>]);
<a href=#143 id=143 data-nosnippet>143</a> <span class="macro">assert_matches!</span>(region.format, Format::Csv { .. });
<a href=#144 id=144 data-nosnippet>144</a> <span class="macro">assert_eq!</span>(region.options, new_test_options());
<a href=#145 id=145 data-nosnippet>145</a> <span class="macro">assert_eq!</span>(region.metadata.region_id, region_id);
<a href=#146 id=146 data-nosnippet>146</a> <span class="macro">assert_eq!</span>(region.metadata.primary_key, <span class="macro">vec!</span>[<span class="number">1</span>]);
<a href=#147 id=147 data-nosnippet>147</a>
<a href=#148 id=148 data-nosnippet>148</a> <span class="macro">assert!</span>(
<a href=#149 id=149 data-nosnippet>149</a> object_store
<a href=#150 id=150 data-nosnippet>150</a> .exists(<span class="string">"create_region_dir/1_0000000000/manifest/_file_manifest"</span>)
<a href=#151 id=151 data-nosnippet>151</a> .<span class="kw">await
<a href=#152 id=152 data-nosnippet>152</a> </span>.unwrap()
<a href=#153 id=153 data-nosnippet>153</a> );
<a href=#154 id=154 data-nosnippet>154</a>
<a href=#155 id=155 data-nosnippet>155</a> <span class="comment">// Object exists, should fail
<a href=#156 id=156 data-nosnippet>156</a> </span><span class="kw">let </span>err = FileRegion::create(region_id, request, <span class="kw-2">&amp;</span>object_store)
<a href=#157 id=157 data-nosnippet>157</a> .<span class="kw">await
<a href=#158 id=158 data-nosnippet>158</a> </span>.unwrap_err();
<a href=#159 id=159 data-nosnippet>159</a> <span class="macro">assert_matches!</span>(err, Error::ManifestExists { .. });
<a href=#160 id=160 data-nosnippet>160</a> }
<a href=#161 id=161 data-nosnippet>161</a>
<a href=#162 id=162 data-nosnippet>162</a> <span class="attr">#[tokio::test]
<a href=#163 id=163 data-nosnippet>163</a> </span><span class="kw">async fn </span>test_persisted_local_region_rejected_when_disabled() {
<a href=#164 id=164 data-nosnippet>164</a> <span class="kw">let </span>(_dir, object_store) = new_test_object_store(<span class="string">"test_disabled_local_region"</span>);
<a href=#165 id=165 data-nosnippet>165</a> <span class="kw">let </span>request = RegionCreateRequest {
<a href=#166 id=166 data-nosnippet>166</a> engine: <span class="string">"file"</span>.to_string(),
<a href=#167 id=167 data-nosnippet>167</a> column_metadatas: new_test_column_metadata(),
<a href=#168 id=168 data-nosnippet>168</a> primary_key: <span class="macro">vec!</span>[<span class="number">1</span>],
<a href=#169 id=169 data-nosnippet>169</a> options: new_test_options(),
<a href=#170 id=170 data-nosnippet>170</a> table_dir: <span class="string">"disabled_local_region/"</span>.to_string(),
<a href=#171 id=171 data-nosnippet>171</a> path_type: PathType::Bare,
<a href=#172 id=172 data-nosnippet>172</a> partition_expr_json: <span class="prelude-val">Some</span>(<span class="string">""</span>.to_string()),
<a href=#173 id=173 data-nosnippet>173</a> requirements: Default::default(),
<a href=#174 id=174 data-nosnippet>174</a> };
<a href=#175 id=175 data-nosnippet>175</a> <span class="kw">let </span>region = FileRegion::create(RegionId::new(<span class="number">1</span>, <span class="number">0</span>), request, <span class="kw-2">&amp;</span>object_store)
<a href=#176 id=176 data-nosnippet>176</a> .<span class="kw">await
<a href=#177 id=177 data-nosnippet>177</a> </span>.unwrap();
<a href=#178 id=178 data-nosnippet>178</a>
<a href=#179 id=179 data-nosnippet>179</a> <span class="kw">let </span>error = <span class="kw">match </span>region
<a href=#180 id=180 data-nosnippet>180</a> .query(ScanRequest::default(), <span class="kw-2">&amp;</span>LocalFileAccess::Disabled)
<a href=#181 id=181 data-nosnippet>181</a> .<span class="kw">await
<a href=#182 id=182 data-nosnippet>182</a> </span>{
<a href=#183 id=183 data-nosnippet>183</a> <span class="prelude-val">Ok</span>(<span class="kw">_</span>) =&gt; <span class="macro">panic!</span>(<span class="string">"local file query must be rejected"</span>),
<a href=#184 id=184 data-nosnippet>184</a> <span class="prelude-val">Err</span>(error) =&gt; error,
<a href=#185 id=185 data-nosnippet>185</a> };
<a href=#186 id=186 data-nosnippet>186</a> <span class="macro">assert_matches!</span>(
<a href=#187 id=187 data-nosnippet>187</a> <span class="kw-2">&amp;</span>error,
<a href=#188 id=188 data-nosnippet>188</a> Error::BuildBackend {
<a href=#189 id=189 data-nosnippet>189</a> source: common_datasource::error::Error::LocalFileAccessDisabled { .. },
<a href=#190 id=190 data-nosnippet>190</a> ..
<a href=#191 id=191 data-nosnippet>191</a> }
<a href=#192 id=192 data-nosnippet>192</a> );
<a href=#193 id=193 data-nosnippet>193</a> <span class="macro">assert_eq!</span>(error.status_code(), StatusCode::InvalidArguments);
<a href=#194 id=194 data-nosnippet>194</a> <span class="macro">assert_eq!</span>(error.retry_hint(), RetryHint::NonRetryable);
<a href=#195 id=195 data-nosnippet>195</a> }
<a href=#196 id=196 data-nosnippet>196</a>
<a href=#197 id=197 data-nosnippet>197</a> <span class="attr">#[tokio::test]
<a href=#198 id=198 data-nosnippet>198</a> </span><span class="kw">async fn </span>test_open_region() {
<a href=#199 id=199 data-nosnippet>199</a> <span class="kw">let </span>(_dir, object_store) = new_test_object_store(<span class="string">"test_open_region"</span>);
<a href=#200 id=200 data-nosnippet>200</a>
<a href=#201 id=201 data-nosnippet>201</a> <span class="kw">let </span>region_dir = <span class="string">"open_region_dir/"</span>.to_string();
<a href=#202 id=202 data-nosnippet>202</a> <span class="kw">let </span>request = RegionCreateRequest {
<a href=#203 id=203 data-nosnippet>203</a> engine: <span class="string">"file"</span>.to_string(),
<a href=#204 id=204 data-nosnippet>204</a> column_metadatas: new_test_column_metadata(),
<a href=#205 id=205 data-nosnippet>205</a> primary_key: <span class="macro">vec!</span>[<span class="number">1</span>],
<a href=#206 id=206 data-nosnippet>206</a> options: new_test_options(),
<a href=#207 id=207 data-nosnippet>207</a> table_dir: region_dir.clone(),
<a href=#208 id=208 data-nosnippet>208</a> path_type: PathType::Bare,
<a href=#209 id=209 data-nosnippet>209</a> partition_expr_json: <span class="prelude-val">Some</span>(<span class="string">""</span>.to_string()),
<a href=#210 id=210 data-nosnippet>210</a> requirements: Default::default(),
<a href=#211 id=211 data-nosnippet>211</a> };
<a href=#212 id=212 data-nosnippet>212</a> <span class="kw">let </span>region_id = RegionId::new(<span class="number">1</span>, <span class="number">0</span>);
<a href=#213 id=213 data-nosnippet>213</a>
<a href=#214 id=214 data-nosnippet>214</a> <span class="kw">let _ </span>= FileRegion::create(region_id, request.clone(), <span class="kw-2">&amp;</span>object_store)
<a href=#215 id=215 data-nosnippet>215</a> .<span class="kw">await
<a href=#216 id=216 data-nosnippet>216</a> </span>.unwrap();
<a href=#217 id=217 data-nosnippet>217</a>
<a href=#218 id=218 data-nosnippet>218</a> <span class="kw">let </span>request = RegionOpenRequest {
<a href=#219 id=219 data-nosnippet>219</a> engine: <span class="string">"file"</span>.to_string(),
<a href=#220 id=220 data-nosnippet>220</a> table_dir: region_dir,
<a href=#221 id=221 data-nosnippet>221</a> path_type: PathType::Bare,
<a href=#222 id=222 data-nosnippet>222</a> options: HashMap::default(),
<a href=#223 id=223 data-nosnippet>223</a> skip_wal_replay: <span class="bool-val">false</span>,
<a href=#224 id=224 data-nosnippet>224</a> checkpoint: <span class="prelude-val">None</span>,
<a href=#225 id=225 data-nosnippet>225</a> requirements: Default::default(),
<a href=#226 id=226 data-nosnippet>226</a> };
<a href=#227 id=227 data-nosnippet>227</a>
<a href=#228 id=228 data-nosnippet>228</a> <span class="kw">let </span>region = FileRegion::open(region_id, request, <span class="kw-2">&amp;</span>object_store)
<a href=#229 id=229 data-nosnippet>229</a> .<span class="kw">await
<a href=#230 id=230 data-nosnippet>230</a> </span>.unwrap();
<a href=#231 id=231 data-nosnippet>231</a>
<a href=#232 id=232 data-nosnippet>232</a> <span class="macro">assert_eq!</span>(region.region_dir, <span class="string">"open_region_dir/1_0000000000/"</span>);
<a href=#233 id=233 data-nosnippet>233</a> <span class="macro">assert_eq!</span>(region.url, <span class="string">"test"</span>);
<a href=#234 id=234 data-nosnippet>234</a> <span class="macro">assert_eq!</span>(region.file_options.files, <span class="macro">vec!</span>[<span class="string">"1.csv"</span>]);
<a href=#235 id=235 data-nosnippet>235</a> <span class="macro">assert_matches!</span>(region.format, Format::Csv { .. });
<a href=#236 id=236 data-nosnippet>236</a> <span class="macro">assert_eq!</span>(region.options, new_test_options());
<a href=#237 id=237 data-nosnippet>237</a> <span class="macro">assert_eq!</span>(region.metadata.region_id, region_id);
<a href=#238 id=238 data-nosnippet>238</a> <span class="macro">assert_eq!</span>(region.metadata.primary_key, <span class="macro">vec!</span>[<span class="number">1</span>]);
<a href=#239 id=239 data-nosnippet>239</a> }
<a href=#240 id=240 data-nosnippet>240</a>
<a href=#241 id=241 data-nosnippet>241</a> <span class="attr">#[tokio::test]
<a href=#242 id=242 data-nosnippet>242</a> </span><span class="kw">async fn </span>test_drop_region() {
<a href=#243 id=243 data-nosnippet>243</a> <span class="kw">let </span>(_dir, object_store) = new_test_object_store(<span class="string">"test_drop_region"</span>);
<a href=#244 id=244 data-nosnippet>244</a>
<a href=#245 id=245 data-nosnippet>245</a> <span class="kw">let </span>region_dir = <span class="string">"drop_region_dir/"</span>.to_string();
<a href=#246 id=246 data-nosnippet>246</a> <span class="kw">let </span>request = RegionCreateRequest {
<a href=#247 id=247 data-nosnippet>247</a> engine: <span class="string">"file"</span>.to_string(),
<a href=#248 id=248 data-nosnippet>248</a> column_metadatas: new_test_column_metadata(),
<a href=#249 id=249 data-nosnippet>249</a> primary_key: <span class="macro">vec!</span>[<span class="number">1</span>],
<a href=#250 id=250 data-nosnippet>250</a> options: new_test_options(),
<a href=#251 id=251 data-nosnippet>251</a> table_dir: region_dir.clone(),
<a href=#252 id=252 data-nosnippet>252</a> path_type: PathType::Bare,
<a href=#253 id=253 data-nosnippet>253</a> partition_expr_json: <span class="prelude-val">Some</span>(<span class="string">""</span>.to_string()),
<a href=#254 id=254 data-nosnippet>254</a> requirements: Default::default(),
<a href=#255 id=255 data-nosnippet>255</a> };
<a href=#256 id=256 data-nosnippet>256</a> <span class="kw">let </span>region_id = RegionId::new(<span class="number">1</span>, <span class="number">0</span>);
<a href=#257 id=257 data-nosnippet>257</a>
<a href=#258 id=258 data-nosnippet>258</a> <span class="kw">let </span>region = FileRegion::create(region_id, request.clone(), <span class="kw-2">&amp;</span>object_store)
<a href=#259 id=259 data-nosnippet>259</a> .<span class="kw">await
<a href=#260 id=260 data-nosnippet>260</a> </span>.unwrap();
<a href=#261 id=261 data-nosnippet>261</a>
<a href=#262 id=262 data-nosnippet>262</a> <span class="macro">assert!</span>(
<a href=#263 id=263 data-nosnippet>263</a> object_store
<a href=#264 id=264 data-nosnippet>264</a> .exists(<span class="string">"drop_region_dir/1_0000000000/manifest/_file_manifest"</span>)
<a href=#265 id=265 data-nosnippet>265</a> .<span class="kw">await
<a href=#266 id=266 data-nosnippet>266</a> </span>.unwrap()
<a href=#267 id=267 data-nosnippet>267</a> );
<a href=#268 id=268 data-nosnippet>268</a>
<a href=#269 id=269 data-nosnippet>269</a> FileRegion::drop(<span class="kw-2">&amp;</span>region, <span class="kw-2">&amp;</span>object_store).<span class="kw">await</span>.unwrap();
<a href=#270 id=270 data-nosnippet>270</a> <span class="macro">assert!</span>(
<a href=#271 id=271 data-nosnippet>271</a> !object_store
<a href=#272 id=272 data-nosnippet>272</a> .exists(<span class="string">"drop_region_dir/1_0000000000/manifest/_file_manifest"</span>)
<a href=#273 id=273 data-nosnippet>273</a> .<span class="kw">await
<a href=#274 id=274 data-nosnippet>274</a> </span>.unwrap()
<a href=#275 id=275 data-nosnippet>275</a> );
<a href=#276 id=276 data-nosnippet>276</a>
<a href=#277 id=277 data-nosnippet>277</a> <span class="kw">let </span>request = RegionOpenRequest {
<a href=#278 id=278 data-nosnippet>278</a> engine: <span class="string">"file"</span>.to_string(),
<a href=#279 id=279 data-nosnippet>279</a> table_dir: region_dir,
<a href=#280 id=280 data-nosnippet>280</a> path_type: PathType::Bare,
<a href=#281 id=281 data-nosnippet>281</a> options: HashMap::default(),
<a href=#282 id=282 data-nosnippet>282</a> skip_wal_replay: <span class="bool-val">false</span>,
<a href=#283 id=283 data-nosnippet>283</a> checkpoint: <span class="prelude-val">None</span>,
<a href=#284 id=284 data-nosnippet>284</a> requirements: Default::default(),
<a href=#285 id=285 data-nosnippet>285</a> };
<a href=#286 id=286 data-nosnippet>286</a> <span class="kw">let </span>err = FileRegion::open(region_id, request, <span class="kw-2">&amp;</span>object_store)
<a href=#287 id=287 data-nosnippet>287</a> .<span class="kw">await
<a href=#288 id=288 data-nosnippet>288</a> </span>.unwrap_err();
<a href=#289 id=289 data-nosnippet>289</a> <span class="macro">assert_matches!</span>(err, Error::LoadRegionManifest { .. });
<a href=#290 id=290 data-nosnippet>290</a> }
<a href=#291 id=291 data-nosnippet>291</a>}
</code></pre></div></section></main></body></html>