index/
target.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use std::any::Any;
16
17use common_error::ext::ErrorExt;
18use common_error::status_code::StatusCode;
19use common_macro::stack_trace_debug;
20use datatypes::json::JSON2_REMAINDER_FIELD_NAME;
21use serde::{Deserialize, Serialize};
22use snafu::{OptionExt, Snafu, ensure};
23use store_api::storage::ColumnId;
24
25/// Identifies a column or a JSON object path within a column.
26///
27/// # Encoding
28///
29/// - Columns are encoded as decimal IDs, e.g. `1`.
30/// - JSON path targets use `json-{column_id}:` followed by path segments encoded
31///   as `{UTF-8 byte length}:{segment}`, e.g. `json-1:7:service4:name`.
32///   IDs and lengths are unsigned decimal integers without leading zeros.
33///   Segments are read by byte length, so their contents need no escaping.
34///
35/// This persisted encoding is independent of the serde representation of these types.
36///
37/// Note: Columns retain decimal ID encoding because existing index files use
38/// these IDs as lookup keys. Changing the encoding would prevent queries from
39/// finding those indexes.
40#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
41pub enum IndexTarget {
42    ColumnId(ColumnId),
43    JsonPath(JsonPathTarget),
44}
45
46/// An index target for a JSON object path.
47#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
48pub struct JsonPathTarget {
49    column_id: ColumnId,
50    #[serde(deserialize_with = "deserialize_json_path")]
51    path: Vec<String>,
52}
53
54impl JsonPathTarget {
55    /// Returns the ID of the root column containing this JSON path.
56    pub fn column_id(&self) -> ColumnId {
57        self.column_id
58    }
59
60    /// Returns the JSON object path.
61    pub fn path(&self) -> &[String] {
62        &self.path
63    }
64}
65
66impl IndexTarget {
67    /// Creates an index target for a JSON object path.
68    pub fn new_json_path(column_id: ColumnId, path: Vec<String>) -> Result<Self, TargetKeyError> {
69        validate_json_path(&path)?;
70        let target = JsonPathTarget { column_id, path };
71        Ok(Self::JsonPath(target))
72    }
73
74    /// Encodes this target using the format described in [`IndexTarget`].
75    pub fn encode(&self) -> String {
76        let target = match self {
77            Self::ColumnId(id) => return id.to_string(),
78            Self::JsonPath(target) => target,
79        };
80        let mut key = format!("json-{}:", target.column_id);
81        for part in &target.path {
82            key.push_str(&part.len().to_string());
83            key.push(':');
84            key.push_str(part);
85        }
86        key
87    }
88
89    /// Decodes an index name using the format described in [`IndexTarget`].
90    pub fn decode(key: &str) -> Result<Self, TargetKeyError> {
91        ensure!(!key.is_empty(), EmptySnafu);
92        if key.bytes().all(|byte| byte.is_ascii_digit()) {
93            let id = key
94                .parse::<ColumnId>()
95                .map_err(|_| InvalidColumnIdSnafu { value: key }.build())?;
96            return Ok(Self::ColumnId(id));
97        }
98
99        let invalid_key = || InvalidJsonTargetSnafu { target_key: key };
100        // JSON target IDs and lengths use decimal digits without signs or leading zeros.
101        let is_canonical_decimal = |value: &str| {
102            !value.is_empty()
103                && value.bytes().all(|byte| byte.is_ascii_digit())
104                && (value.len() == 1 || !value.starts_with('0'))
105        };
106
107        // The prefix contains the root column ID; the remainder contains path segments.
108        let (column_id, mut rest) = key
109            .strip_prefix("json-")
110            .and_then(|rest| rest.split_once(':'))
111            .with_context(invalid_key)?;
112        ensure!(is_canonical_decimal(column_id), invalid_key());
113        let column_id = column_id.parse().ok().with_context(invalid_key)?;
114
115        let mut path = Vec::new();
116        while !rest.is_empty() {
117            let (length, content) = rest.split_once(':').with_context(invalid_key)?;
118            ensure!(is_canonical_decimal(length), invalid_key());
119            let length = length.parse::<usize>().ok().with_context(invalid_key)?;
120            // Read exactly this many bytes, including any delimiters in the segment.
121            // Check bounds and the UTF-8 boundary before allocating the segment string.
122            let (part, remaining) = content.split_at_checked(length).with_context(invalid_key)?;
123            path.push(part.to_string());
124            rest = remaining;
125        }
126        Self::new_json_path(column_id, path)
127    }
128}
129
130/// Errors that can occur when working with index target keys.
131#[derive(Snafu, Clone, PartialEq, Eq)]
132#[stack_trace_debug]
133pub enum TargetKeyError {
134    #[snafu(display("target key cannot be empty"))]
135    Empty,
136
137    #[snafu(display("target key must contain digits only: {key}"))]
138    InvalidCharacters { key: String },
139
140    #[snafu(display("failed to parse column id from '{value}'"))]
141    InvalidColumnId { value: String },
142
143    #[snafu(display("invalid JSON index target key: {target_key}"))]
144    InvalidJsonTarget { target_key: String },
145
146    #[snafu(display("invalid JSON index path: {detail}"))]
147    InvalidJsonPath { detail: String },
148}
149
150fn deserialize_json_path<'de, D>(deserializer: D) -> Result<Vec<String>, D::Error>
151where
152    D: serde::Deserializer<'de>,
153{
154    let path = Vec::<String>::deserialize(deserializer)?;
155    validate_json_path(&path).map_err(serde::de::Error::custom)?;
156    Ok(path)
157}
158
159fn validate_json_path(path: &[String]) -> Result<(), TargetKeyError> {
160    ensure!(
161        !path.is_empty(),
162        InvalidJsonPathSnafu {
163            detail: "path must not be empty",
164        }
165    );
166    for (index, part) in path.iter().enumerate() {
167        ensure!(
168            !part.is_empty(),
169            InvalidJsonPathSnafu {
170                detail: format!("path segment at index {index} must not be empty"),
171            }
172        );
173        ensure!(
174            part != JSON2_REMAINDER_FIELD_NAME,
175            InvalidJsonPathSnafu {
176                detail: format!("path segment at index {index} uses reserved name '{part}'"),
177            }
178        );
179    }
180    Ok(())
181}
182
183impl ErrorExt for TargetKeyError {
184    fn status_code(&self) -> StatusCode {
185        StatusCode::InvalidArguments
186    }
187
188    fn as_any(&self) -> &dyn Any {
189        self
190    }
191}
192
193#[cfg(test)]
194mod tests {
195    use super::*;
196
197    #[test]
198    fn encode_decode_column() {
199        let target = IndexTarget::ColumnId(42);
200        let key = target.encode();
201        assert_eq!(key, "42");
202        assert_eq!(IndexTarget::decode("42").unwrap(), target);
203        let decoded = IndexTarget::decode(&key).unwrap();
204        assert_eq!(decoded, target);
205    }
206
207    #[test]
208    fn encode_decode_column_id_bounds() {
209        for id in [0, 1, ColumnId::MAX] {
210            let target = IndexTarget::ColumnId(id);
211            let key = id.to_string();
212            assert_eq!(target.encode(), key);
213            assert_eq!(IndexTarget::decode(&key).unwrap(), target);
214        }
215        for key in ["-1", "4294967296", "1.5", "\"1\"", "null"] {
216            assert!(IndexTarget::decode(key).is_err(), "{key}");
217        }
218    }
219
220    #[test]
221    fn decode_rejects_empty() {
222        let err = IndexTarget::decode("").unwrap_err();
223        assert!(matches!(err, TargetKeyError::Empty));
224    }
225
226    #[test]
227    fn decode_rejects_invalid_digits() {
228        let err = IndexTarget::decode("1a2").unwrap_err();
229        assert!(matches!(err, TargetKeyError::InvalidJsonTarget { .. }));
230    }
231
232    #[test]
233    fn json_target_encoding_is_stable() {
234        let target =
235            IndexTarget::new_json_path(7, vec!["resource".into(), "service.name".into()]).unwrap();
236        let key = "json-7:8:resource12:service.name";
237        assert_eq!(target.encode(), key);
238        assert_eq!(IndexTarget::decode(key).unwrap(), target);
239        let json = r#"{"JsonPath":{"column_id":7,"path":["resource","service.name"]}}"#;
240        assert_eq!(serde_json::to_string(&target).unwrap(), json);
241        assert_eq!(serde_json::from_str::<IndexTarget>(json).unwrap(), target);
242        let target = IndexTarget::new_json_path(42, vec!["引号\".:[]".into()]).unwrap();
243        assert_eq!(IndexTarget::decode(&target.encode()).unwrap(), target);
244        assert_eq!(
245            serde_json::to_string(&IndexTarget::ColumnId(42)).unwrap(),
246            r#"{"ColumnId":42}"#
247        );
248    }
249
250    #[test]
251    fn json_target_encode_decode_segments() {
252        let cases: &[(ColumnId, &[&str], &str)] = &[
253            (42, &["service", "name"], "json-42:7:service4:name"),
254            (0, &["a"], "json-0:1:a"),
255            (ColumnId::MAX, &["a"], "json-4294967295:1:a"),
256            (42, &["a.b"], "json-42:3:a.b"),
257            (42, &["a", "b"], "json-42:1:a1:b"),
258            (42, &["a:b", "12:xyz"], "json-42:3:a:b6:12:xyz"),
259            (42, &["中文", "😀"], "json-42:6:中文4:😀"),
260            (42, &["\"\\\n\0"], "json-42:4:\"\\\n\0"),
261            (42, &["abcdefghijkl"], "json-42:12:abcdefghijkl"),
262        ];
263        for &(column_id, path, key) in cases {
264            let target = IndexTarget::new_json_path(
265                column_id,
266                path.iter().map(|part| (*part).to_string()).collect(),
267            )
268            .unwrap();
269            assert_eq!(target.encode(), key);
270            assert_eq!(IndexTarget::decode(key).unwrap(), target);
271        }
272    }
273
274    #[test]
275    fn json_target_rejects_malformed_framing() {
276        for key in [
277            "json-",
278            "json-42",
279            "json-:1:a",
280            "json-+42:1:a",
281            "json-042:1:a",
282            "json-4294967296:1:a",
283            "json-42:",
284            "json-42:0:",
285            "json-42:1:a0:",
286            "json-42::a",
287            "json-42:+1:a",
288            "json-42:-1:a",
289            "json-42:01:a",
290            "json-42:1",
291            "json-42:2:a",
292            "json-42:1:中",
293            "json-42:3:中x",
294            "json-42:1:a1:",
295            "json-42:999999999999999999999999999999999999:a",
296        ] {
297            assert!(IndexTarget::decode(key).is_err(), "{key}");
298        }
299        for path in [
300            vec![JSON2_REMAINDER_FIELD_NAME],
301            vec!["nested", JSON2_REMAINDER_FIELD_NAME],
302        ] {
303            let mut key = "json-42:".to_string();
304            for part in path {
305                key.push_str(&format!("{}:{part}", part.len()));
306            }
307            assert!(matches!(
308                IndexTarget::decode(&key),
309                Err(TargetKeyError::InvalidJsonPath { .. })
310            ));
311        }
312    }
313
314    #[test]
315    fn json_target_rejects_invalid_paths_and_keys() {
316        for path in [
317            vec![],
318            vec![String::new()],
319            vec!["a".into(), String::new()],
320            vec![JSON2_REMAINDER_FIELD_NAME.into()],
321            vec!["a".into(), JSON2_REMAINDER_FIELD_NAME.into()],
322            vec!["a".into(), JSON2_REMAINDER_FIELD_NAME.into(), "b".into()],
323        ] {
324            assert!(matches!(
325                IndexTarget::new_json_path(7, path.clone()),
326                Err(TargetKeyError::InvalidJsonPath { .. })
327            ));
328            let json = serde_json::json!({
329                "column_id": 7,
330                "path": path,
331            });
332            assert!(
333                serde_json::from_value::<IndexTarget>(serde_json::json!({"JsonPath": json}))
334                    .is_err()
335            );
336            let key = serde_json::json!({"JsonPath": {"column_id": 7, "path": path}}).to_string();
337            assert!(IndexTarget::decode(&key).is_err());
338        }
339        assert!(
340            serde_json::from_value::<JsonPathTarget>(serde_json::json!({
341                "column_id": 7,
342                "path": [],
343            }))
344            .is_err()
345        );
346        for key in ["j.2.7.e30", "j.1.7", "j.1.7.!", "j.1.x.e30", "j.1.7.e30"] {
347            assert!(IndexTarget::decode(key).is_err(), "{key}");
348        }
349    }
350}
index/target.rs

index/
target.rs
