mirror of
https://github.com/GreptimeTeam/greptimedb.git
synced 2026-11-15 07:17:01 +00:00
refactor(series): hold the frame layout marker at one value
The series frame is the native PromQL path's internal transport and has never shipped, so its layout marker can only ever tell two binaries of different workspace revisions apart - and a workspace builds one revision. Carrying a revision number in it bought nothing and could only make a stale local binary fail against a freshly built one, so it goes back to a single value and stays there. The reader still refuses a frame that declares any other value: that check is what turns a mismatched producer into an error instead of values decoded under the wrong columns. Only the value it expects changes. The two wire fixtures under tests/compatibility/wire/ are renamed to match and their literal metadata maps retargeted; the cases that exist to be the wrong layout now use the value that is not current, so they still test what they were written to test. Verified: datatypes+common-query 565 passed / 0 failed; promql+query 2192 passed / 0 failed / 2 skipped; cargo fmt --check clean. Signed-off-by: discord9 <discord9@163.com>
This commit is contained in:
discord9
@@ -2606,7 +2606,7 @@ async fn interleaved_constructors_accept_without_polling() {
|
||||
/// The unpack stream rejects a batch whose schema differs from the stream's declared schema,
|
||||
/// including drift in metadata the codec itself does not interpret and an empty batch that
|
||||
/// carries the drift; the reserved layout metadata is rejected as well. Since the current layout is
|
||||
/// `v2`, the drift case rewrites the layout to `v1`: any value other than this revision's is refused,
|
||||
/// `v1`, the drift case rewrites the layout to `v2`: any value other than this revision's is refused,
|
||||
/// so a stale or a future spelling cannot be decoded under the current columns.
|
||||
#[tokio::test]
|
||||
async fn unpack_rejects_unrelated_metadata_drift_and_empty_schema_changes() {
|
||||
@@ -2626,9 +2626,9 @@ async fn unpack_rejects_unrelated_metadata_drift_and_empty_schema_changes() {
|
||||
|
||||
let mut unrelated = base.metadata().clone();
|
||||
unrelated.insert("app.unrelated".into(), "x".into());
|
||||
// The current layout is `v2`, so a frame that still declares `v1` is the drift here.
|
||||
// The current layout is `v1`, so a frame that declares `v2` is the drift here.
|
||||
let mut reserved = base.metadata().clone();
|
||||
reserved.insert("greptime.series.layout".into(), "v1".into());
|
||||
reserved.insert("greptime.series.layout".into(), "v2".into());
|
||||
let mut renamed = base.metadata().clone();
|
||||
renamed.insert("app.renamed".into(), "y".into());
|
||||
let mut different_ordering = base.metadata().clone();
|
||||
@@ -2696,7 +2696,7 @@ fn raw_series_ordering_metadata_fixture() {
|
||||
}
|
||||
|
||||
let cases: Vec<Case> = serde_json::from_str(include_str!(
|
||||
"../../../../../tests/compatibility/wire/raw_series_ordering/v2_metadata.json"
|
||||
"../../../../../tests/compatibility/wire/raw_series_ordering/v1_metadata.json"
|
||||
))
|
||||
.unwrap();
|
||||
for case in cases {
|
||||
@@ -3286,11 +3286,11 @@ async fn unpack_reports_the_ordering_the_frame_declares() {
|
||||
|
||||
/// Reserved layout metadata this revision does not write is rejected instead of being read as the
|
||||
/// current layout, so a frame cannot be decoded under a contract it did not declare. The current
|
||||
/// layout is `v2`; a `v1` frame and every unknown value are refused alike.
|
||||
/// layout is `v1`; a `v2` frame and every unknown value are refused alike.
|
||||
#[tokio::test]
|
||||
async fn unpack_rejects_unknown_layout_metadata() {
|
||||
let cases = [
|
||||
("greptime.series.layout", "v1"),
|
||||
("greptime.series.layout", "v2"),
|
||||
("greptime.series.layout", "v3"),
|
||||
("greptime.series.content", "series"),
|
||||
("greptime.series.time", "timestamps_v2"),
|
||||
|
||||
@@ -33,9 +33,10 @@
|
||||
//! Packing accumulates bounded multi-row batches: [`SeriesFrameCursor`] walks one validated chunk
|
||||
//! and [`SeriesFrameBuilder`] emits rows directly as Arrow arrays once a row or estimated byte bound
|
||||
//! is reached. Decoding reuses one prepared
|
||||
//! schema per stream through [`SeriesFrameDecoder`]. The column layout and existing metadata stay
|
||||
//! on the `v2` contract. Raw frames may also declare `greptime.series.ordering=interleaved`; older
|
||||
//! readers that do not recognize this ordering reject it. A key-encoded frame declares a versioned
|
||||
//! schema per stream through [`SeriesFrameDecoder`]. Column layout, metadata, and wire strings stay
|
||||
//! on the single `v1` contract. Raw frames may also declare `greptime.series.ordering=interleaved`;
|
||||
//! older readers that do not recognize this ordering reject it. A key-encoded frame declares a
|
||||
//! versioned
|
||||
//! [`LabelDecodeSpec`] so its labels can be decoded from the received schema alone, and an evaluated
|
||||
//! frame declares its [`GridSpec`] once instead of repeating timestamps.
|
||||
//!
|
||||
@@ -130,12 +131,14 @@ const RESERVED_KEYS: [&str; 9] = [
|
||||
AGGREGATE_COMPONENTS,
|
||||
];
|
||||
|
||||
/// The only layout this revision writes and reads.
|
||||
/// The layout marker every frame this engine writes carries.
|
||||
///
|
||||
/// A frame declaring any other value is refused instead of decoded: the sample representation of an
|
||||
/// unreleased format is not worth a second decoder, and silently guessing would turn a format change
|
||||
/// into wrong values.
|
||||
pub const FRAME_LAYOUT_VERSION: &str = "v2";
|
||||
/// It stays at one value: the frame is the native path's internal transport and has never shipped,
|
||||
/// so there is no released layout to tell apart from this one. A reader that meets another marker
|
||||
/// refuses the frame instead of guessing at it - the check is what turns a stale producer into an
|
||||
/// error rather than into wrong values - but the marker itself does not move when the layout
|
||||
/// changes, because two binaries of one workspace are built from one revision.
|
||||
pub const FRAME_LAYOUT_VERSION: &str = "v1";
|
||||
|
||||
/// Version of the label decode description this revision writes and reads.
|
||||
///
|
||||
|
||||
@@ -582,7 +582,7 @@ fn schema_and_key_explicit_round_trips_preserve_identity_and_bits() {
|
||||
.into(),
|
||||
);
|
||||
let mut expected_key_metadata = std::collections::HashMap::new();
|
||||
expected_key_metadata.insert("greptime.series.layout".to_string(), "v2".to_string());
|
||||
expected_key_metadata.insert("greptime.series.layout".to_string(), "v1".to_string());
|
||||
expected_key_metadata.insert("greptime.series.content".to_string(), "raw".to_string());
|
||||
expected_key_metadata.insert("greptime.series.labels".to_string(), "key".to_string());
|
||||
expected_key_metadata.insert("greptime.series.time".to_string(), "timestamps".to_string());
|
||||
@@ -2061,9 +2061,9 @@ fn rejects_invalid_schema_and_metadata_and_accepts_valid_empty() {
|
||||
let mut metadata = valid.schema().metadata().clone();
|
||||
match metadata_case {
|
||||
"wrong_layout" => {
|
||||
// `v2` is the only layout this revision reads, so the previously unreadable `v1`
|
||||
// is what a wrong layout is spelled as now.
|
||||
metadata.insert("greptime.series.layout".to_string(), "v1".to_string());
|
||||
// `v1` is the only layout this revision reads, so `v2` is what a wrong layout is
|
||||
// spelled as now.
|
||||
metadata.insert("greptime.series.layout".to_string(), "v2".to_string());
|
||||
}
|
||||
"wrong_labels" => {
|
||||
metadata.insert("greptime.series.labels".to_string(), "explicit".to_string());
|
||||
@@ -3390,7 +3390,7 @@ fn cross_chunk_rows_coalesce_into_one_multirow_batch() {
|
||||
["read_key", "is_last", "time", "fields"]
|
||||
);
|
||||
for (key, value) in [
|
||||
("greptime.series.layout", "v2"),
|
||||
("greptime.series.layout", "v1"),
|
||||
("greptime.series.content", "raw"),
|
||||
("greptime.series.labels", "key"),
|
||||
("greptime.series.time", "timestamps"),
|
||||
@@ -4039,8 +4039,8 @@ fn decoder_reuses_cached_schema_and_rejects_schema_evolution() {
|
||||
|
||||
// Reserved metadata and evolved fields are rejected, including by a zero-row batch.
|
||||
for (key, value) in [
|
||||
// `v2` is the layout this decoder expects, so the unreadable `v1` is the wrong one.
|
||||
("greptime.series.layout", "v1"),
|
||||
// `v1` is the layout this decoder expects, so `v2` is the wrong one.
|
||||
("greptime.series.layout", "v2"),
|
||||
("greptime.series.content", "evaluated"),
|
||||
("greptime.series.labels", "explicit"),
|
||||
("greptime.series.time", "grid"),
|
||||
@@ -4229,7 +4229,7 @@ fn schema_with_metadata(metadata: Vec<(String, String)>) -> Schema {
|
||||
/// The five metadata entries every raw v2 frame declares, with the given label mode.
|
||||
fn base_metadata(labels: &str) -> Vec<(String, String)> {
|
||||
[
|
||||
("greptime.series.layout", "v2"),
|
||||
("greptime.series.layout", "v1"),
|
||||
("greptime.series.content", "raw"),
|
||||
("greptime.series.labels", labels),
|
||||
("greptime.series.time", "timestamps"),
|
||||
@@ -4305,7 +4305,7 @@ fn raw_series_label_decode_metadata_fixture() {
|
||||
}
|
||||
|
||||
let cases: Vec<Case> = serde_json::from_str(include_str!(
|
||||
"../../../../../tests/compatibility/wire/raw_series_label_decode/v2_metadata.json"
|
||||
"../../../../../tests/compatibility/wire/raw_series_label_decode/v1_metadata.json"
|
||||
))
|
||||
.unwrap();
|
||||
for case in cases {
|
||||
|
||||
+15
-15
@@ -2,7 +2,7 @@
|
||||
{
|
||||
"name": "key_frame_declares_dense_description",
|
||||
"metadata": {
|
||||
"greptime.series.layout": "v2",
|
||||
"greptime.series.layout": "v1",
|
||||
"greptime.series.content": "raw",
|
||||
"greptime.series.labels": "key",
|
||||
"greptime.series.time": "timestamps",
|
||||
@@ -14,7 +14,7 @@
|
||||
{
|
||||
"name": "key_frame_declares_sparse_description",
|
||||
"metadata": {
|
||||
"greptime.series.layout": "v2",
|
||||
"greptime.series.layout": "v1",
|
||||
"greptime.series.content": "raw",
|
||||
"greptime.series.labels": "key",
|
||||
"greptime.series.time": "timestamps",
|
||||
@@ -26,7 +26,7 @@
|
||||
{
|
||||
"name": "missing_description_rejected",
|
||||
"metadata": {
|
||||
"greptime.series.layout": "v2",
|
||||
"greptime.series.layout": "v1",
|
||||
"greptime.series.content": "raw",
|
||||
"greptime.series.labels": "key",
|
||||
"greptime.series.time": "timestamps",
|
||||
@@ -37,7 +37,7 @@
|
||||
{
|
||||
"name": "unknown_version_rejected",
|
||||
"metadata": {
|
||||
"greptime.series.layout": "v2",
|
||||
"greptime.series.layout": "v1",
|
||||
"greptime.series.content": "raw",
|
||||
"greptime.series.labels": "key",
|
||||
"greptime.series.time": "timestamps",
|
||||
@@ -49,7 +49,7 @@
|
||||
{
|
||||
"name": "missing_field_rejected",
|
||||
"metadata": {
|
||||
"greptime.series.layout": "v2",
|
||||
"greptime.series.layout": "v1",
|
||||
"greptime.series.content": "raw",
|
||||
"greptime.series.labels": "key",
|
||||
"greptime.series.time": "timestamps",
|
||||
@@ -61,7 +61,7 @@
|
||||
{
|
||||
"name": "unknown_field_rejected",
|
||||
"metadata": {
|
||||
"greptime.series.layout": "v2",
|
||||
"greptime.series.layout": "v1",
|
||||
"greptime.series.content": "raw",
|
||||
"greptime.series.labels": "key",
|
||||
"greptime.series.time": "timestamps",
|
||||
@@ -73,7 +73,7 @@
|
||||
{
|
||||
"name": "wrong_field_type_rejected",
|
||||
"metadata": {
|
||||
"greptime.series.layout": "v2",
|
||||
"greptime.series.layout": "v1",
|
||||
"greptime.series.content": "raw",
|
||||
"greptime.series.labels": "key",
|
||||
"greptime.series.time": "timestamps",
|
||||
@@ -85,7 +85,7 @@
|
||||
{
|
||||
"name": "unlisted_logical_tag_rejected",
|
||||
"metadata": {
|
||||
"greptime.series.layout": "v2",
|
||||
"greptime.series.layout": "v1",
|
||||
"greptime.series.content": "raw",
|
||||
"greptime.series.labels": "key",
|
||||
"greptime.series.time": "timestamps",
|
||||
@@ -97,7 +97,7 @@
|
||||
{
|
||||
"name": "explicit_frame_with_description_rejected",
|
||||
"metadata": {
|
||||
"greptime.series.layout": "v2",
|
||||
"greptime.series.layout": "v1",
|
||||
"greptime.series.content": "raw",
|
||||
"greptime.series.labels": "explicit",
|
||||
"greptime.series.time": "timestamps",
|
||||
@@ -109,7 +109,7 @@
|
||||
{
|
||||
"name": "malformed_json_rejected",
|
||||
"metadata": {
|
||||
"greptime.series.layout": "v2",
|
||||
"greptime.series.layout": "v1",
|
||||
"greptime.series.content": "raw",
|
||||
"greptime.series.labels": "key",
|
||||
"greptime.series.time": "timestamps",
|
||||
@@ -121,7 +121,7 @@
|
||||
{
|
||||
"name": "wrong_layout_rejected",
|
||||
"metadata": {
|
||||
"greptime.series.layout": "v3",
|
||||
"greptime.series.layout": "v2",
|
||||
"greptime.series.content": "raw",
|
||||
"greptime.series.labels": "key",
|
||||
"greptime.series.time": "timestamps",
|
||||
@@ -133,7 +133,7 @@
|
||||
{
|
||||
"name": "unregistered_reserved_metadata_rejected",
|
||||
"metadata": {
|
||||
"greptime.series.layout": "v2",
|
||||
"greptime.series.layout": "v1",
|
||||
"greptime.series.content": "raw",
|
||||
"greptime.series.labels": "key",
|
||||
"greptime.series.time": "timestamps",
|
||||
@@ -146,7 +146,7 @@
|
||||
{
|
||||
"name": "evaluated_with_timestamps_rejected",
|
||||
"metadata": {
|
||||
"greptime.series.layout": "v2",
|
||||
"greptime.series.layout": "v1",
|
||||
"greptime.series.content": "evaluated",
|
||||
"greptime.series.labels": "explicit",
|
||||
"greptime.series.time": "timestamps",
|
||||
@@ -158,7 +158,7 @@
|
||||
{
|
||||
"name": "evaluated_with_key_labels_rejected",
|
||||
"metadata": {
|
||||
"greptime.series.layout": "v2",
|
||||
"greptime.series.layout": "v1",
|
||||
"greptime.series.content": "evaluated",
|
||||
"greptime.series.labels": "key",
|
||||
"greptime.series.time": "grid_spans",
|
||||
@@ -171,7 +171,7 @@
|
||||
{
|
||||
"name": "raw_with_grid_rejected",
|
||||
"metadata": {
|
||||
"greptime.series.layout": "v2",
|
||||
"greptime.series.layout": "v1",
|
||||
"greptime.series.content": "raw",
|
||||
"greptime.series.labels": "explicit",
|
||||
"greptime.series.time": "timestamps",
|
||||
@@ -1,7 +1,7 @@
|
||||
# Raw-series ordering schema metadata compatibility
|
||||
|
||||
`v2_metadata.json` is a hand-authored, frozen set of Arrow schema metadata maps
|
||||
for the v2 series frame. It records the established layout/content/labels/time
|
||||
`v1_metadata.json` is a hand-authored, frozen set of Arrow schema metadata maps
|
||||
for the series frame, whose layout marker is the single `v1` this engine writes. It records the established layout/content/labels/time
|
||||
metadata and mandatory ordering entry, for raw frames and for evaluated ones.
|
||||
The Rust unit test parses these literal maps through `series_frame_ordering` and
|
||||
through `SeriesFrameDeclaration::parse`; the fixture is not produced by the
|
||||
@@ -17,7 +17,7 @@ cargo nextest run -p common-query -E 'test(series_frame)'
|
||||
|
||||
The fixture proves that missing ordering metadata and unknown values are rejected,
|
||||
the two original values decode, and that a whole declaration is accepted or
|
||||
refused consistently with its layout and content: a `v3` layout, an unregistered
|
||||
refused consistently with its layout and content: a `v2` layout, an unregistered
|
||||
`greptime.series.*` key, a raw frame that declares a grid, and any content/time or
|
||||
content/labels contradiction are refusals. Evaluated declarations use explicit
|
||||
`contiguous` ordering; streaming round trips for evaluated frames are covered by
|
||||
|
||||
+8
-8
@@ -2,7 +2,7 @@
|
||||
{
|
||||
"name": "missing_ordering_rejected",
|
||||
"metadata": {
|
||||
"greptime.series.layout": "v2",
|
||||
"greptime.series.layout": "v1",
|
||||
"greptime.series.content": "raw",
|
||||
"greptime.series.labels": "explicit",
|
||||
"greptime.series.time": "timestamps"
|
||||
@@ -13,7 +13,7 @@
|
||||
{
|
||||
"name": "known_contiguous",
|
||||
"metadata": {
|
||||
"greptime.series.layout": "v2",
|
||||
"greptime.series.layout": "v1",
|
||||
"greptime.series.content": "raw",
|
||||
"greptime.series.labels": "explicit",
|
||||
"greptime.series.time": "timestamps",
|
||||
@@ -25,7 +25,7 @@
|
||||
{
|
||||
"name": "known_read_key_then_time",
|
||||
"metadata": {
|
||||
"greptime.series.layout": "v2",
|
||||
"greptime.series.layout": "v1",
|
||||
"greptime.series.content": "raw",
|
||||
"greptime.series.labels": "explicit",
|
||||
"greptime.series.time": "timestamps",
|
||||
@@ -37,7 +37,7 @@
|
||||
{
|
||||
"name": "unknown_ordering_rejected",
|
||||
"metadata": {
|
||||
"greptime.series.layout": "v2",
|
||||
"greptime.series.layout": "v1",
|
||||
"greptime.series.content": "raw",
|
||||
"greptime.series.labels": "explicit",
|
||||
"greptime.series.time": "timestamps",
|
||||
@@ -49,7 +49,7 @@
|
||||
{
|
||||
"name": "evaluated_contiguous",
|
||||
"metadata": {
|
||||
"greptime.series.layout": "v2",
|
||||
"greptime.series.layout": "v1",
|
||||
"greptime.series.content": "evaluated",
|
||||
"greptime.series.labels": "explicit",
|
||||
"greptime.series.time": "grid_spans",
|
||||
@@ -62,7 +62,7 @@
|
||||
{
|
||||
"name": "evaluated_read_key_then_time_rejected",
|
||||
"metadata": {
|
||||
"greptime.series.layout": "v2",
|
||||
"greptime.series.layout": "v1",
|
||||
"greptime.series.content": "evaluated",
|
||||
"greptime.series.labels": "explicit",
|
||||
"greptime.series.time": "grid_spans",
|
||||
@@ -75,7 +75,7 @@
|
||||
{
|
||||
"name": "wrong_layout_rejected",
|
||||
"metadata": {
|
||||
"greptime.series.layout": "v3",
|
||||
"greptime.series.layout": "v2",
|
||||
"greptime.series.content": "raw",
|
||||
"greptime.series.labels": "explicit",
|
||||
"greptime.series.time": "timestamps",
|
||||
@@ -87,7 +87,7 @@
|
||||
{
|
||||
"name": "unregistered_reserved_metadata_rejected",
|
||||
"metadata": {
|
||||
"greptime.series.layout": "v2",
|
||||
"greptime.series.layout": "v1",
|
||||
"greptime.series.content": "raw",
|
||||
"greptime.series.labels": "explicit",
|
||||
"greptime.series.time": "timestamps",
|
||||
Reference in New Issue
Block a user