diff --git a/.github/runner-scale-sets/query-regression/README.md b/.github/runner-scale-sets/query-regression/README.md
index f7a9bdc1732..fa8d436d2c2 100644
--- a/.github/runner-scale-sets/query-regression/README.md
+++ b/.github/runner-scale-sets/query-regression/README.md
@@ -108,7 +108,7 @@ agent, which Aliyun public Ubuntu images include.
An allowlisted repository admin commenting `/query-regression` on the PR is
**trust admission for that exact PR revision**. `/query-regression` runs the
-six routine default cases; `/query-regression heavy` runs only the
+nine routine default cases; `/query-regression heavy` runs only the
high-cardinality `prom_remote_write_7913` remote-write case.
`slash-command-dispatch.yml` (`issue_comment` on the default branch) parses
the command and `repository_dispatch`es; `query-regression-slash.yml` admits
diff --git a/.github/scripts/query-regression-run.py b/.github/scripts/query-regression-run.py
index a99706c04a5..6da7c896d67 100644
--- a/.github/scripts/query-regression-run.py
+++ b/.github/scripts/query-regression-run.py
@@ -33,6 +33,7 @@ from typing import Any
DEFAULT_CASES = [
"tests/perf/query_cases/smoke_direct_sst/case.toml",
+ "tests/perf/query_cases/sst_float_bss/case.toml",
"tests/perf/query_cases/prom_remote_write_seeded_random/case.toml",
"tests/perf/query_cases/prom_remote_write_run_heavy/case.toml",
"tests/perf/query_cases/prom_remote_write_mixed_every/case.toml",
diff --git a/.github/workflows/backport.yml b/.github/workflows/backport.yml
index 8cf2da32fcb..013aa320b20 100644
--- a/.github/workflows/backport.yml
+++ b/.github/workflows/backport.yml
@@ -45,6 +45,7 @@ jobs:
PR_TITLE: ${{ github.event.pull_request.title }}
PR_BODY: ${{ github.event.pull_request.body }}
AUTHOR: ${{ github.event.pull_request.merged_by.login }}
+ PR_AUTHOR: ${{ github.event.pull_request.user.login }}
BASE_BRANCH: ${{ github.event.pull_request.base.ref }}
run: |
git config user.name "github-actions[bot]"
@@ -76,6 +77,7 @@ jobs:
git branch -D "${head_branch}"
gh issue create \
--title "Backport failure: PR #${PR_NUMBER} to ${release_branch}" \
+ --assignee "${PR_AUTHOR}" \
--body "Cherry-pick of ${MERGE_COMMIT} (PR #${PR_NUMBER}) failed on ${release_branch}. Manual backport required." \
|| true
continue
diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml
index d7a91cadd43..d63d52a2189 100644
--- a/.github/workflows/integration.yml
+++ b/.github/workflows/integration.yml
@@ -781,10 +781,9 @@ jobs:
kubectl -n kafka-cluster apply \
-f .github/actions/setup-greptimedb-cluster/kafka-wal-helper.yaml
- kubectl -n kafka-cluster wait \
- --for=condition=Ready \
- pod -l app=kafka-wal-helper \
- --timeout=120s
+ kubectl rollout status deployment/kafka-wal-helper \
+ --timeout=120s \
+ -n kafka-cluster
- name: Print etcd info
shell: bash
run: kubectl get all --show-labels -n etcd-cluster
diff --git a/.github/workflows/semantic-pull-request.yml b/.github/workflows/semantic-pull-request.yml
index e2958776e93..b2d3c648df1 100644
--- a/.github/workflows/semantic-pull-request.yml
+++ b/.github/workflows/semantic-pull-request.yml
@@ -6,9 +6,13 @@ on:
- opened
- reopened
- edited
+ - synchronize
+ - ready_for_review
+ merge_group:
+ types: [checks_requested]
concurrency:
- group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+ group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
permissions:
@@ -18,6 +22,9 @@ permissions:
jobs:
check:
+ name: Semantic PR title
+ # Validate before queue admission; report a skipped check for merge groups.
+ if: github.event_name == 'pull_request_target'
runs-on: ubuntu-latest
timeout-minutes: 10
steps:
diff --git a/Cargo.lock b/Cargo.lock
index 3876eb6d7d3..61005e91848 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -221,7 +221,7 @@ checksum = "d301b3b94cb4b2f23d7917810addbbaff90738e0ca2be692bd027e70d7e0330c"
name = "api"
version = "1.3.0-alpha.1"
dependencies = [
- "arrow-schema 58.3.0",
+ "arrow-schema 59.2.0",
"common-base",
"common-decimal",
"common-error",
@@ -323,23 +323,23 @@ dependencies = [
[[package]]
name = "arrow"
-version = "58.3.0"
+version = "59.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "378530e55cd479eda3c14eb345310799717e6f76d0c332041e8487022166b471"
+checksum = "61d285d16bce7d0be61912f7928342b673067b6b7d7ef6cc179258ba7de1fecf"
dependencies = [
- "arrow-arith 58.3.0",
- "arrow-array 58.3.0",
- "arrow-buffer 58.3.0",
- "arrow-cast 58.3.0",
- "arrow-csv 58.3.0",
- "arrow-data 58.3.0",
- "arrow-ipc 58.3.0",
- "arrow-json 58.3.0",
- "arrow-ord 58.3.0",
- "arrow-row 58.3.0",
- "arrow-schema 58.3.0",
- "arrow-select 58.3.0",
- "arrow-string 58.3.0",
+ "arrow-arith 59.2.0",
+ "arrow-array 59.2.0",
+ "arrow-buffer 59.2.0",
+ "arrow-cast 59.2.0",
+ "arrow-csv 59.2.0",
+ "arrow-data 59.2.0",
+ "arrow-ipc 59.2.0",
+ "arrow-json 59.2.0",
+ "arrow-ord 59.2.0",
+ "arrow-row 59.2.0",
+ "arrow-schema 59.2.0",
+ "arrow-select 59.2.0",
+ "arrow-string 59.2.0",
]
[[package]]
@@ -358,14 +358,14 @@ dependencies = [
[[package]]
name = "arrow-arith"
-version = "58.3.0"
+version = "59.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a0ab212d2c1886e802f51c5212d78ebbcbb0bec980fff9dadc1eb8d45cd0b738"
+checksum = "757ef1836251e88222542a7da2623bc1c9cb9e20afefa6db2c41e79991cd91d4"
dependencies = [
- "arrow-array 58.3.0",
- "arrow-buffer 58.3.0",
- "arrow-data 58.3.0",
- "arrow-schema 58.3.0",
+ "arrow-array 59.2.0",
+ "arrow-buffer 59.2.0",
+ "arrow-data 59.2.0",
+ "arrow-schema 59.2.0",
"chrono",
"num-traits",
]
@@ -388,14 +388,14 @@ dependencies = [
[[package]]
name = "arrow-array"
-version = "58.3.0"
+version = "59.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cfd33d3e92f207444098c75b42de99d329562be0cf686b307b097cc52b4e999e"
+checksum = "bc9a4a4b2b5ecd0e04df03471661cb61f28bed3c7fd50994715129b01b2edb97"
dependencies = [
"ahash 0.8.12",
- "arrow-buffer 58.3.0",
- "arrow-data 58.3.0",
- "arrow-schema 58.3.0",
+ "arrow-buffer 59.2.0",
+ "arrow-data 59.2.0",
+ "arrow-schema 59.2.0",
"chrono",
"chrono-tz",
"half",
@@ -418,13 +418,13 @@ dependencies = [
[[package]]
name = "arrow-buffer"
-version = "58.3.0"
+version = "59.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0c6cd424c2693bcdbc150d843dc9d4d137dd2de4782ce6df491ad11a3a0416c0"
+checksum = "c12b576ef18c1deb80925a248b25ad84f419198d791b8e293fc6aaa60441fe90"
dependencies = [
"bytes",
"half",
- "num-bigint 0.4.6",
+ "num-bigint 0.5.1",
"num-traits",
]
@@ -450,18 +450,18 @@ dependencies = [
[[package]]
name = "arrow-cast"
-version = "58.3.0"
+version = "59.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4c5aefb56a2c02e9e2b30746241058b85f8983f0fcff2ba0c6d09006e1cded7f"
+checksum = "68338a9096a5dc9bc11927c58c43a8526d96bf6abd2012ef6c0c9f505991cc79"
dependencies = [
- "arrow-array 58.3.0",
- "arrow-buffer 58.3.0",
- "arrow-data 58.3.0",
- "arrow-ord 58.3.0",
- "arrow-schema 58.3.0",
- "arrow-select 58.3.0",
+ "arrow-array 59.2.0",
+ "arrow-buffer 59.2.0",
+ "arrow-data 59.2.0",
+ "arrow-ord 59.2.0",
+ "arrow-schema 59.2.0",
+ "arrow-select 59.2.0",
"atoi",
- "base64 0.22.1",
+ "base64 0.23.1",
"chrono",
"comfy-table",
"half",
@@ -487,13 +487,13 @@ dependencies = [
[[package]]
name = "arrow-csv"
-version = "58.3.0"
+version = "59.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e94e8cf7e517657a52b91ea1263acf38c4ca62a84655d72458a3359b12ab97de"
+checksum = "25011b52b346407d497ef0030e12b45e4f2d0cc279efc09c4f3d09106db30e36"
dependencies = [
- "arrow-array 58.3.0",
- "arrow-cast 58.3.0",
- "arrow-schema 58.3.0",
+ "arrow-array 59.2.0",
+ "arrow-cast 59.2.0",
+ "arrow-schema 59.2.0",
"chrono",
"csv",
"csv-core",
@@ -514,12 +514,12 @@ dependencies = [
[[package]]
name = "arrow-data"
-version = "58.3.0"
+version = "59.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3c88210023a2bfee1896af366309a3028fc3bcbd6515fa29a7990ee1baa08ee0"
+checksum = "723fe4aeed7604e00b9883a465af4ff0a0e6c44c03e41a68c3d1cbc403e0e44d"
dependencies = [
- "arrow-buffer 58.3.0",
- "arrow-schema 58.3.0",
+ "arrow-buffer 59.2.0",
+ "arrow-schema 59.2.0",
"half",
"num-integer",
"num-traits",
@@ -527,16 +527,17 @@ dependencies = [
[[package]]
name = "arrow-flight"
-version = "58.3.0"
+version = "59.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "28abfe8bf9f124e5fc83b334af4fa58f8d0323ad25312ccb2d1da50178415704"
+checksum = "2bebfacc9d71f0728f6774164e4d4254b5e504d2b46812d0512d8290ec119a64"
dependencies = [
- "arrow-array 58.3.0",
- "arrow-buffer 58.3.0",
- "arrow-cast 58.3.0",
- "arrow-ipc 58.3.0",
- "arrow-schema 58.3.0",
- "base64 0.22.1",
+ "arrow-array 59.2.0",
+ "arrow-buffer 59.2.0",
+ "arrow-cast 59.2.0",
+ "arrow-data 59.2.0",
+ "arrow-ipc 59.2.0",
+ "arrow-schema 59.2.0",
+ "base64 0.23.1",
"bytes",
"futures",
"prost 0.14.1",
@@ -562,17 +563,17 @@ dependencies = [
[[package]]
name = "arrow-ipc"
-version = "58.3.0"
+version = "59.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "238438f0834483703d88896db6fe5a7138b2230debc31b34c0336c2996e3c64f"
+checksum = "149437b14371f5b9ec60f5ddc751483ae99d7a7072653c0075e5e469156eea7b"
dependencies = [
- "arrow-array 58.3.0",
- "arrow-buffer 58.3.0",
- "arrow-data 58.3.0",
- "arrow-schema 58.3.0",
- "arrow-select 58.3.0",
+ "arrow-array 59.2.0",
+ "arrow-buffer 59.2.0",
+ "arrow-data 59.2.0",
+ "arrow-schema 59.2.0",
+ "arrow-select 59.2.0",
"flatbuffers",
- "lz4_flex 0.13.1",
+ "lz4_flex 0.14.0",
"zstd",
]
@@ -589,7 +590,7 @@ dependencies = [
"arrow-schema 56.2.0",
"chrono",
"half",
- "indexmap 2.13.0",
+ "indexmap 2.14.2",
"lexical-core",
"memchr",
"num",
@@ -600,19 +601,19 @@ dependencies = [
[[package]]
name = "arrow-json"
-version = "58.3.0"
+version = "59.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "205ca2119e6d679d5c133c6f30e68f027738d95ed948cf77677ea69c7800036b"
+checksum = "f18b9123ccfec418a663f821c9a034af339711678c11ffe00d3ec07da5ff9f7e"
dependencies = [
- "arrow-array 58.3.0",
- "arrow-buffer 58.3.0",
- "arrow-cast 58.3.0",
- "arrow-ord 58.3.0",
- "arrow-schema 58.3.0",
- "arrow-select 58.3.0",
+ "arrow-array 59.2.0",
+ "arrow-buffer 59.2.0",
+ "arrow-cast 59.2.0",
+ "arrow-ord 59.2.0",
+ "arrow-schema 59.2.0",
+ "arrow-select 59.2.0",
"chrono",
"half",
- "indexmap 2.13.0",
+ "indexmap 2.14.2",
"itoa",
"lexical-core",
"memchr",
@@ -638,25 +639,24 @@ dependencies = [
[[package]]
name = "arrow-ord"
-version = "58.3.0"
+version = "59.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1bffd8fd2579286a5d63bac898159873e5094a79009940bcb42bbfce4f19f1d0"
+checksum = "e6c08dff0686cf23ca4f562803f191ccbeb726dbae6309cd4b4aaf65e0f2c979"
dependencies = [
- "arrow-array 58.3.0",
- "arrow-buffer 58.3.0",
- "arrow-data 58.3.0",
- "arrow-schema 58.3.0",
- "arrow-select 58.3.0",
+ "arrow-array 59.2.0",
+ "arrow-buffer 59.2.0",
+ "arrow-data 59.2.0",
+ "arrow-schema 59.2.0",
+ "arrow-select 59.2.0",
]
[[package]]
name = "arrow-pg"
-version = "0.14.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d7acc179d3edb91fed930ea6aa6def3f73458228079857b4547096040a1cbde8"
+version = "0.15.0"
+source = "git+https://github.com/GreptimeTeam/datafusion-postgres.git?rev=3c77e6c32b8db80635a0d2f4b318a36b31170bc3#3c77e6c32b8db80635a0d2f4b318a36b31170bc3"
dependencies = [
- "arrow 58.3.0",
- "arrow-schema 58.3.0",
+ "arrow 59.2.0",
+ "arrow-schema 59.2.0",
"bytes",
"chrono",
"datafusion",
@@ -682,14 +682,14 @@ dependencies = [
[[package]]
name = "arrow-row"
-version = "58.3.0"
+version = "59.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bab5994731204603c73ba69267616c50f80780774c6bb0476f1f830625115e0c"
+checksum = "bbec439386df71ad570e6758a946111322b9e9dc8db83b5527321f0b4c9119c2"
dependencies = [
- "arrow-array 58.3.0",
- "arrow-buffer 58.3.0",
- "arrow-data 58.3.0",
- "arrow-schema 58.3.0",
+ "arrow-array 59.2.0",
+ "arrow-buffer 59.2.0",
+ "arrow-data 59.2.0",
+ "arrow-schema 59.2.0",
"half",
]
@@ -701,9 +701,9 @@ checksum = "b3aa9e59c611ebc291c28582077ef25c97f1975383f1479b12f3b9ffee2ffabe"
[[package]]
name = "arrow-schema"
-version = "58.3.0"
+version = "59.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f633dbfdf39c039ada1bf9e34c694816eb71fbb7dc78f613993b7245e078a1ed"
+checksum = "e6fed2ca0d1eade57e811cbe73b98ad50cc08a1183e13b2d2aa43a7df593f40e"
dependencies = [
"serde",
"serde_core",
@@ -726,15 +726,15 @@ dependencies = [
[[package]]
name = "arrow-select"
-version = "58.3.0"
+version = "59.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8cd065c54172ac787cf3f2f8d4107e0d3fdc26edba76fdf4f4cc170258942222"
+checksum = "466b19cf75130b891dc1b23a84b343c714c62c64c9c62e365c76aa0ff90a53fb"
dependencies = [
"ahash 0.8.12",
- "arrow-array 58.3.0",
- "arrow-buffer 58.3.0",
- "arrow-data 58.3.0",
- "arrow-schema 58.3.0",
+ "arrow-array 59.2.0",
+ "arrow-buffer 59.2.0",
+ "arrow-data 59.2.0",
+ "arrow-schema 59.2.0",
"num-traits",
]
@@ -757,15 +757,15 @@ dependencies = [
[[package]]
name = "arrow-string"
-version = "58.3.0"
+version = "59.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "29dd7cda3ab9692f43a2e4acc444d760cc17b12bb6d8232ddf64e9bab7c06b42"
+checksum = "c838a25bb3691e919e0f617616ac51a4ff8517a952e29ca133cf0c22b2ce65b1"
dependencies = [
- "arrow-array 58.3.0",
- "arrow-buffer 58.3.0",
- "arrow-data 58.3.0",
- "arrow-schema 58.3.0",
- "arrow-select 58.3.0",
+ "arrow-array 59.2.0",
+ "arrow-buffer 59.2.0",
+ "arrow-data 59.2.0",
+ "arrow-schema 59.2.0",
+ "arrow-select 59.2.0",
"memchr",
"num-traits",
"regex",
@@ -1290,7 +1290,7 @@ dependencies = [
"quote",
"regex",
"rustc-hash 2.1.1",
- "shlex",
+ "shlex 1.3.0",
"syn 2.0.117",
]
@@ -1661,8 +1661,8 @@ name = "catalog"
version = "1.3.0-alpha.1"
dependencies = [
"api",
- "arrow 58.3.0",
- "arrow-schema 58.3.0",
+ "arrow 59.2.0",
+ "arrow-schema 59.2.0",
"async-stream",
"async-trait",
"bytes",
@@ -1731,13 +1731,14 @@ dependencies = [
[[package]]
name = "cc"
-version = "1.2.27"
+version = "1.4.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d487aa071b5f64da6f19a3e848e3578944b726ee5a4854b82172f02aa876bfdc"
+checksum = "005ec2760ca554fae18df7a11195552ec576cd665632a881bc011d5bb2fd4d80"
dependencies = [
+ "find-msvc-tools",
"jobserver",
"libc",
- "shlex",
+ "shlex 2.0.1",
]
[[package]]
@@ -1781,9 +1782,9 @@ dependencies = [
[[package]]
name = "cfg-if"
-version = "1.0.1"
+version = "1.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9555578bc9e57714c812a1f84e4fc5b4d21fcb063490c624de019f7464c91268"
+checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801"
[[package]]
name = "cfg_aliases"
@@ -1803,7 +1804,7 @@ version = "0.13.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7fe45e18904af7af10e4312df7c97251e98af98c70f42f1f2587aecfcbee56bf"
dependencies = [
- "indexmap 2.13.0",
+ "indexmap 2.14.2",
"lazy_static",
"num-traits",
"regex",
@@ -1868,9 +1869,9 @@ dependencies = [
[[package]]
name = "chrono"
-version = "0.4.44"
+version = "0.4.45"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c673075a2e0e5f4a1dde27ce9dee1ea4558c7ffe648f576438a20ca1d2acc4b0"
+checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327"
dependencies = [
"iana-time-zone",
"js-sys",
@@ -2316,6 +2317,15 @@ dependencies = [
"zeroize",
]
+[[package]]
+name = "common-batcher"
+version = "1.3.0-alpha.1"
+dependencies = [
+ "dashmap",
+ "futures",
+ "tokio",
+]
+
[[package]]
name = "common-catalog"
version = "1.3.0-alpha.1"
@@ -2351,8 +2361,8 @@ dependencies = [
name = "common-datasource"
version = "1.3.0-alpha.1"
dependencies = [
- "arrow 58.3.0",
- "arrow-schema 58.3.0",
+ "arrow 59.2.0",
+ "arrow-schema 59.2.0",
"async-compression",
"async-trait",
"bytes",
@@ -2463,9 +2473,9 @@ dependencies = [
"api",
"approx 0.5.1",
"arc-swap",
- "arrow 58.3.0",
- "arrow-cast 58.3.0",
- "arrow-schema 58.3.0",
+ "arrow 59.2.0",
+ "arrow-cast 59.2.0",
+ "arrow-schema 59.2.0",
"async-trait",
"bincode",
"catalog",
@@ -2866,7 +2876,7 @@ dependencies = [
name = "common-sql"
version = "1.3.0-alpha.1"
dependencies = [
- "arrow-schema 58.3.0",
+ "arrow-schema 59.2.0",
"common-base",
"common-decimal",
"common-error",
@@ -2943,7 +2953,7 @@ dependencies = [
name = "common-time"
version = "1.3.0-alpha.1"
dependencies = [
- "arrow 58.3.0",
+ "arrow 59.2.0",
"chrono",
"chrono-tz",
"common-error",
@@ -3552,7 +3562,7 @@ checksum = "7a9bc1a22964ff6a355fbec24cf68266a0ed28f8b84c0864c386474ea3d0e479"
dependencies = [
"cc",
"codespan-reporting 0.13.1",
- "indexmap 2.13.0",
+ "indexmap 2.14.2",
"proc-macro2",
"quote",
"scratch",
@@ -3567,7 +3577,7 @@ checksum = "b1f29a879d35f7906e3c9b77d7a1005a6a0787d330c09dfe4ffb5f617728cb44"
dependencies = [
"clap",
"codespan-reporting 0.13.1",
- "indexmap 2.13.0",
+ "indexmap 2.14.2",
"proc-macro2",
"quote",
"syn 2.0.117",
@@ -3585,7 +3595,7 @@ version = "1.0.190"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d187e019e7b05a1f3e69a8396b70800ee867aa9fc2ab972761173ccee03742df"
dependencies = [
- "indexmap 2.13.0",
+ "indexmap 2.14.2",
"proc-macro2",
"quote",
"syn 2.0.117",
@@ -3747,9 +3757,9 @@ checksum = "04d2cd9c18b9f454ed67da600630b021a8a80bf33f8c95896ab33aaf1c26b728"
[[package]]
name = "dashmap"
-version = "6.1.0"
+version = "6.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5041cc499144891f3790297212f32a74fb938e5136a14943f338ef9e0ae276cf"
+checksum = "e6361d5c062261c78a176addb82d4c821ae42bed6089de0e12603cd25de2059c"
dependencies = [
"cfg-if",
"crossbeam-utils",
@@ -3767,13 +3777,12 @@ checksum = "a4ae5f15dda3c708c0ade84bfee31ccab44a3da4f88015ed22f63732abe300c8"
[[package]]
name = "datafusion"
-version = "53.1.0"
-source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=bb531e754b3b36d7a4d3222f35ebc88204dafd2b#bb531e754b3b36d7a4d3222f35ebc88204dafd2b"
+version = "55.0.0"
+source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=b66dca3260c8e04f314c8036cdfbb08f1a564d95#b66dca3260c8e04f314c8036cdfbb08f1a564d95"
dependencies = [
- "arrow 58.3.0",
- "arrow-schema 58.3.0",
+ "arrow 59.2.0",
+ "arrow-schema 59.2.0",
"async-trait",
- "bytes",
"bzip2",
"chrono",
"datafusion-catalog",
@@ -3803,14 +3812,13 @@ dependencies = [
"datafusion-sql",
"flate2",
"futures",
- "itertools 0.14.0",
+ "indexmap 2.14.2",
+ "itertools 0.15.0",
"liblzma",
"log",
"object_store",
"parking_lot 0.12.4",
"parquet",
- "rand 0.9.4",
- "regex",
"sqlparser",
"tempfile",
"tokio",
@@ -3821,10 +3829,10 @@ dependencies = [
[[package]]
name = "datafusion-catalog"
-version = "53.1.0"
-source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=bb531e754b3b36d7a4d3222f35ebc88204dafd2b#bb531e754b3b36d7a4d3222f35ebc88204dafd2b"
+version = "55.0.0"
+source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=b66dca3260c8e04f314c8036cdfbb08f1a564d95#b66dca3260c8e04f314c8036cdfbb08f1a564d95"
dependencies = [
- "arrow 58.3.0",
+ "arrow 59.2.0",
"async-trait",
"dashmap",
"datafusion-common",
@@ -3836,7 +3844,7 @@ dependencies = [
"datafusion-physical-plan",
"datafusion-session",
"futures",
- "itertools 0.14.0",
+ "itertools 0.15.0",
"log",
"object_store",
"parking_lot 0.12.4",
@@ -3845,10 +3853,10 @@ dependencies = [
[[package]]
name = "datafusion-catalog-listing"
-version = "53.1.0"
-source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=bb531e754b3b36d7a4d3222f35ebc88204dafd2b#bb531e754b3b36d7a4d3222f35ebc88204dafd2b"
+version = "55.0.0"
+source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=b66dca3260c8e04f314c8036cdfbb08f1a564d95#b66dca3260c8e04f314c8036cdfbb08f1a564d95"
dependencies = [
- "arrow 58.3.0",
+ "arrow 59.2.0",
"async-trait",
"datafusion-catalog",
"datafusion-common",
@@ -3860,39 +3868,42 @@ dependencies = [
"datafusion-physical-expr-common",
"datafusion-physical-plan",
"futures",
- "itertools 0.14.0",
+ "itertools 0.15.0",
"log",
"object_store",
+ "percent-encoding",
]
[[package]]
name = "datafusion-common"
-version = "53.1.0"
-source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=bb531e754b3b36d7a4d3222f35ebc88204dafd2b#bb531e754b3b36d7a4d3222f35ebc88204dafd2b"
+version = "55.0.0"
+source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=b66dca3260c8e04f314c8036cdfbb08f1a564d95#b66dca3260c8e04f314c8036cdfbb08f1a564d95"
dependencies = [
- "ahash 0.8.12",
- "arrow 58.3.0",
- "arrow-ipc 58.3.0",
+ "arrow 59.2.0",
+ "arrow-ipc 59.2.0",
+ "arrow-schema 59.2.0",
"chrono",
+ "foldhash 0.2.0",
"half",
- "hashbrown 0.16.1",
- "indexmap 2.13.0",
- "itertools 0.14.0",
+ "hashbrown 0.17.1",
+ "indexmap 2.14.2",
+ "itertools 0.15.0",
"libc",
"log",
+ "num-traits",
"object_store",
"parquet",
- "paste",
"recursive",
"sqlparser",
"tokio",
+ "uuid",
"web-time",
]
[[package]]
name = "datafusion-common-runtime"
-version = "53.1.0"
-source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=bb531e754b3b36d7a4d3222f35ebc88204dafd2b#bb531e754b3b36d7a4d3222f35ebc88204dafd2b"
+version = "55.0.0"
+source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=b66dca3260c8e04f314c8036cdfbb08f1a564d95#b66dca3260c8e04f314c8036cdfbb08f1a564d95"
dependencies = [
"futures",
"log",
@@ -3901,10 +3912,10 @@ dependencies = [
[[package]]
name = "datafusion-datasource"
-version = "53.1.0"
-source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=bb531e754b3b36d7a4d3222f35ebc88204dafd2b#bb531e754b3b36d7a4d3222f35ebc88204dafd2b"
+version = "55.0.0"
+source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=b66dca3260c8e04f314c8036cdfbb08f1a564d95#b66dca3260c8e04f314c8036cdfbb08f1a564d95"
dependencies = [
- "arrow 58.3.0",
+ "arrow 59.2.0",
"async-compression",
"async-trait",
"bytes",
@@ -3918,14 +3929,16 @@ dependencies = [
"datafusion-physical-expr-adapter",
"datafusion-physical-expr-common",
"datafusion-physical-plan",
+ "datafusion-proto-models",
"datafusion-session",
"flate2",
"futures",
"glob",
- "itertools 0.14.0",
+ "itertools 0.15.0",
"liblzma",
"log",
"object_store",
+ "parking_lot 0.12.4",
"rand 0.9.4",
"tokio",
"tokio-util",
@@ -3935,11 +3948,11 @@ dependencies = [
[[package]]
name = "datafusion-datasource-arrow"
-version = "53.1.0"
-source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=bb531e754b3b36d7a4d3222f35ebc88204dafd2b#bb531e754b3b36d7a4d3222f35ebc88204dafd2b"
+version = "55.0.0"
+source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=b66dca3260c8e04f314c8036cdfbb08f1a564d95#b66dca3260c8e04f314c8036cdfbb08f1a564d95"
dependencies = [
- "arrow 58.3.0",
- "arrow-ipc 58.3.0",
+ "arrow 59.2.0",
+ "arrow-ipc 59.2.0",
"async-trait",
"bytes",
"datafusion-common",
@@ -3949,19 +3962,20 @@ dependencies = [
"datafusion-expr",
"datafusion-physical-expr-common",
"datafusion-physical-plan",
+ "datafusion-proto-models",
"datafusion-session",
"futures",
- "itertools 0.14.0",
+ "itertools 0.15.0",
"object_store",
"tokio",
]
[[package]]
name = "datafusion-datasource-csv"
-version = "53.1.0"
-source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=bb531e754b3b36d7a4d3222f35ebc88204dafd2b#bb531e754b3b36d7a4d3222f35ebc88204dafd2b"
+version = "55.0.0"
+source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=b66dca3260c8e04f314c8036cdfbb08f1a564d95#b66dca3260c8e04f314c8036cdfbb08f1a564d95"
dependencies = [
- "arrow 58.3.0",
+ "arrow 59.2.0",
"async-trait",
"bytes",
"datafusion-common",
@@ -3971,6 +3985,7 @@ dependencies = [
"datafusion-expr",
"datafusion-physical-expr-common",
"datafusion-physical-plan",
+ "datafusion-proto-models",
"datafusion-session",
"futures",
"object_store",
@@ -3980,10 +3995,10 @@ dependencies = [
[[package]]
name = "datafusion-datasource-json"
-version = "53.1.0"
-source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=bb531e754b3b36d7a4d3222f35ebc88204dafd2b#bb531e754b3b36d7a4d3222f35ebc88204dafd2b"
+version = "55.0.0"
+source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=b66dca3260c8e04f314c8036cdfbb08f1a564d95#b66dca3260c8e04f314c8036cdfbb08f1a564d95"
dependencies = [
- "arrow 58.3.0",
+ "arrow 59.2.0",
"async-trait",
"bytes",
"datafusion-common",
@@ -3993,20 +4008,21 @@ dependencies = [
"datafusion-expr",
"datafusion-physical-expr-common",
"datafusion-physical-plan",
+ "datafusion-proto-models",
"datafusion-session",
"futures",
"object_store",
- "serde_json",
"tokio",
"tokio-stream",
]
[[package]]
name = "datafusion-datasource-parquet"
-version = "53.1.0"
-source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=bb531e754b3b36d7a4d3222f35ebc88204dafd2b#bb531e754b3b36d7a4d3222f35ebc88204dafd2b"
+version = "55.0.0"
+source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=b66dca3260c8e04f314c8036cdfbb08f1a564d95#b66dca3260c8e04f314c8036cdfbb08f1a564d95"
dependencies = [
- "arrow 58.3.0",
+ "arrow 59.2.0",
+ "arrow-schema 59.2.0",
"async-trait",
"bytes",
"datafusion-common",
@@ -4014,15 +4030,17 @@ dependencies = [
"datafusion-datasource",
"datafusion-execution",
"datafusion-expr",
+ "datafusion-functions",
"datafusion-functions-aggregate-common",
"datafusion-physical-expr",
"datafusion-physical-expr-adapter",
"datafusion-physical-expr-common",
"datafusion-physical-plan",
+ "datafusion-proto-models",
"datafusion-pruning",
"datafusion-session",
"futures",
- "itertools 0.14.0",
+ "itertools 0.15.0",
"log",
"object_store",
"parking_lot 0.12.4",
@@ -4032,18 +4050,18 @@ dependencies = [
[[package]]
name = "datafusion-doc"
-version = "53.1.0"
-source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=bb531e754b3b36d7a4d3222f35ebc88204dafd2b#bb531e754b3b36d7a4d3222f35ebc88204dafd2b"
+version = "55.0.0"
+source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=b66dca3260c8e04f314c8036cdfbb08f1a564d95#b66dca3260c8e04f314c8036cdfbb08f1a564d95"
[[package]]
name = "datafusion-execution"
-version = "53.1.0"
-source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=bb531e754b3b36d7a4d3222f35ebc88204dafd2b#bb531e754b3b36d7a4d3222f35ebc88204dafd2b"
+version = "55.0.0"
+source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=b66dca3260c8e04f314c8036cdfbb08f1a564d95#b66dca3260c8e04f314c8036cdfbb08f1a564d95"
dependencies = [
- "arrow 58.3.0",
- "arrow-buffer 58.3.0",
+ "arrow 59.2.0",
+ "arrow-buffer 59.2.0",
"async-trait",
- "chrono",
+ "bytes",
"dashmap",
"datafusion-common",
"datafusion-expr",
@@ -4052,17 +4070,21 @@ dependencies = [
"log",
"object_store",
"parking_lot 0.12.4",
+ "pin-project-lite",
"rand 0.9.4",
"tempfile",
+ "tokio",
+ "tokio-util",
"url",
]
[[package]]
name = "datafusion-expr"
-version = "53.1.0"
-source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=bb531e754b3b36d7a4d3222f35ebc88204dafd2b#bb531e754b3b36d7a4d3222f35ebc88204dafd2b"
+version = "55.0.0"
+source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=b66dca3260c8e04f314c8036cdfbb08f1a564d95#b66dca3260c8e04f314c8036cdfbb08f1a564d95"
dependencies = [
- "arrow 58.3.0",
+ "arrow 59.2.0",
+ "arrow-schema 59.2.0",
"async-trait",
"chrono",
"datafusion-common",
@@ -4071,9 +4093,10 @@ dependencies = [
"datafusion-functions-aggregate-common",
"datafusion-functions-window-common",
"datafusion-physical-expr-common",
- "indexmap 2.13.0",
- "itertools 0.14.0",
- "paste",
+ "datafusion-proto-common",
+ "datafusion-proto-models",
+ "indexmap 2.14.2",
+ "itertools 0.15.0",
"recursive",
"serde_json",
"sqlparser",
@@ -4081,24 +4104,23 @@ dependencies = [
[[package]]
name = "datafusion-expr-common"
-version = "53.1.0"
-source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=bb531e754b3b36d7a4d3222f35ebc88204dafd2b#bb531e754b3b36d7a4d3222f35ebc88204dafd2b"
+version = "55.0.0"
+source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=b66dca3260c8e04f314c8036cdfbb08f1a564d95#b66dca3260c8e04f314c8036cdfbb08f1a564d95"
dependencies = [
- "arrow 58.3.0",
+ "arrow 59.2.0",
"datafusion-common",
- "indexmap 2.13.0",
- "itertools 0.14.0",
- "paste",
+ "indexmap 2.14.2",
+ "itertools 0.15.0",
]
[[package]]
name = "datafusion-functions"
-version = "53.1.0"
-source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=bb531e754b3b36d7a4d3222f35ebc88204dafd2b#bb531e754b3b36d7a4d3222f35ebc88204dafd2b"
+version = "55.0.0"
+source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=b66dca3260c8e04f314c8036cdfbb08f1a564d95#b66dca3260c8e04f314c8036cdfbb08f1a564d95"
dependencies = [
- "arrow 58.3.0",
- "arrow-buffer 58.3.0",
- "base64 0.22.1",
+ "arrow 59.2.0",
+ "arrow-buffer 59.2.0",
+ "base64 0.23.1",
"blake2",
"blake3",
"chrono",
@@ -4109,26 +4131,25 @@ dependencies = [
"datafusion-expr",
"datafusion-expr-common",
"datafusion-macros",
+ "datafusion-physical-expr-common",
"hex",
- "itertools 0.14.0",
+ "itertools 0.15.0",
"log",
- "md-5 0.10.6",
+ "md-5 0.11.0",
"memchr",
"num-traits",
"rand 0.9.4",
"regex",
- "sha2 0.10.9",
- "unicode-segmentation",
+ "sha2 0.11.0",
"uuid",
]
[[package]]
name = "datafusion-functions-aggregate"
-version = "53.1.0"
-source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=bb531e754b3b36d7a4d3222f35ebc88204dafd2b#bb531e754b3b36d7a4d3222f35ebc88204dafd2b"
+version = "55.0.0"
+source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=b66dca3260c8e04f314c8036cdfbb08f1a564d95#b66dca3260c8e04f314c8036cdfbb08f1a564d95"
dependencies = [
- "ahash 0.8.12",
- "arrow 58.3.0",
+ "arrow 59.2.0",
"datafusion-common",
"datafusion-doc",
"datafusion-execution",
@@ -4138,18 +4159,17 @@ dependencies = [
"datafusion-physical-expr",
"datafusion-physical-expr-common",
"half",
+ "hashbrown 0.17.1",
"log",
"num-traits",
- "paste",
]
[[package]]
name = "datafusion-functions-aggregate-common"
-version = "53.1.0"
-source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=bb531e754b3b36d7a4d3222f35ebc88204dafd2b#bb531e754b3b36d7a4d3222f35ebc88204dafd2b"
+version = "55.0.0"
+source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=b66dca3260c8e04f314c8036cdfbb08f1a564d95#b66dca3260c8e04f314c8036cdfbb08f1a564d95"
dependencies = [
- "ahash 0.8.12",
- "arrow 58.3.0",
+ "arrow 59.2.0",
"datafusion-common",
"datafusion-expr-common",
"datafusion-physical-expr-common",
@@ -4157,11 +4177,11 @@ dependencies = [
[[package]]
name = "datafusion-functions-nested"
-version = "53.1.0"
-source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=bb531e754b3b36d7a4d3222f35ebc88204dafd2b#bb531e754b3b36d7a4d3222f35ebc88204dafd2b"
+version = "55.0.0"
+source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=b66dca3260c8e04f314c8036cdfbb08f1a564d95#b66dca3260c8e04f314c8036cdfbb08f1a564d95"
dependencies = [
- "arrow 58.3.0",
- "arrow-ord 58.3.0",
+ "arrow 59.2.0",
+ "arrow-ord 59.2.0",
"datafusion-common",
"datafusion-doc",
"datafusion-execution",
@@ -4172,34 +4192,34 @@ dependencies = [
"datafusion-functions-aggregate-common",
"datafusion-macros",
"datafusion-physical-expr-common",
- "hashbrown 0.16.1",
- "itertools 0.14.0",
+ "hashbrown 0.17.1",
+ "itertools 0.15.0",
"itoa",
"log",
- "paste",
+ "memchr",
]
[[package]]
name = "datafusion-functions-table"
-version = "53.1.0"
-source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=bb531e754b3b36d7a4d3222f35ebc88204dafd2b#bb531e754b3b36d7a4d3222f35ebc88204dafd2b"
+version = "55.0.0"
+source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=b66dca3260c8e04f314c8036cdfbb08f1a564d95#b66dca3260c8e04f314c8036cdfbb08f1a564d95"
dependencies = [
- "arrow 58.3.0",
+ "arrow 59.2.0",
"async-trait",
"datafusion-catalog",
"datafusion-common",
"datafusion-expr",
+ "datafusion-physical-expr",
"datafusion-physical-plan",
"parking_lot 0.12.4",
- "paste",
]
[[package]]
name = "datafusion-functions-window"
-version = "53.1.0"
-source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=bb531e754b3b36d7a4d3222f35ebc88204dafd2b#bb531e754b3b36d7a4d3222f35ebc88204dafd2b"
+version = "55.0.0"
+source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=b66dca3260c8e04f314c8036cdfbb08f1a564d95#b66dca3260c8e04f314c8036cdfbb08f1a564d95"
dependencies = [
- "arrow 58.3.0",
+ "arrow 59.2.0",
"datafusion-common",
"datafusion-doc",
"datafusion-expr",
@@ -4208,13 +4228,12 @@ dependencies = [
"datafusion-physical-expr",
"datafusion-physical-expr-common",
"log",
- "paste",
]
[[package]]
name = "datafusion-functions-window-common"
-version = "53.1.0"
-source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=bb531e754b3b36d7a4d3222f35ebc88204dafd2b#bb531e754b3b36d7a4d3222f35ebc88204dafd2b"
+version = "55.0.0"
+source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=b66dca3260c8e04f314c8036cdfbb08f1a564d95#b66dca3260c8e04f314c8036cdfbb08f1a564d95"
dependencies = [
"datafusion-common",
"datafusion-physical-expr-common",
@@ -4222,27 +4241,27 @@ dependencies = [
[[package]]
name = "datafusion-macros"
-version = "53.1.0"
-source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=bb531e754b3b36d7a4d3222f35ebc88204dafd2b#bb531e754b3b36d7a4d3222f35ebc88204dafd2b"
+version = "55.0.0"
+source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=b66dca3260c8e04f314c8036cdfbb08f1a564d95#b66dca3260c8e04f314c8036cdfbb08f1a564d95"
dependencies = [
"datafusion-doc",
"quote",
- "syn 2.0.117",
+ "syn 3.0.5",
]
[[package]]
name = "datafusion-optimizer"
-version = "53.1.0"
-source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=bb531e754b3b36d7a4d3222f35ebc88204dafd2b#bb531e754b3b36d7a4d3222f35ebc88204dafd2b"
+version = "55.0.0"
+source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=b66dca3260c8e04f314c8036cdfbb08f1a564d95#b66dca3260c8e04f314c8036cdfbb08f1a564d95"
dependencies = [
- "arrow 58.3.0",
+ "arrow 59.2.0",
"chrono",
"datafusion-common",
"datafusion-expr",
"datafusion-expr-common",
"datafusion-physical-expr",
- "indexmap 2.13.0",
- "itertools 0.14.0",
+ "indexmap 2.14.2",
+ "itertools 0.15.0",
"log",
"recursive",
"regex",
@@ -4251,24 +4270,23 @@ dependencies = [
[[package]]
name = "datafusion-orc"
-version = "0.8.0"
-source = "git+https://github.com/datafusion-contrib/datafusion-orc.git?rev=6c07fa282dc8d62db2aa4ded06ab55485efc811a#6c07fa282dc8d62db2aa4ded06ab55485efc811a"
+version = "0.10.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "637382f27aa0c7d6f30ca18a3e37d46bdf427aa83e1b0366083dc38e362dae99"
dependencies = [
"async-trait",
"bytes",
"datafusion",
"futures",
"futures-util",
- "object_store",
"orc-rust",
"tokio",
]
[[package]]
name = "datafusion-pg-catalog"
-version = "0.17.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f6b82fb8bd291b718d4226eaba85f78c73f0abdedb256a5dc91d9d9e6b0e0dab"
+version = "0.18.3"
+source = "git+https://github.com/GreptimeTeam/datafusion-postgres.git?rev=3c77e6c32b8db80635a0d2f4b318a36b31170bc3#3c77e6c32b8db80635a0d2f4b318a36b31170bc3"
dependencies = [
"arrow-pg",
"async-trait",
@@ -4281,22 +4299,21 @@ dependencies = [
[[package]]
name = "datafusion-physical-expr"
-version = "53.1.0"
-source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=bb531e754b3b36d7a4d3222f35ebc88204dafd2b#bb531e754b3b36d7a4d3222f35ebc88204dafd2b"
+version = "55.0.0"
+source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=b66dca3260c8e04f314c8036cdfbb08f1a564d95#b66dca3260c8e04f314c8036cdfbb08f1a564d95"
dependencies = [
- "ahash 0.8.12",
- "arrow 58.3.0",
+ "arrow 59.2.0",
"datafusion-common",
"datafusion-expr",
"datafusion-expr-common",
"datafusion-functions-aggregate-common",
"datafusion-physical-expr-common",
+ "datafusion-proto-models",
"half",
- "hashbrown 0.16.1",
- "indexmap 2.13.0",
- "itertools 0.14.0",
+ "hashbrown 0.17.1",
+ "indexmap 2.14.2",
+ "itertools 0.15.0",
"parking_lot 0.12.4",
- "paste",
"petgraph 0.8.3",
"recursive",
"tokio",
@@ -4304,40 +4321,41 @@ dependencies = [
[[package]]
name = "datafusion-physical-expr-adapter"
-version = "53.1.0"
-source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=bb531e754b3b36d7a4d3222f35ebc88204dafd2b#bb531e754b3b36d7a4d3222f35ebc88204dafd2b"
+version = "55.0.0"
+source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=b66dca3260c8e04f314c8036cdfbb08f1a564d95#b66dca3260c8e04f314c8036cdfbb08f1a564d95"
dependencies = [
- "arrow 58.3.0",
+ "arrow 59.2.0",
"datafusion-common",
"datafusion-expr",
"datafusion-functions",
"datafusion-physical-expr",
"datafusion-physical-expr-common",
- "itertools 0.14.0",
+ "itertools 0.15.0",
]
[[package]]
name = "datafusion-physical-expr-common"
-version = "53.1.0"
-source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=bb531e754b3b36d7a4d3222f35ebc88204dafd2b#bb531e754b3b36d7a4d3222f35ebc88204dafd2b"
+version = "55.0.0"
+source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=b66dca3260c8e04f314c8036cdfbb08f1a564d95#b66dca3260c8e04f314c8036cdfbb08f1a564d95"
dependencies = [
- "ahash 0.8.12",
- "arrow 58.3.0",
+ "arrow 59.2.0",
"chrono",
"datafusion-common",
"datafusion-expr-common",
- "hashbrown 0.16.1",
- "indexmap 2.13.0",
- "itertools 0.14.0",
+ "datafusion-proto-models",
+ "hashbrown 0.17.1",
+ "indexmap 2.14.2",
+ "itertools 0.15.0",
"parking_lot 0.12.4",
+ "pin-project",
]
[[package]]
name = "datafusion-physical-optimizer"
-version = "53.1.0"
-source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=bb531e754b3b36d7a4d3222f35ebc88204dafd2b#bb531e754b3b36d7a4d3222f35ebc88204dafd2b"
+version = "55.0.0"
+source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=b66dca3260c8e04f314c8036cdfbb08f1a564d95#b66dca3260c8e04f314c8036cdfbb08f1a564d95"
dependencies = [
- "arrow 58.3.0",
+ "arrow 59.2.0",
"datafusion-common",
"datafusion-execution",
"datafusion-expr",
@@ -4346,20 +4364,23 @@ dependencies = [
"datafusion-physical-expr-common",
"datafusion-physical-plan",
"datafusion-pruning",
- "itertools 0.14.0",
+ "datafusion-session",
+ "itertools 0.15.0",
"recursive",
]
[[package]]
name = "datafusion-physical-plan"
-version = "53.1.0"
-source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=bb531e754b3b36d7a4d3222f35ebc88204dafd2b#bb531e754b3b36d7a4d3222f35ebc88204dafd2b"
+version = "55.0.0"
+source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=b66dca3260c8e04f314c8036cdfbb08f1a564d95#b66dca3260c8e04f314c8036cdfbb08f1a564d95"
dependencies = [
- "ahash 0.8.12",
- "arrow 58.3.0",
- "arrow-ord 58.3.0",
- "arrow-schema 58.3.0",
+ "arrow 59.2.0",
+ "arrow-data 59.2.0",
+ "arrow-ipc 59.2.0",
+ "arrow-ord 59.2.0",
+ "arrow-schema 59.2.0",
"async-trait",
+ "bytes",
"datafusion-common",
"datafusion-common-runtime",
"datafusion-execution",
@@ -4369,25 +4390,28 @@ dependencies = [
"datafusion-functions-window-common",
"datafusion-physical-expr",
"datafusion-physical-expr-common",
+ "datafusion-proto-common",
+ "datafusion-proto-models",
"futures",
"half",
- "hashbrown 0.16.1",
- "indexmap 2.13.0",
- "itertools 0.14.0",
+ "hashbrown 0.17.1",
+ "indexmap 2.14.2",
+ "itertools 0.15.0",
"log",
"num-traits",
"parking_lot 0.12.4",
"pin-project-lite",
+ "serde",
+ "serde_json",
"tokio",
]
[[package]]
name = "datafusion-proto"
-version = "53.1.0"
-source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=bb531e754b3b36d7a4d3222f35ebc88204dafd2b#bb531e754b3b36d7a4d3222f35ebc88204dafd2b"
+version = "55.0.0"
+source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=b66dca3260c8e04f314c8036cdfbb08f1a564d95#b66dca3260c8e04f314c8036cdfbb08f1a564d95"
dependencies = [
- "arrow 58.3.0",
- "chrono",
+ "arrow 59.2.0",
"datafusion-catalog",
"datafusion-catalog-listing",
"datafusion-common",
@@ -4403,42 +4427,52 @@ dependencies = [
"datafusion-physical-expr-common",
"datafusion-physical-plan",
"datafusion-proto-common",
+ "datafusion-proto-models",
"object_store",
"prost 0.14.1",
- "rand 0.9.4",
]
[[package]]
name = "datafusion-proto-common"
-version = "53.1.0"
-source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=bb531e754b3b36d7a4d3222f35ebc88204dafd2b#bb531e754b3b36d7a4d3222f35ebc88204dafd2b"
+version = "55.0.0"
+source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=b66dca3260c8e04f314c8036cdfbb08f1a564d95#b66dca3260c8e04f314c8036cdfbb08f1a564d95"
dependencies = [
- "arrow 58.3.0",
+ "arrow 59.2.0",
"datafusion-common",
"prost 0.14.1",
]
[[package]]
-name = "datafusion-pruning"
-version = "53.1.0"
-source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=bb531e754b3b36d7a4d3222f35ebc88204dafd2b#bb531e754b3b36d7a4d3222f35ebc88204dafd2b"
+name = "datafusion-proto-models"
+version = "55.0.0"
+source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=b66dca3260c8e04f314c8036cdfbb08f1a564d95#b66dca3260c8e04f314c8036cdfbb08f1a564d95"
dependencies = [
- "arrow 58.3.0",
+ "datafusion-common",
+ "datafusion-proto-common",
+ "prost 0.14.1",
+]
+
+[[package]]
+name = "datafusion-pruning"
+version = "55.0.0"
+source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=b66dca3260c8e04f314c8036cdfbb08f1a564d95#b66dca3260c8e04f314c8036cdfbb08f1a564d95"
+dependencies = [
+ "arrow 59.2.0",
"datafusion-common",
"datafusion-datasource",
"datafusion-expr-common",
"datafusion-physical-expr",
"datafusion-physical-expr-common",
"datafusion-physical-plan",
- "itertools 0.14.0",
"log",
]
[[package]]
name = "datafusion-session"
-version = "53.1.0"
-source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=bb531e754b3b36d7a4d3222f35ebc88204dafd2b#bb531e754b3b36d7a4d3222f35ebc88204dafd2b"
+version = "55.0.0"
+source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=b66dca3260c8e04f314c8036cdfbb08f1a564d95#b66dca3260c8e04f314c8036cdfbb08f1a564d95"
dependencies = [
+ "arrow-schema 59.2.0",
"async-trait",
"datafusion-common",
"datafusion-execution",
@@ -4449,37 +4483,38 @@ dependencies = [
[[package]]
name = "datafusion-sql"
-version = "53.1.0"
-source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=bb531e754b3b36d7a4d3222f35ebc88204dafd2b#bb531e754b3b36d7a4d3222f35ebc88204dafd2b"
+version = "55.0.0"
+source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=b66dca3260c8e04f314c8036cdfbb08f1a564d95#b66dca3260c8e04f314c8036cdfbb08f1a564d95"
dependencies = [
- "arrow 58.3.0",
+ "arrow 59.2.0",
"bigdecimal 0.4.8",
"chrono",
"datafusion-common",
"datafusion-expr",
"datafusion-functions-nested",
- "indexmap 2.13.0",
+ "indexmap 2.14.2",
"log",
"recursive",
"regex",
"sqlparser",
+ "stacker",
]
[[package]]
name = "datafusion-substrait"
-version = "53.1.0"
-source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=bb531e754b3b36d7a4d3222f35ebc88204dafd2b#bb531e754b3b36d7a4d3222f35ebc88204dafd2b"
+version = "55.0.0"
+source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=b66dca3260c8e04f314c8036cdfbb08f1a564d95#b66dca3260c8e04f314c8036cdfbb08f1a564d95"
dependencies = [
"async-recursion",
"async-trait",
"chrono",
"datafusion",
"half",
- "itertools 0.14.0",
+ "itertools 0.15.0",
"object_store",
"pbjson-types",
"prost 0.14.1",
- "substrait 0.62.2",
+ "substrait 0.63.0",
"tokio",
"url",
]
@@ -4563,9 +4598,9 @@ checksum = "c286de4e81ea2590afc24d754e0f83810c566f50a1388fa75ebd57928c0d9745"
name = "datatypes"
version = "1.3.0-alpha.1"
dependencies = [
- "arrow 58.3.0",
- "arrow-array 58.3.0",
- "arrow-schema 58.3.0",
+ "arrow 59.2.0",
+ "arrow-array 59.2.0",
+ "arrow-schema 59.2.0",
"common-base",
"common-decimal",
"common-error",
@@ -5359,6 +5394,12 @@ dependencies = [
"windows-sys 0.59.0",
]
+[[package]]
+name = "find-msvc-tools"
+version = "0.1.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3e0f1c7c3a72c66fd80abe965175f7523475c0489a87d3ff9d6e8c87d87a9d2d"
+
[[package]]
name = "findshlibs"
version = "0.10.2"
@@ -5459,9 +5500,9 @@ name = "flow"
version = "1.3.0-alpha.1"
dependencies = [
"api",
- "arrow 58.3.0",
+ "arrow 59.2.0",
"arrow-flight",
- "arrow-schema 58.3.0",
+ "arrow-schema 59.2.0",
"async-recursion",
"async-trait",
"bytes",
@@ -5637,6 +5678,7 @@ dependencies = [
"catalog",
"client",
"common-base",
+ "common-batcher",
"common-catalog",
"common-config",
"common-datasource",
@@ -6121,7 +6163,7 @@ dependencies = [
[[package]]
name = "greptime-proto"
version = "0.1.0"
-source = "git+https://github.com/GreptimeTeam/greptime-proto.git?rev=b74667d17827481dbb6222fa0326163627211557#b74667d17827481dbb6222fa0326163627211557"
+source = "git+https://github.com/GreptimeTeam/greptime-proto.git?rev=0c4beffaa2b3968034915867593a717c6ab0c6b4#0c4beffaa2b3968034915867593a717c6ab0c6b4"
dependencies = [
"prost 0.14.1",
"prost-types 0.14.1",
@@ -6155,7 +6197,7 @@ dependencies = [
"futures-sink",
"futures-util",
"http 0.2.12",
- "indexmap 2.13.0",
+ "indexmap 2.14.2",
"slab",
"tokio",
"tokio-util",
@@ -6174,7 +6216,7 @@ dependencies = [
"futures-core",
"futures-sink",
"http 1.5.0",
- "indexmap 2.13.0",
+ "indexmap 2.14.2",
"slab",
"tokio",
"tokio-util",
@@ -6987,12 +7029,12 @@ dependencies = [
[[package]]
name = "indexmap"
-version = "2.13.0"
+version = "2.14.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7714e70437a7dc3ac8eb7e6f8df75fd8eb422675fc7678aff7364301092b1017"
+checksum = "cc4e190f5d26ca7051642629da2c52fc03bde85a03197c99408dcd291734c855"
dependencies = [
"equivalent",
- "hashbrown 0.16.1",
+ "hashbrown 0.17.1",
"serde",
"serde_core",
]
@@ -7023,7 +7065,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "232929e1d75fe899576a3d5c7416ad0d88dbfbb3c3d6aa00873a7408a50ddb88"
dependencies = [
"ahash 0.8.12",
- "indexmap 2.13.0",
+ "indexmap 2.14.2",
"is-terminal",
"itoa",
"log",
@@ -7046,7 +7088,7 @@ dependencies = [
"crossbeam-utils",
"dashmap",
"env_logger",
- "indexmap 2.13.0",
+ "indexmap 2.14.2",
"itoa",
"log",
"num-format",
@@ -7130,12 +7172,6 @@ dependencies = [
"cfg-if",
]
-[[package]]
-name = "integer-encoding"
-version = "3.0.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8bb03732005da905c88227371639bf1ad885cc712789c011c31c5fb3ab3ccf02"
-
[[package]]
name = "integer-encoding"
version = "4.0.2"
@@ -7268,6 +7304,15 @@ dependencies = [
"either",
]
+[[package]]
+name = "itertools"
+version = "0.15.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8b4baf93f58d4425749ca49a51c50ebab072c5df6994d08fed93541c331481dc"
+dependencies = [
+ "either",
+]
+
[[package]]
name = "itoa"
version = "1.0.15"
@@ -7545,7 +7590,7 @@ version = "0.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4ee7893dab2e44ae5f9d0173f26ff4aa327c10b01b06a72b52dd9405b628640d"
dependencies = [
- "indexmap 2.13.0",
+ "indexmap 2.14.2",
]
[[package]]
@@ -8157,7 +8202,7 @@ dependencies = [
"cactus",
"cfgrammar",
"filetime",
- "indexmap 2.13.0",
+ "indexmap 2.14.2",
"lazy_static",
"lrtable",
"num-traits",
@@ -8240,6 +8285,12 @@ name = "lz4_flex"
version = "0.13.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7ef0d4ed8669f8f8826eb00dc878084aa8f253506c4fd5e8f58f5bce72ddb97e"
+
+[[package]]
+name = "lz4_flex"
+version = "0.14.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ecbdfe44b1bd960b68170b417450a628c43f7cf56bb3c5317e61cb230ee7f226"
dependencies = [
"twox-hash",
]
@@ -8367,9 +8418,9 @@ dependencies = [
[[package]]
name = "memchr"
-version = "2.8.0"
+version = "2.8.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79"
+checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98"
[[package]]
name = "memcomparable"
@@ -8672,7 +8723,7 @@ version = "1.3.0-alpha.1"
dependencies = [
"api",
"aquamarine",
- "arrow-schema 58.3.0",
+ "arrow-schema 59.2.0",
"async-channel 1.9.0",
"async-stream",
"async-trait",
@@ -9614,9 +9665,9 @@ dependencies = [
[[package]]
name = "once_cell"
-version = "1.21.3"
+version = "1.21.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d"
+checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50"
[[package]]
name = "once_cell_polyfill"
@@ -10053,10 +10104,11 @@ version = "1.3.0-alpha.1"
dependencies = [
"ahash 0.8.12",
"api",
- "arrow 58.3.0",
- "arrow-ipc 58.3.0",
+ "arrow 59.2.0",
+ "arrow-ipc 59.2.0",
"async-stream",
"async-trait",
+ "axum 0.8.4",
"bytes",
"catalog",
"chrono",
@@ -10117,15 +10169,16 @@ dependencies = [
"tokio-util",
"tonic 0.14.2",
"tracing",
+ "url",
]
[[package]]
name = "orc-rust"
-version = "0.8.0"
+version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "32b9867c4e7343218682ba11aceae1310c80735ec2beaf6124a0f8f848dad197"
+checksum = "e54c163f843c4f0fcfca65b9b73ca6f93c37c8e86736d5a1e578c6e843eed0a0"
dependencies = [
- "arrow 58.3.0",
+ "arrow 59.2.0",
"async-trait",
"bytemuck",
"bytes",
@@ -10348,18 +10401,18 @@ dependencies = [
[[package]]
name = "parquet"
-version = "58.3.0"
+version = "59.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5dafa7d01085b62a47dd0c1829550a0a36710ea9c4fe358a05a85477cec8a908"
+checksum = "7065842956a20c2a536924ce8e4d9955f7422451511b9eb7500d7bfe5077e59c"
dependencies = [
"ahash 0.8.12",
- "arrow-array 58.3.0",
- "arrow-buffer 58.3.0",
- "arrow-data 58.3.0",
- "arrow-ipc 58.3.0",
- "arrow-schema 58.3.0",
- "arrow-select 58.3.0",
- "base64 0.22.1",
+ "arrow-array 59.2.0",
+ "arrow-buffer 59.2.0",
+ "arrow-data 59.2.0",
+ "arrow-ipc 59.2.0",
+ "arrow-schema 59.2.0",
+ "arrow-select 59.2.0",
+ "base64 0.23.1",
"brotli",
"bytes",
"chrono",
@@ -10367,19 +10420,17 @@ dependencies = [
"futures",
"half",
"hashbrown 0.17.1",
- "lz4_flex 0.13.1",
- "num-bigint 0.4.6",
+ "lz4_flex 0.14.0",
+ "num-bigint 0.5.1",
"num-integer",
"num-traits",
"object_store",
"parquet-variant",
"parquet-variant-compute",
"parquet-variant-json",
- "paste",
"seq-macro",
"simdutf8",
"snap",
- "thrift",
"tokio",
"twox-hash",
"zstd",
@@ -10387,15 +10438,15 @@ dependencies = [
[[package]]
name = "parquet-variant"
-version = "58.3.0"
+version = "59.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "74c8db065291f088a2aad8ab831853eae1871c0d311c8d0b83bbc3b7e735d0fc"
+checksum = "3f7e5fff3ed0c07514a7fb8bee3f2ea5a53f36939410ecac4a466620213539a8"
dependencies = [
- "arrow 58.3.0",
- "arrow-schema 58.3.0",
+ "arrow 59.2.0",
+ "arrow-schema 59.2.0",
"chrono",
"half",
- "indexmap 2.13.0",
+ "indexmap 2.14.2",
"num-traits",
"simdutf8",
"uuid",
@@ -10403,15 +10454,15 @@ dependencies = [
[[package]]
name = "parquet-variant-compute"
-version = "58.3.0"
+version = "59.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a530e8d5b5e14efcb39c9a6ec55432ad11f6afb7dc4455a79be0dc615fe3cc31"
+checksum = "ba4d3de89dab8d1aaaf601ae8d71bd07ea88cfca9efc1df5815b982c30f631e1"
dependencies = [
- "arrow 58.3.0",
- "arrow-schema 58.3.0",
+ "arrow 59.2.0",
+ "arrow-schema 59.2.0",
"chrono",
"half",
- "indexmap 2.13.0",
+ "indexmap 2.14.2",
"parquet-variant",
"parquet-variant-json",
"serde_json",
@@ -10420,12 +10471,12 @@ dependencies = [
[[package]]
name = "parquet-variant-json"
-version = "58.3.0"
+version = "59.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "00ed89908289f67caa2ca078f9ff9aacd6229a313ec92b12bf4f48f613dc2b97"
+checksum = "fb19dfe1bd24c17addd761ba4f7000f615e2fa12525871c7baa835dbb3d7f147"
dependencies = [
- "arrow-schema 58.3.0",
- "base64 0.22.1",
+ "arrow-schema 59.2.0",
+ "base64 0.23.1",
"chrono",
"parquet-variant",
"serde_json",
@@ -10639,7 +10690,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b4c5cc86750666a3ed20bdaf5ca2a0344f9c67674cae0515bec2da16fbaa47db"
dependencies = [
"fixedbitset 0.4.2",
- "indexmap 2.13.0",
+ "indexmap 2.14.2",
]
[[package]]
@@ -10649,7 +10700,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3672b37090dbd86368a4145bc067582552b29c27377cad4e0a306c97f9bd7772"
dependencies = [
"fixedbitset 0.5.7",
- "indexmap 2.13.0",
+ "indexmap 2.14.2",
]
[[package]]
@@ -10660,7 +10711,7 @@ checksum = "8701b58ea97060d5e5b155d383a69952a60943f0e6dfe30b04c287beb0b27455"
dependencies = [
"fixedbitset 0.5.7",
"hashbrown 0.15.4",
- "indexmap 2.13.0",
+ "indexmap 2.14.2",
"serde",
]
@@ -10848,8 +10899,8 @@ version = "1.3.0-alpha.1"
dependencies = [
"ahash 0.8.12",
"api",
- "arrow 58.3.0",
- "arrow-schema 58.3.0",
+ "arrow 59.2.0",
+ "arrow-schema 59.2.0",
"async-trait",
"catalog",
"chrono",
@@ -10972,7 +11023,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3d77244ce2d584cd84f6a15f86195b8c9b2a0dfbfd817c09e0464244091a58ed"
dependencies = [
"base64 0.22.1",
- "indexmap 2.13.0",
+ "indexmap 2.14.2",
"quick-xml 0.37.5",
"serde",
"time",
@@ -11453,7 +11504,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "22505a5c94da8e3b7c2996394d1c933236c4d743e81a410bcca4e6989fc066a4"
dependencies = [
"bytes",
- "heck 0.4.1",
+ "heck 0.5.0",
"itertools 0.12.1",
"log",
"multimap",
@@ -11473,7 +11524,7 @@ version = "0.14.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ac6c3320f9abac597dcbc668774ef006702672474aad53c6d596b62e487b40b1"
dependencies = [
- "heck 0.4.1",
+ "heck 0.5.0",
"itertools 0.14.0",
"log",
"multimap",
@@ -11835,8 +11886,8 @@ dependencies = [
"ahash 0.8.12",
"api",
"arc-swap",
- "arrow 58.3.0",
- "arrow-schema 58.3.0",
+ "arrow 59.2.0",
+ "arrow-schema 59.2.0",
"async-recursion",
"async-stream",
"async-trait",
@@ -11863,6 +11914,7 @@ dependencies = [
"datafusion-expr-common",
"datafusion-functions",
"datafusion-optimizer",
+ "datafusion-pg-catalog",
"datafusion-physical-expr",
"datafusion-proto",
"datafusion-sql",
@@ -12317,7 +12369,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4c11639076bf147be211b90e47790db89f4c22b6c8a9ca6e960833869da67166"
dependencies = [
"aho-corasick",
- "indexmap 2.13.0",
+ "indexmap 2.14.2",
"itertools 0.13.0",
"nohash",
"regex",
@@ -12777,7 +12829,7 @@ dependencies = [
"crc32c",
"flate2",
"futures",
- "integer-encoding 4.0.2",
+ "integer-encoding",
"lz4",
"parking_lot 0.12.4",
"rand 0.10.1",
@@ -13485,7 +13537,7 @@ dependencies = [
"chrono",
"hex",
"indexmap 1.9.3",
- "indexmap 2.13.0",
+ "indexmap 2.14.2",
"schemars 0.9.0",
"schemars 1.2.1",
"serde_core",
@@ -13512,7 +13564,7 @@ version = "0.9.34+deprecated"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47"
dependencies = [
- "indexmap 2.13.0",
+ "indexmap 2.14.2",
"itoa",
"ryu",
"serde",
@@ -13525,7 +13577,7 @@ version = "0.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7b4db627b98b36d4203a7b458cf3573730f2bb591b28871d916dfa9efabfd41f"
dependencies = [
- "indexmap 2.13.0",
+ "indexmap 2.14.2",
"itoa",
"ryu",
"serde",
@@ -13538,11 +13590,11 @@ version = "1.3.0-alpha.1"
dependencies = [
"ahash 0.8.12",
"api",
- "arrow 58.3.0",
+ "arrow 59.2.0",
"arrow-flight",
- "arrow-ipc 58.3.0",
+ "arrow-ipc 59.2.0",
"arrow-pg",
- "arrow-schema 58.3.0",
+ "arrow-schema 59.2.0",
"async-trait",
"auth",
"axum 0.8.4",
@@ -13554,6 +13606,7 @@ dependencies = [
"chrono",
"client",
"common-base",
+ "common-batcher",
"common-catalog",
"common-decimal",
"common-error",
@@ -13594,7 +13647,7 @@ dependencies = [
"humantime",
"humantime-serde",
"hyper 1.6.0",
- "indexmap 2.13.0",
+ "indexmap 2.14.2",
"influxdb_line_protocol",
"itertools 0.14.0",
"json5",
@@ -13778,6 +13831,12 @@ version = "1.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64"
+[[package]]
+name = "shlex"
+version = "2.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba"
+
[[package]]
name = "signal-hook-registry"
version = "1.4.5"
@@ -13953,7 +14012,7 @@ version = "0.8.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1961e2ef424c1424204d3a5d6975f934f56b6d50ff5732382d84ebf460e147f7"
dependencies = [
- "heck 0.4.1",
+ "heck 0.5.0",
"proc-macro2",
"quote",
"syn 2.0.117",
@@ -14042,7 +14101,7 @@ name = "sql"
version = "1.3.0-alpha.1"
dependencies = [
"api",
- "arrow-buffer 58.3.0",
+ "arrow-buffer 59.2.0",
"chrono",
"common-base",
"common-catalog",
@@ -14132,13 +14191,11 @@ dependencies = [
[[package]]
name = "sqlparser"
-version = "0.61.0"
-source = "git+https://github.com/GreptimeTeam/sqlparser-rs.git?rev=2aefa08a8d69c96eec2d6d6703598a009bba6e4c#2aefa08a8d69c96eec2d6d6703598a009bba6e4c"
+version = "0.62.0"
+source = "git+https://github.com/GreptimeTeam/sqlparser-rs.git?rev=9e9019bb1c7040ed956f654e39378dd42ab17884#9e9019bb1c7040ed956f654e39378dd42ab17884"
dependencies = [
- "lazy_static",
"log",
"recursive",
- "regex",
"serde",
"sqlparser_derive 0.5.0",
]
@@ -14157,7 +14214,7 @@ dependencies = [
[[package]]
name = "sqlparser_derive"
version = "0.5.0"
-source = "git+https://github.com/GreptimeTeam/sqlparser-rs.git?rev=2aefa08a8d69c96eec2d6d6703598a009bba6e4c#2aefa08a8d69c96eec2d6d6703598a009bba6e4c"
+source = "git+https://github.com/GreptimeTeam/sqlparser-rs.git?rev=9e9019bb1c7040ed956f654e39378dd42ab17884#9e9019bb1c7040ed956f654e39378dd42ab17884"
dependencies = [
"proc-macro2",
"quote",
@@ -14196,7 +14253,7 @@ dependencies = [
"futures-util",
"hashbrown 0.15.4",
"hashlink",
- "indexmap 2.13.0",
+ "indexmap 2.14.2",
"log",
"memchr",
"once_cell",
@@ -14369,15 +14426,15 @@ checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3"
[[package]]
name = "stacker"
-version = "0.1.21"
+version = "0.1.25"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cddb07e32ddb770749da91081d8d0ac3a16f1a569a18b20348cd371f5dead06b"
+checksum = "707f49d46706bacf8a2b00d51dace3f9de527c13eec3778f570c411f89e69967"
dependencies = [
"cc",
"cfg-if",
"libc",
"psm",
- "windows-sys 0.52.0",
+ "windows-sys 0.61.2",
]
[[package]]
@@ -14599,11 +14656,11 @@ dependencies = [
[[package]]
name = "substrait"
-version = "0.62.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "62fc4b483a129b9772ccb9c3f7945a472112fdd9140da87f8a4e7f1d44e045d0"
+version = "0.63.0"
+source = "git+https://github.com/GreptimeTeam/substrait-rs.git?rev=91ec978b0649417ad3da8390e7a515baec723b1b#91ec978b0649417ad3da8390e7a515baec723b1b"
dependencies = [
"heck 0.5.0",
+ "indexmap 2.14.2",
"pbjson",
"pbjson-build",
"pbjson-types",
@@ -14700,6 +14757,17 @@ dependencies = [
"unicode-ident",
]
+[[package]]
+name = "syn"
+version = "3.0.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "12df2e0110f65b775f769bb17ef989067a1d931b2eb822bd4346631eeada89f9"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "unicode-ident",
+]
+
[[package]]
name = "sync_wrapper"
version = "0.1.2"
@@ -15172,6 +15240,7 @@ dependencies = [
"tokio",
"tokio-postgres",
"tokio-stream",
+ "tokio-util",
"tonic 0.14.2",
"tower 0.5.2",
"url",
@@ -15240,17 +15309,6 @@ dependencies = [
"cfg-if",
]
-[[package]]
-name = "thrift"
-version = "0.17.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7e54bc85fc7faa8bc175c4bab5b92ba8d9a3ce893d0e9f42cc455c8ab16a9e09"
-dependencies = [
- "byteorder",
- "integer-encoding 3.0.4",
- "ordered-float 2.10.1",
-]
-
[[package]]
name = "tikv-jemalloc-ctl"
version = "0.7.0"
@@ -15548,7 +15606,7 @@ version = "0.8.23"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dc1beb996b9d83529a9e75c17a1686767d148d70663143c7854d8b4a09ced362"
dependencies = [
- "indexmap 2.13.0",
+ "indexmap 2.14.2",
"serde",
"serde_spanned",
"toml_datetime",
@@ -15570,7 +15628,7 @@ version = "0.19.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1b5bb770da30e5cbfde35a2d7b9b8a2c4b8ef89548a7a6aeab5c9a576e3e7421"
dependencies = [
- "indexmap 2.13.0",
+ "indexmap 2.14.2",
"toml_datetime",
"winnow 0.5.40",
]
@@ -15581,7 +15639,7 @@ version = "0.22.27"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a"
dependencies = [
- "indexmap 2.13.0",
+ "indexmap 2.14.2",
"serde",
"serde_spanned",
"toml_datetime",
@@ -15767,7 +15825,7 @@ dependencies = [
"futures-core",
"futures-util",
"hdrhistogram",
- "indexmap 2.13.0",
+ "indexmap 2.14.2",
"pin-project-lite",
"slab",
"sync_wrapper 1.0.2",
@@ -16232,13 +16290,13 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
[[package]]
name = "uuid"
-version = "1.21.0"
+version = "1.26.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b672338555252d43fd2240c714dc444b8c6fb0a5c5335e65a07bba7742735ddb"
+checksum = "b5772d71c9be8a8a6ac2117d949c5b224c1b72241bb611d9a3012edcf8af7812"
dependencies = [
"getrandom 0.4.1",
"js-sys",
- "rand 0.9.4",
+ "rand 0.10.1",
"serde_core",
"wasm-bindgen",
]
@@ -16377,7 +16435,7 @@ dependencies = [
"hostname 0.4.1",
"iana-time-zone",
"idna",
- "indexmap 2.13.0",
+ "indexmap 2.14.2",
"indoc",
"influxdb-line-protocol",
"ipcrypt-rs",
@@ -16604,7 +16662,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909"
dependencies = [
"anyhow",
- "indexmap 2.13.0",
+ "indexmap 2.14.2",
"wasm-encoder",
"wasmparser",
]
@@ -16643,7 +16701,7 @@ checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe"
dependencies = [
"bitflags 2.12.1",
"hashbrown 0.15.4",
- "indexmap 2.13.0",
+ "indexmap 2.14.2",
"semver",
]
@@ -17282,7 +17340,7 @@ checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21"
dependencies = [
"anyhow",
"heck 0.5.0",
- "indexmap 2.13.0",
+ "indexmap 2.14.2",
"prettyplease",
"syn 2.0.117",
"wasm-metadata",
@@ -17313,7 +17371,7 @@ checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2"
dependencies = [
"anyhow",
"bitflags 2.12.1",
- "indexmap 2.13.0",
+ "indexmap 2.14.2",
"log",
"serde",
"serde_derive",
@@ -17332,7 +17390,7 @@ checksum = "ecc8ac4bc1dc3381b7f59c34f00b67e18f910c2c0f50015669dde7def656a736"
dependencies = [
"anyhow",
"id-arena",
- "indexmap 2.13.0",
+ "indexmap 2.14.2",
"log",
"semver",
"serde",
diff --git a/Cargo.toml b/Cargo.toml
index 29fcdc74320..730901925a9 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -8,6 +8,7 @@ members = [
"src/client",
"src/cmd",
"src/common/base",
+ "src/common/batcher",
"src/common/catalog",
"src/common/config",
"src/common/datasource",
@@ -100,13 +101,13 @@ rust.unexpected_cfgs = { level = "warn", check-cfg = ['cfg(tokio_unstable)'] }
# See for more detaiils: https://github.com/rust-lang/cargo/issues/11329
ahash = { version = "0.8", features = ["compile-time-rng"] }
aquamarine = "0.6"
-arrow = { version = "58.3", features = ["prettyprint"] }
-arrow-array = { version = "58.3", default-features = false, features = ["chrono-tz"] }
-arrow-buffer = "58.3"
-arrow-cast = "58.3"
-arrow-flight = "58.3"
-arrow-ipc = { version = "58.3", default-features = false, features = ["lz4", "zstd"] }
-arrow-schema = { version = "58.3", features = ["serde"] }
+arrow = { version = "=59.2.0", features = ["prettyprint"] }
+arrow-array = { version = "=59.2.0", default-features = false, features = ["chrono-tz"] }
+arrow-buffer = "=59.2.0"
+arrow-cast = "=59.2.0"
+arrow-flight = "=59.2.0"
+arrow-ipc = { version = "=59.2.0", default-features = false, features = ["lz4", "zstd"] }
+arrow-schema = { version = "=59.2.0", features = ["serde"] }
async-stream = "0.3"
async-trait = "0.1"
# Remember to update axum-extra, axum-macros when updating axum
@@ -128,22 +129,22 @@ const_format = "0.2"
criterion = "0.7"
crossbeam-utils = "0.8"
dashmap = "6.1"
-datafusion = "=53.1.0"
-datafusion-common = "=53.1.0"
-datafusion-datasource = "=53.1.0"
-datafusion-expr = "=53.1.0"
-datafusion-expr-common = "=53.1.0"
-datafusion-functions = "=53.1.0"
-datafusion-functions-aggregate-common = "=53.1.0"
-datafusion-functions-window-common = "=53.1.0"
-datafusion-optimizer = "=53.1.0"
-datafusion-orc = { git = "https://github.com/datafusion-contrib/datafusion-orc.git", rev = "6c07fa282dc8d62db2aa4ded06ab55485efc811a" }
-datafusion-pg-catalog = "0.17.3"
-datafusion-physical-expr = "=53.1.0"
-datafusion-physical-plan = "=53.1.0"
-datafusion-proto = "=53.1.0"
-datafusion-sql = "=53.1.0"
-datafusion-substrait = "=53.1.0"
+datafusion = "=55.0.0"
+datafusion-common = "=55.0.0"
+datafusion-datasource = "=55.0.0"
+datafusion-expr = "=55.0.0"
+datafusion-expr-common = "=55.0.0"
+datafusion-functions = "=55.0.0"
+datafusion-functions-aggregate-common = "=55.0.0"
+datafusion-functions-window-common = "=55.0.0"
+datafusion-optimizer = "=55.0.0"
+datafusion-orc = "0.10.0"
+datafusion-pg-catalog = "0.18.3"
+datafusion-physical-expr = "=55.0.0"
+datafusion-physical-plan = "=55.0.0"
+datafusion-proto = "=55.0.0"
+datafusion-sql = "=55.0.0"
+datafusion-substrait = "=55.0.0"
datafusion_object_store = { package = "object_store", version = "0.13.2" }
deadpool = "0.12"
deadpool-postgres = "0.14"
@@ -159,7 +160,7 @@ fs2 = "0.4"
fst = "0.4.7"
futures = "0.3"
futures-util = "0.3"
-greptime-proto = { git = "https://github.com/GreptimeTeam/greptime-proto.git", rev = "b74667d17827481dbb6222fa0326163627211557" }
+greptime-proto = { git = "https://github.com/GreptimeTeam/greptime-proto.git", rev = "0c4beffaa2b3968034915867593a717c6ab0c6b4" }
hex = "0.4"
hostname = "0.4.0"
http = "1"
@@ -195,10 +196,10 @@ otel-arrow-rust = { git = "https://github.com/GreptimeTeam/otel-arrow", rev = "5
"server",
] }
parking_lot = "0.12"
-parquet = { version = "58.3", default-features = false, features = ["arrow", "async", "object_store"] }
-parquet-variant = "58.3"
-parquet-variant-compute = "58.3"
-parquet-variant-json = "58.3"
+parquet = { version = "=59.2.0", default-features = false, features = ["arrow", "async", "object_store"] }
+parquet-variant = "=59.2.0"
+parquet-variant-compute = "=59.2.0"
+parquet-variant-json = "=59.2.0"
paste = "1.0"
pin-project = "1.0"
pretty_assertions = "1.4.0"
@@ -239,7 +240,7 @@ simd-json = "0.15"
similar-asserts = "1.6.0"
smallvec = { version = "1", features = ["serde"] }
snafu = "0.8"
-sqlparser = { version = "0.61.0", default-features = false, features = ["std", "visitor", "serde"] }
+sqlparser = { version = "0.62.0", default-features = false, features = ["std", "visitor", "serde"] }
sqlx = { version = "0.8", default-features = false, features = [
"any",
"macros",
@@ -287,6 +288,7 @@ cli = { path = "src/cli" }
client = { path = "src/client" }
cmd = { path = "src/cmd", default-features = false }
common-base = { path = "src/common/base" }
+common-batcher = { path = "src/common/batcher" }
common-catalog = { path = "src/common/catalog" }
common-config = { path = "src/common/config" }
common-datasource = { path = "src/common/datasource" }
@@ -353,22 +355,25 @@ git = "https://github.com/GreptimeTeam/greptime-meter.git"
rev = "5618e779cf2bb4755b499c630fba4c35e91898cb"
[patch.crates-io]
-datafusion = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "bb531e754b3b36d7a4d3222f35ebc88204dafd2b" }
-datafusion-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "bb531e754b3b36d7a4d3222f35ebc88204dafd2b" }
-datafusion-datasource = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "bb531e754b3b36d7a4d3222f35ebc88204dafd2b" }
-datafusion-expr = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "bb531e754b3b36d7a4d3222f35ebc88204dafd2b" }
-datafusion-expr-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "bb531e754b3b36d7a4d3222f35ebc88204dafd2b" }
-datafusion-functions = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "bb531e754b3b36d7a4d3222f35ebc88204dafd2b" }
-datafusion-functions-aggregate-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "bb531e754b3b36d7a4d3222f35ebc88204dafd2b" }
-datafusion-functions-window-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "bb531e754b3b36d7a4d3222f35ebc88204dafd2b" }
-datafusion-optimizer = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "bb531e754b3b36d7a4d3222f35ebc88204dafd2b" }
-datafusion-physical-expr = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "bb531e754b3b36d7a4d3222f35ebc88204dafd2b" }
-datafusion-physical-expr-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "bb531e754b3b36d7a4d3222f35ebc88204dafd2b" }
-datafusion-physical-plan = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "bb531e754b3b36d7a4d3222f35ebc88204dafd2b" }
-datafusion-proto = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "bb531e754b3b36d7a4d3222f35ebc88204dafd2b" }
-datafusion-sql = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "bb531e754b3b36d7a4d3222f35ebc88204dafd2b" }
-datafusion-substrait = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "bb531e754b3b36d7a4d3222f35ebc88204dafd2b" }
-sqlparser = { git = "https://github.com/GreptimeTeam/sqlparser-rs.git", rev = "2aefa08a8d69c96eec2d6d6703598a009bba6e4c" } # on branch v0.61.x
+substrait = { git = "https://github.com/GreptimeTeam/substrait-rs.git", rev = "91ec978b0649417ad3da8390e7a515baec723b1b" }
+datafusion = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "b66dca3260c8e04f314c8036cdfbb08f1a564d95" }
+datafusion-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "b66dca3260c8e04f314c8036cdfbb08f1a564d95" }
+datafusion-datasource = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "b66dca3260c8e04f314c8036cdfbb08f1a564d95" }
+datafusion-expr = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "b66dca3260c8e04f314c8036cdfbb08f1a564d95" }
+datafusion-expr-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "b66dca3260c8e04f314c8036cdfbb08f1a564d95" }
+datafusion-functions = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "b66dca3260c8e04f314c8036cdfbb08f1a564d95" }
+datafusion-functions-aggregate-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "b66dca3260c8e04f314c8036cdfbb08f1a564d95" }
+datafusion-functions-window-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "b66dca3260c8e04f314c8036cdfbb08f1a564d95" }
+datafusion-optimizer = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "b66dca3260c8e04f314c8036cdfbb08f1a564d95" }
+datafusion-physical-expr = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "b66dca3260c8e04f314c8036cdfbb08f1a564d95" }
+datafusion-physical-expr-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "b66dca3260c8e04f314c8036cdfbb08f1a564d95" }
+datafusion-physical-plan = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "b66dca3260c8e04f314c8036cdfbb08f1a564d95" }
+datafusion-proto = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "b66dca3260c8e04f314c8036cdfbb08f1a564d95" }
+datafusion-sql = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "b66dca3260c8e04f314c8036cdfbb08f1a564d95" }
+datafusion-substrait = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "b66dca3260c8e04f314c8036cdfbb08f1a564d95" }
+datafusion-pg-catalog = { git = "https://github.com/GreptimeTeam/datafusion-postgres.git", rev = "3c77e6c32b8db80635a0d2f4b318a36b31170bc3" }
+arrow-pg = { git = "https://github.com/GreptimeTeam/datafusion-postgres.git", rev = "3c77e6c32b8db80635a0d2f4b318a36b31170bc3" }
+sqlparser = { git = "https://github.com/GreptimeTeam/sqlparser-rs.git", rev = "9e9019bb1c7040ed956f654e39378dd42ab17884" }
# Temporary: use the GreptimeTeam fork of tikv-jemalloc-sys embedding
# jemalloc 5.3.1 + backport of 54f22c83 ("Initialize TSD tcache before
diff --git a/config/config.md b/config/config.md
index 2d85ddd7176..dde0b231066 100644
--- a/config/config.md
+++ b/config/config.md
@@ -165,6 +165,9 @@
| `region_engine.mito.worker_request_batch_size` | Integer | `64` | Max batch size for a worker to handle requests. |
| `region_engine.mito.manifest_checkpoint_distance` | Integer | `10` | Number of meta action updated to trigger a new checkpoint for the manifest. |
| `region_engine.mito.compress_manifest` | Bool | `false` | Whether to compress manifest and checkpoint file by gzip (default false). |
+| `region_engine.mito.experimental_enable_series_index` | Bool | `false` | Under development; do not enable. Whether to enable series indexes.
Indexes are stored on the local filesystem under `{data_home}/series_index`. |
+| `region_engine.mito.experimental_series_index_maintenance_interval` | String | `5m` | Interval between series-index maintenance runs. Zero uses the default of 5 min. |
+| `region_engine.mito.experimental_series_index_bucket_width` | String | `5days` | Requested minimum series-index bucket width (default: 5 days), rounded up to
an exact multiple of each region's compaction time window. |
| `region_engine.mito.max_background_flushes` | Integer | Auto | Max number of running background flush jobs (default: 1/2 of cpu cores). |
| `region_engine.mito.max_background_compactions` | Integer | Auto | Max number of running background compaction jobs (default: 1/4 of cpu cores). |
| `region_engine.mito.max_background_purges` | Integer | Auto | Max number of running background purge jobs (default: number of cpu cores). |
@@ -612,6 +615,9 @@
| `region_engine.mito.experimental_manifest_keep_removed_file_count` | Integer | `256` | Number of removed files to keep in manifest's `removed_files` field before also
remove them from `removed_files`. Mostly for debugging purpose.
If set to 0, it will only use `keep_removed_file_ttl` to decide when to remove files
from `removed_files` field. |
| `region_engine.mito.experimental_manifest_keep_removed_file_ttl` | String | `1h` | How long to keep removed files in the `removed_files` field of manifest
after they are removed from manifest.
files will only be removed from `removed_files` field
if both `keep_removed_file_count` and `keep_removed_file_ttl` is reached. |
| `region_engine.mito.compress_manifest` | Bool | `false` | Whether to compress manifest and checkpoint file by gzip (default false). |
+| `region_engine.mito.experimental_enable_series_index` | Bool | `false` | Under development; do not enable. Whether to enable series indexes.
Indexes are stored on the local filesystem under `{data_home}/series_index`. |
+| `region_engine.mito.experimental_series_index_maintenance_interval` | String | `5m` | Interval between series-index maintenance runs. Zero uses the default of 5 min. |
+| `region_engine.mito.experimental_series_index_bucket_width` | String | `5days` | Requested minimum series-index bucket width (default: 5 days), rounded up to
an exact multiple of each region's compaction time window. |
| `region_engine.mito.max_background_flushes` | Integer | Auto | Max number of running background flush jobs (default: 1/2 of cpu cores). |
| `region_engine.mito.max_background_compactions` | Integer | Auto | Max number of running background compaction jobs (default: 1/4 of cpu cores). |
| `region_engine.mito.max_background_purges` | Integer | Auto | Max number of running background purge jobs (default: number of cpu cores). |
diff --git a/config/datanode.example.toml b/config/datanode.example.toml
index d378d55cc2b..67b6eea7274 100644
--- a/config/datanode.example.toml
+++ b/config/datanode.example.toml
@@ -474,6 +474,17 @@ experimental_manifest_keep_removed_file_ttl = "1h"
## Whether to compress manifest and checkpoint file by gzip (default false).
compress_manifest = false
+## Under development; do not enable. Whether to enable series indexes.
+## Indexes are stored on the local filesystem under `{data_home}/series_index`.
+#+ experimental_enable_series_index = false
+
+## Interval between series-index maintenance runs. Zero uses the default of 5 min.
+#+ experimental_series_index_maintenance_interval = "5m"
+
+## Requested minimum series-index bucket width (default: 5 days), rounded up to
+## an exact multiple of each region's compaction time window.
+#+ experimental_series_index_bucket_width = "5days"
+
## Max number of running background flush jobs (default: 1/2 of cpu cores).
## @toml2docs:none-default="Auto"
#+ max_background_flushes = 4
diff --git a/config/standalone.example.toml b/config/standalone.example.toml
index 0bef0793247..975e0239d08 100644
--- a/config/standalone.example.toml
+++ b/config/standalone.example.toml
@@ -635,6 +635,17 @@ manifest_checkpoint_distance = 10
## Whether to compress manifest and checkpoint file by gzip (default false).
compress_manifest = false
+## Under development; do not enable. Whether to enable series indexes.
+## Indexes are stored on the local filesystem under `{data_home}/series_index`.
+#+ experimental_enable_series_index = false
+
+## Interval between series-index maintenance runs. Zero uses the default of 5 min.
+#+ experimental_series_index_maintenance_interval = "5m"
+
+## Requested minimum series-index bucket width (default: 5 days), rounded up to
+## an exact multiple of each region's compaction time window.
+#+ experimental_series_index_bucket_width = "5days"
+
## Max number of running background flush jobs (default: 1/2 of cpu cores).
## @toml2docs:none-default="Auto"
#+ max_background_flushes = 4
diff --git a/docs/rfcs/2026-09-11-metric-export-import.md b/docs/rfcs/2026-09-11-metric-export-import.md
new file mode 100644
index 00000000000..1a984211e4d
--- /dev/null
+++ b/docs/rfcs/2026-09-11-metric-export-import.md
@@ -0,0 +1,280 @@
+---
+Feature Name: Metric Export and Import Optimization
+Tracking Issue: https://github.com/GreptimeTeam/greptimedb/issues/9120
+Date: 2026-09-11
+Status: Draft for discussion
+---
+
+# Summary
+
+Group the selected Metric logical tables by physical table. Query each physical
+table across all its regions, order the rows by `__table_id`, and project each
+logical schema into the existing logical-table Parquet files. Restore the same
+V2 snapshot with batched logical-table DDL and the existing data COPY path. A
+separate generic COPY fix removes repeated directory scans.
+
+The design preserves `manifest.version = 1` and the current V2 layout. It does
+not introduce physical backup files or require importing source table IDs.
+Merged data writes remain a first-release scope decision, pending representative
+measurements after the COPY fix.
+
+# Motivation and evidence
+
+[Discussion #7394](https://github.com/orgs/GreptimeTeam/discussions/7394)
+includes reports of exports taking days for roughly 100,000 logical tables with
+only several GiB of data. Table count, physical schema width and data volume must
+be measured separately; total database bytes do not describe the transfer cost.
+
+Local experiments support three independent optimizations: shared physical scans,
+batched logical CREATE, and removal of repeated file lookup. The
+[experiment results](2026-09-11-metric-export-import/experiment-results.md)
+contain the measurements, controls and coverage limits. The COPY improvement
+uses per-table writes; merged-write benefits remain unmeasured.
+
+# Terminology
+
+| Term | Meaning in this proposal |
+| --- | --- |
+| Physical table | A Metric Engine table whose data regions store rows for multiple logical tables; its schema includes their column union. |
+| Logical table | A user-facing Metric table with its own schema and an association with a physical table. |
+| Region | A storage partition of a physical table. A logical table's rows may span regions. |
+| Export unit | The selected logical tables sharing one physical table, within one schema and V2 time chunk. This is an export grouping, not a storage object or completion checkpoint. |
+| Query execution partition | A stream of work in the query plan. Frontend `target_partitions` influences the partition count; execution partitions are distinct from storage regions and export units. |
+| Logical-table descriptor | Captured catalog metadata: source ID, name, physical association, column names/types/order, time index and primary key. Capturing and reusing it provides no snapshot or locking guarantee. |
+| Parquet writer | The writer for one logical table's output file in an export unit. |
+| Merged writes | Combining insert requests for multiple target logical tables into a region batch dispatch while retaining each table's identity, schema and routing. |
+
+# Physical export and logical files
+
+## Query and routing
+
+Capture the selected logical-table descriptors and reuse them for schema DDL
+and data export. For each schema and V2 time chunk, group the selected logical
+tables by physical table to form export units. Source schemas, selected data and
+TTL effects must stay stable during export; this proposal does not provide a
+transactionally consistent online backup.
+
+The server must authorize every selected logical table and the destination
+before querying the physical table. Routing filters are not an authorization
+boundary.
+
+For example, suppose `cpu` has `(ts, host, cpu)` and `requests` has
+`(ts, host, service, requests)`, sharing `public.phy`. The physical query is
+equivalent to:
+
+```sql
+SELECT __table_id, ts, host, service, cpu, requests
+FROM public.phy
+WHERE ts >= '2026-09-01T00:00:00Z'
+ AND ts < '2026-09-02T00:00:00Z'
+ORDER BY __table_id;
+```
+
+The implementation builds this plan through the existing query engine, using
+COPY's time-filter semantics and properly resolved columns. It reads the
+physical table directly; it does not first write and reread `phy.parquet`.
+
+```text
+physical table query: selected column union + __table_id, ordered by ID
+ → reject rows whose IDs are absent from the captured descriptor set
+ → contiguous rows for cpu → project (ts, host, cpu) → cpu.parquet
+ → contiguous rows for requests → project (ts, host, service, requests) → requests.parquet
+```
+
+Only one Parquet writer is active per export unit. At an ID transition, close the
+previous writer and open the next. Verify non-null UInt32 IDs and monotonic order
+across input batches. Create a valid zero-row file for every selected empty table.
+Never put `__table_id`, `__tsid`, or another table's columns into logical files.
+
+The captured catalog IDs are the routing whitelist; physical rows are not the
+source of table membership. Dropped tables can leave residual rows. Predicate
+pushdown for selected IDs/ranges is an optional optimization to evaluate against
+residual density and index effectiveness; it does not replace routing checks.
+
+## Physical column union and memory
+
+The physical schema can contain the union of thousands of logical schemas.
+Project the scan to `__table_id` plus the union needed by the selected tables.
+For each ID run, project again to that logical schema **before** expanding Arrow
+dictionary columns. Preserve field order, types, NULLs and empty strings.
+
+Logical projection reduces output conversion but does not eliminate wide scan
+batches or upstream sorting. Account separately for retained scan and merge
+buffers, conversion, codec reservations, row-group metadata and object-store
+buffers. Bound concurrent export units as well as each unit's work.
+
+Input checks after a batch arrives cannot cap the scan's allocation peak, and a
+writer flush threshold does not cap codec allocations. Combine schema validation
+with query memory accounting, bounded conversion slices, row-group limits and
+cancellation. Reject an oversized row explicitly. These controls do not provide
+a hard process RSS guarantee.
+
+## Ordering and multiple regions
+
+The first release covers every selected Metric physical table, any region count,
+and the Metric column types supported by existing Parquet COPY. Issue one query
+per export unit covering all regions of its physical table. The query engine
+provides globally ordered results through `ORDER BY __table_id`, including any
+sorting and merging needed across datanodes. The exporter consumes that stream
+without implementing its own region scheduler.
+
+In the current implementation, `MergeScanExec` declares the requested ordering
+only when `output_partition_count >= regions.len()`. When the region count
+exceeds frontend `target_partitions`, the frontend may need an additional sort
+to satisfy SQL ordering. That sort operates on the selected column union before
+per-logical-table projection. Parquet writer limits and downstream conversion
+slices cannot bound its memory or spill cost.
+
+SQL ordering is the contract. Keep the runtime ID-monotonicity check, but do not
+reject a valid plan because it contains `SortExec`. If query resources are
+exhausted, cancel the query, fail the chunk and retain retry semantics. Region
+count or resource exhaustion must not silently switch an enabled physical
+export to per-logical-table queries. The
+[ordering acceptance gate](#ordering-acceptance-gate) is required before release.
+
+# Snapshot and metadata contract
+
+Retain the current implementation's layout:
+
+```text
+manifest.json # version 1
+schema/schemas.json # schema index
+schema/ddl/public.sql # canonical database/table/view DDL
+data/public/1/cpu.parquet
+data/public/1/requests.parquet
+```
+
+Schema path segments retain existing V2 encoding. Logical data filenames retain
+COPY's raw table-name semantics and import's `file_stem` matching, including dots.
+There is no existing general table-filename escaping contract to reuse. Reject
+path separators for the fast path; changing that contract requires a separate
+compatibility design.
+
+Keep database, physical-table, logical/ordinary-table and view DDL in dependency
+order. Physical DDL describes the table and its options; logical DDL describes
+each logical schema and its physical association. Normal target CREATE procedures
+reconstruct Metric data/metadata regions and logical metadata with newly allocated
+IDs. Do not copy hidden metadata-region records or source IDs as restore data.
+Verify effective options, column mappings and physical routes after recreation.
+
+# Restore path
+
+## Batched logical DDL
+
+Parse the existing DDL and batch contiguous logical CREATE statements sharing
+catalog, schema and the literal `on_physical_table` value. A dot in that option
+does not make it a qualified identifier. Database, physical, ordinary and view
+statements are barriers and retain their order.
+
+Use the existing logical-table batch procedure. Process bounded batches
+sequentially and validate each batch before submission. Preserve normal CREATE
+validation, schema-option inheritance and target metadata resolution. Bound the
+request size and the importer's DDL buffering separately.
+
+The production CLI needs an authenticated server operation that checks CREATE
+authorization for every member and reuses the ordinary DDL path. Transport,
+capability discovery and request limits must be agreed before accepting this
+interface. An older server uses ordinary DDL; do not retry an ambiguously
+completed batch through another path.
+
+## Data COPY and generic directory fix
+
+Read each logical Parquet file through the current COPY path and write through
+normal target logical-table routing. No source-ID remapping belongs in these
+files, and batch DDL does not imply batched data insertion.
+
+The generic fix changes explicit-file lookup from `stat → list parent → find`
+to `stat → use path`, with a sandboxed file-type check for explicit local
+inputs to preserve symlink filtering. Directory COPY retains listing, pattern
+matching and file-type filtering. Backend authorization and error handling
+remain in place. With N files in one directory, this removes roughly N²
+directory-entry work.
+COPY bypasses the common Lister's filename branch; the branch itself is unchanged.
+This fix can ship independently of the Metric design.
+
+## Completion, interruption and retry
+
+Keep V2 completion units: export chunk completion and import `(chunk, schema)`
+data tasks, with the existing DDL-completed flag. Finish and close every required
+file before recording completion. A closed subset of files is not a completed
+chunk. On export retry, replace only the unfinished attempt's owned outputs;
+never touch completed chunks. Cancellation must close/abort writers and leave
+the chunk incomplete.
+
+Batch DDL is not a transaction across the restore. A later failure can leave
+earlier batches created. Keep DDL completion false until all statements succeed,
+then use the current durable state update. Data retries retain existing COPY
+semantics and may replay an incomplete task; there is no new exactly-once
+guarantee or per-file checkpoint in this proposal.
+
+# Rationale and alternatives
+
+| Choice | Reason and trade-off |
+| --- | --- |
+| Keep logical-table Parquet files | Reuses V2 DDL, file lookup and normal target CREATE/routing with fresh IDs. Physical backup files would need a new restore contract for mixed schemas and source metadata. The retained format still has one file per logical table per time chunk. |
+| Scan by physical table | Shares query and scan work among logical tables. Per-logical-table COPY is simpler but retains the table-count overhead; physical scans instead pay for the selected column union and ID routing. |
+| Global ID ordering and one Parquet writer per export unit | Contiguous IDs allow a file to be closed at each table transition. Unordered routing would require many open writers or intermediate buffering/files. Ordering bounds writer state, but query sorting and merging need separate resource controls. |
+| Use query-engine ordering across regions | Reuses distributed planning and SQL semantics. An exporter-owned region scheduler would add merge, cancellation and retry coordination. The chosen approach must handle both streaming merge and additional-sort plans. |
+| Batch DDL while retaining per-table data COPY | Reuses logical CREATE procedures and current data-task replay semantics. Merged writes may reduce dispatch/WAL overhead, but introduce partial-success and checkpoint questions and require a separate benefit comparison. |
+
+# Unresolved questions
+
+- Which authenticated batch-DDL operation and capability-discovery mechanism
+ should the CLI use? Agree request byte/table limits and DDL buffering bounds.
+- What are the query memory, spill-space, writer and export-unit concurrency
+ limits? Agree the test workloads and resource/performance thresholds before
+ running the ordering acceptance benchmark.
+- Which released import-v2 versions and object-store backends are compatibility
+ targets? Test those readers against the retained V2 format.
+- What name and defaults should the `experimental_` export option use?
+- Should merged writes ship in the first release? Compare a bounded merged-write
+ PoC against corrected per-table COPY with the same batch DDL and concurrency.
+ Bound decoded rows/bytes and in-flight requests, preserving normal logical
+ validation and target routing through the existing region batch dispatch.
+ Cover small/larger files, wide schemas and distributed/multi-region targets;
+ report repeated total/data times, rows/s, bytes/s, request counts and memory.
+ Agree a minimum useful gain and memory ceiling before the comparison. Validate
+ partial success, ambiguous completion and checkpoint advancement, then record
+ an explicit include/defer decision.
+
+# Integration and release acceptance
+
+Use existing COPY DATABASE orchestration to form Metric export units. Retain
+ordinary COPY for non-Metric tables and other formats, and direct COPY TABLE
+behavior. Gate the unfinished Metric fast path behind an `experimental_`
+configuration option; disabling it selects the existing path.
+
+The [tracking issue](https://github.com/GreptimeTeam/greptimedb/issues/9120) owns
+the PR breakdown and implementation dependencies. Each implementation slice
+must verify its own correctness and failure behavior. Release acceptance requires:
+
+| Area | Required evidence |
+| --- | --- |
+| Export | Actual CLI export with authorization, multiple physical tables and regions, dense/sparse keys, empty/residual tables, supported Metric Parquet types, and logical schema/value equivalence. |
+| Restore | Authenticated remote batch DDL, per-table authorization, fresh target IDs and reconstructed metadata, dependency ordering, older-server behavior and batch failure/retry. |
+| V2 lifecycle | Supported released readers; local and object-store interruption, owned-output cleanup, durable completion state and import replay. |
+| Scale | Repeated release-build measurements varying table count up to approximately 100,000, rows/file, physical union width, regions and backend. Report median/spread, control execution order/cache, and include schema export and metadata discovery in end-to-end timing. |
+| Open decisions | Agreed batch-DDL interface, resource limits, compatibility targets and experimental option; documented merged-write include/defer decision. |
+
+## Ordering acceptance gate
+
+The first release must pass a distributed export benchmark with region counts
+**below, equal to and above frontend `target_partitions`**, using representative
+row counts and narrow/wide selected column unions. Record actual execution
+partition counts rather than assuming the configured target determines the plan.
+Include multiple physical tables and a logical table whose rows span regions.
+Exercise both streaming-merge and additional-sort plans.
+
+For each case, report frontend and datanode execution plans, peak memory,
+query memory-accounting metrics, spill volume and elapsed time. Run with the
+agreed query memory, spill-space, writer and concurrency limits. Verify global
+ID order and exactly one complete file per selected logical table in each export
+unit, checking schemas and typed rows outside the timed interval.
+
+Passing requires correct exports within the agreed resource/performance
+thresholds on both plan paths. Also force resource exhaustion: cancellation
+must leave the chunk incomplete, and retry with sufficient resources must
+produce complete files without altering completed chunks. Writer-only memory
+measurements and single-region runs cannot satisfy this gate. Until both the
+normal and failure/retry cases pass, multi-region resource acceptance remains
+open.
diff --git a/docs/rfcs/2026-09-11-metric-export-import/experiment-results.md b/docs/rfcs/2026-09-11-metric-export-import/experiment-results.md
new file mode 100644
index 00000000000..56d75530c7d
--- /dev/null
+++ b/docs/rfcs/2026-09-11-metric-export-import/experiment-results.md
@@ -0,0 +1,109 @@
+# Metric export/import experiment results
+
+These experiments support three changes: physical-table export, batched logical
+DDL, and removal of repeated directory listing during COPY. They measure separate
+parts of export/import; their speedups should not be multiplied.
+
+Measurements were taken on September 10–11, 2026, on an ARM64 machine with 16 GiB
+RAM, using local standalone instances, local files, warm caches and unoptimized
+test builds. Each table entry represents one run. The results establish useful
+optimization directions, not production throughput estimates.
+
+## Physical-table export
+
+The comparison uses 64 populated logical tables with 1,000 source rows each,
+plus an empty table. A time filter selects 63,744 rows, written into 65 logical
+Parquet files. Ordinary COPY DATABASE uses parallelism one. Timings cover data
+export and exclude schema export and fixture preparation. Both execution orders
+were measured at narrow and wide physical schemas.
+
+| Physical tag columns | Execution order | Ordinary COPY (ms) | Physical-table export (ms) | Speedup |
+| ---: | --- | ---: | ---: | ---: |
+| 4 | Ordinary COPY first | 889.4 | 262.7 | 3.39x |
+| 4 | Physical export first | 893.7 | 309.5 | 2.89x |
+| 67 | Ordinary COPY first | 1062.0 | 316.0 | 3.36x |
+| 67 | Physical export first | 869.0 | 308.3 | 2.82x |
+
+Physical-table export was **2.82–3.39x faster** in these runs. Each populated
+logical file still contains only its own four columns. Projecting the logical
+schema before dictionary expansion kept the expanded-array peak at 0.148 MiB
+and the Parquet row-group estimate at 1.112 MiB for both widths. Incoming array
+estimates grew from 0.267 MiB to 2.291–2.408 MiB: logical projection does not remove
+the memory cost of a wide physical schema. These estimates are not process RSS
+limits, and a writer flush threshold does not cap codec allocations.
+
+The measured single-region query used `SeriesScan` and `SortPreservingMergeExec`
+without a separate `SortExec`; `__table_id` matches a physical primary-key prefix.
+Only Float64 fields, string tags and timestamps were exercised. These runs do not
+establish sorting memory, spill or performance when region count exceeds
+frontend `target_partitions`; see the RFC's
+[ordering acceptance gate](../2026-09-11-metric-export-import.md#ordering-acceptance-gate).
+The timestamp encoder reserved about 1 MiB even for a small batch, illustrating
+why a writer flush threshold is not an allocation ceiling.
+
+## Batched logical DDL
+
+The restore fixture has logical tables sharing one physical table, one file per
+logical table, two rows per populated table and one empty table. The 10,000-table
+case contains 19,998 rows. The three paths use SQL HTTP per table, an internal call per table, and
+internal batches of up to 128 tables and 1 MiB of rendered SQL, submitted
+sequentially. These request bounds do not bound the whole parsed DDL list in
+memory. The internal single-call control separates transport savings from
+procedure batching.
+
+| Logical tables | SQL HTTP CREATE (s) | Internal single CREATE (s) | Batched CREATE (s) | Logical calls: single → batch |
+| ---: | ---: | ---: | ---: | ---: |
+| 100 | 0.318 | 0.177 | 0.056 | 100 → 1 |
+| 1,000 | 3.091 | 1.601 | 0.544 | 1,000 → 8 |
+| 10,000 | 29.181 | 19.524 | 5.776 | 10,000 → 79 |
+
+These are logical CREATE call durations. At 10,000 tables, the complete DDL phase
+fell from 30.752 s with SQL HTTP to 7.019 s with batching. Total restore fell from
+187.279 s to 157.492 s; unchanged data COPY still took about 150 s. Data-phase
+variation is not a DDL benefit. Batching improves CREATE itself, but the original
+COPY bottleneck limits its effect on total restore.
+
+The batch executor was in-process. A production authenticated server interface
+is still needed; these times do not predict batch HTTP latency.
+
+## COPY file lookup
+
+Both variants below use batch DDL and the same COPY concurrency. Previously,
+COPY listed a file's parent directory even when it already knew the filename.
+With N files in one directory, repeating this for each file causes roughly N²
+directory-entry work. The fix checks the known path directly; directory COPY
+still lists files normally.
+
+| Logical tables / files | COPY before (s) | COPY after (s) | Total restore before (s) | Total restore after (s) |
+| ---: | ---: | ---: | ---: | ---: |
+| 100 | 0.094 | 0.071 | 0.183 | 0.164 |
+| 1,000 | 2.135 | 0.612 | 2.849 | 1.289 |
+| 10,000 | 146.496 | 5.481 | 153.647 | 12.300 |
+
+At 10,000 files, COPY improved **26.73x** and total restore improved **12.49x**.
+Before the fix, backend/file lookup accounted for about 92% of cumulative active
+file-COPY time. A control with profiling disabled completed COPY in 5.449 s.
+
+The **5.481 s result still uses per-table writes**, with only 19,998 rows. It is
+not a merged-write result. Larger files, distributed targets and object stores
+are needed to decide whether merged writes provide enough additional benefit.
+
+## Correctness and remaining coverage
+
+The experiments compared exported/restored schemas and typed rows with the
+source. Checks covered dense/sparse keys, different logical schemas, NULL and
+empty values, empty tables, dropped-table residue and fresh target IDs. The
+unmodified import-v2 command used in the experiment restored the exported files
+into a fresh target with different IDs. This does not establish compatibility
+with all released readers. Batch-DDL checks covered two schemas, ordinary
+tables/views, failure after a successful 128-table batch and retry of the original
+DDL. Existing standalone COPY cases also passed.
+
+The internal exporter assumed a trusted caller. It could leave a partial output
+directory after failure and did not implement completion publication or resume.
+These limitations require production integration work.
+
+Production acceptance still requires release-build measurements with repeated
+runs, larger workloads, distributed/multi-region execution, object stores,
+released-reader compatibility and export cancellation/recovery. These are tracked
+in the [tracking issue](https://github.com/GreptimeTeam/greptimedb/issues/9120).
diff --git a/src/api/src/helper.rs b/src/api/src/helper.rs
index 0a8c999461a..d8e3004aa2b 100644
--- a/src/api/src/helper.rs
+++ b/src/api/src/helper.rs
@@ -449,7 +449,14 @@ impl TryFrom for ColumnDataTypeWrapper {
}),
JsonFormat::Json2(native_type) => {
if native_type.is_null() {
- None
+ Some(ColumnDataTypeExtension {
+ type_ext: Some(TypeExt::JsonNativeType(Box::new(
+ JsonNativeTypeExtension {
+ datatype: ColumnDataType::Json as i32,
+ datatype_extension: None,
+ },
+ ))),
+ })
} else {
let concrete_type =
ConcreteDataType::from_arrow_type(&native_type.as_arrow_type());
@@ -1291,6 +1298,32 @@ mod tests {
assert!(values.bool_values.is_empty());
}
+ #[test]
+ fn test_json2_unknown_type_encoding() {
+ let datatype = ConcreteDataType::json2(JsonNativeType::Null);
+ let wrapper = ColumnDataTypeWrapper::try_from(datatype.clone()).unwrap();
+ assert_eq!(
+ wrapper.to_parts(),
+ (
+ ColumnDataType::Json,
+ Some(ColumnDataTypeExtension {
+ type_ext: Some(TypeExt::JsonNativeType(Box::new(JsonNativeTypeExtension {
+ datatype: ColumnDataType::Json as i32,
+ datatype_extension: None,
+ }))),
+ }),
+ )
+ );
+ assert_eq!(ConcreteDataType::from(wrapper), datatype);
+
+ for extension in [None, Some(ColumnDataTypeExtension::default())] {
+ assert_eq!(
+ ConcreteDataType::from(ColumnDataTypeWrapper::new(ColumnDataType::Json, extension)),
+ datatype
+ );
+ }
+ }
+
#[test]
fn test_concrete_datatype_from_column_datatype() {
assert_eq!(
diff --git a/src/catalog/src/information_extension.rs b/src/catalog/src/information_extension.rs
index c41c816c6d7..9d55f9f7a1e 100644
--- a/src/catalog/src/information_extension.rs
+++ b/src/catalog/src/information_extension.rs
@@ -27,8 +27,10 @@ use common_query::request::QueryRequest;
use common_recordbatch::adapter::{AsyncRecordBatchStreamAdapter, DfRecordBatchStreamAdapter};
use common_recordbatch::util::{ChainedRecordBatchStream, LimitedRecordBatchStream};
use common_recordbatch::{DfSendableRecordBatchStream, SendableRecordBatchStream};
+use datafusion::common::Result;
+use datafusion::common::tree_node::TreeNodeRecursion;
use datafusion::execution::TaskContext;
-use datafusion::physical_expr::{EquivalenceProperties, Partitioning};
+use datafusion::physical_expr::{EquivalenceProperties, Partitioning, PhysicalExpr};
use datafusion::physical_plan::execution_plan::{Boundedness, EmissionType};
use datafusion::physical_plan::{DisplayAs, DisplayFormatType, ExecutionPlan, PlanProperties};
use datatypes::arrow::datatypes::SchemaRef as ArrowSchemaRef;
@@ -157,10 +159,6 @@ impl ExecutionPlan for DistributedInspectExec {
"DistributedInspectExec"
}
- fn as_any(&self) -> &dyn std::any::Any {
- self
- }
-
fn schema(&self) -> ArrowSchemaRef {
self.arrow_schema.clone()
}
@@ -173,6 +171,13 @@ impl ExecutionPlan for DistributedInspectExec {
vec![]
}
+ fn apply_expressions(
+ &self,
+ _f: &mut dyn FnMut(&Arc) -> Result,
+ ) -> Result {
+ Ok(TreeNodeRecursion::Continue)
+ }
+
fn with_new_children(
self: Arc,
children: Vec>,
diff --git a/src/catalog/src/system_schema/information_schema.rs b/src/catalog/src/system_schema/information_schema.rs
index d0539a062c5..8c3f4bf0f5f 100644
--- a/src/catalog/src/system_schema/information_schema.rs
+++ b/src/catalog/src/system_schema/information_schema.rs
@@ -135,6 +135,9 @@ lazy_static! {
GLOBAL_STATUS,
SESSION_STATUS,
PARTITIONS,
+ PLUGINS,
+ USER_PRIVILEGES,
+ PROCESSLIST,
];
}
@@ -237,6 +240,9 @@ impl SystemSchemaProviderInner for InformationSchemaProvider {
TABLE_PRIVILEGES => setup_memory_table!(TABLE_PRIVILEGES),
GLOBAL_STATUS => setup_memory_table!(GLOBAL_STATUS),
SESSION_STATUS => setup_memory_table!(SESSION_STATUS),
+ PLUGINS => setup_memory_table!(PLUGINS),
+ USER_PRIVILEGES => setup_memory_table!(USER_PRIVILEGES),
+ PROCESSLIST => setup_memory_table!(PROCESSLIST),
KEY_COLUMN_USAGE => Some(Arc::new(InformationSchemaKeyColumnUsage::new(
self.catalog_name.clone(),
self.catalog_manager.clone(),
diff --git a/src/catalog/src/system_schema/information_schema/columns.rs b/src/catalog/src/system_schema/information_schema/columns.rs
index 4e68b093cd4..e92efd3bebf 100644
--- a/src/catalog/src/system_schema/information_schema/columns.rs
+++ b/src/catalog/src/system_schema/information_schema/columns.rs
@@ -31,7 +31,7 @@ use datatypes::scalars::ScalarVectorBuilder;
use datatypes::schema::{ColumnSchema, Schema, SchemaRef};
use datatypes::value::Value;
use datatypes::vectors::{
- ConstantVector, Int64Vector, Int64VectorBuilder, StringVector, StringVectorBuilder, VectorRef,
+ Int64Vector, Int64VectorBuilder, StringVector, StringVectorBuilder, VectorRef,
};
use futures::TryStreamExt;
use snafu::{OptionExt, ResultExt};
@@ -408,18 +408,9 @@ impl InformationSchemaColumnsBuilder {
fn finish(&mut self) -> Result {
let rows_num = self.collation_names.len();
- let privileges = Arc::new(ConstantVector::new(
- Arc::new(StringVector::from(vec![DEFAULT_PRIVILEGES])),
- rows_num,
- ));
- let empty_string = Arc::new(ConstantVector::new(
- Arc::new(StringVector::from(vec![EMPTY_STR])),
- rows_num,
- ));
- let srs_ids = Arc::new(ConstantVector::new(
- Arc::new(Int64Vector::from(vec![None])),
- rows_num,
- ));
+ let privileges = Arc::new(StringVector::from(vec![DEFAULT_PRIVILEGES; rows_num]));
+ let empty_string = Arc::new(StringVector::from(vec![EMPTY_STR; rows_num]));
+ let srs_ids = Arc::new(Int64Vector::from(vec![None; rows_num]));
let columns: Vec = vec![
Arc::new(self.catalog_names.finish()),
diff --git a/src/catalog/src/system_schema/information_schema/information_memory_table.rs b/src/catalog/src/system_schema/information_schema/information_memory_table.rs
index 56a84a0da1b..847ad06dc34 100644
--- a/src/catalog/src/system_schema/information_schema/information_memory_table.rs
+++ b/src/catalog/src/system_schema/information_schema/information_memory_table.rs
@@ -383,6 +383,47 @@ pub(super) fn get_schema_columns(table_name: &str) -> (SchemaRef, Vec
vec![],
),
+ // GreptimeDB has no pluggable components.
+ PLUGINS => (
+ string_columns(&[
+ "PLUGIN_NAME",
+ "PLUGIN_VERSION",
+ "PLUGIN_STATUS",
+ "PLUGIN_TYPE",
+ "PLUGIN_TYPE_VERSION",
+ "PLUGIN_LIBRARY",
+ "PLUGIN_LIBRARY_VERSION",
+ "PLUGIN_AUTHOR",
+ "PLUGIN_DESCRIPTION",
+ "PLUGIN_LICENSE",
+ "LOAD_OPTION",
+ ]),
+ vec![],
+ ),
+
+ // Privileges are not exposed through `information_schema`, same as the other
+ // `*_privileges` tables above.
+ USER_PRIVILEGES => (
+ string_columns(&["GRANTEE", "TABLE_CATALOG", "PRIVILEGE_TYPE", "IS_GRANTABLE"]),
+ vec![],
+ ),
+
+ // Sessions are reported through `information_schema.process_list` and
+ // `SHOW PROCESSLIST`; this table carries the MySQL column shape only.
+ PROCESSLIST => (
+ vec![
+ bigint_column("ID"),
+ string_column("USER"),
+ string_column("HOST"),
+ string_column("DB"),
+ string_column("COMMAND"),
+ bigint_column("TIME"),
+ string_column("STATE"),
+ string_column("INFO"),
+ ],
+ vec![],
+ ),
+
_ => unreachable!("Unknown table in information_schema: {}", table_name),
};
diff --git a/src/catalog/src/system_schema/information_schema/key_column_usage.rs b/src/catalog/src/system_schema/information_schema/key_column_usage.rs
index 99b0d7f6b73..4f8f116afaf 100644
--- a/src/catalog/src/system_schema/information_schema/key_column_usage.rs
+++ b/src/catalog/src/system_schema/information_schema/key_column_usage.rs
@@ -26,7 +26,7 @@ use datafusion::physical_plan::streaming::PartitionStream as DfPartitionStream;
use datatypes::prelude::{ConcreteDataType, MutableVector, ScalarVectorBuilder, VectorRef};
use datatypes::schema::{ColumnSchema, Schema, SchemaRef};
use datatypes::value::Value;
-use datatypes::vectors::{ConstantVector, StringVector, StringVectorBuilder, UInt32VectorBuilder};
+use datatypes::vectors::{StringVector, StringVectorBuilder, UInt32VectorBuilder};
use futures_util::TryStreamExt;
use snafu::{OptionExt, ResultExt};
use store_api::storage::{ScanRequest, TableId};
@@ -326,10 +326,7 @@ impl InformationSchemaKeyColumnUsageBuilder {
fn finish(&mut self) -> Result {
let rows_num = self.table_catalog.len();
- let null_string_vector = Arc::new(ConstantVector::new(
- Arc::new(StringVector::from(vec![None as Option<&str>])),
- rows_num,
- ));
+ let null_string_vector = Arc::new(StringVector::from(vec![None as Option<&str>; rows_num]));
let columns: Vec = vec![
Arc::new(self.constraint_catalog.finish()),
Arc::new(self.constraint_schema.finish()),
diff --git a/src/catalog/src/system_schema/information_schema/partitions.rs b/src/catalog/src/system_schema/information_schema/partitions.rs
index b9396fe5543..d2cce93150a 100644
--- a/src/catalog/src/system_schema/information_schema/partitions.rs
+++ b/src/catalog/src/system_schema/information_schema/partitions.rs
@@ -29,8 +29,8 @@ use datatypes::schema::{ColumnSchema, Schema, SchemaRef};
use datatypes::timestamp::TimestampSecond;
use datatypes::value::Value;
use datatypes::vectors::{
- ConstantVector, Int64Vector, Int64VectorBuilder, MutableVector, StringVector,
- StringVectorBuilder, TimestampSecondVector, TimestampSecondVectorBuilder, UInt64VectorBuilder,
+ Int64Vector, Int64VectorBuilder, MutableVector, StringVector, StringVectorBuilder,
+ TimestampSecondVector, TimestampSecondVectorBuilder, UInt64VectorBuilder,
};
use futures::{StreamExt, TryStreamExt};
use partition::manager::PartitionInfo;
@@ -361,22 +361,11 @@ impl InformationSchemaPartitionsBuilder {
fn finish(&mut self) -> Result {
let rows_num = self.catalog_names.len();
- let null_string_vector = Arc::new(ConstantVector::new(
- Arc::new(StringVector::from(vec![None as Option<&str>])),
- rows_num,
- ));
- let null_i64_vector = Arc::new(ConstantVector::new(
- Arc::new(Int64Vector::from(vec![None])),
- rows_num,
- ));
- let null_timestamp_second_vector = Arc::new(ConstantVector::new(
- Arc::new(TimestampSecondVector::from(vec![None])),
- rows_num,
- ));
- let partition_methods = Arc::new(ConstantVector::new(
- Arc::new(StringVector::from(vec![Some("RANGE")])),
- rows_num,
- ));
+ let null_string_vector = Arc::new(StringVector::from(vec![None as Option<&str>; rows_num]));
+ let null_i64_vector = Arc::new(Int64Vector::from(vec![None; rows_num]));
+ let null_timestamp_second_vector =
+ Arc::new(TimestampSecondVector::from(vec![None; rows_num]));
+ let partition_methods = Arc::new(StringVector::from(vec![Some("RANGE"); rows_num]));
let columns: Vec = vec![
Arc::new(self.catalog_names.finish()),
diff --git a/src/catalog/src/system_schema/information_schema/ssts.rs b/src/catalog/src/system_schema/information_schema/ssts.rs
index da5fcd2eed1..e364bddfd3d 100644
--- a/src/catalog/src/system_schema/information_schema/ssts.rs
+++ b/src/catalog/src/system_schema/information_schema/ssts.rs
@@ -336,7 +336,7 @@ mod tests {
};
let plan = table.scan_to_plan(request).unwrap().unwrap();
- assert!(plan.as_any().is::());
+ assert!(plan.as_ref().is::());
assert_eq!(1, plan.schema().fields().len());
}
}
diff --git a/src/catalog/src/system_schema/information_schema/table_constraints.rs b/src/catalog/src/system_schema/information_schema/table_constraints.rs
index 8c975207f21..2208d303c2e 100644
--- a/src/catalog/src/system_schema/information_schema/table_constraints.rs
+++ b/src/catalog/src/system_schema/information_schema/table_constraints.rs
@@ -27,7 +27,7 @@ use datatypes::prelude::{ConcreteDataType, MutableVector};
use datatypes::scalars::ScalarVectorBuilder;
use datatypes::schema::{ColumnSchema, Schema, SchemaRef};
use datatypes::value::Value;
-use datatypes::vectors::{ConstantVector, StringVector, StringVectorBuilder, VectorRef};
+use datatypes::vectors::{StringVector, StringVectorBuilder, VectorRef};
use futures::TryStreamExt;
use snafu::{OptionExt, ResultExt};
use store_api::storage::{ScanRequest, TableId};
@@ -242,14 +242,8 @@ impl InformationSchemaTableConstraintsBuilder {
fn finish(&mut self) -> Result {
let rows_num = self.constraint_names.len();
- let constraint_catalogs = Arc::new(ConstantVector::new(
- Arc::new(StringVector::from(vec!["def"])),
- rows_num,
- ));
- let enforceds = Arc::new(ConstantVector::new(
- Arc::new(StringVector::from(vec!["YES"])),
- rows_num,
- ));
+ let constraint_catalogs = Arc::new(StringVector::from(vec!["def"; rows_num]));
+ let enforceds = Arc::new(StringVector::from(vec!["YES"; rows_num]));
let columns: Vec = vec![
constraint_catalogs,
diff --git a/src/catalog/src/system_schema/information_schema/table_names.rs b/src/catalog/src/system_schema/information_schema/table_names.rs
index f161fecb54b..3e9b2e2d6de 100644
--- a/src/catalog/src/system_schema/information_schema/table_names.rs
+++ b/src/catalog/src/system_schema/information_schema/table_names.rs
@@ -55,3 +55,8 @@ pub const SSTS_INDEX_META: &str = "ssts_index_meta";
pub const TABLE_SEMANTICS: &str = "table_semantics";
pub const STATISTICS: &str = "statistics";
pub const RECYCLE_BIN: &str = "recycle_bin";
+pub const PLUGINS: &str = "plugins";
+pub const USER_PRIVILEGES: &str = "user_privileges";
+/// MySQL's session list. GreptimeDB reports its own sessions through [`PROCESS_LIST`];
+/// this table only exists so MySQL tooling finds the name and column shape it expects.
+pub const PROCESSLIST: &str = "processlist";
diff --git a/src/catalog/src/table_source/dummy_catalog.rs b/src/catalog/src/table_source/dummy_catalog.rs
index 20637c3a3a8..455cc8c34fa 100644
--- a/src/catalog/src/table_source/dummy_catalog.rs
+++ b/src/catalog/src/table_source/dummy_catalog.rs
@@ -14,7 +14,6 @@
//! Dummy catalog for region server.
-use std::any::Any;
use std::fmt;
use std::sync::Arc;
@@ -64,10 +63,6 @@ impl fmt::Debug for DummyCatalogList {
}
impl CatalogProviderList for DummyCatalogList {
- fn as_any(&self) -> &dyn Any {
- self
- }
-
fn register_catalog(
&self,
_name: String,
@@ -98,10 +93,6 @@ struct DummyCatalogProvider {
}
impl CatalogProvider for DummyCatalogProvider {
- fn as_any(&self) -> &dyn Any {
- self
- }
-
fn schema_names(&self) -> Vec {
vec![]
}
@@ -135,10 +126,6 @@ struct DummySchemaProvider {
#[async_trait]
impl SchemaProvider for DummySchemaProvider {
- fn as_any(&self) -> &dyn Any {
- self
- }
-
fn table_names(&self) -> Vec {
vec![]
}
diff --git a/src/client/src/database.rs b/src/client/src/database.rs
index 881730cfaaa..e97fff6db00 100644
--- a/src/client/src/database.rs
+++ b/src/client/src/database.rs
@@ -825,7 +825,17 @@ impl Database {
/// Ingest a stream of [RecordBatch]es that belong to a table, using Arrow Flight's "`DoPut`"
/// method. The return value is also a stream, produces [DoPutResponse]s.
pub async fn do_put(&self, stream: FlightDataStream) -> Result {
+ self.do_put_with_hints(stream, &[]).await
+ }
+
+ /// Ingest a stream of [RecordBatch]es using Arrow Flight's `DoPut` with request hints.
+ pub async fn do_put_with_hints(
+ &self,
+ stream: FlightDataStream,
+ hints: &[(&str, &str)],
+ ) -> Result {
let mut request = tonic::Request::new(stream);
+ Self::put_hints(request.metadata_mut(), hints)?;
if let Some(AuthHeader {
auth_scheme: Some(AuthScheme::Basic(Basic { username, password })),
diff --git a/src/cmd/src/bin/query_perf_fixture/case.rs b/src/cmd/src/bin/query_perf_fixture/case.rs
index a76c49d43d0..0f5239a6b51 100644
--- a/src/cmd/src/bin/query_perf_fixture/case.rs
+++ b/src/cmd/src/bin/query_perf_fixture/case.rs
@@ -105,6 +105,10 @@ pub(super) struct PromRemoteWritePlan {
#[serde(default = "default_visibility_timeout_seconds")]
pub(super) visibility_timeout_seconds: u64,
#[serde(default)]
+ pub(super) base_setup_sql: Vec,
+ #[serde(default)]
+ pub(super) candidate_setup_sql: Vec,
+ #[serde(default)]
pub(super) prom_store: PromStoreConfig,
#[serde(default)]
pub(super) value: ValueConfig,
@@ -200,6 +204,7 @@ pub(super) enum ValuePattern {
QuantizedSignal,
SignalWithSporadicStalls,
MixedSignalRepeated,
+ BoundedMixed,
}
impl std::fmt::Display for ValuePattern {
@@ -436,6 +441,63 @@ pub(super) fn default_parallelism() -> u64 {
1
}
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn remote_write_setup_sql_defaults_to_empty() {
+ let case: CaseFile = toml::from_str(
+ r#"
+[scenario]
+kind = "prom_remote_write_then_query"
+
+[scenario.remote_write]
+metric = "metric"
+"#,
+ )
+ .unwrap();
+ let Scenario::PromRemoteWriteThenQuery(scenario) = case.scenario else {
+ panic!("expected prom_remote_write_then_query scenario");
+ };
+ assert!(scenario.remote_write.base_setup_sql.is_empty());
+ assert!(scenario.remote_write.candidate_setup_sql.is_empty());
+ }
+
+ #[test]
+ fn remote_write_setup_sql_roundtrips() {
+ let case: CaseFile = toml::from_str(
+ r#"
+[scenario]
+kind = "prom_remote_write_then_query"
+
+[scenario.remote_write]
+metric = "metric"
+base_setup_sql = ["CREATE TABLE base_table"]
+candidate_setup_sql = ["CREATE TABLE candidate_table", "ALTER TABLE candidate_table SET 'x'='y'"]
+"#,
+ )
+ .unwrap();
+
+ let roundtrip: CaseFile =
+ serde_json::from_str(&serde_json::to_string(&case).unwrap()).unwrap();
+ let Scenario::PromRemoteWriteThenQuery(scenario) = roundtrip.scenario else {
+ panic!("expected prom_remote_write_then_query scenario");
+ };
+ assert_eq!(
+ scenario.remote_write.base_setup_sql,
+ ["CREATE TABLE base_table"]
+ );
+ assert_eq!(
+ scenario.remote_write.candidate_setup_sql,
+ [
+ "CREATE TABLE candidate_table",
+ "ALTER TABLE candidate_table SET 'x'='y'"
+ ]
+ );
+ }
+}
+
impl Scenario {
pub(super) fn kind(&self) -> &'static str {
match self {
diff --git a/src/cmd/src/bin/query_perf_fixture/direct_sst.rs b/src/cmd/src/bin/query_perf_fixture/direct_sst.rs
index 746ecbd76e4..27a6b4109cd 100644
--- a/src/cmd/src/bin/query_perf_fixture/direct_sst.rs
+++ b/src/cmd/src/bin/query_perf_fixture/direct_sst.rs
@@ -469,6 +469,7 @@ pub(super) async fn run_direct_sst(args: DirectArgs) {
write_buffer_size: DEFAULT_WRITE_BUFFER_SIZE,
row_group_size: scenario.layout.row_group_size,
max_file_size: None,
+ float_field_encoding: Default::default(),
};
let infos = match format {
FormatType::Flat => writer.write_all_flat(source, Some(sequence), &opts).await,
diff --git a/src/cmd/src/bin/query_perf_fixture/inspect_footer.rs b/src/cmd/src/bin/query_perf_fixture/inspect_footer.rs
index d2a2e8006cb..19e972db3b3 100644
--- a/src/cmd/src/bin/query_perf_fixture/inspect_footer.rs
+++ b/src/cmd/src/bin/query_perf_fixture/inspect_footer.rs
@@ -14,18 +14,21 @@
use std::collections::BTreeSet;
use std::fs;
+use std::ops::Range;
use std::path::{Path, PathBuf};
use std::sync::Arc;
use clap::Args as ClapArgs;
use datafusion_object_store::path::Path as StorePath;
-use datafusion_object_store::{ObjectMeta, ObjectStore};
-use futures::{StreamExt, TryStreamExt};
+use datafusion_object_store::{ObjectMeta, ObjectStore, ObjectStoreExt};
+use futures::future::BoxFuture;
+use futures::{FutureExt, StreamExt, TryFutureExt, TryStreamExt};
use object_store::config::ObjectStoreConfig;
use object_store::factory::new_raw_object_store;
use object_store::services::Fs;
-use parquet::arrow::async_reader::ParquetObjectReader;
-use parquet::file::metadata::ParquetMetaDataReader;
+use parquet::arrow::async_reader::AsyncFileReader;
+use parquet::errors::{ParquetError, Result as ParquetResult};
+use parquet::file::metadata::{ParquetMetaData, ParquetMetaDataReader};
use serde::{Deserialize, Serialize};
/// Same shape as `query_regression_runner::model::DestinationConfig`. The two
@@ -100,6 +103,70 @@ struct ListedFile {
relative_path: String,
}
+/// An asynchronous Parquet reader backed directly by an object store.
+///
+/// The file size is retained from the listing so footer reads use bounded
+/// ranges without an additional stat/head request.
+#[derive(Clone, Debug)]
+struct ObjectStoreReader {
+ store: Arc,
+ path: StorePath,
+ file_size: u64,
+}
+
+impl ObjectStoreReader {
+ fn new(store: Arc, path: StorePath, file_size: u64) -> Self {
+ Self {
+ store,
+ path,
+ file_size,
+ }
+ }
+}
+
+fn to_parquet_error(error: datafusion_object_store::Error) -> ParquetError {
+ ParquetError::External(Box::new(error))
+}
+
+impl AsyncFileReader for ObjectStoreReader {
+ fn get_bytes(
+ &mut self,
+ range: Range,
+ ) -> BoxFuture<'_, ParquetResult> {
+ self.store
+ .get_range(&self.path, range)
+ .map_err(to_parquet_error)
+ .boxed()
+ }
+
+ fn get_byte_ranges(
+ &mut self,
+ ranges: Vec>,
+ ) -> BoxFuture<'_, ParquetResult>> {
+ async move {
+ self.store
+ .get_ranges(&self.path, &ranges)
+ .await
+ .map_err(to_parquet_error)
+ }
+ .boxed()
+ }
+
+ fn get_metadata<'a>(
+ &'a mut self,
+ _options: Option<&'a parquet::arrow::arrow_reader::ArrowReaderOptions>,
+ ) -> BoxFuture<'a, ParquetResult>> {
+ let file_size = self.file_size;
+ async move {
+ let metadata = ParquetMetaDataReader::new()
+ .load_and_finish(self, file_size)
+ .await?;
+ Ok(Arc::new(metadata))
+ }
+ .boxed()
+ }
+}
+
pub(super) async fn run_inspect_footer(
args: InspectFooterArgs,
) -> Result<(), Box> {
@@ -217,11 +284,8 @@ async fn inspect_file(
file: &ListedFile,
column: &str,
) -> Result> {
- let mut reader =
- ParquetObjectReader::new(store, file.location.clone()).with_file_size(file.size);
- let metadata = ParquetMetaDataReader::new()
- .load_and_finish(&mut reader, file.size)
- .await?;
+ let mut reader = ObjectStoreReader::new(store, file.location.clone(), file.size);
+ let metadata = reader.get_metadata(None).await?;
let file_metadata = metadata.file_metadata();
let row_groups = metadata.row_groups();
let mut columns = Vec::new();
diff --git a/src/cmd/src/bin/query_perf_fixture/prom_remote_write.rs b/src/cmd/src/bin/query_perf_fixture/prom_remote_write.rs
index 1f36b0c2d23..a59178f4a79 100644
--- a/src/cmd/src/bin/query_perf_fixture/prom_remote_write.rs
+++ b/src/cmd/src/bin/query_perf_fixture/prom_remote_write.rs
@@ -197,8 +197,35 @@ fn deterministic_prom_value(args: &PromRemoteWriteArgs, series_idx: u64, sample_
args.value_base + (series_idx % 97) as f64 + local as f64 * args.value_step
}
}
+ ValuePattern::BoundedMixed => bounded_mixed_value(args, series_idx, local),
}
}
+
+fn bounded_mixed_value(args: &PromRemoteWriteArgs, series_idx: u64, local: u64) -> f64 {
+ const SPANS: [i64; 4] = [10, 1_000, 100_000, 10_000_000];
+ let series = splitmix64(series_idx ^ args.value_seed);
+ let span = SPANS[(series >> 62) as usize] * (1 + ((series >> 32) % 10) as i64);
+ let baseline = (series % args.value_cardinality.max(1)) as f64 * span as f64 / 10.0;
+ let run_length = args.value_run_length.max(1);
+ let group = local / run_length;
+ let phase = local % run_length;
+ let anchor = bounded_mixed_anchor(series, group, span);
+ let next_anchor = bounded_mixed_anchor(series, group + 1, span);
+ let signal = baseline + anchor + (next_anchor - anchor) * phase as f64 / run_length as f64;
+ let value = (args.value_base + args.value_step * signal).round();
+ if series_idx % args.value_mixed_every.max(1) == args.value_mixed_every.max(1) - 1 {
+ let numerator = splitmix64(series ^ local) % 999 + 1;
+ value + numerator as f64 / 1_000.0
+ } else {
+ value
+ }
+}
+
+fn bounded_mixed_anchor(series: u64, group: u64, span: i64) -> f64 {
+ let limit = span - 1;
+ (splitmix64(series ^ group) % (2 * limit as u64 + 1)) as i64 as f64 - limit as f64
+}
+
fn splitmix64(mut value: u64) -> u64 {
value = value.wrapping_add(0x9e3779b97f4a7c15);
let mut mixed = value;
@@ -307,7 +334,7 @@ mod tests {
for chunk_idx in 0..10 {
let mut chunk = args(ValuePattern::SeededRandom);
chunk.samples_per_series = 1_440;
- chunk.value_sample_offset = chunk_idx * 1_440;
+ chunk.value_sample_offset = chunk_idx * 1_439;
chunk.value_total_samples_per_series = Some(14_400);
for sample_idx in 0..chunk.samples_per_series {
assert_eq!(
@@ -319,6 +346,118 @@ mod tests {
}
}
+ #[test]
+ fn bounded_mixed_has_bounded_95_5_per_series_values() {
+ let mut args = args(ValuePattern::BoundedMixed);
+ args.series_count = 1_000;
+ args.samples_per_series = 4_320;
+ args.value_cardinality = 1_000;
+ args.value_seed = 42;
+ args.value_run_length = 60;
+ args.value_mixed_every = 20;
+
+ let mut integral_series = 0;
+ let mut fractional_series = 0;
+ let mut spans = HashSet::new();
+ let mut ranges = HashSet::new();
+ for series_idx in 0..args.series_count {
+ let series = splitmix64(series_idx ^ args.value_seed);
+ let span = [10_i64, 1_000, 100_000, 10_000_000][(series >> 62) as usize]
+ * (1 + ((series >> 32) % 10) as i64);
+ let baseline = (series % args.value_cardinality.max(1)) as f64 * span as f64 / 10.0;
+ let values = (0..args.samples_per_series)
+ .map(|sample_idx| deterministic_prom_value(&args, series_idx, sample_idx))
+ .collect::>();
+
+ assert!(values.iter().all(|value| value.is_finite()));
+ assert!(
+ values
+ .iter()
+ .all(|value| *value >= baseline - span as f64 - 1.0
+ && *value <= baseline + span as f64 + 1.0)
+ );
+ assert!(
+ values.windows(2).any(|pair| pair[1] > pair[0]),
+ "series {series_idx} never rises"
+ );
+ assert!(
+ values.windows(2).any(|pair| pair[1] < pair[0]),
+ "series {series_idx} never falls"
+ );
+ ranges.insert((
+ values
+ .iter()
+ .copied()
+ .fold(f64::INFINITY, f64::min)
+ .to_bits(),
+ values
+ .iter()
+ .copied()
+ .fold(f64::NEG_INFINITY, f64::max)
+ .to_bits(),
+ ));
+ spans.insert(span);
+ if series_idx % args.value_mixed_every == args.value_mixed_every - 1 {
+ fractional_series += 1;
+ assert!(values.iter().all(|value| value.fract() != 0.0));
+ } else {
+ integral_series += 1;
+ assert!(values.iter().all(|value| value.fract() == 0.0));
+ }
+ }
+
+ assert_eq!((integral_series, fractional_series), (950, 50));
+ assert!(spans.len() > 4);
+ assert!(ranges.len() > 1);
+ }
+
+ #[test]
+ fn bounded_mixed_is_seeded_and_chunk_bit_identical() {
+ let mut monolithic = args(ValuePattern::BoundedMixed);
+ monolithic.samples_per_series = 14_400;
+ monolithic.value_total_samples_per_series = Some(14_400);
+ monolithic.value_cardinality = 1_000;
+ monolithic.value_seed = 42;
+ monolithic.value_run_length = 60;
+ monolithic.value_mixed_every = 20;
+
+ let original = (0..600)
+ .map(|sample_idx| deterministic_prom_value(&monolithic, 19, sample_idx))
+ .collect::>();
+ monolithic.value_seed = 43;
+ assert_ne!(
+ original,
+ (0..600)
+ .map(|sample_idx| deterministic_prom_value(&monolithic, 19, sample_idx))
+ .collect::>()
+ );
+
+ monolithic.value_seed = 42;
+ for series_idx in [0, 19] {
+ for chunk_idx in 0..10 {
+ let mut chunk = args(ValuePattern::BoundedMixed);
+ chunk.samples_per_series = 1_440;
+ chunk.value_sample_offset = chunk_idx * 1_439;
+ chunk.value_total_samples_per_series = Some(14_400);
+ chunk.value_cardinality = 1_000;
+ chunk.value_seed = 42;
+ chunk.value_run_length = 60;
+ chunk.value_mixed_every = 20;
+ for sample_idx in 0..chunk.samples_per_series {
+ assert_eq!(
+ deterministic_prom_value(&chunk, series_idx, sample_idx).to_bits(),
+ deterministic_prom_value(
+ &monolithic,
+ series_idx,
+ chunk.value_sample_offset + sample_idx
+ )
+ .to_bits()
+ );
+ }
+ }
+ }
+ }
+
#[test]
fn values_continue_across_series_boundaries() {
let mut args = args(ValuePattern::Unique);
diff --git a/src/cmd/src/bin/query_regression_runner/model.rs b/src/cmd/src/bin/query_regression_runner/model.rs
index 6831a5256a4..5fafdb5635b 100644
--- a/src/cmd/src/bin/query_regression_runner/model.rs
+++ b/src/cmd/src/bin/query_regression_runner/model.rs
@@ -116,6 +116,10 @@ pub(super) struct RemoteWrite {
pub(super) sample_chunk_size: Option,
pub(super) flush_every_sample_chunks: u64,
pub(super) visibility_timeout_seconds: u64,
+ #[serde(default)]
+ pub(super) base_setup_sql: Vec,
+ #[serde(default)]
+ pub(super) candidate_setup_sql: Vec,
pub(super) prom_store: PromStore,
pub(super) value: RemoteValue,
pub(super) storage: Option,
diff --git a/src/cmd/src/bin/query_regression_runner/remote/ingest.rs b/src/cmd/src/bin/query_regression_runner/remote/ingest.rs
index f563402afcd..271e27fac42 100644
--- a/src/cmd/src/bin/query_regression_runner/remote/ingest.rs
+++ b/src/cmd/src/bin/query_regression_runner/remote/ingest.rs
@@ -57,6 +57,7 @@ pub(super) async fn run_prepare_remote(args: PrepareRemoteArgs) -> Result<()> {
args.base_http_port,
&args.fixture_generator,
&remote,
+ &remote.base_setup_sql,
&client,
)
.await?;
@@ -65,6 +66,7 @@ pub(super) async fn run_prepare_remote(args: PrepareRemoteArgs) -> Result<()> {
args.candidate_http_port,
&args.fixture_generator,
&remote,
+ &remote.candidate_setup_sql,
&client,
)
.await?;
@@ -88,6 +90,7 @@ async fn prepare_remote_target(
port: u16,
generator: &Path,
remote: &RemoteWrite,
+ setup_sql: &[String],
client: &Client,
) -> Result {
let create_database = http_post_sql(
@@ -107,6 +110,7 @@ async fn prepare_remote_target(
)
.into());
}
+ let setup = run_setup_sql(client, port, &remote.database, name, setup_sql).await?;
let (remote_write, flushes) = ingest_remote_write(generator, port, remote, client).await?;
let expected_rows = remote
.series_count
@@ -124,6 +128,7 @@ async fn prepare_remote_target(
Ok(json!({
"name": name,
"create_database": create_database,
+ "setup_sql": setup,
"remote_write": remote_write,
"flushes": flushes,
"visibility": visibility,
@@ -131,6 +136,28 @@ async fn prepare_remote_target(
}))
}
+async fn run_setup_sql(
+ client: &Client,
+ port: u16,
+ database: &str,
+ target: &str,
+ setup_sql: &[String],
+) -> Result> {
+ let mut results = Vec::with_capacity(setup_sql.len());
+ for (index, statement) in setup_sql.iter().enumerate() {
+ let result = http_post_sql(client, port, statement, database).await;
+ if !result["ok"].as_bool().unwrap_or(false) {
+ return Err(format!(
+ "setup SQL statement {} failed for {target}: {result}",
+ index + 1
+ )
+ .into());
+ }
+ results.push(result);
+ }
+ Ok(results)
+}
+
#[derive(Clone, Debug, PartialEq, Eq)]
struct SampleChunk {
index: u64,
@@ -434,9 +461,110 @@ fn json_number(value: f64) -> String {
#[cfg(test)]
mod tests {
+ use tokio::io::{AsyncReadExt, AsyncWriteExt};
+ use tokio::net::{TcpListener, TcpStream};
+
use super::*;
use crate::query_regression_runner::model::{PromStore, RemoteValue};
+ async fn read_http_request(stream: &mut TcpStream) -> Vec {
+ let mut request = Vec::new();
+ let mut chunk = [0; 1024];
+ loop {
+ let read = stream.read(&mut chunk).await.unwrap();
+ assert_ne!(
+ read, 0,
+ "HTTP client closed request before sending its body"
+ );
+ request.extend_from_slice(&chunk[..read]);
+ let Some(header_end) = request.windows(4).position(|window| window == b"\r\n\r\n")
+ else {
+ continue;
+ };
+ let headers = std::str::from_utf8(&request[..header_end]).unwrap();
+ let content_length = headers
+ .lines()
+ .find_map(|line| line.strip_prefix("content-length: "))
+ .or_else(|| {
+ headers
+ .lines()
+ .find_map(|line| line.strip_prefix("Content-Length: "))
+ })
+ .unwrap()
+ .parse::()
+ .unwrap();
+ if request.len() >= header_end + 4 + content_length {
+ return request;
+ }
+ }
+ }
+
+ async fn spawn_sql_server(
+ responses: Vec,
+ ) -> (u16, tokio::task::JoinHandle>) {
+ let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
+ let port = listener.local_addr().unwrap().port();
+ let server = tokio::spawn(async move {
+ let mut requests = Vec::with_capacity(responses.len());
+ for body in responses {
+ let (mut stream, _) = listener.accept().await.unwrap();
+ let request = read_http_request(&mut stream).await;
+ requests.push(String::from_utf8(request).unwrap());
+ let response = format!(
+ "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\n\
+ Connection: close\r\nContent-Length: {}\r\n\r\n{body}",
+ body.len()
+ );
+ stream.write_all(response.as_bytes()).await.unwrap();
+ }
+ requests
+ });
+ (port, server)
+ }
+
+ #[tokio::test]
+ async fn setup_sql_runs_in_order_with_configured_database() {
+ let (port, server) = spawn_sql_server(vec!["{\"code\":0}".to_string(); 2]).await;
+ let client = Client::new();
+ let setup = vec![
+ "CREATE TABLE physical (ts TIMESTAMP TIME INDEX) ENGINE=metric".to_string(),
+ "ALTER TABLE physical SET 'compaction.twcs.trigger_file_num'='100'".to_string(),
+ ];
+
+ let results = run_setup_sql(&client, port, "perf", "candidate", &setup)
+ .await
+ .unwrap();
+
+ assert_eq!(results.len(), 2);
+ let requests = server.await.unwrap();
+ assert_eq!(requests.len(), 2);
+ assert!(requests[0].contains("db=perf"));
+ assert!(requests[0].contains("sql=CREATE"));
+ assert!(requests[1].contains("db=perf"));
+ assert!(requests[1].contains("sql=ALTER"));
+ }
+
+ #[tokio::test]
+ async fn setup_sql_stops_at_the_failing_statement() {
+ let (port, server) =
+ spawn_sql_server(vec!["{\"code\":0}".to_string(), "{\"code\":1}".to_string()]).await;
+ let client = Client::new();
+ let setup = vec![
+ "CREATE TABLE physical (ts TIMESTAMP TIME INDEX) ENGINE=metric".to_string(),
+ "invalid SQL".to_string(),
+ "ALTER TABLE physical SET 'compaction.twcs.trigger_file_num'='100'".to_string(),
+ ];
+
+ let error = run_setup_sql(&client, port, "perf", "base", &setup)
+ .await
+ .unwrap_err();
+
+ assert!(error.to_string().contains("statement 2 failed for base"));
+ let requests = server.await.unwrap();
+ assert_eq!(requests.len(), 2);
+ assert!(requests[1].contains("sql=invalid+SQL"));
+ }
+
#[test]
fn schedules_remote_sample_chunks_and_flushes() {
let remote = RemoteWrite {
@@ -452,6 +580,8 @@ mod tests {
sample_chunk_size: Some(2),
flush_every_sample_chunks: 2,
visibility_timeout_seconds: 30,
+ base_setup_sql: Vec::new(),
+ candidate_setup_sql: Vec::new(),
prom_store: PromStore {
pending_rows_flush_interval: "1s".to_string(),
max_batch_rows: 1,
diff --git a/src/cmd/src/bin/query_regression_runner/remote/read_bench.rs b/src/cmd/src/bin/query_regression_runner/remote/read_bench.rs
index 6b20a31c181..b9a05f6a69a 100644
--- a/src/cmd/src/bin/query_regression_runner/remote/read_bench.rs
+++ b/src/cmd/src/bin/query_regression_runner/remote/read_bench.rs
@@ -306,13 +306,25 @@ fn run_bench_command(command: Vec, mut run: Value) -> Result {
}
fn parse_average_duration(stdout: &str) -> Option {
- Regex::new(r"(?i)Average duration[^0-9]*([0-9.]+)\s*ms")
- .ok()?
- .captures(stdout)?
- .get(1)?
- .as_str()
- .parse()
- .ok()
+ let captures = Regex::new(
+ r"(?i)Average:\s+(?:\x1b\[[0-9;]*m)*\d+(?:\x1b\[[0-9;]*m)*\s+rows(?:,\s+(?:\x1b\[[0-9;]*m)*\d+(?:\x1b\[[0-9;]*m)*\s+record batches)?\s+in\s+([0-9]+(?:\.[0-9]+)?)\s*(ns|µs|us|ms|s)\s+over\s+\d+\s+iterations|Average duration[^0-9]*([0-9]+(?:\.[0-9]+)?)\s*ms",
+ )
+ .ok()?
+ .captures(stdout)?;
+ let (value, unit) = match (captures.get(1), captures.get(2), captures.get(3)) {
+ (Some(value), Some(unit), _) => (value.as_str(), unit.as_str()),
+ (_, _, Some(value)) => (value.as_str(), "ms"),
+ _ => return None,
+ };
+ let value = value.parse::().ok()?;
+ let milliseconds = match unit.to_ascii_lowercase().as_str() {
+ "ns" => value / 1_000_000.0,
+ "µs" | "us" => value / 1_000.0,
+ "ms" => value,
+ "s" => value * 1_000.0,
+ _ => return None,
+ };
+ milliseconds.is_finite().then_some(milliseconds)
}
fn bench_median(runs: &[Value]) -> Option {
@@ -328,12 +340,51 @@ mod tests {
use super::*;
#[test]
- fn parses_average_duration_and_groups_scan_paths() {
- assert_eq!(
- parse_average_duration("work\nAverage duration: 12.5 ms\n"),
- Some(12.5)
- );
- assert_eq!(parse_average_duration("completed"), None);
+ fn parses_average_durations() {
+ let cases = [
+ (
+ "parquetbench nanoseconds",
+ "ℹ Average: 42 rows, 2 record batches in 500ns over 4 iterations",
+ Some(0.0005),
+ ),
+ (
+ "scanbench colored microseconds",
+ "\x1b[1;32mSummary\x1b[0m Average: \x1b[36m12\x1b[0m rows in 250µs over 2 iterations",
+ Some(0.25),
+ ),
+ (
+ "scanbench seconds",
+ "Summary Average: 12 rows in 2.5s over 2 iterations",
+ Some(2500.0),
+ ),
+ (
+ "legacy milliseconds",
+ "Average duration: 12.5 ms",
+ Some(12.5),
+ ),
+ ("absent", "Benchmark completed!", None),
+ (
+ "malformed",
+ "Summary Average: 12 rows in unknown over 2 iterations",
+ None,
+ ),
+ ];
+
+ for (name, stdout, expected) in cases {
+ let actual = parse_average_duration(stdout);
+ match (actual, expected) {
+ (Some(actual), Some(expected)) => assert!(
+ (actual - expected).abs() < f64::EPSILON,
+ "{name}: expected {expected}, got {actual}"
+ ),
+ (None, None) => {}
+ (actual, expected) => panic!("{name}: expected {expected:?}, got {actual:?}"),
+ }
+ }
+ }
+
+ #[test]
+ fn groups_scan_paths() {
let targets = vec![
BenchTarget {
relative_path: "data/a.parquet".to_string(),
diff --git a/src/common/batcher/Cargo.toml b/src/common/batcher/Cargo.toml
new file mode 100644
index 00000000000..5d6ace4b2d3
--- /dev/null
+++ b/src/common/batcher/Cargo.toml
@@ -0,0 +1,19 @@
+[package]
+name = "common-batcher"
+version.workspace = true
+edition.workspace = true
+license.workspace = true
+
+[lib]
+path = "src/lib.rs"
+
+[lints]
+workspace = true
+
+[dependencies]
+dashmap.workspace = true
+futures.workspace = true
+tokio.workspace = true
+
+[dev-dependencies]
+tokio = { workspace = true, features = ["test-util"] }
diff --git a/src/common/batcher/src/flush_limiter.rs b/src/common/batcher/src/flush_limiter.rs
new file mode 100644
index 00000000000..98b6b2a2f01
--- /dev/null
+++ b/src/common/batcher/src/flush_limiter.rs
@@ -0,0 +1,73 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::sync::Arc;
+
+use tokio::sync::{AcquireError, OwnedSemaphorePermit, Semaphore};
+
+/// Shared execution budget. One permit represents one complete flush, not one RPC.
+#[derive(Clone)]
+pub struct FlushLimiter {
+ semaphore: Arc,
+}
+
+impl FlushLimiter {
+ /// Returns `None` if the limit is zero or exceeds Tokio's supported semaphore capacity.
+ pub fn try_new(max_concurrent_flushes: usize) -> Option {
+ let permits = max_concurrent_flushes;
+ ((1..=Semaphore::MAX_PERMITS).contains(&permits)).then(|| Self {
+ semaphore: Arc::new(Semaphore::new(permits)),
+ })
+ }
+
+ /// The caller must retain the returned permit until its flush finishes.
+ pub async fn acquire(&self) -> Result {
+ self.semaphore.clone().acquire_owned().await
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use std::future::Future;
+
+ use super::*;
+
+ #[test]
+ fn test_invalid_capacity() {
+ for capacity in [0, Semaphore::MAX_PERMITS + 1, usize::MAX] {
+ assert!(FlushLimiter::try_new(capacity).is_none());
+ }
+ for capacity in [1, Semaphore::MAX_PERMITS] {
+ assert!(FlushLimiter::try_new(capacity).is_some());
+ }
+ }
+
+ #[tokio::test]
+ async fn test_clones_share_flush_budget() {
+ let limiter = FlushLimiter::try_new(1).unwrap();
+ let other = limiter.clone();
+ let permit = limiter.acquire().await.unwrap();
+ let waiting = other.acquire();
+ tokio::pin!(waiting);
+ // Poll admission without a sleep or an assumption about scheduler timing.
+ assert!(
+ std::future::poll_fn(|cx| {
+ std::task::Poll::Ready(waiting.as_mut().poll(cx).is_pending())
+ })
+ .await
+ );
+ drop(permit);
+ let _next = waiting.await.unwrap();
+ }
+}
diff --git a/src/common/batcher/src/flush_policy.rs b/src/common/batcher/src/flush_policy.rs
new file mode 100644
index 00000000000..14804f815e2
--- /dev/null
+++ b/src/common/batcher/src/flush_policy.rs
@@ -0,0 +1,43 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+pub mod timing;
+
+use tokio::time::Instant;
+
+use crate::pending_batch::PendingBatch;
+
+/// The event that caused the worker to reconsider flushing.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum FlushTrigger {
+ /// A complete submission was appended to the batch.
+ Submission,
+ /// The flush timer reached its deadline.
+ Deadline,
+}
+
+/// Decides when a pending batch is ready without owning its payloads or execution.
+///
+/// A deadline wakeup must either allow flushing or produce a future deadline;
+/// otherwise the caller would repeatedly poll an expired timer.
+pub trait FlushPolicy {
+ /// Returns the next timer deadline, or `None` when no timer is needed.
+ fn deadline(&self, batch: &PendingBatch) -> Option;
+
+ /// Returns whether the batch should be flushed for this event at `now`.
+ ///
+ /// Workers never flush empty batches, regardless of this decision.
+ fn should_flush(&self, batch: &PendingBatch, now: Instant, trigger: FlushTrigger)
+ -> bool;
+}
diff --git a/src/common/batcher/src/flush_policy/timing.rs b/src/common/batcher/src/flush_policy/timing.rs
new file mode 100644
index 00000000000..56a9172eaa0
--- /dev/null
+++ b/src/common/batcher/src/flush_policy/timing.rs
@@ -0,0 +1,157 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::time::Duration;
+
+use tokio::time::Instant;
+
+use crate::flush_policy::{FlushPolicy, FlushTrigger};
+use crate::pending_batch::PendingBatch;
+
+/// Decides whether the first-submission deadline or row threshold requires a flush.
+///
+/// The row threshold is checked after appending a complete submission. It is not a
+/// hard batch-size limit and never splits a submission.
+#[derive(Debug, Clone, Copy)]
+pub struct TimingFlushPolicy {
+ flush_interval: Duration,
+ max_batch_rows: usize,
+}
+
+impl TimingFlushPolicy {
+ /// Creates a policy, rejecting zero limits, zero intervals, and unrepresentable deadlines.
+ pub fn try_new(flush_interval: Duration, max_batch_rows: usize) -> Option {
+ if max_batch_rows == 0 || !Self::validate(flush_interval) {
+ return None;
+ }
+ Some(Self {
+ flush_interval,
+ max_batch_rows,
+ })
+ }
+
+ /// Checks that the interval is nonzero and its deadline is representable.
+ pub fn validate(flush_interval: Duration) -> bool {
+ !flush_interval.is_zero() && Instant::now().checked_add(flush_interval).is_some()
+ }
+
+ /// Checks only the row trigger when a caller drives deadline events separately.
+ pub fn reached_row_threshold(&self, batch: &PendingBatch) -> bool {
+ !batch.is_empty() && batch.total_rows() >= self.max_batch_rows
+ }
+}
+
+impl FlushPolicy for TimingFlushPolicy {
+ /// Returns the deadline anchored to the first submission, or `None` for an empty batch.
+ ///
+ /// An unrepresentable deadline falls back to the first submission time so that
+ /// extreme timestamps cannot leave a nonempty batch waiting indefinitely.
+ fn deadline(&self, batch: &PendingBatch) -> Option {
+ batch
+ .first_submitted_at()
+ .map(|first| first.checked_add(self.flush_interval).unwrap_or(first))
+ }
+
+ /// Returns whether a nonempty batch has reached either trigger.
+ fn should_flush(
+ &self,
+ batch: &PendingBatch,
+ now: Instant,
+ trigger: FlushTrigger,
+ ) -> bool {
+ !batch.is_empty()
+ && (self.reached_row_threshold(batch)
+ || (trigger == FlushTrigger::Deadline
+ && self.deadline(batch).is_some_and(|deadline| now >= deadline)))
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn test_validate_matches_construction() {
+ for interval in [Duration::ZERO, Duration::from_secs(1), Duration::MAX] {
+ assert_eq!(
+ TimingFlushPolicy::validate(interval),
+ TimingFlushPolicy::try_new(interval, 1).is_some()
+ );
+ }
+ }
+
+ #[test]
+ fn test_first_submission_deadline() {
+ let policy = TimingFlushPolicy::try_new(Duration::from_millis(10), 100).unwrap();
+ let mut batch = PendingBatch::new();
+ let first = Instant::now();
+ assert_eq!(policy.deadline(&batch), None);
+ assert!(!policy.should_flush(&batch, first, FlushTrigger::Deadline));
+ batch.push(1, 1, first);
+ batch.push(2, 1, first + Duration::from_millis(9));
+ assert_eq!(
+ policy.deadline(&batch),
+ Some(first + Duration::from_millis(10))
+ );
+ assert!(!policy.should_flush(
+ &batch,
+ first + Duration::from_millis(9),
+ FlushTrigger::Deadline
+ ));
+ assert!(policy.should_flush(
+ &batch,
+ first + Duration::from_millis(10),
+ FlushTrigger::Deadline
+ ));
+ assert!(!policy.reached_row_threshold(&batch));
+ assert!(!policy.should_flush(
+ &batch,
+ first + Duration::from_millis(10),
+ FlushTrigger::Submission,
+ ));
+ assert_eq!(batch.take(), vec![1, 2]);
+ let next = first + Duration::from_millis(20);
+ batch.push(3, 0, next);
+ assert!(!policy.should_flush(&batch, next, FlushTrigger::Deadline));
+ assert!(policy.should_flush(
+ &batch,
+ next + Duration::from_millis(10),
+ FlushTrigger::Deadline
+ ));
+ }
+
+ #[test]
+ fn test_row_threshold_preserves_complete_submissions() {
+ let policy = TimingFlushPolicy::try_new(Duration::from_secs(1), 3).unwrap();
+ let now = Instant::now();
+ for rows in [2, 3, 4] {
+ let mut batch = PendingBatch::new();
+ batch.push(vec![0; rows], rows, now);
+ assert_eq!(
+ policy.should_flush(&batch, now, FlushTrigger::Submission),
+ rows >= 3
+ );
+ assert_eq!(policy.reached_row_threshold(&batch), rows >= 3);
+ assert_eq!(batch.take(), vec![vec![0; rows]]);
+ }
+ }
+
+ #[test]
+ fn test_invalid_interval() {
+ assert!(TimingFlushPolicy::try_new(Duration::from_secs(1), 0).is_none());
+ let max_batch_rows = 1;
+ assert!(TimingFlushPolicy::try_new(Duration::ZERO, max_batch_rows).is_none());
+ assert!(TimingFlushPolicy::try_new(Duration::MAX, max_batch_rows).is_none());
+ }
+}
diff --git a/src/common/batcher/src/flush_timer.rs b/src/common/batcher/src/flush_timer.rs
new file mode 100644
index 00000000000..7ae8173201a
--- /dev/null
+++ b/src/common/batcher/src/flush_timer.rs
@@ -0,0 +1,141 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::pin::Pin;
+
+use tokio::time::{Instant, Sleep, sleep_until};
+
+/// Maintains one reusable timer for the current flush deadline.
+///
+/// Creating a timer does not require a Tokio runtime. Arming it requires a Tokio
+/// runtime with time enabled. Disabling retains the allocation for the next arm.
+#[derive(Default)]
+pub struct FlushTimer {
+ deadline: Option,
+ sleep: Option>>,
+}
+
+impl FlushTimer {
+ /// Creates a disabled timer without allocating a Tokio sleep.
+ pub fn new() -> Self {
+ Self::default()
+ }
+
+ /// Arms or disables the timer, resetting its sleep only when the deadline changes.
+ ///
+ /// # Panics
+ ///
+ /// Arming for the first time panics outside a Tokio runtime with time enabled.
+ pub fn set_deadline(&mut self, deadline: Option) {
+ if self.deadline == deadline {
+ return;
+ }
+ self.deadline = deadline;
+ if let Some(deadline) = deadline {
+ match &mut self.sleep {
+ Some(sleep) => sleep.as_mut().reset(deadline),
+ None => self.sleep = Some(Box::pin(sleep_until(deadline))),
+ }
+ }
+ }
+
+ /// Waits until the armed deadline, or indefinitely while disabled.
+ ///
+ /// Cancelling this wait does not change the deadline. An elapsed timer stays
+ /// ready until the caller changes or disables its deadline.
+ pub async fn wait(&mut self) {
+ if self.deadline.is_some()
+ && let Some(sleep) = &mut self.sleep
+ {
+ sleep.as_mut().await;
+ } else {
+ std::future::pending::<()>().await;
+ }
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use std::future::{Future, poll_fn};
+ use std::task::Poll;
+ use std::time::Duration;
+
+ use super::*;
+
+ async fn is_ready(timer: &mut FlushTimer) -> bool {
+ let wait = timer.wait();
+ tokio::pin!(wait);
+ poll_fn(|cx| Poll::Ready(wait.as_mut().poll(cx).is_ready())).await
+ }
+
+ #[test]
+ fn test_create_without_runtime() {
+ let mut timer = FlushTimer::new();
+ timer.set_deadline(None);
+ assert!(timer.sleep.is_none());
+ }
+
+ #[tokio::test(start_paused = true)]
+ async fn test_deadline_and_cancelled_wait() {
+ let mut timer = FlushTimer::new();
+ assert!(!is_ready(&mut timer).await);
+ let deadline = Instant::now() + Duration::from_millis(10);
+ timer.set_deadline(Some(deadline));
+ assert!(!is_ready(&mut timer).await);
+ tokio::time::advance(Duration::from_millis(9)).await;
+ // Each poll drops its wait future. Repeating the same deadline must not
+ // move the timer forward or replace its sleep allocation.
+ let sleep = timer.sleep.as_ref().unwrap().as_ref().get_ref() as *const Sleep;
+ timer.set_deadline(Some(deadline));
+ assert_eq!(
+ sleep,
+ timer.sleep.as_ref().unwrap().as_ref().get_ref() as *const Sleep
+ );
+ assert!(!is_ready(&mut timer).await);
+ tokio::time::advance(Duration::from_millis(1)).await;
+ assert!(is_ready(&mut timer).await);
+ }
+
+ #[tokio::test(start_paused = true)]
+ async fn test_disable_and_rearm_reuses_sleep() {
+ let mut timer = FlushTimer::new();
+ timer.set_deadline(Some(Instant::now()));
+ assert!(is_ready(&mut timer).await);
+ let sleep = timer.sleep.as_ref().unwrap().as_ref().get_ref() as *const Sleep;
+ timer.set_deadline(None);
+ assert!(!is_ready(&mut timer).await);
+ assert!(!is_ready(&mut timer).await);
+ timer.set_deadline(Some(Instant::now() + Duration::from_millis(5)));
+ assert_eq!(
+ sleep,
+ timer.sleep.as_ref().unwrap().as_ref().get_ref() as *const Sleep
+ );
+ assert!(!is_ready(&mut timer).await);
+ tokio::time::advance(Duration::from_millis(5)).await;
+ assert!(is_ready(&mut timer).await);
+ }
+
+ #[tokio::test(start_paused = true)]
+ async fn test_change_armed_deadline() {
+ let mut timer = FlushTimer::new();
+ let now = Instant::now();
+ timer.set_deadline(Some(now + Duration::from_millis(5)));
+ assert!(!is_ready(&mut timer).await);
+ timer.set_deadline(Some(now + Duration::from_millis(10)));
+ tokio::time::advance(Duration::from_millis(5)).await;
+ assert!(!is_ready(&mut timer).await);
+ timer.set_deadline(Some(Instant::now()));
+ assert!(is_ready(&mut timer).await);
+ }
+}
diff --git a/src/mito2/src/sst/parquet/json_align/mod.rs b/src/common/batcher/src/lib.rs
similarity index 61%
rename from src/mito2/src/sst/parquet/json_align/mod.rs
rename to src/common/batcher/src/lib.rs
index 54c97bdbb89..9ad29097aff 100644
--- a/src/mito2/src/sst/parquet/json_align/mod.rs
+++ b/src/common/batcher/src/lib.rs
@@ -12,13 +12,16 @@
// See the License for the specific language governing permissions and
// limitations under the License.
-use datatypes::arrow::record_batch::RecordBatch;
-use futures::stream::BoxStream;
+//! Protocol-independent building blocks for timing-based batching.
+//!
+//! Payload preparation, grouping keys, write execution and request completion
+//! remain the caller's responsibility.
-mod stream;
-
-pub(crate) use stream::NestedSchemaAligner;
-
-use crate::error::Result;
-
-pub(crate) type ProjectedRecordBatchStream = BoxStream<'static, Result>;
+pub mod flush_limiter;
+pub mod flush_policy;
+pub mod flush_timer;
+pub mod notifier;
+pub mod pending_batch;
+pub mod pending_worker;
+pub mod request_limiter;
+pub mod worker_registry;
diff --git a/src/common/batcher/src/notifier.rs b/src/common/batcher/src/notifier.rs
new file mode 100644
index 00000000000..efc795b938a
--- /dev/null
+++ b/src/common/batcher/src/notifier.rs
@@ -0,0 +1,145 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::future::Future;
+use std::num::NonZeroUsize;
+
+use futures::{StreamExt, stream};
+use tokio::sync::Semaphore;
+use tokio::sync::mpsc::error::TrySendError;
+use tokio::sync::mpsc::{self, Receiver, Sender};
+
+/// Best-effort notification admission, independent of the write result.
+///
+/// The caller owns the notification payload, delivery handler and failure metrics.
+pub struct Notifier {
+ sender: Sender,
+}
+
+impl Clone for Notifier {
+ fn clone(&self) -> Self {
+ Self {
+ sender: self.sender.clone(),
+ }
+ }
+}
+
+impl Notifier {
+ /// Returns `None` if the queue capacity is zero or exceeds Tokio's supported capacity.
+ pub fn try_new(capacity: usize) -> Option<(Self, Receiver)> {
+ if capacity == 0 || capacity > Semaphore::MAX_PERMITS {
+ return None;
+ }
+ let (sender, receiver) = mpsc::channel(capacity);
+ Some((Self { sender }, receiver))
+ }
+
+ /// Configured queue capacity, independent of its current occupancy.
+ pub fn max_capacity(&self) -> usize {
+ self.sender.max_capacity()
+ }
+
+ /// Never waits for delivery. Returns the payload on a full or closed queue
+ /// so the caller can record domain-specific diagnostics.
+ pub fn try_notify(&self, notification: T) -> Result<(), TrySendError> {
+ self.sender.try_send(notification)
+ }
+}
+
+/// Delivers queued notifications with bounded concurrency.
+///
+/// The caller schedules this future on its runtime. Closing all senders drains
+/// queued and in-flight notifications. Handler failures are handled by the caller;
+/// this worker does not retry or propagate them into write completion.
+pub async fn run_notifier(
+ receiver: Receiver,
+ max_concurrent_notifications: NonZeroUsize,
+ handler: F,
+) where
+ F: FnMut(T) -> Fut,
+ Fut: Future