rust/greptimedb
1
0
Fork 0
mirror of https://github.com/GreptimeTeam/greptimedb.git synced 2026-11-13 22:36:57 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(mito2): sync main and verify two-phase data cache replay

Browse Source 
Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com>
This commit is contained in:
discord9
2026-09-15 12:25:24 +08:00
parent 922af8db51 ba0f7acd93
commit c1fead51d0
254 changed files with 20681 additions and 2911 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/runner-scale-sets/query-regression/README.md
+7 -7
View File
@@ -51,10 +51,10 @@ Configuration lives in repository variables/secrets:
| vars | `QUERY_REGRESSION_ECS_IMAGE_ID` | Custom image built by `ecs-image/build-ecs-image.py`. |
| vars | `QUERY_REGRESSION_COMMENT_ALLOWLIST` | Comma/whitespace-separated GitHub logins allowed to comment `/query-regression` on a PR. Each login must also have repository `admin` permission. Empty denies all comment commands. |
The system disk is 50 GiB, which covers the image, a 16 GiB swapfile, the
checkout, and cold build caches (target dir, cargo registry, sccache). ENOSPC
stops the runner itself from writing logs, which GitHub reports as `The
operation was canceled` with no telemetry, indistinguishable from a
The system disk is 80 GiB, providing capacity headroom for the image, a 16 GiB
swapfile, the checkout, and cold build caches (target dir, cargo registry,
sccache). ENOSPC stops the runner itself from writing logs, which GitHub reports
as `The operation was canceled` with no telemetry, indistinguishable from a
platform-side cancellation.
cloud-init masks `systemd-oomd`, disables `unattended-upgrades` /
`apt-daily-upgrade`, creates `/swapfile`, and sets `OOMPolicy=continue`
@@ -108,7 +108,7 @@ agent, which Aliyun public Ubuntu images include.
An allowlisted repository admin commenting `/query-regression` on the PR is
**trust admission for that exact PR revision**. `/query-regression` runs the
six routine default cases; `/query-regression heavy` runs only the
nine routine default cases; `/query-regression heavy` runs only the
high-cardinality `prom_remote_write_7913` remote-write case.
`slash-command-dispatch.yml` (`issue_comment` on the default branch) parses
the command and `repository_dispatch`es; `query-regression-slash.yml` admits
@@ -180,7 +180,7 @@ reintroduce those actions unless the image contract changes.
Each run provisions its own ECS instance, so overlapping dispatches proceed
in parallel. There is no workflow `concurrency` group. All Cargo state
(`CARGO_HOME` including registry/git, `CARGO_TARGET_DIR`, sccache, cache
metadata) lives on the 50 GiB system disk and is discarded with the VM.
metadata) lives on the 80 GiB system disk and is discarded with the VM.
`RUSTUP_HOME=/opt/rustup` and `/opt/cargo/bin` are image-owned. The runner
sets `RUSTC_WRAPPER=/usr/local/bin/sccache`,
`SCCACHE_DIR=/home/runner/.cache/sccache`, `SCCACHE_CACHE_SIZE=10G`, and
@@ -189,7 +189,7 @@ at 10G. Base and candidate builds share the target on that disk; Cargo
fingerprints invalidate source and dependency changes.
The workflow reports `du` / `df` in telemetry. It does not try to reclaim
space across runs: a cold 50 GiB disk that fills up fails the build.
space across runs: a cold 80 GiB disk that fills up fails the build.
## Future optional phases
.github/scripts/aliyun-ecs-runner-provision.py
+2 -2
View File
@@ -81,8 +81,8 @@ POLL_INTERVAL_SECONDS = 5
SWAP_FILE = "/swapfile"
SWAP_SIZE_GIB = 16
# Image (~12G), 16G swap, checkout, base+candidate data homes, and cold
# build caches. Deleted with the instance.
SYSTEM_DISK_GIB = 50
# build caches. 80 GiB provides capacity headroom. Deleted with the instance.
SYSTEM_DISK_GIB = 80
@dataclass(frozen=True)
.github/scripts/query-regression-run.py
+2
View File
@@ -33,12 +33,14 @@ from typing import Any
DEFAULT_CASES = [
"tests/perf/query_cases/smoke_direct_sst/case.toml",
"tests/perf/query_cases/sst_float_bss/case.toml",
"tests/perf/query_cases/prom_remote_write_seeded_random/case.toml",
"tests/perf/query_cases/prom_remote_write_run_heavy/case.toml",
"tests/perf/query_cases/prom_remote_write_mixed_every/case.toml",
"tests/perf/query_cases/prom_remote_write_integer_counter/case.toml",
"tests/perf/query_cases/promql_range_boundary/case.toml",
"tests/perf/query_cases/promql_instant_last_row_9034/case.toml",
"tests/perf/query_cases/mito_prefilter_all_match/case.toml",
]
HEAVY_CASES = [
.github/workflows/backport.yml
+2
View File
@@ -45,6 +45,7 @@ jobs:
PR_TITLE: ${{ github.event.pull_request.title }}
PR_BODY: ${{ github.event.pull_request.body }}
AUTHOR: ${{ github.event.pull_request.merged_by.login }}
PR_AUTHOR: ${{ github.event.pull_request.user.login }}
BASE_BRANCH: ${{ github.event.pull_request.base.ref }}
run: |
git config user.name "github-actions[bot]"
@@ -76,6 +77,7 @@ jobs:
git branch -D "${head_branch}"
gh issue create \
--title "Backport failure: PR #${PR_NUMBER} to ${release_branch}" \
--assignee "${PR_AUTHOR}" \
--body "Cherry-pick of ${MERGE_COMMIT} (PR #${PR_NUMBER}) failed on ${release_branch}. Manual backport required." \
|| true
continue
.github/workflows/rust.yml
+29
View File
@@ -122,6 +122,35 @@ jobs:
- name: Run cargo check for riscv64
run: cargo check --locked --workspace --all-targets --features servers/dashboard --target riscv64gc-unknown-linux-gnu
check-windows:
if: ${{ github.repository == 'GreptimeTeam/greptimedb' }}
name: Check Windows build
runs-on: windows-2022
timeout-minutes: 60
steps:
- run: git config --global core.autocrlf false
- uses: actions/checkout@v4
with:
persist-credentials: false
- uses: ./.github/actions/setup-cyborg
- uses: arduino/setup-protoc@v3
with:
repo-token: ${{ secrets.GITHUB_TOKEN }}
- uses: KyleMayes/install-llvm-action@v1
with:
version: "14.0"
- uses: actions-rust-lang/setup-rust-toolchain@v1
- name: Rust Cache
uses: Swatinem/rust-cache@v2
with:
shared-key: "check-windows"
cache-all-crates: "true"
save-if: ${{ github.ref == 'refs/heads/main' }}
- name: Run cargo check for Windows
run: cargo check --locked --workspace --all-targets
env:
RUSTFLAGS: "-D warnings"
conflict-check:
if: ${{ github.repository == 'GreptimeTeam/greptimedb' }}
name: Check for conflict
AGENTS.md
+5
View File
@@ -69,6 +69,11 @@ a module map, read/write paths, change-coupling points, and gotchas:
- [`tests/compatibility/AGENTS.md`](tests/compatibility/AGENTS.md)
- [`tests/perf/AGENTS.md`](tests/perf/AGENTS.md)
## Rust imports
Prefer crate-rooted imports (`use crate::...`) over `use super::...` in
production code. Relative imports using `super` are allowed in tests.
## Read before changing code
- [`.agents/architecture-invariants.md`](.agents/architecture-invariants.md) —
Cargo.lock
Generated
+148 -73
View File
@@ -1172,7 +1172,7 @@ dependencies = [
"cfg-if",
"libc",
"miniz_oxide",
"object",
"object 0.36.7",
"rustc-demangle",
"windows-targets 0.52.6",
]
@@ -1623,7 +1623,7 @@ dependencies = [
"maybe-owned",
"rustix 1.0.7",
"rustix-linux-procfs",
"windows-sys 0.60.2",
"windows-sys 0.61.2",
"winx",
]
@@ -2271,7 +2271,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "117725a109d387c937a1533ce01b450cbde6b88abceea8473c4d7a85853cda3c"
dependencies = [
"lazy_static",
"windows-sys 0.59.0",
"windows-sys 0.48.0",
]
[[package]]
@@ -3768,7 +3768,7 @@ checksum = "a4ae5f15dda3c708c0ade84bfee31ccab44a3da4f88015ed22f63732abe300c8"
[[package]]
name = "datafusion"
version = "53.1.0"
source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=a281d9f0ea4b3fb2ec88bc3553d8dcef307e072a#a281d9f0ea4b3fb2ec88bc3553d8dcef307e072a"
source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=bb531e754b3b36d7a4d3222f35ebc88204dafd2b#bb531e754b3b36d7a4d3222f35ebc88204dafd2b"
dependencies = [
"arrow 58.3.0",
"arrow-schema 58.3.0",
@@ -3822,7 +3822,7 @@ dependencies = [
[[package]]
name = "datafusion-catalog"
version = "53.1.0"
source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=a281d9f0ea4b3fb2ec88bc3553d8dcef307e072a#a281d9f0ea4b3fb2ec88bc3553d8dcef307e072a"
source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=bb531e754b3b36d7a4d3222f35ebc88204dafd2b#bb531e754b3b36d7a4d3222f35ebc88204dafd2b"
dependencies = [
"arrow 58.3.0",
"async-trait",
@@ -3846,7 +3846,7 @@ dependencies = [
[[package]]
name = "datafusion-catalog-listing"
version = "53.1.0"
source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=a281d9f0ea4b3fb2ec88bc3553d8dcef307e072a#a281d9f0ea4b3fb2ec88bc3553d8dcef307e072a"
source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=bb531e754b3b36d7a4d3222f35ebc88204dafd2b#bb531e754b3b36d7a4d3222f35ebc88204dafd2b"
dependencies = [
"arrow 58.3.0",
"async-trait",
@@ -3868,7 +3868,7 @@ dependencies = [
[[package]]
name = "datafusion-common"
version = "53.1.0"
source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=a281d9f0ea4b3fb2ec88bc3553d8dcef307e072a#a281d9f0ea4b3fb2ec88bc3553d8dcef307e072a"
source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=bb531e754b3b36d7a4d3222f35ebc88204dafd2b#bb531e754b3b36d7a4d3222f35ebc88204dafd2b"
dependencies = [
"ahash 0.8.12",
"arrow 58.3.0",
@@ -3892,7 +3892,7 @@ dependencies = [
[[package]]
name = "datafusion-common-runtime"
version = "53.1.0"
source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=a281d9f0ea4b3fb2ec88bc3553d8dcef307e072a#a281d9f0ea4b3fb2ec88bc3553d8dcef307e072a"
source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=bb531e754b3b36d7a4d3222f35ebc88204dafd2b#bb531e754b3b36d7a4d3222f35ebc88204dafd2b"
dependencies = [
"futures",
"log",
@@ -3902,7 +3902,7 @@ dependencies = [
[[package]]
name = "datafusion-datasource"
version = "53.1.0"
source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=a281d9f0ea4b3fb2ec88bc3553d8dcef307e072a#a281d9f0ea4b3fb2ec88bc3553d8dcef307e072a"
source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=bb531e754b3b36d7a4d3222f35ebc88204dafd2b#bb531e754b3b36d7a4d3222f35ebc88204dafd2b"
dependencies = [
"arrow 58.3.0",
"async-compression",
@@ -3936,7 +3936,7 @@ dependencies = [
[[package]]
name = "datafusion-datasource-arrow"
version = "53.1.0"
source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=a281d9f0ea4b3fb2ec88bc3553d8dcef307e072a#a281d9f0ea4b3fb2ec88bc3553d8dcef307e072a"
source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=bb531e754b3b36d7a4d3222f35ebc88204dafd2b#bb531e754b3b36d7a4d3222f35ebc88204dafd2b"
dependencies = [
"arrow 58.3.0",
"arrow-ipc 58.3.0",
@@ -3959,7 +3959,7 @@ dependencies = [
[[package]]
name = "datafusion-datasource-csv"
version = "53.1.0"
source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=a281d9f0ea4b3fb2ec88bc3553d8dcef307e072a#a281d9f0ea4b3fb2ec88bc3553d8dcef307e072a"
source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=bb531e754b3b36d7a4d3222f35ebc88204dafd2b#bb531e754b3b36d7a4d3222f35ebc88204dafd2b"
dependencies = [
"arrow 58.3.0",
"async-trait",
@@ -3981,7 +3981,7 @@ dependencies = [
[[package]]
name = "datafusion-datasource-json"
version = "53.1.0"
source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=a281d9f0ea4b3fb2ec88bc3553d8dcef307e072a#a281d9f0ea4b3fb2ec88bc3553d8dcef307e072a"
source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=bb531e754b3b36d7a4d3222f35ebc88204dafd2b#bb531e754b3b36d7a4d3222f35ebc88204dafd2b"
dependencies = [
"arrow 58.3.0",
"async-trait",
@@ -4004,7 +4004,7 @@ dependencies = [
[[package]]
name = "datafusion-datasource-parquet"
version = "53.1.0"
source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=a281d9f0ea4b3fb2ec88bc3553d8dcef307e072a#a281d9f0ea4b3fb2ec88bc3553d8dcef307e072a"
source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=bb531e754b3b36d7a4d3222f35ebc88204dafd2b#bb531e754b3b36d7a4d3222f35ebc88204dafd2b"
dependencies = [
"arrow 58.3.0",
"async-trait",
@@ -4033,12 +4033,12 @@ dependencies = [
[[package]]
name = "datafusion-doc"
version = "53.1.0"
source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=a281d9f0ea4b3fb2ec88bc3553d8dcef307e072a#a281d9f0ea4b3fb2ec88bc3553d8dcef307e072a"
source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=bb531e754b3b36d7a4d3222f35ebc88204dafd2b#bb531e754b3b36d7a4d3222f35ebc88204dafd2b"
[[package]]
name = "datafusion-execution"
version = "53.1.0"
source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=a281d9f0ea4b3fb2ec88bc3553d8dcef307e072a#a281d9f0ea4b3fb2ec88bc3553d8dcef307e072a"
source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=bb531e754b3b36d7a4d3222f35ebc88204dafd2b#bb531e754b3b36d7a4d3222f35ebc88204dafd2b"
dependencies = [
"arrow 58.3.0",
"arrow-buffer 58.3.0",
@@ -4060,7 +4060,7 @@ dependencies = [
[[package]]
name = "datafusion-expr"
version = "53.1.0"
source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=a281d9f0ea4b3fb2ec88bc3553d8dcef307e072a#a281d9f0ea4b3fb2ec88bc3553d8dcef307e072a"
source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=bb531e754b3b36d7a4d3222f35ebc88204dafd2b#bb531e754b3b36d7a4d3222f35ebc88204dafd2b"
dependencies = [
"arrow 58.3.0",
"async-trait",
@@ -4082,7 +4082,7 @@ dependencies = [
[[package]]
name = "datafusion-expr-common"
version = "53.1.0"
source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=a281d9f0ea4b3fb2ec88bc3553d8dcef307e072a#a281d9f0ea4b3fb2ec88bc3553d8dcef307e072a"
source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=bb531e754b3b36d7a4d3222f35ebc88204dafd2b#bb531e754b3b36d7a4d3222f35ebc88204dafd2b"
dependencies = [
"arrow 58.3.0",
"datafusion-common",
@@ -4094,7 +4094,7 @@ dependencies = [
[[package]]
name = "datafusion-functions"
version = "53.1.0"
source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=a281d9f0ea4b3fb2ec88bc3553d8dcef307e072a#a281d9f0ea4b3fb2ec88bc3553d8dcef307e072a"
source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=bb531e754b3b36d7a4d3222f35ebc88204dafd2b#bb531e754b3b36d7a4d3222f35ebc88204dafd2b"
dependencies = [
"arrow 58.3.0",
"arrow-buffer 58.3.0",
@@ -4125,7 +4125,7 @@ dependencies = [
[[package]]
name = "datafusion-functions-aggregate"
version = "53.1.0"
source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=a281d9f0ea4b3fb2ec88bc3553d8dcef307e072a#a281d9f0ea4b3fb2ec88bc3553d8dcef307e072a"
source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=bb531e754b3b36d7a4d3222f35ebc88204dafd2b#bb531e754b3b36d7a4d3222f35ebc88204dafd2b"
dependencies = [
"ahash 0.8.12",
"arrow 58.3.0",
@@ -4146,7 +4146,7 @@ dependencies = [
[[package]]
name = "datafusion-functions-aggregate-common"
version = "53.1.0"
source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=a281d9f0ea4b3fb2ec88bc3553d8dcef307e072a#a281d9f0ea4b3fb2ec88bc3553d8dcef307e072a"
source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=bb531e754b3b36d7a4d3222f35ebc88204dafd2b#bb531e754b3b36d7a4d3222f35ebc88204dafd2b"
dependencies = [
"ahash 0.8.12",
"arrow 58.3.0",
@@ -4158,7 +4158,7 @@ dependencies = [
[[package]]
name = "datafusion-functions-nested"
version = "53.1.0"
source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=a281d9f0ea4b3fb2ec88bc3553d8dcef307e072a#a281d9f0ea4b3fb2ec88bc3553d8dcef307e072a"
source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=bb531e754b3b36d7a4d3222f35ebc88204dafd2b#bb531e754b3b36d7a4d3222f35ebc88204dafd2b"
dependencies = [
"arrow 58.3.0",
"arrow-ord 58.3.0",
@@ -4182,7 +4182,7 @@ dependencies = [
[[package]]
name = "datafusion-functions-table"
version = "53.1.0"
source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=a281d9f0ea4b3fb2ec88bc3553d8dcef307e072a#a281d9f0ea4b3fb2ec88bc3553d8dcef307e072a"
source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=bb531e754b3b36d7a4d3222f35ebc88204dafd2b#bb531e754b3b36d7a4d3222f35ebc88204dafd2b"
dependencies = [
"arrow 58.3.0",
"async-trait",
@@ -4197,7 +4197,7 @@ dependencies = [
[[package]]
name = "datafusion-functions-window"
version = "53.1.0"
source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=a281d9f0ea4b3fb2ec88bc3553d8dcef307e072a#a281d9f0ea4b3fb2ec88bc3553d8dcef307e072a"
source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=bb531e754b3b36d7a4d3222f35ebc88204dafd2b#bb531e754b3b36d7a4d3222f35ebc88204dafd2b"
dependencies = [
"arrow 58.3.0",
"datafusion-common",
@@ -4214,7 +4214,7 @@ dependencies = [
[[package]]
name = "datafusion-functions-window-common"
version = "53.1.0"
source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=a281d9f0ea4b3fb2ec88bc3553d8dcef307e072a#a281d9f0ea4b3fb2ec88bc3553d8dcef307e072a"
source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=bb531e754b3b36d7a4d3222f35ebc88204dafd2b#bb531e754b3b36d7a4d3222f35ebc88204dafd2b"
dependencies = [
"datafusion-common",
"datafusion-physical-expr-common",
@@ -4223,7 +4223,7 @@ dependencies = [
[[package]]
name = "datafusion-macros"
version = "53.1.0"
source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=a281d9f0ea4b3fb2ec88bc3553d8dcef307e072a#a281d9f0ea4b3fb2ec88bc3553d8dcef307e072a"
source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=bb531e754b3b36d7a4d3222f35ebc88204dafd2b#bb531e754b3b36d7a4d3222f35ebc88204dafd2b"
dependencies = [
"datafusion-doc",
"quote",
@@ -4233,7 +4233,7 @@ dependencies = [
[[package]]
name = "datafusion-optimizer"
version = "53.1.0"
source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=a281d9f0ea4b3fb2ec88bc3553d8dcef307e072a#a281d9f0ea4b3fb2ec88bc3553d8dcef307e072a"
source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=bb531e754b3b36d7a4d3222f35ebc88204dafd2b#bb531e754b3b36d7a4d3222f35ebc88204dafd2b"
dependencies = [
"arrow 58.3.0",
"chrono",
@@ -4282,7 +4282,7 @@ dependencies = [
[[package]]
name = "datafusion-physical-expr"
version = "53.1.0"
source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=a281d9f0ea4b3fb2ec88bc3553d8dcef307e072a#a281d9f0ea4b3fb2ec88bc3553d8dcef307e072a"
source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=bb531e754b3b36d7a4d3222f35ebc88204dafd2b#bb531e754b3b36d7a4d3222f35ebc88204dafd2b"
dependencies = [
"ahash 0.8.12",
"arrow 58.3.0",
@@ -4305,7 +4305,7 @@ dependencies = [
[[package]]
name = "datafusion-physical-expr-adapter"
version = "53.1.0"
source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=a281d9f0ea4b3fb2ec88bc3553d8dcef307e072a#a281d9f0ea4b3fb2ec88bc3553d8dcef307e072a"
source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=bb531e754b3b36d7a4d3222f35ebc88204dafd2b#bb531e754b3b36d7a4d3222f35ebc88204dafd2b"
dependencies = [
"arrow 58.3.0",
"datafusion-common",
@@ -4319,7 +4319,7 @@ dependencies = [
[[package]]
name = "datafusion-physical-expr-common"
version = "53.1.0"
source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=a281d9f0ea4b3fb2ec88bc3553d8dcef307e072a#a281d9f0ea4b3fb2ec88bc3553d8dcef307e072a"
source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=bb531e754b3b36d7a4d3222f35ebc88204dafd2b#bb531e754b3b36d7a4d3222f35ebc88204dafd2b"
dependencies = [
"ahash 0.8.12",
"arrow 58.3.0",
@@ -4335,7 +4335,7 @@ dependencies = [
[[package]]
name = "datafusion-physical-optimizer"
version = "53.1.0"
source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=a281d9f0ea4b3fb2ec88bc3553d8dcef307e072a#a281d9f0ea4b3fb2ec88bc3553d8dcef307e072a"
source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=bb531e754b3b36d7a4d3222f35ebc88204dafd2b#bb531e754b3b36d7a4d3222f35ebc88204dafd2b"
dependencies = [
"arrow 58.3.0",
"datafusion-common",
@@ -4353,7 +4353,7 @@ dependencies = [
[[package]]
name = "datafusion-physical-plan"
version = "53.1.0"
source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=a281d9f0ea4b3fb2ec88bc3553d8dcef307e072a#a281d9f0ea4b3fb2ec88bc3553d8dcef307e072a"
source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=bb531e754b3b36d7a4d3222f35ebc88204dafd2b#bb531e754b3b36d7a4d3222f35ebc88204dafd2b"
dependencies = [
"ahash 0.8.12",
"arrow 58.3.0",
@@ -4384,7 +4384,7 @@ dependencies = [
[[package]]
name = "datafusion-proto"
version = "53.1.0"
source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=a281d9f0ea4b3fb2ec88bc3553d8dcef307e072a#a281d9f0ea4b3fb2ec88bc3553d8dcef307e072a"
source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=bb531e754b3b36d7a4d3222f35ebc88204dafd2b#bb531e754b3b36d7a4d3222f35ebc88204dafd2b"
dependencies = [
"arrow 58.3.0",
"chrono",
@@ -4411,7 +4411,7 @@ dependencies = [
[[package]]
name = "datafusion-proto-common"
version = "53.1.0"
source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=a281d9f0ea4b3fb2ec88bc3553d8dcef307e072a#a281d9f0ea4b3fb2ec88bc3553d8dcef307e072a"
source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=bb531e754b3b36d7a4d3222f35ebc88204dafd2b#bb531e754b3b36d7a4d3222f35ebc88204dafd2b"
dependencies = [
"arrow 58.3.0",
"datafusion-common",
@@ -4421,7 +4421,7 @@ dependencies = [
[[package]]
name = "datafusion-pruning"
version = "53.1.0"
source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=a281d9f0ea4b3fb2ec88bc3553d8dcef307e072a#a281d9f0ea4b3fb2ec88bc3553d8dcef307e072a"
source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=bb531e754b3b36d7a4d3222f35ebc88204dafd2b#bb531e754b3b36d7a4d3222f35ebc88204dafd2b"
dependencies = [
"arrow 58.3.0",
"datafusion-common",
@@ -4437,7 +4437,7 @@ dependencies = [
[[package]]
name = "datafusion-session"
version = "53.1.0"
source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=a281d9f0ea4b3fb2ec88bc3553d8dcef307e072a#a281d9f0ea4b3fb2ec88bc3553d8dcef307e072a"
source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=bb531e754b3b36d7a4d3222f35ebc88204dafd2b#bb531e754b3b36d7a4d3222f35ebc88204dafd2b"
dependencies = [
"async-trait",
"datafusion-common",
@@ -4450,7 +4450,7 @@ dependencies = [
[[package]]
name = "datafusion-sql"
version = "53.1.0"
source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=a281d9f0ea4b3fb2ec88bc3553d8dcef307e072a#a281d9f0ea4b3fb2ec88bc3553d8dcef307e072a"
source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=bb531e754b3b36d7a4d3222f35ebc88204dafd2b#bb531e754b3b36d7a4d3222f35ebc88204dafd2b"
dependencies = [
"arrow 58.3.0",
"bigdecimal 0.4.8",
@@ -4468,7 +4468,7 @@ dependencies = [
[[package]]
name = "datafusion-substrait"
version = "53.1.0"
source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=a281d9f0ea4b3fb2ec88bc3553d8dcef307e072a#a281d9f0ea4b3fb2ec88bc3553d8dcef307e072a"
source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=bb531e754b3b36d7a4d3222f35ebc88204dafd2b#bb531e754b3b36d7a4d3222f35ebc88204dafd2b"
dependencies = [
"async-recursion",
"async-trait",
@@ -5177,7 +5177,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "778e2ac28f6c47af28e4907f13ffd1e1ddbd400980a9abd7c8df189bf578a5ad"
dependencies = [
"libc",
"windows-sys 0.60.2",
"windows-sys 0.52.0",
]
[[package]]
@@ -5265,6 +5265,12 @@ version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7"
[[package]]
name = "fallible-iterator"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649"
[[package]]
name = "fallible-streaming-iterator"
version = "0.1.9"
@@ -5601,6 +5607,20 @@ version = "2.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "28dd6caf6059519a65843af8fe2a3ae298b14b80179855aeb4adc2c1934ee619"
[[package]]
name = "framehop"
version = "0.13.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "09a5a3f0acb82df800ca3aa50c0d60d286c5d13d4cfc3114b3a9663f13b032fe"
dependencies = [
"arrayvec",
"cfg-if",
"fallible-iterator 0.3.0",
"gimli",
"macho-unwind-info",
"pe-unwind-info",
]
[[package]]
name = "frontend"
version = "1.3.0-alpha.1"
@@ -5746,7 +5766,7 @@ checksum = "94e7099f6313ecacbe1256e8ff9d617b75d1bcb16a6fddef94866d225a01a14a"
dependencies = [
"io-lifetimes 2.0.4",
"rustix 1.0.7",
"windows-sys 0.59.0",
"windows-sys 0.52.0",
]
[[package]]
@@ -6062,6 +6082,10 @@ name = "gimli"
version = "0.31.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f"
dependencies = [
"fallible-iterator 0.3.0",
"stable_deref_trait",
]
[[package]]
name = "git2"
@@ -6097,7 +6121,7 @@ dependencies = [
[[package]]
name = "greptime-proto"
version = "0.1.0"
source = "git+https://github.com/GreptimeTeam/greptime-proto.git?rev=32f467fa2ba2b3588a58381a24af83de09fbb00a#32f467fa2ba2b3588a58381a24af83de09fbb00a"
source = "git+https://github.com/GreptimeTeam/greptime-proto.git?rev=0c4beffaa2b3968034915867593a717c6ab0c6b4#0c4beffaa2b3968034915867593a717c6ab0c6b4"
dependencies = [
"prost 0.14.1",
"prost-types 0.14.1",
@@ -7143,7 +7167,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "20fd6de4ccfcc187e38bc21cfa543cb5a302cb86a8b114eb7f0bf0dc9f8ac00f"
dependencies = [
"io-lifetimes 3.0.1",
"windows-sys 0.60.2",
"windows-sys 0.52.0",
]
[[package]]
@@ -7181,7 +7205,7 @@ checksum = "e04d7f318608d35d4b61ddd75cbdaee86b023ebe2bd5a66ee0915f0bf93095a9"
dependencies = [
"hermit-abi",
"libc",
"windows-sys 0.59.0",
"windows-sys 0.52.0",
]
[[package]]
@@ -7884,7 +7908,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "07033963ba89ebaf1584d767badaa2e8fcec21aedea6b8c0346d487d49c28667"
dependencies = [
"cfg-if",
"windows-targets 0.53.5",
"windows-targets 0.48.5",
]
[[package]]
@@ -8230,6 +8254,17 @@ dependencies = [
"thiserror 1.0.69",
]
[[package]]
name = "macho-unwind-info"
version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bb4bdc8b0ce69932332cf76d24af69c3a155242af95c226b2ab6c2e371ed1149"
dependencies = [
"thiserror 2.0.17",
"zerocopy",
"zerocopy-derive",
]
[[package]]
name = "mappings"
version = "0.7.1"
@@ -8346,6 +8381,15 @@ dependencies = [
"thiserror 1.0.69",
]
[[package]]
name = "memmap2"
version = "0.5.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "83faa42c0a078c393f6b29d5db232d8be22776a891f8f56e5284faee4a20b327"
dependencies = [
"libc",
]
[[package]]
name = "memmap2"
version = "0.9.5"
@@ -9175,7 +9219,7 @@ version = "0.50.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5"
dependencies = [
"windows-sys 0.60.2",
"windows-sys 0.61.2",
]
[[package]]
@@ -9443,6 +9487,16 @@ dependencies = [
"objc2-core-foundation",
]
[[package]]
name = "object"
version = "0.29.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "21158b2c33aa6d4561f1c0a6ea283ca92bc54802a93b263e910746d679a7eb53"
dependencies = [
"flate2",
"memchr",
]
[[package]]
name = "object"
version = "0.36.7"
@@ -10003,6 +10057,7 @@ dependencies = [
"arrow-ipc 58.3.0",
"async-stream",
"async-trait",
"axum 0.8.4",
"bytes",
"catalog",
"chrono",
@@ -10063,6 +10118,7 @@ dependencies = [
"tokio-util",
"tonic 0.14.2",
"tracing",
"url",
]
[[package]]
@@ -10484,6 +10540,19 @@ dependencies = [
"hmac 0.12.1",
]
[[package]]
name = "pe-unwind-info"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "500fa4cdeacd98997c5865e3d0d1cb8fe7e9d7d75ecc775e07989a433a9a9a59"
dependencies = [
"arrayvec",
"bitflags 2.12.1",
"thiserror 2.0.17",
"zerocopy",
"zerocopy-derive",
]
[[package]]
name = "peeking_take_while"
version = "1.0.0"
@@ -10997,7 +11066,7 @@ dependencies = [
"base64 0.22.1",
"byteorder",
"bytes",
"fallible-iterator",
"fallible-iterator 0.2.0",
"hmac 0.13.0",
"md-5 0.11.0",
"memchr",
@@ -11015,7 +11084,7 @@ dependencies = [
"array-init",
"bytes",
"chrono",
"fallible-iterator",
"fallible-iterator 0.2.0",
"postgres-protocol",
"serde_core",
"serde_json",
@@ -11038,30 +11107,33 @@ checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391"
[[package]]
name = "pprof"
version = "0.14.0"
version = "0.15.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ebbe2f8898beba44815fdc9e5a4ae9c929e21c5dc29b0c774a15555f7f58d6d0"
checksum = "38a01da47675efa7673b032bf8efd8214f1917d89685e07e395ab125ea42b187"
dependencies = [
"aligned-vec",
"backtrace",
"cfg-if",
"criterion 0.5.1",
"findshlibs",
"framehop",
"inferno 0.11.21",
"libc",
"log",
"memmap2 0.5.10",
"nix 0.26.4",
"object 0.29.0",
"once_cell",
"parking_lot 0.12.4",
"prost 0.12.6",
"prost-build 0.12.6",
"prost-derive 0.12.6",
"protobuf 2.28.0",
"protobuf 3.7.2",
"sha2 0.10.9",
"smallvec",
"spin 0.10.1",
"symbolic-demangle",
"tempfile",
"thiserror 1.0.69",
"thiserror 2.0.17",
]
[[package]]
@@ -11383,7 +11455,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "22505a5c94da8e3b7c2996394d1c933236c4d743e81a410bcca4e6989fc066a4"
dependencies = [
"bytes",
"heck 0.5.0",
"heck 0.4.1",
"itertools 0.12.1",
"log",
"multimap",
@@ -11403,7 +11475,7 @@ version = "0.14.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ac6c3320f9abac597dcbc668774ef006702672474aad53c6d596b62e487b40b1"
dependencies = [
"heck 0.5.0",
"heck 0.4.1",
"itertools 0.14.0",
"log",
"multimap",
@@ -11922,7 +11994,7 @@ dependencies = [
"once_cell",
"socket2 0.5.10",
"tracing",
"windows-sys 0.59.0",
"windows-sys 0.52.0",
]
[[package]]
@@ -12893,7 +12965,7 @@ dependencies = [
"errno",
"libc",
"linux-raw-sys 0.4.15",
"windows-sys 0.59.0",
"windows-sys 0.52.0",
]
[[package]]
@@ -12906,7 +12978,7 @@ dependencies = [
"errno",
"libc",
"linux-raw-sys 0.9.4",
"windows-sys 0.59.0",
"windows-sys 0.52.0",
]
[[package]]
@@ -12997,7 +13069,7 @@ dependencies = [
"security-framework 3.7.0",
"security-framework-sys",
"webpki-root-certs",
"windows-sys 0.60.2",
"windows-sys 0.52.0",
]
[[package]]
@@ -13584,6 +13656,7 @@ dependencies = [
"table",
"tempfile",
"tikv-jemalloc-ctl",
"tikv-jemallocator",
"tokio",
"tokio-postgres",
"tokio-postgres-rustls",
@@ -13882,7 +13955,7 @@ version = "0.8.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1961e2ef424c1424204d3a5d6975f934f56b6d50ff5732382d84ebf460e147f7"
dependencies = [
"heck 0.5.0",
"heck 0.4.1",
"proc-macro2",
"quote",
"syn 2.0.117",
@@ -13911,7 +13984,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "52d1cfed4120b4d927bf7c0f86d2087a4a7d6027c906d9f9d525a80573b9be51"
dependencies = [
"libc",
"windows-sys 0.60.2",
"windows-sys 0.61.2",
]
[[package]]
@@ -13952,6 +14025,9 @@ name = "spin"
version = "0.10.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "023a211cb3138dbc438680b32560ad89f699977624c9f8dbb95a47d5b4c07dd3"
dependencies = [
"lock_api",
]
[[package]]
name = "spki"
@@ -14303,7 +14379,7 @@ dependencies = [
"cfg-if",
"libc",
"psm",
"windows-sys 0.59.0",
"windows-sys 0.52.0",
]
[[package]]
@@ -14582,7 +14658,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6a1150bdda9314f6cfeeea801c23f5593c6e6a6c72e64f67e48d723a12b8efdb"
dependencies = [
"debugid",
"memmap2",
"memmap2 0.9.5",
"stable_deref_trait",
"uuid",
]
@@ -14751,7 +14827,7 @@ dependencies = [
"lru 0.16.4",
"lz4_flex 0.13.1",
"measure_time",
"memmap2",
"memmap2 0.9.5",
"once_cell",
"oneshot",
"rayon",
@@ -14920,7 +14996,7 @@ dependencies = [
"getrandom 0.3.4",
"once_cell",
"rustix 1.0.7",
"windows-sys 0.60.2",
"windows-sys 0.52.0",
]
[[package]]
@@ -15179,9 +15255,9 @@ dependencies = [
[[package]]
name = "tikv-jemalloc-ctl"
version = "0.6.0"
version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f21f216790c8df74ce3ab25b534e0718da5a1916719771d3fec23315c99e468b"
checksum = "3a184c43b8ab2f41df2733b55556e3f5f632f4aeaa205b1bb018f574b7f5f142"
dependencies = [
"libc",
"paste",
@@ -15190,9 +15266,8 @@ dependencies = [
[[package]]
name = "tikv-jemalloc-sys"
version = "0.6.0+5.3.0-1-ge13ca993e8ccb9ba9847cc330696e02839f328f7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cd3c60906412afa9c2b5b5a48ca6a5abe5736aec9eb48ad05037a677e52e4e2d"
version = "0.7.0+5.3.1-161-gff80bf2d0c8f162ea445533354cad8b7e5830f5c"
source = "git+https://github.com/GreptimeTeam/jemallocator?rev=e254a7eaf13a9ede04953ddefd173b5303018530#e254a7eaf13a9ede04953ddefd173b5303018530"
dependencies = [
"cc",
"libc",
@@ -15200,9 +15275,9 @@ dependencies = [
[[package]]
name = "tikv-jemallocator"
version = "0.6.0"
version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4cec5ff18518d81584f477e9bfdf957f5bb0979b0bac3af4ca30b5b3ae2d2865"
checksum = "249f09e49ab1609436f34c776e84231bead18d6a955f119f939bdc1d847561bd"
dependencies = [
"libc",
"tikv-jemalloc-sys",
@@ -15379,7 +15454,7 @@ dependencies = [
"async-trait",
"byteorder",
"bytes",
"fallible-iterator",
"fallible-iterator 0.2.0",
"futures-channel",
"futures-util",
"log",
@@ -16728,7 +16803,7 @@ version = "0.1.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb"
dependencies = [
"windows-sys 0.59.0",
"windows-sys 0.48.0",
]
[[package]]
@@ -17178,7 +17253,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3f3fd376f71958b862e7afb20cfe5a22830e1963462f3a17f49d82a6c1d1f42d"
dependencies = [
"bitflags 2.12.1",
"windows-sys 0.59.0",
"windows-sys 0.52.0",
]
[[package]]
Cargo.toml
+23 -16
View File
@@ -159,7 +159,7 @@ fs2 = "0.4"
fst = "0.4.7"
futures = "0.3"
futures-util = "0.3"
greptime-proto = { git = "https://github.com/GreptimeTeam/greptime-proto.git", rev = "32f467fa2ba2b3588a58381a24af83de09fbb00a" }
greptime-proto = { git = "https://github.com/GreptimeTeam/greptime-proto.git", rev = "0c4beffaa2b3968034915867593a717c6ab0c6b4" }
hex = "0.4"
hostname = "0.4.0"
http = "1"
@@ -353,23 +353,30 @@ git = "https://github.com/GreptimeTeam/greptime-meter.git"
rev = "5618e779cf2bb4755b499c630fba4c35e91898cb"
[patch.crates-io]
datafusion = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "a281d9f0ea4b3fb2ec88bc3553d8dcef307e072a" }
datafusion-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "a281d9f0ea4b3fb2ec88bc3553d8dcef307e072a" }
datafusion-datasource = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "a281d9f0ea4b3fb2ec88bc3553d8dcef307e072a" }
datafusion-expr = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "a281d9f0ea4b3fb2ec88bc3553d8dcef307e072a" }
datafusion-expr-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "a281d9f0ea4b3fb2ec88bc3553d8dcef307e072a" }
datafusion-functions = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "a281d9f0ea4b3fb2ec88bc3553d8dcef307e072a" }
datafusion-functions-aggregate-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "a281d9f0ea4b3fb2ec88bc3553d8dcef307e072a" }
datafusion-functions-window-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "a281d9f0ea4b3fb2ec88bc3553d8dcef307e072a" }
datafusion-optimizer = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "a281d9f0ea4b3fb2ec88bc3553d8dcef307e072a" }
datafusion-physical-expr = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "a281d9f0ea4b3fb2ec88bc3553d8dcef307e072a" }
datafusion-physical-expr-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "a281d9f0ea4b3fb2ec88bc3553d8dcef307e072a" }
datafusion-physical-plan = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "a281d9f0ea4b3fb2ec88bc3553d8dcef307e072a" }
datafusion-proto = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "a281d9f0ea4b3fb2ec88bc3553d8dcef307e072a" }
datafusion-sql = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "a281d9f0ea4b3fb2ec88bc3553d8dcef307e072a" }
datafusion-substrait = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "a281d9f0ea4b3fb2ec88bc3553d8dcef307e072a" }
datafusion = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "bb531e754b3b36d7a4d3222f35ebc88204dafd2b" }
datafusion-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "bb531e754b3b36d7a4d3222f35ebc88204dafd2b" }
datafusion-datasource = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "bb531e754b3b36d7a4d3222f35ebc88204dafd2b" }
datafusion-expr = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "bb531e754b3b36d7a4d3222f35ebc88204dafd2b" }
datafusion-expr-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "bb531e754b3b36d7a4d3222f35ebc88204dafd2b" }
datafusion-functions = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "bb531e754b3b36d7a4d3222f35ebc88204dafd2b" }
datafusion-functions-aggregate-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "bb531e754b3b36d7a4d3222f35ebc88204dafd2b" }
datafusion-functions-window-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "bb531e754b3b36d7a4d3222f35ebc88204dafd2b" }
datafusion-optimizer = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "bb531e754b3b36d7a4d3222f35ebc88204dafd2b" }
datafusion-physical-expr = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "bb531e754b3b36d7a4d3222f35ebc88204dafd2b" }
datafusion-physical-expr-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "bb531e754b3b36d7a4d3222f35ebc88204dafd2b" }
datafusion-physical-plan = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "bb531e754b3b36d7a4d3222f35ebc88204dafd2b" }
datafusion-proto = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "bb531e754b3b36d7a4d3222f35ebc88204dafd2b" }
datafusion-sql = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "bb531e754b3b36d7a4d3222f35ebc88204dafd2b" }
datafusion-substrait = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "bb531e754b3b36d7a4d3222f35ebc88204dafd2b" }
sqlparser = { git = "https://github.com/GreptimeTeam/sqlparser-rs.git", rev = "2aefa08a8d69c96eec2d6d6703598a009bba6e4c" } # on branch v0.61.x
# Temporary: use the GreptimeTeam fork of tikv-jemalloc-sys embedding
# jemalloc 5.3.1 + backport of 54f22c83 ("Initialize TSD tcache before
# enabling it"), fixing the TSD/tcache enable-before-init reentrancy
# window. Remove once tikv/jemallocator ships a jemalloc snapshot that
# includes the fix. See GreptimeTeam/jemalloc#1.
tikv-jemalloc-sys = { git = "https://github.com/GreptimeTeam/jemallocator", rev = "e254a7eaf13a9ede04953ddefd173b5303018530" }
[profile.release]
debug = 1
Makefile
+10 -1
View File
@@ -4,6 +4,8 @@ FEATURES ?=
TARGET_DIR ?=
TARGET ?=
BUILD_BIN ?= greptime
BUILD_PACKAGE ?= $(if $(filter greptime,$(strip $(BUILD_BIN))),cmd)
BUILD_PACKAGE_OPT = $(if $(strip $(BUILD_PACKAGE)),-p ${BUILD_PACKAGE})
CARGO_BUILD_OPTS := --locked
IMAGE_REGISTRY ?= docker.io
IMAGE_NAMESPACE ?= greptime
@@ -84,7 +86,7 @@ endif
.PHONY: build
build: ## Build debug version greptime.
cargo ${CARGO_EXTENSION} build ${CARGO_BUILD_OPTS}
cargo ${CARGO_EXTENSION} build ${BUILD_PACKAGE_OPT} ${CARGO_BUILD_OPTS}
.PHONY: build-by-dev-builder
build-by-dev-builder: ## Build greptime by dev-builder.
@@ -93,6 +95,8 @@ build-by-dev-builder: ## Build greptime by dev-builder.
-v ${PWD}:/greptimedb -v ${CARGO_REGISTRY_CACHE}:/root/.cargo/registry -v ${CARGO_GIT_CACHE}:/root/.cargo/git \
-w /greptimedb ${IMAGE_REGISTRY}/${IMAGE_NAMESPACE}/dev-builder-${BASE_IMAGE}:${DEV_BUILDER_IMAGE_TAG} \
make build \
BUILD_PACKAGE="${BUILD_PACKAGE}" \
BUILD_BIN="${BUILD_BIN}" \
CARGO_EXTENSION="${CARGO_EXTENSION}" \
CARGO_PROFILE=${CARGO_PROFILE} \
FEATURES=${FEATURES} \
@@ -108,6 +112,8 @@ build-riscv64-bin: ## Build greptime binary for riscv64 (linux-gnu) by the riscv
-v ${PWD}:/greptimedb -v ${CARGO_REGISTRY_CACHE}:/root/.cargo/registry -v ${CARGO_GIT_CACHE}:/root/.cargo/git \
-w /greptimedb ${IMAGE_REGISTRY}/${IMAGE_NAMESPACE}/dev-builder-riscv64:${DEV_BUILDER_RISCV64_IMAGE_TAG} \
make build \
BUILD_PACKAGE="${BUILD_PACKAGE}" \
BUILD_BIN="${BUILD_BIN}" \
CARGO_EXTENSION="${CARGO_EXTENSION}" \
CARGO_PROFILE=${CARGO_PROFILE} \
FEATURES=${FEATURES} \
@@ -122,6 +128,8 @@ build-android-bin: ## Build greptime binary for android.
-v ${PWD}:/greptimedb -v ${CARGO_REGISTRY_CACHE}:/root/.cargo/registry -v ${CARGO_GIT_CACHE}:/root/.cargo/git \
-w /greptimedb ${IMAGE_REGISTRY}/${IMAGE_NAMESPACE}/dev-builder-android:${DEV_BUILDER_IMAGE_TAG} \
make build \
BUILD_PACKAGE="${BUILD_PACKAGE}" \
BUILD_BIN="${BUILD_BIN}" \
CARGO_EXTENSION="ndk --platform 23 -t aarch64-linux-android" \
CARGO_PROFILE=release \
FEATURES="${FEATURES}" \
@@ -278,6 +286,7 @@ dashboards: ## Generate the Grafana dashboards for standalone mode and intermedi
@./grafana/scripts/gen-dashboards.sh
##@ Docs
.PHONY: config-docs
config-docs: ## Generate configuration documentation from toml files.
docker run --rm \
-v ${PWD}:/greptimedb \
README.md
+61 -77
View File
@@ -13,95 +13,65 @@ One engine, on your infrastructure.</h2>
> Apache-2.0 licensed core.
<div align="center">
<h3 align="center">
<a href="https://docs.greptime.com/user-guide/overview/">User Guide</a> |
<a href="https://greptimedb.rs/">API Docs</a> |
<a href="https://github.com/GreptimeTeam/greptimedb/issues/7685">Roadmap 2026</a>
</h3>
<a href="https://github.com/GreptimeTeam/greptimedb/releases/latest">
<img src="https://img.shields.io/github/v/release/GreptimeTeam/greptimedb?filter=!*-*&label=stable&color=brightgreen" alt="Stable"/>
</a>
<a href="https://github.com/GreptimeTeam/greptimedb/releases">
<img src="https://img.shields.io/github/v/release/GreptimeTeam/greptimedb?include_prereleases&filter=!*-*-*&label=canary&color=blueviolet" alt="Canary"/>
</a>
<a href="https://github.com/GreptimeTeam/greptimedb/releases">
<img src="https://img.shields.io/github/v/release/GreptimeTeam/greptimedb?include_prereleases&filter=*-nightly-*&label=nightly&color=orange" alt="Nightly"/>
</a>
<a href="https://docs.greptime.com/user-guide/overview/">User Guide</a> &nbsp;·&nbsp;
<a href="https://greptimedb.rs/">API Docs</a> &nbsp;·&nbsp;
<a href="https://github.com/GreptimeTeam/greptimedb/issues/7685">Roadmap 2026</a> &nbsp;·&nbsp;
<a href="https://greptime.com/slack">Slack</a>
<a href="https://github.com/GreptimeTeam/greptimedb/releases/latest"><img src="https://img.shields.io/github/v/release/GreptimeTeam/greptimedb?filter=!*-*&sort=semver&label=stable&color=brightgreen" alt="Stable"/></a>
<a href="https://github.com/GreptimeTeam/greptimedb/releases"><img src="https://img.shields.io/github/v/release/GreptimeTeam/greptimedb?include_prereleases&filter=!*-*-*&sort=semver&label=canary&color=blueviolet" alt="Canary"/></a>
<a href="https://github.com/GreptimeTeam/greptimedb/releases"><img src="https://img.shields.io/github/v/release/GreptimeTeam/greptimedb?include_prereleases&filter=*-nightly-*&label=nightly&color=orange" alt="Nightly"/></a>
<a href="https://hub.docker.com/r/greptime/greptimedb/"><img src="https://img.shields.io/docker/pulls/greptime/greptimedb.svg" alt="Docker Pulls"/></a>
<a href="https://github.com/GreptimeTeam/greptimedb/blob/main/LICENSE"><img src="https://img.shields.io/github/license/GreptimeTeam/greptimedb" alt="License"/></a>
<sub><b>stable</b> for production &nbsp;·&nbsp; <b>canary</b> includes pre-releases &nbsp;·&nbsp; <b>nightly</b> is a weekly snapshot of <code>main</code></sub>
<a href="https://hub.docker.com/r/greptime/greptimedb/">
<img src="https://img.shields.io/docker/pulls/greptime/greptimedb.svg" alt="Docker Pulls"/>
</a>
<a href="https://github.com/GreptimeTeam/greptimedb/actions/workflows/integration.yml">
<img src="https://github.com/GreptimeTeam/greptimedb/actions/workflows/integration.yml/badge.svg" alt="GitHub Actions"/>
</a>
<a href="https://codecov.io/gh/GreptimeTeam/greptimedb">
<img src="https://codecov.io/gh/GreptimeTeam/greptimedb/branch/main/graph/badge.svg?token=FITFDI3J3C" alt="Codecov"/>
</a>
<a href="https://github.com/GreptimeTeam/greptimedb/blob/main/LICENSE">
<img src="https://img.shields.io/github/license/GreptimeTeam/greptimedb" alt="License"/>
</a>
<br/>
<a href="https://greptime.com/slack">
<img src="https://img.shields.io/badge/slack-GreptimeDB-0abd59?logo=slack&style=for-the-badge" alt="Slack"/>
</a>
<a href="https://twitter.com/greptime">
<img src="https://img.shields.io/badge/twitter-follow_us-1d9bf0.svg?style=for-the-badge" alt="Twitter"/>
</a>
<a href="https://www.linkedin.com/company/greptime/">
<img src="https://img.shields.io/badge/linkedin-connect_with_us-0a66c2.svg?style=for-the-badge" alt="LinkedIn"/>
</a>
</div>
- [Introduction](#introduction)
- [Why You Might Use It](#why-you-might-use-it)
- [Overview](#overview)
- [What's Supported](#whats-supported)
- [Compatibility and Migration](#compatibility-and-migration)
- [Limitations and Edition Boundary](#limitations-and-edition-boundary)
- [Architecture](#architecture)
- [Try GreptimeDB](#try-greptimedb)
- [Getting Started](#getting-started)
- [Build From Source](#build-from-source)
- [Tools & Extensions](#tools--extensions)
- [Project Status](#project-status)
- [Community](#community)
- [License](#license)
- [Commercial Support](#commercial-support)
- [Contributing](#contributing)
- [Acknowledgement](#acknowledgement)
## Introduction
**GreptimeDB** is an open-source observability database. Metrics, logs, and traces run on one columnar engine over object storage and share one [table model](https://docs.greptime.com/user-guide/concepts/data-model/): tags, timestamp, and fields. When signals carry common identifiers such as service, host, or trace ID, you can correlate them in SQL without moving data between databases.
Ingest through OpenTelemetry, Prometheus Remote Write, Loki Push, or Elasticsearch Bulk. Use SQL across observability data and PromQL for metrics. Migrate ingestion one signal at a time without rebuilding your collectors.
## One Query Across Signals
OpenTelemetry ingestion writes spans to `opentelemetry_traces` and log records to
`opentelemetry_logs`. Both tables carry `trace_id`, so correlating them is a join:
```sql
-- The slowest failed spans in the last hour,
-- with the log lines emitted inside those same traces.
SELECT
t.service_name,
t.span_name,
t.duration_nano / 1000000 AS duration_ms,
l.timestamp AS log_time,
l.severity_text,
l.body
FROM opentelemetry_traces t
JOIN opentelemetry_logs l ON l.trace_id = t.trace_id
WHERE t.timestamp > now() - INTERVAL '1' HOUR
AND t.span_status_code = 'STATUS_CODE_ERROR'
ORDER BY t.duration_nano DESC
LIMIT 20;
```
Metrics join the same way, on any tag the tables share, such as `service`,
`host`, or `pod`.
## Why You Might Use It
- You run Prometheus plus Loki or Elasticsearch and want one backend instead of three
- You have outgrown Prometheus on cardinality or retention and don't want the Thanos/Mimir operational surface
- You are hitting Loki's query performance limits as log volume grows
- You need long retention on object storage without a separate analytics stack
- You want to query telemetry with SQL, not only a domain query language
- You are storing GenAI or agent telemetry ([OTel GenAI conventions](https://opentelemetry.io/docs/specs/semconv/gen-ai/)) alongside infrastructure signals
- You need the same engine and semantics on resource-constrained devices
Learn more in [Why GreptimeDB](https://docs.greptime.com/user-guide/concepts/why-greptimedb).
## Overview
A quick overview of what GreptimeDB ingests, how it connects to other systems, and what its distributed engine lets you do.
<p align="center">
<a href="https://github.com/GreptimeTeam/greptimedb/raw/main/docs/overview.png" target="_blank" rel="noopener">
<img alt="GreptimeDB Overview" src="docs/overview.png" width="900px">
</a>
</p>
## What's Supported
| | |
@@ -113,6 +83,22 @@ A quick overview of what GreptimeDB ingests, how it connects to other systems, a
Compute and storage are disaggregated: object storage holds the data, while memory and local-disk caches keep recent and frequently queried data close to compute.
<p align="center">
<a href="https://github.com/GreptimeTeam/greptimedb/raw/main/docs/overview.png" target="_blank" rel="noopener">
<img alt="GreptimeDB Overview" src="docs/overview.png" width="900px">
</a>
</p>
## Benchmarks
* [Agent RCA Bench](https://rca-bench.greptime.com/): LLM agents doing root cause analysis
over GreptimeDB versus Prometheus + Loki + Tempo. 40% fewer wrong diagnoses, 48% fewer
input tokens, 45% lower cost
([write-up](https://greptime.com/blogs/2026-09-08-agent-rca-bench-interface-semantic-layer))
* [GreptimeDB tops JSONBench's billion-record cold run test](https://greptime.com/blogs/2025-03-18-jsonbench-greptimedb-performance)
* [TSBS Benchmark](https://github.com/GreptimeTeam/greptimedb/tree/main/docs/benchmarks/tsbs)
* [More benchmark reports](https://docs.greptime.com/user-guide/concepts/features-that-you-concern#how-is-greptimedbs-performance-compared-to-other-solutions)
## Compatibility and Migration
Compatibility is per protocol, and query-side coverage is narrower than ingestion.
@@ -123,11 +109,6 @@ Compatibility is per protocol, and query-side coverage is narrower than ingestio
| **Loki** | Push ingestion; dual-write through Grafana Alloy makes the [cutover gradual](https://greptime.com/blogs/2026-07-23-from-loki-to-greptimedb-dual-write-migration) | LogQL and the rest of the Loki query API |
| **Elasticsearch** | `_bulk` ingestion in the open-source core; [QueryDSL](https://docs.greptime.com/enterprise/elasticsearch-compatible/query/) partially, in Enterprise | Most other Elasticsearch APIs |
**Benchmarks:**
* [GreptimeDB tops JSONBench's billion-record cold run test](https://greptime.com/blogs/2025-03-18-jsonbench-greptimedb-performance)
* [TSBS Benchmark](https://github.com/GreptimeTeam/greptimedb/tree/main/docs/benchmarks/tsbs)
* [More benchmark reports](https://docs.greptime.com/user-guide/concepts/features-that-you-concern#how-is-greptimedbs-performance-compared-to-other-solutions)
## Limitations and Edition Boundary
Cluster deployment, object storage, the Flow engine, and every ingestion protocol listed above are in the Apache-2.0 build. Repartitioning, region migration, and index creation are manual operations there.
@@ -221,11 +202,7 @@ See the [Contribution Guidelines](CONTRIBUTING.md) for the full developer workfl
GreptimeDB is generally available, with stable APIs and regular releases. It runs in production at scale — [OceanBase Cloud](https://greptime.com/blogs/2025-07-22-user-case-obcloud-log-management-greptimedb) operates 80+ GreptimeDB clusters managing 300 TB of logs, cutting log storage cost by 60%+ after migrating from Grafana Loki. See more in [case studies](https://greptime.com/blogs/?category=Use%20Case).
Read the [v1.0 highlights](https://greptime.com/blogs/2025-11-05-greptimedb-v1-highlights) and [2026 roadmap](https://greptime.com/blogs/2026-02-11-greptimedb-roadmap-2026), or browse the [version reference](https://docs.greptime.com/nightly/reference/about-greptimedb-version).
If GreptimeDB is useful to you, please star the repo.
<img alt="Known Users" src="https://greptime.com/logo/img/users.png"/>
Release lines and support windows are in the [version reference](https://docs.greptime.com/nightly/reference/about-greptimedb-version). For where the project is going, read the [v1.0 highlights](https://greptime.com/blogs/2025-11-05-greptimedb-v1-highlights) and the [2026 roadmap](https://greptime.com/blogs/2026-02-11-greptimedb-roadmap-2026).
## Community
@@ -239,6 +216,10 @@ We invite you to engage and contribute!
- [X (Twitter)](https://X.com/greptime)
- [YouTube](https://www.youtube.com/@greptime)
If GreptimeDB is useful to you, please star the repo.
<img alt="Known Users" src="https://greptime.com/logo/img/users.png"/>
## License
GreptimeDB is an open-core project. Its core is licensed under the
@@ -262,6 +243,9 @@ security, and support layer for production deployments.
- Explore [Internal Concepts](https://docs.greptime.com/contributor-guide/overview.html) and [DeepWiki](https://deepwiki.com/GreptimeTeam/greptimedb).
- Pick up a [good first issue](https://github.com/GreptimeTeam/greptimedb/issues?q=is%3Aissue+is%3Aopen+label%3A%22good+first+issue%22) and join the #contributors [Slack](https://greptime.com/slack) channel.
[![Integration CI](https://github.com/GreptimeTeam/greptimedb/actions/workflows/integration.yml/badge.svg)](https://github.com/GreptimeTeam/greptimedb/actions/workflows/integration.yml)
[![Codecov](https://codecov.io/gh/GreptimeTeam/greptimedb/branch/main/graph/badge.svg?token=FITFDI3J3C)](https://codecov.io/gh/GreptimeTeam/greptimedb)
## Acknowledgement
Special thanks to all contributors! See [AUTHOR.md](AUTHOR.md).
config/config.md
+2 -2
View File
@@ -249,7 +249,7 @@
| `pipeline.cache_ttl` | String | `10s` | Time to live of the local pipeline cache. Default is `10s`. |
| `event_recorder` | -- | -- | Configuration options for the event recorder. |
| `event_recorder.ttl` | String | `90d` | TTL for the events table that will be used to store the events. Default is `90d`. |
| `event_recorder.event_types` | Array | -- | Event types to record. Current available event types: `create_database`,<br/>`alter_database`, `drop_database`, `create_flow`, `drop_flow`,<br/>`create_table`, `create_logical_tables`, `alter_table`, `alter_logical_tables`,<br/>`drop_table`, `undrop_table`, `purge_dropped_table`, `truncate_table`,<br/>`create_view`, `drop_view`, `admin_function`, `reconcile_table`,<br/>`reconcile_logical_tables`.<br/>When omitted, all current and future event types are recorded.<br/>Set to an empty array to disable event recording. |
| `event_recorder.event_types` | Array | -- | Event types to record. Current available event types: `create_database`,<br/>`alter_database`, `drop_database`, `create_flow`, `drop_flow`,<br/>`create_table`, `create_logical_tables`, `alter_table`, `alter_logical_tables`,<br/>`drop_table`, `undrop_table`, `purge_dropped_table`, `truncate_table`,<br/>`create_view`, `drop_view`, `admin_function`, `reconcile_catalog`,<br/>`reconcile_database`, `reconcile_table`, `reconcile_logical_tables`.<br/>When omitted, all current and future event types are recorded.<br/>Set to an empty array to disable event recording. |
| `memory` | -- | -- | The memory options. |
| `memory.enable_heap_profiling` | Bool | `true` | Whether to enable heap profiling activation during startup.<br/>When enabled, heap profiling will be activated if the `MALLOC_CONF` environment variable<br/>is set to "prof:true,prof_active:false". The official image adds this env variable.<br/>Default is true. |
@@ -477,7 +477,7 @@
| `wal.create_topic_timeout` | String | `30s` | The timeout for creating a Kafka topic.<br/>**It's only used when the provider is `kafka`**. |
| `event_recorder` | -- | -- | Configuration options for the event recorder. |
| `event_recorder.ttl` | String | `90d` | TTL for the events table that will be used to store the events. Default is `90d`. |
| `event_recorder.event_types` | Array | -- | Event types to record. Current available event types: `region_migration`,<br/>`create_database`, `alter_database`, `drop_database`, `create_flow`,<br/>`drop_flow`, `create_table`, `create_logical_tables`, `alter_table`,<br/>`alter_logical_tables`, `drop_table`, `undrop_table`, `purge_dropped_table`,<br/>`truncate_table`, `create_view`, `drop_view`, `repartition`,<br/>`repartition_group`, `wal_prune`, `batch_gc`, `reconcile_table`,<br/>`reconcile_logical_tables`.<br/>When omitted, all current and future event types are recorded.<br/>Set to an empty array to disable event recording. |
| `event_recorder.event_types` | Array | -- | Event types to record. Current available event types: `region_migration`,<br/>`create_database`, `alter_database`, `drop_database`, `create_flow`,<br/>`drop_flow`, `create_table`, `create_logical_tables`, `alter_table`,<br/>`alter_logical_tables`, `drop_table`, `undrop_table`, `purge_dropped_table`,<br/>`truncate_table`, `create_view`, `drop_view`, `repartition`,<br/>`repartition_group`, `wal_prune`, `batch_gc`, `reconcile_catalog`,<br/>`reconcile_database`, `reconcile_table`, `reconcile_logical_tables`.<br/>When omitted, all current and future event types are recorded.<br/>Set to an empty array to disable event recording. |
| `stats_persistence` | -- | -- | Configuration options for the stats persistence. |
| `stats_persistence.ttl` | String | `0s` | TTL for the stats table that will be used to store the stats.<br/>Set to `0s` to disable stats persistence.<br/>Default is `0s`.<br/>If you want to enable stats persistence, set the TTL to a value greater than 0.<br/>It is recommended to set a small value, e.g., `3h`. |
| `stats_persistence.interval` | String | `10m` | The interval to persist the stats. Default is `10m`.<br/>The minimum value is `10m`, if the value is less than `10m`, it will be overridden to `10m`. |
config/metasrv.example.toml
+2 -2
View File
@@ -328,8 +328,8 @@ ttl = "90d"
## `drop_flow`, `create_table`, `create_logical_tables`, `alter_table`,
## `alter_logical_tables`, `drop_table`, `undrop_table`, `purge_dropped_table`,
## `truncate_table`, `create_view`, `drop_view`, `repartition`,
## `repartition_group`, `wal_prune`, `batch_gc`, `reconcile_table`,
## `reconcile_logical_tables`.
## `repartition_group`, `wal_prune`, `batch_gc`, `reconcile_catalog`,
## `reconcile_database`, `reconcile_table`, `reconcile_logical_tables`.
## When omitted, all current and future event types are recorded.
## Set to an empty array to disable event recording.
#+ event_types = ["region_migration"]
config/standalone.example.toml
+2 -2
View File
@@ -973,8 +973,8 @@ ttl = "90d"
## `alter_database`, `drop_database`, `create_flow`, `drop_flow`,
## `create_table`, `create_logical_tables`, `alter_table`, `alter_logical_tables`,
## `drop_table`, `undrop_table`, `purge_dropped_table`, `truncate_table`,
## `create_view`, `drop_view`, `admin_function`, `reconcile_table`,
## `reconcile_logical_tables`.
## `create_view`, `drop_view`, `admin_function`, `reconcile_catalog`,
## `reconcile_database`, `reconcile_table`, `reconcile_logical_tables`.
## When omitted, all current and future event types are recorded.
## Set to an empty array to disable event recording.
#+ event_types = ["create_database"]
src/catalog/src/system_schema/information_schema/columns.rs
+4 -13
View File
@@ -31,7 +31,7 @@ use datatypes::scalars::ScalarVectorBuilder;
use datatypes::schema::{ColumnSchema, Schema, SchemaRef};
use datatypes::value::Value;
use datatypes::vectors::{
ConstantVector, Int64Vector, Int64VectorBuilder, StringVector, StringVectorBuilder, VectorRef,
Int64Vector, Int64VectorBuilder, StringVector, StringVectorBuilder, VectorRef,
};
use futures::TryStreamExt;
use snafu::{OptionExt, ResultExt};
@@ -408,18 +408,9 @@ impl InformationSchemaColumnsBuilder {
fn finish(&mut self) -> Result<RecordBatch> {
let rows_num = self.collation_names.len();
let privileges = Arc::new(ConstantVector::new(
Arc::new(StringVector::from(vec![DEFAULT_PRIVILEGES])),
rows_num,
));
let empty_string = Arc::new(ConstantVector::new(
Arc::new(StringVector::from(vec![EMPTY_STR])),
rows_num,
));
let srs_ids = Arc::new(ConstantVector::new(
Arc::new(Int64Vector::from(vec![None])),
rows_num,
));
let privileges = Arc::new(StringVector::from(vec![DEFAULT_PRIVILEGES; rows_num]));
let empty_string = Arc::new(StringVector::from(vec![EMPTY_STR; rows_num]));
let srs_ids = Arc::new(Int64Vector::from(vec![None; rows_num]));
let columns: Vec<VectorRef> = vec![
Arc::new(self.catalog_names.finish()),
src/catalog/src/system_schema/information_schema/key_column_usage.rs
+2 -5
View File
@@ -26,7 +26,7 @@ use datafusion::physical_plan::streaming::PartitionStream as DfPartitionStream;
use datatypes::prelude::{ConcreteDataType, MutableVector, ScalarVectorBuilder, VectorRef};
use datatypes::schema::{ColumnSchema, Schema, SchemaRef};
use datatypes::value::Value;
use datatypes::vectors::{ConstantVector, StringVector, StringVectorBuilder, UInt32VectorBuilder};
use datatypes::vectors::{StringVector, StringVectorBuilder, UInt32VectorBuilder};
use futures_util::TryStreamExt;
use snafu::{OptionExt, ResultExt};
use store_api::storage::{ScanRequest, TableId};
@@ -326,10 +326,7 @@ impl InformationSchemaKeyColumnUsageBuilder {
fn finish(&mut self) -> Result<RecordBatch> {
let rows_num = self.table_catalog.len();
let null_string_vector = Arc::new(ConstantVector::new(
Arc::new(StringVector::from(vec![None as Option<&str>])),
rows_num,
));
let null_string_vector = Arc::new(StringVector::from(vec![None as Option<&str>; rows_num]));
let columns: Vec<VectorRef> = vec![
Arc::new(self.constraint_catalog.finish()),
Arc::new(self.constraint_schema.finish()),
src/catalog/src/system_schema/information_schema/partitions.rs
+7 -18
View File
@@ -29,8 +29,8 @@ use datatypes::schema::{ColumnSchema, Schema, SchemaRef};
use datatypes::timestamp::TimestampSecond;
use datatypes::value::Value;
use datatypes::vectors::{
ConstantVector, Int64Vector, Int64VectorBuilder, MutableVector, StringVector,
StringVectorBuilder, TimestampSecondVector, TimestampSecondVectorBuilder, UInt64VectorBuilder,
Int64Vector, Int64VectorBuilder, MutableVector, StringVector, StringVectorBuilder,
TimestampSecondVector, TimestampSecondVectorBuilder, UInt64VectorBuilder,
};
use futures::{StreamExt, TryStreamExt};
use partition::manager::PartitionInfo;
@@ -361,22 +361,11 @@ impl InformationSchemaPartitionsBuilder {
fn finish(&mut self) -> Result<RecordBatch> {
let rows_num = self.catalog_names.len();
let null_string_vector = Arc::new(ConstantVector::new(
Arc::new(StringVector::from(vec![None as Option<&str>])),
rows_num,
));
let null_i64_vector = Arc::new(ConstantVector::new(
Arc::new(Int64Vector::from(vec![None])),
rows_num,
));
let null_timestamp_second_vector = Arc::new(ConstantVector::new(
Arc::new(TimestampSecondVector::from(vec![None])),
rows_num,
));
let partition_methods = Arc::new(ConstantVector::new(
Arc::new(StringVector::from(vec![Some("RANGE")])),
rows_num,
));
let null_string_vector = Arc::new(StringVector::from(vec![None as Option<&str>; rows_num]));
let null_i64_vector = Arc::new(Int64Vector::from(vec![None; rows_num]));
let null_timestamp_second_vector =
Arc::new(TimestampSecondVector::from(vec![None; rows_num]));
let partition_methods = Arc::new(StringVector::from(vec![Some("RANGE"); rows_num]));
let columns: Vec<VectorRef> = vec![
Arc::new(self.catalog_names.finish()),
src/catalog/src/system_schema/information_schema/table_constraints.rs
+3 -9
View File
@@ -27,7 +27,7 @@ use datatypes::prelude::{ConcreteDataType, MutableVector};
use datatypes::scalars::ScalarVectorBuilder;
use datatypes::schema::{ColumnSchema, Schema, SchemaRef};
use datatypes::value::Value;
use datatypes::vectors::{ConstantVector, StringVector, StringVectorBuilder, VectorRef};
use datatypes::vectors::{StringVector, StringVectorBuilder, VectorRef};
use futures::TryStreamExt;
use snafu::{OptionExt, ResultExt};
use store_api::storage::{ScanRequest, TableId};
@@ -242,14 +242,8 @@ impl InformationSchemaTableConstraintsBuilder {
fn finish(&mut self) -> Result<RecordBatch> {
let rows_num = self.constraint_names.len();
let constraint_catalogs = Arc::new(ConstantVector::new(
Arc::new(StringVector::from(vec!["def"])),
rows_num,
));
let enforceds = Arc::new(ConstantVector::new(
Arc::new(StringVector::from(vec!["YES"])),
rows_num,
));
let constraint_catalogs = Arc::new(StringVector::from(vec!["def"; rows_num]));
let enforceds = Arc::new(StringVector::from(vec!["YES"; rows_num]));
let columns: Vec<VectorRef> = vec![
constraint_catalogs,
src/client/src/database.rs
+10
View File
@@ -825,7 +825,17 @@ impl Database {
/// Ingest a stream of [RecordBatch]es that belong to a table, using Arrow Flight's "`DoPut`"
/// method. The return value is also a stream, produces [DoPutResponse]s.
pub async fn do_put(&self, stream: FlightDataStream) -> Result<DoPutResponseStream> {
self.do_put_with_hints(stream, &[]).await
}
/// Ingest a stream of [RecordBatch]es using Arrow Flight's `DoPut` with request hints.
pub async fn do_put_with_hints(
&self,
stream: FlightDataStream,
hints: &[(&str, &str)],
) -> Result<DoPutResponseStream> {
let mut request = tonic::Request::new(stream);
Self::put_hints(request.metadata_mut(), hints)?;
if let Some(AuthHeader {
auth_scheme: Some(AuthScheme::Basic(Basic { username, password })),
src/cmd/Cargo.toml
+2 -2
View File
@@ -119,12 +119,12 @@ tracing-appender.workspace = true
[target.'cfg(unix)'.dependencies]
daemonize = "0.5"
pprof = { version = "0.14", features = [
pprof = { version = "0.15", features = [
"flamegraph",
] }
[target.'cfg(not(windows))'.dependencies]
tikv-jemallocator = "0.6"
tikv-jemallocator = "0.7"
[dev-dependencies]
api.workspace = true
src/cmd/src/bin/query_perf_fixture/case.rs
+62
View File
@@ -105,6 +105,10 @@ pub(super) struct PromRemoteWritePlan {
#[serde(default = "default_visibility_timeout_seconds")]
pub(super) visibility_timeout_seconds: u64,
#[serde(default)]
pub(super) base_setup_sql: Vec<String>,
#[serde(default)]
pub(super) candidate_setup_sql: Vec<String>,
#[serde(default)]
pub(super) prom_store: PromStoreConfig,
#[serde(default)]
pub(super) value: ValueConfig,
@@ -200,6 +204,7 @@ pub(super) enum ValuePattern {
QuantizedSignal,
SignalWithSporadicStalls,
MixedSignalRepeated,
BoundedMixed,
}
impl std::fmt::Display for ValuePattern {
@@ -436,6 +441,63 @@ pub(super) fn default_parallelism() -> u64 {
1
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn remote_write_setup_sql_defaults_to_empty() {
let case: CaseFile = toml::from_str(
r#"
[scenario]
kind = "prom_remote_write_then_query"
[scenario.remote_write]
metric = "metric"
"#,
)
.unwrap();
let Scenario::PromRemoteWriteThenQuery(scenario) = case.scenario else {
panic!("expected prom_remote_write_then_query scenario");
};
assert!(scenario.remote_write.base_setup_sql.is_empty());
assert!(scenario.remote_write.candidate_setup_sql.is_empty());
}
#[test]
fn remote_write_setup_sql_roundtrips() {
let case: CaseFile = toml::from_str(
r#"
[scenario]
kind = "prom_remote_write_then_query"
[scenario.remote_write]
metric = "metric"
base_setup_sql = ["CREATE TABLE base_table"]
candidate_setup_sql = ["CREATE TABLE candidate_table", "ALTER TABLE candidate_table SET 'x'='y'"]
"#,
)
.unwrap();
let roundtrip: CaseFile =
serde_json::from_str(&serde_json::to_string(&case).unwrap()).unwrap();
let Scenario::PromRemoteWriteThenQuery(scenario) = roundtrip.scenario else {
panic!("expected prom_remote_write_then_query scenario");
};
assert_eq!(
scenario.remote_write.base_setup_sql,
["CREATE TABLE base_table"]
);
assert_eq!(
scenario.remote_write.candidate_setup_sql,
[
"CREATE TABLE candidate_table",
"ALTER TABLE candidate_table SET 'x'='y'"
]
);
}
}
impl Scenario {
pub(super) fn kind(&self) -> &'static str {
match self {
src/cmd/src/bin/query_perf_fixture/direct_sst.rs
+1
View File
@@ -469,6 +469,7 @@ pub(super) async fn run_direct_sst(args: DirectArgs) {
write_buffer_size: DEFAULT_WRITE_BUFFER_SIZE,
row_group_size: scenario.layout.row_group_size,
max_file_size: None,
float_field_encoding: Default::default(),
};
let infos = match format {
FormatType::Flat => writer.write_all_flat(source, Some(sequence), &opts).await,
src/cmd/src/bin/query_perf_fixture/prom_remote_write.rs
+140 -1
View File
@@ -197,8 +197,35 @@ fn deterministic_prom_value(args: &PromRemoteWriteArgs, series_idx: u64, sample_
args.value_base + (series_idx % 97) as f64 + local as f64 * args.value_step
}
}
ValuePattern::BoundedMixed => bounded_mixed_value(args, series_idx, local),
}
}
fn bounded_mixed_value(args: &PromRemoteWriteArgs, series_idx: u64, local: u64) -> f64 {
const SPANS: [i64; 4] = [10, 1_000, 100_000, 10_000_000];
let series = splitmix64(series_idx ^ args.value_seed);
let span = SPANS[(series >> 62) as usize] * (1 + ((series >> 32) % 10) as i64);
let baseline = (series % args.value_cardinality.max(1)) as f64 * span as f64 / 10.0;
let run_length = args.value_run_length.max(1);
let group = local / run_length;
let phase = local % run_length;
let anchor = bounded_mixed_anchor(series, group, span);
let next_anchor = bounded_mixed_anchor(series, group + 1, span);
let signal = baseline + anchor + (next_anchor - anchor) * phase as f64 / run_length as f64;
let value = (args.value_base + args.value_step * signal).round();
if series_idx % args.value_mixed_every.max(1) == args.value_mixed_every.max(1) - 1 {
let numerator = splitmix64(series ^ local) % 999 + 1;
value + numerator as f64 / 1_000.0
} else {
value
}
}
fn bounded_mixed_anchor(series: u64, group: u64, span: i64) -> f64 {
let limit = span - 1;
(splitmix64(series ^ group) % (2 * limit as u64 + 1)) as i64 as f64 - limit as f64
}
fn splitmix64(mut value: u64) -> u64 {
value = value.wrapping_add(0x9e3779b97f4a7c15);
let mut mixed = value;
@@ -307,7 +334,7 @@ mod tests {
for chunk_idx in 0..10 {
let mut chunk = args(ValuePattern::SeededRandom);
chunk.samples_per_series = 1_440;
chunk.value_sample_offset = chunk_idx * 1_440;
chunk.value_sample_offset = chunk_idx * 1_439;
chunk.value_total_samples_per_series = Some(14_400);
for sample_idx in 0..chunk.samples_per_series {
assert_eq!(
@@ -319,6 +346,118 @@ mod tests {
}
}
#[test]
fn bounded_mixed_has_bounded_95_5_per_series_values() {
let mut args = args(ValuePattern::BoundedMixed);
args.series_count = 1_000;
args.samples_per_series = 4_320;
args.value_cardinality = 1_000;
args.value_seed = 42;
args.value_run_length = 60;
args.value_mixed_every = 20;
let mut integral_series = 0;
let mut fractional_series = 0;
let mut spans = HashSet::new();
let mut ranges = HashSet::new();
for series_idx in 0..args.series_count {
let series = splitmix64(series_idx ^ args.value_seed);
let span = [10_i64, 1_000, 100_000, 10_000_000][(series >> 62) as usize]
* (1 + ((series >> 32) % 10) as i64);
let baseline = (series % args.value_cardinality.max(1)) as f64 * span as f64 / 10.0;
let values = (0..args.samples_per_series)
.map(|sample_idx| deterministic_prom_value(&args, series_idx, sample_idx))
.collect::<Vec<_>>();
assert!(values.iter().all(|value| value.is_finite()));
assert!(
values
.iter()
.all(|value| *value >= baseline - span as f64 - 1.0
&& *value <= baseline + span as f64 + 1.0)
);
assert!(
values.windows(2).any(|pair| pair[1] > pair[0]),
"series {series_idx} never rises"
);
assert!(
values.windows(2).any(|pair| pair[1] < pair[0]),
"series {series_idx} never falls"
);
ranges.insert((
values
.iter()
.copied()
.fold(f64::INFINITY, f64::min)
.to_bits(),
values
.iter()
.copied()
.fold(f64::NEG_INFINITY, f64::max)
.to_bits(),
));
spans.insert(span);
if series_idx % args.value_mixed_every == args.value_mixed_every - 1 {
fractional_series += 1;
assert!(values.iter().all(|value| value.fract() != 0.0));
} else {
integral_series += 1;
assert!(values.iter().all(|value| value.fract() == 0.0));
}
}
assert_eq!((integral_series, fractional_series), (950, 50));
assert!(spans.len() > 4);
assert!(ranges.len() > 1);
}
#[test]
fn bounded_mixed_is_seeded_and_chunk_bit_identical() {
let mut monolithic = args(ValuePattern::BoundedMixed);
monolithic.samples_per_series = 14_400;
monolithic.value_total_samples_per_series = Some(14_400);
monolithic.value_cardinality = 1_000;
monolithic.value_seed = 42;
monolithic.value_run_length = 60;
monolithic.value_mixed_every = 20;
let original = (0..600)
.map(|sample_idx| deterministic_prom_value(&monolithic, 19, sample_idx))
.collect::<Vec<_>>();
monolithic.value_seed = 43;
assert_ne!(
original,
(0..600)
.map(|sample_idx| deterministic_prom_value(&monolithic, 19, sample_idx))
.collect::<Vec<_>>()
);
monolithic.value_seed = 42;
for series_idx in [0, 19] {
for chunk_idx in 0..10 {
let mut chunk = args(ValuePattern::BoundedMixed);
chunk.samples_per_series = 1_440;
chunk.value_sample_offset = chunk_idx * 1_439;
chunk.value_total_samples_per_series = Some(14_400);
chunk.value_cardinality = 1_000;
chunk.value_seed = 42;
chunk.value_run_length = 60;
chunk.value_mixed_every = 20;
for sample_idx in 0..chunk.samples_per_series {
assert_eq!(
deterministic_prom_value(&chunk, series_idx, sample_idx).to_bits(),
deterministic_prom_value(
&monolithic,
series_idx,
chunk.value_sample_offset + sample_idx
)
.to_bits()
);
}
}
}
}
#[test]
fn values_continue_across_series_boundaries() {
let mut args = args(ValuePattern::Unique);
src/cmd/src/bin/query_regression_runner/model.rs
+4
View File
@@ -116,6 +116,10 @@ pub(super) struct RemoteWrite {
pub(super) sample_chunk_size: Option<u64>,
pub(super) flush_every_sample_chunks: u64,
pub(super) visibility_timeout_seconds: u64,
#[serde(default)]
pub(super) base_setup_sql: Vec<String>,
#[serde(default)]
pub(super) candidate_setup_sql: Vec<String>,
pub(super) prom_store: PromStore,
pub(super) value: RemoteValue,
pub(super) storage: Option<StorageConfig>,
src/cmd/src/bin/query_regression_runner/remote/ingest.rs
+130
View File
@@ -57,6 +57,7 @@ pub(super) async fn run_prepare_remote(args: PrepareRemoteArgs) -> Result<()> {
args.base_http_port,
&args.fixture_generator,
&remote,
&remote.base_setup_sql,
&client,
)
.await?;
@@ -65,6 +66,7 @@ pub(super) async fn run_prepare_remote(args: PrepareRemoteArgs) -> Result<()> {
args.candidate_http_port,
&args.fixture_generator,
&remote,
&remote.candidate_setup_sql,
&client,
)
.await?;
@@ -88,6 +90,7 @@ async fn prepare_remote_target(
port: u16,
generator: &Path,
remote: &RemoteWrite,
setup_sql: &[String],
client: &Client,
) -> Result<Value> {
let create_database = http_post_sql(
@@ -107,6 +110,7 @@ async fn prepare_remote_target(
)
.into());
}
let setup = run_setup_sql(client, port, &remote.database, name, setup_sql).await?;
let (remote_write, flushes) = ingest_remote_write(generator, port, remote, client).await?;
let expected_rows = remote
.series_count
@@ -124,6 +128,7 @@ async fn prepare_remote_target(
Ok(json!({
"name": name,
"create_database": create_database,
"setup_sql": setup,
"remote_write": remote_write,
"flushes": flushes,
"visibility": visibility,
@@ -131,6 +136,28 @@ async fn prepare_remote_target(
}))
}
async fn run_setup_sql(
client: &Client,
port: u16,
database: &str,
target: &str,
setup_sql: &[String],
) -> Result<Vec<Value>> {
let mut results = Vec::with_capacity(setup_sql.len());
for (index, statement) in setup_sql.iter().enumerate() {
let result = http_post_sql(client, port, statement, database).await;
if !result["ok"].as_bool().unwrap_or(false) {
return Err(format!(
"setup SQL statement {} failed for {target}: {result}",
index + 1
)
.into());
}
results.push(result);
}
Ok(results)
}
#[derive(Clone, Debug, PartialEq, Eq)]
struct SampleChunk {
index: u64,
@@ -434,9 +461,110 @@ fn json_number(value: f64) -> String {
#[cfg(test)]
mod tests {
use tokio::io::{AsyncReadExt, AsyncWriteExt};
use tokio::net::{TcpListener, TcpStream};
use super::*;
use crate::query_regression_runner::model::{PromStore, RemoteValue};
async fn read_http_request(stream: &mut TcpStream) -> Vec<u8> {
let mut request = Vec::new();
let mut chunk = [0; 1024];
loop {
let read = stream.read(&mut chunk).await.unwrap();
assert_ne!(
read, 0,
"HTTP client closed request before sending its body"
);
request.extend_from_slice(&chunk[..read]);
let Some(header_end) = request.windows(4).position(|window| window == b"\r\n\r\n")
else {
continue;
};
let headers = std::str::from_utf8(&request[..header_end]).unwrap();
let content_length = headers
.lines()
.find_map(|line| line.strip_prefix("content-length: "))
.or_else(|| {
headers
.lines()
.find_map(|line| line.strip_prefix("Content-Length: "))
})
.unwrap()
.parse::<usize>()
.unwrap();
if request.len() >= header_end + 4 + content_length {
return request;
}
}
}
async fn spawn_sql_server(
responses: Vec<String>,
) -> (u16, tokio::task::JoinHandle<Vec<String>>) {
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
let port = listener.local_addr().unwrap().port();
let server = tokio::spawn(async move {
let mut requests = Vec::with_capacity(responses.len());
for body in responses {
let (mut stream, _) = listener.accept().await.unwrap();
let request = read_http_request(&mut stream).await;
requests.push(String::from_utf8(request).unwrap());
let response = format!(
"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\n\
Connection: close\r\nContent-Length: {}\r\n\r\n{body}",
body.len()
);
stream.write_all(response.as_bytes()).await.unwrap();
}
requests
});
(port, server)
}
#[tokio::test]
async fn setup_sql_runs_in_order_with_configured_database() {
let (port, server) = spawn_sql_server(vec!["{\"code\":0}".to_string(); 2]).await;
let client = Client::new();
let setup = vec![
"CREATE TABLE physical (ts TIMESTAMP TIME INDEX) ENGINE=metric".to_string(),
"ALTER TABLE physical SET 'compaction.twcs.trigger_file_num'='100'".to_string(),
];
let results = run_setup_sql(&client, port, "perf", "candidate", &setup)
.await
.unwrap();
assert_eq!(results.len(), 2);
let requests = server.await.unwrap();
assert_eq!(requests.len(), 2);
assert!(requests[0].contains("db=perf"));
assert!(requests[0].contains("sql=CREATE"));
assert!(requests[1].contains("db=perf"));
assert!(requests[1].contains("sql=ALTER"));
}
#[tokio::test]
async fn setup_sql_stops_at_the_failing_statement() {
let (port, server) =
spawn_sql_server(vec!["{\"code\":0}".to_string(), "{\"code\":1}".to_string()]).await;
let client = Client::new();
let setup = vec![
"CREATE TABLE physical (ts TIMESTAMP TIME INDEX) ENGINE=metric".to_string(),
"invalid SQL".to_string(),
"ALTER TABLE physical SET 'compaction.twcs.trigger_file_num'='100'".to_string(),
];
let error = run_setup_sql(&client, port, "perf", "base", &setup)
.await
.unwrap_err();
assert!(error.to_string().contains("statement 2 failed for base"));
let requests = server.await.unwrap();
assert_eq!(requests.len(), 2);
assert!(requests[1].contains("sql=invalid+SQL"));
}
#[test]
fn schedules_remote_sample_chunks_and_flushes() {
let remote = RemoteWrite {
@@ -452,6 +580,8 @@ mod tests {
sample_chunk_size: Some(2),
flush_every_sample_chunks: 2,
visibility_timeout_seconds: 30,
base_setup_sql: Vec::new(),
candidate_setup_sql: Vec::new(),
prom_store: PromStore {
pending_rows_flush_interval: "1s".to_string(),
max_batch_rows: 1,
src/cmd/src/bin/query_regression_runner/remote/read_bench.rs
+64 -13
View File
@@ -306,13 +306,25 @@ fn run_bench_command(command: Vec<String>, mut run: Value) -> Result<Value> {
}
fn parse_average_duration(stdout: &str) -> Option<f64> {
Regex::new(r"(?i)Average duration[^0-9]*([0-9.]+)\s*ms")
.ok()?
.captures(stdout)?
.get(1)?
.as_str()
.parse()
.ok()
let captures = Regex::new(
r"(?i)Average:\s+(?:\x1b\[[0-9;]*m)*\d+(?:\x1b\[[0-9;]*m)*\s+rows(?:,\s+(?:\x1b\[[0-9;]*m)*\d+(?:\x1b\[[0-9;]*m)*\s+record batches)?\s+in\s+([0-9]+(?:\.[0-9]+)?)\s*(ns|µs|us|ms|s)\s+over\s+\d+\s+iterations|Average duration[^0-9]*([0-9]+(?:\.[0-9]+)?)\s*ms",
)
.ok()?
.captures(stdout)?;
let (value, unit) = match (captures.get(1), captures.get(2), captures.get(3)) {
(Some(value), Some(unit), _) => (value.as_str(), unit.as_str()),
(_, _, Some(value)) => (value.as_str(), "ms"),
_ => return None,
};
let value = value.parse::<f64>().ok()?;
let milliseconds = match unit.to_ascii_lowercase().as_str() {
"ns" => value / 1_000_000.0,
"µs" | "us" => value / 1_000.0,
"ms" => value,
"s" => value * 1_000.0,
_ => return None,
};
milliseconds.is_finite().then_some(milliseconds)
}
fn bench_median(runs: &[Value]) -> Option<f64> {
@@ -328,12 +340,51 @@ mod tests {
use super::*;
#[test]
fn parses_average_duration_and_groups_scan_paths() {
assert_eq!(
parse_average_duration("work\nAverage duration: 12.5 ms\n"),
Some(12.5)
);
assert_eq!(parse_average_duration("completed"), None);
fn parses_average_durations() {
let cases = [
(
"parquetbench nanoseconds",
"ℹ Average: 42 rows, 2 record batches in 500ns over 4 iterations",
Some(0.0005),
),
(
"scanbench colored microseconds",
"\x1b[1;32mSummary\x1b[0m Average: \x1b[36m12\x1b[0m rows in 250µs over 2 iterations",
Some(0.25),
),
(
"scanbench seconds",
"Summary Average: 12 rows in 2.5s over 2 iterations",
Some(2500.0),
),
(
"legacy milliseconds",
"Average duration: 12.5 ms",
Some(12.5),
),
("absent", "Benchmark completed!", None),
(
"malformed",
"Summary Average: 12 rows in unknown over 2 iterations",
None,
),
];
for (name, stdout, expected) in cases {
let actual = parse_average_duration(stdout);
match (actual, expected) {
(Some(actual), Some(expected)) => assert!(
(actual - expected).abs() < f64::EPSILON,
"{name}: expected {expected}, got {actual}"
),
(None, None) => {}
(actual, expected) => panic!("{name}: expected {expected:?}, got {actual:?}"),
}
}
}
#[test]
fn groups_scan_paths() {
let targets = vec![
BenchTarget {
relative_path: "data/a.parquet".to_string(),
src/common/datasource/src/object_store.rs
+18
View File
@@ -50,6 +50,22 @@ pub const AZBLOB_SCHEMA: &str = "AZBLOB";
pub struct BuiltBackend {
pub object_store: ObjectStore,
pub object_path: Option<String>,
local_root: Option<SecureFsRoot>,
}
impl BuiltBackend {
/// Checks an explicit input using the same file-type filter as directory listing.
pub async fn is_file(&self, path: &str) -> object_store::Result<bool> {
// Stat first preserves errors for dangling or inaccessible symlink targets.
if self.object_store.stat(path).await?.mode() != object_store::EntryMode::FILE {
return Ok(false);
}
if let Some(root) = &self.local_root {
root.is_file(path).await
} else {
Ok(true)
}
}
}
/// Controls whether SQL paths may access the local filesystem.
@@ -379,6 +395,7 @@ async fn build_backend_inner(
return Ok(BuiltBackend {
object_store: build_fs_backend(&root)?,
object_path,
local_root: Some(root),
});
}
@@ -418,6 +435,7 @@ async fn build_backend_inner(
Ok(BuiltBackend {
object_store,
object_path,
local_root: None,
})
}
src/common/function/src/aggrs/vector/avg.rs
+5 -9
View File
@@ -183,8 +183,6 @@ mod tests {
use std::sync::Arc;
use arrow::array::StringArray;
use datatypes::scalars::ScalarVector;
use datatypes::vectors::{ConstantVector, StringVector, Vector};
use super::*;
@@ -252,14 +250,12 @@ mod tests {
vec_avg.evaluate().unwrap()
);
// test update with constant vector
// test update with repeated values
let mut vec_avg = VectorAvg::default();
let v: Vec<ArrayRef> = vec![
Arc::new(ConstantVector::new(
Arc::new(StringVector::from_vec(vec!["[1.0,2.0,3.0]".to_string()])),
4,
))
.to_arrow_array(),
let v = vec![
ScalarValue::Utf8(Some("[1.0,2.0,3.0]".to_string()))
.to_array_of_size(4)
.unwrap(),
];
vec_avg.update_batch(&v).unwrap();
assert_eq!(
src/common/function/src/aggrs/vector/product.rs
+5 -10
View File
@@ -159,9 +159,6 @@ impl Accumulator for VectorProduct {
mod tests {
use std::sync::Arc;
use datatypes::scalars::ScalarVector;
use datatypes::vectors::{ConstantVector, StringVector, Vector};
use super::*;
#[test]
@@ -213,14 +210,12 @@ mod tests {
vec_product.update_batch(&v).unwrap();
assert_eq!(ScalarValue::Binary(None), vec_product.evaluate().unwrap());
// test update with constant vector
// test update with repeated values
let mut vec_product = VectorProduct::default();
let v: Vec<ArrayRef> = vec![
Arc::new(ConstantVector::new(
Arc::new(StringVector::from_vec(vec!["[1.0,2.0,3.0]".to_string()])),
4,
))
.to_arrow_array(),
let v = vec![
ScalarValue::Utf8(Some("[1.0,2.0,3.0]".to_string()))
.to_array_of_size(4)
.unwrap(),
];
vec_product.update_batch(&v).unwrap();
src/common/function/src/aggrs/vector/sum.rs
+5 -9
View File
@@ -171,8 +171,6 @@ mod tests {
use std::sync::Arc;
use arrow::array::StringArray;
use datatypes::scalars::ScalarVector;
use datatypes::vectors::{ConstantVector, StringVector, Vector};
use super::*;
@@ -225,14 +223,12 @@ mod tests {
vec_sum.update_batch(&v).unwrap();
assert_eq!(ScalarValue::Binary(None), vec_sum.evaluate().unwrap());
// test update with constant vector
// test update with repeated values
let mut vec_sum = VectorSum::default();
let v: Vec<ArrayRef> = vec![
Arc::new(ConstantVector::new(
Arc::new(StringVector::from_vec(vec!["[1.0,2.0,3.0]".to_string()])),
4,
))
.to_arrow_array(),
let v = vec![
ScalarValue::Utf8(Some("[1.0,2.0,3.0]".to_string()))
.to_array_of_size(4)
.unwrap(),
];
vec_sum.update_batch(&v).unwrap();
assert_eq!(
src/common/function/src/scalars/expression.rs
-2
View File
@@ -12,13 +12,11 @@
// See the License for the specific language governing permissions and
// limitations under the License.
mod binary;
mod ctx;
mod if_func;
mod is_null;
mod unary;
pub use binary::scalar_binary_op;
pub use ctx::EvalContext;
pub use unary::scalar_unary_op;
src/common/function/src/scalars/expression/binary.rs
-90
View File
@@ -1,90 +0,0 @@
// Copyright 2023 Greptime Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use std::iter;
use common_query::error::Result;
use datatypes::prelude::*;
use datatypes::vectors::{ConstantVector, Helper};
use crate::scalars::expression::ctx::EvalContext;
pub fn scalar_binary_op<L: Scalar, R: Scalar, O: Scalar, F>(
l: &VectorRef,
r: &VectorRef,
f: F,
ctx: &mut EvalContext,
) -> Result<<O as Scalar>::VectorType>
where
F: Fn(Option<L::RefType<'_>>, Option<R::RefType<'_>>, &mut EvalContext) -> Option<O>,
{
debug_assert!(
l.len() == r.len(),
"Size of vectors must match to apply binary expression"
);
let result = match (l.is_const(), r.is_const()) {
(false, true) => {
let left: &<L as Scalar>::VectorType = unsafe { Helper::static_cast(l) };
let right: &ConstantVector = unsafe { Helper::static_cast(r) };
let right: &<R as Scalar>::VectorType = unsafe { Helper::static_cast(right.inner()) };
let b = right.get_data(0);
let it = left.iter_data().map(|a| f(a, b.clone(), ctx));
<O as Scalar>::VectorType::from_owned_iterator(it)
}
(false, false) => {
let left: &<L as Scalar>::VectorType = unsafe { Helper::static_cast(l) };
let right: &<R as Scalar>::VectorType = unsafe { Helper::static_cast(r) };
let it = left
.iter_data()
.zip(right.iter_data())
.map(|(a, b)| f(a, b, ctx));
<O as Scalar>::VectorType::from_owned_iterator(it)
}
(true, false) => {
let left: &ConstantVector = unsafe { Helper::static_cast(l) };
let left: &<L as Scalar>::VectorType = unsafe { Helper::static_cast(left.inner()) };
let a = left.get_data(0);
let right: &<R as Scalar>::VectorType = unsafe { Helper::static_cast(r) };
let it = right.iter_data().map(|b| f(a.clone(), b, ctx));
<O as Scalar>::VectorType::from_owned_iterator(it)
}
(true, true) => {
let left: &ConstantVector = unsafe { Helper::static_cast(l) };
let left: &<L as Scalar>::VectorType = unsafe { Helper::static_cast(left.inner()) };
let a = left.get_data(0);
let right: &ConstantVector = unsafe { Helper::static_cast(r) };
let right: &<R as Scalar>::VectorType = unsafe { Helper::static_cast(right.inner()) };
let b = right.get_data(0);
let it = iter::repeat(a)
.zip(iter::repeat(b))
.map(|(a, b)| f(a, b, ctx))
.take(left.len());
<O as Scalar>::VectorType::from_owned_iterator(it)
}
};
if let Some(error) = ctx.error.take() {
return Err(error);
}
Ok(result)
}
src/common/function/src/scalars/test.rs
+31 -1
View File
@@ -14,13 +14,43 @@
use std::fmt;
use common_query::error::Result;
use datafusion::logical_expr::ColumnarValue;
use datafusion_expr::{ScalarFunctionArgs, Signature, Volatility};
use datatypes::arrow::datatypes::DataType;
use datatypes::prelude::{Scalar, ScalarVector, VectorRef};
use datatypes::vectors::{Helper, Vector};
use crate::function::{Function, extract_args};
use crate::scalars::expression::{EvalContext, scalar_binary_op};
use crate::scalars::expression::EvalContext;
fn scalar_binary_op<L: Scalar, R: Scalar, O: Scalar, F>(
l: &VectorRef,
r: &VectorRef,
f: F,
ctx: &mut EvalContext,
) -> Result<<O as Scalar>::VectorType>
where
F: Fn(Option<L::RefType<'_>>, Option<R::RefType<'_>>, &mut EvalContext) -> Option<O>,
{
debug_assert!(
l.len() == r.len(),
"Size of vectors must match to apply binary expression"
);
let left: &<L as Scalar>::VectorType = unsafe { Helper::static_cast(l) };
let right: &<R as Scalar>::VectorType = unsafe { Helper::static_cast(r) };
let result = <O as Scalar>::VectorType::from_owned_iterator(
left.iter_data()
.zip(right.iter_data())
.map(|(a, b)| f(a, b, ctx)),
);
if let Some(error) = ctx.error.take() {
return Err(error);
}
Ok(result)
}
#[derive(Clone)]
pub(crate) struct TestAndFunction {
src/common/function/src/state.rs
+5
View File
@@ -12,12 +12,16 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use common_base::Plugins;
use crate::handlers::{FlowServiceHandlerRef, ProcedureServiceHandlerRef, TableMutationHandlerRef};
/// Shared state for SQL functions.
/// The handlers in state may be `None` in cli command-line or test cases.
#[derive(Clone, Default)]
pub struct FunctionState {
/// Process plugins available to dynamically registered functions.
pub plugins: Plugins,
// The table mutation handler
pub table_mutation_handler: Option<TableMutationHandlerRef>,
// The procedure service handler
@@ -212,6 +216,7 @@ impl FunctionState {
}
Self {
plugins: Plugins::default(),
table_mutation_handler: Some(Arc::new(MockTableMutationHandler)),
procedure_service_handler: Some(Arc::new(MockProcedureServiceHandler)),
flow_service_handler: Some(Arc::new(MockFlowServiceHandler)),
src/common/grpc-expr/src/alter.rs
+133 -30
View File
@@ -49,33 +49,14 @@ const LOCATION_TYPE_AFTER: i32 = LocationType::After as i32;
/// mixed batch — annotation alters skip region dispatch, so they cannot share
/// a statement with options that regions must see.
fn annotation_family_of_keys<'a>(
mut keys: impl Iterator<Item = &'a str>,
keys: impl Iterator<Item = &'a str>,
) -> Result<Option<AnnotationFamily>> {
let Some(first) = keys.next() else {
return Ok(None);
};
let family = AnnotationFamily::of_key(first);
let mut count = 1usize;
for key in keys {
count += 1;
let this = AnnotationFamily::of_key(key);
if this != family {
return error::InvalidTableOptionRequestSnafu {
err_msg: family.or(this).unwrap().mixed_batch_error(),
}
.fail();
table::requests::validate_annotation_keys(keys).map_err(|err| {
error::InvalidTableOptionRequestSnafu {
err_msg: err.to_string(),
}
}
if let Some(family) = family
&& family.requires_single_key()
&& count > 1
{
return error::InvalidTableOptionRequestSnafu {
err_msg: family.mixed_batch_error(),
}
.fail();
}
Ok(family)
.build()
})
}
/// Returns the annotation family when `kind` is a SET/UNSET whose keys all
@@ -583,6 +564,131 @@ mod tests {
assert_eq!("mem_usage".to_string(), drop_names.pop().unwrap());
}
#[test]
fn test_repartition_hints_together() {
let keys = [
REPARTITION_COLUMN_HINT_KEY,
table::requests::REPARTITION_PARTITION_NUM_HINT_KEY,
];
let options = vec![
(keys[0].to_string(), "host".to_string()),
(keys[1].to_string(), "10".to_string()),
];
let set = Kind::SetTableOptions(SetTableOptions {
table_options: options
.iter()
.map(|(key, value)| PbOption {
key: key.clone(),
value: value.clone(),
})
.collect(),
});
let unset = Kind::UnsetTableOptions(UnsetTableOptions {
keys: keys.map(str::to_string).to_vec(),
});
for kind in [set, unset] {
assert_eq!(
annotation_alter_family(&kind).unwrap(),
Some(AnnotationFamily::RepartitionHint)
);
let request = alter_expr_to_request(
1,
AlterTableExpr {
kind: Some(kind),
..Default::default()
},
None,
)
.unwrap();
match request.alter_kind {
AlterKind::SetAnnotations {
family,
options: actual,
} => {
assert_eq!(family, AnnotationFamily::RepartitionHint);
assert_eq!(actual, options);
}
AlterKind::UnsetAnnotations {
family,
keys: actual,
} => {
assert_eq!(family, AnnotationFamily::RepartitionHint);
assert_eq!(actual, keys);
}
other => panic!("unexpected alter kind: {other:?}"),
}
}
}
#[test]
fn test_repartition_partition_num_hint_expr() {
let key = table::requests::REPARTITION_PARTITION_NUM_HINT_KEY;
let set = Kind::SetTableOptions(SetTableOptions {
table_options: vec![PbOption {
key: key.to_string(),
value: "8".to_string(),
}],
});
let unset = Kind::UnsetTableOptions(UnsetTableOptions {
keys: vec![key.to_string()],
});
for kind in [set, unset] {
assert_eq!(
annotation_alter_family(&kind).unwrap(),
Some(AnnotationFamily::RepartitionHint)
);
let request = alter_expr_to_request(
1,
AlterTableExpr {
kind: Some(kind),
..Default::default()
},
None,
)
.unwrap();
match request.alter_kind {
AlterKind::SetAnnotations { family, options } => {
assert_eq!(family, AnnotationFamily::RepartitionHint);
assert_eq!(options, vec![(key.to_string(), "8".to_string())]);
}
AlterKind::UnsetAnnotations { family, keys } => {
assert_eq!(family, AnnotationFamily::RepartitionHint);
assert_eq!(keys, vec![key.to_string()]);
}
other => panic!("unexpected alter kind: {other:?}"),
}
}
for other in [key, table::requests::TTL_KEY] {
for kind in [
Kind::SetTableOptions(SetTableOptions {
table_options: [key, other]
.into_iter()
.map(|key| PbOption {
key: key.to_string(),
value: "8".to_string(),
})
.collect(),
}),
Kind::UnsetTableOptions(UnsetTableOptions {
keys: vec![key.to_string(), other.to_string()],
}),
] {
assert!(annotation_alter_family(&kind).is_err());
assert!(
alter_expr_to_request(
1,
AlterTableExpr {
kind: Some(kind),
..Default::default()
},
None
)
.is_err()
);
}
}
}
#[test]
fn test_set_repartition_column_hint_expr() {
let expr = AlterTableExpr {
@@ -633,7 +739,7 @@ mod tests {
let err = alter_expr_to_request(1, expr, None).unwrap_err();
assert!(
err.to_string()
.contains("repartition.column.hint must be altered separately")
.contains("repartition hints must be altered separately")
);
// Duplicate hint entries are not a meaningful batch either.
@@ -655,10 +761,7 @@ mod tests {
})),
};
let err = alter_expr_to_request(1, dup, None).unwrap_err();
assert!(
err.to_string()
.contains("repartition.column.hint must be altered separately")
);
assert!(err.to_string().contains("duplicate repartition hint keys"));
}
#[test]
src/common/macro/src/admin_fn.rs
+1 -1
View File
@@ -304,7 +304,7 @@ fn build_struct(
.and_then(|cv| match cv {
common_query::prelude::ColumnarValue::Vector(v) => Ok(v),
common_query::prelude::ColumnarValue::Scalar(s) => {
datatypes::vectors::Helper::try_from_scalar_value(s, args.number_rows)
datatypes::vectors::Helper::try_from_scalar_value(s, args.number_rows, None)
.context(common_query::error::FromScalarValueSnafu)
}
})
src/common/macro/src/lib.rs
+4
View File
@@ -75,6 +75,10 @@ pub fn as_aggr_func_creator(args: TokenStream, input: TokenStream) -> TokenStrea
/// - `name`: The name of the generated `ScalarUDF` struct.
/// - `ret`: The return type of the generated UDF function.
/// - `display_name`: The display name of the generated UDF function.
/// - `evaluator`: Optional path to a specialized evaluator with the calling convention
/// `fn(&[ColumnarValue], &str) -> Result<ColumnarValue, DataFusionError>`. When supplied,
/// the generated UDF `calc` delegates directly to it; without it, the default expansion is
/// unchanged.
#[proc_macro_attribute]
pub fn range_fn(args: TokenStream, input: TokenStream) -> TokenStream {
process_range_fn(args, input)
src/common/macro/src/range_fn.rs
+29 -4
View File
@@ -16,7 +16,7 @@ use proc_macro::TokenStream;
use quote::quote;
use syn::spanned::Spanned;
use syn::{
Attribute, Ident, ItemFn, Signature, Type, TypeReference, Visibility, parse_macro_input,
Attribute, Ident, ItemFn, Path, Signature, Type, TypeReference, Visibility, parse_macro_input,
};
use crate::utils::extract_input_types;
@@ -34,6 +34,7 @@ pub(crate) fn process_range_fn(args: TokenStream, input: TokenStream) -> TokenSt
let mut name: Option<Ident> = None;
let mut display_name: Option<Ident> = None;
let mut ret: Option<Ident> = None;
let mut evaluator: Option<Path> = None;
let parser = syn::meta::parser(|meta| {
if meta.path.is_ident("name") {
@@ -45,6 +46,9 @@ pub(crate) fn process_range_fn(args: TokenStream, input: TokenStream) -> TokenSt
} else if meta.path.is_ident("ret") {
ret = Some(meta.value()?.parse()?);
Ok(())
} else if meta.path.is_ident("evaluator") {
evaluator = Some(meta.value()?.parse()?);
Ok(())
} else {
Err(meta.error("unsupported property"))
}
@@ -103,10 +107,19 @@ pub(crate) fn process_range_fn(args: TokenStream, input: TokenStream) -> TokenSt
arg_types,
fn_name.clone(),
ret.expect("ret required"),
evaluator.clone(),
);
// preserve this fn, but remove its `pub` modifier
let input_fn_code: TokenStream = quote! {
#sig { #block }
// Preserve this fn, but remove its `pub` modifier. Specialized evaluators
// do not call it in production, while tests keep it as the slice oracle.
let input_fn_code: TokenStream = if evaluator.is_some() {
quote! {
#[cfg(test)]
#sig { #block }
}
} else {
quote! {
#sig { #block }
}
}
.into();
@@ -161,7 +174,19 @@ fn build_calc_fn(
param_types: Vec<Type>,
fn_name: Ident,
ret_type: Ident,
evaluator: Option<Path>,
) -> TokenStream {
if let Some(evaluator) = evaluator {
return quote! {
impl #name {
fn calc(input: &[ColumnarValue]) -> Result<ColumnarValue, DataFusionError> {
#evaluator(input, Self::name())
}
}
}
.into();
}
let param_names = param_types
.iter()
.enumerate()
src/common/mem-prof/Cargo.toml
+2 -2
View File
@@ -16,7 +16,7 @@ tempfile = "3.4"
tokio.workspace = true
[target.'cfg(not(windows))'.dependencies]
tikv-jemalloc-ctl = { version = "0.6", features = ["use_std", "stats"] }
tikv-jemalloc-ctl = { version = "0.7", features = ["use_std", "stats"] }
jemalloc-pprof-utils = { version = "0.8", package = "pprof_util", features = [
"flamegraph",
"symbolize",
@@ -25,4 +25,4 @@ jemalloc-pprof-mappings = { version = "0.7", package = "mappings" } # for get th
[target.'cfg(not(windows))'.dependencies.tikv-jemalloc-sys]
features = ["stats", "profiling", "unprefixed_malloc_on_supported_platforms"]
version = "0.6"
version = "0.7"
src/common/meta/src/ddl/tests/alter_table.rs
+78 -51
View File
@@ -1123,58 +1123,85 @@ async fn test_on_submit_alter_request_with_exist_poison() {
}
#[tokio::test]
async fn test_semantic_annotation_alter_is_metadata_only() {
let (tx, mut rx) = mpsc::channel(8);
let node_manager = Arc::new(MockDatanodeManager::new(DatanodeWatcher::new(tx)));
let ddl_context = new_ddl_context(node_manager);
let table_id = 1024;
let table_name = "foo";
let task = test_create_table_task(table_name, table_id);
ddl_context
.table_metadata_manager
.create_table_metadata(
task.table_info.clone(),
prepare_table_route(table_id),
HashMap::new(),
)
.await
.unwrap();
async fn test_annotation_alter_is_metadata_only() {
for (key, value) in [
("greptime.semantic.signal_type", "metric"),
(table::requests::REPARTITION_PARTITION_NUM_HINT_KEY, "8"),
] {
let (tx, mut rx) = mpsc::channel(8);
let node_manager = Arc::new(MockDatanodeManager::new(DatanodeWatcher::new(tx)));
let ddl_context = new_ddl_context(node_manager);
let table_id = 1024;
let table_name = "foo";
let task = test_create_table_task(table_name, table_id);
ddl_context
.table_metadata_manager
.create_table_metadata(
task.table_info.clone(),
prepare_table_route(table_id),
HashMap::new(),
)
.await
.unwrap();
let alter_table_task = AlterTableTask {
alter_table: AlterTableExpr {
catalog_name: DEFAULT_CATALOG_NAME.to_string(),
schema_name: DEFAULT_SCHEMA_NAME.to_string(),
table_name: table_name.to_string(),
kind: Some(Kind::SetTableOptions(SetTableOptions {
table_options: vec![api::v1::Option {
key: "greptime.semantic.signal_type".to_string(),
value: "metric".to_string(),
}],
})),
},
};
let mut procedure =
AlterTableProcedure::new(table_id, alter_table_task, ddl_context.clone()).unwrap();
execute_procedure_until_done(&mut procedure).await;
let alter_table_task = AlterTableTask {
alter_table: AlterTableExpr {
catalog_name: DEFAULT_CATALOG_NAME.to_string(),
schema_name: DEFAULT_SCHEMA_NAME.to_string(),
table_name: table_name.to_string(),
kind: Some(Kind::SetTableOptions(SetTableOptions {
table_options: vec![api::v1::Option {
key: key.to_string(),
value: value.to_string(),
}],
})),
},
};
let mut procedure =
AlterTableProcedure::new(table_id, alter_table_task, ddl_context.clone()).unwrap();
execute_procedure_until_done(&mut procedure).await;
// Metadata-only: no region request reaches any datanode.
rx.try_recv().unwrap_err();
// Metadata-only: no region request reaches any datanode.
rx.try_recv().unwrap_err();
let table_info = ddl_context
.table_metadata_manager
.table_info_manager()
.get(table_id)
.await
.unwrap()
.unwrap()
.into_inner()
.table_info;
assert_eq!(
table_info
.meta
.options
.extra_options
.get("greptime.semantic.signal_type"),
Some(&"metric".to_string())
);
let table_info = ddl_context
.table_metadata_manager
.table_info_manager()
.get(table_id)
.await
.unwrap()
.unwrap()
.into_inner()
.table_info;
assert_eq!(
table_info.meta.options.extra_options.get(key),
Some(&value.to_string())
);
for _ in 0..2 {
let task = AlterTableTask {
alter_table: AlterTableExpr {
catalog_name: DEFAULT_CATALOG_NAME.to_string(),
schema_name: DEFAULT_SCHEMA_NAME.to_string(),
table_name: table_name.to_string(),
kind: Some(Kind::UnsetTableOptions(api::v1::UnsetTableOptions {
keys: vec![key.to_string()],
})),
},
};
let mut procedure =
AlterTableProcedure::new(table_id, task, ddl_context.clone()).unwrap();
execute_procedure_until_done(&mut procedure).await;
rx.try_recv().unwrap_err();
let info = ddl_context
.table_metadata_manager
.table_info_manager()
.get(table_id)
.await
.unwrap()
.unwrap()
.into_inner()
.table_info;
assert!(!info.meta.options.extra_options.contains_key(key));
}
}
}
src/common/meta/src/ddl_manager.rs
+20 -15
View File
@@ -2036,20 +2036,25 @@ mod tests {
// The repartition hint drives physical repartitioning; on a logical
// route it stays rejected by the guard.
let err = ddl_manager
.submit_ddl_task(
ExecutorContext {
query_context: Some(QueryContext::default()),
..Default::default()
},
SubmitDdlTaskRequest::new(DdlTask::new_alter_table(set_options_expr(
"logical",
&[("repartition.column.hint", "host")],
))),
)
.await
.unwrap_err();
let msg = common_error::ext::ErrorExt::output_msg(&err);
assert!(msg.contains("non-physical TableRouteValue"), "{msg}");
for (key, value) in [
(table::requests::REPARTITION_COLUMN_HINT_KEY, "host"),
(table::requests::REPARTITION_PARTITION_NUM_HINT_KEY, "8"),
] {
let err = ddl_manager
.submit_ddl_task(
ExecutorContext {
query_context: Some(QueryContext::default()),
..Default::default()
},
SubmitDdlTaskRequest::new(DdlTask::new_alter_table(set_options_expr(
"logical",
&[(key, value)],
))),
)
.await
.unwrap_err();
let msg = common_error::ext::ErrorExt::output_msg(&err);
assert!(msg.contains("non-physical TableRouteValue"), "{msg}");
}
}
}
src/common/meta/src/error.rs
+16
View File
@@ -269,6 +269,20 @@ pub enum Error {
error: prost::DecodeError,
},
#[snafu(display("Failed to decode packed file references"))]
DecodePackedFileRefs {
#[snafu(implicit)]
location: Location,
#[snafu(source)]
error: base64::DecodeError,
},
#[snafu(display("Invalid packed file references framing"))]
InvalidPackedFileRefs {
#[snafu(implicit)]
location: Location,
},
#[snafu(display("Failed to encode object into json"))]
EncodeJson {
#[snafu(implicit)]
@@ -1212,6 +1226,8 @@ impl ErrorExt for Error {
| PayloadNotExist { .. }
| ConvertRawKey { .. }
| DecodeProto { .. }
| DecodePackedFileRefs { .. }
| InvalidPackedFileRefs { .. }
| BuildTableMeta { .. }
| TableRouteNotFound { .. }
| TableRepartNotFound { .. }
src/common/meta/src/heartbeat/handler.rs
+98 -1
View File
@@ -94,7 +94,21 @@ impl HeartbeatResponseHandlerExecutor for HandlerGroupExecutor {
Ok(HandleControl::Done) => break,
Ok(HandleControl::Continue) => {}
Err(e) => {
error!(e;"Error while handling: {:?}", ctx.response);
let mailbox_message_id = ctx
.response
.mailbox_message
.as_ref()
.map(|message| message.id);
let json_payload_len =
ctx.response.mailbox_message.as_ref().and_then(|message| {
message.payload.as_ref().map(|payload| match payload {
api::v1::meta::mailbox_message::Payload::Json(json) => json.len(),
})
});
error!(
%e;
"Error while handling heartbeat response: mailbox_message_id={mailbox_message_id:?}, json_payload_len={json_payload_len:?}"
);
break;
}
}
@@ -102,3 +116,86 @@ impl HeartbeatResponseHandlerExecutor for HandlerGroupExecutor {
Ok(())
}
}
#[cfg(test)]
mod error_log_tests {
use std::fmt::Debug;
use std::sync::{Arc, Mutex};
use api::v1::meta::mailbox_message::Payload;
use api::v1::meta::{HeartbeatResponse, MailboxMessage};
use common_telemetry::tracing::field::{Field, Visit};
use common_telemetry::tracing::{Event, Subscriber};
use common_telemetry::tracing_subscriber::layer::{Context, SubscriberExt};
use common_telemetry::{tracing, tracing_subscriber};
use super::parse_mailbox_message::ParseMailboxMessageHandler;
use super::*;
use crate::heartbeat::mailbox::HeartbeatMailbox;
#[derive(Clone, Default)]
struct LogCapture(Arc<Mutex<Vec<String>>>);
impl<S> tracing_subscriber::Layer<S> for LogCapture
where
S: Subscriber,
{
fn on_event(&self, event: &Event<'_>, _ctx: Context<'_, S>) {
let mut visitor = FieldVisitor::default();
event.record(&mut visitor);
self.0.lock().unwrap().push(visitor.fields.join(", "));
}
}
#[derive(Default)]
struct FieldVisitor {
fields: Vec<String>,
}
impl Visit for FieldVisitor {
fn record_debug(&mut self, field: &Field, value: &dyn Debug) {
self.fields.push(format!("{}={value:?}", field.name()));
}
fn record_str(&mut self, field: &Field, value: &str) {
self.fields.push(format!("{}={value:?}", field.name()));
}
}
#[tokio::test(flavor = "current_thread")]
async fn test_handler_error_log_excludes_mailbox_payload() {
let payload_sentinel = "MALFORMED_PACKED_PAYLOAD_MUST_NOT_BE_LOGGED".repeat(256);
let payload = format!(
r#"{{"PackedGcRegions":{{"regions":[],"packed_file_refs_manifest":"{}"}}"#,
payload_sentinel
);
let payload_len = payload.len();
let capture = LogCapture::default();
let subscriber = tracing_subscriber::registry().with(capture.clone());
let _guard = tracing::subscriber::set_default(subscriber);
let (mailbox_tx, _) = tokio::sync::mpsc::channel(1);
let ctx = HeartbeatResponseHandlerContext::new(
Arc::new(HeartbeatMailbox::new(mailbox_tx)),
HeartbeatResponse {
mailbox_message: Some(MailboxMessage {
id: 42,
subject: "unsafe subject".to_string(),
to: "unsafe recipient".to_string(),
from: "unsafe sender".to_string(),
payload: Some(Payload::Json(payload)),
..Default::default()
}),
..Default::default()
},
);
let executor = HandlerGroupExecutor::new(vec![Arc::new(ParseMailboxMessageHandler)]);
executor.handle(ctx).await.unwrap();
let logs = capture.0.lock().unwrap().join("\n");
assert!(logs.contains("Error while handling heartbeat response"));
assert!(logs.contains("mailbox_message_id=Some(42)"));
assert!(logs.contains(&format!("json_payload_len=Some({payload_len})")));
assert!(!logs.contains(&payload_sentinel));
}
}
src/common/meta/src/instruction.rs
+347 -1
View File
@@ -16,16 +16,19 @@ use std::collections::{HashMap, HashSet};
use std::fmt::{Display, Formatter};
use std::time::Duration;
use base64::Engine;
use common_error::ext::{ErrorExt, RetryHint};
use common_error::status_code::StatusCode;
use serde::{Deserialize, Deserializer, Serialize, Serializer};
use snafu::ResultExt as _;
use store_api::region_engine::SyncRegionFromRequest;
use store_api::region_request::{RegionFlushReason, RegionRequirements};
use store_api::storage::{FileRefsManifest, GcReport, RegionId, RegionNumber};
use store_api::storage::{FileId, FileRef, FileRefsManifest, GcReport, RegionId, RegionNumber};
use strum::Display;
use table::metadata::TableId;
use table::table_name::TableName;
use crate::error::{DecodePackedFileRefsSnafu, InvalidPackedFileRefsSnafu};
use crate::flow_name::FlowName;
use crate::key::schema_name::SchemaName;
use crate::key::{FlowId, FlowPartitionId};
@@ -579,6 +582,24 @@ impl Display for GetFileRefs {
}
}
/// Instruction to get file references using the packed manifest format.
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct GetPackedFileRefs {
/// List of region IDs to get file references from active FileHandles (in-memory).
pub query_regions: Vec<RegionId>,
/// Mapping from the src region IDs (whose file references to look for) to
/// the dst region IDs (where to read the manifests).
/// Key: The source region IDs (where files originally came from).
/// Value: The set of destination region IDs (whose manifests need to be read).
pub related_regions: HashMap<RegionId, HashSet<RegionId>>,
}
impl Display for GetPackedFileRefs {
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
write!(f, "GetPackedFileRefs(region_ids={:?})", self.query_regions)
}
}
/// Instruction to trigger garbage collection for a region.
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct GcRegions {
@@ -590,6 +611,147 @@ pub struct GcRegions {
pub full_file_listing: bool,
}
/// Instruction to trigger garbage collection with a packed file-reference manifest.
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct PackedGcRegions {
/// The region ID to perform GC on.
pub regions: Vec<RegionId>,
/// Packed manifest used by the packed-only GC protocol.
pub packed_file_refs_manifest: PackedFileRefsManifest,
/// Whether to perform a full file listing to find orphan files.
pub full_file_listing: bool,
}
impl Display for PackedGcRegions {
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
let file_refs_count = self.packed_file_refs_manifest.file_refs.len();
write!(
f,
"PackedGcRegions(regions={:?}, file_refs_count={}, full_file_listing={})",
self.regions, file_refs_count, self.full_file_listing
)
}
}
/// A packed, JSON-compatible encoding of a file reference manifest.
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, Default)]
pub struct PackedFileRefsManifest {
/// Required packed file references for each region.
pub file_refs: HashMap<RegionId, PackedRegionFileRefs>,
#[serde(default)]
pub manifest_version: HashMap<RegionId, u64>,
#[serde(default)]
pub cross_region_refs: HashMap<RegionId, HashSet<RegionId>>,
}
/// Packed file references for one region, encoded with STANDARD Base64.
///
/// `files` is required and contains 16-byte UUID records for references whose
/// `index_version` is `None`. `indexed` is required and contains 24-byte records
/// for references whose `index_version` is `Some`: a 16-byte UUID followed by
/// an 8-byte big-endian `u64` version. Decoding fails when either field is not
/// valid STANDARD Base64 or its decoded bytes do not align to the corresponding
/// record width.
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, Default)]
pub struct PackedRegionFileRefs {
/// Required STANDARD Base64-encoded 16-byte UUID records.
pub files: String,
/// Required STANDARD Base64-encoded 16-byte UUID and 8-byte version records.
pub indexed: String,
}
impl PackedFileRefsManifest {
/// Packs a file reference manifest using required [`PackedRegionFileRefs`]
/// STANDARD Base64 fields.
///
/// References whose `index_version` is `None` are encoded as 16-byte UUID
/// `files` records; references whose `index_version` is `Some` are encoded
/// as 24-byte `indexed` records: a 16-byte UUID followed by an 8-byte
/// big-endian `u64` version.
pub fn from_manifest(manifest: &FileRefsManifest) -> Self {
Self {
file_refs: manifest
.file_refs
.iter()
.map(|(region, refs)| (*region, PackedRegionFileRefs::from_refs(refs)))
.collect(),
manifest_version: manifest.manifest_version.clone(),
cross_region_refs: manifest.cross_region_refs.clone(),
}
}
/// Decodes this packed representation into a file reference manifest.
///
/// Returns a typed error for invalid STANDARD Base64 or malformed record
/// framing without retaining encoded payloads.
pub fn into_manifest(self) -> crate::error::Result<FileRefsManifest> {
let mut file_refs = HashMap::new();
for (region, encoded) in self.file_refs {
file_refs.insert(region, encoded.into_refs(region)?);
}
Ok(FileRefsManifest {
file_refs,
manifest_version: self.manifest_version,
cross_region_refs: self.cross_region_refs,
})
}
}
impl PackedRegionFileRefs {
/// Packs file references as the documented STANDARD Base64 record streams.
pub fn from_refs(refs: &HashSet<FileRef>) -> Self {
let engine = base64::engine::general_purpose::STANDARD;
let mut files = Vec::new();
let mut indexed = Vec::new();
for file_ref in refs {
match file_ref.index_version {
None => files.extend_from_slice(file_ref.file_id.as_bytes()),
Some(version) => {
indexed.extend_from_slice(file_ref.file_id.as_bytes());
indexed.extend_from_slice(&version.to_be_bytes());
}
}
}
Self {
files: engine.encode(files),
indexed: engine.encode(indexed),
}
}
/// Decodes the documented STANDARD Base64 record streams for `region`.
fn into_refs(self, region: RegionId) -> crate::error::Result<HashSet<FileRef>> {
let engine = base64::engine::general_purpose::STANDARD;
let files = engine
.decode(self.files)
.context(DecodePackedFileRefsSnafu)?;
let indexed = engine
.decode(self.indexed)
.context(DecodePackedFileRefsSnafu)?;
if files.len() % 16 != 0 || indexed.len() % 24 != 0 {
return InvalidPackedFileRefsSnafu.fail();
}
let mut refs = HashSet::new();
for bytes in files.chunks_exact(16) {
let mut id = [0; 16];
id.copy_from_slice(bytes);
refs.insert(FileRef::new(region, FileId::from_bytes(id), None));
}
for bytes in indexed.chunks_exact(24) {
let mut id = [0; 16];
id.copy_from_slice(&bytes[..16]);
let mut version = [0; 8];
version.copy_from_slice(&bytes[16..]);
refs.insert(FileRef::new(
region,
FileId::from_bytes(id),
Some(u64::from_be_bytes(version)),
));
}
Ok(refs)
}
}
impl Display for GcRegions {
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
write!(
@@ -625,6 +787,26 @@ impl Display for GetFileRefsReply {
}
}
/// Reply for GetPackedFileRefs instruction.
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct GetPackedFileRefsReply {
pub packed_file_refs_manifest: PackedFileRefsManifest,
pub success: bool,
pub error: Option<InstructionError>,
}
impl Display for GetPackedFileRefsReply {
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
write!(
f,
"GetPackedFileRefsReply(success={}, file_refs_count={}, error={:?})",
self.success,
self.packed_file_refs_manifest.file_refs.len(),
self.error
)
}
}
/// Reply for GC instruction.
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct GcRegionsReply {
@@ -827,8 +1009,12 @@ pub enum Instruction {
FlushRegions(FlushRegions),
/// Gets file references for regions.
GetFileRefs(GetFileRefs),
/// Gets file references using the packed manifest format.
GetPackedFileRefs(GetPackedFileRefs),
/// Triggers garbage collection for a region.
GcRegions(GcRegions),
/// Triggers garbage collection with a packed file-reference manifest.
PackedGcRegions(PackedGcRegions),
/// Temporary suspend serving reads or writes
Suspend,
/// Makes regions enter staging state.
@@ -1111,6 +1297,7 @@ pub enum InstructionReply {
DowngradeRegions(DowngradeRegionsReply),
FlushRegions(FlushRegionReply),
GetFileRefs(GetFileRefsReply),
GetPackedFileRefs(GetPackedFileRefsReply),
GcRegions(GcRegionsReply),
EnterStagingRegions(EnterStagingRegionsReply),
SyncRegions(SyncRegionsReply),
@@ -1131,6 +1318,9 @@ impl Display for InstructionReply {
}
Self::FlushRegions(reply) => write!(f, "InstructionReply::FlushRegions({})", reply),
Self::GetFileRefs(reply) => write!(f, "InstructionReply::GetFileRefs({})", reply),
Self::GetPackedFileRefs(reply) => {
write!(f, "InstructionReply::GetPackedFileRefs({})", reply)
}
Self::GcRegions(reply) => write!(f, "InstructionReply::GcRegion({})", reply),
Self::EnterStagingRegions(reply) => {
write!(
@@ -1221,6 +1411,7 @@ impl InstructionReply {
#[cfg(test)]
mod tests {
use std::collections::HashSet;
use std::error::Error as _;
use common_error::mock::MockError;
use common_wal::options::WalOptions;
@@ -1771,6 +1962,161 @@ mod tests {
}
}
#[test]
fn test_legacy_gc_file_refs_compatibility() {
#[derive(Debug, Deserialize)]
struct LegacyGetFileRefs {
query_regions: Vec<RegionId>,
related_regions: HashMap<RegionId, HashSet<RegionId>>,
}
#[derive(Debug, Deserialize)]
struct LegacyGetFileRefsReply {
file_refs_manifest: FileRefsManifest,
success: bool,
error: Option<InstructionError>,
}
#[derive(Debug, Deserialize)]
enum LegacyInstruction {
GetFileRefs(LegacyGetFileRefs),
}
let get_file_refs = Instruction::GetFileRefs(GetFileRefs {
query_regions: vec![RegionId::new(7, 3)],
related_regions: HashMap::new(),
});
let get_file_refs_json = serde_json::to_string(&get_file_refs).unwrap();
let legacy_get_file_refs: LegacyInstruction =
serde_json::from_str(&get_file_refs_json).unwrap();
let LegacyInstruction::GetFileRefs(legacy_get_file_refs) = legacy_get_file_refs;
assert_eq!(legacy_get_file_refs.query_regions.len(), 1);
assert!(legacy_get_file_refs.related_regions.is_empty());
let reply = GetFileRefsReply {
file_refs_manifest: FileRefsManifest::default(),
success: true,
error: None,
};
let reply_json = serde_json::to_string(&reply).unwrap();
let legacy_reply: LegacyGetFileRefsReply = serde_json::from_str(&reply_json).unwrap();
let current_reply: GetFileRefsReply = serde_json::from_str(&reply_json).unwrap();
assert!(legacy_reply.success);
assert!(legacy_reply.error.is_none());
assert!(legacy_reply.file_refs_manifest.file_refs.is_empty());
assert_eq!(current_reply, reply);
let packed = Instruction::PackedGcRegions(PackedGcRegions {
regions: vec![],
packed_file_refs_manifest: PackedFileRefsManifest::default(),
full_file_listing: false,
});
let packed_json = serde_json::to_string(&packed).unwrap();
assert!(serde_json::from_str::<LegacyInstruction>(&packed_json).is_err());
}
#[test]
fn test_packed_gc_regions_round_trip_is_distinct() {
let instruction = Instruction::PackedGcRegions(PackedGcRegions {
regions: vec![RegionId::new(7, 3)],
packed_file_refs_manifest: PackedFileRefsManifest::default(),
full_file_listing: true,
});
let serialized = serde_json::to_string(&instruction).unwrap();
assert!(serialized.contains("PackedGcRegions"));
assert_eq!(
serde_json::from_str::<Instruction>(&serialized).unwrap(),
instruction
);
}
#[test]
fn test_packed_file_refs_manifest_round_trip_and_validation() {
let region = RegionId::new(7, 3);
let file_id = FileId::parse_str("00000000-0000-0000-0000-000000000001").unwrap();
let mut manifest = FileRefsManifest::default();
manifest.file_refs.insert(
region,
HashSet::from([
FileRef::new(region, file_id, None),
FileRef::new(region, file_id, Some(0)),
FileRef::new(region, file_id, Some(u64::MAX)),
]),
);
manifest.manifest_version.insert(region, 42);
manifest
.cross_region_refs
.insert(region, HashSet::from([RegionId::new(7, 4)]));
let packed = PackedFileRefsManifest::from_manifest(&manifest);
assert_eq!(packed.clone().into_manifest().unwrap(), manifest);
let mut missing_files = serde_json::to_value(&packed).unwrap();
missing_files
.get_mut("file_refs")
.and_then(serde_json::Value::as_object_mut)
.and_then(|file_refs| file_refs.values_mut().next())
.and_then(serde_json::Value::as_object_mut)
.unwrap()
.remove("files");
assert!(serde_json::from_value::<PackedFileRefsManifest>(missing_files).is_err());
let mut missing_indexed = serde_json::to_value(&packed).unwrap();
missing_indexed
.get_mut("file_refs")
.and_then(serde_json::Value::as_object_mut)
.and_then(|file_refs| file_refs.values_mut().next())
.and_then(serde_json::Value::as_object_mut)
.unwrap()
.remove("indexed");
assert!(serde_json::from_value::<PackedFileRefsManifest>(missing_indexed).is_err());
let mut malformed = packed.clone();
malformed.file_refs.get_mut(&region).unwrap().indexed = "!".to_string();
let err = malformed.into_manifest().unwrap_err();
assert!(matches!(
&err,
crate::error::Error::DecodePackedFileRefs { .. }
));
assert!(
err.source()
.is_some_and(|source| source.is::<base64::DecodeError>())
);
assert_eq!(err.status_code(), StatusCode::Unexpected);
assert_eq!(err.retry_hint(), RetryHint::NonRetryable);
let mut malformed = packed;
malformed.file_refs.get_mut(&region).unwrap().indexed =
base64::engine::general_purpose::STANDARD.encode([0; 1]);
let err = malformed.into_manifest().unwrap_err();
assert!(matches!(
&err,
crate::error::Error::InvalidPackedFileRefs { .. }
));
assert!(err.source().is_none());
assert_eq!(err.status_code(), StatusCode::Unexpected);
assert_eq!(err.retry_hint(), RetryHint::NonRetryable);
}
#[test]
fn test_packed_file_refs_is_smaller_than_legacy() {
let region = RegionId::new(7, 3);
let mut manifest = FileRefsManifest::default();
manifest.file_refs.insert(
region,
(0..500)
.map(|i| {
let id =
FileId::parse_str(&format!("00000000-0000-0000-0000-{i:012}")).unwrap();
FileRef::new(region, id, None)
})
.collect(),
);
let legacy = serde_json::to_vec(&manifest).unwrap().len();
let packed = serde_json::to_vec(&PackedFileRefsManifest::from_manifest(&manifest))
.unwrap()
.len();
assert!(packed * 2 < legacy, "packed={packed}, legacy={legacy}");
}
#[test]
fn test_serialize_get_file_refs_instruction_reply() {
let mut manifest = FileRefsManifest::default();
src/common/meta/src/reconciliation/event.rs
+353 -5
View File
@@ -28,6 +28,10 @@ use store_api::storage::TableId;
use crate::reconciliation::ResolveStrategy;
/// Stable event type stored for catalog reconciliation procedures.
pub(crate) const RECONCILE_CATALOG_EVENT_TYPE: &str = "reconcile_catalog";
/// Stable event type stored for database reconciliation procedures.
pub(crate) const RECONCILE_DATABASE_EVENT_TYPE: &str = "reconcile_database";
/// Stable event type stored for logical table reconciliation procedures.
pub(crate) const RECONCILE_LOGICAL_TABLES_EVENT_TYPE: &str = "reconcile_logical_tables";
/// Stable event type stored for physical table reconciliation procedures.
@@ -45,6 +49,23 @@ pub(crate) struct ReconciliationLocator {
}
impl ReconciliationLocator {
/// Creates a locator for a catalog.
pub(crate) fn catalog(catalog_name: &str) -> Self {
Self {
catalog_name: Some(catalog_name.to_string()),
..Default::default()
}
}
/// Creates a locator for a database.
pub(crate) fn database(catalog_name: &str, schema_name: &str) -> Self {
Self {
catalog_name: Some(catalog_name.to_string()),
schema_name: Some(schema_name.to_string()),
..Default::default()
}
}
/// Creates a locator for a physical table with its fully qualified name and ID.
pub(crate) fn physical_table(
catalog_name: &str,
@@ -101,6 +122,229 @@ impl ReconciliationLocator {
}
}
#[derive(Debug, Serialize)]
#[serde(untagged)]
enum ReconcileCatalogPayload {
Submitted(CatalogSubmittedPayload),
Result(CatalogResultPayload),
}
#[derive(Debug, Serialize)]
struct CatalogSubmittedPayload {
version: u8,
resolve_strategy: &'static str,
fail_fast: bool,
parallelism: usize,
}
#[derive(Debug, Serialize)]
struct CatalogResultPayload {
version: u8,
complete: bool,
processed_database_count: usize,
succeeded_database_count: usize,
failed_database_count: usize,
}
/// Event representation for catalog reconciliation.
#[derive(Debug)]
pub(crate) struct ReconcileCatalogEvent {
locator: ReconciliationLocator,
payload: Option<ReconcileCatalogPayload>,
}
impl ReconcileCatalogEvent {
/// Builds the bounded intent event emitted when catalog reconciliation is submitted.
pub(crate) fn submitted(
locator: ReconciliationLocator,
resolve_strategy: ResolveStrategy,
fail_fast: bool,
parallelism: usize,
) -> Self {
Self {
locator,
payload: Some(ReconcileCatalogPayload::Submitted(
CatalogSubmittedPayload {
version: PAYLOAD_VERSION,
resolve_strategy: resolve_strategy_name(resolve_strategy),
fail_fast,
parallelism,
},
)),
}
}
/// Builds a terminal event from the existing volatile reconciliation metrics.
///
/// Metrics are best-effort observations from the current process and reset on recovery.
pub(crate) fn result(
locator: ReconciliationLocator,
complete: bool,
succeeded_database_count: usize,
failed_database_count: usize,
) -> Self {
Self {
locator,
payload: Some(ReconcileCatalogPayload::Result(CatalogResultPayload {
version: PAYLOAD_VERSION,
complete,
processed_database_count: succeeded_database_count + failed_database_count,
succeeded_database_count,
failed_database_count,
})),
}
}
/// Builds a catalog lifecycle event whose reconciliation payload is null.
pub(crate) fn lifecycle(locator: ReconciliationLocator) -> Self {
Self {
locator,
payload: None,
}
}
}
impl Event for ReconcileCatalogEvent {
fn event_type(&self) -> &str {
RECONCILE_CATALOG_EVENT_TYPE
}
fn json_payload(&self) -> Result<serde_json::Value> {
match &self.payload {
Some(payload) => serde_json::to_value(payload).context(SerializeEventSnafu),
None => Ok(serde_json::Value::Null),
}
}
fn extra_schema(&self) -> Vec<ColumnSchema> {
ReconciliationLocator::schema()
}
fn extra_rows(&self) -> Result<Vec<Row>> {
Ok(vec![self.locator.row()])
}
fn as_any(&self) -> &dyn Any {
self
}
}
#[derive(Debug, Serialize)]
#[serde(untagged)]
enum ReconcileDatabasePayload {
Submitted(DatabaseSubmittedPayload),
Result(DatabaseResultPayload),
}
#[derive(Debug, Serialize)]
struct DatabaseSubmittedPayload {
version: u8,
resolve_strategy: &'static str,
fail_fast: bool,
parallelism: usize,
is_subprocedure: bool,
}
#[derive(Debug, Serialize)]
struct DatabaseResultPayload {
version: u8,
complete: bool,
processed_table_count: usize,
succeeded_table_count: usize,
failed_table_count: usize,
succeeded_subprocedure_count: usize,
failed_subprocedure_count: usize,
}
/// Event representation for database reconciliation.
#[derive(Debug)]
pub(crate) struct ReconcileDatabaseEvent {
locator: ReconciliationLocator,
payload: Option<ReconcileDatabasePayload>,
}
impl ReconcileDatabaseEvent {
/// Builds the bounded intent event emitted when database reconciliation is submitted.
pub(crate) fn submitted(
locator: ReconciliationLocator,
resolve_strategy: ResolveStrategy,
fail_fast: bool,
parallelism: usize,
is_subprocedure: bool,
) -> Self {
Self {
locator,
payload: Some(ReconcileDatabasePayload::Submitted(
DatabaseSubmittedPayload {
version: PAYLOAD_VERSION,
resolve_strategy: resolve_strategy_name(resolve_strategy),
fail_fast,
parallelism,
is_subprocedure,
},
)),
}
}
/// Builds a terminal event from the existing volatile reconciliation metrics.
///
/// Metrics are best-effort observations from the current process and reset on recovery.
pub(crate) fn result(
locator: ReconciliationLocator,
complete: bool,
succeeded_table_count: usize,
failed_table_count: usize,
succeeded_subprocedure_count: usize,
failed_subprocedure_count: usize,
) -> Self {
Self {
locator,
payload: Some(ReconcileDatabasePayload::Result(DatabaseResultPayload {
version: PAYLOAD_VERSION,
complete,
processed_table_count: succeeded_table_count + failed_table_count,
succeeded_table_count,
failed_table_count,
succeeded_subprocedure_count,
failed_subprocedure_count,
})),
}
}
/// Builds a database lifecycle event whose reconciliation payload is null.
pub(crate) fn lifecycle(locator: ReconciliationLocator) -> Self {
Self {
locator,
payload: None,
}
}
}
impl Event for ReconcileDatabaseEvent {
fn event_type(&self) -> &str {
RECONCILE_DATABASE_EVENT_TYPE
}
fn json_payload(&self) -> Result<serde_json::Value> {
match &self.payload {
Some(payload) => serde_json::to_value(payload).context(SerializeEventSnafu),
None => Ok(serde_json::Value::Null),
}
}
fn extra_schema(&self) -> Vec<ColumnSchema> {
ReconciliationLocator::schema()
}
fn extra_rows(&self) -> Result<Vec<Row>> {
Ok(vec![self.locator.row()])
}
fn as_any(&self) -> &dyn Any {
self
}
}
#[derive(Debug, Serialize)]
#[serde(untagged)]
enum ReconcileTablePayload {
@@ -355,12 +599,14 @@ mod tests {
#[test]
fn reconciliation_events_use_the_shared_locator_contract() {
let catalog = ReconcileCatalogEvent::lifecycle(ReconciliationLocator::catalog("greptime"));
assert_eq!(catalog.event_type(), RECONCILE_CATALOG_EVENT_TYPE);
let table = ReconcileTableEvent::table_lifecycle(ReconciliationLocator::physical_table(
"greptime", "public", "metrics", 42,
));
assert_eq!(table.event_type(), RECONCILE_TABLE_EVENT_TYPE);
assert_eq!(
table
catalog
.extra_schema()
.into_iter()
.map(|column| {
@@ -375,22 +621,22 @@ mod tests {
(
"catalog_name".to_string(),
ColumnDataType::String,
SemanticType::Field
SemanticType::Field,
),
(
"schema_name".to_string(),
ColumnDataType::String,
SemanticType::Field
SemanticType::Field,
),
(
"table_name".to_string(),
ColumnDataType::String,
SemanticType::Field
SemanticType::Field,
),
(
"table_id".to_string(),
ColumnDataType::Uint32,
SemanticType::Field
SemanticType::Field,
),
(
"physical_table_id".to_string(),
@@ -399,6 +645,40 @@ mod tests {
),
]
);
assert_eq!(
catalog.extra_rows().unwrap(),
vec![Row {
values: vec![
ValueData::StringValue("greptime".to_string()).into(),
Value::default(),
Value::default(),
Value::default(),
Value::default(),
],
}]
);
assert_eq!(catalog.json_payload().unwrap(), serde_json::Value::Null);
let database = ReconcileDatabaseEvent::lifecycle(ReconciliationLocator::database(
"greptime", "public",
));
assert_eq!(database.event_type(), RECONCILE_DATABASE_EVENT_TYPE);
assert_eq!(database.extra_schema(), catalog.extra_schema());
assert_eq!(
database.extra_rows().unwrap(),
vec![Row {
values: vec![
ValueData::StringValue("greptime".to_string()).into(),
ValueData::StringValue("public".to_string()).into(),
Value::default(),
Value::default(),
Value::default(),
],
}]
);
assert_eq!(database.json_payload().unwrap(), serde_json::Value::Null);
assert_eq!(table.extra_schema(), catalog.extra_schema());
assert_eq!(
table.extra_rows().unwrap(),
vec![Row {
@@ -458,6 +738,22 @@ mod tests {
(ResolveStrategy::UseMetasrv, "use_metasrv"),
(ResolveStrategy::AbortOnConflict, "abort_on_conflict"),
] {
let catalog = ReconcileCatalogEvent::submitted(
ReconciliationLocator::catalog("greptime"),
strategy,
true,
16,
);
assert_eq!(
catalog.json_payload().unwrap(),
json!({
"version": 1,
"resolve_strategy": expected,
"fail_fast": true,
"parallelism": 16,
})
);
let table = ReconcileTableEvent::table_submitted(
ReconciliationLocator::physical_table("greptime", "public", "metrics", 42),
strategy,
@@ -473,6 +769,24 @@ mod tests {
);
}
let database = ReconcileDatabaseEvent::submitted(
ReconciliationLocator::database("greptime", "public"),
ResolveStrategy::UseMetasrv,
false,
64,
true,
);
assert_eq!(
database.json_payload().unwrap(),
json!({
"version": 1,
"resolve_strategy": "use_metasrv",
"fail_fast": false,
"parallelism": 64,
"is_subprocedure": true,
})
);
let logical_tables = ReconcileLogicalTablesEvent::submitted(
vec![
ReconciliationLocator::logical_table("greptime", "public", "cpu", 43, 42),
@@ -492,6 +806,40 @@ mod tests {
#[test]
fn terminal_payloads_distinguish_complete_and_partial_results() {
let catalog =
ReconcileCatalogEvent::result(ReconciliationLocator::catalog("greptime"), true, 3, 1);
assert_eq!(
catalog.json_payload().unwrap(),
json!({
"version": 1,
"complete": true,
"processed_database_count": 4,
"succeeded_database_count": 3,
"failed_database_count": 1,
})
);
let database = ReconcileDatabaseEvent::result(
ReconciliationLocator::database("greptime", "public"),
false,
5,
2,
4,
1,
);
assert_eq!(
database.json_payload().unwrap(),
json!({
"version": 1,
"complete": false,
"processed_table_count": 7,
"succeeded_table_count": 5,
"failed_table_count": 2,
"succeeded_subprocedure_count": 4,
"failed_subprocedure_count": 1,
})
);
let table = ReconcileTableEvent::table_result(
ReconciliationLocator::physical_table("greptime", "public", "metrics", 42),
false,
src/common/meta/src/reconciliation/reconcile_catalog.rs
+256 -2
View File
@@ -18,8 +18,8 @@ use std::time::Instant;
use common_procedure::error::FromJsonSnafu;
use common_procedure::{
Context as ProcedureContext, Error as ProcedureError, LockKey, Procedure,
Result as ProcedureResult, Status,
Context as ProcedureContext, Error as ProcedureError, EventContext, EventTrigger, LockKey,
Procedure, Result as ProcedureResult, Status,
};
use futures::stream::BoxStream;
use serde::{Deserialize, Serialize};
@@ -31,6 +31,9 @@ use crate::key::TableMetadataManagerRef;
use crate::lock_key::CatalogLock;
use crate::metrics;
use crate::node_manager::NodeManagerRef;
use crate::reconciliation::event::{
RECONCILE_CATALOG_EVENT_TYPE, ReconcileCatalogEvent, ReconciliationLocator,
};
use crate::reconciliation::reconcile_catalog::start::ReconcileCatalogStart;
use crate::reconciliation::reconcile_table::resolve_column_metadata::ResolveStrategy;
use crate::reconciliation::utils::{
@@ -216,6 +219,43 @@ impl Procedure for ReconcileCatalogProcedure {
LockKey::new(vec![CatalogLock::Write(catalog).into()])
}
fn event(&self, ctx: &EventContext<'_>) -> Option<Box<dyn common_event_recorder::Event>> {
if !ctx.event_type_filter.allows(RECONCILE_CATALOG_EVENT_TYPE) {
return None;
}
let persistent_ctx = &self.context.persistent_ctx;
let locator = ReconciliationLocator::catalog(&persistent_ctx.catalog);
let event = match ctx.trigger {
EventTrigger::Submitted => ReconcileCatalogEvent::submitted(
locator,
persistent_ctx.resolve_strategy,
persistent_ctx.fast_fail,
persistent_ctx.parallelism,
),
EventTrigger::Succeeded => self.result_event(locator, true),
EventTrigger::Failed | EventTrigger::Poisoned => self.result_event(locator, false),
_ => ReconcileCatalogEvent::lifecycle(locator),
};
Some(Box::new(event))
}
}
impl ReconcileCatalogProcedure {
fn result_event(
&self,
locator: ReconciliationLocator,
complete: bool,
) -> ReconcileCatalogEvent {
let metrics = &self.context.volatile_ctx.metrics;
ReconcileCatalogEvent::result(
locator,
complete,
metrics.succeeded_databases,
metrics.failed_databases,
)
}
}
#[async_trait::async_trait]
@@ -235,3 +275,217 @@ pub(crate) trait State: Sync + Send + Debug {
fn as_any(&self) -> &dyn Any;
}
#[cfg(test)]
mod tests {
use std::sync::Arc;
use common_event_recorder::{EventTypeFilter, EventTypeFilterRef};
use common_procedure::{
ChildSubmissionOutcome, EventContext, EventTrigger, Procedure, ProcedureId, ProcedureState,
RetryPhase,
};
use serde_json::{Value, json};
use super::*;
use crate::reconciliation::event::RECONCILE_DATABASE_EVENT_TYPE;
use crate::test_util::{MockDatanodeManager, new_ddl_context};
struct CatalogEventHarness {
procedure_id: ProcedureId,
lifecycle_state: ProcedureState,
event_type_filter: EventTypeFilterRef,
}
impl CatalogEventHarness {
fn all() -> Self {
Self {
procedure_id: ProcedureId::random(),
lifecycle_state: ProcedureState::Running,
event_type_filter: Arc::new(EventTypeFilter::All),
}
}
fn selected(event_types: impl IntoIterator<Item = &'static str>) -> Self {
Self {
event_type_filter: Arc::new(EventTypeFilter::Only(
event_types.into_iter().map(str::to_string).collect(),
)),
..Self::all()
}
}
fn event(
&self,
procedure: &dyn Procedure,
trigger: EventTrigger,
) -> Option<Box<dyn common_event_recorder::Event>> {
procedure.event(&EventContext {
procedure_id: self.procedure_id,
lifecycle_state: &self.lifecycle_state,
trigger,
event_type_filter: self.event_type_filter.clone(),
event_context: None,
})
}
}
#[test]
fn catalog_submitted_event_reports_intent() {
let submitted = CatalogEventHarness::all()
.event(&test_procedure(), EventTrigger::Submitted)
.unwrap();
assert_eq!(submitted.event_type(), RECONCILE_CATALOG_EVENT_TYPE);
assert_eq!(
submitted.json_payload().unwrap(),
json!({
"version": 1,
"resolve_strategy": "use_latest",
"fail_fast": false,
"parallelism": 8,
})
);
}
#[test]
fn catalog_non_terminal_lifecycle_events_have_null_payloads() {
let events = CatalogEventHarness::all();
let mut procedure = test_procedure();
procedure.context.volatile_ctx.metrics = populated_metrics();
for trigger in [
EventTrigger::Recovered,
EventTrigger::ChildSubmitted {
procedure_id: ProcedureId::random(),
outcome: ChildSubmissionOutcome::Accepted,
},
EventTrigger::Retrying {
phase: RetryPhase::Execute,
attempt: 1,
},
EventTrigger::RollingBack,
] {
assert_eq!(
events
.event(&procedure, trigger)
.unwrap()
.json_payload()
.unwrap(),
Value::Null
);
}
}
#[test]
fn catalog_terminal_events_report_existing_metrics() {
let events = CatalogEventHarness::all();
let mut procedure = test_procedure();
procedure.context.volatile_ctx.metrics = populated_metrics();
for (trigger, complete) in [
(EventTrigger::Succeeded, true),
(EventTrigger::Failed, false),
(EventTrigger::Poisoned, false),
] {
assert_eq!(
events
.event(&procedure, trigger)
.unwrap()
.json_payload()
.unwrap(),
json!({
"version": 1,
"complete": complete,
"processed_database_count": 4,
"succeeded_database_count": 3,
"failed_database_count": 1,
})
);
}
}
#[test]
fn catalog_event_filtering_uses_the_catalog_event_type() {
let procedure = test_procedure();
assert!(
CatalogEventHarness::selected([RECONCILE_CATALOG_EVENT_TYPE])
.event(&procedure, EventTrigger::Submitted)
.is_some()
);
assert!(
CatalogEventHarness::selected([RECONCILE_DATABASE_EVENT_TYPE])
.event(&procedure, EventTrigger::Submitted)
.is_none()
);
assert!(
CatalogEventHarness::selected([])
.event(&procedure, EventTrigger::Submitted)
.is_none()
);
}
#[test]
fn catalog_recovery_preserves_locator_and_resets_metrics() {
let events = CatalogEventHarness::all();
let mut procedure = test_procedure();
procedure.context.volatile_ctx.metrics = populated_metrics();
let original_dump = procedure.dump().unwrap();
procedure.context.volatile_ctx.metrics = ReconcileCatalogMetrics::default();
assert_eq!(procedure.dump().unwrap(), original_dump);
let loaded = ReconcileCatalogProcedure::from_json(test_context(), &original_dump).unwrap();
assert_eq!(loaded.dump().unwrap(), original_dump);
assert_eq!(
events
.event(&loaded, EventTrigger::Recovered)
.unwrap()
.extra_rows()
.unwrap(),
events
.event(&procedure, EventTrigger::Submitted)
.unwrap()
.extra_rows()
.unwrap(),
);
assert_eq!(
events
.event(&loaded, EventTrigger::Succeeded)
.unwrap()
.json_payload()
.unwrap(),
json!({
"version": 1,
"complete": true,
"processed_database_count": 0,
"succeeded_database_count": 0,
"failed_database_count": 0,
})
);
}
fn populated_metrics() -> ReconcileCatalogMetrics {
ReconcileCatalogMetrics {
succeeded_databases: 3,
failed_databases: 1,
}
}
fn test_procedure() -> ReconcileCatalogProcedure {
ReconcileCatalogProcedure::new(
test_context(),
"greptime".to_string(),
false,
ResolveStrategy::UseLatest,
8,
)
}
fn test_context() -> Context {
let ddl_context = new_ddl_context(Arc::new(MockDatanodeManager::new(())));
Context {
node_manager: ddl_context.node_manager,
table_metadata_manager: ddl_context.table_metadata_manager,
cache_invalidator: ddl_context.cache_invalidator,
}
}
}
src/common/meta/src/reconciliation/reconcile_database.rs
+272 -2
View File
@@ -25,8 +25,8 @@ use std::time::Instant;
use async_trait::async_trait;
use common_procedure::error::{FromJsonSnafu, ToJsonSnafu};
use common_procedure::{
Context as ProcedureContext, Error as ProcedureError, LockKey, Procedure,
Result as ProcedureResult, Status,
Context as ProcedureContext, Error as ProcedureError, EventContext, EventTrigger, LockKey,
Procedure, Result as ProcedureResult, Status,
};
use futures::stream::BoxStream;
use serde::{Deserialize, Serialize};
@@ -41,6 +41,9 @@ use crate::key::table_name::TableNameValue;
use crate::lock_key::{CatalogLock, SchemaLock};
use crate::metrics;
use crate::node_manager::NodeManagerRef;
use crate::reconciliation::event::{
RECONCILE_DATABASE_EVENT_TYPE, ReconcileDatabaseEvent, ReconciliationLocator,
};
use crate::reconciliation::reconcile_database::start::ReconcileDatabaseStart;
use crate::reconciliation::reconcile_table::resolve_column_metadata::ResolveStrategy;
use crate::reconciliation::utils::{
@@ -264,6 +267,47 @@ impl Procedure for ReconcileDatabaseProcedure {
SchemaLock::write(catalog, schema).into(),
])
}
fn event(&self, ctx: &EventContext<'_>) -> Option<Box<dyn common_event_recorder::Event>> {
if !ctx.event_type_filter.allows(RECONCILE_DATABASE_EVENT_TYPE) {
return None;
}
let persistent_ctx = &self.context.persistent_ctx;
let locator =
ReconciliationLocator::database(&persistent_ctx.catalog, &persistent_ctx.schema);
let event = match ctx.trigger {
EventTrigger::Submitted => ReconcileDatabaseEvent::submitted(
locator,
persistent_ctx.resolve_strategy,
persistent_ctx.fail_fast,
persistent_ctx.parallelism,
persistent_ctx.is_subprocedure,
),
EventTrigger::Succeeded => self.result_event(locator, true),
EventTrigger::Failed | EventTrigger::Poisoned => self.result_event(locator, false),
_ => ReconcileDatabaseEvent::lifecycle(locator),
};
Some(Box::new(event))
}
}
impl ReconcileDatabaseProcedure {
fn result_event(
&self,
locator: ReconciliationLocator,
complete: bool,
) -> ReconcileDatabaseEvent {
let metrics = self.context.metrics();
ReconcileDatabaseEvent::result(
locator,
complete,
metrics.succeeded_tables,
metrics.failed_tables,
metrics.succeeded_procedures,
metrics.failed_procedures,
)
}
}
#[async_trait::async_trait]
@@ -283,3 +327,229 @@ pub(crate) trait State: Sync + Send + Debug {
fn as_any(&self) -> &dyn Any;
}
#[cfg(test)]
mod tests {
use std::sync::Arc;
use common_event_recorder::{EventTypeFilter, EventTypeFilterRef};
use common_procedure::{
ChildSubmissionOutcome, EventContext, EventTrigger, Procedure, ProcedureId, ProcedureState,
RetryPhase,
};
use serde_json::{Value, json};
use super::*;
use crate::reconciliation::event::RECONCILE_CATALOG_EVENT_TYPE;
use crate::test_util::{MockDatanodeManager, new_ddl_context};
struct DatabaseEventHarness {
procedure_id: ProcedureId,
lifecycle_state: ProcedureState,
event_type_filter: EventTypeFilterRef,
}
impl DatabaseEventHarness {
fn all() -> Self {
Self {
procedure_id: ProcedureId::random(),
lifecycle_state: ProcedureState::Running,
event_type_filter: Arc::new(EventTypeFilter::All),
}
}
fn selected(event_types: impl IntoIterator<Item = &'static str>) -> Self {
Self {
event_type_filter: Arc::new(EventTypeFilter::Only(
event_types.into_iter().map(str::to_string).collect(),
)),
..Self::all()
}
}
fn event(
&self,
procedure: &dyn Procedure,
trigger: EventTrigger,
) -> Option<Box<dyn common_event_recorder::Event>> {
procedure.event(&EventContext {
procedure_id: self.procedure_id,
lifecycle_state: &self.lifecycle_state,
trigger,
event_type_filter: self.event_type_filter.clone(),
event_context: None,
})
}
}
#[test]
fn database_submitted_events_cover_root_and_child_intent() {
let events = DatabaseEventHarness::all();
let root = test_procedure(false);
let child = test_procedure(true);
for (procedure, is_subprocedure) in [(&root, false), (&child, true)] {
let submitted = events.event(procedure, EventTrigger::Submitted).unwrap();
assert_eq!(submitted.event_type(), RECONCILE_DATABASE_EVENT_TYPE);
assert_eq!(
submitted.json_payload().unwrap(),
json!({
"version": 1,
"resolve_strategy": "use_metasrv",
"fail_fast": false,
"parallelism": 8,
"is_subprocedure": is_subprocedure,
})
);
}
}
#[test]
fn database_non_terminal_lifecycle_events_have_null_payloads() {
let events = DatabaseEventHarness::all();
let mut procedure = test_procedure(true);
procedure.context.volatile_ctx.metrics = populated_metrics();
for trigger in [
EventTrigger::Recovered,
EventTrigger::ChildSubmitted {
procedure_id: ProcedureId::random(),
outcome: ChildSubmissionOutcome::Accepted,
},
EventTrigger::Retrying {
phase: RetryPhase::Execute,
attempt: 2,
},
EventTrigger::RollingBack,
] {
assert_eq!(
events
.event(&procedure, trigger)
.unwrap()
.json_payload()
.unwrap(),
Value::Null
);
}
}
#[test]
fn database_terminal_events_report_existing_metrics() {
let events = DatabaseEventHarness::all();
let mut procedure = test_procedure(true);
procedure.context.volatile_ctx.metrics = populated_metrics();
for (trigger, complete) in [
(EventTrigger::Succeeded, true),
(EventTrigger::Failed, false),
(EventTrigger::Poisoned, false),
] {
assert_eq!(
events
.event(&procedure, trigger)
.unwrap()
.json_payload()
.unwrap(),
json!({
"version": 1,
"complete": complete,
"processed_table_count": 8,
"succeeded_table_count": 5,
"failed_table_count": 3,
"succeeded_subprocedure_count": 3,
"failed_subprocedure_count": 2,
})
);
}
}
#[test]
fn database_event_filtering_uses_the_database_event_type() {
let procedure = test_procedure(false);
assert!(
DatabaseEventHarness::selected([RECONCILE_DATABASE_EVENT_TYPE])
.event(&procedure, EventTrigger::Submitted)
.is_some()
);
assert!(
DatabaseEventHarness::selected([RECONCILE_CATALOG_EVENT_TYPE])
.event(&procedure, EventTrigger::Submitted)
.is_none()
);
assert!(
DatabaseEventHarness::selected([])
.event(&procedure, EventTrigger::Submitted)
.is_none()
);
}
#[test]
fn database_recovery_preserves_locator_and_resets_metrics() {
let events = DatabaseEventHarness::all();
let mut procedure = test_procedure(false);
procedure.context.volatile_ctx.metrics = populated_metrics();
let original_dump = procedure.dump().unwrap();
procedure.context.volatile_ctx.metrics = ReconcileDatabaseMetrics::default();
assert_eq!(procedure.dump().unwrap(), original_dump);
let loaded = ReconcileDatabaseProcedure::from_json(test_context(), &original_dump).unwrap();
assert_eq!(loaded.dump().unwrap(), original_dump);
assert_eq!(
events
.event(&loaded, EventTrigger::Recovered)
.unwrap()
.extra_rows()
.unwrap(),
events
.event(&procedure, EventTrigger::Submitted)
.unwrap()
.extra_rows()
.unwrap(),
);
assert_eq!(
events
.event(&loaded, EventTrigger::Succeeded)
.unwrap()
.json_payload()
.unwrap(),
json!({
"version": 1,
"complete": true,
"processed_table_count": 0,
"succeeded_table_count": 0,
"failed_table_count": 0,
"succeeded_subprocedure_count": 0,
"failed_subprocedure_count": 0,
})
);
}
fn populated_metrics() -> ReconcileDatabaseMetrics {
ReconcileDatabaseMetrics {
succeeded_tables: 5,
failed_tables: 3,
succeeded_procedures: 3,
failed_procedures: 2,
}
}
fn test_procedure(is_subprocedure: bool) -> ReconcileDatabaseProcedure {
ReconcileDatabaseProcedure::new(
test_context(),
"greptime".to_string(),
"public".to_string(),
false,
8,
ResolveStrategy::UseMetasrv,
is_subprocedure,
)
}
fn test_context() -> Context {
let ddl_context = new_ddl_context(Arc::new(MockDatanodeManager::new(())));
Context {
node_manager: ddl_context.node_manager,
table_metadata_manager: ddl_context.table_metadata_manager,
cache_invalidator: ddl_context.cache_invalidator,
}
}
}
src/common/pprof/Cargo.toml
+7 -1
View File
@@ -11,11 +11,17 @@ snafu.workspace = true
tokio.workspace = true
[target.'cfg(unix)'.dependencies]
pprof = { version = "0.14", features = [
pprof = { version = "0.15", features = [
"flamegraph",
"prost-codec",
"protobuf",
] }
# framehop-unwinder is only available on x86_64/aarch64 Linux/macOS.
# Enabling it on other targets (e.g. riscv64) would disable the
# backtrace-rs fallback without providing a TraceImpl, breaking the build.
[target.'cfg(all(unix, any(target_arch = "x86_64", target_arch = "aarch64"), any(target_os = "linux", target_os = "macos")))'.dependencies]
pprof = { version = "0.15", features = ["framehop-unwinder"] }
[lints]
workspace = true
src/common/telemetry/src/logging/file_retention.rs
+7 -2
View File
@@ -453,7 +453,9 @@ pub(crate) fn build_file_appender(
#[cfg(test)]
mod tests {
use std::fs::{self, File};
#[cfg(unix)]
use std::fs;
use std::fs::File;
use std::io::Write;
#[cfg(unix)]
use std::os::unix::fs::symlink;
@@ -464,8 +466,11 @@ mod tests {
use tempfile::TempDir;
use super::FileIndex;
#[cfg(unix)]
use crate::logging::LoggingOptions;
use crate::logging::file_retention::{DirectoryRetention, LogFileKind, build_file_appender};
#[cfg(unix)]
use crate::logging::file_retention::build_file_appender;
use crate::logging::file_retention::{DirectoryRetention, LogFileKind};
fn write_file(path: &Path, contents: &[u8]) {
let mut file = File::create(path).unwrap();
src/datanode/src/heartbeat.rs
+14 -3
View File
@@ -18,6 +18,7 @@ use std::sync::atomic::{AtomicBool, Ordering};
use std::time::Duration;
use api::v1::meta::heartbeat_request::NodeWorkloads;
use api::v1::meta::mailbox_message::Payload;
use api::v1::meta::{DatanodeWorkloads, HeartbeatRequest, NodeInfo, Peer, RegionRole, RegionStat};
use common_base::Plugins;
use common_meta::cache_invalidator::CacheInvalidatorRef;
@@ -146,7 +147,15 @@ impl HeartbeatTask {
None
}) {
if let Some(msg) = res.mailbox_message.as_ref() {
info!("Received mailbox message: {msg:?}, meta_client id: {client_id:?}");
let payload_len = msg
.payload
.as_ref()
.map(|Payload::Json(payload)| payload.len())
.unwrap_or_default();
info!(
message_id = msg.id,
payload_len, client_id, "Received mailbox message"
);
}
if let Some(lease) = res.region_lease.as_ref() {
metrics::LAST_RECEIVED_HEARTBEAT_ELAPSED
@@ -193,7 +202,8 @@ impl HeartbeatTask {
ctx: HeartbeatResponseHandlerContext,
handler_executor: HeartbeatResponseHandlerExecutorRef,
) -> Result<()> {
trace!("Heartbeat response: {:?}", ctx.response);
let mailbox_message_id = ctx.response.mailbox_message.as_ref().map(|msg| msg.id);
trace!(?mailbox_message_id, "Handling heartbeat response");
handler_executor
.handle(ctx)
.await
@@ -379,7 +389,8 @@ impl HeartbeatTask {
.set(last_sent.elapsed().as_millis() as i64);
// Resets the timer.
last_sent = Instant::now();
debug!("Sending heartbeat request: {:?}", req);
let mailbox_message_id = req.mailbox_message.as_ref().map(|msg| msg.id);
debug!(?mailbox_message_id, "Sending heartbeat request");
if let Err(e) = tx.send(req).await {
error!(e; "Failed to send heartbeat to metasrv");
match Self::create_streams(
src/datanode/src/heartbeat/handler.rs
+12 -2
View File
@@ -41,9 +41,9 @@ use crate::heartbeat::handler::apply_staging_manifest::ApplyStagingManifestsHand
use crate::heartbeat::handler::close_region::CloseRegionsHandler;
use crate::heartbeat::handler::downgrade_region::DowngradeRegionsHandler;
use crate::heartbeat::handler::enter_staging::EnterStagingRegionsHandler;
use crate::heartbeat::handler::file_ref::GetFileRefsHandler;
use crate::heartbeat::handler::file_ref::{GetFileRefsHandler, GetPackedFileRefsHandler};
use crate::heartbeat::handler::flush_region::FlushRegionsHandler;
use crate::heartbeat::handler::gc_worker::GcRegionsHandler;
use crate::heartbeat::handler::gc_worker::{GcRegionsHandler, PackedGcRegionsHandler};
use crate::heartbeat::handler::open_region::OpenRegionsHandler;
use crate::heartbeat::handler::remap_manifest::RemapManifestHandler;
use crate::heartbeat::handler::sync_region::SyncRegionHandler;
@@ -135,7 +135,11 @@ impl RegionHeartbeatResponseHandler {
.into(),
))),
Instruction::GetFileRefs(_) => Ok(Some(Box::new(GetFileRefsHandler.into()))),
Instruction::GetPackedFileRefs(_) => {
Ok(Some(Box::new(GetPackedFileRefsHandler.into())))
}
Instruction::GcRegions(_) => Ok(Some(Box::new(GcRegionsHandler.into()))),
Instruction::PackedGcRegions(_) => Ok(Some(Box::new(PackedGcRegionsHandler.into()))),
Instruction::InvalidateCaches(_) => InvalidHeartbeatResponseSnafu.fail(),
Instruction::Suspend => Ok(None),
Instruction::EnterStagingRegions(_) => {
@@ -159,7 +163,9 @@ pub enum InstructionHandlers {
DowngradeRegions(DowngradeRegionsHandler),
UpgradeRegions(UpgradeRegionsHandler),
GetFileRefs(GetFileRefsHandler),
GetPackedFileRefs(GetPackedFileRefsHandler),
GcRegions(GcRegionsHandler),
PackedGcRegions(PackedGcRegionsHandler),
EnterStagingRegions(EnterStagingRegionsHandler),
SyncRegions(SyncRegionHandler),
RemapManifest(RemapManifestHandler),
@@ -192,7 +198,9 @@ impl_from_handler!(
DowngradeRegionsHandler => DowngradeRegions,
UpgradeRegionsHandler => UpgradeRegions,
GetFileRefsHandler => GetFileRefs,
GetPackedFileRefsHandler => GetPackedFileRefs,
GcRegionsHandler => GcRegions,
PackedGcRegionsHandler => PackedGcRegions,
EnterStagingRegionsHandler => EnterStagingRegions,
SyncRegionHandler => SyncRegions,
RemapManifestHandler => RemapManifest,
@@ -240,7 +248,9 @@ dispatch_instr!(
DowngradeRegions => DowngradeRegions,
UpgradeRegions => UpgradeRegions,
GetFileRefs => GetFileRefs,
GetPackedFileRefs => GetPackedFileRefs,
GcRegions => GcRegions,
PackedGcRegions => PackedGcRegions,
EnterStagingRegions => EnterStagingRegions,
SyncRegions => SyncRegions,
RemapManifest => RemapManifest,
src/datanode/src/heartbeat/handler/file_ref.rs
+55 -1
View File
@@ -13,7 +13,10 @@
// limitations under the License.
use common_error::ext::ErrorExt;
use common_meta::instruction::{GetFileRefs, GetFileRefsReply, InstructionError, InstructionReply};
use common_meta::instruction::{
GetFileRefs, GetFileRefsReply, GetPackedFileRefs, GetPackedFileRefsReply, InstructionError,
InstructionReply, PackedFileRefsManifest,
};
use store_api::storage::FileRefsManifest;
use crate::heartbeat::handler::{HandlerContext, InstructionHandler};
@@ -65,3 +68,54 @@ impl InstructionHandler for GetFileRefsHandler {
}
}
}
pub struct GetPackedFileRefsHandler;
#[async_trait::async_trait]
impl InstructionHandler for GetPackedFileRefsHandler {
type Instruction = GetPackedFileRefs;
async fn handle(
&self,
ctx: &HandlerContext,
get_file_refs: Self::Instruction,
) -> Option<InstructionReply> {
let region_server = &ctx.region_server;
let Some(mito_engine) = region_server.mito_engine() else {
return Some(InstructionReply::GetPackedFileRefs(
GetPackedFileRefsReply {
packed_file_refs_manifest: PackedFileRefsManifest::default(),
success: false,
error: Some(InstructionError::legacy_internal_retryable(
"MitoEngine not found",
)),
},
));
};
match mito_engine
.get_snapshot_of_file_refs(get_file_refs.query_regions, get_file_refs.related_regions)
.await
{
Ok(all_file_refs) => Some(InstructionReply::GetPackedFileRefs(
GetPackedFileRefsReply {
packed_file_refs_manifest: PackedFileRefsManifest::from_manifest(
&all_file_refs,
),
success: true,
error: None,
},
)),
Err(e) => Some(InstructionReply::GetPackedFileRefs(
GetPackedFileRefsReply {
packed_file_refs_manifest: PackedFileRefsManifest::default(),
success: false,
error: Some(InstructionError::new(
e.status_code(),
format!("Failed to get file refs: {}", e.output_msg()),
e.retry_hint(),
)),
},
)),
}
}
}
src/datanode/src/heartbeat/handler/gc_worker.rs
+33 -2
View File
@@ -15,7 +15,7 @@
use std::collections::{BTreeMap, HashMap};
use std::sync::Arc;
use common_meta::instruction::{GcRegions, GcRegionsReply, InstructionReply};
use common_meta::instruction::{GcRegions, GcRegionsReply, InstructionReply, PackedGcRegions};
use common_meta::key::table_info::TableInfoManager;
use common_meta::key::table_route::TableRouteManager;
use common_telemetry::{debug, warn};
@@ -34,6 +34,7 @@ use crate::error::{GcMitoEngineSnafu, GetMetadataSnafu, Result, UnexpectedSnafu}
use crate::heartbeat::handler::{HandlerContext, InstructionHandler};
pub struct GcRegionsHandler;
pub struct PackedGcRegionsHandler;
#[async_trait::async_trait]
impl InstructionHandler for GcRegionsHandler {
@@ -67,7 +68,7 @@ impl InstructionHandler for GcRegionsHandler {
.push(rid);
}
let file_refs_manifest = gc_regions.file_refs_manifest.clone();
let file_refs_manifest = gc_regions.file_refs_manifest;
let full_file_listing = gc_regions.full_file_listing;
let ctx_clone = ctx.clone();
@@ -138,6 +139,36 @@ impl InstructionHandler for GcRegionsHandler {
}
}
#[async_trait::async_trait]
impl InstructionHandler for PackedGcRegionsHandler {
type Instruction = PackedGcRegions;
async fn handle(
&self,
ctx: &HandlerContext,
instruction: Self::Instruction,
) -> Option<InstructionReply> {
let manifest = match instruction.packed_file_refs_manifest.into_manifest() {
Ok(manifest) => manifest,
Err(err) => {
return Some(InstructionReply::GcRegions(GcRegionsReply {
result: Err(common_meta::instruction::InstructionError::from_error(&err)),
}));
}
};
GcRegionsHandler
.handle(
ctx,
GcRegions {
regions: instruction.regions,
file_refs_manifest: manifest,
full_file_listing: instruction.full_file_listing,
},
)
.await
}
}
impl GcRegionsHandler {
/// Create a GC worker for the given table and region IDs.
async fn create_gc_worker(
src/datanode/src/region_server.rs
+1
View File
@@ -2018,6 +2018,7 @@ mod tests {
assert!(RegionServerInner::is_ingest_request(&RegionRequest::Put(
RegionPutRequest {
skip_wal: false,
rows: rows(),
hint: None,
partition_expr_version: None,
src/datatypes/src/schema/constraint.rs
+91 -10
View File
@@ -23,7 +23,7 @@ use crate::error::{self, Result};
use crate::types::cast;
use crate::value::Value;
use crate::vectors::operations::VectorOp;
use crate::vectors::{TimestampMillisecondVector, VectorRef};
use crate::vectors::{Helper, TimestampMillisecondVector, VectorRef};
pub const CURRENT_TIMESTAMP: &str = "current_timestamp";
pub const CURRENT_TIMESTAMP_FN: &str = "current_timestamp()";
@@ -151,15 +151,20 @@ impl ColumnDefaultConstraint {
ColumnDefaultConstraint::Value(v) => {
ensure!(is_nullable || !v.is_null(), error::NullDefaultSnafu);
// TODO(yingwen):
// 1. For null value, we could use NullVector once it supports custom logical type.
// 2. For non null value, we could use ConstantVector, but it would cause all codes
// attempt to downcast the vector fail if they don't check whether the vector is const
// first.
let mut mutable_vector = data_type.create_mutable_vector(1);
mutable_vector.try_push_value_ref(&v.as_value_ref())?;
let base_vector = mutable_vector.to_vector();
Ok(base_vector.replicate(&[num_rows]))
if let Ok(vector) = v.try_to_scalar_value(data_type).and_then(|scalar| {
Helper::try_from_scalar_value(scalar, num_rows, Some(data_type))
}) {
return Ok(vector);
}
// Some extension values, such as JSON nested in a struct, cannot safely
// round-trip through ScalarValue. Preserve their logical type with the
// type-specific vector builder instead.
let mut mutable_vector = data_type.create_mutable_vector(num_rows);
for _ in 0..num_rows {
mutable_vector.try_push_value_ref(&v.as_value_ref())?;
}
Ok(mutable_vector.to_vector())
}
}
}
@@ -391,6 +396,82 @@ mod tests {
assert_eq!(Value::Int32(10), v);
}
#[test]
fn test_struct_default_null_fields_and_json() {
use crate::types::{StructField, StructType};
use crate::value::StructValue;
let inner_type = StructType::from([StructField::new(
"x",
ConcreteDataType::int32_datatype(),
true,
)]);
let inner = Value::Struct(StructValue::new(vec![Value::Null], inner_type));
let json = crate::json::JsonSettings::default()
.encode(serde_json::json!({"answer": 42}))
.unwrap();
let nested_type = StructType::from([StructField::new("nested", inner.data_type(), true)]);
let json_type = StructType::from([StructField::new("json", json.data_type(), true)]);
let values = [
inner.clone(),
Value::Struct(StructValue::new(vec![inner], nested_type)),
Value::Struct(StructValue::new(vec![], StructType::default())),
Value::Struct(StructValue::new(vec![json], json_type)),
];
for value in values {
let data_type = value.data_type();
// JSON children are read back as their underlying struct values.
let expected = serde_json::Value::try_from(value.clone()).unwrap();
for num_rows in [1, 3] {
let vector = ColumnDefaultConstraint::Value(value.clone())
.create_default_vector(&data_type, false, num_rows)
.unwrap();
assert_eq!(data_type, vector.data_type());
assert_eq!(
data_type.as_arrow_type(),
*vector.to_arrow_array().data_type()
);
assert_eq!(num_rows, vector.len());
assert_eq!(0, vector.null_count());
for row in 0..num_rows {
assert_eq!(
expected,
serde_json::Value::try_from(vector.get(row)).unwrap()
);
}
}
}
}
#[test]
fn test_string_default_preserves_batch_schema() {
use arrow::datatypes::{Field, Schema};
use arrow::record_batch::RecordBatch;
for data_type in [
ConcreteDataType::large_string_datatype(),
ConcreteDataType::utf8_view_datatype(),
] {
let schema = Arc::new(Schema::new(vec![Field::new(
"tag",
data_type.as_arrow_type(),
true,
)]));
for value in [Value::from("greptime"), Value::Null] {
let vector = ColumnDefaultConstraint::Value(value.clone())
.create_default_vector(&data_type, true, 3)
.unwrap();
let batch =
RecordBatch::try_new(schema.clone(), vec![vector.to_arrow_array()]).unwrap();
assert_eq!(3, batch.num_rows());
assert_eq!(data_type, vector.data_type());
for row in 0..batch.num_rows() {
assert_eq!(value, vector.get(row));
}
}
}
}
#[test]
fn test_create_default_vector_by_func() {
let constraint = ColumnDefaultConstraint::Function(CURRENT_TIMESTAMP.to_string());
src/datatypes/src/types/cast.rs
+1 -1
View File
@@ -26,7 +26,7 @@ pub fn cast(src_value: Value, dest_type: &ConcreteDataType) -> Result<Value> {
}
let src_type = src_value.data_type();
let scalar_value = src_value.try_to_scalar_value(&src_type)?;
let new_value = Helper::try_from_scalar_value(scalar_value, 1)?
let new_value = Helper::try_from_scalar_value(scalar_value, 1, None)?
.cast(dest_type)?
.get(0);
Ok(new_value)
src/datatypes/src/value.rs
+79 -3
View File
@@ -1085,11 +1085,21 @@ impl StructValue {
}
fn try_to_scalar_value(&self, output_type: &StructType) -> Result<ScalarValue> {
let output_fields = output_type.fields();
ensure!(
self.items.len() == output_fields.len(),
InconsistentStructFieldsAndItemsSnafu {
field_len: output_fields.len(),
item_len: self.items.len()
}
);
let arrays = self
.items
.iter()
.map(|value| {
let scalar_value = value.try_to_scalar_value(&value.data_type())?;
.zip(output_fields.iter())
.map(|(value, field)| {
// Null values need the declared field type to produce a typed null array.
let scalar_value = value.try_to_scalar_value(field.data_type())?;
scalar_value
.to_array()
.context(ConvertScalarToArrowArraySnafu)
@@ -1097,7 +1107,13 @@ impl StructValue {
.collect::<Result<Vec<Arc<dyn Array>>>>()?;
let fields = output_type.as_arrow_fields();
let struct_array = StructArray::new(fields, arrays, None);
let struct_array =
StructArray::try_new_with_length(fields, arrays, None, 1).map_err(|error| {
error::ToScalarValueSnafu {
reason: error.to_string(),
}
.build()
})?;
Ok(ScalarValue::Struct(Arc::new(struct_array)))
}
}
@@ -3082,6 +3098,66 @@ pub(crate) mod tests {
}
}
#[test]
fn test_struct_scalar_null_fields() {
let struct_type = StructType::from([
StructField::new("x", ConcreteDataType::int32_datatype(), true),
StructField::new("name", ConcreteDataType::string_datatype(), true),
]);
let value = StructValue::new(vec![Value::Null, Value::from("hello")], struct_type.clone());
let ScalarValue::Struct(array) = value.try_to_scalar_value(&struct_type).unwrap() else {
panic!("Expected struct scalar");
};
assert_eq!(1, array.len());
assert_eq!(0, array.null_count());
assert_eq!(&struct_type.as_arrow_fields(), array.fields());
assert_eq!(
ScalarValue::Int32(None),
ScalarValue::try_from_array(array.column(0), 0).unwrap()
);
let nested_type = StructType::from([StructField::new(
"nested",
ConcreteDataType::struct_datatype(struct_type),
true,
)]);
for child in [Value::Struct(value), Value::Null] {
let nested = StructValue::new(vec![child.clone()], nested_type.clone());
let ScalarValue::Struct(array) = nested.try_to_scalar_value(&nested_type).unwrap()
else {
panic!("Expected struct scalar");
};
let vector = crate::vectors::Helper::try_into_vector(array.column(0).clone()).unwrap();
assert_eq!(child, vector.get(0));
}
let empty_type = StructType::default();
let empty = StructValue::new(vec![], empty_type.clone());
let ScalarValue::Struct(array) = empty.try_to_scalar_value(&empty_type).unwrap() else {
panic!("Expected struct scalar");
};
assert_eq!(1, array.len());
assert_eq!(0, array.num_columns());
assert_eq!(0, array.null_count());
}
#[test]
fn test_struct_scalar_invalid_fields() {
let struct_type = StructType::from([StructField::new(
"x",
ConcreteDataType::int32_datatype(),
false,
)]);
for child in [Value::Null, Value::from("wrong type")] {
let value = StructValue::new(vec![child], struct_type.clone());
assert!(value.try_to_scalar_value(&struct_type).is_err());
}
let value = StructValue::new(vec![Value::Int32(1)], struct_type.clone());
assert!(value.try_to_scalar_value(&StructType::default()).is_err());
let empty = StructValue::new(vec![], StructType::default());
assert!(empty.try_to_scalar_value(&struct_type).is_err());
}
#[test]
fn test_timestamp_to_scalar_value() {
assert_eq!(
src/datatypes/src/vectors.rs
-7
View File
@@ -27,7 +27,6 @@ use crate::vectors::operations::VectorOp;
mod binary;
mod boolean;
mod constant;
mod date;
mod decimal;
mod dictionary;
@@ -48,7 +47,6 @@ mod validity;
pub use binary::{BinaryVector, BinaryVectorBuilder};
pub use boolean::{BooleanVector, BooleanVectorBuilder};
pub use constant::ConstantVector;
pub use date::{DateVector, DateVectorBuilder};
pub use decimal::{Decimal128Vector, Decimal128VectorBuilder};
pub(crate) use dictionary::StringDictionaryVectorBuilder;
@@ -126,11 +124,6 @@ pub trait Vector: Send + Sync + Serializable + Debug + VectorOp {
/// This is `O(1)`.
fn null_count(&self) -> usize;
/// Returns true when it's a ConstantColumn
fn is_const(&self) -> bool {
false
}
/// Returns whether row is null.
fn is_null(&self, row: usize) -> bool;
src/datatypes/src/vectors/binary.rs
-1
View File
@@ -479,7 +479,6 @@ mod tests {
assert_eq!(2, v.len());
assert_eq!("BinaryVector", v.vector_type_name());
assert!(!v.is_const());
assert!(v.validity().is_all_valid());
assert!(!v.only_null());
assert_eq!(128, v.memory_size());
src/datatypes/src/vectors/boolean.rs
-5
View File
@@ -40,10 +40,6 @@ impl BooleanVector {
pub fn as_boolean_array(&self) -> &BooleanArray {
&self.array
}
pub(crate) fn false_count(&self) -> usize {
self.array.false_count()
}
}
impl From<Vec<bool>> for BooleanVector {
@@ -251,7 +247,6 @@ mod tests {
let v = BooleanVector::from(bools.clone());
assert_eq!(9, v.len());
assert_eq!("BooleanVector", v.vector_type_name());
assert!(!v.is_const());
assert!(v.validity().is_all_valid());
assert!(!v.only_null());
assert_eq!(2, v.memory_size());
src/datatypes/src/vectors/constant.rs
-252
View File
@@ -1,252 +0,0 @@
// Copyright 2023 Greptime Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use std::any::Any;
use std::fmt;
use std::sync::Arc;
use arrow::array::{Array, ArrayRef, UInt32Array};
use snafu::{ResultExt, ensure};
use crate::data_type::ConcreteDataType;
use crate::error::{self, Result, SerializeSnafu};
use crate::serialize::Serializable;
use crate::value::{Value, ValueRef};
use crate::vectors::{BooleanVector, Helper, UInt32Vector, Validity, Vector, VectorRef};
#[derive(Clone)]
pub struct ConstantVector {
length: usize,
vector: VectorRef,
}
impl ConstantVector {
/// Create a new [ConstantVector].
///
/// # Panics
/// Panics if `vector.len() != 1`.
pub fn new(vector: VectorRef, length: usize) -> Self {
assert_eq!(1, vector.len());
// Avoid const recursion.
if vector.is_const() {
let vec: &ConstantVector = unsafe { Helper::static_cast(&vector) };
return Self::new(vec.inner().clone(), length);
}
Self { vector, length }
}
pub fn inner(&self) -> &VectorRef {
&self.vector
}
/// Returns the constant value.
pub fn get_constant_ref(&self) -> ValueRef<'_> {
self.vector.get_ref(0)
}
pub(crate) fn replicate_vector(&self, offsets: &[usize]) -> VectorRef {
assert_eq!(offsets.len(), self.len());
if offsets.is_empty() {
return self.slice(0, 0);
}
Arc::new(ConstantVector::new(
self.vector.clone(),
*offsets.last().unwrap(),
))
}
pub(crate) fn filter_vector(&self, filter: &BooleanVector) -> Result<VectorRef> {
let length = self.len() - filter.false_count();
if length == self.len() {
return Ok(Arc::new(self.clone()));
}
Ok(Arc::new(ConstantVector::new(self.inner().clone(), length)))
}
pub(crate) fn cast_vector(&self, to_type: &ConcreteDataType) -> Result<VectorRef> {
Ok(Arc::new(ConstantVector::new(
self.inner().cast(to_type)?,
self.length,
)))
}
pub(crate) fn take_vector(&self, indices: &UInt32Vector) -> Result<VectorRef> {
if indices.is_empty() {
return Ok(self.slice(0, 0));
}
ensure!(
indices.null_count() == 0,
error::UnsupportedOperationSnafu {
op: "taking a null index",
vector_type: self.vector_type_name(),
}
);
let len = self.len();
let arr = indices.to_arrow_array();
let indices_arr = arr.as_any().downcast_ref::<UInt32Array>().unwrap();
if !arrow::compute::min_boolean(
&arrow::compute::kernels::cmp::lt(indices_arr, &UInt32Array::new_scalar(len as u32))
.unwrap(),
)
.unwrap()
{
panic!(
"Array index out of bounds, cannot take index out of the length of the array: {len}"
);
}
Ok(Arc::new(ConstantVector::new(
self.inner().clone(),
indices.len(),
)))
}
}
impl Vector for ConstantVector {
fn data_type(&self) -> ConcreteDataType {
self.vector.data_type()
}
fn vector_type_name(&self) -> String {
"ConstantVector".to_string()
}
fn as_any(&self) -> &dyn Any {
self
}
fn len(&self) -> usize {
self.length
}
fn to_arrow_array(&self) -> ArrayRef {
let v = self.vector.replicate(&[self.length]);
v.to_arrow_array()
}
fn to_boxed_arrow_array(&self) -> Box<dyn Array> {
let v = self.vector.replicate(&[self.length]);
v.to_boxed_arrow_array()
}
fn is_const(&self) -> bool {
true
}
fn validity(&self) -> Validity {
if self.vector.is_null(0) {
Validity::all_null(self.length)
} else {
Validity::all_valid(self.length)
}
}
fn memory_size(&self) -> usize {
self.vector.memory_size()
}
fn is_null(&self, _row: usize) -> bool {
self.vector.is_null(0)
}
fn only_null(&self) -> bool {
self.vector.is_null(0)
}
fn slice(&self, _offset: usize, length: usize) -> VectorRef {
Arc::new(Self {
vector: self.vector.clone(),
length,
})
}
fn get(&self, _index: usize) -> Value {
self.vector.get(0)
}
fn get_ref(&self, _index: usize) -> ValueRef<'_> {
self.vector.get_ref(0)
}
fn null_count(&self) -> usize {
if self.only_null() { self.len() } else { 0 }
}
}
impl fmt::Debug for ConstantVector {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
write!(f, "ConstantVector([{:?}; {}])", self.get(0), self.len())
}
}
impl Serializable for ConstantVector {
fn serialize_to_json(&self) -> Result<Vec<serde_json::Value>> {
std::iter::repeat_n(self.get(0), self.len())
.map(serde_json::Value::try_from)
.collect::<serde_json::Result<_>>()
.context(SerializeSnafu)
}
}
#[cfg(test)]
mod tests {
use arrow::datatypes::DataType as ArrowDataType;
use super::*;
use crate::vectors::Int32Vector;
#[test]
fn test_constant_vector_misc() {
let a = Int32Vector::from_slice(vec![1]);
let c = ConstantVector::new(Arc::new(a), 10);
assert_eq!("ConstantVector", c.vector_type_name());
assert!(c.is_const());
assert_eq!(10, c.len());
assert!(c.validity().is_all_valid());
assert!(!c.only_null());
assert_eq!(4, c.memory_size());
for i in 0..10 {
assert!(!c.is_null(i));
assert_eq!(Value::Int32(1), c.get(i));
}
let arrow_arr = c.to_arrow_array();
assert_eq!(10, arrow_arr.len());
assert_eq!(&ArrowDataType::Int32, arrow_arr.data_type());
}
#[test]
fn test_debug_null_array() {
let a = Int32Vector::from_slice(vec![1]);
let c = ConstantVector::new(Arc::new(a), 10);
let s = format!("{c:?}");
assert_eq!(s, "ConstantVector([Int32(1); 10])");
}
#[test]
fn test_serialize_json() {
let a = Int32Vector::from_slice(vec![1]);
let c = ConstantVector::new(Arc::new(a), 10);
let s = serde_json::to_string(&c.serialize_to_json().unwrap()).unwrap();
assert_eq!(s, "[1,1,1,1,1,1,1,1,1,1]");
}
}
src/datatypes/src/vectors/decimal.rs
-37
View File
@@ -385,43 +385,6 @@ impl Decimal128VectorBuilder {
vectors::impl_try_from_arrow_array_for_vector!(Decimal128Array, Decimal128Vector);
pub(crate) fn replicate_decimal128(
vector: &Decimal128Vector,
offsets: &[usize],
) -> Decimal128Vector {
assert_eq!(offsets.len(), vector.len());
if offsets.is_empty() {
return vector.get_slice(0, 0);
}
// Safety: safe to unwrap because we the vector ensures precision and scale are valid.
let mut builder = Decimal128VectorBuilder::with_capacity(*offsets.last().unwrap())
.with_precision_and_scale(vector.precision(), vector.scale())
.unwrap();
let mut previous_offset = 0;
for (offset, value) in offsets.iter().zip(vector.array.iter()) {
let repeat_times = *offset - previous_offset;
match value {
Some(data) => {
unsafe {
// Safety: std::iter::Repeat and std::iter::Take implement TrustedLen.
builder
.mutable_array
.append_trusted_len_iter(std::iter::repeat_n(data, repeat_times));
}
}
None => {
builder.mutable_array.append_nulls(repeat_times);
}
}
previous_offset = *offset;
}
builder.finish()
}
#[cfg(test)]
pub mod tests {
use arrow_array::Decimal128Array;
src/datatypes/src/vectors/dictionary.rs
+1 -47
View File
@@ -17,8 +17,7 @@ use std::fmt;
use std::sync::Arc;
use arrow::array::{
Array, ArrayBuilder, ArrayRef, DictionaryArray, PrimitiveArray, PrimitiveBuilder,
StringDictionaryBuilder,
Array, ArrayBuilder, ArrayRef, DictionaryArray, PrimitiveArray, StringDictionaryBuilder,
};
use arrow::datatypes::{ArrowDictionaryKeyType, ArrowNativeType, UInt32Type};
use serde_json::Value as JsonValue;
@@ -320,36 +319,6 @@ impl<'a, K: ArrowDictionaryKeyType> Iterator for DictionaryIter<'a, K> {
}
impl<K: ArrowDictionaryKeyType> VectorOp for DictionaryVector<K> {
fn replicate(&self, offsets: &[usize]) -> VectorRef {
let keys = self.array.keys();
let mut replicated_keys = PrimitiveBuilder::new();
let mut previous_offset = 0;
let mut key_iter = keys.iter().chain(std::iter::repeat(None));
for &offset in offsets {
let key = key_iter.next().unwrap();
// repeat this key (offset - previous_offset) times
let repeat_count = offset - previous_offset;
for _ in 0..repeat_count {
replicated_keys.append_option(key);
}
previous_offset = offset;
}
let new_keys = replicated_keys.finish();
let new_array = DictionaryArray::try_new(new_keys, self.values().clone())
.expect("Failed to create replicated dictionary array");
Arc::new(Self {
array: new_array,
key_type: self.key_type.clone(),
item_type: self.item_type.clone(),
item_vector: self.item_vector.clone(),
})
}
fn filter(&self, filter: &vectors::BooleanVector) -> Result<VectorRef> {
let key_array: ArrayRef = Arc::new(self.array.keys().clone());
let key_vector = Helper::try_into_vector(&key_array)?;
@@ -485,21 +454,6 @@ mod tests {
assert_eq!(sliced.get(2), Value::Null);
}
#[test]
fn test_replicate() {
let dict_vec = create_test_dictionary();
// Replicate with offsets [0, 2, 5] - should get values at these indices
let offsets = vec![0, 2, 5];
let replicated = dict_vec.replicate(&offsets);
assert_eq!(replicated.len(), 5);
assert_eq!(replicated.get(0), Value::String("b".to_string().into()));
assert_eq!(replicated.get(1), Value::String("b".to_string().into()));
assert_eq!(replicated.get(2), Value::String("c".to_string().into()));
assert_eq!(replicated.get(3), Value::String("c".to_string().into()));
assert_eq!(replicated.get(4), Value::String("c".to_string().into()));
}
#[test]
fn test_filter() {
let dict_vec = create_test_dictionary();
src/datatypes/src/vectors/eq.rs
-68
View File
@@ -18,7 +18,6 @@ use common_time::interval::IntervalUnit;
use crate::data_type::DataType;
use crate::types::{DurationType, TimeType, TimestampType};
use crate::vectors::constant::ConstantVector;
use crate::vectors::struct_vector::StructVector;
use crate::vectors::{
BinaryVector, BooleanVector, DateVector, Decimal128Vector, DurationMicrosecondVector,
@@ -58,23 +57,6 @@ fn equal(lhs: &dyn Vector, rhs: &dyn Vector) -> bool {
return false;
}
if lhs.is_const() || rhs.is_const() {
// Length has been checked before, so we only need to compare inner
// vector here.
return equal(
&**lhs
.as_any()
.downcast_ref::<ConstantVector>()
.unwrap()
.inner(),
&**rhs
.as_any()
.downcast_ref::<ConstantVector>()
.unwrap()
.inner(),
);
}
use crate::data_type::ConcreteDataType::*;
let lhs_type = lhs.data_type();
@@ -191,10 +173,6 @@ mod tests {
Some(b"world".to_vec()),
])));
assert_vector_ref_eq(Arc::new(BooleanVector::from(vec![true, false])));
assert_vector_ref_eq(Arc::new(ConstantVector::new(
Arc::new(BooleanVector::from(vec![true])),
5,
)));
assert_vector_ref_eq(Arc::new(BooleanVector::from(vec![true, false])));
assert_vector_ref_eq(Arc::new(DateVector::from(vec![Some(100), Some(120)])));
assert_vector_ref_eq(Arc::new(TimestampSecondVector::from_values([100, 120])));
@@ -259,22 +237,6 @@ mod tests {
])));
}
// Regression: second arm must downcast `rhs` (was `lhs`), or same-length ConstantVectors
// with different inners compare equal.
#[test]
fn test_constant_vector_eq_compares_both_inners() {
assert_vector_ref_ne(
Arc::new(ConstantVector::new(
Arc::new(BooleanVector::from(vec![true])),
5,
)),
Arc::new(ConstantVector::new(
Arc::new(BooleanVector::from(vec![false])),
5,
)),
);
}
#[test]
fn test_vector_ne() {
assert_vector_ref_ne(
@@ -289,36 +251,6 @@ mod tests {
Arc::new(Int32Vector::from_slice([1, 2, 3, 4])),
Arc::new(BooleanVector::from(vec![true, true])),
);
assert_vector_ref_ne(
Arc::new(ConstantVector::new(
Arc::new(BooleanVector::from(vec![true])),
5,
)),
Arc::new(ConstantVector::new(
Arc::new(BooleanVector::from(vec![true])),
4,
)),
);
assert_vector_ref_ne(
Arc::new(ConstantVector::new(
Arc::new(BooleanVector::from(vec![true])),
5,
)),
Arc::new(ConstantVector::new(
Arc::new(BooleanVector::from(vec![false])),
4,
)),
);
assert_vector_ref_ne(
Arc::new(ConstantVector::new(
Arc::new(BooleanVector::from(vec![true])),
5,
)),
Arc::new(ConstantVector::new(
Arc::new(Int32Vector::from_slice(vec![1])),
4,
)),
);
assert_vector_ref_ne(Arc::new(NullVector::new(5)), Arc::new(NullVector::new(8)));
assert_vector_ref_ne(
src/datatypes/src/vectors/helper.rs
+163 -143
View File
@@ -30,21 +30,21 @@ use datafusion_common::ScalarValue;
use snafu::{OptionExt, ResultExt};
use crate::data_type::ConcreteDataType;
use crate::error::{self, ConvertArrowArrayToScalarsSnafu, Result};
use crate::error::{self, ConvertScalarToArrowArraySnafu, Result};
use crate::prelude::DataType;
use crate::scalars::{Scalar, ScalarVectorBuilder};
use crate::scalars::Scalar;
use crate::types::StructType;
use crate::value::{ListValue, ListValueRef, Value};
use crate::value::Value;
use crate::vectors::struct_vector::StructVector;
use crate::vectors::{
BinaryVector, BooleanVector, ConstantVector, DateVector, Decimal128Vector, DictionaryVector,
BinaryVector, BooleanVector, DateVector, Decimal128Vector, DictionaryVector,
DurationMicrosecondVector, DurationMillisecondVector, DurationNanosecondVector,
DurationSecondVector, Float32Vector, Float64Vector, Int8Vector, Int16Vector, Int32Vector,
Int64Vector, IntervalDayTimeVector, IntervalMonthDayNanoVector, IntervalYearMonthVector,
ListVector, ListVectorBuilder, MutableVector, NullVector, StringVector, TimeMicrosecondVector,
TimeMillisecondVector, TimeNanosecondVector, TimeSecondVector, TimestampMicrosecondVector,
TimestampMillisecondVector, TimestampNanosecondVector, TimestampSecondVector, UInt8Vector,
UInt16Vector, UInt32Vector, UInt64Vector, Vector, VectorRef,
ListVector, NullVector, StringVector, TimeMicrosecondVector, TimeMillisecondVector,
TimeNanosecondVector, TimeSecondVector, TimestampMicrosecondVector, TimestampMillisecondVector,
TimestampNanosecondVector, TimestampSecondVector, UInt8Vector, UInt16Vector, UInt32Vector,
UInt64Vector, Vector, VectorRef,
};
/// Helper functions for `Vector`.
@@ -102,135 +102,50 @@ impl Helper {
})
}
/// Try to cast an arrow scalar value into vector
pub fn try_from_scalar_value(value: ScalarValue, length: usize) -> Result<VectorRef> {
let vector = match value {
ScalarValue::Null => ConstantVector::new(Arc::new(NullVector::new(1)), length),
ScalarValue::Boolean(v) => {
ConstantVector::new(Arc::new(BooleanVector::from(vec![v])), length)
/// Materializes an Arrow scalar as a vector of the given length.
///
/// With a type hint, casts the array to the requested representation and returns
/// an error if the vector cannot preserve that concrete type. Without a hint,
/// normalizes the scalar to a supported GreptimeDB representation.
pub fn try_from_scalar_value(
value: ScalarValue,
length: usize,
type_hint: Option<&ConcreteDataType>,
) -> Result<VectorRef> {
if let Some(data_type) = type_hint {
let mut array = value
.to_array_of_size(length)
.context(ConvertScalarToArrowArraySnafu)?;
let arrow_type = data_type.as_arrow_type();
if array.data_type() != &arrow_type {
array = compute::cast(&array, &arrow_type).context(error::ArrowComputeSnafu)?;
}
ScalarValue::Float16(v) => ConstantVector::new(
Arc::new(Float32Vector::from(vec![v.map(f32::from)])),
length,
),
ScalarValue::Float32(v) => {
ConstantVector::new(Arc::new(Float32Vector::from(vec![v])), length)
let vector = Self::try_into_vector(array)?;
if &vector.data_type() != data_type {
return error::CastTypeSnafu {
msg: format!(
"Scalar materialization produced {:?}, expected {data_type:?}",
vector.data_type()
),
}
.fail();
}
ScalarValue::Float64(v) => {
ConstantVector::new(Arc::new(Float64Vector::from(vec![v])), length)
return Ok(vector);
}
let value = match value {
// GreptimeDB doesn't support Float16 vectors.
ScalarValue::Float16(v) => ScalarValue::Float32(v.map(f32::from)),
ScalarValue::LargeUtf8(v) => ScalarValue::Utf8(v),
ScalarValue::LargeBinary(v) | ScalarValue::FixedSizeBinary(_, v) => {
ScalarValue::Binary(v)
}
ScalarValue::Int8(v) => {
ConstantVector::new(Arc::new(Int8Vector::from(vec![v])), length)
}
ScalarValue::Int16(v) => {
ConstantVector::new(Arc::new(Int16Vector::from(vec![v])), length)
}
ScalarValue::Int32(v) => {
ConstantVector::new(Arc::new(Int32Vector::from(vec![v])), length)
}
ScalarValue::Int64(v) => {
ConstantVector::new(Arc::new(Int64Vector::from(vec![v])), length)
}
ScalarValue::UInt8(v) => {
ConstantVector::new(Arc::new(UInt8Vector::from(vec![v])), length)
}
ScalarValue::UInt16(v) => {
ConstantVector::new(Arc::new(UInt16Vector::from(vec![v])), length)
}
ScalarValue::UInt32(v) => {
ConstantVector::new(Arc::new(UInt32Vector::from(vec![v])), length)
}
ScalarValue::UInt64(v) => {
ConstantVector::new(Arc::new(UInt64Vector::from(vec![v])), length)
}
ScalarValue::Utf8(v) | ScalarValue::LargeUtf8(v) => {
ConstantVector::new(Arc::new(StringVector::from(vec![v])), length)
}
ScalarValue::Binary(v)
| ScalarValue::LargeBinary(v)
| ScalarValue::FixedSizeBinary(_, v) => {
ConstantVector::new(Arc::new(BinaryVector::from(vec![v])), length)
}
ScalarValue::List(array) => {
let item_type = Arc::new(ConcreteDataType::try_from(&array.value_type())?);
let mut builder = ListVectorBuilder::with_type_capacity(item_type.clone(), 1);
let scalar_values = ScalarValue::convert_array_to_scalar_vec(array.as_ref())
.context(ConvertArrowArrayToScalarsSnafu)?;
let values = scalar_values
.into_iter()
.flat_map(|v| v.unwrap_or_else(|| vec![ScalarValue::Null]))
.map(ScalarValue::try_into)
.collect::<Result<Vec<Value>>>()?;
builder.push(Some(ListValueRef::Ref {
val: &ListValue::new(values, item_type),
}));
let list_vector = builder.to_vector();
ConstantVector::new(list_vector, length)
}
ScalarValue::Date32(v) => {
ConstantVector::new(Arc::new(DateVector::from(vec![v])), length)
}
ScalarValue::TimestampSecond(v, _) => {
// Timezone is unimplemented now.
ConstantVector::new(Arc::new(TimestampSecondVector::from(vec![v])), length)
}
ScalarValue::TimestampMillisecond(v, _) => {
// Timezone is unimplemented now.
ConstantVector::new(Arc::new(TimestampMillisecondVector::from(vec![v])), length)
}
ScalarValue::TimestampMicrosecond(v, _) => {
// Timezone is unimplemented now.
ConstantVector::new(Arc::new(TimestampMicrosecondVector::from(vec![v])), length)
}
ScalarValue::TimestampNanosecond(v, _) => {
// Timezone is unimplemented now.
ConstantVector::new(Arc::new(TimestampNanosecondVector::from(vec![v])), length)
}
ScalarValue::Time32Second(v) => {
ConstantVector::new(Arc::new(TimeSecondVector::from(vec![v])), length)
}
ScalarValue::Time32Millisecond(v) => {
ConstantVector::new(Arc::new(TimeMillisecondVector::from(vec![v])), length)
}
ScalarValue::Time64Microsecond(v) => {
ConstantVector::new(Arc::new(TimeMicrosecondVector::from(vec![v])), length)
}
ScalarValue::Time64Nanosecond(v) => {
ConstantVector::new(Arc::new(TimeNanosecondVector::from(vec![v])), length)
}
ScalarValue::IntervalYearMonth(v) => {
ConstantVector::new(Arc::new(IntervalYearMonthVector::from(vec![v])), length)
}
ScalarValue::IntervalDayTime(v) => {
ConstantVector::new(Arc::new(IntervalDayTimeVector::from(vec![v])), length)
}
ScalarValue::IntervalMonthDayNano(v) => {
ConstantVector::new(Arc::new(IntervalMonthDayNanoVector::from(vec![v])), length)
}
ScalarValue::DurationSecond(v) => {
ConstantVector::new(Arc::new(DurationSecondVector::from(vec![v])), length)
}
ScalarValue::DurationMillisecond(v) => {
ConstantVector::new(Arc::new(DurationMillisecondVector::from(vec![v])), length)
}
ScalarValue::DurationMicrosecond(v) => {
ConstantVector::new(Arc::new(DurationMicrosecondVector::from(vec![v])), length)
}
ScalarValue::DurationNanosecond(v) => {
ConstantVector::new(Arc::new(DurationNanosecondVector::from(vec![v])), length)
}
ScalarValue::Decimal128(v, p, s) => {
let vector = Decimal128Vector::from(vec![v]).with_precision_and_scale(p, s)?;
ConstantVector::new(Arc::new(vector), length)
}
ScalarValue::Struct(v) => {
let struct_type = StructType::from(v.fields());
ConstantVector::new(
Arc::new(StructVector::try_new(struct_type, (*v).clone())?),
length,
)
}
ScalarValue::Decimal32(_, _, _)
// Timezones are not supported by GreptimeDB vectors.
ScalarValue::TimestampSecond(v, _) => ScalarValue::TimestampSecond(v, None),
ScalarValue::TimestampMillisecond(v, _) => ScalarValue::TimestampMillisecond(v, None),
ScalarValue::TimestampMicrosecond(v, _) => ScalarValue::TimestampMicrosecond(v, None),
ScalarValue::TimestampNanosecond(v, _) => ScalarValue::TimestampNanosecond(v, None),
value @ (ScalarValue::Decimal32(_, _, _)
| ScalarValue::Decimal64(_, _, _)
| ScalarValue::Decimal256(_, _, _)
| ScalarValue::FixedSizeList(_)
@@ -241,15 +156,19 @@ impl Helper {
| ScalarValue::BinaryView(_)
| ScalarValue::Map(_)
| ScalarValue::Date64(_)
| ScalarValue::RunEndEncoded(_, _, _) => {
| ScalarValue::RunEndEncoded(_, _, _)) => {
return error::ConversionSnafu {
from: format!("Unsupported scalar value: {value}"),
}
.fail();
}
value => value,
};
Ok(Arc::new(vector))
let array = value
.to_array_of_size(length)
.context(ConvertScalarToArrowArraySnafu)?;
Self::try_into_vector(array)
}
/// Try to cast an arrow array into vector
@@ -457,12 +376,15 @@ mod tests {
};
use arrow::buffer::Buffer;
use arrow::datatypes::{Int32Type, IntervalMonthDayNano};
use arrow_array::{BinaryArray, DictionaryArray, FixedSizeBinaryArray, LargeStringArray};
use arrow_schema::DataType;
use arrow_array::{
BinaryArray, DictionaryArray, FixedSizeBinaryArray, LargeStringArray, StructArray,
};
use arrow_schema::{DataType, Field, Fields};
use common_decimal::Decimal128;
use common_time::time::Time;
use common_time::timestamp::TimeUnit;
use common_time::{Date, Duration};
use datafusion_common::scalar::ScalarStructBuilder;
use super::*;
use crate::value::Value;
@@ -496,7 +418,7 @@ mod tests {
#[test]
fn test_try_from_scalar_date_value() {
let vector = Helper::try_from_scalar_value(ScalarValue::Date32(Some(42)), 3).unwrap();
let vector = Helper::try_from_scalar_value(ScalarValue::Date32(Some(42)), 3, None).unwrap();
assert_eq!(ConcreteDataType::date_datatype(), vector.data_type());
assert_eq!(3, vector.len());
for i in 0..vector.len() {
@@ -507,7 +429,7 @@ mod tests {
#[test]
fn test_try_from_scalar_duration_value() {
let vector =
Helper::try_from_scalar_value(ScalarValue::DurationSecond(Some(42)), 3).unwrap();
Helper::try_from_scalar_value(ScalarValue::DurationSecond(Some(42)), 3, None).unwrap();
assert_eq!(
ConcreteDataType::duration_second_datatype(),
vector.data_type()
@@ -524,7 +446,8 @@ mod tests {
#[test]
fn test_try_from_scalar_decimal128_value() {
let vector =
Helper::try_from_scalar_value(ScalarValue::Decimal128(Some(42), 3, 1), 3).unwrap();
Helper::try_from_scalar_value(ScalarValue::Decimal128(Some(42), 3, 1), 3, None)
.unwrap();
assert_eq!(
ConcreteDataType::decimal128_datatype(3, 1),
vector.data_type()
@@ -542,7 +465,7 @@ mod tests {
&ArrowDataType::Int32,
true,
));
let vector = Helper::try_from_scalar_value(value, 3).unwrap();
let vector = Helper::try_from_scalar_value(value, 3, None).unwrap();
assert_eq!(
ConcreteDataType::list_datatype(Arc::new(ConcreteDataType::int32_datatype())),
vector.data_type()
@@ -555,6 +478,101 @@ mod tests {
}
}
#[test]
fn test_try_from_scalar_value_materializes_values() {
let vector = Helper::try_from_scalar_value(ScalarValue::Int32(Some(42)), 4, None).unwrap();
assert_eq!(ConcreteDataType::int32_datatype(), vector.data_type());
assert_eq!(4, vector.len());
assert_eq!(0, vector.null_count());
for i in 0..vector.len() {
assert_eq!(Value::Int32(42), vector.get(i));
}
let empty = Helper::try_from_scalar_value(ScalarValue::Int32(Some(42)), 0, None).unwrap();
assert_eq!(ConcreteDataType::int32_datatype(), empty.data_type());
assert!(empty.is_empty());
let nulls = Helper::try_from_scalar_value(ScalarValue::Int32(None), 3, None).unwrap();
assert_eq!(3, nulls.len());
assert_eq!(3, nulls.null_count());
for i in 0..nulls.len() {
assert_eq!(Value::Null, nulls.get(i));
}
}
#[test]
fn test_try_from_scalar_struct_value() {
let fields = Fields::from(vec![
Field::new("id", ArrowDataType::Int32, false),
Field::new("name", ArrowDataType::Utf8, true),
]);
let value = ScalarValue::Struct(Arc::new(StructArray::new(
fields.clone(),
vec![
ScalarValue::Int32(Some(7)).to_array().unwrap(),
ScalarValue::Utf8(Some("greptime".to_string()))
.to_array()
.unwrap(),
],
None,
)));
let vector = Helper::try_from_scalar_value(value, 3, None).unwrap();
assert_eq!(
ConcreteDataType::struct_datatype(StructType::from(&fields)),
vector.data_type()
);
assert_eq!(3, vector.len());
for i in 0..vector.len() {
let Value::Struct(value) = vector.get(i) else {
panic!("expected struct value");
};
assert_eq!(
&[Value::Int32(7), Value::String("greptime".into())],
value.items()
);
}
let null = ScalarStructBuilder::new_null(fields);
let vector = Helper::try_from_scalar_value(null, 2, None).unwrap();
assert_eq!(2, vector.len());
assert_eq!(2, vector.null_count());
assert_eq!(Value::Null, vector.get(0));
assert_eq!(Value::Null, vector.get(1));
}
#[test]
fn test_try_from_scalar_value_normalizes_arrow_types() {
let string = Helper::try_from_scalar_value(
ScalarValue::LargeUtf8(Some("greptime".to_string())),
2,
None,
)
.unwrap();
assert_eq!(ConcreteDataType::string_datatype(), string.data_type());
assert_eq!(&ArrowDataType::Utf8, string.to_arrow_array().data_type());
let binary = Helper::try_from_scalar_value(
ScalarValue::FixedSizeBinary(2, Some(vec![1, 2])),
2,
None,
)
.unwrap();
assert_eq!(ConcreteDataType::binary_datatype(), binary.data_type());
assert_eq!(&ArrowDataType::Binary, binary.to_arrow_array().data_type());
let timestamp = Helper::try_from_scalar_value(
ScalarValue::TimestampMillisecond(Some(42), Some("UTC".into())),
2,
None,
)
.unwrap();
assert_eq!(
&ArrowDataType::Timestamp(arrow::datatypes::TimeUnit::Millisecond, None),
timestamp.to_arrow_array().data_type()
);
}
#[test]
fn test_like_utf8() {
fn assert_vector(expected: Vec<&str>, actual: &VectorRef) {
@@ -745,7 +763,8 @@ mod tests {
#[test]
fn test_try_from_scalar_time_value() {
let vector = Helper::try_from_scalar_value(ScalarValue::Time32Second(Some(42)), 3).unwrap();
let vector =
Helper::try_from_scalar_value(ScalarValue::Time32Second(Some(42)), 3, None).unwrap();
assert_eq!(ConcreteDataType::time_second_datatype(), vector.data_type());
assert_eq!(3, vector.len());
for i in 0..vector.len() {
@@ -758,6 +777,7 @@ mod tests {
let vector = Helper::try_from_scalar_value(
ScalarValue::IntervalMonthDayNano(Some(IntervalMonthDayNano::new(1, 1, 2000))),
3,
None,
)
.unwrap();
src/datatypes/src/vectors/null.rs
-11
View File
@@ -260,16 +260,6 @@ impl ScalarVectorBuilder for NullVectorBuilder {
}
}
pub(crate) fn replicate_null(vector: &NullVector, offsets: &[usize]) -> VectorRef {
assert_eq!(offsets.len(), vector.len());
if offsets.is_empty() {
return vector.slice(0, 0);
}
Arc::new(NullVector::new(*offsets.last().unwrap()))
}
#[cfg(test)]
mod tests {
use serde_json;
@@ -290,7 +280,6 @@ mod tests {
assert_eq!(vector2.null_count(), 16);
assert_eq!("NullVector", v.vector_type_name());
assert!(!v.is_const());
assert!(v.validity().is_all_null());
assert!(v.only_null());
src/datatypes/src/vectors/operations.rs
-48
View File
@@ -14,14 +14,12 @@
mod cast;
mod filter;
mod replicate;
mod take;
use std::sync::Arc;
use crate::error::{self, Result};
use crate::types::LogicalPrimitiveType;
use crate::vectors::constant::ConstantVector;
use crate::vectors::{
BinaryVector, BooleanVector, ConcreteDataType, Decimal128Vector, ListVector, NullVector,
PrimitiveVector, StringVector, UInt32Vector, Vector, VectorRef,
@@ -29,14 +27,6 @@ use crate::vectors::{
/// Vector compute operations.
pub trait VectorOp {
/// Copies each element according `offsets` parameter.
/// - `i-th` element should be copied `offsets[i] - offsets[i - 1]` times
/// - `0-th` element would be copied `offsets[0]` times
///
/// # Panics
/// Panics if `offsets.len() != self.len()`.
fn replicate(&self, offsets: &[usize]) -> VectorRef;
/// Filters the vector, returns elements matching the `filter` (i.e. where the values are true).
///
/// Note that the nulls of `filter` are interpreted as `false` will lead to these elements being masked out.
@@ -57,10 +47,6 @@ pub trait VectorOp {
macro_rules! impl_scalar_vector_op {
($($VectorType: ident),+) => {$(
impl VectorOp for $VectorType {
fn replicate(&self, offsets: &[usize]) -> VectorRef {
replicate::replicate_scalar(self, offsets)
}
fn filter(&self, filter: &BooleanVector) -> Result<VectorRef> {
filter::filter_non_constant!(self, $VectorType, filter)
}
@@ -92,10 +78,6 @@ macro_rules! impl_scalar_vector_op {
impl_scalar_vector_op!(BinaryVector, BooleanVector, StringVector);
impl VectorOp for ListVector {
fn replicate(&self, offsets: &[usize]) -> VectorRef {
replicate::replicate_list(self, offsets)
}
fn filter(&self, filter: &BooleanVector) -> Result<VectorRef> {
filter::filter_non_constant!(self, ListVector, filter)
}
@@ -110,10 +92,6 @@ impl VectorOp for ListVector {
}
impl VectorOp for Decimal128Vector {
fn replicate(&self, offsets: &[usize]) -> VectorRef {
std::sync::Arc::new(replicate::replicate_decimal128(self, offsets))
}
fn filter(&self, filter: &BooleanVector) -> Result<VectorRef> {
filter::filter_non_constant!(self, Decimal128Vector, filter)
}
@@ -128,10 +106,6 @@ impl VectorOp for Decimal128Vector {
}
impl<T: LogicalPrimitiveType> VectorOp for PrimitiveVector<T> {
fn replicate(&self, offsets: &[usize]) -> VectorRef {
std::sync::Arc::new(replicate::replicate_primitive(self, offsets))
}
fn filter(&self, filter: &BooleanVector) -> Result<VectorRef> {
filter::filter_non_constant!(self, PrimitiveVector<T>, filter)
}
@@ -146,10 +120,6 @@ impl<T: LogicalPrimitiveType> VectorOp for PrimitiveVector<T> {
}
impl VectorOp for NullVector {
fn replicate(&self, offsets: &[usize]) -> VectorRef {
replicate::replicate_null(self, offsets)
}
fn filter(&self, filter: &BooleanVector) -> Result<VectorRef> {
filter::filter_non_constant!(self, NullVector, filter)
}
@@ -166,21 +136,3 @@ impl VectorOp for NullVector {
take::take_indices!(self, NullVector, indices)
}
}
impl VectorOp for ConstantVector {
fn replicate(&self, offsets: &[usize]) -> VectorRef {
self.replicate_vector(offsets)
}
fn filter(&self, filter: &BooleanVector) -> Result<VectorRef> {
self.filter_vector(filter)
}
fn cast(&self, to_type: &ConcreteDataType) -> Result<VectorRef> {
self.cast_vector(to_type)
}
fn take(&self, indices: &UInt32Vector) -> Result<VectorRef> {
self.take_vector(indices)
}
}
src/datatypes/src/vectors/operations/filter.rs
-21
View File
@@ -39,7 +39,6 @@ mod tests {
TimestampMicrosecond, TimestampMillisecond, TimestampNanosecond, TimestampSecond,
};
use crate::types::WrapperType;
use crate::vectors::constant::ConstantVector;
use crate::vectors::{
BooleanVector, Int32Vector, NullVector, StringVector, VectorOp, VectorRef,
};
@@ -67,26 +66,6 @@ mod tests {
check_filter_primitive(&[5, 7], &[5, 6, 7], &[true, false, true]);
}
fn check_filter_constant(expect_length: usize, input_length: usize, filter: &[bool]) {
let v = ConstantVector::new(Arc::new(Int32Vector::from_slice([123])), input_length);
let filter = BooleanVector::from_slice(filter);
let out = v.filter(&filter).unwrap();
assert!(out.is_const());
assert_eq!(expect_length, out.len());
}
#[test]
fn test_filter_constant() {
check_filter_constant(0, 0, &[]);
check_filter_constant(1, 1, &[true]);
check_filter_constant(0, 1, &[false]);
check_filter_constant(1, 2, &[false, true]);
check_filter_constant(2, 2, &[true, true]);
check_filter_constant(1, 4, &[false, false, false, true]);
check_filter_constant(2, 4, &[false, true, false, true]);
}
#[test]
fn test_filter_scalar() {
let v = StringVector::from_slice(&["0", "1", "2", "3"]);
src/datatypes/src/vectors/operations/replicate.rs
-232
View File
@@ -1,232 +0,0 @@
// Copyright 2023 Greptime Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::prelude::*;
pub(crate) use crate::vectors::decimal::replicate_decimal128;
pub(crate) use crate::vectors::null::replicate_null;
pub(crate) use crate::vectors::primitive::replicate_primitive;
use crate::vectors::{ListVector, ListVectorBuilder};
pub(crate) fn replicate_scalar<C: ScalarVector>(c: &C, offsets: &[usize]) -> VectorRef {
assert_eq!(offsets.len(), c.len());
if offsets.is_empty() {
return c.slice(0, 0);
}
let mut builder = <<C as ScalarVector>::Builder>::with_capacity(c.len());
let mut previous_offset = 0;
for (i, offset) in offsets.iter().enumerate() {
let data = c.get_data(i);
for _ in previous_offset..*offset {
builder.push(data.clone());
}
previous_offset = *offset;
}
builder.to_vector()
}
pub(crate) fn replicate_list(c: &ListVector, offsets: &[usize]) -> VectorRef {
assert_eq!(offsets.len(), c.len());
if offsets.is_empty() {
return c.slice(0, 0);
}
let mut builder =
ListVectorBuilder::with_type_capacity(c.item_type(), *offsets.last().unwrap());
let mut previous_offset = 0;
for (i, offset) in offsets.iter().enumerate() {
let data = c.get_data(i);
for _ in previous_offset..*offset {
builder.push(data.clone());
}
previous_offset = *offset;
}
builder.to_vector()
}
#[cfg(test)]
mod tests {
use std::sync::Arc;
use common_time::timestamp::TimeUnit;
use common_time::{Date, Timestamp};
use paste::paste;
use super::*;
use crate::value::{ListValue, ListValueRef};
use crate::vectors::constant::ConstantVector;
use crate::vectors::{
Decimal128Vector, Int32Vector, ListVectorBuilder, NullVector, StringVector, VectorOp,
};
#[test]
fn test_replicate_primitive() {
let v = Int32Vector::from_iterator(0..5);
let offsets = [0, 1, 2, 3, 4];
let v = v.replicate(&offsets);
assert_eq!(4, v.len());
for i in 0..4 {
assert_eq!(Value::Int32(i as i32 + 1), v.get(i));
}
}
#[test]
fn test_replicate_nullable_primitive() {
let v = Int32Vector::from(vec![None, Some(1), None, Some(2)]);
let offsets = [2, 4, 6, 8];
let v = v.replicate(&offsets);
assert_eq!(8, v.len());
let expect: VectorRef = Arc::new(Int32Vector::from(vec![
None,
None,
Some(1),
Some(1),
None,
None,
Some(2),
Some(2),
]));
assert_eq!(expect, v);
}
#[test]
fn test_replicate_scalar() {
let v = StringVector::from_slice(&["0", "1", "2", "3"]);
let offsets = [1, 3, 5, 6];
let v = v.replicate(&offsets);
assert_eq!(6, v.len());
let expect: VectorRef = Arc::new(StringVector::from_slice(&["0", "1", "1", "2", "2", "3"]));
assert_eq!(expect, v);
}
#[test]
fn test_replicate_list() {
let item_type = Arc::new(ConcreteDataType::int32_datatype());
let first = ListValue::new(vec![Value::Int32(1), Value::Int32(2)], item_type.clone());
let second = ListValue::new(vec![Value::Int32(3)], item_type.clone());
let mut builder = ListVectorBuilder::with_type_capacity(item_type, 2);
builder.push(Some(ListValueRef::Ref { val: &first }));
builder.push(Some(ListValueRef::Ref { val: &second }));
let v = builder.finish();
let v = v.replicate(&[1, 3]);
let mut expect_builder =
ListVectorBuilder::with_type_capacity(Arc::new(ConcreteDataType::int32_datatype()), 3);
expect_builder.push(Some(ListValueRef::Ref { val: &first }));
expect_builder.push(Some(ListValueRef::Ref { val: &second }));
expect_builder.push(Some(ListValueRef::Ref { val: &second }));
let expect = expect_builder.to_vector();
assert_eq!(expect, v);
}
#[test]
fn test_replicate_constant() {
let v = Arc::new(StringVector::from_slice(&["hello"]));
let cv = ConstantVector::new(v.clone(), 2);
let offsets = [1, 4];
let cv = cv.replicate(&offsets);
assert_eq!(4, cv.len());
let expect: VectorRef = Arc::new(ConstantVector::new(v, 4));
assert_eq!(expect, cv);
}
#[test]
fn test_replicate_null() {
let v = NullVector::new(0);
let offsets = [];
let v = v.replicate(&offsets);
assert!(v.is_empty());
let v = NullVector::new(3);
let offsets = [1, 3, 5];
let v = v.replicate(&offsets);
assert_eq!(5, v.len());
}
macro_rules! impl_replicate_date_like_test {
($VectorType: ident, $ValueType: ident, $method: ident) => {{
use $crate::vectors::$VectorType;
let v = $VectorType::from_iterator((0..5).map($ValueType::$method));
let offsets = [0, 1, 2, 3, 4];
let v = v.replicate(&offsets);
assert_eq!(4, v.len());
for i in 0..4 {
assert_eq!(
Value::$ValueType($ValueType::$method((i as i32 + 1).into())),
v.get(i)
);
}
}};
}
macro_rules! impl_replicate_timestamp_test {
($unit: ident) => {{
paste!{
use $crate::vectors::[<Timestamp $unit Vector>];
use $crate::timestamp::[<Timestamp $unit>];
let v = [<Timestamp $unit Vector>]::from_iterator((0..5).map([<Timestamp $unit>]::from));
let offsets = [0, 1, 2, 3, 4];
let v = v.replicate(&offsets);
assert_eq!(4, v.len());
for i in 0..4 {
assert_eq!(
Value::Timestamp(Timestamp::new(i as i64 + 1, TimeUnit::$unit)),
v.get(i)
);
}
}
}};
}
#[test]
fn test_replicate_date_like() {
impl_replicate_date_like_test!(DateVector, Date, new);
impl_replicate_timestamp_test!(Second);
impl_replicate_timestamp_test!(Millisecond);
impl_replicate_timestamp_test!(Microsecond);
impl_replicate_timestamp_test!(Nanosecond);
}
#[test]
fn test_replicate_decimal() {
let data = vec![100];
// create a decimal vector
let v = Decimal128Vector::from_values(data.clone())
.with_precision_and_scale(10, 2)
.unwrap();
let offsets = [5];
let v = v.replicate(&offsets);
assert_eq!(5, v.len());
let expect: VectorRef = Arc::new(
Decimal128Vector::from_values(vec![100; 5])
.with_precision_and_scale(10, 2)
.unwrap(),
);
assert_eq!(expect, v);
}
}
src/datatypes/src/vectors/operations/take.rs
+1 -25
View File
@@ -43,8 +43,7 @@ mod tests {
use crate::types::{LogicalPrimitiveType, WrapperType};
use crate::vectors::operations::VectorOp;
use crate::vectors::{
BooleanVector, ConstantVector, Int32Vector, NullVector, PrimitiveVector, StringVector,
UInt32Vector,
BooleanVector, Int32Vector, NullVector, PrimitiveVector, StringVector, UInt32Vector,
};
fn check_take_primitive<T>(
@@ -119,29 +118,6 @@ mod tests {
take_time_like_test!(TimestampNanosecondVector, TimestampNanosecond, from_native);
}
fn check_take_constant(expect_length: usize, input_length: usize, indices: &[u32]) {
let v = ConstantVector::new(Arc::new(Int32Vector::from_slice([111])), input_length);
let indices = UInt32Vector::from_slice(indices);
let out = v.take(&indices).unwrap();
assert!(out.is_const());
assert_eq!(expect_length, out.len());
}
#[test]
fn test_take_constant() {
check_take_constant(2, 5, &[3, 4]);
check_take_constant(3, 10, &[1, 2, 3]);
check_take_constant(4, 10, &[1, 5, 3, 6]);
check_take_constant(5, 10, &[1, 9, 8, 7, 3]);
}
#[test]
#[should_panic]
fn test_take_constant_out_of_index() {
check_take_constant(2, 5, &[3, 5]);
}
#[test]
#[should_panic]
fn test_take_out_of_index() {
src/datatypes/src/vectors/primitive.rs
-35
View File
@@ -366,40 +366,6 @@ where
}
}
pub(crate) fn replicate_primitive<T: LogicalPrimitiveType>(
vector: &PrimitiveVector<T>,
offsets: &[usize],
) -> PrimitiveVector<T> {
assert_eq!(offsets.len(), vector.len());
if offsets.is_empty() {
return vector.get_slice(0, 0);
}
let mut builder = PrimitiveVectorBuilder::<T>::with_capacity(*offsets.last().unwrap());
let mut previous_offset = 0;
for (offset, value) in offsets.iter().zip(vector.array.iter()) {
let repeat_times = *offset - previous_offset;
match value {
Some(data) => {
unsafe {
// Safety: std::iter::Repeat and std::iter::Take implement TrustedLen.
builder
.mutable_array
.append_trusted_len_iter(std::iter::repeat_n(data, repeat_times));
}
}
None => {
builder.mutable_array.append_nulls(repeat_times);
}
}
previous_offset = *offset;
}
builder.finish()
}
#[cfg(test)]
mod tests {
use std::vec;
@@ -433,7 +399,6 @@ mod tests {
fn check_vec(v: Int32Vector) {
assert_eq!(4, v.len());
assert_eq!("Int32Vector", v.vector_type_name());
assert!(!v.is_const());
assert!(v.validity().is_all_valid());
assert!(!v.only_null());
src/datatypes/src/vectors/string.rs
-1
View File
@@ -604,7 +604,6 @@ mod tests {
let v = StringVector::from(strs.clone());
assert_eq!(3, v.len());
assert_eq!("StringVector", v.vector_type_name());
assert!(!v.is_const());
assert!(v.validity().is_all_valid());
assert!(!v.only_null());
assert_eq!(1040, v.memory_size());
src/datatypes/src/vectors/struct_vector.rs
+2 -64
View File
@@ -15,7 +15,7 @@
use std::any::Any;
use std::sync::Arc;
use arrow::array::{MutableArrayData, NullBufferBuilder};
use arrow::array::NullBufferBuilder;
use arrow::compute::TakeOptions;
use arrow::datatypes::DataType as ArrowDataType;
use arrow_array::{Array, ArrayRef, StructArray};
@@ -143,31 +143,6 @@ impl Vector for StructVector {
}
impl VectorOp for StructVector {
fn replicate(&self, offsets: &[usize]) -> VectorRef {
assert_eq!(offsets.len(), self.len());
assert!(offsets.is_sorted(), "offsets must be non-decreasing");
let Some(&output_len) = offsets.last() else {
return self.slice(0, 0);
};
let source = self.array.to_data();
let mut output = MutableArrayData::new(vec![&source], false, output_len);
let mut previous_offset = 0;
for (index, &offset) in offsets.iter().enumerate() {
for _ in previous_offset..offset {
output.extend(0, index, index + 1);
}
previous_offset = offset;
}
Arc::new(StructVector {
array: StructArray::from(output.freeze()),
fields: self.fields.clone(),
})
}
fn cast(&self, _to_type: &ConcreteDataType) -> Result<VectorRef> {
UnsupportedOperationSnafu {
op: "cast",
@@ -466,10 +441,6 @@ impl ScalarVectorBuilder for StructVectorBuilder {
#[cfg(test)]
mod tests {
use arrow::array::{DictionaryArray, Int8Array, StringArray};
use arrow::buffer::NullBuffer;
use arrow::datatypes::Int8Type;
use super::*;
use crate::json::JsonSettings;
use crate::schema::{ColumnDefaultConstraint, ColumnSchema};
@@ -555,7 +526,7 @@ mod tests {
}
#[test]
fn test_replicate_preserves_json2_identity() {
fn test_default_vector_preserves_json2_identity() {
let json = JsonSettings::default()
.encode(serde_json::json!({"answer": 42}))
.unwrap();
@@ -576,39 +547,6 @@ mod tests {
assert_eq!(replicated.len(), 2);
}
#[test]
fn test_replicate_preserves_dictionary_and_nulls() {
let fields = StructType::new(Arc::new(vec![StructField::new(
"label",
ConcreteDataType::dictionary_datatype(
ConcreteDataType::int8_datatype(),
ConcreteDataType::string_datatype(),
),
true,
)]));
let dictionary = DictionaryArray::<Int8Type>::new(
Int8Array::from(vec![Some(0), Some(1)]),
Arc::new(StringArray::from(vec!["a", "b"])),
);
let array = StructArray::new(
fields.as_arrow_fields(),
vec![Arc::new(dictionary)],
Some(NullBuffer::from(vec![true, false])),
);
let vector = StructVector::try_new(fields.clone(), array).unwrap();
let replicated = vector.replicate(&[2, 3]);
assert_eq!(
replicated.data_type(),
ConcreteDataType::struct_datatype(fields)
);
assert_eq!(replicated.len(), 3);
assert_eq!(replicated.null_count(), 1);
assert_eq!(replicated.get(0), replicated.get(1));
assert!(replicated.is_null(2));
}
#[test]
fn test_deep_nested_struct_list() {
// level 1: struct
src/flow/src/batching_mode/frontend_client.rs
+123 -1
View File
@@ -183,13 +183,21 @@ impl DatabaseWithPeer {
/// Try sending a "SELECT 1" to the database
async fn try_select_one(&self) -> Result<(), Error> {
// notice here use `sql` for `SELECT 1` return 1 row
let _ = self
let output = self
.database
.sql("SELECT 1")
.await
.with_context(|_| InvalidRequestSnafu {
context: format!("Failed to handle `SELECT 1` request at {:?}", self.peer),
})?;
if let OutputData::Stream(stream) = output.data {
common_recordbatch::util::collect(stream)
.await
.map_err(BoxedError::new)
.context(ExternalSnafu)?;
}
Ok(())
}
}
@@ -698,9 +706,11 @@ mod tests {
use api::v1::query_request::Query;
use arrow_flight::flight_service_server::FlightServiceServer;
use arrow_flight::{FlightData, Ticket};
use common_grpc::flight::FlightEncoder;
use common_query::{Output, OutputData};
use common_recordbatch::adapter::RecordBatchMetrics;
use common_recordbatch::{OrderOption, RecordBatch, RecordBatchStream};
use datatypes::arrow::datatypes::Schema as ArrowSchema;
use datatypes::prelude::{ConcreteDataType, VectorRef};
use datatypes::schema::{ColumnSchema, Schema};
use datatypes::vectors::Int32Vector;
@@ -775,6 +785,14 @@ mod tests {
#[derive(Debug)]
struct SlowFlight;
struct DelayedEofFlight {
schema_sent: Mutex<Option<tokio::sync::oneshot::Sender<()>>>,
release: Arc<tokio::sync::Notify>,
}
#[derive(Debug)]
struct LateStreamErrorFlight;
struct WaitForConcurrentFlight {
barrier: Arc<tokio::sync::Barrier>,
}
@@ -870,6 +888,45 @@ mod tests {
}
}
#[async_trait::async_trait]
impl FlightCraft for DelayedEofFlight {
async fn do_get(
&self,
_request: TonicRequest<Ticket>,
) -> std::result::Result<TonicResponse<TonicStream<FlightData>>, Status> {
let schema = FlightEncoder::default().encode_schema(&ArrowSchema::empty());
let schema_sent = self.schema_sent.lock().unwrap().take();
let schema_stream = futures::stream::once(async move {
if let Some(schema_sent) = schema_sent {
let _ = schema_sent.send(());
}
Ok(schema)
});
let release = self.release.clone();
let delayed_eof = futures::stream::unfold(release, |release| async move {
release.notified().await;
None::<(std::result::Result<FlightData, Status>, _)>
});
Ok(TonicResponse::new(Box::pin(
schema_stream.chain(delayed_eof),
)))
}
}
#[async_trait::async_trait]
impl FlightCraft for LateStreamErrorFlight {
async fn do_get(
&self,
_request: TonicRequest<Ticket>,
) -> std::result::Result<TonicResponse<TonicStream<FlightData>>, Status> {
let schema = FlightEncoder::default().encode_schema(&ArrowSchema::empty());
let stream =
futures::stream::iter([Ok(schema), Err(Status::unavailable("late stream error"))]);
Ok(TonicResponse::new(Box::pin(stream)))
}
}
#[async_trait::async_trait]
impl FlightCraft for WaitForConcurrentFlight {
async fn do_get(
@@ -1056,6 +1113,71 @@ mod tests {
assert!(format!("{err:?}").contains("Invalid value for flow.return_region_seq"));
}
#[tokio::test]
async fn test_try_select_one_waits_for_stream_eof() {
let (schema_sent, schema_sent_rx) = tokio::sync::oneshot::channel();
let release = Arc::new(tokio::sync::Notify::new());
let (addr, server) = start_flight_server(DelayedEofFlight {
schema_sent: Mutex::new(Some(schema_sent)),
release: release.clone(),
})
.await;
let database = Database::new(
DEFAULT_CATALOG_NAME,
DEFAULT_SCHEMA_NAME,
Client::with_urls([addr.as_str()]),
);
let db = DatabaseWithPeer::new(
database,
Peer {
id: 1,
addr: addr.clone(),
},
);
let mut probe = tokio::spawn(async move { db.try_select_one().await });
timeout(Duration::from_secs(1), schema_sent_rx)
.await
.expect("server should send the schema")
.expect("schema signal should be sent");
assert!(
timeout(Duration::from_millis(100), &mut probe)
.await
.is_err(),
"SELECT 1 must wait for the delayed stream tail and EOF"
);
release.notify_one();
timeout(Duration::from_secs(1), &mut probe)
.await
.expect("SELECT 1 should complete after EOF")
.expect("probe task should not panic")
.expect("SELECT 1 should succeed after EOF");
server.abort();
}
#[tokio::test]
async fn test_try_select_one_propagates_late_stream_error() {
let (addr, server) = start_flight_server(LateStreamErrorFlight).await;
let database = Database::new(
DEFAULT_CATALOG_NAME,
DEFAULT_SCHEMA_NAME,
Client::with_urls([addr.as_str()]),
);
let db = DatabaseWithPeer::new(
database,
Peer {
id: 1,
addr: addr.clone(),
},
);
let err = db.try_select_one().await.unwrap_err();
server.abort();
assert!(format!("{err:?}").contains("late stream error"));
}
#[tokio::test]
async fn test_check_all_frontends_without_auth_fails_fast_on_unauthenticated_frontend() {
let (addr, server) = start_flight_server(RejectUnauthenticatedFlight).await;
src/flow/src/expr/scalar.rs
+1
View File
@@ -216,6 +216,7 @@ impl ScalarExpr {
msg: "Failed to convert literal to scalar value",
})?,
batch.row_count(),
None,
)
.context(DataTypeSnafu {
msg: "Failed to convert scalar value to vector ref when parsing literal",
src/flow/src/server.rs
+1
View File
@@ -124,6 +124,7 @@ impl flow_server::Flow for FlowService {
api::v1::region::InsertRequest {
region_id: insert.region_id,
rows: insert.rows,
skip_wal: false,
partition_expr_version: insert.partition_expr_version,
}
})
src/frontend/src/instance.rs
+7
View File
@@ -1329,6 +1329,7 @@ impl PrometheusHandler for Instance {
self.check_query_target_permission(targets, &query_ctx)
.await?;
let requires_output_ordering = query.requires_output_ordering();
let (query, stmt) = query.into_parts();
let QueryStatement::Promql(eval_stmt, _) = &stmt else {
@@ -1342,6 +1343,12 @@ impl PrometheusHandler for Instance {
.map_err(BoxedError::new)
.context(ExecuteQuerySnafu)?;
let plan = if requires_output_ordering {
plan
} else {
promql::remove_output_sort(plan)
};
interceptor.pre_execute(&query, &eval_stmt.expr, Some(&plan), query_ctx.clone())?;
// Take the EvalStmt from the original QueryStatement and use it to create the CatalogQueryStatement.
src/frontend/src/instance/grpc.rs
+1
View File
@@ -453,6 +453,7 @@ impl Instance {
request.flight_data,
request.record_batch,
request.schema_bytes,
ctx.skip_wal(),
)
.await
.context(TableOperationSnafu)?;
src/frontend/src/instance/promql.rs
+93
View File
@@ -12,6 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use std::sync::Arc;
use std::time::SystemTime;
use auth::PermissionTableTarget;
@@ -21,6 +22,7 @@ use common_catalog::consts::INFORMATION_SCHEMA_NAME;
use common_catalog::format_full_table_name;
use common_recordbatch::util;
use common_telemetry::tracing;
use datafusion_expr::LogicalPlan;
use promql_parser::label::{Matcher, Matchers};
use query::promql;
use query::promql::planner::PromPlanner;
@@ -35,6 +37,21 @@ use crate::error::{
};
use crate::instance::Instance;
/// Strips the output sort a PromQL plan ends with, keeping the plan schema intact.
///
/// Only the sort the caller would observe is removed: recursion stops at any other
/// node, so ordering consumed by windows, limits or PromQL extension nodes stays.
pub(super) fn remove_output_sort(plan: LogicalPlan) -> LogicalPlan {
match plan {
LogicalPlan::Sort(sort) if sort.fetch.is_none() => Arc::unwrap_or_clone(sort.input),
LogicalPlan::Projection(mut projection) => {
projection.input = Arc::new(remove_output_sort(Arc::unwrap_or_clone(projection.input)));
LogicalPlan::Projection(projection)
}
plan => plan,
}
}
impl Instance {
/// Handles metric names query request, returns the names.
#[tracing::instrument(skip_all)]
@@ -180,3 +197,79 @@ impl Instance {
Ok(results)
}
}
#[cfg(test)]
mod tests {
use datafusion::arrow::array::Int64Array;
use datafusion::prelude::{SessionConfig, SessionContext};
use datafusion_expr::{LogicalPlanBuilder, Sort, col, lit};
use super::*;
#[tokio::test]
async fn remove_output_sort_keeps_schema_and_row_selection() {
let input =
LogicalPlanBuilder::values(vec![vec![lit(3_i64)], vec![lit(1_i64)], vec![lit(2_i64)]])
.unwrap()
.build()
.unwrap();
let sort = Sort {
expr: vec![col("column1").sort(true, false)],
input: Arc::new(input),
fetch: None,
};
let projected = LogicalPlanBuilder::from(LogicalPlan::Sort(sort.clone()))
.project(vec![col("column1").alias("sample")])
.unwrap()
.project(vec![col("sample")])
.unwrap()
.build()
.unwrap();
// The root sort is removed, but the sort feeding the limit selects the rows.
let limited = LogicalPlanBuilder::from(LogicalPlan::Sort(sort.clone()))
.limit(0, Some(2))
.unwrap()
.sort(vec![col("column1").sort(false, false)])
.unwrap()
.build()
.unwrap();
let fetched = LogicalPlan::Sort(Sort {
fetch: Some(2),
..sort.clone()
});
let context =
SessionContext::new_with_config(SessionConfig::new().with_target_partitions(1));
for (name, plan, expected) in [
("root", LogicalPlan::Sort(sort), vec![3, 1, 2]),
("projection", projected, vec![3, 1, 2]),
("sort below limit", limited, vec![1, 2]),
("fetch", fetched, vec![1, 2]),
] {
let schema = plan.schema().clone();
let plan = remove_output_sort(plan);
assert_eq!(plan.schema(), &schema, "{name}");
let output = context
.execute_logical_plan(plan)
.await
.unwrap()
.collect()
.await
.unwrap();
let values = output
.iter()
.flat_map(|batch| {
batch
.column(0)
.as_any()
.downcast_ref::<Int64Array>()
.unwrap()
.values()
.iter()
.copied()
})
.collect::<Vec<_>>();
assert_eq!(values, expected, "{name}");
}
}
}
src/meta-srv/src/error.rs
+2 -2
View File
@@ -368,9 +368,9 @@ pub enum Error {
location: Location,
},
#[snafu(display("Failed to deserialize from json: {}", input))]
#[snafu(display("Failed to deserialize from json payload of length {}", input_len))]
DeserializeFromJson {
input: String,
input_len: usize,
#[snafu(source)]
error: serde_json::error::Error,
#[snafu(implicit)]
src/meta-srv/src/event/gc.rs
+3
View File
@@ -218,6 +218,7 @@ mod tests {
use common_event_recorder::testing::assert_event_contract;
use common_event_recorder::{EventTypeFilter, PersistentEventContext, TriggerReason};
use common_meta::key::TableMetadataManager;
use common_meta::key::runtime_switch::RuntimeSwitchManager;
use common_meta::kv_backend::memory::MemoryKvBackend;
use common_meta::sequence::SequenceBuilder;
use common_procedure::{
@@ -588,11 +589,13 @@ mod tests {
fn batch_gc_procedure() -> BatchGcProcedure {
let kv_backend = Arc::new(MemoryKvBackend::new());
let table_metadata_manager = Arc::new(TableMetadataManager::new(kv_backend.clone()));
let runtime_switch_manager = Arc::new(RuntimeSwitchManager::new(kv_backend.clone()));
let mailbox_sequence = SequenceBuilder::new("test_batch_gc_event", kv_backend).build();
let mailbox = MailboxContext::new(mailbox_sequence);
BatchGcProcedure::new(
mailbox.mailbox().clone(),
table_metadata_manager,
runtime_switch_manager,
"localhost".to_string(),
vec![RegionId::new(1024, 1)],
true,
src/meta-srv/src/gc/ctx.rs
+6
View File
@@ -27,6 +27,7 @@ use common_meta::ddl_manager::DdlManagerRef;
#[cfg(feature = "enterprise")]
use common_meta::key::DroppedTableName;
use common_meta::key::TableMetadataManagerRef;
use common_meta::key::runtime_switch::RuntimeSwitchManagerRef;
use common_meta::key::table_repart::TableRepartValue;
use common_meta::key::table_route::PhysicalTableRouteValue;
#[cfg(feature = "enterprise")]
@@ -155,6 +156,8 @@ pub(crate) struct DefaultGcSchedulerCtx {
pub(crate) table_metadata_manager: TableMetadataManagerRef,
/// Procedure manager.
pub(crate) procedure_manager: ProcedureManagerRef,
/// Runtime switch manager used by recovered and newly submitted GC procedures.
pub(crate) runtime_switch_manager: RuntimeSwitchManagerRef,
/// DDL manager used to submit the existing purge procedure.
#[cfg(feature = "enterprise")]
pub(crate) ddl_manager: DdlManagerRef,
@@ -176,6 +179,7 @@ impl DefaultGcSchedulerCtx {
pub fn try_new(
table_metadata_manager: TableMetadataManagerRef,
procedure_manager: ProcedureManagerRef,
runtime_switch_manager: RuntimeSwitchManagerRef,
#[cfg(feature = "enterprise")] ddl_manager: DdlManagerRef,
meta_peer_client: MetaPeerClientRef,
mailbox: MailboxRef,
@@ -184,6 +188,7 @@ impl DefaultGcSchedulerCtx {
Ok(Self {
table_metadata_manager,
procedure_manager,
runtime_switch_manager,
#[cfg(feature = "enterprise")]
ddl_manager,
#[cfg(feature = "enterprise")]
@@ -320,6 +325,7 @@ impl DefaultGcSchedulerCtx {
let procedure = BatchGcProcedure::new(
self.mailbox.clone(),
self.table_metadata_manager.clone(),
self.runtime_switch_manager.clone(),
self.server_addr.clone(),
region_ids.to_vec(),
full_file_listing,
src/meta-srv/src/gc/procedure.rs
+288 -99
View File
@@ -18,8 +18,14 @@ use std::sync::Arc;
use std::time::Duration;
use api::v1::meta::MailboxMessage;
use common_meta::instruction::{self, GcRegions, GetFileRefs, GetFileRefsReply, InstructionReply};
use common_error::ext::PlainError;
use common_error::status_code::StatusCode;
use common_meta::instruction::{
self, GetPackedFileRefs, GetPackedFileRefsReply, InstructionReply, PackedFileRefsManifest,
PackedGcRegions, PackedRegionFileRefs,
};
use common_meta::key::TableMetadataManagerRef;
use common_meta::key::runtime_switch::RuntimeSwitchManagerRef;
use common_meta::key::table_repart::TableRepartValue;
use common_meta::key::table_route::PhysicalTableRouteValue;
use common_meta::lock_key::{RegionLock, TableLock};
@@ -34,6 +40,7 @@ use common_telemetry::tracing::Instrument as _;
use common_telemetry::tracing_context::TracingContext;
use common_telemetry::{debug, error, info, warn};
use futures::future::join_all;
use futures::stream::{FuturesUnordered, StreamExt};
use itertools::Itertools as _;
use serde::{Deserialize, Serialize};
use snafu::ResultExt as _;
@@ -49,14 +56,14 @@ use crate::metrics::{METRIC_META_GC_DATANODE_CALLS_TOTAL, METRIC_META_GC_FAILED_
use crate::procedure::utils::{instruction_error_result, instruction_to_error};
use crate::service::mailbox::{Channel, MailboxReceiver, MailboxRef};
async fn send_get_file_refs_inner(
async fn send_get_packed_file_refs_inner(
mailbox: &MailboxRef,
server_addr: &str,
peer: &Peer,
instruction: GetFileRefs,
instruction: GetPackedFileRefs,
timeout: Duration,
) -> Result<MailboxReceiver> {
let instruction = instruction::Instruction::GetFileRefs(instruction);
let instruction = instruction::Instruction::GetPackedFileRefs(instruction);
let tracing_ctx = TracingContext::from_current_span();
let msg = MailboxMessage::json_message(
&format!("Get file references: {}", instruction),
@@ -75,25 +82,25 @@ async fn send_get_file_refs_inner(
.await
}
async fn recv_get_file_refs_reply(
async fn recv_get_packed_file_refs_reply(
peer: &Peer,
mailbox_rx: MailboxReceiver,
) -> Result<GetFileRefsReply> {
) -> Result<GetPackedFileRefsReply> {
let reply = match mailbox_rx.await {
Ok(reply_msg) => HeartbeatMailbox::json_reply(&reply_msg)?,
Err(e) => {
error!(
e; "Failed to receive reply from datanode {} for GetFileRefs instruction",
e; "Failed to receive reply from datanode {} for GetPackedFileRefs instruction",
peer,
);
return Err(e);
}
};
let InstructionReply::GetFileRefs(reply) = reply else {
let InstructionReply::GetPackedFileRefs(reply) = reply else {
return error::UnexpectedInstructionReplySnafu {
mailbox_message: format!("{:?}", reply),
reason: "Unexpected reply of the GetFileRefs instruction",
mailbox_message: "unexpected instruction reply for GetPackedFileRefs".to_string(),
reason: "Unexpected reply of the GetPackedFileRefs instruction",
}
.fail();
};
@@ -104,12 +111,11 @@ async fn recv_get_file_refs_reply(
async fn send_gc_regions_inner(
mailbox: &MailboxRef,
peer: &Peer,
gc_regions: &GcRegions,
instruction: instruction::Instruction,
server_addr: &str,
timeout: Duration,
description: &str,
) -> Result<MailboxReceiver> {
let instruction = instruction::Instruction::GcRegions(gc_regions.clone());
let tracing_ctx = TracingContext::from_current_span();
let msg = MailboxMessage::json_message(
&format!("{}: {}", description, instruction),
@@ -128,9 +134,43 @@ async fn send_gc_regions_inner(
.await
}
fn scoped_gc_instruction(
regions: Vec<RegionId>,
full_manifest: &FileRefsManifest,
full_file_listing: bool,
) -> instruction::Instruction {
let packed_file_refs_manifest = PackedFileRefsManifest {
file_refs: regions
.iter()
.filter_map(|region| {
full_manifest
.file_refs
.get(region)
.map(|refs| (*region, PackedRegionFileRefs::from_refs(refs)))
})
.collect(),
manifest_version: regions
.iter()
.filter_map(|region| {
full_manifest
.manifest_version
.get(region)
.map(|version| (*region, *version))
})
.collect(),
cross_region_refs: HashMap::new(),
};
instruction::Instruction::PackedGcRegions(PackedGcRegions {
regions,
packed_file_refs_manifest,
full_file_listing,
})
}
async fn recv_gc_regions_reply(
peer: &Peer,
gc_regions: &GcRegions,
regions: &[RegionId],
description: &str,
mailbox_rx: MailboxReceiver,
) -> Result<GcReport> {
@@ -147,7 +187,7 @@ async fn recv_gc_regions_reply(
let InstructionReply::GcRegions(reply) = reply else {
return error::UnexpectedInstructionReplySnafu {
mailbox_message: format!("{:?}", reply),
mailbox_message: "unexpected instruction reply for GcRegions".to_string(),
reason: "Unexpected reply of the GcRegions instruction",
}
.fail();
@@ -158,14 +198,16 @@ async fn recv_gc_regions_reply(
Ok(report) => Ok(report),
Err(e) => {
error!(
e; "Datanode {} reported error during GC for regions {:?}",
peer, gc_regions
e; "Datanode {} reported error during GC for {} regions",
peer, regions.len()
);
instruction_error_result(
&e,
format!(
"Datanode {} reported error during GC for regions {:?}: {}",
peer, gc_regions, e
"Datanode {} reported error during GC for {} regions: {}",
peer,
regions.len(),
e
),
)
}
@@ -177,6 +219,7 @@ async fn recv_gc_regions_reply(
pub struct BatchGcProcedure {
mailbox: MailboxRef,
table_metadata_manager: TableMetadataManagerRef,
runtime_switch_manager: RuntimeSwitchManagerRef,
data: BatchGcData,
}
@@ -217,9 +260,11 @@ pub enum State {
impl BatchGcProcedure {
pub const TYPE_NAME: &'static str = "metasrv-procedure::BatchGcProcedure";
#[allow(clippy::too_many_arguments)]
pub fn new(
mailbox: MailboxRef,
table_metadata_manager: TableMetadataManagerRef,
runtime_switch_manager: RuntimeSwitchManagerRef,
server_addr: String,
regions: Vec<RegionId>,
full_file_listing: bool,
@@ -229,6 +274,7 @@ impl BatchGcProcedure {
Self {
mailbox,
table_metadata_manager,
runtime_switch_manager,
data: BatchGcData {
state: State::Start,
server_addr,
@@ -251,6 +297,7 @@ impl BatchGcProcedure {
pub fn new_update_repartition_for_test(
mailbox: MailboxRef,
table_metadata_manager: TableMetadataManagerRef,
runtime_switch_manager: RuntimeSwitchManagerRef,
server_addr: String,
regions: Vec<RegionId>,
file_refs: FileRefsManifest,
@@ -259,6 +306,7 @@ impl BatchGcProcedure {
Self {
mailbox,
table_metadata_manager,
runtime_switch_manager,
data: BatchGcData {
state: State::UpdateRepartition,
server_addr,
@@ -286,6 +334,22 @@ impl BatchGcProcedure {
})
}
async fn check_maintenance_mode(&self) -> ProcedureResult<()> {
let enabled = self
.runtime_switch_manager
.maintenance_mode()
.await
.context(error::RuntimeSwitchManagerSnafu)
.map_err(ProcedureError::retry_later)?;
if enabled {
return Err(ProcedureError::external(PlainError::new(
"maintenance mode is enabled".to_string(),
StatusCode::IllegalState,
)));
}
Ok(())
}
fn merge_gc_report(&mut self, report: GcReport) {
let accumulated = self.data.gc_report.get_or_insert_default();
// Deleted objects are cumulative, while these sets describe the latest outcome
@@ -644,10 +708,10 @@ impl BatchGcProcedure {
}
}
// Send GetFileRefs instructions to each datanode
// Send packed GetFileRefs instructions to each datanode
let mut all_file_refs: HashMap<RegionId, HashSet<_>> = HashMap::new();
let mut all_manifest_versions = HashMap::new();
let mut all_cross_region_refs = HashMap::new();
let mut all_cross_region_refs: HashMap<RegionId, HashSet<RegionId>> = HashMap::new();
let mut peers = HashSet::new();
peers.extend(datanode2query_regions.keys().cloned());
@@ -655,33 +719,34 @@ impl BatchGcProcedure {
let mailbox = &self.mailbox;
let server_addr = &self.data.server_addr;
let mut tasks = Vec::new();
// Each future owns both send and receive, so a completed reply is merged and
// dropped immediately rather than retained in a second join_all buffer.
let mut tasks = FuturesUnordered::new();
for peer in peers {
let regions = datanode2query_regions.remove(&peer).unwrap_or_default();
let related_regions_for_peer =
datanode2related_regions.remove(&peer).unwrap_or_default();
if regions.is_empty() && related_regions_for_peer.is_empty() {
continue;
}
tasks.push(async move {
let instruction = GetFileRefs {
query_regions: regions.clone(),
related_regions: related_regions_for_peer.clone(),
let instruction = GetPackedFileRefs {
query_regions: regions,
related_regions: related_regions_for_peer,
};
let reply =
send_get_file_refs_inner(mailbox, server_addr, &peer, instruction, timeout)
.await;
(peer, regions, related_regions_for_peer, reply)
let rx = send_get_packed_file_refs_inner(
mailbox,
server_addr,
&peer,
instruction,
timeout,
)
.await?;
let reply = recv_get_packed_file_refs_reply(&peer, rx).await?;
Ok::<_, crate::error::Error>((peer, reply))
});
}
let mut recv_tasks = Vec::new();
// store error to make sure metrics doesn't ignore other peers
let mut first_error = None;
let mut record_get_file_refs_error = |e| {
METRIC_META_GC_DATANODE_CALLS_TOTAL
@@ -691,80 +756,56 @@ impl BatchGcProcedure {
first_error = Some(e);
}
};
for (peer, regions, related_regions_for_peer, reply) in join_all(tasks).await {
match reply {
Ok(mailbox_rx) => {
recv_tasks.push(async move {
let reply = recv_get_file_refs_reply(&peer, mailbox_rx).await;
(peer, regions, related_regions_for_peer, reply)
});
}
Err(e) => record_get_file_refs_error(e),
}
}
let replies = join_all(recv_tasks).await;
for (peer, regions, related_regions_for_peer, reply) in replies {
let reply = match reply {
while let Some(result) = tasks.next().await {
let (peer, reply) = match result {
Ok(reply) => reply,
Err(e) => {
record_get_file_refs_error(e);
continue;
}
};
debug!(
"Got file references from datanode: {:?}, query_regions: {:?}, related_regions: {:?}, reply: {:?}",
peer, regions, related_regions_for_peer, reply
);
if !reply.success {
METRIC_META_GC_DATANODE_CALLS_TOTAL
.with_label_values(&["get_file_refs", "error"])
.inc();
let err = if let Some(error) = &reply.error {
instruction_to_error(
error,
format!(
"Failed to get file references from datanode {}: {:?}",
peer, error
),
format!("Failed to get file references from datanode {peer}"),
)
} else {
error::UnexpectedSnafu {
violated: format!(
"Failed to get file references from datanode {}: {:?}",
peer, reply.error
),
violated:
"Datanode returned an unsuccessful GetPackedFileRefs reply without an error"
.to_string(),
}
.build()
};
record_get_file_refs_error(err);
continue;
}
let manifest = match reply.packed_file_refs_manifest.into_manifest() {
Ok(manifest) => manifest,
Err(err) => {
record_get_file_refs_error(error::Error::Other {
source: common_error::ext::BoxedError::new(err),
location: snafu::Location::new(file!(), line!(), 0),
});
continue;
}
};
METRIC_META_GC_DATANODE_CALLS_TOTAL
.with_label_values(&["get_file_refs", "success"])
.inc();
// Merge the file references from this datanode
for (region_id, file_refs) in reply.file_refs_manifest.file_refs {
all_file_refs
.entry(region_id)
.or_default()
.extend(file_refs);
for (region_id, refs) in manifest.file_refs {
all_file_refs.entry(region_id).or_default().extend(refs);
}
// region manifest version should be the smallest one among all peers, so outdated region can be detected
for (region_id, version) in reply.file_refs_manifest.manifest_version {
for (region_id, version) in manifest.manifest_version {
let entry = all_manifest_versions.entry(region_id).or_insert(version);
*entry = (*entry).min(version);
}
for (region_id, related_region_ids) in reply.file_refs_manifest.cross_region_refs {
let entry = all_cross_region_refs
for (region_id, related) in manifest.cross_region_refs {
all_cross_region_refs
.entry(region_id)
.or_insert_with(HashSet::new);
entry.extend(related_region_ids);
.or_default()
.extend(related);
}
}
@@ -813,28 +854,23 @@ impl BatchGcProcedure {
let tasks = datanode2regions
.into_iter()
.map(|(peer, regions_for_peer)| {
let gc_regions = GcRegions {
regions: regions_for_peer.clone(),
// file_refs_manifest could be somewhere large. But still intentionally clone per datanode here:
// this path is admin-triggered or scheduler-triggered, peer count is expected to be bounded, and
// and abnormal manifest growth should be addressed at the source
file_refs_manifest: file_refs.clone(),
full_file_listing,
};
let region_count = gc_regions.regions.len() as u64;
let region_count = regions_for_peer.len() as u64;
let regions = regions_for_peer.clone();
let instruction =
scoped_gc_instruction(regions_for_peer, file_refs, full_file_listing);
async move {
let report = send_gc_regions_inner(
mailbox,
&peer,
&gc_regions,
instruction,
server_addr,
timeout,
"Batch GC",
)
.await;
(peer, gc_regions, region_count, report)
(peer, regions, region_count, report)
}
});
@@ -851,12 +887,12 @@ impl BatchGcProcedure {
first_error = Some(e);
}
};
for (peer, gc_regions, region_count, report) in join_all(tasks).await {
for (peer, regions, region_count, report) in join_all(tasks).await {
match report {
Ok(mailbox_rx) => {
recv_tasks.push(async move {
let report =
recv_gc_regions_reply(&peer, &gc_regions, "Batch GC", mailbox_rx).await;
recv_gc_regions_reply(&peer, &regions, "Batch GC", mailbox_rx).await;
(peer, region_count, report)
});
}
@@ -887,12 +923,12 @@ impl BatchGcProcedure {
if need_retry.is_empty() {
info!(
"GC report from datanode {}: successfully deleted files for regions {:?}",
"GC report from datanode {}: successfully deleted files for region IDs {:?}",
peer, success
);
} else {
warn!(
"GC report from datanode {}: successfully deleted files for regions {:?}, need retry for regions {:?}",
"GC report from datanode {}: successfully deleted files for region IDs {:?}, need retry for region IDs {:?}",
peer, success, need_retry
);
}
@@ -921,6 +957,8 @@ impl Procedure for BatchGcProcedure {
}
async fn execute(&mut self, ctx: &ProcedureContext) -> ProcedureResult<Status> {
self.check_maintenance_mode().await?;
match self.data.state {
State::Start => {
let _regions_span = common_telemetry::tracing::debug_span!(
@@ -1122,11 +1160,15 @@ mod tests {
use api::v1::meta::MailboxMessage;
use api::v1::meta::mailbox_message::Payload;
use common_meta::instruction::{GcRegionsReply, InstructionReply};
use common_meta::instruction::{GcRegionsReply, Instruction, InstructionReply};
use common_meta::key::TableMetadataManager;
use common_meta::key::runtime_switch::RuntimeSwitchManager;
use common_meta::kv_backend::memory::MemoryKvBackend;
use common_meta::kv_backend::test_util::MockKvBackendBuilder;
use common_meta::peer::Peer;
use common_meta::sequence::SequenceBuilder;
use common_procedure::Context as ProcedureContext;
use common_procedure_test::MockContextProvider;
use common_time::util::current_time_millis;
use store_api::storage::FileId;
use tokio::sync::mpsc;
@@ -1135,6 +1177,38 @@ mod tests {
use crate::procedure::test_util::{MailboxContext, send_mock_reply};
use crate::service::mailbox::Channel;
#[test]
fn test_scoped_gc_instruction_selects_and_scopes_manifest() {
let region = RegionId::new(7, 3);
let other = RegionId::new(7, 4);
let mut manifest = FileRefsManifest::default();
manifest.file_refs.insert(region, HashSet::new());
manifest.file_refs.insert(other, HashSet::new());
manifest.manifest_version.insert(region, 42);
let packed = scoped_gc_instruction(vec![region], &manifest, true);
let Instruction::PackedGcRegions(packed) = packed else {
panic!("expected packed instruction");
};
assert_eq!(packed.regions, vec![region]);
assert_eq!(
packed.packed_file_refs_manifest.manifest_version[&region],
42
);
assert!(
!packed
.packed_file_refs_manifest
.file_refs
.contains_key(&other)
);
assert!(
packed
.packed_file_refs_manifest
.cross_region_refs
.is_empty()
);
}
#[test]
fn test_done_with_gc_report_keeps_report() {
let region_id = RegionId::new(1024, 1);
@@ -1215,6 +1289,7 @@ mod tests {
let kv_backend = Arc::new(MemoryKvBackend::new());
let table_metadata_manager = Arc::new(TableMetadataManager::new(kv_backend.clone()));
let runtime_switch_manager = Arc::new(RuntimeSwitchManager::new(kv_backend.clone()));
let mailbox_sequence =
SequenceBuilder::new("test_batch_gc_partial_report", kv_backend).build();
let mut mailbox = MailboxContext::new(mailbox_sequence);
@@ -1230,6 +1305,7 @@ mod tests {
let mut procedure = BatchGcProcedure::new(
mailbox.mailbox().clone(),
table_metadata_manager,
runtime_switch_manager,
"localhost".to_string(),
vec![first_region, second_region],
true,
@@ -1265,11 +1341,15 @@ mod tests {
fn batch_gc_procedure() -> BatchGcProcedure {
let kv_backend = Arc::new(MemoryKvBackend::new());
let table_metadata_manager = Arc::new(TableMetadataManager::new(kv_backend.clone()));
let mailbox_sequence = SequenceBuilder::new("test_batch_gc_procedure", kv_backend).build();
let runtime_switch_manager = Arc::new(RuntimeSwitchManager::new(kv_backend));
let mailbox_sequence =
SequenceBuilder::new("test_batch_gc_procedure", Arc::new(MemoryKvBackend::new()))
.build();
let mailbox = MailboxContext::new(mailbox_sequence);
BatchGcProcedure::new(
mailbox.mailbox().clone(),
table_metadata_manager,
runtime_switch_manager,
"localhost".to_string(),
vec![RegionId::new(1024, 1)],
true,
@@ -1277,4 +1357,113 @@ mod tests {
HashMap::new(),
)
}
#[tokio::test]
async fn test_maintenance_mode_gates_every_gc_state() {
let states = [
State::Start,
State::Acquiring,
State::Gcing,
State::UpdateRepartition,
];
for state in states {
let kv_backend = Arc::new(MemoryKvBackend::new());
let table_metadata_manager = Arc::new(TableMetadataManager::new(kv_backend.clone()));
let runtime_switch_manager = Arc::new(RuntimeSwitchManager::new(kv_backend.clone()));
runtime_switch_manager.set_maintenance_mode().await.unwrap();
let mailbox_sequence =
SequenceBuilder::new("test_batch_gc_maintenance_gate", kv_backend).build();
let mut mailbox = MailboxContext::new(mailbox_sequence);
let (tx, mut rx) = mpsc::channel(1);
mailbox
.insert_heartbeat_response_receiver(Channel::Datanode(1), tx)
.await;
let mut procedure = BatchGcProcedure::new(
mailbox.mailbox().clone(),
table_metadata_manager,
runtime_switch_manager,
"localhost".to_string(),
vec![RegionId::new(1024, 1)],
true,
Duration::from_secs(10),
HashMap::new(),
);
procedure.data.state = state.clone();
let dump_before = procedure.dump().unwrap();
let ctx = ProcedureContext {
procedure_id: common_procedure::ProcedureId::random(),
provider: Arc::new(MockContextProvider::default()),
event_context: None,
};
let err = procedure.execute(&ctx).await.unwrap_err();
assert!(!err.is_retry_later());
assert_eq!(
common_error::ext::ErrorExt::retry_hint(&err),
common_error::ext::RetryHint::NonRetryable
);
assert_eq!(procedure.data.state, state);
assert_eq!(procedure.dump().unwrap(), dump_before);
if matches!(state, State::Acquiring | State::Gcing) {
assert!(rx.try_recv().is_err());
}
}
}
#[tokio::test]
async fn test_maintenance_mode_read_error_retries_without_advancing_state() {
let calls = Arc::new(std::sync::atomic::AtomicUsize::new(0));
let failing_once_calls = calls.clone();
let kv_backend = Arc::new(
MockKvBackendBuilder::default()
.range_fn(Arc::new(move |_| {
if failing_once_calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst) == 0 {
common_meta::error::UnexpectedSnafu {
err_msg: "maintenance read failed",
}
.fail()
} else {
Ok(common_meta::rpc::store::RangeResponse {
kvs: vec![],
more: false,
})
}
}))
.build()
.unwrap(),
);
let table_metadata_manager = Arc::new(TableMetadataManager::new(kv_backend.clone()));
let runtime_switch_manager = Arc::new(RuntimeSwitchManager::new(kv_backend.clone()));
let mailbox_sequence =
SequenceBuilder::new("test_batch_gc_maintenance_read_error", kv_backend).build();
let mailbox = MailboxContext::new(mailbox_sequence);
let mut procedure = BatchGcProcedure::new(
mailbox.mailbox().clone(),
table_metadata_manager,
runtime_switch_manager,
"localhost".to_string(),
vec![RegionId::new(1024, 1)],
true,
Duration::from_secs(10),
HashMap::new(),
);
let ctx = ProcedureContext {
procedure_id: common_procedure::ProcedureId::random(),
provider: Arc::new(MockContextProvider::default()),
event_context: None,
};
let dump_before = procedure.dump().unwrap();
assert!(procedure.execute(&ctx).await.unwrap_err().is_retry_later());
assert_eq!(procedure.data.state, State::Start);
assert_eq!(procedure.dump().unwrap(), dump_before);
assert!(matches!(
procedure.execute(&ctx).await.unwrap(),
Status::Executing { .. }
));
assert_eq!(procedure.data.state, State::Acquiring);
}
}
src/meta-srv/src/handler.rs
+41 -8
View File
@@ -428,9 +428,11 @@ impl HeartbeatMailbox {
.as_ref()
.with_context(|| UnexpectedInstructionReplySnafu {
mailbox_message: msg.to_string(),
reason: format!("empty payload, msg: {msg:?}"),
reason: "empty JSON payload".to_string(),
})?;
serde_json::from_str(payload).context(DeserializeFromJsonSnafu { input: payload })
serde_json::from_str(payload).context(DeserializeFromJsonSnafu {
input_len: payload.len(),
})
}
/// Parses the [Instruction] from [MailboxMessage].
@@ -443,9 +445,11 @@ impl HeartbeatMailbox {
.as_ref()
.with_context(|| UnexpectedInstructionReplySnafu {
mailbox_message: msg.to_string(),
reason: format!("empty payload, msg: {msg:?}"),
reason: "empty JSON payload".to_string(),
})?;
serde_json::from_str(payload).context(DeserializeFromJsonSnafu { input: payload })
serde_json::from_str(payload).context(DeserializeFromJsonSnafu {
input_len: payload.len(),
})
}
pub fn create(pushers: Pushers, sequence: Sequence) -> MailboxRef {
@@ -526,7 +530,12 @@ impl Mailbox for HeartbeatMailbox {
msg.id = message_id;
let pusher_id = ch.pusher_id();
debug!("Sending mailbox message {msg:?} to {pusher_id}");
let payload_len = msg
.payload
.as_ref()
.map(|Payload::Json(payload)| payload.len())
.unwrap_or_default();
debug!(message_id, payload_len, %pusher_id, "Sending mailbox message");
let (tx, rx) = oneshot::channel();
let _ = self.senders.insert(message_id, tx);
@@ -548,7 +557,12 @@ impl Mailbox for HeartbeatMailbox {
msg.id = message_id;
let pusher_id = ch.pusher_id();
debug!("Sending mailbox message {msg:?} to {pusher_id}");
let payload_len = msg
.payload
.as_ref()
.map(|Payload::Json(payload)| payload.len())
.unwrap_or_default();
debug!(message_id, payload_len, %pusher_id, "Sending one-way mailbox message");
self.pushers.push(pusher_id, msg).await?;
@@ -560,7 +574,18 @@ impl Mailbox for HeartbeatMailbox {
}
async fn on_recv(&self, id: MessageId, maybe_msg: Result<MailboxMessage>) -> Result<()> {
debug!("Received mailbox message {maybe_msg:?}");
let payload_len = maybe_msg
.as_ref()
.ok()
.and_then(|msg| msg.payload.as_ref())
.map(|Payload::Json(payload)| payload.len())
.unwrap_or_default();
debug!(
message_id = id,
payload_len,
success = maybe_msg.is_ok(),
"Received mailbox message"
);
let _ = self.timeouts.remove(&id);
@@ -568,7 +593,15 @@ impl Mailbox for HeartbeatMailbox {
tx.send(maybe_msg)
.map_err(|_| error::MailboxClosedSnafu { id }.build())?;
} else if let Ok(finally_msg) = maybe_msg {
warn!("The response arrived too late: {finally_msg:?}");
let payload_len = finally_msg
.payload
.as_ref()
.map(|Payload::Json(payload)| payload.len())
.unwrap_or_default();
warn!(
message_id = id,
payload_len, "The mailbox response arrived too late"
);
}
Ok(())
src/meta-srv/src/key.rs
+3 -1
View File
@@ -87,7 +87,9 @@ impl FromStr for LeaseValue {
type Err = error::Error;
fn from_str(value: &str) -> crate::Result<Self> {
serde_json::from_str(value).context(error::DeserializeFromJsonSnafu { input: value })
serde_json::from_str(value).context(error::DeserializeFromJsonSnafu {
input_len: value.len(),
})
}
}
src/meta-srv/src/metasrv/builder.rs
+1
View File
@@ -530,6 +530,7 @@ impl MetasrvBuilder {
let gc_scheduler_ctx = DefaultGcSchedulerCtx::try_new(
table_metadata_manager.clone(),
procedure_manager.clone(),
runtime_switch_manager.clone(),
#[cfg(feature = "enterprise")]
ddl_manager.clone(),
meta_peer_client.clone(),
src/meta-srv/src/service/heartbeat.rs
+13 -2
View File
@@ -206,7 +206,13 @@ where
/// Handles the incoming heartbeat request, and returns whether to continue the session.
async fn handle_request(&mut self, req: HeartbeatRequest, is_handshake: bool) -> bool {
debug!("Receiving heartbeat request: {:?}", req);
let mailbox_message_id = req.mailbox_message.as_ref().map(|msg| msg.id);
debug!(
sender_id = %self.sender_id,
?mailbox_message_id,
is_handshake,
"Receiving heartbeat request"
);
let sender_id = self.sender_id.to_string();
METRIC_META_HEARTBEAT_RECV
@@ -224,7 +230,12 @@ where
let is_not_leader = res.as_ref().is_ok_and(|r| r.is_not_leader());
debug!("Sending heartbeat response: {:?}", res);
debug!(
sender_id = %self.sender_id,
success = res.is_ok(),
is_not_leader,
"Sending heartbeat response"
);
if self.tx.send(res).await.is_err() {
info!(
src/metric-engine/src/engine.rs
+1
View File
@@ -636,6 +636,7 @@ mod test {
.handle_request(
logical_region_id,
RegionRequest::Put(RegionPutRequest {
skip_wal: false,
rows: Rows {
schema: row_schema_with_tags(&["job"]),
rows: build_rows(1, 5),
src/metric-engine/src/engine/bulk_insert.rs
+134 -16
View File
@@ -124,6 +124,7 @@ impl MetricEngineInner {
let aligned_schema_version = Some(self.physical_schema_version(data_region_id).await?);
let request = RegionBulkInsertsRequest {
skip_wal: request.skip_wal,
region_id: data_region_id,
payload: modified_batch,
raw_data: ArrowIpc {
@@ -241,18 +242,28 @@ mod tests {
use common_error::ext::ErrorExt;
use common_query::prelude::{greptime_timestamp, greptime_value};
use common_recordbatch::RecordBatches;
use datafusion::parquet::basic::Encoding;
use datatypes::arrow::array::{Float64Array, StringArray, TimestampMillisecondArray};
use datatypes::arrow::datatypes::{DataType, Field, Schema as ArrowSchema, TimeUnit};
use datatypes::arrow::record_batch::RecordBatch;
use mito2::config::MitoConfig;
use store_api::metric_engine_consts::PRIMARY_KEY_ENCODING;
use mito2::sst::parquet::metadata::MetadataLoader;
use mito2::sst::parquet::reader::MetadataCacheMetrics;
use store_api::metric_engine_consts::{
METRIC_ENGINE_NAME, PHYSICAL_TABLE_METADATA_KEY, PRIMARY_KEY_ENCODING,
};
use store_api::mito_engine_options::EXPERIMENTAL_SST_FLOAT_FIELD_ENCODING;
use store_api::path_utils::table_dir;
use store_api::region_engine::RegionEngine;
use store_api::region_request::{RegionBulkInsertsRequest, RegionPutRequest, RegionRequest};
use store_api::region_request::{
PathType, RegionBulkInsertsRequest, RegionCloseRequest, RegionFlushRequest,
RegionOpenRequest, RegionPutRequest, RegionRequest,
};
use store_api::storage::{RegionId, ScanRequest};
use super::record_batch_to_ipc;
use crate::batch_modifier::{TagColumnInfo, modify_batch_sparse};
use crate::engine::MetricEngine;
use crate::error::Error;
use crate::test_util::{self, TestEnv};
@@ -287,9 +298,14 @@ mod tests {
.unwrap()
}
fn build_bulk_request(logical_region_id: RegionId, batch: RecordBatch) -> RegionRequest {
fn build_bulk_request(
logical_region_id: RegionId,
batch: RecordBatch,
skip_wal: bool,
) -> RegionRequest {
let (schema, data_header, payload) = record_batch_to_ipc(&batch).unwrap();
RegionRequest::BulkInserts(RegionBulkInsertsRequest {
skip_wal,
region_id: logical_region_id,
payload: batch,
raw_data: ArrowIpc {
@@ -332,6 +348,7 @@ mod tests {
let batch = build_logical_batch(0, 0);
let request = RegionRequest::BulkInserts(RegionBulkInsertsRequest {
skip_wal: false,
region_id: logical_region_id,
payload: batch,
raw_data: ArrowIpc::default(),
@@ -348,6 +365,11 @@ mod tests {
#[tokio::test]
async fn test_bulk_insert_physical_region_passthrough() {
check_bulk_insert_physical_region_passthrough(false).await;
check_bulk_insert_physical_region_passthrough(true).await;
}
async fn check_bulk_insert_physical_region_passthrough(skip_wal: bool) {
// Use flat format so that BulkMemtable is used (supports write_bulk).
let mito_config = MitoConfig {
default_flat_format: true,
@@ -355,12 +377,13 @@ mod tests {
};
let env = TestEnv::with_mito_config("", mito_config, Default::default()).await;
env.init_metric_region().await;
env.metric().inner.flush_task.stop().await.unwrap();
let physical_region_id = env.default_physical_region_id();
let logical_region_id = env.default_logical_region_id();
// First, do a normal logical bulk insert so we can compare results.
let logical_batch = build_logical_batch(0, 3);
let logical_request = build_bulk_request(logical_region_id, logical_batch.clone());
let logical_request = build_bulk_request(logical_region_id, logical_batch, skip_wal);
let response = env
.metric()
.handle_request(logical_region_id, logical_request)
@@ -385,7 +408,7 @@ mod tests {
&non_tag_indices,
)
.unwrap();
let request = build_bulk_request(physical_region_id, physical_batch);
let request = build_bulk_request(physical_region_id, physical_batch, skip_wal);
let response = env
.metric()
.handle_request(physical_region_id, request)
@@ -401,6 +424,48 @@ mod tests {
.unwrap();
let batches = RecordBatches::try_collect(stream).await.unwrap();
assert_eq!(batches.iter().map(|b| b.num_rows()).sum::<usize>(), 6);
// Closing without a flush must recover only WAL-backed data. Recreate
// the wrapper too, so its metadata cache cannot hide missing metadata.
let stat = env.mito().region_statistic(physical_region_id).unwrap();
assert_eq!(stat.sst_num, 0);
env.metric()
.handle_request(
physical_region_id,
RegionRequest::Close(RegionCloseRequest {
flush_on_close: false,
}),
)
.await
.unwrap();
let reopened = MetricEngine::try_new(env.mito(), Default::default()).unwrap();
reopened.inner.flush_task.stop().await.unwrap();
reopened
.handle_request(
physical_region_id,
RegionRequest::Open(RegionOpenRequest {
engine: METRIC_ENGINE_NAME.to_string(),
table_dir: TestEnv::default_table_dir(),
path_type: PathType::Bare,
options: [(PHYSICAL_TABLE_METADATA_KEY.to_string(), String::new())]
.into_iter()
.collect(),
skip_wal_replay: false,
checkpoint: None,
requirements: Default::default(),
}),
)
.await
.unwrap();
let stream = reopened
.scan_to_stream(logical_region_id, ScanRequest::default())
.await
.unwrap();
let batches = RecordBatches::try_collect(stream).await.unwrap();
assert_eq!(
batches.iter().map(|b| b.num_rows()).sum::<usize>(),
if skip_wal { 0 } else { 6 },
);
}
#[tokio::test]
@@ -415,7 +480,7 @@ mod tests {
let physical_region_id = env.default_physical_region_id();
let batch = build_logical_batch(0, 0);
let request = build_bulk_request(physical_region_id, batch);
let request = build_bulk_request(physical_region_id, batch, false);
let response = env
.metric()
.handle_request(physical_region_id, request)
@@ -449,7 +514,7 @@ mod tests {
)
.unwrap();
let request = build_bulk_request(logical_region_id, batch);
let request = build_bulk_request(logical_region_id, batch, false);
let err = env
.metric()
.handle_request(logical_region_id, request)
@@ -499,7 +564,7 @@ mod tests {
)
.unwrap();
let request = build_bulk_request(logical_region_id, batch);
let request = build_bulk_request(logical_region_id, batch, false);
let response = env
.metric()
.handle_request(logical_region_id, request)
@@ -522,7 +587,7 @@ mod tests {
env.init_metric_region().await;
let logical_region_id = env.default_logical_region_id();
let request = build_bulk_request(logical_region_id, build_logical_batch(0, 3));
let request = build_bulk_request(logical_region_id, build_logical_batch(0, 3), false);
let response = env
.metric()
.handle_request(logical_region_id, request)
@@ -530,7 +595,7 @@ mod tests {
.unwrap();
assert_eq!(response.affected_rows, 3);
let request = build_bulk_request(logical_region_id, build_logical_batch(3, 5));
let request = build_bulk_request(logical_region_id, build_logical_batch(3, 5), false);
let response = env
.metric()
.handle_request(logical_region_id, request)
@@ -550,16 +615,68 @@ mod tests {
#[tokio::test]
async fn test_bulk_insert_sparse_encoding() {
let env = TestEnv::new().await;
env.init_metric_region().await;
let physical_region_id = env.default_physical_region_id();
env.create_physical_region(
physical_region_id,
&TestEnv::default_table_dir(),
vec![(
EXPERIMENTAL_SST_FLOAT_FIELD_ENCODING.to_string(),
"byte_stream_split".to_string(),
)],
)
.await;
let logical_region_id = env.default_logical_region_id();
env.create_logical_region(physical_region_id, logical_region_id)
.await;
let request = build_bulk_request(logical_region_id, build_logical_batch(0, 4));
let rows = 4;
let request = build_bulk_request(logical_region_id, build_logical_batch(0, rows), false);
let response = env
.metric()
.handle_request(logical_region_id, request)
.await
.unwrap();
assert_eq!(response.affected_rows, 4);
assert_eq!(response.affected_rows, rows);
let data_region_id = crate::utils::to_data_region_id(physical_region_id);
env.mito()
.handle_request(
data_region_id,
RegionRequest::Flush(RegionFlushRequest::default()),
)
.await
.unwrap();
let region = env.mito().find_region(data_region_id).unwrap();
let entry = region
.manifest_sst_entries()
.await
.into_iter()
.find(|entry| entry.visible && entry.file_path.ends_with(".parquet"))
.unwrap();
let mut cache_metrics = MetadataCacheMetrics::default();
let footer = MetadataLoader::new(
region.access_layer().object_store().clone(),
&entry.file_path,
entry.file_size,
)
.load(&mut cache_metrics)
.await
.unwrap();
let field_column_index = footer
.file_metadata()
.schema_descr()
.columns()
.iter()
.position(|column| column.name() == greptime_value())
.unwrap();
assert!(!footer.row_groups().is_empty());
assert!(footer.row_groups().iter().all(|row_group| {
row_group
.column(field_column_index)
.encodings()
.any(|encoding| encoding == Encoding::BYTE_STREAM_SPLIT)
}));
let stream = env
.metric()
@@ -567,7 +684,7 @@ mod tests {
.await
.unwrap();
let batches = RecordBatches::try_collect(stream).await.unwrap();
assert_eq!(batches.iter().map(|b| b.num_rows()).sum::<usize>(), 4);
assert_eq!(batches.iter().map(|b| b.num_rows()).sum::<usize>(), rows);
}
#[tokio::test]
@@ -575,7 +692,7 @@ mod tests {
let env = TestEnv::new().await;
let logical_region_id = init_dense_metric_region(&env).await;
let request = build_bulk_request(logical_region_id, build_logical_batch(0, 2));
let request = build_bulk_request(logical_region_id, build_logical_batch(0, 2), false);
let err = env
.metric()
.handle_request(logical_region_id, request)
@@ -599,6 +716,7 @@ mod tests {
.handle_request(
logical_region_id,
RegionRequest::Put(RegionPutRequest {
skip_wal: false,
rows: api::v1::Rows { schema, rows },
hint: None,
partition_expr_version: None,
@@ -616,7 +734,7 @@ mod tests {
let env_bulk = TestEnv::new().await;
env_bulk.init_metric_region().await;
let request = build_bulk_request(logical_region_id, build_logical_batch(0, 5));
let request = build_bulk_request(logical_region_id, build_logical_batch(0, 5), false);
env_bulk
.metric()
.handle_request(logical_region_id, request)
src/metric-engine/src/engine/flush.rs
+1
View File
@@ -90,6 +90,7 @@ mod tests {
let schema = row_schema_with_tags(&["job"]);
let rows = build_rows(1, 10);
let request = RegionRequest::Put(RegionPutRequest {
skip_wal: false,
rows: Rows { schema, rows },
hint: None,
partition_expr_version: None,
src/metric-engine/src/engine/put.rs
+367 -27
View File
@@ -142,6 +142,8 @@ impl MetricEngineInner {
let data_region_id = to_data_region_id(physical_region_id);
let primary_key_encoding = self.get_primary_key_encoding(data_region_id)?;
// TODO(weny): Consolidate validation and merging to avoid redundant request traversals,
// while ensuring the entire batch is validated before writing.
// Validate all requests
self.validate_batch_requests(physical_region_id, &mut requests)
.await?;
@@ -176,6 +178,18 @@ impl MetricEngineInner {
physical_region_id: RegionId,
requests: &mut [(RegionId, RegionPutRequest)],
) -> Result<()> {
let skip_wal = requests
.first()
.is_some_and(|(_, request)| request.skip_wal);
ensure!(
requests
.iter()
.all(|(_, request)| request.skip_wal == skip_wal),
InvalidRequestSnafu {
region_id: physical_region_id,
reason: "inconsistent WAL policy in batch"
}
);
for (logical_region_id, request) in requests {
self.verify_rows(
*logical_region_id,
@@ -194,6 +208,9 @@ impl MetricEngineInner {
physical_region_id: RegionId,
requests: Vec<(RegionId, RegionPutRequest)>,
) -> Result<(RegionPutRequest, AffectedRows)> {
let skip_wal = requests
.first()
.is_some_and(|(_, request)| request.skip_wal);
let total_rows: usize = requests.iter().map(|(_, req)| req.rows.rows.len()).sum();
let mut modified_requests = Vec::with_capacity(requests.len());
let mut total_affected_rows: AffectedRows = 0;
@@ -233,6 +250,7 @@ impl MetricEngineInner {
}
let merged_request = RegionPutRequest {
skip_wal,
rows: Rows {
schema,
rows: merged_rows,
@@ -256,6 +274,9 @@ impl MetricEngineInner {
data_region_id: RegionId,
requests: Vec<(RegionId, RegionPutRequest)>,
) -> Result<(RegionPutRequest, AffectedRows)> {
let skip_wal = requests
.first()
.is_some_and(|(_, request)| request.skip_wal);
// Build union schema from all requests
let merged_schema =
Self::build_union_schema(requests.iter().map(|(_, req)| req.rows.schema.as_slice()));
@@ -291,6 +312,7 @@ impl MetricEngineInner {
};
let merged_request = RegionPutRequest {
skip_wal,
rows: final_rows,
hint: None,
partition_expr_version: merged_version,
@@ -769,25 +791,272 @@ mod tests {
use common_function::utils::partition_expr_version;
use common_query::prelude::{greptime_native_histogram, greptime_timestamp, greptime_value};
use common_recordbatch::RecordBatches;
use datatypes::arrow::array::{Float64Array, TimestampMillisecondArray};
use datatypes::prelude::ConcreteDataType;
use datatypes::schema::{ColumnDefaultConstraint, ColumnSchema};
use datatypes::value::Value as PartitionValue;
use partition::expr::col;
use store_api::metadata::ColumnMetadata;
use store_api::metric_engine_consts::{
DATA_SCHEMA_TABLE_ID_COLUMN_NAME, DATA_SCHEMA_TSID_COLUMN_NAME, PRIMARY_KEY_ENCODING,
DATA_SCHEMA_TABLE_ID_COLUMN_NAME, DATA_SCHEMA_TSID_COLUMN_NAME, METRIC_ENGINE_NAME,
PHYSICAL_TABLE_METADATA_KEY, PRIMARY_KEY_ENCODING,
};
use store_api::path_utils::table_dir;
use store_api::region_engine::RegionEngine;
use store_api::region_request::{
EnterStagingRequest, RegionRequest, StagingPartitionDirective,
EnterStagingRequest, PathType, RegionCloseRequest, RegionOpenRequest, RegionRequest,
StagingPartitionDirective,
};
use store_api::storage::ScanRequest;
use store_api::storage::consts::PRIMARY_KEY_COLUMN_NAME;
use super::*;
use crate::engine::MetricEngine;
use crate::test_util::{self, TestEnv};
async fn scan_timestamp_values(engine: &MetricEngine, region_id: RegionId) -> Vec<(i64, f64)> {
let stream = engine
.scan_to_stream(region_id, ScanRequest::default())
.await
.unwrap();
let batches = RecordBatches::try_collect(stream).await.unwrap();
let mut rows = Vec::new();
for batch in batches.iter() {
let batch = batch.df_record_batch();
let timestamp_index = batch.schema().index_of(greptime_timestamp()).unwrap();
let value_index = batch.schema().index_of(greptime_value()).unwrap();
let timestamps = batch
.column(timestamp_index)
.as_any()
.downcast_ref::<TimestampMillisecondArray>()
.unwrap();
let values = batch
.column(value_index)
.as_any()
.downcast_ref::<Float64Array>()
.unwrap();
rows.extend(
timestamps
.values()
.iter()
.copied()
.zip(values.values().iter().copied()),
);
}
rows.sort_unstable_by_key(|(timestamp, _)| *timestamp);
rows
}
#[tokio::test]
async fn test_batch_partition_versions() {
check_batch_partition_versions("sparse").await;
check_batch_partition_versions("dense").await;
}
async fn check_batch_partition_versions(encoding: &str) {
let env = TestEnv::new().await;
let physical_region_id = env.default_physical_region_id();
let logical_region_id = env.default_logical_region_id();
env.create_physical_region(
physical_region_id,
&TestEnv::default_table_dir(),
vec![(PRIMARY_KEY_ENCODING.to_string(), encoding.to_string())],
)
.await;
create_logical_region_with_tags(&env, physical_region_id, logical_region_id, &["job"])
.await;
let build_requests = |versions: [Option<u64>; 3]| {
versions
.into_iter()
.map(|partition_expr_version| {
(
logical_region_id,
RegionPutRequest {
skip_wal: false,
rows: Rows {
schema: test_util::row_schema_with_tags(&["job"]),
rows: test_util::build_rows(1, 1),
},
hint: None,
partition_expr_version,
},
)
})
.collect::<Vec<_>>()
};
// Conflicting explicit versions must fail before any data is written.
let err = env
.metric()
.inner
.put_regions_batch_single_physical(
physical_region_id,
build_requests([None, Some(10), Some(11)]),
)
.await
.unwrap_err();
assert!(
err.to_string()
.contains("inconsistent partition expr version")
);
assert!(
scan_timestamp_values(&env.metric(), logical_region_id)
.await
.is_empty()
);
for (versions, expected) in [
([None, None, None], None),
([None, Some(7), None], Some(7)),
([Some(7), None, Some(7)], Some(7)),
] {
let mut requests = build_requests(versions);
let engine = env.metric();
engine
.inner
.validate_batch_requests(physical_region_id, &mut requests)
.await
.unwrap();
let (merged, _) = match encoding {
"sparse" => engine
.inner
.merge_sparse_batch(physical_region_id, requests),
"dense" => engine
.inner
.merge_dense_batch(to_data_region_id(physical_region_id), requests),
_ => unreachable!(),
}
.unwrap();
assert_eq!(merged.partition_expr_version, expected);
}
}
#[tokio::test]
async fn test_put_skip_wal_batch_recovery() {
check_put_skip_wal_batch_recovery("sparse", false).await;
check_put_skip_wal_batch_recovery("sparse", true).await;
check_put_skip_wal_batch_recovery("dense", false).await;
check_put_skip_wal_batch_recovery("dense", true).await;
}
async fn check_put_skip_wal_batch_recovery(encoding: &str, skip_wal: bool) {
let env = TestEnv::new().await;
let engine = env.metric();
engine.inner.flush_task.stop().await.unwrap();
let physical_region_id = env.default_physical_region_id();
let logical_region_id = env.default_logical_region_id();
env.create_physical_region(
physical_region_id,
&TestEnv::default_table_dir(),
vec![(PRIMARY_KEY_ENCODING.to_string(), encoding.to_string())],
)
.await;
create_logical_region_with_tags(&env, physical_region_id, logical_region_id, &["job"])
.await;
let metadata_before = engine.get_metadata(logical_region_id).await.unwrap();
let requests = [skip_wal; 3]
.into_iter()
.enumerate()
.map(|(index, skip_wal)| {
let timestamp = index as i64 + 1;
let value = timestamp as f64 * 10.0;
// Every request updates the same key at timestamp zero and
// also inserts a distinct key to verify merge order.
let rows = [0, timestamp]
.into_iter()
.map(|timestamp| Row {
values: vec![
Value {
value_data: Some(ValueData::TimestampMillisecondValue(timestamp)),
},
Value {
value_data: Some(ValueData::F64Value(value)),
},
Value {
value_data: Some(ValueData::StringValue("tag_0".to_string())),
},
],
})
.collect();
(
logical_region_id,
RegionPutRequest {
rows: Rows {
schema: test_util::row_schema_with_tags(&["job"]),
rows,
},
hint: None,
partition_expr_version: None,
skip_wal,
},
)
});
let affected_rows = engine.inner.put_regions_batch(requests).await.unwrap();
assert_eq!(affected_rows, 6);
assert_eq!(
scan_timestamp_values(&engine, logical_region_id).await,
vec![(0, 30.0), (1, 10.0), (2, 20.0), (3, 30.0)]
);
// Neither data nor metadata has an SST to hide missing WAL.
for region_id in [
to_data_region_id(physical_region_id),
crate::utils::to_metadata_region_id(physical_region_id),
] {
let stat = env.mito().region_statistic(region_id).unwrap();
assert!(stat.memtable_size > 0);
assert_eq!(stat.sst_num, 0);
}
engine
.handle_request(
physical_region_id,
RegionRequest::Close(RegionCloseRequest {
flush_on_close: false,
}),
)
.await
.unwrap();
// Recreate the wrapper as well, discarding its metadata cache.
let reopened = MetricEngine::try_new(env.mito(), Default::default()).unwrap();
reopened.inner.flush_task.stop().await.unwrap();
reopened
.handle_request(
physical_region_id,
RegionRequest::Open(RegionOpenRequest {
engine: METRIC_ENGINE_NAME.to_string(),
table_dir: TestEnv::default_table_dir(),
path_type: PathType::Bare,
options: [
(PHYSICAL_TABLE_METADATA_KEY.to_string(), String::new()),
(PRIMARY_KEY_ENCODING.to_string(), encoding.to_string()),
]
.into_iter()
.collect(),
skip_wal_replay: false,
checkpoint: None,
requirements: Default::default(),
}),
)
.await
.unwrap();
let recovered_metadata = reopened.get_metadata(logical_region_id).await.unwrap();
assert_eq!(
metadata_before.column_metadatas,
recovered_metadata.column_metadatas
);
let expected = if skip_wal {
vec![]
} else {
vec![(0, 30.0), (1, 10.0), (2, 20.0), (3, 30.0)]
};
assert_eq!(
scan_timestamp_values(&reopened, logical_region_id).await,
expected,
"encoding={encoding}, skip_wal={skip_wal}"
);
}
fn assert_merged_schema(rows: &Rows, expect_sparse: bool) {
let column_names: HashSet<String> = rows
.schema
@@ -874,6 +1143,45 @@ mod tests {
.unwrap()
}
fn check_batch_merge_wal_policy(
env: &TestEnv,
physical_region_id: RegionId,
mut requests: Vec<(RegionId, RegionPutRequest)>,
expect_sparse: bool,
skip_wal: bool,
) {
for (_, request) in &mut requests {
request.skip_wal = skip_wal;
}
let (merged_request, affected_rows) = if expect_sparse {
let (merged_request, affected_rows) = env
.metric()
.inner
.merge_sparse_batch(physical_region_id, requests)
.unwrap();
let hint = merged_request
.hint
.as_ref()
.expect("missing sparse write hint");
assert_eq!(
hint.primary_key_encoding,
PrimaryKeyEncodingProto::Sparse as i32
);
(merged_request, affected_rows)
} else {
let (merged_request, affected_rows) = env
.metric()
.inner
.merge_dense_batch(to_data_region_id(physical_region_id), requests)
.unwrap();
assert!(merged_request.hint.is_none());
(merged_request, affected_rows)
};
assert_merged_schema(&merged_request.rows, expect_sparse);
assert_eq!(merged_request.skip_wal, skip_wal);
assert_eq!(affected_rows, 5);
}
async fn run_batch_write_with_schema_variants(
env: &TestEnv,
physical_region_id: RegionId,
@@ -921,6 +1229,7 @@ mod tests {
(
logical_region_1,
RegionPutRequest {
skip_wal: false,
rows: Rows {
schema: schema_1.clone(),
rows: rows_1,
@@ -932,6 +1241,7 @@ mod tests {
(
logical_region_2,
RegionPutRequest {
skip_wal: false,
rows: Rows {
schema: schema_2.clone(),
rows: rows_2,
@@ -943,32 +1253,41 @@ mod tests {
]
};
let merged_request = if expect_sparse {
let (merged_request, _) = env
.metric()
.inner
.merge_sparse_batch(physical_region_id, build_requests())
.unwrap();
let hint = merged_request
.hint
.as_ref()
.expect("missing sparse write hint");
assert_eq!(
hint.primary_key_encoding,
PrimaryKeyEncodingProto::Sparse as i32
);
merged_request
} else {
let (merged_request, _) = env
.metric()
.inner
.merge_dense_batch(data_region_id, build_requests())
.unwrap();
assert!(merged_request.hint.is_none());
merged_request
};
check_batch_merge_wal_policy(
env,
physical_region_id,
build_requests(),
expect_sparse,
false,
);
check_batch_merge_wal_policy(
env,
physical_region_id,
build_requests(),
expect_sparse,
true,
);
assert_merged_schema(&merged_request.rows, expect_sparse);
for policies in [[false, true], [true, false]] {
let mut mixed_requests = build_requests();
for ((_, request), skip_wal) in mixed_requests.iter_mut().zip(policies) {
request.skip_wal = skip_wal;
}
let err = env
.metric()
.inner
.put_regions_batch(mixed_requests.into_iter())
.await
.unwrap_err();
assert!(err.to_string().contains("inconsistent WAL policy in batch"));
for logical_region_id in [logical_region_1, logical_region_2] {
assert!(
scan_timestamp_values(&env.metric(), logical_region_id)
.await
.is_empty()
);
}
}
let affected_rows = env
.metric()
@@ -1095,6 +1414,7 @@ mod tests {
let schema = test_util::row_schema_with_tags(&["job"]);
let rows = test_util::build_rows(1, 5);
let request = RegionRequest::Put(RegionPutRequest {
skip_wal: false,
rows: Rows { schema, rows },
hint: None,
partition_expr_version: None,
@@ -1170,6 +1490,7 @@ mod tests {
let schema = test_util::row_schema_with_tags(columns);
let rows = test_util::build_rows(3, 100);
let request = RegionRequest::Put(RegionPutRequest {
skip_wal: false,
rows: Rows { schema, rows },
hint: None,
partition_expr_version: None,
@@ -1193,6 +1514,7 @@ mod tests {
let schema = test_util::row_schema_with_tags(&["abc"]);
let rows = test_util::build_rows(1, 100);
let request = RegionRequest::Put(RegionPutRequest {
skip_wal: false,
rows: Rows { schema, rows },
hint: None,
partition_expr_version: None,
@@ -1214,6 +1536,7 @@ mod tests {
let schema = test_util::row_schema_with_tags(&["def"]);
let rows = test_util::build_rows(1, 100);
let request = RegionRequest::Put(RegionPutRequest {
skip_wal: false,
rows: Rows { schema, rows },
hint: None,
partition_expr_version: None,
@@ -1274,6 +1597,7 @@ mod tests {
(
logical_region_1,
RegionPutRequest {
skip_wal: false,
rows: Rows {
schema: schema.clone(),
rows: rows1,
@@ -1285,6 +1609,7 @@ mod tests {
(
logical_region_2,
RegionPutRequest {
skip_wal: false,
rows: Rows {
schema: schema.clone(),
rows: rows2,
@@ -1296,6 +1621,7 @@ mod tests {
(
logical_region_3,
RegionPutRequest {
skip_wal: false,
rows: Rows {
schema: schema.clone(),
rows: rows3,
@@ -1347,6 +1673,7 @@ mod tests {
(
logical_region_1,
RegionPutRequest {
skip_wal: false,
rows: Rows {
schema: schema.clone(),
rows: test_util::build_rows(1, 3),
@@ -1358,6 +1685,7 @@ mod tests {
(
nonexistent_region,
RegionPutRequest {
skip_wal: false,
rows: Rows {
schema: schema.clone(),
rows: test_util::build_rows(1, 2),
@@ -1369,6 +1697,7 @@ mod tests {
(
logical_region_2,
RegionPutRequest {
skip_wal: false,
rows: Rows {
schema: schema.clone(),
rows: test_util::build_rows(1, 5),
@@ -1407,6 +1736,7 @@ mod tests {
let requests = vec![(
physical_region_id,
RegionPutRequest {
skip_wal: false,
rows: Rows {
schema,
rows: test_util::build_rows(1, 1),
@@ -1441,6 +1771,7 @@ mod tests {
(
logical_region_id,
RegionPutRequest {
skip_wal: false,
rows: Rows {
schema: schema.clone(),
rows: test_util::build_rows(1, 1),
@@ -1452,6 +1783,7 @@ mod tests {
(
physical_region_id,
RegionPutRequest {
skip_wal: false,
rows: Rows {
schema,
rows: test_util::build_rows(1, 1),
@@ -1487,6 +1819,7 @@ mod tests {
let requests = vec![(
logical_region_id,
RegionPutRequest {
skip_wal: false,
rows: Rows {
schema,
rows: test_util::build_rows(1, 5),
@@ -1565,6 +1898,7 @@ mod tests {
.handle_request(
logical_region_id,
RegionRequest::Put(RegionPutRequest {
skip_wal: false,
rows,
hint: None,
partition_expr_version: Some(1),
@@ -1607,6 +1941,7 @@ mod tests {
.handle_request(
logical_region_id,
RegionRequest::Put(RegionPutRequest {
skip_wal: false,
rows: rows.clone(),
hint: None,
partition_expr_version: Some(expected_version.wrapping_add(1)),
@@ -1621,6 +1956,7 @@ mod tests {
.handle_request(
logical_region_id,
RegionRequest::Put(RegionPutRequest {
skip_wal: false,
rows: rows.clone(),
hint: None,
partition_expr_version: None,
@@ -1635,6 +1971,7 @@ mod tests {
.handle_request(
logical_region_id,
RegionRequest::Put(RegionPutRequest {
skip_wal: false,
rows,
hint: None,
partition_expr_version: Some(expected_version),
@@ -1703,6 +2040,7 @@ mod tests {
.handle_request(
logical_region_id,
RegionRequest::Put(RegionPutRequest {
skip_wal: false,
rows: Rows { schema, rows },
hint: None,
partition_expr_version: None,
@@ -1759,6 +2097,7 @@ mod tests {
.handle_request(
logical_region_id,
RegionRequest::Put(RegionPutRequest {
skip_wal: false,
rows: Rows { schema, rows },
hint: None,
partition_expr_version: None,
@@ -1813,6 +2152,7 @@ mod tests {
.handle_request(
logical_region_id,
RegionRequest::Put(RegionPutRequest {
skip_wal: false,
rows: Rows { schema, rows },
hint: None,
partition_expr_version: None,
src/metric-engine/src/engine/read.rs
+1
View File
@@ -500,6 +500,7 @@ mod test {
let schema = test_util::row_schema_with_tags(&["job"]);
let put = |rows| {
RegionRequest::Put(RegionPutRequest {
skip_wal: false,
rows: Rows {
schema: schema.clone(),
rows: test_util::build_rows(1, rows),
src/metric-engine/src/metadata_region.rs
+33
View File
@@ -585,6 +585,8 @@ impl MetadataRegion {
};
RegionPutRequest {
// Metadata must remain recoverable regardless of the user write policy.
skip_wal: false,
rows,
hint: None,
partition_expr_version: None,
@@ -819,6 +821,37 @@ mod test {
use crate::test_util::TestEnv;
use crate::utils::to_metadata_region_id;
#[test]
fn test_metadata_put_always_writes_wal() {
// Both metadata put paths use this constructor rather than forwarding
// a user insert request, so its WAL policy must always be independent.
for entries in [
vec![],
vec![("region", "")],
vec![("region", ""), ("column", "metadata")],
] {
let request = MetadataRegion::build_put_request_from_iter(
entries
.iter()
.map(|(key, value)| (key.to_string(), value.to_string())),
);
assert!(!request.skip_wal);
assert!(request.hint.is_none());
assert!(request.partition_expr_version.is_none());
let expected_rows = entries
.into_iter()
.map(|(key, value)| {
row(vec![
ValueData::TimestampMillisecondValue(0),
ValueData::StringValue(key.to_string()),
ValueData::StringValue(value.to_string()),
])
})
.collect::<Vec<_>>();
assert_eq!(request.rows.rows, expected_rows);
}
}
#[test]
fn test_concat_table_key() {
let region_id = RegionId::new(1234, 7844);
src/mito2/AGENTS.md
+3 -1
View File
@@ -26,7 +26,7 @@ snapshot isolation). It implements the `RegionEngine` trait from `store-api`.
| `compaction` | `src/mito2/src/compaction/` | Compaction scheduler (`scheduler.rs` + `scheduler/`), TWCS picker, strict-window manual picker, compactor, memory control |
| `access_layer` | `src/mito2/src/access_layer.rs` | SST read/write over the object store |
| `sst` | `src/mito2/src/sst/` | Parquet format, file metadata, index layout |
| `series_index` | `src/mito2/src/series_index/` | Series index writer/searcher, catalogs, immutable snapshots, file lifecycle, and background maintenance |
| `series_index` | `src/mito2/src/series_index/` | Series index writer/searcher, bucket planning and SST builders, catalogs, immutable snapshots, file lifecycle, and background maintenance |
| `read` | `src/mito2/src/read/` | `ScanRegion`, merge, dedup, projection, streaming |
| `manifest` | `src/mito2/src/manifest/` | `RegionManifestManager`, manifest actions/edits |
| `cache` | `src/mito2/src/cache.rs` | Write/file/page caches |
@@ -62,6 +62,8 @@ filtered `RecordBatch` stream.
- **Manifest format** (`manifest/action.rs`): affects crash recovery and
follower replay. Keep it backward compatible.
- **SST/Parquet layout** (`sst/`): readers must stay compatible with existing files.
- **Series-index coverage** (`series_index/catalog.rs`): `SeriesIndexEntry` stores
compaction-window width and SST summaries keyed by aligned start in both catalogs and Parquet footers.
- **Request types** (`request.rs`): usually tied to proto definitions consumed by `datanode`.
- **WAL/memtable encoding** (`wal/`, `memtable/`): breaks replay if changed incompatibly.
src/mito2/src/access_layer.rs
+41 -3
View File
@@ -12,6 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use std::path::Path;
use std::sync::Arc;
use std::time::{Duration, Instant};
@@ -655,14 +656,17 @@ impl TempFileCleaner {
}
pub(crate) async fn new_fs_cache_store(root: &str) -> Result<ObjectStore> {
let atomic_write_dir = join_dir(root, ATOMIC_WRITE_DIR);
clean_dir(&atomic_write_dir).await?;
// Preserve native filesystem prefixes such as Windows UNC shares.
let atomic_write_dir = Path::new(root).join(ATOMIC_WRITE_DIR);
clean_dir(&atomic_write_dir.to_string_lossy()).await?;
// Compatible code. Remove this after a major release.
let old_atomic_temp_dir = join_dir(root, OLD_ATOMIC_WRITE_DIR);
clean_dir(&old_atomic_temp_dir).await?;
let builder = Fs::default().root(root).atomic_write_dir(&atomic_write_dir);
let builder = Fs::default()
.root(root)
.atomic_write_dir(&atomic_write_dir.to_string_lossy());
let store = ObjectStore::new(builder).context(OpenDalSnafu)?;
Ok(with_instrument_layers(store, false))
@@ -759,3 +763,37 @@ impl FilePathProvider for RegionFilePathFactory {
location::sst_file_path(&self.table_dir, file_id, self.path_type)
}
}
#[cfg(test)]
mod tests {
use std::path::PathBuf;
use super::*;
#[tokio::test]
async fn test_new_fs_cache_store() {
let root = common_test_util::temp_dir::create_temp_dir("fs-cache-store");
let dirs = [
root.path().join(ATOMIC_WRITE_DIR),
PathBuf::from(join_dir(
root.path().to_str().unwrap(),
OLD_ATOMIC_WRITE_DIR,
)),
];
for dir in &dirs {
tokio::fs::create_dir_all(dir).await.unwrap();
tokio::fs::write(dir.join("stale"), b"stale").await.unwrap();
}
let store = new_fs_cache_store(root.path().to_str().unwrap())
.await
.unwrap();
for dir in &dirs {
assert!(!dir.join("stale").exists());
}
store.write("index", "contents").await.unwrap();
assert_eq!(
tokio::fs::read(root.path().join("index")).await.unwrap(),
b"contents"
);
}
}
src/mito2/src/compaction/compactor.rs
+1
View File
@@ -655,6 +655,7 @@ where
write_buffer_size: compaction_region.engine_config.sst_write_buffer_size,
max_file_size: picker_output.max_file_size,
row_group_size: compaction_region.region_options.row_group_size(),
float_field_encoding: compaction_region.region_options.float_field_encoding,
};
let merger = self.merger.clone();
let compaction_region = compaction_region.clone();
src/mito2/src/compaction/json2.rs
+15 -2
View File
@@ -52,6 +52,8 @@ pub(crate) type Json2RewritePlans = HashMap<String, Json2RewritePlan>;
#[derive(Clone)]
struct Json2LeafPathStats {
rows: u64,
/// Number of input schemas that explicitly represent this leaf.
sources: usize,
data_type: JsonNativeType,
is_type_conflicted: bool,
}
@@ -59,8 +61,8 @@ struct Json2LeafPathStats {
/// Builds the JSON2 rewrite plans for a compaction.
///
/// Type hints from current region metadata are always retained. Existing explicit dynamic paths
/// from all input SST schemas are ranked once to produce a fixed layout; paths found only in a v2
/// remainder are deliberately not promoted. [`rewrite_json2_batch`] decodes inputs and rewrites
/// shared by every input SST schema are ranked once to produce a fixed layout; paths that may
/// reside in a v2 remainder are deliberately not promoted. [`rewrite_json2_batch`] decodes inputs and rewrites
/// them according to these plans. Non-JSON2 columns are omitted from the returned map.
///
/// Returns an error when a JSON2 column has invalid or missing extension metadata, or when its
@@ -144,6 +146,14 @@ pub(crate) fn collect_json2_rewrite_plans(
collect_json2_path_stats(field, *rows, &hint_paths, &mut stats)?;
}
// A leaf absent from an input's explicit schema may have arbitrary values
// (including scalar ancestors) in its remainder. Only common explicit leaves
// are safe to promote without inspecting rows. Keep unsafe paths opaque for
// the entire output SST so existing readers never miss remainder values when
// projecting an explicit leaf.
for stat in stats.values_mut() {
stat.is_type_conflicted |= stat.sources != schemas.len();
}
let mut hints = settings.type_hints().to_vec();
hints.extend(select_dynamic_hints(settings, &hint_paths, &stats));
let target_layout = JsonSettings::try_new(hints, Some(0)).context(DataTypeMismatchSnafu)?;
@@ -187,12 +197,14 @@ fn collect_json2_path_stats<'a>(
path,
Json2LeafPathStats {
rows,
sources: 1,
data_type,
is_type_conflicted: false,
},
);
continue;
};
stat.sources += 1;
if stat.data_type != data_type {
stat.is_type_conflicted = true;
} else {
@@ -368,6 +380,7 @@ mod tests {
)?;
let stat = |rows, data_type, is_type_conflicted| Json2LeafPathStats {
rows,
sources: 1,
data_type,
is_type_conflicted,
};
src/mito2/src/compaction/window.rs
+1
View File
@@ -337,6 +337,7 @@ mod tests {
primary_key_encoding: None,
write_buffer_size: None,
preserve_row_sequence: false,
float_field_encoding: Default::default(),
},
compaction_time_window: None,
}
src/mito2/src/config.rs
+4 -45
View File
@@ -89,11 +89,9 @@ pub struct MitoConfig {
// Background job configs:
/// Max number of running background index build jobs (default: 1/8 of cpu cores).
pub max_background_index_builds: usize,
// TODO: Document both series-index settings in the example configs and regenerate configuration
// docs before exposing the feature.
/// Under development; do not enable. Root directory for loading series indexes, currently stored
/// on the local filesystem. Empty disables the feature. Relative paths resolve under `data_home`.
pub experimental_series_index_root: String,
/// Under development; do not enable. Whether to enable series indexes (default false).
/// Indexes are stored on the local filesystem under `{data_home}/series_index`.
pub experimental_enable_series_index: bool,
/// Interval between series-index maintenance runs (default 5 min). Zero uses the default.
#[serde(with = "humantime_serde")]
pub experimental_series_index_maintenance_interval: Duration,
@@ -214,7 +212,7 @@ impl Default for MitoConfig {
experimental_manifest_keep_removed_file_ttl: Duration::from_secs(60 * 60),
compress_manifest: false,
max_background_index_builds: divide_num_cpus(8),
experimental_series_index_root: String::new(),
experimental_enable_series_index: false,
experimental_series_index_maintenance_interval:
DEFAULT_SERIES_INDEX_MAINTENANCE_INTERVAL,
max_background_flushes: divide_num_cpus(2),
@@ -312,15 +310,6 @@ impl MitoConfig {
DEFAULT_SERIES_INDEX_MAINTENANCE_INTERVAL;
}
if !self.experimental_series_index_root.trim().is_empty()
&& Path::new(&self.experimental_series_index_root).is_relative()
{
self.experimental_series_index_root = Path::new(data_home)
.join(&self.experimental_series_index_root)
.display()
.to_string();
}
if self.global_write_buffer_reject_size <= self.global_write_buffer_size {
self.global_write_buffer_reject_size = self.global_write_buffer_size * 2;
warn!(
@@ -432,36 +421,6 @@ mod tests {
assert_eq!(ReadableSize::mb(128), config.prefilter_result_cache_size);
}
#[test]
fn test_series_index_config() {
assert!(
MitoConfig::default()
.experimental_series_index_root
.is_empty()
);
let mut config: MitoConfig = toml::from_str(
"experimental_series_index_root = 'indexes'
experimental_series_index_maintenance_interval = '30s'",
)
.unwrap();
config.sanitize("/data").unwrap();
assert_eq!(config.experimental_series_index_root, "/data/indexes");
assert_eq!(
config.experimental_series_index_maintenance_interval,
Duration::from_secs(30)
);
let restored: MitoConfig = toml::from_str(&toml::to_string(&config).unwrap()).unwrap();
assert_eq!(config, restored);
let mut config: MitoConfig =
toml::from_str("experimental_series_index_maintenance_interval = '0s'").unwrap();
config.sanitize("/data").unwrap();
assert_eq!(
config.experimental_series_index_maintenance_interval,
MitoConfig::default().experimental_series_index_maintenance_interval
);
}
#[test]
fn test_experimental_series_scan_v2_config() {
assert!(MitoConfig::default().experimental_series_scan_v2);
src/mito2/src/engine.rs
+2
View File
@@ -225,6 +225,7 @@ impl<'a, S: LogStore> MitoEngineBuilder<'a, S> {
// so the engine (and thus the GC worker) can fire `on_region_gc`.
let region_hook = self.plugins.get::<RegionHookRef>();
let workers = WorkerGroup::start(
self.data_home,
config.clone(),
self.log_store.clone(),
self.object_store_manager,
@@ -1620,6 +1621,7 @@ impl MitoEngine {
Ok(MitoEngine {
inner: Arc::new(EngineInner {
workers: WorkerGroup::start_for_test(
data_home,
config.clone(),
log_store,
object_store_manager,
src/mito2/src/engine/alter_test.rs
+2
View File
@@ -3003,6 +3003,7 @@ fn build_schema_v0_bulk_request(region_id: RegionId) -> RegionBulkInsertsRequest
let raw_data = encode_arrow_ipc(&payload);
RegionBulkInsertsRequest {
skip_wal: false,
region_id,
payload,
raw_data,
@@ -3411,6 +3412,7 @@ async fn test_alter_time_index_widen_sparse_compaction() {
};
let put_sparse = |rows| {
RegionRequest::Put(RegionPutRequest {
skip_wal: false,
rows,
hint: Some(WriteHint {
primary_key_encoding: api::v1::PrimaryKeyEncoding::Sparse.into(),

Some files were not shown because too many files have changed in this diff Show More