diff --git a/.github/actions/setup-chaos/action.yml b/.github/actions/setup-chaos/action.yml
index 76eb48cf4a..4b3fb86744 100644
--- a/.github/actions/setup-chaos/action.yml
+++ b/.github/actions/setup-chaos/action.yml
@@ -1,15 +1,16 @@
-name: Setup Kind
-description: Deploy Kind
+name: Setup Chaos Mesh
+description: Install and wait for Chaos Mesh
runs:
using: composite
steps:
- uses: actions/checkout@v4
- - name: Create kind cluster
+ - name: Install Chaos Mesh
shell: bash
run: |
helm repo add chaos-mesh https://charts.chaos-mesh.org
+ helm repo update chaos-mesh
kubectl create ns chaos-mesh
- helm install chaos-mesh chaos-mesh/chaos-mesh -n=chaos-mesh --version 2.6.3
+ helm install chaos-mesh chaos-mesh/chaos-mesh -n=chaos-mesh --set chaosDaemon.runtime=containerd --set chaosDaemon.socketPath=/run/containerd/containerd.sock --version 2.8.0
- name: Print Chaos-mesh
if: always()
shell: bash
diff --git a/.github/actions/setup-greptimedb-cluster/with-minio-repartition-gc.yaml b/.github/actions/setup-greptimedb-cluster/with-minio-repartition-gc.yaml
index b836606ad6..5b7f43d81b 100644
--- a/.github/actions/setup-greptimedb-cluster/with-minio-repartition-gc.yaml
+++ b/.github/actions/setup-greptimedb-cluster/with-minio-repartition-gc.yaml
@@ -1,3 +1,8 @@
+logging:
+ level: "info"
+ format: "json"
+ filters:
+ - mito2::sst::file=debug
meta:
configData: |-
[runtime]
diff --git a/.github/cargo-blacklist.txt b/.github/cargo-blacklist.txt
index d2f071130e..32e7878a86 100644
--- a/.github/cargo-blacklist.txt
+++ b/.github/cargo-blacklist.txt
@@ -1,3 +1,2 @@
native-tls
openssl
-aws-lc-sys
diff --git a/.github/scripts/check-version.sh b/.github/scripts/check-version.sh
index 28c2812ded..1efa3bb4db 100755
--- a/.github/scripts/check-version.sh
+++ b/.github/scripts/check-version.sh
@@ -30,13 +30,72 @@ CLEAN_LATEST=$(echo "$LATEST_VERSION" | sed 's/^v//' | sed 's/-nightly-.*//')
echo "Current version: $CLEAN_CURRENT"
echo "Latest release version: $CLEAN_LATEST"
-# Use sort -V to compare versions
-HIGHER_VERSION=$(printf "%s\n%s" "$CLEAN_CURRENT" "$CLEAN_LATEST" | sort -V | tail -n1)
+# Function to extract base version (without pre-release suffix)
+get_base_version() {
+ echo "$1" | sed -E 's/-(alpha|beta|rc|pre).*//'
+}
-if [ "$HIGHER_VERSION" = "$CLEAN_CURRENT" ]; then
+# Function to check if a version is pre-release
+is_prerelease() {
+ [[ "$1" =~ -(alpha|beta|rc|pre) ]]
+}
+
+# Compare versions properly considering pre-release
+compare_versions() {
+ local current=$1
+ local latest=$2
+
+ # Extract base versions
+ local current_base=$(get_base_version "$current")
+ local latest_base=$(get_base_version "$latest")
+
+ # Compare base versions first
+ HIGHER_BASE=$(printf "%s\n%s" "$current_base" "$latest_base" | sort -V | tail -n1)
+
+ if [ "$HIGHER_BASE" = "$latest_base" ] && [ "$current_base" != "$latest_base" ]; then
+ # Latest has higher base version
+ echo "current_older"
+ return
+ elif [ "$HIGHER_BASE" = "$current_base" ] && [ "$current_base" != "$latest_base" ]; then
+ # Current has higher base version
+ echo "current_newer"
+ return
+ fi
+
+ # Base versions are equal, compare pre-release status
+ if [ "$current_base" = "$latest_base" ]; then
+ # If current is pre-release and latest is not, current is older
+ if is_prerelease "$current" && ! is_prerelease "$latest"; then
+ echo "current_older"
+ return
+ fi
+
+ # If latest is pre-release and current is not, current is newer
+ if ! is_prerelease "$current" && is_prerelease "$latest"; then
+ echo "current_newer"
+ return
+ fi
+ fi
+
+ # Both are same type or different base versions already handled, use sort -V
+ HIGHER_VERSION=$(printf "%s\n%s" "$current" "$latest" | sort -V | tail -n1)
+ if [ "$HIGHER_VERSION" = "$current" ]; then
+ echo "current_newer_or_equal"
+ else
+ echo "current_older"
+ fi
+}
+
+RESULT=$(compare_versions "$CLEAN_CURRENT" "$CLEAN_LATEST")
+
+if [ "$RESULT" = "current_newer" ] || [ "$RESULT" = "current_newer_or_equal" ]; then
echo "Current version ($CLEAN_CURRENT) is NEWER than or EQUAL to latest ($CLEAN_LATEST)"
- echo "is-current-version-latest=true" >> $GITHUB_OUTPUT
+ if [ -n "$GITHUB_OUTPUT" ]; then
+ echo "is-current-version-latest=true" >> $GITHUB_OUTPUT
+ fi
else
echo "Current version ($CLEAN_CURRENT) is OLDER than latest ($CLEAN_LATEST)"
- echo "is-current-version-latest=false" >> $GITHUB_OUTPUT
+ if [ -n "$GITHUB_OUTPUT" ]; then
+ echo "is-current-version-latest=false" >> $GITHUB_OUTPUT
+ fi
fi
diff --git a/.github/scripts/upload-artifacts-to-s3.sh b/.github/scripts/upload-artifacts-to-s3.sh
index 1ddf32044b..9c92859e3b 100755
--- a/.github/scripts/upload-artifacts-to-s3.sh
+++ b/.github/scripts/upload-artifacts-to-s3.sh
@@ -38,6 +38,11 @@ function upload_artifacts() {
curl -X PUT \
-u "$PROXY_USERNAME:$PROXY_PASSWORD" \
-F "file=@$file" \
+ --max-time 3600 \
+ --connect-timeout 20 \
+ --retry 5 \
+ --retry-delay 10 \
+ --retry-max-time 3000 \
"$TARGET_URL"
done
}
@@ -54,6 +59,11 @@ function update_version_info() {
curl -X PUT \
-u "$PROXY_USERNAME:$PROXY_PASSWORD" \
-F "file=@latest-version.txt" \
+ --max-time 3600 \
+ --connect-timeout 20 \
+ --retry 5 \
+ --retry-delay 10 \
+ --retry-max-time 3000 \
"$TARGET_URL"
fi
@@ -66,6 +76,11 @@ function update_version_info() {
curl -X PUT \
-u "$PROXY_USERNAME:$PROXY_PASSWORD" \
-F "file=@latest-nightly-version.txt" \
+ --max-time 3600 \
+ --connect-timeout 20 \
+ --retry 5 \
+ --retry-delay 10 \
+ --retry-max-time 3000 \
"$TARGET_URL"
fi
fi
diff --git a/.github/workflows/bump-helm-charts-version.yml b/.github/workflows/bump-helm-charts-version.yml
new file mode 100644
index 0000000000..5921ec8a8c
--- /dev/null
+++ b/.github/workflows/bump-helm-charts-version.yml
@@ -0,0 +1,29 @@
+name: Bump helm charts version
+
+on:
+ workflow_dispatch:
+ inputs:
+ version:
+ description: The version to bump (e.g. v1.0.0)
+ required: true
+ type: string
+
+jobs:
+ bump-helm-charts-version:
+ name: Bump helm charts version
+ runs-on: ubuntu-latest
+ permissions:
+ contents: write
+ pull-requests: write
+ steps:
+ - name: Checkout repository
+ uses: actions/checkout@v4
+ with:
+ fetch-depth: 0
+
+ - name: Bump helm charts version
+ env:
+ GITHUB_TOKEN: ${{ secrets.HELM_CHARTS_REPO_TOKEN }}
+ VERSION: ${{ inputs.version }}
+ run: |
+ ./.github/scripts/update-helm-charts-version.sh
diff --git a/.github/workflows/bump-homebrew-greptime-version.yml b/.github/workflows/bump-homebrew-greptime-version.yml
new file mode 100644
index 0000000000..af8ca8fc99
--- /dev/null
+++ b/.github/workflows/bump-homebrew-greptime-version.yml
@@ -0,0 +1,29 @@
+name: Bump homebrew greptime version
+
+on:
+ workflow_dispatch:
+ inputs:
+ version:
+ description: The version to bump (e.g. v1.0.0)
+ required: true
+ type: string
+
+jobs:
+ bump-homebrew-greptime-version:
+ name: Bump homebrew greptime version
+ runs-on: ubuntu-latest
+ permissions:
+ contents: write
+ pull-requests: write
+ steps:
+ - name: Checkout repository
+ uses: actions/checkout@v4
+ with:
+ fetch-depth: 0
+
+ - name: Bump homebrew greptime version
+ env:
+ GITHUB_TOKEN: ${{ secrets.HOMEBREW_GREPTIME_REPO_TOKEN }}
+ VERSION: ${{ inputs.version }}
+ run: |
+ ./.github/scripts/update-homebrew-greptme-version.sh
diff --git a/.github/workflows/develop.yml b/.github/workflows/develop.yml
index b6ab0f8926..d0d2804c6a 100644
--- a/.github/workflows/develop.yml
+++ b/.github/workflows/develop.yml
@@ -506,6 +506,12 @@ jobs:
minio: true
kafka: false
values: "with-minio.yaml"
+ - target: "fuzz_repartition_table_chaos"
+ mode:
+ name: "Local WAL repartition chaos"
+ minio: true
+ kafka: false
+ values: "with-minio-repartition-gc.yaml"
steps:
- name: Remove unused software
run: |
diff --git a/.github/workflows/run-multi-lang-tests.yml b/.github/workflows/run-multi-lang-tests.yml
index 4fc405b5b2..52fdca2117 100644
--- a/.github/workflows/run-multi-lang-tests.yml
+++ b/.github/workflows/run-multi-lang-tests.yml
@@ -127,7 +127,7 @@ jobs:
run: |
./bins/greptime standalone start \
--http-addr 0.0.0.0:${{ inputs.http-port }} \
- --rpc-addr 0.0.0.0:4001 \
+ --grpc-bind-addr 0.0.0.0:4001 \
--mysql-addr 0.0.0.0:${{ inputs.mysql-port }} \
--postgres-addr 0.0.0.0:${{ inputs.postgres-port }} \
--user-provider=static_user_provider:cmd:${{ inputs.username }}=${{ inputs.password }} > /tmp/greptimedb.log 2>&1 &
diff --git a/.gitignore b/.gitignore
index 1e6194369f..0789f3ec10 100644
--- a/.gitignore
+++ b/.gitignore
@@ -71,6 +71,8 @@ AGENTS.md
.codex
.gemini
.opencode
+.worktrees/
# local design docs
docs/specs/
+.vs/
diff --git a/Cargo.lock b/Cargo.lock
index edb8ce04d4..49871a94c1 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -200,9 +200,9 @@ dependencies = [
[[package]]
name = "anyhow"
-version = "1.0.98"
+version = "1.0.102"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e16d2d3311acee920a9eb8d33b8cbc1787ce4a264e85f964c2404b969bdcd487"
+checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c"
[[package]]
name = "anymap2"
@@ -982,6 +982,29 @@ dependencies = [
"cc",
]
+[[package]]
+name = "aws-lc-rs"
+version = "1.16.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0ec6fb3fe69024a75fa7e1bfb48aa6cf59706a101658ea01bfd33b2b248a038f"
+dependencies = [
+ "aws-lc-sys",
+ "untrusted 0.7.1",
+ "zeroize",
+]
+
+[[package]]
+name = "aws-lc-sys"
+version = "0.40.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f50037ee5e1e41e7b8f9d161680a725bd1626cb6f8c7e901f91f942850852fe7"
+dependencies = [
+ "cc",
+ "cmake",
+ "dunce",
+ "fs_extra",
+]
+
[[package]]
name = "axum"
version = "0.6.20"
@@ -1129,9 +1152,9 @@ dependencies = [
[[package]]
name = "backon"
-version = "1.5.1"
+version = "1.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "302eaff5357a264a2c42f127ecb8bac761cf99749fc3dc95677e2743991f99e7"
+checksum = "cffb0e931875b666fc4fcb20fee52e9bbd1ef836fd9e9e04ec21555f9f85f7ef"
dependencies = [
"fastrand",
"gloo-timers",
@@ -1252,7 +1275,7 @@ version = "0.72.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4f72209734318d0b619a5e0f5129918b848c416e122a3c4ce054e03cb87b726f"
dependencies = [
- "bitflags 2.9.1",
+ "bitflags 2.11.1",
"cexpr",
"clang-sys",
"itertools 0.13.0",
@@ -1287,11 +1310,11 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
[[package]]
name = "bitflags"
-version = "2.9.1"
+version = "2.11.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1b8e56985ec62d17e9c1001dc89c88ecd7dc08e47eba5ec7c29c7b5eeecde967"
+checksum = "c4512299f36f043ab09a583e57bceb5a5aab7a73db1805848e8fef3c9e8c78b3"
dependencies = [
- "serde",
+ "serde_core",
]
[[package]]
@@ -1654,6 +1677,12 @@ version = "0.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4f4c707c6a209cbe82d10abd08e1ea8995e9ea937d2550646e02798948992be0"
+[[package]]
+name = "cesu8"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6d43a04d8753f35258c91f8ec639f792891f748a1edbd759cf1dcea3382ad83c"
+
[[package]]
name = "cexpr"
version = "0.6.0"
@@ -1923,6 +1952,7 @@ dependencies = [
"common-wal",
"datatypes",
"etcd-client",
+ "fs2",
"futures",
"humantime",
"meta-client",
@@ -1933,7 +1963,7 @@ dependencies = [
"paste",
"query",
"rand 0.9.1",
- "reqwest",
+ "reqwest 0.13.2",
"serde",
"serde_json",
"servers",
@@ -2084,7 +2114,7 @@ dependencies = [
"query",
"rand 0.9.1",
"regex",
- "reqwest",
+ "reqwest 0.13.2",
"serde",
"serde_json",
"servers",
@@ -2139,7 +2169,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "117725a109d387c937a1533ce01b450cbde6b88abceea8473c4d7a85853cda3c"
dependencies = [
"lazy_static",
- "windows-sys 0.59.0",
+ "windows-sys 0.48.0",
+]
+
+[[package]]
+name = "combine"
+version = "4.6.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ba5a308b75df32fe02788e748662718f03fde005016435c444eea572398219fd"
+dependencies = [
+ "bytes",
+ "memchr",
]
[[package]]
@@ -2227,7 +2267,6 @@ dependencies = [
"futures",
"lazy_static",
"object-store",
- "object_store_opendal",
"orc-rust",
"parquet",
"paste",
@@ -2348,8 +2387,8 @@ dependencies = [
"geohash",
"h3o",
"hyperloglogplus",
+ "icu_properties",
"jsonb",
- "jsonpath-rust 0.7.5",
"memchr",
"mito-codec",
"nalgebra",
@@ -2381,7 +2420,7 @@ dependencies = [
"common-test-util",
"common-version",
"hyper 0.14.32",
- "reqwest",
+ "reqwest 0.13.2",
"serde",
"tempfile",
"tokio",
@@ -2623,6 +2662,7 @@ version = "1.0.0"
dependencies = [
"api",
"async-trait",
+ "base64 0.22.1",
"bytes",
"common-base",
"common-error",
@@ -2632,10 +2672,13 @@ dependencies = [
"datafusion",
"datafusion-common",
"datafusion-expr",
+ "datafusion-proto",
"datatypes",
"futures-util",
"once_cell",
+ "prost 0.14.1",
"serde",
+ "serde_json",
"snafu 0.8.6",
"sqlparser",
"sqlparser_derive 0.1.1",
@@ -2755,7 +2798,7 @@ dependencies = [
"once_cell",
"opentelemetry 0.30.0",
"opentelemetry-otlp",
- "opentelemetry-semantic-conventions",
+ "opentelemetry-semantic-conventions 0.30.0",
"opentelemetry_sdk 0.30.0",
"parking_lot 0.12.4",
"prometheus 0.14.0",
@@ -3318,6 +3361,22 @@ dependencies = [
"memchr",
]
+[[package]]
+name = "ctor"
+version = "0.6.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "424e0138278faeb2b401f174ad17e715c829512d74f3d1e81eb43365c2e0590e"
+dependencies = [
+ "ctor-proc-macro",
+ "dtor",
+]
+
+[[package]]
+name = "ctor-proc-macro"
+version = "0.0.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "52560adf09603e58c9a7ee1fe1dcb95a16927b17c127f0ac02d6e768a0e25bc1"
+
[[package]]
name = "ctr"
version = "0.9.2"
@@ -3555,7 +3614,7 @@ dependencies = [
"itertools 0.14.0",
"liblzma",
"log",
- "object_store",
+ "object_store 0.12.5",
"parking_lot 0.12.4",
"parquet",
"rand 0.9.1",
@@ -3587,7 +3646,7 @@ dependencies = [
"futures",
"itertools 0.14.0",
"log",
- "object_store",
+ "object_store 0.12.5",
"parking_lot 0.12.4",
"tokio",
]
@@ -3611,7 +3670,7 @@ dependencies = [
"futures",
"itertools 0.14.0",
"log",
- "object_store",
+ "object_store 0.12.5",
]
[[package]]
@@ -3629,7 +3688,7 @@ dependencies = [
"itertools 0.14.0",
"libc",
"log",
- "object_store",
+ "object_store 0.12.5",
"parquet",
"paste",
"recursive",
@@ -3674,7 +3733,7 @@ dependencies = [
"itertools 0.14.0",
"liblzma",
"log",
- "object_store",
+ "object_store 0.12.5",
"rand 0.9.1",
"tokio",
"tokio-util",
@@ -3701,7 +3760,7 @@ dependencies = [
"datafusion-session",
"futures",
"itertools 0.14.0",
- "object_store",
+ "object_store 0.12.5",
"tokio",
]
@@ -3722,7 +3781,7 @@ dependencies = [
"datafusion-physical-plan",
"datafusion-session",
"futures",
- "object_store",
+ "object_store 0.12.5",
"regex",
"tokio",
]
@@ -3744,7 +3803,7 @@ dependencies = [
"datafusion-physical-plan",
"datafusion-session",
"futures",
- "object_store",
+ "object_store 0.12.5",
"serde_json",
"tokio",
"tokio-stream",
@@ -3773,7 +3832,7 @@ dependencies = [
"futures",
"itertools 0.14.0",
"log",
- "object_store",
+ "object_store 0.12.5",
"parking_lot 0.12.4",
"parquet",
"tokio",
@@ -3799,7 +3858,7 @@ dependencies = [
"datafusion-physical-expr-common",
"futures",
"log",
- "object_store",
+ "object_store 0.12.5",
"parking_lot 0.12.4",
"rand 0.9.1",
"tempfile",
@@ -4008,7 +4067,7 @@ dependencies = [
"datafusion",
"futures",
"futures-util",
- "object_store",
+ "object_store 0.12.5",
"orc-rust",
"tokio",
]
@@ -4129,6 +4188,43 @@ dependencies = [
"tokio",
]
+[[package]]
+name = "datafusion-proto"
+version = "52.1.0"
+source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=02b82535e0160c4545667f36a03e1ff9d1d2e51f#02b82535e0160c4545667f36a03e1ff9d1d2e51f"
+dependencies = [
+ "arrow 57.3.0",
+ "chrono",
+ "datafusion-catalog",
+ "datafusion-catalog-listing",
+ "datafusion-common",
+ "datafusion-datasource",
+ "datafusion-datasource-arrow",
+ "datafusion-datasource-csv",
+ "datafusion-datasource-json",
+ "datafusion-datasource-parquet",
+ "datafusion-execution",
+ "datafusion-expr",
+ "datafusion-functions-table",
+ "datafusion-physical-expr",
+ "datafusion-physical-expr-common",
+ "datafusion-physical-plan",
+ "datafusion-proto-common",
+ "object_store 0.12.5",
+ "prost 0.14.1",
+ "rand 0.9.1",
+]
+
+[[package]]
+name = "datafusion-proto-common"
+version = "52.1.0"
+source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=02b82535e0160c4545667f36a03e1ff9d1d2e51f#02b82535e0160c4545667f36a03e1ff9d1d2e51f"
+dependencies = [
+ "arrow 57.3.0",
+ "datafusion-common",
+ "prost 0.14.1",
+]
+
[[package]]
name = "datafusion-pruning"
version = "52.1.0"
@@ -4187,7 +4283,7 @@ dependencies = [
"datafusion",
"half",
"itertools 0.14.0",
- "object_store",
+ "object_store 0.12.5",
"pbjson-types",
"prost 0.14.1",
"substrait 0.62.2",
@@ -4248,7 +4344,7 @@ dependencies = [
"prometheus 0.14.0",
"prost 0.14.1",
"query",
- "reqwest",
+ "reqwest 0.13.2",
"serde",
"serde_json",
"servers",
@@ -4660,6 +4756,27 @@ version = "2.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ea8a8b81cacc08888170eef4d13b775126db426d0b348bee9d18c2c1eaf123cf"
+[[package]]
+name = "dtor"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "404d02eeb088a82cfd873006cb713fe411306c7d182c344905e101fb1167d301"
+dependencies = [
+ "dtor-proc-macro",
+]
+
+[[package]]
+name = "dtor-proc-macro"
+version = "0.0.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f678cf4a922c215c63e0de95eb1ff08a958a81d47e485cf9da1e27bf6305cfa5"
+
+[[package]]
+name = "dunce"
+version = "1.0.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813"
+
[[package]]
name = "duration-str"
version = "0.11.3"
@@ -4963,7 +5080,6 @@ dependencies = [
"datatypes",
"futures",
"object-store",
- "object_store_opendal",
"serde",
"serde_json",
"snafu 0.8.6",
@@ -5014,7 +5130,7 @@ version = "25.2.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1045398c1bfd89168b5fd3f1fc11f6e70b34f6f66300c87d44d3de849463abf1"
dependencies = [
- "bitflags 2.9.1",
+ "bitflags 2.11.1",
"rustc_version",
]
@@ -5256,6 +5372,7 @@ dependencies = [
"meta-srv",
"num_cpus",
"opentelemetry-proto 0.31.0",
+ "opentelemetry-semantic-conventions 0.31.0",
"operator",
"otel-arrow-rust",
"partition",
@@ -5265,7 +5382,7 @@ dependencies = [
"prost 0.14.1",
"query",
"rand 0.9.1",
- "reqwest",
+ "reqwest 0.13.2",
"serde",
"serde_json",
"servers",
@@ -5361,6 +5478,12 @@ dependencies = [
"windows-sys 0.52.0",
]
+[[package]]
+name = "fs_extra"
+version = "1.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c"
+
[[package]]
name = "fsevent-sys"
version = "4.1.0"
@@ -5653,7 +5776,7 @@ version = "0.20.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7b88256088d75a56f8ecfa070513a775dd9107f6530ef14919dac831af9cfe2b"
dependencies = [
- "bitflags 2.9.1",
+ "bitflags 2.11.1",
"libc",
"libgit2-sys",
"log",
@@ -5681,7 +5804,7 @@ dependencies = [
[[package]]
name = "greptime-proto"
version = "0.1.0"
-source = "git+https://github.com/GreptimeTeam/greptime-proto.git?rev=092ba1d01e2da676dca66cca7eebb55009da8ef8#092ba1d01e2da676dca66cca7eebb55009da8ef8"
+source = "git+https://github.com/GreptimeTeam/greptime-proto.git?rev=dfd2a6d7d3d9c718cb159fcf9abae144b74fc503#dfd2a6d7d3d9c718cb159fcf9abae144b74fc503"
dependencies = [
"prost 0.14.1",
"prost-types 0.14.1",
@@ -5691,7 +5814,6 @@ dependencies = [
"strum_macros 0.25.3",
"tonic 0.14.2",
"tonic-prost",
- "tonic-prost-build",
]
[[package]]
@@ -6607,7 +6729,7 @@ version = "0.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f37dccff2791ab604f9babef0ba14fbe0be30bd368dc541e2b08d07c8aa908f3"
dependencies = [
- "bitflags 2.9.1",
+ "bitflags 2.11.1",
"inotify-sys",
"libc",
]
@@ -6786,6 +6908,93 @@ dependencies = [
"regex",
]
+[[package]]
+name = "jiff"
+version = "0.2.23"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1a3546dc96b6d42c5f24902af9e2538e82e39ad350b0c766eb3fbf2d8f3d8359"
+dependencies = [
+ "jiff-static",
+ "jiff-tzdb-platform",
+ "js-sys",
+ "log",
+ "portable-atomic",
+ "portable-atomic-util",
+ "serde_core",
+ "wasm-bindgen",
+ "windows-sys 0.61.2",
+]
+
+[[package]]
+name = "jiff-static"
+version = "0.2.23"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2a8c8b344124222efd714b73bb41f8b5120b27a7cc1c75593a6ff768d9d05aa4"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.117",
+]
+
+[[package]]
+name = "jiff-tzdb"
+version = "0.1.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c900ef84826f1338a557697dc8fc601df9ca9af4ac137c7fb61d4c6f2dfd3076"
+
+[[package]]
+name = "jiff-tzdb-platform"
+version = "0.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "875a5a69ac2bab1a891711cf5eccbec1ce0341ea805560dcd90b7a2e925132e8"
+dependencies = [
+ "jiff-tzdb",
+]
+
+[[package]]
+name = "jni"
+version = "0.21.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1a87aa2bb7d2af34197c04845522473242e1aa17c12f4935d5856491a7fb8c97"
+dependencies = [
+ "cesu8",
+ "cfg-if",
+ "combine",
+ "jni-sys 0.3.1",
+ "log",
+ "thiserror 1.0.69",
+ "walkdir",
+ "windows-sys 0.45.0",
+]
+
+[[package]]
+name = "jni-sys"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "41a652e1f9b6e0275df1f15b32661cf0d4b78d4d87ddec5e0c3c20f097433258"
+dependencies = [
+ "jni-sys 0.4.1",
+]
+
+[[package]]
+name = "jni-sys"
+version = "0.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c6377a88cb3910bee9b0fa88d4f42e1d2da8e79915598f65fb0c7ee14c878af2"
+dependencies = [
+ "jni-sys-macros",
+]
+
+[[package]]
+name = "jni-sys-macros"
+version = "0.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264"
+dependencies = [
+ "quote",
+ "syn 2.0.117",
+]
+
[[package]]
name = "jobserver"
version = "0.1.33"
@@ -6798,10 +7007,12 @@ dependencies = [
[[package]]
name = "js-sys"
-version = "0.3.77"
+version = "0.3.95"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1cfaf33c695fc6e08064efbc1f72ec937429614f25eef83af942d0e227c3a28f"
+checksum = "2964e92d1d9dc3364cae4d718d93f227e3abb088e747d92e0395bfdedf1c12ca"
dependencies = [
+ "cfg-if",
+ "futures-util",
"once_cell",
"wasm-bindgen",
]
@@ -6886,16 +7097,18 @@ dependencies = [
[[package]]
name = "jsonwebtoken"
-version = "9.3.1"
+version = "10.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5a87cc7a48537badeae96744432de36f4be2b4a34a05a5ef32e9dd8a1c169dde"
+checksum = "0529410abe238729a60b108898784df8984c87f6054c9c4fcacc47e4803c1ce1"
dependencies = [
+ "aws-lc-rs",
"base64 0.22.1",
+ "getrandom 0.2.16",
"js-sys",
"pem",
- "ring",
"serde",
"serde_json",
+ "signature",
"simple_asn1",
]
@@ -7290,7 +7503,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "07033963ba89ebaf1584d767badaa2e8fcec21aedea6b8c0346d487d49c28667"
dependencies = [
"cfg-if",
- "windows-targets 0.52.6",
+ "windows-targets 0.48.5",
]
[[package]]
@@ -7325,7 +7538,7 @@ version = "0.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1580801010e535496706ba011c15f8532df6b42297d2e471fec38ceadd8c0638"
dependencies = [
- "bitflags 2.9.1",
+ "bitflags 2.11.1",
"libc",
"redox_syscall 0.5.13",
]
@@ -7679,6 +7892,15 @@ version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ae960838283323069879657ca3de837e9f7bbb4c7bf6ea7f1b290d5e9476d2e0"
+[[package]]
+name = "mea"
+version = "0.6.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6747f54621d156e1b47eb6b25f39a941b9fc347f98f67d25d8881ff99e8ed832"
+dependencies = [
+ "slab",
+]
+
[[package]]
name = "measure_time"
version = "0.9.0"
@@ -7735,6 +7957,7 @@ dependencies = [
"common-meta",
"common-options",
"common-telemetry",
+ "common-time",
"datatypes",
"futures",
"futures-util",
@@ -7826,7 +8049,7 @@ dependencies = [
"toml 0.8.23",
"tonic 0.14.2",
"tower 0.5.2",
- "tower-http 0.6.6",
+ "tower-http 0.6.8",
"tracing",
"tracing-subscriber",
"typetag",
@@ -7985,9 +8208,11 @@ version = "1.0.0"
dependencies = [
"api",
"aquamarine",
+ "arrow-schema 57.3.0",
"async-channel 1.9.0",
"async-stream",
"async-trait",
+ "base64 0.22.1",
"bytemuck",
"bytes",
"chrono",
@@ -8249,7 +8474,7 @@ dependencies = [
"base64 0.21.7",
"bigdecimal 0.4.8",
"bindgen",
- "bitflags 2.9.1",
+ "bitflags 2.11.1",
"bitvec",
"btoi",
"byteorder",
@@ -8287,7 +8512,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "34a9141e735d5bb02414a7ac03add09522466d4db65bdd827069f76ae0850e58"
dependencies = [
"base64 0.22.1",
- "bitflags 2.9.1",
+ "bitflags 2.11.1",
"btoi",
"byteorder",
"bytes",
@@ -8399,7 +8624,7 @@ version = "0.28.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ab2156c4fce2f8df6c499cc1c763e4394b7482525bf2a9701c9d79d215f519e4"
dependencies = [
- "bitflags 2.9.1",
+ "bitflags 2.11.1",
"cfg-if",
"cfg_aliases 0.1.1",
"libc",
@@ -8411,7 +8636,7 @@ version = "0.30.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "74523f3a35e05aba87a1d978330aef40f67b0304ac79c1c00b294c9830543db6"
dependencies = [
- "bitflags 2.9.1",
+ "bitflags 2.11.1",
"cfg-if",
"cfg_aliases 0.2.1",
"libc",
@@ -8448,7 +8673,7 @@ version = "8.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2fee8403b3d66ac7b26aee6e40a897d85dc5ce26f44da36b8b73e987cc52e943"
dependencies = [
- "bitflags 2.9.1",
+ "bitflags 2.11.1",
"filetime",
"fsevent-sys",
"inotify",
@@ -8708,7 +8933,9 @@ name = "object-store"
version = "1.0.0"
dependencies = [
"anyhow",
+ "async-trait",
"bytes",
+ "chrono",
"common-base",
"common-error",
"common-macro",
@@ -8718,11 +8945,16 @@ dependencies = [
"futures",
"humantime-serde",
"lazy_static",
+ "object_store 0.12.5",
+ "object_store 0.13.2",
+ "object_store_opendal",
"opendal",
"prometheus 0.14.0",
- "reqwest",
+ "rand 0.9.1",
+ "reqwest 0.13.2",
"serde",
"snafu 0.8.6",
+ "tempfile",
"tokio",
"uuid",
]
@@ -8752,15 +8984,43 @@ dependencies = [
]
[[package]]
-name = "object_store_opendal"
-version = "0.54.0"
+name = "object_store"
+version = "0.13.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5ce697ee723fdc3eaf6c457abf4059034be15167022b18b619993802cd1443d5"
+checksum = "622acbc9100d3c10e2ee15804b0caa40e55c933d5aa53814cd520805b7958a49"
dependencies = [
"async-trait",
"bytes",
+ "chrono",
+ "futures-channel",
+ "futures-core",
+ "futures-util",
+ "http 1.3.1",
+ "humantime",
+ "itertools 0.14.0",
+ "parking_lot 0.12.4",
+ "percent-encoding",
+ "thiserror 2.0.17",
+ "tokio",
+ "tracing",
+ "url",
+ "walkdir",
+ "wasm-bindgen-futures",
+ "web-time",
+]
+
+[[package]]
+name = "object_store_opendal"
+version = "0.56.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "08298874eee5935c95bcaa393148834f9c53d904461ca15584a041d8a1c907c2"
+dependencies = [
+ "async-trait",
+ "bytes",
+ "chrono",
"futures",
- "object_store",
+ "mea",
+ "object_store 0.13.2",
"opendal",
"pin-project",
"tokio",
@@ -8810,7 +9070,7 @@ version = "6.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "336b9c63443aceef14bea841b899035ae3abe89b7c486aaf4c5bd8aafedac3f0"
dependencies = [
- "bitflags 2.9.1",
+ "bitflags 2.11.1",
"libc",
"once_cell",
"onig_sys",
@@ -8840,33 +9100,245 @@ checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381"
[[package]]
name = "opendal"
-version = "0.54.1"
+version = "0.56.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "42afda58fa2cf50914402d132cc1caacff116a85d10c72ab2082bb7c50021754"
+checksum = "97b31d3d8e99a85d83b73ec26647f5607b80578ed9375810b6e44ffa3590a236"
+dependencies = [
+ "ctor",
+ "opendal-core",
+ "opendal-layer-concurrent-limit",
+ "opendal-layer-logging",
+ "opendal-layer-prometheus",
+ "opendal-layer-retry",
+ "opendal-layer-timeout",
+ "opendal-layer-tracing",
+ "opendal-service-azblob",
+ "opendal-service-fs",
+ "opendal-service-gcs",
+ "opendal-service-http",
+ "opendal-service-oss",
+ "opendal-service-s3",
+]
+
+[[package]]
+name = "opendal-core"
+version = "0.56.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1849dd2687e173e776d3af5fce1ba3ae47b9dd37a09d1c4deba850ef45fe00ca"
dependencies = [
"anyhow",
- "backon",
"base64 0.22.1",
"bytes",
- "chrono",
- "crc32c",
"futures",
- "getrandom 0.2.16",
"http 1.3.1",
"http-body 1.0.1",
+ "jiff",
"log",
"md-5",
+ "mea",
"percent-encoding",
- "prometheus 0.14.0",
"quick-xml 0.38.4",
- "reqsign",
- "reqwest",
+ "reqsign-core",
+ "reqwest 0.13.2",
"serde",
"serde_json",
- "sha2",
"tokio",
- "tracing",
+ "url",
"uuid",
+ "web-time",
+]
+
+[[package]]
+name = "opendal-layer-concurrent-limit"
+version = "0.56.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "048b1b29c503263bdd80a9afe46a68cd02ea9bd361185b1feab4b151078998e9"
+dependencies = [
+ "futures",
+ "http 1.3.1",
+ "mea",
+ "opendal-core",
+]
+
+[[package]]
+name = "opendal-layer-logging"
+version = "0.56.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d2645adc988b12eda106e2679ae529facfbbaa868ceb706f6f8125c6af15c47b"
+dependencies = [
+ "log",
+ "opendal-core",
+]
+
+[[package]]
+name = "opendal-layer-observe-metrics-common"
+version = "0.56.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9130f0ac11569edc0f70b0e64078b9a12e37a128849d27ea62b0ca7568e8eb97"
+dependencies = [
+ "futures",
+ "http 1.3.1",
+ "opendal-core",
+]
+
+[[package]]
+name = "opendal-layer-prometheus"
+version = "0.56.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "eef98056f8198b5053005e1fbe7d163562f56d9f3be1b73a3792667e7cbaf7be"
+dependencies = [
+ "opendal-core",
+ "opendal-layer-observe-metrics-common",
+ "prometheus 0.14.0",
+]
+
+[[package]]
+name = "opendal-layer-retry"
+version = "0.56.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4eac134ffa4ddda6131a640a84a5315996424b9416c85052f8c64c1a33b70ad4"
+dependencies = [
+ "backon",
+ "log",
+ "opendal-core",
+]
+
+[[package]]
+name = "opendal-layer-timeout"
+version = "0.56.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "619586ab7480c2e3009f6d18eabab18957bc094778fd130bcc38924970a90f4c"
+dependencies = [
+ "opendal-core",
+ "tokio",
+]
+
+[[package]]
+name = "opendal-layer-tracing"
+version = "0.56.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f569b1cfae56851662e0db0e2d8df2f417d40aa281536c13f0c5211e7b56cd6f"
+dependencies = [
+ "futures",
+ "http 1.3.1",
+ "opendal-core",
+ "tracing",
+]
+
+[[package]]
+name = "opendal-service-azblob"
+version = "0.56.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7452bf3ec61cfd81ac9ad9ada17825931e9e371d44a045c6bfab9596c0a2ac3b"
+dependencies = [
+ "base64 0.22.1",
+ "bytes",
+ "http 1.3.1",
+ "log",
+ "opendal-core",
+ "opendal-service-azure-common",
+ "quick-xml 0.38.4",
+ "reqsign-azure-storage",
+ "reqsign-core",
+ "reqsign-file-read-tokio",
+ "serde",
+ "sha2",
+ "uuid",
+]
+
+[[package]]
+name = "opendal-service-azure-common"
+version = "0.56.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ffb0e45d6c8dcf66ce2da20e241bcb80e6e540e109a4ff20f318f6c9b4c54e0c"
+dependencies = [
+ "http 1.3.1",
+ "opendal-core",
+]
+
+[[package]]
+name = "opendal-service-fs"
+version = "0.56.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cf0be0417abeeb0053376d816b90fceb9ca98f20dfb54ebf1f2a282729f83663"
+dependencies = [
+ "bytes",
+ "log",
+ "opendal-core",
+ "serde",
+ "tokio",
+ "xattr",
+]
+
+[[package]]
+name = "opendal-service-gcs"
+version = "0.56.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "70a49477a10163431896d106136117f5670717f9c9e49cf6f710528800c6633a"
+dependencies = [
+ "async-trait",
+ "bytes",
+ "http 1.3.1",
+ "log",
+ "opendal-core",
+ "percent-encoding",
+ "quick-xml 0.38.4",
+ "reqsign-core",
+ "reqsign-file-read-tokio",
+ "reqsign-google",
+ "serde",
+ "serde_json",
+ "tokio",
+]
+
+[[package]]
+name = "opendal-service-http"
+version = "0.56.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6fe73e6978feec293acfb92bfa94bdb9cf1b5be3f7c3f93a4333a25455826005"
+dependencies = [
+ "http 1.3.1",
+ "log",
+ "opendal-core",
+ "serde",
+]
+
+[[package]]
+name = "opendal-service-oss"
+version = "0.56.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "29c8a917829ad06d21b639558532cb0101fe49b040d946d673a73018683fac05"
+dependencies = [
+ "bytes",
+ "http 1.3.1",
+ "log",
+ "opendal-core",
+ "quick-xml 0.38.4",
+ "reqsign-aliyun-oss",
+ "reqsign-core",
+ "reqsign-file-read-tokio",
+ "serde",
+]
+
+[[package]]
+name = "opendal-service-s3"
+version = "0.56.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9dadddeb9bb50b0d30927dd914c298c4ddca47e4c1cfa7674d311f0cf9b051c8"
+dependencies = [
+ "base64 0.22.1",
+ "bytes",
+ "crc32c",
+ "http 1.3.1",
+ "log",
+ "md-5",
+ "opendal-core",
+ "quick-xml 0.38.4",
+ "reqsign-aws-v4",
+ "reqsign-core",
+ "reqsign-file-read-tokio",
+ "serde",
+ "url",
]
[[package]]
@@ -8883,7 +9355,7 @@ dependencies = [
[[package]]
name = "opensrv-mysql"
version = "0.8.0"
-source = "git+https://github.com/datafuselabs/opensrv?tag=v0.10.0#074bd8fb81da3c9e6d6a098a482f3380478b9c0b"
+source = "git+https://github.com/GreptimeTeam/opensrv?rev=6c5a451544194b7bb60a8318d155d4f892b49f2c#6c5a451544194b7bb60a8318d155d4f892b49f2c"
dependencies = [
"async-trait",
"byteorder",
@@ -8940,7 +9412,7 @@ dependencies = [
"bytes",
"http 1.3.1",
"opentelemetry 0.30.0",
- "reqwest",
+ "reqwest 0.12.28",
]
[[package]]
@@ -8955,7 +9427,7 @@ dependencies = [
"opentelemetry-proto 0.30.0",
"opentelemetry_sdk 0.30.0",
"prost 0.13.5",
- "reqwest",
+ "reqwest 0.12.28",
"thiserror 2.0.17",
"tokio",
"tonic 0.13.1",
@@ -8997,6 +9469,12 @@ version = "0.30.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "83d059a296a47436748557a353c5e6c5705b9470ef6c95cfc52c21a8814ddac2"
+[[package]]
+name = "opentelemetry-semantic-conventions"
+version = "0.31.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e62e29dfe041afb8ed2a6c9737ab57db4907285d999ef8ad3a59092a36bdc846"
+
[[package]]
name = "opentelemetry_sdk"
version = "0.30.0"
@@ -9078,7 +9556,6 @@ dependencies = [
"meter-macros",
"moka",
"object-store",
- "object_store_opendal",
"partition",
"path-slash",
"prometheus 0.14.0",
@@ -9343,7 +9820,7 @@ dependencies = [
"num-bigint",
"num-integer",
"num-traits",
- "object_store",
+ "object_store 0.12.5",
"paste",
"seq-macro",
"simdutf8",
@@ -9591,6 +10068,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "24bd4e6b1bfddc5c6420dee6602ec80946700b4c31ddcb64ee190ad6d979c210"
dependencies = [
"async-trait",
+ "aws-lc-rs",
"base64 0.22.1",
"bytes",
"chrono",
@@ -9602,7 +10080,6 @@ dependencies = [
"pg_interval_2",
"postgres-types",
"rand 0.10.0",
- "ring",
"rust_decimal",
"rustls-pki-types",
"ryu",
@@ -9919,6 +10396,15 @@ version = "1.11.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f84267b20a16ea918e43c6a88433c2d54fa145c92a811b5b047ccbe153674483"
+[[package]]
+name = "portable-atomic-util"
+version = "0.2.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c2a106d1259c23fac8e543272398ae0e3c0b8d33c88ed73d0cc71b0f1d902618"
+dependencies = [
+ "portable-atomic",
+]
+
[[package]]
name = "postgres-protocol"
version = "0.6.8"
@@ -10137,7 +10623,7 @@ version = "0.17.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cc5b72d8145275d844d4b5f6d4e1eef00c8cd889edb6035c21675d1bb1f45c9f"
dependencies = [
- "bitflags 2.9.1",
+ "bitflags 2.11.1",
"hex",
"procfs-core",
"rustix 0.38.44",
@@ -10149,7 +10635,7 @@ version = "0.17.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "239df02d8349b06fc07398a3a1697b06418223b1c7725085e801e7c0fc6a12ec"
dependencies = [
- "bitflags 2.9.1",
+ "bitflags 2.11.1",
"hex",
]
@@ -10244,7 +10730,7 @@ version = "1.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bee689443a2bd0a16ab0348b52ee43e3b2d1b1f931c8aa5c9f8de4c86fbe8c40"
dependencies = [
- "bitflags 2.9.1",
+ "bitflags 2.11.1",
"num-traits",
"rand 0.9.1",
"rand_chacha 0.9.0",
@@ -10321,7 +10807,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ac6c3320f9abac597dcbc668774ef006702672474aad53c6d596b62e487b40b1"
dependencies = [
"heck 0.5.0",
- "itertools 0.14.0",
+ "itertools 0.11.0",
"log",
"multimap",
"once_cell",
@@ -10369,7 +10855,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d"
dependencies = [
"anyhow",
- "itertools 0.14.0",
+ "itertools 0.11.0",
"proc-macro2",
"quote",
"syn 2.0.117",
@@ -10382,7 +10868,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9120690fafc389a67ba3803df527d0ec9cbbc9cc45e4cc20b332996dfb672425"
dependencies = [
"anyhow",
- "itertools 0.14.0",
+ "itertools 0.11.0",
"proc-macro2",
"quote",
"syn 2.0.117",
@@ -10558,7 +11044,7 @@ dependencies = [
"async-walkdir",
"auto_impl",
"base64 0.22.1",
- "bitflags 2.9.1",
+ "bitflags 2.11.1",
"bytes",
"common-base",
"common-error",
@@ -10587,7 +11073,7 @@ version = "0.13.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1e8bbe1a966bd2f362681a44f6edce3c2310ac21e4d5067a6e7ec396297a6ea0"
dependencies = [
- "bitflags 2.9.1",
+ "bitflags 2.11.1",
"memchr",
"unicase",
]
@@ -10643,11 +11129,13 @@ dependencies = [
"datafusion",
"datafusion-common",
"datafusion-expr",
+ "datafusion-expr-common",
"datafusion-functions",
"datafusion-optimizer",
"datafusion-physical-expr",
"datafusion-sql",
"datatypes",
+ "either",
"fastrand",
"futures",
"futures-util",
@@ -10704,7 +11192,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "331e97a1af0bf59823e6eadffe373d7b27f485be8748f71471c662c1f269b7fb"
dependencies = [
"memchr",
- "serde",
]
[[package]]
@@ -10717,6 +11204,16 @@ dependencies = [
"serde",
]
+[[package]]
+name = "quick-xml"
+version = "0.39.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "958f21e8e7ceb5a1aa7fa87fab28e7c75976e0bfe7e23ff069e0a260f894067d"
+dependencies = [
+ "memchr",
+ "serde",
+]
+
[[package]]
name = "quinn"
version = "0.11.8"
@@ -10743,6 +11240,7 @@ version = "0.11.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "434b42fec591c96ef50e21e886936e66d3cc3f737104fdb9b737c40ffb94c098"
dependencies = [
+ "aws-lc-rs",
"bytes",
"getrandom 0.3.3",
"lru-slab",
@@ -11007,7 +11505,7 @@ version = "0.5.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0d04b7d0ee6b4a0207a0a7adb104d23ecb0b47d6beae7152d0fa34b692b29fd6"
dependencies = [
- "bitflags 2.9.1",
+ "bitflags 2.11.1",
]
[[package]]
@@ -11111,42 +11609,123 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "51743d3e274e2b18df81c4dc6caf8a5b8e15dbe799e0dca05c7617380094e884"
[[package]]
-name = "reqsign"
-version = "0.16.5"
+name = "reqsign-aliyun-oss"
+version = "3.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "43451dbf3590a7590684c25fb8d12ecdcc90ed3ac123433e500447c7d77ed701"
+checksum = "57ac2757f3140aa2e213b554148ae0b52733e624fc6723f0cc6bb3d440176c95"
dependencies = [
"anyhow",
- "async-trait",
- "base64 0.22.1",
- "chrono",
"form_urlencoded",
- "getrandom 0.2.16",
- "hex",
- "hmac",
- "home",
"http 1.3.1",
- "jsonwebtoken",
"log",
- "once_cell",
"percent-encoding",
- "quick-xml 0.37.5",
- "rand 0.8.5",
- "reqwest",
- "rsa",
+ "reqsign-core",
"rust-ini 0.21.1",
"serde",
"serde_json",
+]
+
+[[package]]
+name = "reqsign-aws-v4"
+version = "3.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "44eaca382e94505a49f1a4849658d153aebf79d9c1a58e5dd3b10361511e9f43"
+dependencies = [
+ "anyhow",
+ "bytes",
+ "form_urlencoded",
+ "http 1.3.1",
+ "log",
+ "percent-encoding",
+ "quick-xml 0.39.2",
+ "reqsign-core",
+ "rust-ini 0.21.1",
+ "serde",
+ "serde_json",
+ "serde_urlencoded",
"sha1",
+]
+
+[[package]]
+name = "reqsign-azure-storage"
+version = "3.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7a321980405d596bd34aaf95c4722a3de4128a67fd19e74a81a83aa3fdf082e6"
+dependencies = [
+ "anyhow",
+ "base64 0.22.1",
+ "bytes",
+ "form_urlencoded",
+ "http 1.3.1",
+ "jsonwebtoken",
+ "log",
+ "pem",
+ "percent-encoding",
+ "reqsign-core",
+ "rsa",
+ "serde",
+ "serde_json",
+ "sha1",
+]
+
+[[package]]
+name = "reqsign-core"
+version = "3.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b10302cf0a7d7e7352ba211fc92c3c5bebf1286153e49cc5aa87348078a8e102"
+dependencies = [
+ "anyhow",
+ "base64 0.22.1",
+ "bytes",
+ "form_urlencoded",
+ "futures",
+ "hex",
+ "hmac",
+ "http 1.3.1",
+ "jiff",
+ "log",
+ "percent-encoding",
+ "sha1",
+ "sha2",
+ "windows-sys 0.61.2",
+]
+
+[[package]]
+name = "reqsign-file-read-tokio"
+version = "3.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e2d89295b3d17abea31851cc8de55d843d89c52132c864963c38d41920613dc5"
+dependencies = [
+ "anyhow",
+ "reqsign-core",
+ "tokio",
+]
+
+[[package]]
+name = "reqsign-google"
+version = "3.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "35cc609b49c69e76ecaceb775a03f792d1ed3e7755ab3548d4534fd801e3242e"
+dependencies = [
+ "form_urlencoded",
+ "http 1.3.1",
+ "jsonwebtoken",
+ "log",
+ "percent-encoding",
+ "reqsign-aws-v4",
+ "reqsign-core",
+ "rsa",
+ "serde",
+ "serde_json",
"sha2",
"tokio",
]
[[package]]
name = "reqwest"
-version = "0.12.24"
+version = "0.12.28"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9d0946410b9f7b082a427e4ef5c8ff541a88b357bc6c637c40db3a68ac70a36f"
+checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147"
dependencies = [
"base64 0.22.1",
"bytes",
@@ -11161,13 +11740,54 @@ dependencies = [
"hyper-util",
"js-sys",
"log",
+ "percent-encoding",
+ "pin-project-lite",
+ "quinn",
+ "rustls",
+ "rustls-pki-types",
+ "serde",
+ "serde_json",
+ "serde_urlencoded",
+ "sync_wrapper 1.0.2",
+ "tokio",
+ "tokio-rustls",
+ "tokio-util",
+ "tower 0.5.2",
+ "tower-http 0.6.8",
+ "tower-service",
+ "url",
+ "wasm-bindgen",
+ "wasm-bindgen-futures",
+ "wasm-streams 0.4.2",
+ "web-sys",
+ "webpki-roots 1.0.1",
+]
+
+[[package]]
+name = "reqwest"
+version = "0.13.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ab3f43e3283ab1488b624b44b0e988d0acea0b3214e694730a055cb6b2efa801"
+dependencies = [
+ "base64 0.22.1",
+ "bytes",
+ "futures-core",
+ "futures-util",
+ "http 1.3.1",
+ "http-body 1.0.1",
+ "http-body-util",
+ "hyper 1.6.0",
+ "hyper-rustls",
+ "hyper-util",
+ "js-sys",
+ "log",
"mime_guess",
"percent-encoding",
"pin-project-lite",
"quinn",
"rustls",
- "rustls-native-certs 0.8.1",
"rustls-pki-types",
+ "rustls-platform-verifier",
"serde",
"serde_json",
"serde_urlencoded",
@@ -11176,14 +11796,13 @@ dependencies = [
"tokio-rustls",
"tokio-util",
"tower 0.5.2",
- "tower-http 0.6.6",
+ "tower-http 0.6.8",
"tower-service",
"url",
"wasm-bindgen",
"wasm-bindgen-futures",
- "wasm-streams",
+ "wasm-streams 0.5.0",
"web-sys",
- "webpki-roots 1.0.1",
]
[[package]]
@@ -11205,7 +11824,7 @@ dependencies = [
"cfg-if",
"getrandom 0.2.16",
"libc",
- "untrusted",
+ "untrusted 0.9.0",
"windows-sys 0.52.0",
]
@@ -11508,7 +12127,7 @@ version = "0.38.44"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154"
dependencies = [
- "bitflags 2.9.1",
+ "bitflags 2.11.1",
"errno",
"libc",
"linux-raw-sys 0.4.15",
@@ -11521,7 +12140,7 @@ version = "1.0.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c71e83d6afe7ff64890ec6b71d6a69bb8a610ab78ce364b3352876bb4c801266"
dependencies = [
- "bitflags 2.9.1",
+ "bitflags 2.11.1",
"errno",
"libc",
"linux-raw-sys 0.9.4",
@@ -11534,6 +12153,7 @@ version = "0.23.28"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7160e3e10bf4535308537f3c4e1641468cd0e485175d6163087c0393c7d46643"
dependencies = [
+ "aws-lc-rs",
"log",
"once_cell",
"ring",
@@ -11565,7 +12185,7 @@ dependencies = [
"openssl-probe",
"rustls-pki-types",
"schannel",
- "security-framework 3.2.0",
+ "security-framework 3.7.0",
]
[[package]]
@@ -11588,14 +12208,42 @@ dependencies = [
]
[[package]]
-name = "rustls-webpki"
-version = "0.103.10"
+name = "rustls-platform-verifier"
+version = "0.6.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "df33b2b81ac578cabaf06b89b0631153a3f416b0a886e8a7a1707fb51abbd1ef"
+checksum = "1d99feebc72bae7ab76ba994bb5e121b8d83d910ca40b36e0921f53becc41784"
dependencies = [
+ "core-foundation 0.10.1",
+ "core-foundation-sys",
+ "jni",
+ "log",
+ "once_cell",
+ "rustls",
+ "rustls-native-certs 0.8.1",
+ "rustls-platform-verifier-android",
+ "rustls-webpki",
+ "security-framework 3.7.0",
+ "security-framework-sys",
+ "webpki-root-certs",
+ "windows-sys 0.61.2",
+]
+
+[[package]]
+name = "rustls-platform-verifier-android"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f"
+
+[[package]]
+name = "rustls-webpki"
+version = "0.103.13"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e"
+dependencies = [
+ "aws-lc-rs",
"ring",
"rustls-pki-types",
- "untrusted",
+ "untrusted 0.9.0",
]
[[package]]
@@ -11777,7 +12425,7 @@ version = "2.11.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02"
dependencies = [
- "bitflags 2.9.1",
+ "bitflags 2.11.1",
"core-foundation 0.9.4",
"core-foundation-sys",
"libc",
@@ -11786,11 +12434,11 @@ dependencies = [
[[package]]
name = "security-framework"
-version = "3.2.0"
+version = "3.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "271720403f46ca04f7ba6f55d438f8bd878d6b8ca0a1046e8228c4145bcbb316"
+checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d"
dependencies = [
- "bitflags 2.9.1",
+ "bitflags 2.11.1",
"core-foundation 0.10.1",
"core-foundation-sys",
"libc",
@@ -11799,9 +12447,9 @@ dependencies = [
[[package]]
name = "security-framework-sys"
-version = "2.14.0"
+version = "2.17.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "49db231d56a190491cb4aeda9527f1ad45345af50b0851622a7adb8c03b01c32"
+checksum = "6ce2691df843ecc5d231c0b14ece2acc3efb62c0a398c7e1d875f3983ce020e3"
dependencies = [
"core-foundation-sys",
"libc",
@@ -12034,6 +12682,7 @@ dependencies = [
"datafusion-pg-catalog",
"datatypes",
"derive_builder 0.20.2",
+ "either",
"futures",
"futures-util",
"headers",
@@ -12078,8 +12727,9 @@ dependencies = [
"quoted-string",
"rand 0.9.1",
"regex",
- "reqwest",
+ "reqwest 0.13.2",
"rust-embed",
+ "rust_decimal",
"rustls",
"rustls-pemfile",
"rustls-pki-types",
@@ -12106,8 +12756,9 @@ dependencies = [
"tokio-util",
"tonic 0.14.2",
"tonic-reflection",
+ "tonic-web",
"tower 0.5.2",
- "tower-http 0.6.6",
+ "tower-http 0.6.8",
"tracing",
"tracing-opentelemetry",
"urlencoding",
@@ -12136,6 +12787,7 @@ dependencies = [
"derive_more",
"snafu 0.8.6",
"sql",
+ "uuid",
]
[[package]]
@@ -12526,7 +13178,7 @@ dependencies = [
"local-ip-address",
"mysql",
"num_cpus",
- "reqwest",
+ "reqwest 0.12.28",
"serde",
"serde_json",
"sha2",
@@ -12670,7 +13322,7 @@ checksum = "aa003f0038df784eb8fecbbac13affe3da23b45194bd57dba231c8f48199c526"
dependencies = [
"atoi",
"base64 0.22.1",
- "bitflags 2.9.1",
+ "bitflags 2.11.1",
"byteorder",
"bytes",
"chrono",
@@ -12714,7 +13366,7 @@ checksum = "db58fcd5a53cf07c184b154801ff91347e4c30d17a3562a635ff028ad5deda46"
dependencies = [
"atoi",
"base64 0.22.1",
- "bitflags 2.9.1",
+ "bitflags 2.11.1",
"byteorder",
"chrono",
"crc",
@@ -13446,7 +14098,8 @@ dependencies = [
"paste",
"rand 0.9.1",
"rand_chacha 0.9.0",
- "reqwest",
+ "reqwest 0.13.2",
+ "rustls",
"schemars",
"serde",
"serde_json",
@@ -13471,6 +14124,7 @@ dependencies = [
"async-trait",
"auth",
"axum 0.8.4",
+ "base64 0.22.1",
"cache",
"catalog",
"chrono",
@@ -13486,6 +14140,7 @@ dependencies = [
"common-grpc",
"common-memory-manager",
"common-meta",
+ "common-options",
"common-procedure",
"common-procedure-test",
"common-query",
@@ -13501,6 +14156,7 @@ dependencies = [
"datanode",
"datatypes",
"dotenv",
+ "either",
"flate2",
"flow",
"frontend",
@@ -13516,7 +14172,6 @@ dependencies = [
"meta-client",
"meta-srv",
"mito2",
- "moka",
"mysql_async",
"object-store",
"opentelemetry-proto 0.31.0",
@@ -13525,6 +14180,7 @@ dependencies = [
"partition",
"paste",
"pipeline",
+ "plugins",
"prost 0.14.1",
"query",
"rand 0.9.1",
@@ -14090,6 +14746,24 @@ dependencies = [
"tonic-prost",
]
+[[package]]
+name = "tonic-web"
+version = "0.14.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "29453d84de05f4f1b573db22e6f9f6c95c189a6089a440c9a098aa9dea009299"
+dependencies = [
+ "base64 0.22.1",
+ "bytes",
+ "http 1.3.1",
+ "http-body 1.0.1",
+ "pin-project",
+ "tokio-stream",
+ "tonic 0.14.2",
+ "tower-layer",
+ "tower-service",
+ "tracing",
+]
+
[[package]]
name = "tower"
version = "0.4.13"
@@ -14137,7 +14811,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1e9cd434a998747dd2c4276bc96ee2e0c7a2eadf3cae88e52be55a05fa9053f5"
dependencies = [
"base64 0.21.7",
- "bitflags 2.9.1",
+ "bitflags 2.11.1",
"bytes",
"http 1.3.1",
"http-body 1.0.1",
@@ -14151,13 +14825,13 @@ dependencies = [
[[package]]
name = "tower-http"
-version = "0.6.6"
+version = "0.6.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "adc82fd73de2a9722ac5da747f12383d2bfdb93591ee6c58486e0097890f05f2"
+checksum = "d4e6559d53cc268e5031cd8429d05415bc4cb4aefc4aa5d6cc35fbf5b924a1f8"
dependencies = [
"async-compression",
"base64 0.22.1",
- "bitflags 2.9.1",
+ "bitflags 2.11.1",
"bytes",
"futures-core",
"futures-util",
@@ -14511,6 +15185,12 @@ version = "0.2.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861"
+[[package]]
+name = "untrusted"
+version = "0.7.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a"
+
[[package]]
name = "untrusted"
version = "0.9.0"
@@ -14831,48 +15511,32 @@ checksum = "b8dad83b4f25e74f184f64c43b150b91efe7647395b42289f38e50566d82855b"
[[package]]
name = "wasm-bindgen"
-version = "0.2.100"
+version = "0.2.118"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1edc8929d7499fc4e8f0be2262a241556cfc54a0bea223790e71446f2aab1ef5"
+checksum = "0bf938a0bacb0469e83c1e148908bd7d5a6010354cf4fb73279b7447422e3a89"
dependencies = [
"cfg-if",
"once_cell",
"rustversion",
"wasm-bindgen-macro",
-]
-
-[[package]]
-name = "wasm-bindgen-backend"
-version = "0.2.100"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2f0a0651a5c2bc21487bde11ee802ccaf4c51935d0d3d42a6101f98161700bc6"
-dependencies = [
- "bumpalo",
- "log",
- "proc-macro2",
- "quote",
- "syn 2.0.117",
"wasm-bindgen-shared",
]
[[package]]
name = "wasm-bindgen-futures"
-version = "0.4.50"
+version = "0.4.68"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "555d470ec0bc3bb57890405e5d4322cc9ea83cebb085523ced7be4144dac1e61"
+checksum = "f371d383f2fb139252e0bfac3b81b265689bf45b6874af544ffa4c975ac1ebf8"
dependencies = [
- "cfg-if",
"js-sys",
- "once_cell",
"wasm-bindgen",
- "web-sys",
]
[[package]]
name = "wasm-bindgen-macro"
-version = "0.2.100"
+version = "0.2.118"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7fe63fc6d09ed3792bd0897b314f53de8e16568c2b3f7982f468c0bf9bd0b407"
+checksum = "eeff24f84126c0ec2db7a449f0c2ec963c6a49efe0698c4242929da037ca28ed"
dependencies = [
"quote",
"wasm-bindgen-macro-support",
@@ -14880,22 +15544,22 @@ dependencies = [
[[package]]
name = "wasm-bindgen-macro-support"
-version = "0.2.100"
+version = "0.2.118"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de"
+checksum = "9d08065faf983b2b80a79fd87d8254c409281cf7de75fc4b773019824196c904"
dependencies = [
+ "bumpalo",
"proc-macro2",
"quote",
"syn 2.0.117",
- "wasm-bindgen-backend",
"wasm-bindgen-shared",
]
[[package]]
name = "wasm-bindgen-shared"
-version = "0.2.100"
+version = "0.2.118"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1a05d73b933a847d6cccdda8f838a22ff101ad9bf93e33684f39c1f5f0eece3d"
+checksum = "5fd04d9e306f1907bd13c6361b5c6bfc7b3b3c095ed3f8a9246390f8dbdee129"
dependencies = [
"unicode-ident",
]
@@ -14935,13 +15599,26 @@ dependencies = [
"web-sys",
]
+[[package]]
+name = "wasm-streams"
+version = "0.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9d1ec4f6517c9e11ae630e200b2b65d193279042e28edd4a2cda233e46670bbb"
+dependencies = [
+ "futures-util",
+ "js-sys",
+ "wasm-bindgen",
+ "wasm-bindgen-futures",
+ "web-sys",
+]
+
[[package]]
name = "wasmparser"
version = "0.244.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe"
dependencies = [
- "bitflags 2.9.1",
+ "bitflags 2.11.1",
"hashbrown 0.15.4",
"indexmap 2.13.0",
"semver",
@@ -14949,9 +15626,9 @@ dependencies = [
[[package]]
name = "web-sys"
-version = "0.3.77"
+version = "0.3.95"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "33b6dd2ef9186f1f2072e409e99cd22a975331a6b3591b12c764e0e55c60d5d2"
+checksum = "4f2dfbb17949fa2088e5d39408c48368947b86f7834484e87b73de55bc14d97d"
dependencies = [
"js-sys",
"wasm-bindgen",
@@ -14974,7 +15651,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ed63aea5ce73d0ff405984102c42de94fc55a6b75765d621c65262469b3c9b53"
dependencies = [
"ring",
- "untrusted",
+ "untrusted 0.9.0",
+]
+
+[[package]]
+name = "webpki-root-certs"
+version = "1.0.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f31141ce3fc3e300ae89b78c0dd67f9708061d1d2eda54b8209346fd6be9a92c"
+dependencies = [
+ "rustls-pki-types",
]
[[package]]
@@ -15038,7 +15724,7 @@ version = "0.1.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb"
dependencies = [
- "windows-sys 0.59.0",
+ "windows-sys 0.48.0",
]
[[package]]
@@ -15208,6 +15894,15 @@ dependencies = [
"windows-link 0.1.3",
]
+[[package]]
+name = "windows-sys"
+version = "0.45.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0"
+dependencies = [
+ "windows-targets 0.42.2",
+]
+
[[package]]
name = "windows-sys"
version = "0.48.0"
@@ -15244,6 +15939,21 @@ dependencies = [
"windows-link 0.2.1",
]
+[[package]]
+name = "windows-targets"
+version = "0.42.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071"
+dependencies = [
+ "windows_aarch64_gnullvm 0.42.2",
+ "windows_aarch64_msvc 0.42.2",
+ "windows_i686_gnu 0.42.2",
+ "windows_i686_msvc 0.42.2",
+ "windows_x86_64_gnu 0.42.2",
+ "windows_x86_64_gnullvm 0.42.2",
+ "windows_x86_64_msvc 0.42.2",
+]
+
[[package]]
name = "windows-targets"
version = "0.48.5"
@@ -15284,6 +15994,12 @@ dependencies = [
"windows-link 0.1.3",
]
+[[package]]
+name = "windows_aarch64_gnullvm"
+version = "0.42.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8"
+
[[package]]
name = "windows_aarch64_gnullvm"
version = "0.48.5"
@@ -15296,6 +16012,12 @@ version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3"
+[[package]]
+name = "windows_aarch64_msvc"
+version = "0.42.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43"
+
[[package]]
name = "windows_aarch64_msvc"
version = "0.48.5"
@@ -15308,6 +16030,12 @@ version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469"
+[[package]]
+name = "windows_i686_gnu"
+version = "0.42.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f"
+
[[package]]
name = "windows_i686_gnu"
version = "0.48.5"
@@ -15326,6 +16054,12 @@ version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66"
+[[package]]
+name = "windows_i686_msvc"
+version = "0.42.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060"
+
[[package]]
name = "windows_i686_msvc"
version = "0.48.5"
@@ -15338,6 +16072,12 @@ version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66"
+[[package]]
+name = "windows_x86_64_gnu"
+version = "0.42.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36"
+
[[package]]
name = "windows_x86_64_gnu"
version = "0.48.5"
@@ -15350,6 +16090,12 @@ version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78"
+[[package]]
+name = "windows_x86_64_gnullvm"
+version = "0.42.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3"
+
[[package]]
name = "windows_x86_64_gnullvm"
version = "0.48.5"
@@ -15362,6 +16108,12 @@ version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d"
+[[package]]
+name = "windows_x86_64_msvc"
+version = "0.42.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0"
+
[[package]]
name = "windows_x86_64_msvc"
version = "0.48.5"
@@ -15427,7 +16179,7 @@ version = "0.39.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6f42320e61fe2cfd34354ecb597f86f413484a798ba44a8ca1165c58d42da6c1"
dependencies = [
- "bitflags 2.9.1",
+ "bitflags 2.11.1",
]
[[package]]
@@ -15468,7 +16220,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2"
dependencies = [
"anyhow",
- "bitflags 2.9.1",
+ "bitflags 2.11.1",
"indexmap 2.13.0",
"log",
"serde",
diff --git a/Cargo.toml b/Cargo.toml
index 227608bf64..6778e98c61 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -110,6 +110,7 @@ arrow-schema = { version = "57.3", features = ["serde"] }
async-stream = "0.3"
async-trait = "0.1"
# Remember to update axum-extra, axum-macros when updating axum
+arrow_object_store = { package = "object_store", version = "0.13.2" }
axum = "0.8"
axum-extra = "0.10"
axum-macros = "0.5"
@@ -131,6 +132,7 @@ datafusion = "=52.1"
datafusion-common = "=52.1"
datafusion-datasource = "=52.1"
datafusion-expr = "=52.1"
+datafusion-expr-common = "=52.1"
datafusion-functions = "=52.1"
datafusion-functions-aggregate-common = "=52.1"
datafusion-functions-window-common = "=52.1"
@@ -139,8 +141,10 @@ datafusion-orc = "0.7"
datafusion-pg-catalog = "0.15.1"
datafusion-physical-expr = "=52.1"
datafusion-physical-plan = "=52.1"
+datafusion-proto = "=52.1"
datafusion-sql = "=52.1"
datafusion-substrait = "=52.1"
+datafusion_object_store = { package = "object_store", version = "0.12.5" }
deadpool = "0.12"
deadpool-postgres = "0.14"
derive_builder = "0.20"
@@ -151,16 +155,18 @@ etcd-client = { version = "0.17", features = [
"tls",
"tls-roots",
] }
+fs2 = "0.4"
fst = "0.4.7"
futures = "0.3"
futures-util = "0.3"
-greptime-proto = { git = "https://github.com/GreptimeTeam/greptime-proto.git", rev = "092ba1d01e2da676dca66cca7eebb55009da8ef8" }
+greptime-proto = { git = "https://github.com/GreptimeTeam/greptime-proto.git", rev = "dfd2a6d7d3d9c718cb159fcf9abae144b74fc503" }
hex = "0.4"
http = "1"
humantime = "2.1"
humantime-serde = "1.1"
hyper = "1.1"
hyper-util = "0.1"
+icu_properties = "2.0.1"
itertools = "0.14"
jsonb = { version = "0.4.4", default-features = false }
lazy_static = "1.4"
@@ -173,7 +179,7 @@ nalgebra = "0.33"
nix = { version = "0.30.1", default-features = false, features = ["event", "fs", "process"] }
notify = "8.0"
num_cpus = "1.16"
-object_store_opendal = "0.54"
+object_store_opendal = "0.56"
once_cell = "1.18"
opentelemetry-proto = { version = "0.31", features = [
"gen-tonic",
@@ -200,14 +206,17 @@ rand = "0.9"
ratelimit = "0.10"
regex = "1.12"
regex-automata = "0.4"
-reqwest = { version = "0.12", default-features = false, features = [
+reqwest = { version = "0.13", default-features = false, features = [
+ "form",
"json",
- "rustls-tls-native-roots",
+ "query",
+ "rustls",
"stream",
"multipart",
] }
url = "2.3"
# Branch: feat/request-timeout
+hostname = "0.4.0"
rskafka = { git = "https://github.com/GreptimeTeam/rskafka.git", rev = "f5688f83e7da591cda3f2674c2408b4c0ed4ed50", features = [
"transport-tls",
] }
@@ -215,8 +224,6 @@ rstest = "0.25"
rstest_reuse = "0.7"
rust_decimal = "1.33"
rustc-hash = "2.0"
-# It is worth noting that we should try to avoid using aws-lc-rs until it can be compiled on various platforms.
-hostname = "0.4.0"
rustls = { version = "0.23.25", default-features = false }
sea-query = "0.32"
serde = { version = "1.0", features = ["derive"] }
@@ -231,7 +238,8 @@ sqlx = { version = "0.8", default-features = false, features = [
"any",
"macros",
"json",
- "runtime-tokio-rustls",
+ "runtime-tokio",
+ "tls-rustls-aws-lc-rs",
"rust_decimal",
] }
strum = { version = "0.27", features = ["derive"] }
@@ -243,7 +251,7 @@ tokio-rustls = { version = "0.26.2", default-features = false }
tokio-stream = "0.1"
tokio-util = { version = "0.7", features = ["io-util", "compat"] }
toml = "0.8.8"
-tonic = { version = "0.14", features = ["tls-ring", "gzip", "zstd"] }
+tonic = { version = "0.14", features = ["tls-aws-lc", "gzip", "zstd"] }
tower = "0.5"
tower-http = "0.6"
tracing = "0.1"
@@ -251,7 +259,7 @@ tracing-appender = "0.2"
tracing-opentelemetry = "0.31.0"
tracing-subscriber = { version = "0.3", features = ["env-filter", "json", "fmt"] }
typetag = "0.2"
-uuid = { version = "1.17", features = ["serde", "v4", "fast-rng"] }
+uuid = { version = "1.17", features = ["serde", "v4", "v7", "fast-rng"] }
vrl = "0.25"
zstd = "0.13"
# DO_NOT_REMOVE_THIS: END_OF_EXTERNAL_DEPENDENCIES
@@ -334,6 +342,7 @@ rev = "5618e779cf2bb4755b499c630fba4c35e91898cb"
datafusion = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "02b82535e0160c4545667f36a03e1ff9d1d2e51f" }
datafusion-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "02b82535e0160c4545667f36a03e1ff9d1d2e51f" }
datafusion-expr = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "02b82535e0160c4545667f36a03e1ff9d1d2e51f" }
+datafusion-expr-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "02b82535e0160c4545667f36a03e1ff9d1d2e51f" }
datafusion-functions = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "02b82535e0160c4545667f36a03e1ff9d1d2e51f" }
datafusion-functions-aggregate-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "02b82535e0160c4545667f36a03e1ff9d1d2e51f" }
datafusion-functions-window-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "02b82535e0160c4545667f36a03e1ff9d1d2e51f" }
@@ -341,6 +350,7 @@ datafusion-optimizer = { git = "https://github.com/GreptimeTeam/datafusion.git",
datafusion-physical-expr = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "02b82535e0160c4545667f36a03e1ff9d1d2e51f" }
datafusion-physical-expr-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "02b82535e0160c4545667f36a03e1ff9d1d2e51f" }
datafusion-physical-plan = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "02b82535e0160c4545667f36a03e1ff9d1d2e51f" }
+datafusion-proto = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "02b82535e0160c4545667f36a03e1ff9d1d2e51f" }
datafusion-datasource = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "02b82535e0160c4545667f36a03e1ff9d1d2e51f" }
datafusion-sql = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "02b82535e0160c4545667f36a03e1ff9d1d2e51f" }
datafusion-substrait = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "02b82535e0160c4545667f36a03e1ff9d1d2e51f" }
diff --git a/README.md b/README.md
index 290fe0263d..4ed99fa306 100644
--- a/README.md
+++ b/README.md
@@ -131,7 +131,7 @@ docker run -p 127.0.0.1:4000-4003:4000-4003 \
--name greptime --rm \
greptime/greptimedb:latest standalone start \
--http-addr 0.0.0.0:4000 \
- --rpc-bind-addr 0.0.0.0:4001 \
+ --grpc-bind-addr 0.0.0.0:4001 \
--mysql-addr 0.0.0.0:4002 \
--postgres-addr 0.0.0.0:4003
```
@@ -175,17 +175,16 @@ cargo run -- standalone start
## Project Status
-> **Status:** RC — marching toward v1.0 GA!
-> **GA (v1.0):** March 2026
+> **Status:** [v1.0 GA](https://github.com/GreptimeTeam/greptimedb/releases/tag/v1.0.0) — generally available and production-ready! 🎉
- Deployed in production handling billions of data points daily
- Stable APIs, actively maintained, with regular releases ([version info](https://docs.greptime.com/nightly/reference/about-greptimedb-version))
-GreptimeDB v1.0 represents a major milestone toward maturity — marking stable APIs, production readiness, and proven performance.
+GreptimeDB v1.0 marks a major milestone — stable APIs, production readiness, and proven performance at scale.
-**Roadmap:** [v1.0 highlights and release plan](https://greptime.com/blogs/2025-11-05-greptimedb-v1-highlights) and [2026 roadmap](https://greptime.com/blogs/2026-02-11-greptimedb-roadmap-2026).
+**Learn more:** [v1.0 highlights](https://greptime.com/blogs/2025-11-05-greptimedb-v1-highlights) and [2026 roadmap](https://greptime.com/blogs/2026-02-11-greptimedb-roadmap-2026).
-For production use, we recommend using the latest stable release.
+For production use, we recommend v1.0 or later.
If you find this project useful, a ⭐ would mean a lot to us!
diff --git a/cliff.toml b/cliff.toml
index 4245203e92..2b35ddab5c 100644
--- a/cliff.toml
+++ b/cliff.toml
@@ -12,7 +12,9 @@ footer = ""
body = """
# {{ version }}
+{% if timestamp -%}
Release date: {{ timestamp | date(format="%B %d, %Y") }}
+{% endif -%}
{%- set breakings = commits | filter(attribute="breaking", value=true) -%}
{%- if breakings | length > 0 %}
@@ -118,7 +120,10 @@ filter_commits = false
# regex for skipping tags
# skip_tags = ""
# regex for ignoring tags
-ignore_tags = ".*-nightly-.*"
+# Ignore nightly tags and build-suffixed release tags such as
+# v1.0.0-rc.2-13cdfa9b5-20260325-1774407105 so their commits are merged into
+# the next visible release section instead of creating extra headings.
+ignore_tags = ".*-nightly-.*|^v[0-9]+\\.[0-9]+\\.[0-9]+(-(alpha|beta|rc)\\.[0-9]+)?-[0-9a-f]{7,}-[0-9]{8}-[0-9]+$"
# sort the tags topologically
topo_order = false
# sort the commits inside sections by oldest/newest order
diff --git a/config/config.md b/config/config.md
index 82297d484e..dfb4db1cf1 100644
--- a/config/config.md
+++ b/config/config.md
@@ -14,6 +14,7 @@
| --- | -----| ------- | ----------- |
| `default_timezone` | String | Unset | The default timezone of the server. |
| `default_column_prefix` | String | Unset | The default column prefix for auto-created time index and value columns. |
+| `user_provider` | String | Unset | The user provider for authentication.
Examples: "static_user_provider:file:/path/to/users", "static_user_provider:cmd:greptime_user=greptime_pwd" |
| `max_in_flight_write_bytes` | String | Unset | Maximum total memory for all concurrent write request bodies and messages (HTTP, gRPC, Flight).
Set to 0 to disable the limit. Default: "0" (unlimited) |
| `write_bytes_exhausted_policy` | String | Unset | Policy when write bytes quota is exhausted.
Options: "wait" (default, 10s timeout), "wait()" (e.g., "wait(30s)"), "fail" |
| `init_regions_in_background` | Bool | `false` | Initialize all regions in the background during the startup.
By default, it provides services after all regions have been initialized. |
@@ -231,14 +232,12 @@
| --- | -----| ------- | ----------- |
| `default_timezone` | String | Unset | The default timezone of the server. |
| `default_column_prefix` | String | Unset | The default column prefix for auto-created time index and value columns. |
+| `user_provider` | String | Unset | The user provider for authentication.
Examples: "static_user_provider:file:/path/to/users", "static_user_provider:cmd:greptime_user=greptime_pwd" |
| `max_in_flight_write_bytes` | String | Unset | Maximum total memory for all concurrent write request bodies and messages (HTTP, gRPC, Flight).
Set to 0 to disable the limit. Default: "0" (unlimited) |
| `write_bytes_exhausted_policy` | String | Unset | Policy when write bytes quota is exhausted.
Options: "wait" (default, 10s timeout), "wait()" (e.g., "wait(30s)"), "fail" |
| `runtime` | -- | -- | The runtime options. |
| `runtime.global_rt_size` | Integer | `8` | The number of threads to execute the runtime for global read operations. |
| `runtime.compact_rt_size` | Integer | `4` | The number of threads to execute the runtime for global write operations. |
-| `heartbeat` | -- | -- | The heartbeat options. |
-| `heartbeat.interval` | String | `18s` | Interval for sending heartbeat messages to the metasrv. |
-| `heartbeat.retry_interval` | String | `3s` | Interval for retrying to send heartbeat messages to the metasrv. |
| `http` | -- | -- | The HTTP server options. |
| `http.addr` | String | `127.0.0.1:4000` | The address to bind the HTTP server. |
| `http.timeout` | String | `0s` | HTTP request timeout. Set to 0 to disable timeout. |
@@ -259,7 +258,7 @@
| `grpc.tls.watch` | Bool | `false` | Watch for Certificate and key file change and auto reload.
For now, gRPC tls config does not support auto reload. |
| `internal_grpc` | -- | -- | The internal gRPC server options. Internal gRPC port for nodes inside cluster to access frontend. |
| `internal_grpc.bind_addr` | String | `127.0.0.1:4010` | The address to bind the gRPC server. |
-| `internal_grpc.server_addr` | String | `127.0.0.1:4010` | The address advertised to the metasrv, and used for connections from outside the host.
If left empty or unset, the server will automatically use the IP address of the first network interface
on the host, with the same port number as the one specified in `grpc.bind_addr`. |
+| `internal_grpc.server_addr` | String | `127.0.0.1:4010` | The address advertised to the metasrv, and used for connections from outside the host.
If left empty or unset, the server will automatically use the IP address of the first network interface
on the host, with the same port number as the one specified in `internal_grpc.bind_addr`. |
| `internal_grpc.runtime_size` | Integer | `8` | The number of server worker threads. |
| `internal_grpc.flight_compression` | String | `arrow_ipc` | Compression mode for frontend side Arrow IPC service. Available options:
- `none`: disable all compression
- `transport`: only enable gRPC transport compression (zstd)
- `arrow_ipc`: only enable Arrow IPC compression (lz4)
- `all`: enable all compression.
Default to `none` |
| `internal_grpc.tls` | -- | -- | internal gRPC server TLS options, see `mysql.tls` section. |
@@ -362,7 +361,7 @@
| `region_failure_detector_initialization_delay` | String | `10m` | The delay before starting region failure detection.
This delay helps prevent Metasrv from triggering unnecessary region failovers before all Datanodes are fully started.
Especially useful when the cluster is not deployed with GreptimeDB Operator and maintenance mode is not enabled. |
| `allow_region_failover_on_local_wal` | Bool | `false` | Whether to allow region failover on local WAL.
**This option is not recommended to be set to true, because it may lead to data loss during failover.** |
| `node_max_idle_time` | String | `24hours` | Max allowed idle time before removing node info from metasrv memory. |
-| `heartbeat_interval` | String | `3s` | Base heartbeat interval for calculating distributed time constants.
The frontend heartbeat interval is 6 times of the base heartbeat interval.
The flownode/datanode heartbeat interval is 1 times of the base heartbeat interval.
e.g., If the base heartbeat interval is 3s, the frontend heartbeat interval is 18s, the flownode/datanode heartbeat interval is 3s.
If you change this value, you need to change the heartbeat interval of the flownode/frontend/datanode accordingly. |
+| `heartbeat_interval` | String | `3s` | Base heartbeat interval for calculating distributed time constants.
The frontend heartbeat interval is 6 times of the base heartbeat interval.
The flownode/datanode heartbeat interval is 1 times of the base heartbeat interval.
e.g., If the base heartbeat interval is 3s, the frontend heartbeat interval is 18s, the flownode/datanode heartbeat interval is 3s.
Heartbeat intervals are negotiated from metasrv during handshake; local node configs do not override this. |
| `enable_telemetry` | Bool | `true` | Whether to enable greptimedb telemetry. Enabled by default. |
| `runtime` | -- | -- | The runtime options. |
| `runtime.global_rt_size` | Integer | `8` | The number of threads to execute the runtime for global read operations. |
@@ -378,7 +377,7 @@
| `backend_client.connect_timeout` | String | `3s` | The connect timeout for backend client. |
| `grpc` | -- | -- | The gRPC server options. |
| `grpc.bind_addr` | String | `127.0.0.1:3002` | The address to bind the gRPC server. |
-| `grpc.server_addr` | String | `127.0.0.1:3002` | The communication server address for the frontend and datanode to connect to metasrv.
If left empty or unset, the server will automatically use the IP address of the first network interface
on the host, with the same port number as the one specified in `bind_addr`. |
+| `grpc.server_addr` | String | `127.0.0.1:3002` | The communication server address for the frontend and datanode to connect to metasrv.
If left empty or unset, the server will automatically use the IP address of the first network interface
on the host, with the same port number as the one specified in `grpc.bind_addr`. |
| `grpc.runtime_size` | Integer | `8` | The number of server worker threads. |
| `grpc.max_recv_message_size` | String | `512MB` | The maximum receive message size for gRPC server. |
| `grpc.max_send_message_size` | String | `512MB` | The maximum send message size for gRPC server. |
@@ -471,9 +470,6 @@
| `runtime` | -- | -- | The runtime options. |
| `runtime.global_rt_size` | Integer | `8` | The number of threads to execute the runtime for global read operations. |
| `runtime.compact_rt_size` | Integer | `4` | The number of threads to execute the runtime for global write operations. |
-| `heartbeat` | -- | -- | The heartbeat options. |
-| `heartbeat.interval` | String | `3s` | Interval for sending heartbeat messages to the metasrv. |
-| `heartbeat.retry_interval` | String | `3s` | Interval for retrying to send heartbeat messages to the metasrv. |
| `meta_client` | -- | -- | The metasrv client options. |
| `meta_client.metasrv_addrs` | Array | -- | The addresses of the metasrv. |
| `meta_client.timeout` | String | `3s` | Operation timeout. |
@@ -624,6 +620,7 @@
| Key | Type | Default | Descriptions |
| --- | -----| ------- | ----------- |
| `node_id` | Integer | Unset | The flownode identifier and should be unique in the cluster. |
+| `user_provider` | String | Unset | The user provider for authentication.
Examples: "static_user_provider:file:/path/to/users", "static_user_provider:cmd:greptime_user=greptime_pwd" |
| `flow` | -- | -- | flow engine options. |
| `flow.num_workers` | Integer | `0` | The number of flow worker in flownode.
Not setting(or set to 0) this value will use the number of CPU cores divided by 2. |
| `flow.batching_mode` | -- | -- | -- |
@@ -633,7 +630,6 @@
| `flow.batching_mode.grpc_conn_timeout` | String | `5s` | The gRPC connection timeout |
| `flow.batching_mode.experimental_grpc_max_retries` | Integer | `3` | The gRPC max retry number |
| `flow.batching_mode.experimental_frontend_scan_timeout` | String | `30s` | Flow wait for available frontend timeout,
if failed to find available frontend after frontend_scan_timeout elapsed, return error
which prevent flownode from starting |
-| `flow.batching_mode.experimental_frontend_activity_timeout` | String | `60s` | Frontend activity timeout
if frontend is down(not sending heartbeat) for more than frontend_activity_timeout,
it will be removed from the list that flownode use to connect |
| `flow.batching_mode.experimental_max_filter_num_per_query` | Integer | `20` | Maximum number of filters allowed in a single query |
| `flow.batching_mode.experimental_time_window_merge_threshold` | Integer | `3` | Time window merge distance |
| `flow.batching_mode.read_preference` | String | `Leader` | Read preference of the Frontend client. |
@@ -661,9 +657,6 @@
| `meta_client.metadata_cache_max_capacity` | Integer | `100000` | The configuration about the cache of the metadata. |
| `meta_client.metadata_cache_ttl` | String | `10m` | TTL of the metadata cache. |
| `meta_client.metadata_cache_tti` | String | `5m` | -- |
-| `heartbeat` | -- | -- | The heartbeat options. |
-| `heartbeat.interval` | String | `3s` | Interval for sending heartbeat messages to the metasrv. |
-| `heartbeat.retry_interval` | String | `3s` | Interval for retrying to send heartbeat messages to the metasrv. |
| `logging` | -- | -- | The logging options. |
| `logging.dir` | String | `./greptimedb_data/logs` | The directory to store the log files. If set to empty, logs will not be written to files. |
| `logging.level` | String | Unset | The log level. Can be `info`/`debug`/`warn`/`error`. |
diff --git a/config/datanode.example.toml b/config/datanode.example.toml
index 6effec4c87..fbe205c17a 100644
--- a/config/datanode.example.toml
+++ b/config/datanode.example.toml
@@ -80,14 +80,6 @@ watch = false
## The number of threads to execute the runtime for global write operations.
#+ compact_rt_size = 4
-## The heartbeat options.
-[heartbeat]
-## Interval for sending heartbeat messages to the metasrv.
-interval = "3s"
-
-## Interval for retrying to send heartbeat messages to the metasrv.
-retry_interval = "3s"
-
## The metasrv client options.
[meta_client]
## The addresses of the metasrv.
diff --git a/config/flownode.example.toml b/config/flownode.example.toml
index b13acfc447..2c053e6e8c 100644
--- a/config/flownode.example.toml
+++ b/config/flownode.example.toml
@@ -2,6 +2,11 @@
## @toml2docs:none-default
node_id = 14
+## The user provider for authentication.
+## Examples: "static_user_provider:file:/path/to/users", "static_user_provider:cmd:greptime_user=greptime_pwd"
+## @toml2docs:none-default
+#+ user_provider = "static_user_provider:file:/path/to/users"
+
## flow engine options.
[flow]
## The number of flow worker in flownode.
@@ -22,10 +27,6 @@ node_id = 14
## if failed to find available frontend after frontend_scan_timeout elapsed, return error
## which prevent flownode from starting
#+experimental_frontend_scan_timeout="30s"
-## Frontend activity timeout
-## if frontend is down(not sending heartbeat) for more than frontend_activity_timeout,
-## it will be removed from the list that flownode use to connect
-#+experimental_frontend_activity_timeout="60s"
## Maximum number of filters allowed in a single query
#+experimental_max_filter_num_per_query=20
## Time window merge distance
@@ -96,14 +97,6 @@ metadata_cache_ttl = "10m"
# TTI of the metadata cache.
metadata_cache_tti = "5m"
-## The heartbeat options.
-[heartbeat]
-## Interval for sending heartbeat messages to the metasrv.
-interval = "3s"
-
-## Interval for retrying to send heartbeat messages to the metasrv.
-retry_interval = "3s"
-
## The logging options.
[logging]
## The directory to store the log files. If set to empty, logs will not be written to files.
diff --git a/config/frontend.example.toml b/config/frontend.example.toml
index 97b5851672..60115e93bb 100644
--- a/config/frontend.example.toml
+++ b/config/frontend.example.toml
@@ -6,6 +6,11 @@ default_timezone = "UTC"
## @toml2docs:none-default
default_column_prefix = "greptime"
+## The user provider for authentication.
+## Examples: "static_user_provider:file:/path/to/users", "static_user_provider:cmd:greptime_user=greptime_pwd"
+## @toml2docs:none-default
+#+ user_provider = "static_user_provider:file:/path/to/users"
+
## Maximum total memory for all concurrent write request bodies and messages (HTTP, gRPC, Flight).
## Set to 0 to disable the limit. Default: "0" (unlimited)
## @toml2docs:none-default
@@ -23,14 +28,6 @@ default_column_prefix = "greptime"
## The number of threads to execute the runtime for global write operations.
#+ compact_rt_size = 4
-## The heartbeat options.
-[heartbeat]
-## Interval for sending heartbeat messages to the metasrv.
-interval = "18s"
-
-## Interval for retrying to send heartbeat messages to the metasrv.
-retry_interval = "3s"
-
## The HTTP server options.
[http]
## The address to bind the HTTP server.
@@ -100,7 +97,7 @@ watch = false
bind_addr = "127.0.0.1:4010"
## The address advertised to the metasrv, and used for connections from outside the host.
## If left empty or unset, the server will automatically use the IP address of the first network interface
-## on the host, with the same port number as the one specified in `grpc.bind_addr`.
+## on the host, with the same port number as the one specified in `internal_grpc.bind_addr`.
server_addr = "127.0.0.1:4010"
## The number of server worker threads.
runtime_size = 8
diff --git a/config/metasrv.example.toml b/config/metasrv.example.toml
index fed6e3b814..8e27e5be26 100644
--- a/config/metasrv.example.toml
+++ b/config/metasrv.example.toml
@@ -79,7 +79,7 @@ node_max_idle_time = "24hours"
## The frontend heartbeat interval is 6 times of the base heartbeat interval.
## The flownode/datanode heartbeat interval is 1 times of the base heartbeat interval.
## e.g., If the base heartbeat interval is 3s, the frontend heartbeat interval is 18s, the flownode/datanode heartbeat interval is 3s.
-## If you change this value, you need to change the heartbeat interval of the flownode/frontend/datanode accordingly.
+## Heartbeat intervals are negotiated from metasrv during handshake; local node configs do not override this.
#+ heartbeat_interval = "3s"
## Whether to enable greptimedb telemetry. Enabled by default.
@@ -136,7 +136,7 @@ ca_cert_path = ""
bind_addr = "127.0.0.1:3002"
## The communication server address for the frontend and datanode to connect to metasrv.
## If left empty or unset, the server will automatically use the IP address of the first network interface
-## on the host, with the same port number as the one specified in `bind_addr`.
+## on the host, with the same port number as the one specified in `grpc.bind_addr`.
server_addr = "127.0.0.1:3002"
## The number of server worker threads.
runtime_size = 8
diff --git a/config/standalone.example.toml b/config/standalone.example.toml
index d14bbe63d5..aa745bb6ba 100644
--- a/config/standalone.example.toml
+++ b/config/standalone.example.toml
@@ -6,6 +6,11 @@ default_timezone = "UTC"
## @toml2docs:none-default
default_column_prefix = "greptime"
+## The user provider for authentication.
+## Examples: "static_user_provider:file:/path/to/users", "static_user_provider:cmd:greptime_user=greptime_pwd"
+## @toml2docs:none-default
+#+ user_provider = "static_user_provider:file:/path/to/users"
+
## Maximum total memory for all concurrent write request bodies and messages (HTTP, gRPC, Flight).
## Set to 0 to disable the limit. Default: "0" (unlimited)
## @toml2docs:none-default
diff --git a/docker/docker-compose/cluster-with-etcd.yaml b/docker/docker-compose/cluster-with-etcd.yaml
index 34eee7f4b5..39f421e637 100644
--- a/docker/docker-compose/cluster-with-etcd.yaml
+++ b/docker/docker-compose/cluster-with-etcd.yaml
@@ -85,8 +85,8 @@ services:
command:
- metasrv
- start
- - --rpc-bind-addr=0.0.0.0:3002
- - --rpc-server-addr=metasrv:3002
+ - --grpc-bind-addr=0.0.0.0:3002
+ - --grpc-server-addr=metasrv:3002
- --store-addrs=etcd0:2379
- --http-addr=0.0.0.0:3000
healthcheck:
@@ -111,8 +111,8 @@ services:
- start
- --node-id=0
- --data-home=/greptimedb_data
- - --rpc-bind-addr=0.0.0.0:3001
- - --rpc-server-addr=datanode0:3001
+ - --grpc-bind-addr=0.0.0.0:3001
+ - --grpc-server-addr=datanode0:3001
- --metasrv-addrs=metasrv:3002
- --http-addr=0.0.0.0:5000
volumes:
@@ -141,7 +141,7 @@ services:
- start
- --metasrv-addrs=metasrv:3002
- --http-addr=0.0.0.0:4000
- - --rpc-bind-addr=0.0.0.0:4001
+ - --grpc-bind-addr=0.0.0.0:4001
- --mysql-addr=0.0.0.0:4002
- --postgres-addr=0.0.0.0:4003
healthcheck:
@@ -166,8 +166,8 @@ services:
- start
- --node-id=0
- --metasrv-addrs=metasrv:3002
- - --rpc-bind-addr=0.0.0.0:4004
- - --rpc-server-addr=flownode0:4004
+ - --grpc-bind-addr=0.0.0.0:4004
+ - --grpc-server-addr=flownode0:4004
- --http-addr=0.0.0.0:4005
depends_on:
frontend0:
diff --git a/docs/rfcs/remote-dyn-filter-rfc.md b/docs/rfcs/remote-dyn-filter-rfc.md
new file mode 100644
index 0000000000..5f2243b15c
--- /dev/null
+++ b/docs/rfcs/remote-dyn-filter-rfc.md
@@ -0,0 +1,136 @@
+---
+Feature Name: Remote Dynamic Filter Propagation
+Tracking Issue: N/A
+Date: 2026-04-08
+Author: @discord9
+---
+
+# RFC: Remote Dynamic Filter Propagation
+
+# Summary
+
+This RFC proposes a remote dynamic filter propagation mechanism for distributed queries. It lets frontend-produced dynamic filters reach remote datanode scans through a lightweight control plane, while preserving one rule: remote dynamic filters are an optimization only, never a correctness dependency.
+
+# Motivation
+
+Today, dynamic filters can improve local execution, but they do not automatically propagate to remote datanode scans in distributed queries. As a result, the frontend may already know that a probe-side scan can be narrowed, while the remote scan still runs with a weaker predicate and loses pruning opportunities.
+
+We want a minimal design that:
+
+- propagates dynamic filter updates to remote scans,
+- keeps filter identity and lifecycle stable across register/update/unregister,
+- and safely degrades when encoding, routing, RPC, or apply logic fails.
+
+# Details
+
+The high-level flow is:
+
+1. A join on the frontend produces an alive dynamic filter.
+2. `MergeScanExec` identifies the remote subscribers, generates a stable `filter_id`, and registers the alive filter into a query-scoped registry.
+3. The initial remote read establishes the corresponding registration on the datanode side.
+4. The frontend registry watches for dynamic filter updates via `wait_update` or generation changes.
+5. Later updates and unregister messages are sent through the existing region unary RPC path.
+6. The datanode applies these updates to query-scoped remote filter state and scan wrappers.
+7. Query finish, cancel, or no-consumer conditions trigger unregister and cleanup.
+
+## Identity
+
+The logical identity of a remote dynamic filter is `query_id + filter_id`.
+
+Region and scan metadata are routing information, not part of filter-state identity. `filter_id` only needs to be stable and unique within a single query.
+
+The current recommendation is to derive `filter_id` from:
+
+- `region_id`
+- `producer-local ordinal`
+- `canonicalized children fingerprint`
+
+The following should not be included:
+
+- `partition`
+- transport metadata
+- memory addresses or temporary runtime object ids
+
+## Transport
+
+This design reuses the existing region unary gRPC path:
+
+- `RegionRequest.body.remote_dyn_filter`
+- `RemoteDynFilterRequest.oneof action`
+ - `update`
+ - `unregister`
+
+The initial remote read is responsible for register and scan setup. The unary RPC path is only for later `update` and `unregister` messages.
+
+## Frontend registry
+
+The frontend uses a query-engine runtime map:
+
+- implementation near `src/query/src/dist_plan/remote_dyn_filter_registry.rs`
+- storage model: `query_id -> Arc`
+
+This registry should not live on a single `MergeScanExec`, and it should not be stored in `QueryContext.mutable_session_data`. It is a query execution runtime object that owns watcher tasks, cleanup tail, and fanout state.
+
+The registry lifecycle has three states:
+
+- `Active`: accepts registrations and sends updates
+- `Closing`: query ended; stop new registrations, send final cleanup messages, drain in-flight RPCs
+- `Closed`: watchers stopped, state removable from the runtime map
+
+The registry may outlive the main query execution briefly for cleanup, but it is not intended to be a long-lived global object.
+
+## Propagation policy
+
+Remote dynamic filters should remain a selective optimization, not an automatic fanout for every filter update.
+
+The frontend may skip remote propagation when the encoded filter becomes too large, fanout cost is too high, or the expected pruning benefit is too small. In those cases, execution should continue with local-only dynamic filtering semantics.
+
+## Responsibilities
+
+On the frontend:
+
+- the join produces alive dynamic filters,
+- `MergeScanExec` bridges producers to remote subscribers,
+- the registry watches updates and fans out RPCs.
+
+On the datanode:
+
+- the unary handler receives `update` and `unregister`,
+- query-scoped remote filter state is keyed by `query_id + filter_id`,
+- remote wrappers apply updates through existing predicate and scan refresh paths.
+
+## Failure semantics
+
+All failures must degrade safely:
+
+- encode failure -> local-only filter
+- RPC failure -> log/metric and degrade
+- early update or missing target -> explicit buffer, drop+metric, or retry policy
+- decode or remap failure -> disable remote optimization only
+
+# Alternatives
+
+## Registry on `MergeScanExec`
+
+Rejected because lifecycle and cleanup would become fragmented across multiple bridge or exec instances in the same query.
+
+## Registry in `QueryContext.mutable_session_data`
+
+Rejected because this is the wrong ownership model. The registry is not session metadata; it is a query runtime object with watcher tasks and cleanup behavior.
+
+## Long-lived global manager
+
+Rejected for now because it is heavier than necessary. A query-engine runtime map is sufficient for the current design.
+
+# Drawbacks
+
+- The design introduces extra query runtime state and cleanup logic on both frontend and datanode.
+- The initial version only covers the current minimal filter form and leaves larger membership propagation to later work.
+- A clear policy is still required for updates that arrive before scan registration.
+
+# Unresolved questions
+
+1. Should children fingerprint canonicalization become a shared helper?
+2. What is the strict timing relationship between `is_complete` and final unregister?
+3. Does the runtime map need a background sweep task, or is explicit reap enough?
+4. How should large build-side membership evolve beyond `IN` in later work?
diff --git a/grafana/dashboards/metrics/cluster/dashboard.json b/grafana/dashboards/metrics/cluster/dashboard.json
index 918eb3c4e8..01253bd83b 100644
--- a/grafana/dashboards/metrics/cluster/dashboard.json
+++ b/grafana/dashboards/metrics/cluster/dashboard.json
@@ -25,7 +25,7 @@
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 1,
- "id": 34,
+ "id": 33,
"links": [],
"panels": [
{
@@ -927,7 +927,7 @@
"type": "stat"
},
{
- "collapsed": true,
+ "collapsed": false,
"gridPos": {
"h": 1,
"w": 24,
@@ -935,226 +935,226 @@
"y": 9
},
"id": 275,
- "panels": [
+ "panels": [],
+ "title": "Ingestion",
+ "type": "row"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Total ingestion rate.\n\nHere we listed 3 primary protocols:\n\n- Prometheus remote write\n- Greptime's gRPC API (when using our ingest SDK)\n- Log ingestion http API\n",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "rowsps"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 6,
+ "w": 24,
+ "x": 0,
+ "y": 10
+ },
+ "id": 193,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${metrics}"
},
- "description": "Total ingestion rate.\n\nHere we listed 3 primary protocols:\n\n- Prometheus remote write\n- Greptime's gRPC API (when using our ingest SDK)\n- Log ingestion http API\n",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "rowsps"
- },
- "overrides": []
+ "editorMode": "code",
+ "expr": "sum(rate(greptime_table_operator_ingest_rows{instance=~\"$frontend\"}[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "ingestion",
+ "range": true,
+ "refId": "C"
+ }
+ ],
+ "title": "Total Ingestion Rate",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Total ingestion rate.\n\nHere we listed 3 primary protocols:\n\n- Prometheus remote write\n- Greptime's gRPC API (when using our ingest SDK)\n- Log ingestion http API\n",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
},
- "gridPos": {
- "h": 6,
- "w": 24,
- "x": 0,
- "y": 18
- },
- "id": 193,
- "options": {
- "legend": {
- "calcs": [
- "mean"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
},
- "tooltip": {
- "hideZeros": false,
- "mode": "single",
- "sort": "none"
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
}
},
- "pluginVersion": "12.0.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
},
- "editorMode": "code",
- "expr": "sum(rate(greptime_table_operator_ingest_rows{instance=~\"$frontend\"}[$__rate_interval]))",
- "hide": false,
- "instant": false,
- "legendFormat": "ingestion",
- "range": true,
- "refId": "C"
- }
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "rowsps"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 6,
+ "w": 24,
+ "x": 0,
+ "y": 16
+ },
+ "id": 277,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean"
],
- "title": "Total Ingestion Rate",
- "type": "timeseries"
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum(rate(greptime_servers_http_logs_ingestion_counter[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "http-logs",
+ "range": true,
+ "refId": "http_logs"
},
{
"datasource": {
"type": "prometheus",
"uid": "${metrics}"
},
- "description": "Total ingestion rate.\n\nHere we listed 3 primary protocols:\n\n- Prometheus remote write\n- Greptime's gRPC API (when using our ingest SDK)\n- Log ingestion http API\n",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "barWidthFactor": 0.6,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "rowsps"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 6,
- "w": 24,
- "x": 0,
- "y": 70
- },
- "id": 277,
- "options": {
- "legend": {
- "calcs": [
- "mean"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "12.0.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "sum(rate(greptime_servers_http_logs_ingestion_counter[$__rate_interval]))",
- "hide": false,
- "instant": false,
- "legendFormat": "http-logs",
- "range": true,
- "refId": "http_logs"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "sum(rate(greptime_servers_prometheus_remote_write_samples[$__rate_interval]))",
- "hide": false,
- "instant": false,
- "legendFormat": "prometheus-remote-write",
- "range": true,
- "refId": "prometheus-remote-write"
- }
- ],
- "title": "Ingestion Rate by Type",
- "type": "timeseries"
+ "editorMode": "code",
+ "expr": "sum(rate(greptime_servers_prometheus_remote_write_samples[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "prometheus-remote-write",
+ "range": true,
+ "refId": "prometheus-remote-write"
}
],
- "title": "Ingestion",
- "type": "row"
+ "title": "Ingestion Rate by Type",
+ "type": "timeseries"
},
{
"collapsed": true,
@@ -1162,7 +1162,7 @@
"h": 1,
"w": 24,
"x": 0,
- "y": 10
+ "y": 22
},
"id": 276,
"panels": [
@@ -1231,7 +1231,7 @@
"h": 6,
"w": 24,
"x": 0,
- "y": 19
+ "y": 246
},
"id": 255,
"options": {
@@ -1303,7 +1303,7 @@
"h": 1,
"w": 24,
"x": 0,
- "y": 11
+ "y": 23
},
"id": 274,
"panels": [
@@ -1371,7 +1371,7 @@
"h": 10,
"w": 12,
"x": 0,
- "y": 20
+ "y": 247
},
"id": 256,
"options": {
@@ -1486,7 +1486,7 @@
"h": 10,
"w": 12,
"x": 12,
- "y": 20
+ "y": 247
},
"id": 262,
"options": {
@@ -1603,7 +1603,7 @@
"h": 10,
"w": 12,
"x": 0,
- "y": 30
+ "y": 257
},
"id": 266,
"options": {
@@ -1718,7 +1718,7 @@
"h": 10,
"w": 12,
"x": 12,
- "y": 30
+ "y": 257
},
"id": 268,
"options": {
@@ -1835,7 +1835,7 @@
"h": 10,
"w": 12,
"x": 0,
- "y": 40
+ "y": 267
},
"id": 269,
"options": {
@@ -1950,7 +1950,7 @@
"h": 10,
"w": 12,
"x": 12,
- "y": 40
+ "y": 267
},
"id": 271,
"options": {
@@ -2067,7 +2067,7 @@
"h": 10,
"w": 12,
"x": 0,
- "y": 50
+ "y": 277
},
"id": 272,
"options": {
@@ -2182,7 +2182,7 @@
"h": 10,
"w": 12,
"x": 12,
- "y": 50
+ "y": 277
},
"id": 273,
"options": {
@@ -2245,7 +2245,7 @@
"h": 1,
"w": 24,
"x": 0,
- "y": 12
+ "y": 24
},
"id": 280,
"panels": [
@@ -2314,7 +2314,7 @@
"h": 8,
"w": 12,
"x": 0,
- "y": 61
+ "y": 288
},
"id": 281,
"options": {
@@ -2415,7 +2415,7 @@
"h": 8,
"w": 12,
"x": 12,
- "y": 61
+ "y": 288
},
"id": 282,
"options": {
@@ -2516,7 +2516,7 @@
"h": 8,
"w": 12,
"x": 0,
- "y": 69
+ "y": 296
},
"id": 283,
"options": {
@@ -2617,7 +2617,7 @@
"h": 8,
"w": 12,
"x": 12,
- "y": 69
+ "y": 296
},
"id": 284,
"options": {
@@ -2716,7 +2716,7 @@
"h": 8,
"w": 12,
"x": 0,
- "y": 77
+ "y": 304
},
"id": 285,
"options": {
@@ -2817,7 +2817,7 @@
"h": 8,
"w": 12,
"x": 12,
- "y": 77
+ "y": 304
},
"id": 286,
"options": {
@@ -2919,7 +2919,7 @@
"h": 8,
"w": 12,
"x": 0,
- "y": 85
+ "y": 312
},
"id": 287,
"options": {
@@ -3020,7 +3020,7 @@
"h": 8,
"w": 12,
"x": 12,
- "y": 85
+ "y": 312
},
"id": 288,
"options": {
@@ -3064,7 +3064,7 @@
"h": 1,
"w": 24,
"x": 0,
- "y": 13
+ "y": 25
},
"id": 289,
"panels": [
@@ -3133,7 +3133,7 @@
"h": 6,
"w": 24,
"x": 0,
- "y": 62
+ "y": 289
},
"id": 292,
"options": {
@@ -3234,7 +3234,7 @@
"h": 8,
"w": 12,
"x": 0,
- "y": 68
+ "y": 295
},
"id": 290,
"options": {
@@ -3335,7 +3335,7 @@
"h": 8,
"w": 12,
"x": 12,
- "y": 68
+ "y": 295
},
"id": 291,
"options": {
@@ -3464,7 +3464,7 @@
"h": 8,
"w": 12,
"x": 0,
- "y": 76
+ "y": 303
},
"id": 336,
"options": {
@@ -3518,2519 +3518,2520 @@
"type": "row"
},
{
- "collapsed": false,
+ "collapsed": true,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
- "y": 14
+ "y": 26
},
"id": 293,
- "panels": [],
+ "panels": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Request QPS per Instance.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "ops"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 193
+ },
+ "id": 294,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by(instance, pod, type) (rate(greptime_mito_handle_request_elapsed_count{instance=~\"$datanode\"}[$__rate_interval]))",
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-[{{type}}]",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Request OPS per Instance",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Request P99 per Instance.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "points",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "s"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 193
+ },
+ "id": 295,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.99, sum by(instance, pod, le, type) (rate(greptime_mito_handle_request_elapsed_bucket{instance=~\"$datanode\"}[$__rate_interval])))",
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-[{{type}}]",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Request P99 per Instance",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Write Buffer per Instance.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "decbytes"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 225
+ },
+ "id": 296,
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "greptime_mito_write_buffer_bytes{instance=~\"$datanode\"}",
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Write Buffer per Instance",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Ingestion size by row counts.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "rowsps"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 225
+ },
+ "id": 297,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by (instance, pod) (rate(greptime_mito_write_rows_total{instance=~\"$datanode\"}[$__rate_interval]))",
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Write Rows per Instance",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Flush QPS per Instance.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "ops"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 233
+ },
+ "id": 298,
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by(instance, pod, reason) (rate(greptime_mito_flush_requests_total{instance=~\"$datanode\"}[$__rate_interval]))",
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-[{{reason}}]",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Flush OPS per Instance",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Write Stall per Instance.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ }
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 233
+ },
+ "id": 299,
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by(instance, pod) (greptime_mito_write_stall_total{instance=~\"$datanode\"})",
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Write Stall per Instance",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Read Stage OPS per Instance.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "ops"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 241
+ },
+ "id": 300,
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by(instance, pod) (rate(greptime_mito_read_stage_elapsed_count{instance=~\"$datanode\", stage=\"total\"}[$__rate_interval]))",
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Read Stage OPS per Instance",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Read Stage P99 per Instance.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "points",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "s"
+ },
+ "overrides": [
+ {
+ "__systemRef": "hideSeriesFrom",
+ "matcher": {
+ "id": "byNames",
+ "options": {
+ "mode": "exclude",
+ "names": [
+ "[10.39.145.87:4000]-[mycluster-datanode-1]-[cache_miss_read]"
+ ],
+ "prefix": "All except:",
+ "readOnly": true
+ }
+ },
+ "properties": [
+ {
+ "id": "custom.hideFrom",
+ "value": {
+ "legend": false,
+ "tooltip": false,
+ "viz": true
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 241
+ },
+ "id": 301,
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.99, sum by(instance, pod, le, stage) (rate(greptime_mito_read_stage_elapsed_bucket{instance=~\"$datanode\"}[$__rate_interval])))",
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Read Stage P99 per Instance",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Write Stage P99 per Instance.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "points",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "s"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 249
+ },
+ "id": 302,
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true,
+ "sortBy": "Last *",
+ "sortDesc": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.99, sum by(instance, pod, le, stage) (rate(greptime_mito_write_stage_elapsed_bucket{instance=~\"$datanode\"}[$__rate_interval])))",
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Write Stage P99 per Instance",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Compaction OPS per Instance.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "ops"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 249
+ },
+ "id": 303,
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by(instance, pod) (rate(greptime_mito_compaction_total_elapsed_count{instance=~\"$datanode\"}[$__rate_interval]))",
+ "instant": false,
+ "legendFormat": "[{{ instance }}]-[{{pod}}]",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Compaction OPS per Instance",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Compaction latency by stage",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "points",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "s"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 257
+ },
+ "id": 304,
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.99, sum by(instance, pod, le, stage) (rate(greptime_mito_compaction_stage_elapsed_bucket{instance=~\"$datanode\"}[$__rate_interval])))",
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]-p99",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by(instance, pod, stage) (rate(greptime_mito_compaction_stage_elapsed_sum{instance=~\"$datanode\"}[$__rate_interval]))/sum by(instance, pod, stage) (rate(greptime_mito_compaction_stage_elapsed_count{instance=~\"$datanode\"}[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]-avg",
+ "range": true,
+ "refId": "B"
+ }
+ ],
+ "title": "Compaction Elapsed Time per Instance by Stage",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Compaction P99 per Instance.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "points",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "s"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 257
+ },
+ "id": 305,
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.99, sum by(instance, pod, le,stage) (rate(greptime_mito_compaction_total_elapsed_bucket{instance=~\"$datanode\"}[$__rate_interval])))",
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]-compaction",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Compaction P99 per Instance",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Write-ahead logs write size as bytes. This chart includes stats of p95 and p99 size by instance, total WAL write rate.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "bytes"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 265
+ },
+ "id": 306,
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.95, sum by(le,instance, pod) (rate(raft_engine_write_size_bucket[$__rate_interval])))",
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-req-size-p95",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.99, sum by(le,instance,pod) (rate(raft_engine_write_size_bucket[$__rate_interval])))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-req-size-p99",
+ "range": true,
+ "refId": "B"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by (instance, pod)(rate(raft_engine_write_size_sum[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-throughput",
+ "range": true,
+ "refId": "C"
+ }
+ ],
+ "title": "WAL write size",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Cached Bytes per Instance.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "points",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "decbytes"
+ },
+ "overrides": [
+ {
+ "__systemRef": "hideSeriesFrom",
+ "matcher": {
+ "id": "byNames",
+ "options": {
+ "mode": "exclude",
+ "names": [
+ "[10.39.145.79:4000]-[mycluster-datanode-0]-[file]"
+ ],
+ "prefix": "All except:",
+ "readOnly": true
+ }
+ },
+ "properties": [
+ {
+ "id": "custom.hideFrom",
+ "value": {
+ "legend": false,
+ "tooltip": false,
+ "viz": true
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 265
+ },
+ "id": 307,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "greptime_mito_cache_bytes{instance=~\"$datanode\"}",
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-[{{type}}]",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Cached Bytes per Instance",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Ongoing compaction task count",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "points",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "none"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 273
+ },
+ "id": 308,
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "greptime_mito_inflight_compaction_count",
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Inflight Compaction",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Raft engine (local disk) log store sync latency, p99",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "points",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "s"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 273
+ },
+ "id": 310,
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.99, sum by(le, type, node, instance, pod) (rate(raft_engine_sync_log_duration_seconds_bucket[$__rate_interval])))",
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-p99",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "WAL sync duration seconds",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Write-ahead log operations latency at p99",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "points",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "s"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 281
+ },
+ "id": 311,
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.99, sum by(le,logstore,optype,instance, pod) (rate(greptime_logstore_op_elapsed_bucket[$__rate_interval])))",
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-[{{logstore}}]-[{{optype}}]-p99",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Log Store op duration seconds",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Ongoing flush task count",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "points",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "none"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 281
+ },
+ "id": 312,
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "greptime_mito_inflight_flush_count",
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Inflight Flush",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Compaction oinput output bytes",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "bytes"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 289
+ },
+ "id": 335,
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by(instance, pod) (greptime_mito_compaction_input_bytes)",
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-input",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by(instance, pod) (greptime_mito_compaction_output_bytes)",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-output",
+ "range": true,
+ "refId": "B"
+ }
+ ],
+ "title": "Compaction Input/Output Bytes",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Per-stage elapsed time for region worker to handle bulk insert region requests.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "points",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "s"
+ },
+ "overrides": [
+ {
+ "__systemRef": "hideSeriesFrom",
+ "matcher": {
+ "id": "byNames",
+ "options": {
+ "mode": "exclude",
+ "names": [
+ "[192.168.50.8:4000]-[]-[prepare_bulk_request]-AVG",
+ "[minipc-1:4000]-[]-[bulk_extend]-AVG",
+ "[minipc-1:4000]-[]-[prepare_bulk_request]-AVG",
+ "[minipc-1:4000]-[]-[process_bulk_req]-AVG",
+ "[minipc-1:4000]-[]-[write_bulk]-AVG"
+ ],
+ "prefix": "All except:",
+ "readOnly": true
+ }
+ },
+ "properties": [
+ {
+ "id": "custom.hideFrom",
+ "value": {
+ "legend": false,
+ "tooltip": false,
+ "viz": true
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 289
+ },
+ "id": 337,
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.95, sum by(le,instance, stage, pod) (rate(greptime_region_worker_handle_write_bucket[$__rate_interval])))",
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]-P95",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by(instance, stage, pod) (rate(greptime_region_worker_handle_write_sum[$__rate_interval]))/sum by(instance, stage, pod) (rate(greptime_region_worker_handle_write_count[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]-AVG",
+ "range": true,
+ "refId": "B"
+ }
+ ],
+ "title": "Region Worker Handle Bulk Insert Requests",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Compaction oinput output bytes",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "none"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 297
+ },
+ "id": 348,
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by(instance, pod) (greptime_mito_memtable_active_series_count)",
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-series",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by(instance, pod) (greptime_mito_memtable_field_builder_count)",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-field_builders",
+ "range": true,
+ "refId": "B"
+ }
+ ],
+ "title": "Active Series and Field Builders Count",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Per-stage elapsed time for region worker to decode requests.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "points",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "s"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 297
+ },
+ "id": 338,
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "disableTextWrap": false,
+ "editorMode": "builder",
+ "expr": "histogram_quantile(0.95, sum by(le, instance, stage, pod) (rate(greptime_datanode_convert_region_request_bucket[$__rate_interval])))",
+ "fullMetaSearch": false,
+ "includeNullMetadata": true,
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]-P95",
+ "range": true,
+ "refId": "A",
+ "useBackend": false
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by(le,instance, stage, pod) (rate(greptime_datanode_convert_region_request_sum[$__rate_interval]))/sum by(le,instance, stage, pod) (rate(greptime_datanode_convert_region_request_count[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]-AVG",
+ "range": true,
+ "refId": "B"
+ }
+ ],
+ "title": "Region Worker Convert Requests",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "The local cache miss of the datanode.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ }
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 305
+ },
+ "id": 349,
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by (instance,pod, type) (rate(greptime_mito_cache_miss{instance=~\"$datanode\"}[$__rate_interval]))",
+ "legendFormat": "[{{instance}}]-[{{pod}}]-[{{type}}]",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Cache Miss",
+ "type": "timeseries"
+ }
+ ],
"title": "Mito Engine",
"type": "row"
},
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Request QPS per Instance.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "barWidthFactor": 0.6,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "ops"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 15
- },
- "id": 294,
- "options": {
- "legend": {
- "calcs": [
- "mean"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "multi",
- "sort": "desc"
- }
- },
- "pluginVersion": "11.6.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "sum by(instance, pod, type) (rate(greptime_mito_handle_request_elapsed_count{instance=~\"$datanode\"}[$__rate_interval]))",
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]-[{{type}}]",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Request OPS per Instance",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Request P99 per Instance.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "barWidthFactor": 0.6,
- "drawStyle": "points",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "s"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 15
- },
- "id": 295,
- "options": {
- "legend": {
- "calcs": [
- "mean"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "multi",
- "sort": "desc"
- }
- },
- "pluginVersion": "11.6.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.99, sum by(instance, pod, le, type) (rate(greptime_mito_handle_request_elapsed_bucket{instance=~\"$datanode\"}[$__rate_interval])))",
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]-[{{type}}]",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Request P99 per Instance",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Write Buffer per Instance.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "barWidthFactor": 0.6,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "decbytes"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 23
- },
- "id": 296,
- "options": {
- "legend": {
- "calcs": [
- "lastNotNull"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "multi",
- "sort": "desc"
- }
- },
- "pluginVersion": "11.6.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "greptime_mito_write_buffer_bytes{instance=~\"$datanode\"}",
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Write Buffer per Instance",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Ingestion size by row counts.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "barWidthFactor": 0.6,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "rowsps"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 23
- },
- "id": 297,
- "options": {
- "legend": {
- "calcs": [
- "mean"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "11.6.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "sum by (instance, pod) (rate(greptime_mito_write_rows_total{instance=~\"$datanode\"}[$__rate_interval]))",
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Write Rows per Instance",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Flush QPS per Instance.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "barWidthFactor": 0.6,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "ops"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 31
- },
- "id": 298,
- "options": {
- "legend": {
- "calcs": [
- "lastNotNull"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "multi",
- "sort": "desc"
- }
- },
- "pluginVersion": "11.6.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "sum by(instance, pod, reason) (rate(greptime_mito_flush_requests_total{instance=~\"$datanode\"}[$__rate_interval]))",
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]-[{{reason}}]",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Flush OPS per Instance",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Write Stall per Instance.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "barWidthFactor": 0.6,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- }
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 31
- },
- "id": 299,
- "options": {
- "legend": {
- "calcs": [
- "lastNotNull"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "multi",
- "sort": "desc"
- }
- },
- "pluginVersion": "11.6.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "sum by(instance, pod) (greptime_mito_write_stall_total{instance=~\"$datanode\"})",
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Write Stall per Instance",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Read Stage OPS per Instance.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "barWidthFactor": 0.6,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "ops"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 39
- },
- "id": 300,
- "options": {
- "legend": {
- "calcs": [
- "lastNotNull"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "multi",
- "sort": "desc"
- }
- },
- "pluginVersion": "11.6.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "sum by(instance, pod) (rate(greptime_mito_read_stage_elapsed_count{instance=~\"$datanode\", stage=\"total\"}[$__rate_interval]))",
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Read Stage OPS per Instance",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Read Stage P99 per Instance.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "barWidthFactor": 0.6,
- "drawStyle": "points",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "s"
- },
- "overrides": [
- {
- "__systemRef": "hideSeriesFrom",
- "matcher": {
- "id": "byNames",
- "options": {
- "mode": "exclude",
- "names": [
- "[10.39.145.87:4000]-[mycluster-datanode-1]-[cache_miss_read]"
- ],
- "prefix": "All except:",
- "readOnly": true
- }
- },
- "properties": [
- {
- "id": "custom.hideFrom",
- "value": {
- "legend": false,
- "tooltip": false,
- "viz": true
- }
- }
- ]
- }
- ]
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 39
- },
- "id": 301,
- "options": {
- "legend": {
- "calcs": [
- "lastNotNull"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "multi",
- "sort": "desc"
- }
- },
- "pluginVersion": "11.6.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.99, sum by(instance, pod, le, stage) (rate(greptime_mito_read_stage_elapsed_bucket{instance=~\"$datanode\"}[$__rate_interval])))",
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Read Stage P99 per Instance",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Write Stage P99 per Instance.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "barWidthFactor": 0.6,
- "drawStyle": "points",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "s"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 47
- },
- "id": 302,
- "options": {
- "legend": {
- "calcs": [
- "lastNotNull"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true,
- "sortBy": "Last *",
- "sortDesc": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "multi",
- "sort": "desc"
- }
- },
- "pluginVersion": "11.6.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.99, sum by(instance, pod, le, stage) (rate(greptime_mito_write_stage_elapsed_bucket{instance=~\"$datanode\"}[$__rate_interval])))",
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Write Stage P99 per Instance",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Compaction OPS per Instance.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "barWidthFactor": 0.6,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "ops"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 47
- },
- "id": 303,
- "options": {
- "legend": {
- "calcs": [
- "lastNotNull"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "multi",
- "sort": "desc"
- }
- },
- "pluginVersion": "11.6.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "sum by(instance, pod) (rate(greptime_mito_compaction_total_elapsed_count{instance=~\"$datanode\"}[$__rate_interval]))",
- "instant": false,
- "legendFormat": "[{{ instance }}]-[{{pod}}]",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Compaction OPS per Instance",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Compaction latency by stage",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "barWidthFactor": 0.6,
- "drawStyle": "points",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "s"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 55
- },
- "id": 304,
- "options": {
- "legend": {
- "calcs": [
- "lastNotNull"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "11.6.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.99, sum by(instance, pod, le, stage) (rate(greptime_mito_compaction_stage_elapsed_bucket{instance=~\"$datanode\"}[$__rate_interval])))",
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]-p99",
- "range": true,
- "refId": "A"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "sum by(instance, pod, stage) (rate(greptime_mito_compaction_stage_elapsed_sum{instance=~\"$datanode\"}[$__rate_interval]))/sum by(instance, pod, stage) (rate(greptime_mito_compaction_stage_elapsed_count{instance=~\"$datanode\"}[$__rate_interval]))",
- "hide": false,
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]-avg",
- "range": true,
- "refId": "B"
- }
- ],
- "title": "Compaction Elapsed Time per Instance by Stage",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Compaction P99 per Instance.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "barWidthFactor": 0.6,
- "drawStyle": "points",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "s"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 55
- },
- "id": 305,
- "options": {
- "legend": {
- "calcs": [
- "lastNotNull"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "multi",
- "sort": "desc"
- }
- },
- "pluginVersion": "11.6.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.99, sum by(instance, pod, le,stage) (rate(greptime_mito_compaction_total_elapsed_bucket{instance=~\"$datanode\"}[$__rate_interval])))",
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]-compaction",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Compaction P99 per Instance",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Write-ahead logs write size as bytes. This chart includes stats of p95 and p99 size by instance, total WAL write rate.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "barWidthFactor": 0.6,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "bytes"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 63
- },
- "id": 306,
- "options": {
- "legend": {
- "calcs": [
- "lastNotNull"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "11.6.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.95, sum by(le,instance, pod) (rate(raft_engine_write_size_bucket[$__rate_interval])))",
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]-req-size-p95",
- "range": true,
- "refId": "A"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.99, sum by(le,instance,pod) (rate(raft_engine_write_size_bucket[$__rate_interval])))",
- "hide": false,
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]-req-size-p99",
- "range": true,
- "refId": "B"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "sum by (instance, pod)(rate(raft_engine_write_size_sum[$__rate_interval]))",
- "hide": false,
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]-throughput",
- "range": true,
- "refId": "C"
- }
- ],
- "title": "WAL write size",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Cached Bytes per Instance.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "barWidthFactor": 0.6,
- "drawStyle": "points",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "decbytes"
- },
- "overrides": [
- {
- "__systemRef": "hideSeriesFrom",
- "matcher": {
- "id": "byNames",
- "options": {
- "mode": "exclude",
- "names": [
- "[10.39.145.79:4000]-[mycluster-datanode-0]-[file]"
- ],
- "prefix": "All except:",
- "readOnly": true
- }
- },
- "properties": [
- {
- "id": "custom.hideFrom",
- "value": {
- "legend": false,
- "tooltip": false,
- "viz": true
- }
- }
- ]
- }
- ]
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 63
- },
- "id": 307,
- "options": {
- "legend": {
- "calcs": [],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "multi",
- "sort": "desc"
- }
- },
- "pluginVersion": "11.6.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "greptime_mito_cache_bytes{instance=~\"$datanode\"}",
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]-[{{type}}]",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Cached Bytes per Instance",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Ongoing compaction task count",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "barWidthFactor": 0.6,
- "drawStyle": "points",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "none"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 71
- },
- "id": 308,
- "options": {
- "legend": {
- "calcs": [
- "lastNotNull"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "11.6.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "greptime_mito_inflight_compaction_count",
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Inflight Compaction",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Raft engine (local disk) log store sync latency, p99",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "barWidthFactor": 0.6,
- "drawStyle": "points",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "s"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 71
- },
- "id": 310,
- "options": {
- "legend": {
- "calcs": [
- "lastNotNull"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "11.6.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.99, sum by(le, type, node, instance, pod) (rate(raft_engine_sync_log_duration_seconds_bucket[$__rate_interval])))",
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]-p99",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "WAL sync duration seconds",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Write-ahead log operations latency at p99",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "barWidthFactor": 0.6,
- "drawStyle": "points",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "s"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 79
- },
- "id": 311,
- "options": {
- "legend": {
- "calcs": [
- "lastNotNull"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "11.6.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.99, sum by(le,logstore,optype,instance, pod) (rate(greptime_logstore_op_elapsed_bucket[$__rate_interval])))",
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]-[{{logstore}}]-[{{optype}}]-p99",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Log Store op duration seconds",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Ongoing flush task count",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "barWidthFactor": 0.6,
- "drawStyle": "points",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "none"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 79
- },
- "id": 312,
- "options": {
- "legend": {
- "calcs": [
- "lastNotNull"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "11.6.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "greptime_mito_inflight_flush_count",
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Inflight Flush",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Compaction oinput output bytes",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "barWidthFactor": 0.6,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "bytes"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 87
- },
- "id": 335,
- "options": {
- "legend": {
- "calcs": [
- "lastNotNull"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "11.6.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "sum by(instance, pod) (greptime_mito_compaction_input_bytes)",
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]-input",
- "range": true,
- "refId": "A"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "sum by(instance, pod) (greptime_mito_compaction_output_bytes)",
- "hide": false,
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]-output",
- "range": true,
- "refId": "B"
- }
- ],
- "title": "Compaction Input/Output Bytes",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Per-stage elapsed time for region worker to handle bulk insert region requests.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "barWidthFactor": 0.6,
- "drawStyle": "points",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "s"
- },
- "overrides": [
- {
- "__systemRef": "hideSeriesFrom",
- "matcher": {
- "id": "byNames",
- "options": {
- "mode": "exclude",
- "names": [
- "[192.168.50.8:4000]-[]-[prepare_bulk_request]-AVG",
- "[minipc-1:4000]-[]-[bulk_extend]-AVG",
- "[minipc-1:4000]-[]-[prepare_bulk_request]-AVG",
- "[minipc-1:4000]-[]-[process_bulk_req]-AVG",
- "[minipc-1:4000]-[]-[write_bulk]-AVG"
- ],
- "prefix": "All except:",
- "readOnly": true
- }
- },
- "properties": [
- {
- "id": "custom.hideFrom",
- "value": {
- "legend": false,
- "tooltip": false,
- "viz": true
- }
- }
- ]
- }
- ]
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 87
- },
- "id": 337,
- "options": {
- "legend": {
- "calcs": [
- "lastNotNull"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "11.6.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.95, sum by(le,instance, stage, pod) (rate(greptime_region_worker_handle_write_bucket[$__rate_interval])))",
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]-P95",
- "range": true,
- "refId": "A"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "sum by(instance, stage, pod) (rate(greptime_region_worker_handle_write_sum[$__rate_interval]))/sum by(instance, stage, pod) (rate(greptime_region_worker_handle_write_count[$__rate_interval]))",
- "hide": false,
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]-AVG",
- "range": true,
- "refId": "B"
- }
- ],
- "title": "Region Worker Handle Bulk Insert Requests",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Compaction oinput output bytes",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "barWidthFactor": 0.6,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "none"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 95
- },
- "id": 348,
- "options": {
- "legend": {
- "calcs": [
- "lastNotNull"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "11.6.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "sum by(instance, pod) (greptime_mito_memtable_active_series_count)",
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]-series",
- "range": true,
- "refId": "A"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "sum by(instance, pod) (greptime_mito_memtable_field_builder_count)",
- "hide": false,
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]-field_builders",
- "range": true,
- "refId": "B"
- }
- ],
- "title": "Active Series and Field Builders Count",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Per-stage elapsed time for region worker to decode requests.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "barWidthFactor": 0.6,
- "drawStyle": "points",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "s"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 95
- },
- "id": 338,
- "options": {
- "legend": {
- "calcs": [
- "lastNotNull"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "11.6.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "disableTextWrap": false,
- "editorMode": "builder",
- "expr": "histogram_quantile(0.95, sum by(le, instance, stage, pod) (rate(greptime_datanode_convert_region_request_bucket[$__rate_interval])))",
- "fullMetaSearch": false,
- "includeNullMetadata": true,
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]-P95",
- "range": true,
- "refId": "A",
- "useBackend": false
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "sum by(le,instance, stage, pod) (rate(greptime_datanode_convert_region_request_sum[$__rate_interval]))/sum by(le,instance, stage, pod) (rate(greptime_datanode_convert_region_request_count[$__rate_interval]))",
- "hide": false,
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]-AVG",
- "range": true,
- "refId": "B"
- }
- ],
- "title": "Region Worker Convert Requests",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "The local cache miss of the datanode.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "barWidthFactor": 0.6,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- }
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 103
- },
- "id": 349,
- "options": {
- "legend": {
- "calcs": [
- "lastNotNull"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "11.6.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "sum by (instance,pod, type) (rate(greptime_mito_cache_miss{instance=~\"$datanode\"}[$__rate_interval]))",
- "legendFormat": "[{{instance}}]-[{{pod}}]-[{{type}}]",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Cache Miss",
- "type": "timeseries"
- },
{
"collapsed": true,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
- "y": 111
+ "y": 27
},
"id": 313,
"panels": [
@@ -6098,7 +6099,7 @@
"h": 10,
"w": 24,
"x": 0,
- "y": 232
+ "y": 459
},
"id": 314,
"options": {
@@ -6198,7 +6199,7 @@
"h": 7,
"w": 12,
"x": 0,
- "y": 242
+ "y": 469
},
"id": 315,
"options": {
@@ -6298,7 +6299,7 @@
"h": 7,
"w": 12,
"x": 12,
- "y": 242
+ "y": 469
},
"id": 316,
"options": {
@@ -6398,7 +6399,7 @@
"h": 7,
"w": 12,
"x": 0,
- "y": 249
+ "y": 476
},
"id": 317,
"options": {
@@ -6498,7 +6499,7 @@
"h": 7,
"w": 12,
"x": 12,
- "y": 249
+ "y": 476
},
"id": 318,
"options": {
@@ -6598,7 +6599,7 @@
"h": 7,
"w": 12,
"x": 0,
- "y": 256
+ "y": 483
},
"id": 319,
"options": {
@@ -6699,7 +6700,7 @@
"h": 7,
"w": 12,
"x": 12,
- "y": 256
+ "y": 483
},
"id": 320,
"options": {
@@ -6800,7 +6801,7 @@
"h": 7,
"w": 12,
"x": 0,
- "y": 263
+ "y": 490
},
"id": 321,
"options": {
@@ -6900,7 +6901,7 @@
"h": 7,
"w": 12,
"x": 12,
- "y": 263
+ "y": 490
},
"id": 322,
"options": {
@@ -7000,7 +7001,7 @@
"h": 7,
"w": 12,
"x": 0,
- "y": 270
+ "y": 497
},
"id": 323,
"options": {
@@ -7103,7 +7104,7 @@
"h": 7,
"w": 12,
"x": 12,
- "y": 270
+ "y": 497
},
"id": 334,
"options": {
@@ -7144,1827 +7145,1829 @@
"type": "row"
},
{
- "collapsed": false,
+ "collapsed": true,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
- "y": 112
+ "y": 28
},
"id": 351,
- "panels": [],
+ "panels": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Triggered region flush total",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "none"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 195
+ },
+ "id": 350,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "meta_triggered_region_flush_total",
+ "instant": false,
+ "legendFormat": "{{pod}}-{{topic_name}}",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Triggered region flush total",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Triggered region checkpoint total",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "none"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 195
+ },
+ "id": 352,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "meta_triggered_region_checkpoint_total",
+ "instant": false,
+ "legendFormat": "{{pod}}-{{topic_name}}",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Triggered region checkpoint total",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Topic estimated max replay size",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "bytes"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 227
+ },
+ "id": 353,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "meta_topic_estimated_replay_size",
+ "instant": false,
+ "legendFormat": "{{pod}}-{{topic_name}}",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Topic estimated replay size",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Kafka logstore's bytes traffic",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "bytes"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 227
+ },
+ "id": 354,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "rate(greptime_logstore_kafka_client_bytes_total[$__rate_interval])",
+ "instant": false,
+ "legendFormat": "{{pod}}-{{logstore}}",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Kafka logstore's bytes traffic",
+ "type": "timeseries"
+ }
+ ],
"title": "Remote WAL",
"type": "row"
},
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Triggered region flush total",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "barWidthFactor": 0.6,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "none"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 113
- },
- "id": 350,
- "options": {
- "legend": {
- "calcs": [],
- "displayMode": "list",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "11.6.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "meta_triggered_region_flush_total",
- "instant": false,
- "legendFormat": "{{pod}}-{{topic_name}}",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Triggered region flush total",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Triggered region checkpoint total",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "barWidthFactor": 0.6,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "none"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 113
- },
- "id": 352,
- "options": {
- "legend": {
- "calcs": [],
- "displayMode": "list",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "11.6.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "meta_triggered_region_checkpoint_total",
- "instant": false,
- "legendFormat": "{{pod}}-{{topic_name}}",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Triggered region checkpoint total",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Topic estimated max replay size",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "barWidthFactor": 0.6,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "bytes"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 121
- },
- "id": 353,
- "options": {
- "legend": {
- "calcs": [],
- "displayMode": "list",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "11.6.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "meta_topic_estimated_replay_size",
- "instant": false,
- "legendFormat": "{{pod}}-{{topic_name}}",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Topic estimated replay size",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Kafka logstore's bytes traffic",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "barWidthFactor": 0.6,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "bytes"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 121
- },
- "id": 354,
- "options": {
- "legend": {
- "calcs": [],
- "displayMode": "list",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "11.6.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "rate(greptime_logstore_kafka_client_bytes_total[$__rate_interval])",
- "instant": false,
- "legendFormat": "{{pod}}-{{logstore}}",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Kafka logstore's bytes traffic",
- "type": "timeseries"
- },
- {
- "collapsed": false,
- "gridPos": {
- "h": 1,
- "w": 24,
- "x": 0,
- "y": 129
- },
- "id": 324,
- "panels": [],
- "title": "Metasrv",
- "type": "row"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Counter of region migration by source and destination",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "thresholds"
- },
- "custom": {
- "axisPlacement": "auto",
- "fillOpacity": 70,
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "lineWidth": 1
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- }
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 130
- },
- "id": 325,
- "options": {
- "colWidth": 0.9,
- "legend": {
- "displayMode": "list",
- "placement": "bottom",
- "showLegend": false
- },
- "rowHeight": 0.9,
- "showValue": "auto",
- "tooltip": {
- "hideZeros": false,
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "11.6.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "greptime_meta_region_migration_stat{datanode_type=\"src\"}",
- "instant": false,
- "legendFormat": "from-datanode-{{datanode_id}}",
- "range": true,
- "refId": "A"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "greptime_meta_region_migration_stat{datanode_type=\"desc\"}",
- "hide": false,
- "instant": false,
- "legendFormat": "to-datanode-{{datanode_id}}",
- "range": true,
- "refId": "B"
- }
- ],
- "title": "Region migration datanode",
- "type": "status-history"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Counter of region migration error",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "barWidthFactor": 0.6,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "none"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 130
- },
- "id": 326,
- "options": {
- "legend": {
- "calcs": [],
- "displayMode": "list",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "11.6.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "greptime_meta_region_migration_error",
- "instant": false,
- "legendFormat": "{{pod}}-{{state}}-{{error_type}}",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Region migration error",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Gauge of load information of each datanode, collected via heartbeat between datanode and metasrv. This information is for metasrv to schedule workloads.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "barWidthFactor": 0.6,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "binBps"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 138
- },
- "id": 327,
- "options": {
- "legend": {
- "calcs": [],
- "displayMode": "list",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "11.6.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "greptime_datanode_load",
- "instant": false,
- "legendFormat": "Datanode-{{datanode_id}}-writeload",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Datanode load",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Displays the rate of SQL executions processed by the Meta service using the RDS backend.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "barWidthFactor": 0.6,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "none"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 138
- },
- "id": 339,
- "options": {
- "legend": {
- "calcs": [],
- "displayMode": "list",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "11.6.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "rate(greptime_meta_rds_pg_sql_execute_elapsed_ms_count[$__rate_interval])",
- "instant": false,
- "legendFormat": "{{pod}} {{op}} {{type}} {{result}} ",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Rate of SQL Executions (RDS)",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Measures the response time of SQL executions via the RDS backend. ",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "barWidthFactor": 0.6,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "ms"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 146
- },
- "id": 340,
- "options": {
- "legend": {
- "calcs": [],
- "displayMode": "list",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "11.6.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "disableTextWrap": false,
- "editorMode": "code",
- "expr": "histogram_quantile(0.90, sum by(pod, op, type, result, le) (rate(greptime_meta_rds_pg_sql_execute_elapsed_ms_bucket[$__rate_interval])))",
- "fullMetaSearch": false,
- "includeNullMetadata": true,
- "instant": false,
- "legendFormat": "{{pod}} {{op}} {{type}} {{result}} p90",
- "range": true,
- "refId": "A",
- "useBackend": false
- }
- ],
- "title": "SQL Execution Latency (RDS)",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Shows latency of Meta handlers by pod and handler name, useful for monitoring handler performance and detecting latency spikes.\n",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "barWidthFactor": 0.6,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "s"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 146
- },
- "id": 341,
- "options": {
- "legend": {
- "calcs": [],
- "displayMode": "list",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "11.6.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "disableTextWrap": false,
- "editorMode": "code",
- "expr": "histogram_quantile(0.90, sum by(pod, le, name) (\n rate(greptime_meta_handler_execute_bucket[$__rate_interval])\n))",
- "fullMetaSearch": false,
- "includeNullMetadata": true,
- "instant": false,
- "legendFormat": "{{pod}} {{name}} p90",
- "range": true,
- "refId": "A",
- "useBackend": false
- }
- ],
- "title": "Handler Execution Latency",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Shows p90 heartbeat message sizes, helping track network usage and identify anomalies in heartbeat payload.\n",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "barWidthFactor": 0.6,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "bytes"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 154
- },
- "id": 342,
- "options": {
- "legend": {
- "calcs": [],
- "displayMode": "list",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "11.6.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "disableTextWrap": false,
- "editorMode": "code",
- "expr": "histogram_quantile(0.9, sum by(pod, le) (greptime_meta_heartbeat_stat_memory_size_bucket))",
- "fullMetaSearch": false,
- "includeNullMetadata": true,
- "instant": false,
- "legendFormat": "{{pod}}",
- "range": true,
- "refId": "A",
- "useBackend": false
- }
- ],
- "title": "Heartbeat Packet Size",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Gauge of load information of each datanode, collected via heartbeat between datanode and metasrv. This information is for metasrv to schedule workloads.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "barWidthFactor": 0.6,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "s"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 154
- },
- "id": 345,
- "options": {
- "legend": {
- "calcs": [],
- "displayMode": "list",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "11.6.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "disableTextWrap": false,
- "editorMode": "code",
- "expr": "rate(greptime_meta_heartbeat_rate[$__rate_interval])",
- "fullMetaSearch": false,
- "includeNullMetadata": true,
- "instant": false,
- "legendFormat": "{{pod}}",
- "range": true,
- "refId": "A",
- "useBackend": false
- }
- ],
- "title": "Meta Heartbeat Receive Rate",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Gauge of load information of each datanode, collected via heartbeat between datanode and metasrv. This information is for metasrv to schedule workloads.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "barWidthFactor": 0.6,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "s"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 162
- },
- "id": 343,
- "options": {
- "legend": {
- "calcs": [],
- "displayMode": "list",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "11.6.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "disableTextWrap": false,
- "editorMode": "code",
- "expr": "histogram_quantile(0.99, sum by(pod, le, op, target) (greptime_meta_kv_request_elapsed_bucket))",
- "fullMetaSearch": false,
- "includeNullMetadata": true,
- "instant": false,
- "legendFormat": "{{pod}}-{{op}} p99",
- "range": true,
- "refId": "A",
- "useBackend": false
- }
- ],
- "title": "Meta KV Ops Latency",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Gauge of load information of each datanode, collected via heartbeat between datanode and metasrv. This information is for metasrv to schedule workloads.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "barWidthFactor": 0.6,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "none"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 162
- },
- "id": 346,
- "options": {
- "legend": {
- "calcs": [],
- "displayMode": "list",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "11.6.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "disableTextWrap": false,
- "editorMode": "code",
- "expr": "rate(greptime_meta_kv_request_elapsed_count[$__rate_interval])",
- "fullMetaSearch": false,
- "includeNullMetadata": true,
- "instant": false,
- "legendFormat": "{{pod}}-{{op}} p99",
- "range": true,
- "refId": "A",
- "useBackend": false
- }
- ],
- "title": "Rate of meta KV Ops",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Gauge of load information of each datanode, collected via heartbeat between datanode and metasrv. This information is for metasrv to schedule workloads.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "barWidthFactor": 0.6,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "s"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 170
- },
- "id": 347,
- "options": {
- "legend": {
- "calcs": [],
- "displayMode": "list",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "11.6.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "disableTextWrap": false,
- "editorMode": "code",
- "expr": "histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_create_tables_bucket))",
- "fullMetaSearch": false,
- "includeNullMetadata": true,
- "instant": false,
- "legendFormat": "CreateLogicalTables-{{step}} p90",
- "range": true,
- "refId": "A",
- "useBackend": false
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "disableTextWrap": false,
- "editorMode": "code",
- "expr": "histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_create_table))",
- "fullMetaSearch": false,
- "hide": false,
- "includeNullMetadata": true,
- "instant": false,
- "legendFormat": "CreateTable-{{step}} p90",
- "range": true,
- "refId": "B",
- "useBackend": false
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "disableTextWrap": false,
- "editorMode": "code",
- "expr": "histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_create_view))",
- "fullMetaSearch": false,
- "hide": false,
- "includeNullMetadata": true,
- "instant": false,
- "legendFormat": "CreateView-{{step}} p90",
- "range": true,
- "refId": "C",
- "useBackend": false
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "disableTextWrap": false,
- "editorMode": "code",
- "expr": "histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_create_flow))",
- "fullMetaSearch": false,
- "hide": false,
- "includeNullMetadata": true,
- "instant": false,
- "legendFormat": "CreateFlow-{{step}} p90",
- "range": true,
- "refId": "D",
- "useBackend": false
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "disableTextWrap": false,
- "editorMode": "code",
- "expr": "histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_drop_table))",
- "fullMetaSearch": false,
- "hide": false,
- "includeNullMetadata": true,
- "instant": false,
- "legendFormat": "DropTable-{{step}} p90",
- "range": true,
- "refId": "E",
- "useBackend": false
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "disableTextWrap": false,
- "editorMode": "code",
- "expr": "histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_alter_table))",
- "fullMetaSearch": false,
- "hide": false,
- "includeNullMetadata": true,
- "instant": false,
- "legendFormat": "AlterTable-{{step}} p90",
- "range": true,
- "refId": "F",
- "useBackend": false
- }
- ],
- "title": "DDL Latency",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Reconciliation stats",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "barWidthFactor": 0.6,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "s"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 170
- },
- "id": 355,
- "options": {
- "legend": {
- "calcs": [],
- "displayMode": "list",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "11.6.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "greptime_meta_reconciliation_stats",
- "hide": false,
- "instant": false,
- "legendFormat": "{{pod}}-{{table_type}}-{{type}}",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Reconciliation stats",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Elapsed of Reconciliation steps",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "barWidthFactor": 0.6,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "s"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 178
- },
- "id": 356,
- "options": {
- "legend": {
- "calcs": [],
- "displayMode": "list",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "11.6.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.9, greptime_meta_reconciliation_procedure_bucket)",
- "hide": false,
- "instant": false,
- "legendFormat": "{{procedure_name}}-{{step}}-P90",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Reconciliation steps",
- "type": "timeseries"
- },
{
"collapsed": true,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
- "y": 186
+ "y": 29
+ },
+ "id": 324,
+ "panels": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Counter of region migration by source and destination",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "custom": {
+ "axisPlacement": "auto",
+ "fillOpacity": 70,
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineWidth": 1
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ }
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 196
+ },
+ "id": 325,
+ "options": {
+ "colWidth": 0.9,
+ "legend": {
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": false
+ },
+ "rowHeight": 0.9,
+ "showValue": "auto",
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "greptime_meta_region_migration_stat{datanode_type=\"src\"}",
+ "instant": false,
+ "legendFormat": "from-datanode-{{datanode_id}}",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "greptime_meta_region_migration_stat{datanode_type=\"desc\"}",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "to-datanode-{{datanode_id}}",
+ "range": true,
+ "refId": "B"
+ }
+ ],
+ "title": "Region migration datanode",
+ "type": "status-history"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Counter of region migration error",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "none"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 196
+ },
+ "id": 326,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "greptime_meta_region_migration_error",
+ "instant": false,
+ "legendFormat": "{{pod}}-{{state}}-{{error_type}}",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Region migration error",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Gauge of load information of each datanode, collected via heartbeat between datanode and metasrv. This information is for metasrv to schedule workloads.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "binBps"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 228
+ },
+ "id": 327,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "greptime_datanode_load",
+ "instant": false,
+ "legendFormat": "Datanode-{{datanode_id}}-writeload",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Datanode load",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Displays the rate of SQL executions processed by the Meta service using the RDS backend.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "none"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 228
+ },
+ "id": 339,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "rate(greptime_meta_rds_pg_sql_execute_elapsed_ms_count[$__rate_interval])",
+ "instant": false,
+ "legendFormat": "{{pod}} {{op}} {{type}} {{result}} ",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Rate of SQL Executions (RDS)",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Measures the response time of SQL executions via the RDS backend. ",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "ms"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 236
+ },
+ "id": 340,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "disableTextWrap": false,
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.90, sum by(pod, op, type, result, le) (rate(greptime_meta_rds_pg_sql_execute_elapsed_ms_bucket[$__rate_interval])))",
+ "fullMetaSearch": false,
+ "includeNullMetadata": true,
+ "instant": false,
+ "legendFormat": "{{pod}} {{op}} {{type}} {{result}} p90",
+ "range": true,
+ "refId": "A",
+ "useBackend": false
+ }
+ ],
+ "title": "SQL Execution Latency (RDS)",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Shows latency of Meta handlers by pod and handler name, useful for monitoring handler performance and detecting latency spikes.\n",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "s"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 236
+ },
+ "id": 341,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "disableTextWrap": false,
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.90, sum by(pod, le, name) (\n rate(greptime_meta_handler_execute_bucket[$__rate_interval])\n))",
+ "fullMetaSearch": false,
+ "includeNullMetadata": true,
+ "instant": false,
+ "legendFormat": "{{pod}} {{name}} p90",
+ "range": true,
+ "refId": "A",
+ "useBackend": false
+ }
+ ],
+ "title": "Handler Execution Latency",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Shows p90 heartbeat message sizes, helping track network usage and identify anomalies in heartbeat payload.\n",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "bytes"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 244
+ },
+ "id": 342,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "disableTextWrap": false,
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.9, sum by(pod, le) (greptime_meta_heartbeat_stat_memory_size_bucket))",
+ "fullMetaSearch": false,
+ "includeNullMetadata": true,
+ "instant": false,
+ "legendFormat": "{{pod}}",
+ "range": true,
+ "refId": "A",
+ "useBackend": false
+ }
+ ],
+ "title": "Heartbeat Packet Size",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Gauge of load information of each datanode, collected via heartbeat between datanode and metasrv. This information is for metasrv to schedule workloads.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "s"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 244
+ },
+ "id": 345,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "disableTextWrap": false,
+ "editorMode": "code",
+ "expr": "rate(greptime_meta_heartbeat_rate[$__rate_interval])",
+ "fullMetaSearch": false,
+ "includeNullMetadata": true,
+ "instant": false,
+ "legendFormat": "{{pod}}",
+ "range": true,
+ "refId": "A",
+ "useBackend": false
+ }
+ ],
+ "title": "Meta Heartbeat Receive Rate",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Gauge of load information of each datanode, collected via heartbeat between datanode and metasrv. This information is for metasrv to schedule workloads.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "s"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 252
+ },
+ "id": 343,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "disableTextWrap": false,
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.99, sum by(pod, le, op, target) (greptime_meta_kv_request_elapsed_bucket))",
+ "fullMetaSearch": false,
+ "includeNullMetadata": true,
+ "instant": false,
+ "legendFormat": "{{pod}}-{{op}} p99",
+ "range": true,
+ "refId": "A",
+ "useBackend": false
+ }
+ ],
+ "title": "Meta KV Ops Latency",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Gauge of load information of each datanode, collected via heartbeat between datanode and metasrv. This information is for metasrv to schedule workloads.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "none"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 252
+ },
+ "id": 346,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "disableTextWrap": false,
+ "editorMode": "code",
+ "expr": "rate(greptime_meta_kv_request_elapsed_count[$__rate_interval])",
+ "fullMetaSearch": false,
+ "includeNullMetadata": true,
+ "instant": false,
+ "legendFormat": "{{pod}}-{{op}} p99",
+ "range": true,
+ "refId": "A",
+ "useBackend": false
+ }
+ ],
+ "title": "Rate of meta KV Ops",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Gauge of load information of each datanode, collected via heartbeat between datanode and metasrv. This information is for metasrv to schedule workloads.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "s"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 260
+ },
+ "id": 347,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "disableTextWrap": false,
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_create_tables_bucket))",
+ "fullMetaSearch": false,
+ "includeNullMetadata": true,
+ "instant": false,
+ "legendFormat": "CreateLogicalTables-{{step}} p90",
+ "range": true,
+ "refId": "A",
+ "useBackend": false
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "disableTextWrap": false,
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_create_table))",
+ "fullMetaSearch": false,
+ "hide": false,
+ "includeNullMetadata": true,
+ "instant": false,
+ "legendFormat": "CreateTable-{{step}} p90",
+ "range": true,
+ "refId": "B",
+ "useBackend": false
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "disableTextWrap": false,
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_create_view))",
+ "fullMetaSearch": false,
+ "hide": false,
+ "includeNullMetadata": true,
+ "instant": false,
+ "legendFormat": "CreateView-{{step}} p90",
+ "range": true,
+ "refId": "C",
+ "useBackend": false
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "disableTextWrap": false,
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_create_flow))",
+ "fullMetaSearch": false,
+ "hide": false,
+ "includeNullMetadata": true,
+ "instant": false,
+ "legendFormat": "CreateFlow-{{step}} p90",
+ "range": true,
+ "refId": "D",
+ "useBackend": false
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "disableTextWrap": false,
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_drop_table))",
+ "fullMetaSearch": false,
+ "hide": false,
+ "includeNullMetadata": true,
+ "instant": false,
+ "legendFormat": "DropTable-{{step}} p90",
+ "range": true,
+ "refId": "E",
+ "useBackend": false
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "disableTextWrap": false,
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_alter_table))",
+ "fullMetaSearch": false,
+ "hide": false,
+ "includeNullMetadata": true,
+ "instant": false,
+ "legendFormat": "AlterTable-{{step}} p90",
+ "range": true,
+ "refId": "F",
+ "useBackend": false
+ }
+ ],
+ "title": "DDL Latency",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Reconciliation stats",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "s"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 260
+ },
+ "id": 355,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "greptime_meta_reconciliation_stats",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "{{pod}}-{{table_type}}-{{type}}",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Reconciliation stats",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Elapsed of Reconciliation steps",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "s"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 268
+ },
+ "id": 356,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.9, greptime_meta_reconciliation_procedure_bucket)",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "{{procedure_name}}-{{step}}-P90",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Reconciliation steps",
+ "type": "timeseries"
+ }
+ ],
+ "title": "Metasrv",
+ "type": "row"
+ },
+ {
+ "collapsed": true,
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 30
},
"id": 328,
"panels": [
@@ -9031,7 +9034,7 @@
"h": 8,
"w": 12,
"x": 0,
- "y": 3055
+ "y": 3282
},
"id": 329,
"options": {
@@ -9126,7 +9129,7 @@
"h": 8,
"w": 12,
"x": 12,
- "y": 3055
+ "y": 3282
},
"id": 330,
"options": {
@@ -9234,7 +9237,7 @@
"h": 8,
"w": 9,
"x": 0,
- "y": 3063
+ "y": 3290
},
"id": 331,
"options": {
@@ -9342,7 +9345,7 @@
"h": 8,
"w": 9,
"x": 9,
- "y": 3063
+ "y": 3290
},
"id": 332,
"options": {
@@ -9437,7 +9440,7 @@
"h": 8,
"w": 6,
"x": 18,
- "y": 3063
+ "y": 3290
},
"id": 333,
"options": {
@@ -9474,12 +9477,12 @@
"type": "row"
},
{
- "collapsed": true,
+ "collapsed": false,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
- "y": 187
+ "y": 31
},
"id": 357,
"panels": [],
@@ -9550,7 +9553,7 @@
"h": 8,
"w": 24,
"x": 0,
- "y": 188
+ "y": 32
},
"id": 358,
"options": {
@@ -9648,7 +9651,7 @@
"h": 8,
"w": 12,
"x": 0,
- "y": 196
+ "y": 40
},
"id": 359,
"options": {
@@ -9761,7 +9764,7 @@
"h": 8,
"w": 12,
"x": 12,
- "y": 196
+ "y": 40
},
"id": 360,
"options": {
@@ -9861,7 +9864,7 @@
"h": 8,
"w": 12,
"x": 0,
- "y": 204
+ "y": 48
},
"id": 361,
"options": {
@@ -9974,7 +9977,7 @@
"h": 8,
"w": 12,
"x": 12,
- "y": 204
+ "y": 48
},
"id": 364,
"options": {
@@ -10074,7 +10077,7 @@
"h": 8,
"w": 12,
"x": 0,
- "y": 212
+ "y": 56
},
"id": 363,
"options": {
@@ -10187,7 +10190,7 @@
"h": 8,
"w": 12,
"x": 12,
- "y": 212
+ "y": 56
},
"id": 362,
"options": {
@@ -10221,6 +10224,1330 @@
],
"title": "Save Alert Failure Rate",
"type": "timeseries"
+ },
+ {
+ "collapsed": false,
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 64
+ },
+ "id": 366,
+ "panels": [],
+ "title": "Hotspot",
+ "type": "row"
+ },
+ {
+ "datasource": {
+ "type": "mysql",
+ "uid": "${information_schema}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "custom": {
+ "align": "auto",
+ "cellOptions": {
+ "type": "auto",
+ "wrapText": false
+ },
+ "inspect": false
+ },
+ "fieldMinMax": false,
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ }
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "disk_size"
+ },
+ "properties": [
+ {
+ "id": "unit",
+ "value": "bytes"
+ }
+ ]
+ }
+ ]
+ },
+ "gridPos": {
+ "h": 18,
+ "w": 24,
+ "x": 0,
+ "y": 65
+ },
+ "id": 365,
+ "options": {
+ "cellHeight": "sm",
+ "footer": {
+ "countRows": false,
+ "enablePagination": true,
+ "fields": [
+ "region_rows",
+ "disk_size"
+ ],
+ "reducer": [
+ "sum"
+ ],
+ "show": true
+ },
+ "showHeader": true,
+ "sortBy": []
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "dataset": "information_schema",
+ "datasource": {
+ "type": "mysql",
+ "uid": "${information_schema}"
+ },
+ "editorMode": "code",
+ "format": "table",
+ "rawQuery": true,
+ "rawSql": "WITH table_stats AS (\n SELECT\n table_id,\n COUNT(*) AS region_count,\n SUM(disk_size) AS total_disk_size,\n SUM(region_rows) as total_region_rows\n FROM information_schema.region_statistics\n WHERE region_role = 'Leader'\n GROUP BY table_id\n HAVING COUNT(*) > 1\n)\n\nSELECT\n t.table_schema,\n t.table_name,\n\n r.region_id,\n t.table_id,\n r.region_number,\n\n p.partition_description,\n\n\n ROUND(\n r.disk_size * 100.0\n / NULLIF(ts.total_disk_size, 0),\n 2\n ) AS disk_size_share_percent,\n\n r.disk_size,\n \n ROUND(\n r.region_rows * 100.0\n / NULLIF(ts.total_region_rows, 0),\n 2\n ) AS region_rows_share_percent,\n r.region_rows\n\nFROM information_schema.region_statistics r\n\nJOIN table_stats ts\n ON r.table_id = ts.table_id\n\nJOIN information_schema.tables t\n ON r.table_id = t.table_id\n\nLEFT JOIN information_schema.partitions p\n ON p.table_schema = t.table_schema\n AND p.table_name = t.table_name\n AND p.greptime_partition_id = r.region_id\n\nWHERE r.region_role = 'Leader'\n\nORDER BY region_rows_share_percent DESC\nLIMIT 100;",
+ "refId": "A",
+ "sql": {
+ "columns": [
+ {
+ "parameters": [],
+ "type": "function"
+ }
+ ],
+ "groupBy": [
+ {
+ "property": {
+ "type": "string"
+ },
+ "type": "groupBy"
+ }
+ ],
+ "limit": 50
+ }
+ }
+ ],
+ "title": "Hotspot Regions",
+ "type": "table"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Write load of each datanode over time.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "fieldMinMax": false,
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "binBps"
+ },
+ "overrides": [
+ {
+ "__systemRef": "hideSeriesFrom",
+ "matcher": {
+ "id": "byNames",
+ "options": {
+ "mode": "exclude",
+ "names": [
+ "total_region_rows",
+ "total_written_bytes_since_open",
+ "total_disk_size",
+ "total_sst_size",
+ "total_sst_num",
+ "total_memtable_size",
+ "total_manifest_size",
+ "total_index_size",
+ "cluster_disk_ratio_percent",
+ "cluster_sst_ratio_percent",
+ "cluster_rows_ratio_percent",
+ "max_region_disk_size",
+ "max_region_rows",
+ "leader_region_count",
+ "disk_ratio_percent",
+ "data_size"
+ ],
+ "prefix": "All except:",
+ "readOnly": true
+ }
+ },
+ "properties": []
+ }
+ ]
+ },
+ "gridPos": {
+ "h": 11,
+ "w": 12,
+ "x": 0,
+ "y": 83
+ },
+ "id": 368,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "disableTextWrap": false,
+ "editorMode": "builder",
+ "expr": "greptime_datanode_history_load",
+ "fullMetaSearch": false,
+ "includeNullMetadata": true,
+ "instant": false,
+ "legendFormat": "datanode-{{datanode_id}}({{instance}})",
+ "range": true,
+ "refId": "A",
+ "useBackend": false
+ }
+ ],
+ "title": "Datanode Load(Write)",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Distribution of write load across datanodes.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ }
+ },
+ "fieldMinMax": false,
+ "mappings": [],
+ "min": 0,
+ "unit": "binBps"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 11,
+ "w": 12,
+ "x": 12,
+ "y": 83
+ },
+ "id": 369,
+ "options": {
+ "displayLabels": [
+ "percent"
+ ],
+ "legend": {
+ "displayMode": "list",
+ "placement": "right",
+ "showLegend": true
+ },
+ "pieType": "donut",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "disableTextWrap": false,
+ "editorMode": "builder",
+ "exemplar": false,
+ "expr": "greptime_datanode_history_load",
+ "format": "time_series",
+ "fullMetaSearch": false,
+ "includeNullMetadata": true,
+ "instant": true,
+ "legendFormat": "datanode-{{datanode_id}}({{instance}})",
+ "range": false,
+ "refId": "A",
+ "useBackend": false
+ }
+ ],
+ "title": "Datanode Load(Write) Distribution",
+ "type": "piechart"
+ },
+ {
+ "datasource": {
+ "type": "mysql",
+ "uid": "${information_schema}"
+ },
+ "description": "Distribution of leader regions and data size across datanodes.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ }
+ },
+ "fieldMinMax": false,
+ "mappings": [],
+ "unit": "bytes"
+ },
+ "overrides": [
+ {
+ "__systemRef": "hideSeriesFrom",
+ "matcher": {
+ "id": "byNames",
+ "options": {
+ "mode": "exclude",
+ "names": [
+ "total_region_rows",
+ "total_written_bytes_since_open",
+ "total_disk_size",
+ "total_sst_size",
+ "total_sst_num",
+ "total_memtable_size",
+ "total_manifest_size",
+ "total_index_size",
+ "cluster_disk_ratio_percent",
+ "cluster_sst_ratio_percent",
+ "cluster_rows_ratio_percent",
+ "max_region_disk_size",
+ "max_region_rows",
+ "leader_region_count",
+ "disk_ratio_percent",
+ "data_size"
+ ],
+ "prefix": "All except:",
+ "readOnly": true
+ }
+ },
+ "properties": [
+ {
+ "id": "custom.hideFrom",
+ "value": {
+ "legend": false,
+ "tooltip": false,
+ "viz": true
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "gridPos": {
+ "h": 11,
+ "w": 12,
+ "x": 0,
+ "y": 94
+ },
+ "id": 367,
+ "options": {
+ "displayLabels": [
+ "value"
+ ],
+ "legend": {
+ "displayMode": "list",
+ "placement": "right",
+ "showLegend": true,
+ "values": [
+ "percent"
+ ]
+ },
+ "pieType": "pie",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "/^data_size$/",
+ "values": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "dataset": "information_schema",
+ "datasource": {
+ "type": "mysql",
+ "uid": "${information_schema}"
+ },
+ "editorMode": "code",
+ "format": "table",
+ "rawQuery": true,
+ "rawSql": "WITH leader_regions AS (\n SELECT\n CONCAT(\n 'datanode-',\n p.peer_id,\n ' (',\n p.peer_addr,\n ')'\n ) AS datanode,\n r.disk_size\n FROM information_schema.region_statistics r\n JOIN information_schema.region_peers p\n ON r.region_id = p.region_id\n WHERE r.region_role = 'Leader'\n AND p.is_leader = 'Yes'\n)\n\nSELECT\n datanode,\n COUNT(*) AS leader_region_count,\n SUM(disk_size) AS data_size\nFROM leader_regions\nGROUP BY datanode\nORDER BY data_size DESC;",
+ "refId": "A",
+ "sql": {
+ "columns": [
+ {
+ "parameters": [],
+ "type": "function"
+ }
+ ],
+ "groupBy": [
+ {
+ "property": {
+ "type": "string"
+ },
+ "type": "groupBy"
+ }
+ ],
+ "limit": 50
+ }
+ }
+ ],
+ "title": "Datanode Data Distribution",
+ "type": "piechart"
+ },
+ {
+ "collapsed": false,
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 105
+ },
+ "id": 370,
+ "panels": [],
+ "title": "Autopilot",
+ "type": "row"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Region balancer action count",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 106
+ },
+ "id": 371,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by (result) (\n changes(greptime_region_balancer_actions_total[$__rate_interval])\n)",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "{{result}}",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Region Balancer Actions",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Region balancer gate stop count by gate and reason",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 106
+ },
+ "id": 372,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by (gate, reason) (changes(greptime_region_balancer_gate_stop_total[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "{{gate}} / {{reason}}",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Region Balancer Gate Stops",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Region balancer datanode count by state",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 114
+ },
+ "id": 373,
+ "options": {
+ "colorMode": "value",
+ "graphMode": "area",
+ "justifyMode": "auto",
+ "orientation": "auto",
+ "percentChangeColorMode": "standard",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "showPercentChange": false,
+ "textMode": "auto",
+ "wideLayout": true
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by (state) (greptime_region_balancer_datanodes)",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "{{state}}",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Region Balancer Datanodes",
+ "type": "stat"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Region balancer region count by state",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 114
+ },
+ "id": 374,
+ "options": {
+ "colorMode": "value",
+ "graphMode": "area",
+ "justifyMode": "auto",
+ "orientation": "auto",
+ "percentChangeColorMode": "standard",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "showPercentChange": false,
+ "textMode": "auto",
+ "wideLayout": true
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by (state) (greptime_region_balancer_regions)",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "{{state}}",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Region Balancer Regions",
+ "type": "stat"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Region balancer datanode stability statistics by state",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ }
+ ]
+ },
+ "unit": "binBps"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byRegexp",
+ "options": ".*count$"
+ },
+ "properties": [
+ {
+ "id": "unit",
+ "value": "short"
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byRegexp",
+ "options": ".*ratio$"
+ },
+ "properties": [
+ {
+ "id": "unit",
+ "value": "percentunit"
+ }
+ ]
+ }
+ ]
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 24,
+ "x": 0,
+ "y": 122
+ },
+ "id": 375,
+ "options": {
+ "colorMode": "value",
+ "graphMode": "area",
+ "justifyMode": "auto",
+ "orientation": "auto",
+ "percentChangeColorMode": "standard",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "showPercentChange": false,
+ "textMode": "auto",
+ "wideLayout": true
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "builder",
+ "expr": "sum by (state) (greptime_region_balancer_datanode_stability)",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "{{state}}",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Region Balancer Datanode Stability",
+ "type": "stat"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Auto repartition action count by result",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": -1,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineStyle": {
+ "fill": "solid"
+ },
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 130
+ },
+ "id": 376,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by (result) (changes(greptime_auto_repartition_actions_total[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "{{result}}",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Auto Repartition Actions",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Auto repartition gate stop count by gate and reason",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 130
+ },
+ "id": 377,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by (gate, reason) (changes(greptime_auto_repartition_gate_stop_total[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "{{gate}} / {{reason}}",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Auto Repartition Gate Stops",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Auto repartition sampling elapsed time by stage",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "points",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "s"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 138
+ },
+ "id": 378,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.99, sum by (le, stage) (rate(greptime_auto_repartition_sampling_elapsed_bucket[$__rate_interval])))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "{{stage}}",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Auto Repartition Sampling P99",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Auto repartition executor elapsed time by stage",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "points",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "smooth",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "s"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 138
+ },
+ "id": 379,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.99, sum by (le, stage) (rate(greptime_auto_repartition_executor_elapsed_bucket[$__rate_interval])))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "{{stage}}",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Auto Repartition Executor P99",
+ "type": "timeseries"
}
],
"preload": false,
@@ -10359,7 +11686,7 @@
},
"timepicker": {},
"timezone": "",
- "title": "GreptimeDB",
- "uid": "dejf3k5e7g2kgb",
- "version": 15
-}
+ "title": "GreptimeDB Copy",
+ "uid": "dflfbxbwvvchsa",
+ "version": 67
+}
\ No newline at end of file
diff --git a/grafana/dashboards/metrics/cluster/dashboard.md b/grafana/dashboards/metrics/cluster/dashboard.md
index c0bc49770b..2204c6d0a7 100644
--- a/grafana/dashboards/metrics/cluster/dashboard.md
+++ b/grafana/dashboards/metrics/cluster/dashboard.md
@@ -142,3 +142,101 @@
rate(greptime_trigger_save_alert_record_elapsed_bucket[$__rate_interval])
)` | `timeseries` | Elapsed time to persist trigger alert records. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{storage_type}}]-p99` |
| Save Alert Failure Rate | `rate(greptime_trigger_save_alert_record_failure_count[$__rate_interval])` | `timeseries` | Rate of failures when persisting trigger alert records. | `prometheus` | `none` | `__auto` |
+# Hotspot
+| Title | Query | Type | Description | Datasource | Unit | Legend Format |
+| --- | --- | --- | --- | --- | --- | --- |
+| Hotspot Regions | `WITH table_stats AS (
+ SELECT
+ table_id,
+ COUNT(*) AS region_count,
+ SUM(disk_size) AS total_disk_size,
+ SUM(region_rows) as total_region_rows
+ FROM information_schema.region_statistics
+ WHERE region_role = 'Leader'
+ GROUP BY table_id
+ HAVING COUNT(*) > 1
+)
+
+SELECT
+ t.table_schema,
+ t.table_name,
+
+ r.region_id,
+ t.table_id,
+ r.region_number,
+
+ p.partition_description,
+
+
+ ROUND(
+ r.disk_size * 100.0
+ / NULLIF(ts.total_disk_size, 0),
+ 2
+ ) AS disk_size_share_percent,
+
+ r.disk_size,
+
+ ROUND(
+ r.region_rows * 100.0
+ / NULLIF(ts.total_region_rows, 0),
+ 2
+ ) AS region_rows_share_percent,
+ r.region_rows
+
+FROM information_schema.region_statistics r
+
+JOIN table_stats ts
+ ON r.table_id = ts.table_id
+
+JOIN information_schema.tables t
+ ON r.table_id = t.table_id
+
+LEFT JOIN information_schema.partitions p
+ ON p.table_schema = t.table_schema
+ AND p.table_name = t.table_name
+ AND p.greptime_partition_id = r.region_id
+
+WHERE r.region_role = 'Leader'
+
+ORDER BY region_rows_share_percent DESC
+LIMIT 100;` | `table` | | `mysql` | -- | -- |
+| Datanode Load(Write) | `greptime_datanode_history_load` | `timeseries` | Write load of each datanode over time. | `prometheus` | `binBps` | `datanode-{{datanode_id}}({{instance}})` |
+| Datanode Load(Write) Distribution | `greptime_datanode_history_load` | `piechart` | Distribution of write load across datanodes. | `prometheus` | `binBps` | `datanode-{{datanode_id}}({{instance}})` |
+| Datanode Data Distribution | `WITH leader_regions AS (
+ SELECT
+ CONCAT(
+ 'datanode-',
+ p.peer_id,
+ ' (',
+ p.peer_addr,
+ ')'
+ ) AS datanode,
+ r.disk_size
+ FROM information_schema.region_statistics r
+ JOIN information_schema.region_peers p
+ ON r.region_id = p.region_id
+ WHERE r.region_role = 'Leader'
+ AND p.is_leader = 'Yes'
+)
+
+SELECT
+ datanode,
+ COUNT(*) AS leader_region_count,
+ SUM(disk_size) AS data_size
+FROM leader_regions
+GROUP BY datanode
+ORDER BY data_size DESC;` | `piechart` | Distribution of leader regions and data size across datanodes. | `mysql` | `bytes` | -- |
+# Autopilot
+| Title | Query | Type | Description | Datasource | Unit | Legend Format |
+| --- | --- | --- | --- | --- | --- | --- |
+| Region Balancer Actions | `sum by (result) (
+ changes(greptime_region_balancer_actions_total[$__rate_interval])
+)` | `timeseries` | Region balancer action count | `prometheus` | `short` | `{{result}}` |
+| Region Balancer Gate Stops | `sum by (gate, reason) (changes(greptime_region_balancer_gate_stop_total[$__rate_interval]))` | `timeseries` | Region balancer gate stop count by gate and reason | `prometheus` | `short` | `{{gate}} / {{reason}}` |
+| Region Balancer Datanodes | `sum by (state) (greptime_region_balancer_datanodes)` | `stat` | Region balancer datanode count by state | `prometheus` | `short` | `{{state}}` |
+| Region Balancer Regions | `sum by (state) (greptime_region_balancer_regions)` | `stat` | Region balancer region count by state | `prometheus` | `short` | `{{state}}` |
+| Region Balancer Datanode Stability | `sum by (state) (greptime_region_balancer_datanode_stability)` | `stat` | Region balancer datanode stability statistics by state | `prometheus` | `binBps` | `{{state}}` |
+| Auto Repartition Actions | `sum by (result) (changes(greptime_auto_repartition_actions_total[$__rate_interval]))` | `timeseries` | Auto repartition action count by result | `prometheus` | `short` | `{{result}}` |
+| Auto Repartition Gate Stops | `sum by (gate, reason) (changes(greptime_auto_repartition_gate_stop_total[$__rate_interval]))` | `timeseries` | Auto repartition gate stop count by gate and reason | `prometheus` | `short` | `{{gate}} / {{reason}}` |
+| Auto Repartition Sampling P99 | `histogram_quantile(0.99, sum by (le, stage) (rate(greptime_auto_repartition_sampling_elapsed_bucket[$__rate_interval])))` | `timeseries` | Auto repartition sampling elapsed time by stage | `prometheus` | `s` | `{{stage}}` |
+| Auto Repartition Executor P99 | `histogram_quantile(0.99, sum by (le, stage) (rate(greptime_auto_repartition_executor_elapsed_bucket[$__rate_interval])))` | `timeseries` | Auto repartition executor elapsed time by stage | `prometheus` | `s` | `{{stage}}` |
diff --git a/grafana/dashboards/metrics/cluster/dashboard.yaml b/grafana/dashboards/metrics/cluster/dashboard.yaml
index 934cf58c0c..ca8f88267f 100644
--- a/grafana/dashboards/metrics/cluster/dashboard.yaml
+++ b/grafana/dashboards/metrics/cluster/dashboard.yaml
@@ -1153,3 +1153,160 @@ groups:
type: prometheus
uid: ${metrics}
legendFormat: __auto
+ - title: Hotspot
+ panels:
+ - title: Hotspot Regions
+ type: table
+ queries:
+ - expr: "WITH table_stats AS (\n SELECT\n table_id,\n COUNT(*) AS region_count,\n SUM(disk_size) AS total_disk_size,\n SUM(region_rows) as total_region_rows\n FROM information_schema.region_statistics\n WHERE region_role = 'Leader'\n GROUP BY table_id\n HAVING COUNT(*) > 1\n)\n\nSELECT\n t.table_schema,\n t.table_name,\n\n r.region_id,\n t.table_id,\n r.region_number,\n\n p.partition_description,\n\n\n ROUND(\n r.disk_size * 100.0\n / NULLIF(ts.total_disk_size, 0),\n 2\n ) AS disk_size_share_percent,\n\n r.disk_size,\n \n ROUND(\n r.region_rows * 100.0\n / NULLIF(ts.total_region_rows, 0),\n 2\n ) AS region_rows_share_percent,\n r.region_rows\n\nFROM information_schema.region_statistics r\n\nJOIN table_stats ts\n ON r.table_id = ts.table_id\n\nJOIN information_schema.tables t\n ON r.table_id = t.table_id\n\nLEFT JOIN information_schema.partitions p\n ON p.table_schema = t.table_schema\n AND p.table_name = t.table_name\n AND p.greptime_partition_id = r.region_id\n\nWHERE r.region_role = 'Leader'\n\nORDER BY region_rows_share_percent DESC\nLIMIT 100;"
+ datasource:
+ type: mysql
+ uid: ${information_schema}
+ - title: Datanode Load(Write)
+ type: timeseries
+ description: Write load of each datanode over time.
+ unit: binBps
+ queries:
+ - expr: greptime_datanode_history_load
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: datanode-{{datanode_id}}({{instance}})
+ - title: Datanode Load(Write) Distribution
+ type: piechart
+ description: Distribution of write load across datanodes.
+ unit: binBps
+ queries:
+ - expr: greptime_datanode_history_load
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: datanode-{{datanode_id}}({{instance}})
+ - title: Datanode Data Distribution
+ type: piechart
+ description: Distribution of leader regions and data size across datanodes.
+ unit: bytes
+ queries:
+ - expr: |-
+ WITH leader_regions AS (
+ SELECT
+ CONCAT(
+ 'datanode-',
+ p.peer_id,
+ ' (',
+ p.peer_addr,
+ ')'
+ ) AS datanode,
+ r.disk_size
+ FROM information_schema.region_statistics r
+ JOIN information_schema.region_peers p
+ ON r.region_id = p.region_id
+ WHERE r.region_role = 'Leader'
+ AND p.is_leader = 'Yes'
+ )
+
+ SELECT
+ datanode,
+ COUNT(*) AS leader_region_count,
+ SUM(disk_size) AS data_size
+ FROM leader_regions
+ GROUP BY datanode
+ ORDER BY data_size DESC;
+ datasource:
+ type: mysql
+ uid: ${information_schema}
+ - title: Autopilot
+ panels:
+ - title: Region Balancer Actions
+ type: timeseries
+ description: Region balancer action count
+ unit: short
+ queries:
+ - expr: |-
+ sum by (result) (
+ changes(greptime_region_balancer_actions_total[$__rate_interval])
+ )
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: '{{result}}'
+ - title: Region Balancer Gate Stops
+ type: timeseries
+ description: Region balancer gate stop count by gate and reason
+ unit: short
+ queries:
+ - expr: sum by (gate, reason) (changes(greptime_region_balancer_gate_stop_total[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: '{{gate}} / {{reason}}'
+ - title: Region Balancer Datanodes
+ type: stat
+ description: Region balancer datanode count by state
+ unit: short
+ queries:
+ - expr: sum by (state) (greptime_region_balancer_datanodes)
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: '{{state}}'
+ - title: Region Balancer Regions
+ type: stat
+ description: Region balancer region count by state
+ unit: short
+ queries:
+ - expr: sum by (state) (greptime_region_balancer_regions)
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: '{{state}}'
+ - title: Region Balancer Datanode Stability
+ type: stat
+ description: Region balancer datanode stability statistics by state
+ unit: binBps
+ queries:
+ - expr: sum by (state) (greptime_region_balancer_datanode_stability)
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: '{{state}}'
+ - title: Auto Repartition Actions
+ type: timeseries
+ description: Auto repartition action count by result
+ unit: short
+ queries:
+ - expr: sum by (result) (changes(greptime_auto_repartition_actions_total[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: '{{result}}'
+ - title: Auto Repartition Gate Stops
+ type: timeseries
+ description: Auto repartition gate stop count by gate and reason
+ unit: short
+ queries:
+ - expr: sum by (gate, reason) (changes(greptime_auto_repartition_gate_stop_total[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: '{{gate}} / {{reason}}'
+ - title: Auto Repartition Sampling P99
+ type: timeseries
+ description: Auto repartition sampling elapsed time by stage
+ unit: s
+ queries:
+ - expr: histogram_quantile(0.99, sum by (le, stage) (rate(greptime_auto_repartition_sampling_elapsed_bucket[$__rate_interval])))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: '{{stage}}'
+ - title: Auto Repartition Executor P99
+ type: timeseries
+ description: Auto repartition executor elapsed time by stage
+ unit: s
+ queries:
+ - expr: histogram_quantile(0.99, sum by (le, stage) (rate(greptime_auto_repartition_executor_elapsed_bucket[$__rate_interval])))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: '{{stage}}'
diff --git a/grafana/dashboards/metrics/standalone/dashboard.json b/grafana/dashboards/metrics/standalone/dashboard.json
index 28f38f9dd7..c8313ecb7a 100644
--- a/grafana/dashboards/metrics/standalone/dashboard.json
+++ b/grafana/dashboards/metrics/standalone/dashboard.json
@@ -25,7 +25,7 @@
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 1,
- "id": 34,
+ "id": 33,
"links": [],
"panels": [
{
@@ -927,7 +927,7 @@
"type": "stat"
},
{
- "collapsed": true,
+ "collapsed": false,
"gridPos": {
"h": 1,
"w": 24,
@@ -935,226 +935,226 @@
"y": 9
},
"id": 275,
- "panels": [
+ "panels": [],
+ "title": "Ingestion",
+ "type": "row"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Total ingestion rate.\n\nHere we listed 3 primary protocols:\n\n- Prometheus remote write\n- Greptime's gRPC API (when using our ingest SDK)\n- Log ingestion http API\n",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "rowsps"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 6,
+ "w": 24,
+ "x": 0,
+ "y": 10
+ },
+ "id": 193,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${metrics}"
},
- "description": "Total ingestion rate.\n\nHere we listed 3 primary protocols:\n\n- Prometheus remote write\n- Greptime's gRPC API (when using our ingest SDK)\n- Log ingestion http API\n",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "rowsps"
- },
- "overrides": []
+ "editorMode": "code",
+ "expr": "sum(rate(greptime_table_operator_ingest_rows{}[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "ingestion",
+ "range": true,
+ "refId": "C"
+ }
+ ],
+ "title": "Total Ingestion Rate",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Total ingestion rate.\n\nHere we listed 3 primary protocols:\n\n- Prometheus remote write\n- Greptime's gRPC API (when using our ingest SDK)\n- Log ingestion http API\n",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
},
- "gridPos": {
- "h": 6,
- "w": 24,
- "x": 0,
- "y": 18
- },
- "id": 193,
- "options": {
- "legend": {
- "calcs": [
- "mean"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
},
- "tooltip": {
- "hideZeros": false,
- "mode": "single",
- "sort": "none"
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
}
},
- "pluginVersion": "12.0.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
},
- "editorMode": "code",
- "expr": "sum(rate(greptime_table_operator_ingest_rows{}[$__rate_interval]))",
- "hide": false,
- "instant": false,
- "legendFormat": "ingestion",
- "range": true,
- "refId": "C"
- }
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "rowsps"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 6,
+ "w": 24,
+ "x": 0,
+ "y": 16
+ },
+ "id": 277,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean"
],
- "title": "Total Ingestion Rate",
- "type": "timeseries"
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum(rate(greptime_servers_http_logs_ingestion_counter[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "http-logs",
+ "range": true,
+ "refId": "http_logs"
},
{
"datasource": {
"type": "prometheus",
"uid": "${metrics}"
},
- "description": "Total ingestion rate.\n\nHere we listed 3 primary protocols:\n\n- Prometheus remote write\n- Greptime's gRPC API (when using our ingest SDK)\n- Log ingestion http API\n",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "barWidthFactor": 0.6,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "rowsps"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 6,
- "w": 24,
- "x": 0,
- "y": 70
- },
- "id": 277,
- "options": {
- "legend": {
- "calcs": [
- "mean"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "12.0.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "sum(rate(greptime_servers_http_logs_ingestion_counter[$__rate_interval]))",
- "hide": false,
- "instant": false,
- "legendFormat": "http-logs",
- "range": true,
- "refId": "http_logs"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "sum(rate(greptime_servers_prometheus_remote_write_samples[$__rate_interval]))",
- "hide": false,
- "instant": false,
- "legendFormat": "prometheus-remote-write",
- "range": true,
- "refId": "prometheus-remote-write"
- }
- ],
- "title": "Ingestion Rate by Type",
- "type": "timeseries"
+ "editorMode": "code",
+ "expr": "sum(rate(greptime_servers_prometheus_remote_write_samples[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "prometheus-remote-write",
+ "range": true,
+ "refId": "prometheus-remote-write"
}
],
- "title": "Ingestion",
- "type": "row"
+ "title": "Ingestion Rate by Type",
+ "type": "timeseries"
},
{
"collapsed": true,
@@ -1162,7 +1162,7 @@
"h": 1,
"w": 24,
"x": 0,
- "y": 10
+ "y": 22
},
"id": 276,
"panels": [
@@ -1231,7 +1231,7 @@
"h": 6,
"w": 24,
"x": 0,
- "y": 19
+ "y": 246
},
"id": 255,
"options": {
@@ -1303,7 +1303,7 @@
"h": 1,
"w": 24,
"x": 0,
- "y": 11
+ "y": 23
},
"id": 274,
"panels": [
@@ -1371,7 +1371,7 @@
"h": 10,
"w": 12,
"x": 0,
- "y": 20
+ "y": 247
},
"id": 256,
"options": {
@@ -1486,7 +1486,7 @@
"h": 10,
"w": 12,
"x": 12,
- "y": 20
+ "y": 247
},
"id": 262,
"options": {
@@ -1603,7 +1603,7 @@
"h": 10,
"w": 12,
"x": 0,
- "y": 30
+ "y": 257
},
"id": 266,
"options": {
@@ -1718,7 +1718,7 @@
"h": 10,
"w": 12,
"x": 12,
- "y": 30
+ "y": 257
},
"id": 268,
"options": {
@@ -1835,7 +1835,7 @@
"h": 10,
"w": 12,
"x": 0,
- "y": 40
+ "y": 267
},
"id": 269,
"options": {
@@ -1950,7 +1950,7 @@
"h": 10,
"w": 12,
"x": 12,
- "y": 40
+ "y": 267
},
"id": 271,
"options": {
@@ -2067,7 +2067,7 @@
"h": 10,
"w": 12,
"x": 0,
- "y": 50
+ "y": 277
},
"id": 272,
"options": {
@@ -2182,7 +2182,7 @@
"h": 10,
"w": 12,
"x": 12,
- "y": 50
+ "y": 277
},
"id": 273,
"options": {
@@ -2245,7 +2245,7 @@
"h": 1,
"w": 24,
"x": 0,
- "y": 12
+ "y": 24
},
"id": 280,
"panels": [
@@ -2314,7 +2314,7 @@
"h": 8,
"w": 12,
"x": 0,
- "y": 61
+ "y": 288
},
"id": 281,
"options": {
@@ -2415,7 +2415,7 @@
"h": 8,
"w": 12,
"x": 12,
- "y": 61
+ "y": 288
},
"id": 282,
"options": {
@@ -2516,7 +2516,7 @@
"h": 8,
"w": 12,
"x": 0,
- "y": 69
+ "y": 296
},
"id": 283,
"options": {
@@ -2617,7 +2617,7 @@
"h": 8,
"w": 12,
"x": 12,
- "y": 69
+ "y": 296
},
"id": 284,
"options": {
@@ -2716,7 +2716,7 @@
"h": 8,
"w": 12,
"x": 0,
- "y": 77
+ "y": 304
},
"id": 285,
"options": {
@@ -2817,7 +2817,7 @@
"h": 8,
"w": 12,
"x": 12,
- "y": 77
+ "y": 304
},
"id": 286,
"options": {
@@ -2919,7 +2919,7 @@
"h": 8,
"w": 12,
"x": 0,
- "y": 85
+ "y": 312
},
"id": 287,
"options": {
@@ -3020,7 +3020,7 @@
"h": 8,
"w": 12,
"x": 12,
- "y": 85
+ "y": 312
},
"id": 288,
"options": {
@@ -3064,7 +3064,7 @@
"h": 1,
"w": 24,
"x": 0,
- "y": 13
+ "y": 25
},
"id": 289,
"panels": [
@@ -3133,7 +3133,7 @@
"h": 6,
"w": 24,
"x": 0,
- "y": 62
+ "y": 289
},
"id": 292,
"options": {
@@ -3234,7 +3234,7 @@
"h": 8,
"w": 12,
"x": 0,
- "y": 68
+ "y": 295
},
"id": 290,
"options": {
@@ -3335,7 +3335,7 @@
"h": 8,
"w": 12,
"x": 12,
- "y": 68
+ "y": 295
},
"id": 291,
"options": {
@@ -3464,7 +3464,7 @@
"h": 8,
"w": 12,
"x": 0,
- "y": 76
+ "y": 303
},
"id": 336,
"options": {
@@ -3518,2519 +3518,2520 @@
"type": "row"
},
{
- "collapsed": false,
+ "collapsed": true,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
- "y": 14
+ "y": 26
},
"id": 293,
- "panels": [],
+ "panels": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Request QPS per Instance.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "ops"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 193
+ },
+ "id": 294,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by(instance, pod, type) (rate(greptime_mito_handle_request_elapsed_count{}[$__rate_interval]))",
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-[{{type}}]",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Request OPS per Instance",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Request P99 per Instance.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "points",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "s"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 193
+ },
+ "id": 295,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.99, sum by(instance, pod, le, type) (rate(greptime_mito_handle_request_elapsed_bucket{}[$__rate_interval])))",
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-[{{type}}]",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Request P99 per Instance",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Write Buffer per Instance.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "decbytes"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 225
+ },
+ "id": 296,
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "greptime_mito_write_buffer_bytes{}",
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Write Buffer per Instance",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Ingestion size by row counts.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "rowsps"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 225
+ },
+ "id": 297,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by (instance, pod) (rate(greptime_mito_write_rows_total{}[$__rate_interval]))",
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Write Rows per Instance",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Flush QPS per Instance.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "ops"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 233
+ },
+ "id": 298,
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by(instance, pod, reason) (rate(greptime_mito_flush_requests_total{}[$__rate_interval]))",
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-[{{reason}}]",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Flush OPS per Instance",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Write Stall per Instance.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ }
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 233
+ },
+ "id": 299,
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by(instance, pod) (greptime_mito_write_stall_total{})",
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Write Stall per Instance",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Read Stage OPS per Instance.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "ops"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 241
+ },
+ "id": 300,
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by(instance, pod) (rate(greptime_mito_read_stage_elapsed_count{ stage=\"total\"}[$__rate_interval]))",
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Read Stage OPS per Instance",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Read Stage P99 per Instance.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "points",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "s"
+ },
+ "overrides": [
+ {
+ "__systemRef": "hideSeriesFrom",
+ "matcher": {
+ "id": "byNames",
+ "options": {
+ "mode": "exclude",
+ "names": [
+ "[10.39.145.87:4000]-[mycluster-datanode-1]-[cache_miss_read]"
+ ],
+ "prefix": "All except:",
+ "readOnly": true
+ }
+ },
+ "properties": [
+ {
+ "id": "custom.hideFrom",
+ "value": {
+ "legend": false,
+ "tooltip": false,
+ "viz": true
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 241
+ },
+ "id": 301,
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.99, sum by(instance, pod, le, stage) (rate(greptime_mito_read_stage_elapsed_bucket{}[$__rate_interval])))",
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Read Stage P99 per Instance",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Write Stage P99 per Instance.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "points",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "s"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 249
+ },
+ "id": 302,
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true,
+ "sortBy": "Last *",
+ "sortDesc": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.99, sum by(instance, pod, le, stage) (rate(greptime_mito_write_stage_elapsed_bucket{}[$__rate_interval])))",
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Write Stage P99 per Instance",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Compaction OPS per Instance.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "ops"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 249
+ },
+ "id": 303,
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by(instance, pod) (rate(greptime_mito_compaction_total_elapsed_count{}[$__rate_interval]))",
+ "instant": false,
+ "legendFormat": "[{{ instance }}]-[{{pod}}]",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Compaction OPS per Instance",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Compaction latency by stage",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "points",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "s"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 257
+ },
+ "id": 304,
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.99, sum by(instance, pod, le, stage) (rate(greptime_mito_compaction_stage_elapsed_bucket{}[$__rate_interval])))",
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]-p99",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by(instance, pod, stage) (rate(greptime_mito_compaction_stage_elapsed_sum{}[$__rate_interval]))/sum by(instance, pod, stage) (rate(greptime_mito_compaction_stage_elapsed_count{}[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]-avg",
+ "range": true,
+ "refId": "B"
+ }
+ ],
+ "title": "Compaction Elapsed Time per Instance by Stage",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Compaction P99 per Instance.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "points",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "s"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 257
+ },
+ "id": 305,
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.99, sum by(instance, pod, le,stage) (rate(greptime_mito_compaction_total_elapsed_bucket{}[$__rate_interval])))",
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]-compaction",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Compaction P99 per Instance",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Write-ahead logs write size as bytes. This chart includes stats of p95 and p99 size by instance, total WAL write rate.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "bytes"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 265
+ },
+ "id": 306,
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.95, sum by(le,instance, pod) (rate(raft_engine_write_size_bucket[$__rate_interval])))",
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-req-size-p95",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.99, sum by(le,instance,pod) (rate(raft_engine_write_size_bucket[$__rate_interval])))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-req-size-p99",
+ "range": true,
+ "refId": "B"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by (instance, pod)(rate(raft_engine_write_size_sum[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-throughput",
+ "range": true,
+ "refId": "C"
+ }
+ ],
+ "title": "WAL write size",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Cached Bytes per Instance.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "points",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "decbytes"
+ },
+ "overrides": [
+ {
+ "__systemRef": "hideSeriesFrom",
+ "matcher": {
+ "id": "byNames",
+ "options": {
+ "mode": "exclude",
+ "names": [
+ "[10.39.145.79:4000]-[mycluster-datanode-0]-[file]"
+ ],
+ "prefix": "All except:",
+ "readOnly": true
+ }
+ },
+ "properties": [
+ {
+ "id": "custom.hideFrom",
+ "value": {
+ "legend": false,
+ "tooltip": false,
+ "viz": true
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 265
+ },
+ "id": 307,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "greptime_mito_cache_bytes{}",
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-[{{type}}]",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Cached Bytes per Instance",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Ongoing compaction task count",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "points",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "none"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 273
+ },
+ "id": 308,
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "greptime_mito_inflight_compaction_count",
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Inflight Compaction",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Raft engine (local disk) log store sync latency, p99",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "points",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "s"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 273
+ },
+ "id": 310,
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.99, sum by(le, type, node, instance, pod) (rate(raft_engine_sync_log_duration_seconds_bucket[$__rate_interval])))",
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-p99",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "WAL sync duration seconds",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Write-ahead log operations latency at p99",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "points",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "s"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 281
+ },
+ "id": 311,
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.99, sum by(le,logstore,optype,instance, pod) (rate(greptime_logstore_op_elapsed_bucket[$__rate_interval])))",
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-[{{logstore}}]-[{{optype}}]-p99",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Log Store op duration seconds",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Ongoing flush task count",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "points",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "none"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 281
+ },
+ "id": 312,
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "greptime_mito_inflight_flush_count",
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Inflight Flush",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Compaction oinput output bytes",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "bytes"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 289
+ },
+ "id": 335,
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by(instance, pod) (greptime_mito_compaction_input_bytes)",
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-input",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by(instance, pod) (greptime_mito_compaction_output_bytes)",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-output",
+ "range": true,
+ "refId": "B"
+ }
+ ],
+ "title": "Compaction Input/Output Bytes",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Per-stage elapsed time for region worker to handle bulk insert region requests.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "points",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "s"
+ },
+ "overrides": [
+ {
+ "__systemRef": "hideSeriesFrom",
+ "matcher": {
+ "id": "byNames",
+ "options": {
+ "mode": "exclude",
+ "names": [
+ "[192.168.50.8:4000]-[]-[prepare_bulk_request]-AVG",
+ "[minipc-1:4000]-[]-[bulk_extend]-AVG",
+ "[minipc-1:4000]-[]-[prepare_bulk_request]-AVG",
+ "[minipc-1:4000]-[]-[process_bulk_req]-AVG",
+ "[minipc-1:4000]-[]-[write_bulk]-AVG"
+ ],
+ "prefix": "All except:",
+ "readOnly": true
+ }
+ },
+ "properties": [
+ {
+ "id": "custom.hideFrom",
+ "value": {
+ "legend": false,
+ "tooltip": false,
+ "viz": true
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 289
+ },
+ "id": 337,
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.95, sum by(le,instance, stage, pod) (rate(greptime_region_worker_handle_write_bucket[$__rate_interval])))",
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]-P95",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by(instance, stage, pod) (rate(greptime_region_worker_handle_write_sum[$__rate_interval]))/sum by(instance, stage, pod) (rate(greptime_region_worker_handle_write_count[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]-AVG",
+ "range": true,
+ "refId": "B"
+ }
+ ],
+ "title": "Region Worker Handle Bulk Insert Requests",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Compaction oinput output bytes",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "none"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 297
+ },
+ "id": 348,
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by(instance, pod) (greptime_mito_memtable_active_series_count)",
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-series",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by(instance, pod) (greptime_mito_memtable_field_builder_count)",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-field_builders",
+ "range": true,
+ "refId": "B"
+ }
+ ],
+ "title": "Active Series and Field Builders Count",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Per-stage elapsed time for region worker to decode requests.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "points",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "s"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 297
+ },
+ "id": 338,
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "disableTextWrap": false,
+ "editorMode": "builder",
+ "expr": "histogram_quantile(0.95, sum by(le, instance, stage, pod) (rate(greptime_datanode_convert_region_request_bucket[$__rate_interval])))",
+ "fullMetaSearch": false,
+ "includeNullMetadata": true,
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]-P95",
+ "range": true,
+ "refId": "A",
+ "useBackend": false
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by(le,instance, stage, pod) (rate(greptime_datanode_convert_region_request_sum[$__rate_interval]))/sum by(le,instance, stage, pod) (rate(greptime_datanode_convert_region_request_count[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]-AVG",
+ "range": true,
+ "refId": "B"
+ }
+ ],
+ "title": "Region Worker Convert Requests",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "The local cache miss of the datanode.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ }
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 305
+ },
+ "id": 349,
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by (instance,pod, type) (rate(greptime_mito_cache_miss{}[$__rate_interval]))",
+ "legendFormat": "[{{instance}}]-[{{pod}}]-[{{type}}]",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Cache Miss",
+ "type": "timeseries"
+ }
+ ],
"title": "Mito Engine",
"type": "row"
},
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Request QPS per Instance.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "barWidthFactor": 0.6,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "ops"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 15
- },
- "id": 294,
- "options": {
- "legend": {
- "calcs": [
- "mean"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "multi",
- "sort": "desc"
- }
- },
- "pluginVersion": "11.6.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "sum by(instance, pod, type) (rate(greptime_mito_handle_request_elapsed_count{}[$__rate_interval]))",
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]-[{{type}}]",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Request OPS per Instance",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Request P99 per Instance.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "barWidthFactor": 0.6,
- "drawStyle": "points",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "s"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 15
- },
- "id": 295,
- "options": {
- "legend": {
- "calcs": [
- "mean"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "multi",
- "sort": "desc"
- }
- },
- "pluginVersion": "11.6.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.99, sum by(instance, pod, le, type) (rate(greptime_mito_handle_request_elapsed_bucket{}[$__rate_interval])))",
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]-[{{type}}]",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Request P99 per Instance",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Write Buffer per Instance.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "barWidthFactor": 0.6,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "decbytes"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 23
- },
- "id": 296,
- "options": {
- "legend": {
- "calcs": [
- "lastNotNull"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "multi",
- "sort": "desc"
- }
- },
- "pluginVersion": "11.6.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "greptime_mito_write_buffer_bytes{}",
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Write Buffer per Instance",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Ingestion size by row counts.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "barWidthFactor": 0.6,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "rowsps"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 23
- },
- "id": 297,
- "options": {
- "legend": {
- "calcs": [
- "mean"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "11.6.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "sum by (instance, pod) (rate(greptime_mito_write_rows_total{}[$__rate_interval]))",
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Write Rows per Instance",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Flush QPS per Instance.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "barWidthFactor": 0.6,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "ops"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 31
- },
- "id": 298,
- "options": {
- "legend": {
- "calcs": [
- "lastNotNull"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "multi",
- "sort": "desc"
- }
- },
- "pluginVersion": "11.6.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "sum by(instance, pod, reason) (rate(greptime_mito_flush_requests_total{}[$__rate_interval]))",
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]-[{{reason}}]",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Flush OPS per Instance",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Write Stall per Instance.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "barWidthFactor": 0.6,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- }
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 31
- },
- "id": 299,
- "options": {
- "legend": {
- "calcs": [
- "lastNotNull"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "multi",
- "sort": "desc"
- }
- },
- "pluginVersion": "11.6.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "sum by(instance, pod) (greptime_mito_write_stall_total{})",
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Write Stall per Instance",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Read Stage OPS per Instance.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "barWidthFactor": 0.6,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "ops"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 39
- },
- "id": 300,
- "options": {
- "legend": {
- "calcs": [
- "lastNotNull"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "multi",
- "sort": "desc"
- }
- },
- "pluginVersion": "11.6.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "sum by(instance, pod) (rate(greptime_mito_read_stage_elapsed_count{ stage=\"total\"}[$__rate_interval]))",
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Read Stage OPS per Instance",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Read Stage P99 per Instance.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "barWidthFactor": 0.6,
- "drawStyle": "points",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "s"
- },
- "overrides": [
- {
- "__systemRef": "hideSeriesFrom",
- "matcher": {
- "id": "byNames",
- "options": {
- "mode": "exclude",
- "names": [
- "[10.39.145.87:4000]-[mycluster-datanode-1]-[cache_miss_read]"
- ],
- "prefix": "All except:",
- "readOnly": true
- }
- },
- "properties": [
- {
- "id": "custom.hideFrom",
- "value": {
- "legend": false,
- "tooltip": false,
- "viz": true
- }
- }
- ]
- }
- ]
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 39
- },
- "id": 301,
- "options": {
- "legend": {
- "calcs": [
- "lastNotNull"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "multi",
- "sort": "desc"
- }
- },
- "pluginVersion": "11.6.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.99, sum by(instance, pod, le, stage) (rate(greptime_mito_read_stage_elapsed_bucket{}[$__rate_interval])))",
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Read Stage P99 per Instance",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Write Stage P99 per Instance.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "barWidthFactor": 0.6,
- "drawStyle": "points",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "s"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 47
- },
- "id": 302,
- "options": {
- "legend": {
- "calcs": [
- "lastNotNull"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true,
- "sortBy": "Last *",
- "sortDesc": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "multi",
- "sort": "desc"
- }
- },
- "pluginVersion": "11.6.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.99, sum by(instance, pod, le, stage) (rate(greptime_mito_write_stage_elapsed_bucket{}[$__rate_interval])))",
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Write Stage P99 per Instance",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Compaction OPS per Instance.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "barWidthFactor": 0.6,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "ops"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 47
- },
- "id": 303,
- "options": {
- "legend": {
- "calcs": [
- "lastNotNull"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "multi",
- "sort": "desc"
- }
- },
- "pluginVersion": "11.6.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "sum by(instance, pod) (rate(greptime_mito_compaction_total_elapsed_count{}[$__rate_interval]))",
- "instant": false,
- "legendFormat": "[{{ instance }}]-[{{pod}}]",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Compaction OPS per Instance",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Compaction latency by stage",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "barWidthFactor": 0.6,
- "drawStyle": "points",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "s"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 55
- },
- "id": 304,
- "options": {
- "legend": {
- "calcs": [
- "lastNotNull"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "11.6.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.99, sum by(instance, pod, le, stage) (rate(greptime_mito_compaction_stage_elapsed_bucket{}[$__rate_interval])))",
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]-p99",
- "range": true,
- "refId": "A"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "sum by(instance, pod, stage) (rate(greptime_mito_compaction_stage_elapsed_sum{}[$__rate_interval]))/sum by(instance, pod, stage) (rate(greptime_mito_compaction_stage_elapsed_count{}[$__rate_interval]))",
- "hide": false,
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]-avg",
- "range": true,
- "refId": "B"
- }
- ],
- "title": "Compaction Elapsed Time per Instance by Stage",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Compaction P99 per Instance.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "barWidthFactor": 0.6,
- "drawStyle": "points",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "s"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 55
- },
- "id": 305,
- "options": {
- "legend": {
- "calcs": [
- "lastNotNull"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "multi",
- "sort": "desc"
- }
- },
- "pluginVersion": "11.6.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.99, sum by(instance, pod, le,stage) (rate(greptime_mito_compaction_total_elapsed_bucket{}[$__rate_interval])))",
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]-compaction",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Compaction P99 per Instance",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Write-ahead logs write size as bytes. This chart includes stats of p95 and p99 size by instance, total WAL write rate.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "barWidthFactor": 0.6,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "bytes"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 63
- },
- "id": 306,
- "options": {
- "legend": {
- "calcs": [
- "lastNotNull"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "11.6.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.95, sum by(le,instance, pod) (rate(raft_engine_write_size_bucket[$__rate_interval])))",
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]-req-size-p95",
- "range": true,
- "refId": "A"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.99, sum by(le,instance,pod) (rate(raft_engine_write_size_bucket[$__rate_interval])))",
- "hide": false,
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]-req-size-p99",
- "range": true,
- "refId": "B"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "sum by (instance, pod)(rate(raft_engine_write_size_sum[$__rate_interval]))",
- "hide": false,
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]-throughput",
- "range": true,
- "refId": "C"
- }
- ],
- "title": "WAL write size",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Cached Bytes per Instance.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "barWidthFactor": 0.6,
- "drawStyle": "points",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "decbytes"
- },
- "overrides": [
- {
- "__systemRef": "hideSeriesFrom",
- "matcher": {
- "id": "byNames",
- "options": {
- "mode": "exclude",
- "names": [
- "[10.39.145.79:4000]-[mycluster-datanode-0]-[file]"
- ],
- "prefix": "All except:",
- "readOnly": true
- }
- },
- "properties": [
- {
- "id": "custom.hideFrom",
- "value": {
- "legend": false,
- "tooltip": false,
- "viz": true
- }
- }
- ]
- }
- ]
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 63
- },
- "id": 307,
- "options": {
- "legend": {
- "calcs": [],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "multi",
- "sort": "desc"
- }
- },
- "pluginVersion": "11.6.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "greptime_mito_cache_bytes{}",
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]-[{{type}}]",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Cached Bytes per Instance",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Ongoing compaction task count",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "barWidthFactor": 0.6,
- "drawStyle": "points",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "none"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 71
- },
- "id": 308,
- "options": {
- "legend": {
- "calcs": [
- "lastNotNull"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "11.6.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "greptime_mito_inflight_compaction_count",
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Inflight Compaction",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Raft engine (local disk) log store sync latency, p99",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "barWidthFactor": 0.6,
- "drawStyle": "points",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "s"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 71
- },
- "id": 310,
- "options": {
- "legend": {
- "calcs": [
- "lastNotNull"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "11.6.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.99, sum by(le, type, node, instance, pod) (rate(raft_engine_sync_log_duration_seconds_bucket[$__rate_interval])))",
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]-p99",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "WAL sync duration seconds",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Write-ahead log operations latency at p99",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "barWidthFactor": 0.6,
- "drawStyle": "points",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "s"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 79
- },
- "id": 311,
- "options": {
- "legend": {
- "calcs": [
- "lastNotNull"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "11.6.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.99, sum by(le,logstore,optype,instance, pod) (rate(greptime_logstore_op_elapsed_bucket[$__rate_interval])))",
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]-[{{logstore}}]-[{{optype}}]-p99",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Log Store op duration seconds",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Ongoing flush task count",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "barWidthFactor": 0.6,
- "drawStyle": "points",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "none"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 79
- },
- "id": 312,
- "options": {
- "legend": {
- "calcs": [
- "lastNotNull"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "11.6.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "greptime_mito_inflight_flush_count",
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Inflight Flush",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Compaction oinput output bytes",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "barWidthFactor": 0.6,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "bytes"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 87
- },
- "id": 335,
- "options": {
- "legend": {
- "calcs": [
- "lastNotNull"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "11.6.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "sum by(instance, pod) (greptime_mito_compaction_input_bytes)",
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]-input",
- "range": true,
- "refId": "A"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "sum by(instance, pod) (greptime_mito_compaction_output_bytes)",
- "hide": false,
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]-output",
- "range": true,
- "refId": "B"
- }
- ],
- "title": "Compaction Input/Output Bytes",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Per-stage elapsed time for region worker to handle bulk insert region requests.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "barWidthFactor": 0.6,
- "drawStyle": "points",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "s"
- },
- "overrides": [
- {
- "__systemRef": "hideSeriesFrom",
- "matcher": {
- "id": "byNames",
- "options": {
- "mode": "exclude",
- "names": [
- "[192.168.50.8:4000]-[]-[prepare_bulk_request]-AVG",
- "[minipc-1:4000]-[]-[bulk_extend]-AVG",
- "[minipc-1:4000]-[]-[prepare_bulk_request]-AVG",
- "[minipc-1:4000]-[]-[process_bulk_req]-AVG",
- "[minipc-1:4000]-[]-[write_bulk]-AVG"
- ],
- "prefix": "All except:",
- "readOnly": true
- }
- },
- "properties": [
- {
- "id": "custom.hideFrom",
- "value": {
- "legend": false,
- "tooltip": false,
- "viz": true
- }
- }
- ]
- }
- ]
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 87
- },
- "id": 337,
- "options": {
- "legend": {
- "calcs": [
- "lastNotNull"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "11.6.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.95, sum by(le,instance, stage, pod) (rate(greptime_region_worker_handle_write_bucket[$__rate_interval])))",
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]-P95",
- "range": true,
- "refId": "A"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "sum by(instance, stage, pod) (rate(greptime_region_worker_handle_write_sum[$__rate_interval]))/sum by(instance, stage, pod) (rate(greptime_region_worker_handle_write_count[$__rate_interval]))",
- "hide": false,
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]-AVG",
- "range": true,
- "refId": "B"
- }
- ],
- "title": "Region Worker Handle Bulk Insert Requests",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Compaction oinput output bytes",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "barWidthFactor": 0.6,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "none"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 95
- },
- "id": 348,
- "options": {
- "legend": {
- "calcs": [
- "lastNotNull"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "11.6.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "sum by(instance, pod) (greptime_mito_memtable_active_series_count)",
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]-series",
- "range": true,
- "refId": "A"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "sum by(instance, pod) (greptime_mito_memtable_field_builder_count)",
- "hide": false,
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]-field_builders",
- "range": true,
- "refId": "B"
- }
- ],
- "title": "Active Series and Field Builders Count",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Per-stage elapsed time for region worker to decode requests.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "barWidthFactor": 0.6,
- "drawStyle": "points",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "s"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 95
- },
- "id": 338,
- "options": {
- "legend": {
- "calcs": [
- "lastNotNull"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "11.6.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "disableTextWrap": false,
- "editorMode": "builder",
- "expr": "histogram_quantile(0.95, sum by(le, instance, stage, pod) (rate(greptime_datanode_convert_region_request_bucket[$__rate_interval])))",
- "fullMetaSearch": false,
- "includeNullMetadata": true,
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]-P95",
- "range": true,
- "refId": "A",
- "useBackend": false
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "sum by(le,instance, stage, pod) (rate(greptime_datanode_convert_region_request_sum[$__rate_interval]))/sum by(le,instance, stage, pod) (rate(greptime_datanode_convert_region_request_count[$__rate_interval]))",
- "hide": false,
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]-AVG",
- "range": true,
- "refId": "B"
- }
- ],
- "title": "Region Worker Convert Requests",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "The local cache miss of the datanode.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "barWidthFactor": 0.6,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- }
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 103
- },
- "id": 349,
- "options": {
- "legend": {
- "calcs": [
- "lastNotNull"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "11.6.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "sum by (instance,pod, type) (rate(greptime_mito_cache_miss{}[$__rate_interval]))",
- "legendFormat": "[{{instance}}]-[{{pod}}]-[{{type}}]",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Cache Miss",
- "type": "timeseries"
- },
{
"collapsed": true,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
- "y": 111
+ "y": 27
},
"id": 313,
"panels": [
@@ -6098,7 +6099,7 @@
"h": 10,
"w": 24,
"x": 0,
- "y": 232
+ "y": 459
},
"id": 314,
"options": {
@@ -6198,7 +6199,7 @@
"h": 7,
"w": 12,
"x": 0,
- "y": 242
+ "y": 469
},
"id": 315,
"options": {
@@ -6298,7 +6299,7 @@
"h": 7,
"w": 12,
"x": 12,
- "y": 242
+ "y": 469
},
"id": 316,
"options": {
@@ -6398,7 +6399,7 @@
"h": 7,
"w": 12,
"x": 0,
- "y": 249
+ "y": 476
},
"id": 317,
"options": {
@@ -6498,7 +6499,7 @@
"h": 7,
"w": 12,
"x": 12,
- "y": 249
+ "y": 476
},
"id": 318,
"options": {
@@ -6598,7 +6599,7 @@
"h": 7,
"w": 12,
"x": 0,
- "y": 256
+ "y": 483
},
"id": 319,
"options": {
@@ -6699,7 +6700,7 @@
"h": 7,
"w": 12,
"x": 12,
- "y": 256
+ "y": 483
},
"id": 320,
"options": {
@@ -6800,7 +6801,7 @@
"h": 7,
"w": 12,
"x": 0,
- "y": 263
+ "y": 490
},
"id": 321,
"options": {
@@ -6900,7 +6901,7 @@
"h": 7,
"w": 12,
"x": 12,
- "y": 263
+ "y": 490
},
"id": 322,
"options": {
@@ -7000,7 +7001,7 @@
"h": 7,
"w": 12,
"x": 0,
- "y": 270
+ "y": 497
},
"id": 323,
"options": {
@@ -7103,7 +7104,7 @@
"h": 7,
"w": 12,
"x": 12,
- "y": 270
+ "y": 497
},
"id": 334,
"options": {
@@ -7144,1827 +7145,1829 @@
"type": "row"
},
{
- "collapsed": false,
+ "collapsed": true,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
- "y": 112
+ "y": 28
},
"id": 351,
- "panels": [],
+ "panels": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Triggered region flush total",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "none"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 195
+ },
+ "id": 350,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "meta_triggered_region_flush_total",
+ "instant": false,
+ "legendFormat": "{{pod}}-{{topic_name}}",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Triggered region flush total",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Triggered region checkpoint total",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "none"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 195
+ },
+ "id": 352,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "meta_triggered_region_checkpoint_total",
+ "instant": false,
+ "legendFormat": "{{pod}}-{{topic_name}}",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Triggered region checkpoint total",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Topic estimated max replay size",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "bytes"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 227
+ },
+ "id": 353,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "meta_topic_estimated_replay_size",
+ "instant": false,
+ "legendFormat": "{{pod}}-{{topic_name}}",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Topic estimated replay size",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Kafka logstore's bytes traffic",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "bytes"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 227
+ },
+ "id": 354,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "rate(greptime_logstore_kafka_client_bytes_total[$__rate_interval])",
+ "instant": false,
+ "legendFormat": "{{pod}}-{{logstore}}",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Kafka logstore's bytes traffic",
+ "type": "timeseries"
+ }
+ ],
"title": "Remote WAL",
"type": "row"
},
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Triggered region flush total",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "barWidthFactor": 0.6,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "none"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 113
- },
- "id": 350,
- "options": {
- "legend": {
- "calcs": [],
- "displayMode": "list",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "11.6.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "meta_triggered_region_flush_total",
- "instant": false,
- "legendFormat": "{{pod}}-{{topic_name}}",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Triggered region flush total",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Triggered region checkpoint total",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "barWidthFactor": 0.6,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "none"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 113
- },
- "id": 352,
- "options": {
- "legend": {
- "calcs": [],
- "displayMode": "list",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "11.6.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "meta_triggered_region_checkpoint_total",
- "instant": false,
- "legendFormat": "{{pod}}-{{topic_name}}",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Triggered region checkpoint total",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Topic estimated max replay size",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "barWidthFactor": 0.6,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "bytes"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 121
- },
- "id": 353,
- "options": {
- "legend": {
- "calcs": [],
- "displayMode": "list",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "11.6.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "meta_topic_estimated_replay_size",
- "instant": false,
- "legendFormat": "{{pod}}-{{topic_name}}",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Topic estimated replay size",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Kafka logstore's bytes traffic",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "barWidthFactor": 0.6,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "bytes"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 121
- },
- "id": 354,
- "options": {
- "legend": {
- "calcs": [],
- "displayMode": "list",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "11.6.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "rate(greptime_logstore_kafka_client_bytes_total[$__rate_interval])",
- "instant": false,
- "legendFormat": "{{pod}}-{{logstore}}",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Kafka logstore's bytes traffic",
- "type": "timeseries"
- },
- {
- "collapsed": false,
- "gridPos": {
- "h": 1,
- "w": 24,
- "x": 0,
- "y": 129
- },
- "id": 324,
- "panels": [],
- "title": "Metasrv",
- "type": "row"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Counter of region migration by source and destination",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "thresholds"
- },
- "custom": {
- "axisPlacement": "auto",
- "fillOpacity": 70,
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "lineWidth": 1
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- }
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 130
- },
- "id": 325,
- "options": {
- "colWidth": 0.9,
- "legend": {
- "displayMode": "list",
- "placement": "bottom",
- "showLegend": false
- },
- "rowHeight": 0.9,
- "showValue": "auto",
- "tooltip": {
- "hideZeros": false,
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "11.6.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "greptime_meta_region_migration_stat{datanode_type=\"src\"}",
- "instant": false,
- "legendFormat": "from-datanode-{{datanode_id}}",
- "range": true,
- "refId": "A"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "greptime_meta_region_migration_stat{datanode_type=\"desc\"}",
- "hide": false,
- "instant": false,
- "legendFormat": "to-datanode-{{datanode_id}}",
- "range": true,
- "refId": "B"
- }
- ],
- "title": "Region migration datanode",
- "type": "status-history"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Counter of region migration error",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "barWidthFactor": 0.6,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "none"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 130
- },
- "id": 326,
- "options": {
- "legend": {
- "calcs": [],
- "displayMode": "list",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "11.6.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "greptime_meta_region_migration_error",
- "instant": false,
- "legendFormat": "{{pod}}-{{state}}-{{error_type}}",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Region migration error",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Gauge of load information of each datanode, collected via heartbeat between datanode and metasrv. This information is for metasrv to schedule workloads.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "barWidthFactor": 0.6,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "binBps"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 138
- },
- "id": 327,
- "options": {
- "legend": {
- "calcs": [],
- "displayMode": "list",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "11.6.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "greptime_datanode_load",
- "instant": false,
- "legendFormat": "Datanode-{{datanode_id}}-writeload",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Datanode load",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Displays the rate of SQL executions processed by the Meta service using the RDS backend.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "barWidthFactor": 0.6,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "none"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 138
- },
- "id": 339,
- "options": {
- "legend": {
- "calcs": [],
- "displayMode": "list",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "11.6.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "rate(greptime_meta_rds_pg_sql_execute_elapsed_ms_count[$__rate_interval])",
- "instant": false,
- "legendFormat": "{{pod}} {{op}} {{type}} {{result}} ",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Rate of SQL Executions (RDS)",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Measures the response time of SQL executions via the RDS backend. ",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "barWidthFactor": 0.6,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "ms"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 146
- },
- "id": 340,
- "options": {
- "legend": {
- "calcs": [],
- "displayMode": "list",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "11.6.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "disableTextWrap": false,
- "editorMode": "code",
- "expr": "histogram_quantile(0.90, sum by(pod, op, type, result, le) (rate(greptime_meta_rds_pg_sql_execute_elapsed_ms_bucket[$__rate_interval])))",
- "fullMetaSearch": false,
- "includeNullMetadata": true,
- "instant": false,
- "legendFormat": "{{pod}} {{op}} {{type}} {{result}} p90",
- "range": true,
- "refId": "A",
- "useBackend": false
- }
- ],
- "title": "SQL Execution Latency (RDS)",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Shows latency of Meta handlers by pod and handler name, useful for monitoring handler performance and detecting latency spikes.\n",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "barWidthFactor": 0.6,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "s"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 146
- },
- "id": 341,
- "options": {
- "legend": {
- "calcs": [],
- "displayMode": "list",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "11.6.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "disableTextWrap": false,
- "editorMode": "code",
- "expr": "histogram_quantile(0.90, sum by(pod, le, name) (\n rate(greptime_meta_handler_execute_bucket[$__rate_interval])\n))",
- "fullMetaSearch": false,
- "includeNullMetadata": true,
- "instant": false,
- "legendFormat": "{{pod}} {{name}} p90",
- "range": true,
- "refId": "A",
- "useBackend": false
- }
- ],
- "title": "Handler Execution Latency",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Shows p90 heartbeat message sizes, helping track network usage and identify anomalies in heartbeat payload.\n",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "barWidthFactor": 0.6,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "bytes"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 154
- },
- "id": 342,
- "options": {
- "legend": {
- "calcs": [],
- "displayMode": "list",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "11.6.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "disableTextWrap": false,
- "editorMode": "code",
- "expr": "histogram_quantile(0.9, sum by(pod, le) (greptime_meta_heartbeat_stat_memory_size_bucket))",
- "fullMetaSearch": false,
- "includeNullMetadata": true,
- "instant": false,
- "legendFormat": "{{pod}}",
- "range": true,
- "refId": "A",
- "useBackend": false
- }
- ],
- "title": "Heartbeat Packet Size",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Gauge of load information of each datanode, collected via heartbeat between datanode and metasrv. This information is for metasrv to schedule workloads.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "barWidthFactor": 0.6,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "s"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 154
- },
- "id": 345,
- "options": {
- "legend": {
- "calcs": [],
- "displayMode": "list",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "11.6.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "disableTextWrap": false,
- "editorMode": "code",
- "expr": "rate(greptime_meta_heartbeat_rate[$__rate_interval])",
- "fullMetaSearch": false,
- "includeNullMetadata": true,
- "instant": false,
- "legendFormat": "{{pod}}",
- "range": true,
- "refId": "A",
- "useBackend": false
- }
- ],
- "title": "Meta Heartbeat Receive Rate",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Gauge of load information of each datanode, collected via heartbeat between datanode and metasrv. This information is for metasrv to schedule workloads.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "barWidthFactor": 0.6,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "s"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 162
- },
- "id": 343,
- "options": {
- "legend": {
- "calcs": [],
- "displayMode": "list",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "11.6.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "disableTextWrap": false,
- "editorMode": "code",
- "expr": "histogram_quantile(0.99, sum by(pod, le, op, target) (greptime_meta_kv_request_elapsed_bucket))",
- "fullMetaSearch": false,
- "includeNullMetadata": true,
- "instant": false,
- "legendFormat": "{{pod}}-{{op}} p99",
- "range": true,
- "refId": "A",
- "useBackend": false
- }
- ],
- "title": "Meta KV Ops Latency",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Gauge of load information of each datanode, collected via heartbeat between datanode and metasrv. This information is for metasrv to schedule workloads.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "barWidthFactor": 0.6,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "none"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 162
- },
- "id": 346,
- "options": {
- "legend": {
- "calcs": [],
- "displayMode": "list",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "11.6.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "disableTextWrap": false,
- "editorMode": "code",
- "expr": "rate(greptime_meta_kv_request_elapsed_count[$__rate_interval])",
- "fullMetaSearch": false,
- "includeNullMetadata": true,
- "instant": false,
- "legendFormat": "{{pod}}-{{op}} p99",
- "range": true,
- "refId": "A",
- "useBackend": false
- }
- ],
- "title": "Rate of meta KV Ops",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Gauge of load information of each datanode, collected via heartbeat between datanode and metasrv. This information is for metasrv to schedule workloads.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "barWidthFactor": 0.6,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "s"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 170
- },
- "id": 347,
- "options": {
- "legend": {
- "calcs": [],
- "displayMode": "list",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "11.6.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "disableTextWrap": false,
- "editorMode": "code",
- "expr": "histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_create_tables_bucket))",
- "fullMetaSearch": false,
- "includeNullMetadata": true,
- "instant": false,
- "legendFormat": "CreateLogicalTables-{{step}} p90",
- "range": true,
- "refId": "A",
- "useBackend": false
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "disableTextWrap": false,
- "editorMode": "code",
- "expr": "histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_create_table))",
- "fullMetaSearch": false,
- "hide": false,
- "includeNullMetadata": true,
- "instant": false,
- "legendFormat": "CreateTable-{{step}} p90",
- "range": true,
- "refId": "B",
- "useBackend": false
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "disableTextWrap": false,
- "editorMode": "code",
- "expr": "histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_create_view))",
- "fullMetaSearch": false,
- "hide": false,
- "includeNullMetadata": true,
- "instant": false,
- "legendFormat": "CreateView-{{step}} p90",
- "range": true,
- "refId": "C",
- "useBackend": false
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "disableTextWrap": false,
- "editorMode": "code",
- "expr": "histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_create_flow))",
- "fullMetaSearch": false,
- "hide": false,
- "includeNullMetadata": true,
- "instant": false,
- "legendFormat": "CreateFlow-{{step}} p90",
- "range": true,
- "refId": "D",
- "useBackend": false
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "disableTextWrap": false,
- "editorMode": "code",
- "expr": "histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_drop_table))",
- "fullMetaSearch": false,
- "hide": false,
- "includeNullMetadata": true,
- "instant": false,
- "legendFormat": "DropTable-{{step}} p90",
- "range": true,
- "refId": "E",
- "useBackend": false
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "disableTextWrap": false,
- "editorMode": "code",
- "expr": "histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_alter_table))",
- "fullMetaSearch": false,
- "hide": false,
- "includeNullMetadata": true,
- "instant": false,
- "legendFormat": "AlterTable-{{step}} p90",
- "range": true,
- "refId": "F",
- "useBackend": false
- }
- ],
- "title": "DDL Latency",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Reconciliation stats",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "barWidthFactor": 0.6,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "s"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 170
- },
- "id": 355,
- "options": {
- "legend": {
- "calcs": [],
- "displayMode": "list",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "11.6.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "greptime_meta_reconciliation_stats",
- "hide": false,
- "instant": false,
- "legendFormat": "{{pod}}-{{table_type}}-{{type}}",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Reconciliation stats",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Elapsed of Reconciliation steps",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "barWidthFactor": 0.6,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "s"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 178
- },
- "id": 356,
- "options": {
- "legend": {
- "calcs": [],
- "displayMode": "list",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "11.6.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.9, greptime_meta_reconciliation_procedure_bucket)",
- "hide": false,
- "instant": false,
- "legendFormat": "{{procedure_name}}-{{step}}-P90",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Reconciliation steps",
- "type": "timeseries"
- },
{
"collapsed": true,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
- "y": 186
+ "y": 29
+ },
+ "id": 324,
+ "panels": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Counter of region migration by source and destination",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "custom": {
+ "axisPlacement": "auto",
+ "fillOpacity": 70,
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineWidth": 1
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ }
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 196
+ },
+ "id": 325,
+ "options": {
+ "colWidth": 0.9,
+ "legend": {
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": false
+ },
+ "rowHeight": 0.9,
+ "showValue": "auto",
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "greptime_meta_region_migration_stat{datanode_type=\"src\"}",
+ "instant": false,
+ "legendFormat": "from-datanode-{{datanode_id}}",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "greptime_meta_region_migration_stat{datanode_type=\"desc\"}",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "to-datanode-{{datanode_id}}",
+ "range": true,
+ "refId": "B"
+ }
+ ],
+ "title": "Region migration datanode",
+ "type": "status-history"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Counter of region migration error",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "none"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 196
+ },
+ "id": 326,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "greptime_meta_region_migration_error",
+ "instant": false,
+ "legendFormat": "{{pod}}-{{state}}-{{error_type}}",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Region migration error",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Gauge of load information of each datanode, collected via heartbeat between datanode and metasrv. This information is for metasrv to schedule workloads.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "binBps"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 228
+ },
+ "id": 327,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "greptime_datanode_load",
+ "instant": false,
+ "legendFormat": "Datanode-{{datanode_id}}-writeload",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Datanode load",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Displays the rate of SQL executions processed by the Meta service using the RDS backend.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "none"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 228
+ },
+ "id": 339,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "rate(greptime_meta_rds_pg_sql_execute_elapsed_ms_count[$__rate_interval])",
+ "instant": false,
+ "legendFormat": "{{pod}} {{op}} {{type}} {{result}} ",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Rate of SQL Executions (RDS)",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Measures the response time of SQL executions via the RDS backend. ",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "ms"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 236
+ },
+ "id": 340,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "disableTextWrap": false,
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.90, sum by(pod, op, type, result, le) (rate(greptime_meta_rds_pg_sql_execute_elapsed_ms_bucket[$__rate_interval])))",
+ "fullMetaSearch": false,
+ "includeNullMetadata": true,
+ "instant": false,
+ "legendFormat": "{{pod}} {{op}} {{type}} {{result}} p90",
+ "range": true,
+ "refId": "A",
+ "useBackend": false
+ }
+ ],
+ "title": "SQL Execution Latency (RDS)",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Shows latency of Meta handlers by pod and handler name, useful for monitoring handler performance and detecting latency spikes.\n",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "s"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 236
+ },
+ "id": 341,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "disableTextWrap": false,
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.90, sum by(pod, le, name) (\n rate(greptime_meta_handler_execute_bucket[$__rate_interval])\n))",
+ "fullMetaSearch": false,
+ "includeNullMetadata": true,
+ "instant": false,
+ "legendFormat": "{{pod}} {{name}} p90",
+ "range": true,
+ "refId": "A",
+ "useBackend": false
+ }
+ ],
+ "title": "Handler Execution Latency",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Shows p90 heartbeat message sizes, helping track network usage and identify anomalies in heartbeat payload.\n",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "bytes"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 244
+ },
+ "id": 342,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "disableTextWrap": false,
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.9, sum by(pod, le) (greptime_meta_heartbeat_stat_memory_size_bucket))",
+ "fullMetaSearch": false,
+ "includeNullMetadata": true,
+ "instant": false,
+ "legendFormat": "{{pod}}",
+ "range": true,
+ "refId": "A",
+ "useBackend": false
+ }
+ ],
+ "title": "Heartbeat Packet Size",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Gauge of load information of each datanode, collected via heartbeat between datanode and metasrv. This information is for metasrv to schedule workloads.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "s"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 244
+ },
+ "id": 345,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "disableTextWrap": false,
+ "editorMode": "code",
+ "expr": "rate(greptime_meta_heartbeat_rate[$__rate_interval])",
+ "fullMetaSearch": false,
+ "includeNullMetadata": true,
+ "instant": false,
+ "legendFormat": "{{pod}}",
+ "range": true,
+ "refId": "A",
+ "useBackend": false
+ }
+ ],
+ "title": "Meta Heartbeat Receive Rate",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Gauge of load information of each datanode, collected via heartbeat between datanode and metasrv. This information is for metasrv to schedule workloads.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "s"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 252
+ },
+ "id": 343,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "disableTextWrap": false,
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.99, sum by(pod, le, op, target) (greptime_meta_kv_request_elapsed_bucket))",
+ "fullMetaSearch": false,
+ "includeNullMetadata": true,
+ "instant": false,
+ "legendFormat": "{{pod}}-{{op}} p99",
+ "range": true,
+ "refId": "A",
+ "useBackend": false
+ }
+ ],
+ "title": "Meta KV Ops Latency",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Gauge of load information of each datanode, collected via heartbeat between datanode and metasrv. This information is for metasrv to schedule workloads.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "none"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 252
+ },
+ "id": 346,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "disableTextWrap": false,
+ "editorMode": "code",
+ "expr": "rate(greptime_meta_kv_request_elapsed_count[$__rate_interval])",
+ "fullMetaSearch": false,
+ "includeNullMetadata": true,
+ "instant": false,
+ "legendFormat": "{{pod}}-{{op}} p99",
+ "range": true,
+ "refId": "A",
+ "useBackend": false
+ }
+ ],
+ "title": "Rate of meta KV Ops",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Gauge of load information of each datanode, collected via heartbeat between datanode and metasrv. This information is for metasrv to schedule workloads.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "s"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 260
+ },
+ "id": 347,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "disableTextWrap": false,
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_create_tables_bucket))",
+ "fullMetaSearch": false,
+ "includeNullMetadata": true,
+ "instant": false,
+ "legendFormat": "CreateLogicalTables-{{step}} p90",
+ "range": true,
+ "refId": "A",
+ "useBackend": false
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "disableTextWrap": false,
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_create_table))",
+ "fullMetaSearch": false,
+ "hide": false,
+ "includeNullMetadata": true,
+ "instant": false,
+ "legendFormat": "CreateTable-{{step}} p90",
+ "range": true,
+ "refId": "B",
+ "useBackend": false
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "disableTextWrap": false,
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_create_view))",
+ "fullMetaSearch": false,
+ "hide": false,
+ "includeNullMetadata": true,
+ "instant": false,
+ "legendFormat": "CreateView-{{step}} p90",
+ "range": true,
+ "refId": "C",
+ "useBackend": false
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "disableTextWrap": false,
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_create_flow))",
+ "fullMetaSearch": false,
+ "hide": false,
+ "includeNullMetadata": true,
+ "instant": false,
+ "legendFormat": "CreateFlow-{{step}} p90",
+ "range": true,
+ "refId": "D",
+ "useBackend": false
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "disableTextWrap": false,
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_drop_table))",
+ "fullMetaSearch": false,
+ "hide": false,
+ "includeNullMetadata": true,
+ "instant": false,
+ "legendFormat": "DropTable-{{step}} p90",
+ "range": true,
+ "refId": "E",
+ "useBackend": false
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "disableTextWrap": false,
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_alter_table))",
+ "fullMetaSearch": false,
+ "hide": false,
+ "includeNullMetadata": true,
+ "instant": false,
+ "legendFormat": "AlterTable-{{step}} p90",
+ "range": true,
+ "refId": "F",
+ "useBackend": false
+ }
+ ],
+ "title": "DDL Latency",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Reconciliation stats",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "s"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 260
+ },
+ "id": 355,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "greptime_meta_reconciliation_stats",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "{{pod}}-{{table_type}}-{{type}}",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Reconciliation stats",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Elapsed of Reconciliation steps",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "s"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 268
+ },
+ "id": 356,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.9, greptime_meta_reconciliation_procedure_bucket)",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "{{procedure_name}}-{{step}}-P90",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Reconciliation steps",
+ "type": "timeseries"
+ }
+ ],
+ "title": "Metasrv",
+ "type": "row"
+ },
+ {
+ "collapsed": true,
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 30
},
"id": 328,
"panels": [
@@ -9031,7 +9034,7 @@
"h": 8,
"w": 12,
"x": 0,
- "y": 3055
+ "y": 3282
},
"id": 329,
"options": {
@@ -9126,7 +9129,7 @@
"h": 8,
"w": 12,
"x": 12,
- "y": 3055
+ "y": 3282
},
"id": 330,
"options": {
@@ -9234,7 +9237,7 @@
"h": 8,
"w": 9,
"x": 0,
- "y": 3063
+ "y": 3290
},
"id": 331,
"options": {
@@ -9342,7 +9345,7 @@
"h": 8,
"w": 9,
"x": 9,
- "y": 3063
+ "y": 3290
},
"id": 332,
"options": {
@@ -9437,7 +9440,7 @@
"h": 8,
"w": 6,
"x": 18,
- "y": 3063
+ "y": 3290
},
"id": 333,
"options": {
@@ -9474,12 +9477,12 @@
"type": "row"
},
{
- "collapsed": true,
+ "collapsed": false,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
- "y": 187
+ "y": 31
},
"id": 357,
"panels": [],
@@ -9550,7 +9553,7 @@
"h": 8,
"w": 24,
"x": 0,
- "y": 188
+ "y": 32
},
"id": 358,
"options": {
@@ -9648,7 +9651,7 @@
"h": 8,
"w": 12,
"x": 0,
- "y": 196
+ "y": 40
},
"id": 359,
"options": {
@@ -9761,7 +9764,7 @@
"h": 8,
"w": 12,
"x": 12,
- "y": 196
+ "y": 40
},
"id": 360,
"options": {
@@ -9861,7 +9864,7 @@
"h": 8,
"w": 12,
"x": 0,
- "y": 204
+ "y": 48
},
"id": 361,
"options": {
@@ -9974,7 +9977,7 @@
"h": 8,
"w": 12,
"x": 12,
- "y": 204
+ "y": 48
},
"id": 364,
"options": {
@@ -10074,7 +10077,7 @@
"h": 8,
"w": 12,
"x": 0,
- "y": 212
+ "y": 56
},
"id": 363,
"options": {
@@ -10187,7 +10190,7 @@
"h": 8,
"w": 12,
"x": 12,
- "y": 212
+ "y": 56
},
"id": 362,
"options": {
@@ -10221,6 +10224,1330 @@
],
"title": "Save Alert Failure Rate",
"type": "timeseries"
+ },
+ {
+ "collapsed": false,
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 64
+ },
+ "id": 366,
+ "panels": [],
+ "title": "Hotspot",
+ "type": "row"
+ },
+ {
+ "datasource": {
+ "type": "mysql",
+ "uid": "${information_schema}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "custom": {
+ "align": "auto",
+ "cellOptions": {
+ "type": "auto",
+ "wrapText": false
+ },
+ "inspect": false
+ },
+ "fieldMinMax": false,
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ }
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "disk_size"
+ },
+ "properties": [
+ {
+ "id": "unit",
+ "value": "bytes"
+ }
+ ]
+ }
+ ]
+ },
+ "gridPos": {
+ "h": 18,
+ "w": 24,
+ "x": 0,
+ "y": 65
+ },
+ "id": 365,
+ "options": {
+ "cellHeight": "sm",
+ "footer": {
+ "countRows": false,
+ "enablePagination": true,
+ "fields": [
+ "region_rows",
+ "disk_size"
+ ],
+ "reducer": [
+ "sum"
+ ],
+ "show": true
+ },
+ "showHeader": true,
+ "sortBy": []
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "dataset": "information_schema",
+ "datasource": {
+ "type": "mysql",
+ "uid": "${information_schema}"
+ },
+ "editorMode": "code",
+ "format": "table",
+ "rawQuery": true,
+ "rawSql": "WITH table_stats AS (\n SELECT\n table_id,\n COUNT(*) AS region_count,\n SUM(disk_size) AS total_disk_size,\n SUM(region_rows) as total_region_rows\n FROM information_schema.region_statistics\n WHERE region_role = 'Leader'\n GROUP BY table_id\n HAVING COUNT(*) > 1\n)\n\nSELECT\n t.table_schema,\n t.table_name,\n\n r.region_id,\n t.table_id,\n r.region_number,\n\n p.partition_description,\n\n\n ROUND(\n r.disk_size * 100.0\n / NULLIF(ts.total_disk_size, 0),\n 2\n ) AS disk_size_share_percent,\n\n r.disk_size,\n \n ROUND(\n r.region_rows * 100.0\n / NULLIF(ts.total_region_rows, 0),\n 2\n ) AS region_rows_share_percent,\n r.region_rows\n\nFROM information_schema.region_statistics r\n\nJOIN table_stats ts\n ON r.table_id = ts.table_id\n\nJOIN information_schema.tables t\n ON r.table_id = t.table_id\n\nLEFT JOIN information_schema.partitions p\n ON p.table_schema = t.table_schema\n AND p.table_name = t.table_name\n AND p.greptime_partition_id = r.region_id\n\nWHERE r.region_role = 'Leader'\n\nORDER BY region_rows_share_percent DESC\nLIMIT 100;",
+ "refId": "A",
+ "sql": {
+ "columns": [
+ {
+ "parameters": [],
+ "type": "function"
+ }
+ ],
+ "groupBy": [
+ {
+ "property": {
+ "type": "string"
+ },
+ "type": "groupBy"
+ }
+ ],
+ "limit": 50
+ }
+ }
+ ],
+ "title": "Hotspot Regions",
+ "type": "table"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Write load of each datanode over time.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "fieldMinMax": false,
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "binBps"
+ },
+ "overrides": [
+ {
+ "__systemRef": "hideSeriesFrom",
+ "matcher": {
+ "id": "byNames",
+ "options": {
+ "mode": "exclude",
+ "names": [
+ "total_region_rows",
+ "total_written_bytes_since_open",
+ "total_disk_size",
+ "total_sst_size",
+ "total_sst_num",
+ "total_memtable_size",
+ "total_manifest_size",
+ "total_index_size",
+ "cluster_disk_ratio_percent",
+ "cluster_sst_ratio_percent",
+ "cluster_rows_ratio_percent",
+ "max_region_disk_size",
+ "max_region_rows",
+ "leader_region_count",
+ "disk_ratio_percent",
+ "data_size"
+ ],
+ "prefix": "All except:",
+ "readOnly": true
+ }
+ },
+ "properties": []
+ }
+ ]
+ },
+ "gridPos": {
+ "h": 11,
+ "w": 12,
+ "x": 0,
+ "y": 83
+ },
+ "id": 368,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "disableTextWrap": false,
+ "editorMode": "builder",
+ "expr": "greptime_datanode_history_load",
+ "fullMetaSearch": false,
+ "includeNullMetadata": true,
+ "instant": false,
+ "legendFormat": "datanode-{{datanode_id}}({{instance}})",
+ "range": true,
+ "refId": "A",
+ "useBackend": false
+ }
+ ],
+ "title": "Datanode Load(Write)",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Distribution of write load across datanodes.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ }
+ },
+ "fieldMinMax": false,
+ "mappings": [],
+ "min": 0,
+ "unit": "binBps"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 11,
+ "w": 12,
+ "x": 12,
+ "y": 83
+ },
+ "id": 369,
+ "options": {
+ "displayLabels": [
+ "percent"
+ ],
+ "legend": {
+ "displayMode": "list",
+ "placement": "right",
+ "showLegend": true
+ },
+ "pieType": "donut",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "disableTextWrap": false,
+ "editorMode": "builder",
+ "exemplar": false,
+ "expr": "greptime_datanode_history_load",
+ "format": "time_series",
+ "fullMetaSearch": false,
+ "includeNullMetadata": true,
+ "instant": true,
+ "legendFormat": "datanode-{{datanode_id}}({{instance}})",
+ "range": false,
+ "refId": "A",
+ "useBackend": false
+ }
+ ],
+ "title": "Datanode Load(Write) Distribution",
+ "type": "piechart"
+ },
+ {
+ "datasource": {
+ "type": "mysql",
+ "uid": "${information_schema}"
+ },
+ "description": "Distribution of leader regions and data size across datanodes.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ }
+ },
+ "fieldMinMax": false,
+ "mappings": [],
+ "unit": "bytes"
+ },
+ "overrides": [
+ {
+ "__systemRef": "hideSeriesFrom",
+ "matcher": {
+ "id": "byNames",
+ "options": {
+ "mode": "exclude",
+ "names": [
+ "total_region_rows",
+ "total_written_bytes_since_open",
+ "total_disk_size",
+ "total_sst_size",
+ "total_sst_num",
+ "total_memtable_size",
+ "total_manifest_size",
+ "total_index_size",
+ "cluster_disk_ratio_percent",
+ "cluster_sst_ratio_percent",
+ "cluster_rows_ratio_percent",
+ "max_region_disk_size",
+ "max_region_rows",
+ "leader_region_count",
+ "disk_ratio_percent",
+ "data_size"
+ ],
+ "prefix": "All except:",
+ "readOnly": true
+ }
+ },
+ "properties": [
+ {
+ "id": "custom.hideFrom",
+ "value": {
+ "legend": false,
+ "tooltip": false,
+ "viz": true
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "gridPos": {
+ "h": 11,
+ "w": 12,
+ "x": 0,
+ "y": 94
+ },
+ "id": 367,
+ "options": {
+ "displayLabels": [
+ "value"
+ ],
+ "legend": {
+ "displayMode": "list",
+ "placement": "right",
+ "showLegend": true,
+ "values": [
+ "percent"
+ ]
+ },
+ "pieType": "pie",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "/^data_size$/",
+ "values": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "dataset": "information_schema",
+ "datasource": {
+ "type": "mysql",
+ "uid": "${information_schema}"
+ },
+ "editorMode": "code",
+ "format": "table",
+ "rawQuery": true,
+ "rawSql": "WITH leader_regions AS (\n SELECT\n CONCAT(\n 'datanode-',\n p.peer_id,\n ' (',\n p.peer_addr,\n ')'\n ) AS datanode,\n r.disk_size\n FROM information_schema.region_statistics r\n JOIN information_schema.region_peers p\n ON r.region_id = p.region_id\n WHERE r.region_role = 'Leader'\n AND p.is_leader = 'Yes'\n)\n\nSELECT\n datanode,\n COUNT(*) AS leader_region_count,\n SUM(disk_size) AS data_size\nFROM leader_regions\nGROUP BY datanode\nORDER BY data_size DESC;",
+ "refId": "A",
+ "sql": {
+ "columns": [
+ {
+ "parameters": [],
+ "type": "function"
+ }
+ ],
+ "groupBy": [
+ {
+ "property": {
+ "type": "string"
+ },
+ "type": "groupBy"
+ }
+ ],
+ "limit": 50
+ }
+ }
+ ],
+ "title": "Datanode Data Distribution",
+ "type": "piechart"
+ },
+ {
+ "collapsed": false,
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 105
+ },
+ "id": 370,
+ "panels": [],
+ "title": "Autopilot",
+ "type": "row"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Region balancer action count",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 106
+ },
+ "id": 371,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by (result) (\n changes(greptime_region_balancer_actions_total[$__rate_interval])\n)",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "{{result}}",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Region Balancer Actions",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Region balancer gate stop count by gate and reason",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 106
+ },
+ "id": 372,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by (gate, reason) (changes(greptime_region_balancer_gate_stop_total[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "{{gate}} / {{reason}}",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Region Balancer Gate Stops",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Region balancer datanode count by state",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 114
+ },
+ "id": 373,
+ "options": {
+ "colorMode": "value",
+ "graphMode": "area",
+ "justifyMode": "auto",
+ "orientation": "auto",
+ "percentChangeColorMode": "standard",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "showPercentChange": false,
+ "textMode": "auto",
+ "wideLayout": true
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by (state) (greptime_region_balancer_datanodes)",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "{{state}}",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Region Balancer Datanodes",
+ "type": "stat"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Region balancer region count by state",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 114
+ },
+ "id": 374,
+ "options": {
+ "colorMode": "value",
+ "graphMode": "area",
+ "justifyMode": "auto",
+ "orientation": "auto",
+ "percentChangeColorMode": "standard",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "showPercentChange": false,
+ "textMode": "auto",
+ "wideLayout": true
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by (state) (greptime_region_balancer_regions)",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "{{state}}",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Region Balancer Regions",
+ "type": "stat"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Region balancer datanode stability statistics by state",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ }
+ ]
+ },
+ "unit": "binBps"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byRegexp",
+ "options": ".*count$"
+ },
+ "properties": [
+ {
+ "id": "unit",
+ "value": "short"
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byRegexp",
+ "options": ".*ratio$"
+ },
+ "properties": [
+ {
+ "id": "unit",
+ "value": "percentunit"
+ }
+ ]
+ }
+ ]
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 24,
+ "x": 0,
+ "y": 122
+ },
+ "id": 375,
+ "options": {
+ "colorMode": "value",
+ "graphMode": "area",
+ "justifyMode": "auto",
+ "orientation": "auto",
+ "percentChangeColorMode": "standard",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "showPercentChange": false,
+ "textMode": "auto",
+ "wideLayout": true
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "builder",
+ "expr": "sum by (state) (greptime_region_balancer_datanode_stability)",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "{{state}}",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Region Balancer Datanode Stability",
+ "type": "stat"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Auto repartition action count by result",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": -1,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineStyle": {
+ "fill": "solid"
+ },
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 130
+ },
+ "id": 376,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by (result) (changes(greptime_auto_repartition_actions_total[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "{{result}}",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Auto Repartition Actions",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Auto repartition gate stop count by gate and reason",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 130
+ },
+ "id": 377,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by (gate, reason) (changes(greptime_auto_repartition_gate_stop_total[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "{{gate}} / {{reason}}",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Auto Repartition Gate Stops",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Auto repartition sampling elapsed time by stage",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "points",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "s"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 138
+ },
+ "id": 378,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.99, sum by (le, stage) (rate(greptime_auto_repartition_sampling_elapsed_bucket[$__rate_interval])))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "{{stage}}",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Auto Repartition Sampling P99",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Auto repartition executor elapsed time by stage",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "points",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "smooth",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "s"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 138
+ },
+ "id": 379,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.99, sum by (le, stage) (rate(greptime_auto_repartition_executor_elapsed_bucket[$__rate_interval])))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "{{stage}}",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Auto Repartition Executor P99",
+ "type": "timeseries"
}
],
"preload": false,
@@ -10359,7 +11686,7 @@
},
"timepicker": {},
"timezone": "",
- "title": "GreptimeDB",
- "uid": "dejf3k5e7g2kgb",
- "version": 15
-}
+ "title": "GreptimeDB Copy",
+ "uid": "dflfbxbwvvchsa",
+ "version": 67
+}
\ No newline at end of file
diff --git a/grafana/dashboards/metrics/standalone/dashboard.md b/grafana/dashboards/metrics/standalone/dashboard.md
index 63b45208c5..672a2a8364 100644
--- a/grafana/dashboards/metrics/standalone/dashboard.md
+++ b/grafana/dashboards/metrics/standalone/dashboard.md
@@ -142,3 +142,101 @@
rate(greptime_trigger_save_alert_record_elapsed_bucket[$__rate_interval])
)` | `timeseries` | Elapsed time to persist trigger alert records. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{storage_type}}]-p99` |
| Save Alert Failure Rate | `rate(greptime_trigger_save_alert_record_failure_count[$__rate_interval])` | `timeseries` | Rate of failures when persisting trigger alert records. | `prometheus` | `none` | `__auto` |
+# Hotspot
+| Title | Query | Type | Description | Datasource | Unit | Legend Format |
+| --- | --- | --- | --- | --- | --- | --- |
+| Hotspot Regions | `WITH table_stats AS (
+ SELECT
+ table_id,
+ COUNT(*) AS region_count,
+ SUM(disk_size) AS total_disk_size,
+ SUM(region_rows) as total_region_rows
+ FROM information_schema.region_statistics
+ WHERE region_role = 'Leader'
+ GROUP BY table_id
+ HAVING COUNT(*) > 1
+)
+
+SELECT
+ t.table_schema,
+ t.table_name,
+
+ r.region_id,
+ t.table_id,
+ r.region_number,
+
+ p.partition_description,
+
+
+ ROUND(
+ r.disk_size * 100.0
+ / NULLIF(ts.total_disk_size, 0),
+ 2
+ ) AS disk_size_share_percent,
+
+ r.disk_size,
+
+ ROUND(
+ r.region_rows * 100.0
+ / NULLIF(ts.total_region_rows, 0),
+ 2
+ ) AS region_rows_share_percent,
+ r.region_rows
+
+FROM information_schema.region_statistics r
+
+JOIN table_stats ts
+ ON r.table_id = ts.table_id
+
+JOIN information_schema.tables t
+ ON r.table_id = t.table_id
+
+LEFT JOIN information_schema.partitions p
+ ON p.table_schema = t.table_schema
+ AND p.table_name = t.table_name
+ AND p.greptime_partition_id = r.region_id
+
+WHERE r.region_role = 'Leader'
+
+ORDER BY region_rows_share_percent DESC
+LIMIT 100;` | `table` | | `mysql` | -- | -- |
+| Datanode Load(Write) | `greptime_datanode_history_load` | `timeseries` | Write load of each datanode over time. | `prometheus` | `binBps` | `datanode-{{datanode_id}}({{instance}})` |
+| Datanode Load(Write) Distribution | `greptime_datanode_history_load` | `piechart` | Distribution of write load across datanodes. | `prometheus` | `binBps` | `datanode-{{datanode_id}}({{instance}})` |
+| Datanode Data Distribution | `WITH leader_regions AS (
+ SELECT
+ CONCAT(
+ 'datanode-',
+ p.peer_id,
+ ' (',
+ p.peer_addr,
+ ')'
+ ) AS datanode,
+ r.disk_size
+ FROM information_schema.region_statistics r
+ JOIN information_schema.region_peers p
+ ON r.region_id = p.region_id
+ WHERE r.region_role = 'Leader'
+ AND p.is_leader = 'Yes'
+)
+
+SELECT
+ datanode,
+ COUNT(*) AS leader_region_count,
+ SUM(disk_size) AS data_size
+FROM leader_regions
+GROUP BY datanode
+ORDER BY data_size DESC;` | `piechart` | Distribution of leader regions and data size across datanodes. | `mysql` | `bytes` | -- |
+# Autopilot
+| Title | Query | Type | Description | Datasource | Unit | Legend Format |
+| --- | --- | --- | --- | --- | --- | --- |
+| Region Balancer Actions | `sum by (result) (
+ changes(greptime_region_balancer_actions_total[$__rate_interval])
+)` | `timeseries` | Region balancer action count | `prometheus` | `short` | `{{result}}` |
+| Region Balancer Gate Stops | `sum by (gate, reason) (changes(greptime_region_balancer_gate_stop_total[$__rate_interval]))` | `timeseries` | Region balancer gate stop count by gate and reason | `prometheus` | `short` | `{{gate}} / {{reason}}` |
+| Region Balancer Datanodes | `sum by (state) (greptime_region_balancer_datanodes)` | `stat` | Region balancer datanode count by state | `prometheus` | `short` | `{{state}}` |
+| Region Balancer Regions | `sum by (state) (greptime_region_balancer_regions)` | `stat` | Region balancer region count by state | `prometheus` | `short` | `{{state}}` |
+| Region Balancer Datanode Stability | `sum by (state) (greptime_region_balancer_datanode_stability)` | `stat` | Region balancer datanode stability statistics by state | `prometheus` | `binBps` | `{{state}}` |
+| Auto Repartition Actions | `sum by (result) (changes(greptime_auto_repartition_actions_total[$__rate_interval]))` | `timeseries` | Auto repartition action count by result | `prometheus` | `short` | `{{result}}` |
+| Auto Repartition Gate Stops | `sum by (gate, reason) (changes(greptime_auto_repartition_gate_stop_total[$__rate_interval]))` | `timeseries` | Auto repartition gate stop count by gate and reason | `prometheus` | `short` | `{{gate}} / {{reason}}` |
+| Auto Repartition Sampling P99 | `histogram_quantile(0.99, sum by (le, stage) (rate(greptime_auto_repartition_sampling_elapsed_bucket[$__rate_interval])))` | `timeseries` | Auto repartition sampling elapsed time by stage | `prometheus` | `s` | `{{stage}}` |
+| Auto Repartition Executor P99 | `histogram_quantile(0.99, sum by (le, stage) (rate(greptime_auto_repartition_executor_elapsed_bucket[$__rate_interval])))` | `timeseries` | Auto repartition executor elapsed time by stage | `prometheus` | `s` | `{{stage}}` |
diff --git a/grafana/dashboards/metrics/standalone/dashboard.yaml b/grafana/dashboards/metrics/standalone/dashboard.yaml
index 32f771f66f..c23501c074 100644
--- a/grafana/dashboards/metrics/standalone/dashboard.yaml
+++ b/grafana/dashboards/metrics/standalone/dashboard.yaml
@@ -1153,3 +1153,160 @@ groups:
type: prometheus
uid: ${metrics}
legendFormat: __auto
+ - title: Hotspot
+ panels:
+ - title: Hotspot Regions
+ type: table
+ queries:
+ - expr: "WITH table_stats AS (\n SELECT\n table_id,\n COUNT(*) AS region_count,\n SUM(disk_size) AS total_disk_size,\n SUM(region_rows) as total_region_rows\n FROM information_schema.region_statistics\n WHERE region_role = 'Leader'\n GROUP BY table_id\n HAVING COUNT(*) > 1\n)\n\nSELECT\n t.table_schema,\n t.table_name,\n\n r.region_id,\n t.table_id,\n r.region_number,\n\n p.partition_description,\n\n\n ROUND(\n r.disk_size * 100.0\n / NULLIF(ts.total_disk_size, 0),\n 2\n ) AS disk_size_share_percent,\n\n r.disk_size,\n \n ROUND(\n r.region_rows * 100.0\n / NULLIF(ts.total_region_rows, 0),\n 2\n ) AS region_rows_share_percent,\n r.region_rows\n\nFROM information_schema.region_statistics r\n\nJOIN table_stats ts\n ON r.table_id = ts.table_id\n\nJOIN information_schema.tables t\n ON r.table_id = t.table_id\n\nLEFT JOIN information_schema.partitions p\n ON p.table_schema = t.table_schema\n AND p.table_name = t.table_name\n AND p.greptime_partition_id = r.region_id\n\nWHERE r.region_role = 'Leader'\n\nORDER BY region_rows_share_percent DESC\nLIMIT 100;"
+ datasource:
+ type: mysql
+ uid: ${information_schema}
+ - title: Datanode Load(Write)
+ type: timeseries
+ description: Write load of each datanode over time.
+ unit: binBps
+ queries:
+ - expr: greptime_datanode_history_load
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: datanode-{{datanode_id}}({{instance}})
+ - title: Datanode Load(Write) Distribution
+ type: piechart
+ description: Distribution of write load across datanodes.
+ unit: binBps
+ queries:
+ - expr: greptime_datanode_history_load
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: datanode-{{datanode_id}}({{instance}})
+ - title: Datanode Data Distribution
+ type: piechart
+ description: Distribution of leader regions and data size across datanodes.
+ unit: bytes
+ queries:
+ - expr: |-
+ WITH leader_regions AS (
+ SELECT
+ CONCAT(
+ 'datanode-',
+ p.peer_id,
+ ' (',
+ p.peer_addr,
+ ')'
+ ) AS datanode,
+ r.disk_size
+ FROM information_schema.region_statistics r
+ JOIN information_schema.region_peers p
+ ON r.region_id = p.region_id
+ WHERE r.region_role = 'Leader'
+ AND p.is_leader = 'Yes'
+ )
+
+ SELECT
+ datanode,
+ COUNT(*) AS leader_region_count,
+ SUM(disk_size) AS data_size
+ FROM leader_regions
+ GROUP BY datanode
+ ORDER BY data_size DESC;
+ datasource:
+ type: mysql
+ uid: ${information_schema}
+ - title: Autopilot
+ panels:
+ - title: Region Balancer Actions
+ type: timeseries
+ description: Region balancer action count
+ unit: short
+ queries:
+ - expr: |-
+ sum by (result) (
+ changes(greptime_region_balancer_actions_total[$__rate_interval])
+ )
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: '{{result}}'
+ - title: Region Balancer Gate Stops
+ type: timeseries
+ description: Region balancer gate stop count by gate and reason
+ unit: short
+ queries:
+ - expr: sum by (gate, reason) (changes(greptime_region_balancer_gate_stop_total[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: '{{gate}} / {{reason}}'
+ - title: Region Balancer Datanodes
+ type: stat
+ description: Region balancer datanode count by state
+ unit: short
+ queries:
+ - expr: sum by (state) (greptime_region_balancer_datanodes)
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: '{{state}}'
+ - title: Region Balancer Regions
+ type: stat
+ description: Region balancer region count by state
+ unit: short
+ queries:
+ - expr: sum by (state) (greptime_region_balancer_regions)
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: '{{state}}'
+ - title: Region Balancer Datanode Stability
+ type: stat
+ description: Region balancer datanode stability statistics by state
+ unit: binBps
+ queries:
+ - expr: sum by (state) (greptime_region_balancer_datanode_stability)
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: '{{state}}'
+ - title: Auto Repartition Actions
+ type: timeseries
+ description: Auto repartition action count by result
+ unit: short
+ queries:
+ - expr: sum by (result) (changes(greptime_auto_repartition_actions_total[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: '{{result}}'
+ - title: Auto Repartition Gate Stops
+ type: timeseries
+ description: Auto repartition gate stop count by gate and reason
+ unit: short
+ queries:
+ - expr: sum by (gate, reason) (changes(greptime_auto_repartition_gate_stop_total[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: '{{gate}} / {{reason}}'
+ - title: Auto Repartition Sampling P99
+ type: timeseries
+ description: Auto repartition sampling elapsed time by stage
+ unit: s
+ queries:
+ - expr: histogram_quantile(0.99, sum by (le, stage) (rate(greptime_auto_repartition_sampling_elapsed_bucket[$__rate_interval])))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: '{{stage}}'
+ - title: Auto Repartition Executor P99
+ type: timeseries
+ description: Auto repartition executor elapsed time by stage
+ unit: s
+ queries:
+ - expr: histogram_quantile(0.99, sum by (le, stage) (rate(greptime_auto_repartition_executor_elapsed_bucket[$__rate_interval])))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: '{{stage}}'
diff --git a/src/api/src/helper.rs b/src/api/src/helper.rs
index 4664c0434b..3b305d90c5 100644
--- a/src/api/src/helper.rs
+++ b/src/api/src/helper.rs
@@ -444,7 +444,7 @@ impl TryFrom for ColumnDataTypeWrapper {
JsonFormat::Jsonb => Some(ColumnDataTypeExtension {
type_ext: Some(TypeExt::JsonType(JsonTypeExtension::JsonBinary.into())),
}),
- JsonFormat::Native(native_type) => {
+ JsonFormat::Json2(native_type) => {
if native_type.is_null() {
None
} else {
@@ -919,6 +919,7 @@ pub fn encode_json_value(value: JsonValue) -> v1::JsonValue {
.collect::>();
Some(json_value::Value::Object(JsonObject { entries }))
}
+ JsonVariant::Variant(x) => Some(json_value::Value::Variant(x)),
};
v1::JsonValue { value }
}
@@ -952,6 +953,7 @@ fn decode_json_value(value: &v1::JsonValue) -> JsonValueRef<'_> {
})
.collect::>()
.into(),
+ json_value::Value::Variant(x) => x.as_slice().into(),
}
}
diff --git a/src/cache/Cargo.toml b/src/cache/Cargo.toml
index 3c128fe1ad..814eb9e484 100644
--- a/src/cache/Cargo.toml
+++ b/src/cache/Cargo.toml
@@ -9,6 +9,6 @@ catalog.workspace = true
common-error.workspace = true
common-macro.workspace = true
common-meta.workspace = true
-moka.workspace = true
+moka = { workspace = true, features = ["future"] }
partition.workspace = true
snafu.workspace = true
diff --git a/src/catalog/src/kvbackend.rs b/src/catalog/src/kvbackend.rs
index 334acc999c..77eb3f2048 100644
--- a/src/catalog/src/kvbackend.rs
+++ b/src/catalog/src/kvbackend.rs
@@ -20,6 +20,6 @@ mod table_cache;
pub use builder::{
CatalogManagerConfigurator, CatalogManagerConfiguratorRef, KvBackendCatalogManagerBuilder,
};
-pub use client::{CachedKvBackend, CachedKvBackendBuilder, MetaKvBackend};
+pub use client::{CachedKvBackend, CachedKvBackendBuilder, new_read_only_meta_kv_backend};
pub use manager::KvBackendCatalogManager;
pub use table_cache::{TableCache, TableCacheRef, new_table_cache};
diff --git a/src/catalog/src/kvbackend/client.rs b/src/catalog/src/kvbackend/client.rs
index f74509217f..69b31d6232 100644
--- a/src/catalog/src/kvbackend/client.rs
+++ b/src/catalog/src/kvbackend/client.rs
@@ -21,7 +21,10 @@ use std::time::Duration;
use common_error::ext::BoxedError;
use common_meta::cache_invalidator::KvCacheInvalidator;
use common_meta::error::Error::CacheNotGet;
-use common_meta::error::{CacheNotGetSnafu, Error, ExternalSnafu, GetKvCacheSnafu, Result};
+use common_meta::error::{
+ CacheNotGetSnafu, Error, ExternalSnafu, GetKvCacheSnafu, Result, UnsupportedSnafu,
+};
+use common_meta::kv_backend::read_only::ReadOnlyKvBackend;
use common_meta::kv_backend::txn::{Txn, TxnResponse};
use common_meta::kv_backend::{KvBackend, KvBackendRef, TxnService};
use common_meta::rpc::KeyValue;
@@ -357,19 +360,35 @@ impl CachedKvBackend {
}
#[derive(Debug)]
-pub struct MetaKvBackend {
- pub client: Arc,
+pub(crate) struct MetaKvBackend {
+ client: Arc,
}
impl MetaKvBackend {
/// Constructs a [MetaKvBackend].
- pub fn new(client: Arc) -> MetaKvBackend {
+ fn new(client: Arc) -> MetaKvBackend {
MetaKvBackend { client }
}
}
+pub fn new_read_only_meta_kv_backend(client: Arc) -> KvBackendRef {
+ Arc::new(ReadOnlyKvBackend::new(Arc::new(MetaKvBackend::new(client))))
+}
+
+#[async_trait::async_trait]
impl TxnService for MetaKvBackend {
type Error = Error;
+
+ async fn txn(&self, _txn: Txn) -> Result {
+ UnsupportedSnafu {
+ operation: "MetaKvBackend txn",
+ }
+ .fail()
+ }
+
+ fn max_txn_ops(&self) -> usize {
+ usize::MAX
+ }
}
/// Implement `KvBackend` trait for `MetaKvBackend` instead of opendal's `Accessor` since
@@ -465,6 +484,9 @@ mod tests {
use std::sync::atomic::{AtomicU32, Ordering};
use async_trait::async_trait;
+ use common_meta::kv_backend::memory::MemoryKvBackend;
+ use common_meta::kv_backend::read_only::ReadOnlyKvBackend;
+ use common_meta::kv_backend::txn::{Txn, TxnOp};
use common_meta::kv_backend::{KvBackend, TxnService};
use common_meta::rpc::KeyValue;
use common_meta::rpc::store::{
@@ -473,8 +495,9 @@ mod tests {
PutResponse, RangeRequest, RangeResponse,
};
use dashmap::DashMap;
+ use meta_client::client::MetaClientBuilder;
- use super::CachedKvBackend;
+ use super::{CachedKvBackend, new_read_only_meta_kv_backend};
#[derive(Default)]
pub struct SimpleKvBackend {
@@ -579,6 +602,62 @@ mod tests {
}
}
+ #[tokio::test]
+ async fn test_cached_kv_backend_rejects_writes_with_read_only_inner() {
+ let inner = Arc::new(MemoryKvBackend::::new());
+ let cached_kv = CachedKvBackend::wrap(Arc::new(ReadOnlyKvBackend::new(inner)));
+
+ let err = cached_kv
+ .put(PutRequest {
+ key: b"k1".to_vec(),
+ value: b"v1".to_vec(),
+ prev_kv: false,
+ })
+ .await
+ .unwrap_err();
+
+ assert!(matches!(
+ err,
+ common_meta::error::Error::ReadOnlyKvBackend { .. }
+ ));
+ }
+
+ #[tokio::test]
+ async fn test_read_only_meta_kv_backend_rejects_writes() {
+ let meta_client = Arc::new(MetaClientBuilder::frontend_default_options().build());
+ let backend = new_read_only_meta_kv_backend(meta_client);
+
+ let err = backend
+ .put(PutRequest {
+ key: b"k1".to_vec(),
+ value: b"v1".to_vec(),
+ prev_kv: false,
+ })
+ .await
+ .unwrap_err();
+
+ assert!(matches!(
+ err,
+ common_meta::error::Error::ReadOnlyKvBackend { .. }
+ ));
+ }
+
+ #[tokio::test]
+ async fn test_read_only_meta_kv_backend_does_not_emulate_txn() {
+ let meta_client = Arc::new(MetaClientBuilder::frontend_default_options().build());
+ let backend = new_read_only_meta_kv_backend(meta_client);
+
+ let result = backend
+ .txn(Txn::new().and_then(vec![TxnOp::Get(b"k1".to_vec())]))
+ .await;
+ let err = match result {
+ Ok(_) => panic!("expected unsupported txn error"),
+ Err(err) => err,
+ };
+
+ assert!(matches!(err, common_meta::error::Error::Unsupported { .. }));
+ }
+
async fn add_some_vals(kv_backend: &impl KvBackend) {
kv_backend
.put(PutRequest {
diff --git a/src/catalog/src/process_manager.rs b/src/catalog/src/process_manager.rs
index 8796d948e1..49be14933c 100644
--- a/src/catalog/src/process_manager.rs
+++ b/src/catalog/src/process_manager.rs
@@ -58,6 +58,8 @@ pub enum QueryStatement {
Sql(Statement),
// The optional string is the alias of the PromQL query.
Promql(EvalStmt, Option),
+ /// Logical plan with original query string
+ Plan(String),
}
impl Display for QueryStatement {
@@ -71,6 +73,7 @@ impl Display for QueryStatement {
write!(f, "{}", eval_stmt)
}
}
+ QueryStatement::Plan(query) => write!(f, "{}", query),
}
}
}
@@ -170,7 +173,7 @@ impl ProcessManager {
let mut processes = vec![];
if let Some(remote_frontend_selector) = self.frontend_selector.as_ref() {
let frontends = remote_frontend_selector
- .select(|node| node.peer.addr != self.server_addr)
+ .select(|peer| peer.addr != self.server_addr)
.await
.context(error::InvokeFrontendSnafu)?;
for mut f in frontends {
@@ -208,7 +211,7 @@ impl ProcessManager {
.frontend_selector
.as_ref()
.context(error::MetaClientMissingSnafu)?
- .select(|node| node.peer.addr == server_addr)
+ .select(|peer| peer.addr == server_addr)
.await
.context(error::InvokeFrontendSnafu)?;
ensure!(
@@ -369,6 +372,9 @@ impl SlowQueryTimer {
QueryStatement::Sql(stmt) => {
slow_query_event.query = stmt.to_string();
}
+ QueryStatement::Plan(query) => {
+ slow_query_event.query = query.clone();
+ }
}
match self.record_type {
diff --git a/src/catalog/src/system_schema.rs b/src/catalog/src/system_schema.rs
index 00b96e5292..6c6a5fc202 100644
--- a/src/catalog/src/system_schema.rs
+++ b/src/catalog/src/system_schema.rs
@@ -139,12 +139,16 @@ impl DataSource for SystemTableDataSource {
&self,
request: ScanRequest,
) -> std::result::Result {
- let projected_schema = match &request.projection {
+ let projection = request
+ .projection_input
+ .as_ref()
+ .map(|input| input.projection.clone());
+
+ let projected_schema = match projection.as_ref() {
Some(projection) => self.try_project(projection)?,
None => self.table.schema(),
};
- let projection = request.projection.clone();
let stream = self
.table
.to_stream(request)
diff --git a/src/catalog/src/system_schema/information_schema/flows.rs b/src/catalog/src/system_schema/information_schema/flows.rs
index a751716620..8002b17888 100644
--- a/src/catalog/src/system_schema/information_schema/flows.rs
+++ b/src/catalog/src/system_schema/information_schema/flows.rs
@@ -16,6 +16,7 @@ use std::sync::{Arc, Weak};
use common_catalog::consts::INFORMATION_SCHEMA_FLOW_TABLE_ID;
use common_error::ext::BoxedError;
+use common_meta::ddl::create_flow::FlowType;
use common_meta::key::FlowId;
use common_meta::key::flow::FlowMetadataManager;
use common_meta::key::flow::flow_info::FlowInfoValue;
@@ -71,6 +72,7 @@ pub const CREATED_TIME: &str = "created_time";
pub const UPDATED_TIME: &str = "updated_time";
pub const LAST_EXECUTION_TIME: &str = "last_execution_time";
pub const SOURCE_TABLE_NAMES: &str = "source_table_names";
+pub const FLOWNODE_ADDRS: &str = "flownode_addrs";
/// The `information_schema.flows` to provides information about flows in databases.
#[derive(Debug)]
@@ -95,7 +97,8 @@ impl InformationSchemaFlows {
}
}
- /// for complex fields(including [`SOURCE_TABLE_IDS`], [`FLOWNODE_IDS`] and [`OPTIONS`]), it will be serialized to json string for now
+ /// for complex fields(including [`SOURCE_TABLE_IDS`], [`FLOWNODE_IDS`], [`OPTIONS`] and
+ /// [`FLOWNODE_ADDRS`]), it will be serialized to json string for now
/// TODO(discord9): use a better way to store complex fields like json type
pub(crate) fn schema() -> SchemaRef {
Arc::new(Schema::new(
@@ -119,6 +122,7 @@ impl InformationSchemaFlows {
true,
),
(SOURCE_TABLE_NAMES, CDT::string_datatype(), true),
+ (FLOWNODE_ADDRS, CDT::string_datatype(), true),
]
.into_iter()
.map(|(name, ty, nullable)| ColumnSchema::new(name, ty, nullable))
@@ -165,6 +169,10 @@ impl InformationSchemaFlows {
expire_after: flow_info.expire_after(),
eval_interval: flow_info.eval_interval(),
comment,
+ flow_options: sql::statements::OptionMap::from_filtered_string_map(
+ flow_info.options(),
+ &[FlowType::FLOW_TYPE_KEY],
+ ),
query,
};
@@ -239,6 +247,7 @@ struct InformationSchemaFlowsBuilder {
updated_time: TimestampMillisecondVectorBuilder,
last_execution_time: TimestampMillisecondVectorBuilder,
source_table_names: StringVectorBuilder,
+ flownode_addr_groups: StringVectorBuilder,
}
impl InformationSchemaFlowsBuilder {
@@ -269,6 +278,7 @@ impl InformationSchemaFlowsBuilder {
updated_time: TimestampMillisecondVectorBuilder::with_capacity(INIT_CAPACITY),
last_execution_time: TimestampMillisecondVectorBuilder::with_capacity(INIT_CAPACITY),
source_table_names: StringVectorBuilder::with_capacity(INIT_CAPACITY),
+ flownode_addr_groups: StringVectorBuilder::with_capacity(INIT_CAPACITY),
}
}
@@ -378,6 +388,21 @@ impl InformationSchemaFlowsBuilder {
.get(&flow_id)
.map(|v| TimestampMillisecond::new(*v))
}));
+ let flownode_addrs = self
+ .flow_metadata_manager
+ .flownode_addrs(flow_id)
+ .await
+ .map_err(BoxedError::new)
+ .context(InternalSnafu)?;
+ if flownode_addrs.is_empty() {
+ self.flownode_addr_groups.push(None);
+ } else {
+ let flownode_addrs_json =
+ serde_json::to_string(&flownode_addrs).with_context(|_| JsonSnafu {
+ input: format!("{:?}", flownode_addrs),
+ })?;
+ self.flownode_addr_groups.push(Some(&flownode_addrs_json));
+ }
let mut source_table_names = vec![];
let catalog_manager = self
@@ -413,6 +438,7 @@ impl InformationSchemaFlowsBuilder {
Arc::new(self.updated_time.finish()),
Arc::new(self.last_execution_time.finish()),
Arc::new(self.source_table_names.finish()),
+ Arc::new(self.flownode_addr_groups.finish()),
];
RecordBatch::new(self.schema.clone(), columns).context(CreateRecordBatchSnafu)
}
diff --git a/src/catalog/src/system_schema/information_schema/ssts.rs b/src/catalog/src/system_schema/information_schema/ssts.rs
index 1c0d507a29..520fc6d485 100644
--- a/src/catalog/src/system_schema/information_schema/ssts.rs
+++ b/src/catalog/src/system_schema/information_schema/ssts.rs
@@ -63,7 +63,7 @@ impl InformationTable for InformationSchemaSstsManifest {
}
fn to_stream(&self, request: ScanRequest) -> Result {
- let schema = if let Some(p) = &request.projection {
+ let schema = if let Some(p) = request.projection_indices() {
Arc::new(self.schema.try_project(p).context(ProjectSchemaSnafu)?)
} else {
self.schema.clone()
@@ -117,7 +117,7 @@ impl InformationTable for InformationSchemaSstsStorage {
}
fn to_stream(&self, request: ScanRequest) -> Result {
- let schema = if let Some(p) = &request.projection {
+ let schema = if let Some(p) = request.projection_indices() {
Arc::new(self.schema.try_project(p).context(ProjectSchemaSnafu)?)
} else {
self.schema.clone()
@@ -172,7 +172,7 @@ impl InformationTable for InformationSchemaSstsIndexMeta {
}
fn to_stream(&self, request: ScanRequest) -> Result {
- let schema = if let Some(p) = &request.projection {
+ let schema = if let Some(p) = request.projection_indices() {
Arc::new(self.schema.try_project(p).context(ProjectSchemaSnafu)?)
} else {
self.schema.clone()
diff --git a/src/cli/Cargo.toml b/src/cli/Cargo.toml
index 1eb2736007..8c4150cd46 100644
--- a/src/cli/Cargo.toml
+++ b/src/cli/Cargo.toml
@@ -44,6 +44,7 @@ common-version.workspace = true
common-wal.workspace = true
datatypes.workspace = true
etcd-client.workspace = true
+fs2.workspace = true
futures.workspace = true
humantime.workspace = true
meta-client.workspace = true
diff --git a/src/cli/src/common/object_store.rs b/src/cli/src/common/object_store.rs
index 0b8372e509..129fb29cb7 100644
--- a/src/cli/src/common/object_store.rs
+++ b/src/cli/src/common/object_store.rs
@@ -220,18 +220,8 @@ impl PrefixedAzblobConnection {
name: "AzBlob",
required: [
(&self.azblob_container, "container"),
- (&self.azblob_root, "root"),
- (&self.azblob_account_name, "account name"),
(&self.azblob_endpoint, "endpoint"),
- ],
- custom_validator: |missing: &mut Vec<&str>| {
- // account_key is only required if sas_token is not provided
- if self.azblob_sas_token.is_none()
- && self.azblob_account_key.is_empty()
- {
- missing.push("account key (when sas_token is not provided)");
- }
- }
+ ]
)
}
}
diff --git a/src/cli/src/common/store.rs b/src/cli/src/common/store.rs
index 373c96a37a..34baaba4ff 100644
--- a/src/cli/src/common/store.rs
+++ b/src/cli/src/common/store.rs
@@ -153,17 +153,11 @@ impl StoreConfig {
BackendImpl::PostgresStore => {
let table_name = &self.meta_table_name;
let tls_config = self.tls_config();
- let pool = meta_srv::utils::postgres::create_postgres_pool(
+ Ok(meta_srv::utils::postgres::build_postgres_kv_backend(
store_addrs,
None,
tls_config,
- )
- .await
- .map_err(BoxedError::new)?;
- let schema_name = self.meta_schema_name.as_deref();
- Ok(common_meta::kv_backend::rds::PgStore::with_pg_pool(
- pool,
- schema_name,
+ self.meta_schema_name.as_deref(),
table_name,
max_txn_ops,
self.auto_create_schema,
@@ -175,12 +169,9 @@ impl StoreConfig {
BackendImpl::MysqlStore => {
let table_name = &self.meta_table_name;
let tls_config = self.tls_config();
- let pool =
- meta_srv::utils::mysql::create_mysql_pool(store_addrs, tls_config.as_ref())
- .await
- .map_err(BoxedError::new)?;
- Ok(common_meta::kv_backend::rds::MySqlStore::with_mysql_pool(
- pool,
+ Ok(meta_srv::utils::mysql::build_mysql_kv_backend(
+ store_addrs,
+ tls_config.as_ref(),
table_name,
max_txn_ops,
)
diff --git a/src/cli/src/data/export.rs b/src/cli/src/data/export.rs
index 051c07da35..148c27316e 100644
--- a/src/cli/src/data/export.rs
+++ b/src/cli/src/data/export.rs
@@ -1084,7 +1084,7 @@ mod tests {
#[tokio::test]
async fn test_export_command_build_with_azblob_empty_account_name() {
- // Test Azure Blob with empty account_name
+ // account_name is optional for Azure Blob validation
let cmd = ExportCommand::parse_from([
"export",
"--addr",
@@ -1092,30 +1092,19 @@ mod tests {
"--azblob",
"--azblob-container",
"test-container",
- "--azblob-root",
- "test-root",
"--azblob-account-name",
"", // Empty account name
- "--azblob-account-key",
- MOCK_AZBLOB_ACCOUNT_KEY_B64,
"--azblob-endpoint",
"https://account.blob.core.windows.net",
]);
let result = cmd.build().await;
- assert!(result.is_err());
- if let Err(err) = result {
- assert!(
- err.to_string().contains("AzBlob account name must be set"),
- "Actual error: {}",
- err
- );
- }
+ assert!(result.is_ok(), "Empty account_name should succeed");
}
#[tokio::test]
async fn test_export_command_build_with_azblob_missing_account_key() {
- // Missing account key
+ // account_key is optional for Azure Blob validation
let cmd = ExportCommand::parse_from([
"export",
"--addr",
@@ -1123,24 +1112,12 @@ mod tests {
"--azblob",
"--azblob-container",
"test-container",
- "--azblob-root",
- "test-root",
- "--azblob-account-name",
- "test-account",
"--azblob-endpoint",
"https://account.blob.core.windows.net",
]);
let result = cmd.build().await;
- assert!(result.is_err());
- if let Err(err) = result {
- assert!(
- err.to_string()
- .contains("AzBlob account key (when sas_token is not provided) must be set"),
- "Actual error: {}",
- err
- );
- }
+ assert!(result.is_ok(), "Missing account_key should succeed");
}
// ==================== Gap 3: Boundary cases ====================
@@ -1238,21 +1215,58 @@ mod tests {
"--azblob",
"--azblob-container",
"test-container",
- "--azblob-root",
- "test-root",
- "--azblob-account-name",
- "test-account",
- "--azblob-account-key",
- MOCK_AZBLOB_ACCOUNT_KEY_B64,
"--azblob-endpoint",
"https://account.blob.core.windows.net",
- // No sas_token
]);
let result = cmd.build().await;
assert!(result.is_ok(), "Minimal AzBlob config should succeed");
}
+ #[tokio::test]
+ async fn test_export_command_build_with_azblob_missing_endpoint() {
+ let cmd = ExportCommand::parse_from([
+ "export",
+ "--addr",
+ "127.0.0.1:4000",
+ "--azblob",
+ "--azblob-container",
+ "test-container",
+ ]);
+
+ let result = cmd.build().await;
+ assert!(result.is_err());
+ if let Err(err) = result {
+ assert!(
+ err.to_string().contains("AzBlob endpoint must be set"),
+ "Actual error: {}",
+ err
+ );
+ }
+ }
+
+ #[tokio::test]
+ async fn test_export_command_build_with_azblob_missing_container() {
+ let cmd = ExportCommand::parse_from([
+ "export",
+ "--addr",
+ "127.0.0.1:4000",
+ "--azblob",
+ "--azblob-endpoint",
+ "https://account.blob.core.windows.net",
+ ]);
+
+ let result = cmd.build().await;
+ assert!(result.is_err());
+ if let Err(err) = result {
+ assert!(
+ err.to_string().contains("AzBlob container must be set"),
+ "Actual error: {}",
+ err
+ );
+ }
+ }
+
#[tokio::test]
async fn test_export_command_build_with_local_and_s3() {
// Both output-dir and S3 - S3 should take precedence
@@ -1287,7 +1301,7 @@ mod tests {
#[tokio::test]
async fn test_export_command_build_with_azblob_only_sas_token() {
- // Azure Blob with sas_token but no account_key - should succeed
+ // Azure Blob with sas_token but no credentials - should still succeed
let cmd = ExportCommand::parse_from([
"export",
"--addr",
@@ -1295,15 +1309,10 @@ mod tests {
"--azblob",
"--azblob-container",
"test-container",
- "--azblob-root",
- "test-root",
- "--azblob-account-name",
- "test-account",
"--azblob-endpoint",
"https://account.blob.core.windows.net",
"--azblob-sas-token",
"test-sas-token",
- // No account_key
]);
let result = cmd.build().await;
@@ -1324,10 +1333,6 @@ mod tests {
"--azblob",
"--azblob-container",
"test-container",
- "--azblob-root",
- "test-root",
- "--azblob-account-name",
- "test-account",
"--azblob-account-key",
"", // Empty account_key is OK if sas_token is provided
"--azblob-endpoint",
diff --git a/src/cli/src/data/export_v2.rs b/src/cli/src/data/export_v2.rs
index 1921ffe4b4..9bdf3c63f5 100644
--- a/src/cli/src/data/export_v2.rs
+++ b/src/cli/src/data/export_v2.rs
@@ -41,7 +41,7 @@
mod chunker;
mod command;
mod coordinator;
-mod data;
+pub(crate) mod data;
pub mod error;
pub mod extractor;
pub mod manifest;
diff --git a/src/cli/src/data/export_v2/command.rs b/src/cli/src/data/export_v2/command.rs
index ddcb323fef..a7da6d3862 100644
--- a/src/cli/src/data/export_v2/command.rs
+++ b/src/cli/src/data/export_v2/command.rs
@@ -34,7 +34,7 @@ use crate::data::export_v2::error::{
};
use crate::data::export_v2::extractor::SchemaExtractor;
use crate::data::export_v2::manifest::{
- ChunkMeta, DataFormat, MANIFEST_VERSION, Manifest, TimeRange,
+ ChunkMeta, DataFormat, MANIFEST_FILE, MANIFEST_VERSION, Manifest, TimeRange,
};
use crate::data::path::ddl_path_for_schema;
use crate::data::snapshot_storage::{OpenDalStorage, SnapshotStorage, validate_uri};
@@ -46,16 +46,73 @@ use crate::database::{DatabaseClient, parse_proxy_opts};
pub enum ExportV2Command {
/// Create a new snapshot.
Create(ExportCreateCommand),
+ /// List snapshots under a parent location.
+ List(ExportListCommand),
}
impl ExportV2Command {
pub async fn build(&self) -> std::result::Result, BoxedError> {
match self {
ExportV2Command::Create(cmd) => cmd.build().await,
+ ExportV2Command::List(cmd) => cmd.build().await,
}
}
}
+/// List snapshots under a parent location.
+#[derive(Debug, Parser)]
+pub struct ExportListCommand {
+ /// Parent storage location whose direct subdirectories are snapshots.
+ #[clap(long)]
+ location: String,
+
+ /// Object store configuration for remote storage backends.
+ #[clap(flatten)]
+ storage: ObjectStoreConfig,
+}
+
+impl ExportListCommand {
+ pub async fn build(&self) -> std::result::Result, BoxedError> {
+ validate_uri(&self.location).map_err(BoxedError::new)?;
+ let storage = OpenDalStorage::from_parent_uri(&self.location, &self.storage)
+ .map_err(BoxedError::new)?;
+
+ Ok(Box::new(ExportList {
+ location: self.location.clone(),
+ storage,
+ }))
+ }
+}
+
+/// Export list tool implementation.
+pub struct ExportList {
+ location: String,
+ storage: OpenDalStorage,
+}
+
+#[async_trait]
+impl Tool for ExportList {
+ async fn do_work(&self) -> std::result::Result<(), BoxedError> {
+ self.run().await.map_err(BoxedError::new)
+ }
+}
+
+impl ExportList {
+ async fn run(&self) -> Result<()> {
+ let result = scan_snapshots(&self.storage).await?;
+
+ println!("Scanning: {}", self.location);
+ if result.snapshots.is_empty() {
+ println!("No snapshots found.");
+ } else {
+ print_snapshot_list(&result.snapshots, result.unreadable.len());
+ }
+ print_unreadable_warnings(&result.unreadable);
+
+ Ok(())
+ }
+}
+
/// Create a new snapshot.
#[derive(Debug, Parser)]
pub struct ExportCreateCommand {
@@ -628,10 +685,138 @@ fn format_chunk_plan(chunks: &[ChunkMeta]) -> String {
format!("[{}]", items.join(", "))
}
+#[derive(Debug)]
+struct SnapshotListEntry {
+ path: String,
+ manifest: Manifest,
+}
+
+#[derive(Debug, Default)]
+struct SnapshotScanResult {
+ snapshots: Vec,
+ unreadable: Vec,
+}
+
+async fn scan_snapshots(storage: &OpenDalStorage) -> Result {
+ let mut result = SnapshotScanResult::default();
+ for dir in storage.list_direct_child_dirs().await? {
+ let manifest_path = format!("{}/{}", dir.trim_matches('/'), MANIFEST_FILE);
+ let Some(data) = storage.read_file_if_exists(&manifest_path).await? else {
+ continue;
+ };
+
+ match serde_json::from_slice::(&data) {
+ Ok(manifest) => result.snapshots.push(SnapshotListEntry {
+ path: format!("{}/", dir.trim_matches('/')),
+ manifest,
+ }),
+ Err(_) => result
+ .unreadable
+ .push(format!("{}/", dir.trim_matches('/'))),
+ }
+ }
+
+ result
+ .snapshots
+ .sort_by_key(|entry| std::cmp::Reverse(entry.manifest.created_at));
+ result.unreadable.sort();
+ Ok(result)
+}
+
+fn print_snapshot_list(snapshots: &[SnapshotListEntry], unreadable_count: usize) {
+ if unreadable_count == 0 {
+ println!("Found {} snapshots:", snapshots.len());
+ } else {
+ println!(
+ "Found {} snapshots ({} {} skipped: unreadable manifest):",
+ snapshots.len(),
+ unreadable_count,
+ directory_word(unreadable_count)
+ );
+ }
+ println!();
+ println!(
+ " {:<24} {:<36} {:<19} {:<9} {:<7} {:<6} Status",
+ "Path", "ID", "Created", "Catalog", "Schemas", "Chunks"
+ );
+ println!(
+ " {:<24} {:<36} {:<19} {:<9} {:<7} {:<6} {:<10}",
+ "-".repeat(24),
+ "-".repeat(36),
+ "-".repeat(19),
+ "-".repeat(9),
+ "-".repeat(7),
+ "-".repeat(6),
+ "-".repeat(10)
+ );
+ for entry in snapshots {
+ let manifest = &entry.manifest;
+ println!(
+ " {:<24} {:<36} {:<19} {:<9} {:<7} {:<6} {}",
+ entry.path,
+ manifest.snapshot_id,
+ manifest.created_at.format("%Y-%m-%d %H:%M:%S"),
+ manifest.catalog,
+ manifest.schemas.len(),
+ format_list_chunks(manifest),
+ snapshot_status(manifest)
+ );
+ }
+}
+
+fn print_unreadable_warnings(unreadable: &[String]) {
+ if unreadable.is_empty() {
+ return;
+ }
+
+ println!();
+ println!(
+ "Warning: {} {} had corrupt/unreadable manifest.json:",
+ unreadable.len(),
+ directory_word(unreadable.len())
+ );
+ for path in unreadable {
+ println!(" - {}", path);
+ }
+}
+
+fn directory_word(count: usize) -> &'static str {
+ if count == 1 {
+ "directory"
+ } else {
+ "directories"
+ }
+}
+
+fn snapshot_status(manifest: &Manifest) -> &'static str {
+ if manifest.schema_only {
+ "schema-only"
+ } else if manifest.is_complete() {
+ "complete"
+ } else {
+ "incomplete"
+ }
+}
+
+fn format_list_chunks(manifest: &Manifest) -> String {
+ let total = manifest.chunks.len();
+ if total == 0 {
+ return "0".to_string();
+ }
+
+ format!(
+ "{}/{}",
+ manifest.completed_count() + manifest.skipped_count(),
+ total
+ )
+}
+
#[cfg(test)]
mod tests {
use chrono::TimeZone;
use clap::Parser;
+ use tempfile::tempdir;
+ use url::Url;
use super::*;
use crate::data::path::ddl_path_for_schema;
@@ -886,4 +1071,124 @@ mod tests {
.to_string();
assert!(error.contains("time_range"));
}
+
+ #[tokio::test]
+ async fn test_scan_snapshots_sorts_and_tracks_unreadable_manifests() {
+ let dir = tempdir().unwrap();
+ write_test_manifest(
+ dir.path(),
+ "older",
+ test_manifest(
+ chrono::Utc.with_ymd_and_hms(2026, 1, 1, 0, 0, 0).unwrap(),
+ false,
+ true,
+ ),
+ );
+ write_test_manifest(
+ dir.path(),
+ "newer",
+ test_manifest(
+ chrono::Utc.with_ymd_and_hms(2026, 2, 1, 0, 0, 0).unwrap(),
+ false,
+ true,
+ ),
+ );
+
+ std::fs::create_dir_all(dir.path().join("empty-dir")).unwrap();
+ std::fs::create_dir_all(dir.path().join("not-snapshot")).unwrap();
+ std::fs::write(dir.path().join("not-snapshot").join("data.txt"), "x").unwrap();
+ std::fs::create_dir_all(dir.path().join("broken")).unwrap();
+ std::fs::write(dir.path().join("broken").join(MANIFEST_FILE), "{not-json").unwrap();
+
+ let uri = Url::from_directory_path(dir.path()).unwrap().to_string();
+ let storage = OpenDalStorage::from_file_uri(&uri).unwrap();
+ let result = scan_snapshots(&storage).await.unwrap();
+
+ assert_eq!(result.snapshots.len(), 2);
+ assert_eq!(
+ result.snapshots[0].manifest.created_at,
+ chrono::Utc.with_ymd_and_hms(2026, 2, 1, 0, 0, 0).unwrap()
+ );
+ assert_eq!(
+ result.snapshots[1].manifest.created_at,
+ chrono::Utc.with_ymd_and_hms(2026, 1, 1, 0, 0, 0).unwrap()
+ );
+ assert_eq!(result.unreadable, vec!["broken/".to_string()]);
+ assert_eq!(result.snapshots[0].path, "newer/");
+ assert_eq!(result.snapshots[1].path, "older/");
+ }
+
+ #[test]
+ fn test_snapshot_list_status_and_chunk_summary() {
+ let schema_only = test_manifest(
+ chrono::Utc.with_ymd_and_hms(2026, 1, 1, 0, 0, 0).unwrap(),
+ true,
+ true,
+ );
+ assert_eq!(snapshot_status(&schema_only), "schema-only");
+ assert_eq!(format_list_chunks(&schema_only), "0");
+
+ let complete = test_manifest(
+ chrono::Utc.with_ymd_and_hms(2026, 1, 1, 0, 0, 0).unwrap(),
+ false,
+ true,
+ );
+ assert_eq!(snapshot_status(&complete), "complete");
+ assert_eq!(format_list_chunks(&complete), "2/2");
+
+ let incomplete = test_manifest(
+ chrono::Utc.with_ymd_and_hms(2026, 1, 1, 0, 0, 0).unwrap(),
+ false,
+ false,
+ );
+ assert_eq!(snapshot_status(&incomplete), "incomplete");
+ assert_eq!(format_list_chunks(&incomplete), "1/2");
+ }
+
+ fn write_test_manifest(root: &std::path::Path, dir: &str, manifest: Manifest) {
+ let snapshot_dir = root.join(dir);
+ std::fs::create_dir_all(&snapshot_dir).unwrap();
+ std::fs::write(
+ snapshot_dir.join(MANIFEST_FILE),
+ serde_json::to_vec_pretty(&manifest).unwrap(),
+ )
+ .unwrap();
+ }
+
+ fn test_manifest(
+ created_at: chrono::DateTime,
+ schema_only: bool,
+ complete: bool,
+ ) -> Manifest {
+ let mut manifest = Manifest::new_for_export(
+ "greptime".to_string(),
+ vec!["public".to_string(), "analytics".to_string()],
+ schema_only,
+ TimeRange::unbounded(),
+ DataFormat::Parquet,
+ None,
+ )
+ .unwrap();
+ manifest.created_at = created_at;
+ manifest.updated_at = created_at;
+
+ if !schema_only {
+ manifest.chunks.clear();
+ let mut first = ChunkMeta::new(1, TimeRange::unbounded());
+ first.mark_completed(vec!["data/public/chunk_1/file.parquet".to_string()], None);
+ manifest.chunks.push(first);
+
+ if complete {
+ manifest
+ .chunks
+ .push(ChunkMeta::skipped(2, TimeRange::unbounded()));
+ } else {
+ manifest
+ .chunks
+ .push(ChunkMeta::new(2, TimeRange::unbounded()));
+ }
+ }
+
+ manifest
+ }
}
diff --git a/src/cli/src/data/export_v2/data.rs b/src/cli/src/data/export_v2/data.rs
index 25d70ee118..dce924a717 100644
--- a/src/cli/src/data/export_v2/data.rs
+++ b/src/cli/src/data/export_v2/data.rs
@@ -38,18 +38,64 @@ pub(super) struct CopyTarget {
secrets: Vec>,
}
+pub(crate) struct CopySource {
+ pub(crate) location: String,
+ pub(crate) connection: String,
+ secrets: Vec>,
+}
+
impl CopyTarget {
fn mask_sql(&self, sql: &str) -> String {
mask_secrets(sql, &self.secrets)
}
}
+impl CopySource {
+ fn mask_sql(&self, sql: &str) -> String {
+ mask_secrets(sql, &self.secrets)
+ }
+}
+
pub(super) fn build_copy_target(
snapshot_uri: &str,
storage: &ObjectStoreConfig,
schema: &str,
chunk_id: u32,
) -> Result {
+ let location = build_copy_location(snapshot_uri, storage, schema, chunk_id)?;
+ Ok(CopyTarget {
+ location: location.location,
+ connection: location.connection,
+ secrets: location.secrets,
+ })
+}
+
+pub(crate) fn build_copy_source(
+ snapshot_uri: &str,
+ storage: &ObjectStoreConfig,
+ schema: &str,
+ chunk_id: u32,
+) -> Result {
+ let location = build_copy_location(snapshot_uri, storage, schema, chunk_id)?;
+ Ok(CopySource {
+ location: location.location,
+ connection: location.connection,
+ secrets: location.secrets,
+ })
+}
+
+struct CopyLocation {
+ location: String,
+ connection: String,
+ secrets: Vec>,
+}
+
+fn build_copy_location(
+ snapshot_uri: &str,
+ storage: &ObjectStoreConfig,
+ schema: &str,
+ chunk_id: u32,
+) -> Result {
let url = Url::parse(snapshot_uri).context(UrlParseSnafu)?;
let scheme = StorageScheme::from_uri(snapshot_uri)?;
let suffix = data_dir_for_schema_chunk(schema, chunk_id);
@@ -64,7 +110,7 @@ pub(super) fn build_copy_target(
.build()
})?;
let location = normalize_path(&format!("{}/{}", root.to_string_lossy(), suffix));
- Ok(CopyTarget {
+ Ok(CopyLocation {
location,
connection: String::new(),
secrets: Vec::new(),
@@ -74,7 +120,7 @@ pub(super) fn build_copy_target(
let (bucket, root) = extract_bucket_root(&url, snapshot_uri)?;
let location = format!("s3://{}/{}", bucket, join_root(&root, &suffix));
let (connection, secrets) = build_s3_connection(storage);
- Ok(CopyTarget {
+ Ok(CopyLocation {
location,
connection,
secrets,
@@ -84,7 +130,7 @@ pub(super) fn build_copy_target(
let (bucket, root) = extract_bucket_root(&url, snapshot_uri)?;
let location = format!("oss://{}/{}", bucket, join_root(&root, &suffix));
let (connection, secrets) = build_oss_connection(storage);
- Ok(CopyTarget {
+ Ok(CopyLocation {
location,
connection,
secrets,
@@ -94,7 +140,7 @@ pub(super) fn build_copy_target(
let (bucket, root) = extract_bucket_root(&url, snapshot_uri)?;
let location = format!("gcs://{}/{}", bucket, join_root(&root, &suffix));
let (connection, secrets) = build_gcs_connection(storage, snapshot_uri)?;
- Ok(CopyTarget {
+ Ok(CopyLocation {
location,
connection,
secrets,
@@ -104,7 +150,7 @@ pub(super) fn build_copy_target(
let (bucket, root) = extract_bucket_root(&url, snapshot_uri)?;
let location = format!("azblob://{}/{}", bucket, join_root(&root, &suffix));
let (connection, secrets) = build_azblob_connection(storage);
- Ok(CopyTarget {
+ Ok(CopyLocation {
location,
connection,
secrets,
@@ -138,6 +184,30 @@ pub(super) async fn execute_copy_database(
Ok(())
}
+pub(crate) async fn execute_copy_database_from(
+ database_client: &DatabaseClient,
+ catalog: &str,
+ schema: &str,
+ source: &CopySource,
+ format: DataFormat,
+) -> Result<()> {
+ let sql = format!(
+ r#"COPY DATABASE "{}"."{}" FROM '{}' WITH (FORMAT='{}'){};"#,
+ escape_sql_identifier(catalog),
+ escape_sql_identifier(schema),
+ escape_sql_literal(&source.location),
+ format,
+ source.connection
+ );
+ let safe_sql = source.mask_sql(&sql);
+ info!("Executing sql: {}", safe_sql);
+ database_client
+ .sql_in_public(&sql)
+ .await
+ .context(DatabaseSnafu)?;
+ Ok(())
+}
+
fn build_with_options(options: &CopyOptions) -> String {
let mut parts = vec![format!("FORMAT='{}'", options.format)];
if let Some(start) = options.time_range.start {
@@ -328,6 +398,10 @@ fn mask_secrets(sql: &str, secrets: &[Option]) -> String {
if let Some(secret) = secret
&& !secret.is_empty()
{
+ let escaped = escape_sql_literal(secret);
+ if escaped != *secret {
+ masked = masked.replace(&escaped, "[REDACTED]");
+ }
masked = masked.replace(secret, "[REDACTED]");
}
}
@@ -430,6 +504,17 @@ mod tests {
assert!(!masked.contains("sig=secret-token"));
}
+ #[test]
+ fn test_mask_secrets_redacts_sql_escaped_literals() {
+ let sql =
+ "COPY DATABASE \"greptime\".\"public\" TO 's3://bucket' CONNECTION (SECRET='ab''cd');";
+ let masked = mask_secrets(sql, &[Some("ab'cd".to_string())]);
+
+ assert!(!masked.contains("ab'cd"));
+ assert!(!masked.contains("ab''cd"));
+ assert!(masked.contains("SECRET='[REDACTED]'"));
+ }
+
#[test]
fn test_build_copy_target_decodes_file_uri_path() {
let storage = ObjectStoreConfig::default();
diff --git a/src/cli/src/data/export_v2/tests.rs b/src/cli/src/data/export_v2/tests.rs
index bd28801a0d..92f45c6aec 100644
--- a/src/cli/src/data/export_v2/tests.rs
+++ b/src/cli/src/data/export_v2/tests.rs
@@ -17,17 +17,49 @@ use std::time::Duration;
use clap::Parser;
use common_error::ext::BoxedError;
+use serde_json::Value;
use snafu::ResultExt;
use tempfile::tempdir;
use url::Url;
use super::command::ExportCreateCommand;
use crate::common::ObjectStoreConfig;
+use crate::data::export_v2::manifest::ChunkStatus;
use crate::data::import_v2::ImportV2Command;
-use crate::data::snapshot_storage::OpenDalStorage;
+use crate::data::snapshot_storage::{OpenDalStorage, SnapshotStorage};
+use crate::data::sql::escape_sql_identifier;
use crate::database::DatabaseClient;
use crate::error::{FileIoSnafu, InvalidArgumentsSnafu, OtherSnafu, Result};
+async fn query_count(database_client: &DatabaseClient, schema: &str, table: &str) -> Result {
+ let sql = format!("SELECT COUNT(*) FROM {}", escape_sql_identifier(table));
+ let rows = database_client.sql(&sql, schema).await?;
+ let first_row = rows.as_ref().and_then(|rows| rows.first()).ok_or_else(|| {
+ InvalidArgumentsSnafu {
+ msg: format!("empty result for query: {sql}"),
+ }
+ .build()
+ })?;
+ let first_value = first_row.first().ok_or_else(|| {
+ InvalidArgumentsSnafu {
+ msg: format!("no first column for query: {sql}"),
+ }
+ .build()
+ })?;
+ match first_value {
+ Value::Number(n) => n.as_u64().ok_or_else(|| {
+ InvalidArgumentsSnafu {
+ msg: format!("count is not u64 for query: {sql}"),
+ }
+ .build()
+ }),
+ _ => InvalidArgumentsSnafu {
+ msg: format!("unexpected count type for query: {sql}"),
+ }
+ .fail(),
+ }
+}
+
#[tokio::test]
#[ignore]
async fn export_import_v2_schema_parity_e2e() -> Result<()> {
@@ -339,3 +371,515 @@ async fn import_v2_ddl_dry_run_e2e() -> Result<()> {
Ok(())
}
+
+#[tokio::test]
+#[ignore]
+async fn export_import_v2_data_roundtrip_e2e() -> Result<()> {
+ let addr = env::var("GREPTIME_ADDR").unwrap_or_else(|_| "127.0.0.1:4000".to_string());
+ let catalog = env::var("GREPTIME_CATALOG").unwrap_or_else(|_| "greptime".to_string());
+ let auth_basic = env::var("GREPTIME_AUTH_BASIC").ok();
+ let schema = "test_db_data_roundtrip";
+
+ let database_client = DatabaseClient::new(
+ addr.clone(),
+ catalog.clone(),
+ auth_basic.clone(),
+ Duration::from_secs(60),
+ None,
+ false,
+ );
+
+ database_client
+ .sql_in_public(&format!("DROP DATABASE IF EXISTS {schema}"))
+ .await?;
+ database_client
+ .sql_in_public(&format!("CREATE DATABASE {schema}"))
+ .await?;
+ database_client
+ .sql(
+ "CREATE TABLE metrics (\
+ ts TIMESTAMP TIME INDEX, \
+ host STRING PRIMARY KEY, \
+ cpu DOUBLE \
+ ) ENGINE=mito",
+ schema,
+ )
+ .await?;
+ database_client
+ .sql(
+ "INSERT INTO metrics (ts, host, cpu) VALUES \
+ ('2025-01-01T00:00:00Z', 'h1', 1.0), \
+ ('2025-01-01T01:00:00Z', 'h2', 2.0)",
+ schema,
+ )
+ .await?;
+
+ let expected_rows = query_count(&database_client, schema, "metrics").await?;
+
+ let src_dir = tempdir().context(FileIoSnafu)?;
+ let src_uri = Url::from_directory_path(src_dir.path())
+ .map_err(|_| {
+ InvalidArgumentsSnafu {
+ msg: "invalid temp dir path".to_string(),
+ }
+ .build()
+ })?
+ .to_string();
+
+ let mut export_args = vec![
+ "export-v2-create",
+ "--addr",
+ &addr,
+ "--to",
+ &src_uri,
+ "--catalog",
+ &catalog,
+ "--schemas",
+ schema,
+ ];
+ if let Some(auth) = &auth_basic {
+ export_args.push("--auth-basic");
+ export_args.push(auth);
+ }
+ let export_cmd = ExportCreateCommand::parse_from(export_args);
+ export_cmd
+ .build()
+ .await
+ .context(OtherSnafu)?
+ .do_work()
+ .await
+ .context(OtherSnafu)?;
+
+ database_client
+ .sql_in_public(&format!("DROP DATABASE IF EXISTS {schema}"))
+ .await?;
+
+ let mut import_args = vec![
+ "import-v2",
+ "--addr",
+ &addr,
+ "--from",
+ &src_uri,
+ "--catalog",
+ &catalog,
+ "--schemas",
+ schema,
+ ];
+ if let Some(auth) = &auth_basic {
+ import_args.push("--auth-basic");
+ import_args.push(auth);
+ }
+ let import_cmd = ImportV2Command::parse_from(import_args);
+ import_cmd
+ .build()
+ .await
+ .context(OtherSnafu)?
+ .do_work()
+ .await
+ .context(OtherSnafu)?;
+
+ let actual_rows = query_count(&database_client, schema, "metrics").await?;
+ assert_eq!(actual_rows, expected_rows);
+
+ database_client
+ .sql_in_public(&format!("DROP DATABASE IF EXISTS {schema}"))
+ .await?;
+
+ Ok(())
+}
+
+#[tokio::test]
+#[ignore]
+async fn import_v2_fails_on_incomplete_snapshot_e2e() -> Result<()> {
+ let addr = env::var("GREPTIME_ADDR").unwrap_or_else(|_| "127.0.0.1:4000".to_string());
+ let catalog = env::var("GREPTIME_CATALOG").unwrap_or_else(|_| "greptime".to_string());
+ let auth_basic = env::var("GREPTIME_AUTH_BASIC").ok();
+ let schema = "test_db_incomplete_snapshot";
+
+ let database_client = DatabaseClient::new(
+ addr.clone(),
+ catalog.clone(),
+ auth_basic.clone(),
+ Duration::from_secs(60),
+ None,
+ false,
+ );
+
+ database_client
+ .sql_in_public(&format!("DROP DATABASE IF EXISTS {schema}"))
+ .await?;
+ database_client
+ .sql_in_public(&format!("CREATE DATABASE {schema}"))
+ .await?;
+ database_client
+ .sql(
+ "CREATE TABLE metrics (\
+ ts TIMESTAMP TIME INDEX, \
+ host STRING PRIMARY KEY, \
+ cpu DOUBLE \
+ ) ENGINE=mito",
+ schema,
+ )
+ .await?;
+ database_client
+ .sql(
+ "INSERT INTO metrics (ts, host, cpu) VALUES ('2025-01-01T00:00:00Z', 'h1', 1.0)",
+ schema,
+ )
+ .await?;
+
+ let src_dir = tempdir().context(FileIoSnafu)?;
+ let src_uri = Url::from_directory_path(src_dir.path())
+ .map_err(|_| {
+ InvalidArgumentsSnafu {
+ msg: "invalid temp dir path".to_string(),
+ }
+ .build()
+ })?
+ .to_string();
+
+ let mut export_args = vec![
+ "export-v2-create",
+ "--addr",
+ &addr,
+ "--to",
+ &src_uri,
+ "--catalog",
+ &catalog,
+ "--schemas",
+ schema,
+ ];
+ if let Some(auth) = &auth_basic {
+ export_args.push("--auth-basic");
+ export_args.push(auth);
+ }
+ let export_cmd = ExportCreateCommand::parse_from(export_args);
+ export_cmd
+ .build()
+ .await
+ .context(OtherSnafu)?
+ .do_work()
+ .await
+ .context(OtherSnafu)?;
+
+ let storage_config = ObjectStoreConfig::default();
+ let storage = OpenDalStorage::from_uri(&src_uri, &storage_config)
+ .map_err(BoxedError::new)
+ .context(OtherSnafu)?;
+ let mut manifest = storage
+ .read_manifest()
+ .await
+ .map_err(BoxedError::new)
+ .context(OtherSnafu)?;
+ if let Some(first_chunk) = manifest.chunks.first_mut() {
+ first_chunk.status = ChunkStatus::Failed;
+ }
+ storage
+ .write_manifest(&manifest)
+ .await
+ .map_err(BoxedError::new)
+ .context(OtherSnafu)?;
+
+ database_client
+ .sql_in_public(&format!("DROP DATABASE IF EXISTS {schema}"))
+ .await?;
+
+ let mut import_args = vec![
+ "import-v2",
+ "--addr",
+ &addr,
+ "--from",
+ &src_uri,
+ "--catalog",
+ &catalog,
+ "--schemas",
+ schema,
+ ];
+ if let Some(auth) = &auth_basic {
+ import_args.push("--auth-basic");
+ import_args.push(auth);
+ }
+ let import_cmd = ImportV2Command::parse_from(import_args);
+ let err = import_cmd
+ .build()
+ .await
+ .context(OtherSnafu)?
+ .do_work()
+ .await
+ .expect_err("import should fail on incomplete snapshot");
+ assert!(err.to_string().contains("Incomplete snapshot"));
+
+ Ok(())
+}
+
+#[tokio::test]
+#[ignore]
+async fn import_v2_schema_filter_data_e2e() -> Result<()> {
+ let addr = env::var("GREPTIME_ADDR").unwrap_or_else(|_| "127.0.0.1:4000".to_string());
+ let catalog = env::var("GREPTIME_CATALOG").unwrap_or_else(|_| "greptime".to_string());
+ let auth_basic = env::var("GREPTIME_AUTH_BASIC").ok();
+ let schema_a = "test_db_filter_a";
+ let schema_b = "test_db_filter_b";
+
+ let database_client = DatabaseClient::new(
+ addr.clone(),
+ catalog.clone(),
+ auth_basic.clone(),
+ Duration::from_secs(60),
+ None,
+ false,
+ );
+
+ for schema in [schema_a, schema_b] {
+ database_client
+ .sql_in_public(&format!("DROP DATABASE IF EXISTS {schema}"))
+ .await?;
+ database_client
+ .sql_in_public(&format!("CREATE DATABASE {schema}"))
+ .await?;
+ database_client
+ .sql(
+ "CREATE TABLE metrics (\
+ ts TIMESTAMP TIME INDEX, \
+ host STRING PRIMARY KEY, \
+ cpu DOUBLE \
+ ) ENGINE=mito",
+ schema,
+ )
+ .await?;
+ }
+ database_client
+ .sql(
+ "INSERT INTO metrics (ts, host, cpu) VALUES ('2025-01-01T00:00:00Z', 'a1', 1.0)",
+ schema_a,
+ )
+ .await?;
+ database_client
+ .sql(
+ "INSERT INTO metrics (ts, host, cpu) VALUES ('2025-01-01T00:00:00Z', 'b1', 2.0)",
+ schema_b,
+ )
+ .await?;
+
+ let expected_rows_a = query_count(&database_client, schema_a, "metrics").await?;
+
+ let src_dir = tempdir().context(FileIoSnafu)?;
+ let src_uri = Url::from_directory_path(src_dir.path())
+ .map_err(|_| {
+ InvalidArgumentsSnafu {
+ msg: "invalid temp dir path".to_string(),
+ }
+ .build()
+ })?
+ .to_string();
+
+ let mut export_args = vec![
+ "export-v2-create",
+ "--addr",
+ &addr,
+ "--to",
+ &src_uri,
+ "--catalog",
+ &catalog,
+ "--schemas",
+ schema_a,
+ "--schemas",
+ schema_b,
+ ];
+ if let Some(auth) = &auth_basic {
+ export_args.push("--auth-basic");
+ export_args.push(auth);
+ }
+ let export_cmd = ExportCreateCommand::parse_from(export_args);
+ export_cmd
+ .build()
+ .await
+ .context(OtherSnafu)?
+ .do_work()
+ .await
+ .context(OtherSnafu)?;
+
+ for schema in [schema_a, schema_b] {
+ database_client
+ .sql_in_public(&format!("DROP DATABASE IF EXISTS {schema}"))
+ .await?;
+ }
+
+ let mut import_args = vec![
+ "import-v2",
+ "--addr",
+ &addr,
+ "--from",
+ &src_uri,
+ "--catalog",
+ &catalog,
+ "--schemas",
+ schema_a,
+ ];
+ if let Some(auth) = &auth_basic {
+ import_args.push("--auth-basic");
+ import_args.push(auth);
+ }
+ let import_cmd = ImportV2Command::parse_from(import_args);
+ import_cmd
+ .build()
+ .await
+ .context(OtherSnafu)?
+ .do_work()
+ .await
+ .context(OtherSnafu)?;
+
+ let actual_rows_a = query_count(&database_client, schema_a, "metrics").await?;
+ assert_eq!(actual_rows_a, expected_rows_a);
+
+ let schema_b_query = database_client
+ .sql("SELECT COUNT(*) FROM metrics", schema_b)
+ .await;
+ assert!(schema_b_query.is_err(), "schema_b should not be imported");
+
+ for schema in [schema_a, schema_b] {
+ database_client
+ .sql_in_public(&format!("DROP DATABASE IF EXISTS {schema}"))
+ .await?;
+ }
+
+ Ok(())
+}
+
+#[tokio::test]
+#[ignore]
+async fn export_import_v2_skipped_chunk_e2e() -> Result<()> {
+ let addr = env::var("GREPTIME_ADDR").unwrap_or_else(|_| "127.0.0.1:4000".to_string());
+ let catalog = env::var("GREPTIME_CATALOG").unwrap_or_else(|_| "greptime".to_string());
+ let auth_basic = env::var("GREPTIME_AUTH_BASIC").ok();
+ let schema = "test_db_skipped_chunk";
+
+ let database_client = DatabaseClient::new(
+ addr.clone(),
+ catalog.clone(),
+ auth_basic.clone(),
+ Duration::from_secs(60),
+ None,
+ false,
+ );
+
+ database_client
+ .sql_in_public(&format!("DROP DATABASE IF EXISTS {schema}"))
+ .await?;
+ database_client
+ .sql_in_public(&format!("CREATE DATABASE {schema}"))
+ .await?;
+ database_client
+ .sql(
+ "CREATE TABLE metrics (\
+ ts TIMESTAMP TIME INDEX, \
+ host STRING PRIMARY KEY, \
+ cpu DOUBLE \
+ ) ENGINE=mito",
+ schema,
+ )
+ .await?;
+ database_client
+ .sql(
+ "INSERT INTO metrics (ts, host, cpu) VALUES \
+ ('2025-01-01T00:00:00Z', 'h1', 1.0), \
+ ('2025-01-01T01:00:00Z', 'h2', 2.0)",
+ schema,
+ )
+ .await?;
+
+ let src_dir = tempdir().context(FileIoSnafu)?;
+ let src_uri = Url::from_directory_path(src_dir.path())
+ .map_err(|_| {
+ InvalidArgumentsSnafu {
+ msg: "invalid temp dir path".to_string(),
+ }
+ .build()
+ })?
+ .to_string();
+
+ let mut export_args = vec![
+ "export-v2-create",
+ "--addr",
+ &addr,
+ "--to",
+ &src_uri,
+ "--catalog",
+ &catalog,
+ "--schemas",
+ schema,
+ "--start-time",
+ "2025-01-01T00:00:00Z",
+ "--end-time",
+ "2025-01-01T02:00:00Z",
+ "--chunk-time-window",
+ "1h",
+ ];
+ if let Some(auth) = &auth_basic {
+ export_args.push("--auth-basic");
+ export_args.push(auth);
+ }
+ let export_cmd = ExportCreateCommand::parse_from(export_args);
+ export_cmd
+ .build()
+ .await
+ .context(OtherSnafu)?
+ .do_work()
+ .await
+ .context(OtherSnafu)?;
+
+ let storage_config = ObjectStoreConfig::default();
+ let storage = OpenDalStorage::from_uri(&src_uri, &storage_config)
+ .map_err(BoxedError::new)
+ .context(OtherSnafu)?;
+ let mut manifest = storage
+ .read_manifest()
+ .await
+ .map_err(BoxedError::new)
+ .context(OtherSnafu)?;
+ assert_eq!(manifest.chunks.len(), 2);
+ manifest.chunks[0].status = ChunkStatus::Skipped;
+ manifest.chunks[0].files.clear();
+ storage
+ .write_manifest(&manifest)
+ .await
+ .map_err(BoxedError::new)
+ .context(OtherSnafu)?;
+
+ database_client
+ .sql_in_public(&format!("DROP DATABASE IF EXISTS {schema}"))
+ .await?;
+
+ let mut import_args = vec![
+ "import-v2",
+ "--addr",
+ &addr,
+ "--from",
+ &src_uri,
+ "--catalog",
+ &catalog,
+ "--schemas",
+ schema,
+ ];
+ if let Some(auth) = &auth_basic {
+ import_args.push("--auth-basic");
+ import_args.push(auth);
+ }
+ let import_cmd = ImportV2Command::parse_from(import_args);
+ import_cmd
+ .build()
+ .await
+ .context(OtherSnafu)?
+ .do_work()
+ .await
+ .context(OtherSnafu)?;
+
+ let actual_rows = query_count(&database_client, schema, "metrics").await?;
+ assert_eq!(actual_rows, 1);
+
+ database_client
+ .sql_in_public(&format!("DROP DATABASE IF EXISTS {schema}"))
+ .await?;
+
+ Ok(())
+}
diff --git a/src/cli/src/data/import_v2.rs b/src/cli/src/data/import_v2.rs
index 772e18cc93..b197f59b3b 100644
--- a/src/cli/src/data/import_v2.rs
+++ b/src/cli/src/data/import_v2.rs
@@ -35,7 +35,9 @@
//! ```
mod command;
+pub(crate) mod coordinator;
pub mod error;
pub mod executor;
+pub(crate) mod state;
pub use command::ImportV2Command;
diff --git a/src/cli/src/data/import_v2/command.rs b/src/cli/src/data/import_v2/command.rs
index 6a9d440071..9ee60252a0 100644
--- a/src/cli/src/data/import_v2/command.rs
+++ b/src/cli/src/data/import_v2/command.rs
@@ -21,17 +21,24 @@ use async_trait::async_trait;
use clap::Parser;
use common_error::ext::BoxedError;
use common_telemetry::info;
-use snafu::ResultExt;
+use snafu::{OptionExt, ResultExt};
use crate::Tool;
use crate::common::ObjectStoreConfig;
-use crate::data::export_v2::manifest::MANIFEST_VERSION;
+use crate::data::export_v2::data::{build_copy_source, execute_copy_database_from};
+use crate::data::export_v2::manifest::{ChunkMeta, ChunkStatus, DataFormat, MANIFEST_VERSION};
+use crate::data::import_v2::coordinator::{
+ ImportResumeConfig, ImportTaskExecutor, build_import_tasks, chunk_has_schema_files,
+ import_with_resume_session, prepare_import_resume,
+};
use crate::data::import_v2::error::{
- FullSnapshotImportNotSupportedSnafu, ManifestVersionMismatchSnafu, Result,
+ ChunkImportFailedSnafu, EmptyChunkManifestSnafu, ImportStatePathUnavailableSnafu,
+ IncompleteSnapshotSnafu, ManifestVersionMismatchSnafu, MissingChunkDataSnafu, Result,
SchemaNotInSnapshotSnafu, SnapshotStorageSnafu,
};
use crate::data::import_v2::executor::{DdlExecutor, DdlStatement};
-use crate::data::path::ddl_path_for_schema;
+use crate::data::import_v2::state::{ImportTaskKey, default_state_path};
+use crate::data::path::{data_dir_for_schema_chunk, ddl_path_for_schema};
use crate::data::snapshot_storage::{OpenDalStorage, SnapshotStorage, validate_uri};
use crate::database::{DatabaseClient, parse_proxy_opts};
@@ -116,8 +123,11 @@ impl ImportV2Command {
);
Ok(Box::new(Import {
+ catalog: self.catalog.clone(),
schemas,
dry_run: self.dry_run,
+ snapshot_uri: self.from.clone(),
+ storage_config: self.storage.clone(),
storage: Box::new(storage),
database_client,
}))
@@ -126,8 +136,11 @@ impl ImportV2Command {
/// Import tool implementation.
pub struct Import {
+ catalog: String,
schemas: Option>,
dry_run: bool,
+ snapshot_uri: String,
+ storage_config: ObjectStoreConfig,
storage: Box,
database_client: DatabaseClient,
}
@@ -164,13 +177,6 @@ impl Import {
info!("Snapshot contains {} schema(s)", manifest.schemas.len());
- if !manifest.schema_only && !manifest.chunks.is_empty() {
- return FullSnapshotImportNotSupportedSnafu {
- chunk_count: manifest.chunks.len(),
- }
- .fail();
- }
-
// 2. Determine schemas to import
let schemas_to_import = match &self.schemas {
Some(filter) => canonicalize_schema_filter(filter, &manifest.schemas)?,
@@ -184,6 +190,14 @@ impl Import {
info!("Generated {} DDL statements", ddl_statements.len());
+ let data_tasks = if !manifest.schema_only && !manifest.chunks.is_empty() {
+ validate_data_snapshot(self.storage.as_ref(), &manifest.chunks, &schemas_to_import)
+ .await?;
+ build_import_tasks(&manifest.chunks, &schemas_to_import)
+ } else {
+ Vec::new()
+ };
+
// 4. Dry-run mode: print DDL and exit
if self.dry_run {
info!("Dry-run mode - DDL statements to execute:");
@@ -193,17 +207,76 @@ impl Import {
println!("{};", stmt.sql);
println!();
}
+ if !manifest.schema_only && !manifest.chunks.is_empty() {
+ for line in format_data_import_plan(&manifest.chunks, &schemas_to_import) {
+ println!("{line}");
+ }
+ println!();
+ }
return Ok(());
}
- // 5. Execute DDL
- let executor = DdlExecutor::new(&self.database_client);
- executor.execute_strict(&ddl_statements).await?;
+ let mut resume_session = if !data_tasks.is_empty() {
+ let state_path = default_state_path(
+ &manifest.snapshot_id.to_string(),
+ self.database_client.addr(),
+ &self.catalog,
+ &schemas_to_import,
+ )
+ .context(ImportStatePathUnavailableSnafu {
+ snapshot_id: manifest.snapshot_id.to_string(),
+ })?;
+ Some(
+ prepare_import_resume(ImportResumeConfig {
+ snapshot_id: manifest.snapshot_id.to_string(),
+ target_addr: self.database_client.addr().to_string(),
+ catalog: self.catalog.clone(),
+ schemas: schemas_to_import.clone(),
+ state_path,
+ tasks: data_tasks,
+ })
+ .await?,
+ )
+ } else {
+ None
+ };
- info!(
- "Import completed: {} DDL statements executed",
- ddl_statements.len()
- );
+ let skip_ddl = resume_session
+ .as_ref()
+ .map(|session| session.should_skip_ddl())
+ .unwrap_or(false);
+
+ // 5. Execute DDL unless a previous run already completed it.
+ let ddl_executed = if skip_ddl {
+ info!(
+ "Existing import state has DDL marked completed; skipping DDL execution and resuming data import"
+ );
+ false
+ } else {
+ let executor = DdlExecutor::new(&self.database_client);
+ executor.execute_strict(&ddl_statements).await?;
+ if let Some(session) = resume_session.as_mut() {
+ session.mark_ddl_completed().await?;
+ }
+ true
+ };
+
+ if let Some(resume_session) = resume_session {
+ let executor = CopyDatabaseImportTaskExecutor {
+ import: self,
+ format: manifest.format,
+ };
+ import_with_resume_session(resume_session, &executor).await?;
+ }
+
+ if ddl_executed {
+ info!(
+ "Import completed: {} DDL statements executed",
+ ddl_statements.len()
+ );
+ } else {
+ info!("Import completed: DDL execution skipped");
+ }
Ok(())
}
@@ -228,6 +301,40 @@ impl Import {
}
}
+struct CopyDatabaseImportTaskExecutor<'a> {
+ import: &'a Import,
+ format: DataFormat,
+}
+
+#[async_trait]
+impl ImportTaskExecutor for CopyDatabaseImportTaskExecutor<'_> {
+ async fn import_task(&self, task: &ImportTaskKey) -> Result<()> {
+ let source = build_copy_source(
+ &self.import.snapshot_uri,
+ &self.import.storage_config,
+ &task.schema,
+ task.chunk_id,
+ )
+ .context(ChunkImportFailedSnafu {
+ chunk_id: task.chunk_id,
+ schema: task.schema.clone(),
+ })?;
+
+ execute_copy_database_from(
+ &self.import.database_client,
+ &self.import.catalog,
+ &task.schema,
+ &source,
+ self.format,
+ )
+ .await
+ .context(ChunkImportFailedSnafu {
+ chunk_id: task.chunk_id,
+ schema: task.schema.clone(),
+ })
+ }
+}
+
fn parse_ddl_statements(content: &str) -> Vec {
let mut statements = Vec::new();
let mut current = String::new();
@@ -395,21 +502,122 @@ fn canonicalize_schema_filter(
Ok(canonicalized)
}
+fn validate_chunk_statuses(chunks: &[ChunkMeta]) -> Result<()> {
+ let invalid_chunk = chunks
+ .iter()
+ .find(|chunk| !matches!(chunk.status, ChunkStatus::Completed | ChunkStatus::Skipped));
+
+ if let Some(chunk) = invalid_chunk {
+ return IncompleteSnapshotSnafu {
+ chunk_id: chunk.id,
+ status: chunk.status,
+ }
+ .fail();
+ }
+
+ Ok(())
+}
+
+fn format_data_import_plan(chunks: &[ChunkMeta], schemas: &[String]) -> Vec {
+ let mut lines = vec!["-- Data import plan:".to_string()];
+ for chunk in chunks {
+ lines.push(format!("-- Chunk {}: {:?}", chunk.id, chunk.status));
+ for schema in schemas {
+ if chunk_has_schema_files(chunk, schema) {
+ lines.push(format!("-- {} -> COPY DATABASE FROM", schema));
+ }
+ }
+ }
+ lines
+}
+
+async fn validate_data_snapshot(
+ storage: &dyn SnapshotStorage,
+ chunks: &[ChunkMeta],
+ schemas: &[String],
+) -> Result<()> {
+ validate_chunk_statuses(chunks)?;
+ let actual_prefixes = collect_chunk_data_prefixes(storage).await?;
+
+ for chunk in chunks {
+ if chunk.status == ChunkStatus::Skipped {
+ continue;
+ }
+ if chunk.files.is_empty() {
+ return EmptyChunkManifestSnafu { chunk_id: chunk.id }.fail();
+ }
+ for schema in schemas {
+ validate_chunk_schema_files(chunk, schema, &actual_prefixes)?;
+ }
+ }
+
+ Ok(())
+}
+
+async fn collect_chunk_data_prefixes(storage: &dyn SnapshotStorage) -> Result> {
+ let files = storage
+ .list_files_recursive("data/")
+ .await
+ .context(SnapshotStorageSnafu)?;
+ let mut prefixes = HashSet::new();
+
+ for path in files {
+ let normalized = path.trim_start_matches('/');
+ let mut parts = normalized.splitn(4, '/');
+ let Some(root) = parts.next() else {
+ continue;
+ };
+ let Some(schema) = parts.next() else {
+ continue;
+ };
+ let Some(chunk_id) = parts.next() else {
+ continue;
+ };
+ if root != "data" {
+ continue;
+ }
+ prefixes.insert(format!("data/{schema}/{chunk_id}/"));
+ }
+
+ Ok(prefixes)
+}
+
+fn validate_chunk_schema_files(
+ chunk: &ChunkMeta,
+ schema: &str,
+ actual_prefixes: &HashSet,
+) -> Result {
+ if !chunk_has_schema_files(chunk, schema) {
+ return Ok(false);
+ }
+
+ let prefix = data_dir_for_schema_chunk(schema, chunk.id);
+ if !actual_prefixes.contains(&prefix) {
+ return MissingChunkDataSnafu {
+ chunk_id: chunk.id,
+ schema: schema.to_string(),
+ path: prefix,
+ }
+ .fail();
+ }
+
+ Ok(true)
+}
+
#[cfg(test)]
mod tests {
- use std::time::Duration;
+ use std::collections::{HashMap, HashSet};
use async_trait::async_trait;
use super::*;
- use crate::Tool;
- use crate::data::export_v2::manifest::{ChunkMeta, DataFormat, Manifest, TimeRange};
+ use crate::data::export_v2::manifest::{ChunkMeta, ChunkStatus, Manifest, TimeRange};
use crate::data::export_v2::schema::SchemaSnapshot;
use crate::data::snapshot_storage::SnapshotStorage;
- use crate::database::DatabaseClient;
struct StubStorage {
manifest: Manifest,
+ files_by_prefix: HashMap>,
}
#[async_trait]
@@ -454,9 +662,14 @@ mod tests {
async fn list_files_recursive(
&self,
- _prefix: &str,
+ prefix: &str,
) -> crate::data::export_v2::error::Result> {
- unimplemented!("not needed in import_v2::command tests")
+ Ok(self
+ .files_by_prefix
+ .iter()
+ .filter(|(candidate, _)| candidate.starts_with(prefix))
+ .flat_map(|(_, files)| files.clone())
+ .collect())
}
async fn delete_snapshot(&self) -> crate::data::export_v2::error::Result<()> {
@@ -464,48 +677,6 @@ mod tests {
}
}
- fn test_database_client() -> DatabaseClient {
- DatabaseClient::new(
- "127.0.0.1:4000".to_string(),
- "greptime".to_string(),
- None,
- Duration::from_secs(1),
- None,
- false,
- )
- }
-
- #[tokio::test]
- async fn test_import_rejects_full_snapshot_before_schema_execution() {
- let mut manifest = Manifest::new_full(
- "greptime".to_string(),
- vec!["public".to_string()],
- TimeRange::unbounded(),
- DataFormat::Parquet,
- );
- manifest
- .chunks
- .push(ChunkMeta::new(1, TimeRange::unbounded()));
-
- let import = Import {
- schemas: None,
- dry_run: false,
- storage: Box::new(StubStorage { manifest }),
- database_client: test_database_client(),
- };
-
- let error = import
- .do_work()
- .await
- .expect_err("full snapshot import should fail");
-
- assert!(
- error
- .to_string()
- .contains("Importing data from full snapshots is not implemented yet")
- );
- }
-
#[test]
fn test_parse_ddl_statements() {
let content = r#"
@@ -640,4 +811,164 @@ CREATE VIEW v AS SELECT 1;
assert!(!starts_with_keyword("CREATED TABLE t", "CREATE"));
assert!(!starts_with_keyword("TABLESPACE foo", "TABLE"));
}
+
+ #[test]
+ fn test_validate_chunk_statuses_rejects_failed_chunk() {
+ let mut failed = ChunkMeta::new(3, TimeRange::unbounded());
+ failed.status = ChunkStatus::Failed;
+
+ let error = validate_chunk_statuses(&[failed]).expect_err("failed chunk should error");
+ assert!(error.to_string().contains("Incomplete snapshot"));
+ }
+
+ #[test]
+ fn test_validate_chunk_statuses_accepts_completed_and_skipped_chunks() {
+ let mut completed = ChunkMeta::new(1, TimeRange::unbounded());
+ completed.status = ChunkStatus::Completed;
+ let skipped = ChunkMeta::skipped(2, TimeRange::unbounded());
+
+ assert!(validate_chunk_statuses(&[completed, skipped]).is_ok());
+ }
+
+ #[test]
+ fn test_chunk_has_schema_files_matches_encoded_schema_prefix() {
+ let mut chunk = ChunkMeta::new(7, TimeRange::unbounded());
+ chunk.files = vec![
+ "data/public/7/a.parquet".to_string(),
+ "data/%E6%B5%8B%E8%AF%95/7/b.parquet".to_string(),
+ ];
+
+ assert!(chunk_has_schema_files(&chunk, "public"));
+ assert!(chunk_has_schema_files(&chunk, "测试"));
+ assert!(!chunk_has_schema_files(&chunk, "metrics"));
+ }
+
+ #[test]
+ fn test_format_data_import_plan_includes_matching_schemas_only() {
+ let mut completed = ChunkMeta::new(1, TimeRange::unbounded());
+ completed.status = ChunkStatus::Completed;
+ completed.files = vec![
+ "data/public/1/a.parquet".to_string(),
+ "data/%E6%B5%8B%E8%AF%95/1/b.parquet".to_string(),
+ ];
+ let skipped = ChunkMeta::skipped(2, TimeRange::unbounded());
+
+ let lines = format_data_import_plan(
+ &[completed, skipped],
+ &[
+ "public".to_string(),
+ "测试".to_string(),
+ "metrics".to_string(),
+ ],
+ );
+
+ assert_eq!(lines[0], "-- Data import plan:");
+ assert!(lines.contains(&"-- Chunk 1: Completed".to_string()));
+ assert!(lines.contains(&"-- public -> COPY DATABASE FROM".to_string()));
+ assert!(lines.contains(&"-- 测试 -> COPY DATABASE FROM".to_string()));
+ assert!(!lines.contains(&"-- metrics -> COPY DATABASE FROM".to_string()));
+ assert!(lines.contains(&"-- Chunk 2: Skipped".to_string()));
+ }
+
+ #[tokio::test]
+ async fn test_collect_chunk_data_prefixes_indexes_present_prefixes() {
+ let storage = StubStorage {
+ manifest: Manifest::new_schema_only("greptime".to_string(), vec!["public".to_string()]),
+ files_by_prefix: HashMap::from([
+ (
+ "data/public/7/".to_string(),
+ vec!["data/public/7/a.parquet".to_string()],
+ ),
+ (
+ "data/%E6%B5%8B%E8%AF%95/9/".to_string(),
+ vec!["data/%E6%B5%8B%E8%AF%95/9/b.parquet".to_string()],
+ ),
+ ]),
+ };
+
+ let prefixes = collect_chunk_data_prefixes(&storage).await.unwrap();
+
+ assert!(prefixes.contains("data/public/7/"));
+ assert!(prefixes.contains("data/%E6%B5%8B%E8%AF%95/9/"));
+ }
+
+ #[test]
+ fn test_validate_chunk_schema_files_accepts_present_prefix() {
+ let mut chunk = ChunkMeta::new(7, TimeRange::unbounded());
+ chunk.files = vec!["data/public/7/a.parquet".to_string()];
+ let actual_prefixes = HashSet::from(["data/public/7/".to_string()]);
+
+ assert!(validate_chunk_schema_files(&chunk, "public", &actual_prefixes).unwrap());
+ }
+
+ #[test]
+ fn test_validate_chunk_schema_files_rejects_missing_prefix() {
+ let mut chunk = ChunkMeta::new(7, TimeRange::unbounded());
+ chunk.files = vec!["data/public/7/a.parquet".to_string()];
+
+ let error = validate_chunk_schema_files(&chunk, "public", &HashSet::new())
+ .expect_err("missing chunk prefix should fail")
+ .to_string();
+ assert!(error.contains("marked completed but no files were found"));
+ }
+
+ #[test]
+ fn test_validate_chunk_schema_files_skips_absent_schema() {
+ let mut chunk = ChunkMeta::new(7, TimeRange::unbounded());
+ chunk.files = vec!["data/public/7/a.parquet".to_string()];
+
+ assert!(!validate_chunk_schema_files(&chunk, "metrics", &HashSet::new()).unwrap());
+ }
+
+ #[tokio::test]
+ async fn test_validate_data_snapshot_rejects_failed_chunk_before_dry_run() {
+ let mut failed = ChunkMeta::new(3, TimeRange::unbounded());
+ failed.status = ChunkStatus::Failed;
+
+ let storage = StubStorage {
+ manifest: Manifest::new_schema_only("greptime".to_string(), vec!["public".to_string()]),
+ files_by_prefix: HashMap::new(),
+ };
+
+ let error = validate_data_snapshot(&storage, &[failed], &["public".to_string()])
+ .await
+ .expect_err("failed chunk should reject dry-run validation")
+ .to_string();
+ assert!(error.contains("Incomplete snapshot"));
+ }
+
+ #[tokio::test]
+ async fn test_validate_data_snapshot_rejects_missing_chunk_prefix_before_dry_run() {
+ let mut completed = ChunkMeta::new(7, TimeRange::unbounded());
+ completed.status = ChunkStatus::Completed;
+ completed.files = vec!["data/public/7/a.parquet".to_string()];
+
+ let storage = StubStorage {
+ manifest: Manifest::new_schema_only("greptime".to_string(), vec!["public".to_string()]),
+ files_by_prefix: HashMap::new(),
+ };
+
+ let error = validate_data_snapshot(&storage, &[completed], &["public".to_string()])
+ .await
+ .expect_err("missing chunk prefix should reject dry-run validation")
+ .to_string();
+ assert!(error.contains("marked completed but no files were found"));
+ }
+
+ #[tokio::test]
+ async fn test_validate_data_snapshot_rejects_completed_chunk_with_empty_manifest() {
+ let mut completed = ChunkMeta::new(7, TimeRange::unbounded());
+ completed.status = ChunkStatus::Completed;
+
+ let storage = StubStorage {
+ manifest: Manifest::new_schema_only("greptime".to_string(), vec!["public".to_string()]),
+ files_by_prefix: HashMap::new(),
+ };
+
+ let error = validate_data_snapshot(&storage, &[completed], &["public".to_string()])
+ .await
+ .expect_err("empty completed chunk should reject validation")
+ .to_string();
+ assert!(error.contains("file manifest is empty"));
+ }
}
diff --git a/src/cli/src/data/import_v2/coordinator.rs b/src/cli/src/data/import_v2/coordinator.rs
new file mode 100644
index 0000000000..fe2d016464
--- /dev/null
+++ b/src/cli/src/data/import_v2/coordinator.rs
@@ -0,0 +1,695 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::collections::BTreeSet;
+use std::path::{Path, PathBuf};
+use std::time::Instant;
+
+use async_trait::async_trait;
+use common_telemetry::{info, warn};
+
+use crate::data::export_v2::manifest::{ChunkMeta, ChunkStatus};
+use crate::data::import_v2::error::{
+ ImportStateDdlIncompleteSnafu, ImportStateMismatchSnafu, Result,
+};
+use crate::data::import_v2::state::{
+ ImportState, ImportStateLockGuard, ImportTaskKey, ImportTaskStatus, canonical_schema_selection,
+ delete_import_state, load_import_state, save_import_state, try_acquire_import_state_lock,
+};
+use crate::data::path::data_dir_for_schema_chunk;
+
+#[async_trait]
+pub(crate) trait ImportTaskExecutor {
+ async fn import_task(&self, task: &ImportTaskKey) -> Result<()>;
+}
+
+pub(crate) struct ImportResumeConfig {
+ pub(crate) snapshot_id: String,
+ pub(crate) target_addr: String,
+ pub(crate) catalog: String,
+ pub(crate) schemas: Vec,
+ pub(crate) state_path: PathBuf,
+ pub(crate) tasks: Vec,
+}
+
+pub(crate) struct ImportResumeSession {
+ config: ImportResumeConfig,
+ state: ImportState,
+ lock: ImportStateLockGuard,
+}
+
+impl ImportResumeSession {
+ pub(crate) fn should_skip_ddl(&self) -> bool {
+ self.state.ddl_completed
+ }
+
+ /// Marks DDL as completed and persists the state. Must be called after a
+ /// successful DDL run on a fresh session, so that crashes after this point
+ /// resume into the data-import phase instead of replaying DDL.
+ pub(crate) async fn mark_ddl_completed(&mut self) -> Result<()> {
+ self.state.mark_ddl_completed();
+ save_import_state(&self.config.state_path, &self.state).await
+ }
+}
+
+pub(crate) fn chunk_has_schema_files(chunk: &ChunkMeta, schema: &str) -> bool {
+ let prefix = data_dir_for_schema_chunk(schema, chunk.id);
+ chunk.files.iter().any(|path| {
+ let normalized = path.trim_start_matches('/');
+ normalized.starts_with(&prefix)
+ })
+}
+
+pub(crate) fn build_import_tasks(chunks: &[ChunkMeta], schemas: &[String]) -> Vec {
+ let mut tasks = Vec::new();
+ for chunk in chunks {
+ if chunk.status == ChunkStatus::Skipped {
+ continue;
+ }
+ // TODO: build a per-chunk schema index if chunk file manifests become large.
+ for schema in schemas {
+ if chunk_has_schema_files(chunk, schema) {
+ tasks.push(ImportTaskKey::new(chunk.id, schema.clone()));
+ }
+ }
+ }
+ tasks
+}
+
+pub(crate) async fn prepare_import_resume(
+ config: ImportResumeConfig,
+) -> Result {
+ // Validate the request before touching the state file or acquiring the
+ // lock. Duplicate task keys would corrupt the resume bookkeeping because
+ // status lookups use linear `find()` and only ever see the first match.
+ validate_config_tasks(&config)?;
+
+ let lock = try_acquire_import_state_lock(&config.state_path)?;
+ let state = match load_import_state(&config.state_path).await? {
+ Some(loaded) => {
+ validate_state_matches(&loaded, &config)?;
+ loaded
+ }
+ None => {
+ // Persist a fresh state immediately so that any crash after this
+ // point is recoverable as a resume. `ddl_completed=false` on a
+ // loaded state therefore means a previous run reached this point
+ // but did not confirm DDL completion - DDL must be (re-)run before
+ // data import is allowed.
+ let fresh = ImportState::new(
+ &config.snapshot_id,
+ &config.target_addr,
+ &config.catalog,
+ &config.schemas,
+ config.tasks.clone(),
+ );
+ save_import_state(&config.state_path, &fresh).await?;
+ fresh
+ }
+ };
+
+ Ok(ImportResumeSession {
+ config,
+ state,
+ lock,
+ })
+}
+
+pub(crate) async fn import_with_resume_session(
+ session: ImportResumeSession,
+ executor: &E,
+) -> Result<()>
+where
+ E: ImportTaskExecutor + Sync,
+{
+ let ImportResumeSession {
+ config,
+ mut state,
+ lock,
+ } = session;
+
+ // The state machine requires DDL to be explicitly marked completed before
+ // data import; otherwise a caller could import data and leave a state that
+ // replays DDL on the next resume. Surface the misuse instead of silently
+ // importing.
+ if !state.ddl_completed {
+ return ImportStateDdlIncompleteSnafu {
+ path: config.state_path.display().to_string(),
+ }
+ .fail();
+ }
+
+ let completed = state
+ .tasks
+ .iter()
+ .filter(|task| task.status == ImportTaskStatus::Completed)
+ .count();
+ info!(
+ "Import resume state: {} completed, {} pending, path: {}",
+ completed,
+ state.tasks.len().saturating_sub(completed),
+ config.state_path.display()
+ );
+
+ let import_start = Instant::now();
+ for (idx, task) in config.tasks.iter().enumerate() {
+ if state.task_status(task.chunk_id, &task.schema) == Some(ImportTaskStatus::Completed) {
+ info!(
+ "[{}/{}] Chunk {} schema {}: already completed, skipped",
+ idx + 1,
+ config.tasks.len(),
+ task.chunk_id,
+ task.schema
+ );
+ continue;
+ }
+
+ info!(
+ "[{}/{}] Chunk {} schema {}: importing...",
+ idx + 1,
+ config.tasks.len(),
+ task.chunk_id,
+ task.schema
+ );
+ state.set_task_status(
+ task.chunk_id,
+ &task.schema,
+ ImportTaskStatus::InProgress,
+ None,
+ )?;
+ save_import_state(&config.state_path, &state).await?;
+
+ let task_start = Instant::now();
+ let result = executor.import_task(task).await;
+
+ match result {
+ Ok(()) => {
+ // The task itself succeeded. If we cannot persist the
+ // Completed marker, the next resume will replay it (potentially
+ // duplicating data depending on engine semantics), but we must
+ // not pretend the import as a whole failed - return the persist
+ // error so the operator notices, after logging the success.
+ update_status_and_save(
+ &config,
+ &mut state,
+ task,
+ ImportTaskStatus::Completed,
+ None,
+ )
+ .await?;
+ info!(
+ "[{}/{}] Chunk {} schema {}: done in {:?}",
+ idx + 1,
+ config.tasks.len(),
+ task.chunk_id,
+ task.schema,
+ task_start.elapsed()
+ );
+ }
+ Err(task_error) => {
+ // Persist Failed best-effort, but always surface the original
+ // task error to the caller. State persistence problems are
+ // logged so they are not silently lost.
+ if let Err(persist_error) = update_status_and_save(
+ &config,
+ &mut state,
+ task,
+ ImportTaskStatus::Failed,
+ Some(task_error.to_string()),
+ )
+ .await
+ {
+ warn!(
+ "Failed to persist Failed status for chunk {} schema {} after task error ({}); state file may be out of date: {}",
+ task.chunk_id, task.schema, task_error, persist_error
+ );
+ }
+ return Err(task_error);
+ }
+ }
+ }
+
+ delete_import_state(&config.state_path).await?;
+ info!("Data import finished in {:?}", import_start.elapsed());
+ drop(lock);
+ Ok(())
+}
+
+async fn update_status_and_save(
+ config: &ImportResumeConfig,
+ state: &mut ImportState,
+ task: &ImportTaskKey,
+ status: ImportTaskStatus,
+ error_message: Option,
+) -> Result<()> {
+ // set_task_status only fails if the task isn't in the state; that would
+ // indicate a logic bug since `task` came from the same config. Surface it
+ // instead of swallowing.
+ state.set_task_status(task.chunk_id, &task.schema, status, error_message)?;
+ save_import_state(&config.state_path, state).await
+}
+
+fn validate_state_matches(state: &ImportState, config: &ImportResumeConfig) -> Result<()> {
+ if state.snapshot_id != config.snapshot_id {
+ return state_mismatch(
+ config,
+ format!(
+ "snapshot_id differs (state: {}, requested: {})",
+ state.snapshot_id, config.snapshot_id
+ ),
+ );
+ }
+ // Target addresses are compared literally; hostname normalization is left to the caller.
+ if state.target_addr != config.target_addr {
+ return state_mismatch(
+ config,
+ format!(
+ "target_addr differs (state: {}, requested: {})",
+ state.target_addr, config.target_addr
+ ),
+ );
+ }
+ if state.catalog != config.catalog {
+ return state_mismatch(
+ config,
+ format!(
+ "catalog differs (state: {}, requested: {})",
+ state.catalog, config.catalog
+ ),
+ );
+ }
+
+ let requested_schemas = canonical_schema_selection(&config.schemas);
+ if state.schemas != requested_schemas {
+ return state_mismatch(
+ config,
+ format!(
+ "schemas differ (state: {:?}, requested: {:?})",
+ state.schemas, requested_schemas
+ ),
+ );
+ }
+
+ if task_set_from_state(state, &config.state_path)? != task_set_from_config(config)? {
+ return state_mismatch(config, "task set differs".to_string());
+ }
+
+ Ok(())
+}
+
+fn state_mismatch(config: &ImportResumeConfig, reason: String) -> Result<()> {
+ ImportStateMismatchSnafu {
+ path: config.state_path.display().to_string(),
+ reason,
+ }
+ .fail()
+}
+
+fn task_set_from_state<'a>(
+ state: &'a ImportState,
+ state_path: &Path,
+) -> Result> {
+ let mut tasks = BTreeSet::new();
+ for task in &state.tasks {
+ if !tasks.insert((task.chunk_id, task.schema.as_str())) {
+ return ImportStateMismatchSnafu {
+ path: state_path.display().to_string(),
+ reason: format!(
+ "duplicate task key in state (chunk_id: {}, schema: {})",
+ task.chunk_id, task.schema
+ ),
+ }
+ .fail();
+ }
+ }
+ Ok(tasks)
+}
+
+fn task_set_from_config(config: &ImportResumeConfig) -> Result> {
+ let mut tasks = BTreeSet::new();
+ for task in &config.tasks {
+ if !tasks.insert((task.chunk_id, task.schema.as_str())) {
+ return ImportStateMismatchSnafu {
+ path: config.state_path.display().to_string(),
+ reason: format!(
+ "duplicate task key in request (chunk_id: {}, schema: {})",
+ task.chunk_id, task.schema
+ ),
+ }
+ .fail();
+ }
+ }
+ Ok(tasks)
+}
+
+fn validate_config_tasks(config: &ImportResumeConfig) -> Result<()> {
+ task_set_from_config(config).map(|_| ())
+}
+
+#[cfg(test)]
+mod tests {
+ use std::sync::atomic::{AtomicUsize, Ordering};
+ use std::sync::{Arc, Mutex};
+
+ use super::*;
+ use crate::data::export_v2::manifest::{ChunkMeta, TimeRange};
+ use crate::data::import_v2::error::TestTaskFailedSnafu;
+
+ #[derive(Debug, Clone, Copy)]
+ enum FailureMode {
+ Fatal,
+ RetryableThenSuccess { failures: usize },
+ }
+
+ struct RecordingExecutor {
+ imported: Arc>>,
+ fail_task: Option,
+ failure_mode: Option,
+ attempts: Arc,
+ }
+
+ #[async_trait]
+ impl ImportTaskExecutor for RecordingExecutor {
+ async fn import_task(&self, task: &ImportTaskKey) -> Result<()> {
+ let attempt = self.attempts.fetch_add(1, Ordering::SeqCst);
+ if self.fail_task.as_ref() == Some(task) {
+ match self.failure_mode {
+ Some(FailureMode::Fatal) => {
+ return TestTaskFailedSnafu {
+ message: "fatal failure".to_string(),
+ retryable: false,
+ }
+ .fail();
+ }
+ Some(FailureMode::RetryableThenSuccess { failures }) if attempt < failures => {
+ return TestTaskFailedSnafu {
+ message: "retryable failure".to_string(),
+ retryable: true,
+ }
+ .fail();
+ }
+ _ => {}
+ }
+ }
+ self.imported.lock().unwrap().push(task.clone());
+ Ok(())
+ }
+ }
+
+ fn recording_executor(imported: Arc>>) -> RecordingExecutor {
+ RecordingExecutor {
+ imported,
+ fail_task: None,
+ failure_mode: None,
+ attempts: Arc::new(AtomicUsize::new(0)),
+ }
+ }
+
+ fn config(path: PathBuf, tasks: Vec) -> ImportResumeConfig {
+ ImportResumeConfig {
+ snapshot_id: "snapshot-1".to_string(),
+ target_addr: "127.0.0.1:4000".to_string(),
+ catalog: "greptime".to_string(),
+ schemas: vec!["public".to_string(), "analytics".to_string()],
+ state_path: path,
+ tasks,
+ }
+ }
+
+ async fn run_import_with_resume(config: ImportResumeConfig, executor: &E) -> Result<()>
+ where
+ E: ImportTaskExecutor + Sync,
+ {
+ // Mirror the production caller: mark DDL completed for fresh sessions
+ // so the data-import guard is satisfied. Tests that want to exercise
+ // the unsafe path drive prepare/import directly.
+ let mut session = prepare_import_resume(config).await?;
+ if !session.should_skip_ddl() {
+ session.mark_ddl_completed().await?;
+ }
+ import_with_resume_session(session, executor).await
+ }
+
+ #[test]
+ fn test_build_import_tasks_skips_skipped_chunks_and_missing_schema_files() {
+ let mut completed = ChunkMeta::new(1, TimeRange::unbounded());
+ completed.status = ChunkStatus::Completed;
+ completed.files = vec!["data/public/1/file.parquet".to_string()];
+ let mut skipped = ChunkMeta::new(2, TimeRange::unbounded());
+ skipped.status = ChunkStatus::Skipped;
+ skipped.files = vec!["data/public/2/file.parquet".to_string()];
+
+ let tasks = build_import_tasks(
+ &[completed, skipped],
+ &["public".to_string(), "analytics".to_string()],
+ );
+
+ assert_eq!(tasks, vec![ImportTaskKey::new(1, "public")]);
+ }
+
+ #[tokio::test]
+ async fn test_import_with_resume_skips_completed_tasks() {
+ let dir = tempfile::tempdir().unwrap();
+ let path = dir.path().join("import_state.json");
+ let tasks = vec![
+ ImportTaskKey::new(1, "public"),
+ ImportTaskKey::new(2, "analytics"),
+ ];
+ let mut state = ImportState::new(
+ "snapshot-1",
+ "127.0.0.1:4000",
+ "greptime",
+ &["public".to_string(), "analytics".to_string()],
+ tasks.clone(),
+ );
+ state.mark_ddl_completed();
+ state
+ .set_task_status(1, "public", ImportTaskStatus::Completed, None)
+ .unwrap();
+ save_import_state(&path, &state).await.unwrap();
+
+ let imported = Arc::new(Mutex::new(Vec::new()));
+ let executor = recording_executor(imported.clone());
+
+ run_import_with_resume(config(path.clone(), tasks), &executor)
+ .await
+ .unwrap();
+
+ assert_eq!(
+ imported.lock().unwrap().clone(),
+ vec![ImportTaskKey::new(2, "analytics")]
+ );
+ assert!(load_import_state(&path).await.unwrap().is_none());
+ }
+
+ #[tokio::test]
+ async fn test_import_with_resume_persists_failed_task() {
+ let dir = tempfile::tempdir().unwrap();
+ let path = dir.path().join("import_state.json");
+ let failed_task = ImportTaskKey::new(1, "public");
+ let tasks = vec![failed_task.clone()];
+ let imported = Arc::new(Mutex::new(Vec::new()));
+ let executor = RecordingExecutor {
+ imported,
+ fail_task: Some(failed_task.clone()),
+ failure_mode: Some(FailureMode::Fatal),
+ attempts: Arc::new(AtomicUsize::new(0)),
+ };
+
+ let error = run_import_with_resume(config(path.clone(), tasks), &executor)
+ .await
+ .unwrap_err();
+ assert!(matches!(
+ error,
+ crate::data::import_v2::error::Error::TestTaskFailed {
+ retryable: false,
+ ..
+ }
+ ));
+
+ let state = load_import_state(&path).await.unwrap().unwrap();
+ assert_eq!(
+ state.task_status(failed_task.chunk_id, &failed_task.schema),
+ Some(ImportTaskStatus::Failed)
+ );
+ }
+
+ #[tokio::test]
+ async fn test_import_with_resume_rejects_mismatched_state_identity() {
+ let dir = tempfile::tempdir().unwrap();
+ let path = dir.path().join("import_state.json");
+ let tasks = vec![ImportTaskKey::new(1, "public")];
+ let state = ImportState::new(
+ "snapshot-1",
+ "127.0.0.1:4001",
+ "greptime",
+ &["public".to_string(), "analytics".to_string()],
+ tasks.clone(),
+ );
+ save_import_state(&path, &state).await.unwrap();
+
+ let imported = Arc::new(Mutex::new(Vec::new()));
+ let executor = recording_executor(imported);
+
+ let error = run_import_with_resume(config(path, tasks), &executor)
+ .await
+ .unwrap_err();
+
+ assert!(matches!(
+ error,
+ crate::data::import_v2::error::Error::ImportStateMismatch { .. }
+ ));
+ }
+
+ #[tokio::test]
+ async fn test_prepare_import_resume_reports_existing_state_before_ddl() {
+ let dir = tempfile::tempdir().unwrap();
+ let tasks = vec![ImportTaskKey::new(1, "public")];
+
+ let fresh_session =
+ prepare_import_resume(config(dir.path().join("fresh_state.json"), tasks.clone()))
+ .await
+ .unwrap();
+ assert!(!fresh_session.should_skip_ddl());
+ drop(fresh_session);
+
+ let existing_path = dir.path().join("existing_state.json");
+ let mut state = ImportState::new(
+ "snapshot-1",
+ "127.0.0.1:4000",
+ "greptime",
+ &["public".to_string(), "analytics".to_string()],
+ tasks.clone(),
+ );
+ state.mark_ddl_completed();
+ save_import_state(&existing_path, &state).await.unwrap();
+
+ let resume_session = prepare_import_resume(config(existing_path, tasks))
+ .await
+ .unwrap();
+ assert!(resume_session.should_skip_ddl());
+ }
+
+ #[tokio::test]
+ async fn test_import_with_resume_rejects_duplicate_state_tasks() {
+ let dir = tempfile::tempdir().unwrap();
+ let path = dir.path().join("import_state.json");
+ let tasks = vec![ImportTaskKey::new(1, "public")];
+ let mut state = ImportState::new(
+ "snapshot-1",
+ "127.0.0.1:4000",
+ "greptime",
+ &["public".to_string(), "analytics".to_string()],
+ tasks.clone(),
+ );
+ state.tasks.push(state.tasks[0].clone());
+ save_import_state(&path, &state).await.unwrap();
+
+ let imported = Arc::new(Mutex::new(Vec::new()));
+ let executor = recording_executor(imported);
+
+ let error = run_import_with_resume(config(path, tasks), &executor)
+ .await
+ .unwrap_err();
+
+ assert!(matches!(
+ error,
+ crate::data::import_v2::error::Error::ImportStateMismatch { .. }
+ ));
+ }
+
+ #[tokio::test]
+ async fn test_import_with_resume_rejects_data_import_when_ddl_incomplete() {
+ let dir = tempfile::tempdir().unwrap();
+ let path = dir.path().join("import_state.json");
+ let tasks = vec![ImportTaskKey::new(1, "public")];
+
+ // prepare creates fresh state with ddl_completed=false; calling
+ // import_with_resume_session directly (without mark_ddl_completed)
+ // must be rejected.
+ let session = prepare_import_resume(config(path, tasks)).await.unwrap();
+ let imported = Arc::new(Mutex::new(Vec::new()));
+ let executor = recording_executor(imported.clone());
+
+ let error = import_with_resume_session(session, &executor)
+ .await
+ .unwrap_err();
+
+ assert!(matches!(
+ error,
+ crate::data::import_v2::error::Error::ImportStateDdlIncomplete { .. }
+ ));
+ assert!(imported.lock().unwrap().is_empty());
+ }
+
+ #[tokio::test]
+ async fn test_prepare_import_resume_rejects_duplicate_request_tasks_on_fresh_state() {
+ let dir = tempfile::tempdir().unwrap();
+ let path = dir.path().join("import_state.json");
+ let task = ImportTaskKey::new(1, "public");
+ // No state file yet - duplicate detection must run before the fresh
+ // state is persisted, otherwise corrupted bookkeeping would be
+ // written to disk and observed only on a later resume.
+ let error =
+ match prepare_import_resume(config(path.clone(), vec![task.clone(), task])).await {
+ Ok(_) => panic!("duplicate request tasks should be rejected"),
+ Err(error) => error,
+ };
+
+ assert!(matches!(
+ error,
+ crate::data::import_v2::error::Error::ImportStateMismatch { .. }
+ ));
+ assert!(load_import_state(&path).await.unwrap().is_none());
+ }
+
+ #[tokio::test]
+ async fn test_import_with_resume_does_not_retry_retryable_task_error() {
+ let dir = tempfile::tempdir().unwrap();
+ let path = dir.path().join("import_state.json");
+ let failed_task = ImportTaskKey::new(1, "public");
+ let tasks = vec![failed_task.clone()];
+ let imported = Arc::new(Mutex::new(Vec::new()));
+ let attempts = Arc::new(AtomicUsize::new(0));
+ let executor = RecordingExecutor {
+ imported: imported.clone(),
+ fail_task: Some(failed_task.clone()),
+ // If task import were retried, the second attempt would succeed.
+ // COPY DATABASE FROM failures are ambiguous, so retryable errors
+ // must still stop immediately to avoid duplicate rows.
+ failure_mode: Some(FailureMode::RetryableThenSuccess { failures: 1 }),
+ attempts: attempts.clone(),
+ };
+
+ let error = run_import_with_resume(config(path.clone(), tasks), &executor)
+ .await
+ .unwrap_err();
+
+ assert!(matches!(
+ error,
+ crate::data::import_v2::error::Error::TestTaskFailed {
+ retryable: true,
+ ..
+ }
+ ));
+ assert_eq!(attempts.load(Ordering::SeqCst), 1);
+ assert!(imported.lock().unwrap().is_empty());
+
+ let state = load_import_state(&path).await.unwrap().unwrap();
+ assert_eq!(
+ state.task_status(failed_task.chunk_id, &failed_task.schema),
+ Some(ImportTaskStatus::Failed)
+ );
+ }
+}
diff --git a/src/cli/src/data/import_v2/error.rs b/src/cli/src/data/import_v2/error.rs
index 169f11c0fa..165f1c0118 100644
--- a/src/cli/src/data/import_v2/error.rs
+++ b/src/cli/src/data/import_v2/error.rs
@@ -19,6 +19,8 @@ use common_error::status_code::StatusCode;
use common_macro::stack_trace_debug;
use snafu::{Location, Snafu};
+use crate::data::export_v2::manifest::ChunkStatus;
+
#[derive(Snafu)]
#[snafu(visibility(pub))]
#[stack_trace_debug]
@@ -45,12 +47,44 @@ pub enum Error {
location: Location,
},
+ #[snafu(display("Incomplete snapshot: chunk {} has status {:?}", chunk_id, status))]
+ IncompleteSnapshot {
+ chunk_id: u32,
+ status: ChunkStatus,
+ #[snafu(implicit)]
+ location: Location,
+ },
+
#[snafu(display(
- "Importing data from full snapshots is not implemented yet (snapshot has {} chunk(s))",
- chunk_count
+ "Snapshot is inconsistent: chunk {} is marked completed but its file manifest is empty",
+ chunk_id
))]
- FullSnapshotImportNotSupported {
- chunk_count: usize,
+ EmptyChunkManifest {
+ chunk_id: u32,
+ #[snafu(implicit)]
+ location: Location,
+ },
+
+ #[snafu(display(
+ "Snapshot is inconsistent: chunk {} for schema '{}' is marked completed but no files were found under '{}'",
+ chunk_id,
+ schema,
+ path
+ ))]
+ MissingChunkData {
+ chunk_id: u32,
+ schema: String,
+ path: String,
+ #[snafu(implicit)]
+ location: Location,
+ },
+
+ #[snafu(display("Chunk {} import failed for schema '{}'", chunk_id, schema))]
+ ChunkImportFailed {
+ chunk_id: u32,
+ schema: String,
+ #[snafu(source)]
+ error: crate::data::export_v2::error::Error,
#[snafu(implicit)]
location: Location,
},
@@ -70,6 +104,83 @@ pub enum Error {
#[snafu(implicit)]
location: Location,
},
+
+ #[snafu(display("Failed to parse import state file"))]
+ ImportStateParse {
+ #[snafu(source)]
+ error: serde_json::Error,
+ #[snafu(implicit)]
+ location: Location,
+ },
+
+ #[snafu(display("Import state I/O failed at '{}': {}", path, error))]
+ ImportStateIo {
+ path: String,
+ #[snafu(source)]
+ error: std::io::Error,
+ #[snafu(implicit)]
+ location: Location,
+ },
+
+ #[snafu(display("Import state is already locked at '{}'", path))]
+ ImportStateLocked {
+ path: String,
+ #[snafu(implicit)]
+ location: Location,
+ },
+
+ #[snafu(display(
+ "Failed to determine import state path for snapshot '{}'. Set HOME, USERPROFILE, or run from a valid current directory.",
+ snapshot_id
+ ))]
+ ImportStatePathUnavailable {
+ snapshot_id: String,
+ #[snafu(implicit)]
+ location: Location,
+ },
+
+ #[snafu(display(
+ "Import state at '{}' does not match current import: {}. Either rerun with matching import arguments, or delete the state file to start over (DDL will be re-executed).",
+ path,
+ reason
+ ))]
+ ImportStateMismatch {
+ path: String,
+ reason: String,
+ #[snafu(implicit)]
+ location: Location,
+ },
+
+ #[cfg(test)]
+ #[snafu(display("Test task failed: {}", message))]
+ TestTaskFailed {
+ message: String,
+ retryable: bool,
+ #[snafu(implicit)]
+ location: Location,
+ },
+
+ #[snafu(display(
+ "Import state references unknown task: chunk {}, schema '{}'",
+ chunk_id,
+ schema
+ ))]
+ ImportStateUnknownTask {
+ chunk_id: u32,
+ schema: String,
+ #[snafu(implicit)]
+ location: Location,
+ },
+
+ #[snafu(display(
+ "Import state at '{}' is not ready for data import: DDL has not been marked completed",
+ path
+ ))]
+ ImportStateDdlIncomplete {
+ path: String,
+ #[snafu(implicit)]
+ location: Location,
+ },
}
pub type Result = std::result::Result;
@@ -80,9 +191,28 @@ impl ErrorExt for Error {
Error::SnapshotNotFound { .. }
| Error::SchemaNotInSnapshot { .. }
| Error::ManifestVersionMismatch { .. }
- | Error::FullSnapshotImportNotSupported { .. } => StatusCode::InvalidArguments,
+ | Error::IncompleteSnapshot { .. }
+ | Error::EmptyChunkManifest { .. }
+ | Error::MissingChunkData { .. } => StatusCode::InvalidArguments,
+ Error::ImportStatePathUnavailable { .. }
+ | Error::ImportStateUnknownTask { .. }
+ | Error::ImportStateDdlIncomplete { .. } => StatusCode::Unexpected,
+ Error::ImportStateMismatch { .. } => StatusCode::InvalidArguments,
+ #[cfg(test)]
+ Error::TestTaskFailed { retryable, .. } => {
+ if *retryable {
+ StatusCode::StorageUnavailable
+ } else {
+ StatusCode::InvalidArguments
+ }
+ }
Error::Database { error, .. } => error.status_code(),
- Error::SnapshotStorage { error, .. } => error.status_code(),
+ Error::SnapshotStorage { error, .. } | Error::ChunkImportFailed { error, .. } => {
+ error.status_code()
+ }
+ Error::ImportStateParse { .. } => StatusCode::Internal,
+ Error::ImportStateIo { .. } => StatusCode::StorageUnavailable,
+ Error::ImportStateLocked { .. } => StatusCode::IllegalState,
}
}
diff --git a/src/cli/src/data/import_v2/state.rs b/src/cli/src/data/import_v2/state.rs
new file mode 100644
index 0000000000..ba431ac62f
--- /dev/null
+++ b/src/cli/src/data/import_v2/state.rs
@@ -0,0 +1,804 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::path::{Path, PathBuf};
+use std::sync::atomic::{AtomicU64, Ordering};
+
+use chrono::{DateTime, Utc};
+use fs2::FileExt;
+use serde::{Deserialize, Serialize};
+use snafu::{IntoError, OptionExt, ResultExt};
+use tokio::io::AsyncWriteExt;
+
+use crate::data::import_v2::error::{
+ ImportStateIoSnafu, ImportStateLockedSnafu, ImportStateParseSnafu, ImportStateUnknownTaskSnafu,
+ Result,
+};
+use crate::data::path::encode_path_segment;
+
+const IMPORT_STATE_ROOT: &str = ".greptime";
+const IMPORT_STATE_DIR: &str = "import_state";
+static IMPORT_STATE_TMP_ID: AtomicU64 = AtomicU64::new(0);
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub(crate) enum ImportTaskStatus {
+ Pending,
+ InProgress,
+ Completed,
+ Failed,
+}
+
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+pub(crate) struct ImportTaskKey {
+ pub(crate) chunk_id: u32,
+ pub(crate) schema: String,
+}
+
+impl ImportTaskKey {
+ pub(crate) fn new(chunk_id: u32, schema: impl Into) -> Self {
+ Self {
+ chunk_id,
+ schema: schema.into(),
+ }
+ }
+}
+
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+pub(crate) struct ImportTaskState {
+ pub(crate) chunk_id: u32,
+ pub(crate) schema: String,
+ pub(crate) status: ImportTaskStatus,
+ #[serde(skip_serializing_if = "Option::is_none")]
+ pub(crate) error: Option,
+}
+
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+pub(crate) struct ImportState {
+ pub(crate) snapshot_id: String,
+ pub(crate) target_addr: String,
+ pub(crate) catalog: String,
+ pub(crate) schemas: Vec,
+ #[serde(default)]
+ pub(crate) ddl_completed: bool,
+ pub(crate) updated_at: DateTime,
+ // Tasks are (chunk-schema) tuples and can reach the tens of thousands;
+ // linear scans here are accepted because per-task work is dominated by
+ // network I/O and an fsync, but if the bound grows further this should be
+ // backed by a HashMap<(chunk_id, schema), index> rebuilt after load.
+ pub(crate) tasks: Vec,
+}
+
+impl ImportState {
+ pub(crate) fn new(
+ snapshot_id: impl Into,
+ target_addr: impl Into,
+ catalog: impl Into,
+ schemas: &[String],
+ tasks: I,
+ ) -> Self
+ where
+ I: IntoIterator- ,
+ {
+ Self {
+ snapshot_id: snapshot_id.into(),
+ target_addr: target_addr.into(),
+ catalog: catalog.into(),
+ schemas: canonical_schema_selection(schemas),
+ ddl_completed: false,
+ updated_at: Utc::now(),
+ tasks: tasks
+ .into_iter()
+ .map(|task| ImportTaskState {
+ chunk_id: task.chunk_id,
+ schema: task.schema,
+ status: ImportTaskStatus::Pending,
+ error: None,
+ })
+ .collect(),
+ }
+ }
+
+ pub(crate) fn mark_ddl_completed(&mut self) {
+ self.ddl_completed = true;
+ self.updated_at = Utc::now();
+ }
+
+ pub(crate) fn task_status(&self, chunk_id: u32, schema: &str) -> Option {
+ self.tasks
+ .iter()
+ .find(|task| task.chunk_id == chunk_id && task.schema == schema)
+ .map(|task| task.status)
+ }
+
+ pub(crate) fn set_task_status(
+ &mut self,
+ chunk_id: u32,
+ schema: &str,
+ status: ImportTaskStatus,
+ error: Option,
+ ) -> Result<()> {
+ let task = self
+ .tasks
+ .iter_mut()
+ .find(|task| task.chunk_id == chunk_id && task.schema == schema)
+ .context(ImportStateUnknownTaskSnafu {
+ chunk_id,
+ schema: schema.to_string(),
+ })?;
+ task.status = status;
+ task.error = error;
+ self.updated_at = Utc::now();
+ Ok(())
+ }
+}
+
+#[derive(Debug)]
+pub(crate) struct ImportStateLockGuard {
+ file: std::fs::File,
+}
+
+impl Drop for ImportStateLockGuard {
+ fn drop(&mut self) {
+ let _ = self.file.unlock();
+ }
+}
+
+pub(crate) fn default_state_path(
+ snapshot_id: &str,
+ target_addr: &str,
+ catalog: &str,
+ schemas: &[String],
+) -> Option {
+ let home = default_home_dir_with(|key| std::env::var_os(key));
+ let cwd = std::env::current_dir().ok();
+ default_state_path_with(
+ home.as_deref(),
+ cwd.as_deref(),
+ snapshot_id,
+ target_addr,
+ catalog,
+ schemas,
+ )
+}
+
+fn default_home_dir_with(get: F) -> Option
+where
+ F: Fn(&str) -> Option,
+{
+ get("HOME")
+ .or_else(|| get("USERPROFILE"))
+ .map(PathBuf::from)
+ .or_else(|| {
+ let drive = get("HOMEDRIVE")?;
+ let path = get("HOMEPATH")?;
+ Some(PathBuf::from(drive).join(path))
+ })
+}
+
+fn default_state_path_with(
+ home: Option<&Path>,
+ cwd: Option<&Path>,
+ snapshot_id: &str,
+ target_addr: &str,
+ catalog: &str,
+ schemas: &[String],
+) -> Option {
+ let file_name = import_state_file_name(snapshot_id, target_addr, catalog, schemas);
+ match (home, cwd) {
+ (Some(home), _) => Some(
+ home.join(IMPORT_STATE_ROOT)
+ .join(IMPORT_STATE_DIR)
+ .join(file_name),
+ ),
+ (None, Some(cwd)) => Some(cwd.join(file_name)),
+ (None, None) => None,
+ }
+}
+
+fn import_state_file_name(
+ snapshot_id: &str,
+ target_addr: &str,
+ catalog: &str,
+ schemas: &[String],
+) -> String {
+ format!(
+ ".import_state_{}_{}_{}.json",
+ encode_path_segment(snapshot_id),
+ encode_path_segment(target_addr),
+ import_identity_hash(catalog, schemas)
+ )
+}
+
+pub(crate) fn canonical_schema_selection(schemas: &[String]) -> Vec {
+ let mut canonicalized = schemas
+ .iter()
+ .map(|schema| schema.to_ascii_lowercase())
+ .collect::>();
+ canonicalized.sort();
+ canonicalized.dedup();
+ canonicalized
+}
+
+/// FNV-1a over `(catalog, schemas)`. The output is part of the persisted state
+/// filename, so we cannot use `std::collections::hash_map::DefaultHasher` -
+/// Rust does not guarantee its algorithm across releases, which would make a
+/// state file written by one toolchain undiscoverable by another.
+fn import_identity_hash(catalog: &str, schemas: &[String]) -> String {
+ const FNV_OFFSET: u64 = 0xcbf29ce484222325;
+ const FNV_PRIME: u64 = 0x100000001b3;
+
+ fn hash_bytes(mut hash: u64, bytes: &[u8]) -> u64 {
+ for byte in bytes {
+ hash ^= u64::from(*byte);
+ hash = hash.wrapping_mul(FNV_PRIME);
+ }
+ hash
+ }
+
+ let mut hash = FNV_OFFSET;
+ hash = hash_bytes(hash, catalog.as_bytes());
+ // 0xff cannot appear in valid UTF-8, so it works as an unambiguous
+ // field separator between adjacent identifiers.
+ hash = hash_bytes(hash, &[0xff]);
+ for schema in canonical_schema_selection(schemas) {
+ hash = hash_bytes(hash, schema.as_bytes());
+ hash = hash_bytes(hash, &[0xff]);
+ }
+ format!("{hash:016x}")
+}
+
+pub(crate) async fn load_import_state(path: &Path) -> Result> {
+ match tokio::fs::read(path).await {
+ Ok(bytes) => {
+ let mut state: ImportState =
+ serde_json::from_slice(&bytes).context(ImportStateParseSnafu)?;
+ normalize_import_state_for_resume(&mut state);
+ Ok(Some(state))
+ }
+ Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(None),
+ Err(source) => Err(source).context(ImportStateIoSnafu {
+ path: path.display().to_string(),
+ }),
+ }
+}
+
+/// Caller must hold the lock acquired via `try_acquire_import_state_lock`.
+pub(crate) async fn save_import_state(path: &Path, state: &ImportState) -> Result<()> {
+ if let Some(parent) = path.parent() {
+ tokio::fs::create_dir_all(parent)
+ .await
+ .context(ImportStateIoSnafu {
+ path: parent.display().to_string(),
+ })?;
+ }
+
+ let bytes =
+ serde_json::to_vec_pretty(state).expect("ImportState should always be serializable");
+ let tmp_path = unique_tmp_path(path);
+ let mut file = tokio::fs::File::create(&tmp_path)
+ .await
+ .context(ImportStateIoSnafu {
+ path: tmp_path.display().to_string(),
+ })?;
+ file.write_all(&bytes).await.context(ImportStateIoSnafu {
+ path: tmp_path.display().to_string(),
+ })?;
+ file.sync_all().await.context(ImportStateIoSnafu {
+ path: tmp_path.display().to_string(),
+ })?;
+ // Close before rename; Windows forbids renaming an open file.
+ drop(file);
+
+ tokio::fs::rename(&tmp_path, path)
+ .await
+ .context(ImportStateIoSnafu {
+ path: path.display().to_string(),
+ })?;
+ sync_parent_dir(path).await?;
+ Ok(())
+}
+
+pub(crate) fn try_acquire_import_state_lock(path: &Path) -> Result {
+ if let Some(parent) = path.parent() {
+ std::fs::create_dir_all(parent).context(ImportStateIoSnafu {
+ path: parent.display().to_string(),
+ })?;
+ }
+
+ let lock_path = import_state_lock_path(path);
+ let file = std::fs::OpenOptions::new()
+ .create(true)
+ .read(true)
+ .write(true)
+ .truncate(false)
+ .open(&lock_path)
+ .context(ImportStateIoSnafu {
+ path: lock_path.display().to_string(),
+ })?;
+ file.try_lock_exclusive().map_err(|error| {
+ if is_lock_contention(&error) {
+ ImportStateLockedSnafu {
+ path: lock_path.display().to_string(),
+ }
+ .build()
+ } else {
+ ImportStateIoSnafu {
+ path: lock_path.display().to_string(),
+ }
+ .into_error(error)
+ }
+ })?;
+
+ Ok(ImportStateLockGuard { file })
+}
+
+fn is_lock_contention(error: &std::io::Error) -> bool {
+ error.kind() == std::io::ErrorKind::WouldBlock
+ || error.raw_os_error() == fs2::lock_contended_error().raw_os_error()
+}
+
+fn unique_tmp_path(path: &Path) -> PathBuf {
+ let pid = std::process::id();
+ let seq = IMPORT_STATE_TMP_ID.fetch_add(1, Ordering::Relaxed);
+ let file_name = path.file_name().unwrap_or_default().to_string_lossy();
+ path.with_file_name(format!("{file_name}.{pid}.{seq}.tmp"))
+}
+
+fn import_state_lock_path(path: &Path) -> PathBuf {
+ let file_name = path.file_name().unwrap_or_default().to_string_lossy();
+ path.with_file_name(format!("{file_name}.lock"))
+}
+
+fn normalize_import_state_for_resume(state: &mut ImportState) {
+ for task in &mut state.tasks {
+ if task.status == ImportTaskStatus::InProgress {
+ task.status = ImportTaskStatus::Pending;
+ task.error = None;
+ }
+ }
+}
+
+pub(crate) async fn delete_import_state(path: &Path) -> Result<()> {
+ match tokio::fs::remove_file(path).await {
+ Ok(()) => {
+ sync_parent_dir(path).await?;
+ Ok(())
+ }
+ Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()),
+ Err(source) => Err(source).context(ImportStateIoSnafu {
+ path: path.display().to_string(),
+ }),
+ }
+}
+
+#[cfg(unix)]
+async fn sync_parent_dir(path: &Path) -> Result<()> {
+ let Some(parent) = path.parent() else {
+ return Ok(());
+ };
+
+ let dir = tokio::fs::File::open(parent)
+ .await
+ .context(ImportStateIoSnafu {
+ path: parent.display().to_string(),
+ })?;
+ dir.sync_all().await.context(ImportStateIoSnafu {
+ path: parent.display().to_string(),
+ })?;
+ Ok(())
+}
+
+#[cfg(not(unix))]
+async fn sync_parent_dir(_path: &Path) -> Result<()> {
+ Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+ use std::process::Command;
+
+ use chrono::Utc;
+ use tempfile::tempdir;
+
+ use super::*;
+
+ const CHILD_LOCK_PATH_ENV: &str = "GREPTIME_IMPORT_STATE_LOCK_PATH";
+ const CHILD_LOCK_TEST: &str =
+ "data::import_v2::state::tests::test_try_acquire_import_state_lock_child_process";
+
+ fn schemas() -> Vec {
+ vec!["public".to_string(), "analytics".to_string()]
+ }
+
+ fn tasks() -> Vec {
+ vec![
+ ImportTaskKey::new(1, "public"),
+ ImportTaskKey::new(2, "analytics"),
+ ]
+ }
+
+ #[test]
+ fn test_import_state_new_initializes_pending_tasks() {
+ let state = ImportState::new(
+ "snapshot-1",
+ "127.0.0.1:4000",
+ "greptime",
+ &schemas(),
+ tasks(),
+ );
+
+ assert_eq!(state.snapshot_id, "snapshot-1");
+ assert_eq!(state.target_addr, "127.0.0.1:4000");
+ assert_eq!(state.catalog, "greptime");
+ assert_eq!(state.schemas, vec!["analytics", "public"]);
+ assert_eq!(state.tasks.len(), 2);
+ assert_eq!(state.tasks[0].status, ImportTaskStatus::Pending);
+ assert_eq!(state.tasks[1].status, ImportTaskStatus::Pending);
+ }
+
+ #[test]
+ fn test_set_task_status_updates_timestamp_and_error() {
+ let mut state = ImportState::new(
+ "snapshot-1",
+ "127.0.0.1:4000",
+ "greptime",
+ &schemas(),
+ tasks(),
+ );
+ let before = state.updated_at;
+ state.updated_at = Utc::now() - chrono::Duration::seconds(10);
+
+ state
+ .set_task_status(
+ 1,
+ "public",
+ ImportTaskStatus::Failed,
+ Some("timeout".to_string()),
+ )
+ .unwrap();
+ assert_eq!(
+ state.task_status(1, "public"),
+ Some(ImportTaskStatus::Failed)
+ );
+ assert_eq!(state.tasks[0].error.as_deref(), Some("timeout"));
+ assert!(state.updated_at > before);
+ }
+
+ #[test]
+ fn test_set_task_status_rejects_unknown_task() {
+ let mut state = ImportState::new(
+ "snapshot-1",
+ "127.0.0.1:4000",
+ "greptime",
+ &schemas(),
+ tasks(),
+ );
+
+ let error = state
+ .set_task_status(99, "public", ImportTaskStatus::Completed, None)
+ .unwrap_err();
+
+ assert!(matches!(
+ error,
+ crate::data::import_v2::error::Error::ImportStateUnknownTask { chunk_id, schema, .. }
+ if chunk_id == 99 && schema == "public"
+ ));
+ }
+
+ #[tokio::test]
+ async fn test_save_and_load_import_state_round_trip() {
+ let dir = tempdir().unwrap();
+ let path = dir.path().join("import_state.json");
+ let mut state = ImportState::new(
+ "snapshot-1",
+ "127.0.0.1:4000",
+ "greptime",
+ &schemas(),
+ tasks(),
+ );
+ state
+ .set_task_status(2, "analytics", ImportTaskStatus::Completed, None)
+ .unwrap();
+
+ save_import_state(&path, &state).await.unwrap();
+ let loaded = load_import_state(&path).await.unwrap().unwrap();
+
+ assert_eq!(loaded.snapshot_id, state.snapshot_id);
+ assert_eq!(loaded.target_addr, state.target_addr);
+ assert_eq!(loaded.catalog, state.catalog);
+ assert_eq!(loaded.schemas, state.schemas);
+ assert_eq!(loaded.tasks, state.tasks);
+ }
+
+ #[tokio::test]
+ async fn test_save_import_state_overwrites_existing_file() {
+ let dir = tempdir().unwrap();
+ let path = dir.path().join("import_state.json");
+ let mut state = ImportState::new(
+ "snapshot-1",
+ "127.0.0.1:4000",
+ "greptime",
+ &schemas(),
+ tasks(),
+ );
+ save_import_state(&path, &state).await.unwrap();
+
+ state
+ .set_task_status(1, "public", ImportTaskStatus::Completed, None)
+ .unwrap();
+ save_import_state(&path, &state).await.unwrap();
+
+ let loaded = load_import_state(&path).await.unwrap().unwrap();
+ assert_eq!(
+ loaded.task_status(1, "public"),
+ Some(ImportTaskStatus::Completed)
+ );
+ }
+
+ #[test]
+ fn test_load_import_state_resets_in_progress_to_pending() {
+ let mut state = ImportState::new(
+ "snapshot-1",
+ "127.0.0.1:4000",
+ "greptime",
+ &schemas(),
+ tasks(),
+ );
+ state
+ .set_task_status(
+ 2,
+ "analytics",
+ ImportTaskStatus::InProgress,
+ Some("running".to_string()),
+ )
+ .unwrap();
+
+ normalize_import_state_for_resume(&mut state);
+
+ assert_eq!(
+ state.task_status(1, "public"),
+ Some(ImportTaskStatus::Pending)
+ );
+ assert_eq!(
+ state.task_status(2, "analytics"),
+ Some(ImportTaskStatus::Pending)
+ );
+ assert_eq!(state.tasks[1].error, None);
+ }
+
+ #[test]
+ fn test_unique_tmp_path_generates_distinct_paths() {
+ let dir = tempdir().unwrap();
+ let path = dir.path().join("import_state.json");
+
+ let first = unique_tmp_path(&path);
+ let second = unique_tmp_path(&path);
+
+ assert_ne!(first, second);
+ assert!(first.starts_with(dir.path()));
+ assert!(second.starts_with(dir.path()));
+ assert!(
+ first
+ .file_name()
+ .unwrap()
+ .to_string_lossy()
+ .ends_with(".tmp")
+ );
+ assert!(
+ second
+ .file_name()
+ .unwrap()
+ .to_string_lossy()
+ .ends_with(".tmp")
+ );
+ }
+
+ #[test]
+ fn test_lock_contention_detection_accepts_platform_error() {
+ let error = fs2::lock_contended_error();
+
+ assert!(is_lock_contention(&error));
+ }
+
+ #[test]
+ fn test_try_acquire_import_state_lock_rejects_second_holder() {
+ let dir = tempdir().unwrap();
+ let path = dir.path().join("import_state.json");
+
+ let _first = try_acquire_import_state_lock(&path).unwrap();
+ // Import state locking guards concurrent CLI processes, so validate cross-process exclusion.
+ let output = Command::new(std::env::current_exe().unwrap())
+ .arg(CHILD_LOCK_TEST)
+ .arg("--ignored")
+ .arg("--exact")
+ .env(CHILD_LOCK_PATH_ENV, &path)
+ .output()
+ .unwrap();
+
+ assert!(
+ output.status.success(),
+ "child lock test failed\nstdout:\n{}\nstderr:\n{}",
+ String::from_utf8_lossy(&output.stdout),
+ String::from_utf8_lossy(&output.stderr)
+ );
+ let stdout = String::from_utf8_lossy(&output.stdout);
+ assert!(
+ stdout.contains("1 passed"),
+ "child lock test did not run the expected ignored test\nstdout:\n{stdout}"
+ );
+ }
+
+ #[test]
+ #[ignore = "spawned by test_try_acquire_import_state_lock_rejects_second_holder"]
+ fn test_try_acquire_import_state_lock_child_process() {
+ let path = std::env::var_os(CHILD_LOCK_PATH_ENV)
+ .expect("child lock path must be set by the parent test");
+ let path = PathBuf::from(path);
+ let error = try_acquire_import_state_lock(&path).unwrap_err();
+
+ assert!(matches!(
+ error,
+ crate::data::import_v2::error::Error::ImportStateLocked { .. }
+ ));
+ }
+
+ #[tokio::test]
+ async fn test_delete_import_state_ignores_missing_file() {
+ let dir = tempdir().unwrap();
+ let path = dir.path().join("missing.json");
+
+ delete_import_state(&path).await.unwrap();
+ }
+
+ #[test]
+ fn test_default_state_path_prefers_home_and_encodes_snapshot_id() {
+ let home = tempdir().unwrap();
+ let cwd = tempdir().unwrap();
+
+ let path = default_state_path_with(
+ Some(home.path()),
+ Some(cwd.path()),
+ "../snapshot",
+ "127.0.0.1:4000",
+ "greptime",
+ &schemas(),
+ )
+ .unwrap();
+
+ assert_eq!(
+ path.parent().unwrap(),
+ home.path().join(IMPORT_STATE_ROOT).join(IMPORT_STATE_DIR)
+ );
+ let file_name = path.file_name().unwrap().to_string_lossy();
+ assert!(file_name.starts_with(".import_state_%2E%2E%2Fsnapshot_127%2E0%2E0%2E1%3A4000_"));
+ assert!(file_name.ends_with(".json"));
+ }
+
+ #[test]
+ fn test_default_state_path_falls_back_to_cwd_when_home_missing() {
+ let cwd = tempdir().unwrap();
+
+ let path = default_state_path_with(
+ None,
+ Some(cwd.path()),
+ "snapshot-1",
+ "target-a",
+ "greptime",
+ &schemas(),
+ )
+ .unwrap();
+
+ assert_eq!(path.parent().unwrap(), cwd.path());
+ let file_name = path.file_name().unwrap().to_string_lossy();
+ assert!(file_name.starts_with(".import_state_snapshot-1_target-a_"));
+ assert!(file_name.ends_with(".json"));
+ }
+
+ #[test]
+ fn test_default_state_path_isolated_by_target_addr() {
+ let cwd = tempdir().unwrap();
+
+ let first = default_state_path_with(
+ None,
+ Some(cwd.path()),
+ "snapshot-1",
+ "127.0.0.1:4000",
+ "greptime",
+ &schemas(),
+ )
+ .unwrap();
+ let second = default_state_path_with(
+ None,
+ Some(cwd.path()),
+ "snapshot-1",
+ "127.0.0.1:4001",
+ "greptime",
+ &schemas(),
+ )
+ .unwrap();
+
+ assert_ne!(first, second);
+ }
+
+ #[test]
+ fn test_default_state_path_isolated_by_catalog_and_schemas() {
+ let cwd = tempdir().unwrap();
+ let public_only = vec!["public".to_string()];
+ let analytics_only = vec!["analytics".to_string()];
+
+ let first = default_state_path_with(
+ None,
+ Some(cwd.path()),
+ "snapshot-1",
+ "127.0.0.1:4000",
+ "greptime",
+ &public_only,
+ )
+ .unwrap();
+ let second = default_state_path_with(
+ None,
+ Some(cwd.path()),
+ "snapshot-1",
+ "127.0.0.1:4000",
+ "other",
+ &public_only,
+ )
+ .unwrap();
+ let third = default_state_path_with(
+ None,
+ Some(cwd.path()),
+ "snapshot-1",
+ "127.0.0.1:4000",
+ "greptime",
+ &analytics_only,
+ )
+ .unwrap();
+
+ assert_ne!(first, second);
+ assert_ne!(first, third);
+ }
+
+ #[test]
+ fn test_default_home_dir_prefers_home() {
+ let detected = default_home_dir_with(|key| match key {
+ "HOME" => Some(std::ffi::OsString::from("/tmp/home")),
+ "USERPROFILE" => Some(std::ffi::OsString::from("/tmp/userprofile")),
+ _ => None,
+ });
+
+ assert_eq!(detected, Some(PathBuf::from("/tmp/home")));
+ }
+
+ #[test]
+ fn test_default_home_dir_falls_back_to_userprofile() {
+ let detected = default_home_dir_with(|key| match key {
+ "USERPROFILE" => Some(std::ffi::OsString::from("/tmp/userprofile")),
+ _ => None,
+ });
+
+ assert_eq!(detected, Some(PathBuf::from("/tmp/userprofile")));
+ }
+
+ #[test]
+ fn test_default_home_dir_falls_back_to_home_drive_and_path() {
+ let detected = default_home_dir_with(|key| match key {
+ "HOMEDRIVE" => Some(std::ffi::OsString::from("/tmp")),
+ "HOMEPATH" => Some(std::ffi::OsString::from("windows-home")),
+ _ => None,
+ });
+
+ assert_eq!(detected, Some(PathBuf::from("/tmp").join("windows-home")));
+ }
+}
diff --git a/src/cli/src/data/snapshot_storage.rs b/src/cli/src/data/snapshot_storage.rs
index 6bc71153df..2ee07b5586 100644
--- a/src/cli/src/data/snapshot_storage.rs
+++ b/src/cli/src/data/snapshot_storage.rs
@@ -17,6 +17,8 @@
//! This module provides a unified interface for reading and writing snapshot data
//! to various storage backends (S3, OSS, GCS, Azure Blob, local filesystem).
+use std::collections::BTreeSet;
+
use async_trait::async_trait;
use futures::TryStreamExt;
use object_store::services::{Azblob, Fs, Gcs, Oss, S3};
@@ -75,7 +77,10 @@ impl StorageScheme {
}
/// Extracts bucket/container and root path from a URI.
-fn extract_remote_location(uri: &str) -> Result {
+fn extract_remote_location_with_root_policy(
+ uri: &str,
+ allow_empty_root: bool,
+) -> Result {
let url = Url::parse(uri).context(UrlParseSnafu)?;
let bucket_or_container = url.host_str().unwrap_or("").to_string();
if bucket_or_container.is_empty() {
@@ -87,7 +92,7 @@ fn extract_remote_location(uri: &str) -> Result {
}
let root = url.path().trim_start_matches('/').to_string();
- if root.is_empty() {
+ if root.is_empty() && !allow_empty_root {
return InvalidUriSnafu {
uri,
reason: "snapshot URI must include a non-empty path after the bucket/container",
@@ -268,13 +273,21 @@ impl OpenDalStorage {
}
fn from_s3_uri(uri: &str, storage: &ObjectStoreConfig) -> Result {
+ Self::from_s3_uri_with_root_policy(uri, storage, false)
+ }
+
+ fn from_s3_uri_with_root_policy(
+ uri: &str,
+ storage: &ObjectStoreConfig,
+ allow_empty_root: bool,
+ ) -> Result {
Self::ensure_backend_enabled(
uri,
storage.enable_s3,
"s3:// requires --s3 and related options",
)?;
- let location = extract_remote_location(uri)?;
+ let location = extract_remote_location_with_root_policy(uri, allow_empty_root)?;
let mut config = storage.s3.clone();
config.s3_bucket = location.bucket_or_container;
config.s3_root = location.root;
@@ -291,13 +304,21 @@ impl OpenDalStorage {
}
fn from_oss_uri(uri: &str, storage: &ObjectStoreConfig) -> Result {
+ Self::from_oss_uri_with_root_policy(uri, storage, false)
+ }
+
+ fn from_oss_uri_with_root_policy(
+ uri: &str,
+ storage: &ObjectStoreConfig,
+ allow_empty_root: bool,
+ ) -> Result {
Self::ensure_backend_enabled(
uri,
storage.enable_oss,
"oss:// requires --oss and related options",
)?;
- let location = extract_remote_location(uri)?;
+ let location = extract_remote_location_with_root_policy(uri, allow_empty_root)?;
let mut config = storage.oss.clone();
config.oss_bucket = location.bucket_or_container;
config.oss_root = location.root;
@@ -314,17 +335,30 @@ impl OpenDalStorage {
}
fn from_gcs_uri(uri: &str, storage: &ObjectStoreConfig) -> Result {
+ Self::from_gcs_uri_with_root_policy(uri, storage, false)
+ }
+
+ fn from_gcs_uri_with_root_policy(
+ uri: &str,
+ storage: &ObjectStoreConfig,
+ allow_empty_root: bool,
+ ) -> Result {
Self::ensure_backend_enabled(
uri,
storage.enable_gcs,
"gs:// or gcs:// requires --gcs and related options",
)?;
- let location = extract_remote_location(uri)?;
+ let location = extract_remote_location_with_root_policy(uri, allow_empty_root)?;
let mut config = storage.gcs.clone();
config.gcs_bucket = location.bucket_or_container;
config.gcs_root = location.root;
- Self::validate_remote_config(uri, "gcs", config.validate())?;
+ // GCS validate() rejects empty root, unlike S3/OSS/Azblob.
+ if allow_empty_root && config.gcs_root.is_empty() {
+ Self::validate_gcs_parent_config(uri, &config)?;
+ } else {
+ Self::validate_remote_config(uri, "gcs", config.validate())?;
+ }
let conn: GcsConnection = config.into();
let object_store = ObjectStore::new(Gcs::from(&conn))
@@ -336,14 +370,43 @@ impl OpenDalStorage {
))
}
+ fn validate_gcs_parent_config(
+ uri: &str,
+ config: &crate::common::PrefixedGcsConnection,
+ ) -> Result<()> {
+ if config.gcs_bucket.is_empty() {
+ return InvalidUriSnafu {
+ uri,
+ reason: "invalid gcs config: GCS bucket must be set when --gcs is enabled.",
+ }
+ .fail();
+ }
+ if config.gcs_scope.is_empty() {
+ return InvalidUriSnafu {
+ uri,
+ reason: "invalid gcs config: GCS scope must be set when --gcs is enabled.",
+ }
+ .fail();
+ }
+ Ok(())
+ }
+
fn from_azblob_uri(uri: &str, storage: &ObjectStoreConfig) -> Result {
+ Self::from_azblob_uri_with_root_policy(uri, storage, false)
+ }
+
+ fn from_azblob_uri_with_root_policy(
+ uri: &str,
+ storage: &ObjectStoreConfig,
+ allow_empty_root: bool,
+ ) -> Result {
Self::ensure_backend_enabled(
uri,
storage.enable_azblob,
"azblob:// requires --azblob and related options",
)?;
- let location = extract_remote_location(uri)?;
+ let location = extract_remote_location_with_root_policy(uri, allow_empty_root)?;
let mut config = storage.azblob.clone();
config.azblob_container = location.bucket_or_container;
config.azblob_root = location.root;
@@ -370,6 +433,21 @@ impl OpenDalStorage {
}
}
+ /// Creates storage rooted at a snapshot parent URI.
+ ///
+ /// Parent-oriented commands such as `export-v2 list` may scan bucket/container
+ /// roots. Snapshot-oriented commands must keep using `from_uri`, which rejects
+ /// empty remote roots to avoid unsafe snapshot operations at bucket scope.
+ pub fn from_parent_uri(uri: &str, storage: &ObjectStoreConfig) -> Result {
+ match StorageScheme::from_uri(uri)? {
+ StorageScheme::File => Self::from_file_uri_with_config(uri, storage),
+ StorageScheme::S3 => Self::from_s3_uri_with_root_policy(uri, storage, true),
+ StorageScheme::Oss => Self::from_oss_uri_with_root_policy(uri, storage, true),
+ StorageScheme::Gcs => Self::from_gcs_uri_with_root_policy(uri, storage, true),
+ StorageScheme::Azblob => Self::from_azblob_uri_with_root_policy(uri, storage, true),
+ }
+ }
+
/// Reads a file as bytes.
async fn read_file(&self, path: &str) -> Result> {
let data = self
@@ -382,6 +460,17 @@ impl OpenDalStorage {
Ok(data.to_vec())
}
+ /// Reads a file as bytes if it exists.
+ pub(crate) async fn read_file_if_exists(&self, path: &str) -> Result>> {
+ match self.object_store.read(path).await {
+ Ok(data) => Ok(Some(data.to_vec())),
+ Err(error) if error.kind() == ErrorKind::NotFound => Ok(None),
+ Err(error) => Err(error).context(StorageOperationSnafu {
+ operation: format!("read {}", path),
+ }),
+ }
+ }
+
/// Writes bytes to a file.
async fn write_file(&self, path: &str, data: Vec) -> Result<()> {
self.object_store
@@ -404,6 +493,37 @@ impl OpenDalStorage {
}
}
+ /// Lists direct child directory names under the storage root.
+ pub(crate) async fn list_direct_child_dirs(&self) -> Result> {
+ let mut lister = match self.object_store.lister_with("/").recursive(false).await {
+ Ok(lister) => lister,
+ Err(error) if error.kind() == ErrorKind::NotFound => return Ok(Vec::new()),
+ Err(error) => {
+ return Err(error).context(StorageOperationSnafu {
+ operation: "list /",
+ });
+ }
+ };
+
+ let mut dirs = BTreeSet::new();
+ while let Some(entry) = lister.try_next().await.context(StorageOperationSnafu {
+ operation: "list /",
+ })? {
+ let path = entry.path().trim_matches('/');
+ if path.is_empty() {
+ continue;
+ }
+
+ if entry.metadata().is_dir()
+ && let Some(name) = path.split('/').next()
+ {
+ dirs.insert(name.to_string());
+ }
+ }
+
+ Ok(dirs.into_iter().collect())
+ }
+
#[cfg(test)]
pub async fn read_schema(&self) -> Result {
let schemas_path = schema_index_path();
@@ -486,7 +606,8 @@ impl SnapshotStorage for OpenDalStorage {
async fn delete_snapshot(&self) -> Result<()> {
self.object_store
- .remove_all("/")
+ .delete_with("/")
+ .recursive(true)
.await
.context(StorageOperationSnafu {
operation: "delete snapshot",
@@ -556,11 +677,35 @@ mod tests {
#[test]
fn test_extract_remote_location_requires_non_empty_root() {
- assert!(extract_remote_location("s3://bucket").is_err());
- assert!(extract_remote_location("s3://bucket/").is_err());
- assert!(extract_remote_location("oss://bucket").is_err());
- assert!(extract_remote_location("gs://bucket").is_err());
- assert!(extract_remote_location("azblob://container").is_err());
+ assert!(extract_remote_location_with_root_policy("s3://bucket", false).is_err());
+ assert!(extract_remote_location_with_root_policy("s3://bucket/", false).is_err());
+ assert!(extract_remote_location_with_root_policy("oss://bucket", false).is_err());
+ assert!(extract_remote_location_with_root_policy("gs://bucket", false).is_err());
+ assert!(extract_remote_location_with_root_policy("azblob://container", false).is_err());
+ }
+
+ #[test]
+ fn test_extract_remote_location_allows_empty_root_when_permitted() {
+ let location = extract_remote_location_with_root_policy("s3://bucket", true).unwrap();
+ assert_eq!(location.bucket_or_container, "bucket");
+ assert_eq!(location.root, "");
+
+ let location =
+ extract_remote_location_with_root_policy("azblob://container/", true).unwrap();
+ assert_eq!(location.bucket_or_container, "container");
+ assert_eq!(location.root, "");
+ }
+
+ #[test]
+ fn test_parent_storage_allows_s3_bucket_root() {
+ let mut storage = ObjectStoreConfig {
+ enable_s3: true,
+ ..Default::default()
+ };
+ storage.s3.s3_region = Some("us-east-1".to_string());
+
+ assert!(OpenDalStorage::from_uri("s3://bucket", &storage).is_err());
+ assert!(OpenDalStorage::from_parent_uri("s3://bucket", &storage).is_ok());
}
#[cfg(not(windows))]
diff --git a/src/cli/src/database.rs b/src/cli/src/database.rs
index fa3f6faefb..cba27129dd 100644
--- a/src/cli/src/database.rs
+++ b/src/cli/src/database.rs
@@ -89,6 +89,10 @@ impl DatabaseClient {
}
}
+ pub fn addr(&self) -> &str {
+ &self.addr
+ }
+
pub async fn sql_in_public(&self, sql: &str) -> Result>>> {
self.sql(sql, DEFAULT_SCHEMA_NAME).await
}
diff --git a/src/cli/src/metadata.rs b/src/cli/src/metadata.rs
index 5800293cb6..beb62a2deb 100644
--- a/src/cli/src/metadata.rs
+++ b/src/cli/src/metadata.rs
@@ -21,7 +21,7 @@ use clap::Subcommand;
use common_error::ext::BoxedError;
use crate::Tool;
-use crate::metadata::control::{DelCommand, GetCommand};
+use crate::metadata::control::{DelCommand, GetCommand, PutCommand};
use crate::metadata::repair::RepairCommand;
use crate::metadata::snapshot::SnapshotCommand;
@@ -37,6 +37,8 @@ pub enum MetadataCommand {
#[clap(subcommand)]
Del(DelCommand),
#[clap(subcommand)]
+ Put(PutCommand),
+ #[clap(subcommand)]
Repair(RepairCommand),
}
@@ -47,6 +49,7 @@ impl MetadataCommand {
MetadataCommand::Repair(cmd) => cmd.build().await,
MetadataCommand::Get(cmd) => cmd.build().await,
MetadataCommand::Del(cmd) => cmd.build().await,
+ MetadataCommand::Put(cmd) => cmd.build().await,
}
}
}
diff --git a/src/cli/src/metadata/control.rs b/src/cli/src/metadata/control.rs
index 3fba582ce5..eaf5b167b5 100644
--- a/src/cli/src/metadata/control.rs
+++ b/src/cli/src/metadata/control.rs
@@ -14,9 +14,12 @@
mod del;
mod get;
+mod put;
+mod selector;
#[cfg(test)]
mod test_utils;
mod utils;
pub(crate) use del::DelCommand;
pub(crate) use get::GetCommand;
+pub(crate) use put::PutCommand;
diff --git a/src/cli/src/metadata/control/del/table.rs b/src/cli/src/metadata/control/del/table.rs
index d1346928fe..c92e573224 100644
--- a/src/cli/src/metadata/control/del/table.rs
+++ b/src/cli/src/metadata/control/del/table.rs
@@ -14,111 +14,59 @@
use async_trait::async_trait;
use clap::Parser;
-use client::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME};
-use common_catalog::format_full_table_name;
use common_error::ext::BoxedError;
use common_meta::ddl::utils::get_region_wal_options;
use common_meta::key::TableMetadataManager;
-use common_meta::key::table_name::TableNameManager;
use common_meta::kv_backend::KvBackendRef;
use store_api::storage::TableId;
use crate::Tool;
use crate::common::StoreConfig;
-use crate::error::{InvalidArgumentsSnafu, TableNotFoundSnafu};
+use crate::error::TableNotFoundSnafu;
use crate::metadata::control::del::CLI_TOMBSTONE_PREFIX;
-use crate::metadata::control::utils::get_table_id_by_name;
+use crate::metadata::control::selector::TableSelector;
/// Delete table metadata logically from the metadata store.
#[derive(Debug, Default, Parser)]
pub struct DelTableCommand {
- /// The table id to delete from the metadata store.
- #[clap(long)]
- table_id: Option,
-
- /// The table name to delete from the metadata store.
- #[clap(long)]
- table_name: Option,
-
- /// The schema name of the table.
- #[clap(long, default_value = DEFAULT_SCHEMA_NAME)]
- schema_name: String,
-
- /// The catalog name of the table.
- #[clap(long, default_value = DEFAULT_CATALOG_NAME)]
- catalog_name: String,
+ #[clap(flatten)]
+ selector: TableSelector,
/// The store config.
#[clap(flatten)]
store: StoreConfig,
}
-impl DelTableCommand {
- fn validate(&self) -> Result<(), BoxedError> {
- if matches!(
- (&self.table_id, &self.table_name),
- (Some(_), Some(_)) | (None, None)
- ) {
- return Err(BoxedError::new(
- InvalidArgumentsSnafu {
- msg: "You must specify either --table-id or --table-name.",
- }
- .build(),
- ));
- }
- Ok(())
- }
-}
-
impl DelTableCommand {
pub async fn build(&self) -> Result, BoxedError> {
- self.validate()?;
+ self.selector.validate()?;
let kv_backend = self.store.build().await?;
Ok(Box::new(DelTableTool {
- table_id: self.table_id,
- table_name: self.table_name.clone(),
- schema_name: self.schema_name.clone(),
- catalog_name: self.catalog_name.clone(),
- table_name_manager: TableNameManager::new(kv_backend.clone()),
+ selector: self.selector.clone(),
table_metadata_deleter: TableMetadataDeleter::new(kv_backend),
}))
}
}
struct DelTableTool {
- table_id: Option,
- table_name: Option,
- schema_name: String,
- catalog_name: String,
- table_name_manager: TableNameManager,
+ selector: TableSelector,
table_metadata_deleter: TableMetadataDeleter,
}
#[async_trait]
impl Tool for DelTableTool {
async fn do_work(&self) -> Result<(), BoxedError> {
- let table_id = if let Some(table_name) = &self.table_name {
- let catalog_name = &self.catalog_name;
- let schema_name = &self.schema_name;
-
- let Some(table_id) = get_table_id_by_name(
- &self.table_name_manager,
- catalog_name,
- schema_name,
- table_name,
+ let Some(table_id) = self
+ .selector
+ .resolve_table_id(
+ self.table_metadata_deleter
+ .table_metadata_manager
+ .table_name_manager(),
)
.await?
- else {
- println!(
- "Table({}) not found",
- format_full_table_name(catalog_name, schema_name, table_name)
- );
- return Ok(());
- };
- table_id
- } else {
- // Safety: we have validated that table_id or table_name is not None
- self.table_id.unwrap()
+ else {
+ println!("Table({}) not found", self.selector.formatted_table_name());
+ return Ok(());
};
self.table_metadata_deleter.delete(table_id).await?;
println!("Table({}) deleted", table_id);
@@ -182,6 +130,7 @@ mod tests {
use std::collections::HashMap;
use std::sync::Arc;
+ use clap::Parser;
use common_error::ext::ErrorExt;
use common_error::status_code::StatusCode;
use common_meta::key::TableMetadataManager;
@@ -192,9 +141,83 @@ mod tests {
use common_meta::rpc::store::RangeRequest;
use crate::metadata::control::del::CLI_TOMBSTONE_PREFIX;
- use crate::metadata::control::del::table::TableMetadataDeleter;
+ use crate::metadata::control::del::table::{DelTableCommand, TableMetadataDeleter};
use crate::metadata::control::test_utils::prepare_physical_table_metadata;
+ #[tokio::test]
+ async fn test_del_table_selector_requires_single_target() {
+ let command = DelTableCommand::parse_from([
+ "table",
+ "--backend",
+ "memory-store",
+ "--store-addrs",
+ "memory://",
+ ]);
+
+ let err = match command.build().await {
+ Ok(_) => panic!("expected validation failure"),
+ Err(err) => err,
+ };
+ assert!(
+ err.output_msg()
+ .contains("You must specify either --table-id or --table-name.")
+ );
+ }
+
+ #[tokio::test]
+ async fn test_del_table_selector_rejects_both_targets() {
+ let command = DelTableCommand::parse_from([
+ "table",
+ "--table-id",
+ "1024",
+ "--table-name",
+ "my_table",
+ "--backend",
+ "memory-store",
+ "--store-addrs",
+ "memory://",
+ ]);
+
+ let err = match command.build().await {
+ Ok(_) => panic!("expected validation failure"),
+ Err(err) => err,
+ };
+ assert!(
+ err.output_msg()
+ .contains("You must specify either --table-id or --table-name.")
+ );
+ }
+
+ #[tokio::test]
+ async fn test_del_table_command_builds_tool_with_table_id() {
+ let command = DelTableCommand::parse_from([
+ "table",
+ "--table-id",
+ "1024",
+ "--backend",
+ "memory-store",
+ "--store-addrs",
+ "memory://",
+ ]);
+
+ let _tool = command.build().await.unwrap();
+ }
+
+ #[tokio::test]
+ async fn test_del_table_command_builds_tool_with_table_name() {
+ let command = DelTableCommand::parse_from([
+ "table",
+ "--table-name",
+ "my_table",
+ "--backend",
+ "memory-store",
+ "--store-addrs",
+ "memory://",
+ ]);
+
+ let _tool = command.build().await.unwrap();
+ }
+
#[tokio::test]
async fn test_delete_table_not_found() {
let kv_backend = Arc::new(MemoryKvBackend::new()) as KvBackendRef;
diff --git a/src/cli/src/metadata/control/get.rs b/src/cli/src/metadata/control/get.rs
index 1de2bb370e..8a1de07acf 100644
--- a/src/cli/src/metadata/control/get.rs
+++ b/src/cli/src/metadata/control/get.rs
@@ -16,8 +16,6 @@ use std::cmp::min;
use async_trait::async_trait;
use clap::{Parser, Subcommand};
-use client::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME};
-use common_catalog::format_full_table_name;
use common_error::ext::BoxedError;
use common_meta::key::TableMetadataManager;
use common_meta::key::table_info::TableInfoKey;
@@ -29,8 +27,8 @@ use futures::TryStreamExt;
use crate::Tool;
use crate::common::StoreConfig;
-use crate::error::InvalidArgumentsSnafu;
-use crate::metadata::control::utils::{decode_key_value, get_table_id_by_name, json_formatter};
+use crate::metadata::control::selector::TableSelector;
+use crate::metadata::control::utils::{decode_key_value, json_formatter};
/// Getting metadata from metadata store.
#[derive(Subcommand)]
@@ -120,21 +118,8 @@ impl Tool for GetKeyTool {
/// Get table metadata from the metadata store via table id.
#[derive(Debug, Default, Parser)]
pub struct GetTableCommand {
- /// Get table metadata by table id.
- #[clap(long)]
- table_id: Option,
-
- /// Get table metadata by table name.
- #[clap(long)]
- table_name: Option,
-
- /// The schema name of the table.
- #[clap(long, default_value = DEFAULT_SCHEMA_NAME)]
- schema_name: String,
-
- /// The catalog name of the table.
- #[clap(long, default_value = DEFAULT_CATALOG_NAME)]
- catalog_name: String,
+ #[clap(flatten)]
+ selector: TableSelector,
/// Pretty print the output.
#[clap(long, default_value = "false")]
@@ -144,29 +129,9 @@ pub struct GetTableCommand {
store: StoreConfig,
}
-impl GetTableCommand {
- pub fn validate(&self) -> Result<(), BoxedError> {
- if matches!(
- (&self.table_id, &self.table_name),
- (Some(_), Some(_)) | (None, None)
- ) {
- return Err(BoxedError::new(
- InvalidArgumentsSnafu {
- msg: "You must specify either --table-id or --table-name.",
- }
- .build(),
- ));
- }
- Ok(())
- }
-}
-
struct GetTableTool {
kvbackend: KvBackendRef,
- table_id: Option,
- table_name: Option,
- schema_name: String,
- catalog_name: String,
+ selector: TableSelector,
pretty: bool,
}
@@ -178,24 +143,9 @@ impl Tool for GetTableTool {
let table_info_manager = table_metadata_manager.table_info_manager();
let table_route_manager = table_metadata_manager.table_route_manager();
- let table_id = if let Some(table_name) = &self.table_name {
- let catalog_name = &self.catalog_name;
- let schema_name = &self.schema_name;
-
- let Some(table_id) =
- get_table_id_by_name(table_name_manager, catalog_name, schema_name, table_name)
- .await?
- else {
- println!(
- "Table({}) not found",
- format_full_table_name(catalog_name, schema_name, table_name)
- );
- return Ok(());
- };
- table_id
- } else {
- // Safety: we have validated that table_id or table_name is not None
- self.table_id.unwrap()
+ let Some(table_id) = self.selector.resolve_table_id(table_name_manager).await? else {
+ println!("Table({}) not found", self.selector.formatted_table_name());
+ return Ok(());
};
let table_info = table_info_manager
@@ -233,15 +183,94 @@ impl Tool for GetTableTool {
impl GetTableCommand {
pub async fn build(&self) -> Result, BoxedError> {
- self.validate()?;
+ self.selector.validate()?;
let kvbackend = self.store.build().await?;
Ok(Box::new(GetTableTool {
kvbackend,
- table_id: self.table_id,
- table_name: self.table_name.clone(),
- schema_name: self.schema_name.clone(),
- catalog_name: self.catalog_name.clone(),
+ selector: self.selector.clone(),
pretty: self.pretty,
}))
}
}
+
+#[cfg(test)]
+mod tests {
+ use clap::Parser;
+ use common_error::ext::ErrorExt;
+
+ use super::GetTableCommand;
+
+ #[tokio::test]
+ async fn test_get_table_selector_requires_single_target() {
+ let command = GetTableCommand::parse_from([
+ "table",
+ "--backend",
+ "memory-store",
+ "--store-addrs",
+ "memory://",
+ ]);
+
+ let err = match command.build().await {
+ Ok(_) => panic!("expected validation failure"),
+ Err(err) => err,
+ };
+ assert!(
+ err.output_msg()
+ .contains("You must specify either --table-id or --table-name.")
+ );
+ }
+
+ #[tokio::test]
+ async fn test_get_table_selector_rejects_both_targets() {
+ let command = GetTableCommand::parse_from([
+ "table",
+ "--table-id",
+ "1024",
+ "--table-name",
+ "my_table",
+ "--backend",
+ "memory-store",
+ "--store-addrs",
+ "memory://",
+ ]);
+
+ let err = match command.build().await {
+ Ok(_) => panic!("expected validation failure"),
+ Err(err) => err,
+ };
+ assert!(
+ err.output_msg()
+ .contains("You must specify either --table-id or --table-name.")
+ );
+ }
+
+ #[tokio::test]
+ async fn test_get_table_command_builds_tool_with_table_id() {
+ let command = GetTableCommand::parse_from([
+ "table",
+ "--table-id",
+ "1024",
+ "--backend",
+ "memory-store",
+ "--store-addrs",
+ "memory://",
+ ]);
+
+ let _tool = command.build().await.unwrap();
+ }
+
+ #[tokio::test]
+ async fn test_get_table_command_builds_tool_with_table_name() {
+ let command = GetTableCommand::parse_from([
+ "table",
+ "--table-name",
+ "my_table",
+ "--backend",
+ "memory-store",
+ "--store-addrs",
+ "memory://",
+ ]);
+
+ let _tool = command.build().await.unwrap();
+ }
+}
diff --git a/src/cli/src/metadata/control/put.rs b/src/cli/src/metadata/control/put.rs
new file mode 100644
index 0000000000..c2c535c57a
--- /dev/null
+++ b/src/cli/src/metadata/control/put.rs
@@ -0,0 +1,56 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+mod key;
+mod table;
+
+use clap::Subcommand;
+use common_error::ext::BoxedError;
+use snafu::ResultExt;
+use tokio::io::{AsyncRead, AsyncReadExt};
+
+use crate::Tool;
+use crate::error::FileIoSnafu;
+use crate::metadata::control::put::key::PutKeyCommand;
+use crate::metadata::control::put::table::PutTableCommand;
+
+pub(crate) async fn read_value(mut reader: R) -> Result, BoxedError>
+where
+ R: AsyncRead + Unpin,
+{
+ let mut value = Vec::new();
+ reader
+ .read_to_end(&mut value)
+ .await
+ .context(FileIoSnafu)
+ .map_err(BoxedError::new)?;
+ Ok(value)
+}
+
+/// Subcommand for putting metadata into the metadata store.
+#[derive(Subcommand)]
+pub enum PutCommand {
+ Key(PutKeyCommand),
+ #[clap(subcommand)]
+ Table(PutTableCommand),
+}
+
+impl PutCommand {
+ pub async fn build(&self) -> Result, BoxedError> {
+ match self {
+ PutCommand::Key(cmd) => cmd.build().await,
+ PutCommand::Table(cmd) => cmd.build().await,
+ }
+ }
+}
diff --git a/src/cli/src/metadata/control/put/key.rs b/src/cli/src/metadata/control/put/key.rs
new file mode 100644
index 0000000000..7becfd72dc
--- /dev/null
+++ b/src/cli/src/metadata/control/put/key.rs
@@ -0,0 +1,444 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use async_trait::async_trait;
+use clap::Parser;
+use common_error::ext::BoxedError;
+use common_meta::key::catalog_name::{CatalogNameKey, CatalogNameValue};
+use common_meta::key::flow::flow_state::FlowStateValue;
+use common_meta::key::flow::{
+ flow_info_key_prefix, flow_name_key_prefix, flow_route_key_prefix, flow_state_full_key,
+ flownode_flow_key_prefix, table_flow_key_prefix,
+};
+use common_meta::key::node_address::{NodeAddressKey, NodeAddressValue};
+use common_meta::key::schema_name::{SchemaNameKey, SchemaNameValue};
+use common_meta::key::table_repart::{TableRepartKey, TableRepartValue};
+use common_meta::key::topic_name::{TopicNameKey, TopicNameValue};
+use common_meta::key::topic_region::{TopicRegionKey, TopicRegionValue};
+use common_meta::key::view_info::{ViewInfoKey, ViewInfoValue};
+use common_meta::key::{
+ CATALOG_NAME_KEY_PREFIX, DATANODE_TABLE_KEY_PREFIX, KAFKA_TOPIC_KEY_PREFIX, MetadataKey,
+ MetadataValue, NODE_ADDRESS_PREFIX, SCHEMA_NAME_KEY_PREFIX, TABLE_INFO_KEY_PREFIX,
+ TABLE_NAME_KEY_PREFIX, TABLE_REPART_PREFIX, TABLE_ROUTE_PREFIX, TOPIC_REGION_PREFIX,
+ VIEW_INFO_KEY_PREFIX,
+};
+use common_meta::kv_backend::KvBackendRef;
+use common_meta::rpc::store::PutRequest;
+
+use crate::Tool;
+use crate::common::StoreConfig;
+use crate::error::InvalidArgumentsSnafu;
+use crate::metadata::control::put::read_value;
+
+/// Put a key-value pair into the metadata store.
+#[derive(Debug, Default, Parser)]
+pub struct PutKeyCommand {
+ /// The key to put into the metadata store.
+ key: String,
+
+ /// Read the value to put into the metadata store from standard input.
+ #[clap(long, required = true)]
+ value_stdin: bool,
+
+ /// Skip metadata validation before writing.
+ #[clap(long)]
+ no_validate: bool,
+
+ #[clap(flatten)]
+ store: StoreConfig,
+}
+
+impl PutKeyCommand {
+ pub async fn build(&self) -> Result, BoxedError> {
+ let kv_backend = self.store.build().await?;
+ self.build_tool(tokio::io::stdin(), kv_backend).await
+ }
+
+ async fn build_tool(
+ &self,
+ reader: R,
+ kv_backend: KvBackendRef,
+ ) -> Result, BoxedError>
+ where
+ R: tokio::io::AsyncRead + Unpin,
+ {
+ Ok(Box::new(PutKeyTool {
+ kv_backend,
+ key: self.key.clone(),
+ value: read_value(reader).await?,
+ no_validate: self.no_validate,
+ }))
+ }
+}
+
+struct PutKeyTool {
+ kv_backend: KvBackendRef,
+ key: String,
+ value: Vec,
+ no_validate: bool,
+}
+
+#[async_trait]
+impl Tool for PutKeyTool {
+ async fn do_work(&self) -> Result<(), BoxedError> {
+ if !self.no_validate {
+ validate_metadata_value(&self.key, &self.value)?;
+ }
+
+ let request = PutRequest::new()
+ .with_key(self.key.as_bytes())
+ .with_value(self.value.clone());
+ self.kv_backend
+ .put(request)
+ .await
+ .map_err(BoxedError::new)?;
+
+ println!("Key({}) updated", self.key);
+ Ok(())
+ }
+}
+
+fn validate_metadata_value(key: &str, value: &[u8]) -> Result<(), BoxedError> {
+ if let Some(reason) = unsupported_direct_put_reason(key) {
+ return Err(BoxedError::new(
+ InvalidArgumentsSnafu {
+ msg: format!("{reason}, use --no-validate to bypass"),
+ }
+ .build(),
+ ));
+ }
+
+ if key == flow_state_full_key() {
+ validate_value(key, value, FlowStateValue::try_from_raw_value)?;
+ return Ok(());
+ } else if matches_key_prefix(key, VIEW_INFO_KEY_PREFIX) {
+ validate_key(ViewInfoKey::from_bytes(key.as_bytes()), key)?;
+ validate_value(key, value, ViewInfoValue::try_from_raw_value)?;
+ return Ok(());
+ } else if matches_key_prefix(key, CATALOG_NAME_KEY_PREFIX) {
+ validate_key(CatalogNameKey::from_bytes(key.as_bytes()), key)?;
+ validate_value(key, value, CatalogNameValue::try_from_raw_value)?;
+ return Ok(());
+ } else if matches_key_prefix(key, SCHEMA_NAME_KEY_PREFIX) {
+ validate_key(SchemaNameKey::from_bytes(key.as_bytes()), key)?;
+ validate_value(key, value, SchemaNameValue::try_from_raw_value)?;
+ return Ok(());
+ } else if matches_key_prefix(key, TABLE_REPART_PREFIX) {
+ validate_key(TableRepartKey::from_bytes(key.as_bytes()), key)?;
+ validate_value(key, value, TableRepartValue::try_from_raw_value)?;
+ return Ok(());
+ } else if matches_key_prefix(key, NODE_ADDRESS_PREFIX) {
+ validate_key(NodeAddressKey::from_bytes(key.as_bytes()), key)?;
+ validate_value(key, value, NodeAddressValue::try_from_raw_value)?;
+ return Ok(());
+ } else if matches_key_prefix(key, KAFKA_TOPIC_KEY_PREFIX) {
+ validate_key(TopicNameKey::from_bytes(key.as_bytes()), key)?;
+ validate_value(key, value, TopicNameValue::try_from_raw_value)?;
+ return Ok(());
+ } else if matches_key_prefix(key, TOPIC_REGION_PREFIX) {
+ validate_key(TopicRegionKey::from_bytes(key.as_bytes()), key)?;
+ validate_value(key, value, TopicRegionValue::try_from_raw_value)?;
+ return Ok(());
+ }
+
+ Err(BoxedError::new(
+ InvalidArgumentsSnafu {
+ msg: format!(
+ "Unsupported metadata key for validation: {key}, use --no-validate to bypass"
+ ),
+ }
+ .build(),
+ ))
+}
+
+/// Returns the rejection reason for keys that should not be updated by `put key`.
+///
+/// These keys may be decodable, but they are not safe to update via raw KV writes.
+/// `__table_route/*` is the canonical example.
+fn unsupported_direct_put_reason(key: &str) -> Option {
+ let flow_info_prefix = flow_info_key_prefix();
+ let flow_name_prefix = flow_name_key_prefix();
+ let flow_route_prefix = flow_route_key_prefix();
+ let table_flow_prefix = table_flow_key_prefix();
+ let flownode_flow_prefix = flownode_flow_key_prefix();
+
+ let (prefix, target) = [
+ (TABLE_ROUTE_PREFIX, "table route metadata"),
+ (TABLE_INFO_KEY_PREFIX, "table info metadata"),
+ (TABLE_NAME_KEY_PREFIX, "table name metadata"),
+ (DATANODE_TABLE_KEY_PREFIX, "datanode table metadata"),
+ (&flow_info_prefix, "flow info metadata"),
+ (&flow_name_prefix, "flow name metadata"),
+ (&flow_route_prefix, "flow route metadata"),
+ (&table_flow_prefix, "flow source table metadata"),
+ (&flownode_flow_prefix, "flownode flow metadata"),
+ ]
+ .into_iter()
+ .find(|(prefix, _)| matches_key_prefix(key, prefix))?;
+
+ Some(format!(
+ "Direct put is not supported for {target} ({prefix}*); use a dedicated metadata update interface instead"
+ ))
+}
+
+fn matches_key_prefix(key: &str, prefix: &str) -> bool {
+ key == prefix
+ || key
+ .strip_prefix(prefix)
+ .is_some_and(|rest| rest.starts_with('/'))
+}
+
+fn validate_value(key: &str, value: &[u8], parser: F) -> Result<(), BoxedError>
+where
+ F: FnOnce(&[u8]) -> common_meta::error::Result,
+{
+ parser(value).map_err(|e| {
+ BoxedError::new(
+ InvalidArgumentsSnafu {
+ msg: format!("Invalid metadata value for key: {key}: {e}"),
+ }
+ .build(),
+ )
+ })?;
+ Ok(())
+}
+
+fn validate_key(result: common_meta::error::Result, key: &str) -> Result<(), BoxedError> {
+ result.map_err(|e| {
+ BoxedError::new(
+ InvalidArgumentsSnafu {
+ msg: format!("Invalid metadata key: {key}: {e}"),
+ }
+ .build(),
+ )
+ })?;
+ Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+ use std::collections::BTreeMap;
+ use std::sync::Arc;
+
+ use clap::Parser;
+ use common_error::ext::{BoxedError, ErrorExt};
+ use common_meta::key::flow::flow_state::FlowStateValue;
+ use common_meta::key::flow::flow_state_full_key;
+ use common_meta::key::schema_name::SchemaNameValue;
+ use common_meta::key::topic_name::TopicNameValue;
+ use common_meta::key::{KAFKA_TOPIC_KEY_PREFIX, MetadataValue, SCHEMA_NAME_KEY_PREFIX};
+ use common_meta::kv_backend::KvBackendRef;
+ use common_meta::kv_backend::memory::MemoryKvBackend;
+ use tokio::io::BufReader;
+
+ use super::{
+ PutKeyCommand, PutKeyTool, TABLE_ROUTE_PREFIX, matches_key_prefix,
+ unsupported_direct_put_reason, validate_metadata_value,
+ };
+ use crate::Tool;
+
+ impl PutKeyCommand {
+ async fn build_for_test(
+ &self,
+ reader: R,
+ kv_backend: KvBackendRef,
+ ) -> Result, BoxedError>
+ where
+ R: tokio::io::AsyncRead + Unpin,
+ {
+ self.build_tool(reader, kv_backend).await
+ }
+ }
+
+ #[test]
+ fn test_validate_supported_key_success() {
+ let value = SchemaNameValue::default().try_as_raw_value().unwrap();
+
+ validate_metadata_value(&format!("{SCHEMA_NAME_KEY_PREFIX}/greptime/public"), &value)
+ .unwrap();
+ }
+
+ #[test]
+ fn test_validate_supported_key_invalid_value() {
+ let err = validate_metadata_value(
+ &format!("{KAFKA_TOPIC_KEY_PREFIX}/test-topic"),
+ b"not-a-valid-json-value",
+ )
+ .unwrap_err();
+
+ assert!(err.output_msg().contains("Invalid metadata value for key"));
+ }
+
+ #[test]
+ fn test_validate_complex_key_fails() {
+ let value = serde_json::to_vec(&BTreeMap::::new()).unwrap();
+ let err =
+ validate_metadata_value(&format!("{TABLE_ROUTE_PREFIX}/1024"), &value).unwrap_err();
+
+ assert!(
+ err.output_msg()
+ .contains("Direct put is not supported for table route metadata")
+ );
+ }
+
+ #[test]
+ fn test_validate_unknown_key_fails() {
+ let err = validate_metadata_value("__unknown/foo", b"{}").unwrap_err();
+
+ assert!(
+ err.output_msg()
+ .contains("Unsupported metadata key for validation")
+ );
+ }
+
+ #[test]
+ fn test_validate_invalid_supported_key_fails() {
+ let value = SchemaNameValue::default().try_as_raw_value().unwrap();
+ let err = validate_metadata_value("__schema_name/greptime", &value).unwrap_err();
+
+ assert!(
+ err.output_msg()
+ .contains("Invalid metadata key: __schema_name/greptime")
+ );
+ }
+
+ #[test]
+ fn test_unsupported_direct_put_reason_covers_complex_keys() {
+ let cases = [
+ "__table_route/1024",
+ "__table_info/1024",
+ "__table_name/greptime/public/demo",
+ "__dn_table/1/1024",
+ "__flow/route/1/1",
+ ];
+
+ for key in cases {
+ assert!(unsupported_direct_put_reason(key).is_some(), "key: {key}");
+ }
+ }
+
+ #[test]
+ fn test_matches_key_prefix() {
+ assert!(matches_key_prefix("__table_route", "__table_route"));
+ assert!(matches_key_prefix("__table_route/1024", "__table_route"));
+ assert!(!matches_key_prefix(
+ "__table_route_extra/1024",
+ "__table_route"
+ ));
+ assert!(!matches_key_prefix("__table_routex", "__table_route"));
+ assert!(!matches_key_prefix(
+ "__topic_name/kafka_backup/foo",
+ "__topic_name/kafka"
+ ));
+ }
+
+ #[test]
+ fn test_validate_exact_flow_state_key() {
+ let value = FlowStateValue::new(BTreeMap::new(), BTreeMap::new())
+ .try_as_raw_value()
+ .unwrap();
+
+ validate_metadata_value(&flow_state_full_key(), &value).unwrap();
+ }
+
+ #[tokio::test]
+ async fn test_put_key_tool_writes_supported_key() {
+ let kv_backend = Arc::new(MemoryKvBackend::new()) as KvBackendRef;
+ let value = TopicNameValue::new(42).try_as_raw_value().unwrap();
+ let key = format!("{KAFKA_TOPIC_KEY_PREFIX}/test-topic");
+ let tool = PutKeyTool {
+ kv_backend: kv_backend.clone(),
+ key: key.clone(),
+ value: value.clone(),
+ no_validate: false,
+ };
+
+ tool.do_work().await.unwrap();
+
+ let stored = kv_backend.get(key.as_bytes()).await.unwrap().unwrap();
+ assert_eq!(stored.value, value);
+ }
+
+ #[tokio::test]
+ async fn test_put_key_tool_bypasses_validation() {
+ let kv_backend = Arc::new(MemoryKvBackend::new()) as KvBackendRef;
+ let key = format!("{TABLE_ROUTE_PREFIX}/1024");
+ let value = b"not-json".to_vec();
+ let tool = PutKeyTool {
+ kv_backend: kv_backend.clone(),
+ key: key.clone(),
+ value: value.clone(),
+ no_validate: true,
+ };
+
+ tool.do_work().await.unwrap();
+
+ let stored = kv_backend.get(key.as_bytes()).await.unwrap().unwrap();
+ assert_eq!(stored.value, value);
+ }
+
+ #[test]
+ fn test_put_key_command_requires_value_stdin() {
+ let err = PutKeyCommand::try_parse_from([
+ "key",
+ "__topic_name/kafka/test-cli-topic",
+ "--backend",
+ "memory-store",
+ "--store-addrs",
+ "memory://",
+ ])
+ .unwrap_err();
+
+ assert_eq!(err.kind(), clap::error::ErrorKind::MissingRequiredArgument);
+ }
+
+ #[tokio::test]
+ async fn test_put_key_command_builds_tool_with_stdin() {
+ let value = TopicNameValue::new(7).try_as_raw_value().unwrap();
+ let command = PutKeyCommand::parse_from([
+ "key",
+ "__topic_name/kafka/test-cli-topic",
+ "--value-stdin",
+ "--backend",
+ "memory-store",
+ "--store-addrs",
+ "memory://",
+ ]);
+
+ let tool = command
+ .build_for_test(
+ BufReader::new(value.as_slice()),
+ Arc::new(MemoryKvBackend::new()) as KvBackendRef,
+ )
+ .await
+ .unwrap();
+ tool.do_work().await.unwrap();
+ }
+
+ #[tokio::test]
+ async fn test_put_key_command_validate_failure() {
+ let tool = PutKeyTool {
+ kv_backend: Arc::new(MemoryKvBackend::new()) as KvBackendRef,
+ key: "__table_route/1024".to_string(),
+ value: b"{}".to_vec(),
+ no_validate: false,
+ };
+ let err = tool.do_work().await.unwrap_err();
+
+ assert!(
+ err.output_msg()
+ .contains("Direct put is not supported for table route metadata")
+ );
+ }
+}
diff --git a/src/cli/src/metadata/control/put/table.rs b/src/cli/src/metadata/control/put/table.rs
new file mode 100644
index 0000000000..29e5c546d5
--- /dev/null
+++ b/src/cli/src/metadata/control/put/table.rs
@@ -0,0 +1,687 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::collections::HashSet;
+
+use async_trait::async_trait;
+use clap::{Parser, Subcommand};
+use common_error::ext::BoxedError;
+use common_meta::key::datanode_table::{DatanodeTableKey, RegionInfo};
+use common_meta::key::table_info::TableInfoValue;
+use common_meta::key::table_route::TableRouteValue;
+use common_meta::key::{
+ DeserializedValueWithBytes, MetadataValue, RegionDistribution, TableMetadataManager,
+};
+use common_meta::kv_backend::KvBackendRef;
+use common_meta::rpc::router::{RegionRoute, region_distribution};
+use snafu::{OptionExt, ensure};
+use store_api::storage::TableId;
+use table::metadata::TableInfo;
+
+use crate::Tool;
+use crate::common::StoreConfig;
+use crate::error::{Error, InvalidArgumentsSnafu, TableNotFoundSnafu, UnexpectedSnafu};
+use crate::metadata::control::put::read_value;
+use crate::metadata::control::selector::TableSelector;
+
+/// Put table metadata into the metadata store.
+#[derive(Subcommand)]
+pub enum PutTableCommand {
+ Info(PutTableInfoCommand),
+ Route(PutTableRouteCommand),
+}
+
+impl PutTableCommand {
+ pub async fn build(&self) -> Result, BoxedError> {
+ match self {
+ PutTableCommand::Info(cmd) => cmd.build().await,
+ PutTableCommand::Route(cmd) => cmd.build().await,
+ }
+ }
+}
+
+/// Put table info into the metadata store.
+#[derive(Debug, Parser)]
+pub struct PutTableInfoCommand {
+ #[clap(flatten)]
+ selector: TableSelector,
+
+ /// Read the JSON-encoded [`TableInfoValue`] from standard input.
+ #[clap(long, required = true)]
+ value_stdin: bool,
+
+ #[clap(flatten)]
+ store: StoreConfig,
+}
+
+impl PutTableInfoCommand {
+ pub async fn build(&self) -> Result, BoxedError> {
+ let kv_backend = self.store.build().await?;
+ self.build_tool(tokio::io::stdin(), kv_backend).await
+ }
+
+ async fn build_tool(
+ &self,
+ reader: R,
+ kv_backend: KvBackendRef,
+ ) -> Result, BoxedError>
+ where
+ R: tokio::io::AsyncRead + Unpin,
+ {
+ self.selector.validate()?;
+ Ok(Box::new(PutTableInfoTool {
+ kv_backend,
+ selector: self.selector.clone(),
+ value: read_value(reader).await?,
+ }))
+ }
+}
+
+struct PutTableInfoTool {
+ kv_backend: KvBackendRef,
+ selector: TableSelector,
+ value: Vec,
+}
+
+#[async_trait]
+impl Tool for PutTableInfoTool {
+ async fn do_work(&self) -> Result<(), BoxedError> {
+ let table_metadata_manager = TableMetadataManager::new(self.kv_backend.clone());
+ let Some(table_id) = self
+ .selector
+ .resolve_table_id(table_metadata_manager.table_name_manager())
+ .await?
+ else {
+ return Err(BoxedError::new(
+ UnexpectedSnafu {
+ msg: format!("Table({}) not found", self.selector.formatted_table_name()),
+ }
+ .build(),
+ ));
+ };
+
+ let (current_table_info, current_table_route) =
+ load_table_metadata(&table_metadata_manager, table_id).await?;
+ let new_table_info = TableInfoValue::try_from_raw_value(&self.value)
+ .map_err(|e| {
+ BoxedError::new(
+ InvalidArgumentsSnafu {
+ msg: format!("Invalid table info JSON: {e}"),
+ }
+ .build(),
+ )
+ })?
+ .table_info;
+ validate_table_info(table_id, ¤t_table_info.table_info, &new_table_info)
+ .map_err(BoxedError::new)?;
+
+ let region_distribution =
+ physical_region_distribution(current_table_route.get_inner_ref())?;
+
+ if current_table_info.table_info != new_table_info {
+ table_metadata_manager
+ .update_table_info(¤t_table_info, region_distribution, new_table_info)
+ .await
+ .map_err(BoxedError::new)?;
+ println!("Table({table_id}) info updated");
+ }
+
+ Ok(())
+ }
+}
+
+/// Put table route into the metadata store.
+#[derive(Debug, Parser)]
+pub struct PutTableRouteCommand {
+ #[clap(flatten)]
+ selector: TableSelector,
+
+ /// Read the JSON-encoded [`TableRouteValue`] from standard input.
+ #[clap(long, required = true)]
+ value_stdin: bool,
+
+ #[clap(flatten)]
+ store: StoreConfig,
+}
+
+impl PutTableRouteCommand {
+ pub async fn build(&self) -> Result, BoxedError> {
+ let kv_backend = self.store.build().await?;
+ self.build_tool(tokio::io::stdin(), kv_backend).await
+ }
+
+ async fn build_tool(
+ &self,
+ reader: R,
+ kv_backend: KvBackendRef,
+ ) -> Result, BoxedError>
+ where
+ R: tokio::io::AsyncRead + Unpin,
+ {
+ self.selector.validate()?;
+ Ok(Box::new(PutTableRouteTool {
+ kv_backend,
+ selector: self.selector.clone(),
+ value: read_value(reader).await?,
+ }))
+ }
+}
+
+struct PutTableRouteTool {
+ kv_backend: KvBackendRef,
+ selector: TableSelector,
+ value: Vec,
+}
+
+#[async_trait]
+impl Tool for PutTableRouteTool {
+ async fn do_work(&self) -> Result<(), BoxedError> {
+ let table_metadata_manager = TableMetadataManager::new(self.kv_backend.clone());
+ let Some(table_id) = self
+ .selector
+ .resolve_table_id(table_metadata_manager.table_name_manager())
+ .await?
+ else {
+ return Err(BoxedError::new(
+ UnexpectedSnafu {
+ msg: format!("Table({}) not found", self.selector.formatted_table_name()),
+ }
+ .build(),
+ ));
+ };
+
+ let (current_table_info, current_table_route) =
+ load_table_metadata(&table_metadata_manager, table_id).await?;
+ let current_region_routes = current_table_route
+ .region_routes()
+ .map_err(BoxedError::new)?;
+ let new_table_route = TableRouteValue::try_from_raw_value(&self.value).map_err(|e| {
+ BoxedError::new(
+ InvalidArgumentsSnafu {
+ msg: format!("Invalid table route JSON: {e}"),
+ }
+ .build(),
+ )
+ })?;
+ let new_region_routes = new_table_route.region_routes().map_err(BoxedError::new)?;
+ validate_table_route(table_id, new_region_routes, current_region_routes)
+ .map_err(BoxedError::new)?;
+ let region_info =
+ load_region_info(&table_metadata_manager, table_id, current_region_routes).await?;
+ let new_region_options = current_table_info.table_info.to_region_options();
+ let new_region_wal_options = region_info.region_wal_options.clone();
+
+ if current_table_route.get_inner_ref() != &new_table_route {
+ table_metadata_manager
+ .update_table_route(
+ table_id,
+ region_info,
+ ¤t_table_route,
+ new_region_routes.clone(),
+ &new_region_options,
+ &new_region_wal_options,
+ )
+ .await
+ .map_err(BoxedError::new)?;
+ println!("Table({table_id}) route updated");
+ }
+
+ Ok(())
+ }
+}
+
+fn validate_table_route(
+ table_id: TableId,
+ new_region_routes: &[RegionRoute],
+ current_region_route: &[RegionRoute],
+) -> Result<(), Error> {
+ let current_region_ids = current_region_route
+ .iter()
+ .map(|r| r.region.id)
+ .collect::>();
+ for route in new_region_routes {
+ ensure!(
+ route.region.id.table_id() == table_id,
+ InvalidArgumentsSnafu {
+ msg: format!(
+ "Invalid table route: all region routes must have table id {table_id}, but got {}",
+ route.region.id.table_id()
+ ),
+ }
+ );
+ // Ensure the region in new route exists in current route
+ current_region_ids
+ .contains(&route.region.id)
+ .then_some(())
+ .context(InvalidArgumentsSnafu {
+ msg: format!(
+ "Invalid table route: region {} does not exist in current routes",
+ route.region.id
+ ),
+ })?;
+ }
+
+ Ok(())
+}
+
+fn validate_table_info(
+ table_id: TableId,
+ current_table_info: &TableInfo,
+ new_table_info: &TableInfo,
+) -> Result<(), Error> {
+ ensure!(
+ new_table_info.ident.table_id == table_id,
+ InvalidArgumentsSnafu {
+ msg: format!(
+ "Invalid table info: expected table id {table_id}, got {}",
+ new_table_info.ident.table_id
+ ),
+ }
+ );
+
+ ensure!(
+ current_table_info.catalog_name == new_table_info.catalog_name,
+ InvalidArgumentsSnafu {
+ msg: format!(
+ "Invalid table info: catalog name is immutable, expected {}, got {}",
+ current_table_info.catalog_name, new_table_info.catalog_name
+ ),
+ }
+ );
+
+ ensure!(
+ current_table_info.schema_name == new_table_info.schema_name,
+ InvalidArgumentsSnafu {
+ msg: format!(
+ "Invalid table info: schema name is immutable, expected {}, got {}",
+ current_table_info.schema_name, new_table_info.schema_name
+ ),
+ }
+ );
+
+ ensure!(
+ current_table_info.name == new_table_info.name,
+ InvalidArgumentsSnafu {
+ msg: format!(
+ "Invalid table info: table name is immutable, expected {}, got {}",
+ current_table_info.name, new_table_info.name
+ ),
+ }
+ );
+
+ Ok(())
+}
+
+async fn load_region_info(
+ table_metadata_manager: &TableMetadataManager,
+ table_id: TableId,
+ region_routes: &[RegionRoute],
+) -> Result {
+ let datanode_id = region_distribution(region_routes)
+ .into_keys()
+ .next()
+ .ok_or_else(|| {
+ BoxedError::new(
+ UnexpectedSnafu {
+ msg: format!(
+ "Missing datanode assignment for physical table route: {table_id}"
+ ),
+ }
+ .build(),
+ )
+ })?;
+
+ table_metadata_manager
+ .datanode_table_manager()
+ .get(&DatanodeTableKey::new(datanode_id, table_id))
+ .await
+ .map_err(BoxedError::new)?
+ .map(|value| value.region_info)
+ .ok_or_else(|| {
+ BoxedError::new(
+ UnexpectedSnafu {
+ msg: format!(
+ "Missing datanode table metadata for physical table route: {table_id}"
+ ),
+ }
+ .build(),
+ )
+ })
+}
+
+async fn load_table_metadata(
+ table_metadata_manager: &TableMetadataManager,
+ table_id: TableId,
+) -> Result<
+ (
+ DeserializedValueWithBytes,
+ DeserializedValueWithBytes,
+ ),
+ BoxedError,
+> {
+ let (table_info, table_route) = table_metadata_manager
+ .get_full_table_info(table_id)
+ .await
+ .map_err(BoxedError::new)?;
+ let table_info =
+ table_info.ok_or_else(|| BoxedError::new(TableNotFoundSnafu { table_id }.build()))?;
+ let table_route =
+ table_route.ok_or_else(|| BoxedError::new(TableNotFoundSnafu { table_id }.build()))?;
+ Ok((table_info, table_route))
+}
+
+fn physical_region_distribution(
+ table_route: &TableRouteValue,
+) -> Result, BoxedError> {
+ if !table_route.is_physical() {
+ return Ok(None);
+ }
+
+ table_route
+ .region_routes()
+ .map(|routes| Some(region_distribution(routes)))
+ .map_err(BoxedError::new)
+}
+
+#[cfg(test)]
+mod tests {
+ use std::collections::HashMap;
+ use std::sync::Arc;
+
+ use clap::Parser;
+ use common_error::ext::{BoxedError, ErrorExt};
+ use common_meta::key::TableMetadataManager;
+ use common_meta::key::datanode_table::{DatanodeTableKey, DatanodeTableManager};
+ use common_meta::key::table_info::TableInfoValue;
+ use common_meta::key::table_route::TableRouteValue;
+ use common_meta::kv_backend::KvBackendRef;
+ use common_meta::kv_backend::memory::MemoryKvBackend;
+ use common_meta::peer::Peer;
+ use common_meta::rpc::router::RegionRoute;
+ use store_api::storage::RegionId;
+ use tokio::io::BufReader;
+
+ use super::{
+ PutTableInfoCommand, PutTableInfoTool, PutTableRouteCommand, PutTableRouteTool,
+ validate_table_route,
+ };
+ use crate::Tool;
+ use crate::metadata::control::selector::TableSelector;
+ use crate::metadata::control::test_utils::prepare_physical_table_metadata;
+
+ impl PutTableInfoCommand {
+ async fn build_for_test(
+ &self,
+ reader: R,
+ kv_backend: KvBackendRef,
+ ) -> Result, BoxedError>
+ where
+ R: tokio::io::AsyncRead + Unpin,
+ {
+ self.build_tool(reader, kv_backend).await
+ }
+ }
+
+ impl PutTableRouteCommand {
+ async fn build_for_test(
+ &self,
+ reader: R,
+ kv_backend: KvBackendRef,
+ ) -> Result, BoxedError>
+ where
+ R: tokio::io::AsyncRead + Unpin,
+ {
+ self.build_tool(reader, kv_backend).await
+ }
+ }
+
+ #[tokio::test]
+ async fn test_put_table_selector_validation() {
+ let command = PutTableInfoCommand::parse_from([
+ "info",
+ "--value-stdin",
+ "--backend",
+ "memory-store",
+ "--store-addrs",
+ "memory://",
+ ]);
+
+ let err = match command.build().await {
+ Ok(_) => panic!("expected validation failure"),
+ Err(err) => err,
+ };
+ assert!(
+ err.output_msg()
+ .contains("You must specify either --table-id or --table-name.")
+ );
+ }
+
+ #[tokio::test]
+ async fn test_put_table_command_builds_tool_with_table_name() {
+ let command = PutTableInfoCommand::parse_from([
+ "info",
+ "--table-name",
+ "my_table",
+ "--value-stdin",
+ "--backend",
+ "memory-store",
+ "--store-addrs",
+ "memory://",
+ ]);
+
+ let _tool = command
+ .build_for_test(
+ BufReader::new(&b"{}"[..]),
+ Arc::new(MemoryKvBackend::new()) as KvBackendRef,
+ )
+ .await
+ .unwrap();
+ }
+
+ #[tokio::test]
+ async fn test_put_table_info_rejects_table_name_change() {
+ let kv_backend = Arc::new(MemoryKvBackend::new()) as KvBackendRef;
+ let table_metadata_manager = TableMetadataManager::new(kv_backend.clone());
+ let table_id = 1024;
+ let (table_info, table_route) =
+ prepare_physical_table_metadata("old_table", table_id).await;
+ table_metadata_manager
+ .create_table_metadata(
+ table_info.clone(),
+ TableRouteValue::Physical(table_route),
+ HashMap::new(),
+ )
+ .await
+ .unwrap();
+
+ let mut new_table_info = table_info;
+ new_table_info.name = "new_table".to_string();
+ let tool = PutTableInfoTool {
+ kv_backend: kv_backend.clone(),
+ selector: TableSelector::with_table_id(table_id),
+ value: serde_json::to_vec(&TableInfoValue::new(new_table_info)).unwrap(),
+ };
+
+ let err = tool.do_work().await.unwrap_err();
+ assert!(
+ err.output_msg()
+ .contains("Invalid table info: table name is immutable")
+ );
+ }
+
+ #[tokio::test]
+ async fn test_put_table_info_rejects_schema_change() {
+ let kv_backend = Arc::new(MemoryKvBackend::new()) as KvBackendRef;
+ let table_metadata_manager = TableMetadataManager::new(kv_backend.clone());
+ let table_id = 1024;
+ let (table_info, table_route) =
+ prepare_physical_table_metadata("old_table", table_id).await;
+ table_metadata_manager
+ .create_table_metadata(
+ table_info.clone(),
+ TableRouteValue::Physical(table_route),
+ HashMap::new(),
+ )
+ .await
+ .unwrap();
+
+ let mut new_table_info = table_info;
+ new_table_info.schema_name = "another_schema".to_string();
+ let tool = PutTableInfoTool {
+ kv_backend,
+ selector: TableSelector::with_table_id(table_id),
+ value: serde_json::to_vec(&TableInfoValue::new(new_table_info)).unwrap(),
+ };
+
+ let err = tool.do_work().await.unwrap_err();
+ assert!(
+ err.output_msg()
+ .contains("Invalid table info: schema name is immutable")
+ );
+ }
+
+ #[tokio::test]
+ async fn test_put_table_route_updates_route_and_datanode_table() {
+ let kv_backend = Arc::new(MemoryKvBackend::new()) as KvBackendRef;
+ let table_metadata_manager = TableMetadataManager::new(kv_backend.clone());
+ let table_id = 1024;
+ let (table_info, table_route) = prepare_physical_table_metadata("my_table", table_id).await;
+ table_metadata_manager
+ .create_table_metadata(
+ table_info,
+ TableRouteValue::Physical(table_route.clone()),
+ HashMap::new(),
+ )
+ .await
+ .unwrap();
+
+ let mut region_routes = table_route.region_routes.clone();
+ region_routes[0].leader_peer = Some(Peer::empty(2));
+ let new_table_route = TableRouteValue::physical(region_routes);
+ let tool = PutTableRouteTool {
+ kv_backend: kv_backend.clone(),
+ selector: TableSelector::with_table_id(table_id),
+ value: serde_json::to_vec(&new_table_route).unwrap(),
+ };
+
+ tool.do_work().await.unwrap();
+
+ let (_, current_route) = table_metadata_manager
+ .get_full_table_info(table_id)
+ .await
+ .unwrap();
+ let current_route = current_route.unwrap().into_inner();
+ assert_eq!(
+ current_route.region_routes().unwrap(),
+ new_table_route.region_routes().unwrap()
+ );
+
+ let datanode_table_manager = DatanodeTableManager::new(kv_backend);
+ let updated = datanode_table_manager
+ .get(&DatanodeTableKey::new(2, table_id))
+ .await
+ .unwrap();
+ assert!(updated.is_some());
+ }
+
+ #[tokio::test]
+ async fn test_put_table_route_rejects_logical_route() {
+ let kv_backend = Arc::new(MemoryKvBackend::new()) as KvBackendRef;
+ let table_metadata_manager = TableMetadataManager::new(kv_backend.clone());
+ let table_id = 1024;
+ let (table_info, table_route) = prepare_physical_table_metadata("my_table", table_id).await;
+ table_metadata_manager
+ .create_table_metadata(
+ table_info,
+ TableRouteValue::Physical(table_route),
+ HashMap::new(),
+ )
+ .await
+ .unwrap();
+
+ let tool = PutTableRouteTool {
+ kv_backend,
+ selector: TableSelector::with_table_id(table_id),
+ value: serde_json::to_vec(&TableRouteValue::logical(table_id + 1)).unwrap(),
+ };
+
+ let err = tool.do_work().await.unwrap_err();
+ assert!(err.output_msg().contains("non-physical TableRouteValue."));
+ }
+
+ #[test]
+ fn test_validate_table_route_rejects_new_region_not_in_current_route() {
+ let table_id = 1024;
+ let current_region_routes = vec![
+ RegionRoute {
+ region: common_meta::rpc::router::Region {
+ id: RegionId::new(table_id, 1),
+ ..Default::default()
+ },
+ ..Default::default()
+ },
+ RegionRoute {
+ region: common_meta::rpc::router::Region {
+ id: RegionId::new(table_id, 2),
+ ..Default::default()
+ },
+ ..Default::default()
+ },
+ ];
+ let new_region_routes = vec![
+ RegionRoute {
+ region: common_meta::rpc::router::Region {
+ id: RegionId::new(table_id, 1),
+ ..Default::default()
+ },
+ ..Default::default()
+ },
+ RegionRoute {
+ region: common_meta::rpc::router::Region {
+ id: RegionId::new(table_id, 3),
+ ..Default::default()
+ },
+ ..Default::default()
+ },
+ ];
+
+ let err =
+ validate_table_route(table_id, ¤t_region_routes, &new_region_routes).unwrap_err();
+
+ assert!(err.to_string().contains("does not exist in current routes"));
+ }
+
+ #[tokio::test]
+ async fn test_put_table_command_builds_tool() {
+ let value = serde_json::to_vec(&TableRouteValue::logical(1025)).unwrap();
+ let command = PutTableRouteCommand::parse_from([
+ "route",
+ "--table-id",
+ "1024",
+ "--value-stdin",
+ "--backend",
+ "memory-store",
+ "--store-addrs",
+ "memory://",
+ ]);
+
+ let _tool = command
+ .build_for_test(
+ BufReader::new(value.as_slice()),
+ Arc::new(MemoryKvBackend::new()) as KvBackendRef,
+ )
+ .await
+ .unwrap();
+ }
+}
diff --git a/src/cli/src/metadata/control/selector.rs b/src/cli/src/metadata/control/selector.rs
new file mode 100644
index 0000000000..6d3c47fa9c
--- /dev/null
+++ b/src/cli/src/metadata/control/selector.rs
@@ -0,0 +1,100 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use clap::Args;
+use client::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME};
+use common_catalog::format_full_table_name;
+use common_error::ext::BoxedError;
+use common_meta::key::table_name::TableNameManager;
+use store_api::storage::TableId;
+
+use crate::error::InvalidArgumentsSnafu;
+use crate::metadata::control::utils::get_table_id_by_name;
+
+/// Selects a table by id or by fully qualified name.
+#[derive(Debug, Clone, Default, Args)]
+pub(crate) struct TableSelector {
+ /// The table id to select from the metadata store.
+ #[clap(long)]
+ table_id: Option,
+
+ /// The table name to select from the metadata store.
+ #[clap(long)]
+ table_name: Option,
+
+ /// The schema name of the table.
+ #[clap(long, default_value = DEFAULT_SCHEMA_NAME)]
+ schema_name: String,
+
+ /// The catalog name of the table.
+ #[clap(long, default_value = DEFAULT_CATALOG_NAME)]
+ catalog_name: String,
+}
+
+impl TableSelector {
+ pub(crate) fn validate(&self) -> Result<(), BoxedError> {
+ if matches!(
+ (&self.table_id, &self.table_name),
+ (Some(_), Some(_)) | (None, None)
+ ) {
+ return Err(BoxedError::new(
+ InvalidArgumentsSnafu {
+ msg: "You must specify either --table-id or --table-name.",
+ }
+ .build(),
+ ));
+ }
+
+ Ok(())
+ }
+
+ pub(crate) async fn resolve_table_id(
+ &self,
+ table_name_manager: &TableNameManager,
+ ) -> Result, BoxedError> {
+ if let Some(table_id) = self.table_id {
+ return Ok(Some(table_id));
+ }
+
+ get_table_id_by_name(
+ table_name_manager,
+ &self.catalog_name,
+ &self.schema_name,
+ self.table_name
+ .as_deref()
+ .expect("validated table selector"),
+ )
+ .await
+ }
+
+ pub(crate) fn formatted_table_name(&self) -> String {
+ format_full_table_name(
+ &self.catalog_name,
+ &self.schema_name,
+ self.table_name.as_deref().unwrap_or_default(),
+ )
+ }
+}
+
+#[cfg(test)]
+impl TableSelector {
+ pub(crate) fn with_table_id(table_id: u32) -> Self {
+ Self {
+ table_id: Some(table_id),
+ table_name: None,
+ schema_name: DEFAULT_SCHEMA_NAME.to_string(),
+ catalog_name: DEFAULT_CATALOG_NAME.to_string(),
+ }
+ }
+}
diff --git a/src/cli/src/metadata/snapshot.rs b/src/cli/src/metadata/snapshot.rs
index 648d3a687d..59b2599ad9 100644
--- a/src/cli/src/metadata/snapshot.rs
+++ b/src/cli/src/metadata/snapshot.rs
@@ -16,7 +16,7 @@ use async_trait::async_trait;
use clap::{Parser, Subcommand};
use common_error::ext::BoxedError;
use common_meta::snapshot::MetadataSnapshotManager;
-use object_store::{ObjectStore, Scheme};
+use object_store::{ObjectStore, services};
use crate::Tool;
use crate::common::{ObjectStoreConfig, StoreConfig, new_fs_object_store};
@@ -276,7 +276,7 @@ fn build_object_store_and_resolve_file_path(
None => new_fs_object_store(fs_root)?,
};
- let file_path = if matches!(object_store.info().scheme(), Scheme::Fs) {
+ let file_path = if object_store.info().scheme() == services::FS_SCHEME {
resolve_relative_path_with_current_dir(file_path).map_err(BoxedError::new)?
} else {
file_path.to_string()
diff --git a/src/client/src/database.rs b/src/client/src/database.rs
index e12c2ec0fc..13ff9fc651 100644
--- a/src/client/src/database.rs
+++ b/src/client/src/database.rs
@@ -14,7 +14,9 @@
use std::pin::Pin;
use std::str::FromStr;
-use std::sync::Arc;
+use std::sync::atomic::{AtomicBool, Ordering};
+use std::sync::{Arc, RwLock};
+use std::task::{Context, Poll};
use api::v1::auth_header::AuthScheme;
use api::v1::ddl_request::Expr as DdlExpr;
@@ -25,6 +27,7 @@ use api::v1::{
AlterTableExpr, AuthHeader, Basic, CreateTableExpr, DdlRequest, GreptimeRequest,
InsertRequests, QueryRequest, RequestHeader, RowInsertRequests,
};
+use arc_swap::ArcSwapOption;
use arrow_flight::{FlightData, Ticket};
use async_stream::stream;
use base64::Engine;
@@ -33,17 +36,18 @@ use common_catalog::build_db_string;
use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME};
use common_error::ext::BoxedError;
use common_grpc::flight::do_put::DoPutResponse;
-use common_grpc::flight::{FlightDecoder, FlightMessage};
+use common_grpc::flight::{FLOW_EXTENSIONS_METADATA_KEY, FlightDecoder, FlightMessage};
use common_query::Output;
+use common_recordbatch::adapter::RecordBatchMetrics;
use common_recordbatch::error::ExternalSnafu;
-use common_recordbatch::{RecordBatch, RecordBatchStreamWrapper};
+use common_recordbatch::{OrderOption, RecordBatch, RecordBatchStream, RecordBatchStreamWrapper};
use common_telemetry::tracing::Span;
use common_telemetry::tracing_context::W3cTrace;
use common_telemetry::{error, warn};
use futures::future;
use futures_util::{Stream, StreamExt, TryStreamExt};
use prost::Message;
-use snafu::{OptionExt, ResultExt, ensure};
+use snafu::{OptionExt, ResultExt};
use tonic::metadata::{AsciiMetadataKey, AsciiMetadataValue, MetadataMap, MetadataValue};
use tonic::transport::Channel;
@@ -57,6 +61,313 @@ type FlightDataStream = Pin + Send>>;
type DoPutResponseStream = Pin>>>;
+/// Terminal metrics associated with a query output.
+///
+/// For streaming outputs, metrics are only final after the stream is fully
+/// drained and [`Self::is_ready`] returns `true`.
+#[derive(Debug, Clone, Default)]
+pub struct OutputMetrics {
+ inner: Arc,
+}
+
+#[derive(Debug, Default)]
+struct OutputMetricsInner {
+ metrics: RwLock>,
+ ready: AtomicBool,
+}
+
+impl OutputMetrics {
+ fn new() -> Self {
+ Self::default()
+ }
+
+ /// Replaces the current terminal metrics snapshot.
+ pub fn update(&self, metrics: Option) {
+ *self.inner.metrics.write().unwrap() = metrics;
+ }
+
+ /// Marks the terminal metrics as final for this output.
+ pub fn mark_ready(&self) {
+ let _ = self
+ .inner
+ .ready
+ .compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire);
+ }
+
+ /// Returns whether terminal metrics are final.
+ ///
+ /// Streaming outputs become ready only after the stream reaches EOF.
+ pub fn is_ready(&self) -> bool {
+ self.inner.ready.load(Ordering::Acquire)
+ }
+
+ /// Returns the latest terminal metrics snapshot, if any.
+ pub fn get(&self) -> Option {
+ self.inner.metrics.read().unwrap().clone()
+ }
+
+ /// Returns proved per-region watermarks.
+ ///
+ /// Entries whose watermark is `None` are intentionally omitted because they
+ /// represent participating regions whose terminal sequence bound was not
+ /// provable.
+ pub fn region_watermark_map(&self) -> Option> {
+ Some(
+ self.get()?
+ .region_watermarks
+ .into_iter()
+ .filter_map(|entry| entry.watermark.map(|seq| (entry.region_id, seq)))
+ .collect::>(),
+ )
+ }
+
+ /// Returns all regions that participated in terminal metric collection,
+ /// including entries whose watermark is `None`.
+ pub fn participating_regions(&self) -> Option> {
+ Some(
+ self.get()?
+ .region_watermarks
+ .into_iter()
+ .map(|entry| entry.region_id)
+ .collect::>(),
+ )
+ }
+}
+
+/// Query output together with a handle for its terminal metrics.
+///
+/// The contained [`OutputMetrics`] lets callers read stream terminal metrics
+/// after consuming `output`. For non-stream outputs, metrics are ready
+/// immediately.
+#[derive(Debug)]
+pub struct OutputWithMetrics {
+ pub output: Output,
+ pub metrics: OutputMetrics,
+}
+
+impl OutputWithMetrics {
+ /// Wraps an output with a terminal metrics handle.
+ ///
+ /// Stream outputs update the handle as the stream is consumed. Non-stream
+ /// outputs are marked ready immediately.
+ pub fn from_output(output: Output) -> Self {
+ let terminal_metrics = OutputMetrics::new();
+ let output = attach_terminal_metrics(output, &terminal_metrics);
+ Self {
+ output,
+ metrics: terminal_metrics,
+ }
+ }
+
+ /// Returns proved per-region watermarks from the terminal metrics.
+ pub fn region_watermark_map(&self) -> Option> {
+ self.metrics.region_watermark_map()
+ }
+
+ /// Returns all regions participating in terminal metric collection.
+ pub fn participating_regions(&self) -> Option> {
+ self.metrics.participating_regions()
+ }
+
+ /// Drops the terminal metrics handle and returns the original output.
+ pub fn into_output(self) -> Output {
+ self.output
+ }
+}
+
+fn parse_terminal_metrics(metrics_json: &str) -> Result {
+ serde_json::from_str(metrics_json).map_err(|e| {
+ IllegalFlightMessagesSnafu {
+ reason: format!("Invalid terminal metrics message: {e}"),
+ }
+ .build()
+ })
+}
+
+struct StreamWithMetrics {
+ stream: common_recordbatch::SendableRecordBatchStream,
+ metrics: OutputMetrics,
+}
+
+impl StreamWithMetrics {
+ fn new(stream: common_recordbatch::SendableRecordBatchStream, metrics: OutputMetrics) -> Self {
+ Self { stream, metrics }
+ }
+
+ fn sync_terminal_metrics(&self) {
+ self.metrics.update(self.stream.metrics());
+ }
+}
+
+impl RecordBatchStream for StreamWithMetrics {
+ fn name(&self) -> &str {
+ self.stream.name()
+ }
+
+ fn schema(&self) -> datatypes::schema::SchemaRef {
+ self.stream.schema()
+ }
+
+ fn output_ordering(&self) -> Option<&[OrderOption]> {
+ self.stream.output_ordering()
+ }
+
+ fn metrics(&self) -> Option {
+ self.sync_terminal_metrics();
+ self.metrics.get()
+ }
+}
+
+impl Stream for StreamWithMetrics {
+ type Item = common_recordbatch::error::Result;
+
+ fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> {
+ let polled = Pin::new(&mut self.stream).poll_next(cx);
+ if let Poll::Ready(None) = &polled {
+ self.sync_terminal_metrics();
+ self.metrics.mark_ready();
+ }
+ polled
+ }
+
+ fn size_hint(&self) -> (usize, Option) {
+ self.stream.size_hint()
+ }
+}
+
+fn attach_terminal_metrics(output: Output, terminal_metrics: &OutputMetrics) -> Output {
+ let Output { data, meta } = output;
+ let data = match data {
+ common_query::OutputData::Stream(stream) => {
+ terminal_metrics.update(stream.metrics());
+ common_query::OutputData::Stream(Box::pin(StreamWithMetrics::new(
+ stream,
+ terminal_metrics.clone(),
+ )))
+ }
+ other => {
+ terminal_metrics.mark_ready();
+ other
+ }
+ };
+ Output::new(data, meta)
+}
+
+async fn output_from_flight_message_stream
(
+ mut flight_message_stream: S,
+) -> Result
+where
+ S: Stream- > + Send + Unpin + 'static,
+{
+ let Some(first_flight_message) = flight_message_stream.next().await else {
+ return IllegalFlightMessagesSnafu {
+ reason: "Expect the response not to be empty",
+ }
+ .fail();
+ };
+
+ let first_flight_message = first_flight_message?;
+
+ match first_flight_message {
+ FlightMessage::AffectedRows { rows, metrics } => {
+ let terminal_metrics = OutputMetrics::new();
+ if let Some(metrics) = metrics {
+ terminal_metrics.update(Some(parse_terminal_metrics(&metrics)?));
+ }
+ let next_message = flight_message_stream.next().await.transpose()?;
+ match next_message {
+ None => terminal_metrics.mark_ready(),
+ Some(FlightMessage::Metrics(s)) if terminal_metrics.get().is_none() => {
+ terminal_metrics.update(Some(parse_terminal_metrics(&s)?));
+ terminal_metrics.mark_ready();
+ }
+ Some(FlightMessage::Metrics(_)) => {
+ return IllegalFlightMessagesSnafu {
+ reason: "'AffectedRows' Flight metadata already carries Metrics and cannot be followed by another Metrics message",
+ }
+ .fail();
+ }
+ Some(other) => {
+ return IllegalFlightMessagesSnafu {
+ reason: format!(
+ "'AffectedRows' Flight message can only be followed by a Metrics message, got {other:?}"
+ ),
+ }
+ .fail();
+ }
+ }
+ Ok(OutputWithMetrics {
+ output: Output::new_with_affected_rows(rows),
+ metrics: terminal_metrics,
+ })
+ }
+ FlightMessage::RecordBatch(_) | FlightMessage::Metrics(_) => IllegalFlightMessagesSnafu {
+ reason: "The first flight message cannot be a RecordBatch or Metrics message",
+ }
+ .fail(),
+ FlightMessage::Schema(schema) => {
+ let metrics = Arc::new(ArcSwapOption::from(None));
+ let metrics_ref = metrics.clone();
+ let schema = Arc::new(
+ datatypes::schema::Schema::try_from(schema).context(error::ConvertSchemaSnafu)?,
+ );
+ let schema_cloned = schema.clone();
+ let stream = Box::pin(stream!({
+ while let Some(flight_message_item) = flight_message_stream.next().await {
+ let flight_message = match flight_message_item {
+ Ok(message) => message,
+ Err(e) => {
+ yield Err(BoxedError::new(e)).context(ExternalSnafu);
+ break;
+ }
+ };
+
+ match flight_message {
+ FlightMessage::RecordBatch(arrow_batch) => {
+ yield Ok(RecordBatch::from_df_record_batch(
+ schema_cloned.clone(),
+ arrow_batch,
+ ))
+ }
+ FlightMessage::Metrics(s) => {
+ match parse_terminal_metrics(&s) {
+ Ok(m) => {
+ metrics_ref.swap(Some(Arc::new(m)));
+ }
+ Err(e) => {
+ yield Err(BoxedError::new(e)).context(ExternalSnafu);
+ }
+ };
+ }
+ FlightMessage::AffectedRows { .. } | FlightMessage::Schema(_) => {
+ yield IllegalFlightMessagesSnafu {
+ reason: format!(
+ "A Schema message must be succeeded exclusively by a set of RecordBatch messages, flight_message: {:?}",
+ flight_message
+ )
+ }
+ .fail()
+ .map_err(BoxedError::new)
+ .context(ExternalSnafu);
+ break;
+ }
+ }
+ }
+ }));
+ let record_batch_stream = RecordBatchStreamWrapper {
+ schema,
+ stream,
+ output_ordering: None,
+ metrics,
+ span: Span::current(),
+ };
+ Ok(OutputWithMetrics::from_output(Output::new_with_stream(
+ Box::pin(record_batch_stream),
+ )))
+ }
+ }
+}
+
#[derive(Clone, Debug, Default)]
pub struct Database {
// The "catalog" and "schema" to be used in processing the requests at the server side.
@@ -238,6 +549,22 @@ impl Database {
Ok(())
}
+ fn put_flow_extensions(
+ metadata: &mut MetadataMap,
+ flow_extensions: &[(&str, &str)],
+ ) -> Result<()> {
+ if flow_extensions.is_empty() {
+ return Ok(());
+ }
+
+ let value = serde_json::to_string(&flow_extensions.to_vec())
+ .expect("flow extension pairs should serialize");
+ let key = AsciiMetadataKey::from_static(FLOW_EXTENSIONS_METADATA_KEY);
+ let value = AsciiMetadataValue::from_str(&value).context(InvalidTonicMetadataValueSnafu)?;
+ metadata.insert(key, value);
+ Ok(())
+ }
+
/// Make a request to the database.
pub async fn handle(&self, request: Request) -> Result {
let mut client = make_database_client(&self.client)?;
@@ -333,15 +660,58 @@ impl Database {
let request = Request::Query(QueryRequest {
query: Some(Query::Sql(sql.as_ref().to_string())),
});
- self.do_get(request, hints).await
+ self.do_get(request, hints, &[])
+ .await
+ .map(OutputWithMetrics::into_output)
+ }
+
+ /// Executes a SQL query and returns the output with terminal metrics.
+ ///
+ /// For stream outputs, callers must consume the stream before reading final
+ /// terminal metrics from [`OutputWithMetrics::metrics`].
+ pub async fn sql_with_terminal_metrics
(
+ &self,
+ sql: S,
+ hints: &[(&str, &str)],
+ ) -> Result
+ where
+ S: AsRef,
+ {
+ self.query_with_terminal_metrics_and_flow_extensions(
+ QueryRequest {
+ query: Some(Query::Sql(sql.as_ref().to_string())),
+ },
+ hints,
+ &[],
+ )
+ .await
}
/// Executes a logical plan directly without SQL parsing.
pub async fn logical_plan(&self, logical_plan: Vec) -> Result