diff --git a/.github/workflows/dev-build.yml b/.github/workflows/dev-build.yml
index d03fbeff14..c3af006f54 100644
--- a/.github/workflows/dev-build.yml
+++ b/.github/workflows/dev-build.yml
@@ -30,7 +30,7 @@ on:
linux_arm64_runner:
type: choice
description: The runner uses to build linux-arm64 artifacts
- default: ec2-c6g.4xlarge-arm64
+ default: ec2-c6g.8xlarge-arm64
options:
- ec2-c6g.xlarge-arm64 # 4C8G
- ec2-c6g.2xlarge-arm64 # 8C16G
diff --git a/.github/workflows/nightly-build.yml b/.github/workflows/nightly-build.yml
index 14ebb6e715..54af32a94b 100644
--- a/.github/workflows/nightly-build.yml
+++ b/.github/workflows/nightly-build.yml
@@ -27,7 +27,7 @@ on:
linux_arm64_runner:
type: choice
description: The runner uses to build linux-arm64 artifacts
- default: ec2-c6g.4xlarge-arm64
+ default: ec2-c6g.8xlarge-arm64
options:
- ec2-c6g.xlarge-arm64 # 4C8G
- ec2-c6g.2xlarge-arm64 # 8C16G
diff --git a/.github/workflows/nightly-jsonbench.yaml b/.github/workflows/nightly-jsonbench.yaml
new file mode 100644
index 0000000000..a9ce4dd363
--- /dev/null
+++ b/.github/workflows/nightly-jsonbench.yaml
@@ -0,0 +1,223 @@
+name: Nightly JSONBench
+
+on:
+ workflow_run:
+ workflows: [ "GreptimeDB Nightly Build" ]
+ types: [ completed ]
+ workflow_dispatch:
+ inputs:
+ run_id:
+ description: The nightly build workflow run id to download GreptimeDB artifacts from
+ required: true
+ type: string
+
+permissions:
+ actions: read
+ contents: read
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+ cancel-in-progress: true
+
+jobs:
+ resolve-artifact:
+ name: Resolve GreptimeDB nightly artifact
+ if: ${{ github.repository == 'GreptimeTeam/greptimedb' && (github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success') }}
+ runs-on: ubuntu-latest
+ outputs:
+ artifact-name: ${{ steps.find-artifact.outputs.artifact-name }}
+ run-id: ${{ steps.resolve-run-id.outputs.run-id }}
+ steps:
+ - name: Resolve nightly build run id
+ id: resolve-run-id
+ shell: bash
+ env:
+ EVENT_NAME: ${{ github.event_name }}
+ WORKFLOW_RUN_ID: ${{ github.event.workflow_run.id }}
+ INPUT_RUN_ID: ${{ inputs.run_id }}
+ run: |
+ set -euo pipefail
+
+ if [[ "${EVENT_NAME}" == "workflow_dispatch" ]]; then
+ run_id="${INPUT_RUN_ID}"
+ else
+ run_id="${WORKFLOW_RUN_ID}"
+ fi
+
+ if [[ ! "${run_id}" =~ ^[0-9]+$ ]]; then
+ echo "Invalid workflow run id: ${run_id}"
+ exit 1
+ fi
+
+ echo "run-id=${run_id}" >> "${GITHUB_OUTPUT}"
+
+ - name: Find GreptimeDB nightly artifact
+ id: find-artifact
+ shell: bash
+ env:
+ GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+ RUN_ID: ${{ steps.resolve-run-id.outputs.run-id }}
+ run: |
+ set -euo pipefail
+
+ artifact_name=$(gh api "repos/${GITHUB_REPOSITORY}/actions/runs/${RUN_ID}/artifacts" --paginate \
+ --jq '.artifacts[] | select(.name | test("^greptime-linux-arm64-nightly-[0-9]{8}-[0-9a-f]+$")) | .name' \
+ | head -n 1)
+
+ if [[ -z "${artifact_name}" ]]; then
+ echo "Cannot find linux arm64 nightly artifact in workflow run ${RUN_ID}."
+ exit 1
+ fi
+
+ echo "Download GreptimeDB artifact: ${artifact_name}"
+ echo "artifact-name=${artifact_name}" >> "${GITHUB_OUTPUT}"
+
+ allocate-runner:
+ name: Allocate runner
+ if: ${{ github.repository == 'GreptimeTeam/greptimedb' && (github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success') }}
+ needs: [ resolve-artifact ]
+ runs-on: ubuntu-latest
+ outputs:
+ linux-arm64-runner: ${{ steps.start-linux-arm64-runner.outputs.label }}
+
+ # The following EC2 resource id will be used for resource releasing.
+ linux-arm64-ec2-runner-label: ${{ steps.start-linux-arm64-runner.outputs.label }}
+ linux-arm64-ec2-runner-instance-id: ${{ steps.start-linux-arm64-runner.outputs.ec2-instance-id }}
+ steps:
+ - name: Checkout
+ uses: actions/checkout@v4
+ with:
+ fetch-depth: 0
+ persist-credentials: false
+
+ - name: Allocate Linux ARM64 runner
+ uses: ./.github/actions/start-runner
+ id: start-linux-arm64-runner
+ with:
+ runner: ${{ vars.DEFAULT_ARM64_RUNNER }}
+ aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+ aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+ aws-region: ${{ vars.EC2_RUNNER_REGION }}
+ github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
+ image-id: ${{ vars.EC2_RUNNER_LINUX_ARM64_IMAGE_ID }}
+ security-group-id: ${{ vars.EC2_RUNNER_SECURITY_GROUP_ID }}
+ subnet-id: ${{ vars.EC2_RUNNER_SUBNET_ID }}
+
+ jsonbench:
+ name: Run JSONBench
+ if: ${{ github.repository == 'GreptimeTeam/greptimedb' && (github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success') }}
+ needs: [ resolve-artifact, allocate-runner ]
+ runs-on: ${{ needs.allocate-runner.outputs.linux-arm64-runner }}
+ timeout-minutes: 120
+ env:
+ JSONBENCH_OUTPUT_PREFIX: _linux-arm64
+ steps:
+ - name: Download GreptimeDB nightly artifact
+ uses: actions/download-artifact@v4
+ with:
+ name: ${{ needs.resolve-artifact.outputs.artifact-name }}
+ path: greptimedb-artifact
+ github-token: ${{ secrets.GITHUB_TOKEN }}
+ run-id: ${{ needs.resolve-artifact.outputs.run-id }}
+
+ - name: Prepare GreptimeDB binary
+ shell: bash
+ run: |
+ set -euo pipefail
+
+ tar -xzf "greptimedb-artifact/${{ needs.resolve-artifact.outputs.artifact-name }}.tar.gz"
+ cp "${{ needs.resolve-artifact.outputs.artifact-name }}/greptime" ./greptime
+ chmod +x ./greptime
+ rm -rf greptimedb-artifact "${{ needs.resolve-artifact.outputs.artifact-name }}"
+
+ - name: Run JSONBench
+ env:
+ # TODO(LFC): Change to "3" (100m) when JSON2 ingestion performance is optimized.
+ JSONBENCH_DATASET: 2
+ shell: bash
+ run: |
+ set -euo pipefail
+
+ export JSONBENCH_DATA_DIR="/root/data/bluesky"
+ echo "Use JSONBench data directory ${JSONBENCH_DATA_DIR}"
+
+ echo "Cloning JSONBench"
+ git clone --branch greptimedb-new-json --depth 1 https://github.com/GreptimeTeam/JSONBench.git JSONBench
+
+ echo "Downloading JSONBench dataset choice ${JSONBENCH_DATASET} to ${JSONBENCH_DATA_DIR}"
+ mkdir -p "${JSONBENCH_DATA_DIR}"
+ printf "${JSONBENCH_DATASET}\n" | ./JSONBench/download_data.sh
+ downloaded_files=$(find "${JSONBENCH_DATA_DIR}" -type f | wc -l)
+ echo "Downloaded JSONBench dataset files: ${downloaded_files}"
+
+ export GREPTIMEDB_STANDALONE__WAL__DIR=greptimedb_data/wal
+ export GREPTIMEDB_STANDALONE__STORAGE__DATA_HOME=greptimedb_data
+ export GREPTIMEDB_STANDALONE__LOGGING__DIR=greptimedb_data/logs
+ export GREPTIMEDB_STANDALONE__LOGGING__APPEND_STDOUT=false
+ export GREPTIMEDB_STANDALONE__HTTP__BODY_LIMIT=1GB
+ export GREPTIMEDB_STANDALONE__HTTP__TIMEOUT=500s
+
+ echo "Starting GreptimeDB standalone"
+ ./greptime standalone start > greptimedb.log 2>&1 &
+ greptime_pid=$!
+ trap 'kill "${greptime_pid}" 2>/dev/null || true' EXIT
+
+ echo "Waiting for GreptimeDB health check"
+ until curl -s --fail -o /dev/null http://localhost:4000/health; do
+ if ! kill -0 "${greptime_pid}" 2>/dev/null; then
+ cat greptimedb.log
+ exit 1
+ fi
+ sleep 1
+ done
+ echo "GreptimeDB is ready"
+
+ cp ./greptime JSONBench/greptimedb/greptime
+
+ cd JSONBench/greptimedb
+ echo "Running JSONBench main.sh with dataset choice ${JSONBENCH_DATASET} and install=false"
+ ./main.sh ${JSONBENCH_DATASET} "${JSONBENCH_DATA_DIR}" success.log error.log "${JSONBENCH_OUTPUT_PREFIX}" false
+ echo "JSONBench finished"
+
+ - name: Upload JSONBench results
+ if: always()
+ uses: actions/upload-artifact@v4
+ with:
+ name: jsonbench-results
+ path: |
+ ./greptimedb.log
+ ./JSONBench/greptimedb/*.log
+ ./JSONBench/greptimedb/*.total_size
+ ./JSONBench/greptimedb/*.data_size
+ ./JSONBench/greptimedb/*.index_size
+ ./JSONBench/greptimedb/*.count
+ ./JSONBench/greptimedb/*.results_runtime
+ ./JSONBench/greptimedb/*.query_results
+ if-no-files-found: ignore
+ retention-days: 7
+
+ stop-linux-arm64-runner:
+ name: Stop Linux ARM64 runner
+ # It's always run as the last job in the workflow to make sure that the runner is released.
+ if: ${{ always() && needs.allocate-runner.outputs.linux-arm64-ec2-runner-instance-id != '' }}
+ runs-on: ubuntu-latest
+ needs: [
+ allocate-runner,
+ jsonbench,
+ ]
+ steps:
+ - name: Checkout
+ uses: actions/checkout@v4
+ with:
+ fetch-depth: 0
+ persist-credentials: false
+
+ - name: Stop Linux ARM64 runner
+ uses: ./.github/actions/stop-runner
+ with:
+ label: ${{ needs.allocate-runner.outputs.linux-arm64-ec2-runner-label }}
+ ec2-instance-id: ${{ needs.allocate-runner.outputs.linux-arm64-ec2-runner-instance-id }}
+ aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+ aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+ aws-region: ${{ vars.EC2_RUNNER_REGION }}
+ github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
diff --git a/Cargo.lock b/Cargo.lock
index 63ba289947..e3d0dfb69e 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -79,8 +79,9 @@ checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75"
dependencies = [
"cfg-if",
"const-random",
- "getrandom 0.3.3",
+ "getrandom 0.3.4",
"once_cell",
+ "serde",
"version_check",
"zerocopy",
]
@@ -771,7 +772,7 @@ version = "4.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ef1e3e699d84ab1b0911a1010c5c106aa34ae89aeac103be5ce0c3859db1e891"
dependencies = [
- "term",
+ "term 1.0.2",
]
[[package]]
@@ -1427,6 +1428,12 @@ dependencies = [
"syn 2.0.117",
]
+[[package]]
+name = "borrow-or-share"
+version = "0.2.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dc0b364ead1874514c8c2855ab558056ebfeb775653e7ae45ff72f28f8f3166c"
+
[[package]]
name = "borsh"
version = "1.5.7"
@@ -1525,6 +1532,12 @@ dependencies = [
"syn 1.0.109",
]
+[[package]]
+name = "bytecount"
+version = "0.6.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "175812e0be2bccb6abe50bb8d566126198344f707e304f45c648fd8f2cc0365e"
+
[[package]]
name = "bytemuck"
version = "1.23.1"
@@ -1635,7 +1648,7 @@ dependencies = [
"paste",
"prometheus 0.14.0",
"promql-parser",
- "rand 0.9.1",
+ "rand 0.9.4",
"serde",
"serde_json",
"session",
@@ -1973,7 +1986,7 @@ dependencies = [
"partition",
"paste",
"query",
- "rand 0.9.1",
+ "rand 0.9.4",
"reqwest 0.13.2",
"serde",
"serde_json",
@@ -2020,7 +2033,7 @@ dependencies = [
"prometheus 0.14.0",
"prost 0.14.1",
"query",
- "rand 0.9.1",
+ "rand 0.9.4",
"serde_json",
"snafu 0.8.6",
"store-api",
@@ -2031,6 +2044,15 @@ dependencies = [
"tracing",
]
+[[package]]
+name = "clipboard-win"
+version = "5.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bde03770d3df201d4fb868f2c9c59e66a3e4e2bd06692a0fe701e7103c7e84d4"
+dependencies = [
+ "error-code",
+]
+
[[package]]
name = "clocksource"
version = "0.8.1"
@@ -2123,7 +2145,7 @@ dependencies = [
"prometheus 0.14.0",
"prost 0.14.1",
"query",
- "rand 0.9.1",
+ "rand 0.9.4",
"regex",
"reqwest 0.13.2",
"serde",
@@ -2278,6 +2300,7 @@ dependencies = [
"futures",
"lazy_static",
"object-store",
+ "object_store_opendal",
"orc-rust",
"parquet",
"paste",
@@ -2461,7 +2484,7 @@ dependencies = [
"hyper-util",
"lazy_static",
"prost 0.14.1",
- "rand 0.9.1",
+ "rand 0.9.4",
"serde",
"serde_json",
"snafu 0.8.6",
@@ -2580,7 +2603,7 @@ dependencies = [
"prometheus 0.14.0",
"prost 0.14.1",
"prost-types 0.14.1",
- "rand 0.9.1",
+ "rand 0.9.4",
"regex",
"rskafka",
"rustls",
@@ -2648,7 +2671,7 @@ dependencies = [
"futures-util",
"humantime-serde",
"object-store",
- "rand 0.9.1",
+ "rand 0.9.4",
"serde",
"serde_json",
"smallvec",
@@ -2832,7 +2855,7 @@ dependencies = [
"common-query",
"common-recordbatch",
"once_cell",
- "rand 0.9.1",
+ "rand 0.9.4",
"tempfile",
]
@@ -2848,7 +2871,7 @@ dependencies = [
"humantime",
"humantime-serde",
"once_cell",
- "rand 0.9.1",
+ "rand 0.9.4",
"serde",
"serde_json",
"snafu 0.8.6",
@@ -3593,6 +3616,12 @@ dependencies = [
"parking_lot_core 0.9.11",
]
+[[package]]
+name = "data-encoding"
+version = "2.11.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a4ae5f15dda3c708c0ade84bfee31ccab44a3da4f88015ed22f63732abe300c8"
+
[[package]]
name = "datafusion"
version = "53.1.0"
@@ -3637,7 +3666,7 @@ dependencies = [
"object_store",
"parking_lot 0.12.4",
"parquet",
- "rand 0.9.1",
+ "rand 0.9.4",
"regex",
"sqlparser",
"tempfile",
@@ -3754,7 +3783,7 @@ dependencies = [
"liblzma",
"log",
"object_store",
- "rand 0.9.1",
+ "rand 0.9.4",
"tokio",
"tokio-util",
"url",
@@ -3880,7 +3909,7 @@ dependencies = [
"log",
"object_store",
"parking_lot 0.12.4",
- "rand 0.9.1",
+ "rand 0.9.4",
"tempfile",
"url",
]
@@ -3943,7 +3972,7 @@ dependencies = [
"md-5 0.10.6",
"memchr",
"num-traits",
- "rand 0.9.1",
+ "rand 0.9.4",
"regex",
"sha2 0.10.9",
"unicode-segmentation",
@@ -4233,7 +4262,7 @@ dependencies = [
"datafusion-proto-common",
"object_store",
"prost 0.14.1",
- "rand 0.9.1",
+ "rand 0.9.4",
]
[[package]]
@@ -4684,6 +4713,27 @@ dependencies = [
"crypto-common 0.2.1",
]
+[[package]]
+name = "dirs-next"
+version = "2.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b98cf8ebf19c3d1b223e151f99a4f9f0690dca41414773390fc824184ac833e1"
+dependencies = [
+ "cfg-if",
+ "dirs-sys-next",
+]
+
+[[package]]
+name = "dirs-sys-next"
+version = "0.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4ebda144c4fe02d1f7ea1a7d9641b6fc6b580adcfa024ae48797ecdeb6825b4d"
+dependencies = [
+ "libc",
+ "redox_users",
+ "winapi",
+]
+
[[package]]
name = "displaydoc"
version = "0.2.5"
@@ -4712,14 +4762,14 @@ dependencies = [
[[package]]
name = "dns-lookup"
-version = "2.0.4"
+version = "3.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e5766087c2235fec47fafa4cfecc81e494ee679d0fd4a59887ea0919bfb0e4fc"
+checksum = "6e39034cee21a2f5bbb66ba0e3689819c4bb5d00382a282006e802a7ffa6c41d"
dependencies = [
"cfg-if",
"libc",
- "socket2 0.5.10",
- "windows-sys 0.48.0",
+ "socket2 0.6.0",
+ "windows-sys 0.60.2",
]
[[package]]
@@ -4739,31 +4789,30 @@ dependencies = [
[[package]]
name = "domain"
-version = "0.11.0"
+version = "0.12.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a11dd7f04a6a6d2aea0153c6e31f5ea7af8b2efdf52cdaeea7a9a592c7fefef9"
+checksum = "8c469892dddfeff64ecfdbc64cf059c77fb0decaeccd4d5d484394bdd6312bac"
dependencies = [
"bumpalo",
"bytes",
"domain-macros",
"futures-util",
- "hashbrown 0.14.5",
+ "hashbrown 0.17.1",
+ "jiff",
"log",
- "moka",
"octseq",
- "rand 0.8.5",
+ "rand 0.10.1",
"serde",
"smallvec",
- "time",
"tokio",
"tracing",
]
[[package]]
name = "domain-macros"
-version = "0.11.0"
+version = "0.12.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0e197fdfd2cdb5fdeb7f8ddcf3aed5d5d04ecde2890d448b14ffb716f7376b70"
+checksum = "6fef7ef74e413e36d5364db163ca577ccb56f2f74377705d5f920ee3e1544127"
dependencies = [
"proc-macro2",
"quote",
@@ -4848,6 +4897,15 @@ dependencies = [
"serde",
]
+[[package]]
+name = "email_address"
+version = "0.2.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e079f19b08ca6239f47f8ba8509c11cf3ea30095831f7fed61441475edd8c449"
+dependencies = [
+ "serde",
+]
+
[[package]]
name = "ena"
version = "0.14.3"
@@ -4961,6 +5019,12 @@ dependencies = [
"windows-sys 0.59.0",
]
+[[package]]
+name = "error-code"
+version = "3.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dea2df4cf52843e0452895c455a1a2cfbb842a1e7329671acf418fdc53ed4c59"
+
[[package]]
name = "etcd-client"
version = "0.17.0"
@@ -5017,6 +5081,12 @@ dependencies = [
"pin-project-lite",
]
+[[package]]
+name = "exitcode"
+version = "1.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "de853764b47027c2e862a995c34978ffa63c1501f2e15f987ba11bd4f9bba193"
+
[[package]]
name = "fail"
version = "0.5.1"
@@ -5042,9 +5112,9 @@ checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a"
[[package]]
name = "fancy-regex"
-version = "0.14.0"
+version = "0.17.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6e24cb5a94bcae1e5408b0effca5cd7172ea3c5755049c5f3af4cd283a165298"
+checksum = "72cf461f865c862bb7dc573f643dd6a2b6842f7c30b07882b56bd148cc2761b8"
dependencies = [
"bit-set",
"regex-automata",
@@ -5102,6 +5172,7 @@ dependencies = [
"datatypes",
"futures",
"object-store",
+ "object_store_opendal",
"serde",
"serde_json",
"snafu 0.8.6",
@@ -5272,7 +5343,7 @@ dependencies = [
"prometheus 0.14.0",
"prost 0.14.1",
"query",
- "rand 0.9.1",
+ "rand 0.9.4",
"serde",
"serde_json",
"servers",
@@ -5297,6 +5368,17 @@ dependencies = [
"bitflags 1.3.2",
]
+[[package]]
+name = "fluent-uri"
+version = "0.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bc74ac4d8359ae70623506d512209619e5cf8f347124910440dbc221714b328e"
+dependencies = [
+ "borrow-or-share",
+ "ref-cast",
+ "serde",
+]
+
[[package]]
name = "flume"
version = "0.11.1"
@@ -5335,6 +5417,16 @@ dependencies = [
"percent-encoding",
]
+[[package]]
+name = "fraction"
+version = "0.15.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e076045bb43dac435333ed5f04caf35c7463631d0dae2deb2638d94dd0a5b872"
+dependencies = [
+ "lazy_static",
+ "num",
+]
+
[[package]]
name = "fragile"
version = "2.0.1"
@@ -5403,7 +5495,7 @@ dependencies = [
"promql-parser",
"prost 0.14.1",
"query",
- "rand 0.9.1",
+ "rand 0.9.4",
"reqwest 0.13.2",
"serde",
"serde_json",
@@ -5754,21 +5846,21 @@ dependencies = [
"cfg-if",
"js-sys",
"libc",
- "wasi 0.11.1+wasi-snapshot-preview1",
+ "wasi",
"wasm-bindgen",
]
[[package]]
name = "getrandom"
-version = "0.3.3"
+version = "0.3.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "26145e563e54f2cadc477553f1ec5ee650b00862f0a58bcd12cbdc5f0ea2d2f4"
+checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd"
dependencies = [
"cfg-if",
"js-sys",
"libc",
"r-efi",
- "wasi 0.14.2+wasi-0.2.4",
+ "wasip2",
"wasm-bindgen",
]
@@ -5826,7 +5918,7 @@ dependencies = [
[[package]]
name = "greptime-proto"
version = "0.1.0"
-source = "git+https://github.com/GreptimeTeam/greptime-proto.git?rev=7224c2ad6d11db612fbdb621c36135fc37ffce35#7224c2ad6d11db612fbdb621c36135fc37ffce35"
+source = "git+https://github.com/GreptimeTeam/greptime-proto.git?rev=912f6c21017d5ab6c529568d36e7b8c30a94d84b#912f6c21017d5ab6c529568d36e7b8c30a94d84b"
dependencies = [
"prost 0.14.1",
"prost-types 0.14.1",
@@ -5840,9 +5932,9 @@ dependencies = [
[[package]]
name = "grok"
-version = "2.1.0"
+version = "2.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6c52724b609896f661a3f4641dd3a44dc602958ef615857c12d00756b4e9355b"
+checksum = "6ddab6a9c8bb998cb2fc3101fde8ef561b7c4970db3957be7a8eee1e168f666b"
dependencies = [
"glob",
"onig",
@@ -5982,6 +6074,9 @@ name = "hashbrown"
version = "0.17.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a"
+dependencies = [
+ "allocator-api2",
+]
[[package]]
name = "hashlink"
@@ -6648,7 +6743,7 @@ dependencies = [
"pin-project",
"prost 0.14.1",
"puffin",
- "rand 0.9.1",
+ "rand 0.9.4",
"rand_chacha 0.9.0",
"regex",
"regex-automata",
@@ -6839,6 +6934,15 @@ dependencies = [
"derive_utils",
]
+[[package]]
+name = "ipcrypt-rs"
+version = "0.9.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "96e4f67dbfc0f75d7b65953ecf0be3fd84ee0cb1ae72a00a4aa9a2f5518a2c80"
+dependencies = [
+ "aes",
+]
+
[[package]]
name = "ipnet"
version = "2.11.0"
@@ -7014,6 +7118,36 @@ dependencies = [
"windows-sys 0.45.0",
]
+[[package]]
+name = "jni"
+version = "0.22.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5efd9a482cf3a427f00d6b35f14332adc7902ce91efb778580e180ff90fa3498"
+dependencies = [
+ "cfg-if",
+ "combine",
+ "jni-macros",
+ "jni-sys 0.4.1",
+ "log",
+ "simd_cesu8",
+ "thiserror 2.0.17",
+ "walkdir",
+ "windows-link 0.2.1",
+]
+
+[[package]]
+name = "jni-macros"
+version = "0.22.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a00109accc170f0bdb141fed3e393c565b6f5e072365c3bd58f5b062591560a3"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "rustc_version",
+ "simd_cesu8",
+ "syn 2.0.117",
+]
+
[[package]]
name = "jni-sys"
version = "0.3.1"
@@ -7048,7 +7182,7 @@ version = "0.1.33"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "38f262f097c174adebe41eb73d66ae9c06b2844fb0da69969647bbddd9b0538a"
dependencies = [
- "getrandom 0.3.3",
+ "getrandom 0.3.4",
"libc",
]
@@ -7137,11 +7271,38 @@ version = "0.4.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1c6e529149475ca0b2820835d3dce8fcc41c6b943ca608d32f35b449255e4627"
dependencies = [
- "fluent-uri",
+ "fluent-uri 0.1.4",
"serde",
"serde_json",
]
+[[package]]
+name = "jsonschema"
+version = "0.38.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "89f50532ce4a0ba3ae930212908d8ec50e7806065c059fe9c75da2ece6132294"
+dependencies = [
+ "ahash 0.8.12",
+ "bytecount",
+ "data-encoding",
+ "email_address",
+ "fancy-regex",
+ "fraction",
+ "getrandom 0.3.4",
+ "idna",
+ "itoa",
+ "num-cmp",
+ "num-traits",
+ "percent-encoding",
+ "referencing",
+ "regex",
+ "regex-syntax",
+ "serde",
+ "serde_json",
+ "unicode-general-category",
+ "uuid-simd",
+]
+
[[package]]
name = "jsonwebtoken"
version = "10.3.0"
@@ -7335,7 +7496,7 @@ dependencies = [
"regex-syntax",
"sha3",
"string_cache",
- "term",
+ "term 1.0.2",
"unicode-xid",
"walkdir",
]
@@ -7734,7 +7895,7 @@ dependencies = [
"protobuf 2.28.0",
"protobuf-build",
"raft-engine",
- "rand 0.9.1",
+ "rand 0.9.4",
"rskafka",
"serde",
"serde_json",
@@ -8041,7 +8202,7 @@ dependencies = [
"futures-util",
"humantime-serde",
"meta-srv",
- "rand 0.9.1",
+ "rand 0.9.4",
"serde",
"serde_json",
"session",
@@ -8109,7 +8270,7 @@ dependencies = [
"partition",
"prometheus 0.14.0",
"prost 0.14.1",
- "rand 0.9.1",
+ "rand 0.9.4",
"regex",
"rskafka",
"serde",
@@ -8251,7 +8412,7 @@ checksum = "78bed444cc8a2160f01cbcf811ef18cac863ad68ae8ca62092e8db51d51c761c"
dependencies = [
"libc",
"log",
- "wasi 0.11.1+wasi-snapshot-preview1",
+ "wasi",
"windows-sys 0.59.0",
]
@@ -8320,6 +8481,7 @@ dependencies = [
"datafusion-common",
"datafusion-expr",
"datatypes",
+ "derive_more",
"dotenv",
"either",
"futures",
@@ -8340,7 +8502,7 @@ dependencies = [
"prometheus 0.14.0",
"prost 0.14.1",
"puffin",
- "rand 0.9.1",
+ "rand 0.9.4",
"rayon",
"regex",
"roaring",
@@ -8653,6 +8815,12 @@ dependencies = [
"winapi",
]
+[[package]]
+name = "ndk-context"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "27b02d87554356db9e9a873add8782d4ea6e3e58ea071a9adb9a2e8ddb884a8b"
+
[[package]]
name = "neli"
version = "0.6.5"
@@ -8755,6 +8923,15 @@ dependencies = [
"memchr",
]
+[[package]]
+name = "nom-language"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2de2bc5b451bfedaef92c90b8939a8fff5770bdcc1fafd6239d086aab8fa6b29"
+dependencies = [
+ "nom 8.0.0",
+]
+
[[package]]
name = "notify"
version = "8.0.0"
@@ -8838,6 +9015,12 @@ dependencies = [
"zeroize",
]
+[[package]]
+name = "num-cmp"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "63335b2e2c34fae2fb0aa2cecfd9f0832a1e24b3b32ecec612c3426d46dc8aaa"
+
[[package]]
name = "num-complex"
version = "0.4.6"
@@ -8998,6 +9181,31 @@ dependencies = [
"libc",
]
+[[package]]
+name = "objc2"
+version = "0.6.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3a12a8ed07aefc768292f076dc3ac8c48f3781c8f2d5851dd3d98950e8c5a89f"
+dependencies = [
+ "objc2-encode",
+]
+
+[[package]]
+name = "objc2-encode"
+version = "4.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ef25abbcd74fb2609453eb695bd2f860d389e457f67dc17cafc8b8cbc89d0c33"
+
+[[package]]
+name = "objc2-foundation"
+version = "0.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e3e0adef53c21f888deb4fa59fc59f7eb17404926ee8a6f59f5df0fd7f9f3272"
+dependencies = [
+ "bitflags 2.11.1",
+ "objc2",
+]
+
[[package]]
name = "object"
version = "0.36.7"
@@ -9037,7 +9245,7 @@ dependencies = [
"object_store_opendal",
"opendal",
"prometheus 0.14.0",
- "rand 0.9.1",
+ "rand 0.9.4",
"reqwest 0.13.2",
"serde",
"snafu 0.8.6",
@@ -9074,8 +9282,9 @@ dependencies = [
[[package]]
name = "object_store_opendal"
-version = "0.56.0"
-source = "git+https://github.com/apache/opendal.git?rev=4ad2d85296ffa6fdc2882f97d3c760ee243913f7#4ad2d85296ffa6fdc2882f97d3c760ee243913f7"
+version = "0.57.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0eb12a624a41fce745838d0ef3701ff6c47797c13cd18ad3612fd2a3134fdbd8"
dependencies = [
"async-trait",
"bytes",
@@ -9090,9 +9299,9 @@ dependencies = [
[[package]]
name = "octseq"
-version = "0.5.2"
+version = "0.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "126c3ca37c9c44cec575247f43a3e4374d8927684f129d2beeb0d2cef262fe12"
+checksum = "182eab3e1cd9cdc0ecf1ce3342d9844f3dc7d098f0694569bfdf327b612d69fd"
dependencies = [
"bytes",
"serde",
@@ -9162,8 +9371,9 @@ checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381"
[[package]]
name = "opendal"
-version = "0.56.0"
-source = "git+https://github.com/apache/opendal.git?rev=4ad2d85296ffa6fdc2882f97d3c760ee243913f7#4ad2d85296ffa6fdc2882f97d3c760ee243913f7"
+version = "0.57.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "96c9c85ce253ff87225e7669979d877a20c98a06604ec9d6dd5f4473e08f1ae1"
dependencies = [
"ctor",
"opendal-core",
@@ -9183,8 +9393,9 @@ dependencies = [
[[package]]
name = "opendal-core"
-version = "0.56.0"
-source = "git+https://github.com/apache/opendal.git?rev=4ad2d85296ffa6fdc2882f97d3c760ee243913f7#4ad2d85296ffa6fdc2882f97d3c760ee243913f7"
+version = "0.57.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c4f8607c90e2c963a91467f50fb49fbc7fb3d573f88cea219ca59ccd3740b309"
dependencies = [
"anyhow",
"base64 0.22.1",
@@ -9210,8 +9421,9 @@ dependencies = [
[[package]]
name = "opendal-layer-concurrent-limit"
-version = "0.56.0"
-source = "git+https://github.com/apache/opendal.git?rev=4ad2d85296ffa6fdc2882f97d3c760ee243913f7#4ad2d85296ffa6fdc2882f97d3c760ee243913f7"
+version = "0.57.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0d6f81ba6960e3fae1882f253b114b21d7e444e1534f209c7737a79f6243eb6f"
dependencies = [
"futures",
"http 1.3.1",
@@ -9221,8 +9433,9 @@ dependencies = [
[[package]]
name = "opendal-layer-logging"
-version = "0.56.0"
-source = "git+https://github.com/apache/opendal.git?rev=4ad2d85296ffa6fdc2882f97d3c760ee243913f7#4ad2d85296ffa6fdc2882f97d3c760ee243913f7"
+version = "0.57.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "58ada45c6d81d1aa4c9305d0c7d4bc317c59c85866a0908a2d75a7a978aa5ee2"
dependencies = [
"log",
"opendal-core",
@@ -9230,8 +9443,9 @@ dependencies = [
[[package]]
name = "opendal-layer-observe-metrics-common"
-version = "0.56.0"
-source = "git+https://github.com/apache/opendal.git?rev=4ad2d85296ffa6fdc2882f97d3c760ee243913f7#4ad2d85296ffa6fdc2882f97d3c760ee243913f7"
+version = "0.57.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "628b0228fdbd13c3d9d50eee4341f2eb82ca5b44991e4c68f07c84cc823e2d12"
dependencies = [
"futures",
"http 1.3.1",
@@ -9240,8 +9454,9 @@ dependencies = [
[[package]]
name = "opendal-layer-prometheus"
-version = "0.56.0"
-source = "git+https://github.com/apache/opendal.git?rev=4ad2d85296ffa6fdc2882f97d3c760ee243913f7#4ad2d85296ffa6fdc2882f97d3c760ee243913f7"
+version = "0.57.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0487bdb1357097ec8654781bad03ef310282517738e2864ebde69e27aaafc5ec"
dependencies = [
"opendal-core",
"opendal-layer-observe-metrics-common",
@@ -9250,8 +9465,9 @@ dependencies = [
[[package]]
name = "opendal-layer-retry"
-version = "0.56.0"
-source = "git+https://github.com/apache/opendal.git?rev=4ad2d85296ffa6fdc2882f97d3c760ee243913f7#4ad2d85296ffa6fdc2882f97d3c760ee243913f7"
+version = "0.57.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7b2a25a718afb81fad81cb9a0580a1cb989221fa2317f888c6a37f8dad408eb7"
dependencies = [
"backon",
"log",
@@ -9260,8 +9476,9 @@ dependencies = [
[[package]]
name = "opendal-layer-timeout"
-version = "0.56.0"
-source = "git+https://github.com/apache/opendal.git?rev=4ad2d85296ffa6fdc2882f97d3c760ee243913f7#4ad2d85296ffa6fdc2882f97d3c760ee243913f7"
+version = "0.57.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1e91f731724c213af81e9d03517859c8fc47b4578e64ad61ae4f099f10fe36e3"
dependencies = [
"opendal-core",
"tokio",
@@ -9269,8 +9486,9 @@ dependencies = [
[[package]]
name = "opendal-layer-tracing"
-version = "0.56.0"
-source = "git+https://github.com/apache/opendal.git?rev=4ad2d85296ffa6fdc2882f97d3c760ee243913f7#4ad2d85296ffa6fdc2882f97d3c760ee243913f7"
+version = "0.57.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "90c6fc9df6da1f0dafbdf55fa48525f1643aefbe7da8f46936e869e2a5b8a34f"
dependencies = [
"futures",
"http 1.3.1",
@@ -9280,8 +9498,9 @@ dependencies = [
[[package]]
name = "opendal-service-azblob"
-version = "0.56.0"
-source = "git+https://github.com/apache/opendal.git?rev=4ad2d85296ffa6fdc2882f97d3c760ee243913f7#4ad2d85296ffa6fdc2882f97d3c760ee243913f7"
+version = "0.57.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0030644366ef5d8cbe3a4a5822bf99a4aafddc1666e9d24b44d158d9062fc76a"
dependencies = [
"base64 0.22.1",
"bytes",
@@ -9300,8 +9519,9 @@ dependencies = [
[[package]]
name = "opendal-service-azure-common"
-version = "0.56.0"
-source = "git+https://github.com/apache/opendal.git?rev=4ad2d85296ffa6fdc2882f97d3c760ee243913f7#4ad2d85296ffa6fdc2882f97d3c760ee243913f7"
+version = "0.57.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9b489f13c42e69d69bdd72952b634356ec43a7881a20259b38b540fcecdf4051"
dependencies = [
"http 1.3.1",
"opendal-core",
@@ -9309,8 +9529,9 @@ dependencies = [
[[package]]
name = "opendal-service-fs"
-version = "0.56.0"
-source = "git+https://github.com/apache/opendal.git?rev=4ad2d85296ffa6fdc2882f97d3c760ee243913f7#4ad2d85296ffa6fdc2882f97d3c760ee243913f7"
+version = "0.57.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "22e89a665fef0e6bd249cf5ea47fc174b7ba892159bee4b9382528b1ca873a2c"
dependencies = [
"bytes",
"log",
@@ -9322,8 +9543,9 @@ dependencies = [
[[package]]
name = "opendal-service-gcs"
-version = "0.56.0"
-source = "git+https://github.com/apache/opendal.git?rev=4ad2d85296ffa6fdc2882f97d3c760ee243913f7#4ad2d85296ffa6fdc2882f97d3c760ee243913f7"
+version = "0.57.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "48de101aac565ed06af4b47903c24eafd249075553ec1fb18256751c45148d47"
dependencies = [
"async-trait",
"bytes",
@@ -9342,8 +9564,9 @@ dependencies = [
[[package]]
name = "opendal-service-http"
-version = "0.56.0"
-source = "git+https://github.com/apache/opendal.git?rev=4ad2d85296ffa6fdc2882f97d3c760ee243913f7#4ad2d85296ffa6fdc2882f97d3c760ee243913f7"
+version = "0.57.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fb6af628a0bf14075b957179444927e1df40dc7addef382b585a05ef015a077b"
dependencies = [
"http 1.3.1",
"log",
@@ -9353,8 +9576,9 @@ dependencies = [
[[package]]
name = "opendal-service-oss"
-version = "0.56.0"
-source = "git+https://github.com/apache/opendal.git?rev=4ad2d85296ffa6fdc2882f97d3c760ee243913f7#4ad2d85296ffa6fdc2882f97d3c760ee243913f7"
+version = "0.57.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "328fa55e8888cbdfe00826bfea2a79042422b720e8369e9e021e46121dea5ace"
dependencies = [
"bytes",
"http 1.3.1",
@@ -9369,8 +9593,9 @@ dependencies = [
[[package]]
name = "opendal-service-s3"
-version = "0.56.0"
-source = "git+https://github.com/apache/opendal.git?rev=4ad2d85296ffa6fdc2882f97d3c760ee243913f7#4ad2d85296ffa6fdc2882f97d3c760ee243913f7"
+version = "0.57.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "313d46c9f5ae70bca26b7c3e3fbb9b639292625f28af73aa016f47e788af9deb"
dependencies = [
"base64 0.22.1",
"bytes",
@@ -9532,7 +9757,7 @@ dependencies = [
"futures-util",
"opentelemetry 0.30.0",
"percent-encoding",
- "rand 0.9.1",
+ "rand 0.9.4",
"serde_json",
"thiserror 2.0.17",
"tokio",
@@ -9550,7 +9775,7 @@ dependencies = [
"futures-util",
"opentelemetry 0.31.0",
"percent-encoding",
- "rand 0.9.1",
+ "rand 0.9.4",
"thiserror 2.0.17",
]
@@ -9728,7 +9953,7 @@ dependencies = [
"paste",
"prost 0.14.1",
"prost-build 0.14.1",
- "rand 0.9.1",
+ "rand 0.9.4",
"replace_with",
"serde",
"smallvec",
@@ -10134,7 +10359,7 @@ dependencies = [
"md5",
"pg_interval_2",
"postgres-types",
- "rand 0.10.0",
+ "rand 0.10.1",
"rust_decimal",
"rustls-pki-types",
"ryu",
@@ -10511,7 +10736,7 @@ dependencies = [
"hmac",
"md-5 0.10.6",
"memchr",
- "rand 0.9.1",
+ "rand 0.9.4",
"sha2 0.10.9",
"stringprep",
]
@@ -10660,6 +10885,19 @@ dependencies = [
"syn 2.0.117",
]
+[[package]]
+name = "prettytable-rs"
+version = "0.10.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "eea25e07510aa6ab6547308ebe3c036016d162b8da920dbb079e3ba8acf3d95a"
+dependencies = [
+ "encode_unicode",
+ "is-terminal",
+ "lazy_static",
+ "term 0.7.0",
+ "unicode-width 0.1.14",
+]
+
[[package]]
name = "proc-macro-crate"
version = "1.3.1"
@@ -10825,7 +11063,7 @@ checksum = "bee689443a2bd0a16ab0348b52ee43e3b2d1b1f931c8aa5c9f8de4c86fbe8c40"
dependencies = [
"bitflags 2.11.1",
"num-traits",
- "rand 0.9.1",
+ "rand 0.9.4",
"rand_chacha 0.9.0",
"rand_xorshift",
"regex-syntax",
@@ -11252,7 +11490,7 @@ dependencies = [
"promql",
"promql-parser",
"prost 0.14.1",
- "rand 0.9.1",
+ "rand 0.9.4",
"regex",
"serde",
"serde_json",
@@ -11325,9 +11563,9 @@ checksum = "434b42fec591c96ef50e21e886936e66d3cc3f737104fdb9b737c40ffb94c098"
dependencies = [
"aws-lc-rs",
"bytes",
- "getrandom 0.3.3",
+ "getrandom 0.3.4",
"lru-slab",
- "rand 0.9.1",
+ "rand 0.9.4",
"ring",
"rustc-hash 2.1.1",
"rustls",
@@ -11433,9 +11671,9 @@ dependencies = [
[[package]]
name = "rand"
-version = "0.9.1"
+version = "0.9.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9fbfd9d094a40bf3ae768db9361049ace4c0e04a4fd6b359518bd7b73a73dd97"
+checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea"
dependencies = [
"rand_chacha 0.9.0",
"rand_core 0.9.3",
@@ -11443,9 +11681,9 @@ dependencies = [
[[package]]
name = "rand"
-version = "0.10.0"
+version = "0.10.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bc266eb313df6c5c09c1c7b1fbe2510961e5bcd3add930c1e31f7ed9da0feff8"
+checksum = "d2e8e8bcc7961af1fdac401278c6a831614941f6164ee3bf4ce61b7edb162207"
dependencies = [
"chacha20 0.10.0",
"getrandom 0.4.1",
@@ -11488,7 +11726,7 @@ version = "0.9.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38"
dependencies = [
- "getrandom 0.3.3",
+ "getrandom 0.3.4",
]
[[package]]
@@ -11581,6 +11819,17 @@ dependencies = [
"bitflags 2.11.1",
]
+[[package]]
+name = "redox_users"
+version = "0.4.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ba009ff324d1fc1b900bd1fdb31564febe58a8ccc8a6fdbb93b543d33b13ca43"
+dependencies = [
+ "getrandom 0.2.16",
+ "libredox",
+ "thiserror 1.0.69",
+]
+
[[package]]
name = "ref-cast"
version = "1.0.24"
@@ -11601,6 +11850,21 @@ dependencies = [
"syn 2.0.117",
]
+[[package]]
+name = "referencing"
+version = "0.38.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "15a8af0c6bb8eaf8b07cb06fc31ff30ca6fe19fb99afa476c276d8b24f365b0b"
+dependencies = [
+ "ahash 0.8.12",
+ "fluent-uri 0.4.1",
+ "getrandom 0.3.4",
+ "hashbrown 0.16.1",
+ "parking_lot 0.12.4",
+ "percent-encoding",
+ "serde_json",
+]
+
[[package]]
name = "regex"
version = "1.12.2"
@@ -11666,6 +11930,15 @@ version = "1.9.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ba39f3699c378cd8970968dcbff9c43159ea4cfbd88d43c00b22f2ef10a435d2"
+[[package]]
+name = "relative-path"
+version = "2.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bca40a312222d8ba74837cb474edef44b37f561da5f773981007a10bbaa992b0"
+dependencies = [
+ "serde",
+]
+
[[package]]
name = "rend"
version = "0.4.2"
@@ -11805,6 +12078,7 @@ dependencies = [
"futures-channel",
"futures-core",
"futures-util",
+ "h2 0.4.11",
"http 1.3.1",
"http-body 1.0.1",
"http-body-util",
@@ -11817,6 +12091,7 @@ dependencies = [
"pin-project-lite",
"quinn",
"rustls",
+ "rustls-native-certs 0.8.1",
"rustls-pki-types",
"serde",
"serde_json",
@@ -11878,6 +12153,50 @@ dependencies = [
"web-sys",
]
+[[package]]
+name = "reqwest-middleware"
+version = "0.4.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "57f17d28a6e6acfe1733fe24bcd30774d13bffa4b8a22535b4c8c98423088d4e"
+dependencies = [
+ "anyhow",
+ "async-trait",
+ "http 1.3.1",
+ "reqwest 0.12.28",
+ "serde",
+ "thiserror 1.0.69",
+ "tower-service",
+]
+
+[[package]]
+name = "reqwest-retry"
+version = "0.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "105747e3a037fe5bf17458d794de91149e575b6183fc72c85623a44abb9683f5"
+dependencies = [
+ "anyhow",
+ "async-trait",
+ "futures",
+ "getrandom 0.2.16",
+ "http 1.3.1",
+ "hyper 1.6.0",
+ "reqwest 0.12.28",
+ "reqwest-middleware",
+ "retry-policies",
+ "thiserror 2.0.17",
+ "tokio",
+ "wasmtimer",
+]
+
+[[package]]
+name = "retry-policies"
+version = "0.5.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dc05fbf560421a0357a750cbe78c7ca19d4923918490daabba313d5dbc871e47"
+dependencies = [
+ "rand 0.10.1",
+]
+
[[package]]
name = "rgb"
version = "0.8.50"
@@ -11965,9 +12284,12 @@ dependencies = [
[[package]]
name = "roxmltree"
-version = "0.20.0"
+version = "0.21.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6c20b6793b5c2fa6553b250154b78d6d0db37e72700ae35fad9387a46f487c97"
+checksum = "f1964b10c76125c36f8afe190065a4bf9a87bf324842c05701330bba9f1cacbb"
+dependencies = [
+ "memchr",
+]
[[package]]
name = "rsa"
@@ -12022,7 +12344,7 @@ dependencies = [
"integer-encoding 4.0.2",
"lz4",
"parking_lot 0.12.4",
- "rand 0.9.1",
+ "rand 0.9.4",
"rsasl",
"rustls",
"snap",
@@ -12068,7 +12390,7 @@ dependencies = [
"proc-macro2",
"quote",
"regex",
- "relative-path",
+ "relative-path 1.9.3",
"rustc_version",
"syn 2.0.117",
"unicode-ident",
@@ -12289,7 +12611,7 @@ checksum = "1d99feebc72bae7ab76ba994bb5e121b8d83d910ca40b36e0921f53becc41784"
dependencies = [
"core-foundation 0.10.1",
"core-foundation-sys",
- "jni",
+ "jni 0.21.1",
"log",
"once_cell",
"rustls",
@@ -12326,6 +12648,25 @@ version = "1.0.21"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8a0d197bd2c9dc6e53b84da9556a69ba4cdfab8619eb41a8bd1cc2027a0f6b1d"
+[[package]]
+name = "rustyline"
+version = "17.0.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e902948a25149d50edc1a8e0141aad50f54e22ba83ff988cf8f7c9ef07f50564"
+dependencies = [
+ "bitflags 2.11.1",
+ "cfg-if",
+ "clipboard-win",
+ "libc",
+ "log",
+ "memchr",
+ "nix 0.30.1",
+ "unicode-segmentation",
+ "unicode-width 0.2.1",
+ "utf8parse",
+ "windows-sys 0.60.2",
+]
+
[[package]]
name = "ryu"
version = "1.0.20"
@@ -12704,6 +13045,19 @@ dependencies = [
"unsafe-libyaml",
]
+[[package]]
+name = "serde_yaml_ng"
+version = "0.10.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7b4db627b98b36d4203a7b458cf3573730f2bb591b28871d916dfa9efabfd41f"
+dependencies = [
+ "indexmap 2.13.0",
+ "itoa",
+ "ryu",
+ "serde",
+ "unsafe-libyaml",
+]
+
[[package]]
name = "servers"
version = "1.1.0"
@@ -12800,7 +13154,7 @@ dependencies = [
"prost 0.14.1",
"query",
"quoted-string",
- "rand 0.9.1",
+ "rand 0.9.4",
"regex",
"reqwest 0.13.2",
"rust-embed",
@@ -12978,7 +13332,7 @@ version = "0.15.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c962f626b54771990066e5435ec8331d1462576cd2d1e62f24076ae014f92112"
dependencies = [
- "getrandom 0.3.3",
+ "getrandom 0.3.4",
"halfbrown",
"ref-cast",
"serde",
@@ -12987,6 +13341,16 @@ dependencies = [
"value-trait",
]
+[[package]]
+name = "simd_cesu8"
+version = "1.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "94f90157bb87cddf702797c5dadfa0be7d266cdf49e22da2fcaa32eff75b2c33"
+dependencies = [
+ "rustc_version",
+ "simdutf8",
+]
+
[[package]]
name = "simdutf8"
version = "0.1.5"
@@ -13878,12 +14242,12 @@ dependencies = [
[[package]]
name = "syslog_loose"
-version = "0.21.0"
+version = "0.22.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "161028c00842709450114c39db3b29f44c898055ed8833bb9b535aba7facf30e"
+checksum = "d6ec4df26907adce53e94eac201a9ba38744baea3bc97f34ffd591d5646231a6"
dependencies = [
"chrono",
- "nom 7.1.3",
+ "nom 8.0.0",
]
[[package]]
@@ -14102,9 +14466,9 @@ checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369"
[[package]]
name = "tar"
-version = "0.4.45"
+version = "0.4.46"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "22692a6476a21fa75fdfc11d452fda482af402c008cdbaf3476414e122040973"
+checksum = "3f6221d9a6003c78398e3b239969f352578258df48c8eb051caadae0015bc840"
dependencies = [
"filetime",
"libc",
@@ -14127,12 +14491,23 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2d31c77bdf42a745371d260a26ca7163f1e0924b64afa0b688e61b5a9fa02f16"
dependencies = [
"fastrand",
- "getrandom 0.3.3",
+ "getrandom 0.3.4",
"once_cell",
"rustix 1.0.7",
"windows-sys 0.61.2",
]
+[[package]]
+name = "term"
+version = "0.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c59df8ac95d96ff9bede18eb7300b0fda5e5d8d90960e76f8e14ae765eedbf1f"
+dependencies = [
+ "dirs-next",
+ "rustversion",
+ "winapi",
+]
+
[[package]]
name = "term"
version = "1.0.2"
@@ -14185,7 +14560,7 @@ dependencies = [
"nix 0.28.0",
"partition",
"paste",
- "rand 0.9.1",
+ "rand 0.9.4",
"rand_chacha 0.9.0",
"reqwest 0.13.2",
"rustls",
@@ -14272,7 +14647,7 @@ dependencies = [
"plugins",
"prost 0.14.1",
"query",
- "rand 0.9.1",
+ "rand 0.9.4",
"rstest",
"rstest_reuse",
"sea-query",
@@ -14562,7 +14937,7 @@ dependencies = [
"pin-project-lite",
"postgres-protocol",
"postgres-types",
- "rand 0.9.1",
+ "rand 0.9.4",
"socket2 0.5.10",
"tokio",
"tokio-util",
@@ -15213,6 +15588,12 @@ version = "0.3.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5c1cb5db39152898a79168971543b1cb5020dff7fe43c8dc468b0885f5e29df5"
+[[package]]
+name = "unicode-general-category"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0b993bddc193ae5bd0d623b49ec06ac3e9312875fdae725a975c51db1cc1677f"
+
[[package]]
name = "unicode-ident"
version = "1.0.22"
@@ -15346,11 +15727,21 @@ checksum = "b672338555252d43fd2240c714dc444b8c6fb0a5c5335e65a07bba7742735ddb"
dependencies = [
"getrandom 0.4.1",
"js-sys",
- "rand 0.9.1",
+ "rand 0.9.4",
"serde_core",
"wasm-bindgen",
]
+[[package]]
+name = "uuid-simd"
+version = "0.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "23b082222b4f6619906941c17eb2297fff4c2fb96cb60164170522942a200bd8"
+dependencies = [
+ "outref",
+ "vsimd",
+]
+
[[package]]
name = "valuable"
version = "0.1.1"
@@ -15433,9 +15824,9 @@ dependencies = [
[[package]]
name = "vrl"
-version = "0.25.0"
+version = "0.33.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4f49394b948406ea1564aa00152e011d87a38ad35d277ebddda257a9ee39c419"
+checksum = "925a4d3321b18a200c82c3ec02ee2be2b4bf16db07a5ce7e2a9a888b795ea862"
dependencies = [
"aes",
"aes-siv",
@@ -15465,8 +15856,10 @@ dependencies = [
"domain",
"dyn-clone",
"encoding_rs",
+ "exitcode",
"fancy-regex",
"flate2",
+ "getrandom 0.3.4",
"grok",
"hex",
"hmac",
@@ -15476,12 +15869,15 @@ dependencies = [
"indexmap 2.13.0",
"indoc",
"influxdb-line-protocol",
+ "ipcrypt-rs",
"itertools 0.14.0",
+ "jsonschema",
"lalrpop",
"lalrpop-util",
"lz4_flex 0.11.6",
"md-5 0.10.6",
- "nom 7.1.3",
+ "nom 8.0.0",
+ "nom-language",
"ofb",
"onig",
"ordered-float 4.6.0",
@@ -15490,20 +15886,27 @@ dependencies = [
"percent-encoding",
"pest",
"pest_derive",
+ "prettytable-rs",
"prost 0.13.5",
"prost-reflect",
"psl",
"psl-types",
"publicsuffix",
"quoted_printable",
- "rand 0.8.5",
+ "rand 0.9.4",
"regex",
+ "relative-path 2.0.1",
+ "reqwest 0.12.28",
+ "reqwest-middleware",
+ "reqwest-retry",
"roxmltree",
"rust_decimal",
+ "rustyline",
"seahash",
"serde",
"serde_json",
"serde_yaml",
+ "serde_yaml_ng",
"sha-1",
"sha2 0.10.9",
"sha3",
@@ -15511,6 +15914,8 @@ dependencies = [
"snafu 0.8.6",
"snap",
"strip-ansi-escapes",
+ "strum 0.26.3",
+ "strum_macros 0.26.4",
"syslog_loose",
"termcolor",
"thiserror 2.0.17",
@@ -15521,7 +15926,9 @@ dependencies = [
"url",
"utf8-width",
"uuid",
+ "webbrowser",
"woothee",
+ "xxhash-rust",
"zstd",
]
@@ -15565,15 +15972,6 @@ version = "0.11.1+wasi-snapshot-preview1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b"
-[[package]]
-name = "wasi"
-version = "0.14.2+wasi-0.2.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9683f9a5a998d873c0d21fcbe3c083009670149a8fab228644b8bd36b2c48cb3"
-dependencies = [
- "wit-bindgen-rt",
-]
-
[[package]]
name = "wasip2"
version = "1.0.2+wasi-0.2.9"
@@ -15714,6 +16112,20 @@ dependencies = [
"semver",
]
+[[package]]
+name = "wasmtimer"
+version = "0.4.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1c598d6b99ea013e35844697fc4670d08339d5cda15588f193c6beedd12f644b"
+dependencies = [
+ "futures",
+ "js-sys",
+ "parking_lot 0.12.4",
+ "pin-utils",
+ "slab",
+ "wasm-bindgen",
+]
+
[[package]]
name = "web-sys"
version = "0.3.95"
@@ -15734,6 +16146,22 @@ dependencies = [
"wasm-bindgen",
]
+[[package]]
+name = "webbrowser"
+version = "1.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0fc95580916af1e68ff6a7be07446fc5db73ebf71cf092de939bbf5f7e189f72"
+dependencies = [
+ "core-foundation 0.10.1",
+ "jni 0.22.4",
+ "log",
+ "ndk-context",
+ "objc2",
+ "objc2-foundation",
+ "url",
+ "web-sys",
+]
+
[[package]]
name = "webpki"
version = "0.22.4"
@@ -16020,6 +16448,15 @@ dependencies = [
"windows-targets 0.52.6",
]
+[[package]]
+name = "windows-sys"
+version = "0.60.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb"
+dependencies = [
+ "windows-targets 0.53.5",
+]
+
[[package]]
name = "windows-sys"
version = "0.61.2"
@@ -16068,13 +16505,30 @@ dependencies = [
"windows_aarch64_gnullvm 0.52.6",
"windows_aarch64_msvc 0.52.6",
"windows_i686_gnu 0.52.6",
- "windows_i686_gnullvm",
+ "windows_i686_gnullvm 0.52.6",
"windows_i686_msvc 0.52.6",
"windows_x86_64_gnu 0.52.6",
"windows_x86_64_gnullvm 0.52.6",
"windows_x86_64_msvc 0.52.6",
]
+[[package]]
+name = "windows-targets"
+version = "0.53.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3"
+dependencies = [
+ "windows-link 0.2.1",
+ "windows_aarch64_gnullvm 0.53.1",
+ "windows_aarch64_msvc 0.53.1",
+ "windows_i686_gnu 0.53.1",
+ "windows_i686_gnullvm 0.53.1",
+ "windows_i686_msvc 0.53.1",
+ "windows_x86_64_gnu 0.53.1",
+ "windows_x86_64_gnullvm 0.53.1",
+ "windows_x86_64_msvc 0.53.1",
+]
+
[[package]]
name = "windows-threading"
version = "0.1.0"
@@ -16102,6 +16556,12 @@ version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3"
+[[package]]
+name = "windows_aarch64_gnullvm"
+version = "0.53.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53"
+
[[package]]
name = "windows_aarch64_msvc"
version = "0.42.2"
@@ -16120,6 +16580,12 @@ version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469"
+[[package]]
+name = "windows_aarch64_msvc"
+version = "0.53.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006"
+
[[package]]
name = "windows_i686_gnu"
version = "0.42.2"
@@ -16138,12 +16604,24 @@ version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b"
+[[package]]
+name = "windows_i686_gnu"
+version = "0.53.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3"
+
[[package]]
name = "windows_i686_gnullvm"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66"
+[[package]]
+name = "windows_i686_gnullvm"
+version = "0.53.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c"
+
[[package]]
name = "windows_i686_msvc"
version = "0.42.2"
@@ -16162,6 +16640,12 @@ version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66"
+[[package]]
+name = "windows_i686_msvc"
+version = "0.53.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2"
+
[[package]]
name = "windows_x86_64_gnu"
version = "0.42.2"
@@ -16180,6 +16664,12 @@ version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78"
+[[package]]
+name = "windows_x86_64_gnu"
+version = "0.53.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499"
+
[[package]]
name = "windows_x86_64_gnullvm"
version = "0.42.2"
@@ -16198,6 +16688,12 @@ version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d"
+[[package]]
+name = "windows_x86_64_gnullvm"
+version = "0.53.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1"
+
[[package]]
name = "windows_x86_64_msvc"
version = "0.42.2"
@@ -16216,6 +16712,12 @@ version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"
+[[package]]
+name = "windows_x86_64_msvc"
+version = "0.53.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650"
+
[[package]]
name = "winnow"
version = "0.5.40"
@@ -16263,15 +16765,6 @@ dependencies = [
"wit-parser",
]
-[[package]]
-name = "wit-bindgen-rt"
-version = "0.39.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6f42320e61fe2cfd34354ecb597f86f413484a798ba44a8ca1165c58d42da6c1"
-dependencies = [
- "bitflags 2.11.1",
-]
-
[[package]]
name = "wit-bindgen-rust"
version = "0.51.0"
@@ -16425,6 +16918,12 @@ dependencies = [
"rustix 1.0.7",
]
+[[package]]
+name = "xxhash-rust"
+version = "0.8.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fdd20c5420375476fbd4394763288da7eb0cc0b8c11deed431a91562af7335d3"
+
[[package]]
name = "yaml-rust"
version = "0.4.5"
diff --git a/Cargo.toml b/Cargo.toml
index 32407f31cf..47b04be366 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -158,7 +158,7 @@ fs2 = "0.4"
fst = "0.4.7"
futures = "0.3"
futures-util = "0.3"
-greptime-proto = { git = "https://github.com/GreptimeTeam/greptime-proto.git", rev = "7224c2ad6d11db612fbdb621c36135fc37ffce35" }
+greptime-proto = { git = "https://github.com/GreptimeTeam/greptime-proto.git", rev = "912f6c21017d5ab6c529568d36e7b8c30a94d84b" }
hex = "0.4"
http = "1"
humantime = "2.1"
@@ -178,7 +178,7 @@ nalgebra = "0.33"
nix = { version = "0.30.1", default-features = false, features = ["event", "fs", "process"] }
notify = "8.0"
num_cpus = "1.16"
-object_store_opendal = { git = "https://github.com/apache/opendal.git", rev = "4ad2d85296ffa6fdc2882f97d3c760ee243913f7" }
+object_store_opendal = "0.57"
once_cell = "1.18"
opentelemetry-proto = { version = "0.31", features = [
"gen-tonic",
@@ -259,7 +259,7 @@ tracing-opentelemetry = "0.31.0"
tracing-subscriber = { version = "0.3", features = ["env-filter", "json", "fmt"] }
typetag = "0.2"
uuid = { version = "1.17", features = ["serde", "v4", "v7", "fast-rng"] }
-vrl = "0.25"
+vrl = "0.33"
zstd = "0.13"
# DO_NOT_REMOVE_THIS: END_OF_EXTERNAL_DEPENDENCIES
diff --git a/config/config.md b/config/config.md
index 0fae0caaa4..df06d2153c 100644
--- a/config/config.md
+++ b/config/config.md
@@ -14,6 +14,7 @@
| --- | -----| ------- | ----------- |
| `default_timezone` | String | Unset | The default timezone of the server. |
| `default_column_prefix` | String | Unset | The default column prefix for auto-created time index and value columns. |
+| `auto_create_table` | Bool | `true` | Server-side global switch for auto table creation on write.
When `false`, a missing table is never auto-created even if the request sets the `auto_create_table` hint to `true`. Default: `true`. |
| `user_provider` | String | Unset | The user provider for authentication.
Examples: "static_user_provider:file:/path/to/users", "static_user_provider:cmd:greptime_user=greptime_pwd" |
| `max_in_flight_write_bytes` | String | Unset | Maximum total memory for all concurrent write request bodies and messages (HTTP, gRPC, Flight).
Set to 0 to disable the limit. Default: "0" (unlimited) |
| `write_bytes_exhausted_policy` | String | Unset | Policy when write bytes quota is exhausted.
Options: "wait" (default, 10s timeout), "wait()" (e.g., "wait(30s)"), "fail" |
@@ -230,6 +231,7 @@
| --- | -----| ------- | ----------- |
| `default_timezone` | String | Unset | The default timezone of the server. |
| `default_column_prefix` | String | Unset | The default column prefix for auto-created time index and value columns. |
+| `auto_create_table` | Bool | `true` | Server-side global switch for auto table creation on write.
When `false`, a missing table is never auto-created even if the request sets the `auto_create_table` hint to `true`. Default: `true`. |
| `user_provider` | String | Unset | The user provider for authentication.
Examples: "static_user_provider:file:/path/to/users", "static_user_provider:cmd:greptime_user=greptime_pwd" |
| `max_in_flight_write_bytes` | String | Unset | Maximum total memory for all concurrent write request bodies and messages (HTTP, gRPC, Flight).
Set to 0 to disable the limit. Default: "0" (unlimited) |
| `write_bytes_exhausted_policy` | String | Unset | Policy when write bytes quota is exhausted.
Options: "wait" (default, 10s timeout), "wait()" (e.g., "wait(30s)"), "fail" |
@@ -449,6 +451,7 @@
| `init_regions_in_background` | Bool | `false` | Initialize all regions in the background during the startup.
By default, it provides services after all regions have been initialized. |
| `init_regions_parallelism` | Integer | `16` | Parallelism of initializing regions. |
| `max_concurrent_queries` | Integer | `0` | The maximum concurrent queries allowed to be executed. Zero means unlimited. |
+| `concurrent_query_limiter_timeout` | String | `100ms` | Timeout to acquire a permit from the concurrent query limiter when `max_concurrent_queries` is reached. |
| `enable_telemetry` | Bool | `true` | Enable telemetry to collect anonymous usage data. Enabled by default. |
| `http` | -- | -- | The HTTP server options. |
| `http.addr` | String | `127.0.0.1:4000` | The address to bind the HTTP server. |
@@ -628,6 +631,7 @@
| `flow.batching_mode.experimental_frontend_scan_timeout` | String | `30s` | Flow wait for available frontend timeout,
if failed to find available frontend after frontend_scan_timeout elapsed, return error
which prevent flownode from starting |
| `flow.batching_mode.experimental_max_filter_num_per_query` | Integer | `20` | Maximum number of filters allowed in a single query |
| `flow.batching_mode.experimental_time_window_merge_threshold` | Integer | `3` | Time window merge distance |
+| `flow.batching_mode.experimental_enable_incremental_read` | Bool | `false` | Whether to enable experimental flow incremental source reads.
When disabled, batching flows always execute full-snapshot queries.
Can be overridden per flow with WITH (experimental_enable_incremental_read = 'true'). |
| `flow.batching_mode.read_preference` | String | `Leader` | Read preference of the Frontend client. |
| `flow.batching_mode.frontend_tls` | -- | -- | -- |
| `flow.batching_mode.frontend_tls.enabled` | Bool | `false` | Whether to enable TLS for client. |
diff --git a/config/datanode.example.toml b/config/datanode.example.toml
index d558918daf..9351c4e85d 100644
--- a/config/datanode.example.toml
+++ b/config/datanode.example.toml
@@ -20,6 +20,9 @@ init_regions_parallelism = 16
## The maximum concurrent queries allowed to be executed. Zero means unlimited.
max_concurrent_queries = 0
+## Timeout to acquire a permit from the concurrent query limiter when `max_concurrent_queries` is reached.
+concurrent_query_limiter_timeout = "100ms"
+
## Enable telemetry to collect anonymous usage data. Enabled by default.
#+ enable_telemetry = true
diff --git a/config/flownode.example.toml b/config/flownode.example.toml
index 2c053e6e8c..ff8a9e4a50 100644
--- a/config/flownode.example.toml
+++ b/config/flownode.example.toml
@@ -31,6 +31,10 @@ node_id = 14
#+experimental_max_filter_num_per_query=20
## Time window merge distance
#+experimental_time_window_merge_threshold=3
+## Whether to enable experimental flow incremental source reads.
+## When disabled, batching flows always execute full-snapshot queries.
+## Can be overridden per flow with WITH (experimental_enable_incremental_read = 'true').
+#+experimental_enable_incremental_read=false
## Read preference of the Frontend client.
#+read_preference="Leader"
[flow.batching_mode.frontend_tls]
diff --git a/config/frontend.example.toml b/config/frontend.example.toml
index 39f38fbef9..a044aebda6 100644
--- a/config/frontend.example.toml
+++ b/config/frontend.example.toml
@@ -6,6 +6,10 @@ default_timezone = "UTC"
## @toml2docs:none-default
default_column_prefix = "greptime"
+## Server-side global switch for auto table creation on write.
+## When `false`, a missing table is never auto-created even if the request sets the `auto_create_table` hint to `true`. Default: `true`.
+#+ auto_create_table = true
+
## The user provider for authentication.
## Examples: "static_user_provider:file:/path/to/users", "static_user_provider:cmd:greptime_user=greptime_pwd"
## @toml2docs:none-default
diff --git a/config/standalone.example.toml b/config/standalone.example.toml
index d5c42e744c..5740e0e1cf 100644
--- a/config/standalone.example.toml
+++ b/config/standalone.example.toml
@@ -6,6 +6,10 @@ default_timezone = "UTC"
## @toml2docs:none-default
default_column_prefix = "greptime"
+## Server-side global switch for auto table creation on write.
+## When `false`, a missing table is never auto-created even if the request sets the `auto_create_table` hint to `true`. Default: `true`.
+#+ auto_create_table = true
+
## The user provider for authentication.
## Examples: "static_user_provider:file:/path/to/users", "static_user_provider:cmd:greptime_user=greptime_pwd"
## @toml2docs:none-default
diff --git a/docs/rfcs/2026-05-28-table-semantic-layer.md b/docs/rfcs/2026-05-28-table-semantic-layer.md
new file mode 100644
index 0000000000..e4d899d704
--- /dev/null
+++ b/docs/rfcs/2026-05-28-table-semantic-layer.md
@@ -0,0 +1,157 @@
+---
+Feature Name: Table Semantic Layer
+Tracking Issue: TBD
+Date: 2026-05-28
+Author: "Dennis Zhuang "
+---
+
+# Summary
+
+Attach a thin layer of semantic metadata to each table so machine consumers — LLM agents, alert generators, dashboard builders, MCP servers, ETL pipelines — can align it with the observability concepts they already know (OTel instrument kinds, Prometheus naming conventions, UCUM units, semantic conventions, severity numbers, OTel ↔ Prometheus translation rules).
+
+The mechanism reuses what already exists in `table_options` (the same slot that today carries `table_data_model` and `otlp_metric_compat`): a reserved `greptime.semantic.*` namespace, plus standard SQL column `COMMENT` for field-level supplements, plus an `information_schema.semantic_tables` view as the discovery entry point. No new protocol, no new DDL keyword.
+
+Per-table identity only. Cross-table relationships are deferred.
+
+# Motivation
+
+GreptimeDB already ingests OTLP metrics / traces / logs and Prometheus remote write. Each protocol carries rich metadata on the wire (instrument kind, temporality, unit, scope, resource, semantic-conventions version), and most of it is dropped when rows land in a table:
+
+- An `opentelemetry_traces` table looks like any wide table; signal type, source, and field provenance must be guessed from naming.
+- The OTel-to-Prometheus translation in v0.16+ actively drops scope attributes and most resource attributes; the table never records *what was dropped*.
+- Prometheus remote write v1 metadata is unreliable by protocol, but downstream tables do not flag whether `counter` typing was *declared* or *inferred* from the `_total` suffix.
+- Mixed-temporality data (OTel delta + Prometheus cumulative in the same table) is unrecoverable from schema alone.
+
+The audience is broader than LLM agents. Alert generators need to choose between `rate()` and absolute thresholds, and need units to pick sensible bounds. Dashboard builders pick visualisations by signal type. MCP servers surface a structured tool catalog instead of free-text descriptions. ETL pipelines need lineage to know whether a `service_name` column is `resource.service.name` or a free-form label. All of them currently guess from column names; the metadata to remove the guess already exists at ingest time, we just do not preserve it.
+
+# Goals
+
+1. Tag every ingested table with a stable identity using existing SQL surfaces — no new protocol, no new DDL keyword.
+2. Record the lossy transformations the ingestion path performs (dropped attributes, scope handling, type inference vs. declaration).
+3. Expose one `information_schema` view as the consumer-facing discovery entry point.
+4. Keep the layer optional and additive — tables without these options keep working unchanged.
+
+# Non-Goals
+
+- Cross-table relationship modelling. Deferred to a follow-up RFC.
+- Bespoke storage. Reuse `table_options` and column `COMMENT`.
+- Semantic enforcement at query time. The layer is descriptive, not coercive.
+- New wire protocol. Upstream standardisation is mentioned only as a future direction.
+
+# Proposal
+
+## Three mechanisms
+
+1. **`greptime.semantic.*` table options** — table-level identity and lineage. Carried inside the existing `table_options` blob. This is the same slot that today carries `table_data_model = 'greptime_trace_v1'` and `otlp_metric_compat = 'prom'`, so the mechanism is generalising what the OTLP trace auto-create path already does.
+2. **Column `COMMENT`** — column-level supplements ("this column is `resource.service.name`"; "this column carries delta values"). Standard SQL.
+3. **`information_schema.semantic_tables` view** — a denormalised projection of the options, registered through the existing `with_extra_table_factories()` hook. Tables without a `greptime.semantic.*` option do not appear in the view.
+
+## Vocabulary
+
+All keys are flat strings under the `greptime.semantic.` prefix; values are strings; unknown keys are tolerated so the vocabulary can grow without coordinated rollouts.
+
+**Common (all signals)**
+
+| Key | Example |
+| --- | --- |
+| `greptime.semantic.signal_type` | `trace` / `log` / `metric` / `event` |
+| `greptime.semantic.source` | `opentelemetry` / `prometheus` / `elasticsearch` / `loki` / `custom` |
+| `greptime.semantic.source_version` | protocol or SDK version, e.g. `v2` (Prom remote write), `1.30.0` (optional) |
+| `greptime.semantic.pipeline` | `greptime_trace_v1` (subsumes the existing `table_data_model` value) |
+
+**Trace**: `greptime.semantic.trace.conventions` (e.g. `otel-semconv-1.27`, lifted from `schema_url`, which is the version of the OpenTelemetry semantic conventions used in this table), `greptime.semantic.trace.has_events`, `greptime.semantic.trace.has_links`.
+
+**Metric** — v1 assumes one metric type per table, which is how both Prom RW and the post-v0.16 OTel ingestion path land data today; mixed-type tables are a follow-up.
+
+| Key | Example |
+| --- | --- |
+| `greptime.semantic.metric.type` | `counter` / `gauge` / `histogram` / `summary` / `updown_counter` / `gauge_histogram` / `info` / `stateset` |
+| `greptime.semantic.metric.unit` | UCUM, e.g. `s`, `By`, `{request}` |
+| `greptime.semantic.metric.temporality` | `cumulative` / `delta` (OTel only) |
+| `greptime.semantic.metric.monotonic` | `true` / `false` |
+| `greptime.semantic.metric.metadata_quality` | `declared` (OTLP / Prom RW v2 / exposition) or `inferred` (Prom RW v1, name-suffix guess) |
+| `greptime.semantic.metric.original_name` | Pre-translation OTel name when the table name was Prometheus-ised |
+
+`metadata_quality = inferred` is the load-bearing field for confidence-aware tooling: an inferred counter should be re-checked before betting on `rate()`-style semantics.
+
+**Log**: `greptime.semantic.log.severity_scheme` (`otlp` / `syslog` / `custom`), `greptime.semantic.log.body_format` (`string` / `json` / `mixed`).
+
+**Resource / scope preservation**: `greptime.semantic.resource.attributes_preserved` (JSON array string of attrs promoted to columns), `greptime.semantic.resource.attributes_dropped` (boolean), `greptime.semantic.scope.preserved` (boolean). These answer the most common downstream question: "is this data missing because it was dropped, or because it lives on a different column than I think?" List-shaped values use JSON array strings rather than comma-separated text to avoid escaping and ordering ambiguity.
+
+## Conflict and update semantics
+
+Two design decisions worth pinning down up front, because they constrain everything else:
+
+- **Conflict.** Some table-level keys (`trace.conventions` lifted from `schema_url`, `metric.temporality`, ...) cannot represent the truth when a long-lived table sees rows from multiple sources. v1 records `mixed` or `unknown` rather than a fictitious single value. Downstream consumers must treat any single-valued semantic key as best-effort, not strong evidence.
+- **Update.** Semantic options are stamped at table creation. v1 does not specify an update path; promoting `metadata_quality` from `inferred` to `declared`, refreshing `resource.attributes_preserved`, or revising `trace.conventions` on later writes is deferred. If real usage shows update is needed, it lands as a separate RFC.
+
+## `information_schema.semantic_tables`
+
+A consumer's first SQL on connect:
+
+```sql
+SELECT table_catalog, table_schema, table_name, signal_type, source, pipeline
+FROM information_schema.semantic_tables;
+```
+
+returns one row per semantic-tagged table. The view exposes a stable set of core columns (`table_catalog`, `table_schema`, `table_name`, `signal_type`, `source`, `source_version`, `pipeline`) plus a `semantic_options` JSON column carrying the rest of the `greptime.semantic.*` keys verbatim. Future keys appear inside `semantic_options` without forcing a view-schema change; only widely-used keys are ever promoted to first-class columns.
+
+# Implementation Plan
+
+Four phases, each independently shippable.
+
+1. **Identity.** Stamp `signal_type` and `source` on every auto-create path. The OTLP paths already have natural injection points; Prom remote write is the one non-trivial path because metric-engine logical tables share physical storage (see Open Question 2).
+2. **Metric specifics.** Add type / unit / temporality / monotonic / metadata_quality / original_name at OTel metric and Prom RW ingestion sites; the data is already at hand inside the OTel translator.
+3. **Resource / scope lineage.** Record what the OTel-to-Prometheus translation kept and dropped.
+4. **`information_schema.semantic_tables` view + documentation** as a stable user-facing contract.
+
+# Relationship to OpenTelemetry standardisation
+
+OTel today standardises what producers emit and how data collectors are managed; the read side — what a backend exposes back to clients — is deliberately vendor turf. OTLP is one-way; OpAMP is agent management; OTEP-0243 (App Telemetry Schema) is producer-side; `schema_url` is producer-stated with no reverse. Adjacent precedents — Prometheus `/api/v1/metadata`, Loki labels API, Tempo tags, Jaeger services, ad-hoc MCP servers — are all vendor-specific.
+
+This is a real gap. The shape we propose locally (signal-agnostic, `schema_url`-aware, structured around a small vocabulary) is deliberately close to what a future upstream OTEP for a backend-catalog read API could look like, with Weaver's *Resolved Telemetry Schema* as the natural data model. We do not commit to driving such an OTEP here; we do commit to keeping the local shape close enough that a future upstream proposal does not force a breaking migration.
+
+# Alternatives
+
+- **New DDL syntax (`SEMANTIC trace WITH (...)`).** Cleaner-looking but non-standard and forces every client to learn it. The metadata is not interesting enough to justify a new keyword.
+- **Dedicated `_semantic` system table.** Doubles the storage path for what is static per-table KV and adds lifecycle questions (drop, backfill). A view over `table_options` covers the same access pattern.
+- **Column comments only.** Discovery (`WHERE signal_type = 'trace'`) becomes a full-text problem. Comments are good for column-level supplements, not for identity.
+- **Encode everything into the table name.** What we do today. Every new field becomes a new naming convention.
+
+# Open Questions
+
+1. **Namespace prefix.** `greptime.semantic.*` vs. bare `semantic.*`. v1 picks the vendored prefix; alias or migrate if a community standard later emerges.
+2. **Prom RW injection point.** Metric-engine logical tables share physical storage, so per-logical-table options need a hook that does not exist as cleanly as the OTLP trace branch. A short spike before Phase 1 lands for Prom RW.
+3. **Mixed-type metric tables.** When ingestion modes that pack multiple metric types into one table appear, `metric.type` migrates from table-level to row-level. v1 leaves a `metric.type = 'mixed'` marker and punts.
+4. **Stability surface.** Top-level keys (`signal_type`, `source`) are stable; sub-namespaces (`metric.*`, ...) are evolving until v1.0 of the layer is declared.
+
+# Future Work
+
+- **Cross-table relationships.** Paired trace/services tables, metric/info pairing, JOIN hints. Its own RFC.
+- **Producer SDK/client identity.** An optional `greptime.semantic.source.sdk` key recording the emitting client (e.g. `opentelemetry-go`, `opentelemetry-java`, `opentelemetry-collector`). Because a single table can receive data from multiple SDKs (a shared trace table is the common case), mixed producers collapse to `mixed`, following the same conflict rule as the table-level keys above.
+- **Backfill** for tables created before this feature shipped.
+- **Upstream proposal.** Carry the shape into a community proposal — likely an OTEP for an OTLP-Catalog read API plus an MCP binding — informed by Greptime's local usage data.
+
+# References
+
+OpenTelemetry:
+- [OTLP specification](https://opentelemetry.io/docs/specs/otlp/)
+- [OTel Schemas (`schema_url`)](https://opentelemetry.io/docs/specs/otel/schemas/)
+- [Semantic Conventions](https://opentelemetry.io/docs/specs/semconv/)
+- [OTEP-0243: App Telemetry Schema](https://github.com/open-telemetry/oteps/blob/main/text/0243-app-telemetry-schema-vision-roadmap.md)
+- [OpAMP specification](https://github.com/open-telemetry/opamp-spec/blob/main/specification.md)
+- [Weaver: Resolved Telemetry Schema](https://github.com/open-telemetry/weaver)
+- [2025 Stability Proposal](https://opentelemetry.io/blog/2025/stability-proposal-announcement/)
+
+Prometheus / OpenMetrics:
+- [Prometheus Remote Write 1.0](https://prometheus.io/docs/specs/prw/remote_write_spec/)
+- [Prometheus Remote Write 2.0](https://prometheus.io/docs/specs/prw/remote_write_spec_2_0/)
+- [Prometheus exposition formats](https://prometheus.io/docs/instrumenting/exposition_formats/)
+- [Prometheus HTTP API: `/api/v1/metadata`](https://prometheus.io/docs/prometheus/latest/querying/api/#querying-metric-metadata)
+
+Units and conventions:
+- [UCUM — Unified Code for Units of Measure](https://ucum.org/)
+
+GreptimeDB:
+- [OTLP ingestion guide](https://docs.greptime.com/user-guide/ingest-data/for-observability/opentelemetry/)
+- [Trace data model](https://docs.greptime.com/user-guide/traces/data-model/)
diff --git a/src/auth/src/permission.rs b/src/auth/src/permission.rs
index 88adfda633..8914635290 100644
--- a/src/auth/src/permission.rs
+++ b/src/auth/src/permission.rs
@@ -16,6 +16,7 @@ use std::fmt::Debug;
use std::sync::Arc;
use api::v1::greptime_request::Request;
+use api::v1::query_request::Query;
use common_telemetry::debug;
use sql::statements::statement::Statement;
@@ -42,10 +43,12 @@ impl<'a> PermissionReq<'a> {
/// Returns true if the permission request is for read operations.
pub fn is_readonly(&self) -> bool {
match self {
- PermissionReq::GrpcRequest(Request::Query(_))
- | PermissionReq::PromQuery
- | PermissionReq::LogQuery
- | PermissionReq::PromStoreRead => true,
+ PermissionReq::GrpcRequest(Request::Query(query_request)) => {
+ !matches!(query_request.query, Some(Query::InsertIntoPlan(_)))
+ }
+ PermissionReq::PromQuery | PermissionReq::LogQuery | PermissionReq::PromStoreRead => {
+ true
+ }
PermissionReq::SqlStatement(stmt) => stmt.is_readonly(),
PermissionReq::GrpcRequest(_)
@@ -196,4 +199,14 @@ mod tests {
assert!(matches!(read_result, PermissionResp::Reject));
assert!(matches!(write_result, PermissionResp::Allow));
}
+
+ #[test]
+ fn test_grpc_insert_into_plan_is_write_request() {
+ let request = Request::Query(api::v1::QueryRequest {
+ query: Some(Query::InsertIntoPlan(api::v1::InsertIntoPlan::default())),
+ });
+ let req = PermissionReq::GrpcRequest(&request);
+
+ assert!(req.is_write());
+ }
}
diff --git a/src/catalog/src/kvbackend/table_cache.rs b/src/catalog/src/kvbackend/table_cache.rs
index 42b3fbc74b..13f74a48c9 100644
--- a/src/catalog/src/kvbackend/table_cache.rs
+++ b/src/catalog/src/kvbackend/table_cache.rs
@@ -14,7 +14,9 @@
use std::sync::Arc;
-use common_meta::cache::{CacheContainer, Initializer, TableInfoCacheRef, TableNameCacheRef};
+use common_meta::cache::{
+ CacheContainer, InitStrategy, Initializer, TableInfoCacheRef, TableNameCacheRef,
+};
use common_meta::error::{Result as MetaResult, ValueNotExistSnafu};
use common_meta::instruction::CacheIdent;
use futures::future::BoxFuture;
@@ -38,7 +40,14 @@ pub fn new_table_cache(
) -> TableCache {
let init = init_factory(table_info_cache, table_name_cache);
- CacheContainer::new(name, cache, Box::new(invalidator), init, filter)
+ CacheContainer::with_strategy(
+ name,
+ cache,
+ Box::new(invalidator),
+ init,
+ filter,
+ InitStrategy::VersionChecked,
+ )
}
fn init_factory(
diff --git a/src/cli/src/data/export_v2/command.rs b/src/cli/src/data/export_v2/command.rs
index db0f576a4e..bb027bbef1 100644
--- a/src/cli/src/data/export_v2/command.rs
+++ b/src/cli/src/data/export_v2/command.rs
@@ -1077,7 +1077,9 @@ async fn verify_snapshot(storage: &OpenDalStorage) -> Result {
));
}
let data_files = storage.list_files_recursive("data/").await?;
- if let Some(path) = data_files.first() {
+ // Report the lexicographically smallest path so the message is stable
+ // regardless of listing order across backends.
+ if let Some(path) = data_files.iter().min() {
report.push_error(format!(
"Schema-only snapshot should not contain data files (found '{}')",
path
@@ -1103,75 +1105,113 @@ fn summarize_chunks(manifest: &Manifest) -> VerifyChunkSummary {
}
}
+/// A data file declared by a completed chunk that is expected to exist in storage.
+#[derive(Debug)]
+struct ChunkFile {
+ chunk_id: u32,
+ path: String,
+}
+
+/// Expected snapshot contents derived purely from the manifest (no object-store IO).
+///
+/// Separating planning from scanning makes it obvious which problems come from
+/// the manifest alone and which require comparing against actual storage.
+#[derive(Debug, Default)]
+struct VerifyPlan {
+ /// Valid data files declared by completed chunks; each must exist in storage.
+ files_to_check: Vec,
+ /// All syntactically-safe data paths declared by any chunk, regardless of
+ /// status. Used as the orphan-detection baseline so a listed-but-invalid
+ /// file is not also reported as unexpected.
+ claimed_data_files: HashSet,
+ /// Total data-file references in completed chunks (valid + invalid).
+ data_files_total: usize,
+ /// Problems detectable from the manifest alone.
+ problems: Vec,
+}
+
+/// Actual data files discovered under `data/` (the only object-store IO in
+/// chunk/data-file verification).
+#[derive(Debug)]
+struct VerifyDataScan {
+ existing_data_files: HashSet,
+}
+
+/// Result of reconciling the manifest plan against the storage scan.
+#[derive(Debug, Default)]
+struct VerifyOutcome {
+ data_files_total: usize,
+ data_files_verified: usize,
+ problems: Vec,
+}
+
async fn verify_chunks_and_data_files(
storage: &OpenDalStorage,
report: &mut VerifyReport,
) -> Result<()> {
- let existing_files: HashSet<_> = storage
- .list_files_recursive("data/")
- .await?
- .into_iter()
- .collect();
- let mut data_files_total = 0;
- let mut data_files_verified = 0;
- let mut problems = Vec::new();
- let mut seen_chunk_ids = HashSet::new();
- let mut claimed_data_files = HashSet::new();
+ let plan = build_verify_plan(&report.manifest);
+ let scan = scan_data_files(storage).await?;
+ let outcome = reconcile_plan_with_scan(plan, &scan);
- for chunk in &report.manifest.chunks {
+ report.data_files_total = outcome.data_files_total;
+ report.data_files_verified = outcome.data_files_verified;
+ report.problems.extend(outcome.problems);
+
+ Ok(())
+}
+
+/// Builds the expected-state plan from the manifest. Pure; performs no IO.
+fn build_verify_plan(manifest: &Manifest) -> VerifyPlan {
+ let mut plan = VerifyPlan::default();
+ let mut seen_chunk_ids = HashSet::new();
+
+ for chunk in &manifest.chunks {
if !seen_chunk_ids.insert(chunk.id) {
- problems.push(VerifyProblem {
+ plan.problems.push(VerifyProblem {
severity: VerifySeverity::Error,
message: format!("Chunk {}: duplicate chunk id", chunk.id),
});
}
for file in &chunk.files {
if let Some(path) = safe_manifest_data_file_path(file) {
- claimed_data_files.insert(path.to_string());
+ plan.claimed_data_files.insert(path.to_string());
}
}
match chunk.status {
ChunkStatus::Completed => {
if chunk.files.is_empty() {
- problems.push(VerifyProblem {
+ plan.problems.push(VerifyProblem {
severity: VerifySeverity::Error,
message: format!("Chunk {}: completed chunk has no data files", chunk.id),
});
continue;
}
- let allowed_prefixes = report
- .manifest
+ let allowed_prefixes = manifest
.schemas
.iter()
.map(|schema| data_dir_for_schema_chunk(schema, chunk.id))
.collect::>();
for file in &chunk.files {
- data_files_total += 1;
- let Some(path) = valid_manifest_data_file_path(file, &allowed_prefixes) else {
- problems.push(VerifyProblem {
+ plan.data_files_total += 1;
+ match valid_manifest_data_file_path(file, &allowed_prefixes) {
+ Some(path) => plan.files_to_check.push(ChunkFile {
+ chunk_id: chunk.id,
+ path: path.to_string(),
+ }),
+ None => plan.problems.push(VerifyProblem {
severity: VerifySeverity::Error,
message: format!(
"Chunk {}: invalid data file path '{}'",
chunk.id, file
),
- });
- continue;
- };
-
- if existing_files.contains(path) {
- data_files_verified += 1;
- } else {
- problems.push(VerifyProblem {
- severity: VerifySeverity::Error,
- message: format!("Chunk {}: missing file '{}'", chunk.id, path),
- });
+ }),
}
}
}
ChunkStatus::Skipped => {
if !chunk.files.is_empty() {
- problems.push(VerifyProblem {
+ plan.problems.push(VerifyProblem {
severity: VerifySeverity::Error,
message: format!(
"Chunk {}: skipped chunk should not list data files",
@@ -1181,20 +1221,20 @@ async fn verify_chunks_and_data_files(
}
}
ChunkStatus::Pending => {
- problems.push(VerifyProblem {
+ plan.problems.push(VerifyProblem {
severity: VerifySeverity::Error,
message: format!("Chunk {}: status is 'pending'", chunk.id),
});
}
ChunkStatus::InProgress => {
- problems.push(VerifyProblem {
+ plan.problems.push(VerifyProblem {
severity: VerifySeverity::Error,
message: format!("Chunk {}: status is 'in_progress'", chunk.id),
});
}
ChunkStatus::Failed => {
let reason = chunk.error.as_deref().unwrap_or("unknown error");
- problems.push(VerifyProblem {
+ plan.problems.push(VerifyProblem {
severity: VerifySeverity::Error,
message: format!("Chunk {}: status is 'failed' (error: {})", chunk.id, reason),
});
@@ -1202,20 +1242,60 @@ async fn verify_chunks_and_data_files(
}
}
- for path in &existing_files {
- if !claimed_data_files.contains(path) {
+ plan
+}
+
+/// Lists all data files under `data/`. This is the only object-store IO in
+/// chunk/data-file verification.
+async fn scan_data_files(storage: &OpenDalStorage) -> Result {
+ let existing_data_files = storage
+ .list_files_recursive("data/")
+ .await?
+ .into_iter()
+ .collect();
+ Ok(VerifyDataScan {
+ existing_data_files,
+ })
+}
+
+/// Reconciles the manifest plan against the storage scan. Pure; performs no IO.
+///
+/// Emits missing-file problems for expected files absent from storage and
+/// unexpected-file problems for storage files no chunk claims. Unexpected files
+/// are sorted by path so output is deterministic regardless of listing order.
+fn reconcile_plan_with_scan(plan: VerifyPlan, scan: &VerifyDataScan) -> VerifyOutcome {
+ let mut problems = plan.problems;
+ let mut data_files_verified = 0;
+
+ for file in &plan.files_to_check {
+ if scan.existing_data_files.contains(&file.path) {
+ data_files_verified += 1;
+ } else {
problems.push(VerifyProblem {
severity: VerifySeverity::Error,
- message: format!("Unexpected data file '{}' is not listed in manifest", path),
+ message: format!("Chunk {}: missing file '{}'", file.chunk_id, file.path),
});
}
}
- report.data_files_total = data_files_total;
- report.data_files_verified = data_files_verified;
- report.problems.extend(problems);
+ let mut orphans: Vec<&String> = scan
+ .existing_data_files
+ .iter()
+ .filter(|path| !plan.claimed_data_files.contains(*path))
+ .collect();
+ orphans.sort();
+ for path in orphans {
+ problems.push(VerifyProblem {
+ severity: VerifySeverity::Error,
+ message: format!("Unexpected data file '{}' is not listed in manifest", path),
+ });
+ }
- Ok(())
+ VerifyOutcome {
+ data_files_total: plan.data_files_total,
+ data_files_verified,
+ problems,
+ }
}
fn valid_manifest_data_file_path<'a>(
@@ -2294,6 +2374,90 @@ mod tests {
);
}
+ #[test]
+ fn test_build_verify_plan_classifies_chunks_without_io() {
+ let mut manifest = test_manifest(
+ chrono::Utc.with_ymd_and_hms(2026, 1, 1, 0, 0, 0).unwrap(),
+ false,
+ true,
+ );
+ // test_manifest(complete) gives: chunk 1 completed (1 file), chunk 2 skipped.
+ let mut failed = ChunkMeta::new(3, TimeRange::unbounded());
+ failed.mark_failed("boom".to_string());
+ manifest.chunks.push(failed);
+ manifest
+ .chunks
+ .push(ChunkMeta::new(4, TimeRange::unbounded()));
+
+ let plan = build_verify_plan(&manifest);
+
+ assert_eq!(plan.files_to_check.len(), 1);
+ assert_eq!(plan.files_to_check[0].chunk_id, 1);
+ assert_eq!(plan.files_to_check[0].path, "data/public/1/file.parquet");
+ assert_eq!(plan.data_files_total, 1);
+ assert!(
+ plan.claimed_data_files
+ .contains("data/public/1/file.parquet")
+ );
+ assert_eq!(plan.problems.len(), 2);
+ assert!(
+ plan.problems
+ .iter()
+ .any(|problem| problem.message.contains("status is 'failed'"))
+ );
+ assert!(
+ plan.problems
+ .iter()
+ .any(|problem| problem.message.contains("status is 'pending'"))
+ );
+ }
+
+ #[tokio::test]
+ async fn test_verify_snapshot_produces_deterministic_problem_output() {
+ let dir = tempdir().unwrap();
+ let manifest = test_manifest(
+ chrono::Utc.with_ymd_and_hms(2026, 1, 1, 0, 0, 0).unwrap(),
+ false,
+ true,
+ );
+ write_root_manifest(dir.path(), manifest);
+ write_snapshot_file(dir.path(), "schema/schemas.json", b"[]");
+ write_default_ddl_files(dir.path());
+ write_snapshot_file(dir.path(), "data/public/1/file.parquet", b"data");
+ // Many orphan files under a known chunk prefix to stress ordering.
+ for i in 0..50 {
+ write_snapshot_file(
+ dir.path(),
+ &format!("data/public/1/orphan_{:02}.parquet", i),
+ b"x",
+ );
+ }
+
+ let storage = file_storage_for_dir(dir.path());
+ let messages = |report: &VerifyReport| {
+ report
+ .problems
+ .iter()
+ .map(|problem| problem.message.clone())
+ .collect::>()
+ };
+ let first = messages(&verify_snapshot(&storage).await.unwrap());
+ let second = messages(&verify_snapshot(&storage).await.unwrap());
+
+ // Output is identical across runs despite HashSet-based scanning.
+ assert_eq!(first, second);
+
+ let orphans = first
+ .iter()
+ .filter(|message| message.contains("Unexpected data file"))
+ .cloned()
+ .collect::>();
+ assert_eq!(orphans.len(), 50);
+ let mut sorted = orphans.clone();
+ sorted.sort();
+ assert_eq!(orphans, sorted);
+ }
+
fn write_test_manifest(root: &std::path::Path, dir: &str, manifest: Manifest) {
let snapshot_dir = root.join(dir);
std::fs::create_dir_all(&snapshot_dir).unwrap();
diff --git a/src/cmd/src/datanode.rs b/src/cmd/src/datanode.rs
index 3c106bd43f..7f2057bc97 100644
--- a/src/cmd/src/datanode.rs
+++ b/src/cmd/src/datanode.rs
@@ -79,7 +79,7 @@ impl App for Instance {
}
async fn start(&mut self) -> Result<()> {
- plugins::start_datanode_plugins(self.datanode.plugins())
+ plugins::start_datanode_plugins(&self.datanode)
.await
.context(StartDatanodeSnafu)?;
diff --git a/src/cmd/src/datanode/scanbench.rs b/src/cmd/src/datanode/scanbench.rs
index b26705991c..b2a715ad31 100644
--- a/src/cmd/src/datanode/scanbench.rs
+++ b/src/cmd/src/datanode/scanbench.rs
@@ -524,6 +524,7 @@ impl ScanbenchCommand {
options: HashMap::default(),
skip_wal_replay: !self.enable_wal,
checkpoint: None,
+ requirements: Default::default(),
};
engine
diff --git a/src/cmd/src/flownode.rs b/src/cmd/src/flownode.rs
index 6228cbd3f3..32d9070ec6 100644
--- a/src/cmd/src/flownode.rs
+++ b/src/cmd/src/flownode.rs
@@ -90,7 +90,7 @@ impl App for Instance {
}
async fn start(&mut self) -> Result<()> {
- plugins::start_flownode_plugins(self.flownode.flow_engine().plugins().clone())
+ plugins::start_flownode_plugins(&self.flownode)
.await
.context(StartFlownodeSnafu)?;
diff --git a/src/cmd/src/frontend.rs b/src/cmd/src/frontend.rs
index da2c111e7c..cbc07d10e9 100644
--- a/src/cmd/src/frontend.rs
+++ b/src/cmd/src/frontend.rs
@@ -95,8 +95,7 @@ impl App for Instance {
}
async fn start(&mut self) -> Result<()> {
- let plugins = self.frontend.instance.plugins().clone();
- plugins::start_frontend_plugins(plugins)
+ plugins::start_frontend_plugins(&self.frontend.instance)
.await
.context(error::StartFrontendSnafu)?;
diff --git a/src/cmd/src/metasrv.rs b/src/cmd/src/metasrv.rs
index bf3cb2f5e7..e30b115ada 100644
--- a/src/cmd/src/metasrv.rs
+++ b/src/cmd/src/metasrv.rs
@@ -68,7 +68,7 @@ impl App for Instance {
}
async fn start(&mut self) -> Result<()> {
- plugins::start_metasrv_plugins(self.instance.plugins())
+ plugins::start_metasrv_plugins(&self.instance)
.await
.context(StartMetaServerSnafu)?;
diff --git a/src/cmd/src/standalone.rs b/src/cmd/src/standalone.rs
index e0f2c673ff..7d99e99554 100644
--- a/src/cmd/src/standalone.rs
+++ b/src/cmd/src/standalone.rs
@@ -164,7 +164,7 @@ impl App for Instance {
.start(self.leader_services_context.clone())
.await?;
- plugins::start_frontend_plugins(self.frontend.instance.plugins().clone())
+ plugins::start_frontend_plugins(&self.frontend.instance)
.await
.context(error::StartFrontendSnafu)?;
diff --git a/src/cmd/tests/load_config_test.rs b/src/cmd/tests/load_config_test.rs
index 6cffcd67c2..cee29e4456 100644
--- a/src/cmd/tests/load_config_test.rs
+++ b/src/cmd/tests/load_config_test.rs
@@ -114,6 +114,7 @@ fn test_load_frontend_example_config() {
component: FrontendOptions {
default_timezone: Some("UTC".to_string()),
default_column_prefix: Some("greptime".to_string()),
+ auto_create_table: true,
meta_client: Some(MetaClientOptions {
metasrv_addrs: vec!["127.0.0.1:3002".to_string()],
timeout: Duration::from_secs(3),
@@ -267,6 +268,7 @@ fn test_load_standalone_example_config() {
component: StandaloneOptions {
default_timezone: Some("UTC".to_string()),
default_column_prefix: Some("greptime".to_string()),
+ auto_create_table: true,
wal: DatanodeWalConfig::RaftEngine(RaftEngineConfig {
dir: Some(format!("{}/{}", DEFAULT_DATA_HOME, WAL_DIR)),
sync_period: Some(Duration::from_secs(10)),
diff --git a/src/common/datasource/Cargo.toml b/src/common/datasource/Cargo.toml
index 470b5371f7..8b4053db2f 100644
--- a/src/common/datasource/Cargo.toml
+++ b/src/common/datasource/Cargo.toml
@@ -33,6 +33,7 @@ datatypes.workspace = true
futures.workspace = true
lazy_static.workspace = true
object-store.workspace = true
+object_store_opendal.workspace = true
orc-rust = { version = "0.8", default-features = false, features = ["async"] }
parquet.workspace = true
paste.workspace = true
diff --git a/src/common/datasource/src/file_format.rs b/src/common/datasource/src/file_format.rs
index a6a358c9e4..d9d7b8b648 100644
--- a/src/common/datasource/src/file_format.rs
+++ b/src/common/datasource/src/file_format.rs
@@ -61,6 +61,7 @@ pub const FORMAT_COMPRESSION_TYPE: &str = "compression_type";
pub const FORMAT_DELIMITER: &str = "delimiter";
pub const FORMAT_SCHEMA_INFER_MAX_RECORD: &str = "schema_infer_max_record";
pub const FORMAT_HAS_HEADER: &str = "has_header";
+pub const FORMAT_SKIP_BAD_RECORDS: &str = "skip_bad_records";
pub const FORMAT_TYPE: &str = "format";
pub const FILE_PATTERN: &str = "pattern";
pub const TIMESTAMP_FORMAT: &str = "timestamp_format";
@@ -316,7 +317,7 @@ pub async fn file_to_stream(
.with_file_compression_type(df_compression)
.build();
- let store = Arc::new(object_store::compat::OpendalStore::new(store.clone()));
+ let store = Arc::new(object_store_opendal::OpendalStore::new(store.clone()));
let file_opener = config.file_source().create_file_opener(store, &config, 0)?;
let stream = FileStream::new(&config, 0, file_opener, &ExecutionPlanMetricsSet::new())?;
diff --git a/src/common/datasource/src/file_format/csv.rs b/src/common/datasource/src/file_format/csv.rs
index 77ea553f35..2b39051b48 100644
--- a/src/common/datasource/src/file_format/csv.rs
+++ b/src/common/datasource/src/file_format/csv.rs
@@ -13,15 +13,24 @@
// limitations under the License.
use std::collections::HashMap;
+use std::io;
use std::str::FromStr;
+use std::sync::Arc;
+use std::task::Poll;
use arrow::csv::reader::Format;
use arrow::csv::{self, WriterBuilder};
+use arrow::error::ArrowError;
use arrow::record_batch::RecordBatch;
-use arrow_schema::Schema;
+use arrow_schema::{Schema, SchemaRef};
use async_trait::async_trait;
+use bytes::{Buf, Bytes};
use common_runtime;
+use common_telemetry::warn;
use datafusion::physical_plan::SendableRecordBatchStream;
+use datafusion::physical_plan::stream::RecordBatchStreamAdapter;
+use futures::StreamExt;
+use futures::stream::BoxStream;
use object_store::ObjectStore;
use snafu::ResultExt;
use tokio_util::compat::FuturesAsyncReadCompatExt;
@@ -34,9 +43,12 @@ use crate::file_format::{self, FileFormat, stream_to_file};
use crate::share_buffer::SharedBuffer;
use crate::util::normalize_infer_schema;
+const SKIP_BAD_RECORDS_BATCH_SIZE: usize = 1;
+
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct CsvFormat {
pub has_header: bool,
+ pub skip_bad_records: bool,
pub delimiter: u8,
pub schema_infer_max_record: Option,
pub compression_type: CompressionType,
@@ -76,13 +88,11 @@ impl TryFrom<&HashMap> for CsvFormat {
})?);
};
if let Some(has_header) = value.get(file_format::FORMAT_HAS_HEADER) {
- format.has_header = has_header.parse().map_err(|_| {
- error::ParseFormatSnafu {
- key: file_format::FORMAT_HAS_HEADER,
- value: has_header,
- }
- .build()
- })?;
+ format.has_header = parse_bool(file_format::FORMAT_HAS_HEADER, has_header)?;
+ };
+ if let Some(skip_bad_records) = value.get(file_format::FORMAT_SKIP_BAD_RECORDS) {
+ format.skip_bad_records =
+ parse_bool(file_format::FORMAT_SKIP_BAD_RECORDS, skip_bad_records)?;
};
if let Some(timestamp_format) = value.get(file_format::TIMESTAMP_FORMAT) {
format.timestamp_format = Some(timestamp_format.clone());
@@ -97,10 +107,17 @@ impl TryFrom<&HashMap> for CsvFormat {
}
}
+fn parse_bool(key: &'static str, value: &str) -> Result {
+ value
+ .parse()
+ .map_err(|_| error::ParseFormatSnafu { key, value }.build())
+}
+
impl Default for CsvFormat {
fn default() -> Self {
Self {
has_header: true,
+ skip_bad_records: false,
delimiter: b',',
schema_infer_max_record: Some(file_format::DEFAULT_SCHEMA_INFER_MAX_RECORD),
compression_type: CompressionType::Uncompressed,
@@ -189,10 +206,136 @@ impl DfRecordBatchEncoder for csv::Writer {
}
}
+/// Builds a CSV stream that can skip selected record-level parse/cast errors.
+///
+/// This recovery path intentionally uses one-record batches. It is slower than
+/// normal CSV scanning, but keeps each parse/cast failure isolated to a single
+/// record. Arrow's CSV decoder clears buffered rows before type parsing, so a
+/// failed multi-row flush cannot be safely retried row by row without replaying
+/// input bytes.
+pub async fn tolerant_csv_stream(
+ store: &ObjectStore,
+ path: &str,
+ schema: SchemaRef,
+ projection: Vec,
+ format: &CsvFormat,
+) -> Result {
+ let meta = store
+ .stat(path)
+ .await
+ .context(error::ReadObjectSnafu { path })?;
+
+ let reader = store
+ .reader(path)
+ .await
+ .context(error::ReadObjectSnafu { path })?
+ .into_bytes_stream(0..meta.content_length())
+ .await
+ .context(error::ReadObjectSnafu { path })?;
+
+ let reader = format.compression_type.convert_stream(reader).boxed();
+ tolerant_csv_stream_from_reader(
+ reader,
+ path,
+ schema,
+ projection,
+ format.has_header,
+ format.delimiter,
+ )
+}
+
+fn tolerant_csv_stream_from_reader(
+ reader: BoxStream<'static, io::Result>,
+ path: &str,
+ schema: SchemaRef,
+ projection: Vec,
+ has_header: bool,
+ delimiter: u8,
+) -> Result {
+ let projected_schema = Arc::new(
+ schema
+ .project(&projection)
+ .context(error::InferSchemaSnafu)?,
+ );
+ let mut decoder = csv::ReaderBuilder::new(schema)
+ .with_header(has_header)
+ .with_delimiter(delimiter)
+ .with_batch_size(SKIP_BAD_RECORDS_BATCH_SIZE)
+ .with_projection(projection)
+ .build_decoder();
+
+ let path = path.to_string();
+ let mut upstream = reader.fuse();
+ let mut buffered = Bytes::new();
+ let mut input_finished = false;
+ let stream = futures::stream::poll_fn(move |cx| {
+ loop {
+ while !input_finished {
+ if buffered.is_empty() {
+ match futures::ready!(upstream.poll_next_unpin(cx)) {
+ Some(Ok(bytes)) if bytes.is_empty() => continue,
+ Some(Ok(bytes)) => buffered = bytes,
+ Some(Err(error)) => return Poll::Ready(Some(Err(error.into()))),
+ None => input_finished = true,
+ }
+ }
+
+ let decoded = decoder.decode(buffered.as_ref())?;
+ if decoded > 0 {
+ buffered.advance(decoded);
+ continue;
+ }
+
+ if decoder.capacity() == 0 || input_finished {
+ break;
+ }
+
+ if buffered.is_empty() {
+ continue;
+ }
+
+ return Poll::Ready(Some(Err(ArrowError::ParseError(
+ "CSV decoder made no progress while input bytes remain".to_string(),
+ ))));
+ }
+
+ match decoder.flush() {
+ Ok(Some(batch)) => return Poll::Ready(Some(Ok(batch))),
+ Ok(None) if input_finished => return Poll::Ready(None),
+ Ok(None) => continue,
+ Err(error) if is_skippable_arrow_error(&error) => {
+ warn!(
+ "Skipping bad CSV record while copying from {}: {}",
+ path, error
+ );
+ }
+ Err(error) => return Poll::Ready(Some(Err(error))),
+ }
+ }
+ })
+ .map(|result: std::result::Result| result.map_err(Into::into));
+
+ Ok(Box::pin(RecordBatchStreamAdapter::new(
+ projected_schema,
+ stream,
+ )))
+}
+
+pub fn is_skippable_arrow_error(error: &ArrowError) -> bool {
+ matches!(
+ error,
+ ArrowError::ParseError(_)
+ | ArrowError::CastError(_)
+ | ArrowError::ComputeError(_)
+ | ArrowError::InvalidArgumentError(_)
+ )
+}
+
#[cfg(test)]
mod tests {
use std::sync::Arc;
+ use arrow_schema::{DataType, Field};
use common_recordbatch::adapter::DfRecordBatchStreamAdapter;
use common_recordbatch::{RecordBatch, RecordBatches};
use common_test_util::find_workspace_path;
@@ -205,7 +348,7 @@ mod tests {
use super::*;
use crate::file_format::{
FORMAT_COMPRESSION_TYPE, FORMAT_DELIMITER, FORMAT_HAS_HEADER,
- FORMAT_SCHEMA_INFER_MAX_RECORD, FileFormat, file_to_stream,
+ FORMAT_SCHEMA_INFER_MAX_RECORD, FORMAT_SKIP_BAD_RECORDS, FileFormat, file_to_stream,
};
use crate::test_util::{format_schema, test_store};
@@ -331,11 +474,29 @@ mod tests {
schema_infer_max_record: Some(2000),
delimiter: b'\t',
has_header: false,
+ skip_bad_records: false,
timestamp_format: None,
time_format: None,
date_format: None
}
);
+
+ let map = HashMap::from([(FORMAT_SKIP_BAD_RECORDS.to_string(), "true".to_string())]);
+ let format = CsvFormat::try_from(&map).unwrap();
+
+ assert_eq!(
+ format,
+ CsvFormat {
+ skip_bad_records: true,
+ ..CsvFormat::default()
+ }
+ );
+ }
+
+ #[test]
+ fn test_try_from_rejects_invalid_bool_options() {
+ let map = HashMap::from([(FORMAT_SKIP_BAD_RECORDS.to_string(), "yes".to_string())]);
+ assert!(CsvFormat::try_from(&map).is_err());
}
#[tokio::test]
@@ -496,4 +657,63 @@ mod tests {
assert_eq!(expected, pretty_print);
}
}
+
+ #[tokio::test]
+ async fn test_tolerant_csv_stream_continues_after_parse_error() {
+ let temp_dir = common_test_util::temp_dir::create_temp_dir("test_tolerant_csv_stream");
+ let csv_file_path = temp_dir.path().join("input.csv");
+ std::fs::write(
+ &csv_file_path,
+ "id,name,value\n1,Alice,10.5\nbad,Bad,20.0\nworse,Bad,21.0\n2,Bob,30.5",
+ )
+ .unwrap();
+
+ let store = test_store("/");
+ let schema = Arc::new(arrow_schema::Schema::new(vec![
+ Field::new("id", DataType::UInt32, false),
+ Field::new("name", DataType::Utf8, false),
+ Field::new("value", DataType::Float64, false),
+ ]));
+ let path = csv_file_path.to_str().unwrap();
+
+ let stream =
+ tolerant_csv_stream(&store, path, schema, vec![0, 1, 2], &CsvFormat::default())
+ .await
+ .unwrap();
+ let batches = stream.try_collect::>().await.unwrap();
+ let pretty_print = arrow::util::pretty::pretty_format_batches(&batches)
+ .unwrap()
+ .to_string();
+ let expected = r#"+----+-------+-------+
+| id | name | value |
++----+-------+-------+
+| 1 | Alice | 10.5 |
+| 2 | Bob | 30.5 |
++----+-------+-------+"#;
+ assert_eq!(expected, pretty_print);
+ }
+
+ #[tokio::test]
+ async fn test_tolerant_csv_stream_fails_on_structural_csv_error() {
+ let temp_dir =
+ common_test_util::temp_dir::create_temp_dir("test_tolerant_csv_stream_csv_error");
+ let csv_file_path = temp_dir.path().join("input.csv");
+ std::fs::write(&csv_file_path, "id,name,value\n1,Alice,10.5\n2,Bob\n").unwrap();
+
+ let store = test_store("/");
+ let schema = Arc::new(arrow_schema::Schema::new(vec![
+ Field::new("id", DataType::UInt32, false),
+ Field::new("name", DataType::Utf8, false),
+ Field::new("value", DataType::Float64, false),
+ ]));
+ let path = csv_file_path.to_str().unwrap();
+
+ let stream =
+ tolerant_csv_stream(&store, path, schema, vec![0, 1, 2], &CsvFormat::default())
+ .await
+ .unwrap();
+ let error = stream.try_collect::>().await.unwrap_err();
+
+ assert!(error.to_string().contains("incorrect number of fields"));
+ }
}
diff --git a/src/common/datasource/src/file_format/tests.rs b/src/common/datasource/src/file_format/tests.rs
index 93ab3b4409..a925f73d48 100644
--- a/src/common/datasource/src/file_format/tests.rs
+++ b/src/common/datasource/src/file_format/tests.rs
@@ -44,7 +44,7 @@ struct Test<'a> {
impl Test<'_> {
async fn run(self, store: &ObjectStore) {
- let store = Arc::new(object_store::compat::OpendalStore::new(store.clone()));
+ let store = Arc::new(object_store_opendal::OpendalStore::new(store.clone()));
let file_opener = self
.file_source
.create_file_opener(store, &self.config, 0)
diff --git a/src/common/datasource/src/object_store/oss.rs b/src/common/datasource/src/object_store/oss.rs
index aded3eca2c..aacc17ac5e 100644
--- a/src/common/datasource/src/object_store/oss.rs
+++ b/src/common/datasource/src/object_store/oss.rs
@@ -27,12 +27,14 @@ const ACCESS_KEY_ID: &str = "access_key_id";
const ACCESS_KEY_SECRET: &str = "access_key_secret";
const ROOT: &str = "root";
const ALLOW_ANONYMOUS: &str = "allow_anonymous";
+const SKIP_SIGNATURE: &str = "skip_signature";
/// Check if the key is supported in OSS configuration.
pub fn is_supported_in_oss(key: &str) -> bool {
[
ROOT,
ALLOW_ANONYMOUS,
+ SKIP_SIGNATURE,
BUCKET,
ENDPOINT,
ACCESS_KEY_ID,
@@ -61,18 +63,23 @@ pub fn build_oss_backend(
builder = builder.access_key_secret(access_key_secret);
}
- if let Some(allow_anonymous) = connection.get(ALLOW_ANONYMOUS) {
- let allow = allow_anonymous.as_str().parse::().map_err(|e| {
+ if let Some((key, value)) = connection
+ .get(SKIP_SIGNATURE)
+ .map(|value| (SKIP_SIGNATURE, value))
+ .or_else(|| {
+ connection
+ .get(ALLOW_ANONYMOUS)
+ .map(|value| (ALLOW_ANONYMOUS, value))
+ })
+ {
+ let skip_signature = value.as_str().parse::().map_err(|e| {
error::InvalidConnectionSnafu {
- msg: format!(
- "failed to parse the option {}={}, {}",
- ALLOW_ANONYMOUS, allow_anonymous, e
- ),
+ msg: format!("failed to parse the option {}={}, {}", key, value, e),
}
.build()
})?;
- if allow {
- builder = builder.allow_anonymous();
+ if skip_signature {
+ builder = builder.skip_signature();
}
}
@@ -93,6 +100,7 @@ mod tests {
fn test_is_supported_in_oss() {
assert!(is_supported_in_oss(ROOT));
assert!(is_supported_in_oss(ALLOW_ANONYMOUS));
+ assert!(is_supported_in_oss(SKIP_SIGNATURE));
assert!(is_supported_in_oss(BUCKET));
assert!(is_supported_in_oss(ENDPOINT));
assert!(is_supported_in_oss(ACCESS_KEY_ID));
diff --git a/src/common/datasource/src/test_util.rs b/src/common/datasource/src/test_util.rs
index 0a13d9c6e8..ea2b0c768c 100644
--- a/src/common/datasource/src/test_util.rs
+++ b/src/common/datasource/src/test_util.rs
@@ -103,7 +103,7 @@ pub async fn setup_stream_to_json_test(origin_path: &str, threshold: impl Fn(usi
test_util::TEST_BATCH_SIZE,
schema.clone(),
FileCompressionType::UNCOMPRESSED,
- Arc::new(object_store::compat::OpendalStore::new(store.clone())),
+ Arc::new(object_store_opendal::OpendalStore::new(store.clone())),
true,
);
@@ -157,7 +157,7 @@ pub async fn setup_stream_to_csv_test(
let csv_opener = csv_source
.create_file_opener(
- Arc::new(object_store::compat::OpendalStore::new(store.clone())),
+ Arc::new(object_store_opendal::OpendalStore::new(store.clone())),
&config,
0,
)
diff --git a/src/common/meta/src/cache.rs b/src/common/meta/src/cache.rs
index f16290937a..c26a0fab76 100644
--- a/src/common/meta/src/cache.rs
+++ b/src/common/meta/src/cache.rs
@@ -17,7 +17,7 @@ mod flow;
mod registry;
mod table;
-pub use container::{CacheContainer, Initializer, Invalidator, TokenFilter};
+pub use container::{CacheContainer, InitStrategy, Initializer, Invalidator, TokenFilter};
pub use flow::{TableFlownodeSetCache, TableFlownodeSetCacheRef, new_table_flownode_set_cache};
pub use registry::{
CacheRegistry, CacheRegistryBuilder, CacheRegistryRef, LayeredCacheRegistry,
diff --git a/src/common/meta/src/cache/flow/table_flownode.rs b/src/common/meta/src/cache/flow/table_flownode.rs
index ebe3664202..4d3513a21d 100644
--- a/src/common/meta/src/cache/flow/table_flownode.rs
+++ b/src/common/meta/src/cache/flow/table_flownode.rs
@@ -210,7 +210,7 @@ mod tests {
use crate::cache::flow::table_flownode::{FlowIdent, new_table_flownode_set_cache};
use crate::instruction::{CacheIdent, CreateFlow, DropFlow};
use crate::key::flow::FlowMetadataManager;
- use crate::key::flow::flow_info::FlowInfoValue;
+ use crate::key::flow::flow_info::{FlowInfoValue, FlowStatus};
use crate::key::flow::flow_route::FlowRouteValue;
use crate::kv_backend::memory::MemoryKvBackend;
use crate::peer::Peer;
@@ -242,11 +242,14 @@ mod tests {
catalog_name: DEFAULT_CATALOG_NAME.to_string(),
query_context: None,
flow_name: "my_flow".to_string(),
+ all_source_table_names: vec![],
+ unresolved_source_table_names: vec![],
raw_sql: "sql".to_string(),
expire_after: Some(300),
eval_interval_secs: None,
comment: "comment".to_string(),
options: Default::default(),
+ status: FlowStatus::Active,
created_time: chrono::Utc::now(),
updated_time: chrono::Utc::now(),
},
diff --git a/src/common/meta/src/ddl/create_flow.rs b/src/common/meta/src/ddl/create_flow.rs
index 7120e50425..8a419176c9 100644
--- a/src/common/meta/src/ddl/create_flow.rs
+++ b/src/common/meta/src/ddl/create_flow.rs
@@ -14,7 +14,7 @@
mod metadata;
-use std::collections::BTreeMap;
+use std::collections::{BTreeMap, HashMap};
use std::fmt;
use api::v1::ExpireAfter;
@@ -34,13 +34,14 @@ use serde::{Deserialize, Serialize};
use snafu::{ResultExt, ensure};
use strum::AsRefStr;
use table::metadata::TableId;
+use table::table_name::TableName;
use crate::cache_invalidator::Context;
use crate::ddl::DdlContext;
use crate::ddl::utils::{add_peer_context_if_needed, map_to_procedure_error};
use crate::error::{self, Result, UnexpectedSnafu};
use crate::instruction::{CacheIdent, CreateFlow, DropFlow};
-use crate::key::flow::flow_info::FlowInfoValue;
+use crate::key::flow::flow_info::{FlowInfoValue, FlowStatus};
use crate::key::flow::flow_route::FlowRouteValue;
use crate::key::table_name::TableNameKey;
use crate::key::{DeserializedValueWithBytes, FlowId, FlowPartitionId};
@@ -67,6 +68,7 @@ impl CreateFlowProcedure {
flow_id: None,
peers: vec![],
source_table_ids: vec![],
+ unresolved_source_table_names: vec![],
flow_context: query_context.into(), // Convert to FlowQueryContext
state: CreateFlowState::Prepare,
prev_flow_info_value: None,
@@ -89,6 +91,8 @@ impl CreateFlowProcedure {
let create_if_not_exists = self.data.task.create_if_not_exists;
let or_replace = self.data.task.or_replace;
+ validate_flow_options(&self.data.task)?;
+
let flow_name_value = self
.context
.flow_metadata_manager
@@ -167,6 +171,21 @@ impl CreateFlowProcedure {
}
self.collect_source_tables().await?;
+ ensure!(
+ self.data.unresolved_source_table_names.is_empty()
+ || defer_on_missing_source(&self.data.task)?,
+ error::UnsupportedSnafu {
+ operation: format!(
+ "Create flow with missing source tables requires WITH ('{DEFER_ON_MISSING_SOURCE_KEY}'='true'): {}",
+ self.data
+ .unresolved_source_table_names
+ .iter()
+ .map(ToString::to_string)
+ .join(", ")
+ )
+ }
+ );
+ self.ensure_supported_replace_transition()?;
// Validate that source and sink tables are not the same
let sink_table_name = &self.data.task.sink_table_name;
@@ -189,13 +208,38 @@ impl CreateFlowProcedure {
if self.data.flow_id.is_none() {
self.allocate_flow_id().await?;
}
- self.data.state = CreateFlowState::CreateFlows;
- // determine flow type
self.data.flow_type = Some(get_flow_type_from_options(&self.data.task)?);
+ self.data.state = if self.data.is_pending() {
+ self.data.peers.clear();
+ CreateFlowState::CreateMetadata
+ } else {
+ CreateFlowState::CreateFlows
+ };
+
Ok(Status::executing(true))
}
+ fn ensure_supported_replace_transition(&self) -> Result<()> {
+ if !self.data.task.or_replace {
+ return Ok(());
+ }
+
+ let Some(prev_flow_info) = self.data.prev_flow_info_value.as_ref() else {
+ return Ok(());
+ };
+ let prev_pending = prev_flow_info.get_inner_ref().is_pending();
+ let new_pending = self.data.is_pending();
+ ensure!(
+ prev_pending == new_pending,
+ error::UnsupportedSnafu {
+ operation: "Replacing between pending and active flow states is not supported yet"
+ }
+ );
+
+ Ok(())
+ }
+
async fn on_flownode_create_flows(&mut self) -> Result {
// Safety: must be allocated.
let mut create_flow = Vec::with_capacity(self.data.peers.len());
@@ -365,6 +409,63 @@ pub fn get_flow_type_from_options(flow_task: &CreateFlowTask) -> Result Result {
+ flow_task
+ .flow_options
+ .get(DEFER_ON_MISSING_SOURCE_KEY)
+ .map(|value| {
+ value
+ .trim()
+ .to_ascii_lowercase()
+ .parse::()
+ .map_err(|_| {
+ error::UnexpectedSnafu {
+ err_msg: format!(
+ "Invalid flow option '{DEFER_ON_MISSING_SOURCE_KEY}': {value}"
+ ),
+ }
+ .build()
+ })
+ })
+ .transpose()
+ .map(|value| value.unwrap_or(false))
+}
+
+pub fn validate_flow_options(flow_task: &CreateFlowTask) -> Result<()> {
+ for key in flow_task.flow_options.keys() {
+ match key.as_str() {
+ DEFER_ON_MISSING_SOURCE_KEY
+ | FLOW_EXPERIMENTAL_ENABLE_INCREMENTAL_READ_KEY
+ | FlowType::FLOW_TYPE_KEY => {}
+ unknown => {
+ return UnexpectedSnafu {
+ err_msg: format!(
+ "Unknown flow option '{unknown}', supported user options: {DEFER_ON_MISSING_SOURCE_KEY}, {FLOW_EXPERIMENTAL_ENABLE_INCREMENTAL_READ_KEY}"
+ ),
+ }
+ .fail();
+ }
+ }
+ }
+
+ defer_on_missing_source(flow_task)?;
+ get_flow_type_from_options(flow_task)?;
+ Ok(())
+}
+
+fn user_runtime_flow_options(options: &HashMap) -> HashMap {
+ let mut options = options.clone();
+ options.remove(DEFER_ON_MISSING_SOURCE_KEY);
+ options
+}
+
+fn metadata_flow_options(options: &HashMap) -> HashMap {
+ options.clone()
+}
+
/// The state of [CreateFlowProcedure].
#[derive(Debug, Clone, Serialize, Deserialize, AsRefStr, PartialEq)]
pub enum CreateFlowState {
@@ -388,6 +489,9 @@ pub enum FlowType {
Streaming,
}
+pub const FLOW_EXPERIMENTAL_ENABLE_INCREMENTAL_READ_KEY: &str =
+ "experimental_enable_incremental_read";
+
impl FlowType {
pub const BATCHING: &str = "batching";
pub const STREAMING: &str = "streaming";
@@ -411,6 +515,8 @@ pub struct CreateFlowData {
pub(crate) flow_id: Option,
pub(crate) peers: Vec,
pub(crate) source_table_ids: Vec,
+ #[serde(default)]
+ pub(crate) unresolved_source_table_names: Vec,
/// Use alias for backward compatibility with QueryContext serialized data
#[serde(alias = "query_context")]
pub(crate) flow_context: FlowQueryContext,
@@ -424,6 +530,16 @@ pub struct CreateFlowData {
pub(crate) flow_type: Option,
}
+impl CreateFlowData {
+ pub(crate) fn is_pending(&self) -> bool {
+ !self.unresolved_source_table_names.is_empty()
+ }
+
+ pub(crate) fn is_active(&self) -> bool {
+ !self.is_pending()
+ }
+}
+
impl From<&CreateFlowData> for CreateRequest {
fn from(value: &CreateFlowData) -> Self {
let flow_id = value.flow_id.unwrap();
@@ -446,7 +562,7 @@ impl From<&CreateFlowData> for CreateRequest {
.map(|seconds| api::v1::EvalInterval { seconds }),
comment: value.task.comment.clone(),
sql: value.task.sql.clone(),
- flow_options: value.task.flow_options.clone(),
+ flow_options: user_runtime_flow_options(&value.task.flow_options),
};
let flow_type = value.flow_type.unwrap_or_default().to_string();
@@ -466,9 +582,9 @@ impl From<&CreateFlowData> for (FlowInfoValue, Vec<(FlowPartitionId, FlowRouteVa
eval_interval_secs: eval_interval,
comment,
sql,
- flow_options: mut options,
..
} = value.task.clone();
+ let mut options = metadata_flow_options(&value.task.flow_options);
let flownode_ids = value
.peers
@@ -484,7 +600,7 @@ impl From<&CreateFlowData> for (FlowInfoValue, Vec<(FlowPartitionId, FlowRouteVa
.collect::>();
let flow_type = value.flow_type.unwrap_or_default().to_string();
- options.insert("flow_type".to_string(), flow_type);
+ options.insert(FlowType::FLOW_TYPE_KEY.to_string(), flow_type);
let mut create_time = chrono::Utc::now();
if let Some(prev_flow_value) = value.prev_flow_info_value.as_ref()
@@ -495,6 +611,8 @@ impl From<&CreateFlowData> for (FlowInfoValue, Vec<(FlowPartitionId, FlowRouteVa
let flow_info: FlowInfoValue = FlowInfoValue {
source_table_ids: value.source_table_ids.clone(),
+ all_source_table_names: value.task.source_table_names.clone(),
+ unresolved_source_table_names: value.unresolved_source_table_names.clone(),
sink_table_name,
flownode_ids,
catalog_name,
@@ -506,6 +624,11 @@ impl From<&CreateFlowData> for (FlowInfoValue, Vec<(FlowPartitionId, FlowRouteVa
eval_interval_secs: eval_interval,
comment,
options,
+ status: if value.is_active() {
+ FlowStatus::Active
+ } else {
+ FlowStatus::PendingSources
+ },
created_time: create_time,
updated_time: chrono::Utc::now(),
};
diff --git a/src/common/meta/src/ddl/create_flow/metadata.rs b/src/common/meta/src/ddl/create_flow/metadata.rs
index 27b85b7946..f97ecfdf4a 100644
--- a/src/common/meta/src/ddl/create_flow/metadata.rs
+++ b/src/common/meta/src/ddl/create_flow/metadata.rs
@@ -12,10 +12,8 @@
// See the License for the specific language governing permissions and
// limitations under the License.
-use snafu::OptionExt;
-
use crate::ddl::create_flow::CreateFlowProcedure;
-use crate::error::{self, Result};
+use crate::error::Result;
use crate::key::table_name::TableNameKey;
impl CreateFlowProcedure {
@@ -34,9 +32,8 @@ impl CreateFlowProcedure {
Ok(())
}
- /// Ensures all source tables exist and collects source table ids
+ /// Collects source table ids and keeps track of missing tables.
pub(crate) async fn collect_source_tables(&mut self) -> Result<()> {
- // Ensures all source tables exist.
let keys = self
.data
.task
@@ -52,22 +49,24 @@ impl CreateFlowProcedure {
.batch_get(keys)
.await?;
- let source_table_ids = self
+ let mut resolved = Vec::with_capacity(self.data.task.source_table_names.len());
+ let mut unresolved = Vec::new();
+
+ for (name, table_id) in self
.data
.task
.source_table_names
.iter()
.zip(source_table_ids)
- .map(|(name, table_id)| {
- Ok(table_id
- .with_context(|| error::TableNotFoundSnafu {
- table_name: name.to_string(),
- })?
- .table_id())
- })
- .collect::>>()?;
+ {
+ match table_id {
+ Some(table_id) => resolved.push(table_id.table_id()),
+ None => unresolved.push(name.clone()),
+ }
+ }
- self.data.source_table_ids = source_table_ids;
+ self.data.source_table_ids = resolved;
+ self.data.unresolved_source_table_names = unresolved;
Ok(())
}
}
diff --git a/src/common/meta/src/ddl/drop_flow/metadata.rs b/src/common/meta/src/ddl/drop_flow/metadata.rs
index 0437098be3..7afd00f9d5 100644
--- a/src/common/meta/src/ddl/drop_flow/metadata.rs
+++ b/src/common/meta/src/ddl/drop_flow/metadata.rs
@@ -43,7 +43,7 @@ impl DropFlowProcedure {
.map(|(_, value)| value)
.collect::>();
ensure!(
- !flow_route_values.is_empty(),
+ flow_info_value.is_pending() || !flow_route_values.is_empty(),
error::FlowRouteNotFoundSnafu {
flow_name: format_full_flow_name(catalog_name, flow_name),
}
diff --git a/src/common/meta/src/ddl/tests/create_flow.rs b/src/common/meta/src/ddl/tests/create_flow.rs
index 344fc05024..7150be39cb 100644
--- a/src/common/meta/src/ddl/tests/create_flow.rs
+++ b/src/common/meta/src/ddl/tests/create_flow.rs
@@ -16,12 +16,18 @@ use std::assert_matches;
use std::collections::HashMap;
use std::sync::Arc;
+use api::v1::flow::CreateRequest;
use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME};
+use common_procedure::Status;
use common_procedure_test::execute_procedure_until_done;
use table::table_name::TableName;
use crate::ddl::DdlContext;
-use crate::ddl::create_flow::{CreateFlowData, CreateFlowProcedure, CreateFlowState, FlowType};
+use crate::ddl::create_flow::{
+ CreateFlowData, CreateFlowProcedure, CreateFlowState, DEFER_ON_MISSING_SOURCE_KEY,
+ FLOW_EXPERIMENTAL_ENABLE_INCREMENTAL_READ_KEY, FlowType, defer_on_missing_source,
+ validate_flow_options,
+};
use crate::ddl::test_util::create_table::test_create_table_task;
use crate::ddl::test_util::flownode_handler::NaiveFlownodeHandler;
use crate::error;
@@ -63,6 +69,11 @@ pub(crate) fn test_create_flow_task(
}
}
+fn enable_defer_on_missing_source(task: &mut CreateFlowTask) {
+ task.flow_options
+ .insert(DEFER_ON_MISSING_SOURCE_KEY.to_string(), "true".to_string());
+}
+
#[tokio::test]
async fn test_create_flow_source_table_not_found() {
let source_table_names = vec![TableName::new(
@@ -78,7 +89,277 @@ async fn test_create_flow_source_table_not_found() {
let query_ctx = test_query_context();
let mut procedure = CreateFlowProcedure::new(task, query_ctx, ddl_context);
let err = procedure.on_prepare().await.unwrap_err();
- assert_matches!(err, error::Error::TableNotFound { .. });
+ assert_matches!(err, error::Error::Unsupported { .. });
+ assert!(
+ err.to_string()
+ .contains("requires WITH ('defer_on_missing_source'='true')")
+ );
+}
+
+#[tokio::test]
+async fn test_create_pending_flow_source_table_not_found_with_defer() {
+ let source_table_names = vec![TableName::new(
+ DEFAULT_CATALOG_NAME,
+ DEFAULT_SCHEMA_NAME,
+ "my_table",
+ )];
+ let sink_table_name =
+ TableName::new(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, "my_sink_table");
+ let mut task = test_create_flow_task("my_flow", source_table_names, sink_table_name, false);
+ enable_defer_on_missing_source(&mut task);
+ let node_manager = Arc::new(MockFlownodeManager::new(NaiveFlownodeHandler));
+ let ddl_context = new_ddl_context(node_manager);
+ let query_ctx = test_query_context();
+ let mut procedure = CreateFlowProcedure::new(task, query_ctx, ddl_context.clone());
+ let status = procedure.on_prepare().await.unwrap();
+ assert_matches!(status, Status::Executing { persist: true, .. });
+ assert_eq!(procedure.data.unresolved_source_table_names.len(), 1);
+ assert_eq!(procedure.data.source_table_ids, Vec::::new());
+
+ let output = execute_procedure_until_done(&mut procedure).await.unwrap();
+ let flow_id = *output.downcast_ref::().unwrap();
+ let flow_info = ddl_context
+ .flow_metadata_manager
+ .flow_info_manager()
+ .get(flow_id)
+ .await
+ .unwrap()
+ .unwrap();
+ assert_eq!(flow_info.source_table_ids(), Vec::::new());
+ assert_eq!(
+ flow_info
+ .options()
+ .get(DEFER_ON_MISSING_SOURCE_KEY)
+ .map(String::as_str),
+ Some("true")
+ );
+}
+
+#[tokio::test]
+async fn test_create_pending_flow_source_table_not_found_with_defer_false() {
+ let source_table_names = vec![TableName::new(
+ DEFAULT_CATALOG_NAME,
+ DEFAULT_SCHEMA_NAME,
+ "my_table",
+ )];
+ let sink_table_name =
+ TableName::new(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, "my_sink_table");
+ let mut task = test_create_flow_task("my_flow", source_table_names, sink_table_name, false);
+ task.flow_options
+ .insert(DEFER_ON_MISSING_SOURCE_KEY.to_string(), "false".to_string());
+ let node_manager = Arc::new(MockFlownodeManager::new(NaiveFlownodeHandler));
+ let ddl_context = new_ddl_context(node_manager);
+ let query_ctx = test_query_context();
+ let mut procedure = CreateFlowProcedure::new(task, query_ctx, ddl_context);
+ let err = procedure.on_prepare().await.unwrap_err();
+ assert_matches!(err, error::Error::Unsupported { .. });
+ assert!(
+ err.to_string()
+ .contains("requires WITH ('defer_on_missing_source'='true')")
+ );
+}
+
+#[tokio::test]
+async fn test_create_pending_flow_records_partial_source_resolution() {
+ let existing_source = TableName::new(
+ DEFAULT_CATALOG_NAME,
+ DEFAULT_SCHEMA_NAME,
+ "partial_existing_source_table",
+ );
+ let missing_source = TableName::new(
+ DEFAULT_CATALOG_NAME,
+ DEFAULT_SCHEMA_NAME,
+ "partial_missing_source_table",
+ );
+ let sink_table_name = TableName::new(
+ DEFAULT_CATALOG_NAME,
+ DEFAULT_SCHEMA_NAME,
+ "partial_pending_sink_table",
+ );
+ let node_manager = Arc::new(MockFlownodeManager::new(NaiveFlownodeHandler));
+ let ddl_context = new_ddl_context(node_manager);
+
+ let existing_table_id = 3026;
+ let create_table_task =
+ test_create_table_task("partial_existing_source_table", existing_table_id);
+ ddl_context
+ .table_metadata_manager
+ .create_table_metadata(
+ create_table_task.table_info.clone(),
+ TableRouteValue::physical(vec![]),
+ HashMap::new(),
+ )
+ .await
+ .unwrap();
+
+ let mut task = test_create_flow_task(
+ "partial_pending_flow",
+ vec![existing_source.clone(), missing_source.clone()],
+ sink_table_name,
+ false,
+ );
+ enable_defer_on_missing_source(&mut task);
+ let query_ctx = test_query_context();
+ let mut procedure = CreateFlowProcedure::new(task, query_ctx, ddl_context.clone());
+ let status = procedure.on_prepare().await.unwrap();
+ assert_matches!(status, Status::Executing { persist: true, .. });
+ assert_eq!(procedure.data.source_table_ids, vec![existing_table_id]);
+ assert_eq!(
+ procedure.data.unresolved_source_table_names,
+ vec![missing_source.clone()]
+ );
+
+ let output = execute_procedure_until_done(&mut procedure).await.unwrap();
+ let flow_id = *output.downcast_ref::().unwrap();
+ let flow_info = ddl_context
+ .flow_metadata_manager
+ .flow_info_manager()
+ .get(flow_id)
+ .await
+ .unwrap()
+ .unwrap();
+
+ assert!(flow_info.is_pending());
+ assert_eq!(flow_info.source_table_ids(), &[existing_table_id]);
+ let expected_all_sources = vec![existing_source, missing_source.clone()];
+ assert_eq!(
+ flow_info.all_source_table_names(),
+ expected_all_sources.as_slice()
+ );
+ assert_eq!(flow_info.unresolved_source_table_names(), &[missing_source]);
+ assert!(flow_info.flownode_ids().is_empty());
+}
+
+#[test]
+fn test_defer_on_missing_source_defaults_false() {
+ let task = test_create_flow_task(
+ "my_flow",
+ vec![],
+ TableName::new(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, "my_sink_table"),
+ false,
+ );
+
+ assert!(!defer_on_missing_source(&task).unwrap());
+}
+
+#[test]
+fn test_defer_on_missing_source_true() {
+ let mut task = test_create_flow_task(
+ "my_flow",
+ vec![],
+ TableName::new(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, "my_sink_table"),
+ false,
+ );
+ task.flow_options
+ .insert(DEFER_ON_MISSING_SOURCE_KEY.to_string(), "true".to_string());
+
+ assert!(defer_on_missing_source(&task).unwrap());
+}
+
+#[test]
+fn test_defer_on_missing_source_invalid_value() {
+ let mut task = test_create_flow_task(
+ "my_flow",
+ vec![],
+ TableName::new(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, "my_sink_table"),
+ false,
+ );
+ task.flow_options.insert(
+ DEFER_ON_MISSING_SOURCE_KEY.to_string(),
+ "invalid".to_string(),
+ );
+
+ let err = defer_on_missing_source(&task).unwrap_err();
+ assert!(
+ err.to_string()
+ .contains("Invalid flow option 'defer_on_missing_source': invalid")
+ );
+}
+
+#[test]
+fn test_validate_flow_options_allows_incremental_read_option() {
+ let mut task = test_create_flow_task(
+ "my_flow",
+ vec![],
+ TableName::new(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, "my_sink_table"),
+ false,
+ );
+ task.flow_options.insert(
+ FLOW_EXPERIMENTAL_ENABLE_INCREMENTAL_READ_KEY.to_string(),
+ "true".to_string(),
+ );
+
+ validate_flow_options(&task).unwrap();
+}
+
+#[tokio::test]
+async fn test_create_flow_rejects_unknown_option_in_meta_task() {
+ let mut task = test_create_flow_task(
+ "my_flow",
+ vec![],
+ TableName::new(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, "my_sink_table"),
+ false,
+ );
+ task.flow_options
+ .insert("unknown_option".to_string(), "value".to_string());
+ let node_manager = Arc::new(MockFlownodeManager::new(NaiveFlownodeHandler));
+ let ddl_context = new_ddl_context(node_manager);
+ let query_ctx = test_query_context();
+ let mut procedure = CreateFlowProcedure::new(task, query_ctx, ddl_context);
+
+ let err = procedure.on_prepare().await.unwrap_err();
+ assert_matches!(err, error::Error::Unexpected { .. });
+ assert!(
+ err.to_string()
+ .contains("Unknown flow option 'unknown_option'")
+ );
+}
+
+#[test]
+fn test_create_request_strips_defer_on_missing_source_runtime_option() {
+ let mut task = test_create_flow_task(
+ "my_flow",
+ vec![],
+ TableName::new(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, "my_sink_table"),
+ false,
+ );
+ enable_defer_on_missing_source(&mut task);
+
+ let data = CreateFlowData {
+ state: CreateFlowState::CreateFlows,
+ task,
+ flow_id: Some(1024),
+ peers: vec![],
+ source_table_ids: vec![],
+ unresolved_source_table_names: vec![],
+ flow_context: FlowQueryContext {
+ catalog: DEFAULT_CATALOG_NAME.to_string(),
+ schema: DEFAULT_SCHEMA_NAME.to_string(),
+ timezone: "UTC".to_string(),
+ extensions: HashMap::new(),
+ channel: 0,
+ snapshot_seqs: HashMap::new(),
+ sst_min_sequences: HashMap::new(),
+ },
+ prev_flow_info_value: None,
+ did_replace: false,
+ flow_type: Some(FlowType::Batching),
+ };
+
+ let request: CreateRequest = (&data).into();
+
+ assert!(
+ !request
+ .flow_options
+ .contains_key(DEFER_ON_MISSING_SOURCE_KEY)
+ );
+ assert_eq!(
+ request
+ .flow_options
+ .get(FlowType::FLOW_TYPE_KEY)
+ .map(String::as_str),
+ Some(FlowType::BATCHING)
+ );
}
pub(crate) async fn create_test_flow(
@@ -101,6 +382,27 @@ pub(crate) async fn create_test_flow(
*flow_id
}
+pub(crate) async fn create_test_pending_flow(
+ ddl_context: &DdlContext,
+ flow_name: &str,
+ source_table_names: Vec,
+ sink_table_name: TableName,
+) -> FlowId {
+ let mut task = test_create_flow_task(
+ flow_name,
+ source_table_names.clone(),
+ sink_table_name.clone(),
+ false,
+ );
+ enable_defer_on_missing_source(&mut task);
+ let query_ctx = test_query_context();
+ let mut procedure = CreateFlowProcedure::new(task, query_ctx, ddl_context.clone());
+ let output = execute_procedure_until_done(&mut procedure).await.unwrap();
+ let flow_id = output.downcast_ref::().unwrap();
+
+ *flow_id
+}
+
#[tokio::test]
async fn test_create_flow() {
let table_id = 1024;
@@ -154,6 +456,201 @@ async fn test_create_flow() {
assert_matches!(err, error::Error::FlowAlreadyExists { .. });
}
+#[tokio::test]
+async fn test_replace_pending_flow_with_active_flow_is_unsupported() {
+ let source_table_name = TableName::new(
+ DEFAULT_CATALOG_NAME,
+ DEFAULT_SCHEMA_NAME,
+ "replace_pending_source_table",
+ );
+ let sink_table_name = TableName::new(
+ DEFAULT_CATALOG_NAME,
+ DEFAULT_SCHEMA_NAME,
+ "replace_pending_sink_table",
+ );
+ let node_manager = Arc::new(MockFlownodeManager::new(NaiveFlownodeHandler));
+ let ddl_context = new_ddl_context(node_manager);
+
+ let pending_flow_id = create_test_pending_flow(
+ &ddl_context,
+ "replace_pending_flow",
+ vec![source_table_name.clone()],
+ sink_table_name.clone(),
+ )
+ .await;
+
+ let pending_flow = ddl_context
+ .flow_metadata_manager
+ .flow_info_manager()
+ .get(pending_flow_id)
+ .await
+ .unwrap()
+ .unwrap();
+ assert!(pending_flow.is_pending());
+ assert!(pending_flow.flownode_ids().is_empty());
+
+ let create_table_task = test_create_table_task("replace_pending_source_table", 1026);
+ ddl_context
+ .table_metadata_manager
+ .create_table_metadata(
+ create_table_task.table_info.clone(),
+ TableRouteValue::physical(vec![]),
+ HashMap::new(),
+ )
+ .await
+ .unwrap();
+
+ let mut replace_task = test_create_flow_task(
+ "replace_pending_flow",
+ vec![source_table_name],
+ sink_table_name,
+ false,
+ );
+ replace_task.or_replace = true;
+ let query_ctx = test_query_context();
+ let mut procedure = CreateFlowProcedure::new(replace_task, query_ctx, ddl_context.clone());
+ let err = procedure.on_prepare().await.unwrap_err();
+ assert_matches!(err, error::Error::Unsupported { .. });
+ assert!(
+ err.to_string()
+ .contains("Replacing between pending and active flow states")
+ );
+}
+
+#[tokio::test]
+async fn test_replace_active_flow_with_pending_flow_is_unsupported() {
+ let existing_source_table = TableName::new(
+ DEFAULT_CATALOG_NAME,
+ DEFAULT_SCHEMA_NAME,
+ "replace_active_source_table",
+ );
+ let missing_source_table = TableName::new(
+ DEFAULT_CATALOG_NAME,
+ DEFAULT_SCHEMA_NAME,
+ "replace_missing_source_table",
+ );
+ let sink_table_name = TableName::new(
+ DEFAULT_CATALOG_NAME,
+ DEFAULT_SCHEMA_NAME,
+ "replace_active_sink_table",
+ );
+
+ let node_manager = Arc::new(MockFlownodeManager::new(NaiveFlownodeHandler));
+ let ddl_context = new_ddl_context(node_manager);
+
+ let create_table_task = test_create_table_task("replace_active_source_table", 2026);
+ ddl_context
+ .table_metadata_manager
+ .create_table_metadata(
+ create_table_task.table_info.clone(),
+ TableRouteValue::physical(vec![]),
+ HashMap::new(),
+ )
+ .await
+ .unwrap();
+
+ let _flow_id = create_test_flow(
+ &ddl_context,
+ "replace_active_flow_to_pending",
+ vec![existing_source_table],
+ sink_table_name.clone(),
+ )
+ .await;
+
+ let mut replace_task = test_create_flow_task(
+ "replace_active_flow_to_pending",
+ vec![missing_source_table],
+ sink_table_name,
+ false,
+ );
+ enable_defer_on_missing_source(&mut replace_task);
+ replace_task.or_replace = true;
+ let query_ctx = test_query_context();
+ let mut procedure = CreateFlowProcedure::new(replace_task, query_ctx, ddl_context.clone());
+ let err = procedure.on_prepare().await.unwrap_err();
+ assert_matches!(err, error::Error::Unsupported { .. });
+ assert!(
+ err.to_string()
+ .contains("Replacing between pending and active flow states")
+ );
+}
+
+#[tokio::test]
+async fn test_replace_pending_flow_with_pending_flow_updates_metadata() {
+ let first_missing_source = TableName::new(
+ DEFAULT_CATALOG_NAME,
+ DEFAULT_SCHEMA_NAME,
+ "replace_pending_first_missing_source",
+ );
+ let second_missing_source = TableName::new(
+ DEFAULT_CATALOG_NAME,
+ DEFAULT_SCHEMA_NAME,
+ "replace_pending_second_missing_source",
+ );
+ let sink_table_name = TableName::new(
+ DEFAULT_CATALOG_NAME,
+ DEFAULT_SCHEMA_NAME,
+ "replace_pending_to_pending_sink_table",
+ );
+ let node_manager = Arc::new(MockFlownodeManager::new(NaiveFlownodeHandler));
+ let ddl_context = new_ddl_context(node_manager);
+
+ let original_flow_id = create_test_pending_flow(
+ &ddl_context,
+ "replace_pending_to_pending_flow",
+ vec![first_missing_source.clone()],
+ sink_table_name.clone(),
+ )
+ .await;
+
+ let original_flow = ddl_context
+ .flow_metadata_manager
+ .flow_info_manager()
+ .get(original_flow_id)
+ .await
+ .unwrap()
+ .unwrap();
+ assert!(original_flow.is_pending());
+ assert_eq!(
+ original_flow.unresolved_source_table_names(),
+ &[first_missing_source]
+ );
+ assert!(original_flow.flownode_ids().is_empty());
+
+ let mut replace_task = test_create_flow_task(
+ "replace_pending_to_pending_flow",
+ vec![second_missing_source.clone()],
+ sink_table_name,
+ false,
+ );
+ enable_defer_on_missing_source(&mut replace_task);
+ replace_task.or_replace = true;
+ let query_ctx = test_query_context();
+ let mut procedure = CreateFlowProcedure::new(replace_task, query_ctx, ddl_context.clone());
+ let output = execute_procedure_until_done(&mut procedure).await.unwrap();
+ let replaced_flow_id = *output.downcast_ref::().unwrap();
+ assert_eq!(replaced_flow_id, original_flow_id);
+
+ let replaced_flow = ddl_context
+ .flow_metadata_manager
+ .flow_info_manager()
+ .get(replaced_flow_id)
+ .await
+ .unwrap()
+ .unwrap();
+ assert!(replaced_flow.is_pending());
+ assert_eq!(replaced_flow.source_table_ids(), Vec::::new());
+ assert_eq!(
+ replaced_flow.unresolved_source_table_names(),
+ std::slice::from_ref(&second_missing_source)
+ );
+ assert_eq!(
+ replaced_flow.all_source_table_names(),
+ &[second_missing_source]
+ );
+ assert!(replaced_flow.flownode_ids().is_empty());
+}
+
#[tokio::test]
async fn test_create_flow_same_source_and_sink_table() {
let table_id = 1024;
@@ -228,6 +725,7 @@ fn test_create_flow_data_serialization_backward_compatibility() {
"flow_id": null,
"peers": [],
"source_table_ids": [],
+ "unresolved_source_table_names": [],
"query_context": {
"current_catalog": "old_catalog",
"current_schema": "old_schema",
@@ -265,6 +763,7 @@ fn test_create_flow_data_new_format_serialization() {
flow_id: None,
peers: vec![],
source_table_ids: vec![],
+ unresolved_source_table_names: vec![],
flow_context,
prev_flow_info_value: None,
did_replace: false,
@@ -327,6 +826,7 @@ fn test_flow_info_conversion_with_flow_context() {
flow_id: Some(123),
peers: vec![],
source_table_ids: vec![456, 789],
+ unresolved_source_table_names: vec![],
flow_context,
prev_flow_info_value: None,
did_replace: false,
diff --git a/src/common/meta/src/ddl/tests/drop_flow.rs b/src/common/meta/src/ddl/tests/drop_flow.rs
index af34da4809..400fd2e118 100644
--- a/src/common/meta/src/ddl/tests/drop_flow.rs
+++ b/src/common/meta/src/ddl/tests/drop_flow.rs
@@ -23,7 +23,7 @@ use table::table_name::TableName;
use crate::ddl::drop_flow::DropFlowProcedure;
use crate::ddl::test_util::create_table::test_create_table_task;
use crate::ddl::test_util::flownode_handler::NaiveFlownodeHandler;
-use crate::ddl::tests::create_flow::create_test_flow;
+use crate::ddl::tests::create_flow::{create_test_flow, create_test_pending_flow};
use crate::error;
use crate::key::table_route::TableRouteValue;
use crate::rpc::ddl::DropFlowTask;
@@ -91,3 +91,45 @@ async fn test_drop_flow() {
let err = procedure.on_prepare().await.unwrap_err();
assert_matches!(err, error::Error::FlowNotFound { .. });
}
+
+#[tokio::test]
+async fn test_drop_pending_flow_without_routes() {
+ let source_table_name = TableName::new(
+ DEFAULT_CATALOG_NAME,
+ DEFAULT_SCHEMA_NAME,
+ "drop_pending_missing_source_table",
+ );
+ let sink_table_name = TableName::new(
+ DEFAULT_CATALOG_NAME,
+ DEFAULT_SCHEMA_NAME,
+ "drop_pending_sink_table",
+ );
+ let node_manager = Arc::new(MockFlownodeManager::new(NaiveFlownodeHandler));
+ let ddl_context = new_ddl_context(node_manager);
+
+ let flow_id = create_test_pending_flow(
+ &ddl_context,
+ "drop_pending_flow",
+ vec![source_table_name],
+ sink_table_name,
+ )
+ .await;
+ let flow_info = ddl_context
+ .flow_metadata_manager
+ .flow_info_manager()
+ .get(flow_id)
+ .await
+ .unwrap()
+ .unwrap();
+ assert!(flow_info.is_pending());
+ assert!(flow_info.flownode_ids().is_empty());
+
+ let task = test_drop_flow_task("drop_pending_flow", flow_id, false);
+ let mut procedure = DropFlowProcedure::new(task, ddl_context.clone());
+ execute_procedure_until_done(&mut procedure).await;
+
+ let task = test_drop_flow_task("drop_pending_flow", flow_id, false);
+ let mut procedure = DropFlowProcedure::new(task, ddl_context);
+ let err = procedure.on_prepare().await.unwrap_err();
+ assert_matches!(err, error::Error::FlowNotFound { .. });
+}
diff --git a/src/common/meta/src/instruction.rs b/src/common/meta/src/instruction.rs
index 3fa6b1bad0..6872b9ad55 100644
--- a/src/common/meta/src/instruction.rs
+++ b/src/common/meta/src/instruction.rs
@@ -18,7 +18,7 @@ use std::time::Duration;
use serde::{Deserialize, Deserializer, Serialize, Serializer};
use store_api::region_engine::SyncRegionFromRequest;
-use store_api::region_request::RegionFlushReason;
+use store_api::region_request::{RegionFlushReason, RegionRequirements};
use store_api::storage::{FileRefsManifest, GcReport, RegionId, RegionNumber};
use strum::Display;
use table::metadata::TableId;
@@ -179,12 +179,24 @@ impl Display for OpenRegion {
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
write!(
f,
- "OpenRegion(region_ident={}, region_storage_path={})",
- self.region_ident, self.region_storage_path
+ "OpenRegion(region_ident={}, region_storage_path={}, reason={:?})",
+ self.region_ident, self.region_storage_path, self.reason
)
}
}
+/// The reason why an open region instruction is triggered.
+#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
+pub enum OpenRegionReason {
+ /// Open triggered before region migration.
+ RegionMigration,
+ /// Open triggered by region failover.
+ RegionFailover,
+ /// Open triggered when adding a follower region.
+ #[cfg(feature = "enterprise")]
+ RegionFollower,
+}
+
#[serde_with::serde_as]
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
pub struct OpenRegion {
@@ -196,6 +208,10 @@ pub struct OpenRegion {
pub region_wal_options: HashMap,
#[serde(default)]
pub skip_wal_replay: bool,
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub reason: Option,
+ #[serde(default)]
+ pub requirements: RegionRequirements,
}
impl OpenRegion {
@@ -205,6 +221,8 @@ impl OpenRegion {
region_options: HashMap,
region_wal_options: HashMap,
skip_wal_replay: bool,
+ reason: Option,
+ requirements: RegionRequirements,
) -> Self {
Self {
region_ident,
@@ -212,6 +230,8 @@ impl OpenRegion {
region_options,
region_wal_options,
skip_wal_replay,
+ reason,
+ requirements,
}
}
}
@@ -1126,11 +1146,13 @@ mod tests {
HashMap::new(),
HashMap::new(),
false,
+ None,
+ RegionRequirements::empty(),
)]);
let serialized = serde_json::to_string(&open_region).unwrap();
assert_eq!(
- r#"{"OpenRegions":[{"region_ident":{"datanode_id":2,"table_id":1024,"region_number":1,"engine":"mito2"},"region_storage_path":"test/foo","region_options":{},"region_wal_options":{},"skip_wal_replay":false}]}"#,
+ r#"{"OpenRegions":[{"region_ident":{"datanode_id":2,"table_id":1024,"region_number":1,"engine":"mito2"},"region_storage_path":"test/foo","region_options":{},"region_wal_options":{},"skip_wal_replay":false,"requirements":{"object_storage":false}}]}"#,
serialized
);
@@ -1213,6 +1235,8 @@ mod tests {
HashMap::new(),
HashMap::new(),
false,
+ None,
+ RegionRequirements::empty(),
)]);
assert_eq!(open_region_instruction, open_region);
@@ -1368,10 +1392,41 @@ mod tests {
region_options,
region_wal_options: HashMap::new(),
skip_wal_replay: false,
+ reason: None,
+ requirements: RegionRequirements::empty(),
};
assert_eq!(expected, deserialized);
}
+ #[test]
+ fn test_serialize_open_region_with_reason_and_requirements() {
+ let open_region = OpenRegion::new(
+ RegionIdent {
+ datanode_id: 2,
+ table_id: 1024,
+ region_number: 1,
+ engine: "mito2".to_string(),
+ },
+ "test/foo",
+ HashMap::new(),
+ HashMap::new(),
+ false,
+ Some(OpenRegionReason::RegionMigration),
+ RegionRequirements::object_storage(),
+ );
+
+ let serialized = serde_json::to_string(&open_region).unwrap();
+ assert!(serialized.contains(r#""reason":"RegionMigration""#));
+ assert!(serialized.contains(r#""object_storage":true"#));
+
+ let deserialized: OpenRegion = serde_json::from_str(&serialized).unwrap();
+ assert_eq!(Some(OpenRegionReason::RegionMigration), deserialized.reason);
+ assert_eq!(
+ RegionRequirements::object_storage(),
+ deserialized.requirements
+ );
+ }
+
#[test]
fn test_flush_regions_creation() {
let region_id = RegionId::new(1024, 1);
diff --git a/src/common/meta/src/key/flow.rs b/src/common/meta/src/key/flow.rs
index d581b92685..bc9aaaa6b3 100644
--- a/src/common/meta/src/key/flow.rs
+++ b/src/common/meta/src/key/flow.rs
@@ -459,6 +459,7 @@ mod tests {
use super::*;
use crate::FlownodeId;
+ use crate::key::flow::flow_info::FlowStatus;
use crate::key::flow::table_flow::TableFlowKey;
use crate::key::node_address::{NodeAddressKey, NodeAddressValue};
use crate::key::{FlowPartitionId, MetadataValue};
@@ -522,6 +523,8 @@ mod tests {
query_context: None,
flow_name: flow_name.to_string(),
source_table_ids,
+ all_source_table_names: vec![],
+ unresolved_source_table_names: vec![],
sink_table_name,
flownode_ids,
raw_sql: "raw".to_string(),
@@ -529,6 +532,7 @@ mod tests {
eval_interval_secs: None,
comment: "hi".to_string(),
options: Default::default(),
+ status: FlowStatus::Active,
created_time: chrono::Utc::now(),
updated_time: chrono::Utc::now(),
}
@@ -774,6 +778,8 @@ mod tests {
query_context: None,
flow_name: "flow".to_string(),
source_table_ids: vec![1024, 1025, 1026],
+ all_source_table_names: vec![],
+ unresolved_source_table_names: vec![],
sink_table_name: another_sink_table_name,
flownode_ids: [(0, 1u64)].into(),
raw_sql: "raw".to_string(),
@@ -781,6 +787,7 @@ mod tests {
eval_interval_secs: None,
comment: "hi".to_string(),
options: Default::default(),
+ status: FlowStatus::Active,
created_time: chrono::Utc::now(),
updated_time: chrono::Utc::now(),
};
@@ -1151,6 +1158,8 @@ mod tests {
query_context: None,
flow_name: "flow".to_string(),
source_table_ids: vec![1024, 1025, 1026],
+ all_source_table_names: vec![],
+ unresolved_source_table_names: vec![],
sink_table_name: another_sink_table_name,
flownode_ids: [(0, 1u64)].into(),
raw_sql: "raw".to_string(),
@@ -1158,6 +1167,7 @@ mod tests {
eval_interval_secs: None,
comment: "hi".to_string(),
options: Default::default(),
+ status: FlowStatus::Active,
created_time: chrono::Utc::now(),
updated_time: chrono::Utc::now(),
};
diff --git a/src/common/meta/src/key/flow/flow_info.rs b/src/common/meta/src/key/flow/flow_info.rs
index d501822c3c..b1056902da 100644
--- a/src/common/meta/src/key/flow/flow_info.rs
+++ b/src/common/meta/src/key/flow/flow_info.rs
@@ -16,6 +16,8 @@ use std::collections::{BTreeMap, HashMap};
use std::sync::Arc;
use chrono::{DateTime, Utc};
+use futures::TryStreamExt;
+use futures::stream::BoxStream;
use lazy_static::lazy_static;
use regex::Regex;
use serde::{Deserialize, Serialize};
@@ -27,12 +29,27 @@ use crate::FlownodeId;
use crate::error::{self, Result};
use crate::key::flow::FlowScoped;
use crate::key::txn_helper::TxnOpGetResponseSet;
-use crate::key::{DeserializedValueWithBytes, FlowId, FlowPartitionId, MetadataKey, MetadataValue};
+use crate::key::{
+ BytesAdapter, DeserializedValueWithBytes, FlowId, FlowPartitionId, MetadataKey, MetadataValue,
+};
use crate::kv_backend::KvBackendRef;
use crate::kv_backend::txn::{Compare, CompareOp, Txn, TxnOp};
+use crate::range_stream::{DEFAULT_PAGE_SIZE, PaginationStream};
+use crate::rpc::KeyValue;
+use crate::rpc::store::RangeRequest;
pub const FLOW_INFO_KEY_PREFIX: &str = "info";
+/// The lifecycle status of a flow stored in metadata.
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, Default)]
+pub enum FlowStatus {
+ /// The flow metadata exists, but at least one source table did not exist at create time.
+ PendingSources,
+ /// The flow has resolved source tables and can be scheduled on flownodes.
+ #[default]
+ Active,
+}
+
lazy_static! {
static ref FLOW_INFO_KEY_PATTERN: Regex =
Regex::new(&format!("^{FLOW_INFO_KEY_PREFIX}/([0-9]+)$")).unwrap();
@@ -114,7 +131,12 @@ impl<'a> MetadataKey<'a, FlowInfoKeyInner> for FlowInfoKeyInner {
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
pub struct FlowInfoValue {
/// The source tables used by the flow.
+ #[serde(default)]
pub source_table_ids: Vec,
+ #[serde(default)]
+ pub all_source_table_names: Vec,
+ #[serde(default)]
+ pub unresolved_source_table_names: Vec,
/// The sink table used by the flow.
pub sink_table_name: TableName,
/// Which flow nodes this flow is running on.
@@ -145,6 +167,8 @@ pub struct FlowInfoValue {
pub comment: String,
/// The options.
pub options: HashMap,
+ #[serde(default)]
+ pub status: FlowStatus,
/// The created time
#[serde(default)]
pub created_time: DateTime,
@@ -154,6 +178,14 @@ pub struct FlowInfoValue {
}
impl FlowInfoValue {
+ pub fn is_pending(&self) -> bool {
+ self.status == FlowStatus::PendingSources
+ }
+
+ pub fn is_active(&self) -> bool {
+ self.status == FlowStatus::Active
+ }
+
/// Returns the `flownode_id`.
pub fn flownode_ids(&self) -> &BTreeMap {
&self.flownode_ids
@@ -173,6 +205,14 @@ impl FlowInfoValue {
&self.source_table_ids
}
+ pub fn all_source_table_names(&self) -> &[TableName] {
+ &self.all_source_table_names
+ }
+
+ pub fn unresolved_source_table_names(&self) -> &[TableName] {
+ &self.unresolved_source_table_names
+ }
+
pub fn catalog_name(&self) -> &String {
&self.catalog_name
}
@@ -209,6 +249,10 @@ impl FlowInfoValue {
&self.options
}
+ pub fn status(&self) -> &FlowStatus {
+ &self.status
+ }
+
pub fn created_time(&self) -> &DateTime {
&self.created_time
}
@@ -225,6 +269,12 @@ pub struct FlowInfoManager {
kv_backend: KvBackendRef,
}
+pub fn flow_info_decoder(kv: KeyValue) -> Result<(FlowInfoKey, FlowInfoValue)> {
+ let key = FlowInfoKey::from_bytes(&kv.key)?;
+ let value = FlowInfoValue::try_from_raw_value(&kv.value)?;
+ Ok((key, value))
+}
+
impl FlowInfoManager {
/// Returns a new [FlowInfoManager].
pub fn new(kv_backend: KvBackendRef) -> Self {
@@ -254,6 +304,23 @@ impl FlowInfoManager {
.transpose()
}
+ pub fn flow_infos(&self) -> BoxStream<'static, Result<(FlowId, FlowInfoValue)>> {
+ let start_key = FlowScoped::new(BytesAdapter::from(
+ format!("{FLOW_INFO_KEY_PREFIX}/").into_bytes(),
+ ))
+ .to_bytes();
+ let req = RangeRequest::new().with_prefix(start_key);
+ let stream = PaginationStream::new(
+ self.kv_backend.clone(),
+ req,
+ DEFAULT_PAGE_SIZE,
+ flow_info_decoder,
+ )
+ .into_stream();
+
+ Box::pin(stream.map_ok(|(key, value)| (key.flow_id(), value)))
+ }
+
/// Builds a create flow transaction.
/// It is expected that the `__flow/info/{flow_id}` wasn't occupied.
/// Otherwise, the transaction will retrieve existing value.
diff --git a/src/common/meta/src/key/topic_name.rs b/src/common/meta/src/key/topic_name.rs
index 99ae631a72..79454c7cd2 100644
--- a/src/common/meta/src/key/topic_name.rs
+++ b/src/common/meta/src/key/topic_name.rs
@@ -12,6 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
+use std::collections::HashMap;
use std::fmt::{self, Display};
use serde::{Deserialize, Serialize};
@@ -27,7 +28,7 @@ use crate::key::{
use crate::kv_backend::KvBackendRef;
use crate::kv_backend::txn::{Txn, TxnOp};
use crate::rpc::KeyValue;
-use crate::rpc::store::{BatchPutRequest, RangeRequest};
+use crate::rpc::store::{BatchGetRequest, BatchPutRequest, RangeRequest};
#[derive(Debug, Clone, PartialEq)]
pub struct TopicNameKey<'a> {
@@ -205,6 +206,25 @@ impl TopicNameManager {
.transpose()
}
+ /// Batch get values for specific topics.
+ pub async fn batch_get(
+ &self,
+ topics: Vec>,
+ ) -> Result> {
+ let raw_keys = topics.iter().map(|key| key.to_bytes()).collect::>();
+ let req = BatchGetRequest { keys: raw_keys };
+ let resp = self.kv_backend.batch_get(req).await?;
+
+ resp.kvs
+ .into_iter()
+ .map(|kv| {
+ let key = TopicNameKey::from_bytes(&kv.key)?;
+ let value = TopicNameValue::try_from_raw_value(&kv.value)?;
+ Ok((key.topic.to_string(), value))
+ })
+ .collect::>>()
+ }
+
/// Update the topic name key and value in the kv backend.
pub async fn update(
&self,
@@ -295,5 +315,17 @@ mod tests {
let err = manager.update(topic, 3, Some(value)).await.unwrap_err();
assert_matches!(err, error::Error::Unexpected { .. });
}
+
+ let batch_topics = topics
+ .iter()
+ .take(2)
+ .map(|topic| TopicNameKey::new(topic))
+ .chain(std::iter::once(TopicNameKey::new("missing-topic")))
+ .collect::>();
+ let values = manager.batch_get(batch_topics).await.unwrap();
+ assert_eq!(values.len(), 2);
+ for topic in topics.iter().take(2) {
+ assert_eq!(values.get(topic).unwrap().pruned_entry_id, 1);
+ }
}
}
diff --git a/src/common/meta/src/key/topic_region.rs b/src/common/meta/src/key/topic_region.rs
index 9feac10804..e7db23cfc2 100644
--- a/src/common/meta/src/key/topic_region.rs
+++ b/src/common/meta/src/key/topic_region.rs
@@ -126,6 +126,34 @@ impl ReplayCheckpoint {
metadata_entry_id,
}
}
+
+ /// Merges the checkpoint with the topic pruned entry id.
+ pub fn merge_with_topic_pruned_entry_id(
+ checkpoint: Option,
+ pruned_entry_id: Option,
+ is_metric_engine: bool,
+ ) -> Option {
+ match (checkpoint, pruned_entry_id) {
+ (Some(checkpoint), Some(pruned_entry_id)) => Some(Self {
+ entry_id: checkpoint.entry_id.max(pruned_entry_id),
+ metadata_entry_id: if is_metric_engine {
+ Some(
+ checkpoint
+ .metadata_entry_id
+ .unwrap_or_default()
+ .max(pruned_entry_id),
+ )
+ } else {
+ checkpoint.metadata_entry_id
+ },
+ }),
+ (None, Some(pruned_entry_id)) => Some(Self {
+ entry_id: pruned_entry_id,
+ metadata_entry_id: is_metric_engine.then_some(pruned_entry_id),
+ }),
+ (checkpoint, None) => checkpoint,
+ }
+ }
}
impl TopicRegionValue {
@@ -369,6 +397,58 @@ mod tests {
use super::*;
use crate::kv_backend::memory::MemoryKvBackend;
+ #[test]
+ fn test_merge_checkpoint_with_topic_pruned_entry_id_missing_pruned() {
+ let checkpoint = Some(ReplayCheckpoint::new(10, None));
+
+ assert_eq!(
+ ReplayCheckpoint::merge_with_topic_pruned_entry_id(checkpoint, None, true),
+ checkpoint
+ );
+ }
+
+ #[test]
+ fn test_merge_checkpoint_with_topic_pruned_entry_id_creates_checkpoint() {
+ assert_eq!(
+ ReplayCheckpoint::merge_with_topic_pruned_entry_id(None, Some(10), true),
+ Some(ReplayCheckpoint::new(10, Some(10)))
+ );
+ }
+
+ #[test]
+ fn test_merge_checkpoint_with_topic_pruned_entry_id_updates_both_ids() {
+ let checkpoint = ReplayCheckpoint::new(10, Some(5));
+
+ assert_eq!(
+ ReplayCheckpoint::merge_with_topic_pruned_entry_id(Some(checkpoint), Some(20), true),
+ Some(ReplayCheckpoint::new(20, Some(20)))
+ );
+ }
+
+ #[test]
+ fn test_merge_checkpoint_with_topic_pruned_entry_id_preserves_larger_ids() {
+ let checkpoint = ReplayCheckpoint::new(30, Some(40));
+
+ assert_eq!(
+ ReplayCheckpoint::merge_with_topic_pruned_entry_id(Some(checkpoint), Some(20), true),
+ Some(checkpoint)
+ );
+ }
+
+ #[test]
+ fn test_merge_checkpoint_with_topic_pruned_entry_id_for_mito() {
+ assert_eq!(
+ ReplayCheckpoint::merge_with_topic_pruned_entry_id(None, Some(10), false),
+ Some(ReplayCheckpoint::new(10, None))
+ );
+
+ let checkpoint = ReplayCheckpoint::new(5, Some(8));
+ assert_eq!(
+ ReplayCheckpoint::merge_with_topic_pruned_entry_id(Some(checkpoint), Some(10), false),
+ Some(ReplayCheckpoint::new(10, Some(8)))
+ );
+ }
+
#[tokio::test]
async fn test_topic_region_manager() {
let kv_backend = Arc::new(MemoryKvBackend::default());
diff --git a/src/datanode/src/config.rs b/src/datanode/src/config.rs
index 2ce306006b..b757c95121 100644
--- a/src/datanode/src/config.rs
+++ b/src/datanode/src/config.rs
@@ -14,6 +14,8 @@
//! Datanode configurations
+use std::time::Duration;
+
use common_base::readable_size::ReadableSize;
use common_config::{Configurable, DEFAULT_DATA_HOME};
use common_options::memory::MemoryOptions;
@@ -75,6 +77,10 @@ pub struct DatanodeOptions {
pub wal: DatanodeWalConfig,
pub storage: StorageConfig,
pub max_concurrent_queries: usize,
+ /// Timeout to acquire a permit from the concurrent query limiter when
+ /// `max_concurrent_queries` is reached. Only effective when the limiter is enabled.
+ #[serde(with = "humantime_serde")]
+ pub concurrent_query_limiter_timeout: Duration,
/// Options for different store engines.
pub region_engine: Vec,
pub logging: LoggingOptions,
@@ -131,6 +137,7 @@ impl Default for DatanodeOptions {
wal: DatanodeWalConfig::default(),
storage: StorageConfig::default(),
max_concurrent_queries: 0,
+ concurrent_query_limiter_timeout: Duration::from_millis(100),
region_engine: vec![
RegionEngineConfig::Mito(MitoConfig::default()),
RegionEngineConfig::File(FileEngineConfig::default()),
diff --git a/src/datanode/src/datanode.rs b/src/datanode/src/datanode.rs
index 9a2fe3d982..12d7c5109c 100644
--- a/src/datanode/src/datanode.rs
+++ b/src/datanode/src/datanode.rs
@@ -445,8 +445,7 @@ impl DatanodeBuilder {
event_listener,
table_provider_factory,
opts.max_concurrent_queries,
- //TODO: revaluate the hardcoded timeout on the next version of datanode concurrency limiter.
- Duration::from_millis(100),
+ opts.concurrent_query_limiter_timeout,
opts.grpc.flight_compression,
);
diff --git a/src/datanode/src/heartbeat/handler.rs b/src/datanode/src/heartbeat/handler.rs
index 10948a3e7c..79e0baaef3 100644
--- a/src/datanode/src/heartbeat/handler.rs
+++ b/src/datanode/src/heartbeat/handler.rs
@@ -313,7 +313,7 @@ mod tests {
use mito2::test_util::{CreateRequestBuilder, TestEnv};
use store_api::path_utils::table_dir;
use store_api::region_engine::RegionRole;
- use store_api::region_request::{RegionCloseRequest, RegionRequest};
+ use store_api::region_request::{RegionCloseRequest, RegionRequest, RegionRequirements};
use store_api::storage::RegionId;
use tokio::sync::mpsc::{self, Receiver};
@@ -442,6 +442,8 @@ mod tests {
HashMap::new(),
HashMap::new(),
false,
+ None,
+ RegionRequirements::empty(),
)])
}
diff --git a/src/datanode/src/heartbeat/handler/open_region.rs b/src/datanode/src/heartbeat/handler/open_region.rs
index 56c07a3efe..9c483e588d 100644
--- a/src/datanode/src/heartbeat/handler/open_region.rs
+++ b/src/datanode/src/heartbeat/handler/open_region.rs
@@ -14,6 +14,7 @@
use common_meta::instruction::{InstructionReply, OpenRegion, SimpleReply};
use common_meta::wal_provider::prepare_wal_options;
+use common_telemetry::info;
use store_api::path_utils::table_dir;
use store_api::region_request::{PathType, RegionOpenRequest};
use store_api::storage::RegionId;
@@ -41,8 +42,13 @@ impl InstructionHandler for OpenRegionsHandler {
mut region_options,
region_wal_options,
skip_wal_replay,
+ reason,
+ requirements,
} = open_region;
let region_id = RegionId::new(region_ident.table_id, region_ident.region_number);
+ info!(
+ "Received open region instruction, region_id: {region_id}, reason: {reason:?}"
+ );
prepare_wal_options(&mut region_options, region_id, ®ion_wal_options);
let request = RegionOpenRequest {
engine: region_ident.engine,
@@ -51,6 +57,7 @@ impl InstructionHandler for OpenRegionsHandler {
options: region_options,
skip_wal_replay,
checkpoint: None,
+ requirements,
};
(region_id, request)
})
@@ -85,7 +92,7 @@ mod tests {
use mito2::engine::MITO_ENGINE_NAME;
use mito2::test_util::{CreateRequestBuilder, TestEnv};
use store_api::path_utils::table_dir;
- use store_api::region_request::{RegionCloseRequest, RegionRequest};
+ use store_api::region_request::{RegionCloseRequest, RegionRequest, RegionRequirements};
use store_api::storage::RegionId;
use crate::heartbeat::handler::RegionHeartbeatResponseHandler;
@@ -98,17 +105,21 @@ mod tests {
) -> Instruction {
let region_idents = region_ids
.into_iter()
- .map(|region_id| OpenRegion {
- region_ident: RegionIdent {
- datanode_id: 0,
- table_id: region_id.table_id(),
- region_number: region_id.region_number(),
- engine: MITO_ENGINE_NAME.to_string(),
- },
- region_storage_path: storage_path.to_string(),
- region_options: HashMap::new(),
- region_wal_options: HashMap::new(),
- skip_wal_replay: false,
+ .map(|region_id| {
+ OpenRegion::new(
+ RegionIdent {
+ datanode_id: 0,
+ table_id: region_id.table_id(),
+ region_number: region_id.region_number(),
+ engine: MITO_ENGINE_NAME.to_string(),
+ },
+ storage_path,
+ HashMap::new(),
+ HashMap::new(),
+ false,
+ None,
+ RegionRequirements::empty(),
+ )
})
.collect();
diff --git a/src/datanode/src/region_server.rs b/src/datanode/src/region_server.rs
index d5711e1761..ce831353d1 100644
--- a/src/datanode/src/region_server.rs
+++ b/src/datanode/src/region_server.rs
@@ -49,6 +49,7 @@ use common_telemetry::{debug, error, info, warn};
use dashmap::DashMap;
use datafusion::datasource::TableProvider;
use datafusion_common::tree_node::TreeNode;
+use datatypes::schema::SchemaRef;
use either::Either;
use futures_util::Stream;
use futures_util::future::try_join_all;
@@ -82,7 +83,7 @@ use store_api::region_request::{
RegionOpenRequest, RegionRequest,
};
use store_api::storage::RegionId;
-use tokio::sync::{Semaphore, SemaphorePermit};
+use tokio::sync::{OwnedSemaphorePermit, Semaphore};
use tokio::time::timeout;
use tonic::{Request, Response, Result as TonicResult};
@@ -257,7 +258,7 @@ impl RegionServer {
request: api::v1::region::QueryRequest,
query_ctx: QueryContextRef,
) -> Result {
- let _permit = if let Some(p) = &self.inner.parallelism {
+ let permit = if let Some(p) = &self.inner.parallelism {
Some(p.acquire().await?)
} else {
None
@@ -298,14 +299,13 @@ impl RegionServer {
)
.await?;
- Ok(wrap_flow_region_watermark_stream(
- stream, region_id, &query_ctx,
- ))
+ let stream = wrap_flow_region_watermark_stream(stream, region_id, &query_ctx);
+ Ok(maybe_guard_stream(stream, permit))
}
#[tracing::instrument(skip_all)]
pub async fn handle_read(&self, request: QueryRequest) -> Result {
- let _permit = if let Some(p) = &self.inner.parallelism {
+ let permit = if let Some(p) = &self.inner.parallelism {
Some(p.acquire().await?)
} else {
None
@@ -332,9 +332,8 @@ impl RegionServer {
.handle_read(QueryRequest { plan, ..request }, query_ctx.clone())
.await?;
- Ok(wrap_flow_region_watermark_stream(
- stream, region_id, &query_ctx,
- ))
+ let stream = wrap_flow_region_watermark_stream(stream, region_id, &query_ctx);
+ Ok(maybe_guard_stream(stream, permit))
}
/// Returns all opened and reportable regions.
@@ -1058,7 +1057,7 @@ struct RegionServerInner {
}
struct RegionServerParallelism {
- semaphore: Semaphore,
+ semaphore: Arc,
timeout: Duration,
}
@@ -1071,19 +1070,68 @@ impl RegionServerParallelism {
return None;
}
Some(RegionServerParallelism {
- semaphore: Semaphore::new(max_concurrent_queries),
+ semaphore: Arc::new(Semaphore::new(max_concurrent_queries)),
timeout: concurrent_query_limiter_timeout,
})
}
- pub async fn acquire(&self) -> Result> {
- timeout(self.timeout, self.semaphore.acquire())
+ pub async fn acquire(&self) -> Result {
+ timeout(self.timeout, self.semaphore.clone().acquire_owned())
.await
.context(ConcurrentQueryLimiterTimeoutSnafu)?
.context(ConcurrentQueryLimiterClosedSnafu)
}
}
+/// Wraps a record batch stream and holds a concurrency permit until the stream is
+/// fully consumed (dropped), so `max_concurrent_queries` bounds the number of
+/// in-flight read streams, not just query planning.
+struct PermitGuardedStream {
+ inner: SendableRecordBatchStream,
+ _permit: OwnedSemaphorePermit,
+}
+
+impl RecordBatchStream for PermitGuardedStream {
+ fn name(&self) -> &str {
+ self.inner.name()
+ }
+
+ fn schema(&self) -> SchemaRef {
+ self.inner.schema()
+ }
+
+ fn output_ordering(&self) -> Option<&[OrderOption]> {
+ self.inner.output_ordering()
+ }
+
+ fn metrics(&self) -> Option {
+ self.inner.metrics()
+ }
+}
+
+impl Stream for PermitGuardedStream {
+ type Item = common_recordbatch::error::Result;
+
+ fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> {
+ self.inner.as_mut().poll_next(cx)
+ }
+}
+
+/// Wraps `stream` so it holds `permit` until fully consumed. Returns `stream`
+/// unchanged when no permit was acquired (limiter disabled).
+fn maybe_guard_stream(
+ stream: SendableRecordBatchStream,
+ permit: Option,
+) -> SendableRecordBatchStream {
+ match permit {
+ Some(permit) => Box::pin(PermitGuardedStream {
+ inner: stream,
+ _permit: permit,
+ }),
+ None => stream,
+ }
+}
+
enum CurrentEngine {
Engine(RegionEngineRef),
EarlyReturn(AffectedRows),
@@ -2057,6 +2105,7 @@ mod tests {
options: Default::default(),
skip_wal_replay: false,
checkpoint: None,
+ requirements: Default::default(),
}),
)
.await
@@ -2235,6 +2284,7 @@ mod tests {
options: Default::default(),
skip_wal_replay: false,
checkpoint: None,
+ requirements: Default::default(),
},
),
(
@@ -2246,6 +2296,7 @@ mod tests {
options: Default::default(),
skip_wal_replay: false,
checkpoint: None,
+ requirements: Default::default(),
},
),
],
@@ -2268,6 +2319,7 @@ mod tests {
options: Default::default(),
skip_wal_replay: false,
checkpoint: None,
+ requirements: Default::default(),
},
),
(
@@ -2279,6 +2331,7 @@ mod tests {
options: Default::default(),
skip_wal_replay: false,
checkpoint: None,
+ requirements: Default::default(),
},
),
],
diff --git a/src/datanode/src/utils.rs b/src/datanode/src/utils.rs
index c5cd008c28..88f4f1aa60 100644
--- a/src/datanode/src/utils.rs
+++ b/src/datanode/src/utils.rs
@@ -16,11 +16,16 @@ use std::collections::HashMap;
use common_meta::DatanodeId;
use common_meta::key::datanode_table::DatanodeTableManager;
-use common_meta::key::topic_region::{TopicRegionKey, TopicRegionManager, TopicRegionValue};
+use common_meta::key::topic_name::{TopicNameKey, TopicNameManager, TopicNameValue};
+use common_meta::key::topic_region::{
+ ReplayCheckpoint as MetadataReplayCheckpoint, TopicRegionKey, TopicRegionManager,
+ TopicRegionValue,
+};
use common_meta::kv_backend::KvBackendRef;
use common_meta::wal_provider::{extract_topic_from_wal_options, prepare_wal_options};
use futures::TryStreamExt;
use snafu::ResultExt;
+use store_api::metric_engine_consts::METRIC_ENGINE_NAME;
use store_api::path_utils::table_dir;
use store_api::region_request::{PathType, RegionOpenRequest, ReplayCheckpoint};
use store_api::storage::{RegionId, RegionNumber};
@@ -63,14 +68,45 @@ fn group_region_by_topic(
}
}
+fn region_pruned_entry_ids(
+ topic_regions: &HashMap>,
+ topic_name_values: &HashMap,
+) -> HashMap {
+ topic_regions
+ .iter()
+ .flat_map(|(topic, region_ids)| {
+ topic_name_values
+ .get(topic)
+ .into_iter()
+ .flat_map(move |value| {
+ region_ids
+ .iter()
+ .map(move |region_id| (*region_id, value.pruned_entry_id))
+ })
+ })
+ .collect()
+}
+
fn get_replay_checkpoint(
region_id: RegionId,
topic_region_values: &Option>,
+ pruned_entry_id: Option,
+ is_metric_engine: bool,
) -> Option {
- let topic_region_values = topic_region_values.as_ref()?;
- let topic_region_value = topic_region_values.get(®ion_id);
- let replay_checkpoint = topic_region_value.and_then(|value| value.checkpoint);
- replay_checkpoint.map(|checkpoint| ReplayCheckpoint {
+ let checkpoint = topic_region_values
+ .as_ref()
+ .and_then(|values| values.get(®ion_id))
+ .and_then(|value| value.checkpoint)
+ .map(|checkpoint| {
+ MetadataReplayCheckpoint::new(checkpoint.entry_id, checkpoint.metadata_entry_id)
+ });
+
+ MetadataReplayCheckpoint::merge_with_topic_pruned_entry_id(
+ checkpoint,
+ pruned_entry_id,
+ is_metric_engine,
+ )
+ .map(|checkpoint| ReplayCheckpoint {
entry_id: checkpoint.entry_id,
metadata_entry_id: checkpoint.metadata_entry_id,
})
@@ -88,7 +124,8 @@ pub async fn build_region_open_requests(
.await
.context(GetMetadataSnafu)?;
- let topic_region_manager = TopicRegionManager::new(kv_backend);
+ let topic_region_manager = TopicRegionManager::new(kv_backend.clone());
+ let topic_name_manager = TopicNameManager::new(kv_backend);
let mut topic_regions = HashMap::>::new();
let mut regions = vec![];
#[cfg(feature = "enterprise")]
@@ -161,10 +198,36 @@ pub async fn build_region_open_requests(
None
};
+ let topic_name_values = if !topic_regions.is_empty() {
+ let topics = topic_regions
+ .keys()
+ .map(|topic| TopicNameKey::new(topic))
+ .collect::>();
+ Some(
+ topic_name_manager
+ .batch_get(topics)
+ .await
+ .context(GetMetadataSnafu)?,
+ )
+ } else {
+ None
+ };
+ let region_pruned_entry_ids = topic_name_values
+ .as_ref()
+ .map(|values| region_pruned_entry_ids(&topic_regions, values));
+
let mut leader_region_requests = Vec::with_capacity(regions.len());
for (region_id, engine, store_path, options) in regions {
let table_dir = table_dir(&store_path, region_id.table_id());
- let checkpoint = get_replay_checkpoint(region_id, &topic_region_values);
+ let pruned_entry_id = region_pruned_entry_ids
+ .as_ref()
+ .and_then(|values| values.get(®ion_id).copied());
+ let checkpoint = get_replay_checkpoint(
+ region_id,
+ &topic_region_values,
+ pruned_entry_id,
+ engine == METRIC_ENGINE_NAME,
+ );
info!("region_id: {}, checkpoint: {:?}", region_id, checkpoint);
leader_region_requests.push((
region_id,
@@ -175,6 +238,7 @@ pub async fn build_region_open_requests(
options,
skip_wal_replay: false,
checkpoint,
+ requirements: Default::default(),
},
));
}
@@ -193,6 +257,7 @@ pub async fn build_region_open_requests(
options,
skip_wal_replay: true,
checkpoint: None,
+ requirements: Default::default(),
},
));
}
diff --git a/src/datatypes/src/json.rs b/src/datatypes/src/json.rs
index db657abbcb..33104084ad 100644
--- a/src/datatypes/src/json.rs
+++ b/src/datatypes/src/json.rs
@@ -26,12 +26,12 @@ use std::sync::Arc;
use serde::{Deserialize, Serialize};
use serde_json::{Map, Value as Json};
-use snafu::{OptionExt, ResultExt, ensure};
+use snafu::{OptionExt, ResultExt};
use crate::error::{self, InvalidJsonSnafu, Result, SerializeSnafu};
use crate::json::value::{JsonValue, JsonVariant};
use crate::types::json_type::{JsonNativeType, JsonNumberType, JsonObjectType};
-use crate::types::{StructField, StructType};
+use crate::types::{JsonType, StructField, StructType};
use crate::value::{ListValue, StructValue, Value};
/// The configuration of JSON encoding
@@ -305,33 +305,47 @@ fn encode_json_array_with_context<'a>(
) -> Result {
let json_array_len = json_array.len();
let mut items = Vec::with_capacity(json_array_len);
- let mut element_type = item_type.cloned();
for (index, value) in json_array.into_iter().enumerate() {
let array_context = context.with_key(&index.to_string());
- let item_value =
- encode_json_value_with_context(value, element_type.as_ref(), &array_context)?;
- let item_type = item_value.json_type().native_type().clone();
- items.push(item_value.into_variant());
-
- // Determine the common type for the list
- if let Some(current_type) = &element_type {
- // It's valid for json array to have different types of items, for example,
- // ["a string", 1]. However, the `JsonValue` will be converted to Arrow list array,
- // which requires all items have exactly same type. So we forbid the different types
- // case here. Besides, it's not common for items in a json array to differ. So I think
- // we are good here.
- ensure!(
- item_type == *current_type,
- error::InvalidJsonSnafu {
- value: "all items in json array must have the same type"
- }
- );
- } else {
- element_type = Some(item_type);
- }
+ let item_value = encode_json_value_with_context(value, None, &array_context)?;
+ items.push(item_value);
}
+ // In specification, it's valid for a JSON array to have different types of items, for example,
+ // ["a string", 1]. However, in implementation, the `JsonValue` will be converted to Arrow list
+ // array, which requires all items have exactly the same type. So we merge out the maybe
+ // different item types to a unified type, and align all the item values to it.
+
+ let provided_item_type = item_type.map(|x| JsonType::new_json2(x.clone()));
+ let merged_item_type = if let Some((first, rests)) = items.split_first() {
+ let mut merged = first.json_type().clone();
+ for rest in rests.iter().map(|x| x.json_type()) {
+ if matches!(merged.native_type(), JsonNativeType::Variant) {
+ break;
+ }
+ merged.merge(rest)?;
+ }
+ Some(merged)
+ } else {
+ None
+ };
+ let unified_item_type = match (provided_item_type, merged_item_type) {
+ (Some(mut x), Some(y)) => {
+ x.merge(&y)?;
+ Some(x)
+ }
+ (x, y) => x.or(y),
+ };
+ if let Some(unified_item_type) = unified_item_type {
+ for item in &mut items {
+ item.try_align(&unified_item_type)?;
+ }
+ }
+ let items = items
+ .into_iter()
+ .map(|x| x.into_variant())
+ .collect::>();
Ok(JsonValue::new(JsonVariant::Array(items)))
}
@@ -1050,11 +1064,8 @@ mod tests {
fn test_encode_json_array_mixed_types() {
let json = json!([1, "hello", true, 3.15]);
let settings = JsonStructureSettings::Structured(None);
- let result = settings.encode_with_type(json, None);
- assert_eq!(
- result.unwrap_err().to_string(),
- "Invalid JSON: all items in json array must have the same type"
- );
+ let value = settings.encode_with_type(json, None).unwrap();
+ assert_eq!(value.data_type().to_string(), r#"Json2[""]"#);
}
#[test]
@@ -1276,12 +1287,12 @@ mod tests {
#[test]
fn test_encode_json_array_with_item_type() {
let json = json!([1, 2, 3]);
- let item_type = Arc::new(ConcreteDataType::uint64_datatype());
+ let item_type = Arc::new(ConcreteDataType::int64_datatype());
let settings = JsonStructureSettings::Structured(None);
let result = settings
.encode_with_type(
json,
- Some(&JsonNativeType::Array(Box::new(JsonNativeType::u64()))),
+ Some(&JsonNativeType::Array(Box::new(JsonNativeType::i64()))),
)
.unwrap()
.into_json_inner()
@@ -1289,9 +1300,9 @@ mod tests {
if let Value::List(list_value) = result {
assert_eq!(list_value.items().len(), 3);
- assert_eq!(list_value.items()[0], Value::UInt64(1));
- assert_eq!(list_value.items()[1], Value::UInt64(2));
- assert_eq!(list_value.items()[2], Value::UInt64(3));
+ assert_eq!(list_value.items()[0], Value::Int64(1));
+ assert_eq!(list_value.items()[1], Value::Int64(2));
+ assert_eq!(list_value.items()[2], Value::Int64(3));
assert_eq!(list_value.datatype(), item_type);
} else {
panic!("Expected List value");
@@ -2249,10 +2260,10 @@ mod tests {
)])),
);
- let decoded_struct = settings.decode_struct(array_struct);
+ let decoded_struct = settings.decode_struct(array_struct).unwrap();
assert_eq!(
- decoded_struct.unwrap_err().to_string(),
- "Invalid JSON: all items in json array must have the same type"
+ format!("{decoded_struct:?}"),
+ r#"StructValue { items: [List(ListValue { items: [Binary(Bytes(b"1")), Binary(Bytes(b"\"hello\"")), Binary(Bytes(b"true")), Binary(Bytes(b"3.15"))], datatype: Binary(BinaryType { repr_type: Binary }) })], fields: StructType { fields: [StructField { name: "value", data_type: List(ListType { item_type: Binary(BinaryType { repr_type: Binary }) }), nullable: true, metadata: {} }] } }"#
);
}
diff --git a/src/datatypes/src/json/value.rs b/src/datatypes/src/json/value.rs
index f3b652a549..4350630003 100644
--- a/src/datatypes/src/json/value.rs
+++ b/src/datatypes/src/json/value.rs
@@ -65,6 +65,14 @@ impl JsonNumber {
JsonNumber::Float(n) => n.0,
}
}
+
+ fn native_type(&self) -> JsonNativeType {
+ match self {
+ JsonNumber::PosInt(_) => JsonNativeType::u64(),
+ JsonNumber::NegInt(_) => JsonNativeType::i64(),
+ JsonNumber::Float(_) => JsonNativeType::f64(),
+ }
+ }
}
impl From for JsonNumber {
@@ -147,26 +155,14 @@ impl JsonVariant {
match self {
JsonVariant::Null => JsonNativeType::Null,
JsonVariant::Bool(_) => JsonNativeType::Bool,
- JsonVariant::Number(n) => match n {
- JsonNumber::PosInt(_) => JsonNativeType::u64(),
- JsonNumber::NegInt(_) => JsonNativeType::i64(),
- JsonNumber::Float(_) => JsonNativeType::f64(),
- },
+ JsonVariant::Number(n) => n.native_type(),
JsonVariant::String(_) => JsonNativeType::String,
JsonVariant::Array(array) => {
- let item_type = if let Some(first) = array.first() {
- first.native_type()
- } else {
- JsonNativeType::Null
- };
- JsonNativeType::Array(Box::new(item_type))
+ json_array_native_type(array.iter().map(JsonVariant::native_type))
+ }
+ JsonVariant::Object(object) => {
+ json_object_native_type(object.iter().map(|(k, v)| (k, v.native_type())))
}
- JsonVariant::Object(object) => JsonNativeType::Object(
- object
- .iter()
- .map(|(k, v)| (k.clone(), v.native_type()))
- .collect(),
- ),
JsonVariant::Variant(_) => JsonNativeType::Variant,
}
}
@@ -469,6 +465,7 @@ impl JsonValue {
.collect::>()?,
),
+ (JsonVariant::Object(kvs), _) if kvs.is_empty() => JsonVariant::Null,
(JsonVariant::Object(mut kvs), JsonNativeType::Object(expected)) => {
ensure!(
expected.keys().len() >= kvs.keys().len()
@@ -517,7 +514,7 @@ impl JsonValue {
let x = std::mem::take(&mut self.json_variant);
self.json_variant = helper(x, expected.native_type())?;
- self.json_type = OnceLock::from(expected.clone());
+ self.json_type = OnceLock::new();
Ok(())
}
}
@@ -623,35 +620,55 @@ pub enum JsonVariantRef<'a> {
}
impl JsonVariantRef<'_> {
- fn json_type(&self) -> JsonType {
- fn native_type(v: &JsonVariantRef<'_>) -> JsonNativeType {
- match v {
- JsonVariantRef::Null => JsonNativeType::Null,
- JsonVariantRef::Bool(_) => JsonNativeType::Bool,
- JsonVariantRef::Number(n) => match n {
- JsonNumber::PosInt(_) => JsonNativeType::u64(),
- JsonNumber::NegInt(_) => JsonNativeType::i64(),
- JsonNumber::Float(_) => JsonNativeType::f64(),
- },
- JsonVariantRef::String(_) => JsonNativeType::String,
- JsonVariantRef::Array(array) => {
- let item_type = if let Some(first) = array.first() {
- native_type(first)
- } else {
- JsonNativeType::Null
- };
- JsonNativeType::Array(Box::new(item_type))
- }
- JsonVariantRef::Object(object) => JsonNativeType::Object(
- object
- .iter()
- .map(|(k, v)| (k.to_string(), native_type(v)))
- .collect(),
- ),
- JsonVariantRef::Variant(_) => JsonNativeType::Variant,
+ fn native_type(&self) -> JsonNativeType {
+ match self {
+ JsonVariantRef::Null => JsonNativeType::Null,
+ JsonVariantRef::Bool(_) => JsonNativeType::Bool,
+ JsonVariantRef::Number(n) => n.native_type(),
+ JsonVariantRef::String(_) => JsonNativeType::String,
+ JsonVariantRef::Array(array) => {
+ json_array_native_type(array.iter().map(JsonVariantRef::native_type))
}
+ JsonVariantRef::Object(object) => {
+ json_object_native_type(object.iter().map(|(k, v)| (*k, v.native_type())))
+ }
+ JsonVariantRef::Variant(_) => JsonNativeType::Variant,
}
- JsonType::new_json2(native_type(self))
+ }
+
+ fn json_type(&self) -> JsonType {
+ JsonType::new_json2(self.native_type())
+ }
+}
+
+fn json_array_native_type(items: I) -> JsonNativeType
+where
+ I: IntoIterator- ,
+{
+ let mut iter = items.into_iter();
+ let mut item_type = match iter.next() {
+ Some(t) => t,
+ None => return JsonNativeType::Array(Box::new(JsonNativeType::Null)),
+ };
+ for x in iter {
+ if matches!(item_type, JsonNativeType::Variant) {
+ break;
+ }
+ item_type.merge(&x);
+ }
+ JsonNativeType::Array(Box::new(item_type))
+}
+
+fn json_object_native_type(fields: I) -> JsonNativeType
+where
+ I: IntoIterator
- ,
+ K: Into,
+{
+ let mut fields = fields.into_iter().peekable();
+ if fields.peek().is_none() {
+ JsonNativeType::Null
+ } else {
+ JsonNativeType::Object(fields.map(|(k, v)| (k.into(), v)).collect())
}
}
@@ -941,7 +958,6 @@ mod tests {
("name".to_string(), JsonVariant::Null),
])))
);
- assert_eq!(value.json_type(), &expected);
// Object alignment should fail if the expected type misses any field from the value.
let expected = JsonType::new_json2(JsonNativeType::Object(JsonObjectType::from([(
diff --git a/src/datatypes/src/types/json_type.rs b/src/datatypes/src/types/json_type.rs
index e8d06543ed..652847da43 100644
--- a/src/datatypes/src/types/json_type.rs
+++ b/src/datatypes/src/types/json_type.rs
@@ -115,6 +115,14 @@ impl JsonNativeType {
(JsonNativeType::Null, that) => that.clone(),
(this, JsonNativeType::Null) => this,
(this, that) if this == *that => this,
+
+ (JsonNativeType::Number(x), JsonNativeType::Number(y)) => {
+ JsonNativeType::Number(match (x, y) {
+ (x, y) if x == *y => x,
+ (JsonNumberType::F64, _) | (_, JsonNumberType::F64) => JsonNumberType::F64,
+ _ => JsonNumberType::I64,
+ })
+ }
_ => JsonNativeType::Variant,
};
}
@@ -822,7 +830,7 @@ mod tests {
test(
"1.5",
&mut JsonType::new_json2(JsonNativeType::i64()),
- Ok(r#""""#),
+ Ok(r#""""#),
)?;
// Object merge should preserve existing fields and append missing fields.
diff --git a/src/datatypes/src/vectors/json/builder.rs b/src/datatypes/src/vectors/json/builder.rs
index be79a921c7..7ca1ff2f6a 100644
--- a/src/datatypes/src/vectors/json/builder.rs
+++ b/src/datatypes/src/vectors/json/builder.rs
@@ -89,7 +89,9 @@ impl MutableVector for JsonVectorBuilder {
.fail();
};
let json_type = value.json_type();
- self.merged_type.merge(json_type)?;
+ if !self.merged_type.is_include(json_type) {
+ self.merged_type.merge(json_type)?;
+ }
let value = JsonValue::new(JsonVariant::from(value.variant().clone()));
self.values.push(value);
diff --git a/src/file-engine/Cargo.toml b/src/file-engine/Cargo.toml
index 6c8c9e887d..9d031cb279 100644
--- a/src/file-engine/Cargo.toml
+++ b/src/file-engine/Cargo.toml
@@ -29,6 +29,7 @@ datafusion-expr.workspace = true
datatypes.workspace = true
futures.workspace = true
object-store.workspace = true
+object_store_opendal.workspace = true
serde = { version = "1.0", features = ["derive"] }
serde_json.workspace = true
snafu.workspace = true
diff --git a/src/file-engine/src/engine.rs b/src/file-engine/src/engine.rs
index 175ebef237..2ddbb6c414 100644
--- a/src/file-engine/src/engine.rs
+++ b/src/file-engine/src/engine.rs
@@ -32,7 +32,7 @@ use store_api::region_engine::{
};
use store_api::region_request::{
AffectedRows, RegionCloseRequest, RegionCreateRequest, RegionDropRequest, RegionOpenRequest,
- RegionRequest,
+ RegionRequest, RegionRequirements,
};
use store_api::storage::{RegionId, ScanRequest, SequenceNumber};
use tokio::sync::Mutex;
@@ -186,6 +186,24 @@ struct EngineInner {
type EngineInnerRef = Arc;
+fn ensure_open_requirements(
+ requirements: RegionRequirements,
+ object_store: &ObjectStore,
+) -> EngineResult<()> {
+ if !requirements.object_storage {
+ return Ok(());
+ }
+
+ ensure!(
+ object_store::util::is_object_storage(object_store),
+ UnsupportedSnafu {
+ operation: "open region with object storage requirement on non-object storage"
+ }
+ );
+
+ Ok(())
+}
+
impl EngineInner {
fn new(object_store: ObjectStore) -> Self {
Self {
@@ -289,6 +307,8 @@ impl EngineInner {
return Ok(0);
}
+ ensure_open_requirements(request.requirements, &self.object_store)?;
+
let res = FileRegion::open(region_id, request, &self.object_store).await;
let region = res.inspect_err(|err| {
error!(
@@ -356,3 +376,53 @@ impl EngineInner {
self.regions.read().unwrap().contains_key(®ion_id)
}
}
+
+#[cfg(test)]
+mod tests {
+ use object_store::services::{Fs, S3};
+
+ use super::*;
+ use crate::error::Error;
+
+ fn build_fs_object_store() -> ObjectStore {
+ ObjectStore::new(Fs::default().root("/tmp"))
+ .unwrap()
+ .finish()
+ }
+
+ fn build_s3_object_store() -> ObjectStore {
+ ObjectStore::new(
+ S3::default()
+ .bucket("test-bucket")
+ .region("us-east-1")
+ .disable_ec2_metadata(),
+ )
+ .unwrap()
+ .finish()
+ }
+
+ #[test]
+ fn test_empty_open_requirements_are_supported() {
+ ensure_open_requirements(RegionRequirements::empty(), &build_fs_object_store()).unwrap();
+ }
+
+ #[test]
+ fn test_object_storage_open_requirement_rejects_fs_object_store() {
+ let err = ensure_open_requirements(
+ RegionRequirements::object_storage(),
+ &build_fs_object_store(),
+ )
+ .unwrap_err();
+
+ assert!(matches!(err, Error::Unsupported { .. }));
+ }
+
+ #[test]
+ fn test_object_storage_open_requirement_accepts_s3_object_store() {
+ ensure_open_requirements(
+ RegionRequirements::object_storage(),
+ &build_s3_object_store(),
+ )
+ .unwrap();
+ }
+}
diff --git a/src/file-engine/src/query/file_stream.rs b/src/file-engine/src/query/file_stream.rs
index eec8f8961d..a480a50374 100644
--- a/src/file-engine/src/query/file_stream.rs
+++ b/src/file-engine/src/query/file_stream.rs
@@ -61,7 +61,7 @@ fn build_record_batch_stream(
.with_file_group(FileGroup::new(files))
.build();
- let store = Arc::new(object_store::compat::OpendalStore::new(
+ let store = Arc::new(object_store_opendal::OpendalStore::new(
scan_plan_config.store.clone(),
));
diff --git a/src/file-engine/src/region.rs b/src/file-engine/src/region.rs
index 3808b33a67..aceec21aa5 100644
--- a/src/file-engine/src/region.rs
+++ b/src/file-engine/src/region.rs
@@ -181,6 +181,7 @@ mod tests {
options: HashMap::default(),
skip_wal_replay: false,
checkpoint: None,
+ requirements: Default::default(),
};
let region = FileRegion::open(region_id, request, &object_store)
@@ -238,6 +239,7 @@ mod tests {
options: HashMap::default(),
skip_wal_replay: false,
checkpoint: None,
+ requirements: Default::default(),
};
let err = FileRegion::open(region_id, request, &object_store)
.await
diff --git a/src/flow/src/batching_mode.rs b/src/flow/src/batching_mode.rs
index 580762a142..a8bd139d98 100644
--- a/src/flow/src/batching_mode.rs
+++ b/src/flow/src/batching_mode.rs
@@ -23,7 +23,6 @@ use session::ReadPreference;
mod checkpoint;
pub(crate) mod engine;
pub(crate) mod frontend_client;
-mod incremental_filter;
mod state;
mod table_creator;
mod task;
@@ -55,6 +54,10 @@ pub struct BatchingModeOptions {
pub experimental_max_filter_num_per_query: usize,
/// Time window merge distance
pub experimental_time_window_merge_threshold: usize,
+ /// Whether to enable experimental flow incremental source reads.
+ ///
+ /// When disabled, batching flows always execute full-snapshot queries.
+ pub experimental_enable_incremental_read: bool,
/// Read preference of the Frontend client.
pub read_preference: ReadPreference,
/// TLS option for client connections to frontends.
@@ -72,6 +75,7 @@ impl Default for BatchingModeOptions {
experimental_frontend_scan_timeout: Duration::from_secs(30),
experimental_max_filter_num_per_query: 20,
experimental_time_window_merge_threshold: 3,
+ experimental_enable_incremental_read: false,
read_preference: Default::default(),
frontend_tls: None,
}
diff --git a/src/flow/src/batching_mode/engine.rs b/src/flow/src/batching_mode/engine.rs
index f37e54d80b..319ddcf2e7 100644
--- a/src/flow/src/batching_mode/engine.rs
+++ b/src/flow/src/batching_mode/engine.rs
@@ -21,7 +21,7 @@ use std::time::Duration;
use api::v1::flow::DirtyWindowRequests;
use catalog::CatalogManagerRef;
use common_error::ext::BoxedError;
-use common_meta::ddl::create_flow::FlowType;
+use common_meta::ddl::create_flow::{FLOW_EXPERIMENTAL_ENABLE_INCREMENTAL_READ_KEY, FlowType};
use common_meta::key::TableMetadataManagerRef;
use common_meta::key::flow::FlowMetadataManagerRef;
use common_meta::key::flow::flow_state::FlowStat;
@@ -38,6 +38,7 @@ use session::context::QueryContext;
use snafu::{OptionExt, ResultExt, ensure};
use sql::parsers::utils::is_tql;
use store_api::metric_engine_consts::is_metric_engine_internal_column;
+use store_api::mito_engine_options::APPEND_MODE_KEY;
use store_api::storage::{RegionId, TableId};
use table::table_reference::TableReference;
use tokio::sync::{RwLock, oneshot};
@@ -428,6 +429,55 @@ async fn get_table_info(
}
impl BatchingEngine {
+ fn batch_opts_for_flow_options(
+ &self,
+ flow_options: &HashMap,
+ ) -> Result, Error> {
+ let mut batch_opts = (*self.batch_opts).clone();
+ if let Some(enable_incremental_read) =
+ flow_options.get(FLOW_EXPERIMENTAL_ENABLE_INCREMENTAL_READ_KEY)
+ {
+ batch_opts.experimental_enable_incremental_read = enable_incremental_read
+ .parse::()
+ .map_err(|_| {
+ InvalidQuerySnafu {
+ reason: format!(
+ "Invalid flow option {FLOW_EXPERIMENTAL_ENABLE_INCREMENTAL_READ_KEY}: {enable_incremental_read}"
+ ),
+ }
+ .build()
+ })?;
+ }
+
+ Ok(Arc::new(batch_opts))
+ }
+
+ fn table_options_enable_append_mode(extra_options: &HashMap) -> bool {
+ extra_options
+ .get(APPEND_MODE_KEY)
+ .is_some_and(|value| value.eq_ignore_ascii_case("true"))
+ }
+
+ fn ensure_incremental_source_append_only(
+ batch_opts: &BatchingModeOptions,
+ table_name: &[String; 3],
+ extra_options: &HashMap,
+ ) -> Result<(), Error> {
+ if batch_opts.experimental_enable_incremental_read {
+ ensure!(
+ Self::table_options_enable_append_mode(extra_options),
+ UnsupportedSnafu {
+ reason: format!(
+ "Flow incremental read requires append-only source table, but source table `{}` is not append-only. Consider setting append_mode='true' on the source table or disabling experimental_enable_incremental_read",
+ table_name.join(".")
+ ),
+ }
+ );
+ }
+
+ Ok(())
+ }
+
pub async fn create_flow_inner(&self, args: CreateFlowArgs) -> Result, Error> {
let CreateFlowArgs {
flow_id,
@@ -494,6 +544,8 @@ impl BatchingEngine {
}
);
+ let batch_opts = self.batch_opts_for_flow_options(&flow_options)?;
+
let mut source_table_names = Vec::with_capacity(2);
for src_id in source_table_ids {
// also check table option to see if ttl!=instant
@@ -509,6 +561,11 @@ impl BatchingEngine {
),
}
);
+ Self::ensure_incremental_source_append_only(
+ &batch_opts,
+ &table_name,
+ &table_info.table_info.meta.options.extra_options,
+ )?;
source_table_names.push(table_name);
}
@@ -563,7 +620,7 @@ impl BatchingEngine {
query_ctx,
catalog_manager: self.catalog_manager.clone(),
shutdown_rx: rx,
- batch_opts: self.batch_opts.clone(),
+ batch_opts,
flow_eval_interval: eval_interval.map(|secs| Duration::from_secs(secs as u64)),
};
@@ -573,8 +630,11 @@ impl BatchingEngine {
let engine = self.query_engine.clone();
let frontend = self.frontend_client.clone();
- // check execute once first to detect any error early
+ // Create sink table if needed, then validate an existing/created sink schema before
+ // spawning the background task. This catches user-created sink schema mismatches at
+ // CREATE FLOW time instead of surfacing them later in the execution loop.
task.check_or_create_sink_table(&engine, &frontend).await?;
+ task.validate_sink_table_schema(&engine).await?;
let (start_tx, start_rx) = oneshot::channel();
@@ -808,7 +868,7 @@ impl BatchingEngine {
});
let res = task
- .gen_exec_once(
+ .execute_once_serialized(
&self.query_engine,
&self.frontend_client,
cur_dirty_window_cnt,
@@ -946,6 +1006,76 @@ mod tests {
)
}
+ #[tokio::test]
+ async fn test_flow_option_overrides_incremental_read_switch() {
+ let engine = new_test_engine().await;
+
+ let default_opts = engine.batch_opts_for_flow_options(&HashMap::new()).unwrap();
+ assert!(!default_opts.experimental_enable_incremental_read);
+
+ let enabled_opts = engine
+ .batch_opts_for_flow_options(&HashMap::from([(
+ FLOW_EXPERIMENTAL_ENABLE_INCREMENTAL_READ_KEY.to_string(),
+ "true".to_string(),
+ )]))
+ .unwrap();
+ assert!(enabled_opts.experimental_enable_incremental_read);
+ }
+
+ #[test]
+ fn test_table_options_enable_append_mode() {
+ assert!(!BatchingEngine::table_options_enable_append_mode(
+ &HashMap::new()
+ ));
+ assert!(!BatchingEngine::table_options_enable_append_mode(
+ &HashMap::from([(APPEND_MODE_KEY.to_string(), "false".to_string())])
+ ));
+ assert!(BatchingEngine::table_options_enable_append_mode(
+ &HashMap::from([(APPEND_MODE_KEY.to_string(), "TRUE".to_string())])
+ ));
+ }
+
+ #[test]
+ fn test_incremental_source_append_only_enforcement() {
+ let table_name = [
+ "greptime".to_string(),
+ "public".to_string(),
+ "numbers".to_string(),
+ ];
+ let disabled_opts = BatchingModeOptions::default();
+ let enabled_opts = BatchingModeOptions {
+ experimental_enable_incremental_read: true,
+ ..Default::default()
+ };
+ let non_append_options = HashMap::new();
+ let append_options = HashMap::from([(APPEND_MODE_KEY.to_string(), "true".to_string())]);
+
+ BatchingEngine::ensure_incremental_source_append_only(
+ &disabled_opts,
+ &table_name,
+ &non_append_options,
+ )
+ .expect("disabled incremental read should not require append-only source");
+ BatchingEngine::ensure_incremental_source_append_only(
+ &enabled_opts,
+ &table_name,
+ &append_options,
+ )
+ .expect("append-only source should be accepted when incremental read is enabled");
+
+ let err = BatchingEngine::ensure_incremental_source_append_only(
+ &enabled_opts,
+ &table_name,
+ &non_append_options,
+ )
+ .expect_err("non-append source should be rejected when incremental read is enabled");
+ assert!(
+ err.to_string()
+ .contains("Flow incremental read requires append-only source table"),
+ "{err}"
+ );
+ }
+
async fn new_test_task(flow_id: FlowId) -> (BatchingTask, oneshot::Sender<()>) {
let query_engine = create_test_query_engine();
let ctx = QueryContext::arc();
diff --git a/src/flow/src/batching_mode/incremental_filter.rs b/src/flow/src/batching_mode/incremental_filter.rs
deleted file mode 100644
index ddc58d0378..0000000000
--- a/src/flow/src/batching_mode/incremental_filter.rs
+++ /dev/null
@@ -1,222 +0,0 @@
-// Copyright 2023 Greptime Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use common_telemetry::tracing::debug;
-use datafusion_expr::Expr;
-use datatypes::schema::Schema;
-
-use crate::batching_mode::state::FilterExprInfo;
-use crate::batching_mode::utils::IncrementalAggregateAnalysis;
-use crate::{Error, FlowId};
-
-pub(super) fn build_sink_dirty_time_window_filter_expr(
- flow_id: FlowId,
- analysis: &IncrementalAggregateAnalysis,
- sink_schema: &Schema,
- dirty_filter: Option<&FilterExprInfo>,
-) -> Result, Error> {
- let Some(dirty_filter) = dirty_filter else {
- return Ok(None);
- };
-
- let Some(sink_filter_col) =
- infer_sink_time_window_filter_col(flow_id, analysis, sink_schema, dirty_filter)
- else {
- return Ok(None);
- };
-
- dirty_filter.predicate_for_col(&sink_filter_col)
-}
-
-fn infer_sink_time_window_filter_col(
- flow_id: FlowId,
- analysis: &IncrementalAggregateAnalysis,
- sink_schema: &Schema,
- dirty_filter: &FilterExprInfo,
-) -> Option {
- if analysis.group_key_names.is_empty() {
- return None;
- }
-
- let is_timestamp_group_key = |name: &str| {
- analysis.group_key_names.iter().any(|key| key == name)
- && sink_schema
- .column_schema_by_name(name)
- .is_some_and(|col| col.data_type.is_timestamp())
- };
-
- if is_timestamp_group_key(&dirty_filter.col_name) {
- return Some(dirty_filter.col_name.clone());
- }
-
- let candidates = analysis
- .group_key_names
- .iter()
- .filter(|name| is_timestamp_group_key(name))
- .cloned()
- .collect::>();
-
- match candidates.as_slice() {
- [name] => Some(name.clone()),
- [] => {
- debug!(
- "Flow {} cannot infer sink dirty-window filter column: no timestamp group key in {:?}",
- flow_id, analysis.group_key_names
- );
- None
- }
- _ => {
- debug!(
- "Flow {} cannot infer sink dirty-window filter column: ambiguous timestamp group keys {:?}",
- flow_id, candidates
- );
- None
- }
- }
-}
-
-#[cfg(test)]
-mod test {
- use datatypes::prelude::ConcreteDataType;
- use datatypes::schema::ColumnSchema;
- use pretty_assertions::assert_eq;
-
- use super::*;
- use crate::adapter::AUTO_CREATED_UPDATE_AT_TS_COL;
- use crate::batching_mode::state::FilterExprInfo;
- use crate::batching_mode::utils::IncrementalAggregateAnalysis;
-
- fn test_analysis_with_group_keys(group_key_names: Vec<&str>) -> IncrementalAggregateAnalysis {
- IncrementalAggregateAnalysis {
- group_key_names: group_key_names
- .into_iter()
- .map(|name| name.to_string())
- .collect(),
- merge_columns: vec![],
- literal_columns: vec![],
- output_field_names: vec![],
- unsupported_exprs: vec![],
- }
- }
-
- fn test_dirty_filter(col_name: &str) -> FilterExprInfo {
- FilterExprInfo {
- expr: datafusion_expr::col(col_name),
- col_name: col_name.to_string(),
- time_ranges: vec![],
- window_size: chrono::Duration::seconds(1),
- }
- }
-
- fn test_sink_schema(columns: Vec<(&str, ConcreteDataType)>) -> Schema {
- Schema::new(
- columns
- .into_iter()
- .map(|(name, data_type)| ColumnSchema::new(name, data_type, true))
- .collect(),
- )
- }
-
- #[test]
- fn test_infer_sink_time_window_filter_col_uses_matching_source_group_key() {
- let analysis = test_analysis_with_group_keys(vec!["ts", "host"]);
- let sink_schema = test_sink_schema(vec![
- ("ts", ConcreteDataType::timestamp_millisecond_datatype()),
- ("host", ConcreteDataType::string_datatype()),
- ]);
- let dirty_filter = test_dirty_filter("ts");
-
- assert_eq!(
- Some("ts".to_string()),
- infer_sink_time_window_filter_col(1, &analysis, &sink_schema, &dirty_filter)
- );
- }
-
- #[test]
- fn test_infer_sink_time_window_filter_col_uses_unique_timestamp_group_key() {
- let analysis = test_analysis_with_group_keys(vec!["host", "time_window"]);
- let sink_schema = test_sink_schema(vec![
- ("host", ConcreteDataType::string_datatype()),
- (
- "time_window",
- ConcreteDataType::timestamp_millisecond_datatype(),
- ),
- (
- AUTO_CREATED_UPDATE_AT_TS_COL,
- ConcreteDataType::timestamp_millisecond_datatype(),
- ),
- ]);
- let dirty_filter = test_dirty_filter("ts");
-
- assert_eq!(
- Some("time_window".to_string()),
- infer_sink_time_window_filter_col(1, &analysis, &sink_schema, &dirty_filter)
- );
- }
-
- #[test]
- fn test_infer_sink_time_window_filter_col_skips_global_aggregate() {
- let analysis = test_analysis_with_group_keys(vec![]);
- let sink_schema = test_sink_schema(vec![
- ("number", ConcreteDataType::uint32_datatype()),
- (
- "time_window",
- ConcreteDataType::timestamp_millisecond_datatype(),
- ),
- ]);
- let dirty_filter = test_dirty_filter("ts");
-
- assert_eq!(
- None,
- infer_sink_time_window_filter_col(1, &analysis, &sink_schema, &dirty_filter)
- );
- }
-
- #[test]
- fn test_infer_sink_time_window_filter_col_skips_without_timestamp_group_key() {
- let analysis = test_analysis_with_group_keys(vec!["host", "device"]);
- let sink_schema = test_sink_schema(vec![
- ("host", ConcreteDataType::string_datatype()),
- ("device", ConcreteDataType::string_datatype()),
- (
- AUTO_CREATED_UPDATE_AT_TS_COL,
- ConcreteDataType::timestamp_millisecond_datatype(),
- ),
- ]);
- let dirty_filter = test_dirty_filter("ts");
-
- assert_eq!(
- None,
- infer_sink_time_window_filter_col(1, &analysis, &sink_schema, &dirty_filter)
- );
- }
-
- #[test]
- fn test_infer_sink_time_window_filter_col_skips_ambiguous_timestamp_group_keys() {
- let analysis = test_analysis_with_group_keys(vec!["ts", "time_window"]);
- let sink_schema = test_sink_schema(vec![
- ("ts", ConcreteDataType::timestamp_millisecond_datatype()),
- (
- "time_window",
- ConcreteDataType::timestamp_millisecond_datatype(),
- ),
- ]);
- let dirty_filter = test_dirty_filter("source_ts");
-
- assert_eq!(
- None,
- infer_sink_time_window_filter_col(1, &analysis, &sink_schema, &dirty_filter)
- );
- }
-}
diff --git a/src/flow/src/batching_mode/state.rs b/src/flow/src/batching_mode/state.rs
index 42b71a4ec7..c5fcc74143 100644
--- a/src/flow/src/batching_mode/state.rs
+++ b/src/flow/src/batching_mode/state.rs
@@ -66,12 +66,20 @@ pub struct TaskState {
}
impl TaskState {
pub fn new(query_ctx: QueryContextRef, shutdown_rx: oneshot::Receiver<()>) -> Self {
+ Self::with_dirty_time_windows(query_ctx, shutdown_rx, DirtyTimeWindows::default())
+ }
+
+ pub fn with_dirty_time_windows(
+ query_ctx: QueryContextRef,
+ shutdown_rx: oneshot::Receiver<()>,
+ dirty_time_windows: DirtyTimeWindows,
+ ) -> Self {
Self {
query_ctx,
last_update_time: Instant::now(),
last_query_duration: Duration::from_secs(0),
last_exec_time_millis: None,
- dirty_time_windows: Default::default(),
+ dirty_time_windows,
checkpoint_mode: CheckpointMode::FullSnapshot,
checkpoints: Default::default(),
incremental_disabled: false,
@@ -264,6 +272,16 @@ impl DirtyTimeWindows {
time_window_merge_threshold,
}
}
+
+ #[cfg(test)]
+ pub(crate) fn max_filter_num_per_query(&self) -> usize {
+ self.max_filter_num_per_query
+ }
+
+ #[cfg(test)]
+ pub(crate) fn time_window_merge_threshold(&self) -> usize {
+ self.time_window_merge_threshold
+ }
}
impl Default for DirtyTimeWindows {
@@ -681,7 +699,7 @@ impl DirtyTimeWindows {
}
}
-fn to_df_literal(value: Timestamp) -> Result {
+pub(crate) fn to_df_literal(value: Timestamp) -> Result {
let value = Value::from(value);
let value = value
.try_to_scalar_value(&value.data_type())
diff --git a/src/flow/src/batching_mode/task.rs b/src/flow/src/batching_mode/task.rs
index 3cdf7899a6..3cd96b7525 100644
--- a/src/flow/src/batching_mode/task.rs
+++ b/src/flow/src/batching_mode/task.rs
@@ -27,7 +27,7 @@ use datafusion::datasource::DefaultTableSource;
use datafusion::sql::unparser::expr_to_sql;
use datafusion_common::DFSchemaRef;
use datafusion_common::tree_node::{Transformed, TreeNode};
-use datafusion_expr::{DmlStatement, LogicalPlan, WriteOp};
+use datafusion_expr::{DmlStatement, LogicalPlan, WriteOp, col, lit};
use datatypes::schema::Schema;
use query::QueryEngineRef;
use query::options::FLOW_INCREMENTAL_MODE;
@@ -38,14 +38,16 @@ use sql::parsers::utils::is_tql;
use store_api::mito_engine_options::MERGE_MODE_KEY;
use substrait::{DFLogicalSubstraitConvertor, SubstraitPlan};
use table::table::adapter::DfTableProviderAdapter;
-use tokio::sync::oneshot;
use tokio::sync::oneshot::error::TryRecvError;
+use tokio::sync::{Mutex, oneshot};
use tokio::time::Instant;
use crate::batching_mode::BatchingModeOptions;
use crate::batching_mode::checkpoint::checkpoint_mode_label;
use crate::batching_mode::frontend_client::{FrontendClient, PeerDesc};
-use crate::batching_mode::state::{CheckpointMode, DirtyTimeWindows, FilterExprInfo, TaskState};
+use crate::batching_mode::state::{
+ CheckpointMode, DirtyTimeWindows, FilterExprInfo, TaskState, to_df_literal,
+};
use crate::batching_mode::table_creator::{QueryType, create_table_with_expr};
use crate::batching_mode::time_window::TimeWindowExpr;
use crate::batching_mode::utils::{
@@ -67,12 +69,6 @@ use crate::{Error, FlowId};
mod ckpt;
mod inc;
-/// Maximum number of dirty time-window predicates attached to one incremental
-/// SQL query. This keeps generated OR filters bounded so Substrait encoding and
-/// downstream planning remain predictable; if the backlog is larger, the flow
-/// drains one capped batch and postpones checkpoint advancement to a later run.
-const MAX_INCREMENTAL_DIRTY_WINDOW_FILTERS: usize = 4096;
-
/// The task's config, immutable once created
#[derive(Clone)]
pub struct TaskConfig {
@@ -113,6 +109,10 @@ fn is_merge_mode_last_non_null(options: &HashMap) -> bool {
pub struct BatchingTask {
pub config: Arc,
pub state: Arc>,
+ /// Serializes plan generation, execution, checkpoint advancement, and dirty
+ /// window restoration for this flow. Without this, a manual flush and the
+ /// background loop can process the same checkpoint range concurrently.
+ execution_lock: Arc>,
}
/// Arguments for creating batching task
@@ -150,6 +150,16 @@ pub enum DirtyRestore {
Unscoped(DirtyTimeWindows),
}
+struct ExecuteOnceOutcome {
+ new_query: Option,
+ /// Execution result of the generated insert plan.
+ ///
+ /// `Ok(Some((affected_rows, elapsed)))` means a query was executed.
+ /// `Ok(None)` means no query was generated because there was no dirty signal.
+ /// `Err(_)` means plan generation or execution failed.
+ result: Result, Error>,
+}
+
impl BatchingTask {
#[allow(clippy::too_many_arguments)]
pub fn try_new(
@@ -168,6 +178,18 @@ impl BatchingTask {
flow_eval_interval,
}: TaskArgs<'_>,
) -> Result {
+ let mut state = TaskState::with_dirty_time_windows(
+ query_ctx.clone(),
+ shutdown_rx,
+ DirtyTimeWindows::new(
+ batch_opts.experimental_max_filter_num_per_query,
+ batch_opts.experimental_time_window_merge_threshold,
+ ),
+ );
+ if !batch_opts.experimental_enable_incremental_read {
+ state.disable_incremental();
+ }
+
Ok(Self {
config: Arc::new(TaskConfig {
flow_id,
@@ -182,7 +204,8 @@ impl BatchingTask {
batch_opts,
flow_eval_interval,
}),
- state: Arc::new(RwLock::new(TaskState::new(query_ctx, shutdown_rx))),
+ state: Arc::new(RwLock::new(state)),
+ execution_lock: Arc::new(Mutex::new(())),
})
}
@@ -242,6 +265,36 @@ impl BatchingTask {
Ok(None)
}
+ /// Validates that the sink table schema can accept this flow's output.
+ ///
+ /// This is a dry-run of the same schema matching logic used by runtime insert-plan
+ /// generation, but without adding dirty-window filters or executing the query. It is used
+ /// during CREATE FLOW to catch existing sink table mismatches early.
+ pub async fn validate_sink_table_schema(&self, engine: &QueryEngineRef) -> Result<(), Error> {
+ let (table, _) = get_table_info_df_schema(
+ self.config.catalog_manager.clone(),
+ self.config.sink_table_name.clone(),
+ )
+ .await?;
+
+ let table_meta = &table.table_info().meta;
+ let merge_mode_last_non_null =
+ is_merge_mode_last_non_null(&table_meta.options.extra_options);
+ let primary_key_indices = table_meta.primary_key_indices.clone();
+ let query_ctx = self.state.read().unwrap().query_ctx.clone();
+
+ gen_plan_with_matching_schema(
+ &self.config.query,
+ query_ctx,
+ engine.clone(),
+ table_meta.schema.clone(),
+ &primary_key_indices,
+ merge_mode_last_non_null,
+ )
+ .await
+ .map(|_| ())
+ }
+
async fn is_table_exist(&self, table_name: &[String; 3]) -> Result {
self.config
.catalog_manager
@@ -251,40 +304,75 @@ impl BatchingTask {
.context(ExternalSnafu)
}
- pub async fn gen_exec_once(
+ pub(crate) async fn execute_once_serialized(
&self,
engine: &QueryEngineRef,
frontend_client: &Arc,
max_window_cnt: Option,
) -> Result, Error> {
- if let Some(new_query) = self.gen_insert_plan(engine, max_window_cnt).await? {
+ let outcome = self
+ .execute_once_serialized_with_outcome(engine, frontend_client, max_window_cnt)
+ .await;
+ outcome.result
+ }
+
+ /// Executes one flow evaluation under `execution_lock` and keeps the
+ /// generated query context for the background loop's error logging/backoff.
+ async fn execute_once_serialized_with_outcome(
+ &self,
+ engine: &QueryEngineRef,
+ frontend_client: &Arc,
+ max_window_cnt: Option,
+ ) -> ExecuteOnceOutcome {
+ let _execution_guard = self.execution_lock.lock().await;
+ self.execute_once_unlocked(engine, frontend_client, max_window_cnt)
+ .await
+ }
+
+ /// Executes one flow evaluation. Caller must hold `execution_lock`.
+ async fn execute_once_unlocked(
+ &self,
+ engine: &QueryEngineRef,
+ frontend_client: &Arc,
+ max_window_cnt: Option,
+ ) -> ExecuteOnceOutcome {
+ let new_query = match self.gen_insert_plan_unlocked(engine, max_window_cnt).await {
+ Ok(new_query) => new_query,
+ Err(err) => {
+ return ExecuteOnceOutcome {
+ new_query: None,
+ result: Err(err),
+ };
+ }
+ };
+
+ if let Some(new_query) = new_query {
debug!("Generate new query: {}", new_query.plan);
- let dirty_filter = match &new_query.dirty_restore {
- DirtyRestore::Scoped(f) => Some(f),
- _ => None,
- };
- match self
- .execute_logical_plan(
+ let res = self
+ .execute_logical_plan_unlocked(
frontend_client,
&new_query.plan,
- dirty_filter,
new_query.can_advance_checkpoints,
)
- .await
- {
- Ok(result) => Ok(result),
- Err(err) => {
- self.handle_executed_query_failure(Some(&new_query));
- Err(err)
- }
+ .await;
+ if res.is_err() {
+ self.handle_executed_query_failure(Some(&new_query));
+ }
+ ExecuteOnceOutcome {
+ new_query: Some(new_query),
+ result: res,
}
} else {
debug!("Generate no query");
- Ok(None)
+ ExecuteOnceOutcome {
+ new_query: None,
+ result: Ok(None),
+ }
}
}
- pub async fn gen_insert_plan(
+ /// Generates the insert plan. Caller must reach this through the serialized path.
+ async fn gen_insert_plan_unlocked(
&self,
engine: &QueryEngineRef,
max_window_cnt: Option,
@@ -388,11 +476,11 @@ impl BatchingTask {
Ok(())
}
- pub async fn execute_logical_plan(
+ /// Executes the insert plan. Caller must reach this through the serialized path.
+ async fn execute_logical_plan_unlocked(
&self,
frontend_client: &Arc,
plan: &LogicalPlan,
- dirty_filter: Option<&FilterExprInfo>,
can_advance_checkpoints: bool,
) -> Result, Error> {
let instant = Instant::now();
@@ -426,8 +514,7 @@ impl BatchingTask {
// For incremental-mode SQL queries, attempt to rewrite the delta aggregate
// plan into a safe delta-LEFT-JOIN-sink form before deciding on extensions.
let incremental_plan = if can_advance_checkpoints {
- self.prepare_plan_for_incremental(&plan, dirty_filter)
- .await?
+ self.prepare_plan_for_incremental(&plan).await?
} else {
None
};
@@ -580,6 +667,112 @@ impl BatchingTask {
})
}
+ fn restore_unscoped_dirty_windows(&self, dirty_windows: &DirtyTimeWindows) {
+ self.state
+ .write()
+ .unwrap()
+ .dirty_time_windows
+ .add_dirty_windows(dirty_windows);
+ }
+
+ fn restore_unscoped_dirty_windows_on_err(
+ &self,
+ dirty_windows: &DirtyTimeWindows,
+ result: Result,
+ ) -> Result {
+ result.inspect_err(|_| {
+ self.restore_unscoped_dirty_windows(dirty_windows);
+ })
+ }
+
+ fn drain_dirty_windows_signal(&self) -> (bool, DirtyTimeWindows) {
+ let mut state = self.state.write().unwrap();
+ let dirty_windows_to_restore = state.dirty_time_windows.clone();
+ let is_dirty = !dirty_windows_to_restore.is_empty();
+ state.dirty_time_windows.clean();
+ (is_dirty, dirty_windows_to_restore)
+ }
+
+ #[allow(clippy::too_many_arguments)]
+ async fn gen_unfiltered_plan_info(
+ &self,
+ engine: QueryEngineRef,
+ query_ctx: QueryContextRef,
+ sink_table_schema: Arc,
+ primary_key_indices: &[usize],
+ allow_partial: bool,
+ dirty_windows_to_restore: DirtyTimeWindows,
+ retention_filter: Option<(&str, Timestamp, &'static str)>,
+ ) -> Result {
+ let mut plan = self.restore_unscoped_dirty_windows_on_err(
+ &dirty_windows_to_restore,
+ gen_plan_with_matching_schema(
+ &self.config.query,
+ query_ctx,
+ engine,
+ sink_table_schema,
+ primary_key_indices,
+ allow_partial,
+ )
+ .await,
+ )?;
+
+ if let Some((col_name, lower_bound, context)) = retention_filter {
+ let lower = self.restore_unscoped_dirty_windows_on_err(
+ &dirty_windows_to_restore,
+ to_df_literal(lower_bound),
+ )?;
+ let retention_filter = col(col_name).gt_eq(lit(lower));
+ let mut add_filter = AddFilterRewriter::new(retention_filter);
+ plan = self.restore_unscoped_dirty_windows_on_err(
+ &dirty_windows_to_restore,
+ plan.clone()
+ .rewrite(&mut add_filter)
+ .with_context(|_| DatafusionSnafu {
+ context: format!(
+ "Failed to apply {context} expire_after filter to plan:\n {}\n",
+ plan
+ ),
+ })
+ .map(|rewrite| rewrite.data),
+ )?;
+ }
+
+ Ok(PlanInfo {
+ plan,
+ dirty_restore: DirtyRestore::Unscoped(dirty_windows_to_restore),
+ can_advance_checkpoints: true,
+ })
+ }
+
+ async fn gen_unfiltered_plan_info_if_dirty(
+ &self,
+ engine: QueryEngineRef,
+ query_ctx: QueryContextRef,
+ sink_table_schema: Arc,
+ primary_key_indices: &[usize],
+ allow_partial: bool,
+ retention_filter: Option<(&str, Timestamp, &'static str)>,
+ ) -> Result, Error> {
+ let (is_dirty, dirty_windows_to_restore) = self.drain_dirty_windows_signal();
+ if !is_dirty {
+ debug!("Flow id={:?}, no new data, not update", self.config.flow_id);
+ return Ok(None);
+ }
+
+ self.gen_unfiltered_plan_info(
+ engine,
+ query_ctx,
+ sink_table_schema,
+ primary_key_indices,
+ allow_partial,
+ dirty_windows_to_restore,
+ retention_filter,
+ )
+ .await
+ .map(Some)
+ }
+
fn handle_executed_query_failure(&self, query: Option<&PlanInfo>) {
if let Some(query) = query {
self.restore_dirty_windows_after_failure(query);
@@ -626,33 +819,11 @@ impl BatchingTask {
let min_refresh = self.config.batch_opts.experimental_min_refresh_duration;
- let new_query = match self.gen_insert_plan(&engine, max_window_cnt).await {
- Ok(new_query) => new_query,
- Err(err) => {
- common_telemetry::error!(err; "Failed to generate query for flow={}", self.config.flow_id);
- // also sleep for a little while before try again to prevent flooding logs
- tokio::time::sleep(min_refresh).await;
- continue;
- }
- };
+ let outcome = self
+ .execute_once_serialized_with_outcome(&engine, &frontend_client, max_window_cnt)
+ .await;
- let res = if let Some(new_query) = &new_query {
- let dirty_filter = match &new_query.dirty_restore {
- DirtyRestore::Scoped(f) => Some(f),
- _ => None,
- };
- self.execute_logical_plan(
- &frontend_client,
- &new_query.plan,
- dirty_filter,
- new_query.can_advance_checkpoints,
- )
- .await
- } else {
- Ok(None)
- };
-
- match res {
+ match outcome.result {
// normal execute, sleep for some time before doing next query
Ok(Some(_)) => {
// can increase max_window_cnt to query more windows next time
@@ -703,11 +874,10 @@ impl BatchingTask {
}
// TODO(discord9): this error should have better place to go, but for now just print error, also more context is needed
Err(err) => {
- self.handle_executed_query_failure(new_query.as_ref());
METRIC_FLOW_BATCHING_ENGINE_ERROR_CNT
.with_label_values(&[&flow_id_str])
.inc();
- match new_query {
+ match outcome.new_query {
Some(query) => {
common_telemetry::error!(err; "Failed to execute query for flow={} with query: {}", self.config.flow_id, query.plan);
// TODO(discord9): add some backoff here? half the query time window or what
@@ -743,6 +913,20 @@ impl BatchingTask {
create_table_with_expr(&plan, &self.config.sink_table_name, &self.config.query_type)
}
+ fn should_use_unfiltered_incremental_delta(&self) -> bool {
+ let state = self.state.read().unwrap();
+ state.checkpoint_mode() == CheckpointMode::Incremental
+ && !state.is_incremental_disabled()
+ && matches!(self.config.query_type, QueryType::Sql)
+ }
+
+ fn should_use_unfiltered_full_snapshot_seeding(&self) -> bool {
+ let state = self.state.read().unwrap();
+ state.checkpoint_mode() == CheckpointMode::FullSnapshot
+ && !state.is_incremental_disabled()
+ && matches!(self.config.query_type, QueryType::Sql)
+ }
+
/// will merge and use the first ten time window in query
async fn gen_query_with_time_window(
&self,
@@ -775,7 +959,7 @@ impl BatchingTask {
let (expire_lower_bound, expire_upper_bound) =
match (expire_time_window_bound, &self.config.query_type) {
(Some((Some(l), Some(u))), QueryType::Sql) => (l, u),
- (None, QueryType::Sql) => {
+ (None, QueryType::Sql) if self.config.flow_eval_interval.is_none() => {
// if it's sql query and no time window lower/upper bound is found, just return the original query(with auto columns)
// use sink_table_meta to add to query the `update_at` and `__ts_placeholder` column's value too for compatibility reason
debug!(
@@ -783,83 +967,36 @@ impl BatchingTask {
self.config.flow_id
);
// clean dirty time window too, this could be from create flow's check_execute
- let (is_dirty, dirty_windows_to_restore) = {
- let mut state = self.state.write().unwrap();
- let dirty_windows_to_restore = state.dirty_time_windows.clone();
- let is_dirty = !dirty_windows_to_restore.is_empty();
- state.dirty_time_windows.clean();
- (is_dirty, dirty_windows_to_restore)
- };
-
- if !is_dirty {
- // no dirty data, hence no need to update
- debug!("Flow id={:?}, no new data, not update", self.config.flow_id);
- return Ok(None);
- }
-
- let plan = match gen_plan_with_matching_schema(
- &self.config.query,
- query_ctx,
- engine,
- sink_table_schema.clone(),
- primary_key_indices,
- allow_partial,
- )
- .await
- {
- Ok(plan) => plan,
- Err(err) => {
- self.state
- .write()
- .unwrap()
- .dirty_time_windows
- .add_dirty_windows(&dirty_windows_to_restore);
- return Err(err);
- }
- };
-
- return Ok(Some(PlanInfo {
- plan,
- dirty_restore: DirtyRestore::Unscoped(dirty_windows_to_restore),
- can_advance_checkpoints: true,
- }));
+ return self
+ .gen_unfiltered_plan_info_if_dirty(
+ engine,
+ query_ctx,
+ sink_table_schema.clone(),
+ primary_key_indices,
+ allow_partial,
+ None,
+ )
+ .await;
}
_ => {
// Clean dirty windows for full-query/non-scoped paths,
- // such as TQL, that cannot use a time-window filter.
- let dirty_windows_to_restore = {
- let mut state = self.state.write().unwrap();
- let dirty_windows_to_restore = state.dirty_time_windows.clone();
- state.dirty_time_windows.clean();
- dirty_windows_to_restore
- };
+ // such as TQL or evaluation-interval SQL without a recognized
+ // time-window expression, that cannot use a time-window filter.
+ let (_, dirty_windows_to_restore) = self.drain_dirty_windows_signal();
- let plan = match gen_plan_with_matching_schema(
- &self.config.query,
- query_ctx,
- engine,
- sink_table_schema.clone(),
- primary_key_indices,
- allow_partial,
- )
- .await
- {
- Ok(plan) => plan,
- Err(err) => {
- self.state
- .write()
- .unwrap()
- .dirty_time_windows
- .add_dirty_windows(&dirty_windows_to_restore);
- return Err(err);
- }
- };
+ let plan_info = self
+ .gen_unfiltered_plan_info(
+ engine,
+ query_ctx,
+ sink_table_schema.clone(),
+ primary_key_indices,
+ allow_partial,
+ dirty_windows_to_restore,
+ None,
+ )
+ .await?;
- return Ok(Some(PlanInfo {
- plan,
- dirty_restore: DirtyRestore::Unscoped(dirty_windows_to_restore),
- can_advance_checkpoints: true,
- }));
+ return Ok(Some(plan_info));
}
};
@@ -889,22 +1026,61 @@ impl BatchingTask {
),
})?;
+ if self.should_use_unfiltered_full_snapshot_seeding() {
+ // A full-snapshot query that can seed/refresh incremental
+ // checkpoints must not use dirty-window predicates. Rows can be
+ // written after dirty windows are drained but before the source scan
+ // snapshot opens; a stale dirty-window filter could exclude those
+ // rows while the returned watermark includes them, causing the next
+ // incremental read to skip them forever. Execute an unfiltered full
+ // snapshot instead, and keep dirty windows only as the scheduling and
+ // failure-restoration signal.
+ let retention_filter = self
+ .config
+ .expire_after
+ .map(|_| (col_name.as_str(), expire_lower_bound, "full-snapshot"));
+ return self
+ .gen_unfiltered_plan_info_if_dirty(
+ engine,
+ query_ctx,
+ sink_table_schema.clone(),
+ primary_key_indices,
+ allow_partial,
+ retention_filter,
+ )
+ .await;
+ }
+
+ if self.should_use_unfiltered_incremental_delta() {
+ // In incremental mode, source correctness is defined by the
+ // per-region sequence range `(checkpoint, scan-open snapshot]`, not
+ // by dirty-window predicates. Dirty windows are only a scheduling
+ // signal here. Applying a stale dirty-window filter to the source can
+ // exclude rows that are inside the returned watermark and make a
+ // checkpoint advance skip them forever. The sink side is also left
+ // unfiltered by dirty windows; the incremental rewrite joins the
+ // delta groups with the full sink state for correctness. Future
+ // dynamic filters can prune sink reads as a pure optimization.
+ let retention_filter = self
+ .config
+ .expire_after
+ .map(|_| (col_name.as_str(), expire_lower_bound, "incremental"));
+ return self
+ .gen_unfiltered_plan_info_if_dirty(
+ engine,
+ query_ctx,
+ sink_table_schema.clone(),
+ primary_key_indices,
+ allow_partial,
+ retention_filter,
+ )
+ .await;
+ }
+
let (expr, can_advance_checkpoints) = {
let mut state = self.state.write().unwrap();
- let window_cnt = if state.checkpoint_mode() == CheckpointMode::Incremental
- && !state.is_incremental_disabled()
- && matches!(self.config.query_type, QueryType::Sql)
- {
- // Incremental scans are bounded by region sequence checkpoints,
- // so the dirty-window filter only narrows sink-side/time-window
- // work. Drain more windows than normal, but keep a hard cap to
- // avoid building a huge OR filter after a long downtime. If
- // windows remain, checkpoints won't advance this round.
- MAX_INCREMENTAL_DIRTY_WINDOW_FILTERS
- } else {
- max_window_cnt
- .unwrap_or(self.config.batch_opts.experimental_max_filter_num_per_query)
- };
+ let window_cnt = max_window_cnt
+ .unwrap_or(self.config.batch_opts.experimental_max_filter_num_per_query);
let expr = state.dirty_time_windows.gen_filter_exprs(
&col_name,
Some(expire_lower_bound),
diff --git a/src/flow/src/batching_mode/task/inc.rs b/src/flow/src/batching_mode/task/inc.rs
index 4fb64a676e..9af54c1ba7 100644
--- a/src/flow/src/batching_mode/task/inc.rs
+++ b/src/flow/src/batching_mode/task/inc.rs
@@ -26,8 +26,7 @@ use snafu::ResultExt;
use table::metadata::TableId;
use crate::Error;
-use crate::batching_mode::incremental_filter::build_sink_dirty_time_window_filter_expr;
-use crate::batching_mode::state::{CheckpointMode, FilterExprInfo};
+use crate::batching_mode::state::CheckpointMode;
use crate::batching_mode::table_creator::QueryType;
use crate::batching_mode::task::BatchingTask;
use crate::batching_mode::utils::{
@@ -74,7 +73,6 @@ impl BatchingTask {
pub(super) async fn prepare_plan_for_incremental(
&self,
plan: &LogicalPlan,
- dirty_filter: Option<&FilterExprInfo>,
) -> Result, Error> {
let is_incremental_sql = {
let state = self.state.read().unwrap();
@@ -152,31 +150,12 @@ impl BatchingTask {
return Ok(None);
}
};
- let sink_schema = sink_table.table_info().meta.schema.clone();
- let sink_dirty_filter = match build_sink_dirty_time_window_filter_expr(
- self.config.flow_id,
- &analysis,
- &sink_schema,
- dirty_filter,
- ) {
- Ok(filter) => filter,
- Err(err) => {
- warn!(
- "Flow {} failed to build sink dirty time window filter; \
- falling back to full snapshot for this round: {:?}",
- self.config.flow_id, err
- );
- self.state.write().unwrap().mark_full_snapshot();
- return Ok(None);
- }
- };
-
let rewritten_inner = match rewrite_incremental_aggregate_with_sink_merge(
&inner_plan,
&analysis,
sink_table,
&self.config.sink_table_name,
- sink_dirty_filter,
+ None,
)
.await
{
diff --git a/src/flow/src/batching_mode/task/test.rs b/src/flow/src/batching_mode/task/test.rs
index 959aeb00c9..c42d564ce2 100644
--- a/src/flow/src/batching_mode/task/test.rs
+++ b/src/flow/src/batching_mode/task/test.rs
@@ -25,7 +25,9 @@ use datatypes::data_type::ConcreteDataType as CDT;
use datatypes::schema::ColumnSchema;
use datatypes::vectors::{TimestampMillisecondVector, UInt32Vector, VectorRef};
use pretty_assertions::assert_eq;
-use query::options::{FLOW_INCREMENTAL_AFTER_SEQS, FLOW_INCREMENTAL_MODE_MEMTABLE_ONLY};
+use query::options::{
+ FLOW_INCREMENTAL_AFTER_SEQS, FLOW_INCREMENTAL_MODE_MEMTABLE_ONLY, QueryOptions,
+};
use session::context::QueryContext;
use table::test_util::MemTable;
@@ -38,6 +40,13 @@ use crate::batching_mode::state::CheckpointMode;
use crate::batching_mode::time_window::find_time_window_expr;
use crate::test_utils::create_test_query_engine;
+fn incremental_batch_opts() -> Arc {
+ Arc::new(BatchingModeOptions {
+ experimental_enable_incremental_read: true,
+ ..Default::default()
+ })
+}
+
async fn new_test_task_and_plan_with_missing_sink() -> (BatchingTask, LogicalPlan) {
new_test_task_engine_and_plan_with_query(
"SELECT number, ts FROM numbers_with_ts",
@@ -60,6 +69,15 @@ impl TestTaskParts {
}
async fn new_test_task_engine_and_plan_with_query(query: &str, sink_table: &str) -> TestTaskParts {
+ new_test_task_engine_and_plan_with_query_and_opts(query, sink_table, incremental_batch_opts())
+ .await
+}
+
+async fn new_test_task_engine_and_plan_with_query_and_opts(
+ query: &str,
+ sink_table: &str,
+ batch_opts: Arc,
+) -> TestTaskParts {
let query_engine = create_test_query_engine();
let ctx = QueryContext::arc();
let plan = sql_to_df_plan(
@@ -91,7 +109,7 @@ async fn new_test_task_engine_and_plan_with_query(query: &str, sink_table: &str)
query_ctx: ctx,
catalog_manager: query_engine.engine_state().catalog_manager().clone(),
shutdown_rx: rx,
- batch_opts: Arc::new(BatchingModeOptions::default()),
+ batch_opts,
flow_eval_interval: None,
})
.unwrap();
@@ -103,6 +121,75 @@ async fn new_test_task_engine_and_plan_with_query(query: &str, sink_table: &str)
}
}
+#[tokio::test]
+async fn test_incremental_read_is_disabled_by_default() {
+ let task = new_test_task_engine_and_plan_with_query_and_opts(
+ "SELECT number, ts FROM numbers_with_ts",
+ "numbers_with_ts",
+ Arc::new(BatchingModeOptions::default()),
+ )
+ .await
+ .task;
+
+ assert!(task.state.read().unwrap().is_incremental_disabled());
+}
+
+#[tokio::test]
+async fn test_dirty_time_windows_uses_batch_opts() {
+ let task = new_test_task_engine_and_plan_with_query_and_opts(
+ "SELECT number, ts FROM numbers_with_ts",
+ "numbers_with_ts",
+ Arc::new(BatchingModeOptions {
+ experimental_max_filter_num_per_query: 7,
+ experimental_time_window_merge_threshold: 11,
+ ..Default::default()
+ }),
+ )
+ .await
+ .task;
+
+ let state = task.state.read().unwrap();
+ assert_eq!(7, state.dirty_time_windows.max_filter_num_per_query());
+ assert_eq!(11, state.dirty_time_windows.time_window_merge_threshold());
+}
+
+#[tokio::test]
+async fn test_execute_once_serialized_waits_for_execution_lock() {
+ let TestTaskParts {
+ task, query_engine, ..
+ } = new_test_task_engine_and_plan_with_query(
+ "SELECT number, ts FROM numbers_with_ts",
+ "missing_sink",
+ )
+ .await;
+ let (frontend_client, _handler) =
+ FrontendClient::from_empty_grpc_handler(QueryOptions::default());
+ let frontend_client = Arc::new(frontend_client);
+
+ let guard = task.execution_lock.clone().lock_owned().await;
+ let task_to_run = task.clone();
+ let query_engine_to_run = query_engine.clone();
+ let frontend_client_to_run = frontend_client.clone();
+ let exec = tokio::spawn(async move {
+ task_to_run
+ .execute_once_serialized(&query_engine_to_run, &frontend_client_to_run, None)
+ .await
+ });
+
+ tokio::time::sleep(Duration::from_millis(20)).await;
+ assert!(
+ !exec.is_finished(),
+ "execute_once_serialized should wait for execution_lock"
+ );
+
+ drop(guard);
+ tokio::time::timeout(Duration::from_secs(1), exec)
+ .await
+ .expect("execute_once_serialized should finish once execution_lock is released")
+ .expect("execute_once_serialized task should not panic")
+ .expect_err("missing sink should fail after acquiring execution_lock");
+}
+
async fn new_time_window_test_task_with_query(query: &str) -> TestTaskParts {
let query_engine = create_test_query_engine();
let ctx = QueryContext::arc();
@@ -147,7 +234,7 @@ async fn new_time_window_test_task_with_query(query: &str) -> TestTaskParts {
query_ctx: ctx,
catalog_manager: query_engine.engine_state().catalog_manager().clone(),
shutdown_rx: rx,
- batch_opts: Arc::new(BatchingModeOptions::default()),
+ batch_opts: incremental_batch_opts(),
flow_eval_interval: None,
})
.unwrap();
@@ -226,6 +313,14 @@ fn dirty_range(start: i64, end: i64) -> DirtyTimeWindows {
dirty
}
+fn expire_after_for_retention_filter_test() -> i64 {
+ let now_secs = SystemTime::now()
+ .duration_since(UNIX_EPOCH)
+ .expect("Time went backwards")
+ .as_secs();
+ (now_secs - 10) as i64
+}
+
async fn assert_unscoped_failure_restore(
consumed_dirty_windows: DirtyTimeWindows,
current_dirty_windows: DirtyTimeWindows,
@@ -626,6 +721,7 @@ async fn test_full_snapshot_scoped_plan_marks_checkpoint_advance_safe_only_after
.await;
{
let mut state = task.state.write().unwrap();
+ state.disable_incremental();
state
.dirty_time_windows
.add_window(Timestamp::new_second(0), Some(Timestamp::new_second(5)));
@@ -657,7 +753,7 @@ async fn test_full_snapshot_scoped_plan_marks_checkpoint_advance_safe_only_after
}
#[tokio::test]
-async fn test_incremental_scoped_plan_consumes_all_dirty_windows_for_checkpoint_safety() {
+async fn test_incremental_plan_consumes_dirty_signal_for_checkpoint_safety() {
let TestTaskParts {
task,
query_engine,
@@ -692,6 +788,224 @@ async fn test_incremental_scoped_plan_consumes_all_dirty_windows_for_checkpoint_
assert!(task.state.read().unwrap().dirty_time_windows.is_empty());
}
+#[tokio::test]
+async fn test_full_snapshot_seeding_for_incremental_does_not_add_dirty_window_filter() {
+ let TestTaskParts {
+ task,
+ query_engine,
+ ..
+ } = new_time_window_test_task_with_query(
+ "SELECT max(number) AS number, date_bin(INTERVAL '5 second', ts) AS time_window FROM numbers_with_ts GROUP BY time_window",
+ )
+ .await;
+ {
+ let mut state = task.state.write().unwrap();
+ assert_eq!(state.checkpoint_mode(), CheckpointMode::FullSnapshot);
+ assert!(!state.is_incremental_disabled());
+ state
+ .dirty_time_windows
+ .add_window(Timestamp::new_second(0), Some(Timestamp::new_second(5)));
+ state
+ .dirty_time_windows
+ .add_window(Timestamp::new_second(30), Some(Timestamp::new_second(35)));
+ }
+ let sink_schema = Arc::new(Schema::new(vec![
+ ColumnSchema::new("number", CDT::uint32_datatype(), false),
+ ColumnSchema::new("time_window", CDT::timestamp_millisecond_datatype(), false)
+ .with_time_index(true),
+ ]));
+
+ let plan = task
+ .gen_query_with_time_window(query_engine, &sink_schema, &[], false, Some(1))
+ .await
+ .unwrap()
+ .unwrap();
+
+ let plan_text = plan.plan.to_string();
+ assert!(plan.can_advance_checkpoints);
+ assert!(task.state.read().unwrap().dirty_time_windows.is_empty());
+ assert!(!plan_text.contains("Filter:"), "{plan_text}");
+}
+
+#[tokio::test]
+async fn test_full_snapshot_seeding_applies_expire_after_retention_filter() {
+ let TestTaskParts {
+ mut task,
+ query_engine,
+ ..
+ } = new_time_window_test_task_with_query(
+ "SELECT max(number) AS number, date_bin(INTERVAL '5 second', ts) AS time_window FROM numbers_with_ts GROUP BY time_window",
+ )
+ .await;
+ {
+ let mut state = task.state.write().unwrap();
+ assert_eq!(state.checkpoint_mode(), CheckpointMode::FullSnapshot);
+ assert!(!state.is_incremental_disabled());
+ state
+ .dirty_time_windows
+ .add_window(Timestamp::new_second(0), Some(Timestamp::new_second(5)));
+ }
+ let sink_schema = Arc::new(Schema::new(vec![
+ ColumnSchema::new("number", CDT::uint32_datatype(), false),
+ ColumnSchema::new("time_window", CDT::timestamp_millisecond_datatype(), false)
+ .with_time_index(true),
+ ]));
+
+ Arc::get_mut(&mut task.config)
+ .expect("test task config should be uniquely owned")
+ .expire_after = Some(expire_after_for_retention_filter_test());
+ let plan = task
+ .gen_query_with_time_window(query_engine, &sink_schema, &[], false, Some(1))
+ .await
+ .unwrap()
+ .unwrap();
+
+ assert!(plan.can_advance_checkpoints);
+ assert!(task.state.read().unwrap().dirty_time_windows.is_empty());
+ let plan_text = plan.plan.to_string();
+ assert!(
+ plan_text.contains("Filter: ts >= TimestampMillisecond("),
+ "{plan_text}"
+ );
+}
+
+#[tokio::test]
+async fn test_incremental_plan_does_not_add_dirty_window_filter() {
+ let TestTaskParts {
+ task,
+ query_engine,
+ ..
+ } = new_time_window_test_task_with_query(
+ "SELECT max(number) AS number, date_bin(INTERVAL '5 second', ts) AS time_window FROM numbers_with_ts GROUP BY time_window",
+ )
+ .await;
+ {
+ let mut state = task.state.write().unwrap();
+ state.advance_checkpoints(HashMap::from([(1_u64, 10_u64)]));
+ state
+ .dirty_time_windows
+ .add_window(Timestamp::new_second(0), Some(Timestamp::new_second(5)));
+ }
+ let sink_schema = Arc::new(Schema::new(vec![
+ ColumnSchema::new("number", CDT::uint32_datatype(), false),
+ ColumnSchema::new("time_window", CDT::timestamp_millisecond_datatype(), false)
+ .with_time_index(true),
+ ]));
+
+ let plan = task
+ .gen_query_with_time_window(query_engine, &sink_schema, &[], false, Some(1))
+ .await
+ .unwrap()
+ .unwrap();
+
+ let plan_text = plan.plan.to_string();
+ assert!(plan.can_advance_checkpoints);
+ assert!(!plan_text.contains("Filter:"), "{plan_text}");
+}
+
+#[tokio::test]
+async fn test_incremental_delta_applies_expire_after_retention_filter() {
+ let TestTaskParts {
+ mut task,
+ query_engine,
+ ..
+ } = new_time_window_test_task_with_query(
+ "SELECT max(number) AS number, date_bin(INTERVAL '5 second', ts) AS time_window FROM numbers_with_ts GROUP BY time_window",
+ )
+ .await;
+ {
+ let mut state = task.state.write().unwrap();
+ state.advance_checkpoints(HashMap::from([(1_u64, 10_u64)]));
+ state
+ .dirty_time_windows
+ .add_window(Timestamp::new_second(0), Some(Timestamp::new_second(5)));
+ }
+ let sink_schema = Arc::new(Schema::new(vec![
+ ColumnSchema::new("number", CDT::uint32_datatype(), false),
+ ColumnSchema::new("time_window", CDT::timestamp_millisecond_datatype(), false)
+ .with_time_index(true),
+ ]));
+
+ Arc::get_mut(&mut task.config)
+ .expect("test task config should be uniquely owned")
+ .expire_after = Some(expire_after_for_retention_filter_test());
+ let plan = task
+ .gen_query_with_time_window(query_engine, &sink_schema, &[], false, Some(1))
+ .await
+ .unwrap()
+ .unwrap();
+
+ assert!(plan.can_advance_checkpoints);
+ assert!(task.state.read().unwrap().dirty_time_windows.is_empty());
+ let plan_text = plan.plan.to_string();
+ assert!(
+ plan_text.contains("Filter: ts >= TimestampMillisecond("),
+ "{plan_text}"
+ );
+}
+
+#[tokio::test]
+async fn test_non_scoped_path_generates_plan_with_empty_dirty_signal() {
+ let TestTaskParts {
+ mut task,
+ query_engine,
+ ..
+ } = new_test_task_engine_and_plan_with_query(
+ "SELECT number, ts FROM numbers_with_ts",
+ "missing_sink",
+ )
+ .await;
+ Arc::get_mut(&mut task.config)
+ .expect("test task config should be uniquely owned")
+ .query_type = QueryType::Tql;
+ task.state.write().unwrap().dirty_time_windows.clean();
+ let sink_schema = Arc::new(Schema::new(vec![
+ ColumnSchema::new("number", CDT::uint32_datatype(), false),
+ ColumnSchema::new("ts", CDT::timestamp_millisecond_datatype(), false).with_time_index(true),
+ ]));
+
+ let plan = task
+ .gen_query_with_time_window(query_engine, &sink_schema, &[], false, None)
+ .await
+ .unwrap()
+ .expect("non-scoped path should generate a plan even with an empty dirty signal");
+
+ assert!(plan.can_advance_checkpoints);
+ assert!(task.state.read().unwrap().dirty_time_windows.is_empty());
+}
+
+#[tokio::test]
+async fn test_no_time_window_sql_with_eval_interval_generates_plan_without_dirty_signal() {
+ let TestTaskParts {
+ mut task,
+ query_engine,
+ ..
+ } = new_test_task_engine_and_plan_with_query(
+ "SELECT number, ts FROM numbers_with_ts",
+ "missing_sink",
+ )
+ .await;
+ Arc::get_mut(&mut task.config)
+ .expect("test task config should be uniquely owned")
+ .flow_eval_interval = Some(Duration::from_secs(60));
+ task.state.write().unwrap().dirty_time_windows.clean();
+ let sink_schema = Arc::new(Schema::new(vec![
+ ColumnSchema::new("number", CDT::uint32_datatype(), false),
+ ColumnSchema::new("ts", CDT::timestamp_millisecond_datatype(), false).with_time_index(true),
+ ]));
+
+ let plan = task
+ .gen_query_with_time_window(query_engine, &sink_schema, &[], false, None)
+ .await
+ .unwrap()
+ .expect(
+ "eval-interval SQL without a time-window expr should run by interval, not dirty signal",
+ );
+
+ assert!(plan.can_advance_checkpoints);
+ assert!(task.state.read().unwrap().dirty_time_windows.is_empty());
+}
+
#[tokio::test]
async fn test_executed_query_failure_restores_scoped_dirty_windows_for_flush_path() {
let (task, plan) = new_test_task_and_plan_with_missing_sink().await;
@@ -773,7 +1087,7 @@ async fn test_prepare_plan_for_incremental_disables_on_non_aggregate() {
query_ctx: ctx,
catalog_manager: query_engine.engine_state().catalog_manager().clone(),
shutdown_rx: rx,
- batch_opts: Arc::new(BatchingModeOptions::default()),
+ batch_opts: incremental_batch_opts(),
flow_eval_interval: None,
})
.unwrap();
@@ -788,10 +1102,7 @@ async fn test_prepare_plan_for_incremental_disables_on_non_aggregate() {
CheckpointMode::Incremental
);
- let incremental_plan = task
- .prepare_plan_for_incremental(&dml_plan, None)
- .await
- .unwrap();
+ let incremental_plan = task.prepare_plan_for_incremental(&dml_plan).await.unwrap();
assert!(incremental_plan.is_none());
let state = task.state.read().unwrap();
assert!(state.is_incremental_disabled());
@@ -852,7 +1163,7 @@ async fn test_prepare_plan_for_incremental_falls_back_without_disable_on_rewrite
query_ctx: ctx,
catalog_manager: query_engine.engine_state().catalog_manager().clone(),
shutdown_rx: rx,
- batch_opts: Arc::new(BatchingModeOptions::default()),
+ batch_opts: incremental_batch_opts(),
flow_eval_interval: None,
})
.unwrap();
@@ -866,10 +1177,7 @@ async fn test_prepare_plan_for_incremental_falls_back_without_disable_on_rewrite
CheckpointMode::Incremental
);
- let incremental_plan = task
- .prepare_plan_for_incremental(&dml_plan, None)
- .await
- .unwrap();
+ let incremental_plan = task.prepare_plan_for_incremental(&dml_plan).await.unwrap();
assert!(incremental_plan.is_none());
let state = task.state.read().unwrap();
assert!(!state.is_incremental_disabled());
@@ -928,7 +1236,7 @@ async fn test_prepare_plan_for_incremental_group_by_without_merge_columns_uses_o
query_ctx: ctx,
catalog_manager: query_engine.engine_state().catalog_manager().clone(),
shutdown_rx: rx,
- batch_opts: Arc::new(BatchingModeOptions::default()),
+ batch_opts: incremental_batch_opts(),
flow_eval_interval: None,
})
.unwrap();
@@ -939,7 +1247,7 @@ async fn test_prepare_plan_for_incremental_group_by_without_merge_columns_uses_o
.advance_checkpoints(HashMap::from([(1_u64, 10_u64)]));
let incremental_plan = task
- .prepare_plan_for_incremental(&dml_plan, None)
+ .prepare_plan_for_incremental(&dml_plan)
.await
.unwrap()
.expect("plain GROUP BY is incremental-safe without a rewrite");
@@ -962,7 +1270,7 @@ async fn test_auto_created_sql_aggregate_sink_reaches_incremental_safe() {
task.state.write().unwrap().dirty_time_windows.set_dirty();
let plan_info = task
- .gen_insert_plan(&query_engine, None)
+ .gen_insert_plan_unlocked(&query_engine, None)
.await
.unwrap()
.unwrap();
@@ -973,7 +1281,7 @@ async fn test_auto_created_sql_aggregate_sink_reaches_incremental_safe() {
.unwrap()
.advance_checkpoints(HashMap::from([(1_u64, 10_u64)]));
let incremental_plan = task
- .prepare_plan_for_incremental(&plan_info.plan, None)
+ .prepare_plan_for_incremental(&plan_info.plan)
.await
.unwrap();
let incremental_safe = incremental_plan.is_some();
@@ -1078,11 +1386,11 @@ async fn test_insert_plan_matching_failure_restores_consumed_dirty_marker() {
register_number_only_sink(&query_engine, sink_table);
task.state.write().unwrap().dirty_time_windows.set_dirty();
- let result = task.gen_insert_plan(&query_engine, None).await;
+ let result = task.gen_insert_plan_unlocked(&query_engine, None).await;
assert!(result.is_err());
let _err = match result {
- Ok(_) => panic!("gen_insert_plan should fail with a sink column mismatch"),
+ Ok(_) => panic!("gen_insert_plan_unlocked should fail with a sink column mismatch"),
Err(err) => err,
};
let state = task.state.read().unwrap();
diff --git a/src/flow/src/batching_mode/utils.rs b/src/flow/src/batching_mode/utils.rs
index e86b1ee3be..5e033c6ae7 100644
--- a/src/flow/src/batching_mode/utils.rs
+++ b/src/flow/src/batching_mode/utils.rs
@@ -33,9 +33,10 @@ use datafusion_common::{
};
use datafusion_expr::logical_plan::{Aggregate, TableScan};
use datafusion_expr::{
- Distinct, JoinType, LogicalPlan, LogicalPlanBuilder, Operator, Projection, and, binary_expr,
- bitwise_and, bitwise_or, bitwise_xor, is_null, or, when,
+ Distinct, ExprSchemable, JoinType, LogicalPlan, LogicalPlanBuilder, Operator, Projection, and,
+ binary_expr, bitwise_and, bitwise_or, bitwise_xor, is_null, or, when,
};
+use datatypes::prelude::ConcreteDataType;
use datatypes::schema::{ColumnSchema, SchemaRef};
use query::QueryEngineRef;
use query::parser::{DEFAULT_LOOKBACK_STRING, PromQuery, QueryLanguageParser, QueryStatement};
@@ -955,7 +956,7 @@ pub(crate) async fn gen_plan_with_matching_schema(
.clone()
.rewrite(&mut add_auto_column)
.with_context(|_| DatafusionSnafu {
- context: format!("Failed to rewrite plan:\n {}\n", plan),
+ context: "Failed to rewrite plan".to_string(),
})?
.data;
Ok(plan)
@@ -1090,33 +1091,23 @@ impl ColumnMatcherRewriter {
}
/// modify the exprs in place so that it matches the schema and some auto columns are added
- fn modify_project_exprs(&mut self, mut exprs: Vec) -> DfResult> {
+ fn modify_project_exprs(
+ &mut self,
+ mut exprs: Vec,
+ input_schema: &DFSchema,
+ ) -> DfResult> {
if self.allow_partial {
return self.modify_project_exprs_with_partial(exprs);
}
+ let original_exprs = exprs.clone();
+
let all_names = self
.schema
.column_schemas()
.iter()
.map(|c| c.name.clone())
.collect::>();
- // first match by position
- for (idx, expr) in exprs.iter_mut().enumerate() {
- if !all_names.contains(&expr.qualified_name().1)
- && let Some(col_name) = self
- .schema
- .column_schemas()
- .get(idx)
- .map(|c| c.name.clone())
- {
- // if the data type mismatched, later check_execute will error out
- // hence no need to check it here, beside, optimize pass might be able to cast it
- // so checking here is not necessary
- *expr = expr.clone().alias(col_name);
- }
- }
-
// add columns if have different column count
let query_col_cnt = exprs.len();
let table_col_cnt = self.schema.column_schemas().len();
@@ -1140,10 +1131,9 @@ impl ColumnMatcherRewriter {
// is the update at column
exprs.push(datafusion::prelude::now().alias(&last_col_schema.name));
} else {
- // helpful error message
- return Err(DataFusionError::Plan(format!(
- "Expect the last column in table to be timestamp column, found column {} with type {:?}",
- last_col_schema.name, last_col_schema.data_type
+ return Err(DataFusionError::Plan(format_flow_sink_schema_mismatch(
+ &original_exprs,
+ self.schema.as_ref(),
)));
}
} else if query_col_cnt + 2 == table_col_cnt {
@@ -1170,14 +1160,110 @@ impl ColumnMatcherRewriter {
)));
}
} else {
- return Err(DataFusionError::Plan(format!(
- "Expect table have 0,1 or 2 columns more than query columns, found {} query columns {:?}, {} table columns {:?}",
- query_col_cnt,
- exprs,
- table_col_cnt,
- self.schema.column_schemas()
+ return Err(DataFusionError::Plan(format_flow_sink_schema_mismatch(
+ &original_exprs,
+ self.schema.as_ref(),
)));
}
+
+ self.match_extra_output_columns(exprs, input_schema, &original_exprs, &all_names)
+ }
+
+ /// Match flow output columns whose names are not in the sink schema by the same position only.
+ ///
+ /// This keeps the legacy "omit output aliases and map by position" behavior, but only when the
+ /// sink column at the same index is actually missing from the flow output. If the extra output
+ /// would be aliased to a sink column that already exists elsewhere, report a schema mismatch
+ /// instead of guessing another sink column by type.
+ ///
+ /// In particular, this intentionally rejects cross-position remaps like
+ /// `record_time_window2 -> record_time_window`: they are easy to confuse with real schema
+ /// mismatches and should be fixed by giving the flow output the sink column name explicitly.
+ fn match_extra_output_columns(
+ &self,
+ mut exprs: Vec,
+ input_schema: &DFSchema,
+ original_exprs: &[Expr],
+ all_names: &BTreeSet,
+ ) -> DfResult> {
+ let mut output_names = exprs
+ .iter()
+ .map(|expr| expr.qualified_name().1)
+ .collect::>();
+ let output_name_set = output_names.iter().cloned().collect::>();
+ let extra_expr_indices = output_names
+ .iter()
+ .enumerate()
+ .filter_map(|(idx, name)| (!all_names.contains(name)).then_some(idx))
+ .collect::>();
+ let missing_sink_indices = self
+ .schema
+ .column_schemas()
+ .iter()
+ .enumerate()
+ .filter_map(|(idx, column)| (!output_name_set.contains(&column.name)).then_some(idx))
+ .collect::>();
+
+ if extra_expr_indices.is_empty() && missing_sink_indices.is_empty() {
+ return Ok(exprs);
+ }
+
+ if extra_expr_indices.len() != missing_sink_indices.len() {
+ return Err(DataFusionError::Plan(format_flow_sink_schema_mismatch(
+ original_exprs,
+ self.schema.as_ref(),
+ )));
+ }
+
+ let mut positional_matches = Vec::new();
+ for expr_idx in extra_expr_indices {
+ if !missing_sink_indices.contains(&expr_idx) {
+ return Err(DataFusionError::Plan(format_flow_sink_schema_mismatch(
+ original_exprs,
+ self.schema.as_ref(),
+ )));
+ }
+
+ let target_col_schema = &self.schema.column_schemas()[expr_idx];
+ let expr_type =
+ ConcreteDataType::from_arrow_type(&exprs[expr_idx].get_type(input_schema)?);
+ if is_obviously_incompatible_positional_match(&expr_type, &target_col_schema.data_type)
+ {
+ return Err(DataFusionError::Plan(format!(
+ "Cannot match flow output column '{}' to sink column '{}' by position: incompatible data types, flow output type is {:?}, sink column type is {:?}. {}",
+ output_names[expr_idx],
+ target_col_schema.name,
+ expr_type,
+ target_col_schema.data_type,
+ format_flow_sink_schema_mismatch(original_exprs, self.schema.as_ref())
+ )));
+ }
+
+ let target_name = target_col_schema.name.clone();
+ positional_matches.push(format!(
+ "{} -> {} (flow output type: {:?}, sink column type: {:?})",
+ output_names[expr_idx], target_name, expr_type, target_col_schema.data_type
+ ));
+ exprs[expr_idx] = exprs[expr_idx].clone().alias(target_name.clone());
+ output_names[expr_idx] = target_name;
+ }
+
+ if !positional_matches.is_empty() {
+ debug!(
+ "Matched flow output columns to sink columns by position: {:?}",
+ positional_matches
+ );
+ }
+
+ let duplicated_output_names = duplicate_names(&output_names);
+ if !duplicated_output_names.is_empty() {
+ return Err(DataFusionError::Plan(format!(
+ "Flow output schema contains duplicate column(s) after schema matching {:?}. {}",
+ duplicated_output_names,
+ format_flow_sink_schema_mismatch(&exprs, self.schema.as_ref())
+ )));
+ }
+
Ok(exprs)
}
@@ -1186,12 +1272,9 @@ impl ColumnMatcherRewriter {
let query_col_cnt = exprs.len();
if query_col_cnt > table_col_cnt {
- return Err(DataFusionError::Plan(format!(
- "Expect query column count <= table column count, found {} query columns {:?}, {} table columns {:?}",
- query_col_cnt,
- exprs,
- table_col_cnt,
- self.schema.column_schemas()
+ return Err(DataFusionError::Plan(format_flow_sink_schema_mismatch(
+ &exprs,
+ self.schema.as_ref(),
)));
}
@@ -1209,8 +1292,9 @@ impl ColumnMatcherRewriter {
.collect();
if !missing.is_empty() {
return Err(DataFusionError::Plan(format!(
- "Column(s) {:?} required by sink table are missing from flow output when merge_mode=last_non_null",
- missing
+ "Column(s) {:?} required by sink table are missing from flow output when merge_mode=last_non_null. {}",
+ missing,
+ format_flow_sink_schema_mismatch(&exprs, self.schema.as_ref())
)));
}
@@ -1250,8 +1334,9 @@ impl ColumnMatcherRewriter {
if !remap.is_empty() {
let extra: Vec<_> = remap.keys().cloned().collect();
return Err(DataFusionError::Plan(format!(
- "Flow output has extra column(s) {:?} not found in sink schema when merge_mode=last_non_null",
- extra
+ "Flow output has extra column(s) {:?} not found in sink schema when merge_mode=last_non_null. {}",
+ extra,
+ format_flow_sink_schema_mismatch(&exprs, self.schema.as_ref())
)));
}
@@ -1281,6 +1366,80 @@ impl ColumnMatcherRewriter {
}
}
+fn is_obviously_incompatible_positional_match(
+ expr_type: &ConcreteDataType,
+ sink_type: &ConcreteDataType,
+) -> bool {
+ // This is a coarse type-family guard for legacy positional aliasing, not a strict type equality
+ // check. For example, numeric width/sign differences are allowed here and left to downstream
+ // coercion, and untyped NULL can be coerced to any target type. Clearly different families such
+ // as timestamp vs string are rejected early.
+ if expr_type.is_null() || expr_type == sink_type {
+ return false;
+ }
+
+ expr_type.is_timestamp() != sink_type.is_timestamp()
+ || expr_type.is_string() != sink_type.is_string()
+ || expr_type.is_boolean() != sink_type.is_boolean()
+ || expr_type.is_json() != sink_type.is_json()
+ || expr_type.is_vector() != sink_type.is_vector()
+}
+
+fn duplicate_names(names: &[String]) -> Vec {
+ let mut seen = HashSet::new();
+ let mut duplicated = BTreeSet::new();
+ for name in names {
+ if !seen.insert(name.as_str()) {
+ duplicated.insert(name.as_str());
+ }
+ }
+ duplicated.into_iter().map(str::to_string).collect()
+}
+
+fn format_flow_sink_schema_mismatch(
+ query_exprs: &[Expr],
+ sink_schema: &datatypes::schema::Schema,
+) -> String {
+ let flow_output_columns = query_exprs
+ .iter()
+ .map(|expr| expr.qualified_name().1)
+ .collect::>();
+ let sink_table_columns = sink_schema
+ .column_schemas()
+ .iter()
+ .map(|col| col.name.clone())
+ .collect::>();
+
+ let flow_output_set = flow_output_columns.iter().cloned().collect::>();
+ let sink_table_set = sink_table_columns.iter().cloned().collect::>();
+
+ let mut extra_flow_columns = flow_output_columns
+ .iter()
+ .filter(|name| !sink_table_set.contains(*name))
+ .cloned()
+ .collect::>();
+ extra_flow_columns.sort();
+ extra_flow_columns.dedup();
+
+ let mut missing_sink_columns = sink_table_columns
+ .iter()
+ .filter(|name| !flow_output_set.contains(*name))
+ .cloned()
+ .collect::>();
+ missing_sink_columns.sort();
+ missing_sink_columns.dedup();
+
+ format!(
+ "Flow output schema does not match sink table schema: found {} flow output columns and {} sink table columns. flow output columns: {:?}, sink table columns: {:?}, extra flow columns not in sink: {:?}, missing sink columns from flow output: {:?}",
+ flow_output_columns.len(),
+ sink_table_columns.len(),
+ flow_output_columns,
+ sink_table_columns,
+ extra_flow_columns,
+ missing_sink_columns
+ )
+}
+
impl TreeNodeRewriter for ColumnMatcherRewriter {
type Node = LogicalPlan;
fn f_down(&mut self, mut node: Self::Node) -> DfResult> {
@@ -1327,7 +1486,7 @@ impl TreeNodeRewriter for ColumnMatcherRewriter {
// if not, wrap it in a projection
if let LogicalPlan::Projection(project) = &node {
let exprs = project.expr.clone();
- let exprs = self.modify_project_exprs(exprs)?;
+ let exprs = self.modify_project_exprs(exprs, project.input.schema())?;
self.is_rewritten = true;
let new_plan =
@@ -1341,7 +1500,7 @@ impl TreeNodeRewriter for ColumnMatcherRewriter {
field.name(),
)));
}
- let exprs = self.modify_project_exprs(exprs)?;
+ let exprs = self.modify_project_exprs(exprs, node.schema())?;
self.is_rewritten = true;
let new_plan =
LogicalPlan::Projection(Projection::try_new(exprs, Arc::new(node.clone()))?);
diff --git a/src/flow/src/batching_mode/utils/test.rs b/src/flow/src/batching_mode/utils/test.rs
index 5b9cf7f507..9ca1186fb6 100644
--- a/src/flow/src/batching_mode/utils/test.rs
+++ b/src/flow/src/batching_mode/utils/test.rs
@@ -14,6 +14,7 @@
use std::sync::Arc;
+use catalog::RegisterTableRequest;
use common_recordbatch::RecordBatch;
use common_time::Timestamp;
use datafusion_common::tree_node::TreeNode as _;
@@ -29,7 +30,9 @@ use substrait::{DFLogicalSubstraitConvertor, SubstraitPlan};
use table::test_util::MemTable;
use super::*;
+use crate::batching_mode::BatchingModeOptions;
use crate::batching_mode::state::FilterExprInfo;
+use crate::batching_mode::task::{BatchingTask, TaskArgs};
use crate::test_utils::create_test_query_engine;
fn u32_table(table_name: &str, columns: Vec<&str>, rows: usize) -> TableRef {
@@ -432,9 +435,7 @@ async fn test_add_auto_column_rewriter() {
// error datatype mismatch
(
"SELECT number, ts FROM numbers_with_ts",
- Err(
- "Expect the last column in table to be timestamp column, found column atat with type Int8",
- ),
+ Err("missing sink columns from flow output: [\"atat\"]"),
vec![
ColumnSchema::new("number", ConcreteDataType::int32_datatype(), true),
ColumnSchema::new(
@@ -498,6 +499,383 @@ async fn test_add_auto_column_rewriter() {
}
}
+#[tokio::test]
+async fn test_gen_plan_with_matching_schema_reports_extra_flow_columns_before_positional_alias() {
+ let query_engine = create_test_query_engine();
+ let ctx = QueryContext::arc();
+ let sink_schema = Arc::new(Schema::new(vec![
+ ColumnSchema::new("number", ConcreteDataType::uint32_datatype(), true),
+ ColumnSchema::new(
+ "ts",
+ ConcreteDataType::timestamp_millisecond_datatype(),
+ false,
+ )
+ .with_time_index(true),
+ ColumnSchema::new(
+ "max(numbers_with_ts.number)",
+ ConcreteDataType::uint32_datatype(),
+ true,
+ ),
+ ]));
+
+ let err = gen_plan_with_matching_schema(
+ "SELECT number, number AS extra, ts, max(number) FROM numbers_with_ts GROUP BY number, ts",
+ ctx,
+ query_engine,
+ sink_schema,
+ &[],
+ false,
+ )
+ .await
+ .unwrap_err()
+ .to_string();
+
+ assert!(
+ err.contains("Flow output schema does not match sink table schema"),
+ "{err}"
+ );
+ assert!(err.contains("flow output columns"), "{err}");
+ assert!(err.contains("sink table columns"), "{err}");
+ assert!(err.contains("extra flow columns not in sink"), "{err}");
+ assert!(err.contains("extra"), "{err}");
+ assert!(
+ !err.contains("extra AS ts"),
+ "schema error should not primarily expose positional alias: {err}"
+ );
+}
+
+#[tokio::test]
+async fn test_gen_plan_with_matching_schema_rejects_positional_alias_type_mismatch() {
+ let query_engine = create_test_query_engine();
+ let ctx = QueryContext::arc();
+ let sink_schema = Arc::new(Schema::new(vec![
+ ColumnSchema::new("number", ConcreteDataType::uint32_datatype(), true),
+ ColumnSchema::new(
+ "event_time",
+ ConcreteDataType::timestamp_millisecond_datatype(),
+ false,
+ )
+ .with_time_index(true),
+ ColumnSchema::new(
+ "max(numbers_with_ts.number)",
+ ConcreteDataType::uint32_datatype(),
+ true,
+ ),
+ ]));
+
+ let err = gen_plan_with_matching_schema(
+ "SELECT number, number AS not_time, max(number) FROM numbers_with_ts GROUP BY number",
+ ctx,
+ query_engine,
+ sink_schema,
+ &[],
+ false,
+ )
+ .await
+ .unwrap_err()
+ .to_string();
+
+ assert!(
+ err.contains(
+ "Cannot match flow output column 'not_time' to sink column 'event_time' by position"
+ ),
+ "{err}"
+ );
+ assert!(err.contains("incompatible data types"), "{err}");
+ assert!(
+ !err.contains("not_time AS event_time"),
+ "schema error should not expose an incompatible positional alias: {err}"
+ );
+}
+
+#[tokio::test]
+async fn test_gen_plan_with_matching_schema_rejects_cross_position_extra_column_match() {
+ let query_engine = create_test_query_engine();
+ let ctx = QueryContext::arc();
+ let sink_schema = Arc::new(Schema::new(vec![
+ ColumnSchema::new("number", ConcreteDataType::uint32_datatype(), true),
+ ColumnSchema::new(
+ "time_window",
+ ConcreteDataType::timestamp_millisecond_datatype(),
+ false,
+ )
+ .with_time_index(true),
+ ColumnSchema::new(
+ "ts",
+ ConcreteDataType::timestamp_millisecond_datatype(),
+ true,
+ ),
+ ]));
+
+ let err = gen_plan_with_matching_schema(
+ "SELECT number, ts, date_bin('5 minutes', ts) AS time_window2 FROM numbers_with_ts GROUP BY number, ts, time_window2",
+ ctx,
+ query_engine,
+ sink_schema,
+ &[],
+ false,
+ )
+ .await
+ .unwrap_err()
+ .to_string();
+
+ assert!(
+ err.contains("Flow output schema does not match sink table schema"),
+ "{err}"
+ );
+ assert!(err.contains("time_window2"), "{err}");
+ assert!(err.contains("time_window"), "{err}");
+ assert!(!err.contains("DuplicateUnqualifiedField"), "{err}");
+}
+
+#[tokio::test]
+async fn test_gen_plan_with_matching_schema_accepts_out_of_order_matching_names() {
+ let query_engine = create_test_query_engine();
+ let ctx = QueryContext::arc();
+ let sink_schema = Arc::new(Schema::new(vec![
+ ColumnSchema::new("number", ConcreteDataType::uint32_datatype(), true),
+ ColumnSchema::new(
+ "time_window",
+ ConcreteDataType::timestamp_millisecond_datatype(),
+ false,
+ )
+ .with_time_index(true),
+ ColumnSchema::new(
+ "ts",
+ ConcreteDataType::timestamp_millisecond_datatype(),
+ true,
+ ),
+ ]));
+
+ let plan = gen_plan_with_matching_schema(
+ "SELECT number, ts, date_bin('5 minutes', ts) AS time_window FROM numbers_with_ts GROUP BY number, ts, time_window",
+ ctx,
+ query_engine,
+ sink_schema,
+ &[],
+ false,
+ )
+ .await
+ .unwrap();
+ let output_names = plan
+ .schema()
+ .fields()
+ .iter()
+ .map(|field| field.name().clone())
+ .collect::>();
+
+ assert_eq!(
+ output_names,
+ vec![
+ "number".to_string(),
+ "ts".to_string(),
+ "time_window".to_string()
+ ]
+ );
+ assert!(duplicate_names(&output_names).is_empty());
+}
+
+#[tokio::test]
+async fn test_gen_plan_with_matching_schema_allows_numeric_positional_alias() {
+ let query_engine = create_test_query_engine();
+ let ctx = QueryContext::arc();
+ let sink_schema = Arc::new(Schema::new(vec![
+ ColumnSchema::new("renamed_number", ConcreteDataType::int64_datatype(), true),
+ ColumnSchema::new(
+ "ts",
+ ConcreteDataType::timestamp_millisecond_datatype(),
+ false,
+ )
+ .with_time_index(true),
+ ]));
+
+ let plan = gen_plan_with_matching_schema(
+ "SELECT number, ts FROM numbers_with_ts",
+ ctx,
+ query_engine,
+ sink_schema,
+ &[],
+ false,
+ )
+ .await
+ .unwrap();
+ let sql = df_plan_to_sql(&plan).unwrap();
+
+ assert_eq!(
+ "SELECT numbers_with_ts.number AS renamed_number, numbers_with_ts.ts FROM numbers_with_ts",
+ sql
+ );
+}
+
+#[tokio::test]
+async fn test_gen_plan_with_matching_schema_allows_null_positional_alias() {
+ let query_engine = create_test_query_engine();
+ let ctx = QueryContext::arc();
+ let sink_schema = Arc::new(Schema::new(vec![
+ ColumnSchema::new("number", ConcreteDataType::uint32_datatype(), true),
+ ColumnSchema::new("label", ConcreteDataType::string_datatype(), true),
+ ]));
+
+ let plan = gen_plan_with_matching_schema(
+ "SELECT number, NULL AS label_placeholder FROM numbers_with_ts",
+ ctx,
+ query_engine,
+ sink_schema,
+ &[],
+ false,
+ )
+ .await
+ .unwrap();
+ let output_names = plan
+ .schema()
+ .fields()
+ .iter()
+ .map(|field| field.name().clone())
+ .collect::>();
+ let sql = df_plan_to_sql(&plan).unwrap();
+
+ assert_eq!(
+ output_names,
+ vec!["number".to_string(), "label".to_string()]
+ );
+ assert!(sql.contains("NULL AS label"), "{sql}");
+}
+
+#[tokio::test]
+async fn test_gen_plan_with_matching_schema_accepts_matching_flow_schema() {
+ let query_engine = create_test_query_engine();
+ let ctx = QueryContext::arc();
+ let sink_schema = Arc::new(Schema::new(vec![
+ ColumnSchema::new("number", ConcreteDataType::uint32_datatype(), true),
+ ColumnSchema::new("extra", ConcreteDataType::uint32_datatype(), true),
+ ColumnSchema::new(
+ "ts",
+ ConcreteDataType::timestamp_millisecond_datatype(),
+ false,
+ )
+ .with_time_index(true),
+ ColumnSchema::new(
+ "max(numbers_with_ts.number)",
+ ConcreteDataType::uint32_datatype(),
+ true,
+ ),
+ ]));
+
+ let plan = gen_plan_with_matching_schema(
+ "SELECT number, number AS extra, ts, max(number) FROM numbers_with_ts GROUP BY number, ts",
+ ctx,
+ query_engine,
+ sink_schema,
+ &[],
+ false,
+ )
+ .await
+ .unwrap();
+ let sql = df_plan_to_sql(&plan).unwrap();
+
+ assert_eq!(
+ "SELECT numbers_with_ts.number, numbers_with_ts.number AS extra, numbers_with_ts.ts, max(numbers_with_ts.number) FROM numbers_with_ts GROUP BY numbers_with_ts.number, numbers_with_ts.ts",
+ sql
+ );
+}
+
+#[tokio::test]
+async fn test_validate_sink_table_schema_rejects_existing_sink_missing_flow_column() {
+ let query_engine = create_test_query_engine();
+ let query_ctx = QueryContext::arc();
+ let sql = "SELECT number, number AS extra, max(number) FROM numbers_with_ts GROUP BY number";
+ let plan = sql_to_df_plan(query_ctx.clone(), query_engine.clone(), sql, true)
+ .await
+ .unwrap();
+
+ let catalog_manager = catalog::memory::new_memory_catalog_manager().unwrap();
+ let sink_table_name = [
+ "greptime".to_string(),
+ "public".to_string(),
+ "existing_sink".to_string(),
+ ];
+ let sink_table = u32_table(
+ "existing_sink",
+ vec!["number", "max(numbers_with_ts.number)"],
+ 0,
+ );
+ catalog_manager
+ .register_table_sync(RegisterTableRequest {
+ catalog: sink_table_name[0].clone(),
+ schema: sink_table_name[1].clone(),
+ table_name: sink_table_name[2].clone(),
+ table_id: 4096,
+ table: sink_table,
+ })
+ .unwrap();
+
+ let (_shutdown_tx, shutdown_rx) = tokio::sync::oneshot::channel();
+ let task = BatchingTask::try_new(TaskArgs {
+ flow_id: 1,
+ query: sql,
+ plan,
+ time_window_expr: None,
+ expire_after: None,
+ sink_table_name,
+ source_table_names: vec![[
+ "greptime".to_string(),
+ "public".to_string(),
+ "numbers_with_ts".to_string(),
+ ]],
+ query_ctx,
+ catalog_manager,
+ shutdown_rx,
+ batch_opts: Arc::new(BatchingModeOptions::default()),
+ flow_eval_interval: None,
+ })
+ .unwrap();
+
+ let err = task
+ .validate_sink_table_schema(&query_engine)
+ .await
+ .unwrap_err()
+ .to_string();
+
+ assert!(
+ err.contains("Flow output schema does not match sink table schema"),
+ "{err}"
+ );
+ assert!(err.contains("extra"), "{err}");
+}
+
+#[tokio::test]
+async fn test_gen_plan_with_matching_schema_allow_partial_fills_nullable_columns() {
+ let query_engine = create_test_query_engine();
+ let ctx = QueryContext::arc();
+ let sink_schema = Arc::new(Schema::new(vec![
+ ColumnSchema::new("number", ConcreteDataType::uint32_datatype(), false),
+ ColumnSchema::new(
+ "ts",
+ ConcreteDataType::timestamp_millisecond_datatype(),
+ false,
+ )
+ .with_time_index(true),
+ ColumnSchema::new("optional_value", ConcreteDataType::uint32_datatype(), true),
+ ]));
+
+ let plan = gen_plan_with_matching_schema(
+ "SELECT number, ts FROM numbers_with_ts",
+ ctx,
+ query_engine,
+ sink_schema,
+ &[0],
+ true,
+ )
+ .await
+ .unwrap();
+ let sql = df_plan_to_sql(&plan).unwrap();
+
+ assert_eq!(
+ "SELECT numbers_with_ts.number, numbers_with_ts.ts, NULL AS optional_value FROM numbers_with_ts",
+ sql
+ );
+}
+
#[tokio::test]
async fn test_find_group_by_exprs() {
let testcases = vec![
@@ -1288,9 +1666,10 @@ async fn test_rewrite_incremental_aggregate_with_left_join() {
#[tokio::test]
async fn test_rewrite_incremental_aggregate_filters_sink_dirty_time_window() {
- // This verifies the rewrite placement when callers supply an already
- // inferred sink dirty-window predicate. The task-level inference rules are
- // covered by `infer_sink_time_window_filter_col` tests in task.rs.
+ // This verifies the rewrite placement when callers supply a sink predicate.
+ // The production incremental flow path currently leaves sink scans
+ // unfiltered for correctness and relies on future dynamic filters for
+ // pruning.
let query_engine = create_test_query_engine();
let ctx = QueryContext::arc();
let sql = "SELECT max(number) AS number, date_bin(INTERVAL '1 second', ts) AS time_window FROM numbers_with_ts GROUP BY time_window";
@@ -1490,3 +1869,118 @@ async fn test_analyze_incremental_aggregate_plan_rejects_cast_wrapped_alias() {
);
}
}
+
+#[tokio::test]
+async fn test_gen_plan_with_matching_schema_last_non_null_rejects_missing_primary_key_column() {
+ let query_engine = create_test_query_engine();
+ let ctx = QueryContext::arc();
+ // Sink table with primary_key_indices=[0] ("number"), time_index="ts", and merge_mode=last_non_null.
+ // The flow query omits "number", which is a required primary-key column.
+ let sink_schema = Arc::new(Schema::new(vec![
+ ColumnSchema::new("number", ConcreteDataType::uint32_datatype(), true),
+ ColumnSchema::new(
+ "ts",
+ ConcreteDataType::timestamp_millisecond_datatype(),
+ false,
+ )
+ .with_time_index(true),
+ ColumnSchema::new("optional_value", ConcreteDataType::uint32_datatype(), true),
+ ]));
+
+ let err = gen_plan_with_matching_schema(
+ "SELECT ts FROM numbers_with_ts",
+ ctx,
+ query_engine,
+ sink_schema,
+ &[0],
+ true,
+ )
+ .await
+ .unwrap_err()
+ .to_string();
+
+ assert!(
+ err.contains(
+ "required by sink table are missing from flow output when merge_mode=last_non_null"
+ ),
+ "{err}"
+ );
+ assert!(err.contains("number"), "{err}");
+}
+
+#[tokio::test]
+async fn test_gen_plan_with_matching_schema_last_non_null_rejects_missing_time_index_column() {
+ let query_engine = create_test_query_engine();
+ let ctx = QueryContext::arc();
+ // Sink table with primary_key_indices=[0] ("number"), time_index="ts", and merge_mode=last_non_null.
+ // The flow query omits "ts", which is a required time-index column.
+ let sink_schema = Arc::new(Schema::new(vec![
+ ColumnSchema::new("number", ConcreteDataType::uint32_datatype(), true),
+ ColumnSchema::new(
+ "ts",
+ ConcreteDataType::timestamp_millisecond_datatype(),
+ false,
+ )
+ .with_time_index(true),
+ ColumnSchema::new("optional_value", ConcreteDataType::uint32_datatype(), true),
+ ]));
+
+ let err = gen_plan_with_matching_schema(
+ "SELECT number FROM numbers_with_ts",
+ ctx,
+ query_engine,
+ sink_schema,
+ &[0],
+ true,
+ )
+ .await
+ .unwrap_err()
+ .to_string();
+
+ assert!(
+ err.contains(
+ "required by sink table are missing from flow output when merge_mode=last_non_null"
+ ),
+ "{err}"
+ );
+ assert!(err.contains("ts"), "{err}");
+}
+
+#[tokio::test]
+async fn test_gen_plan_with_matching_schema_last_non_null_rejects_extra_flow_column() {
+ let query_engine = create_test_query_engine();
+ let ctx = QueryContext::arc();
+ // Sink table with merge_mode=last_non_null.
+ // Sink has 3 columns: number (pk), ts (time_index), optional_value (nullable).
+ // Flow outputs: number, number AS extra, ts → "extra" is not in sink schema.
+ // query_col_cnt(3) <= table_col_cnt(3), so the extra branch is reached.
+ let sink_schema = Arc::new(Schema::new(vec![
+ ColumnSchema::new("number", ConcreteDataType::uint32_datatype(), true),
+ ColumnSchema::new(
+ "ts",
+ ConcreteDataType::timestamp_millisecond_datatype(),
+ false,
+ )
+ .with_time_index(true),
+ ColumnSchema::new("optional_value", ConcreteDataType::uint32_datatype(), true),
+ ]));
+
+ let err = gen_plan_with_matching_schema(
+ "SELECT number, number AS extra, ts FROM numbers_with_ts",
+ ctx,
+ query_engine,
+ sink_schema,
+ &[0],
+ true,
+ )
+ .await
+ .unwrap_err()
+ .to_string();
+
+ assert!(err.contains("extra column(s)"), "{err}");
+ assert!(err.contains("extra"), "{err}");
+ assert!(
+ err.contains("Flow output schema does not match sink table schema"),
+ "{err}"
+ );
+}
diff --git a/src/flow/src/server.rs b/src/flow/src/server.rs
index 913128f386..c2d986f645 100644
--- a/src/flow/src/server.rs
+++ b/src/flow/src/server.rs
@@ -566,11 +566,15 @@ impl FrontendInvoker {
name: TABLE_FLOWNODE_SET_CACHE_NAME,
})?;
+ // TODO(auto_create_table): flow sink tables are created through a controlled
+ // `CREATE FLOW` path, not client writes, so they are intentionally exempt from
+ // the frontend's global auto-create switch. Revisit if flow should honor it.
let inserter = Arc::new(Inserter::new(
catalog_manager.clone(),
partition_manager.clone(),
node_manager.clone(),
table_flownode_cache,
+ true,
));
let deleter = Arc::new(Deleter::new(
diff --git a/src/frontend/src/frontend.rs b/src/frontend/src/frontend.rs
index fb3b096f06..918185cb8f 100644
--- a/src/frontend/src/frontend.rs
+++ b/src/frontend/src/frontend.rs
@@ -44,6 +44,11 @@ pub struct FrontendOptions {
pub node_id: Option,
pub default_timezone: Option,
pub default_column_prefix: Option,
+ /// Server-side global switch for auto table creation on write.
+ /// Acts as an upper bound: when `false`, missing tables are never auto-created
+ /// even if a request sets the `auto_create_table` hint to `true`. When `true`
+ /// (default), the per-request hint still applies. Default: `true`.
+ pub auto_create_table: bool,
/// Maximum total memory for all concurrent write request bodies and messages (HTTP, gRPC, Flight).
/// Set to 0 to disable the limit. Default: "0" (unlimited)
pub max_in_flight_write_bytes: ReadableSize,
@@ -82,6 +87,7 @@ impl Default for FrontendOptions {
node_id: None,
default_timezone: None,
default_column_prefix: None,
+ auto_create_table: true,
max_in_flight_write_bytes: ReadableSize(0),
write_bytes_exhausted_policy: OnExhaustedPolicy::default(),
http: HttpOptions::default(),
diff --git a/src/frontend/src/instance/builder.rs b/src/frontend/src/instance/builder.rs
index ff857ed768..526d8aac73 100644
--- a/src/frontend/src/instance/builder.rs
+++ b/src/frontend/src/instance/builder.rs
@@ -185,6 +185,7 @@ impl FrontendBuilder {
partition_manager.clone(),
node_manager.clone(),
table_flownode_cache,
+ self.options.auto_create_table,
));
let deleter = Arc::new(Deleter::new(
self.catalog_manager.clone(),
diff --git a/src/frontend/src/instance/otlp.rs b/src/frontend/src/instance/otlp.rs
index 434a413fed..757f657c1e 100644
--- a/src/frontend/src/instance/otlp.rs
+++ b/src/frontend/src/instance/otlp.rs
@@ -43,7 +43,12 @@ use servers::query_handler::{
};
use session::context::QueryContextRef;
use snafu::{IntoError, ResultExt};
-use table::requests::{OTLP_METRIC_COMPAT_KEY, OTLP_METRIC_COMPAT_PROM};
+use table::requests::{
+ OTLP_METRIC_COMPAT_KEY, OTLP_METRIC_COMPAT_PROM, SEMANTIC_PIPELINE, SEMANTIC_SIGNAL_TYPE,
+ SEMANTIC_SOURCE, SEMANTIC_TRACE_CONVENTIONS, SEMANTIC_TRACE_HAS_EVENTS,
+ SEMANTIC_TRACE_HAS_LINKS, SEMANTIC_VALUE_UNKNOWN, SIGNAL_TYPE_LOG, SIGNAL_TYPE_METRIC,
+ SIGNAL_TYPE_TRACE, SOURCE_OPENTELEMETRY, TABLE_DATA_MODEL_TRACE_V1,
+};
use crate::instance::Instance;
use crate::instance::otlp::trace_semconv::trace_semconv_fixed_type;
@@ -131,12 +136,14 @@ impl OpenTelemetryProtocolHandler for Instance {
let (requests, rows) = otlp::metrics::to_grpc_insert_requests(request, &mut metric_ctx)?;
OTLP_METRICS_ROWS.inc_by(rows as u64);
- let ctx = if !is_legacy {
+ let ctx = {
let mut c = (*ctx).clone();
- c.set_extension(OTLP_METRIC_COMPAT_KEY, OTLP_METRIC_COMPAT_PROM.to_string());
+ c.set_extension(SEMANTIC_SIGNAL_TYPE, SIGNAL_TYPE_METRIC);
+ c.set_extension(SEMANTIC_SOURCE, SOURCE_OPENTELEMETRY);
+ if !is_legacy {
+ c.set_extension(OTLP_METRIC_COMPAT_KEY, OTLP_METRIC_COMPAT_PROM.to_string());
+ }
Arc::new(c)
- } else {
- ctx
};
// If the user uses the legacy path, it is by default without metric engine.
@@ -211,6 +218,15 @@ impl OpenTelemetryProtocolHandler for Instance {
.get::>();
interceptor_ref.pre_execute(ctx.clone())?;
+ // `as_req_iter` clones this ctx into each `temp_ctx`, so identity set here
+ // reaches the context that drives table auto-create.
+ let ctx = {
+ let mut c = (*ctx).clone();
+ c.set_extension(SEMANTIC_SIGNAL_TYPE, SIGNAL_TYPE_LOG);
+ c.set_extension(SEMANTIC_SOURCE, SOURCE_OPENTELEMETRY);
+ Arc::new(c)
+ };
+
let opt_req = otlp::logs::to_grpc_insert_requests(
request,
pipeline,
@@ -256,6 +272,23 @@ impl Instance {
ctx: QueryContextRef,
) -> ServerResult {
let is_trace_v1_model = matches!(pipeline, PipelineWay::OtlpTraceDirectV1);
+
+ // Only the main span table gets the identity; the derived `_services` /
+ // `_operations` lookup tables keep the unstamped `ctx`.
+ let main_ctx = {
+ let mut c = (*ctx).clone();
+ c.set_extension(SEMANTIC_SIGNAL_TYPE, SIGNAL_TYPE_TRACE);
+ c.set_extension(SEMANTIC_SOURCE, SOURCE_OPENTELEMETRY);
+ if is_trace_v1_model {
+ c.set_extension(SEMANTIC_PIPELINE, TABLE_DATA_MODEL_TRACE_V1);
+ c.set_extension(SEMANTIC_TRACE_HAS_EVENTS, "true");
+ c.set_extension(SEMANTIC_TRACE_HAS_LINKS, "true");
+ // schema_url is row-level, so conventions is unknown at table level.
+ c.set_extension(SEMANTIC_TRACE_CONVENTIONS, SEMANTIC_VALUE_UNKNOWN);
+ }
+ Arc::new(c)
+ };
+
let ingest_ctx = TraceChunkIngestContext {
pipeline_handler,
pipeline,
@@ -278,7 +311,7 @@ impl Instance {
.map(|chunk| chunk.collect::>())
.collect::>();
for chunk in chunks {
- self.ingest_trace_chunk(&ingest_ctx, chunk, ctx.clone(), &mut ingest_state)
+ self.ingest_trace_chunk(&ingest_ctx, chunk, main_ctx.clone(), &mut ingest_state)
.await?;
}
}
diff --git a/src/frontend/src/server.rs b/src/frontend/src/server.rs
index e66ae718ba..6b120ccba5 100644
--- a/src/frontend/src/server.rs
+++ b/src/frontend/src/server.rs
@@ -288,7 +288,6 @@ where
let http_server = builder
.with_metrics_handler(MetricsHandler)
- .with_plugins(self.plugins.clone())
.with_greptime_config_options(toml)
.build();
Ok(http_server)
diff --git a/src/meta-client/src/client.rs b/src/meta-client/src/client.rs
index cbd9b43151..de41caf19e 100644
--- a/src/meta-client/src/client.rs
+++ b/src/meta-client/src/client.rs
@@ -1344,7 +1344,7 @@ mod tests {
// Generates rough 10MB data, which is larger than the default grpc message size limit.
for i in 0..10 {
- let data: Vec = (0..1024 * 1024).map(|_| rng.random()).collect();
+ let data: Vec = (0..1024 * 1024).map(|_| rng.random::()).collect();
in_memory
.put(
PutRequest::new()
diff --git a/src/meta-srv/src/procedure/region_migration.rs b/src/meta-srv/src/procedure/region_migration.rs
index 6563da63bc..c0ab352fc6 100644
--- a/src/meta-srv/src/procedure/region_migration.rs
+++ b/src/meta-srv/src/procedure/region_migration.rs
@@ -39,6 +39,7 @@ use common_meta::ddl::RegionFailureDetectorControllerRef;
use common_meta::instruction::CacheIdent;
use common_meta::key::datanode_table::{DatanodeTableKey, DatanodeTableValue};
use common_meta::key::table_route::TableRouteValue;
+use common_meta::key::topic_name::TopicNameKey;
use common_meta::key::topic_region::{ReplayCheckpoint, TopicRegionKey};
use common_meta::key::{DeserializedValueWithBytes, TableMetadataManagerRef};
use common_meta::kv_backend::{KvBackendRef, ResettableKvBackendRef};
@@ -661,11 +662,15 @@ impl Context {
.await;
}
- /// Fetches the replay checkpoints for the given topic region keys.
- pub async fn get_replay_checkpoints(
+ /// Fetches replay checkpoints and merges them with topic pruned entry ids.
+ pub async fn get_replay_checkpoints_with_topic_pruned_entry_ids(
&self,
- topic_region_keys: Vec>,
+ region_topics: &[(RegionId, String, bool)],
) -> Result> {
+ let topic_region_keys = region_topics
+ .iter()
+ .map(|(region_id, topic, _)| TopicRegionKey::new(*region_id, topic))
+ .collect::>();
let topic_region_values = self
.table_metadata_manager
.topic_region_manager()
@@ -673,9 +678,37 @@ impl Context {
.await
.context(error::TableMetadataManagerSnafu)?;
- let replay_checkpoints = topic_region_values
+ let topic_name_keys = region_topics
+ .iter()
+ .map(|(_, topic, _)| topic.as_str())
+ .collect::>()
.into_iter()
- .flat_map(|(key, value)| value.checkpoint.map(|value| (key, value)))
+ .map(TopicNameKey::new)
+ .collect::>();
+ let topic_name_values = self
+ .table_metadata_manager
+ .topic_name_manager()
+ .batch_get(topic_name_keys)
+ .await
+ .context(error::TableMetadataManagerSnafu)?;
+
+ let replay_checkpoints = region_topics
+ .iter()
+ .filter_map(|(region_id, topic, is_metric_engine)| {
+ let checkpoint = topic_region_values
+ .get(region_id)
+ .and_then(|value| value.checkpoint);
+ let pruned_entry_id = topic_name_values
+ .get(topic)
+ .map(|value| value.pruned_entry_id);
+
+ ReplayCheckpoint::merge_with_topic_pruned_entry_id(
+ checkpoint,
+ pruned_entry_id,
+ *is_metric_engine,
+ )
+ .map(|checkpoint| (*region_id, checkpoint))
+ })
.collect::>();
Ok(replay_checkpoints)
diff --git a/src/meta-srv/src/procedure/region_migration/open_candidate_region.rs b/src/meta-srv/src/procedure/region_migration/open_candidate_region.rs
index 0c0e5de5d7..792c66bdc9 100644
--- a/src/meta-srv/src/procedure/region_migration/open_candidate_region.rs
+++ b/src/meta-srv/src/procedure/region_migration/open_candidate_region.rs
@@ -18,7 +18,9 @@ use std::ops::Div;
use api::v1::meta::MailboxMessage;
use common_meta::RegionIdent;
use common_meta::distributed_time_constants::default_distributed_time_constants;
-use common_meta::instruction::{Instruction, InstructionReply, OpenRegion, SimpleReply};
+use common_meta::instruction::{
+ Instruction, InstructionReply, OpenRegion, OpenRegionReason, SimpleReply,
+};
use common_meta::key::datanode_table::RegionInfo;
use common_procedure::{Context as ProcedureContext, Status};
use common_telemetry::info;
@@ -26,12 +28,13 @@ use common_telemetry::tracing_context::TracingContext;
use serde::{Deserialize, Serialize};
use snafu::{OptionExt, ResultExt};
use store_api::region_engine::RegionRole;
+use store_api::region_request::RegionRequirements;
use tokio::time::Instant;
use crate::error::{self, Result};
use crate::handler::HeartbeatMailbox;
use crate::procedure::region_migration::flush_leader_region::PreFlushRegion;
-use crate::procedure::region_migration::{Context, State};
+use crate::procedure::region_migration::{Context, RegionMigrationTriggerReason, State};
use crate::service::mailbox::Channel;
#[derive(Debug, Serialize, Deserialize)]
@@ -67,6 +70,10 @@ impl OpenCandidateRegion {
let region_ids = ctx.persistent_ctx.region_ids.clone();
let from_peer_id = ctx.persistent_ctx.from_peer.id;
let to_peer_id = ctx.persistent_ctx.to_peer.id;
+ let reason = match ctx.persistent_ctx.trigger_reason {
+ RegionMigrationTriggerReason::Failover => OpenRegionReason::RegionFailover,
+ _ => OpenRegionReason::RegionMigration,
+ };
let datanode_table_values = ctx.get_from_peer_datanode_table_values().await?;
let mut open_regions = Vec::with_capacity(region_ids.len());
@@ -97,6 +104,8 @@ impl OpenCandidateRegion {
region_options,
region_wal_options,
true,
+ Some(reason),
+ RegionRequirements::object_storage(),
));
}
@@ -233,18 +242,20 @@ mod tests {
}
fn new_mock_open_instruction(datanode_id: DatanodeId, region_id: RegionId) -> Instruction {
- Instruction::OpenRegions(vec![OpenRegion {
- region_ident: RegionIdent {
+ Instruction::OpenRegions(vec![OpenRegion::new(
+ RegionIdent {
datanode_id,
table_id: region_id.table_id(),
region_number: region_id.region_number(),
engine: MITO2_ENGINE.to_string(),
},
- region_storage_path: "/bar/foo/region/".to_string(),
- region_options: Default::default(),
- region_wal_options: Default::default(),
- skip_wal_replay: true,
- }])
+ "/bar/foo/region/",
+ Default::default(),
+ Default::default(),
+ true,
+ Some(OpenRegionReason::RegionMigration),
+ RegionRequirements::object_storage(),
+ )])
}
#[tokio::test]
@@ -263,6 +274,57 @@ mod tests {
assert!(!err.is_retryable());
}
+ #[tokio::test]
+ async fn test_build_open_region_instruction_reason() {
+ let state = OpenCandidateRegion;
+ let mut persistent_context = new_persistent_context();
+ let from_peer_id = persistent_context.from_peer.id;
+ let region_id = persistent_context.region_ids[0];
+ let env = TestingEnv::new();
+
+ let table_info = new_test_table_info(1024);
+ let region_routes = vec![RegionRoute {
+ region: Region::new_test(region_id),
+ leader_peer: Some(Peer::empty(from_peer_id)),
+ ..Default::default()
+ }];
+ env.table_metadata_manager()
+ .create_table_metadata(
+ table_info,
+ TableRouteValue::physical(region_routes),
+ HashMap::default(),
+ )
+ .await
+ .unwrap();
+
+ let mut ctx = env
+ .context_factory()
+ .new_context(persistent_context.clone());
+ let instruction = state.build_open_region_instruction(&mut ctx).await.unwrap();
+ let open_regions = instruction.into_open_regions().unwrap();
+ assert_eq!(
+ Some(OpenRegionReason::RegionMigration),
+ open_regions[0].reason
+ );
+ assert_eq!(
+ RegionRequirements::object_storage(),
+ open_regions[0].requirements
+ );
+
+ persistent_context.trigger_reason = RegionMigrationTriggerReason::Failover;
+ let mut ctx = env.context_factory().new_context(persistent_context);
+ let instruction = state.build_open_region_instruction(&mut ctx).await.unwrap();
+ let open_regions = instruction.into_open_regions().unwrap();
+ assert_eq!(
+ Some(OpenRegionReason::RegionFailover),
+ open_regions[0].reason
+ );
+ assert_eq!(
+ RegionRequirements::object_storage(),
+ open_regions[0].requirements
+ );
+ }
+
#[tokio::test]
async fn test_datanode_is_unreachable() {
let state = OpenCandidateRegion;
diff --git a/src/meta-srv/src/procedure/region_migration/upgrade_candidate_region.rs b/src/meta-srv/src/procedure/region_migration/upgrade_candidate_region.rs
index 4c60215cf7..2ffd64bb47 100644
--- a/src/meta-srv/src/procedure/region_migration/upgrade_candidate_region.rs
+++ b/src/meta-srv/src/procedure/region_migration/upgrade_candidate_region.rs
@@ -21,7 +21,6 @@ use common_meta::ddl::utils::parse_region_wal_options;
use common_meta::instruction::{
Instruction, InstructionReply, UpgradeRegion, UpgradeRegionReply, UpgradeRegionsReply,
};
-use common_meta::key::topic_region::TopicRegionKey;
use common_meta::lock_key::RemoteWalLock;
use common_meta::wal_provider::extract_topic_from_wal_options;
use common_procedure::{Context as ProcedureContext, Status};
@@ -30,6 +29,7 @@ use common_telemetry::{error, info};
use common_wal::options::WalOptions;
use serde::{Deserialize, Serialize};
use snafu::{OptionExt, ResultExt, ensure};
+use store_api::metric_engine_consts::METRIC_ENGINE_NAME;
use tokio::time::{Instant, sleep};
use crate::error::{self, Result};
@@ -133,17 +133,14 @@ impl UpgradeCandidateRegion {
&datanode_table_value.region_info.region_wal_options,
)
{
- region_topic.push((*region_id, topic));
+ let is_metric_engine =
+ datanode_table_value.region_info.engine == METRIC_ENGINE_NAME;
+ region_topic.push((*region_id, topic, is_metric_engine));
}
}
let replay_checkpoints = ctx
- .get_replay_checkpoints(
- region_topic
- .iter()
- .map(|(region_id, topic)| TopicRegionKey::new(*region_id, topic))
- .collect(),
- )
+ .get_replay_checkpoints_with_topic_pruned_entry_ids(®ion_topic)
.await?;
// Build upgrade regions instruction.
let mut upgrade_regions = Vec::with_capacity(region_ids.len());
@@ -358,8 +355,11 @@ mod tests {
use common_meta::key::table_route::TableRouteValue;
use common_meta::key::test_utils::new_test_table_info;
+ use common_meta::key::topic_name::TopicNameKey;
+ use common_meta::key::topic_region::{ReplayCheckpoint, TopicRegionKey, TopicRegionValue};
use common_meta::peer::Peer;
use common_meta::rpc::router::{Region, RegionRoute};
+ use common_wal::options::KafkaWalOptions;
use store_api::storage::RegionId;
use super::*;
@@ -382,9 +382,28 @@ mod tests {
)
}
+ fn kafka_wal_options(topic: &str) -> HashMap {
+ HashMap::from([(
+ 1,
+ serde_json::to_string(&WalOptions::Kafka(KafkaWalOptions {
+ topic: topic.to_string(),
+ }))
+ .unwrap(),
+ )])
+ }
+
async fn prepare_table_metadata(ctx: &Context, wal_options: HashMap) {
+ prepare_table_metadata_with_engine(ctx, wal_options, "engine").await;
+ }
+
+ async fn prepare_table_metadata_with_engine(
+ ctx: &Context,
+ wal_options: HashMap,
+ engine: &str,
+ ) {
let region_id = ctx.persistent_ctx.region_ids[0];
- let table_info = new_test_table_info(region_id.table_id());
+ let mut table_info = new_test_table_info(region_id.table_id());
+ table_info.meta.engine = engine.to_string();
let region_routes = vec![RegionRoute {
region: Region::new_test(region_id),
leader_peer: Some(ctx.persistent_ctx.from_peer.clone()),
@@ -401,6 +420,104 @@ mod tests {
.unwrap();
}
+ #[tokio::test]
+ async fn test_build_upgrade_region_instruction_merges_topic_pruned_entry_id() {
+ let state = UpgradeCandidateRegion::default();
+ let persistent_context = new_persistent_context();
+ let env = TestingEnv::new();
+ let mut ctx = env.context_factory().new_context(persistent_context);
+ let region_id = ctx.persistent_ctx.region_ids[0];
+ let topic = "test_topic";
+ prepare_table_metadata(&ctx, kafka_wal_options(topic)).await;
+ ctx.table_metadata_manager
+ .topic_region_manager()
+ .batch_put(&[(
+ TopicRegionKey::new(region_id, topic),
+ Some(TopicRegionValue::new(Some(ReplayCheckpoint::new(10, None)))),
+ )])
+ .await
+ .unwrap();
+ ctx.table_metadata_manager
+ .topic_name_manager()
+ .batch_put(vec![TopicNameKey::new(topic)])
+ .await
+ .unwrap();
+ let prev = ctx
+ .table_metadata_manager
+ .topic_name_manager()
+ .get(topic)
+ .await
+ .unwrap();
+ ctx.table_metadata_manager
+ .topic_name_manager()
+ .update(topic, 20, prev)
+ .await
+ .unwrap();
+
+ let instruction = state
+ .build_upgrade_region_instruction(&mut ctx, Duration::from_secs(1))
+ .await
+ .unwrap();
+ let Instruction::UpgradeRegions(upgrade_regions) = instruction else {
+ unreachable!()
+ };
+
+ assert_eq!(upgrade_regions.len(), 1);
+ assert_eq!(upgrade_regions[0].replay_entry_id, Some(20));
+ assert_eq!(upgrade_regions[0].metadata_replay_entry_id, None);
+ }
+
+ #[tokio::test]
+ async fn test_build_upgrade_region_instruction_merges_metric_metadata_pruned_entry_id() {
+ let state = UpgradeCandidateRegion::default();
+ let persistent_context = new_persistent_context();
+ let env = TestingEnv::new();
+ let mut ctx = env.context_factory().new_context(persistent_context);
+ let region_id = ctx.persistent_ctx.region_ids[0];
+ let topic = "test_topic";
+ prepare_table_metadata_with_engine(&ctx, kafka_wal_options(topic), METRIC_ENGINE_NAME)
+ .await;
+ ctx.table_metadata_manager
+ .topic_region_manager()
+ .batch_put(&[(
+ TopicRegionKey::new(region_id, topic),
+ Some(TopicRegionValue::new(Some(ReplayCheckpoint::new(
+ 10,
+ Some(5),
+ )))),
+ )])
+ .await
+ .unwrap();
+ ctx.table_metadata_manager
+ .topic_name_manager()
+ .batch_put(vec![TopicNameKey::new(topic)])
+ .await
+ .unwrap();
+ let prev = ctx
+ .table_metadata_manager
+ .topic_name_manager()
+ .get(topic)
+ .await
+ .unwrap();
+ ctx.table_metadata_manager
+ .topic_name_manager()
+ .update(topic, 20, prev)
+ .await
+ .unwrap();
+
+ let instruction = state
+ .build_upgrade_region_instruction(&mut ctx, Duration::from_secs(1))
+ .await
+ .unwrap();
+ let Instruction::UpgradeRegions(upgrade_regions) = instruction else {
+ unreachable!()
+ };
+
+ assert_eq!(upgrade_regions.len(), 1);
+ assert_eq!(upgrade_regions[0].replay_entry_id, Some(20));
+ assert_eq!(upgrade_regions[0].metadata_replay_entry_id, Some(20));
+ }
+
#[tokio::test]
async fn test_datanode_is_unreachable() {
let state = UpgradeCandidateRegion::default();
diff --git a/src/meta-srv/src/procedure/repartition.rs b/src/meta-srv/src/procedure/repartition.rs
index c1819cb364..8c7c1dfeff 100644
--- a/src/meta-srv/src/procedure/repartition.rs
+++ b/src/meta-srv/src/procedure/repartition.rs
@@ -440,7 +440,17 @@ impl Context {
};
let _ = self
.cache_invalidator
- .invalidate(&ctx, &[CacheIdent::TableId(table_id)])
+ .invalidate(
+ &ctx,
+ &[
+ CacheIdent::TableId(table_id),
+ CacheIdent::TableName(TableName {
+ catalog_name: self.persistent_ctx.catalog_name.clone(),
+ schema_name: self.persistent_ctx.schema_name.clone(),
+ table_name: self.persistent_ctx.table_name.clone(),
+ }),
+ ],
+ )
.await;
Ok(())
}
diff --git a/src/meta-srv/src/procedure/repartition/update_partition_metadata.rs b/src/meta-srv/src/procedure/repartition/update_partition_metadata.rs
index cc9ca1c9bb..8d68a6e8fe 100644
--- a/src/meta-srv/src/procedure/repartition/update_partition_metadata.rs
+++ b/src/meta-srv/src/procedure/repartition/update_partition_metadata.rs
@@ -95,10 +95,19 @@ impl State for UpdatePartitionMetadata {
let mut new_table_info = table_info_value.table_info.clone();
new_table_info.meta.partition_key_indices = partition_key_indices;
+ common_telemetry::info!(
+ "Update table partition metadata, table_id: {}, partition_key_indices: {:?}, partition_columns: {:?}",
+ table_id,
+ new_table_info.meta.partition_key_indices,
+ new_table_info
+ .meta
+ .partition_column_names()
+ .cloned()
+ .collect::>(),
+ );
ctx.update_table_info(&table_info_value, table_info_value.update(new_table_info))
.await?;
- // We don't invalidate cache here because the subsequent AllocateRegion step
- // will update the table route and invalidate the cache accordingly.
+ ctx.invalidate_table_cache().await?;
Ok((
Box::new(AllocateRegion::new(self.plan_entries.clone())),
diff --git a/src/meta-srv/src/region/flush_trigger.rs b/src/meta-srv/src/region/flush_trigger.rs
index cdaa4339f4..d13fb9f244 100644
--- a/src/meta-srv/src/region/flush_trigger.rs
+++ b/src/meta-srv/src/region/flush_trigger.rs
@@ -45,6 +45,9 @@ const TICKER_INTERVAL: Duration = Duration::from_secs(60);
/// The duration of the recent period.
const RECENT_DURATION: Duration = Duration::from_secs(300);
+/// The interval to periodically persist region checkpoints regardless of replay size.
+const PERIODIC_CHECKPOINT_PERSIST_INTERVAL: Duration = Duration::from_secs(60 * 60);
+
/// [`Event`] represents various types of events that can be processed by the region flush ticker.
///
/// Variants:
@@ -84,6 +87,8 @@ pub struct RegionFlushTrigger {
flush_trigger_size: ReadableSize,
/// The checkpoint trigger size.
checkpoint_trigger_size: ReadableSize,
+ /// The last timestamp in milliseconds when a region checkpoint was persisted.
+ last_checkpoint_persist_millis_by_region: HashMap,
/// The receiver of events.
receiver: Receiver,
}
@@ -123,6 +128,7 @@ impl RegionFlushTrigger {
server_addr,
flush_trigger_size,
checkpoint_trigger_size,
+ last_checkpoint_persist_millis_by_region: HashMap::new(),
receiver: rx,
};
(region_flush_trigger, region_flush_ticker)
@@ -147,14 +153,15 @@ impl RegionFlushTrigger {
}
}
- async fn handle_tick(&self) {
+ async fn handle_tick(&mut self) {
if let Err(e) = self.trigger_flush().await {
error!(e; "Failed to trigger flush");
}
}
- async fn trigger_flush(&self) -> Result<()> {
+ async fn trigger_flush(&mut self) -> Result<()> {
let now = Instant::now();
+ let now_millis = current_time_millis();
let topics = self
.table_metadata_manager
.topic_name_manager()
@@ -162,17 +169,19 @@ impl RegionFlushTrigger {
.await
.context(error::TableMetadataManagerSnafu)?;
+ let mut active_region_ids = HashSet::new();
for topic in &topics {
- let Some((latest_entry_id, avg_record_size)) = self.retrieve_topic_stat(topic) else {
- continue;
- };
if let Err(e) = self
- .flush_regions_in_topic(topic, latest_entry_id, avg_record_size)
+ .handle_topic(topic, now_millis, &mut active_region_ids)
.await
{
- error!(e; "Failed to flush regions in topic: {}", topic);
+ error!(e; "Failed to handle regions in topic: {}", topic);
}
}
+ retain_checkpoint_persist_records(
+ &mut self.last_checkpoint_persist_millis_by_region,
+ &active_region_ids,
+ );
debug!(
"Triggered flush for {} topics in {:?}",
@@ -182,6 +191,79 @@ impl RegionFlushTrigger {
Ok(())
}
+ async fn handle_topic(
+ &mut self,
+ topic: &str,
+ now_millis: i64,
+ active_region_ids: &mut HashSet,
+ ) -> Result<()> {
+ let topic_regions = self
+ .table_metadata_manager
+ .topic_region_manager()
+ .regions(topic)
+ .await
+ .context(error::TableMetadataManagerSnafu)?;
+
+ if topic_regions.is_empty() {
+ debug!("No regions found for topic: {}", topic);
+ return Ok(());
+ }
+ active_region_ids.extend(topic_regions.keys().copied());
+
+ let topic_stat = self.retrieve_topic_stat(topic);
+ let size_based_regions = topic_stat
+ .map(|(latest_entry_id, avg_record_size)| {
+ filter_regions_by_replay_size(
+ topic,
+ topic_regions.iter().map(|(region_id, value)| {
+ (*region_id, value.min_entry_id().unwrap_or_default())
+ }),
+ avg_record_size as u64,
+ latest_entry_id,
+ self.checkpoint_trigger_size,
+ )
+ })
+ .unwrap_or_default();
+ // Periodic checkpoint persistence is intentionally independent of topic stats freshness:
+ // Kafka retention can advance even when recent write stats are unavailable.
+ let periodic_regions = filter_regions_for_periodic_checkpoint(
+ topic_regions.keys().copied(),
+ &self.last_checkpoint_persist_millis_by_region,
+ now_millis,
+ PERIODIC_CHECKPOINT_PERSIST_INTERVAL,
+ );
+ let regions_to_persist = merge_region_ids(size_based_regions, periodic_regions);
+ let region_manifests = self
+ .leader_region_registry
+ .batch_get(topic_regions.keys().cloned());
+
+ match self
+ .persist_region_checkpoints(
+ topic,
+ ®ions_to_persist,
+ &topic_regions,
+ ®ion_manifests,
+ )
+ .await
+ {
+ // Only mark regions that were actually written to KV. If the checkpoint is stale,
+ // already persisted, or the write fails, the next tick should retry.
+ Ok(region_ids) => mark_checkpoint_persisted(
+ &mut self.last_checkpoint_persist_millis_by_region,
+ ®ion_ids,
+ now_millis,
+ ),
+ Err(err) => error!(err; "Failed to persist region checkpoints for topic: {}", topic),
+ }
+
+ if let Some((latest_entry_id, avg_record_size)) = topic_stat {
+ self.flush_regions_in_topic(topic, latest_entry_id, avg_record_size, region_manifests)
+ .await?;
+ }
+
+ Ok(())
+ }
+
/// Retrieves the latest entry id and average record size of a topic.
///
/// Returns `None` if the topic is not found or the latest entry id is not recent.
@@ -226,7 +308,7 @@ impl RegionFlushTrigger {
region_ids: &[RegionId],
topic_regions: &HashMap,
leader_regions: &HashMap,
- ) -> Result<()> {
+ ) -> Result> {
let regions = region_ids
.iter()
.flat_map(|region_id| match leader_regions.get(region_id) {
@@ -237,27 +319,26 @@ impl RegionFlushTrigger {
.cloned()
.and_then(|value| value.checkpoint),
)
- .map(|checkpoint| {
- (
- TopicRegionKey::new(*region_id, topic),
- Some(TopicRegionValue::new(Some(checkpoint))),
- )
- }),
+ .map(|checkpoint| (*region_id, TopicRegionValue::new(Some(checkpoint)))),
None => None,
})
.collect::>();
- // The`chunks` will panic if chunks_size is zero, so we return early if there are no regions to persist.
+ // `chunks` will panic if chunk size is zero, so return early if there are no regions to persist.
if regions.is_empty() {
- return Ok(());
+ return Ok(Vec::new());
}
let max_txn_ops = self.table_metadata_manager.kv_backend().max_txn_ops();
let batch_size = max_txn_ops.min(regions.len());
for batch in regions.chunks(batch_size) {
+ let batch = batch
+ .iter()
+ .map(|(region_id, value)| (TopicRegionKey::new(*region_id, topic), Some(*value)))
+ .collect::>();
self.table_metadata_manager
.topic_region_manager()
- .batch_put(batch)
+ .batch_put(&batch)
.await
.context(error::TableMetadataManagerSnafu)?;
}
@@ -266,7 +347,10 @@ impl RegionFlushTrigger {
.with_label_values(&[topic])
.inc_by(regions.len() as u64);
- Ok(())
+ Ok(regions
+ .into_iter()
+ .map(|(region_id, _)| region_id)
+ .collect())
}
async fn flush_regions_in_topic(
@@ -274,45 +358,8 @@ impl RegionFlushTrigger {
topic: &str,
latest_entry_id: u64,
avg_record_size: usize,
+ region_manifests: HashMap,
) -> Result<()> {
- let topic_regions = self
- .table_metadata_manager
- .topic_region_manager()
- .regions(topic)
- .await
- .context(error::TableMetadataManagerSnafu)?;
-
- if topic_regions.is_empty() {
- debug!("No regions found for topic: {}", topic);
- return Ok(());
- }
-
- // Filters regions need to persist checkpoints.
- let regions_to_persist = filter_regions_by_replay_size(
- topic,
- topic_regions
- .iter()
- .map(|(region_id, value)| (*region_id, value.min_entry_id().unwrap_or_default())),
- avg_record_size as u64,
- latest_entry_id,
- self.checkpoint_trigger_size,
- );
- let region_manifests = self
- .leader_region_registry
- .batch_get(topic_regions.keys().cloned());
-
- if let Err(err) = self
- .persist_region_checkpoints(
- topic,
- ®ions_to_persist,
- &topic_regions,
- ®ion_manifests,
- )
- .await
- {
- error!(err; "Failed to persist region checkpoints for topic: {}", topic);
- }
-
let regions = region_manifests
.into_iter()
.map(|(region_id, region)| (region_id, region.manifest.prunable_entry_id()))
@@ -447,6 +494,56 @@ fn filter_regions_by_replay_size>(
regions_to_flush
}
+/// Filters regions that need periodic checkpoint persistence.
+fn filter_regions_for_periodic_checkpoint(
+ regions: I,
+ last_persisted: &HashMap,
+ now_millis: i64,
+ interval: Duration,
+) -> Vec
+where
+ I: Iterator
- ,
+{
+ let interval_millis = interval.as_millis() as i64;
+ regions
+ .filter(|region_id| {
+ last_persisted
+ .get(region_id)
+ .is_none_or(|last_persist_millis| {
+ now_millis.saturating_sub(*last_persist_millis) >= interval_millis
+ })
+ })
+ .collect()
+}
+
+/// Merges two region id lists and removes duplicates.
+fn merge_region_ids(left: Vec, right: Vec) -> Vec {
+ left.into_iter()
+ .chain(right)
+ .collect::>()
+ .into_iter()
+ .collect()
+}
+
+/// Marks checkpoint persistence timestamps for regions.
+fn mark_checkpoint_persisted(
+ last_persisted: &mut HashMap,
+ region_ids: &[RegionId],
+ now_millis: i64,
+) {
+ for region_id in region_ids {
+ last_persisted.insert(*region_id, now_millis);
+ }
+}
+
+/// Retains checkpoint persistence records for active regions.
+fn retain_checkpoint_persist_records(
+ last_persisted: &mut HashMap,
+ active_region_ids: &HashSet,
+) {
+ last_persisted.retain(|region_id, _| active_region_ids.contains(region_id));
+}
+
/// Group regions by leader.
///
/// The regions are grouped by the leader of the region.
@@ -527,6 +624,71 @@ mod tests {
assert!(!is_recent(now - 1001, now, Duration::from_secs(1)));
}
+ #[test]
+ fn test_filter_regions_for_periodic_checkpoint() {
+ let now_millis = 10_000;
+ let interval = Duration::from_secs(5);
+ let regions = vec![region_id(1, 1), region_id(1, 2), region_id(1, 3)];
+ let last_persisted = HashMap::from([
+ (region_id(1, 1), now_millis - 4_000),
+ (region_id(1, 2), now_millis - 5_000),
+ ]);
+
+ let result = filter_regions_for_periodic_checkpoint(
+ regions.into_iter(),
+ &last_persisted,
+ now_millis,
+ interval,
+ );
+
+ assert_eq!(result, vec![region_id(1, 2), region_id(1, 3)]);
+ }
+
+ #[test]
+ fn test_merge_region_ids() {
+ let merged = merge_region_ids(
+ vec![region_id(1, 1), region_id(1, 2)],
+ vec![region_id(1, 2), region_id(1, 3)],
+ );
+ let merged = merged.into_iter().collect::>();
+
+ assert_eq!(
+ merged,
+ HashSet::from([region_id(1, 1), region_id(1, 2), region_id(1, 3)])
+ );
+ }
+
+ #[test]
+ fn test_mark_checkpoint_persisted() {
+ let now_millis = 10_000;
+ let mut last_persisted = HashMap::from([(region_id(1, 1), 1_000)]);
+
+ mark_checkpoint_persisted(
+ &mut last_persisted,
+ &[region_id(1, 1), region_id(1, 2)],
+ now_millis,
+ );
+
+ assert_eq!(last_persisted.get(®ion_id(1, 1)), Some(&now_millis));
+ assert_eq!(last_persisted.get(®ion_id(1, 2)), Some(&now_millis));
+ }
+
+ #[test]
+ fn test_retain_checkpoint_persist_records() {
+ let mut last_persisted = HashMap::from([
+ (region_id(1, 1), 1_000),
+ (region_id(1, 2), 2_000),
+ (region_id(1, 3), 3_000),
+ ]);
+ let active_regions = HashSet::from([region_id(1, 1), region_id(1, 3)]);
+
+ retain_checkpoint_persist_records(&mut last_persisted, &active_regions);
+
+ assert_eq!(last_persisted.len(), 2);
+ assert!(last_persisted.contains_key(®ion_id(1, 1)));
+ assert!(last_persisted.contains_key(®ion_id(1, 3)));
+ }
+
fn region_id(table: u32, region: u32) -> RegionId {
RegionId::new(table, region)
}
@@ -735,6 +897,76 @@ mod tests {
assert!(result.is_none());
}
+ #[tokio::test]
+ async fn test_persist_region_checkpoints_returns_written_region_ids() {
+ let kv_backend = Arc::new(MemoryKvBackend::new());
+ let table_metadata_manager = Arc::new(TableMetadataManager::new(kv_backend.clone()));
+ let leader_region_registry = Arc::new(LeaderRegionRegistry::new());
+ let topic_stats_registry = Arc::new(TopicStatsRegistry::default());
+ let mailbox_sequence = SequenceBuilder::new(
+ "test_persist_region_checkpoints_returns_written_region_ids",
+ kv_backend,
+ )
+ .build();
+ let mailbox_ctx = MailboxContext::new(mailbox_sequence);
+
+ let (trigger, _ticker) = RegionFlushTrigger::new(
+ table_metadata_manager.clone(),
+ leader_region_registry,
+ topic_stats_registry,
+ mailbox_ctx.mailbox().clone(),
+ "127.0.0.1:3002".to_string(),
+ ReadableSize(1),
+ ReadableSize(1),
+ );
+
+ let topic = "test_topic";
+ let region_to_write = region_id(1, 1);
+ let region_already_persisted = region_id(1, 2);
+ let region_without_leader = region_id(1, 3);
+ let topic_regions = HashMap::from([
+ (region_to_write, TopicRegionValue::new(None)),
+ (
+ region_already_persisted,
+ TopicRegionValue::new(Some(ReplayCheckpoint::new(100, None))),
+ ),
+ (region_without_leader, TopicRegionValue::new(None)),
+ ]);
+ let leader_regions = HashMap::from([
+ (region_to_write, mito_leader_region(100)),
+ (region_already_persisted, mito_leader_region(100)),
+ ]);
+
+ let written_region_ids = trigger
+ .persist_region_checkpoints(
+ topic,
+ &[
+ region_to_write,
+ region_already_persisted,
+ region_without_leader,
+ ],
+ &topic_regions,
+ &leader_regions,
+ )
+ .await
+ .unwrap();
+
+ assert_eq!(written_region_ids, vec![region_to_write]);
+ let persisted = table_metadata_manager
+ .topic_region_manager()
+ .get(TopicRegionKey::new(region_to_write, topic))
+ .await
+ .unwrap()
+ .unwrap();
+ assert_eq!(persisted.checkpoint, Some(ReplayCheckpoint::new(100, None)));
+ let skipped = table_metadata_manager
+ .topic_region_manager()
+ .get(TopicRegionKey::new(region_already_persisted, topic))
+ .await
+ .unwrap();
+ assert!(skipped.is_none());
+ }
+
#[tokio::test]
async fn test_send_flush_instructions_payload_includes_remote_wal_prune_reason() {
let kv_backend = Arc::new(MemoryKvBackend::new());
diff --git a/src/metric-engine/src/engine.rs b/src/metric-engine/src/engine.rs
index ef4d802cfc..fa9ef804cc 100644
--- a/src/metric-engine/src/engine.rs
+++ b/src/metric-engine/src/engine.rs
@@ -620,6 +620,7 @@ mod test {
options: physical_region_option,
skip_wal_replay: false,
checkpoint: None,
+ requirements: Default::default(),
};
engine
.handle_request(physical_region_id, RegionRequest::Open(open_request))
@@ -644,6 +645,7 @@ mod test {
options: HashMap::new(),
skip_wal_replay: false,
checkpoint: None,
+ requirements: Default::default(),
};
engine
.handle_request(
@@ -721,6 +723,7 @@ mod test {
options: physical_region_option,
skip_wal_replay: false,
checkpoint: None,
+ requirements: Default::default(),
};
// Opening an already opened region should succeed.
// Since the region is already open, no metadata recovery operations will be performed.
@@ -749,6 +752,7 @@ mod test {
options: physical_region_option,
skip_wal_replay: false,
checkpoint: None,
+ requirements: Default::default(),
};
let err = metric_engine
.handle_request(physical_region_id, RegionRequest::Open(open_request))
@@ -854,6 +858,7 @@ mod test {
options: options.clone(),
skip_wal_replay: true,
checkpoint: None,
+ requirements: Default::default(),
},
)
})
diff --git a/src/metric-engine/src/engine/bulk_insert.rs b/src/metric-engine/src/engine/bulk_insert.rs
index 6d33d39149..d700d858ac 100644
--- a/src/metric-engine/src/engine/bulk_insert.rs
+++ b/src/metric-engine/src/engine/bulk_insert.rs
@@ -18,9 +18,10 @@ use api::v1::{ArrowIpc, SemanticType};
use bytes::Bytes;
use common_grpc::flight::{FlightEncoder, FlightMessage};
use datatypes::arrow::record_batch::RecordBatch;
-use snafu::{OptionExt, ensure};
+use snafu::{OptionExt, ResultExt, ensure};
use store_api::codec::PrimaryKeyEncoding;
use store_api::metadata::RegionMetadataRef;
+use store_api::region_engine::RegionEngine;
use store_api::region_request::{AffectedRows, RegionBulkInsertsRequest, RegionRequest};
use store_api::storage::RegionId;
@@ -71,8 +72,11 @@ impl MetricEngineInner {
async fn bulk_insert_physical_region(
&self,
region_id: RegionId,
- request: RegionBulkInsertsRequest,
+ mut request: RegionBulkInsertsRequest,
) -> Result {
+ // Simply set the aligned schema to the data region schema version to avoid filling missing columns
+ // because that schema should be constant and callers have ensured request has the same schema.
+ request.aligned_schema_version = Some(self.physical_schema_version(region_id).await?);
self.data_region
.write_data(region_id, RegionRequest::BulkInserts(request))
.await
@@ -117,6 +121,8 @@ impl MetricEngineInner {
let (schema, data_header, payload) = record_batch_to_ipc(&modified_batch)?;
let partition_expr_version = request.partition_expr_version;
+ let aligned_schema_version = Some(self.physical_schema_version(data_region_id).await?);
+
let request = RegionBulkInsertsRequest {
region_id: data_region_id,
payload: modified_batch,
@@ -126,12 +132,22 @@ impl MetricEngineInner {
payload,
},
partition_expr_version,
+ aligned_schema_version,
};
self.data_region
.write_data(data_region_id, RegionRequest::BulkInserts(request))
.await
}
+ async fn physical_schema_version(&self, region_id: RegionId) -> Result {
+ Ok(self
+ .mito
+ .get_metadata(region_id)
+ .await
+ .context(error::MitoReadOperationSnafu)?
+ .schema_version)
+ }
+
fn resolve_tag_columns_from_metadata(
&self,
logical_region_id: RegionId,
@@ -282,6 +298,7 @@ mod tests {
payload,
},
partition_expr_version: None,
+ aligned_schema_version: None,
})
}
@@ -319,6 +336,7 @@ mod tests {
payload: batch,
raw_data: ArrowIpc::default(),
partition_expr_version: None,
+ aligned_schema_version: None,
});
let response = env
.metric()
diff --git a/src/metric-engine/src/engine/open.rs b/src/metric-engine/src/engine/open.rs
index 59b1cfd928..8fcdfcd821 100644
--- a/src/metric-engine/src/engine/open.rs
+++ b/src/metric-engine/src/engine/open.rs
@@ -222,6 +222,7 @@ impl MetricEngineInner {
entry_id: checkpoint.metadata_entry_id.unwrap_or_default(),
metadata_entry_id: None,
}),
+ requirements: request.requirements,
};
let mut data_region_options = request.options;
@@ -239,6 +240,7 @@ impl MetricEngineInner {
entry_id: checkpoint.entry_id,
metadata_entry_id: None,
}),
+ requirements: request.requirements,
};
(open_metadata_region_request, open_data_region_request)
diff --git a/src/metric-engine/src/engine/sync/region.rs b/src/metric-engine/src/engine/sync/region.rs
index cbe6515a19..d1f92bef64 100644
--- a/src/metric-engine/src/engine/sync/region.rs
+++ b/src/metric-engine/src/engine/sync/region.rs
@@ -321,6 +321,7 @@ mod tests {
options: physical_region_option,
skip_wal_replay: false,
checkpoint: None,
+ requirements: Default::default(),
}),
)
.await
diff --git a/src/metric-engine/src/test_util.rs b/src/metric-engine/src/test_util.rs
index ec55a01903..8d4a822b6b 100644
--- a/src/metric-engine/src/test_util.rs
+++ b/src/metric-engine/src/test_util.rs
@@ -144,6 +144,7 @@ impl TestEnv {
options: physical_region_option,
skip_wal_replay: true,
checkpoint: None,
+ requirements: Default::default(),
}),
)
.await
diff --git a/src/mito2/Cargo.toml b/src/mito2/Cargo.toml
index 3e3a18a24d..ea281f2c32 100644
--- a/src/mito2/Cargo.toml
+++ b/src/mito2/Cargo.toml
@@ -8,6 +8,7 @@ license.workspace = true
default = []
test = ["common-test-util", "rstest", "rstest_reuse", "rskafka"]
testing = ["test"]
+test-shared-fs-region-migration = []
enterprise = []
vector_index = ["dep:roaring", "index/vector_index"]
@@ -50,6 +51,7 @@ datafusion-common.workspace = true
datafusion-expr.workspace = true
datatypes.workspace = true
dashmap.workspace = true
+derive_more.workspace = true
dotenv.workspace = true
either.workspace = true
futures.workspace = true
diff --git a/src/mito2/src/cache.rs b/src/mito2/src/cache.rs
index eee1cfae0a..89e84fadcc 100644
--- a/src/mito2/src/cache.rs
+++ b/src/mito2/src/cache.rs
@@ -23,9 +23,10 @@ pub(crate) mod manifest_cache;
pub(crate) mod test_util;
pub(crate) mod write_cache;
+use std::collections::{BTreeMap, HashMap};
use std::mem;
use std::ops::Range;
-use std::sync::Arc;
+use std::sync::{Arc, RwLock};
use bytes::Bytes;
use common_base::readable_size::ReadableSize;
@@ -527,20 +528,33 @@ impl CacheStrategy {
}
}
- /// Calls [CacheManager::get_pages()].
+ /// Calls [CacheManager::get_page_ranges()].
/// It returns None if the strategy is [CacheStrategy::Compaction] or [CacheStrategy::Disabled].
- pub fn get_pages(&self, page_key: &PageKey) -> Option> {
+ pub fn get_page_ranges(
+ &self,
+ file_id: FileId,
+ row_group_idx: usize,
+ ranges: &[Range],
+ ) -> Option {
match self {
- CacheStrategy::EnableAll(cache_manager) => cache_manager.get_pages(page_key),
+ CacheStrategy::EnableAll(cache_manager) => {
+ cache_manager.get_page_ranges(file_id, row_group_idx, ranges)
+ }
CacheStrategy::Compaction(_) | CacheStrategy::Disabled => None,
}
}
- /// Calls [CacheManager::put_pages()].
+ /// Calls [CacheManager::put_page_ranges()].
/// It does nothing if the strategy isn't [CacheStrategy::EnableAll].
- pub fn put_pages(&self, page_key: PageKey, pages: Arc) {
+ pub fn put_page_ranges(
+ &self,
+ file_id: FileId,
+ row_group_idx: usize,
+ ranges: &[Range],
+ pages: &[Bytes],
+ ) {
if let CacheStrategy::EnableAll(cache_manager) = self {
- cache_manager.put_pages(page_key, pages);
+ cache_manager.put_page_ranges(file_id, row_group_idx, ranges, pages);
}
}
@@ -720,8 +734,8 @@ pub struct CacheManager {
sst_meta_cache: Option,
/// Cache for vectors.
vector_cache: Option,
- /// Cache for SST pages.
- page_cache: Option,
+ /// Cache for SST byte ranges.
+ page_cache: Option>,
/// A Cache for writing files to object stores.
write_cache: Option,
/// Cache for inverted index.
@@ -904,21 +918,35 @@ impl CacheManager {
}
}
- /// Gets pages for the row group.
- pub fn get_pages(&self, page_key: &PageKey) -> Option> {
- self.page_cache.as_ref().and_then(|page_cache| {
- let value = page_cache.get(page_key);
- update_hit_miss(value, PAGE_TYPE)
+ /// Gets cached byte fragments for the requested ranges.
+ pub fn get_page_ranges(
+ &self,
+ file_id: FileId,
+ row_group_idx: usize,
+ ranges: &[Range],
+ ) -> Option {
+ self.page_cache.as_ref().map(|page_cache| {
+ let lookup = page_cache.lookup(file_id, row_group_idx, ranges);
+ if lookup.cached_bytes > 0 {
+ CACHE_HIT.with_label_values(&[PAGE_TYPE]).inc();
+ }
+ if !lookup.missing_ranges.is_empty() {
+ CACHE_MISS.with_label_values(&[PAGE_TYPE]).inc();
+ }
+ lookup
})
}
- /// Puts pages of the row group into the cache.
- pub fn put_pages(&self, page_key: PageKey, pages: Arc) {
+ /// Puts byte fragments into the page cache.
+ pub fn put_page_ranges(
+ &self,
+ file_id: FileId,
+ row_group_idx: usize,
+ ranges: &[Range],
+ pages: &[Bytes],
+ ) {
if let Some(cache) = &self.page_cache {
- CACHE_BYTES
- .with_label_values(&[PAGE_TYPE])
- .add(page_cache_weight(&page_key, &pages).into());
- cache.insert(page_key, pages);
+ cache.insert_ranges(file_id, row_group_idx, ranges, pages);
}
}
@@ -1192,19 +1220,8 @@ impl CacheManagerBuilder {
})
.build()
});
- let page_cache = (self.page_cache_size != 0).then(|| {
- Cache::builder()
- .max_capacity(self.page_cache_size)
- .weigher(page_cache_weight)
- .eviction_listener(|k, v, cause| {
- let size = page_cache_weight(&k, &v);
- CACHE_BYTES.with_label_values(&[PAGE_TYPE]).sub(size.into());
- CACHE_EVICTION
- .with_label_values(&[PAGE_TYPE, removal_cause_str(cause)])
- .inc();
- })
- .build()
- });
+ let page_cache =
+ (self.page_cache_size != 0).then(|| PageRangeCache::new(self.page_cache_size));
let inverted_index_cache = InvertedIndexCache::new(
self.index_metadata_size,
self.index_content_size,
@@ -1288,8 +1305,8 @@ fn vector_cache_weight(_k: &(ConcreteDataType, Value), v: &VectorRef) -> u32 {
(mem::size_of::() + mem::size_of::() + v.memory_size()) as u32
}
-fn page_cache_weight(k: &PageKey, v: &Arc) -> u32 {
- (k.estimated_size() + v.estimated_size()) as u32
+fn page_cache_weight(k: &PageFragmentKey, v: &Bytes) -> u32 {
+ (k.estimated_size() + mem::size_of::() + v.len()) as u32
}
fn selector_result_cache_weight(k: &SelectorResultKey, v: &Arc) -> u32 {
@@ -1321,73 +1338,281 @@ impl SstMetaKey {
}
}
-/// Path to column pages in the SST file.
-#[derive(Debug, Clone, PartialEq, Eq, Hash)]
-pub struct ColumnPagePath {
- /// Region id of the SST file to cache.
- region_id: RegionId,
- /// Id of the SST file to cache.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+struct PageFragmentGroupKey {
+ file_id: FileId,
+ row_group_idx: usize,
+}
+
+/// Cache key for one byte fragment in an SST row group.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub struct PageFragmentKey {
+ /// Id of the SST file.
file_id: FileId,
/// Index of the row group.
row_group_idx: usize,
- /// Index of the column in the row group.
- column_idx: usize,
+ /// Start offset of the cached byte fragment.
+ start: u64,
+ /// End offset of the cached byte fragment.
+ end: u64,
}
-/// Cache key to pages in a row group (after projection).
-///
-/// Different projections will have different cache keys.
-/// We cache all ranges together because they may refer to the same `Bytes`.
-#[derive(Debug, Clone, PartialEq, Eq, Hash)]
-pub struct PageKey {
- /// Id of the SST file to cache.
- file_id: FileId,
- /// Index of the row group.
- row_group_idx: usize,
- /// Byte ranges of the pages to cache.
- ranges: Vec>,
-}
-
-impl PageKey {
- /// Creates a key for a list of pages.
- pub fn new(file_id: FileId, row_group_idx: usize, ranges: Vec>) -> PageKey {
- PageKey {
+impl PageFragmentKey {
+ fn new(file_id: FileId, row_group_idx: usize, range: &Range) -> PageFragmentKey {
+ PageFragmentKey {
file_id,
row_group_idx,
- ranges,
+ start: range.start,
+ end: range.end,
+ }
+ }
+
+ fn group_key(&self) -> PageFragmentGroupKey {
+ PageFragmentGroupKey {
+ file_id: self.file_id,
+ row_group_idx: self.row_group_idx,
}
}
/// Returns memory used by the key (estimated).
fn estimated_size(&self) -> usize {
- mem::size_of::() + mem::size_of_val(self.ranges.as_slice())
+ mem::size_of::()
}
}
-/// Cached row group pages for a column.
-// We don't use enum here to make it easier to mock and use the struct.
-#[derive(Default)]
-pub struct PageValue {
- /// Compressed page in the row group.
- pub compressed: Vec,
- /// Total size of the pages (may be larger than sum of compressed bytes due to gaps).
- pub page_size: u64,
+/// One cached byte fragment that overlaps a requested range.
+#[derive(Clone)]
+pub struct PageRangePart {
+ /// Range covered by `bytes`.
+ pub range: Range,
+ /// Bytes for `range`.
+ pub bytes: Bytes,
}
-impl PageValue {
- /// Creates a new value from a range of compressed pages.
- pub fn new(bytes: Vec, page_size: u64) -> PageValue {
- PageValue {
- compressed: bytes,
- page_size,
+/// Result of looking up request ranges in the page range cache.
+pub struct PageRangeLookup {
+ /// Cached fragments grouped by the original requested range index.
+ pub cached_parts: Vec>,
+ /// Ranges that are not covered by cached fragments and need fetching.
+ pub missing_ranges: Vec>,
+ /// Number of cached fragments used.
+ pub cached_range_count: usize,
+ /// Number of requested bytes served from cached fragments.
+ pub cached_bytes: u64,
+}
+
+impl PageRangeLookup {
+ pub fn is_fully_cached(&self) -> bool {
+ self.missing_ranges.is_empty()
+ }
+}
+
+type PageFragmentRangeIndex = BTreeMap<(u64, u64), PageFragmentKey>;
+type PageFragmentIndex = HashMap;
+
+/// Byte-fragment cache for Parquet row-group reads.
+pub struct PageRangeCache {
+ cache: Cache,
+ index: RwLock,
+}
+
+impl PageRangeCache {
+ fn new(capacity: u64) -> Arc {
+ Arc::new_cyclic(|weak_cache: &std::sync::Weak| {
+ let cache = Cache::builder()
+ .max_capacity(capacity)
+ .weigher(page_cache_weight)
+ .eviction_listener({
+ let weak_cache = weak_cache.clone();
+ move |k, v, cause| {
+ let size = page_cache_weight(&k, &v);
+ CACHE_BYTES.with_label_values(&[PAGE_TYPE]).sub(size.into());
+ CACHE_EVICTION
+ .with_label_values(&[PAGE_TYPE, removal_cause_str(cause)])
+ .inc();
+
+ if let Some(cache) = weak_cache.upgrade()
+ && !matches!(cause, RemovalCause::Replaced)
+ {
+ cache.remove_index_entry(*k);
+ }
+ }
+ })
+ .build();
+
+ PageRangeCache {
+ cache,
+ index: RwLock::new(HashMap::new()),
+ }
+ })
+ }
+
+ fn lookup(
+ &self,
+ file_id: FileId,
+ row_group_idx: usize,
+ ranges: &[Range],
+ ) -> PageRangeLookup {
+ let mut cached_parts = Vec::with_capacity(ranges.len());
+ let mut missing_ranges = Vec::new();
+ let mut cached_range_count = 0;
+ let mut cached_bytes = 0;
+
+ for range in ranges {
+ if range.start >= range.end {
+ cached_parts.push(Vec::new());
+ continue;
+ }
+
+ let mut parts = Vec::new();
+ let candidates = self.find_index_candidates(file_id, row_group_idx, range);
+ let mut stale_keys = Vec::new();
+
+ for fragment_key in candidates {
+ if let Some(bytes) = self.cache.get(&fragment_key) {
+ let part_start = range.start.max(fragment_key.start);
+ let part_end = range.end.min(fragment_key.end);
+ let slice_start = (part_start - fragment_key.start) as usize;
+ let slice_end = (part_end - fragment_key.start) as usize;
+ parts.push(PageRangePart {
+ range: part_start..part_end,
+ bytes: bytes.slice(slice_start..slice_end),
+ });
+ } else {
+ stale_keys.push(fragment_key);
+ }
+ }
+ for key in stale_keys {
+ self.remove_uncached_index_entry(key);
+ }
+
+ let mut cursor = range.start;
+ let mut compacted_parts: Vec = Vec::with_capacity(parts.len());
+ for part in parts {
+ if part.range.end <= cursor {
+ continue;
+ }
+
+ let part = if part.range.start < cursor {
+ let offset = (cursor - part.range.start) as usize;
+ PageRangePart {
+ range: cursor..part.range.end,
+ bytes: part.bytes.slice(offset..),
+ }
+ } else {
+ part
+ };
+
+ if cursor < part.range.start {
+ missing_ranges.push(cursor..part.range.start);
+ }
+ cached_bytes += part.range.end - part.range.start;
+ cached_range_count += 1;
+ cursor = part.range.end;
+ compacted_parts.push(part);
+
+ if cursor >= range.end {
+ break;
+ }
+ }
+
+ if cursor < range.end {
+ missing_ranges.push(cursor..range.end);
+ }
+ cached_parts.push(compacted_parts);
+ }
+
+ PageRangeLookup {
+ cached_parts,
+ missing_ranges,
+ cached_range_count,
+ cached_bytes,
}
}
- /// Returns memory used by the value (estimated).
- fn estimated_size(&self) -> usize {
- mem::size_of::()
- + self.page_size as usize
- + self.compressed.iter().map(mem::size_of_val).sum::()
+ fn insert_ranges(
+ &self,
+ file_id: FileId,
+ row_group_idx: usize,
+ ranges: &[Range],
+ pages: &[Bytes],
+ ) {
+ for (range, bytes) in ranges.iter().zip(pages) {
+ if range.start >= range.end || bytes.len() as u64 != range.end - range.start {
+ continue;
+ }
+
+ let key = PageFragmentKey::new(file_id, row_group_idx, range);
+ let bytes = Bytes::copy_from_slice(bytes.as_ref());
+ let size = page_cache_weight(&key, &bytes);
+ CACHE_BYTES.with_label_values(&[PAGE_TYPE]).add(size.into());
+ self.cache.insert(key, bytes);
+ let mut index = self.index.write().unwrap();
+ index
+ .entry(key.group_key())
+ .or_default()
+ .insert((key.start, key.end), key);
+ }
+ }
+
+ fn find_index_candidates(
+ &self,
+ file_id: FileId,
+ row_group_idx: usize,
+ range: &Range,
+ ) -> Vec {
+ let group_key = PageFragmentGroupKey {
+ file_id,
+ row_group_idx,
+ };
+ let index = self.index.read().unwrap();
+ index
+ .get(&group_key)
+ .map(|ranges| {
+ ranges
+ .range(..(range.end, 0))
+ .filter_map(|(_, fragment_key)| {
+ (fragment_key.end > range.start).then_some(*fragment_key)
+ })
+ .collect()
+ })
+ .unwrap_or_default()
+ }
+
+ fn remove_uncached_index_entry(&self, key: PageFragmentKey) {
+ let group_key = key.group_key();
+ let mut index = self.index.write().unwrap();
+ if self.cache.contains_key(&key) {
+ return;
+ }
+
+ Self::remove_index_entry_locked(&mut index, group_key, key);
+ }
+
+ fn remove_index_entry(&self, key: PageFragmentKey) {
+ let group_key = key.group_key();
+ let mut index = self.index.write().unwrap();
+ Self::remove_index_entry_locked(&mut index, group_key, key);
+ }
+
+ fn remove_index_entry_locked(
+ index: &mut PageFragmentIndex,
+ group_key: PageFragmentGroupKey,
+ key: PageFragmentKey,
+ ) {
+ let Some(ranges) = index.get_mut(&group_key) else {
+ return;
+ };
+
+ let removed = ranges
+ .get(&(key.start, key.end))
+ .is_some_and(|current| current == &key);
+ if removed {
+ ranges.remove(&(key.start, key.end));
+ }
+ if ranges.is_empty() {
+ index.remove(&group_key);
+ }
}
}
@@ -1455,8 +1680,6 @@ type SstMetaCache = Cache>;
///
/// e.g. `"hello" => ["hello", "hello", "hello"]`
type VectorCache = Cache<(ConcreteDataType, Value), VectorRef>;
-/// Maps (region, file, row group, column) to [PageValue].
-type PageCache = Cache>;
/// Maps (file id, row group id, time series row selector) to [SelectorResultValue].
type SelectorResultCache = Cache>;
/// Maps partition-range scan key to cached flat batches.
@@ -1514,10 +1737,17 @@ mod tests {
.is_none()
);
- let key = PageKey::new(file_id.file_id(), 1, vec![Range { start: 0, end: 5 }]);
- let pages = Arc::new(PageValue::default());
- cache.put_pages(key.clone(), pages);
- assert!(cache.get_pages(&key).is_none());
+ cache.put_page_ranges(
+ file_id.file_id(),
+ 1,
+ &[Range { start: 0, end: 5 }],
+ &[Bytes::from_static(b"abcde")],
+ );
+ assert!(
+ cache
+ .get_page_ranges(file_id.file_id(), 1, &[Range { start: 0, end: 5 }])
+ .is_none()
+ );
assert!(cache.write_cache().is_none());
}
@@ -1675,11 +1905,168 @@ mod tests {
fn test_page_cache() {
let cache = CacheManager::builder().page_cache_size(1000).build();
let file_id = FileId::random();
- let key = PageKey::new(file_id, 0, vec![(0..10), (10..20)]);
- assert!(cache.get_pages(&key).is_none());
- let pages = Arc::new(PageValue::default());
- cache.put_pages(key.clone(), pages);
- assert!(cache.get_pages(&key).is_some());
+ let uncached = 0..10;
+ assert_eq!(
+ vec![0..10],
+ cache
+ .get_page_ranges(file_id, 0, std::slice::from_ref(&uncached))
+ .unwrap()
+ .missing_ranges
+ );
+
+ let cached = 100..500;
+ cache.put_page_ranges(
+ file_id,
+ 0,
+ std::slice::from_ref(&cached),
+ &[Bytes::from(vec![7; 400])],
+ );
+
+ let subrange = 200..300;
+ let lookup = cache
+ .get_page_ranges(file_id, 0, std::slice::from_ref(&subrange))
+ .unwrap();
+ assert!(lookup.is_fully_cached());
+ assert_eq!(100, lookup.cached_bytes);
+ assert_eq!(1, lookup.cached_parts.len());
+ assert_eq!(200..300, lookup.cached_parts[0][0].range);
+ assert_eq!(100, lookup.cached_parts[0][0].bytes.len());
+
+ let overlapping = 400..600;
+ let lookup = cache
+ .get_page_ranges(file_id, 0, std::slice::from_ref(&overlapping))
+ .unwrap();
+ assert!(!lookup.is_fully_cached());
+ assert_eq!(100, lookup.cached_bytes);
+ assert_eq!(vec![500..600], lookup.missing_ranges);
+ assert_eq!(400..500, lookup.cached_parts[0][0].range);
+ }
+
+ #[test]
+ fn test_page_cache_detaches_fragment_bytes() {
+ let cache = PageRangeCache::new(1000);
+ let file_id = FileId::random();
+ let backing = Bytes::from(vec![1; 1024]);
+ let page = backing.slice(512..522);
+ let page_ptr = page.as_ptr();
+ let range = 0..10;
+
+ cache.insert_ranges(
+ file_id,
+ 0,
+ std::slice::from_ref(&range),
+ std::slice::from_ref(&page),
+ );
+
+ let lookup = cache.lookup(file_id, 0, std::slice::from_ref(&range));
+ assert!(lookup.is_fully_cached());
+ assert_eq!(1, lookup.cached_parts[0].len());
+ assert_eq!(&page[..], &lookup.cached_parts[0][0].bytes[..]);
+ assert_ne!(page_ptr, lookup.cached_parts[0][0].bytes.as_ptr());
+ }
+
+ #[test]
+ fn test_page_cache_replaces_fragment() {
+ let cache = PageRangeCache::new(1000);
+ let file_id = FileId::random();
+ let range = 0..10;
+
+ cache.insert_ranges(
+ file_id,
+ 0,
+ std::slice::from_ref(&range),
+ &[Bytes::from(vec![1; 10])],
+ );
+ cache.insert_ranges(
+ file_id,
+ 0,
+ std::slice::from_ref(&range),
+ &[Bytes::from(vec![2; 10])],
+ );
+ cache.cache.run_pending_tasks();
+ assert_eq!(
+ vec![PageFragmentKey::new(file_id, 0, &range)],
+ cache.find_index_candidates(file_id, 0, &range)
+ );
+
+ let lookup = cache.lookup(file_id, 0, std::slice::from_ref(&range));
+ assert!(lookup.is_fully_cached());
+ assert_eq!(&vec![2; 10][..], &lookup.cached_parts[0][0].bytes[..]);
+ }
+
+ #[test]
+ fn test_page_cache_retains_disjoint_inserts_for_same_row_group() {
+ let cache = PageRangeCache::new(1000);
+ let file_id = FileId::random();
+ let range1 = 0..10;
+ let range2 = 20..30;
+
+ cache.insert_ranges(
+ file_id,
+ 0,
+ std::slice::from_ref(&range1),
+ &[Bytes::from(vec![1; 10])],
+ );
+ cache.insert_ranges(
+ file_id,
+ 0,
+ std::slice::from_ref(&range2),
+ &[Bytes::from(vec![2; 10])],
+ );
+
+ let lookup = cache.lookup(file_id, 0, &[range1, range2]);
+ assert!(lookup.is_fully_cached());
+ assert_eq!(2, lookup.cached_range_count);
+ assert_eq!(&vec![1; 10][..], &lookup.cached_parts[0][0].bytes[..]);
+ assert_eq!(&vec![2; 10][..], &lookup.cached_parts[1][0].bytes[..]);
+ }
+
+ #[test]
+ fn test_page_cache_fragment_eviction() {
+ let file_id = FileId::random();
+ let range = 0..10;
+ let key = PageFragmentKey::new(file_id, 0, &range);
+ let page = Bytes::from(vec![1; 10]);
+ let cache = PageRangeCache::new(page_cache_weight(&key, &page) as u64);
+
+ cache.insert_ranges(
+ file_id,
+ 0,
+ std::slice::from_ref(&range),
+ &[Bytes::from(vec![1; 10])],
+ );
+ assert!(
+ cache
+ .lookup(file_id, 0, std::slice::from_ref(&range))
+ .is_fully_cached()
+ );
+
+ cache.cache.invalidate(&key);
+ cache.cache.run_pending_tasks();
+ assert!(cache.find_index_candidates(file_id, 0, &range).is_empty());
+
+ let lookup = cache.lookup(file_id, 0, std::slice::from_ref(&range));
+ assert!(!lookup.is_fully_cached());
+ assert_eq!(vec![0..10], lookup.missing_ranges);
+ }
+
+ #[test]
+ fn test_page_cache_rejects_oversized_fragment() {
+ let cache = PageRangeCache::new(1);
+ let file_id = FileId::random();
+ let range = 0..10;
+
+ cache.insert_ranges(
+ file_id,
+ 0,
+ std::slice::from_ref(&range),
+ &[Bytes::from(vec![1; 10])],
+ );
+ cache.cache.run_pending_tasks();
+
+ let lookup = cache.lookup(file_id, 0, std::slice::from_ref(&range));
+ assert!(!lookup.is_fully_cached());
+ assert_eq!(vec![0..10], lookup.missing_ranges);
}
#[test]
diff --git a/src/mito2/src/compaction.rs b/src/mito2/src/compaction.rs
index 5cf3c444a8..c91809e253 100644
--- a/src/mito2/src/compaction.rs
+++ b/src/mito2/src/compaction.rs
@@ -150,6 +150,7 @@ impl CompactionScheduler {
}
/// Schedules a compaction for the region.
+ /// Returns whether a compaction is scheduled.
#[allow(clippy::too_many_arguments)]
pub(crate) async fn schedule_compaction(
&mut self,
@@ -161,7 +162,7 @@ impl CompactionScheduler {
manifest_ctx: &ManifestContextRef,
schema_metadata_manager: SchemaMetadataManagerRef,
max_parallelism: usize,
- ) -> Result<()> {
+ ) -> Result