Merge branch 'main' into promql-widen-narrow-join-rule

# Conflicts:
#	src/query/src/promql/planner.rs
#	tests/cases/standalone/common/promql/tsid_binary_join_regression.result
#	tests/cases/standalone/common/promql/tsid_binary_join_regression.sql

.github/workflows/dev-build.yml

@@ -30,7 +30,7 @@ on:
       linux_arm64_runner:
         type: choice
         description: The runner uses to build linux-arm64 artifacts
-        default: ec2-c6g.4xlarge-arm64
+        default: ec2-c6g.8xlarge-arm64
         options:
           - ec2-c6g.xlarge-arm64 # 4C8G
           - ec2-c6g.2xlarge-arm64 # 8C16G

.github/workflows/nightly-build.yml

@@ -27,7 +27,7 @@ on:
       linux_arm64_runner:
         type: choice
         description: The runner uses to build linux-arm64 artifacts
-        default: ec2-c6g.4xlarge-arm64
+        default: ec2-c6g.8xlarge-arm64
         options:
           - ec2-c6g.xlarge-arm64 # 4C8G
           - ec2-c6g.2xlarge-arm64 # 8C16G

.github/workflows/nightly-jsonbench.yaml

@@ -0,0 +1,223 @@
+name: Nightly JSONBench
+
+on:
+  workflow_run:
+    workflows: [ "GreptimeDB Nightly Build" ]
+    types: [ completed ]
+  workflow_dispatch:
+    inputs:
+      run_id:
+        description: The nightly build workflow run id to download GreptimeDB artifacts from
+        required: true
+        type: string
+
+permissions:
+  actions: read
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
+jobs:
+  resolve-artifact:
+    name: Resolve GreptimeDB nightly artifact
+    if: ${{ github.repository == 'GreptimeTeam/greptimedb' && (github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success') }}
+    runs-on: ubuntu-latest
+    outputs:
+      artifact-name: ${{ steps.find-artifact.outputs.artifact-name }}
+      run-id: ${{ steps.resolve-run-id.outputs.run-id }}
+    steps:
+      - name: Resolve nightly build run id
+        id: resolve-run-id
+        shell: bash
+        env:
+          EVENT_NAME: ${{ github.event_name }}
+          WORKFLOW_RUN_ID: ${{ github.event.workflow_run.id }}
+          INPUT_RUN_ID: ${{ inputs.run_id }}
+        run: |
+          set -euo pipefail
+
+          if [[ "${EVENT_NAME}" == "workflow_dispatch" ]]; then
+            run_id="${INPUT_RUN_ID}"
+          else
+            run_id="${WORKFLOW_RUN_ID}"
+          fi
+
+          if [[ ! "${run_id}" =~ ^[0-9]+$ ]]; then
+            echo "Invalid workflow run id: ${run_id}"
+            exit 1
+          fi
+
+          echo "run-id=${run_id}" >> "${GITHUB_OUTPUT}"
+
+      - name: Find GreptimeDB nightly artifact
+        id: find-artifact
+        shell: bash
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          RUN_ID: ${{ steps.resolve-run-id.outputs.run-id }}
+        run: |
+          set -euo pipefail
+
+          artifact_name=$(gh api "repos/${GITHUB_REPOSITORY}/actions/runs/${RUN_ID}/artifacts" --paginate \
+            --jq '.artifacts[] | select(.name | test("^greptime-linux-arm64-nightly-[0-9]{8}-[0-9a-f]+$")) | .name' \
+            | head -n 1)
+
+          if [[ -z "${artifact_name}" ]]; then
+            echo "Cannot find linux arm64 nightly artifact in workflow run ${RUN_ID}."
+            exit 1
+          fi
+
+          echo "Download GreptimeDB artifact: ${artifact_name}"
+          echo "artifact-name=${artifact_name}" >> "${GITHUB_OUTPUT}"
+
+  allocate-runner:
+    name: Allocate runner
+    if: ${{ github.repository == 'GreptimeTeam/greptimedb' && (github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success') }}
+    needs: [ resolve-artifact ]
+    runs-on: ubuntu-latest
+    outputs:
+      linux-arm64-runner: ${{ steps.start-linux-arm64-runner.outputs.label }}
+
+      # The following EC2 resource id will be used for resource releasing.
+      linux-arm64-ec2-runner-label: ${{ steps.start-linux-arm64-runner.outputs.label }}
+      linux-arm64-ec2-runner-instance-id: ${{ steps.start-linux-arm64-runner.outputs.ec2-instance-id }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          persist-credentials: false
+
+      - name: Allocate Linux ARM64 runner
+        uses: ./.github/actions/start-runner
+        id: start-linux-arm64-runner
+        with:
+          runner: ${{ vars.DEFAULT_ARM64_RUNNER }}
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: ${{ vars.EC2_RUNNER_REGION }}
+          github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
+          image-id: ${{ vars.EC2_RUNNER_LINUX_ARM64_IMAGE_ID }}
+          security-group-id: ${{ vars.EC2_RUNNER_SECURITY_GROUP_ID }}
+          subnet-id: ${{ vars.EC2_RUNNER_SUBNET_ID }}
+
+  jsonbench:
+    name: Run JSONBench
+    if: ${{ github.repository == 'GreptimeTeam/greptimedb' && (github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success') }}
+    needs: [ resolve-artifact, allocate-runner ]
+    runs-on: ${{ needs.allocate-runner.outputs.linux-arm64-runner }}
+    timeout-minutes: 120
+    env:
+      JSONBENCH_OUTPUT_PREFIX: _linux-arm64
+    steps:
+      - name: Download GreptimeDB nightly artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: ${{ needs.resolve-artifact.outputs.artifact-name }}
+          path: greptimedb-artifact
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          run-id: ${{ needs.resolve-artifact.outputs.run-id }}
+
+      - name: Prepare GreptimeDB binary
+        shell: bash
+        run: |
+          set -euo pipefail
+
+          tar -xzf "greptimedb-artifact/${{ needs.resolve-artifact.outputs.artifact-name }}.tar.gz"
+          cp "${{ needs.resolve-artifact.outputs.artifact-name }}/greptime" ./greptime
+          chmod +x ./greptime
+          rm -rf greptimedb-artifact "${{ needs.resolve-artifact.outputs.artifact-name }}"
+
+      - name: Run JSONBench
+        env:
+          # TODO(LFC): Change to "3" (100m) when JSON2 ingestion performance is optimized.
+          JSONBENCH_DATASET: 2
+        shell: bash
+        run: |
+          set -euo pipefail
+
+          export JSONBENCH_DATA_DIR="/root/data/bluesky"
+          echo "Use JSONBench data directory ${JSONBENCH_DATA_DIR}"
+
+          echo "Cloning JSONBench"
+          git clone --branch greptimedb-new-json --depth 1 https://github.com/GreptimeTeam/JSONBench.git JSONBench
+
+          echo "Downloading JSONBench dataset choice ${JSONBENCH_DATASET} to ${JSONBENCH_DATA_DIR}"
+          mkdir -p "${JSONBENCH_DATA_DIR}"
+          printf "${JSONBENCH_DATASET}\n" | ./JSONBench/download_data.sh
+          downloaded_files=$(find "${JSONBENCH_DATA_DIR}" -type f | wc -l)
+          echo "Downloaded JSONBench dataset files: ${downloaded_files}"
+
+          export GREPTIMEDB_STANDALONE__WAL__DIR=greptimedb_data/wal
+          export GREPTIMEDB_STANDALONE__STORAGE__DATA_HOME=greptimedb_data
+          export GREPTIMEDB_STANDALONE__LOGGING__DIR=greptimedb_data/logs
+          export GREPTIMEDB_STANDALONE__LOGGING__APPEND_STDOUT=false
+          export GREPTIMEDB_STANDALONE__HTTP__BODY_LIMIT=1GB
+          export GREPTIMEDB_STANDALONE__HTTP__TIMEOUT=500s
+
+          echo "Starting GreptimeDB standalone"
+          ./greptime standalone start > greptimedb.log 2>&1 &
+          greptime_pid=$!
+          trap 'kill "${greptime_pid}" 2>/dev/null || true' EXIT
+
+          echo "Waiting for GreptimeDB health check"
+          until curl -s --fail -o /dev/null http://localhost:4000/health; do
+            if ! kill -0 "${greptime_pid}" 2>/dev/null; then
+              cat greptimedb.log
+              exit 1
+            fi
+            sleep 1
+          done
+          echo "GreptimeDB is ready"
+
+          cp ./greptime JSONBench/greptimedb/greptime
+
+          cd JSONBench/greptimedb
+          echo "Running JSONBench main.sh with dataset choice ${JSONBENCH_DATASET} and install=false"
+          ./main.sh ${JSONBENCH_DATASET} "${JSONBENCH_DATA_DIR}" success.log error.log "${JSONBENCH_OUTPUT_PREFIX}" false
+          echo "JSONBench finished"
+
+      - name: Upload JSONBench results
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: jsonbench-results
+          path: |
+            ./greptimedb.log
+            ./JSONBench/greptimedb/*.log
+            ./JSONBench/greptimedb/*.total_size
+            ./JSONBench/greptimedb/*.data_size
+            ./JSONBench/greptimedb/*.index_size
+            ./JSONBench/greptimedb/*.count
+            ./JSONBench/greptimedb/*.results_runtime
+            ./JSONBench/greptimedb/*.query_results
+          if-no-files-found: ignore
+          retention-days: 7
+
+  stop-linux-arm64-runner:
+    name: Stop Linux ARM64 runner
+    # It's always run as the last job in the workflow to make sure that the runner is released.
+    if: ${{ always() && needs.allocate-runner.outputs.linux-arm64-ec2-runner-instance-id != '' }}
+    runs-on: ubuntu-latest
+    needs: [
+      allocate-runner,
+      jsonbench,
+    ]
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          persist-credentials: false
+
+      - name: Stop Linux ARM64 runner
+        uses: ./.github/actions/stop-runner
+        with:
+          label: ${{ needs.allocate-runner.outputs.linux-arm64-ec2-runner-label }}
+          ec2-instance-id: ${{ needs.allocate-runner.outputs.linux-arm64-ec2-runner-instance-id }}
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: ${{ vars.EC2_RUNNER_REGION }}
+          github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}

Cargo.toml

@@ -158,7 +158,7 @@ fs2 = "0.4"
 fst = "0.4.7"
 futures = "0.3"
 futures-util = "0.3"
-greptime-proto = { git = "https://github.com/GreptimeTeam/greptime-proto.git", rev = "7224c2ad6d11db612fbdb621c36135fc37ffce35" }
+greptime-proto = { git = "https://github.com/GreptimeTeam/greptime-proto.git", rev = "912f6c21017d5ab6c529568d36e7b8c30a94d84b" }
 hex = "0.4"
 http = "1"
 humantime = "2.1"
@@ -178,7 +178,7 @@ nalgebra = "0.33"
 nix = { version = "0.30.1", default-features = false, features = ["event", "fs", "process"] }
 notify = "8.0"
 num_cpus = "1.16"
-object_store_opendal = { git = "https://github.com/apache/opendal.git", rev = "4ad2d85296ffa6fdc2882f97d3c760ee243913f7" }
+object_store_opendal = "0.57"
 once_cell = "1.18"
 opentelemetry-proto = { version = "0.31", features = [
     "gen-tonic",
@@ -259,7 +259,7 @@ tracing-opentelemetry = "0.31.0"
 tracing-subscriber = { version = "0.3", features = ["env-filter", "json", "fmt"] }
 typetag = "0.2"
 uuid = { version = "1.17", features = ["serde", "v4", "v7", "fast-rng"] }
-vrl = "0.25"
+vrl = "0.33"
 zstd = "0.13"
 # DO_NOT_REMOVE_THIS: END_OF_EXTERNAL_DEPENDENCIES
 

config/config.md

@@ -14,6 +14,7 @@
 | --- | -----| ------- | ----------- |
 | `default_timezone` | String | Unset | The default timezone of the server. |
 | `default_column_prefix` | String | Unset | The default column prefix for auto-created time index and value columns. |
+| `auto_create_table` | Bool | `true` | Server-side global switch for auto table creation on write.<br/>When `false`, a missing table is never auto-created even if the request sets the `auto_create_table` hint to `true`. Default: `true`. |
 | `user_provider` | String | Unset | The user provider for authentication.<br/>Examples: "static_user_provider:file:/path/to/users", "static_user_provider:cmd:greptime_user=greptime_pwd" |
 | `max_in_flight_write_bytes` | String | Unset | Maximum total memory for all concurrent write request bodies and messages (HTTP, gRPC, Flight).<br/>Set to 0 to disable the limit. Default: "0" (unlimited) |
 | `write_bytes_exhausted_policy` | String | Unset | Policy when write bytes quota is exhausted.<br/>Options: "wait" (default, 10s timeout), "wait(<duration>)" (e.g., "wait(30s)"), "fail" |
@@ -230,6 +231,7 @@
 | --- | -----| ------- | ----------- |
 | `default_timezone` | String | Unset | The default timezone of the server. |
 | `default_column_prefix` | String | Unset | The default column prefix for auto-created time index and value columns. |
+| `auto_create_table` | Bool | `true` | Server-side global switch for auto table creation on write.<br/>When `false`, a missing table is never auto-created even if the request sets the `auto_create_table` hint to `true`. Default: `true`. |
 | `user_provider` | String | Unset | The user provider for authentication.<br/>Examples: "static_user_provider:file:/path/to/users", "static_user_provider:cmd:greptime_user=greptime_pwd" |
 | `max_in_flight_write_bytes` | String | Unset | Maximum total memory for all concurrent write request bodies and messages (HTTP, gRPC, Flight).<br/>Set to 0 to disable the limit. Default: "0" (unlimited) |
 | `write_bytes_exhausted_policy` | String | Unset | Policy when write bytes quota is exhausted.<br/>Options: "wait" (default, 10s timeout), "wait(<duration>)" (e.g., "wait(30s)"), "fail" |
@@ -449,6 +451,7 @@
 | `init_regions_in_background` | Bool | `false` | Initialize all regions in the background during the startup.<br/>By default, it provides services after all regions have been initialized. |
 | `init_regions_parallelism` | Integer | `16` | Parallelism of initializing regions. |
 | `max_concurrent_queries` | Integer | `0` | The maximum concurrent queries allowed to be executed. Zero means unlimited. |
+| `concurrent_query_limiter_timeout` | String | `100ms` | Timeout to acquire a permit from the concurrent query limiter when `max_concurrent_queries` is reached. |
 | `enable_telemetry` | Bool | `true` | Enable telemetry to collect anonymous usage data. Enabled by default. |
 | `http` | -- | -- | The HTTP server options. |
 | `http.addr` | String | `127.0.0.1:4000` | The address to bind the HTTP server. |
@@ -628,6 +631,7 @@
 | `flow.batching_mode.experimental_frontend_scan_timeout` | String | `30s` | Flow wait for available frontend timeout,<br/>if failed to find available frontend after frontend_scan_timeout elapsed, return error<br/>which prevent flownode from starting |
 | `flow.batching_mode.experimental_max_filter_num_per_query` | Integer | `20` | Maximum number of filters allowed in a single query |
 | `flow.batching_mode.experimental_time_window_merge_threshold` | Integer | `3` | Time window merge distance |
+| `flow.batching_mode.experimental_enable_incremental_read` | Bool | `false` | Whether to enable experimental flow incremental source reads.<br/>When disabled, batching flows always execute full-snapshot queries.<br/>Can be overridden per flow with WITH (experimental_enable_incremental_read = 'true'). |
 | `flow.batching_mode.read_preference` | String | `Leader` | Read preference of the Frontend client. |
 | `flow.batching_mode.frontend_tls` | -- | -- | -- |
 | `flow.batching_mode.frontend_tls.enabled` | Bool | `false` | Whether to enable TLS for client. |

config/datanode.example.toml

@@ -20,6 +20,9 @@ init_regions_parallelism = 16
 ## The maximum concurrent queries allowed to be executed. Zero means unlimited.
 max_concurrent_queries = 0
 
+## Timeout to acquire a permit from the concurrent query limiter when `max_concurrent_queries` is reached.
+concurrent_query_limiter_timeout = "100ms"
+
 ## Enable telemetry to collect anonymous usage data. Enabled by default.
 #+ enable_telemetry = true
 

config/flownode.example.toml

@@ -31,6 +31,10 @@ node_id = 14
 #+experimental_max_filter_num_per_query=20
 ## Time window merge distance
 #+experimental_time_window_merge_threshold=3
+## Whether to enable experimental flow incremental source reads.
+## When disabled, batching flows always execute full-snapshot queries.
+## Can be overridden per flow with WITH (experimental_enable_incremental_read = 'true').
+#+experimental_enable_incremental_read=false
 ## Read preference of the Frontend client.
 #+read_preference="Leader"
 [flow.batching_mode.frontend_tls]

config/frontend.example.toml

@@ -6,6 +6,10 @@ default_timezone = "UTC"
 ## @toml2docs:none-default
 default_column_prefix = "greptime"
 
+## Server-side global switch for auto table creation on write.
+## When `false`, a missing table is never auto-created even if the request sets the `auto_create_table` hint to `true`. Default: `true`.
+#+ auto_create_table = true
+
 ## The user provider for authentication.
 ## Examples: "static_user_provider:file:/path/to/users", "static_user_provider:cmd:greptime_user=greptime_pwd"
 ## @toml2docs:none-default

config/standalone.example.toml

@@ -6,6 +6,10 @@ default_timezone = "UTC"
 ## @toml2docs:none-default
 default_column_prefix = "greptime"
 
+## Server-side global switch for auto table creation on write.
+## When `false`, a missing table is never auto-created even if the request sets the `auto_create_table` hint to `true`. Default: `true`.
+#+ auto_create_table = true
+
 ## The user provider for authentication.
 ## Examples: "static_user_provider:file:/path/to/users", "static_user_provider:cmd:greptime_user=greptime_pwd"
 ## @toml2docs:none-default

docs/rfcs/2026-05-28-table-semantic-layer.md

@@ -0,0 +1,157 @@
+---
+Feature Name: Table Semantic Layer
+Tracking Issue: TBD
+Date: 2026-05-28
+Author: "Dennis Zhuang <killme2008@gmail.com>"
+---
+
+# Summary
+
+Attach a thin layer of semantic metadata to each table so machine consumers — LLM agents, alert generators, dashboard builders, MCP servers, ETL pipelines — can align it with the observability concepts they already know (OTel instrument kinds, Prometheus naming conventions, UCUM units, semantic conventions, severity numbers, OTel ↔ Prometheus translation rules).
+
+The mechanism reuses what already exists in `table_options` (the same slot that today carries `table_data_model` and `otlp_metric_compat`): a reserved `greptime.semantic.*` namespace, plus standard SQL column `COMMENT` for field-level supplements, plus an `information_schema.semantic_tables` view as the discovery entry point. No new protocol, no new DDL keyword.
+
+Per-table identity only. Cross-table relationships are deferred.
+
+# Motivation
+
+GreptimeDB already ingests OTLP metrics / traces / logs and Prometheus remote write. Each protocol carries rich metadata on the wire (instrument kind, temporality, unit, scope, resource, semantic-conventions version), and most of it is dropped when rows land in a table:
+
+- An `opentelemetry_traces` table looks like any wide table; signal type, source, and field provenance must be guessed from naming.
+- The OTel-to-Prometheus translation in v0.16+ actively drops scope attributes and most resource attributes; the table never records *what was dropped*.
+- Prometheus remote write v1 metadata is unreliable by protocol, but downstream tables do not flag whether `counter` typing was *declared* or *inferred* from the `_total` suffix.
+- Mixed-temporality data (OTel delta + Prometheus cumulative in the same table) is unrecoverable from schema alone.
+
+The audience is broader than LLM agents. Alert generators need to choose between `rate()` and absolute thresholds, and need units to pick sensible bounds. Dashboard builders pick visualisations by signal type. MCP servers surface a structured tool catalog instead of free-text descriptions. ETL pipelines need lineage to know whether a `service_name` column is `resource.service.name` or a free-form label. All of them currently guess from column names; the metadata to remove the guess already exists at ingest time, we just do not preserve it.
+
+# Goals
+
+1. Tag every ingested table with a stable identity using existing SQL surfaces — no new protocol, no new DDL keyword.
+2. Record the lossy transformations the ingestion path performs (dropped attributes, scope handling, type inference vs. declaration).
+3. Expose one `information_schema` view as the consumer-facing discovery entry point.
+4. Keep the layer optional and additive — tables without these options keep working unchanged.
+
+# Non-Goals
+
+- Cross-table relationship modelling. Deferred to a follow-up RFC.
+- Bespoke storage. Reuse `table_options` and column `COMMENT`.
+- Semantic enforcement at query time. The layer is descriptive, not coercive.
+- New wire protocol. Upstream standardisation is mentioned only as a future direction.
+
+# Proposal
+
+## Three mechanisms
+
+1. **`greptime.semantic.*` table options** — table-level identity and lineage. Carried inside the existing `table_options` blob. This is the same slot that today carries `table_data_model = 'greptime_trace_v1'` and `otlp_metric_compat = 'prom'`, so the mechanism is generalising what the OTLP trace auto-create path already does.
+2. **Column `COMMENT`** — column-level supplements ("this column is `resource.service.name`"; "this column carries delta values"). Standard SQL.
+3. **`information_schema.semantic_tables` view** — a denormalised projection of the options, registered through the existing `with_extra_table_factories()` hook. Tables without a `greptime.semantic.*` option do not appear in the view.
+
+## Vocabulary
+
+All keys are flat strings under the `greptime.semantic.` prefix; values are strings; unknown keys are tolerated so the vocabulary can grow without coordinated rollouts.
+
+**Common (all signals)**
+
+| Key | Example |
+| --- | --- |
+| `greptime.semantic.signal_type` | `trace` / `log` / `metric` / `event` |
+| `greptime.semantic.source` | `opentelemetry` / `prometheus` / `elasticsearch` / `loki` / `custom` |
+| `greptime.semantic.source_version` | protocol or SDK version, e.g. `v2` (Prom remote write), `1.30.0` (optional) |
+| `greptime.semantic.pipeline` | `greptime_trace_v1` (subsumes the existing `table_data_model` value) |
+
+**Trace**: `greptime.semantic.trace.conventions` (e.g. `otel-semconv-1.27`, lifted from `schema_url`, which is the version of the OpenTelemetry semantic conventions used in this table), `greptime.semantic.trace.has_events`, `greptime.semantic.trace.has_links`.
+
+**Metric** — v1 assumes one metric type per table, which is how both Prom RW and the post-v0.16 OTel ingestion path land data today; mixed-type tables are a follow-up.
+
+| Key | Example |
+| --- | --- |
+| `greptime.semantic.metric.type` | `counter` / `gauge` / `histogram` / `summary` / `updown_counter` / `gauge_histogram` / `info` / `stateset` |
+| `greptime.semantic.metric.unit` | UCUM, e.g. `s`, `By`, `{request}` |
+| `greptime.semantic.metric.temporality` | `cumulative` / `delta` (OTel only) |
+| `greptime.semantic.metric.monotonic` | `true` / `false` |
+| `greptime.semantic.metric.metadata_quality` | `declared` (OTLP / Prom RW v2 / exposition) or `inferred` (Prom RW v1, name-suffix guess) |
+| `greptime.semantic.metric.original_name` | Pre-translation OTel name when the table name was Prometheus-ised |
+
+`metadata_quality = inferred` is the load-bearing field for confidence-aware tooling: an inferred counter should be re-checked before betting on `rate()`-style semantics.
+
+**Log**: `greptime.semantic.log.severity_scheme` (`otlp` / `syslog` / `custom`), `greptime.semantic.log.body_format` (`string` / `json` / `mixed`).
+
+**Resource / scope preservation**: `greptime.semantic.resource.attributes_preserved` (JSON array string of attrs promoted to columns), `greptime.semantic.resource.attributes_dropped` (boolean), `greptime.semantic.scope.preserved` (boolean). These answer the most common downstream question: "is this data missing because it was dropped, or because it lives on a different column than I think?" List-shaped values use JSON array strings rather than comma-separated text to avoid escaping and ordering ambiguity.
+
+## Conflict and update semantics
+
+Two design decisions worth pinning down up front, because they constrain everything else:
+
+- **Conflict.** Some table-level keys (`trace.conventions` lifted from `schema_url`, `metric.temporality`, ...) cannot represent the truth when a long-lived table sees rows from multiple sources. v1 records `mixed` or `unknown` rather than a fictitious single value. Downstream consumers must treat any single-valued semantic key as best-effort, not strong evidence.
+- **Update.** Semantic options are stamped at table creation. v1 does not specify an update path; promoting `metadata_quality` from `inferred` to `declared`, refreshing `resource.attributes_preserved`, or revising `trace.conventions` on later writes is deferred. If real usage shows update is needed, it lands as a separate RFC.
+
+## `information_schema.semantic_tables`
+
+A consumer's first SQL on connect:
+
+```sql
+SELECT table_catalog, table_schema, table_name, signal_type, source, pipeline
+FROM information_schema.semantic_tables;
+```
+
+returns one row per semantic-tagged table. The view exposes a stable set of core columns (`table_catalog`, `table_schema`, `table_name`, `signal_type`, `source`, `source_version`, `pipeline`) plus a `semantic_options` JSON column carrying the rest of the `greptime.semantic.*` keys verbatim. Future keys appear inside `semantic_options` without forcing a view-schema change; only widely-used keys are ever promoted to first-class columns.
+
+# Implementation Plan
+
+Four phases, each independently shippable.
+
+1. **Identity.** Stamp `signal_type` and `source` on every auto-create path. The OTLP paths already have natural injection points; Prom remote write is the one non-trivial path because metric-engine logical tables share physical storage (see Open Question 2).
+2. **Metric specifics.** Add type / unit / temporality / monotonic / metadata_quality / original_name at OTel metric and Prom RW ingestion sites; the data is already at hand inside the OTel translator.
+3. **Resource / scope lineage.** Record what the OTel-to-Prometheus translation kept and dropped.
+4. **`information_schema.semantic_tables` view + documentation** as a stable user-facing contract.
+
+# Relationship to OpenTelemetry standardisation
+
+OTel today standardises what producers emit and how data collectors are managed; the read side — what a backend exposes back to clients — is deliberately vendor turf. OTLP is one-way; OpAMP is agent management; OTEP-0243 (App Telemetry Schema) is producer-side; `schema_url` is producer-stated with no reverse. Adjacent precedents — Prometheus `/api/v1/metadata`, Loki labels API, Tempo tags, Jaeger services, ad-hoc MCP servers — are all vendor-specific.
+
+This is a real gap. The shape we propose locally (signal-agnostic, `schema_url`-aware, structured around a small vocabulary) is deliberately close to what a future upstream OTEP for a backend-catalog read API could look like, with Weaver's *Resolved Telemetry Schema* as the natural data model. We do not commit to driving such an OTEP here; we do commit to keeping the local shape close enough that a future upstream proposal does not force a breaking migration.
+
+# Alternatives
+
+- **New DDL syntax (`SEMANTIC trace WITH (...)`).** Cleaner-looking but non-standard and forces every client to learn it. The metadata is not interesting enough to justify a new keyword.
+- **Dedicated `_semantic` system table.** Doubles the storage path for what is static per-table KV and adds lifecycle questions (drop, backfill). A view over `table_options` covers the same access pattern.
+- **Column comments only.** Discovery (`WHERE signal_type = 'trace'`) becomes a full-text problem. Comments are good for column-level supplements, not for identity.
+- **Encode everything into the table name.** What we do today. Every new field becomes a new naming convention.
+
+# Open Questions
+
+1. **Namespace prefix.** `greptime.semantic.*` vs. bare `semantic.*`. v1 picks the vendored prefix; alias or migrate if a community standard later emerges.
+2. **Prom RW injection point.** Metric-engine logical tables share physical storage, so per-logical-table options need a hook that does not exist as cleanly as the OTLP trace branch. A short spike before Phase 1 lands for Prom RW.
+3. **Mixed-type metric tables.** When ingestion modes that pack multiple metric types into one table appear, `metric.type` migrates from table-level to row-level. v1 leaves a `metric.type = 'mixed'` marker and punts.
+4. **Stability surface.** Top-level keys (`signal_type`, `source`) are stable; sub-namespaces (`metric.*`, ...) are evolving until v1.0 of the layer is declared.
+
+# Future Work
+
+- **Cross-table relationships.** Paired trace/services tables, metric/info pairing, JOIN hints. Its own RFC.
+- **Producer SDK/client identity.** An optional `greptime.semantic.source.sdk` key recording the emitting client (e.g. `opentelemetry-go`, `opentelemetry-java`, `opentelemetry-collector`). Because a single table can receive data from multiple SDKs (a shared trace table is the common case), mixed producers collapse to `mixed`, following the same conflict rule as the table-level keys above.
+- **Backfill** for tables created before this feature shipped.
+- **Upstream proposal.** Carry the shape into a community proposal — likely an OTEP for an OTLP-Catalog read API plus an MCP binding — informed by Greptime's local usage data.
+
+# References
+
+OpenTelemetry:
+- [OTLP specification](https://opentelemetry.io/docs/specs/otlp/)
+- [OTel Schemas (`schema_url`)](https://opentelemetry.io/docs/specs/otel/schemas/)
+- [Semantic Conventions](https://opentelemetry.io/docs/specs/semconv/)
+- [OTEP-0243: App Telemetry Schema](https://github.com/open-telemetry/oteps/blob/main/text/0243-app-telemetry-schema-vision-roadmap.md)
+- [OpAMP specification](https://github.com/open-telemetry/opamp-spec/blob/main/specification.md)
+- [Weaver: Resolved Telemetry Schema](https://github.com/open-telemetry/weaver)
+- [2025 Stability Proposal](https://opentelemetry.io/blog/2025/stability-proposal-announcement/)
+
+Prometheus / OpenMetrics:
+- [Prometheus Remote Write 1.0](https://prometheus.io/docs/specs/prw/remote_write_spec/)
+- [Prometheus Remote Write 2.0](https://prometheus.io/docs/specs/prw/remote_write_spec_2_0/)
+- [Prometheus exposition formats](https://prometheus.io/docs/instrumenting/exposition_formats/)
+- [Prometheus HTTP API: `/api/v1/metadata`](https://prometheus.io/docs/prometheus/latest/querying/api/#querying-metric-metadata)
+
+Units and conventions:
+- [UCUM — Unified Code for Units of Measure](https://ucum.org/)
+
+GreptimeDB:
+- [OTLP ingestion guide](https://docs.greptime.com/user-guide/ingest-data/for-observability/opentelemetry/)
+- [Trace data model](https://docs.greptime.com/user-guide/traces/data-model/)

src/auth/src/permission.rs

@@ -16,6 +16,7 @@ use std::fmt::Debug;
 use std::sync::Arc;
 
 use api::v1::greptime_request::Request;
+use api::v1::query_request::Query;
 use common_telemetry::debug;
 use sql::statements::statement::Statement;
 
@@ -42,10 +43,12 @@ impl<'a> PermissionReq<'a> {
     /// Returns true if the permission request is for read operations.
     pub fn is_readonly(&self) -> bool {
         match self {
-            PermissionReq::GrpcRequest(Request::Query(_))
-            | PermissionReq::PromQuery
-            | PermissionReq::LogQuery
-            | PermissionReq::PromStoreRead => true,
+            PermissionReq::GrpcRequest(Request::Query(query_request)) => {
+                !matches!(query_request.query, Some(Query::InsertIntoPlan(_)))
+            }
+            PermissionReq::PromQuery | PermissionReq::LogQuery | PermissionReq::PromStoreRead => {
+                true
+            }
             PermissionReq::SqlStatement(stmt) => stmt.is_readonly(),
 
             PermissionReq::GrpcRequest(_)
@@ -196,4 +199,14 @@ mod tests {
         assert!(matches!(read_result, PermissionResp::Reject));
         assert!(matches!(write_result, PermissionResp::Allow));
     }
+
+    #[test]
+    fn test_grpc_insert_into_plan_is_write_request() {
+        let request = Request::Query(api::v1::QueryRequest {
+            query: Some(Query::InsertIntoPlan(api::v1::InsertIntoPlan::default())),
+        });
+        let req = PermissionReq::GrpcRequest(&request);
+
+        assert!(req.is_write());
+    }
 }

src/catalog/src/kvbackend/table_cache.rs

@@ -14,7 +14,9 @@
 
 use std::sync::Arc;
 
-use common_meta::cache::{CacheContainer, Initializer, TableInfoCacheRef, TableNameCacheRef};
+use common_meta::cache::{
+    CacheContainer, InitStrategy, Initializer, TableInfoCacheRef, TableNameCacheRef,
+};
 use common_meta::error::{Result as MetaResult, ValueNotExistSnafu};
 use common_meta::instruction::CacheIdent;
 use futures::future::BoxFuture;
@@ -38,7 +40,14 @@ pub fn new_table_cache(
 ) -> TableCache {
     let init = init_factory(table_info_cache, table_name_cache);
 
-    CacheContainer::new(name, cache, Box::new(invalidator), init, filter)
+    CacheContainer::with_strategy(
+        name,
+        cache,
+        Box::new(invalidator),
+        init,
+        filter,
+        InitStrategy::VersionChecked,
+    )
 }
 
 fn init_factory(

src/cli/src/data/export_v2/command.rs

@@ -1077,7 +1077,9 @@ async fn verify_snapshot(storage: &OpenDalStorage) -> Result<VerifyReport> {
             ));
         }
         let data_files = storage.list_files_recursive("data/").await?;
-        if let Some(path) = data_files.first() {
+        // Report the lexicographically smallest path so the message is stable
+        // regardless of listing order across backends.
+        if let Some(path) = data_files.iter().min() {
             report.push_error(format!(
                 "Schema-only snapshot should not contain data files (found '{}')",
                 path
@@ -1103,75 +1105,113 @@ fn summarize_chunks(manifest: &Manifest) -> VerifyChunkSummary {
     }
 }
 
+/// A data file declared by a completed chunk that is expected to exist in storage.
+#[derive(Debug)]
+struct ChunkFile {
+    chunk_id: u32,
+    path: String,
+}
+
+/// Expected snapshot contents derived purely from the manifest (no object-store IO).
+///
+/// Separating planning from scanning makes it obvious which problems come from
+/// the manifest alone and which require comparing against actual storage.
+#[derive(Debug, Default)]
+struct VerifyPlan {
+    /// Valid data files declared by completed chunks; each must exist in storage.
+    files_to_check: Vec<ChunkFile>,
+    /// All syntactically-safe data paths declared by any chunk, regardless of
+    /// status. Used as the orphan-detection baseline so a listed-but-invalid
+    /// file is not also reported as unexpected.
+    claimed_data_files: HashSet<String>,
+    /// Total data-file references in completed chunks (valid + invalid).
+    data_files_total: usize,
+    /// Problems detectable from the manifest alone.
+    problems: Vec<VerifyProblem>,
+}
+
+/// Actual data files discovered under `data/` (the only object-store IO in
+/// chunk/data-file verification).
+#[derive(Debug)]
+struct VerifyDataScan {
+    existing_data_files: HashSet<String>,
+}
+
+/// Result of reconciling the manifest plan against the storage scan.
+#[derive(Debug, Default)]
+struct VerifyOutcome {
+    data_files_total: usize,
+    data_files_verified: usize,
+    problems: Vec<VerifyProblem>,
+}
+
 async fn verify_chunks_and_data_files(
     storage: &OpenDalStorage,
     report: &mut VerifyReport,
 ) -> Result<()> {
-    let existing_files: HashSet<_> = storage
-        .list_files_recursive("data/")
-        .await?
-        .into_iter()
-        .collect();
-    let mut data_files_total = 0;
-    let mut data_files_verified = 0;
-    let mut problems = Vec::new();
-    let mut seen_chunk_ids = HashSet::new();
-    let mut claimed_data_files = HashSet::new();
+    let plan = build_verify_plan(&report.manifest);
+    let scan = scan_data_files(storage).await?;
+    let outcome = reconcile_plan_with_scan(plan, &scan);
 
-    for chunk in &report.manifest.chunks {
+    report.data_files_total = outcome.data_files_total;
+    report.data_files_verified = outcome.data_files_verified;
+    report.problems.extend(outcome.problems);
+
+    Ok(())
+}
+
+/// Builds the expected-state plan from the manifest. Pure; performs no IO.
+fn build_verify_plan(manifest: &Manifest) -> VerifyPlan {
+    let mut plan = VerifyPlan::default();
+    let mut seen_chunk_ids = HashSet::new();
+
+    for chunk in &manifest.chunks {
         if !seen_chunk_ids.insert(chunk.id) {
-            problems.push(VerifyProblem {
+            plan.problems.push(VerifyProblem {
                 severity: VerifySeverity::Error,
                 message: format!("Chunk {}: duplicate chunk id", chunk.id),
             });
         }
         for file in &chunk.files {
             if let Some(path) = safe_manifest_data_file_path(file) {
-                claimed_data_files.insert(path.to_string());
+                plan.claimed_data_files.insert(path.to_string());
             }
         }
 
         match chunk.status {
             ChunkStatus::Completed => {
                 if chunk.files.is_empty() {
-                    problems.push(VerifyProblem {
+                    plan.problems.push(VerifyProblem {
                         severity: VerifySeverity::Error,
                         message: format!("Chunk {}: completed chunk has no data files", chunk.id),
                     });
                     continue;
                 }
-                let allowed_prefixes = report
-                    .manifest
+                let allowed_prefixes = manifest
                     .schemas
                     .iter()
                     .map(|schema| data_dir_for_schema_chunk(schema, chunk.id))
                     .collect::<Vec<_>>();
                 for file in &chunk.files {
-                    data_files_total += 1;
-                    let Some(path) = valid_manifest_data_file_path(file, &allowed_prefixes) else {
-                        problems.push(VerifyProblem {
+                    plan.data_files_total += 1;
+                    match valid_manifest_data_file_path(file, &allowed_prefixes) {
+                        Some(path) => plan.files_to_check.push(ChunkFile {
+                            chunk_id: chunk.id,
+                            path: path.to_string(),
+                        }),
+                        None => plan.problems.push(VerifyProblem {
                             severity: VerifySeverity::Error,
                             message: format!(
                                 "Chunk {}: invalid data file path '{}'",
                                 chunk.id, file
                             ),
-                        });
-                        continue;
-                    };
-
-                    if existing_files.contains(path) {
-                        data_files_verified += 1;
-                    } else {
-                        problems.push(VerifyProblem {
-                            severity: VerifySeverity::Error,
-                            message: format!("Chunk {}: missing file '{}'", chunk.id, path),
-                        });
+                        }),
                     }
                 }
             }
             ChunkStatus::Skipped => {
                 if !chunk.files.is_empty() {
-                    problems.push(VerifyProblem {
+                    plan.problems.push(VerifyProblem {
                         severity: VerifySeverity::Error,
                         message: format!(
                             "Chunk {}: skipped chunk should not list data files",
@@ -1181,20 +1221,20 @@ async fn verify_chunks_and_data_files(
                 }
             }
             ChunkStatus::Pending => {
-                problems.push(VerifyProblem {
+                plan.problems.push(VerifyProblem {
                     severity: VerifySeverity::Error,
                     message: format!("Chunk {}: status is 'pending'", chunk.id),
                 });
             }
             ChunkStatus::InProgress => {
-                problems.push(VerifyProblem {
+                plan.problems.push(VerifyProblem {
                     severity: VerifySeverity::Error,
                     message: format!("Chunk {}: status is 'in_progress'", chunk.id),
                 });
             }
             ChunkStatus::Failed => {
                 let reason = chunk.error.as_deref().unwrap_or("unknown error");
-                problems.push(VerifyProblem {
+                plan.problems.push(VerifyProblem {
                     severity: VerifySeverity::Error,
                     message: format!("Chunk {}: status is 'failed' (error: {})", chunk.id, reason),
                 });
@@ -1202,20 +1242,60 @@ async fn verify_chunks_and_data_files(
         }
     }
 
-    for path in &existing_files {
-        if !claimed_data_files.contains(path) {
+    plan
+}
+
+/// Lists all data files under `data/`. This is the only object-store IO in
+/// chunk/data-file verification.
+async fn scan_data_files(storage: &OpenDalStorage) -> Result<VerifyDataScan> {
+    let existing_data_files = storage
+        .list_files_recursive("data/")
+        .await?
+        .into_iter()
+        .collect();
+    Ok(VerifyDataScan {
+        existing_data_files,
+    })
+}
+
+/// Reconciles the manifest plan against the storage scan. Pure; performs no IO.
+///
+/// Emits missing-file problems for expected files absent from storage and
+/// unexpected-file problems for storage files no chunk claims. Unexpected files
+/// are sorted by path so output is deterministic regardless of listing order.
+fn reconcile_plan_with_scan(plan: VerifyPlan, scan: &VerifyDataScan) -> VerifyOutcome {
+    let mut problems = plan.problems;
+    let mut data_files_verified = 0;
+
+    for file in &plan.files_to_check {
+        if scan.existing_data_files.contains(&file.path) {
+            data_files_verified += 1;
+        } else {
             problems.push(VerifyProblem {
                 severity: VerifySeverity::Error,
-                message: format!("Unexpected data file '{}' is not listed in manifest", path),
+                message: format!("Chunk {}: missing file '{}'", file.chunk_id, file.path),
             });
         }
     }
 
-    report.data_files_total = data_files_total;
-    report.data_files_verified = data_files_verified;
-    report.problems.extend(problems);
+    let mut orphans: Vec<&String> = scan
+        .existing_data_files
+        .iter()
+        .filter(|path| !plan.claimed_data_files.contains(*path))
+        .collect();
+    orphans.sort();
+    for path in orphans {
+        problems.push(VerifyProblem {
+            severity: VerifySeverity::Error,
+            message: format!("Unexpected data file '{}' is not listed in manifest", path),
+        });
+    }
 
-    Ok(())
+    VerifyOutcome {
+        data_files_total: plan.data_files_total,
+        data_files_verified,
+        problems,
+    }
 }
 
 fn valid_manifest_data_file_path<'a>(
@@ -2294,6 +2374,90 @@ mod tests {
         );
     }
 
+    #[test]
+    fn test_build_verify_plan_classifies_chunks_without_io() {
+        let mut manifest = test_manifest(
+            chrono::Utc.with_ymd_and_hms(2026, 1, 1, 0, 0, 0).unwrap(),
+            false,
+            true,
+        );
+        // test_manifest(complete) gives: chunk 1 completed (1 file), chunk 2 skipped.
+        let mut failed = ChunkMeta::new(3, TimeRange::unbounded());
+        failed.mark_failed("boom".to_string());
+        manifest.chunks.push(failed);
+        manifest
+            .chunks
+            .push(ChunkMeta::new(4, TimeRange::unbounded()));
+
+        let plan = build_verify_plan(&manifest);
+
+        assert_eq!(plan.files_to_check.len(), 1);
+        assert_eq!(plan.files_to_check[0].chunk_id, 1);
+        assert_eq!(plan.files_to_check[0].path, "data/public/1/file.parquet");
+        assert_eq!(plan.data_files_total, 1);
+        assert!(
+            plan.claimed_data_files
+                .contains("data/public/1/file.parquet")
+        );
+        assert_eq!(plan.problems.len(), 2);
+        assert!(
+            plan.problems
+                .iter()
+                .any(|problem| problem.message.contains("status is 'failed'"))
+        );
+        assert!(
+            plan.problems
+                .iter()
+                .any(|problem| problem.message.contains("status is 'pending'"))
+        );
+    }
+
+    #[tokio::test]
+    async fn test_verify_snapshot_produces_deterministic_problem_output() {
+        let dir = tempdir().unwrap();
+        let manifest = test_manifest(
+            chrono::Utc.with_ymd_and_hms(2026, 1, 1, 0, 0, 0).unwrap(),
+            false,
+            true,
+        );
+        write_root_manifest(dir.path(), manifest);
+        write_snapshot_file(dir.path(), "schema/schemas.json", b"[]");
+        write_default_ddl_files(dir.path());
+        write_snapshot_file(dir.path(), "data/public/1/file.parquet", b"data");
+        // Many orphan files under a known chunk prefix to stress ordering.
+        for i in 0..50 {
+            write_snapshot_file(
+                dir.path(),
+                &format!("data/public/1/orphan_{:02}.parquet", i),
+                b"x",
+            );
+        }
+
+        let storage = file_storage_for_dir(dir.path());
+        let messages = |report: &VerifyReport| {
+            report
+                .problems
+                .iter()
+                .map(|problem| problem.message.clone())
+                .collect::<Vec<_>>()
+        };
+        let first = messages(&verify_snapshot(&storage).await.unwrap());
+        let second = messages(&verify_snapshot(&storage).await.unwrap());
+
+        // Output is identical across runs despite HashSet-based scanning.
+        assert_eq!(first, second);
+
+        let orphans = first
+            .iter()
+            .filter(|message| message.contains("Unexpected data file"))
+            .cloned()
+            .collect::<Vec<_>>();
+        assert_eq!(orphans.len(), 50);
+        let mut sorted = orphans.clone();
+        sorted.sort();
+        assert_eq!(orphans, sorted);
+    }
+
     fn write_test_manifest(root: &std::path::Path, dir: &str, manifest: Manifest) {
         let snapshot_dir = root.join(dir);
         std::fs::create_dir_all(&snapshot_dir).unwrap();

src/cmd/src/datanode.rs

@@ -79,7 +79,7 @@ impl App for Instance {
     }
 
     async fn start(&mut self) -> Result<()> {
-        plugins::start_datanode_plugins(self.datanode.plugins())
+        plugins::start_datanode_plugins(&self.datanode)
             .await
             .context(StartDatanodeSnafu)?;
 

src/cmd/src/datanode/scanbench.rs

@@ -524,6 +524,7 @@ impl ScanbenchCommand {
             options: HashMap::default(),
             skip_wal_replay: !self.enable_wal,
             checkpoint: None,
+            requirements: Default::default(),
         };
 
         engine

src/cmd/src/flownode.rs

@@ -90,7 +90,7 @@ impl App for Instance {
     }
 
     async fn start(&mut self) -> Result<()> {
-        plugins::start_flownode_plugins(self.flownode.flow_engine().plugins().clone())
+        plugins::start_flownode_plugins(&self.flownode)
             .await
             .context(StartFlownodeSnafu)?;
 

src/cmd/src/frontend.rs

@@ -95,8 +95,7 @@ impl App for Instance {
     }
 
     async fn start(&mut self) -> Result<()> {
-        let plugins = self.frontend.instance.plugins().clone();
-        plugins::start_frontend_plugins(plugins)
+        plugins::start_frontend_plugins(&self.frontend.instance)
             .await
             .context(error::StartFrontendSnafu)?;
 

src/cmd/src/metasrv.rs

@@ -68,7 +68,7 @@ impl App for Instance {
     }
 
     async fn start(&mut self) -> Result<()> {
-        plugins::start_metasrv_plugins(self.instance.plugins())
+        plugins::start_metasrv_plugins(&self.instance)
             .await
             .context(StartMetaServerSnafu)?;
 

src/cmd/src/standalone.rs

@@ -164,7 +164,7 @@ impl App for Instance {
             .start(self.leader_services_context.clone())
             .await?;
 
-        plugins::start_frontend_plugins(self.frontend.instance.plugins().clone())
+        plugins::start_frontend_plugins(&self.frontend.instance)
             .await
             .context(error::StartFrontendSnafu)?;
 

src/cmd/tests/load_config_test.rs

@@ -114,6 +114,7 @@ fn test_load_frontend_example_config() {
         component: FrontendOptions {
             default_timezone: Some("UTC".to_string()),
             default_column_prefix: Some("greptime".to_string()),
+            auto_create_table: true,
             meta_client: Some(MetaClientOptions {
                 metasrv_addrs: vec!["127.0.0.1:3002".to_string()],
                 timeout: Duration::from_secs(3),
@@ -267,6 +268,7 @@ fn test_load_standalone_example_config() {
         component: StandaloneOptions {
             default_timezone: Some("UTC".to_string()),
             default_column_prefix: Some("greptime".to_string()),
+            auto_create_table: true,
             wal: DatanodeWalConfig::RaftEngine(RaftEngineConfig {
                 dir: Some(format!("{}/{}", DEFAULT_DATA_HOME, WAL_DIR)),
                 sync_period: Some(Duration::from_secs(10)),

src/common/datasource/Cargo.toml

@@ -33,6 +33,7 @@ datatypes.workspace = true
 futures.workspace = true
 lazy_static.workspace = true
 object-store.workspace = true
+object_store_opendal.workspace = true
 orc-rust = { version = "0.8", default-features = false, features = ["async"] }
 parquet.workspace = true
 paste.workspace = true

src/common/datasource/src/file_format.rs

@@ -61,6 +61,7 @@ pub const FORMAT_COMPRESSION_TYPE: &str = "compression_type";
 pub const FORMAT_DELIMITER: &str = "delimiter";
 pub const FORMAT_SCHEMA_INFER_MAX_RECORD: &str = "schema_infer_max_record";
 pub const FORMAT_HAS_HEADER: &str = "has_header";
+pub const FORMAT_SKIP_BAD_RECORDS: &str = "skip_bad_records";
 pub const FORMAT_TYPE: &str = "format";
 pub const FILE_PATTERN: &str = "pattern";
 pub const TIMESTAMP_FORMAT: &str = "timestamp_format";
@@ -316,7 +317,7 @@ pub async fn file_to_stream(
             .with_file_compression_type(df_compression)
             .build();
 
-    let store = Arc::new(object_store::compat::OpendalStore::new(store.clone()));
+    let store = Arc::new(object_store_opendal::OpendalStore::new(store.clone()));
     let file_opener = config.file_source().create_file_opener(store, &config, 0)?;
     let stream = FileStream::new(&config, 0, file_opener, &ExecutionPlanMetricsSet::new())?;
 

src/common/datasource/src/file_format/csv.rs

@@ -13,15 +13,24 @@
 // limitations under the License.
 
 use std::collections::HashMap;
+use std::io;
 use std::str::FromStr;
+use std::sync::Arc;
+use std::task::Poll;
 
 use arrow::csv::reader::Format;
 use arrow::csv::{self, WriterBuilder};
+use arrow::error::ArrowError;
 use arrow::record_batch::RecordBatch;
-use arrow_schema::Schema;
+use arrow_schema::{Schema, SchemaRef};
 use async_trait::async_trait;
+use bytes::{Buf, Bytes};
 use common_runtime;
+use common_telemetry::warn;
 use datafusion::physical_plan::SendableRecordBatchStream;
+use datafusion::physical_plan::stream::RecordBatchStreamAdapter;
+use futures::StreamExt;
+use futures::stream::BoxStream;
 use object_store::ObjectStore;
 use snafu::ResultExt;
 use tokio_util::compat::FuturesAsyncReadCompatExt;
@@ -34,9 +43,12 @@ use crate::file_format::{self, FileFormat, stream_to_file};
 use crate::share_buffer::SharedBuffer;
 use crate::util::normalize_infer_schema;
 
+const SKIP_BAD_RECORDS_BATCH_SIZE: usize = 1;
+
 #[derive(Debug, Clone, PartialEq, Eq)]
 pub struct CsvFormat {
     pub has_header: bool,
+    pub skip_bad_records: bool,
     pub delimiter: u8,
     pub schema_infer_max_record: Option<usize>,
     pub compression_type: CompressionType,
@@ -76,13 +88,11 @@ impl TryFrom<&HashMap<String, String>> for CsvFormat {
                 })?);
         };
         if let Some(has_header) = value.get(file_format::FORMAT_HAS_HEADER) {
-            format.has_header = has_header.parse().map_err(|_| {
-                error::ParseFormatSnafu {
-                    key: file_format::FORMAT_HAS_HEADER,
-                    value: has_header,
-                }
-                .build()
-            })?;
+            format.has_header = parse_bool(file_format::FORMAT_HAS_HEADER, has_header)?;
+        };
+        if let Some(skip_bad_records) = value.get(file_format::FORMAT_SKIP_BAD_RECORDS) {
+            format.skip_bad_records =
+                parse_bool(file_format::FORMAT_SKIP_BAD_RECORDS, skip_bad_records)?;
         };
         if let Some(timestamp_format) = value.get(file_format::TIMESTAMP_FORMAT) {
             format.timestamp_format = Some(timestamp_format.clone());
@@ -97,10 +107,17 @@ impl TryFrom<&HashMap<String, String>> for CsvFormat {
     }
 }
 
+fn parse_bool(key: &'static str, value: &str) -> Result<bool> {
+    value
+        .parse()
+        .map_err(|_| error::ParseFormatSnafu { key, value }.build())
+}
+
 impl Default for CsvFormat {
     fn default() -> Self {
         Self {
             has_header: true,
+            skip_bad_records: false,
             delimiter: b',',
             schema_infer_max_record: Some(file_format::DEFAULT_SCHEMA_INFER_MAX_RECORD),
             compression_type: CompressionType::Uncompressed,
@@ -189,10 +206,136 @@ impl DfRecordBatchEncoder for csv::Writer<SharedBuffer> {
     }
 }
 
+/// Builds a CSV stream that can skip selected record-level parse/cast errors.
+///
+/// This recovery path intentionally uses one-record batches. It is slower than
+/// normal CSV scanning, but keeps each parse/cast failure isolated to a single
+/// record. Arrow's CSV decoder clears buffered rows before type parsing, so a
+/// failed multi-row flush cannot be safely retried row by row without replaying
+/// input bytes.
+pub async fn tolerant_csv_stream(
+    store: &ObjectStore,
+    path: &str,
+    schema: SchemaRef,
+    projection: Vec<usize>,
+    format: &CsvFormat,
+) -> Result<SendableRecordBatchStream> {
+    let meta = store
+        .stat(path)
+        .await
+        .context(error::ReadObjectSnafu { path })?;
+
+    let reader = store
+        .reader(path)
+        .await
+        .context(error::ReadObjectSnafu { path })?
+        .into_bytes_stream(0..meta.content_length())
+        .await
+        .context(error::ReadObjectSnafu { path })?;
+
+    let reader = format.compression_type.convert_stream(reader).boxed();
+    tolerant_csv_stream_from_reader(
+        reader,
+        path,
+        schema,
+        projection,
+        format.has_header,
+        format.delimiter,
+    )
+}
+
+fn tolerant_csv_stream_from_reader(
+    reader: BoxStream<'static, io::Result<Bytes>>,
+    path: &str,
+    schema: SchemaRef,
+    projection: Vec<usize>,
+    has_header: bool,
+    delimiter: u8,
+) -> Result<SendableRecordBatchStream> {
+    let projected_schema = Arc::new(
+        schema
+            .project(&projection)
+            .context(error::InferSchemaSnafu)?,
+    );
+    let mut decoder = csv::ReaderBuilder::new(schema)
+        .with_header(has_header)
+        .with_delimiter(delimiter)
+        .with_batch_size(SKIP_BAD_RECORDS_BATCH_SIZE)
+        .with_projection(projection)
+        .build_decoder();
+
+    let path = path.to_string();
+    let mut upstream = reader.fuse();
+    let mut buffered = Bytes::new();
+    let mut input_finished = false;
+    let stream = futures::stream::poll_fn(move |cx| {
+        loop {
+            while !input_finished {
+                if buffered.is_empty() {
+                    match futures::ready!(upstream.poll_next_unpin(cx)) {
+                        Some(Ok(bytes)) if bytes.is_empty() => continue,
+                        Some(Ok(bytes)) => buffered = bytes,
+                        Some(Err(error)) => return Poll::Ready(Some(Err(error.into()))),
+                        None => input_finished = true,
+                    }
+                }
+
+                let decoded = decoder.decode(buffered.as_ref())?;
+                if decoded > 0 {
+                    buffered.advance(decoded);
+                    continue;
+                }
+
+                if decoder.capacity() == 0 || input_finished {
+                    break;
+                }
+
+                if buffered.is_empty() {
+                    continue;
+                }
+
+                return Poll::Ready(Some(Err(ArrowError::ParseError(
+                    "CSV decoder made no progress while input bytes remain".to_string(),
+                ))));
+            }
+
+            match decoder.flush() {
+                Ok(Some(batch)) => return Poll::Ready(Some(Ok(batch))),
+                Ok(None) if input_finished => return Poll::Ready(None),
+                Ok(None) => continue,
+                Err(error) if is_skippable_arrow_error(&error) => {
+                    warn!(
+                        "Skipping bad CSV record while copying from {}: {}",
+                        path, error
+                    );
+                }
+                Err(error) => return Poll::Ready(Some(Err(error))),
+            }
+        }
+    })
+    .map(|result: std::result::Result<RecordBatch, ArrowError>| result.map_err(Into::into));
+
+    Ok(Box::pin(RecordBatchStreamAdapter::new(
+        projected_schema,
+        stream,
+    )))
+}
+
+pub fn is_skippable_arrow_error(error: &ArrowError) -> bool {
+    matches!(
+        error,
+        ArrowError::ParseError(_)
+            | ArrowError::CastError(_)
+            | ArrowError::ComputeError(_)
+            | ArrowError::InvalidArgumentError(_)
+    )
+}
+
 #[cfg(test)]
 mod tests {
     use std::sync::Arc;
 
+    use arrow_schema::{DataType, Field};
     use common_recordbatch::adapter::DfRecordBatchStreamAdapter;
     use common_recordbatch::{RecordBatch, RecordBatches};
     use common_test_util::find_workspace_path;
@@ -205,7 +348,7 @@ mod tests {
     use super::*;
     use crate::file_format::{
         FORMAT_COMPRESSION_TYPE, FORMAT_DELIMITER, FORMAT_HAS_HEADER,
-        FORMAT_SCHEMA_INFER_MAX_RECORD, FileFormat, file_to_stream,
+        FORMAT_SCHEMA_INFER_MAX_RECORD, FORMAT_SKIP_BAD_RECORDS, FileFormat, file_to_stream,
     };
     use crate::test_util::{format_schema, test_store};
 
@@ -331,11 +474,29 @@ mod tests {
                 schema_infer_max_record: Some(2000),
                 delimiter: b'\t',
                 has_header: false,
+                skip_bad_records: false,
                 timestamp_format: None,
                 time_format: None,
                 date_format: None
             }
         );
+
+        let map = HashMap::from([(FORMAT_SKIP_BAD_RECORDS.to_string(), "true".to_string())]);
+        let format = CsvFormat::try_from(&map).unwrap();
+
+        assert_eq!(
+            format,
+            CsvFormat {
+                skip_bad_records: true,
+                ..CsvFormat::default()
+            }
+        );
+    }
+
+    #[test]
+    fn test_try_from_rejects_invalid_bool_options() {
+        let map = HashMap::from([(FORMAT_SKIP_BAD_RECORDS.to_string(), "yes".to_string())]);
+        assert!(CsvFormat::try_from(&map).is_err());
     }
 
     #[tokio::test]
@@ -496,4 +657,63 @@ mod tests {
             assert_eq!(expected, pretty_print);
         }
     }
+
+    #[tokio::test]
+    async fn test_tolerant_csv_stream_continues_after_parse_error() {
+        let temp_dir = common_test_util::temp_dir::create_temp_dir("test_tolerant_csv_stream");
+        let csv_file_path = temp_dir.path().join("input.csv");
+        std::fs::write(
+            &csv_file_path,
+            "id,name,value\n1,Alice,10.5\nbad,Bad,20.0\nworse,Bad,21.0\n2,Bob,30.5",
+        )
+        .unwrap();
+
+        let store = test_store("/");
+        let schema = Arc::new(arrow_schema::Schema::new(vec![
+            Field::new("id", DataType::UInt32, false),
+            Field::new("name", DataType::Utf8, false),
+            Field::new("value", DataType::Float64, false),
+        ]));
+        let path = csv_file_path.to_str().unwrap();
+
+        let stream =
+            tolerant_csv_stream(&store, path, schema, vec![0, 1, 2], &CsvFormat::default())
+                .await
+                .unwrap();
+        let batches = stream.try_collect::<Vec<_>>().await.unwrap();
+        let pretty_print = arrow::util::pretty::pretty_format_batches(&batches)
+            .unwrap()
+            .to_string();
+        let expected = r#"+----+-------+-------+
+| id | name  | value |
++----+-------+-------+
+| 1  | Alice | 10.5  |
+| 2  | Bob   | 30.5  |
++----+-------+-------+"#;
+        assert_eq!(expected, pretty_print);
+    }
+
+    #[tokio::test]
+    async fn test_tolerant_csv_stream_fails_on_structural_csv_error() {
+        let temp_dir =
+            common_test_util::temp_dir::create_temp_dir("test_tolerant_csv_stream_csv_error");
+        let csv_file_path = temp_dir.path().join("input.csv");
+        std::fs::write(&csv_file_path, "id,name,value\n1,Alice,10.5\n2,Bob\n").unwrap();
+
+        let store = test_store("/");
+        let schema = Arc::new(arrow_schema::Schema::new(vec![
+            Field::new("id", DataType::UInt32, false),
+            Field::new("name", DataType::Utf8, false),
+            Field::new("value", DataType::Float64, false),
+        ]));
+        let path = csv_file_path.to_str().unwrap();
+
+        let stream =
+            tolerant_csv_stream(&store, path, schema, vec![0, 1, 2], &CsvFormat::default())
+                .await
+                .unwrap();
+        let error = stream.try_collect::<Vec<_>>().await.unwrap_err();
+
+        assert!(error.to_string().contains("incorrect number of fields"));
+    }
 }

src/common/datasource/src/file_format/tests.rs

@@ -44,7 +44,7 @@ struct Test<'a> {
 
 impl Test<'_> {
     async fn run(self, store: &ObjectStore) {
-        let store = Arc::new(object_store::compat::OpendalStore::new(store.clone()));
+        let store = Arc::new(object_store_opendal::OpendalStore::new(store.clone()));
         let file_opener = self
             .file_source
             .create_file_opener(store, &self.config, 0)

src/common/datasource/src/object_store/oss.rs

@@ -27,12 +27,14 @@ const ACCESS_KEY_ID: &str = "access_key_id";
 const ACCESS_KEY_SECRET: &str = "access_key_secret";
 const ROOT: &str = "root";
 const ALLOW_ANONYMOUS: &str = "allow_anonymous";
+const SKIP_SIGNATURE: &str = "skip_signature";
 
 /// Check if the key is supported in OSS configuration.
 pub fn is_supported_in_oss(key: &str) -> bool {
     [
         ROOT,
         ALLOW_ANONYMOUS,
+        SKIP_SIGNATURE,
         BUCKET,
         ENDPOINT,
         ACCESS_KEY_ID,
@@ -61,18 +63,23 @@ pub fn build_oss_backend(
         builder = builder.access_key_secret(access_key_secret);
     }
 
-    if let Some(allow_anonymous) = connection.get(ALLOW_ANONYMOUS) {
-        let allow = allow_anonymous.as_str().parse::<bool>().map_err(|e| {
+    if let Some((key, value)) = connection
+        .get(SKIP_SIGNATURE)
+        .map(|value| (SKIP_SIGNATURE, value))
+        .or_else(|| {
+            connection
+                .get(ALLOW_ANONYMOUS)
+                .map(|value| (ALLOW_ANONYMOUS, value))
+        })
+    {
+        let skip_signature = value.as_str().parse::<bool>().map_err(|e| {
             error::InvalidConnectionSnafu {
-                msg: format!(
-                    "failed to parse the option {}={}, {}",
-                    ALLOW_ANONYMOUS, allow_anonymous, e
-                ),
+                msg: format!("failed to parse the option {}={}, {}", key, value, e),
             }
             .build()
         })?;
-        if allow {
-            builder = builder.allow_anonymous();
+        if skip_signature {
+            builder = builder.skip_signature();
         }
     }
 
@@ -93,6 +100,7 @@ mod tests {
     fn test_is_supported_in_oss() {
         assert!(is_supported_in_oss(ROOT));
         assert!(is_supported_in_oss(ALLOW_ANONYMOUS));
+        assert!(is_supported_in_oss(SKIP_SIGNATURE));
         assert!(is_supported_in_oss(BUCKET));
         assert!(is_supported_in_oss(ENDPOINT));
         assert!(is_supported_in_oss(ACCESS_KEY_ID));

src/common/datasource/src/test_util.rs

@@ -103,7 +103,7 @@ pub async fn setup_stream_to_json_test(origin_path: &str, threshold: impl Fn(usi
         test_util::TEST_BATCH_SIZE,
         schema.clone(),
         FileCompressionType::UNCOMPRESSED,
-        Arc::new(object_store::compat::OpendalStore::new(store.clone())),
+        Arc::new(object_store_opendal::OpendalStore::new(store.clone())),
         true,
     );
 
@@ -157,7 +157,7 @@ pub async fn setup_stream_to_csv_test(
 
     let csv_opener = csv_source
         .create_file_opener(
-            Arc::new(object_store::compat::OpendalStore::new(store.clone())),
+            Arc::new(object_store_opendal::OpendalStore::new(store.clone())),
             &config,
             0,
         )

src/common/meta/src/cache.rs

@@ -17,7 +17,7 @@ mod flow;
 mod registry;
 mod table;
 
-pub use container::{CacheContainer, Initializer, Invalidator, TokenFilter};
+pub use container::{CacheContainer, InitStrategy, Initializer, Invalidator, TokenFilter};
 pub use flow::{TableFlownodeSetCache, TableFlownodeSetCacheRef, new_table_flownode_set_cache};
 pub use registry::{
     CacheRegistry, CacheRegistryBuilder, CacheRegistryRef, LayeredCacheRegistry,

src/common/meta/src/cache/flow/table_flownode.rs

@@ -210,7 +210,7 @@ mod tests {
     use crate::cache::flow::table_flownode::{FlowIdent, new_table_flownode_set_cache};
     use crate::instruction::{CacheIdent, CreateFlow, DropFlow};
     use crate::key::flow::FlowMetadataManager;
-    use crate::key::flow::flow_info::FlowInfoValue;
+    use crate::key::flow::flow_info::{FlowInfoValue, FlowStatus};
     use crate::key::flow::flow_route::FlowRouteValue;
     use crate::kv_backend::memory::MemoryKvBackend;
     use crate::peer::Peer;
@@ -242,11 +242,14 @@ mod tests {
                     catalog_name: DEFAULT_CATALOG_NAME.to_string(),
                     query_context: None,
                     flow_name: "my_flow".to_string(),
+                    all_source_table_names: vec![],
+                    unresolved_source_table_names: vec![],
                     raw_sql: "sql".to_string(),
                     expire_after: Some(300),
                     eval_interval_secs: None,
                     comment: "comment".to_string(),
                     options: Default::default(),
+                    status: FlowStatus::Active,
                     created_time: chrono::Utc::now(),
                     updated_time: chrono::Utc::now(),
                 },

src/common/meta/src/ddl/create_flow.rs

@@ -14,7 +14,7 @@
 
 mod metadata;
 
-use std::collections::BTreeMap;
+use std::collections::{BTreeMap, HashMap};
 use std::fmt;
 
 use api::v1::ExpireAfter;
@@ -34,13 +34,14 @@ use serde::{Deserialize, Serialize};
 use snafu::{ResultExt, ensure};
 use strum::AsRefStr;
 use table::metadata::TableId;
+use table::table_name::TableName;
 
 use crate::cache_invalidator::Context;
 use crate::ddl::DdlContext;
 use crate::ddl::utils::{add_peer_context_if_needed, map_to_procedure_error};
 use crate::error::{self, Result, UnexpectedSnafu};
 use crate::instruction::{CacheIdent, CreateFlow, DropFlow};
-use crate::key::flow::flow_info::FlowInfoValue;
+use crate::key::flow::flow_info::{FlowInfoValue, FlowStatus};
 use crate::key::flow::flow_route::FlowRouteValue;
 use crate::key::table_name::TableNameKey;
 use crate::key::{DeserializedValueWithBytes, FlowId, FlowPartitionId};
@@ -67,6 +68,7 @@ impl CreateFlowProcedure {
                 flow_id: None,
                 peers: vec![],
                 source_table_ids: vec![],
+                unresolved_source_table_names: vec![],
                 flow_context: query_context.into(), // Convert to FlowQueryContext
                 state: CreateFlowState::Prepare,
                 prev_flow_info_value: None,
@@ -89,6 +91,8 @@ impl CreateFlowProcedure {
         let create_if_not_exists = self.data.task.create_if_not_exists;
         let or_replace = self.data.task.or_replace;
 
+        validate_flow_options(&self.data.task)?;
+
         let flow_name_value = self
             .context
             .flow_metadata_manager
@@ -167,6 +171,21 @@ impl CreateFlowProcedure {
         }
 
         self.collect_source_tables().await?;
+        ensure!(
+            self.data.unresolved_source_table_names.is_empty()
+                || defer_on_missing_source(&self.data.task)?,
+            error::UnsupportedSnafu {
+                operation: format!(
+                    "Create flow with missing source tables requires WITH ('{DEFER_ON_MISSING_SOURCE_KEY}'='true'): {}",
+                    self.data
+                        .unresolved_source_table_names
+                        .iter()
+                        .map(ToString::to_string)
+                        .join(", ")
+                )
+            }
+        );
+        self.ensure_supported_replace_transition()?;
 
         // Validate that source and sink tables are not the same
         let sink_table_name = &self.data.task.sink_table_name;
@@ -189,13 +208,38 @@ impl CreateFlowProcedure {
         if self.data.flow_id.is_none() {
             self.allocate_flow_id().await?;
         }
-        self.data.state = CreateFlowState::CreateFlows;
-        // determine flow type
         self.data.flow_type = Some(get_flow_type_from_options(&self.data.task)?);
 
+        self.data.state = if self.data.is_pending() {
+            self.data.peers.clear();
+            CreateFlowState::CreateMetadata
+        } else {
+            CreateFlowState::CreateFlows
+        };
+
         Ok(Status::executing(true))
     }
 
+    fn ensure_supported_replace_transition(&self) -> Result<()> {
+        if !self.data.task.or_replace {
+            return Ok(());
+        }
+
+        let Some(prev_flow_info) = self.data.prev_flow_info_value.as_ref() else {
+            return Ok(());
+        };
+        let prev_pending = prev_flow_info.get_inner_ref().is_pending();
+        let new_pending = self.data.is_pending();
+        ensure!(
+            prev_pending == new_pending,
+            error::UnsupportedSnafu {
+                operation: "Replacing between pending and active flow states is not supported yet"
+            }
+        );
+
+        Ok(())
+    }
+
     async fn on_flownode_create_flows(&mut self) -> Result<Status> {
         // Safety: must be allocated.
         let mut create_flow = Vec::with_capacity(self.data.peers.len());
@@ -365,6 +409,63 @@ pub fn get_flow_type_from_options(flow_task: &CreateFlowTask) -> Result<FlowType
     }
 }
 
+/// The flow option key for creating pending flow metadata when source tables do not exist.
+pub const DEFER_ON_MISSING_SOURCE_KEY: &str = "defer_on_missing_source";
+
+pub fn defer_on_missing_source(flow_task: &CreateFlowTask) -> Result<bool> {
+    flow_task
+        .flow_options
+        .get(DEFER_ON_MISSING_SOURCE_KEY)
+        .map(|value| {
+            value
+                .trim()
+                .to_ascii_lowercase()
+                .parse::<bool>()
+                .map_err(|_| {
+                    error::UnexpectedSnafu {
+                        err_msg: format!(
+                            "Invalid flow option '{DEFER_ON_MISSING_SOURCE_KEY}': {value}"
+                        ),
+                    }
+                    .build()
+                })
+        })
+        .transpose()
+        .map(|value| value.unwrap_or(false))
+}
+
+pub fn validate_flow_options(flow_task: &CreateFlowTask) -> Result<()> {
+    for key in flow_task.flow_options.keys() {
+        match key.as_str() {
+            DEFER_ON_MISSING_SOURCE_KEY
+            | FLOW_EXPERIMENTAL_ENABLE_INCREMENTAL_READ_KEY
+            | FlowType::FLOW_TYPE_KEY => {}
+            unknown => {
+                return UnexpectedSnafu {
+                    err_msg: format!(
+                        "Unknown flow option '{unknown}', supported user options: {DEFER_ON_MISSING_SOURCE_KEY}, {FLOW_EXPERIMENTAL_ENABLE_INCREMENTAL_READ_KEY}"
+                    ),
+                }
+                .fail();
+            }
+        }
+    }
+
+    defer_on_missing_source(flow_task)?;
+    get_flow_type_from_options(flow_task)?;
+    Ok(())
+}
+
+fn user_runtime_flow_options(options: &HashMap<String, String>) -> HashMap<String, String> {
+    let mut options = options.clone();
+    options.remove(DEFER_ON_MISSING_SOURCE_KEY);
+    options
+}
+
+fn metadata_flow_options(options: &HashMap<String, String>) -> HashMap<String, String> {
+    options.clone()
+}
+
 /// The state of [CreateFlowProcedure].
 #[derive(Debug, Clone, Serialize, Deserialize, AsRefStr, PartialEq)]
 pub enum CreateFlowState {
@@ -388,6 +489,9 @@ pub enum FlowType {
     Streaming,
 }
 
+pub const FLOW_EXPERIMENTAL_ENABLE_INCREMENTAL_READ_KEY: &str =
+    "experimental_enable_incremental_read";
+
 impl FlowType {
     pub const BATCHING: &str = "batching";
     pub const STREAMING: &str = "streaming";
@@ -411,6 +515,8 @@ pub struct CreateFlowData {
     pub(crate) flow_id: Option<FlowId>,
     pub(crate) peers: Vec<Peer>,
     pub(crate) source_table_ids: Vec<TableId>,
+    #[serde(default)]
+    pub(crate) unresolved_source_table_names: Vec<TableName>,
     /// Use alias for backward compatibility with QueryContext serialized data
     #[serde(alias = "query_context")]
     pub(crate) flow_context: FlowQueryContext,
@@ -424,6 +530,16 @@ pub struct CreateFlowData {
     pub(crate) flow_type: Option<FlowType>,
 }
 
+impl CreateFlowData {
+    pub(crate) fn is_pending(&self) -> bool {
+        !self.unresolved_source_table_names.is_empty()
+    }
+
+    pub(crate) fn is_active(&self) -> bool {
+        !self.is_pending()
+    }
+}
+
 impl From<&CreateFlowData> for CreateRequest {
     fn from(value: &CreateFlowData) -> Self {
         let flow_id = value.flow_id.unwrap();
@@ -446,7 +562,7 @@ impl From<&CreateFlowData> for CreateRequest {
                 .map(|seconds| api::v1::EvalInterval { seconds }),
             comment: value.task.comment.clone(),
             sql: value.task.sql.clone(),
-            flow_options: value.task.flow_options.clone(),
+            flow_options: user_runtime_flow_options(&value.task.flow_options),
         };
 
         let flow_type = value.flow_type.unwrap_or_default().to_string();
@@ -466,9 +582,9 @@ impl From<&CreateFlowData> for (FlowInfoValue, Vec<(FlowPartitionId, FlowRouteVa
             eval_interval_secs: eval_interval,
             comment,
             sql,
-            flow_options: mut options,
             ..
         } = value.task.clone();
+        let mut options = metadata_flow_options(&value.task.flow_options);
 
         let flownode_ids = value
             .peers
@@ -484,7 +600,7 @@ impl From<&CreateFlowData> for (FlowInfoValue, Vec<(FlowPartitionId, FlowRouteVa
             .collect::<Vec<_>>();
 
         let flow_type = value.flow_type.unwrap_or_default().to_string();
-        options.insert("flow_type".to_string(), flow_type);
+        options.insert(FlowType::FLOW_TYPE_KEY.to_string(), flow_type);
 
         let mut create_time = chrono::Utc::now();
         if let Some(prev_flow_value) = value.prev_flow_info_value.as_ref()
@@ -495,6 +611,8 @@ impl From<&CreateFlowData> for (FlowInfoValue, Vec<(FlowPartitionId, FlowRouteVa
 
         let flow_info: FlowInfoValue = FlowInfoValue {
             source_table_ids: value.source_table_ids.clone(),
+            all_source_table_names: value.task.source_table_names.clone(),
+            unresolved_source_table_names: value.unresolved_source_table_names.clone(),
             sink_table_name,
             flownode_ids,
             catalog_name,
@@ -506,6 +624,11 @@ impl From<&CreateFlowData> for (FlowInfoValue, Vec<(FlowPartitionId, FlowRouteVa
             eval_interval_secs: eval_interval,
             comment,
             options,
+            status: if value.is_active() {
+                FlowStatus::Active
+            } else {
+                FlowStatus::PendingSources
+            },
             created_time: create_time,
             updated_time: chrono::Utc::now(),
         };

src/common/meta/src/ddl/create_flow/metadata.rs

@@ -12,10 +12,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use snafu::OptionExt;
-
 use crate::ddl::create_flow::CreateFlowProcedure;
-use crate::error::{self, Result};
+use crate::error::Result;
 use crate::key::table_name::TableNameKey;
 
 impl CreateFlowProcedure {
@@ -34,9 +32,8 @@ impl CreateFlowProcedure {
         Ok(())
     }
 
-    /// Ensures all source tables exist and collects source table ids
+    /// Collects source table ids and keeps track of missing tables.
     pub(crate) async fn collect_source_tables(&mut self) -> Result<()> {
-        // Ensures all source tables exist.
         let keys = self
             .data
             .task
@@ -52,22 +49,24 @@ impl CreateFlowProcedure {
             .batch_get(keys)
             .await?;
 
-        let source_table_ids = self
+        let mut resolved = Vec::with_capacity(self.data.task.source_table_names.len());
+        let mut unresolved = Vec::new();
+
+        for (name, table_id) in self
             .data
             .task
             .source_table_names
             .iter()
             .zip(source_table_ids)
-            .map(|(name, table_id)| {
-                Ok(table_id
-                    .with_context(|| error::TableNotFoundSnafu {
-                        table_name: name.to_string(),
-                    })?
-                    .table_id())
-            })
-            .collect::<Result<Vec<_>>>()?;
+        {
+            match table_id {
+                Some(table_id) => resolved.push(table_id.table_id()),
+                None => unresolved.push(name.clone()),
+            }
+        }
 
-        self.data.source_table_ids = source_table_ids;
+        self.data.source_table_ids = resolved;
+        self.data.unresolved_source_table_names = unresolved;
         Ok(())
     }
 }

src/common/meta/src/ddl/drop_flow/metadata.rs

@@ -43,7 +43,7 @@ impl DropFlowProcedure {
             .map(|(_, value)| value)
             .collect::<Vec<_>>();
         ensure!(
-            !flow_route_values.is_empty(),
+            flow_info_value.is_pending() || !flow_route_values.is_empty(),
             error::FlowRouteNotFoundSnafu {
                 flow_name: format_full_flow_name(catalog_name, flow_name),
             }

src/common/meta/src/ddl/tests/create_flow.rs

@@ -16,12 +16,18 @@ use std::assert_matches;
 use std::collections::HashMap;
 use std::sync::Arc;
 
+use api::v1::flow::CreateRequest;
 use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME};
+use common_procedure::Status;
 use common_procedure_test::execute_procedure_until_done;
 use table::table_name::TableName;
 
 use crate::ddl::DdlContext;
-use crate::ddl::create_flow::{CreateFlowData, CreateFlowProcedure, CreateFlowState, FlowType};
+use crate::ddl::create_flow::{
+    CreateFlowData, CreateFlowProcedure, CreateFlowState, DEFER_ON_MISSING_SOURCE_KEY,
+    FLOW_EXPERIMENTAL_ENABLE_INCREMENTAL_READ_KEY, FlowType, defer_on_missing_source,
+    validate_flow_options,
+};
 use crate::ddl::test_util::create_table::test_create_table_task;
 use crate::ddl::test_util::flownode_handler::NaiveFlownodeHandler;
 use crate::error;
@@ -63,6 +69,11 @@ pub(crate) fn test_create_flow_task(
     }
 }
 
+fn enable_defer_on_missing_source(task: &mut CreateFlowTask) {
+    task.flow_options
+        .insert(DEFER_ON_MISSING_SOURCE_KEY.to_string(), "true".to_string());
+}
+
 #[tokio::test]
 async fn test_create_flow_source_table_not_found() {
     let source_table_names = vec![TableName::new(
@@ -78,7 +89,277 @@ async fn test_create_flow_source_table_not_found() {
     let query_ctx = test_query_context();
     let mut procedure = CreateFlowProcedure::new(task, query_ctx, ddl_context);
     let err = procedure.on_prepare().await.unwrap_err();
-    assert_matches!(err, error::Error::TableNotFound { .. });
+    assert_matches!(err, error::Error::Unsupported { .. });
+    assert!(
+        err.to_string()
+            .contains("requires WITH ('defer_on_missing_source'='true')")
+    );
+}
+
+#[tokio::test]
+async fn test_create_pending_flow_source_table_not_found_with_defer() {
+    let source_table_names = vec![TableName::new(
+        DEFAULT_CATALOG_NAME,
+        DEFAULT_SCHEMA_NAME,
+        "my_table",
+    )];
+    let sink_table_name =
+        TableName::new(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, "my_sink_table");
+    let mut task = test_create_flow_task("my_flow", source_table_names, sink_table_name, false);
+    enable_defer_on_missing_source(&mut task);
+    let node_manager = Arc::new(MockFlownodeManager::new(NaiveFlownodeHandler));
+    let ddl_context = new_ddl_context(node_manager);
+    let query_ctx = test_query_context();
+    let mut procedure = CreateFlowProcedure::new(task, query_ctx, ddl_context.clone());
+    let status = procedure.on_prepare().await.unwrap();
+    assert_matches!(status, Status::Executing { persist: true, .. });
+    assert_eq!(procedure.data.unresolved_source_table_names.len(), 1);
+    assert_eq!(procedure.data.source_table_ids, Vec::<u32>::new());
+
+    let output = execute_procedure_until_done(&mut procedure).await.unwrap();
+    let flow_id = *output.downcast_ref::<FlowId>().unwrap();
+    let flow_info = ddl_context
+        .flow_metadata_manager
+        .flow_info_manager()
+        .get(flow_id)
+        .await
+        .unwrap()
+        .unwrap();
+    assert_eq!(flow_info.source_table_ids(), Vec::<u32>::new());
+    assert_eq!(
+        flow_info
+            .options()
+            .get(DEFER_ON_MISSING_SOURCE_KEY)
+            .map(String::as_str),
+        Some("true")
+    );
+}
+
+#[tokio::test]
+async fn test_create_pending_flow_source_table_not_found_with_defer_false() {
+    let source_table_names = vec![TableName::new(
+        DEFAULT_CATALOG_NAME,
+        DEFAULT_SCHEMA_NAME,
+        "my_table",
+    )];
+    let sink_table_name =
+        TableName::new(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, "my_sink_table");
+    let mut task = test_create_flow_task("my_flow", source_table_names, sink_table_name, false);
+    task.flow_options
+        .insert(DEFER_ON_MISSING_SOURCE_KEY.to_string(), "false".to_string());
+    let node_manager = Arc::new(MockFlownodeManager::new(NaiveFlownodeHandler));
+    let ddl_context = new_ddl_context(node_manager);
+    let query_ctx = test_query_context();
+    let mut procedure = CreateFlowProcedure::new(task, query_ctx, ddl_context);
+    let err = procedure.on_prepare().await.unwrap_err();
+    assert_matches!(err, error::Error::Unsupported { .. });
+    assert!(
+        err.to_string()
+            .contains("requires WITH ('defer_on_missing_source'='true')")
+    );
+}
+
+#[tokio::test]
+async fn test_create_pending_flow_records_partial_source_resolution() {
+    let existing_source = TableName::new(
+        DEFAULT_CATALOG_NAME,
+        DEFAULT_SCHEMA_NAME,
+        "partial_existing_source_table",
+    );
+    let missing_source = TableName::new(
+        DEFAULT_CATALOG_NAME,
+        DEFAULT_SCHEMA_NAME,
+        "partial_missing_source_table",
+    );
+    let sink_table_name = TableName::new(
+        DEFAULT_CATALOG_NAME,
+        DEFAULT_SCHEMA_NAME,
+        "partial_pending_sink_table",
+    );
+    let node_manager = Arc::new(MockFlownodeManager::new(NaiveFlownodeHandler));
+    let ddl_context = new_ddl_context(node_manager);
+
+    let existing_table_id = 3026;
+    let create_table_task =
+        test_create_table_task("partial_existing_source_table", existing_table_id);
+    ddl_context
+        .table_metadata_manager
+        .create_table_metadata(
+            create_table_task.table_info.clone(),
+            TableRouteValue::physical(vec![]),
+            HashMap::new(),
+        )
+        .await
+        .unwrap();
+
+    let mut task = test_create_flow_task(
+        "partial_pending_flow",
+        vec![existing_source.clone(), missing_source.clone()],
+        sink_table_name,
+        false,
+    );
+    enable_defer_on_missing_source(&mut task);
+    let query_ctx = test_query_context();
+    let mut procedure = CreateFlowProcedure::new(task, query_ctx, ddl_context.clone());
+    let status = procedure.on_prepare().await.unwrap();
+    assert_matches!(status, Status::Executing { persist: true, .. });
+    assert_eq!(procedure.data.source_table_ids, vec![existing_table_id]);
+    assert_eq!(
+        procedure.data.unresolved_source_table_names,
+        vec![missing_source.clone()]
+    );
+
+    let output = execute_procedure_until_done(&mut procedure).await.unwrap();
+    let flow_id = *output.downcast_ref::<FlowId>().unwrap();
+    let flow_info = ddl_context
+        .flow_metadata_manager
+        .flow_info_manager()
+        .get(flow_id)
+        .await
+        .unwrap()
+        .unwrap();
+
+    assert!(flow_info.is_pending());
+    assert_eq!(flow_info.source_table_ids(), &[existing_table_id]);
+    let expected_all_sources = vec![existing_source, missing_source.clone()];
+    assert_eq!(
+        flow_info.all_source_table_names(),
+        expected_all_sources.as_slice()
+    );
+    assert_eq!(flow_info.unresolved_source_table_names(), &[missing_source]);
+    assert!(flow_info.flownode_ids().is_empty());
+}
+
+#[test]
+fn test_defer_on_missing_source_defaults_false() {
+    let task = test_create_flow_task(
+        "my_flow",
+        vec![],
+        TableName::new(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, "my_sink_table"),
+        false,
+    );
+
+    assert!(!defer_on_missing_source(&task).unwrap());
+}
+
+#[test]
+fn test_defer_on_missing_source_true() {
+    let mut task = test_create_flow_task(
+        "my_flow",
+        vec![],
+        TableName::new(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, "my_sink_table"),
+        false,
+    );
+    task.flow_options
+        .insert(DEFER_ON_MISSING_SOURCE_KEY.to_string(), "true".to_string());
+
+    assert!(defer_on_missing_source(&task).unwrap());
+}
+
+#[test]
+fn test_defer_on_missing_source_invalid_value() {
+    let mut task = test_create_flow_task(
+        "my_flow",
+        vec![],
+        TableName::new(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, "my_sink_table"),
+        false,
+    );
+    task.flow_options.insert(
+        DEFER_ON_MISSING_SOURCE_KEY.to_string(),
+        "invalid".to_string(),
+    );
+
+    let err = defer_on_missing_source(&task).unwrap_err();
+    assert!(
+        err.to_string()
+            .contains("Invalid flow option 'defer_on_missing_source': invalid")
+    );
+}
+
+#[test]
+fn test_validate_flow_options_allows_incremental_read_option() {
+    let mut task = test_create_flow_task(
+        "my_flow",
+        vec![],
+        TableName::new(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, "my_sink_table"),
+        false,
+    );
+    task.flow_options.insert(
+        FLOW_EXPERIMENTAL_ENABLE_INCREMENTAL_READ_KEY.to_string(),
+        "true".to_string(),
+    );
+
+    validate_flow_options(&task).unwrap();
+}
+
+#[tokio::test]
+async fn test_create_flow_rejects_unknown_option_in_meta_task() {
+    let mut task = test_create_flow_task(
+        "my_flow",
+        vec![],
+        TableName::new(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, "my_sink_table"),
+        false,
+    );
+    task.flow_options
+        .insert("unknown_option".to_string(), "value".to_string());
+    let node_manager = Arc::new(MockFlownodeManager::new(NaiveFlownodeHandler));
+    let ddl_context = new_ddl_context(node_manager);
+    let query_ctx = test_query_context();
+    let mut procedure = CreateFlowProcedure::new(task, query_ctx, ddl_context);
+
+    let err = procedure.on_prepare().await.unwrap_err();
+    assert_matches!(err, error::Error::Unexpected { .. });
+    assert!(
+        err.to_string()
+            .contains("Unknown flow option 'unknown_option'")
+    );
+}
+
+#[test]
+fn test_create_request_strips_defer_on_missing_source_runtime_option() {
+    let mut task = test_create_flow_task(
+        "my_flow",
+        vec![],
+        TableName::new(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, "my_sink_table"),
+        false,
+    );
+    enable_defer_on_missing_source(&mut task);
+
+    let data = CreateFlowData {
+        state: CreateFlowState::CreateFlows,
+        task,
+        flow_id: Some(1024),
+        peers: vec![],
+        source_table_ids: vec![],
+        unresolved_source_table_names: vec![],
+        flow_context: FlowQueryContext {
+            catalog: DEFAULT_CATALOG_NAME.to_string(),
+            schema: DEFAULT_SCHEMA_NAME.to_string(),
+            timezone: "UTC".to_string(),
+            extensions: HashMap::new(),
+            channel: 0,
+            snapshot_seqs: HashMap::new(),
+            sst_min_sequences: HashMap::new(),
+        },
+        prev_flow_info_value: None,
+        did_replace: false,
+        flow_type: Some(FlowType::Batching),
+    };
+
+    let request: CreateRequest = (&data).into();
+
+    assert!(
+        !request
+            .flow_options
+            .contains_key(DEFER_ON_MISSING_SOURCE_KEY)
+    );
+    assert_eq!(
+        request
+            .flow_options
+            .get(FlowType::FLOW_TYPE_KEY)
+            .map(String::as_str),
+        Some(FlowType::BATCHING)
+    );
 }
 
 pub(crate) async fn create_test_flow(
@@ -101,6 +382,27 @@ pub(crate) async fn create_test_flow(
     *flow_id
 }
 
+pub(crate) async fn create_test_pending_flow(
+    ddl_context: &DdlContext,
+    flow_name: &str,
+    source_table_names: Vec<TableName>,
+    sink_table_name: TableName,
+) -> FlowId {
+    let mut task = test_create_flow_task(
+        flow_name,
+        source_table_names.clone(),
+        sink_table_name.clone(),
+        false,
+    );
+    enable_defer_on_missing_source(&mut task);
+    let query_ctx = test_query_context();
+    let mut procedure = CreateFlowProcedure::new(task, query_ctx, ddl_context.clone());
+    let output = execute_procedure_until_done(&mut procedure).await.unwrap();
+    let flow_id = output.downcast_ref::<FlowId>().unwrap();
+
+    *flow_id
+}
+
 #[tokio::test]
 async fn test_create_flow() {
     let table_id = 1024;
@@ -154,6 +456,201 @@ async fn test_create_flow() {
     assert_matches!(err, error::Error::FlowAlreadyExists { .. });
 }
 
+#[tokio::test]
+async fn test_replace_pending_flow_with_active_flow_is_unsupported() {
+    let source_table_name = TableName::new(
+        DEFAULT_CATALOG_NAME,
+        DEFAULT_SCHEMA_NAME,
+        "replace_pending_source_table",
+    );
+    let sink_table_name = TableName::new(
+        DEFAULT_CATALOG_NAME,
+        DEFAULT_SCHEMA_NAME,
+        "replace_pending_sink_table",
+    );
+    let node_manager = Arc::new(MockFlownodeManager::new(NaiveFlownodeHandler));
+    let ddl_context = new_ddl_context(node_manager);
+
+    let pending_flow_id = create_test_pending_flow(
+        &ddl_context,
+        "replace_pending_flow",
+        vec![source_table_name.clone()],
+        sink_table_name.clone(),
+    )
+    .await;
+
+    let pending_flow = ddl_context
+        .flow_metadata_manager
+        .flow_info_manager()
+        .get(pending_flow_id)
+        .await
+        .unwrap()
+        .unwrap();
+    assert!(pending_flow.is_pending());
+    assert!(pending_flow.flownode_ids().is_empty());
+
+    let create_table_task = test_create_table_task("replace_pending_source_table", 1026);
+    ddl_context
+        .table_metadata_manager
+        .create_table_metadata(
+            create_table_task.table_info.clone(),
+            TableRouteValue::physical(vec![]),
+            HashMap::new(),
+        )
+        .await
+        .unwrap();
+
+    let mut replace_task = test_create_flow_task(
+        "replace_pending_flow",
+        vec![source_table_name],
+        sink_table_name,
+        false,
+    );
+    replace_task.or_replace = true;
+    let query_ctx = test_query_context();
+    let mut procedure = CreateFlowProcedure::new(replace_task, query_ctx, ddl_context.clone());
+    let err = procedure.on_prepare().await.unwrap_err();
+    assert_matches!(err, error::Error::Unsupported { .. });
+    assert!(
+        err.to_string()
+            .contains("Replacing between pending and active flow states")
+    );
+}
+
+#[tokio::test]
+async fn test_replace_active_flow_with_pending_flow_is_unsupported() {
+    let existing_source_table = TableName::new(
+        DEFAULT_CATALOG_NAME,
+        DEFAULT_SCHEMA_NAME,
+        "replace_active_source_table",
+    );
+    let missing_source_table = TableName::new(
+        DEFAULT_CATALOG_NAME,
+        DEFAULT_SCHEMA_NAME,
+        "replace_missing_source_table",
+    );
+    let sink_table_name = TableName::new(
+        DEFAULT_CATALOG_NAME,
+        DEFAULT_SCHEMA_NAME,
+        "replace_active_sink_table",
+    );
+
+    let node_manager = Arc::new(MockFlownodeManager::new(NaiveFlownodeHandler));
+    let ddl_context = new_ddl_context(node_manager);
+
+    let create_table_task = test_create_table_task("replace_active_source_table", 2026);
+    ddl_context
+        .table_metadata_manager
+        .create_table_metadata(
+            create_table_task.table_info.clone(),
+            TableRouteValue::physical(vec![]),
+            HashMap::new(),
+        )
+        .await
+        .unwrap();
+
+    let _flow_id = create_test_flow(
+        &ddl_context,
+        "replace_active_flow_to_pending",
+        vec![existing_source_table],
+        sink_table_name.clone(),
+    )
+    .await;
+
+    let mut replace_task = test_create_flow_task(
+        "replace_active_flow_to_pending",
+        vec![missing_source_table],
+        sink_table_name,
+        false,
+    );
+    enable_defer_on_missing_source(&mut replace_task);
+    replace_task.or_replace = true;
+    let query_ctx = test_query_context();
+    let mut procedure = CreateFlowProcedure::new(replace_task, query_ctx, ddl_context.clone());
+    let err = procedure.on_prepare().await.unwrap_err();
+    assert_matches!(err, error::Error::Unsupported { .. });
+    assert!(
+        err.to_string()
+            .contains("Replacing between pending and active flow states")
+    );
+}
+
+#[tokio::test]
+async fn test_replace_pending_flow_with_pending_flow_updates_metadata() {
+    let first_missing_source = TableName::new(
+        DEFAULT_CATALOG_NAME,
+        DEFAULT_SCHEMA_NAME,
+        "replace_pending_first_missing_source",
+    );
+    let second_missing_source = TableName::new(
+        DEFAULT_CATALOG_NAME,
+        DEFAULT_SCHEMA_NAME,
+        "replace_pending_second_missing_source",
+    );
+    let sink_table_name = TableName::new(
+        DEFAULT_CATALOG_NAME,
+        DEFAULT_SCHEMA_NAME,
+        "replace_pending_to_pending_sink_table",
+    );
+    let node_manager = Arc::new(MockFlownodeManager::new(NaiveFlownodeHandler));
+    let ddl_context = new_ddl_context(node_manager);
+
+    let original_flow_id = create_test_pending_flow(
+        &ddl_context,
+        "replace_pending_to_pending_flow",
+        vec![first_missing_source.clone()],
+        sink_table_name.clone(),
+    )
+    .await;
+
+    let original_flow = ddl_context
+        .flow_metadata_manager
+        .flow_info_manager()
+        .get(original_flow_id)
+        .await
+        .unwrap()
+        .unwrap();
+    assert!(original_flow.is_pending());
+    assert_eq!(
+        original_flow.unresolved_source_table_names(),
+        &[first_missing_source]
+    );
+    assert!(original_flow.flownode_ids().is_empty());
+
+    let mut replace_task = test_create_flow_task(
+        "replace_pending_to_pending_flow",
+        vec![second_missing_source.clone()],
+        sink_table_name,
+        false,
+    );
+    enable_defer_on_missing_source(&mut replace_task);
+    replace_task.or_replace = true;
+    let query_ctx = test_query_context();
+    let mut procedure = CreateFlowProcedure::new(replace_task, query_ctx, ddl_context.clone());
+    let output = execute_procedure_until_done(&mut procedure).await.unwrap();
+    let replaced_flow_id = *output.downcast_ref::<FlowId>().unwrap();
+    assert_eq!(replaced_flow_id, original_flow_id);
+
+    let replaced_flow = ddl_context
+        .flow_metadata_manager
+        .flow_info_manager()
+        .get(replaced_flow_id)
+        .await
+        .unwrap()
+        .unwrap();
+    assert!(replaced_flow.is_pending());
+    assert_eq!(replaced_flow.source_table_ids(), Vec::<u32>::new());
+    assert_eq!(
+        replaced_flow.unresolved_source_table_names(),
+        std::slice::from_ref(&second_missing_source)
+    );
+    assert_eq!(
+        replaced_flow.all_source_table_names(),
+        &[second_missing_source]
+    );
+    assert!(replaced_flow.flownode_ids().is_empty());
+}
+
 #[tokio::test]
 async fn test_create_flow_same_source_and_sink_table() {
     let table_id = 1024;
@@ -228,6 +725,7 @@ fn test_create_flow_data_serialization_backward_compatibility() {
         "flow_id": null,
         "peers": [],
         "source_table_ids": [],
+        "unresolved_source_table_names": [],
         "query_context": {
             "current_catalog": "old_catalog",
             "current_schema": "old_schema",
@@ -265,6 +763,7 @@ fn test_create_flow_data_new_format_serialization() {
         flow_id: None,
         peers: vec![],
         source_table_ids: vec![],
+        unresolved_source_table_names: vec![],
         flow_context,
         prev_flow_info_value: None,
         did_replace: false,
@@ -327,6 +826,7 @@ fn test_flow_info_conversion_with_flow_context() {
         flow_id: Some(123),
         peers: vec![],
         source_table_ids: vec![456, 789],
+        unresolved_source_table_names: vec![],
         flow_context,
         prev_flow_info_value: None,
         did_replace: false,

src/common/meta/src/ddl/tests/drop_flow.rs

@@ -23,7 +23,7 @@ use table::table_name::TableName;
 use crate::ddl::drop_flow::DropFlowProcedure;
 use crate::ddl::test_util::create_table::test_create_table_task;
 use crate::ddl::test_util::flownode_handler::NaiveFlownodeHandler;
-use crate::ddl::tests::create_flow::create_test_flow;
+use crate::ddl::tests::create_flow::{create_test_flow, create_test_pending_flow};
 use crate::error;
 use crate::key::table_route::TableRouteValue;
 use crate::rpc::ddl::DropFlowTask;
@@ -91,3 +91,45 @@ async fn test_drop_flow() {
     let err = procedure.on_prepare().await.unwrap_err();
     assert_matches!(err, error::Error::FlowNotFound { .. });
 }
+
+#[tokio::test]
+async fn test_drop_pending_flow_without_routes() {
+    let source_table_name = TableName::new(
+        DEFAULT_CATALOG_NAME,
+        DEFAULT_SCHEMA_NAME,
+        "drop_pending_missing_source_table",
+    );
+    let sink_table_name = TableName::new(
+        DEFAULT_CATALOG_NAME,
+        DEFAULT_SCHEMA_NAME,
+        "drop_pending_sink_table",
+    );
+    let node_manager = Arc::new(MockFlownodeManager::new(NaiveFlownodeHandler));
+    let ddl_context = new_ddl_context(node_manager);
+
+    let flow_id = create_test_pending_flow(
+        &ddl_context,
+        "drop_pending_flow",
+        vec![source_table_name],
+        sink_table_name,
+    )
+    .await;
+    let flow_info = ddl_context
+        .flow_metadata_manager
+        .flow_info_manager()
+        .get(flow_id)
+        .await
+        .unwrap()
+        .unwrap();
+    assert!(flow_info.is_pending());
+    assert!(flow_info.flownode_ids().is_empty());
+
+    let task = test_drop_flow_task("drop_pending_flow", flow_id, false);
+    let mut procedure = DropFlowProcedure::new(task, ddl_context.clone());
+    execute_procedure_until_done(&mut procedure).await;
+
+    let task = test_drop_flow_task("drop_pending_flow", flow_id, false);
+    let mut procedure = DropFlowProcedure::new(task, ddl_context);
+    let err = procedure.on_prepare().await.unwrap_err();
+    assert_matches!(err, error::Error::FlowNotFound { .. });
+}

src/common/meta/src/instruction.rs

@@ -18,7 +18,7 @@ use std::time::Duration;
 
 use serde::{Deserialize, Deserializer, Serialize, Serializer};
 use store_api::region_engine::SyncRegionFromRequest;
-use store_api::region_request::RegionFlushReason;
+use store_api::region_request::{RegionFlushReason, RegionRequirements};
 use store_api::storage::{FileRefsManifest, GcReport, RegionId, RegionNumber};
 use strum::Display;
 use table::metadata::TableId;
@@ -179,12 +179,24 @@ impl Display for OpenRegion {
     fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
         write!(
             f,
-            "OpenRegion(region_ident={}, region_storage_path={})",
-            self.region_ident, self.region_storage_path
+            "OpenRegion(region_ident={}, region_storage_path={}, reason={:?})",
+            self.region_ident, self.region_storage_path, self.reason
         )
     }
 }
 
+/// The reason why an open region instruction is triggered.
+#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
+pub enum OpenRegionReason {
+    /// Open triggered before region migration.
+    RegionMigration,
+    /// Open triggered by region failover.
+    RegionFailover,
+    /// Open triggered when adding a follower region.
+    #[cfg(feature = "enterprise")]
+    RegionFollower,
+}
+
 #[serde_with::serde_as]
 #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
 pub struct OpenRegion {
@@ -196,6 +208,10 @@ pub struct OpenRegion {
     pub region_wal_options: HashMap<RegionNumber, String>,
     #[serde(default)]
     pub skip_wal_replay: bool,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub reason: Option<OpenRegionReason>,
+    #[serde(default)]
+    pub requirements: RegionRequirements,
 }
 
 impl OpenRegion {
@@ -205,6 +221,8 @@ impl OpenRegion {
         region_options: HashMap<String, String>,
         region_wal_options: HashMap<RegionNumber, String>,
         skip_wal_replay: bool,
+        reason: Option<OpenRegionReason>,
+        requirements: RegionRequirements,
     ) -> Self {
         Self {
             region_ident,
@@ -212,6 +230,8 @@ impl OpenRegion {
             region_options,
             region_wal_options,
             skip_wal_replay,
+            reason,
+            requirements,
         }
     }
 }
@@ -1126,11 +1146,13 @@ mod tests {
             HashMap::new(),
             HashMap::new(),
             false,
+            None,
+            RegionRequirements::empty(),
         )]);
 
         let serialized = serde_json::to_string(&open_region).unwrap();
         assert_eq!(
-            r#"{"OpenRegions":[{"region_ident":{"datanode_id":2,"table_id":1024,"region_number":1,"engine":"mito2"},"region_storage_path":"test/foo","region_options":{},"region_wal_options":{},"skip_wal_replay":false}]}"#,
+            r#"{"OpenRegions":[{"region_ident":{"datanode_id":2,"table_id":1024,"region_number":1,"engine":"mito2"},"region_storage_path":"test/foo","region_options":{},"region_wal_options":{},"skip_wal_replay":false,"requirements":{"object_storage":false}}]}"#,
             serialized
         );
 
@@ -1213,6 +1235,8 @@ mod tests {
             HashMap::new(),
             HashMap::new(),
             false,
+            None,
+            RegionRequirements::empty(),
         )]);
         assert_eq!(open_region_instruction, open_region);
 
@@ -1368,10 +1392,41 @@ mod tests {
             region_options,
             region_wal_options: HashMap::new(),
             skip_wal_replay: false,
+            reason: None,
+            requirements: RegionRequirements::empty(),
         };
         assert_eq!(expected, deserialized);
     }
 
+    #[test]
+    fn test_serialize_open_region_with_reason_and_requirements() {
+        let open_region = OpenRegion::new(
+            RegionIdent {
+                datanode_id: 2,
+                table_id: 1024,
+                region_number: 1,
+                engine: "mito2".to_string(),
+            },
+            "test/foo",
+            HashMap::new(),
+            HashMap::new(),
+            false,
+            Some(OpenRegionReason::RegionMigration),
+            RegionRequirements::object_storage(),
+        );
+
+        let serialized = serde_json::to_string(&open_region).unwrap();
+        assert!(serialized.contains(r#""reason":"RegionMigration""#));
+        assert!(serialized.contains(r#""object_storage":true"#));
+
+        let deserialized: OpenRegion = serde_json::from_str(&serialized).unwrap();
+        assert_eq!(Some(OpenRegionReason::RegionMigration), deserialized.reason);
+        assert_eq!(
+            RegionRequirements::object_storage(),
+            deserialized.requirements
+        );
+    }
+
     #[test]
     fn test_flush_regions_creation() {
         let region_id = RegionId::new(1024, 1);

src/common/meta/src/key/flow.rs

@@ -459,6 +459,7 @@ mod tests {
 
     use super::*;
     use crate::FlownodeId;
+    use crate::key::flow::flow_info::FlowStatus;
     use crate::key::flow::table_flow::TableFlowKey;
     use crate::key::node_address::{NodeAddressKey, NodeAddressValue};
     use crate::key::{FlowPartitionId, MetadataValue};
@@ -522,6 +523,8 @@ mod tests {
             query_context: None,
             flow_name: flow_name.to_string(),
             source_table_ids,
+            all_source_table_names: vec![],
+            unresolved_source_table_names: vec![],
             sink_table_name,
             flownode_ids,
             raw_sql: "raw".to_string(),
@@ -529,6 +532,7 @@ mod tests {
             eval_interval_secs: None,
             comment: "hi".to_string(),
             options: Default::default(),
+            status: FlowStatus::Active,
             created_time: chrono::Utc::now(),
             updated_time: chrono::Utc::now(),
         }
@@ -774,6 +778,8 @@ mod tests {
             query_context: None,
             flow_name: "flow".to_string(),
             source_table_ids: vec![1024, 1025, 1026],
+            all_source_table_names: vec![],
+            unresolved_source_table_names: vec![],
             sink_table_name: another_sink_table_name,
             flownode_ids: [(0, 1u64)].into(),
             raw_sql: "raw".to_string(),
@@ -781,6 +787,7 @@ mod tests {
             eval_interval_secs: None,
             comment: "hi".to_string(),
             options: Default::default(),
+            status: FlowStatus::Active,
             created_time: chrono::Utc::now(),
             updated_time: chrono::Utc::now(),
         };
@@ -1151,6 +1158,8 @@ mod tests {
             query_context: None,
             flow_name: "flow".to_string(),
             source_table_ids: vec![1024, 1025, 1026],
+            all_source_table_names: vec![],
+            unresolved_source_table_names: vec![],
             sink_table_name: another_sink_table_name,
             flownode_ids: [(0, 1u64)].into(),
             raw_sql: "raw".to_string(),
@@ -1158,6 +1167,7 @@ mod tests {
             eval_interval_secs: None,
             comment: "hi".to_string(),
             options: Default::default(),
+            status: FlowStatus::Active,
             created_time: chrono::Utc::now(),
             updated_time: chrono::Utc::now(),
         };

src/common/meta/src/key/flow/flow_info.rs

@@ -16,6 +16,8 @@ use std::collections::{BTreeMap, HashMap};
 use std::sync::Arc;
 
 use chrono::{DateTime, Utc};
+use futures::TryStreamExt;
+use futures::stream::BoxStream;
 use lazy_static::lazy_static;
 use regex::Regex;
 use serde::{Deserialize, Serialize};
@@ -27,12 +29,27 @@ use crate::FlownodeId;
 use crate::error::{self, Result};
 use crate::key::flow::FlowScoped;
 use crate::key::txn_helper::TxnOpGetResponseSet;
-use crate::key::{DeserializedValueWithBytes, FlowId, FlowPartitionId, MetadataKey, MetadataValue};
+use crate::key::{
+    BytesAdapter, DeserializedValueWithBytes, FlowId, FlowPartitionId, MetadataKey, MetadataValue,
+};
 use crate::kv_backend::KvBackendRef;
 use crate::kv_backend::txn::{Compare, CompareOp, Txn, TxnOp};
+use crate::range_stream::{DEFAULT_PAGE_SIZE, PaginationStream};
+use crate::rpc::KeyValue;
+use crate::rpc::store::RangeRequest;
 
 pub const FLOW_INFO_KEY_PREFIX: &str = "info";
 
+/// The lifecycle status of a flow stored in metadata.
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, Default)]
+pub enum FlowStatus {
+    /// The flow metadata exists, but at least one source table did not exist at create time.
+    PendingSources,
+    /// The flow has resolved source tables and can be scheduled on flownodes.
+    #[default]
+    Active,
+}
+
 lazy_static! {
     static ref FLOW_INFO_KEY_PATTERN: Regex =
         Regex::new(&format!("^{FLOW_INFO_KEY_PREFIX}/([0-9]+)$")).unwrap();
@@ -114,7 +131,12 @@ impl<'a> MetadataKey<'a, FlowInfoKeyInner> for FlowInfoKeyInner {
 #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
 pub struct FlowInfoValue {
     /// The source tables used by the flow.
+    #[serde(default)]
     pub source_table_ids: Vec<TableId>,
+    #[serde(default)]
+    pub all_source_table_names: Vec<TableName>,
+    #[serde(default)]
+    pub unresolved_source_table_names: Vec<TableName>,
     /// The sink table used by the flow.
     pub sink_table_name: TableName,
     /// Which flow nodes this flow is running on.
@@ -145,6 +167,8 @@ pub struct FlowInfoValue {
     pub comment: String,
     /// The options.
     pub options: HashMap<String, String>,
+    #[serde(default)]
+    pub status: FlowStatus,
     /// The created time
     #[serde(default)]
     pub created_time: DateTime<Utc>,
@@ -154,6 +178,14 @@ pub struct FlowInfoValue {
 }
 
 impl FlowInfoValue {
+    pub fn is_pending(&self) -> bool {
+        self.status == FlowStatus::PendingSources
+    }
+
+    pub fn is_active(&self) -> bool {
+        self.status == FlowStatus::Active
+    }
+
     /// Returns the `flownode_id`.
     pub fn flownode_ids(&self) -> &BTreeMap<FlowPartitionId, FlownodeId> {
         &self.flownode_ids
@@ -173,6 +205,14 @@ impl FlowInfoValue {
         &self.source_table_ids
     }
 
+    pub fn all_source_table_names(&self) -> &[TableName] {
+        &self.all_source_table_names
+    }
+
+    pub fn unresolved_source_table_names(&self) -> &[TableName] {
+        &self.unresolved_source_table_names
+    }
+
     pub fn catalog_name(&self) -> &String {
         &self.catalog_name
     }
@@ -209,6 +249,10 @@ impl FlowInfoValue {
         &self.options
     }
 
+    pub fn status(&self) -> &FlowStatus {
+        &self.status
+    }
+
     pub fn created_time(&self) -> &DateTime<Utc> {
         &self.created_time
     }
@@ -225,6 +269,12 @@ pub struct FlowInfoManager {
     kv_backend: KvBackendRef,
 }
 
+pub fn flow_info_decoder(kv: KeyValue) -> Result<(FlowInfoKey, FlowInfoValue)> {
+    let key = FlowInfoKey::from_bytes(&kv.key)?;
+    let value = FlowInfoValue::try_from_raw_value(&kv.value)?;
+    Ok((key, value))
+}
+
 impl FlowInfoManager {
     /// Returns a new [FlowInfoManager].
     pub fn new(kv_backend: KvBackendRef) -> Self {
@@ -254,6 +304,23 @@ impl FlowInfoManager {
             .transpose()
     }
 
+    pub fn flow_infos(&self) -> BoxStream<'static, Result<(FlowId, FlowInfoValue)>> {
+        let start_key = FlowScoped::new(BytesAdapter::from(
+            format!("{FLOW_INFO_KEY_PREFIX}/").into_bytes(),
+        ))
+        .to_bytes();
+        let req = RangeRequest::new().with_prefix(start_key);
+        let stream = PaginationStream::new(
+            self.kv_backend.clone(),
+            req,
+            DEFAULT_PAGE_SIZE,
+            flow_info_decoder,
+        )
+        .into_stream();
+
+        Box::pin(stream.map_ok(|(key, value)| (key.flow_id(), value)))
+    }
+
     /// Builds a create flow transaction.
     /// It is expected that the `__flow/info/{flow_id}` wasn't occupied.
     /// Otherwise, the transaction will retrieve existing value.

src/common/meta/src/key/topic_name.rs

@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use std::collections::HashMap;
 use std::fmt::{self, Display};
 
 use serde::{Deserialize, Serialize};
@@ -27,7 +28,7 @@ use crate::key::{
 use crate::kv_backend::KvBackendRef;
 use crate::kv_backend::txn::{Txn, TxnOp};
 use crate::rpc::KeyValue;
-use crate::rpc::store::{BatchPutRequest, RangeRequest};
+use crate::rpc::store::{BatchGetRequest, BatchPutRequest, RangeRequest};
 
 #[derive(Debug, Clone, PartialEq)]
 pub struct TopicNameKey<'a> {
@@ -205,6 +206,25 @@ impl TopicNameManager {
             .transpose()
     }
 
+    /// Batch get values for specific topics.
+    pub async fn batch_get(
+        &self,
+        topics: Vec<TopicNameKey<'_>>,
+    ) -> Result<HashMap<String, TopicNameValue>> {
+        let raw_keys = topics.iter().map(|key| key.to_bytes()).collect::<Vec<_>>();
+        let req = BatchGetRequest { keys: raw_keys };
+        let resp = self.kv_backend.batch_get(req).await?;
+
+        resp.kvs
+            .into_iter()
+            .map(|kv| {
+                let key = TopicNameKey::from_bytes(&kv.key)?;
+                let value = TopicNameValue::try_from_raw_value(&kv.value)?;
+                Ok((key.topic.to_string(), value))
+            })
+            .collect::<Result<HashMap<_, _>>>()
+    }
+
     /// Update the topic name key and value in the kv backend.
     pub async fn update(
         &self,
@@ -295,5 +315,17 @@ mod tests {
             let err = manager.update(topic, 3, Some(value)).await.unwrap_err();
             assert_matches!(err, error::Error::Unexpected { .. });
         }
+
+        let batch_topics = topics
+            .iter()
+            .take(2)
+            .map(|topic| TopicNameKey::new(topic))
+            .chain(std::iter::once(TopicNameKey::new("missing-topic")))
+            .collect::<Vec<_>>();
+        let values = manager.batch_get(batch_topics).await.unwrap();
+        assert_eq!(values.len(), 2);
+        for topic in topics.iter().take(2) {
+            assert_eq!(values.get(topic).unwrap().pruned_entry_id, 1);
+        }
     }
 }

src/common/meta/src/key/topic_region.rs

@@ -126,6 +126,34 @@ impl ReplayCheckpoint {
             metadata_entry_id,
         }
     }
+
+    /// Merges the checkpoint with the topic pruned entry id.
+    pub fn merge_with_topic_pruned_entry_id(
+        checkpoint: Option<Self>,
+        pruned_entry_id: Option<u64>,
+        is_metric_engine: bool,
+    ) -> Option<Self> {
+        match (checkpoint, pruned_entry_id) {
+            (Some(checkpoint), Some(pruned_entry_id)) => Some(Self {
+                entry_id: checkpoint.entry_id.max(pruned_entry_id),
+                metadata_entry_id: if is_metric_engine {
+                    Some(
+                        checkpoint
+                            .metadata_entry_id
+                            .unwrap_or_default()
+                            .max(pruned_entry_id),
+                    )
+                } else {
+                    checkpoint.metadata_entry_id
+                },
+            }),
+            (None, Some(pruned_entry_id)) => Some(Self {
+                entry_id: pruned_entry_id,
+                metadata_entry_id: is_metric_engine.then_some(pruned_entry_id),
+            }),
+            (checkpoint, None) => checkpoint,
+        }
+    }
 }
 
 impl TopicRegionValue {
@@ -369,6 +397,58 @@ mod tests {
     use super::*;
     use crate::kv_backend::memory::MemoryKvBackend;
 
+    #[test]
+    fn test_merge_checkpoint_with_topic_pruned_entry_id_missing_pruned() {
+        let checkpoint = Some(ReplayCheckpoint::new(10, None));
+
+        assert_eq!(
+            ReplayCheckpoint::merge_with_topic_pruned_entry_id(checkpoint, None, true),
+            checkpoint
+        );
+    }
+
+    #[test]
+    fn test_merge_checkpoint_with_topic_pruned_entry_id_creates_checkpoint() {
+        assert_eq!(
+            ReplayCheckpoint::merge_with_topic_pruned_entry_id(None, Some(10), true),
+            Some(ReplayCheckpoint::new(10, Some(10)))
+        );
+    }
+
+    #[test]
+    fn test_merge_checkpoint_with_topic_pruned_entry_id_updates_both_ids() {
+        let checkpoint = ReplayCheckpoint::new(10, Some(5));
+
+        assert_eq!(
+            ReplayCheckpoint::merge_with_topic_pruned_entry_id(Some(checkpoint), Some(20), true),
+            Some(ReplayCheckpoint::new(20, Some(20)))
+        );
+    }
+
+    #[test]
+    fn test_merge_checkpoint_with_topic_pruned_entry_id_preserves_larger_ids() {
+        let checkpoint = ReplayCheckpoint::new(30, Some(40));
+
+        assert_eq!(
+            ReplayCheckpoint::merge_with_topic_pruned_entry_id(Some(checkpoint), Some(20), true),
+            Some(checkpoint)
+        );
+    }
+
+    #[test]
+    fn test_merge_checkpoint_with_topic_pruned_entry_id_for_mito() {
+        assert_eq!(
+            ReplayCheckpoint::merge_with_topic_pruned_entry_id(None, Some(10), false),
+            Some(ReplayCheckpoint::new(10, None))
+        );
+
+        let checkpoint = ReplayCheckpoint::new(5, Some(8));
+        assert_eq!(
+            ReplayCheckpoint::merge_with_topic_pruned_entry_id(Some(checkpoint), Some(10), false),
+            Some(ReplayCheckpoint::new(10, Some(8)))
+        );
+    }
+
     #[tokio::test]
     async fn test_topic_region_manager() {
         let kv_backend = Arc::new(MemoryKvBackend::default());

src/datanode/src/config.rs

@@ -14,6 +14,8 @@
 
 //! Datanode configurations
 
+use std::time::Duration;
+
 use common_base::readable_size::ReadableSize;
 use common_config::{Configurable, DEFAULT_DATA_HOME};
 use common_options::memory::MemoryOptions;
@@ -75,6 +77,10 @@ pub struct DatanodeOptions {
     pub wal: DatanodeWalConfig,
     pub storage: StorageConfig,
     pub max_concurrent_queries: usize,
+    /// Timeout to acquire a permit from the concurrent query limiter when
+    /// `max_concurrent_queries` is reached. Only effective when the limiter is enabled.
+    #[serde(with = "humantime_serde")]
+    pub concurrent_query_limiter_timeout: Duration,
     /// Options for different store engines.
     pub region_engine: Vec<RegionEngineConfig>,
     pub logging: LoggingOptions,
@@ -131,6 +137,7 @@ impl Default for DatanodeOptions {
             wal: DatanodeWalConfig::default(),
             storage: StorageConfig::default(),
             max_concurrent_queries: 0,
+            concurrent_query_limiter_timeout: Duration::from_millis(100),
             region_engine: vec![
                 RegionEngineConfig::Mito(MitoConfig::default()),
                 RegionEngineConfig::File(FileEngineConfig::default()),

src/datanode/src/datanode.rs

@@ -445,8 +445,7 @@ impl DatanodeBuilder {
             event_listener,
             table_provider_factory,
             opts.max_concurrent_queries,
-            //TODO: revaluate the hardcoded timeout on the next version of datanode concurrency limiter.
-            Duration::from_millis(100),
+            opts.concurrent_query_limiter_timeout,
             opts.grpc.flight_compression,
         );
 

src/datanode/src/heartbeat/handler.rs

@@ -313,7 +313,7 @@ mod tests {
     use mito2::test_util::{CreateRequestBuilder, TestEnv};
     use store_api::path_utils::table_dir;
     use store_api::region_engine::RegionRole;
-    use store_api::region_request::{RegionCloseRequest, RegionRequest};
+    use store_api::region_request::{RegionCloseRequest, RegionRequest, RegionRequirements};
     use store_api::storage::RegionId;
     use tokio::sync::mpsc::{self, Receiver};
 
@@ -442,6 +442,8 @@ mod tests {
             HashMap::new(),
             HashMap::new(),
             false,
+            None,
+            RegionRequirements::empty(),
         )])
     }
 

src/datanode/src/heartbeat/handler/open_region.rs

@@ -14,6 +14,7 @@
 
 use common_meta::instruction::{InstructionReply, OpenRegion, SimpleReply};
 use common_meta::wal_provider::prepare_wal_options;
+use common_telemetry::info;
 use store_api::path_utils::table_dir;
 use store_api::region_request::{PathType, RegionOpenRequest};
 use store_api::storage::RegionId;
@@ -41,8 +42,13 @@ impl InstructionHandler for OpenRegionsHandler {
                     mut region_options,
                     region_wal_options,
                     skip_wal_replay,
+                    reason,
+                    requirements,
                 } = open_region;
                 let region_id = RegionId::new(region_ident.table_id, region_ident.region_number);
+                info!(
+                    "Received open region instruction, region_id: {region_id}, reason: {reason:?}"
+                );
                 prepare_wal_options(&mut region_options, region_id, &region_wal_options);
                 let request = RegionOpenRequest {
                     engine: region_ident.engine,
@@ -51,6 +57,7 @@ impl InstructionHandler for OpenRegionsHandler {
                     options: region_options,
                     skip_wal_replay,
                     checkpoint: None,
+                    requirements,
                 };
                 (region_id, request)
             })
@@ -85,7 +92,7 @@ mod tests {
     use mito2::engine::MITO_ENGINE_NAME;
     use mito2::test_util::{CreateRequestBuilder, TestEnv};
     use store_api::path_utils::table_dir;
-    use store_api::region_request::{RegionCloseRequest, RegionRequest};
+    use store_api::region_request::{RegionCloseRequest, RegionRequest, RegionRequirements};
     use store_api::storage::RegionId;
 
     use crate::heartbeat::handler::RegionHeartbeatResponseHandler;
@@ -98,17 +105,21 @@ mod tests {
     ) -> Instruction {
         let region_idents = region_ids
             .into_iter()
-            .map(|region_id| OpenRegion {
-                region_ident: RegionIdent {
-                    datanode_id: 0,
-                    table_id: region_id.table_id(),
-                    region_number: region_id.region_number(),
-                    engine: MITO_ENGINE_NAME.to_string(),
-                },
-                region_storage_path: storage_path.to_string(),
-                region_options: HashMap::new(),
-                region_wal_options: HashMap::new(),
-                skip_wal_replay: false,
+            .map(|region_id| {
+                OpenRegion::new(
+                    RegionIdent {
+                        datanode_id: 0,
+                        table_id: region_id.table_id(),
+                        region_number: region_id.region_number(),
+                        engine: MITO_ENGINE_NAME.to_string(),
+                    },
+                    storage_path,
+                    HashMap::new(),
+                    HashMap::new(),
+                    false,
+                    None,
+                    RegionRequirements::empty(),
+                )
             })
             .collect();
 

src/datanode/src/region_server.rs

@@ -49,6 +49,7 @@ use common_telemetry::{debug, error, info, warn};
 use dashmap::DashMap;
 use datafusion::datasource::TableProvider;
 use datafusion_common::tree_node::TreeNode;
+use datatypes::schema::SchemaRef;
 use either::Either;
 use futures_util::Stream;
 use futures_util::future::try_join_all;
@@ -82,7 +83,7 @@ use store_api::region_request::{
     RegionOpenRequest, RegionRequest,
 };
 use store_api::storage::RegionId;
-use tokio::sync::{Semaphore, SemaphorePermit};
+use tokio::sync::{OwnedSemaphorePermit, Semaphore};
 use tokio::time::timeout;
 use tonic::{Request, Response, Result as TonicResult};
 
@@ -257,7 +258,7 @@ impl RegionServer {
         request: api::v1::region::QueryRequest,
         query_ctx: QueryContextRef,
     ) -> Result<SendableRecordBatchStream> {
-        let _permit = if let Some(p) = &self.inner.parallelism {
+        let permit = if let Some(p) = &self.inner.parallelism {
             Some(p.acquire().await?)
         } else {
             None
@@ -298,14 +299,13 @@ impl RegionServer {
             )
             .await?;
 
-        Ok(wrap_flow_region_watermark_stream(
-            stream, region_id, &query_ctx,
-        ))
+        let stream = wrap_flow_region_watermark_stream(stream, region_id, &query_ctx);
+        Ok(maybe_guard_stream(stream, permit))
     }
 
     #[tracing::instrument(skip_all)]
     pub async fn handle_read(&self, request: QueryRequest) -> Result<SendableRecordBatchStream> {
-        let _permit = if let Some(p) = &self.inner.parallelism {
+        let permit = if let Some(p) = &self.inner.parallelism {
             Some(p.acquire().await?)
         } else {
             None
@@ -332,9 +332,8 @@ impl RegionServer {
             .handle_read(QueryRequest { plan, ..request }, query_ctx.clone())
             .await?;
 
-        Ok(wrap_flow_region_watermark_stream(
-            stream, region_id, &query_ctx,
-        ))
+        let stream = wrap_flow_region_watermark_stream(stream, region_id, &query_ctx);
+        Ok(maybe_guard_stream(stream, permit))
     }
 
     /// Returns all opened and reportable regions.
@@ -1058,7 +1057,7 @@ struct RegionServerInner {
 }
 
 struct RegionServerParallelism {
-    semaphore: Semaphore,
+    semaphore: Arc<Semaphore>,
     timeout: Duration,
 }
 
@@ -1071,19 +1070,68 @@ impl RegionServerParallelism {
             return None;
         }
         Some(RegionServerParallelism {
-            semaphore: Semaphore::new(max_concurrent_queries),
+            semaphore: Arc::new(Semaphore::new(max_concurrent_queries)),
             timeout: concurrent_query_limiter_timeout,
         })
     }
 
-    pub async fn acquire(&self) -> Result<SemaphorePermit<'_>> {
-        timeout(self.timeout, self.semaphore.acquire())
+    pub async fn acquire(&self) -> Result<OwnedSemaphorePermit> {
+        timeout(self.timeout, self.semaphore.clone().acquire_owned())
             .await
             .context(ConcurrentQueryLimiterTimeoutSnafu)?
             .context(ConcurrentQueryLimiterClosedSnafu)
     }
 }
 
+/// Wraps a record batch stream and holds a concurrency permit until the stream is
+/// fully consumed (dropped), so `max_concurrent_queries` bounds the number of
+/// in-flight read streams, not just query planning.
+struct PermitGuardedStream {
+    inner: SendableRecordBatchStream,
+    _permit: OwnedSemaphorePermit,
+}
+
+impl RecordBatchStream for PermitGuardedStream {
+    fn name(&self) -> &str {
+        self.inner.name()
+    }
+
+    fn schema(&self) -> SchemaRef {
+        self.inner.schema()
+    }
+
+    fn output_ordering(&self) -> Option<&[OrderOption]> {
+        self.inner.output_ordering()
+    }
+
+    fn metrics(&self) -> Option<RecordBatchMetrics> {
+        self.inner.metrics()
+    }
+}
+
+impl Stream for PermitGuardedStream {
+    type Item = common_recordbatch::error::Result<RecordBatch>;
+
+    fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
+        self.inner.as_mut().poll_next(cx)
+    }
+}
+
+/// Wraps `stream` so it holds `permit` until fully consumed. Returns `stream`
+/// unchanged when no permit was acquired (limiter disabled).
+fn maybe_guard_stream(
+    stream: SendableRecordBatchStream,
+    permit: Option<OwnedSemaphorePermit>,
+) -> SendableRecordBatchStream {
+    match permit {
+        Some(permit) => Box::pin(PermitGuardedStream {
+            inner: stream,
+            _permit: permit,
+        }),
+        None => stream,
+    }
+}
+
 enum CurrentEngine {
     Engine(RegionEngineRef),
     EarlyReturn(AffectedRows),
@@ -2057,6 +2105,7 @@ mod tests {
                     options: Default::default(),
                     skip_wal_replay: false,
                     checkpoint: None,
+                    requirements: Default::default(),
                 }),
             )
             .await
@@ -2235,6 +2284,7 @@ mod tests {
                             options: Default::default(),
                             skip_wal_replay: false,
                             checkpoint: None,
+                            requirements: Default::default(),
                         },
                     ),
                     (
@@ -2246,6 +2296,7 @@ mod tests {
                             options: Default::default(),
                             skip_wal_replay: false,
                             checkpoint: None,
+                            requirements: Default::default(),
                         },
                     ),
                 ],
@@ -2268,6 +2319,7 @@ mod tests {
                             options: Default::default(),
                             skip_wal_replay: false,
                             checkpoint: None,
+                            requirements: Default::default(),
                         },
                     ),
                     (
@@ -2279,6 +2331,7 @@ mod tests {
                             options: Default::default(),
                             skip_wal_replay: false,
                             checkpoint: None,
+                            requirements: Default::default(),
                         },
                     ),
                 ],

src/datanode/src/utils.rs

@@ -16,11 +16,16 @@ use std::collections::HashMap;
 
 use common_meta::DatanodeId;
 use common_meta::key::datanode_table::DatanodeTableManager;
-use common_meta::key::topic_region::{TopicRegionKey, TopicRegionManager, TopicRegionValue};
+use common_meta::key::topic_name::{TopicNameKey, TopicNameManager, TopicNameValue};
+use common_meta::key::topic_region::{
+    ReplayCheckpoint as MetadataReplayCheckpoint, TopicRegionKey, TopicRegionManager,
+    TopicRegionValue,
+};
 use common_meta::kv_backend::KvBackendRef;
 use common_meta::wal_provider::{extract_topic_from_wal_options, prepare_wal_options};
 use futures::TryStreamExt;
 use snafu::ResultExt;
+use store_api::metric_engine_consts::METRIC_ENGINE_NAME;
 use store_api::path_utils::table_dir;
 use store_api::region_request::{PathType, RegionOpenRequest, ReplayCheckpoint};
 use store_api::storage::{RegionId, RegionNumber};
@@ -63,14 +68,45 @@ fn group_region_by_topic(
     }
 }
 
+fn region_pruned_entry_ids(
+    topic_regions: &HashMap<String, Vec<RegionId>>,
+    topic_name_values: &HashMap<String, TopicNameValue>,
+) -> HashMap<RegionId, u64> {
+    topic_regions
+        .iter()
+        .flat_map(|(topic, region_ids)| {
+            topic_name_values
+                .get(topic)
+                .into_iter()
+                .flat_map(move |value| {
+                    region_ids
+                        .iter()
+                        .map(move |region_id| (*region_id, value.pruned_entry_id))
+                })
+        })
+        .collect()
+}
+
 fn get_replay_checkpoint(
     region_id: RegionId,
     topic_region_values: &Option<HashMap<RegionId, TopicRegionValue>>,
+    pruned_entry_id: Option<u64>,
+    is_metric_engine: bool,
 ) -> Option<ReplayCheckpoint> {
-    let topic_region_values = topic_region_values.as_ref()?;
-    let topic_region_value = topic_region_values.get(&region_id);
-    let replay_checkpoint = topic_region_value.and_then(|value| value.checkpoint);
-    replay_checkpoint.map(|checkpoint| ReplayCheckpoint {
+    let checkpoint = topic_region_values
+        .as_ref()
+        .and_then(|values| values.get(&region_id))
+        .and_then(|value| value.checkpoint)
+        .map(|checkpoint| {
+            MetadataReplayCheckpoint::new(checkpoint.entry_id, checkpoint.metadata_entry_id)
+        });
+
+    MetadataReplayCheckpoint::merge_with_topic_pruned_entry_id(
+        checkpoint,
+        pruned_entry_id,
+        is_metric_engine,
+    )
+    .map(|checkpoint| ReplayCheckpoint {
         entry_id: checkpoint.entry_id,
         metadata_entry_id: checkpoint.metadata_entry_id,
     })
@@ -88,7 +124,8 @@ pub async fn build_region_open_requests(
         .await
         .context(GetMetadataSnafu)?;
 
-    let topic_region_manager = TopicRegionManager::new(kv_backend);
+    let topic_region_manager = TopicRegionManager::new(kv_backend.clone());
+    let topic_name_manager = TopicNameManager::new(kv_backend);
     let mut topic_regions = HashMap::<String, Vec<RegionId>>::new();
     let mut regions = vec![];
     #[cfg(feature = "enterprise")]
@@ -161,10 +198,36 @@ pub async fn build_region_open_requests(
         None
     };
 
+    let topic_name_values = if !topic_regions.is_empty() {
+        let topics = topic_regions
+            .keys()
+            .map(|topic| TopicNameKey::new(topic))
+            .collect::<Vec<_>>();
+        Some(
+            topic_name_manager
+                .batch_get(topics)
+                .await
+                .context(GetMetadataSnafu)?,
+        )
+    } else {
+        None
+    };
+    let region_pruned_entry_ids = topic_name_values
+        .as_ref()
+        .map(|values| region_pruned_entry_ids(&topic_regions, values));
+
     let mut leader_region_requests = Vec::with_capacity(regions.len());
     for (region_id, engine, store_path, options) in regions {
         let table_dir = table_dir(&store_path, region_id.table_id());
-        let checkpoint = get_replay_checkpoint(region_id, &topic_region_values);
+        let pruned_entry_id = region_pruned_entry_ids
+            .as_ref()
+            .and_then(|values| values.get(&region_id).copied());
+        let checkpoint = get_replay_checkpoint(
+            region_id,
+            &topic_region_values,
+            pruned_entry_id,
+            engine == METRIC_ENGINE_NAME,
+        );
         info!("region_id: {}, checkpoint: {:?}", region_id, checkpoint);
         leader_region_requests.push((
             region_id,
@@ -175,6 +238,7 @@ pub async fn build_region_open_requests(
                 options,
                 skip_wal_replay: false,
                 checkpoint,
+                requirements: Default::default(),
             },
         ));
     }
@@ -193,6 +257,7 @@ pub async fn build_region_open_requests(
                     options,
                     skip_wal_replay: true,
                     checkpoint: None,
+                    requirements: Default::default(),
                 },
             ));
         }

src/datatypes/src/json.rs

@@ -26,12 +26,12 @@ use std::sync::Arc;
 
 use serde::{Deserialize, Serialize};
 use serde_json::{Map, Value as Json};
-use snafu::{OptionExt, ResultExt, ensure};
+use snafu::{OptionExt, ResultExt};
 
 use crate::error::{self, InvalidJsonSnafu, Result, SerializeSnafu};
 use crate::json::value::{JsonValue, JsonVariant};
 use crate::types::json_type::{JsonNativeType, JsonNumberType, JsonObjectType};
-use crate::types::{StructField, StructType};
+use crate::types::{JsonType, StructField, StructType};
 use crate::value::{ListValue, StructValue, Value};
 
 /// The configuration of JSON encoding
@@ -305,33 +305,47 @@ fn encode_json_array_with_context<'a>(
 ) -> Result<JsonValue> {
     let json_array_len = json_array.len();
     let mut items = Vec::with_capacity(json_array_len);
-    let mut element_type = item_type.cloned();
 
     for (index, value) in json_array.into_iter().enumerate() {
         let array_context = context.with_key(&index.to_string());
-        let item_value =
-            encode_json_value_with_context(value, element_type.as_ref(), &array_context)?;
-        let item_type = item_value.json_type().native_type().clone();
-        items.push(item_value.into_variant());
-
-        // Determine the common type for the list
-        if let Some(current_type) = &element_type {
-            // It's valid for json array to have different types of items, for example,
-            // ["a string", 1]. However, the `JsonValue` will be converted to Arrow list array,
-            // which requires all items have exactly same type. So we forbid the different types
-            // case here. Besides, it's not common for items in a json array to differ. So I think
-            // we are good here.
-            ensure!(
-                item_type == *current_type,
-                error::InvalidJsonSnafu {
-                    value: "all items in json array must have the same type"
-                }
-            );
-        } else {
-            element_type = Some(item_type);
-        }
+        let item_value = encode_json_value_with_context(value, None, &array_context)?;
+        items.push(item_value);
     }
 
+    // In specification, it's valid for a JSON array to have different types of items, for example,
+    // ["a string", 1]. However, in implementation, the `JsonValue` will be converted to Arrow list
+    // array, which requires all items have exactly the same type. So we merge out the maybe
+    // different item types to a unified type, and align all the item values to it.
+
+    let provided_item_type = item_type.map(|x| JsonType::new_json2(x.clone()));
+    let merged_item_type = if let Some((first, rests)) = items.split_first() {
+        let mut merged = first.json_type().clone();
+        for rest in rests.iter().map(|x| x.json_type()) {
+            if matches!(merged.native_type(), JsonNativeType::Variant) {
+                break;
+            }
+            merged.merge(rest)?;
+        }
+        Some(merged)
+    } else {
+        None
+    };
+    let unified_item_type = match (provided_item_type, merged_item_type) {
+        (Some(mut x), Some(y)) => {
+            x.merge(&y)?;
+            Some(x)
+        }
+        (x, y) => x.or(y),
+    };
+    if let Some(unified_item_type) = unified_item_type {
+        for item in &mut items {
+            item.try_align(&unified_item_type)?;
+        }
+    }
+    let items = items
+        .into_iter()
+        .map(|x| x.into_variant())
+        .collect::<Vec<_>>();
     Ok(JsonValue::new(JsonVariant::Array(items)))
 }
 
@@ -1050,11 +1064,8 @@ mod tests {
     fn test_encode_json_array_mixed_types() {
         let json = json!([1, "hello", true, 3.15]);
         let settings = JsonStructureSettings::Structured(None);
-        let result = settings.encode_with_type(json, None);
-        assert_eq!(
-            result.unwrap_err().to_string(),
-            "Invalid JSON: all items in json array must have the same type"
-        );
+        let value = settings.encode_with_type(json, None).unwrap();
+        assert_eq!(value.data_type().to_string(), r#"Json2["<Variant>"]"#);
     }
 
     #[test]
@@ -1276,12 +1287,12 @@ mod tests {
     #[test]
     fn test_encode_json_array_with_item_type() {
         let json = json!([1, 2, 3]);
-        let item_type = Arc::new(ConcreteDataType::uint64_datatype());
+        let item_type = Arc::new(ConcreteDataType::int64_datatype());
         let settings = JsonStructureSettings::Structured(None);
         let result = settings
             .encode_with_type(
                 json,
-                Some(&JsonNativeType::Array(Box::new(JsonNativeType::u64()))),
+                Some(&JsonNativeType::Array(Box::new(JsonNativeType::i64()))),
             )
             .unwrap()
             .into_json_inner()
@@ -1289,9 +1300,9 @@ mod tests {
 
         if let Value::List(list_value) = result {
             assert_eq!(list_value.items().len(), 3);
-            assert_eq!(list_value.items()[0], Value::UInt64(1));
-            assert_eq!(list_value.items()[1], Value::UInt64(2));
-            assert_eq!(list_value.items()[2], Value::UInt64(3));
+            assert_eq!(list_value.items()[0], Value::Int64(1));
+            assert_eq!(list_value.items()[1], Value::Int64(2));
+            assert_eq!(list_value.items()[2], Value::Int64(3));
             assert_eq!(list_value.datatype(), item_type);
         } else {
             panic!("Expected List value");
@@ -2249,10 +2260,10 @@ mod tests {
             )])),
         );
 
-        let decoded_struct = settings.decode_struct(array_struct);
+        let decoded_struct = settings.decode_struct(array_struct).unwrap();
         assert_eq!(
-            decoded_struct.unwrap_err().to_string(),
-            "Invalid JSON: all items in json array must have the same type"
+            format!("{decoded_struct:?}"),
+            r#"StructValue { items: [List(ListValue { items: [Binary(Bytes(b"1")), Binary(Bytes(b"\"hello\"")), Binary(Bytes(b"true")), Binary(Bytes(b"3.15"))], datatype: Binary(BinaryType { repr_type: Binary }) })], fields: StructType { fields: [StructField { name: "value", data_type: List(ListType { item_type: Binary(BinaryType { repr_type: Binary }) }), nullable: true, metadata: {} }] } }"#
         );
     }
 

src/datatypes/src/json/value.rs

@@ -65,6 +65,14 @@ impl JsonNumber {
             JsonNumber::Float(n) => n.0,
         }
     }
+
+    fn native_type(&self) -> JsonNativeType {
+        match self {
+            JsonNumber::PosInt(_) => JsonNativeType::u64(),
+            JsonNumber::NegInt(_) => JsonNativeType::i64(),
+            JsonNumber::Float(_) => JsonNativeType::f64(),
+        }
+    }
 }
 
 impl From<u64> for JsonNumber {
@@ -147,26 +155,14 @@ impl JsonVariant {
         match self {
             JsonVariant::Null => JsonNativeType::Null,
             JsonVariant::Bool(_) => JsonNativeType::Bool,
-            JsonVariant::Number(n) => match n {
-                JsonNumber::PosInt(_) => JsonNativeType::u64(),
-                JsonNumber::NegInt(_) => JsonNativeType::i64(),
-                JsonNumber::Float(_) => JsonNativeType::f64(),
-            },
+            JsonVariant::Number(n) => n.native_type(),
             JsonVariant::String(_) => JsonNativeType::String,
             JsonVariant::Array(array) => {
-                let item_type = if let Some(first) = array.first() {
-                    first.native_type()
-                } else {
-                    JsonNativeType::Null
-                };
-                JsonNativeType::Array(Box::new(item_type))
+                json_array_native_type(array.iter().map(JsonVariant::native_type))
+            }
+            JsonVariant::Object(object) => {
+                json_object_native_type(object.iter().map(|(k, v)| (k, v.native_type())))
             }
-            JsonVariant::Object(object) => JsonNativeType::Object(
-                object
-                    .iter()
-                    .map(|(k, v)| (k.clone(), v.native_type()))
-                    .collect(),
-            ),
             JsonVariant::Variant(_) => JsonNativeType::Variant,
         }
     }
@@ -469,6 +465,7 @@ impl JsonValue {
                         .collect::<Result<_>>()?,
                 ),
 
+                (JsonVariant::Object(kvs), _) if kvs.is_empty() => JsonVariant::Null,
                 (JsonVariant::Object(mut kvs), JsonNativeType::Object(expected)) => {
                     ensure!(
                         expected.keys().len() >= kvs.keys().len()
@@ -517,7 +514,7 @@ impl JsonValue {
 
         let x = std::mem::take(&mut self.json_variant);
         self.json_variant = helper(x, expected.native_type())?;
-        self.json_type = OnceLock::from(expected.clone());
+        self.json_type = OnceLock::new();
         Ok(())
     }
 }
@@ -623,35 +620,55 @@ pub enum JsonVariantRef<'a> {
 }
 
 impl JsonVariantRef<'_> {
-    fn json_type(&self) -> JsonType {
-        fn native_type(v: &JsonVariantRef<'_>) -> JsonNativeType {
-            match v {
-                JsonVariantRef::Null => JsonNativeType::Null,
-                JsonVariantRef::Bool(_) => JsonNativeType::Bool,
-                JsonVariantRef::Number(n) => match n {
-                    JsonNumber::PosInt(_) => JsonNativeType::u64(),
-                    JsonNumber::NegInt(_) => JsonNativeType::i64(),
-                    JsonNumber::Float(_) => JsonNativeType::f64(),
-                },
-                JsonVariantRef::String(_) => JsonNativeType::String,
-                JsonVariantRef::Array(array) => {
-                    let item_type = if let Some(first) = array.first() {
-                        native_type(first)
-                    } else {
-                        JsonNativeType::Null
-                    };
-                    JsonNativeType::Array(Box::new(item_type))
-                }
-                JsonVariantRef::Object(object) => JsonNativeType::Object(
-                    object
-                        .iter()
-                        .map(|(k, v)| (k.to_string(), native_type(v)))
-                        .collect(),
-                ),
-                JsonVariantRef::Variant(_) => JsonNativeType::Variant,
+    fn native_type(&self) -> JsonNativeType {
+        match self {
+            JsonVariantRef::Null => JsonNativeType::Null,
+            JsonVariantRef::Bool(_) => JsonNativeType::Bool,
+            JsonVariantRef::Number(n) => n.native_type(),
+            JsonVariantRef::String(_) => JsonNativeType::String,
+            JsonVariantRef::Array(array) => {
+                json_array_native_type(array.iter().map(JsonVariantRef::native_type))
             }
+            JsonVariantRef::Object(object) => {
+                json_object_native_type(object.iter().map(|(k, v)| (*k, v.native_type())))
+            }
+            JsonVariantRef::Variant(_) => JsonNativeType::Variant,
         }
-        JsonType::new_json2(native_type(self))
+    }
+
+    fn json_type(&self) -> JsonType {
+        JsonType::new_json2(self.native_type())
+    }
+}
+
+fn json_array_native_type<I>(items: I) -> JsonNativeType
+where
+    I: IntoIterator<Item = JsonNativeType>,
+{
+    let mut iter = items.into_iter();
+    let mut item_type = match iter.next() {
+        Some(t) => t,
+        None => return JsonNativeType::Array(Box::new(JsonNativeType::Null)),
+    };
+    for x in iter {
+        if matches!(item_type, JsonNativeType::Variant) {
+            break;
+        }
+        item_type.merge(&x);
+    }
+    JsonNativeType::Array(Box::new(item_type))
+}
+
+fn json_object_native_type<I, K>(fields: I) -> JsonNativeType
+where
+    I: IntoIterator<Item = (K, JsonNativeType)>,
+    K: Into<String>,
+{
+    let mut fields = fields.into_iter().peekable();
+    if fields.peek().is_none() {
+        JsonNativeType::Null
+    } else {
+        JsonNativeType::Object(fields.map(|(k, v)| (k.into(), v)).collect())
     }
 }
 
@@ -941,7 +958,6 @@ mod tests {
                 ("name".to_string(), JsonVariant::Null),
             ])))
         );
-        assert_eq!(value.json_type(), &expected);
 
         // Object alignment should fail if the expected type misses any field from the value.
         let expected = JsonType::new_json2(JsonNativeType::Object(JsonObjectType::from([(

src/datatypes/src/types/json_type.rs

@@ -115,6 +115,14 @@ impl JsonNativeType {
             (JsonNativeType::Null, that) => that.clone(),
             (this, JsonNativeType::Null) => this,
             (this, that) if this == *that => this,
+
+            (JsonNativeType::Number(x), JsonNativeType::Number(y)) => {
+                JsonNativeType::Number(match (x, y) {
+                    (x, y) if x == *y => x,
+                    (JsonNumberType::F64, _) | (_, JsonNumberType::F64) => JsonNumberType::F64,
+                    _ => JsonNumberType::I64,
+                })
+            }
             _ => JsonNativeType::Variant,
         };
     }
@@ -822,7 +830,7 @@ mod tests {
         test(
             "1.5",
             &mut JsonType::new_json2(JsonNativeType::i64()),
-            Ok(r#""<Variant>""#),
+            Ok(r#""<Number>""#),
         )?;
 
         // Object merge should preserve existing fields and append missing fields.

src/datatypes/src/vectors/json/builder.rs

@@ -89,7 +89,9 @@ impl MutableVector for JsonVectorBuilder {
             .fail();
         };
         let json_type = value.json_type();
-        self.merged_type.merge(json_type)?;
+        if !self.merged_type.is_include(json_type) {
+            self.merged_type.merge(json_type)?;
+        }
 
         let value = JsonValue::new(JsonVariant::from(value.variant().clone()));
         self.values.push(value);

src/file-engine/Cargo.toml

@@ -29,6 +29,7 @@ datafusion-expr.workspace = true
 datatypes.workspace = true
 futures.workspace = true
 object-store.workspace = true
+object_store_opendal.workspace = true
 serde = { version = "1.0", features = ["derive"] }
 serde_json.workspace = true
 snafu.workspace = true

src/file-engine/src/engine.rs

@@ -32,7 +32,7 @@ use store_api::region_engine::{
 };
 use store_api::region_request::{
     AffectedRows, RegionCloseRequest, RegionCreateRequest, RegionDropRequest, RegionOpenRequest,
-    RegionRequest,
+    RegionRequest, RegionRequirements,
 };
 use store_api::storage::{RegionId, ScanRequest, SequenceNumber};
 use tokio::sync::Mutex;
@@ -186,6 +186,24 @@ struct EngineInner {
 
 type EngineInnerRef = Arc<EngineInner>;
 
+fn ensure_open_requirements(
+    requirements: RegionRequirements,
+    object_store: &ObjectStore,
+) -> EngineResult<()> {
+    if !requirements.object_storage {
+        return Ok(());
+    }
+
+    ensure!(
+        object_store::util::is_object_storage(object_store),
+        UnsupportedSnafu {
+            operation: "open region with object storage requirement on non-object storage"
+        }
+    );
+
+    Ok(())
+}
+
 impl EngineInner {
     fn new(object_store: ObjectStore) -> Self {
         Self {
@@ -289,6 +307,8 @@ impl EngineInner {
             return Ok(0);
         }
 
+        ensure_open_requirements(request.requirements, &self.object_store)?;
+
         let res = FileRegion::open(region_id, request, &self.object_store).await;
         let region = res.inspect_err(|err| {
             error!(
@@ -356,3 +376,53 @@ impl EngineInner {
         self.regions.read().unwrap().contains_key(&region_id)
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use object_store::services::{Fs, S3};
+
+    use super::*;
+    use crate::error::Error;
+
+    fn build_fs_object_store() -> ObjectStore {
+        ObjectStore::new(Fs::default().root("/tmp"))
+            .unwrap()
+            .finish()
+    }
+
+    fn build_s3_object_store() -> ObjectStore {
+        ObjectStore::new(
+            S3::default()
+                .bucket("test-bucket")
+                .region("us-east-1")
+                .disable_ec2_metadata(),
+        )
+        .unwrap()
+        .finish()
+    }
+
+    #[test]
+    fn test_empty_open_requirements_are_supported() {
+        ensure_open_requirements(RegionRequirements::empty(), &build_fs_object_store()).unwrap();
+    }
+
+    #[test]
+    fn test_object_storage_open_requirement_rejects_fs_object_store() {
+        let err = ensure_open_requirements(
+            RegionRequirements::object_storage(),
+            &build_fs_object_store(),
+        )
+        .unwrap_err();
+
+        assert!(matches!(err, Error::Unsupported { .. }));
+    }
+
+    #[test]
+    fn test_object_storage_open_requirement_accepts_s3_object_store() {
+        ensure_open_requirements(
+            RegionRequirements::object_storage(),
+            &build_s3_object_store(),
+        )
+        .unwrap();
+    }
+}

src/file-engine/src/query/file_stream.rs

@@ -61,7 +61,7 @@ fn build_record_batch_stream(
             .with_file_group(FileGroup::new(files))
             .build();
 
-    let store = Arc::new(object_store::compat::OpendalStore::new(
+    let store = Arc::new(object_store_opendal::OpendalStore::new(
         scan_plan_config.store.clone(),
     ));
 

src/file-engine/src/region.rs

@@ -181,6 +181,7 @@ mod tests {
             options: HashMap::default(),
             skip_wal_replay: false,
             checkpoint: None,
+            requirements: Default::default(),
         };
 
         let region = FileRegion::open(region_id, request, &object_store)
@@ -238,6 +239,7 @@ mod tests {
             options: HashMap::default(),
             skip_wal_replay: false,
             checkpoint: None,
+            requirements: Default::default(),
         };
         let err = FileRegion::open(region_id, request, &object_store)
             .await

src/flow/src/batching_mode.rs

@@ -23,7 +23,6 @@ use session::ReadPreference;
 mod checkpoint;
 pub(crate) mod engine;
 pub(crate) mod frontend_client;
-mod incremental_filter;
 mod state;
 mod table_creator;
 mod task;
@@ -55,6 +54,10 @@ pub struct BatchingModeOptions {
     pub experimental_max_filter_num_per_query: usize,
     /// Time window merge distance
     pub experimental_time_window_merge_threshold: usize,
+    /// Whether to enable experimental flow incremental source reads.
+    ///
+    /// When disabled, batching flows always execute full-snapshot queries.
+    pub experimental_enable_incremental_read: bool,
     /// Read preference of the Frontend client.
     pub read_preference: ReadPreference,
     /// TLS option for client connections to frontends.
@@ -72,6 +75,7 @@ impl Default for BatchingModeOptions {
             experimental_frontend_scan_timeout: Duration::from_secs(30),
             experimental_max_filter_num_per_query: 20,
             experimental_time_window_merge_threshold: 3,
+            experimental_enable_incremental_read: false,
             read_preference: Default::default(),
             frontend_tls: None,
         }

src/flow/src/batching_mode/engine.rs

@@ -21,7 +21,7 @@ use std::time::Duration;
 use api::v1::flow::DirtyWindowRequests;
 use catalog::CatalogManagerRef;
 use common_error::ext::BoxedError;
-use common_meta::ddl::create_flow::FlowType;
+use common_meta::ddl::create_flow::{FLOW_EXPERIMENTAL_ENABLE_INCREMENTAL_READ_KEY, FlowType};
 use common_meta::key::TableMetadataManagerRef;
 use common_meta::key::flow::FlowMetadataManagerRef;
 use common_meta::key::flow::flow_state::FlowStat;
@@ -38,6 +38,7 @@ use session::context::QueryContext;
 use snafu::{OptionExt, ResultExt, ensure};
 use sql::parsers::utils::is_tql;
 use store_api::metric_engine_consts::is_metric_engine_internal_column;
+use store_api::mito_engine_options::APPEND_MODE_KEY;
 use store_api::storage::{RegionId, TableId};
 use table::table_reference::TableReference;
 use tokio::sync::{RwLock, oneshot};
@@ -428,6 +429,55 @@ async fn get_table_info(
 }
 
 impl BatchingEngine {
+    fn batch_opts_for_flow_options(
+        &self,
+        flow_options: &HashMap<String, String>,
+    ) -> Result<Arc<BatchingModeOptions>, Error> {
+        let mut batch_opts = (*self.batch_opts).clone();
+        if let Some(enable_incremental_read) =
+            flow_options.get(FLOW_EXPERIMENTAL_ENABLE_INCREMENTAL_READ_KEY)
+        {
+            batch_opts.experimental_enable_incremental_read = enable_incremental_read
+                .parse::<bool>()
+                .map_err(|_| {
+                    InvalidQuerySnafu {
+                        reason: format!(
+                            "Invalid flow option {FLOW_EXPERIMENTAL_ENABLE_INCREMENTAL_READ_KEY}: {enable_incremental_read}"
+                        ),
+                    }
+                    .build()
+                })?;
+        }
+
+        Ok(Arc::new(batch_opts))
+    }
+
+    fn table_options_enable_append_mode(extra_options: &HashMap<String, String>) -> bool {
+        extra_options
+            .get(APPEND_MODE_KEY)
+            .is_some_and(|value| value.eq_ignore_ascii_case("true"))
+    }
+
+    fn ensure_incremental_source_append_only(
+        batch_opts: &BatchingModeOptions,
+        table_name: &[String; 3],
+        extra_options: &HashMap<String, String>,
+    ) -> Result<(), Error> {
+        if batch_opts.experimental_enable_incremental_read {
+            ensure!(
+                Self::table_options_enable_append_mode(extra_options),
+                UnsupportedSnafu {
+                    reason: format!(
+                        "Flow incremental read requires append-only source table, but source table `{}` is not append-only. Consider setting append_mode='true' on the source table or disabling experimental_enable_incremental_read",
+                        table_name.join(".")
+                    ),
+                }
+            );
+        }
+
+        Ok(())
+    }
+
     pub async fn create_flow_inner(&self, args: CreateFlowArgs) -> Result<Option<FlowId>, Error> {
         let CreateFlowArgs {
             flow_id,
@@ -494,6 +544,8 @@ impl BatchingEngine {
             }
         );
 
+        let batch_opts = self.batch_opts_for_flow_options(&flow_options)?;
+
         let mut source_table_names = Vec::with_capacity(2);
         for src_id in source_table_ids {
             // also check table option to see if ttl!=instant
@@ -509,6 +561,11 @@ impl BatchingEngine {
                     ),
                 }
             );
+            Self::ensure_incremental_source_append_only(
+                &batch_opts,
+                &table_name,
+                &table_info.table_info.meta.options.extra_options,
+            )?;
 
             source_table_names.push(table_name);
         }
@@ -563,7 +620,7 @@ impl BatchingEngine {
             query_ctx,
             catalog_manager: self.catalog_manager.clone(),
             shutdown_rx: rx,
-            batch_opts: self.batch_opts.clone(),
+            batch_opts,
             flow_eval_interval: eval_interval.map(|secs| Duration::from_secs(secs as u64)),
         };
 
@@ -573,8 +630,11 @@ impl BatchingEngine {
         let engine = self.query_engine.clone();
         let frontend = self.frontend_client.clone();
 
-        // check execute once first to detect any error early
+        // Create sink table if needed, then validate an existing/created sink schema before
+        // spawning the background task. This catches user-created sink schema mismatches at
+        // CREATE FLOW time instead of surfacing them later in the execution loop.
         task.check_or_create_sink_table(&engine, &frontend).await?;
+        task.validate_sink_table_schema(&engine).await?;
 
         let (start_tx, start_rx) = oneshot::channel();
 
@@ -808,7 +868,7 @@ impl BatchingEngine {
         });
 
         let res = task
-            .gen_exec_once(
+            .execute_once_serialized(
                 &self.query_engine,
                 &self.frontend_client,
                 cur_dirty_window_cnt,
@@ -946,6 +1006,76 @@ mod tests {
         )
     }
 
+    #[tokio::test]
+    async fn test_flow_option_overrides_incremental_read_switch() {
+        let engine = new_test_engine().await;
+
+        let default_opts = engine.batch_opts_for_flow_options(&HashMap::new()).unwrap();
+        assert!(!default_opts.experimental_enable_incremental_read);
+
+        let enabled_opts = engine
+            .batch_opts_for_flow_options(&HashMap::from([(
+                FLOW_EXPERIMENTAL_ENABLE_INCREMENTAL_READ_KEY.to_string(),
+                "true".to_string(),
+            )]))
+            .unwrap();
+        assert!(enabled_opts.experimental_enable_incremental_read);
+    }
+
+    #[test]
+    fn test_table_options_enable_append_mode() {
+        assert!(!BatchingEngine::table_options_enable_append_mode(
+            &HashMap::new()
+        ));
+        assert!(!BatchingEngine::table_options_enable_append_mode(
+            &HashMap::from([(APPEND_MODE_KEY.to_string(), "false".to_string())])
+        ));
+        assert!(BatchingEngine::table_options_enable_append_mode(
+            &HashMap::from([(APPEND_MODE_KEY.to_string(), "TRUE".to_string())])
+        ));
+    }
+
+    #[test]
+    fn test_incremental_source_append_only_enforcement() {
+        let table_name = [
+            "greptime".to_string(),
+            "public".to_string(),
+            "numbers".to_string(),
+        ];
+        let disabled_opts = BatchingModeOptions::default();
+        let enabled_opts = BatchingModeOptions {
+            experimental_enable_incremental_read: true,
+            ..Default::default()
+        };
+        let non_append_options = HashMap::new();
+        let append_options = HashMap::from([(APPEND_MODE_KEY.to_string(), "true".to_string())]);
+
+        BatchingEngine::ensure_incremental_source_append_only(
+            &disabled_opts,
+            &table_name,
+            &non_append_options,
+        )
+        .expect("disabled incremental read should not require append-only source");
+        BatchingEngine::ensure_incremental_source_append_only(
+            &enabled_opts,
+            &table_name,
+            &append_options,
+        )
+        .expect("append-only source should be accepted when incremental read is enabled");
+
+        let err = BatchingEngine::ensure_incremental_source_append_only(
+            &enabled_opts,
+            &table_name,
+            &non_append_options,
+        )
+        .expect_err("non-append source should be rejected when incremental read is enabled");
+        assert!(
+            err.to_string()
+                .contains("Flow incremental read requires append-only source table"),
+            "{err}"
+        );
+    }
+
     async fn new_test_task(flow_id: FlowId) -> (BatchingTask, oneshot::Sender<()>) {
         let query_engine = create_test_query_engine();
         let ctx = QueryContext::arc();

src/flow/src/batching_mode/incremental_filter.rs

@@ -1,222 +0,0 @@
-// Copyright 2023 Greptime Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use common_telemetry::tracing::debug;
-use datafusion_expr::Expr;
-use datatypes::schema::Schema;
-
-use crate::batching_mode::state::FilterExprInfo;
-use crate::batching_mode::utils::IncrementalAggregateAnalysis;
-use crate::{Error, FlowId};
-
-pub(super) fn build_sink_dirty_time_window_filter_expr(
-    flow_id: FlowId,
-    analysis: &IncrementalAggregateAnalysis,
-    sink_schema: &Schema,
-    dirty_filter: Option<&FilterExprInfo>,
-) -> Result<Option<Expr>, Error> {
-    let Some(dirty_filter) = dirty_filter else {
-        return Ok(None);
-    };
-
-    let Some(sink_filter_col) =
-        infer_sink_time_window_filter_col(flow_id, analysis, sink_schema, dirty_filter)
-    else {
-        return Ok(None);
-    };
-
-    dirty_filter.predicate_for_col(&sink_filter_col)
-}
-
-fn infer_sink_time_window_filter_col(
-    flow_id: FlowId,
-    analysis: &IncrementalAggregateAnalysis,
-    sink_schema: &Schema,
-    dirty_filter: &FilterExprInfo,
-) -> Option<String> {
-    if analysis.group_key_names.is_empty() {
-        return None;
-    }
-
-    let is_timestamp_group_key = |name: &str| {
-        analysis.group_key_names.iter().any(|key| key == name)
-            && sink_schema
-                .column_schema_by_name(name)
-                .is_some_and(|col| col.data_type.is_timestamp())
-    };
-
-    if is_timestamp_group_key(&dirty_filter.col_name) {
-        return Some(dirty_filter.col_name.clone());
-    }
-
-    let candidates = analysis
-        .group_key_names
-        .iter()
-        .filter(|name| is_timestamp_group_key(name))
-        .cloned()
-        .collect::<Vec<_>>();
-
-    match candidates.as_slice() {
-        [name] => Some(name.clone()),
-        [] => {
-            debug!(
-                "Flow {} cannot infer sink dirty-window filter column: no timestamp group key in {:?}",
-                flow_id, analysis.group_key_names
-            );
-            None
-        }
-        _ => {
-            debug!(
-                "Flow {} cannot infer sink dirty-window filter column: ambiguous timestamp group keys {:?}",
-                flow_id, candidates
-            );
-            None
-        }
-    }
-}
-
-#[cfg(test)]
-mod test {
-    use datatypes::prelude::ConcreteDataType;
-    use datatypes::schema::ColumnSchema;
-    use pretty_assertions::assert_eq;
-
-    use super::*;
-    use crate::adapter::AUTO_CREATED_UPDATE_AT_TS_COL;
-    use crate::batching_mode::state::FilterExprInfo;
-    use crate::batching_mode::utils::IncrementalAggregateAnalysis;
-
-    fn test_analysis_with_group_keys(group_key_names: Vec<&str>) -> IncrementalAggregateAnalysis {
-        IncrementalAggregateAnalysis {
-            group_key_names: group_key_names
-                .into_iter()
-                .map(|name| name.to_string())
-                .collect(),
-            merge_columns: vec![],
-            literal_columns: vec![],
-            output_field_names: vec![],
-            unsupported_exprs: vec![],
-        }
-    }
-
-    fn test_dirty_filter(col_name: &str) -> FilterExprInfo {
-        FilterExprInfo {
-            expr: datafusion_expr::col(col_name),
-            col_name: col_name.to_string(),
-            time_ranges: vec![],
-            window_size: chrono::Duration::seconds(1),
-        }
-    }
-
-    fn test_sink_schema(columns: Vec<(&str, ConcreteDataType)>) -> Schema {
-        Schema::new(
-            columns
-                .into_iter()
-                .map(|(name, data_type)| ColumnSchema::new(name, data_type, true))
-                .collect(),
-        )
-    }
-
-    #[test]
-    fn test_infer_sink_time_window_filter_col_uses_matching_source_group_key() {
-        let analysis = test_analysis_with_group_keys(vec!["ts", "host"]);
-        let sink_schema = test_sink_schema(vec![
-            ("ts", ConcreteDataType::timestamp_millisecond_datatype()),
-            ("host", ConcreteDataType::string_datatype()),
-        ]);
-        let dirty_filter = test_dirty_filter("ts");
-
-        assert_eq!(
-            Some("ts".to_string()),
-            infer_sink_time_window_filter_col(1, &analysis, &sink_schema, &dirty_filter)
-        );
-    }
-
-    #[test]
-    fn test_infer_sink_time_window_filter_col_uses_unique_timestamp_group_key() {
-        let analysis = test_analysis_with_group_keys(vec!["host", "time_window"]);
-        let sink_schema = test_sink_schema(vec![
-            ("host", ConcreteDataType::string_datatype()),
-            (
-                "time_window",
-                ConcreteDataType::timestamp_millisecond_datatype(),
-            ),
-            (
-                AUTO_CREATED_UPDATE_AT_TS_COL,
-                ConcreteDataType::timestamp_millisecond_datatype(),
-            ),
-        ]);
-        let dirty_filter = test_dirty_filter("ts");
-
-        assert_eq!(
-            Some("time_window".to_string()),
-            infer_sink_time_window_filter_col(1, &analysis, &sink_schema, &dirty_filter)
-        );
-    }
-
-    #[test]
-    fn test_infer_sink_time_window_filter_col_skips_global_aggregate() {
-        let analysis = test_analysis_with_group_keys(vec![]);
-        let sink_schema = test_sink_schema(vec![
-            ("number", ConcreteDataType::uint32_datatype()),
-            (
-                "time_window",
-                ConcreteDataType::timestamp_millisecond_datatype(),
-            ),
-        ]);
-        let dirty_filter = test_dirty_filter("ts");
-
-        assert_eq!(
-            None,
-            infer_sink_time_window_filter_col(1, &analysis, &sink_schema, &dirty_filter)
-        );
-    }
-
-    #[test]
-    fn test_infer_sink_time_window_filter_col_skips_without_timestamp_group_key() {
-        let analysis = test_analysis_with_group_keys(vec!["host", "device"]);
-        let sink_schema = test_sink_schema(vec![
-            ("host", ConcreteDataType::string_datatype()),
-            ("device", ConcreteDataType::string_datatype()),
-            (
-                AUTO_CREATED_UPDATE_AT_TS_COL,
-                ConcreteDataType::timestamp_millisecond_datatype(),
-            ),
-        ]);
-        let dirty_filter = test_dirty_filter("ts");
-
-        assert_eq!(
-            None,
-            infer_sink_time_window_filter_col(1, &analysis, &sink_schema, &dirty_filter)
-        );
-    }
-
-    #[test]
-    fn test_infer_sink_time_window_filter_col_skips_ambiguous_timestamp_group_keys() {
-        let analysis = test_analysis_with_group_keys(vec!["ts", "time_window"]);
-        let sink_schema = test_sink_schema(vec![
-            ("ts", ConcreteDataType::timestamp_millisecond_datatype()),
-            (
-                "time_window",
-                ConcreteDataType::timestamp_millisecond_datatype(),
-            ),
-        ]);
-        let dirty_filter = test_dirty_filter("source_ts");
-
-        assert_eq!(
-            None,
-            infer_sink_time_window_filter_col(1, &analysis, &sink_schema, &dirty_filter)
-        );
-    }
-}

src/flow/src/batching_mode/state.rs

@@ -66,12 +66,20 @@ pub struct TaskState {
 }
 impl TaskState {
     pub fn new(query_ctx: QueryContextRef, shutdown_rx: oneshot::Receiver<()>) -> Self {
+        Self::with_dirty_time_windows(query_ctx, shutdown_rx, DirtyTimeWindows::default())
+    }
+
+    pub fn with_dirty_time_windows(
+        query_ctx: QueryContextRef,
+        shutdown_rx: oneshot::Receiver<()>,
+        dirty_time_windows: DirtyTimeWindows,
+    ) -> Self {
         Self {
             query_ctx,
             last_update_time: Instant::now(),
             last_query_duration: Duration::from_secs(0),
             last_exec_time_millis: None,
-            dirty_time_windows: Default::default(),
+            dirty_time_windows,
             checkpoint_mode: CheckpointMode::FullSnapshot,
             checkpoints: Default::default(),
             incremental_disabled: false,
@@ -264,6 +272,16 @@ impl DirtyTimeWindows {
             time_window_merge_threshold,
         }
     }
+
+    #[cfg(test)]
+    pub(crate) fn max_filter_num_per_query(&self) -> usize {
+        self.max_filter_num_per_query
+    }
+
+    #[cfg(test)]
+    pub(crate) fn time_window_merge_threshold(&self) -> usize {
+        self.time_window_merge_threshold
+    }
 }
 
 impl Default for DirtyTimeWindows {
@@ -681,7 +699,7 @@ impl DirtyTimeWindows {
     }
 }
 
-fn to_df_literal(value: Timestamp) -> Result<datafusion_common::ScalarValue, Error> {
+pub(crate) fn to_df_literal(value: Timestamp) -> Result<datafusion_common::ScalarValue, Error> {
     let value = Value::from(value);
     let value = value
         .try_to_scalar_value(&value.data_type())

src/flow/src/batching_mode/task.rs

@@ -27,7 +27,7 @@ use datafusion::datasource::DefaultTableSource;
 use datafusion::sql::unparser::expr_to_sql;
 use datafusion_common::DFSchemaRef;
 use datafusion_common::tree_node::{Transformed, TreeNode};
-use datafusion_expr::{DmlStatement, LogicalPlan, WriteOp};
+use datafusion_expr::{DmlStatement, LogicalPlan, WriteOp, col, lit};
 use datatypes::schema::Schema;
 use query::QueryEngineRef;
 use query::options::FLOW_INCREMENTAL_MODE;
@@ -38,14 +38,16 @@ use sql::parsers::utils::is_tql;
 use store_api::mito_engine_options::MERGE_MODE_KEY;
 use substrait::{DFLogicalSubstraitConvertor, SubstraitPlan};
 use table::table::adapter::DfTableProviderAdapter;
-use tokio::sync::oneshot;
 use tokio::sync::oneshot::error::TryRecvError;
+use tokio::sync::{Mutex, oneshot};
 use tokio::time::Instant;
 
 use crate::batching_mode::BatchingModeOptions;
 use crate::batching_mode::checkpoint::checkpoint_mode_label;
 use crate::batching_mode::frontend_client::{FrontendClient, PeerDesc};
-use crate::batching_mode::state::{CheckpointMode, DirtyTimeWindows, FilterExprInfo, TaskState};
+use crate::batching_mode::state::{
+    CheckpointMode, DirtyTimeWindows, FilterExprInfo, TaskState, to_df_literal,
+};
 use crate::batching_mode::table_creator::{QueryType, create_table_with_expr};
 use crate::batching_mode::time_window::TimeWindowExpr;
 use crate::batching_mode::utils::{
@@ -67,12 +69,6 @@ use crate::{Error, FlowId};
 mod ckpt;
 mod inc;
 
-/// Maximum number of dirty time-window predicates attached to one incremental
-/// SQL query. This keeps generated OR filters bounded so Substrait encoding and
-/// downstream planning remain predictable; if the backlog is larger, the flow
-/// drains one capped batch and postpones checkpoint advancement to a later run.
-const MAX_INCREMENTAL_DIRTY_WINDOW_FILTERS: usize = 4096;
-
 /// The task's config, immutable once created
 #[derive(Clone)]
 pub struct TaskConfig {
@@ -113,6 +109,10 @@ fn is_merge_mode_last_non_null(options: &HashMap<String, String>) -> bool {
 pub struct BatchingTask {
     pub config: Arc<TaskConfig>,
     pub state: Arc<RwLock<TaskState>>,
+    /// Serializes plan generation, execution, checkpoint advancement, and dirty
+    /// window restoration for this flow. Without this, a manual flush and the
+    /// background loop can process the same checkpoint range concurrently.
+    execution_lock: Arc<Mutex<()>>,
 }
 
 /// Arguments for creating batching task
@@ -150,6 +150,16 @@ pub enum DirtyRestore {
     Unscoped(DirtyTimeWindows),
 }
 
+struct ExecuteOnceOutcome {
+    new_query: Option<PlanInfo>,
+    /// Execution result of the generated insert plan.
+    ///
+    /// `Ok(Some((affected_rows, elapsed)))` means a query was executed.
+    /// `Ok(None)` means no query was generated because there was no dirty signal.
+    /// `Err(_)` means plan generation or execution failed.
+    result: Result<Option<(usize, Duration)>, Error>,
+}
+
 impl BatchingTask {
     #[allow(clippy::too_many_arguments)]
     pub fn try_new(
@@ -168,6 +178,18 @@ impl BatchingTask {
             flow_eval_interval,
         }: TaskArgs<'_>,
     ) -> Result<Self, Error> {
+        let mut state = TaskState::with_dirty_time_windows(
+            query_ctx.clone(),
+            shutdown_rx,
+            DirtyTimeWindows::new(
+                batch_opts.experimental_max_filter_num_per_query,
+                batch_opts.experimental_time_window_merge_threshold,
+            ),
+        );
+        if !batch_opts.experimental_enable_incremental_read {
+            state.disable_incremental();
+        }
+
         Ok(Self {
             config: Arc::new(TaskConfig {
                 flow_id,
@@ -182,7 +204,8 @@ impl BatchingTask {
                 batch_opts,
                 flow_eval_interval,
             }),
-            state: Arc::new(RwLock::new(TaskState::new(query_ctx, shutdown_rx))),
+            state: Arc::new(RwLock::new(state)),
+            execution_lock: Arc::new(Mutex::new(())),
         })
     }
 
@@ -242,6 +265,36 @@ impl BatchingTask {
         Ok(None)
     }
 
+    /// Validates that the sink table schema can accept this flow's output.
+    ///
+    /// This is a dry-run of the same schema matching logic used by runtime insert-plan
+    /// generation, but without adding dirty-window filters or executing the query. It is used
+    /// during CREATE FLOW to catch existing sink table mismatches early.
+    pub async fn validate_sink_table_schema(&self, engine: &QueryEngineRef) -> Result<(), Error> {
+        let (table, _) = get_table_info_df_schema(
+            self.config.catalog_manager.clone(),
+            self.config.sink_table_name.clone(),
+        )
+        .await?;
+
+        let table_meta = &table.table_info().meta;
+        let merge_mode_last_non_null =
+            is_merge_mode_last_non_null(&table_meta.options.extra_options);
+        let primary_key_indices = table_meta.primary_key_indices.clone();
+        let query_ctx = self.state.read().unwrap().query_ctx.clone();
+
+        gen_plan_with_matching_schema(
+            &self.config.query,
+            query_ctx,
+            engine.clone(),
+            table_meta.schema.clone(),
+            &primary_key_indices,
+            merge_mode_last_non_null,
+        )
+        .await
+        .map(|_| ())
+    }
+
     async fn is_table_exist(&self, table_name: &[String; 3]) -> Result<bool, Error> {
         self.config
             .catalog_manager
@@ -251,40 +304,75 @@ impl BatchingTask {
             .context(ExternalSnafu)
     }
 
-    pub async fn gen_exec_once(
+    pub(crate) async fn execute_once_serialized(
         &self,
         engine: &QueryEngineRef,
         frontend_client: &Arc<FrontendClient>,
         max_window_cnt: Option<usize>,
     ) -> Result<Option<(usize, Duration)>, Error> {
-        if let Some(new_query) = self.gen_insert_plan(engine, max_window_cnt).await? {
+        let outcome = self
+            .execute_once_serialized_with_outcome(engine, frontend_client, max_window_cnt)
+            .await;
+        outcome.result
+    }
+
+    /// Executes one flow evaluation under `execution_lock` and keeps the
+    /// generated query context for the background loop's error logging/backoff.
+    async fn execute_once_serialized_with_outcome(
+        &self,
+        engine: &QueryEngineRef,
+        frontend_client: &Arc<FrontendClient>,
+        max_window_cnt: Option<usize>,
+    ) -> ExecuteOnceOutcome {
+        let _execution_guard = self.execution_lock.lock().await;
+        self.execute_once_unlocked(engine, frontend_client, max_window_cnt)
+            .await
+    }
+
+    /// Executes one flow evaluation. Caller must hold `execution_lock`.
+    async fn execute_once_unlocked(
+        &self,
+        engine: &QueryEngineRef,
+        frontend_client: &Arc<FrontendClient>,
+        max_window_cnt: Option<usize>,
+    ) -> ExecuteOnceOutcome {
+        let new_query = match self.gen_insert_plan_unlocked(engine, max_window_cnt).await {
+            Ok(new_query) => new_query,
+            Err(err) => {
+                return ExecuteOnceOutcome {
+                    new_query: None,
+                    result: Err(err),
+                };
+            }
+        };
+
+        if let Some(new_query) = new_query {
             debug!("Generate new query: {}", new_query.plan);
-            let dirty_filter = match &new_query.dirty_restore {
-                DirtyRestore::Scoped(f) => Some(f),
-                _ => None,
-            };
-            match self
-                .execute_logical_plan(
+            let res = self
+                .execute_logical_plan_unlocked(
                     frontend_client,
                     &new_query.plan,
-                    dirty_filter,
                     new_query.can_advance_checkpoints,
                 )
-                .await
-            {
-                Ok(result) => Ok(result),
-                Err(err) => {
-                    self.handle_executed_query_failure(Some(&new_query));
-                    Err(err)
-                }
+                .await;
+            if res.is_err() {
+                self.handle_executed_query_failure(Some(&new_query));
+            }
+            ExecuteOnceOutcome {
+                new_query: Some(new_query),
+                result: res,
             }
         } else {
             debug!("Generate no query");
-            Ok(None)
+            ExecuteOnceOutcome {
+                new_query: None,
+                result: Ok(None),
+            }
         }
     }
 
-    pub async fn gen_insert_plan(
+    /// Generates the insert plan. Caller must reach this through the serialized path.
+    async fn gen_insert_plan_unlocked(
         &self,
         engine: &QueryEngineRef,
         max_window_cnt: Option<usize>,
@@ -388,11 +476,11 @@ impl BatchingTask {
         Ok(())
     }
 
-    pub async fn execute_logical_plan(
+    /// Executes the insert plan. Caller must reach this through the serialized path.
+    async fn execute_logical_plan_unlocked(
         &self,
         frontend_client: &Arc<FrontendClient>,
         plan: &LogicalPlan,
-        dirty_filter: Option<&FilterExprInfo>,
         can_advance_checkpoints: bool,
     ) -> Result<Option<(usize, Duration)>, Error> {
         let instant = Instant::now();
@@ -426,8 +514,7 @@ impl BatchingTask {
         // For incremental-mode SQL queries, attempt to rewrite the delta aggregate
         // plan into a safe delta-LEFT-JOIN-sink form before deciding on extensions.
         let incremental_plan = if can_advance_checkpoints {
-            self.prepare_plan_for_incremental(&plan, dirty_filter)
-                .await?
+            self.prepare_plan_for_incremental(&plan).await?
         } else {
             None
         };
@@ -580,6 +667,112 @@ impl BatchingTask {
         })
     }
 
+    fn restore_unscoped_dirty_windows(&self, dirty_windows: &DirtyTimeWindows) {
+        self.state
+            .write()
+            .unwrap()
+            .dirty_time_windows
+            .add_dirty_windows(dirty_windows);
+    }
+
+    fn restore_unscoped_dirty_windows_on_err<T>(
+        &self,
+        dirty_windows: &DirtyTimeWindows,
+        result: Result<T, Error>,
+    ) -> Result<T, Error> {
+        result.inspect_err(|_| {
+            self.restore_unscoped_dirty_windows(dirty_windows);
+        })
+    }
+
+    fn drain_dirty_windows_signal(&self) -> (bool, DirtyTimeWindows) {
+        let mut state = self.state.write().unwrap();
+        let dirty_windows_to_restore = state.dirty_time_windows.clone();
+        let is_dirty = !dirty_windows_to_restore.is_empty();
+        state.dirty_time_windows.clean();
+        (is_dirty, dirty_windows_to_restore)
+    }
+
+    #[allow(clippy::too_many_arguments)]
+    async fn gen_unfiltered_plan_info(
+        &self,
+        engine: QueryEngineRef,
+        query_ctx: QueryContextRef,
+        sink_table_schema: Arc<Schema>,
+        primary_key_indices: &[usize],
+        allow_partial: bool,
+        dirty_windows_to_restore: DirtyTimeWindows,
+        retention_filter: Option<(&str, Timestamp, &'static str)>,
+    ) -> Result<PlanInfo, Error> {
+        let mut plan = self.restore_unscoped_dirty_windows_on_err(
+            &dirty_windows_to_restore,
+            gen_plan_with_matching_schema(
+                &self.config.query,
+                query_ctx,
+                engine,
+                sink_table_schema,
+                primary_key_indices,
+                allow_partial,
+            )
+            .await,
+        )?;
+
+        if let Some((col_name, lower_bound, context)) = retention_filter {
+            let lower = self.restore_unscoped_dirty_windows_on_err(
+                &dirty_windows_to_restore,
+                to_df_literal(lower_bound),
+            )?;
+            let retention_filter = col(col_name).gt_eq(lit(lower));
+            let mut add_filter = AddFilterRewriter::new(retention_filter);
+            plan = self.restore_unscoped_dirty_windows_on_err(
+                &dirty_windows_to_restore,
+                plan.clone()
+                    .rewrite(&mut add_filter)
+                    .with_context(|_| DatafusionSnafu {
+                        context: format!(
+                            "Failed to apply {context} expire_after filter to plan:\n {}\n",
+                            plan
+                        ),
+                    })
+                    .map(|rewrite| rewrite.data),
+            )?;
+        }
+
+        Ok(PlanInfo {
+            plan,
+            dirty_restore: DirtyRestore::Unscoped(dirty_windows_to_restore),
+            can_advance_checkpoints: true,
+        })
+    }
+
+    async fn gen_unfiltered_plan_info_if_dirty(
+        &self,
+        engine: QueryEngineRef,
+        query_ctx: QueryContextRef,
+        sink_table_schema: Arc<Schema>,
+        primary_key_indices: &[usize],
+        allow_partial: bool,
+        retention_filter: Option<(&str, Timestamp, &'static str)>,
+    ) -> Result<Option<PlanInfo>, Error> {
+        let (is_dirty, dirty_windows_to_restore) = self.drain_dirty_windows_signal();
+        if !is_dirty {
+            debug!("Flow id={:?}, no new data, not update", self.config.flow_id);
+            return Ok(None);
+        }
+
+        self.gen_unfiltered_plan_info(
+            engine,
+            query_ctx,
+            sink_table_schema,
+            primary_key_indices,
+            allow_partial,
+            dirty_windows_to_restore,
+            retention_filter,
+        )
+        .await
+        .map(Some)
+    }
+
     fn handle_executed_query_failure(&self, query: Option<&PlanInfo>) {
         if let Some(query) = query {
             self.restore_dirty_windows_after_failure(query);
@@ -626,33 +819,11 @@ impl BatchingTask {
 
             let min_refresh = self.config.batch_opts.experimental_min_refresh_duration;
 
-            let new_query = match self.gen_insert_plan(&engine, max_window_cnt).await {
-                Ok(new_query) => new_query,
-                Err(err) => {
-                    common_telemetry::error!(err; "Failed to generate query for flow={}", self.config.flow_id);
-                    // also sleep for a little while before try again to prevent flooding logs
-                    tokio::time::sleep(min_refresh).await;
-                    continue;
-                }
-            };
+            let outcome = self
+                .execute_once_serialized_with_outcome(&engine, &frontend_client, max_window_cnt)
+                .await;
 
-            let res = if let Some(new_query) = &new_query {
-                let dirty_filter = match &new_query.dirty_restore {
-                    DirtyRestore::Scoped(f) => Some(f),
-                    _ => None,
-                };
-                self.execute_logical_plan(
-                    &frontend_client,
-                    &new_query.plan,
-                    dirty_filter,
-                    new_query.can_advance_checkpoints,
-                )
-                .await
-            } else {
-                Ok(None)
-            };
-
-            match res {
+            match outcome.result {
                 // normal execute, sleep for some time before doing next query
                 Ok(Some(_)) => {
                     // can increase max_window_cnt to query more windows next time
@@ -703,11 +874,10 @@ impl BatchingTask {
                 }
                 // TODO(discord9): this error should have better place to go, but for now just print error, also more context is needed
                 Err(err) => {
-                    self.handle_executed_query_failure(new_query.as_ref());
                     METRIC_FLOW_BATCHING_ENGINE_ERROR_CNT
                         .with_label_values(&[&flow_id_str])
                         .inc();
-                    match new_query {
+                    match outcome.new_query {
                         Some(query) => {
                             common_telemetry::error!(err; "Failed to execute query for flow={} with query: {}", self.config.flow_id, query.plan);
                             // TODO(discord9): add some backoff here? half the query time window or what
@@ -743,6 +913,20 @@ impl BatchingTask {
         create_table_with_expr(&plan, &self.config.sink_table_name, &self.config.query_type)
     }
 
+    fn should_use_unfiltered_incremental_delta(&self) -> bool {
+        let state = self.state.read().unwrap();
+        state.checkpoint_mode() == CheckpointMode::Incremental
+            && !state.is_incremental_disabled()
+            && matches!(self.config.query_type, QueryType::Sql)
+    }
+
+    fn should_use_unfiltered_full_snapshot_seeding(&self) -> bool {
+        let state = self.state.read().unwrap();
+        state.checkpoint_mode() == CheckpointMode::FullSnapshot
+            && !state.is_incremental_disabled()
+            && matches!(self.config.query_type, QueryType::Sql)
+    }
+
     /// will merge and use the first ten time window in query
     async fn gen_query_with_time_window(
         &self,
@@ -775,7 +959,7 @@ impl BatchingTask {
         let (expire_lower_bound, expire_upper_bound) =
             match (expire_time_window_bound, &self.config.query_type) {
                 (Some((Some(l), Some(u))), QueryType::Sql) => (l, u),
-                (None, QueryType::Sql) => {
+                (None, QueryType::Sql) if self.config.flow_eval_interval.is_none() => {
                     // if it's sql query and no time window lower/upper bound is found, just return the original query(with auto columns)
                     // use sink_table_meta to add to query the `update_at` and `__ts_placeholder` column's value too for compatibility reason
                     debug!(
@@ -783,83 +967,36 @@ impl BatchingTask {
                         self.config.flow_id
                     );
                     // clean dirty time window too, this could be from create flow's check_execute
-                    let (is_dirty, dirty_windows_to_restore) = {
-                        let mut state = self.state.write().unwrap();
-                        let dirty_windows_to_restore = state.dirty_time_windows.clone();
-                        let is_dirty = !dirty_windows_to_restore.is_empty();
-                        state.dirty_time_windows.clean();
-                        (is_dirty, dirty_windows_to_restore)
-                    };
-
-                    if !is_dirty {
-                        // no dirty data, hence no need to update
-                        debug!("Flow id={:?}, no new data, not update", self.config.flow_id);
-                        return Ok(None);
-                    }
-
-                    let plan = match gen_plan_with_matching_schema(
-                        &self.config.query,
-                        query_ctx,
-                        engine,
-                        sink_table_schema.clone(),
-                        primary_key_indices,
-                        allow_partial,
-                    )
-                    .await
-                    {
-                        Ok(plan) => plan,
-                        Err(err) => {
-                            self.state
-                                .write()
-                                .unwrap()
-                                .dirty_time_windows
-                                .add_dirty_windows(&dirty_windows_to_restore);
-                            return Err(err);
-                        }
-                    };
-
-                    return Ok(Some(PlanInfo {
-                        plan,
-                        dirty_restore: DirtyRestore::Unscoped(dirty_windows_to_restore),
-                        can_advance_checkpoints: true,
-                    }));
+                    return self
+                        .gen_unfiltered_plan_info_if_dirty(
+                            engine,
+                            query_ctx,
+                            sink_table_schema.clone(),
+                            primary_key_indices,
+                            allow_partial,
+                            None,
+                        )
+                        .await;
                 }
                 _ => {
                     // Clean dirty windows for full-query/non-scoped paths,
-                    // such as TQL, that cannot use a time-window filter.
-                    let dirty_windows_to_restore = {
-                        let mut state = self.state.write().unwrap();
-                        let dirty_windows_to_restore = state.dirty_time_windows.clone();
-                        state.dirty_time_windows.clean();
-                        dirty_windows_to_restore
-                    };
+                    // such as TQL or evaluation-interval SQL without a recognized
+                    // time-window expression, that cannot use a time-window filter.
+                    let (_, dirty_windows_to_restore) = self.drain_dirty_windows_signal();
 
-                    let plan = match gen_plan_with_matching_schema(
-                        &self.config.query,
-                        query_ctx,
-                        engine,
-                        sink_table_schema.clone(),
-                        primary_key_indices,
-                        allow_partial,
-                    )
-                    .await
-                    {
-                        Ok(plan) => plan,
-                        Err(err) => {
-                            self.state
-                                .write()
-                                .unwrap()
-                                .dirty_time_windows
-                                .add_dirty_windows(&dirty_windows_to_restore);
-                            return Err(err);
-                        }
-                    };
+                    let plan_info = self
+                        .gen_unfiltered_plan_info(
+                            engine,
+                            query_ctx,
+                            sink_table_schema.clone(),
+                            primary_key_indices,
+                            allow_partial,
+                            dirty_windows_to_restore,
+                            None,
+                        )
+                        .await?;
 
-                    return Ok(Some(PlanInfo {
-                        plan,
-                        dirty_restore: DirtyRestore::Unscoped(dirty_windows_to_restore),
-                        can_advance_checkpoints: true,
-                    }));
+                    return Ok(Some(plan_info));
                 }
             };
 
@@ -889,22 +1026,61 @@ impl BatchingTask {
                 ),
             })?;
 
+        if self.should_use_unfiltered_full_snapshot_seeding() {
+            // A full-snapshot query that can seed/refresh incremental
+            // checkpoints must not use dirty-window predicates. Rows can be
+            // written after dirty windows are drained but before the source scan
+            // snapshot opens; a stale dirty-window filter could exclude those
+            // rows while the returned watermark includes them, causing the next
+            // incremental read to skip them forever. Execute an unfiltered full
+            // snapshot instead, and keep dirty windows only as the scheduling and
+            // failure-restoration signal.
+            let retention_filter = self
+                .config
+                .expire_after
+                .map(|_| (col_name.as_str(), expire_lower_bound, "full-snapshot"));
+            return self
+                .gen_unfiltered_plan_info_if_dirty(
+                    engine,
+                    query_ctx,
+                    sink_table_schema.clone(),
+                    primary_key_indices,
+                    allow_partial,
+                    retention_filter,
+                )
+                .await;
+        }
+
+        if self.should_use_unfiltered_incremental_delta() {
+            // In incremental mode, source correctness is defined by the
+            // per-region sequence range `(checkpoint, scan-open snapshot]`, not
+            // by dirty-window predicates. Dirty windows are only a scheduling
+            // signal here. Applying a stale dirty-window filter to the source can
+            // exclude rows that are inside the returned watermark and make a
+            // checkpoint advance skip them forever. The sink side is also left
+            // unfiltered by dirty windows; the incremental rewrite joins the
+            // delta groups with the full sink state for correctness. Future
+            // dynamic filters can prune sink reads as a pure optimization.
+            let retention_filter = self
+                .config
+                .expire_after
+                .map(|_| (col_name.as_str(), expire_lower_bound, "incremental"));
+            return self
+                .gen_unfiltered_plan_info_if_dirty(
+                    engine,
+                    query_ctx,
+                    sink_table_schema.clone(),
+                    primary_key_indices,
+                    allow_partial,
+                    retention_filter,
+                )
+                .await;
+        }
+
         let (expr, can_advance_checkpoints) = {
             let mut state = self.state.write().unwrap();
-            let window_cnt = if state.checkpoint_mode() == CheckpointMode::Incremental
-                && !state.is_incremental_disabled()
-                && matches!(self.config.query_type, QueryType::Sql)
-            {
-                // Incremental scans are bounded by region sequence checkpoints,
-                // so the dirty-window filter only narrows sink-side/time-window
-                // work. Drain more windows than normal, but keep a hard cap to
-                // avoid building a huge OR filter after a long downtime. If
-                // windows remain, checkpoints won't advance this round.
-                MAX_INCREMENTAL_DIRTY_WINDOW_FILTERS
-            } else {
-                max_window_cnt
-                    .unwrap_or(self.config.batch_opts.experimental_max_filter_num_per_query)
-            };
+            let window_cnt = max_window_cnt
+                .unwrap_or(self.config.batch_opts.experimental_max_filter_num_per_query);
             let expr = state.dirty_time_windows.gen_filter_exprs(
                 &col_name,
                 Some(expire_lower_bound),

src/flow/src/batching_mode/task/inc.rs

@@ -26,8 +26,7 @@ use snafu::ResultExt;
 use table::metadata::TableId;
 
 use crate::Error;
-use crate::batching_mode::incremental_filter::build_sink_dirty_time_window_filter_expr;
-use crate::batching_mode::state::{CheckpointMode, FilterExprInfo};
+use crate::batching_mode::state::CheckpointMode;
 use crate::batching_mode::table_creator::QueryType;
 use crate::batching_mode::task::BatchingTask;
 use crate::batching_mode::utils::{
@@ -74,7 +73,6 @@ impl BatchingTask {
     pub(super) async fn prepare_plan_for_incremental(
         &self,
         plan: &LogicalPlan,
-        dirty_filter: Option<&FilterExprInfo>,
     ) -> Result<Option<LogicalPlan>, Error> {
         let is_incremental_sql = {
             let state = self.state.read().unwrap();
@@ -152,31 +150,12 @@ impl BatchingTask {
                 return Ok(None);
             }
         };
-        let sink_schema = sink_table.table_info().meta.schema.clone();
-        let sink_dirty_filter = match build_sink_dirty_time_window_filter_expr(
-            self.config.flow_id,
-            &analysis,
-            &sink_schema,
-            dirty_filter,
-        ) {
-            Ok(filter) => filter,
-            Err(err) => {
-                warn!(
-                    "Flow {} failed to build sink dirty time window filter; \
-                     falling back to full snapshot for this round: {:?}",
-                    self.config.flow_id, err
-                );
-                self.state.write().unwrap().mark_full_snapshot();
-                return Ok(None);
-            }
-        };
-
         let rewritten_inner = match rewrite_incremental_aggregate_with_sink_merge(
             &inner_plan,
             &analysis,
             sink_table,
             &self.config.sink_table_name,
-            sink_dirty_filter,
+            None,
         )
         .await
         {

src/flow/src/batching_mode/task/test.rs

@@ -25,7 +25,9 @@ use datatypes::data_type::ConcreteDataType as CDT;
 use datatypes::schema::ColumnSchema;
 use datatypes::vectors::{TimestampMillisecondVector, UInt32Vector, VectorRef};
 use pretty_assertions::assert_eq;
-use query::options::{FLOW_INCREMENTAL_AFTER_SEQS, FLOW_INCREMENTAL_MODE_MEMTABLE_ONLY};
+use query::options::{
+    FLOW_INCREMENTAL_AFTER_SEQS, FLOW_INCREMENTAL_MODE_MEMTABLE_ONLY, QueryOptions,
+};
 use session::context::QueryContext;
 use table::test_util::MemTable;
 
@@ -38,6 +40,13 @@ use crate::batching_mode::state::CheckpointMode;
 use crate::batching_mode::time_window::find_time_window_expr;
 use crate::test_utils::create_test_query_engine;
 
+fn incremental_batch_opts() -> Arc<BatchingModeOptions> {
+    Arc::new(BatchingModeOptions {
+        experimental_enable_incremental_read: true,
+        ..Default::default()
+    })
+}
+
 async fn new_test_task_and_plan_with_missing_sink() -> (BatchingTask, LogicalPlan) {
     new_test_task_engine_and_plan_with_query(
         "SELECT number, ts FROM numbers_with_ts",
@@ -60,6 +69,15 @@ impl TestTaskParts {
 }
 
 async fn new_test_task_engine_and_plan_with_query(query: &str, sink_table: &str) -> TestTaskParts {
+    new_test_task_engine_and_plan_with_query_and_opts(query, sink_table, incremental_batch_opts())
+        .await
+}
+
+async fn new_test_task_engine_and_plan_with_query_and_opts(
+    query: &str,
+    sink_table: &str,
+    batch_opts: Arc<BatchingModeOptions>,
+) -> TestTaskParts {
     let query_engine = create_test_query_engine();
     let ctx = QueryContext::arc();
     let plan = sql_to_df_plan(
@@ -91,7 +109,7 @@ async fn new_test_task_engine_and_plan_with_query(query: &str, sink_table: &str)
         query_ctx: ctx,
         catalog_manager: query_engine.engine_state().catalog_manager().clone(),
         shutdown_rx: rx,
-        batch_opts: Arc::new(BatchingModeOptions::default()),
+        batch_opts,
         flow_eval_interval: None,
     })
     .unwrap();
@@ -103,6 +121,75 @@ async fn new_test_task_engine_and_plan_with_query(query: &str, sink_table: &str)
     }
 }
 
+#[tokio::test]
+async fn test_incremental_read_is_disabled_by_default() {
+    let task = new_test_task_engine_and_plan_with_query_and_opts(
+        "SELECT number, ts FROM numbers_with_ts",
+        "numbers_with_ts",
+        Arc::new(BatchingModeOptions::default()),
+    )
+    .await
+    .task;
+
+    assert!(task.state.read().unwrap().is_incremental_disabled());
+}
+
+#[tokio::test]
+async fn test_dirty_time_windows_uses_batch_opts() {
+    let task = new_test_task_engine_and_plan_with_query_and_opts(
+        "SELECT number, ts FROM numbers_with_ts",
+        "numbers_with_ts",
+        Arc::new(BatchingModeOptions {
+            experimental_max_filter_num_per_query: 7,
+            experimental_time_window_merge_threshold: 11,
+            ..Default::default()
+        }),
+    )
+    .await
+    .task;
+
+    let state = task.state.read().unwrap();
+    assert_eq!(7, state.dirty_time_windows.max_filter_num_per_query());
+    assert_eq!(11, state.dirty_time_windows.time_window_merge_threshold());
+}
+
+#[tokio::test]
+async fn test_execute_once_serialized_waits_for_execution_lock() {
+    let TestTaskParts {
+        task, query_engine, ..
+    } = new_test_task_engine_and_plan_with_query(
+        "SELECT number, ts FROM numbers_with_ts",
+        "missing_sink",
+    )
+    .await;
+    let (frontend_client, _handler) =
+        FrontendClient::from_empty_grpc_handler(QueryOptions::default());
+    let frontend_client = Arc::new(frontend_client);
+
+    let guard = task.execution_lock.clone().lock_owned().await;
+    let task_to_run = task.clone();
+    let query_engine_to_run = query_engine.clone();
+    let frontend_client_to_run = frontend_client.clone();
+    let exec = tokio::spawn(async move {
+        task_to_run
+            .execute_once_serialized(&query_engine_to_run, &frontend_client_to_run, None)
+            .await
+    });
+
+    tokio::time::sleep(Duration::from_millis(20)).await;
+    assert!(
+        !exec.is_finished(),
+        "execute_once_serialized should wait for execution_lock"
+    );
+
+    drop(guard);
+    tokio::time::timeout(Duration::from_secs(1), exec)
+        .await
+        .expect("execute_once_serialized should finish once execution_lock is released")
+        .expect("execute_once_serialized task should not panic")
+        .expect_err("missing sink should fail after acquiring execution_lock");
+}
+
 async fn new_time_window_test_task_with_query(query: &str) -> TestTaskParts {
     let query_engine = create_test_query_engine();
     let ctx = QueryContext::arc();
@@ -147,7 +234,7 @@ async fn new_time_window_test_task_with_query(query: &str) -> TestTaskParts {
         query_ctx: ctx,
         catalog_manager: query_engine.engine_state().catalog_manager().clone(),
         shutdown_rx: rx,
-        batch_opts: Arc::new(BatchingModeOptions::default()),
+        batch_opts: incremental_batch_opts(),
         flow_eval_interval: None,
     })
     .unwrap();
@@ -226,6 +313,14 @@ fn dirty_range(start: i64, end: i64) -> DirtyTimeWindows {
     dirty
 }
 
+fn expire_after_for_retention_filter_test() -> i64 {
+    let now_secs = SystemTime::now()
+        .duration_since(UNIX_EPOCH)
+        .expect("Time went backwards")
+        .as_secs();
+    (now_secs - 10) as i64
+}
+
 async fn assert_unscoped_failure_restore(
     consumed_dirty_windows: DirtyTimeWindows,
     current_dirty_windows: DirtyTimeWindows,
@@ -626,6 +721,7 @@ async fn test_full_snapshot_scoped_plan_marks_checkpoint_advance_safe_only_after
     .await;
     {
         let mut state = task.state.write().unwrap();
+        state.disable_incremental();
         state
             .dirty_time_windows
             .add_window(Timestamp::new_second(0), Some(Timestamp::new_second(5)));
@@ -657,7 +753,7 @@ async fn test_full_snapshot_scoped_plan_marks_checkpoint_advance_safe_only_after
 }
 
 #[tokio::test]
-async fn test_incremental_scoped_plan_consumes_all_dirty_windows_for_checkpoint_safety() {
+async fn test_incremental_plan_consumes_dirty_signal_for_checkpoint_safety() {
     let TestTaskParts {
         task,
         query_engine,
@@ -692,6 +788,224 @@ async fn test_incremental_scoped_plan_consumes_all_dirty_windows_for_checkpoint_
     assert!(task.state.read().unwrap().dirty_time_windows.is_empty());
 }
 
+#[tokio::test]
+async fn test_full_snapshot_seeding_for_incremental_does_not_add_dirty_window_filter() {
+    let TestTaskParts {
+        task,
+        query_engine,
+        ..
+    } = new_time_window_test_task_with_query(
+        "SELECT max(number) AS number, date_bin(INTERVAL '5 second', ts) AS time_window FROM numbers_with_ts GROUP BY time_window",
+    )
+    .await;
+    {
+        let mut state = task.state.write().unwrap();
+        assert_eq!(state.checkpoint_mode(), CheckpointMode::FullSnapshot);
+        assert!(!state.is_incremental_disabled());
+        state
+            .dirty_time_windows
+            .add_window(Timestamp::new_second(0), Some(Timestamp::new_second(5)));
+        state
+            .dirty_time_windows
+            .add_window(Timestamp::new_second(30), Some(Timestamp::new_second(35)));
+    }
+    let sink_schema = Arc::new(Schema::new(vec![
+        ColumnSchema::new("number", CDT::uint32_datatype(), false),
+        ColumnSchema::new("time_window", CDT::timestamp_millisecond_datatype(), false)
+            .with_time_index(true),
+    ]));
+
+    let plan = task
+        .gen_query_with_time_window(query_engine, &sink_schema, &[], false, Some(1))
+        .await
+        .unwrap()
+        .unwrap();
+
+    let plan_text = plan.plan.to_string();
+    assert!(plan.can_advance_checkpoints);
+    assert!(task.state.read().unwrap().dirty_time_windows.is_empty());
+    assert!(!plan_text.contains("Filter:"), "{plan_text}");
+}
+
+#[tokio::test]
+async fn test_full_snapshot_seeding_applies_expire_after_retention_filter() {
+    let TestTaskParts {
+        mut task,
+        query_engine,
+        ..
+    } = new_time_window_test_task_with_query(
+        "SELECT max(number) AS number, date_bin(INTERVAL '5 second', ts) AS time_window FROM numbers_with_ts GROUP BY time_window",
+    )
+    .await;
+    {
+        let mut state = task.state.write().unwrap();
+        assert_eq!(state.checkpoint_mode(), CheckpointMode::FullSnapshot);
+        assert!(!state.is_incremental_disabled());
+        state
+            .dirty_time_windows
+            .add_window(Timestamp::new_second(0), Some(Timestamp::new_second(5)));
+    }
+    let sink_schema = Arc::new(Schema::new(vec![
+        ColumnSchema::new("number", CDT::uint32_datatype(), false),
+        ColumnSchema::new("time_window", CDT::timestamp_millisecond_datatype(), false)
+            .with_time_index(true),
+    ]));
+
+    Arc::get_mut(&mut task.config)
+        .expect("test task config should be uniquely owned")
+        .expire_after = Some(expire_after_for_retention_filter_test());
+    let plan = task
+        .gen_query_with_time_window(query_engine, &sink_schema, &[], false, Some(1))
+        .await
+        .unwrap()
+        .unwrap();
+
+    assert!(plan.can_advance_checkpoints);
+    assert!(task.state.read().unwrap().dirty_time_windows.is_empty());
+    let plan_text = plan.plan.to_string();
+    assert!(
+        plan_text.contains("Filter: ts >= TimestampMillisecond("),
+        "{plan_text}"
+    );
+}
+
+#[tokio::test]
+async fn test_incremental_plan_does_not_add_dirty_window_filter() {
+    let TestTaskParts {
+        task,
+        query_engine,
+        ..
+    } = new_time_window_test_task_with_query(
+        "SELECT max(number) AS number, date_bin(INTERVAL '5 second', ts) AS time_window FROM numbers_with_ts GROUP BY time_window",
+    )
+    .await;
+    {
+        let mut state = task.state.write().unwrap();
+        state.advance_checkpoints(HashMap::from([(1_u64, 10_u64)]));
+        state
+            .dirty_time_windows
+            .add_window(Timestamp::new_second(0), Some(Timestamp::new_second(5)));
+    }
+    let sink_schema = Arc::new(Schema::new(vec![
+        ColumnSchema::new("number", CDT::uint32_datatype(), false),
+        ColumnSchema::new("time_window", CDT::timestamp_millisecond_datatype(), false)
+            .with_time_index(true),
+    ]));
+
+    let plan = task
+        .gen_query_with_time_window(query_engine, &sink_schema, &[], false, Some(1))
+        .await
+        .unwrap()
+        .unwrap();
+
+    let plan_text = plan.plan.to_string();
+    assert!(plan.can_advance_checkpoints);
+    assert!(!plan_text.contains("Filter:"), "{plan_text}");
+}
+
+#[tokio::test]
+async fn test_incremental_delta_applies_expire_after_retention_filter() {
+    let TestTaskParts {
+        mut task,
+        query_engine,
+        ..
+    } = new_time_window_test_task_with_query(
+        "SELECT max(number) AS number, date_bin(INTERVAL '5 second', ts) AS time_window FROM numbers_with_ts GROUP BY time_window",
+    )
+    .await;
+    {
+        let mut state = task.state.write().unwrap();
+        state.advance_checkpoints(HashMap::from([(1_u64, 10_u64)]));
+        state
+            .dirty_time_windows
+            .add_window(Timestamp::new_second(0), Some(Timestamp::new_second(5)));
+    }
+    let sink_schema = Arc::new(Schema::new(vec![
+        ColumnSchema::new("number", CDT::uint32_datatype(), false),
+        ColumnSchema::new("time_window", CDT::timestamp_millisecond_datatype(), false)
+            .with_time_index(true),
+    ]));
+
+    Arc::get_mut(&mut task.config)
+        .expect("test task config should be uniquely owned")
+        .expire_after = Some(expire_after_for_retention_filter_test());
+    let plan = task
+        .gen_query_with_time_window(query_engine, &sink_schema, &[], false, Some(1))
+        .await
+        .unwrap()
+        .unwrap();
+
+    assert!(plan.can_advance_checkpoints);
+    assert!(task.state.read().unwrap().dirty_time_windows.is_empty());
+    let plan_text = plan.plan.to_string();
+    assert!(
+        plan_text.contains("Filter: ts >= TimestampMillisecond("),
+        "{plan_text}"
+    );
+}
+
+#[tokio::test]
+async fn test_non_scoped_path_generates_plan_with_empty_dirty_signal() {
+    let TestTaskParts {
+        mut task,
+        query_engine,
+        ..
+    } = new_test_task_engine_and_plan_with_query(
+        "SELECT number, ts FROM numbers_with_ts",
+        "missing_sink",
+    )
+    .await;
+    Arc::get_mut(&mut task.config)
+        .expect("test task config should be uniquely owned")
+        .query_type = QueryType::Tql;
+    task.state.write().unwrap().dirty_time_windows.clean();
+    let sink_schema = Arc::new(Schema::new(vec![
+        ColumnSchema::new("number", CDT::uint32_datatype(), false),
+        ColumnSchema::new("ts", CDT::timestamp_millisecond_datatype(), false).with_time_index(true),
+    ]));
+
+    let plan = task
+        .gen_query_with_time_window(query_engine, &sink_schema, &[], false, None)
+        .await
+        .unwrap()
+        .expect("non-scoped path should generate a plan even with an empty dirty signal");
+
+    assert!(plan.can_advance_checkpoints);
+    assert!(task.state.read().unwrap().dirty_time_windows.is_empty());
+}
+
+#[tokio::test]
+async fn test_no_time_window_sql_with_eval_interval_generates_plan_without_dirty_signal() {
+    let TestTaskParts {
+        mut task,
+        query_engine,
+        ..
+    } = new_test_task_engine_and_plan_with_query(
+        "SELECT number, ts FROM numbers_with_ts",
+        "missing_sink",
+    )
+    .await;
+    Arc::get_mut(&mut task.config)
+        .expect("test task config should be uniquely owned")
+        .flow_eval_interval = Some(Duration::from_secs(60));
+    task.state.write().unwrap().dirty_time_windows.clean();
+    let sink_schema = Arc::new(Schema::new(vec![
+        ColumnSchema::new("number", CDT::uint32_datatype(), false),
+        ColumnSchema::new("ts", CDT::timestamp_millisecond_datatype(), false).with_time_index(true),
+    ]));
+
+    let plan = task
+        .gen_query_with_time_window(query_engine, &sink_schema, &[], false, None)
+        .await
+        .unwrap()
+        .expect(
+            "eval-interval SQL without a time-window expr should run by interval, not dirty signal",
+        );
+
+    assert!(plan.can_advance_checkpoints);
+    assert!(task.state.read().unwrap().dirty_time_windows.is_empty());
+}
+
 #[tokio::test]
 async fn test_executed_query_failure_restores_scoped_dirty_windows_for_flush_path() {
     let (task, plan) = new_test_task_and_plan_with_missing_sink().await;
@@ -773,7 +1087,7 @@ async fn test_prepare_plan_for_incremental_disables_on_non_aggregate() {
         query_ctx: ctx,
         catalog_manager: query_engine.engine_state().catalog_manager().clone(),
         shutdown_rx: rx,
-        batch_opts: Arc::new(BatchingModeOptions::default()),
+        batch_opts: incremental_batch_opts(),
         flow_eval_interval: None,
     })
     .unwrap();
@@ -788,10 +1102,7 @@ async fn test_prepare_plan_for_incremental_disables_on_non_aggregate() {
         CheckpointMode::Incremental
     );
 
-    let incremental_plan = task
-        .prepare_plan_for_incremental(&dml_plan, None)
-        .await
-        .unwrap();
+    let incremental_plan = task.prepare_plan_for_incremental(&dml_plan).await.unwrap();
     assert!(incremental_plan.is_none());
     let state = task.state.read().unwrap();
     assert!(state.is_incremental_disabled());
@@ -852,7 +1163,7 @@ async fn test_prepare_plan_for_incremental_falls_back_without_disable_on_rewrite
         query_ctx: ctx,
         catalog_manager: query_engine.engine_state().catalog_manager().clone(),
         shutdown_rx: rx,
-        batch_opts: Arc::new(BatchingModeOptions::default()),
+        batch_opts: incremental_batch_opts(),
         flow_eval_interval: None,
     })
     .unwrap();
@@ -866,10 +1177,7 @@ async fn test_prepare_plan_for_incremental_falls_back_without_disable_on_rewrite
         CheckpointMode::Incremental
     );
 
-    let incremental_plan = task
-        .prepare_plan_for_incremental(&dml_plan, None)
-        .await
-        .unwrap();
+    let incremental_plan = task.prepare_plan_for_incremental(&dml_plan).await.unwrap();
     assert!(incremental_plan.is_none());
     let state = task.state.read().unwrap();
     assert!(!state.is_incremental_disabled());
@@ -928,7 +1236,7 @@ async fn test_prepare_plan_for_incremental_group_by_without_merge_columns_uses_o
         query_ctx: ctx,
         catalog_manager: query_engine.engine_state().catalog_manager().clone(),
         shutdown_rx: rx,
-        batch_opts: Arc::new(BatchingModeOptions::default()),
+        batch_opts: incremental_batch_opts(),
         flow_eval_interval: None,
     })
     .unwrap();
@@ -939,7 +1247,7 @@ async fn test_prepare_plan_for_incremental_group_by_without_merge_columns_uses_o
         .advance_checkpoints(HashMap::from([(1_u64, 10_u64)]));
 
     let incremental_plan = task
-        .prepare_plan_for_incremental(&dml_plan, None)
+        .prepare_plan_for_incremental(&dml_plan)
         .await
         .unwrap()
         .expect("plain GROUP BY is incremental-safe without a rewrite");
@@ -962,7 +1270,7 @@ async fn test_auto_created_sql_aggregate_sink_reaches_incremental_safe() {
     task.state.write().unwrap().dirty_time_windows.set_dirty();
 
     let plan_info = task
-        .gen_insert_plan(&query_engine, None)
+        .gen_insert_plan_unlocked(&query_engine, None)
         .await
         .unwrap()
         .unwrap();
@@ -973,7 +1281,7 @@ async fn test_auto_created_sql_aggregate_sink_reaches_incremental_safe() {
         .unwrap()
         .advance_checkpoints(HashMap::from([(1_u64, 10_u64)]));
     let incremental_plan = task
-        .prepare_plan_for_incremental(&plan_info.plan, None)
+        .prepare_plan_for_incremental(&plan_info.plan)
         .await
         .unwrap();
     let incremental_safe = incremental_plan.is_some();
@@ -1078,11 +1386,11 @@ async fn test_insert_plan_matching_failure_restores_consumed_dirty_marker() {
     register_number_only_sink(&query_engine, sink_table);
     task.state.write().unwrap().dirty_time_windows.set_dirty();
 
-    let result = task.gen_insert_plan(&query_engine, None).await;
+    let result = task.gen_insert_plan_unlocked(&query_engine, None).await;
 
     assert!(result.is_err());
     let _err = match result {
-        Ok(_) => panic!("gen_insert_plan should fail with a sink column mismatch"),
+        Ok(_) => panic!("gen_insert_plan_unlocked should fail with a sink column mismatch"),
         Err(err) => err,
     };
     let state = task.state.read().unwrap();

src/flow/src/batching_mode/utils.rs

@@ -33,9 +33,10 @@ use datafusion_common::{
 };
 use datafusion_expr::logical_plan::{Aggregate, TableScan};
 use datafusion_expr::{
-    Distinct, JoinType, LogicalPlan, LogicalPlanBuilder, Operator, Projection, and, binary_expr,
-    bitwise_and, bitwise_or, bitwise_xor, is_null, or, when,
+    Distinct, ExprSchemable, JoinType, LogicalPlan, LogicalPlanBuilder, Operator, Projection, and,
+    binary_expr, bitwise_and, bitwise_or, bitwise_xor, is_null, or, when,
 };
+use datatypes::prelude::ConcreteDataType;
 use datatypes::schema::{ColumnSchema, SchemaRef};
 use query::QueryEngineRef;
 use query::parser::{DEFAULT_LOOKBACK_STRING, PromQuery, QueryLanguageParser, QueryStatement};
@@ -955,7 +956,7 @@ pub(crate) async fn gen_plan_with_matching_schema(
         .clone()
         .rewrite(&mut add_auto_column)
         .with_context(|_| DatafusionSnafu {
-            context: format!("Failed to rewrite plan:\n {}\n", plan),
+            context: "Failed to rewrite plan".to_string(),
         })?
         .data;
     Ok(plan)
@@ -1090,33 +1091,23 @@ impl ColumnMatcherRewriter {
     }
 
     /// modify the exprs in place so that it matches the schema and some auto columns are added
-    fn modify_project_exprs(&mut self, mut exprs: Vec<Expr>) -> DfResult<Vec<Expr>> {
+    fn modify_project_exprs(
+        &mut self,
+        mut exprs: Vec<Expr>,
+        input_schema: &DFSchema,
+    ) -> DfResult<Vec<Expr>> {
         if self.allow_partial {
             return self.modify_project_exprs_with_partial(exprs);
         }
 
+        let original_exprs = exprs.clone();
+
         let all_names = self
             .schema
             .column_schemas()
             .iter()
             .map(|c| c.name.clone())
             .collect::<BTreeSet<_>>();
-        // first match by position
-        for (idx, expr) in exprs.iter_mut().enumerate() {
-            if !all_names.contains(&expr.qualified_name().1)
-                && let Some(col_name) = self
-                    .schema
-                    .column_schemas()
-                    .get(idx)
-                    .map(|c| c.name.clone())
-            {
-                // if the data type mismatched, later check_execute will error out
-                // hence no need to check it here, beside, optimize pass might be able to cast it
-                // so checking here is not necessary
-                *expr = expr.clone().alias(col_name);
-            }
-        }
-
         // add columns if have different column count
         let query_col_cnt = exprs.len();
         let table_col_cnt = self.schema.column_schemas().len();
@@ -1140,10 +1131,9 @@ impl ColumnMatcherRewriter {
                 // is the update at column
                 exprs.push(datafusion::prelude::now().alias(&last_col_schema.name));
             } else {
-                // helpful error message
-                return Err(DataFusionError::Plan(format!(
-                    "Expect the last column in table to be timestamp column, found column {} with type {:?}",
-                    last_col_schema.name, last_col_schema.data_type
+                return Err(DataFusionError::Plan(format_flow_sink_schema_mismatch(
+                    &original_exprs,
+                    self.schema.as_ref(),
                 )));
             }
         } else if query_col_cnt + 2 == table_col_cnt {
@@ -1170,14 +1160,110 @@ impl ColumnMatcherRewriter {
                 )));
             }
         } else {
-            return Err(DataFusionError::Plan(format!(
-                "Expect table have 0,1 or 2 columns more than query columns, found {} query columns {:?}, {} table columns {:?}",
-                query_col_cnt,
-                exprs,
-                table_col_cnt,
-                self.schema.column_schemas()
+            return Err(DataFusionError::Plan(format_flow_sink_schema_mismatch(
+                &original_exprs,
+                self.schema.as_ref(),
             )));
         }
+
+        self.match_extra_output_columns(exprs, input_schema, &original_exprs, &all_names)
+    }
+
+    /// Match flow output columns whose names are not in the sink schema by the same position only.
+    ///
+    /// This keeps the legacy "omit output aliases and map by position" behavior, but only when the
+    /// sink column at the same index is actually missing from the flow output. If the extra output
+    /// would be aliased to a sink column that already exists elsewhere, report a schema mismatch
+    /// instead of guessing another sink column by type.
+    ///
+    /// In particular, this intentionally rejects cross-position remaps like
+    /// `record_time_window2 -> record_time_window`: they are easy to confuse with real schema
+    /// mismatches and should be fixed by giving the flow output the sink column name explicitly.
+    fn match_extra_output_columns(
+        &self,
+        mut exprs: Vec<Expr>,
+        input_schema: &DFSchema,
+        original_exprs: &[Expr],
+        all_names: &BTreeSet<String>,
+    ) -> DfResult<Vec<Expr>> {
+        let mut output_names = exprs
+            .iter()
+            .map(|expr| expr.qualified_name().1)
+            .collect::<Vec<_>>();
+        let output_name_set = output_names.iter().cloned().collect::<BTreeSet<_>>();
+        let extra_expr_indices = output_names
+            .iter()
+            .enumerate()
+            .filter_map(|(idx, name)| (!all_names.contains(name)).then_some(idx))
+            .collect::<Vec<_>>();
+        let missing_sink_indices = self
+            .schema
+            .column_schemas()
+            .iter()
+            .enumerate()
+            .filter_map(|(idx, column)| (!output_name_set.contains(&column.name)).then_some(idx))
+            .collect::<Vec<_>>();
+
+        if extra_expr_indices.is_empty() && missing_sink_indices.is_empty() {
+            return Ok(exprs);
+        }
+
+        if extra_expr_indices.len() != missing_sink_indices.len() {
+            return Err(DataFusionError::Plan(format_flow_sink_schema_mismatch(
+                original_exprs,
+                self.schema.as_ref(),
+            )));
+        }
+
+        let mut positional_matches = Vec::new();
+        for expr_idx in extra_expr_indices {
+            if !missing_sink_indices.contains(&expr_idx) {
+                return Err(DataFusionError::Plan(format_flow_sink_schema_mismatch(
+                    original_exprs,
+                    self.schema.as_ref(),
+                )));
+            }
+
+            let target_col_schema = &self.schema.column_schemas()[expr_idx];
+            let expr_type =
+                ConcreteDataType::from_arrow_type(&exprs[expr_idx].get_type(input_schema)?);
+            if is_obviously_incompatible_positional_match(&expr_type, &target_col_schema.data_type)
+            {
+                return Err(DataFusionError::Plan(format!(
+                    "Cannot match flow output column '{}' to sink column '{}' by position: incompatible data types, flow output type is {:?}, sink column type is {:?}. {}",
+                    output_names[expr_idx],
+                    target_col_schema.name,
+                    expr_type,
+                    target_col_schema.data_type,
+                    format_flow_sink_schema_mismatch(original_exprs, self.schema.as_ref())
+                )));
+            }
+
+            let target_name = target_col_schema.name.clone();
+            positional_matches.push(format!(
+                "{} -> {} (flow output type: {:?}, sink column type: {:?})",
+                output_names[expr_idx], target_name, expr_type, target_col_schema.data_type
+            ));
+            exprs[expr_idx] = exprs[expr_idx].clone().alias(target_name.clone());
+            output_names[expr_idx] = target_name;
+        }
+
+        if !positional_matches.is_empty() {
+            debug!(
+                "Matched flow output columns to sink columns by position: {:?}",
+                positional_matches
+            );
+        }
+
+        let duplicated_output_names = duplicate_names(&output_names);
+        if !duplicated_output_names.is_empty() {
+            return Err(DataFusionError::Plan(format!(
+                "Flow output schema contains duplicate column(s) after schema matching {:?}. {}",
+                duplicated_output_names,
+                format_flow_sink_schema_mismatch(&exprs, self.schema.as_ref())
+            )));
+        }
+
         Ok(exprs)
     }
 
@@ -1186,12 +1272,9 @@ impl ColumnMatcherRewriter {
         let query_col_cnt = exprs.len();
 
         if query_col_cnt > table_col_cnt {
-            return Err(DataFusionError::Plan(format!(
-                "Expect query column count <= table column count, found {} query columns {:?}, {} table columns {:?}",
-                query_col_cnt,
-                exprs,
-                table_col_cnt,
-                self.schema.column_schemas()
+            return Err(DataFusionError::Plan(format_flow_sink_schema_mismatch(
+                &exprs,
+                self.schema.as_ref(),
             )));
         }
 
@@ -1209,8 +1292,9 @@ impl ColumnMatcherRewriter {
             .collect();
         if !missing.is_empty() {
             return Err(DataFusionError::Plan(format!(
-                "Column(s) {:?} required by sink table are missing from flow output when merge_mode=last_non_null",
-                missing
+                "Column(s) {:?} required by sink table are missing from flow output when merge_mode=last_non_null. {}",
+                missing,
+                format_flow_sink_schema_mismatch(&exprs, self.schema.as_ref())
             )));
         }
 
@@ -1250,8 +1334,9 @@ impl ColumnMatcherRewriter {
         if !remap.is_empty() {
             let extra: Vec<_> = remap.keys().cloned().collect();
             return Err(DataFusionError::Plan(format!(
-                "Flow output has extra column(s) {:?} not found in sink schema when merge_mode=last_non_null",
-                extra
+                "Flow output has extra column(s) {:?} not found in sink schema when merge_mode=last_non_null. {}",
+                extra,
+                format_flow_sink_schema_mismatch(&exprs, self.schema.as_ref())
             )));
         }
 
@@ -1281,6 +1366,80 @@ impl ColumnMatcherRewriter {
     }
 }
 
+fn is_obviously_incompatible_positional_match(
+    expr_type: &ConcreteDataType,
+    sink_type: &ConcreteDataType,
+) -> bool {
+    // This is a coarse type-family guard for legacy positional aliasing, not a strict type equality
+    // check. For example, numeric width/sign differences are allowed here and left to downstream
+    // coercion, and untyped NULL can be coerced to any target type. Clearly different families such
+    // as timestamp vs string are rejected early.
+    if expr_type.is_null() || expr_type == sink_type {
+        return false;
+    }
+
+    expr_type.is_timestamp() != sink_type.is_timestamp()
+        || expr_type.is_string() != sink_type.is_string()
+        || expr_type.is_boolean() != sink_type.is_boolean()
+        || expr_type.is_json() != sink_type.is_json()
+        || expr_type.is_vector() != sink_type.is_vector()
+}
+
+fn duplicate_names(names: &[String]) -> Vec<String> {
+    let mut seen = HashSet::new();
+    let mut duplicated = BTreeSet::new();
+    for name in names {
+        if !seen.insert(name.as_str()) {
+            duplicated.insert(name.as_str());
+        }
+    }
+    duplicated.into_iter().map(str::to_string).collect()
+}
+
+fn format_flow_sink_schema_mismatch(
+    query_exprs: &[Expr],
+    sink_schema: &datatypes::schema::Schema,
+) -> String {
+    let flow_output_columns = query_exprs
+        .iter()
+        .map(|expr| expr.qualified_name().1)
+        .collect::<Vec<_>>();
+    let sink_table_columns = sink_schema
+        .column_schemas()
+        .iter()
+        .map(|col| col.name.clone())
+        .collect::<Vec<_>>();
+
+    let flow_output_set = flow_output_columns.iter().cloned().collect::<HashSet<_>>();
+    let sink_table_set = sink_table_columns.iter().cloned().collect::<HashSet<_>>();
+
+    let mut extra_flow_columns = flow_output_columns
+        .iter()
+        .filter(|name| !sink_table_set.contains(*name))
+        .cloned()
+        .collect::<Vec<_>>();
+    extra_flow_columns.sort();
+    extra_flow_columns.dedup();
+
+    let mut missing_sink_columns = sink_table_columns
+        .iter()
+        .filter(|name| !flow_output_set.contains(*name))
+        .cloned()
+        .collect::<Vec<_>>();
+    missing_sink_columns.sort();
+    missing_sink_columns.dedup();
+
+    format!(
+        "Flow output schema does not match sink table schema: found {} flow output columns and {} sink table columns. flow output columns: {:?}, sink table columns: {:?}, extra flow columns not in sink: {:?}, missing sink columns from flow output: {:?}",
+        flow_output_columns.len(),
+        sink_table_columns.len(),
+        flow_output_columns,
+        sink_table_columns,
+        extra_flow_columns,
+        missing_sink_columns
+    )
+}
+
 impl TreeNodeRewriter for ColumnMatcherRewriter {
     type Node = LogicalPlan;
     fn f_down(&mut self, mut node: Self::Node) -> DfResult<Transformed<Self::Node>> {
@@ -1327,7 +1486,7 @@ impl TreeNodeRewriter for ColumnMatcherRewriter {
         // if not, wrap it in a projection
         if let LogicalPlan::Projection(project) = &node {
             let exprs = project.expr.clone();
-            let exprs = self.modify_project_exprs(exprs)?;
+            let exprs = self.modify_project_exprs(exprs, project.input.schema())?;
 
             self.is_rewritten = true;
             let new_plan =
@@ -1341,7 +1500,7 @@ impl TreeNodeRewriter for ColumnMatcherRewriter {
                     field.name(),
                 )));
             }
-            let exprs = self.modify_project_exprs(exprs)?;
+            let exprs = self.modify_project_exprs(exprs, node.schema())?;
             self.is_rewritten = true;
             let new_plan =
                 LogicalPlan::Projection(Projection::try_new(exprs, Arc::new(node.clone()))?);

src/flow/src/batching_mode/utils/test.rs

@@ -14,6 +14,7 @@
 
 use std::sync::Arc;
 
+use catalog::RegisterTableRequest;
 use common_recordbatch::RecordBatch;
 use common_time::Timestamp;
 use datafusion_common::tree_node::TreeNode as _;
@@ -29,7 +30,9 @@ use substrait::{DFLogicalSubstraitConvertor, SubstraitPlan};
 use table::test_util::MemTable;
 
 use super::*;
+use crate::batching_mode::BatchingModeOptions;
 use crate::batching_mode::state::FilterExprInfo;
+use crate::batching_mode::task::{BatchingTask, TaskArgs};
 use crate::test_utils::create_test_query_engine;
 
 fn u32_table(table_name: &str, columns: Vec<&str>, rows: usize) -> TableRef {
@@ -432,9 +435,7 @@ async fn test_add_auto_column_rewriter() {
         // error datatype mismatch
         (
             "SELECT number, ts FROM numbers_with_ts",
-            Err(
-                "Expect the last column in table to be timestamp column, found column atat with type Int8",
-            ),
+            Err("missing sink columns from flow output: [\"atat\"]"),
             vec![
                 ColumnSchema::new("number", ConcreteDataType::int32_datatype(), true),
                 ColumnSchema::new(
@@ -498,6 +499,383 @@ async fn test_add_auto_column_rewriter() {
     }
 }
 
+#[tokio::test]
+async fn test_gen_plan_with_matching_schema_reports_extra_flow_columns_before_positional_alias() {
+    let query_engine = create_test_query_engine();
+    let ctx = QueryContext::arc();
+    let sink_schema = Arc::new(Schema::new(vec![
+        ColumnSchema::new("number", ConcreteDataType::uint32_datatype(), true),
+        ColumnSchema::new(
+            "ts",
+            ConcreteDataType::timestamp_millisecond_datatype(),
+            false,
+        )
+        .with_time_index(true),
+        ColumnSchema::new(
+            "max(numbers_with_ts.number)",
+            ConcreteDataType::uint32_datatype(),
+            true,
+        ),
+    ]));
+
+    let err = gen_plan_with_matching_schema(
+        "SELECT number, number AS extra, ts, max(number) FROM numbers_with_ts GROUP BY number, ts",
+        ctx,
+        query_engine,
+        sink_schema,
+        &[],
+        false,
+    )
+    .await
+    .unwrap_err()
+    .to_string();
+
+    assert!(
+        err.contains("Flow output schema does not match sink table schema"),
+        "{err}"
+    );
+    assert!(err.contains("flow output columns"), "{err}");
+    assert!(err.contains("sink table columns"), "{err}");
+    assert!(err.contains("extra flow columns not in sink"), "{err}");
+    assert!(err.contains("extra"), "{err}");
+    assert!(
+        !err.contains("extra AS ts"),
+        "schema error should not primarily expose positional alias: {err}"
+    );
+}
+
+#[tokio::test]
+async fn test_gen_plan_with_matching_schema_rejects_positional_alias_type_mismatch() {
+    let query_engine = create_test_query_engine();
+    let ctx = QueryContext::arc();
+    let sink_schema = Arc::new(Schema::new(vec![
+        ColumnSchema::new("number", ConcreteDataType::uint32_datatype(), true),
+        ColumnSchema::new(
+            "event_time",
+            ConcreteDataType::timestamp_millisecond_datatype(),
+            false,
+        )
+        .with_time_index(true),
+        ColumnSchema::new(
+            "max(numbers_with_ts.number)",
+            ConcreteDataType::uint32_datatype(),
+            true,
+        ),
+    ]));
+
+    let err = gen_plan_with_matching_schema(
+        "SELECT number, number AS not_time, max(number) FROM numbers_with_ts GROUP BY number",
+        ctx,
+        query_engine,
+        sink_schema,
+        &[],
+        false,
+    )
+    .await
+    .unwrap_err()
+    .to_string();
+
+    assert!(
+        err.contains(
+            "Cannot match flow output column 'not_time' to sink column 'event_time' by position"
+        ),
+        "{err}"
+    );
+    assert!(err.contains("incompatible data types"), "{err}");
+    assert!(
+        !err.contains("not_time AS event_time"),
+        "schema error should not expose an incompatible positional alias: {err}"
+    );
+}
+
+#[tokio::test]
+async fn test_gen_plan_with_matching_schema_rejects_cross_position_extra_column_match() {
+    let query_engine = create_test_query_engine();
+    let ctx = QueryContext::arc();
+    let sink_schema = Arc::new(Schema::new(vec![
+        ColumnSchema::new("number", ConcreteDataType::uint32_datatype(), true),
+        ColumnSchema::new(
+            "time_window",
+            ConcreteDataType::timestamp_millisecond_datatype(),
+            false,
+        )
+        .with_time_index(true),
+        ColumnSchema::new(
+            "ts",
+            ConcreteDataType::timestamp_millisecond_datatype(),
+            true,
+        ),
+    ]));
+
+    let err = gen_plan_with_matching_schema(
+        "SELECT number, ts, date_bin('5 minutes', ts) AS time_window2 FROM numbers_with_ts GROUP BY number, ts, time_window2",
+        ctx,
+        query_engine,
+        sink_schema,
+        &[],
+        false,
+    )
+    .await
+    .unwrap_err()
+    .to_string();
+
+    assert!(
+        err.contains("Flow output schema does not match sink table schema"),
+        "{err}"
+    );
+    assert!(err.contains("time_window2"), "{err}");
+    assert!(err.contains("time_window"), "{err}");
+    assert!(!err.contains("DuplicateUnqualifiedField"), "{err}");
+}
+
+#[tokio::test]
+async fn test_gen_plan_with_matching_schema_accepts_out_of_order_matching_names() {
+    let query_engine = create_test_query_engine();
+    let ctx = QueryContext::arc();
+    let sink_schema = Arc::new(Schema::new(vec![
+        ColumnSchema::new("number", ConcreteDataType::uint32_datatype(), true),
+        ColumnSchema::new(
+            "time_window",
+            ConcreteDataType::timestamp_millisecond_datatype(),
+            false,
+        )
+        .with_time_index(true),
+        ColumnSchema::new(
+            "ts",
+            ConcreteDataType::timestamp_millisecond_datatype(),
+            true,
+        ),
+    ]));
+
+    let plan = gen_plan_with_matching_schema(
+        "SELECT number, ts, date_bin('5 minutes', ts) AS time_window FROM numbers_with_ts GROUP BY number, ts, time_window",
+        ctx,
+        query_engine,
+        sink_schema,
+        &[],
+        false,
+    )
+    .await
+    .unwrap();
+    let output_names = plan
+        .schema()
+        .fields()
+        .iter()
+        .map(|field| field.name().clone())
+        .collect::<Vec<_>>();
+
+    assert_eq!(
+        output_names,
+        vec![
+            "number".to_string(),
+            "ts".to_string(),
+            "time_window".to_string()
+        ]
+    );
+    assert!(duplicate_names(&output_names).is_empty());
+}
+
+#[tokio::test]
+async fn test_gen_plan_with_matching_schema_allows_numeric_positional_alias() {
+    let query_engine = create_test_query_engine();
+    let ctx = QueryContext::arc();
+    let sink_schema = Arc::new(Schema::new(vec![
+        ColumnSchema::new("renamed_number", ConcreteDataType::int64_datatype(), true),
+        ColumnSchema::new(
+            "ts",
+            ConcreteDataType::timestamp_millisecond_datatype(),
+            false,
+        )
+        .with_time_index(true),
+    ]));
+
+    let plan = gen_plan_with_matching_schema(
+        "SELECT number, ts FROM numbers_with_ts",
+        ctx,
+        query_engine,
+        sink_schema,
+        &[],
+        false,
+    )
+    .await
+    .unwrap();
+    let sql = df_plan_to_sql(&plan).unwrap();
+
+    assert_eq!(
+        "SELECT numbers_with_ts.number AS renamed_number, numbers_with_ts.ts FROM numbers_with_ts",
+        sql
+    );
+}
+
+#[tokio::test]
+async fn test_gen_plan_with_matching_schema_allows_null_positional_alias() {
+    let query_engine = create_test_query_engine();
+    let ctx = QueryContext::arc();
+    let sink_schema = Arc::new(Schema::new(vec![
+        ColumnSchema::new("number", ConcreteDataType::uint32_datatype(), true),
+        ColumnSchema::new("label", ConcreteDataType::string_datatype(), true),
+    ]));
+
+    let plan = gen_plan_with_matching_schema(
+        "SELECT number, NULL AS label_placeholder FROM numbers_with_ts",
+        ctx,
+        query_engine,
+        sink_schema,
+        &[],
+        false,
+    )
+    .await
+    .unwrap();
+    let output_names = plan
+        .schema()
+        .fields()
+        .iter()
+        .map(|field| field.name().clone())
+        .collect::<Vec<_>>();
+    let sql = df_plan_to_sql(&plan).unwrap();
+
+    assert_eq!(
+        output_names,
+        vec!["number".to_string(), "label".to_string()]
+    );
+    assert!(sql.contains("NULL AS label"), "{sql}");
+}
+
+#[tokio::test]
+async fn test_gen_plan_with_matching_schema_accepts_matching_flow_schema() {
+    let query_engine = create_test_query_engine();
+    let ctx = QueryContext::arc();
+    let sink_schema = Arc::new(Schema::new(vec![
+        ColumnSchema::new("number", ConcreteDataType::uint32_datatype(), true),
+        ColumnSchema::new("extra", ConcreteDataType::uint32_datatype(), true),
+        ColumnSchema::new(
+            "ts",
+            ConcreteDataType::timestamp_millisecond_datatype(),
+            false,
+        )
+        .with_time_index(true),
+        ColumnSchema::new(
+            "max(numbers_with_ts.number)",
+            ConcreteDataType::uint32_datatype(),
+            true,
+        ),
+    ]));
+
+    let plan = gen_plan_with_matching_schema(
+        "SELECT number, number AS extra, ts, max(number) FROM numbers_with_ts GROUP BY number, ts",
+        ctx,
+        query_engine,
+        sink_schema,
+        &[],
+        false,
+    )
+    .await
+    .unwrap();
+    let sql = df_plan_to_sql(&plan).unwrap();
+
+    assert_eq!(
+        "SELECT numbers_with_ts.number, numbers_with_ts.number AS extra, numbers_with_ts.ts, max(numbers_with_ts.number) FROM numbers_with_ts GROUP BY numbers_with_ts.number, numbers_with_ts.ts",
+        sql
+    );
+}
+
+#[tokio::test]
+async fn test_validate_sink_table_schema_rejects_existing_sink_missing_flow_column() {
+    let query_engine = create_test_query_engine();
+    let query_ctx = QueryContext::arc();
+    let sql = "SELECT number, number AS extra, max(number) FROM numbers_with_ts GROUP BY number";
+    let plan = sql_to_df_plan(query_ctx.clone(), query_engine.clone(), sql, true)
+        .await
+        .unwrap();
+
+    let catalog_manager = catalog::memory::new_memory_catalog_manager().unwrap();
+    let sink_table_name = [
+        "greptime".to_string(),
+        "public".to_string(),
+        "existing_sink".to_string(),
+    ];
+    let sink_table = u32_table(
+        "existing_sink",
+        vec!["number", "max(numbers_with_ts.number)"],
+        0,
+    );
+    catalog_manager
+        .register_table_sync(RegisterTableRequest {
+            catalog: sink_table_name[0].clone(),
+            schema: sink_table_name[1].clone(),
+            table_name: sink_table_name[2].clone(),
+            table_id: 4096,
+            table: sink_table,
+        })
+        .unwrap();
+
+    let (_shutdown_tx, shutdown_rx) = tokio::sync::oneshot::channel();
+    let task = BatchingTask::try_new(TaskArgs {
+        flow_id: 1,
+        query: sql,
+        plan,
+        time_window_expr: None,
+        expire_after: None,
+        sink_table_name,
+        source_table_names: vec![[
+            "greptime".to_string(),
+            "public".to_string(),
+            "numbers_with_ts".to_string(),
+        ]],
+        query_ctx,
+        catalog_manager,
+        shutdown_rx,
+        batch_opts: Arc::new(BatchingModeOptions::default()),
+        flow_eval_interval: None,
+    })
+    .unwrap();
+
+    let err = task
+        .validate_sink_table_schema(&query_engine)
+        .await
+        .unwrap_err()
+        .to_string();
+
+    assert!(
+        err.contains("Flow output schema does not match sink table schema"),
+        "{err}"
+    );
+    assert!(err.contains("extra"), "{err}");
+}
+
+#[tokio::test]
+async fn test_gen_plan_with_matching_schema_allow_partial_fills_nullable_columns() {
+    let query_engine = create_test_query_engine();
+    let ctx = QueryContext::arc();
+    let sink_schema = Arc::new(Schema::new(vec![
+        ColumnSchema::new("number", ConcreteDataType::uint32_datatype(), false),
+        ColumnSchema::new(
+            "ts",
+            ConcreteDataType::timestamp_millisecond_datatype(),
+            false,
+        )
+        .with_time_index(true),
+        ColumnSchema::new("optional_value", ConcreteDataType::uint32_datatype(), true),
+    ]));
+
+    let plan = gen_plan_with_matching_schema(
+        "SELECT number, ts FROM numbers_with_ts",
+        ctx,
+        query_engine,
+        sink_schema,
+        &[0],
+        true,
+    )
+    .await
+    .unwrap();
+    let sql = df_plan_to_sql(&plan).unwrap();
+
+    assert_eq!(
+        "SELECT numbers_with_ts.number, numbers_with_ts.ts, NULL AS optional_value FROM numbers_with_ts",
+        sql
+    );
+}
+
 #[tokio::test]
 async fn test_find_group_by_exprs() {
     let testcases = vec![
@@ -1288,9 +1666,10 @@ async fn test_rewrite_incremental_aggregate_with_left_join() {
 
 #[tokio::test]
 async fn test_rewrite_incremental_aggregate_filters_sink_dirty_time_window() {
-    // This verifies the rewrite placement when callers supply an already
-    // inferred sink dirty-window predicate. The task-level inference rules are
-    // covered by `infer_sink_time_window_filter_col` tests in task.rs.
+    // This verifies the rewrite placement when callers supply a sink predicate.
+    // The production incremental flow path currently leaves sink scans
+    // unfiltered for correctness and relies on future dynamic filters for
+    // pruning.
     let query_engine = create_test_query_engine();
     let ctx = QueryContext::arc();
     let sql = "SELECT max(number) AS number, date_bin(INTERVAL '1 second', ts) AS time_window FROM numbers_with_ts GROUP BY time_window";
@@ -1490,3 +1869,118 @@ async fn test_analyze_incremental_aggregate_plan_rejects_cast_wrapped_alias() {
         );
     }
 }
+
+#[tokio::test]
+async fn test_gen_plan_with_matching_schema_last_non_null_rejects_missing_primary_key_column() {
+    let query_engine = create_test_query_engine();
+    let ctx = QueryContext::arc();
+    // Sink table with primary_key_indices=[0] ("number"), time_index="ts", and merge_mode=last_non_null.
+    // The flow query omits "number", which is a required primary-key column.
+    let sink_schema = Arc::new(Schema::new(vec![
+        ColumnSchema::new("number", ConcreteDataType::uint32_datatype(), true),
+        ColumnSchema::new(
+            "ts",
+            ConcreteDataType::timestamp_millisecond_datatype(),
+            false,
+        )
+        .with_time_index(true),
+        ColumnSchema::new("optional_value", ConcreteDataType::uint32_datatype(), true),
+    ]));
+
+    let err = gen_plan_with_matching_schema(
+        "SELECT ts FROM numbers_with_ts",
+        ctx,
+        query_engine,
+        sink_schema,
+        &[0],
+        true,
+    )
+    .await
+    .unwrap_err()
+    .to_string();
+
+    assert!(
+        err.contains(
+            "required by sink table are missing from flow output when merge_mode=last_non_null"
+        ),
+        "{err}"
+    );
+    assert!(err.contains("number"), "{err}");
+}
+
+#[tokio::test]
+async fn test_gen_plan_with_matching_schema_last_non_null_rejects_missing_time_index_column() {
+    let query_engine = create_test_query_engine();
+    let ctx = QueryContext::arc();
+    // Sink table with primary_key_indices=[0] ("number"), time_index="ts", and merge_mode=last_non_null.
+    // The flow query omits "ts", which is a required time-index column.
+    let sink_schema = Arc::new(Schema::new(vec![
+        ColumnSchema::new("number", ConcreteDataType::uint32_datatype(), true),
+        ColumnSchema::new(
+            "ts",
+            ConcreteDataType::timestamp_millisecond_datatype(),
+            false,
+        )
+        .with_time_index(true),
+        ColumnSchema::new("optional_value", ConcreteDataType::uint32_datatype(), true),
+    ]));
+
+    let err = gen_plan_with_matching_schema(
+        "SELECT number FROM numbers_with_ts",
+        ctx,
+        query_engine,
+        sink_schema,
+        &[0],
+        true,
+    )
+    .await
+    .unwrap_err()
+    .to_string();
+
+    assert!(
+        err.contains(
+            "required by sink table are missing from flow output when merge_mode=last_non_null"
+        ),
+        "{err}"
+    );
+    assert!(err.contains("ts"), "{err}");
+}
+
+#[tokio::test]
+async fn test_gen_plan_with_matching_schema_last_non_null_rejects_extra_flow_column() {
+    let query_engine = create_test_query_engine();
+    let ctx = QueryContext::arc();
+    // Sink table with merge_mode=last_non_null.
+    // Sink has 3 columns: number (pk), ts (time_index), optional_value (nullable).
+    // Flow outputs: number, number AS extra, ts → "extra" is not in sink schema.
+    // query_col_cnt(3) <= table_col_cnt(3), so the extra branch is reached.
+    let sink_schema = Arc::new(Schema::new(vec![
+        ColumnSchema::new("number", ConcreteDataType::uint32_datatype(), true),
+        ColumnSchema::new(
+            "ts",
+            ConcreteDataType::timestamp_millisecond_datatype(),
+            false,
+        )
+        .with_time_index(true),
+        ColumnSchema::new("optional_value", ConcreteDataType::uint32_datatype(), true),
+    ]));
+
+    let err = gen_plan_with_matching_schema(
+        "SELECT number, number AS extra, ts FROM numbers_with_ts",
+        ctx,
+        query_engine,
+        sink_schema,
+        &[0],
+        true,
+    )
+    .await
+    .unwrap_err()
+    .to_string();
+
+    assert!(err.contains("extra column(s)"), "{err}");
+    assert!(err.contains("extra"), "{err}");
+    assert!(
+        err.contains("Flow output schema does not match sink table schema"),
+        "{err}"
+    );
+}

src/flow/src/server.rs

@@ -566,11 +566,15 @@ impl FrontendInvoker {
                 name: TABLE_FLOWNODE_SET_CACHE_NAME,
             })?;
 
+        // TODO(auto_create_table): flow sink tables are created through a controlled
+        // `CREATE FLOW` path, not client writes, so they are intentionally exempt from
+        // the frontend's global auto-create switch. Revisit if flow should honor it.
         let inserter = Arc::new(Inserter::new(
             catalog_manager.clone(),
             partition_manager.clone(),
             node_manager.clone(),
             table_flownode_cache,
+            true,
         ));
 
         let deleter = Arc::new(Deleter::new(

src/frontend/src/frontend.rs

@@ -44,6 +44,11 @@ pub struct FrontendOptions {
     pub node_id: Option<String>,
     pub default_timezone: Option<String>,
     pub default_column_prefix: Option<String>,
+    /// Server-side global switch for auto table creation on write.
+    /// Acts as an upper bound: when `false`, missing tables are never auto-created
+    /// even if a request sets the `auto_create_table` hint to `true`. When `true`
+    /// (default), the per-request hint still applies. Default: `true`.
+    pub auto_create_table: bool,
     /// Maximum total memory for all concurrent write request bodies and messages (HTTP, gRPC, Flight).
     /// Set to 0 to disable the limit. Default: "0" (unlimited)
     pub max_in_flight_write_bytes: ReadableSize,
@@ -82,6 +87,7 @@ impl Default for FrontendOptions {
             node_id: None,
             default_timezone: None,
             default_column_prefix: None,
+            auto_create_table: true,
             max_in_flight_write_bytes: ReadableSize(0),
             write_bytes_exhausted_policy: OnExhaustedPolicy::default(),
             http: HttpOptions::default(),

src/frontend/src/instance/builder.rs

@@ -185,6 +185,7 @@ impl FrontendBuilder {
             partition_manager.clone(),
             node_manager.clone(),
             table_flownode_cache,
+            self.options.auto_create_table,
         ));
         let deleter = Arc::new(Deleter::new(
             self.catalog_manager.clone(),

src/frontend/src/instance/otlp.rs

@@ -43,7 +43,12 @@ use servers::query_handler::{
 };
 use session::context::QueryContextRef;
 use snafu::{IntoError, ResultExt};
-use table::requests::{OTLP_METRIC_COMPAT_KEY, OTLP_METRIC_COMPAT_PROM};
+use table::requests::{
+    OTLP_METRIC_COMPAT_KEY, OTLP_METRIC_COMPAT_PROM, SEMANTIC_PIPELINE, SEMANTIC_SIGNAL_TYPE,
+    SEMANTIC_SOURCE, SEMANTIC_TRACE_CONVENTIONS, SEMANTIC_TRACE_HAS_EVENTS,
+    SEMANTIC_TRACE_HAS_LINKS, SEMANTIC_VALUE_UNKNOWN, SIGNAL_TYPE_LOG, SIGNAL_TYPE_METRIC,
+    SIGNAL_TYPE_TRACE, SOURCE_OPENTELEMETRY, TABLE_DATA_MODEL_TRACE_V1,
+};
 
 use crate::instance::Instance;
 use crate::instance::otlp::trace_semconv::trace_semconv_fixed_type;
@@ -131,12 +136,14 @@ impl OpenTelemetryProtocolHandler for Instance {
         let (requests, rows) = otlp::metrics::to_grpc_insert_requests(request, &mut metric_ctx)?;
         OTLP_METRICS_ROWS.inc_by(rows as u64);
 
-        let ctx = if !is_legacy {
+        let ctx = {
             let mut c = (*ctx).clone();
-            c.set_extension(OTLP_METRIC_COMPAT_KEY, OTLP_METRIC_COMPAT_PROM.to_string());
+            c.set_extension(SEMANTIC_SIGNAL_TYPE, SIGNAL_TYPE_METRIC);
+            c.set_extension(SEMANTIC_SOURCE, SOURCE_OPENTELEMETRY);
+            if !is_legacy {
+                c.set_extension(OTLP_METRIC_COMPAT_KEY, OTLP_METRIC_COMPAT_PROM.to_string());
+            }
             Arc::new(c)
-        } else {
-            ctx
         };
 
         // If the user uses the legacy path, it is by default without metric engine.
@@ -211,6 +218,15 @@ impl OpenTelemetryProtocolHandler for Instance {
             .get::<OpenTelemetryProtocolInterceptorRef<servers::error::Error>>();
         interceptor_ref.pre_execute(ctx.clone())?;
 
+        // `as_req_iter` clones this ctx into each `temp_ctx`, so identity set here
+        // reaches the context that drives table auto-create.
+        let ctx = {
+            let mut c = (*ctx).clone();
+            c.set_extension(SEMANTIC_SIGNAL_TYPE, SIGNAL_TYPE_LOG);
+            c.set_extension(SEMANTIC_SOURCE, SOURCE_OPENTELEMETRY);
+            Arc::new(c)
+        };
+
         let opt_req = otlp::logs::to_grpc_insert_requests(
             request,
             pipeline,
@@ -256,6 +272,23 @@ impl Instance {
         ctx: QueryContextRef,
     ) -> ServerResult<TraceIngestOutcome> {
         let is_trace_v1_model = matches!(pipeline, PipelineWay::OtlpTraceDirectV1);
+
+        // Only the main span table gets the identity; the derived `_services` /
+        // `_operations` lookup tables keep the unstamped `ctx`.
+        let main_ctx = {
+            let mut c = (*ctx).clone();
+            c.set_extension(SEMANTIC_SIGNAL_TYPE, SIGNAL_TYPE_TRACE);
+            c.set_extension(SEMANTIC_SOURCE, SOURCE_OPENTELEMETRY);
+            if is_trace_v1_model {
+                c.set_extension(SEMANTIC_PIPELINE, TABLE_DATA_MODEL_TRACE_V1);
+                c.set_extension(SEMANTIC_TRACE_HAS_EVENTS, "true");
+                c.set_extension(SEMANTIC_TRACE_HAS_LINKS, "true");
+                // schema_url is row-level, so conventions is unknown at table level.
+                c.set_extension(SEMANTIC_TRACE_CONVENTIONS, SEMANTIC_VALUE_UNKNOWN);
+            }
+            Arc::new(c)
+        };
+
         let ingest_ctx = TraceChunkIngestContext {
             pipeline_handler,
             pipeline,
@@ -278,7 +311,7 @@ impl Instance {
                 .map(|chunk| chunk.collect::<Vec<_>>())
                 .collect::<Vec<_>>();
             for chunk in chunks {
-                self.ingest_trace_chunk(&ingest_ctx, chunk, ctx.clone(), &mut ingest_state)
+                self.ingest_trace_chunk(&ingest_ctx, chunk, main_ctx.clone(), &mut ingest_state)
                     .await?;
             }
         }

src/frontend/src/server.rs

@@ -288,7 +288,6 @@ where
 
         let http_server = builder
             .with_metrics_handler(MetricsHandler)
-            .with_plugins(self.plugins.clone())
             .with_greptime_config_options(toml)
             .build();
         Ok(http_server)

src/meta-client/src/client.rs

@@ -1344,7 +1344,7 @@ mod tests {
 
         // Generates rough 10MB data, which is larger than the default grpc message size limit.
         for i in 0..10 {
-            let data: Vec<u8> = (0..1024 * 1024).map(|_| rng.random()).collect();
+            let data: Vec<u8> = (0..1024 * 1024).map(|_| rng.random::<u8>()).collect();
             in_memory
                 .put(
                     PutRequest::new()

src/meta-srv/src/procedure/region_migration.rs

@@ -39,6 +39,7 @@ use common_meta::ddl::RegionFailureDetectorControllerRef;
 use common_meta::instruction::CacheIdent;
 use common_meta::key::datanode_table::{DatanodeTableKey, DatanodeTableValue};
 use common_meta::key::table_route::TableRouteValue;
+use common_meta::key::topic_name::TopicNameKey;
 use common_meta::key::topic_region::{ReplayCheckpoint, TopicRegionKey};
 use common_meta::key::{DeserializedValueWithBytes, TableMetadataManagerRef};
 use common_meta::kv_backend::{KvBackendRef, ResettableKvBackendRef};
@@ -661,11 +662,15 @@ impl Context {
             .await;
     }
 
-    /// Fetches the replay checkpoints for the given topic region keys.
-    pub async fn get_replay_checkpoints(
+    /// Fetches replay checkpoints and merges them with topic pruned entry ids.
+    pub async fn get_replay_checkpoints_with_topic_pruned_entry_ids(
         &self,
-        topic_region_keys: Vec<TopicRegionKey<'_>>,
+        region_topics: &[(RegionId, String, bool)],
     ) -> Result<HashMap<RegionId, ReplayCheckpoint>> {
+        let topic_region_keys = region_topics
+            .iter()
+            .map(|(region_id, topic, _)| TopicRegionKey::new(*region_id, topic))
+            .collect::<Vec<_>>();
         let topic_region_values = self
             .table_metadata_manager
             .topic_region_manager()
@@ -673,9 +678,37 @@ impl Context {
             .await
             .context(error::TableMetadataManagerSnafu)?;
 
-        let replay_checkpoints = topic_region_values
+        let topic_name_keys = region_topics
+            .iter()
+            .map(|(_, topic, _)| topic.as_str())
+            .collect::<HashSet<_>>()
             .into_iter()
-            .flat_map(|(key, value)| value.checkpoint.map(|value| (key, value)))
+            .map(TopicNameKey::new)
+            .collect::<Vec<_>>();
+        let topic_name_values = self
+            .table_metadata_manager
+            .topic_name_manager()
+            .batch_get(topic_name_keys)
+            .await
+            .context(error::TableMetadataManagerSnafu)?;
+
+        let replay_checkpoints = region_topics
+            .iter()
+            .filter_map(|(region_id, topic, is_metric_engine)| {
+                let checkpoint = topic_region_values
+                    .get(region_id)
+                    .and_then(|value| value.checkpoint);
+                let pruned_entry_id = topic_name_values
+                    .get(topic)
+                    .map(|value| value.pruned_entry_id);
+
+                ReplayCheckpoint::merge_with_topic_pruned_entry_id(
+                    checkpoint,
+                    pruned_entry_id,
+                    *is_metric_engine,
+                )
+                .map(|checkpoint| (*region_id, checkpoint))
+            })
             .collect::<HashMap<_, _>>();
 
         Ok(replay_checkpoints)

src/meta-srv/src/procedure/region_migration/open_candidate_region.rs

@@ -18,7 +18,9 @@ use std::ops::Div;
 use api::v1::meta::MailboxMessage;
 use common_meta::RegionIdent;
 use common_meta::distributed_time_constants::default_distributed_time_constants;
-use common_meta::instruction::{Instruction, InstructionReply, OpenRegion, SimpleReply};
+use common_meta::instruction::{
+    Instruction, InstructionReply, OpenRegion, OpenRegionReason, SimpleReply,
+};
 use common_meta::key::datanode_table::RegionInfo;
 use common_procedure::{Context as ProcedureContext, Status};
 use common_telemetry::info;
@@ -26,12 +28,13 @@ use common_telemetry::tracing_context::TracingContext;
 use serde::{Deserialize, Serialize};
 use snafu::{OptionExt, ResultExt};
 use store_api::region_engine::RegionRole;
+use store_api::region_request::RegionRequirements;
 use tokio::time::Instant;
 
 use crate::error::{self, Result};
 use crate::handler::HeartbeatMailbox;
 use crate::procedure::region_migration::flush_leader_region::PreFlushRegion;
-use crate::procedure::region_migration::{Context, State};
+use crate::procedure::region_migration::{Context, RegionMigrationTriggerReason, State};
 use crate::service::mailbox::Channel;
 
 #[derive(Debug, Serialize, Deserialize)]
@@ -67,6 +70,10 @@ impl OpenCandidateRegion {
         let region_ids = ctx.persistent_ctx.region_ids.clone();
         let from_peer_id = ctx.persistent_ctx.from_peer.id;
         let to_peer_id = ctx.persistent_ctx.to_peer.id;
+        let reason = match ctx.persistent_ctx.trigger_reason {
+            RegionMigrationTriggerReason::Failover => OpenRegionReason::RegionFailover,
+            _ => OpenRegionReason::RegionMigration,
+        };
         let datanode_table_values = ctx.get_from_peer_datanode_table_values().await?;
         let mut open_regions = Vec::with_capacity(region_ids.len());
 
@@ -97,6 +104,8 @@ impl OpenCandidateRegion {
                 region_options,
                 region_wal_options,
                 true,
+                Some(reason),
+                RegionRequirements::object_storage(),
             ));
         }
 
@@ -233,18 +242,20 @@ mod tests {
     }
 
     fn new_mock_open_instruction(datanode_id: DatanodeId, region_id: RegionId) -> Instruction {
-        Instruction::OpenRegions(vec![OpenRegion {
-            region_ident: RegionIdent {
+        Instruction::OpenRegions(vec![OpenRegion::new(
+            RegionIdent {
                 datanode_id,
                 table_id: region_id.table_id(),
                 region_number: region_id.region_number(),
                 engine: MITO2_ENGINE.to_string(),
             },
-            region_storage_path: "/bar/foo/region/".to_string(),
-            region_options: Default::default(),
-            region_wal_options: Default::default(),
-            skip_wal_replay: true,
-        }])
+            "/bar/foo/region/",
+            Default::default(),
+            Default::default(),
+            true,
+            Some(OpenRegionReason::RegionMigration),
+            RegionRequirements::object_storage(),
+        )])
     }
 
     #[tokio::test]
@@ -263,6 +274,57 @@ mod tests {
         assert!(!err.is_retryable());
     }
 
+    #[tokio::test]
+    async fn test_build_open_region_instruction_reason() {
+        let state = OpenCandidateRegion;
+        let mut persistent_context = new_persistent_context();
+        let from_peer_id = persistent_context.from_peer.id;
+        let region_id = persistent_context.region_ids[0];
+        let env = TestingEnv::new();
+
+        let table_info = new_test_table_info(1024);
+        let region_routes = vec![RegionRoute {
+            region: Region::new_test(region_id),
+            leader_peer: Some(Peer::empty(from_peer_id)),
+            ..Default::default()
+        }];
+        env.table_metadata_manager()
+            .create_table_metadata(
+                table_info,
+                TableRouteValue::physical(region_routes),
+                HashMap::default(),
+            )
+            .await
+            .unwrap();
+
+        let mut ctx = env
+            .context_factory()
+            .new_context(persistent_context.clone());
+        let instruction = state.build_open_region_instruction(&mut ctx).await.unwrap();
+        let open_regions = instruction.into_open_regions().unwrap();
+        assert_eq!(
+            Some(OpenRegionReason::RegionMigration),
+            open_regions[0].reason
+        );
+        assert_eq!(
+            RegionRequirements::object_storage(),
+            open_regions[0].requirements
+        );
+
+        persistent_context.trigger_reason = RegionMigrationTriggerReason::Failover;
+        let mut ctx = env.context_factory().new_context(persistent_context);
+        let instruction = state.build_open_region_instruction(&mut ctx).await.unwrap();
+        let open_regions = instruction.into_open_regions().unwrap();
+        assert_eq!(
+            Some(OpenRegionReason::RegionFailover),
+            open_regions[0].reason
+        );
+        assert_eq!(
+            RegionRequirements::object_storage(),
+            open_regions[0].requirements
+        );
+    }
+
     #[tokio::test]
     async fn test_datanode_is_unreachable() {
         let state = OpenCandidateRegion;

src/meta-srv/src/procedure/region_migration/upgrade_candidate_region.rs

@@ -21,7 +21,6 @@ use common_meta::ddl::utils::parse_region_wal_options;
 use common_meta::instruction::{
     Instruction, InstructionReply, UpgradeRegion, UpgradeRegionReply, UpgradeRegionsReply,
 };
-use common_meta::key::topic_region::TopicRegionKey;
 use common_meta::lock_key::RemoteWalLock;
 use common_meta::wal_provider::extract_topic_from_wal_options;
 use common_procedure::{Context as ProcedureContext, Status};
@@ -30,6 +29,7 @@ use common_telemetry::{error, info};
 use common_wal::options::WalOptions;
 use serde::{Deserialize, Serialize};
 use snafu::{OptionExt, ResultExt, ensure};
+use store_api::metric_engine_consts::METRIC_ENGINE_NAME;
 use tokio::time::{Instant, sleep};
 
 use crate::error::{self, Result};
@@ -133,17 +133,14 @@ impl UpgradeCandidateRegion {
                     &datanode_table_value.region_info.region_wal_options,
                 )
             {
-                region_topic.push((*region_id, topic));
+                let is_metric_engine =
+                    datanode_table_value.region_info.engine == METRIC_ENGINE_NAME;
+                region_topic.push((*region_id, topic, is_metric_engine));
             }
         }
 
         let replay_checkpoints = ctx
-            .get_replay_checkpoints(
-                region_topic
-                    .iter()
-                    .map(|(region_id, topic)| TopicRegionKey::new(*region_id, topic))
-                    .collect(),
-            )
+            .get_replay_checkpoints_with_topic_pruned_entry_ids(&region_topic)
             .await?;
         // Build upgrade regions instruction.
         let mut upgrade_regions = Vec::with_capacity(region_ids.len());
@@ -358,8 +355,11 @@ mod tests {
 
     use common_meta::key::table_route::TableRouteValue;
     use common_meta::key::test_utils::new_test_table_info;
+    use common_meta::key::topic_name::TopicNameKey;
+    use common_meta::key::topic_region::{ReplayCheckpoint, TopicRegionKey, TopicRegionValue};
     use common_meta::peer::Peer;
     use common_meta::rpc::router::{Region, RegionRoute};
+    use common_wal::options::KafkaWalOptions;
     use store_api::storage::RegionId;
 
     use super::*;
@@ -382,9 +382,28 @@ mod tests {
         )
     }
 
+    fn kafka_wal_options(topic: &str) -> HashMap<u32, String> {
+        HashMap::from([(
+            1,
+            serde_json::to_string(&WalOptions::Kafka(KafkaWalOptions {
+                topic: topic.to_string(),
+            }))
+            .unwrap(),
+        )])
+    }
+
     async fn prepare_table_metadata(ctx: &Context, wal_options: HashMap<u32, String>) {
+        prepare_table_metadata_with_engine(ctx, wal_options, "engine").await;
+    }
+
+    async fn prepare_table_metadata_with_engine(
+        ctx: &Context,
+        wal_options: HashMap<u32, String>,
+        engine: &str,
+    ) {
         let region_id = ctx.persistent_ctx.region_ids[0];
-        let table_info = new_test_table_info(region_id.table_id());
+        let mut table_info = new_test_table_info(region_id.table_id());
+        table_info.meta.engine = engine.to_string();
         let region_routes = vec![RegionRoute {
             region: Region::new_test(region_id),
             leader_peer: Some(ctx.persistent_ctx.from_peer.clone()),
@@ -401,6 +420,104 @@ mod tests {
             .unwrap();
     }
 
+    #[tokio::test]
+    async fn test_build_upgrade_region_instruction_merges_topic_pruned_entry_id() {
+        let state = UpgradeCandidateRegion::default();
+        let persistent_context = new_persistent_context();
+        let env = TestingEnv::new();
+        let mut ctx = env.context_factory().new_context(persistent_context);
+        let region_id = ctx.persistent_ctx.region_ids[0];
+        let topic = "test_topic";
+        prepare_table_metadata(&ctx, kafka_wal_options(topic)).await;
+        ctx.table_metadata_manager
+            .topic_region_manager()
+            .batch_put(&[(
+                TopicRegionKey::new(region_id, topic),
+                Some(TopicRegionValue::new(Some(ReplayCheckpoint::new(10, None)))),
+            )])
+            .await
+            .unwrap();
+        ctx.table_metadata_manager
+            .topic_name_manager()
+            .batch_put(vec![TopicNameKey::new(topic)])
+            .await
+            .unwrap();
+        let prev = ctx
+            .table_metadata_manager
+            .topic_name_manager()
+            .get(topic)
+            .await
+            .unwrap();
+        ctx.table_metadata_manager
+            .topic_name_manager()
+            .update(topic, 20, prev)
+            .await
+            .unwrap();
+
+        let instruction = state
+            .build_upgrade_region_instruction(&mut ctx, Duration::from_secs(1))
+            .await
+            .unwrap();
+        let Instruction::UpgradeRegions(upgrade_regions) = instruction else {
+            unreachable!()
+        };
+
+        assert_eq!(upgrade_regions.len(), 1);
+        assert_eq!(upgrade_regions[0].replay_entry_id, Some(20));
+        assert_eq!(upgrade_regions[0].metadata_replay_entry_id, None);
+    }
+
+    #[tokio::test]
+    async fn test_build_upgrade_region_instruction_merges_metric_metadata_pruned_entry_id() {
+        let state = UpgradeCandidateRegion::default();
+        let persistent_context = new_persistent_context();
+        let env = TestingEnv::new();
+        let mut ctx = env.context_factory().new_context(persistent_context);
+        let region_id = ctx.persistent_ctx.region_ids[0];
+        let topic = "test_topic";
+        prepare_table_metadata_with_engine(&ctx, kafka_wal_options(topic), METRIC_ENGINE_NAME)
+            .await;
+        ctx.table_metadata_manager
+            .topic_region_manager()
+            .batch_put(&[(
+                TopicRegionKey::new(region_id, topic),
+                Some(TopicRegionValue::new(Some(ReplayCheckpoint::new(
+                    10,
+                    Some(5),
+                )))),
+            )])
+            .await
+            .unwrap();
+        ctx.table_metadata_manager
+            .topic_name_manager()
+            .batch_put(vec![TopicNameKey::new(topic)])
+            .await
+            .unwrap();
+        let prev = ctx
+            .table_metadata_manager
+            .topic_name_manager()
+            .get(topic)
+            .await
+            .unwrap();
+        ctx.table_metadata_manager
+            .topic_name_manager()
+            .update(topic, 20, prev)
+            .await
+            .unwrap();
+
+        let instruction = state
+            .build_upgrade_region_instruction(&mut ctx, Duration::from_secs(1))
+            .await
+            .unwrap();
+        let Instruction::UpgradeRegions(upgrade_regions) = instruction else {
+            unreachable!()
+        };
+
+        assert_eq!(upgrade_regions.len(), 1);
+        assert_eq!(upgrade_regions[0].replay_entry_id, Some(20));
+        assert_eq!(upgrade_regions[0].metadata_replay_entry_id, Some(20));
+    }
+
     #[tokio::test]
     async fn test_datanode_is_unreachable() {
         let state = UpgradeCandidateRegion::default();

src/meta-srv/src/procedure/repartition.rs

@@ -440,7 +440,17 @@ impl Context {
         };
         let _ = self
             .cache_invalidator
-            .invalidate(&ctx, &[CacheIdent::TableId(table_id)])
+            .invalidate(
+                &ctx,
+                &[
+                    CacheIdent::TableId(table_id),
+                    CacheIdent::TableName(TableName {
+                        catalog_name: self.persistent_ctx.catalog_name.clone(),
+                        schema_name: self.persistent_ctx.schema_name.clone(),
+                        table_name: self.persistent_ctx.table_name.clone(),
+                    }),
+                ],
+            )
             .await;
         Ok(())
     }

src/meta-srv/src/procedure/repartition/update_partition_metadata.rs

@@ -95,10 +95,19 @@ impl State for UpdatePartitionMetadata {
 
         let mut new_table_info = table_info_value.table_info.clone();
         new_table_info.meta.partition_key_indices = partition_key_indices;
+        common_telemetry::info!(
+            "Update table partition metadata, table_id: {}, partition_key_indices: {:?}, partition_columns: {:?}",
+            table_id,
+            new_table_info.meta.partition_key_indices,
+            new_table_info
+                .meta
+                .partition_column_names()
+                .cloned()
+                .collect::<Vec<_>>(),
+        );
         ctx.update_table_info(&table_info_value, table_info_value.update(new_table_info))
             .await?;
-        // We don't invalidate cache here because the subsequent AllocateRegion step
-        // will update the table route and invalidate the cache accordingly.
+        ctx.invalidate_table_cache().await?;
 
         Ok((
             Box::new(AllocateRegion::new(self.plan_entries.clone())),

src/meta-srv/src/region/flush_trigger.rs

@@ -45,6 +45,9 @@ const TICKER_INTERVAL: Duration = Duration::from_secs(60);
 /// The duration of the recent period.
 const RECENT_DURATION: Duration = Duration::from_secs(300);
 
+/// The interval to periodically persist region checkpoints regardless of replay size.
+const PERIODIC_CHECKPOINT_PERSIST_INTERVAL: Duration = Duration::from_secs(60 * 60);
+
 /// [`Event`] represents various types of events that can be processed by the region flush ticker.
 ///
 /// Variants:
@@ -84,6 +87,8 @@ pub struct RegionFlushTrigger {
     flush_trigger_size: ReadableSize,
     /// The checkpoint trigger size.
     checkpoint_trigger_size: ReadableSize,
+    /// The last timestamp in milliseconds when a region checkpoint was persisted.
+    last_checkpoint_persist_millis_by_region: HashMap<RegionId, i64>,
     /// The receiver of events.
     receiver: Receiver<Event>,
 }
@@ -123,6 +128,7 @@ impl RegionFlushTrigger {
             server_addr,
             flush_trigger_size,
             checkpoint_trigger_size,
+            last_checkpoint_persist_millis_by_region: HashMap::new(),
             receiver: rx,
         };
         (region_flush_trigger, region_flush_ticker)
@@ -147,14 +153,15 @@ impl RegionFlushTrigger {
         }
     }
 
-    async fn handle_tick(&self) {
+    async fn handle_tick(&mut self) {
         if let Err(e) = self.trigger_flush().await {
             error!(e; "Failed to trigger flush");
         }
     }
 
-    async fn trigger_flush(&self) -> Result<()> {
+    async fn trigger_flush(&mut self) -> Result<()> {
         let now = Instant::now();
+        let now_millis = current_time_millis();
         let topics = self
             .table_metadata_manager
             .topic_name_manager()
@@ -162,17 +169,19 @@ impl RegionFlushTrigger {
             .await
             .context(error::TableMetadataManagerSnafu)?;
 
+        let mut active_region_ids = HashSet::new();
         for topic in &topics {
-            let Some((latest_entry_id, avg_record_size)) = self.retrieve_topic_stat(topic) else {
-                continue;
-            };
             if let Err(e) = self
-                .flush_regions_in_topic(topic, latest_entry_id, avg_record_size)
+                .handle_topic(topic, now_millis, &mut active_region_ids)
                 .await
             {
-                error!(e; "Failed to flush regions in topic: {}", topic);
+                error!(e; "Failed to handle regions in topic: {}", topic);
             }
         }
+        retain_checkpoint_persist_records(
+            &mut self.last_checkpoint_persist_millis_by_region,
+            &active_region_ids,
+        );
 
         debug!(
             "Triggered flush for {} topics in {:?}",
@@ -182,6 +191,79 @@ impl RegionFlushTrigger {
         Ok(())
     }
 
+    async fn handle_topic(
+        &mut self,
+        topic: &str,
+        now_millis: i64,
+        active_region_ids: &mut HashSet<RegionId>,
+    ) -> Result<()> {
+        let topic_regions = self
+            .table_metadata_manager
+            .topic_region_manager()
+            .regions(topic)
+            .await
+            .context(error::TableMetadataManagerSnafu)?;
+
+        if topic_regions.is_empty() {
+            debug!("No regions found for topic: {}", topic);
+            return Ok(());
+        }
+        active_region_ids.extend(topic_regions.keys().copied());
+
+        let topic_stat = self.retrieve_topic_stat(topic);
+        let size_based_regions = topic_stat
+            .map(|(latest_entry_id, avg_record_size)| {
+                filter_regions_by_replay_size(
+                    topic,
+                    topic_regions.iter().map(|(region_id, value)| {
+                        (*region_id, value.min_entry_id().unwrap_or_default())
+                    }),
+                    avg_record_size as u64,
+                    latest_entry_id,
+                    self.checkpoint_trigger_size,
+                )
+            })
+            .unwrap_or_default();
+        // Periodic checkpoint persistence is intentionally independent of topic stats freshness:
+        // Kafka retention can advance even when recent write stats are unavailable.
+        let periodic_regions = filter_regions_for_periodic_checkpoint(
+            topic_regions.keys().copied(),
+            &self.last_checkpoint_persist_millis_by_region,
+            now_millis,
+            PERIODIC_CHECKPOINT_PERSIST_INTERVAL,
+        );
+        let regions_to_persist = merge_region_ids(size_based_regions, periodic_regions);
+        let region_manifests = self
+            .leader_region_registry
+            .batch_get(topic_regions.keys().cloned());
+
+        match self
+            .persist_region_checkpoints(
+                topic,
+                &regions_to_persist,
+                &topic_regions,
+                &region_manifests,
+            )
+            .await
+        {
+            // Only mark regions that were actually written to KV. If the checkpoint is stale,
+            // already persisted, or the write fails, the next tick should retry.
+            Ok(region_ids) => mark_checkpoint_persisted(
+                &mut self.last_checkpoint_persist_millis_by_region,
+                &region_ids,
+                now_millis,
+            ),
+            Err(err) => error!(err; "Failed to persist region checkpoints for topic: {}", topic),
+        }
+
+        if let Some((latest_entry_id, avg_record_size)) = topic_stat {
+            self.flush_regions_in_topic(topic, latest_entry_id, avg_record_size, region_manifests)
+                .await?;
+        }
+
+        Ok(())
+    }
+
     /// Retrieves the latest entry id and average record size of a topic.
     ///
     /// Returns `None` if the topic is not found or the latest entry id is not recent.
@@ -226,7 +308,7 @@ impl RegionFlushTrigger {
         region_ids: &[RegionId],
         topic_regions: &HashMap<RegionId, TopicRegionValue>,
         leader_regions: &HashMap<RegionId, LeaderRegion>,
-    ) -> Result<()> {
+    ) -> Result<Vec<RegionId>> {
         let regions = region_ids
             .iter()
             .flat_map(|region_id| match leader_regions.get(region_id) {
@@ -237,27 +319,26 @@ impl RegionFlushTrigger {
                         .cloned()
                         .and_then(|value| value.checkpoint),
                 )
-                .map(|checkpoint| {
-                    (
-                        TopicRegionKey::new(*region_id, topic),
-                        Some(TopicRegionValue::new(Some(checkpoint))),
-                    )
-                }),
+                .map(|checkpoint| (*region_id, TopicRegionValue::new(Some(checkpoint)))),
                 None => None,
             })
             .collect::<Vec<_>>();
 
-        // The`chunks` will panic if chunks_size is zero, so we return early if there are no regions to persist.
+        // `chunks` will panic if chunk size is zero, so return early if there are no regions to persist.
         if regions.is_empty() {
-            return Ok(());
+            return Ok(Vec::new());
         }
 
         let max_txn_ops = self.table_metadata_manager.kv_backend().max_txn_ops();
         let batch_size = max_txn_ops.min(regions.len());
         for batch in regions.chunks(batch_size) {
+            let batch = batch
+                .iter()
+                .map(|(region_id, value)| (TopicRegionKey::new(*region_id, topic), Some(*value)))
+                .collect::<Vec<_>>();
             self.table_metadata_manager
                 .topic_region_manager()
-                .batch_put(batch)
+                .batch_put(&batch)
                 .await
                 .context(error::TableMetadataManagerSnafu)?;
         }
@@ -266,7 +347,10 @@ impl RegionFlushTrigger {
             .with_label_values(&[topic])
             .inc_by(regions.len() as u64);
 
-        Ok(())
+        Ok(regions
+            .into_iter()
+            .map(|(region_id, _)| region_id)
+            .collect())
     }
 
     async fn flush_regions_in_topic(
@@ -274,45 +358,8 @@ impl RegionFlushTrigger {
         topic: &str,
         latest_entry_id: u64,
         avg_record_size: usize,
+        region_manifests: HashMap<RegionId, LeaderRegion>,
     ) -> Result<()> {
-        let topic_regions = self
-            .table_metadata_manager
-            .topic_region_manager()
-            .regions(topic)
-            .await
-            .context(error::TableMetadataManagerSnafu)?;
-
-        if topic_regions.is_empty() {
-            debug!("No regions found for topic: {}", topic);
-            return Ok(());
-        }
-
-        // Filters regions need to persist checkpoints.
-        let regions_to_persist = filter_regions_by_replay_size(
-            topic,
-            topic_regions
-                .iter()
-                .map(|(region_id, value)| (*region_id, value.min_entry_id().unwrap_or_default())),
-            avg_record_size as u64,
-            latest_entry_id,
-            self.checkpoint_trigger_size,
-        );
-        let region_manifests = self
-            .leader_region_registry
-            .batch_get(topic_regions.keys().cloned());
-
-        if let Err(err) = self
-            .persist_region_checkpoints(
-                topic,
-                &regions_to_persist,
-                &topic_regions,
-                &region_manifests,
-            )
-            .await
-        {
-            error!(err; "Failed to persist region checkpoints for topic: {}", topic);
-        }
-
         let regions = region_manifests
             .into_iter()
             .map(|(region_id, region)| (region_id, region.manifest.prunable_entry_id()))
@@ -447,6 +494,56 @@ fn filter_regions_by_replay_size<I: Iterator<Item = (RegionId, u64)>>(
     regions_to_flush
 }
 
+/// Filters regions that need periodic checkpoint persistence.
+fn filter_regions_for_periodic_checkpoint<I>(
+    regions: I,
+    last_persisted: &HashMap<RegionId, i64>,
+    now_millis: i64,
+    interval: Duration,
+) -> Vec<RegionId>
+where
+    I: Iterator<Item = RegionId>,
+{
+    let interval_millis = interval.as_millis() as i64;
+    regions
+        .filter(|region_id| {
+            last_persisted
+                .get(region_id)
+                .is_none_or(|last_persist_millis| {
+                    now_millis.saturating_sub(*last_persist_millis) >= interval_millis
+                })
+        })
+        .collect()
+}
+
+/// Merges two region id lists and removes duplicates.
+fn merge_region_ids(left: Vec<RegionId>, right: Vec<RegionId>) -> Vec<RegionId> {
+    left.into_iter()
+        .chain(right)
+        .collect::<HashSet<_>>()
+        .into_iter()
+        .collect()
+}
+
+/// Marks checkpoint persistence timestamps for regions.
+fn mark_checkpoint_persisted(
+    last_persisted: &mut HashMap<RegionId, i64>,
+    region_ids: &[RegionId],
+    now_millis: i64,
+) {
+    for region_id in region_ids {
+        last_persisted.insert(*region_id, now_millis);
+    }
+}
+
+/// Retains checkpoint persistence records for active regions.
+fn retain_checkpoint_persist_records(
+    last_persisted: &mut HashMap<RegionId, i64>,
+    active_region_ids: &HashSet<RegionId>,
+) {
+    last_persisted.retain(|region_id, _| active_region_ids.contains(region_id));
+}
+
 /// Group regions by leader.
 ///
 /// The regions are grouped by the leader of the region.
@@ -527,6 +624,71 @@ mod tests {
         assert!(!is_recent(now - 1001, now, Duration::from_secs(1)));
     }
 
+    #[test]
+    fn test_filter_regions_for_periodic_checkpoint() {
+        let now_millis = 10_000;
+        let interval = Duration::from_secs(5);
+        let regions = vec![region_id(1, 1), region_id(1, 2), region_id(1, 3)];
+        let last_persisted = HashMap::from([
+            (region_id(1, 1), now_millis - 4_000),
+            (region_id(1, 2), now_millis - 5_000),
+        ]);
+
+        let result = filter_regions_for_periodic_checkpoint(
+            regions.into_iter(),
+            &last_persisted,
+            now_millis,
+            interval,
+        );
+
+        assert_eq!(result, vec![region_id(1, 2), region_id(1, 3)]);
+    }
+
+    #[test]
+    fn test_merge_region_ids() {
+        let merged = merge_region_ids(
+            vec![region_id(1, 1), region_id(1, 2)],
+            vec![region_id(1, 2), region_id(1, 3)],
+        );
+        let merged = merged.into_iter().collect::<HashSet<_>>();
+
+        assert_eq!(
+            merged,
+            HashSet::from([region_id(1, 1), region_id(1, 2), region_id(1, 3)])
+        );
+    }
+
+    #[test]
+    fn test_mark_checkpoint_persisted() {
+        let now_millis = 10_000;
+        let mut last_persisted = HashMap::from([(region_id(1, 1), 1_000)]);
+
+        mark_checkpoint_persisted(
+            &mut last_persisted,
+            &[region_id(1, 1), region_id(1, 2)],
+            now_millis,
+        );
+
+        assert_eq!(last_persisted.get(&region_id(1, 1)), Some(&now_millis));
+        assert_eq!(last_persisted.get(&region_id(1, 2)), Some(&now_millis));
+    }
+
+    #[test]
+    fn test_retain_checkpoint_persist_records() {
+        let mut last_persisted = HashMap::from([
+            (region_id(1, 1), 1_000),
+            (region_id(1, 2), 2_000),
+            (region_id(1, 3), 3_000),
+        ]);
+        let active_regions = HashSet::from([region_id(1, 1), region_id(1, 3)]);
+
+        retain_checkpoint_persist_records(&mut last_persisted, &active_regions);
+
+        assert_eq!(last_persisted.len(), 2);
+        assert!(last_persisted.contains_key(&region_id(1, 1)));
+        assert!(last_persisted.contains_key(&region_id(1, 3)));
+    }
+
     fn region_id(table: u32, region: u32) -> RegionId {
         RegionId::new(table, region)
     }
@@ -735,6 +897,76 @@ mod tests {
         assert!(result.is_none());
     }
 
+    #[tokio::test]
+    async fn test_persist_region_checkpoints_returns_written_region_ids() {
+        let kv_backend = Arc::new(MemoryKvBackend::new());
+        let table_metadata_manager = Arc::new(TableMetadataManager::new(kv_backend.clone()));
+        let leader_region_registry = Arc::new(LeaderRegionRegistry::new());
+        let topic_stats_registry = Arc::new(TopicStatsRegistry::default());
+        let mailbox_sequence = SequenceBuilder::new(
+            "test_persist_region_checkpoints_returns_written_region_ids",
+            kv_backend,
+        )
+        .build();
+        let mailbox_ctx = MailboxContext::new(mailbox_sequence);
+
+        let (trigger, _ticker) = RegionFlushTrigger::new(
+            table_metadata_manager.clone(),
+            leader_region_registry,
+            topic_stats_registry,
+            mailbox_ctx.mailbox().clone(),
+            "127.0.0.1:3002".to_string(),
+            ReadableSize(1),
+            ReadableSize(1),
+        );
+
+        let topic = "test_topic";
+        let region_to_write = region_id(1, 1);
+        let region_already_persisted = region_id(1, 2);
+        let region_without_leader = region_id(1, 3);
+        let topic_regions = HashMap::from([
+            (region_to_write, TopicRegionValue::new(None)),
+            (
+                region_already_persisted,
+                TopicRegionValue::new(Some(ReplayCheckpoint::new(100, None))),
+            ),
+            (region_without_leader, TopicRegionValue::new(None)),
+        ]);
+        let leader_regions = HashMap::from([
+            (region_to_write, mito_leader_region(100)),
+            (region_already_persisted, mito_leader_region(100)),
+        ]);
+
+        let written_region_ids = trigger
+            .persist_region_checkpoints(
+                topic,
+                &[
+                    region_to_write,
+                    region_already_persisted,
+                    region_without_leader,
+                ],
+                &topic_regions,
+                &leader_regions,
+            )
+            .await
+            .unwrap();
+
+        assert_eq!(written_region_ids, vec![region_to_write]);
+        let persisted = table_metadata_manager
+            .topic_region_manager()
+            .get(TopicRegionKey::new(region_to_write, topic))
+            .await
+            .unwrap()
+            .unwrap();
+        assert_eq!(persisted.checkpoint, Some(ReplayCheckpoint::new(100, None)));
+        let skipped = table_metadata_manager
+            .topic_region_manager()
+            .get(TopicRegionKey::new(region_already_persisted, topic))
+            .await
+            .unwrap();
+        assert!(skipped.is_none());
+    }
+
     #[tokio::test]
     async fn test_send_flush_instructions_payload_includes_remote_wal_prune_reason() {
         let kv_backend = Arc::new(MemoryKvBackend::new());

src/metric-engine/src/engine.rs

@@ -620,6 +620,7 @@ mod test {
             options: physical_region_option,
             skip_wal_replay: false,
             checkpoint: None,
+            requirements: Default::default(),
         };
         engine
             .handle_request(physical_region_id, RegionRequest::Open(open_request))
@@ -644,6 +645,7 @@ mod test {
             options: HashMap::new(),
             skip_wal_replay: false,
             checkpoint: None,
+            requirements: Default::default(),
         };
         engine
             .handle_request(
@@ -721,6 +723,7 @@ mod test {
             options: physical_region_option,
             skip_wal_replay: false,
             checkpoint: None,
+            requirements: Default::default(),
         };
         // Opening an already opened region should succeed.
         // Since the region is already open, no metadata recovery operations will be performed.
@@ -749,6 +752,7 @@ mod test {
             options: physical_region_option,
             skip_wal_replay: false,
             checkpoint: None,
+            requirements: Default::default(),
         };
         let err = metric_engine
             .handle_request(physical_region_id, RegionRequest::Open(open_request))
@@ -854,6 +858,7 @@ mod test {
                         options: options.clone(),
                         skip_wal_replay: true,
                         checkpoint: None,
+                        requirements: Default::default(),
                     },
                 )
             })

src/metric-engine/src/engine/bulk_insert.rs

@@ -18,9 +18,10 @@ use api::v1::{ArrowIpc, SemanticType};
 use bytes::Bytes;
 use common_grpc::flight::{FlightEncoder, FlightMessage};
 use datatypes::arrow::record_batch::RecordBatch;
-use snafu::{OptionExt, ensure};
+use snafu::{OptionExt, ResultExt, ensure};
 use store_api::codec::PrimaryKeyEncoding;
 use store_api::metadata::RegionMetadataRef;
+use store_api::region_engine::RegionEngine;
 use store_api::region_request::{AffectedRows, RegionBulkInsertsRequest, RegionRequest};
 use store_api::storage::RegionId;
 
@@ -71,8 +72,11 @@ impl MetricEngineInner {
     async fn bulk_insert_physical_region(
         &self,
         region_id: RegionId,
-        request: RegionBulkInsertsRequest,
+        mut request: RegionBulkInsertsRequest,
     ) -> Result<AffectedRows> {
+        // Simply set the aligned schema to the data region schema version to avoid filling missing columns
+        // because that schema should be constant and callers have ensured request has the same schema.
+        request.aligned_schema_version = Some(self.physical_schema_version(region_id).await?);
         self.data_region
             .write_data(region_id, RegionRequest::BulkInserts(request))
             .await
@@ -117,6 +121,8 @@ impl MetricEngineInner {
         let (schema, data_header, payload) = record_batch_to_ipc(&modified_batch)?;
 
         let partition_expr_version = request.partition_expr_version;
+        let aligned_schema_version = Some(self.physical_schema_version(data_region_id).await?);
+
         let request = RegionBulkInsertsRequest {
             region_id: data_region_id,
             payload: modified_batch,
@@ -126,12 +132,22 @@ impl MetricEngineInner {
                 payload,
             },
             partition_expr_version,
+            aligned_schema_version,
         };
         self.data_region
             .write_data(data_region_id, RegionRequest::BulkInserts(request))
             .await
     }
 
+    async fn physical_schema_version(&self, region_id: RegionId) -> Result<u64> {
+        Ok(self
+            .mito
+            .get_metadata(region_id)
+            .await
+            .context(error::MitoReadOperationSnafu)?
+            .schema_version)
+    }
+
     fn resolve_tag_columns_from_metadata(
         &self,
         logical_region_id: RegionId,
@@ -282,6 +298,7 @@ mod tests {
                 payload,
             },
             partition_expr_version: None,
+            aligned_schema_version: None,
         })
     }
 
@@ -319,6 +336,7 @@ mod tests {
             payload: batch,
             raw_data: ArrowIpc::default(),
             partition_expr_version: None,
+            aligned_schema_version: None,
         });
         let response = env
             .metric()

src/metric-engine/src/engine/open.rs

@@ -222,6 +222,7 @@ impl MetricEngineInner {
                 entry_id: checkpoint.metadata_entry_id.unwrap_or_default(),
                 metadata_entry_id: None,
             }),
+            requirements: request.requirements,
         };
 
         let mut data_region_options = request.options;
@@ -239,6 +240,7 @@ impl MetricEngineInner {
                 entry_id: checkpoint.entry_id,
                 metadata_entry_id: None,
             }),
+            requirements: request.requirements,
         };
 
         (open_metadata_region_request, open_data_region_request)

src/metric-engine/src/engine/sync/region.rs

@@ -321,6 +321,7 @@ mod tests {
                     options: physical_region_option,
                     skip_wal_replay: false,
                     checkpoint: None,
+                    requirements: Default::default(),
                 }),
             )
             .await

src/metric-engine/src/test_util.rs

@@ -144,6 +144,7 @@ impl TestEnv {
                     options: physical_region_option,
                     skip_wal_replay: true,
                     checkpoint: None,
+                    requirements: Default::default(),
                 }),
             )
             .await

src/mito2/Cargo.toml

@@ -8,6 +8,7 @@ license.workspace = true
 default = []
 test = ["common-test-util", "rstest", "rstest_reuse", "rskafka"]
 testing = ["test"]
+test-shared-fs-region-migration = []
 enterprise = []
 vector_index = ["dep:roaring", "index/vector_index"]
 
@@ -50,6 +51,7 @@ datafusion-common.workspace = true
 datafusion-expr.workspace = true
 datatypes.workspace = true
 dashmap.workspace = true
+derive_more.workspace = true
 dotenv.workspace = true
 either.workspace = true
 futures.workspace = true

src/mito2/src/cache.rs

@@ -23,9 +23,10 @@ pub(crate) mod manifest_cache;
 pub(crate) mod test_util;
 pub(crate) mod write_cache;
 
+use std::collections::{BTreeMap, HashMap};
 use std::mem;
 use std::ops::Range;
-use std::sync::Arc;
+use std::sync::{Arc, RwLock};
 
 use bytes::Bytes;
 use common_base::readable_size::ReadableSize;
@@ -527,20 +528,33 @@ impl CacheStrategy {
         }
     }
 
-    /// Calls [CacheManager::get_pages()].
+    /// Calls [CacheManager::get_page_ranges()].
     /// It returns None if the strategy is [CacheStrategy::Compaction] or [CacheStrategy::Disabled].
-    pub fn get_pages(&self, page_key: &PageKey) -> Option<Arc<PageValue>> {
+    pub fn get_page_ranges(
+        &self,
+        file_id: FileId,
+        row_group_idx: usize,
+        ranges: &[Range<u64>],
+    ) -> Option<PageRangeLookup> {
         match self {
-            CacheStrategy::EnableAll(cache_manager) => cache_manager.get_pages(page_key),
+            CacheStrategy::EnableAll(cache_manager) => {
+                cache_manager.get_page_ranges(file_id, row_group_idx, ranges)
+            }
             CacheStrategy::Compaction(_) | CacheStrategy::Disabled => None,
         }
     }
 
-    /// Calls [CacheManager::put_pages()].
+    /// Calls [CacheManager::put_page_ranges()].
     /// It does nothing if the strategy isn't [CacheStrategy::EnableAll].
-    pub fn put_pages(&self, page_key: PageKey, pages: Arc<PageValue>) {
+    pub fn put_page_ranges(
+        &self,
+        file_id: FileId,
+        row_group_idx: usize,
+        ranges: &[Range<u64>],
+        pages: &[Bytes],
+    ) {
         if let CacheStrategy::EnableAll(cache_manager) = self {
-            cache_manager.put_pages(page_key, pages);
+            cache_manager.put_page_ranges(file_id, row_group_idx, ranges, pages);
         }
     }
 
@@ -720,8 +734,8 @@ pub struct CacheManager {
     sst_meta_cache: Option<SstMetaCache>,
     /// Cache for vectors.
     vector_cache: Option<VectorCache>,
-    /// Cache for SST pages.
-    page_cache: Option<PageCache>,
+    /// Cache for SST byte ranges.
+    page_cache: Option<Arc<PageRangeCache>>,
     /// A Cache for writing files to object stores.
     write_cache: Option<WriteCacheRef>,
     /// Cache for inverted index.
@@ -904,21 +918,35 @@ impl CacheManager {
         }
     }
 
-    /// Gets pages for the row group.
-    pub fn get_pages(&self, page_key: &PageKey) -> Option<Arc<PageValue>> {
-        self.page_cache.as_ref().and_then(|page_cache| {
-            let value = page_cache.get(page_key);
-            update_hit_miss(value, PAGE_TYPE)
+    /// Gets cached byte fragments for the requested ranges.
+    pub fn get_page_ranges(
+        &self,
+        file_id: FileId,
+        row_group_idx: usize,
+        ranges: &[Range<u64>],
+    ) -> Option<PageRangeLookup> {
+        self.page_cache.as_ref().map(|page_cache| {
+            let lookup = page_cache.lookup(file_id, row_group_idx, ranges);
+            if lookup.cached_bytes > 0 {
+                CACHE_HIT.with_label_values(&[PAGE_TYPE]).inc();
+            }
+            if !lookup.missing_ranges.is_empty() {
+                CACHE_MISS.with_label_values(&[PAGE_TYPE]).inc();
+            }
+            lookup
         })
     }
 
-    /// Puts pages of the row group into the cache.
-    pub fn put_pages(&self, page_key: PageKey, pages: Arc<PageValue>) {
+    /// Puts byte fragments into the page cache.
+    pub fn put_page_ranges(
+        &self,
+        file_id: FileId,
+        row_group_idx: usize,
+        ranges: &[Range<u64>],
+        pages: &[Bytes],
+    ) {
         if let Some(cache) = &self.page_cache {
-            CACHE_BYTES
-                .with_label_values(&[PAGE_TYPE])
-                .add(page_cache_weight(&page_key, &pages).into());
-            cache.insert(page_key, pages);
+            cache.insert_ranges(file_id, row_group_idx, ranges, pages);
         }
     }
 
@@ -1192,19 +1220,8 @@ impl CacheManagerBuilder {
                 })
                 .build()
         });
-        let page_cache = (self.page_cache_size != 0).then(|| {
-            Cache::builder()
-                .max_capacity(self.page_cache_size)
-                .weigher(page_cache_weight)
-                .eviction_listener(|k, v, cause| {
-                    let size = page_cache_weight(&k, &v);
-                    CACHE_BYTES.with_label_values(&[PAGE_TYPE]).sub(size.into());
-                    CACHE_EVICTION
-                        .with_label_values(&[PAGE_TYPE, removal_cause_str(cause)])
-                        .inc();
-                })
-                .build()
-        });
+        let page_cache =
+            (self.page_cache_size != 0).then(|| PageRangeCache::new(self.page_cache_size));
         let inverted_index_cache = InvertedIndexCache::new(
             self.index_metadata_size,
             self.index_content_size,
@@ -1288,8 +1305,8 @@ fn vector_cache_weight(_k: &(ConcreteDataType, Value), v: &VectorRef) -> u32 {
     (mem::size_of::<ConcreteDataType>() + mem::size_of::<Value>() + v.memory_size()) as u32
 }
 
-fn page_cache_weight(k: &PageKey, v: &Arc<PageValue>) -> u32 {
-    (k.estimated_size() + v.estimated_size()) as u32
+fn page_cache_weight(k: &PageFragmentKey, v: &Bytes) -> u32 {
+    (k.estimated_size() + mem::size_of::<Bytes>() + v.len()) as u32
 }
 
 fn selector_result_cache_weight(k: &SelectorResultKey, v: &Arc<SelectorResultValue>) -> u32 {
@@ -1321,73 +1338,281 @@ impl SstMetaKey {
     }
 }
 
-/// Path to column pages in the SST file.
-#[derive(Debug, Clone, PartialEq, Eq, Hash)]
-pub struct ColumnPagePath {
-    /// Region id of the SST file to cache.
-    region_id: RegionId,
-    /// Id of the SST file to cache.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+struct PageFragmentGroupKey {
+    file_id: FileId,
+    row_group_idx: usize,
+}
+
+/// Cache key for one byte fragment in an SST row group.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub struct PageFragmentKey {
+    /// Id of the SST file.
     file_id: FileId,
     /// Index of the row group.
     row_group_idx: usize,
-    /// Index of the column in the row group.
-    column_idx: usize,
+    /// Start offset of the cached byte fragment.
+    start: u64,
+    /// End offset of the cached byte fragment.
+    end: u64,
 }
 
-/// Cache key to pages in a row group (after projection).
-///
-/// Different projections will have different cache keys.
-/// We cache all ranges together because they may refer to the same `Bytes`.
-#[derive(Debug, Clone, PartialEq, Eq, Hash)]
-pub struct PageKey {
-    /// Id of the SST file to cache.
-    file_id: FileId,
-    /// Index of the row group.
-    row_group_idx: usize,
-    /// Byte ranges of the pages to cache.
-    ranges: Vec<Range<u64>>,
-}
-
-impl PageKey {
-    /// Creates a key for a list of pages.
-    pub fn new(file_id: FileId, row_group_idx: usize, ranges: Vec<Range<u64>>) -> PageKey {
-        PageKey {
+impl PageFragmentKey {
+    fn new(file_id: FileId, row_group_idx: usize, range: &Range<u64>) -> PageFragmentKey {
+        PageFragmentKey {
             file_id,
             row_group_idx,
-            ranges,
+            start: range.start,
+            end: range.end,
+        }
+    }
+
+    fn group_key(&self) -> PageFragmentGroupKey {
+        PageFragmentGroupKey {
+            file_id: self.file_id,
+            row_group_idx: self.row_group_idx,
         }
     }
 
     /// Returns memory used by the key (estimated).
     fn estimated_size(&self) -> usize {
-        mem::size_of::<Self>() + mem::size_of_val(self.ranges.as_slice())
+        mem::size_of::<Self>()
     }
 }
 
-/// Cached row group pages for a column.
-// We don't use enum here to make it easier to mock and use the struct.
-#[derive(Default)]
-pub struct PageValue {
-    /// Compressed page in the row group.
-    pub compressed: Vec<Bytes>,
-    /// Total size of the pages (may be larger than sum of compressed bytes due to gaps).
-    pub page_size: u64,
+/// One cached byte fragment that overlaps a requested range.
+#[derive(Clone)]
+pub struct PageRangePart {
+    /// Range covered by `bytes`.
+    pub range: Range<u64>,
+    /// Bytes for `range`.
+    pub bytes: Bytes,
 }
 
-impl PageValue {
-    /// Creates a new value from a range of compressed pages.
-    pub fn new(bytes: Vec<Bytes>, page_size: u64) -> PageValue {
-        PageValue {
-            compressed: bytes,
-            page_size,
+/// Result of looking up request ranges in the page range cache.
+pub struct PageRangeLookup {
+    /// Cached fragments grouped by the original requested range index.
+    pub cached_parts: Vec<Vec<PageRangePart>>,
+    /// Ranges that are not covered by cached fragments and need fetching.
+    pub missing_ranges: Vec<Range<u64>>,
+    /// Number of cached fragments used.
+    pub cached_range_count: usize,
+    /// Number of requested bytes served from cached fragments.
+    pub cached_bytes: u64,
+}
+
+impl PageRangeLookup {
+    pub fn is_fully_cached(&self) -> bool {
+        self.missing_ranges.is_empty()
+    }
+}
+
+type PageFragmentRangeIndex = BTreeMap<(u64, u64), PageFragmentKey>;
+type PageFragmentIndex = HashMap<PageFragmentGroupKey, PageFragmentRangeIndex>;
+
+/// Byte-fragment cache for Parquet row-group reads.
+pub struct PageRangeCache {
+    cache: Cache<PageFragmentKey, Bytes>,
+    index: RwLock<PageFragmentIndex>,
+}
+
+impl PageRangeCache {
+    fn new(capacity: u64) -> Arc<PageRangeCache> {
+        Arc::new_cyclic(|weak_cache: &std::sync::Weak<PageRangeCache>| {
+            let cache = Cache::builder()
+                .max_capacity(capacity)
+                .weigher(page_cache_weight)
+                .eviction_listener({
+                    let weak_cache = weak_cache.clone();
+                    move |k, v, cause| {
+                        let size = page_cache_weight(&k, &v);
+                        CACHE_BYTES.with_label_values(&[PAGE_TYPE]).sub(size.into());
+                        CACHE_EVICTION
+                            .with_label_values(&[PAGE_TYPE, removal_cause_str(cause)])
+                            .inc();
+
+                        if let Some(cache) = weak_cache.upgrade()
+                            && !matches!(cause, RemovalCause::Replaced)
+                        {
+                            cache.remove_index_entry(*k);
+                        }
+                    }
+                })
+                .build();
+
+            PageRangeCache {
+                cache,
+                index: RwLock::new(HashMap::new()),
+            }
+        })
+    }
+
+    fn lookup(
+        &self,
+        file_id: FileId,
+        row_group_idx: usize,
+        ranges: &[Range<u64>],
+    ) -> PageRangeLookup {
+        let mut cached_parts = Vec::with_capacity(ranges.len());
+        let mut missing_ranges = Vec::new();
+        let mut cached_range_count = 0;
+        let mut cached_bytes = 0;
+
+        for range in ranges {
+            if range.start >= range.end {
+                cached_parts.push(Vec::new());
+                continue;
+            }
+
+            let mut parts = Vec::new();
+            let candidates = self.find_index_candidates(file_id, row_group_idx, range);
+            let mut stale_keys = Vec::new();
+
+            for fragment_key in candidates {
+                if let Some(bytes) = self.cache.get(&fragment_key) {
+                    let part_start = range.start.max(fragment_key.start);
+                    let part_end = range.end.min(fragment_key.end);
+                    let slice_start = (part_start - fragment_key.start) as usize;
+                    let slice_end = (part_end - fragment_key.start) as usize;
+                    parts.push(PageRangePart {
+                        range: part_start..part_end,
+                        bytes: bytes.slice(slice_start..slice_end),
+                    });
+                } else {
+                    stale_keys.push(fragment_key);
+                }
+            }
+            for key in stale_keys {
+                self.remove_uncached_index_entry(key);
+            }
+
+            let mut cursor = range.start;
+            let mut compacted_parts: Vec<PageRangePart> = Vec::with_capacity(parts.len());
+            for part in parts {
+                if part.range.end <= cursor {
+                    continue;
+                }
+
+                let part = if part.range.start < cursor {
+                    let offset = (cursor - part.range.start) as usize;
+                    PageRangePart {
+                        range: cursor..part.range.end,
+                        bytes: part.bytes.slice(offset..),
+                    }
+                } else {
+                    part
+                };
+
+                if cursor < part.range.start {
+                    missing_ranges.push(cursor..part.range.start);
+                }
+                cached_bytes += part.range.end - part.range.start;
+                cached_range_count += 1;
+                cursor = part.range.end;
+                compacted_parts.push(part);
+
+                if cursor >= range.end {
+                    break;
+                }
+            }
+
+            if cursor < range.end {
+                missing_ranges.push(cursor..range.end);
+            }
+            cached_parts.push(compacted_parts);
+        }
+
+        PageRangeLookup {
+            cached_parts,
+            missing_ranges,
+            cached_range_count,
+            cached_bytes,
         }
     }
 
-    /// Returns memory used by the value (estimated).
-    fn estimated_size(&self) -> usize {
-        mem::size_of::<Self>()
-            + self.page_size as usize
-            + self.compressed.iter().map(mem::size_of_val).sum::<usize>()
+    fn insert_ranges(
+        &self,
+        file_id: FileId,
+        row_group_idx: usize,
+        ranges: &[Range<u64>],
+        pages: &[Bytes],
+    ) {
+        for (range, bytes) in ranges.iter().zip(pages) {
+            if range.start >= range.end || bytes.len() as u64 != range.end - range.start {
+                continue;
+            }
+
+            let key = PageFragmentKey::new(file_id, row_group_idx, range);
+            let bytes = Bytes::copy_from_slice(bytes.as_ref());
+            let size = page_cache_weight(&key, &bytes);
+            CACHE_BYTES.with_label_values(&[PAGE_TYPE]).add(size.into());
+            self.cache.insert(key, bytes);
+            let mut index = self.index.write().unwrap();
+            index
+                .entry(key.group_key())
+                .or_default()
+                .insert((key.start, key.end), key);
+        }
+    }
+
+    fn find_index_candidates(
+        &self,
+        file_id: FileId,
+        row_group_idx: usize,
+        range: &Range<u64>,
+    ) -> Vec<PageFragmentKey> {
+        let group_key = PageFragmentGroupKey {
+            file_id,
+            row_group_idx,
+        };
+        let index = self.index.read().unwrap();
+        index
+            .get(&group_key)
+            .map(|ranges| {
+                ranges
+                    .range(..(range.end, 0))
+                    .filter_map(|(_, fragment_key)| {
+                        (fragment_key.end > range.start).then_some(*fragment_key)
+                    })
+                    .collect()
+            })
+            .unwrap_or_default()
+    }
+
+    fn remove_uncached_index_entry(&self, key: PageFragmentKey) {
+        let group_key = key.group_key();
+        let mut index = self.index.write().unwrap();
+        if self.cache.contains_key(&key) {
+            return;
+        }
+
+        Self::remove_index_entry_locked(&mut index, group_key, key);
+    }
+
+    fn remove_index_entry(&self, key: PageFragmentKey) {
+        let group_key = key.group_key();
+        let mut index = self.index.write().unwrap();
+        Self::remove_index_entry_locked(&mut index, group_key, key);
+    }
+
+    fn remove_index_entry_locked(
+        index: &mut PageFragmentIndex,
+        group_key: PageFragmentGroupKey,
+        key: PageFragmentKey,
+    ) {
+        let Some(ranges) = index.get_mut(&group_key) else {
+            return;
+        };
+
+        let removed = ranges
+            .get(&(key.start, key.end))
+            .is_some_and(|current| current == &key);
+        if removed {
+            ranges.remove(&(key.start, key.end));
+        }
+        if ranges.is_empty() {
+            index.remove(&group_key);
+        }
     }
 }
 
@@ -1455,8 +1680,6 @@ type SstMetaCache = Cache<SstMetaKey, Arc<CachedSstMeta>>;
 ///
 /// e.g. `"hello" => ["hello", "hello", "hello"]`
 type VectorCache = Cache<(ConcreteDataType, Value), VectorRef>;
-/// Maps (region, file, row group, column) to [PageValue].
-type PageCache = Cache<PageKey, Arc<PageValue>>;
 /// Maps (file id, row group id, time series row selector) to [SelectorResultValue].
 type SelectorResultCache = Cache<SelectorResultKey, Arc<SelectorResultValue>>;
 /// Maps partition-range scan key to cached flat batches.
@@ -1514,10 +1737,17 @@ mod tests {
                 .is_none()
         );
 
-        let key = PageKey::new(file_id.file_id(), 1, vec![Range { start: 0, end: 5 }]);
-        let pages = Arc::new(PageValue::default());
-        cache.put_pages(key.clone(), pages);
-        assert!(cache.get_pages(&key).is_none());
+        cache.put_page_ranges(
+            file_id.file_id(),
+            1,
+            &[Range { start: 0, end: 5 }],
+            &[Bytes::from_static(b"abcde")],
+        );
+        assert!(
+            cache
+                .get_page_ranges(file_id.file_id(), 1, &[Range { start: 0, end: 5 }])
+                .is_none()
+        );
 
         assert!(cache.write_cache().is_none());
     }
@@ -1675,11 +1905,168 @@ mod tests {
     fn test_page_cache() {
         let cache = CacheManager::builder().page_cache_size(1000).build();
         let file_id = FileId::random();
-        let key = PageKey::new(file_id, 0, vec![(0..10), (10..20)]);
-        assert!(cache.get_pages(&key).is_none());
-        let pages = Arc::new(PageValue::default());
-        cache.put_pages(key.clone(), pages);
-        assert!(cache.get_pages(&key).is_some());
+        let uncached = 0..10;
+        assert_eq!(
+            vec![0..10],
+            cache
+                .get_page_ranges(file_id, 0, std::slice::from_ref(&uncached))
+                .unwrap()
+                .missing_ranges
+        );
+
+        let cached = 100..500;
+        cache.put_page_ranges(
+            file_id,
+            0,
+            std::slice::from_ref(&cached),
+            &[Bytes::from(vec![7; 400])],
+        );
+
+        let subrange = 200..300;
+        let lookup = cache
+            .get_page_ranges(file_id, 0, std::slice::from_ref(&subrange))
+            .unwrap();
+        assert!(lookup.is_fully_cached());
+        assert_eq!(100, lookup.cached_bytes);
+        assert_eq!(1, lookup.cached_parts.len());
+        assert_eq!(200..300, lookup.cached_parts[0][0].range);
+        assert_eq!(100, lookup.cached_parts[0][0].bytes.len());
+
+        let overlapping = 400..600;
+        let lookup = cache
+            .get_page_ranges(file_id, 0, std::slice::from_ref(&overlapping))
+            .unwrap();
+        assert!(!lookup.is_fully_cached());
+        assert_eq!(100, lookup.cached_bytes);
+        assert_eq!(vec![500..600], lookup.missing_ranges);
+        assert_eq!(400..500, lookup.cached_parts[0][0].range);
+    }
+
+    #[test]
+    fn test_page_cache_detaches_fragment_bytes() {
+        let cache = PageRangeCache::new(1000);
+        let file_id = FileId::random();
+        let backing = Bytes::from(vec![1; 1024]);
+        let page = backing.slice(512..522);
+        let page_ptr = page.as_ptr();
+        let range = 0..10;
+
+        cache.insert_ranges(
+            file_id,
+            0,
+            std::slice::from_ref(&range),
+            std::slice::from_ref(&page),
+        );
+
+        let lookup = cache.lookup(file_id, 0, std::slice::from_ref(&range));
+        assert!(lookup.is_fully_cached());
+        assert_eq!(1, lookup.cached_parts[0].len());
+        assert_eq!(&page[..], &lookup.cached_parts[0][0].bytes[..]);
+        assert_ne!(page_ptr, lookup.cached_parts[0][0].bytes.as_ptr());
+    }
+
+    #[test]
+    fn test_page_cache_replaces_fragment() {
+        let cache = PageRangeCache::new(1000);
+        let file_id = FileId::random();
+        let range = 0..10;
+
+        cache.insert_ranges(
+            file_id,
+            0,
+            std::slice::from_ref(&range),
+            &[Bytes::from(vec![1; 10])],
+        );
+        cache.insert_ranges(
+            file_id,
+            0,
+            std::slice::from_ref(&range),
+            &[Bytes::from(vec![2; 10])],
+        );
+        cache.cache.run_pending_tasks();
+        assert_eq!(
+            vec![PageFragmentKey::new(file_id, 0, &range)],
+            cache.find_index_candidates(file_id, 0, &range)
+        );
+
+        let lookup = cache.lookup(file_id, 0, std::slice::from_ref(&range));
+        assert!(lookup.is_fully_cached());
+        assert_eq!(&vec![2; 10][..], &lookup.cached_parts[0][0].bytes[..]);
+    }
+
+    #[test]
+    fn test_page_cache_retains_disjoint_inserts_for_same_row_group() {
+        let cache = PageRangeCache::new(1000);
+        let file_id = FileId::random();
+        let range1 = 0..10;
+        let range2 = 20..30;
+
+        cache.insert_ranges(
+            file_id,
+            0,
+            std::slice::from_ref(&range1),
+            &[Bytes::from(vec![1; 10])],
+        );
+        cache.insert_ranges(
+            file_id,
+            0,
+            std::slice::from_ref(&range2),
+            &[Bytes::from(vec![2; 10])],
+        );
+
+        let lookup = cache.lookup(file_id, 0, &[range1, range2]);
+        assert!(lookup.is_fully_cached());
+        assert_eq!(2, lookup.cached_range_count);
+        assert_eq!(&vec![1; 10][..], &lookup.cached_parts[0][0].bytes[..]);
+        assert_eq!(&vec![2; 10][..], &lookup.cached_parts[1][0].bytes[..]);
+    }
+
+    #[test]
+    fn test_page_cache_fragment_eviction() {
+        let file_id = FileId::random();
+        let range = 0..10;
+        let key = PageFragmentKey::new(file_id, 0, &range);
+        let page = Bytes::from(vec![1; 10]);
+        let cache = PageRangeCache::new(page_cache_weight(&key, &page) as u64);
+
+        cache.insert_ranges(
+            file_id,
+            0,
+            std::slice::from_ref(&range),
+            &[Bytes::from(vec![1; 10])],
+        );
+        assert!(
+            cache
+                .lookup(file_id, 0, std::slice::from_ref(&range))
+                .is_fully_cached()
+        );
+
+        cache.cache.invalidate(&key);
+        cache.cache.run_pending_tasks();
+        assert!(cache.find_index_candidates(file_id, 0, &range).is_empty());
+
+        let lookup = cache.lookup(file_id, 0, std::slice::from_ref(&range));
+        assert!(!lookup.is_fully_cached());
+        assert_eq!(vec![0..10], lookup.missing_ranges);
+    }
+
+    #[test]
+    fn test_page_cache_rejects_oversized_fragment() {
+        let cache = PageRangeCache::new(1);
+        let file_id = FileId::random();
+        let range = 0..10;
+
+        cache.insert_ranges(
+            file_id,
+            0,
+            std::slice::from_ref(&range),
+            &[Bytes::from(vec![1; 10])],
+        );
+        cache.cache.run_pending_tasks();
+
+        let lookup = cache.lookup(file_id, 0, std::slice::from_ref(&range));
+        assert!(!lookup.is_fully_cached());
+        assert_eq!(vec![0..10], lookup.missing_ranges);
     }
 
     #[test]

src/mito2/src/compaction.rs

@@ -150,6 +150,7 @@ impl CompactionScheduler {
     }
 
     /// Schedules a compaction for the region.
+    /// Returns whether a compaction is scheduled.
     #[allow(clippy::too_many_arguments)]
     pub(crate) async fn schedule_compaction(
         &mut self,
@@ -161,7 +162,7 @@ impl CompactionScheduler {
         manifest_ctx: &ManifestContextRef,
         schema_metadata_manager: SchemaMetadataManagerRef,
         max_parallelism: usize,
-    ) -> Result<()> {
+    ) -> Result<bool> {
         // skip compaction if region is in staging state
         let current_state = manifest_ctx.current_state();
         if current_state == RegionRoleState::Leader(RegionLeaderState::Staging) {
@@ -170,7 +171,7 @@ impl CompactionScheduler {
                 region_id, compact_options
             );
             waiter.send(Ok(0));
-            return Ok(());
+            return Ok(false);
         }
 
         if let Some(status) = self.region_status.get_mut(&region_id) {
@@ -192,7 +193,7 @@ impl CompactionScheduler {
                     );
                 }
             }
-            return Ok(());
+            return Ok(false);
         }
 
         // The region can compact directly.
@@ -209,7 +210,7 @@ impl CompactionScheduler {
             max_parallelism,
         );
 
-        let result = match self
+        match self
             .schedule_compaction_request(request, compact_options)
             .await
         {
@@ -220,14 +221,12 @@ impl CompactionScheduler {
                 status.active_compaction = Some(active_compaction);
                 self.region_status.insert(region_id, status);
 
-                Ok(())
+                self.listener.on_compaction_scheduled(region_id);
+                Ok(true)
             }
-            Ok(None) => Ok(()),
+            Ok(None) => Ok(false),
             Err(e) => Err(e),
-        };
-
-        self.listener.on_compaction_scheduled(region_id);
-        result
+        }
     }
 
     // Handle pending manual compaction request for the region.
@@ -334,6 +333,27 @@ impl CompactionScheduler {
             // And skip try to schedule next compaction task.
             return pending_ddl_requests;
         }
+        Vec::new()
+    }
+
+    pub(crate) fn is_compacting(&self, region_id: RegionId) -> bool {
+        self.region_status
+            .get(&region_id)
+            .map(|status| status.active_compaction.is_some())
+            .unwrap_or(false)
+    }
+
+    /// Schedules next compaction upon a finished compaction.
+    /// Returns whether the compaction is scheduled.
+    pub(crate) async fn schedule_next_compaction(
+        &mut self,
+        region_id: RegionId,
+        manifest_ctx: &ManifestContextRef,
+        schema_metadata_manager: SchemaMetadataManagerRef,
+    ) -> bool {
+        let Some(status) = self.region_status.get_mut(&region_id) else {
+            return false;
+        };
 
         // We should always try to compact the region until picker returns None.
         let request = status.new_compaction_request(
@@ -364,20 +384,21 @@ impl CompactionScheduler {
                     "Successfully scheduled next compaction for region id: {}",
                     region_id
                 );
+                true
             }
             Ok(None) => {
                 // No further compaction tasks can be scheduled; cleanup the `CompactionStatus` for this region.
                 // All DDL requests and pending compaction requests have already been processed.
                 // Safe to remove the region from status tracking.
                 self.region_status.remove(&region_id);
+                false
             }
             Err(e) => {
                 error!(e; "Failed to schedule next compaction for region {}", region_id);
                 self.remove_region_on_failure(region_id, Arc::new(e));
+                false
             }
         }
-
-        Vec::new()
     }
 
     /// Notifies the scheduler that the compaction job is cancelled cooperatively.
@@ -1434,7 +1455,7 @@ mod tests {
         let manifest_ctx = env
             .mock_manifest_context(version_control.current().version.metadata.clone())
             .await;
-        scheduler
+        let scheduled = scheduler
             .schedule_compaction(
                 builder.region_id(),
                 compact_request::Options::Regular(Default::default()),
@@ -1447,6 +1468,7 @@ mod tests {
             )
             .await
             .unwrap();
+        assert!(!scheduled);
         let output = output_rx.await.unwrap().unwrap();
         assert_eq!(output, 0);
         assert!(scheduler.region_status.is_empty());
@@ -1455,7 +1477,7 @@ mod tests {
         let version_control = Arc::new(builder.push_l0_file(0, 1000).build());
         let (output_tx, output_rx) = oneshot::channel();
         let waiter = OptionOutputTx::from(output_tx);
-        scheduler
+        let scheduled = scheduler
             .schedule_compaction(
                 builder.region_id(),
                 compact_request::Options::Regular(Default::default()),
@@ -1468,11 +1490,67 @@ mod tests {
             )
             .await
             .unwrap();
+        assert!(!scheduled);
         let output = output_rx.await.unwrap().unwrap();
         assert_eq!(output, 0);
         assert!(scheduler.region_status.is_empty());
     }
 
+    #[tokio::test]
+    async fn test_schedule_compaction_returns_true_when_task_scheduled() {
+        let job_scheduler = Arc::new(VecScheduler::default());
+        let env = SchedulerEnv::new().await.scheduler(job_scheduler.clone());
+        let (tx, _rx) = mpsc::channel(4);
+        let mut scheduler = env.mock_compaction_scheduler(tx);
+        let mut builder = VersionControlBuilder::new();
+        let region_id = builder.region_id();
+        let end = 1000 * 1000;
+        // Five overlapping L0 files are enough for the regular picker to create a task.
+        let version_control = Arc::new(
+            builder
+                .push_l0_file(0, end)
+                .push_l0_file(10, end)
+                .push_l0_file(50, end)
+                .push_l0_file(80, end)
+                .push_l0_file(90, end)
+                .build(),
+        );
+        let manifest_ctx = env
+            .mock_manifest_context(version_control.current().version.metadata.clone())
+            .await;
+        let (schema_metadata_manager, kv_backend) = mock_schema_metadata_manager();
+        schema_metadata_manager
+            .register_region_table_info(
+                region_id.table_id(),
+                "test_table",
+                "test_catalog",
+                "test_schema",
+                None,
+                kv_backend,
+            )
+            .await;
+
+        let scheduled = scheduler
+            .schedule_compaction(
+                region_id,
+                Options::Regular(Default::default()),
+                &version_control,
+                &env.access_layer,
+                OptionOutputTx::none(),
+                &manifest_ctx,
+                schema_metadata_manager,
+                1,
+            )
+            .await
+            .unwrap();
+
+        // The boolean result is what the worker uses to decide whether to update
+        // last_schedule_compaction_millis.
+        assert!(scheduled);
+        assert_eq!(1, job_scheduler.num_jobs());
+        assert!(scheduler.region_status.contains_key(&region_id));
+    }
+
     #[tokio::test]
     async fn test_schedule_on_finished() {
         common_telemetry::init_default_ut_logging();
@@ -1510,7 +1588,7 @@ mod tests {
         let manifest_ctx = env
             .mock_manifest_context(version_control.current().version.metadata.clone())
             .await;
-        scheduler
+        let scheduled = scheduler
             .schedule_compaction(
                 region_id,
                 compact_request::Options::Regular(Default::default()),
@@ -1524,6 +1602,7 @@ mod tests {
             .await
             .unwrap();
         // Should schedule 1 compaction.
+        assert!(scheduled);
         assert_eq!(1, scheduler.region_status.len());
         assert_eq!(1, job_scheduler.num_jobs());
         let data = version_control.current();
@@ -1542,7 +1621,7 @@ mod tests {
         );
         // The task is pending.
         let (tx, _rx) = oneshot::channel();
-        scheduler
+        let scheduled = scheduler
             .schedule_compaction(
                 region_id,
                 compact_request::Options::Regular(Default::default()),
@@ -1555,6 +1634,7 @@ mod tests {
             )
             .await
             .unwrap();
+        assert!(!scheduled);
         assert_eq!(1, scheduler.region_status.len());
         assert_eq!(1, job_scheduler.num_jobs());
         assert!(
@@ -1570,6 +1650,10 @@ mod tests {
         scheduler
             .on_compaction_finished(region_id, &manifest_ctx, schema_metadata_manager.clone())
             .await;
+        let scheduled = scheduler
+            .schedule_next_compaction(region_id, &manifest_ctx, schema_metadata_manager.clone())
+            .await;
+        assert!(scheduled);
         assert_eq!(1, scheduler.region_status.len());
         assert_eq!(2, job_scheduler.num_jobs());
 
@@ -1582,7 +1666,7 @@ mod tests {
         );
         let (tx, _rx) = oneshot::channel();
         // The task is pending.
-        scheduler
+        let scheduled = scheduler
             .schedule_compaction(
                 region_id,
                 compact_request::Options::Regular(Default::default()),
@@ -1595,6 +1679,7 @@ mod tests {
             )
             .await
             .unwrap();
+        assert!(!scheduled);
         assert_eq!(2, job_scheduler.num_jobs());
         assert!(
             !scheduler
@@ -2328,6 +2413,15 @@ mod tests {
             .await;
 
         assert!(pending_ddls.is_empty());
+        assert!(scheduler.region_status.contains_key(&region_id));
+
+        let (schema_metadata_manager, _kv_backend) = mock_schema_metadata_manager();
+        // With no compactable files, next scheduling returns false and removes
+        // the status without creating a background task.
+        let scheduled = scheduler
+            .schedule_next_compaction(region_id, &manifest_ctx, schema_metadata_manager)
+            .await;
+        assert!(!scheduled);
         assert!(!scheduler.region_status.contains_key(&region_id));
     }
 
@@ -2370,6 +2464,14 @@ mod tests {
             .await;
 
         assert!(pending_ddls.is_empty());
+        assert!(scheduler.region_status.contains_key(&region_id));
+
+        let (schema_metadata_manager, _kv_backend) = mock_schema_metadata_manager();
+        // The failing scheduler simulates a submit error; callers must see false.
+        let scheduled = scheduler
+            .schedule_next_compaction(region_id, &manifest_ctx, schema_metadata_manager)
+            .await;
+        assert!(!scheduled);
         assert!(!scheduler.region_status.contains_key(&region_id));
     }
 

src/mito2/src/compaction/run.rs

@@ -15,6 +15,9 @@
 //! This file contains code to find sorted runs in a set if ranged items and
 //! along with the best way to merge these items to satisfy the desired run count.
 
+use std::cmp::Ordering;
+use std::collections::BinaryHeap;
+
 use bytes::{Buf, Bytes};
 use common_base::BitVec;
 use common_base::readable_size::ReadableSize;
@@ -423,6 +426,133 @@ where
     runs
 }
 
+pub(crate) fn find_sorted_runs_by_time_range<T>(items: &mut [T]) -> Vec<SortedRun<T>>
+where
+    T: Item,
+{
+    if items.is_empty() {
+        return vec![];
+    }
+    sort_ranged_items(items);
+
+    use derive_more::{Eq, PartialEq};
+
+    /// `SortedRun` with a creation sequence `i`.
+    #[derive(PartialEq, Eq)]
+    struct Run<T: Item> {
+        i: usize,
+        #[partial_eq(skip)]
+        run: SortedRun<T>,
+    }
+
+    impl<T: Item> Run<T> {
+        fn new(i: usize, item: &T) -> Run<T> {
+            let mut run = SortedRun::default();
+            run.push_item(item.clone());
+            Run { i, run }
+        }
+
+        fn push_item(&mut self, item: &T) {
+            self.run.push_item(item.clone());
+        }
+    }
+
+    impl<T: Item> PartialOrd for Run<T> {
+        fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
+            Some(self.cmp(other))
+        }
+    }
+
+    /// Sort by run's `end` desc then `start` asc.
+    impl<T: Item> Ord for Run<T> {
+        fn cmp(&self, other: &Self) -> Ordering {
+            let l_run = &self.run;
+            let r_run = &other.run;
+
+            // Safety: `start` and `end` must both exist because it's guaranteed that whenever a
+            // `Run` is created, an item is pushed into it immediately (see its `new` method above).
+            // And there are no other ways to create a `Run` beyond its `new` method in this
+            // function's scope.
+            let l_end = l_run.end.unwrap();
+            let r_end = r_run.end.unwrap();
+            r_end
+                .cmp(&l_end)
+                .then_with(|| {
+                    let l_start = l_run.start.unwrap();
+                    let r_start = r_run.start.unwrap();
+                    l_start.cmp(&r_start)
+                })
+                .then_with(|| self.i.cmp(&other.i))
+        }
+    }
+
+    /// Wrapper around the `Run` above, to support sorting them by their creation sequence `i`.
+    #[derive(PartialEq, Eq)]
+    struct Wrapper<T: Item>(Run<T>);
+
+    impl<T: Item> PartialOrd for Wrapper<T> {
+        fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
+            Some(self.cmp(other))
+        }
+    }
+
+    impl<T: Item> Ord for Wrapper<T> {
+        fn cmp(&self, other: &Self) -> Ordering {
+            other.0.i.cmp(&self.0.i)
+        }
+    }
+
+    // Two heaps for finding a run that is both:
+    // 1. not overlapping with item's range,
+    // 2. and is created earliest,
+    // when iterating the items.
+    //
+    // Heap 1 (`runs_sorted_by_end`) is for storing the runs of which top has the minimal "end"
+    // just about to overlap with the current selected item.
+    //
+    // Heap 2 (`runs_sort_by_index`) is for storing the runs that all have "end"s non-overlap with
+    // the current selected item, and of which top is the earliest created run.
+    //
+    // The finding of a suitable run basically works like this:
+    // 1. moves the runs in heap 1 to heap 2, until the top is overlapping with the current item;
+    // 2. now heap 2 has all the runs that can accept the current item, pop its top;
+    // 3. the top is the earliest created run, push the current item;
+    // 4. because the run has changed, push it back to heap 1;
+    // 5. check the next item. Important: we don't need to push the runs in heap 2 to 1, because
+    //    the items are sorted by "start". When checking the next item, heap 2's runs must all have
+    //    "end"s smaller than next item's "start".
+    //
+    // Actually the heap 2 is only for aligning with the runs selection outcomes in the original
+    // `find_sorted_runs` implementation. If we just need the invariant that each run has the
+    // non-overlapping items, we can get rid of heap 2 and make the codes simpler.
+
+    let mut runs_sort_by_end = BinaryHeap::<Run<T>>::new();
+    let mut runs_sort_by_index = BinaryHeap::<Wrapper<T>>::new();
+    let mut i = 0;
+
+    for item in items {
+        let (start, _) = item.range();
+
+        while let Some(run) = runs_sort_by_end.pop_if(|x| x.run.end.unwrap() <= start) {
+            runs_sort_by_index.push(Wrapper(run));
+        }
+
+        let Some(mut run) = runs_sort_by_index.pop() else {
+            i += 1;
+            runs_sort_by_end.push(Run::new(i, item));
+            continue;
+        };
+
+        run.0.push_item(item);
+        runs_sort_by_end.push(run.0);
+    }
+
+    let mut runs = runs_sort_by_end.into_vec();
+    runs.extend(runs_sort_by_index.into_vec().into_iter().map(|x| x.0));
+    runs.sort_unstable_by_key(|run| run.i);
+    runs.into_iter().map(|x| x.run).collect()
+}
+
 /// Finds a set of files with minimum penalty to merge that can reduce the total num of runs.
 /// The penalty of merging is defined as the size of all overlapping files between two runs.
 pub fn reduce_runs<T: Item>(mut runs: Vec<SortedRun<T>>) -> Vec<T> {
@@ -599,6 +729,8 @@ mod tests {
         expected_runs: &[Vec<(i64, i64)>],
     ) -> Vec<SortedRun<MockFile>> {
         let mut files = build_items(ranges);
+        let mut files_clone = files.clone();
+
         let runs = find_sorted_runs(&mut files);
 
         let result_file_ranges: Vec<Vec<_>> = runs
@@ -606,6 +738,13 @@ mod tests {
             .map(|r| r.items.iter().map(|f| f.range()).collect())
             .collect();
         assert_eq!(&expected_runs, &result_file_ranges);
+
+        let runs_by_time_range = find_sorted_runs_by_time_range(&mut files_clone);
+        let results: Vec<Vec<_>> = runs_by_time_range
+            .iter()
+            .map(|r| r.items.iter().map(|f| f.range()).collect())
+            .collect();
+        assert_eq!(&expected_runs, &results);
         runs
     }
 

src/mito2/src/compaction/twcs.rs

@@ -22,14 +22,15 @@ use common_telemetry::{debug, info};
 use common_time::Timestamp;
 use common_time::timestamp::TimeUnit;
 use common_time::timestamp_millis::BucketAligned;
+use rayon::prelude::*;
 use store_api::storage::RegionId;
 
 use crate::compaction::buckets::infer_time_bucket;
 use crate::compaction::compactor::CompactionRegion;
 use crate::compaction::picker::{Picker, PickerOutput};
 use crate::compaction::run::{
-    FileGroup, Item, Ranged, find_sorted_runs, merge_primary_key_ranges, merge_seq_files,
-    primary_key_ranges_overlap, reduce_runs,
+    FileGroup, Item, Ranged, find_sorted_runs, find_sorted_runs_by_time_range,
+    merge_primary_key_ranges, merge_seq_files, primary_key_ranges_overlap, reduce_runs,
 };
 use crate::compaction::{CompactionOutput, get_expired_ssts};
 use crate::sst::file::{FileHandle, Level, overlaps};
@@ -64,11 +65,10 @@ impl TwcsPicker {
         time_windows: &mut BTreeMap<i64, Window>,
         active_window: Option<i64>,
     ) -> Vec<CompactionOutput> {
-        let mut output = vec![];
-        for (window, files) in time_windows {
-            if files.files.is_empty() {
-                continue;
-            }
+        let find_inputs = |files: &Window,
+                           windows: &BTreeMap<i64, Window>|
+         -> (Vec<FileGroup>, bool) {
+            let window = &files.time_window;
             let mut files_to_merge: Vec<_> = files.files().cloned().collect();
 
             // Filter out large files in append mode - they won't benefit from compaction
@@ -88,13 +88,18 @@ impl TwcsPicker {
                 );
             }
 
-            let sorted_runs = find_sorted_runs(&mut files_to_merge);
+            let sorted_runs = if files_to_merge.len() < 1024 {
+                find_sorted_runs(&mut files_to_merge)
+            } else {
+                find_sorted_runs_by_time_range(&mut files_to_merge)
+            };
             let found_runs = sorted_runs.len();
             // We only remove deletion markers if we found less than 2 runs and not in append mode.
             // because after compaction there will be no overlapping files.
-            let filter_deleted = !files.overlapping && found_runs <= 2 && !self.append_mode;
+            let filter_deleted =
+                found_runs <= 2 && !self.append_mode && !window_has_overlap(files, windows);
             if found_runs == 0 {
-                continue;
+                return (vec![], filter_deleted);
             }
 
             let mut inputs = if found_runs > 1 {
@@ -102,7 +107,7 @@ impl TwcsPicker {
             } else {
                 let run = sorted_runs.last().unwrap();
                 if run.items().len() < self.trigger_file_num {
-                    continue;
+                    return (vec![], filter_deleted);
                 }
                 // no overlapping files, try merge small files
                 merge_seq_files(run.items(), self.max_output_file_size)
@@ -144,6 +149,26 @@ impl TwcsPicker {
                     filter_deleted,
                     &inputs,
                 );
+            }
+            (inputs, filter_deleted)
+        };
+
+        let mut output = vec![];
+        let windows = time_windows
+            .values()
+            .filter(|w| !w.files.is_empty())
+            .collect::<Vec<_>>();
+        let chunk_size = self.max_background_tasks.unwrap_or(windows.len()).max(1);
+        'chunks: for chunk in windows.chunks(chunk_size) {
+            for (inputs, filter_deleted) in chunk
+                .par_iter() // parallelly calculate the inputs
+                .map(|window| find_inputs(window, time_windows))
+                .collect::<Vec<_>>()
+            {
+                if inputs.is_empty() {
+                    continue;
+                }
+
                 output.push(CompactionOutput {
                     output_level: LEVEL_COMPACTED, // always compact to l1
                     inputs: inputs.into_iter().flat_map(|fg| fg.into_files()).collect(),
@@ -158,7 +183,7 @@ impl TwcsPicker {
                         "Region ({:?}) compaction task size larger than max background tasks({}), remaining tasks discarded",
                         region_id, max_background_tasks
                     );
-                    break;
+                    break 'chunks;
                 }
             }
         }
@@ -268,7 +293,6 @@ struct Window {
     // created from the same compaction task.
     files: HashMap<Option<NonZeroU64>, FileGroup>,
     time_window: i64,
-    overlapping: bool,
     primary_key_range: Option<(bytes::Bytes, bytes::Bytes)>,
 }
 
@@ -283,7 +307,6 @@ impl Window {
             end,
             files,
             time_window: 0,
-            overlapping: false,
             primary_key_range,
         }
     }
@@ -346,37 +369,21 @@ fn assign_to_windows<'a>(
             }
         }
     }
-    if windows.is_empty() {
-        return BTreeMap::new();
-    }
+    windows.into_iter().collect()
+}
 
-    let mut windows = windows.into_values().collect::<Vec<_>>();
-    windows.sort_unstable_by(|l, r| l.start.cmp(&r.start).then(l.end.cmp(&r.end).reverse()));
-
-    for idx in 0..windows.len() {
-        let lhs_range = windows[idx].range();
-        for next_idx in idx + 1..windows.len() {
-            let rhs_range = windows[next_idx].range();
-            if rhs_range.0 > lhs_range.1 {
-                break;
-            }
-
-            let windows_overlap = overlaps(&lhs_range, &rhs_range)
-                && match (
-                    &windows[idx].primary_key_range,
-                    &windows[next_idx].primary_key_range,
-                ) {
-                    (Some(lhs), Some(rhs)) => primary_key_ranges_overlap(lhs, rhs),
+fn window_has_overlap(this: &Window, windows: &BTreeMap<i64, Window>) -> bool {
+    windows
+        .values()
+        .filter(|that| this.time_window != that.time_window)
+        .any(|that| {
+            overlaps(&this.range(), &that.range()) && {
+                match (&this.primary_key_range, &that.primary_key_range) {
+                    (Some(l), Some(r)) => primary_key_ranges_overlap(l, r),
                     _ => true,
-                };
-            if windows_overlap {
-                windows[idx].overlapping = true;
-                windows[next_idx].overlapping = true;
+                }
             }
-        }
-    }
-
-    windows.into_iter().map(|w| (w.time_window, w)).collect()
+        })
 }
 
 /// Finds the latest active writing window among all files.
@@ -606,7 +613,8 @@ mod tests {
 
         for (expected_window, overlapping, window_files) in expected_files {
             let actual_window = windows.get(expected_window).unwrap();
-            assert_eq!(*overlapping, actual_window.overlapping);
+            let actual_overlapping = window_has_overlap(actual_window, &windows);
+            assert_eq!(*overlapping, actual_overlapping);
             let mut file_ranges = actual_window
                 .files
                 .values()
@@ -744,7 +752,8 @@ mod tests {
 
         let windows = assign_to_windows(files.iter(), 2);
 
-        assert!(!windows.get(&2).unwrap().overlapping);
+        let overlapping = window_has_overlap(windows.get(&2).unwrap(), &windows);
+        assert!(!overlapping);
     }
 
     #[test]
@@ -773,7 +782,8 @@ mod tests {
 
         let windows = assign_to_windows(files.iter(), 2);
 
-        assert!(!windows.get(&4).unwrap().overlapping);
+        let overlapping = window_has_overlap(windows.get(&4).unwrap(), &windows);
+        assert!(!overlapping);
     }
 
     struct CompactionPickerTestCase {

src/mito2/src/engine/alter_test.rs

@@ -277,6 +277,7 @@ async fn test_alter_region_with_format(flat_format: bool) {
                 options: HashMap::default(),
                 skip_wal_replay: false,
                 checkpoint: None,
+                requirements: Default::default(),
             }),
         )
         .await
@@ -481,6 +482,7 @@ async fn test_put_after_alter_with_format(flat_format: bool) {
                 options: HashMap::default(),
                 skip_wal_replay: false,
                 checkpoint: None,
+                requirements: Default::default(),
             }),
         )
         .await
@@ -844,6 +846,7 @@ async fn test_alter_column_fulltext_options_with_format(flat_format: bool) {
                 options: HashMap::default(),
                 skip_wal_replay: false,
                 checkpoint: None,
+                requirements: Default::default(),
             }),
         )
         .await
@@ -979,6 +982,7 @@ async fn test_alter_column_set_inverted_index_with_format(flat_format: bool) {
                 options: HashMap::default(),
                 skip_wal_replay: false,
                 checkpoint: None,
+                requirements: Default::default(),
             }),
         )
         .await
@@ -1248,6 +1252,7 @@ async fn test_alter_region_sst_format_with_flush() {
                 options: HashMap::default(),
                 skip_wal_replay: false,
                 checkpoint: None,
+                requirements: Default::default(),
             }),
         )
         .await
@@ -1366,6 +1371,7 @@ async fn test_alter_region_sst_format_without_flush() {
                 options: HashMap::default(),
                 skip_wal_replay: false,
                 checkpoint: None,
+                requirements: Default::default(),
             }),
         )
         .await
@@ -1492,6 +1498,7 @@ async fn test_alter_region_sst_format_flat_to_pk_with_flush() {
                 options: HashMap::default(),
                 skip_wal_replay: false,
                 checkpoint: None,
+                requirements: Default::default(),
             }),
         )
         .await
@@ -1610,6 +1617,7 @@ async fn test_alter_region_sst_format_flat_to_pk_without_flush() {
                 options: HashMap::default(),
                 skip_wal_replay: false,
                 checkpoint: None,
+                requirements: Default::default(),
             }),
         )
         .await
@@ -1725,6 +1733,7 @@ async fn test_alter_region_append_mode_with_flush() {
                 options: HashMap::default(),
                 skip_wal_replay: false,
                 checkpoint: None,
+                requirements: Default::default(),
             }),
         )
         .await
@@ -1843,6 +1852,7 @@ async fn test_alter_region_append_mode_without_flush() {
                 options: HashMap::default(),
                 skip_wal_replay: false,
                 checkpoint: None,
+                requirements: Default::default(),
             }),
         )
         .await

src/mito2/src/engine/append_mode_test.rs

@@ -348,6 +348,7 @@ async fn test_alter_append_mode_clears_merge_mode_with_format(flat_format: bool)
                 options,
                 skip_wal_replay: false,
                 checkpoint: None,
+                requirements: Default::default(),
             }),
         )
         .await

src/mito2/src/engine/basic_test.rs

@@ -196,6 +196,7 @@ async fn test_region_replay_with_format(factory: Option<LogStoreFactory>, flat_f
                 options,
                 skip_wal_replay: false,
                 checkpoint: None,
+                requirements: Default::default(),
             }),
         )
         .await

src/mito2/src/engine/batch_catchup_test.rs

@@ -160,6 +160,7 @@ async fn test_batch_catchup_with_format(factory: Option<LogStoreFactory>, flat_f
                     skip_wal_replay: true,
                     path_type: PathType::Bare,
                     checkpoint: None,
+                    requirements: Default::default(),
                 },
             )
         })

src/mito2/src/engine/batch_open_test.rs

@@ -136,6 +136,7 @@ async fn test_batch_open_with_format(factory: Option<LogStoreFactory>, flat_form
                     skip_wal_replay: false,
                     path_type: PathType::Bare,
                     checkpoint: None,
+                    requirements: Default::default(),
                 },
             )
         })
@@ -149,6 +150,7 @@ async fn test_batch_open_with_format(factory: Option<LogStoreFactory>, flat_form
             skip_wal_replay: false,
             path_type: PathType::Bare,
             checkpoint: None,
+            requirements: Default::default(),
         },
     ));
 
@@ -221,6 +223,7 @@ async fn test_batch_open_err_with_format(factory: Option<LogStoreFactory>, flat_
                     skip_wal_replay: false,
                     path_type: PathType::Bare,
                     checkpoint: None,
+                    requirements: Default::default(),
                 },
             )
         })

src/mito2/src/engine/bump_committed_sequence_test.rs

@@ -112,6 +112,7 @@ async fn test_bump_committed_sequence_with_format(flat_format: bool) {
                 options: HashMap::default(),
                 skip_wal_replay: false,
                 checkpoint: None,
+                requirements: Default::default(),
             }),
         )
         .await
@@ -151,6 +152,7 @@ async fn test_bump_committed_sequence_with_format(flat_format: bool) {
                 options: HashMap::default(),
                 skip_wal_replay: false,
                 checkpoint: None,
+                requirements: Default::default(),
             }),
         )
         .await

src/mito2/src/engine/catchup_test.rs

@@ -97,6 +97,7 @@ async fn test_catchup_with_last_entry_id(factory: Option<LogStoreFactory>) {
                 options,
                 skip_wal_replay: false,
                 checkpoint: None,
+                requirements: Default::default(),
             }),
         )
         .await
@@ -218,6 +219,7 @@ async fn test_catchup_with_incorrect_last_entry_id(factory: Option<LogStoreFacto
                 options,
                 skip_wal_replay: false,
                 checkpoint: None,
+                requirements: Default::default(),
             }),
         )
         .await
@@ -321,6 +323,7 @@ async fn test_catchup_without_last_entry_id(factory: Option<LogStoreFactory>) {
                 options,
                 skip_wal_replay: false,
                 checkpoint: None,
+                requirements: Default::default(),
             }),
         )
         .await
@@ -423,6 +426,7 @@ async fn test_catchup_with_manifest_update(factory: Option<LogStoreFactory>) {
                 options,
                 skip_wal_replay: false,
                 checkpoint: None,
+                requirements: Default::default(),
             }),
         )
         .await
@@ -527,6 +531,7 @@ async fn open_region(
                 skip_wal_replay,
                 path_type: PathType::Bare,
                 checkpoint: None,
+                requirements: Default::default(),
             }),
         )
         .await
@@ -622,6 +627,7 @@ async fn test_local_catchup(factory: Option<LogStoreFactory>) {
                 skip_wal_replay: true,
                 path_type: PathType::Bare,
                 checkpoint: None,
+                requirements: Default::default(),
             }),
         )
         .await

src/mito2/src/engine/compaction_test.rs

@@ -1023,6 +1023,7 @@ async fn test_change_region_compaction_window_with_format(flat_format: bool) {
                 options: Default::default(),
                 skip_wal_replay: false,
                 checkpoint: None,
+                requirements: Default::default(),
             }),
         )
         .await
@@ -1125,6 +1126,7 @@ async fn test_open_overwrite_compaction_window_with_format(flat_format: bool) {
                 options,
                 skip_wal_replay: false,
                 checkpoint: None,
+                requirements: Default::default(),
             }),
         )
         .await

src/mito2/src/engine/edit_region_test.rs

@@ -21,6 +21,7 @@ use common_error::ext::ErrorExt;
 use common_error::status_code::StatusCode;
 use common_recordbatch::DfRecordBatch;
 use common_test_util::flight::encode_to_flight_data;
+use common_time::Timestamp;
 use common_time::util::current_time_millis;
 use datatypes::arrow::array::{ArrayRef, Float64Array, StringArray, TimestampMillisecondArray};
 use datatypes::arrow::datatypes::{DataType, Field, Schema, TimeUnit};
@@ -67,7 +68,8 @@ async fn test_edit_region_schedule_compaction_with_format(flat_format: bool) {
         default_flat_format: flat_format,
         ..Default::default()
     };
-    let time_provider = Arc::new(MockTimeProvider::new(current_time_millis()));
+    let initial_time = current_time_millis();
+    let time_provider = Arc::new(MockTimeProvider::new(initial_time));
     let engine = env
         .create_engine_with_time(
             config.clone(),
@@ -99,14 +101,22 @@ async fn test_edit_region_schedule_compaction_with_format(flat_format: bool) {
         .await
         .unwrap();
     let region = engine.get_region(region_id).unwrap();
+    let initial_schedule_time = region.last_schedule_compaction_millis();
+    assert_eq!(initial_time, initial_schedule_time);
 
-    let new_edit = || RegionEdit {
-        files_to_add: vec![FileMeta {
-            region_id: region.region_id,
-            file_id: FileId::random(),
-            level: 0,
-            ..Default::default()
-        }],
+    let new_edit = |file_starts: &[i64]| RegionEdit {
+        files_to_add: file_starts
+            .iter()
+            .map(|start| FileMeta {
+                region_id: region.region_id,
+                file_id: FileId::random(),
+                time_range: (
+                    Timestamp::new_millisecond(*start),
+                    Timestamp::new_millisecond(1000 * 1000),
+                ),
+                ..Default::default()
+            })
+            .collect(),
         files_to_remove: vec![],
         timestamp_ms: None,
         compaction_time_window: None,
@@ -115,19 +125,23 @@ async fn test_edit_region_schedule_compaction_with_format(flat_format: bool) {
         committed_sequence: None,
     };
     engine
-        .edit_region(region.region_id, new_edit())
+        .edit_region(region.region_id, new_edit(&[0, 10, 50, 80]))
         .await
         .unwrap();
     // Asserts that the compaction of the region is not scheduled,
     // because the minimum time interval between two compactions is not passed.
     assert_eq!(rx.try_recv(), Err(oneshot::error::TryRecvError::Empty));
+    assert_eq!(
+        initial_schedule_time,
+        region.last_schedule_compaction_millis()
+    );
 
     // Simulates the time has passed the min compaction interval,
-    time_provider
-        .set_now(current_time_millis() + config.min_compaction_interval.as_millis() as i64);
+    let next_schedule_time = initial_time + config.min_compaction_interval.as_millis() as i64;
+    time_provider.set_now(next_schedule_time);
     // ... then edits the region again,
     engine
-        .edit_region(region.region_id, new_edit())
+        .edit_region(region.region_id, new_edit(&[90]))
         .await
         .unwrap();
     // ... finally asserts that the compaction of the region is scheduled.
@@ -136,6 +150,9 @@ async fn test_edit_region_schedule_compaction_with_format(flat_format: bool) {
         .unwrap()
         .unwrap();
     assert_eq!(region_id, actual);
+    // Wait for the `last_schedule_compaction_millis` to update.
+    tokio::time::sleep(Duration::from_millis(100)).await;
+    assert_eq!(next_schedule_time, region.last_schedule_compaction_millis());
 }
 
 #[tokio::test]
@@ -635,6 +652,7 @@ fn build_bulk_insert_request(
             payload: record_batch.data_body,
         },
         partition_expr_version: None,
+        aligned_schema_version: None,
     }
 }
 

src/mito2/src/engine/flush_test.rs

@@ -14,6 +14,7 @@
 
 //! Flush tests for mito engine.
 
+use std::collections::HashMap;
 use std::sync::Arc;
 use std::sync::atomic::{AtomicI64, Ordering};
 use std::time::Duration;
@@ -21,11 +22,14 @@ use std::time::Duration;
 use api::v1::Rows;
 use common_recordbatch::RecordBatches;
 use common_time::util::current_time_millis;
-use common_wal::options::WAL_OPTIONS_KEY;
+use common_wal::options::{KafkaWalOptions, WAL_OPTIONS_KEY, WalOptions};
 use rstest::rstest;
 use rstest_reuse::{self, apply};
 use store_api::region_engine::RegionEngine;
-use store_api::region_request::{RegionFlushRequest, RegionRequest};
+use store_api::region_request::{
+    PathType, RegionCloseRequest, RegionFlushRequest, RegionOpenRequest, RegionRequest,
+    ReplayCheckpoint,
+};
 use store_api::storage::{RegionId, ScanRequest};
 
 use crate::config::MitoConfig;
@@ -319,8 +323,6 @@ async fn test_flush_empty_with_format(flat_format: bool) {
 async fn test_flush_reopen_region(factory: Option<LogStoreFactory>) {
     use std::collections::HashMap;
 
-    use common_wal::options::{KafkaWalOptions, WalOptions};
-
     common_telemetry::init_default_ut_logging();
     let Some(factory) = factory else {
         return;
@@ -394,6 +396,235 @@ async fn test_flush_reopen_region(factory: Option<LogStoreFactory>) {
     assert_eq!(5, version_data.committed_sequence);
 }
 
+#[apply(single_kafka_log_store_factory)]
+async fn test_skip_remote_wal_replay_sets_topic_latest_entry_id(factory: Option<LogStoreFactory>) {
+    common_telemetry::init_default_ut_logging();
+    let Some(factory) = factory else {
+        return;
+    };
+
+    let mut env = TestEnv::new().await.with_log_store_factory(factory.clone());
+    let engine = env.create_engine(MitoConfig::default()).await;
+    let region_id = RegionId::new(1, 1);
+    env.get_schema_metadata_manager()
+        .register_region_table_info(
+            region_id.table_id(),
+            "test_table",
+            "test_catalog",
+            "test_schema",
+            None,
+            env.get_kv_backend(),
+        )
+        .await;
+
+    let topic = prepare_test_for_kafka_log_store(&factory).await;
+    let request = CreateRequestBuilder::new()
+        .kafka_topic(topic.clone())
+        .build();
+    let table_dir = request.table_dir.clone();
+    let column_schemas = rows_schema(&request);
+    let options = kafka_wal_options(&topic);
+
+    engine
+        .handle_request(region_id, RegionRequest::Create(request))
+        .await
+        .unwrap();
+
+    let rows = Rows {
+        schema: column_schemas,
+        rows: build_rows_for_key("a", 0, 2, 0),
+    };
+    put_rows(&engine, region_id, rows).await;
+    flush_region(&engine, region_id, None).await;
+    // The empty flush updates `topic_latest_entry_id` from KafkaLogStore's topic stats.
+    flush_region(&engine, region_id, None).await;
+
+    let region = engine.get_region(region_id).unwrap();
+    assert_eq!(1, region.topic_latest_entry_id.load(Ordering::Relaxed));
+
+    engine
+        .handle_request(region_id, RegionRequest::Close(RegionCloseRequest {}))
+        .await
+        .unwrap();
+
+    engine
+        .handle_request(
+            region_id,
+            RegionRequest::Open(RegionOpenRequest {
+                engine: String::new(),
+                table_dir,
+                path_type: PathType::Bare,
+                options,
+                skip_wal_replay: true,
+                checkpoint: Some(ReplayCheckpoint {
+                    entry_id: 1,
+                    metadata_entry_id: None,
+                }),
+                requirements: Default::default(),
+            }),
+        )
+        .await
+        .unwrap();
+
+    let region = engine.get_region(region_id).unwrap();
+    assert_eq!(1, region.topic_latest_entry_id.load(Ordering::Relaxed));
+}
+
+#[apply(single_kafka_log_store_factory)]
+async fn test_remote_wal_open_with_replayed_memtable_sets_topic_latest_entry_id(
+    factory: Option<LogStoreFactory>,
+) {
+    common_telemetry::init_default_ut_logging();
+    let Some(factory) = factory else {
+        return;
+    };
+
+    let mut env = TestEnv::new().await.with_log_store_factory(factory.clone());
+    let engine = env.create_engine(MitoConfig::default()).await;
+    let region_id = RegionId::new(1, 1);
+    env.get_schema_metadata_manager()
+        .register_region_table_info(
+            region_id.table_id(),
+            "test_table",
+            "test_catalog",
+            "test_schema",
+            None,
+            env.get_kv_backend(),
+        )
+        .await;
+
+    let topic = prepare_test_for_kafka_log_store(&factory).await;
+    let request = CreateRequestBuilder::new()
+        .kafka_topic(topic.clone())
+        .build();
+    let table_dir = request.table_dir.clone();
+    let column_schemas = rows_schema(&request);
+    let options = kafka_wal_options(&topic);
+
+    engine
+        .handle_request(region_id, RegionRequest::Create(request))
+        .await
+        .unwrap();
+    let rows = Rows {
+        schema: column_schemas.clone(),
+        rows: build_rows_for_key("a", 0, 2, 0),
+    };
+    put_rows(&engine, region_id, rows).await;
+    flush_region(&engine, region_id, None).await;
+
+    let rows = Rows {
+        schema: column_schemas,
+        rows: build_rows_for_key("b", 0, 2, 0),
+    };
+    put_rows(&engine, region_id, rows).await;
+
+    let engine = env.reopen_engine(engine, MitoConfig::default()).await;
+    engine
+        .handle_request(
+            region_id,
+            RegionRequest::Open(RegionOpenRequest {
+                engine: String::new(),
+                table_dir,
+                path_type: PathType::Bare,
+                options,
+                skip_wal_replay: false,
+                checkpoint: None,
+                requirements: Default::default(),
+            }),
+        )
+        .await
+        .unwrap();
+
+    let region = engine.get_region(region_id).unwrap();
+    assert_eq!(1, region.version().flushed_entry_id);
+    assert!(!region.version().memtables.is_empty());
+    assert_eq!(1, region.topic_latest_entry_id.load(Ordering::Relaxed));
+}
+
+#[apply(single_kafka_log_store_factory)]
+async fn test_remote_wal_open_without_replayed_memtable_sets_topic_latest_entry_id(
+    factory: Option<LogStoreFactory>,
+) {
+    common_telemetry::init_default_ut_logging();
+    let Some(factory) = factory else {
+        return;
+    };
+
+    let mut env = TestEnv::new().await.with_log_store_factory(factory.clone());
+    let engine = env.create_engine(MitoConfig::default()).await;
+    let region_id = RegionId::new(1, 1);
+    env.get_schema_metadata_manager()
+        .register_region_table_info(
+            region_id.table_id(),
+            "test_table",
+            "test_catalog",
+            "test_schema",
+            None,
+            env.get_kv_backend(),
+        )
+        .await;
+
+    let topic = prepare_test_for_kafka_log_store(&factory).await;
+    let request = CreateRequestBuilder::new()
+        .kafka_topic(topic.clone())
+        .build();
+    let table_dir = request.table_dir.clone();
+    let column_schemas = rows_schema(&request);
+    let options = kafka_wal_options(&topic);
+
+    engine
+        .handle_request(region_id, RegionRequest::Create(request))
+        .await
+        .unwrap();
+    let rows = Rows {
+        schema: column_schemas,
+        rows: build_rows_for_key("a", 0, 2, 0),
+    };
+    put_rows(&engine, region_id, rows).await;
+    flush_region(&engine, region_id, None).await;
+    // The empty flush updates `topic_latest_entry_id` from KafkaLogStore's topic stats.
+    flush_region(&engine, region_id, None).await;
+    engine
+        .handle_request(region_id, RegionRequest::Close(RegionCloseRequest {}))
+        .await
+        .unwrap();
+
+    engine
+        .handle_request(
+            region_id,
+            RegionRequest::Open(RegionOpenRequest {
+                engine: String::new(),
+                table_dir,
+                path_type: PathType::Bare,
+                options,
+                skip_wal_replay: false,
+                checkpoint: None,
+                requirements: Default::default(),
+            }),
+        )
+        .await
+        .unwrap();
+
+    let region = engine.get_region(region_id).unwrap();
+    assert!(region.version().memtables.is_empty());
+    assert_eq!(1, region.topic_latest_entry_id.load(Ordering::Relaxed));
+}
+
+fn kafka_wal_options(topic: &Option<String>) -> HashMap<String, String> {
+    topic
+        .as_ref()
+        .map(|topic| {
+            HashMap::from([(
+                WAL_OPTIONS_KEY.to_string(),
+                serde_json::to_string(&WalOptions::Kafka(KafkaWalOptions {
+                    topic: topic.clone(),
+                }))
+                .unwrap(),
+            )])
+        })
+        .unwrap_or_default()
+}
+
 #[derive(Debug)]
 pub(crate) struct MockTimeProvider {
     now: AtomicI64,

src/mito2/src/engine/open_test.rs

@@ -64,6 +64,7 @@ async fn test_engine_open_empty_with_format(flat_format: bool) {
                 options: HashMap::default(),
                 skip_wal_replay: false,
                 checkpoint: None,
+                requirements: Default::default(),
             }),
         )
         .await
@@ -110,6 +111,7 @@ async fn test_engine_open_existing_with_format(flat_format: bool) {
                 options: HashMap::default(),
                 skip_wal_replay: false,
                 checkpoint: None,
+                requirements: Default::default(),
             }),
         )
         .await
@@ -237,6 +239,7 @@ async fn test_engine_region_open_with_options_with_format(flat_format: bool) {
                 options: HashMap::from([("ttl".to_string(), "4d".to_string())]),
                 skip_wal_replay: false,
                 checkpoint: None,
+                requirements: Default::default(),
             }),
         )
         .await
@@ -297,6 +300,7 @@ async fn test_engine_region_open_with_custom_store_with_format(flat_format: bool
                 options: HashMap::from([("storage".to_string(), "Gcs".to_string())]),
                 skip_wal_replay: false,
                 checkpoint: None,
+                requirements: Default::default(),
             }),
         )
         .await
@@ -392,6 +396,7 @@ async fn test_open_region_skip_wal_replay_with_format(flat_format: bool) {
                 options: Default::default(),
                 skip_wal_replay: true,
                 checkpoint: None,
+                requirements: Default::default(),
             }),
         )
         .await
@@ -431,6 +436,7 @@ async fn test_open_region_skip_wal_replay_with_format(flat_format: bool) {
                 options: Default::default(),
                 skip_wal_replay: false,
                 checkpoint: None,
+                requirements: Default::default(),
             }),
         )
         .await
@@ -484,6 +490,7 @@ async fn test_open_region_wait_for_opening_region_ok_with_format(flat_format: bo
                     options: HashMap::default(),
                     skip_wal_replay: false,
                     checkpoint: None,
+                    requirements: Default::default(),
                 }),
             )
             .await
@@ -535,6 +542,7 @@ async fn test_open_region_wait_for_opening_region_err_with_format(flat_format: b
                     options: HashMap::default(),
                     skip_wal_replay: false,
                     checkpoint: None,
+                    requirements: Default::default(),
                 }),
             )
             .await
@@ -691,6 +699,7 @@ async fn test_open_backfills_partition_expr_with_fetcher() {
                 options: HashMap::default(),
                 skip_wal_replay: false,
                 checkpoint: None,
+                requirements: Default::default(),
             }),
         )
         .await
@@ -725,6 +734,7 @@ async fn test_open_backfills_partition_expr_with_fetcher() {
                 options: HashMap::default(),
                 skip_wal_replay: false,
                 checkpoint: None,
+                requirements: Default::default(),
             }),
         )
         .await
@@ -766,6 +776,7 @@ async fn test_open_keeps_none_without_fetcher() {
                 options: HashMap::default(),
                 skip_wal_replay: false,
                 checkpoint: None,
+                requirements: Default::default(),
             }),
         )
         .await

src/mito2/src/engine/parallel_test.rs

@@ -52,6 +52,7 @@ async fn scan_in_parallel(
                 skip_wal_replay: false,
                 path_type: PathType::Bare,
                 checkpoint: None,
+                requirements: Default::default(),
             }),
         )
         .await

src/mito2/src/engine/skip_wal_test.rs

@@ -87,6 +87,7 @@ async fn test_close_region_skip_wal(insert: bool) {
                 options: request.options.clone(),
                 skip_wal_replay: false,
                 checkpoint: None,
+                requirements: Default::default(),
             }),
         )
         .await
@@ -154,6 +155,7 @@ async fn test_close_follower_region_skip_wal() {
                 options: request.options.clone(),
                 skip_wal_replay: false,
                 checkpoint: None,
+                requirements: Default::default(),
             }),
         )
         .await
@@ -271,6 +273,7 @@ async fn test_close_region_after_truncate_skip_wal() {
                 options: request.options,
                 skip_wal_replay: false,
                 checkpoint: None,
+                requirements: Default::default(),
             }),
         )
         .await

src/mito2/src/engine/sync_test.rs

@@ -127,6 +127,7 @@ async fn test_sync_after_flush_region_with_format(flat_format: bool) {
                 // Ensure the region is not replayed from the WAL.
                 skip_wal_replay: true,
                 checkpoint: None,
+                requirements: Default::default(),
             }),
         )
         .await
@@ -239,6 +240,7 @@ async fn test_sync_after_alter_region_with_format(flat_format: bool) {
                 // Ensure the region is not replayed from the WAL.
                 skip_wal_replay: true,
                 checkpoint: None,
+                requirements: Default::default(),
             }),
         )
         .await

src/mito2/src/engine/truncate_test.rs

@@ -323,6 +323,7 @@ async fn test_engine_truncate_reopen_with_format(flat_format: bool) {
                 options: HashMap::default(),
                 skip_wal_replay: false,
                 checkpoint: None,
+                requirements: Default::default(),
             }),
         )
         .await
@@ -447,6 +448,7 @@ async fn test_engine_truncate_during_flush_with_format(flat_format: bool) {
                 options: HashMap::default(),
                 skip_wal_replay: false,
                 checkpoint: None,
+                requirements: Default::default(),
             }),
         )
         .await

src/mito2/src/error.rs

@@ -916,6 +916,20 @@ pub enum Error {
         source: Arc<Error>,
     },
 
+    #[snafu(display(
+        "Region {} does not satisfy open requirement '{}': {}",
+        region_id,
+        requirement,
+        reason
+    ))]
+    OpenRegionRequirement {
+        region_id: RegionId,
+        requirement: &'static str,
+        reason: &'static str,
+        #[snafu(implicit)]
+        location: Location,
+    },
+
     #[snafu(display("Failed to parse job id"))]
     ParseJobId {
         #[snafu(implicit)]
@@ -1376,6 +1390,7 @@ impl ErrorExt for Error {
             PrimaryKeyLengthMismatch { .. } => StatusCode::InvalidArguments,
             InvalidSender { .. } => StatusCode::InvalidArguments,
             InvalidSchedulerState { .. } => StatusCode::InvalidArguments,
+            OpenRegionRequirement { .. } => StatusCode::InvalidArguments,
             DeleteSsts { .. } | DeleteIndex { .. } | DeleteIndexes { .. } => {
                 StatusCode::StorageUnavailable
             }