fix(test): use different table name (#5334)

Co-authored-by: ZonaHex <ZonaHex@users.noreply.github.com>

.github/actions/build-linux-artifacts/action.yml

@@ -53,7 +53,7 @@ runs:
       uses: ./.github/actions/build-greptime-binary
       with:
         base-image: ubuntu
-        features: servers/dashboard
+        features: servers/dashboard,pg_kvbackend
         cargo-profile: ${{ inputs.cargo-profile }}
         artifacts-dir: greptime-linux-${{ inputs.arch }}-${{ inputs.version }}
         version: ${{ inputs.version }}
@@ -71,7 +71,7 @@ runs:
       if: ${{ inputs.arch == 'amd64' && inputs.dev-mode == 'false' }} # Builds greptime for centos if the host machine is amd64.
       with:
         base-image: centos
-        features: servers/dashboard
+        features: servers/dashboard,pg_kvbackend
         cargo-profile: ${{ inputs.cargo-profile }}
         artifacts-dir: greptime-linux-${{ inputs.arch }}-centos-${{ inputs.version }}
         version: ${{ inputs.version }}

.github/actions/publish-github-release/action.yml

@@ -9,8 +9,8 @@ runs:
   steps:
     # Download artifacts from previous jobs, the artifacts will be downloaded to:
     # ${WORKING_DIR}
-    #   |- greptime-darwin-amd64-pyo3-v0.5.0/greptime-darwin-amd64-pyo3-v0.5.0.tar.gz
-    #   |- greptime-darwin-amd64-pyo3-v0.5.0.sha256sum/greptime-darwin-amd64-pyo3-v0.5.0.sha256sum
+    #   |- greptime-darwin-amd64-v0.5.0/greptime-darwin-amd64-v0.5.0.tar.gz
+    #   |- greptime-darwin-amd64-v0.5.0.sha256sum/greptime-darwin-amd64-v0.5.0.sha256sum
     #   |- greptime-darwin-amd64-v0.5.0/greptime-darwin-amd64-v0.5.0.tar.gz
     #   |- greptime-darwin-amd64-v0.5.0.sha256sum/greptime-darwin-amd64-v0.5.0.sha256sum
     #   ...

.github/actions/upload-artifacts/action.yml

@@ -30,9 +30,9 @@ runs:
         done
 
     # The compressed artifacts will use the following layout:
-    # greptime-linux-amd64-pyo3-v0.3.0sha256sum
-    # greptime-linux-amd64-pyo3-v0.3.0.tar.gz
-    #   greptime-linux-amd64-pyo3-v0.3.0
+    # greptime-linux-amd64-v0.3.0sha256sum
+    # greptime-linux-amd64-v0.3.0.tar.gz
+    #   greptime-linux-amd64-v0.3.0
     #   └── greptime
     - name: Compress artifacts and calculate checksum
       working-directory: ${{ inputs.working-dir }}

.github/scripts/upload-artifacts-to-s3.sh

@@ -27,11 +27,11 @@ function upload_artifacts() {
   # ├── latest-version.txt
   # ├── latest-nightly-version.txt
   # ├── v0.1.0
-  # │   ├── greptime-darwin-amd64-pyo3-v0.1.0.sha256sum
-  # │   └── greptime-darwin-amd64-pyo3-v0.1.0.tar.gz
+  # │   ├── greptime-darwin-amd64-v0.1.0.sha256sum
+  # │   └── greptime-darwin-amd64-v0.1.0.tar.gz
   # └── v0.2.0
-  #    ├── greptime-darwin-amd64-pyo3-v0.2.0.sha256sum
-  #    └── greptime-darwin-amd64-pyo3-v0.2.0.tar.gz
+  #    ├── greptime-darwin-amd64-v0.2.0.sha256sum
+  #    └── greptime-darwin-amd64-v0.2.0.tar.gz
   find "$ARTIFACTS_DIR" -type f \( -name "*.tar.gz" -o -name "*.sha256sum" \) | while IFS= read -r file; do
     aws s3 cp \
       "$file" "s3://$AWS_S3_BUCKET/$RELEASE_DIRS/$VERSION/$(basename "$file")"

.github/workflows/develop.yml

@@ -51,12 +51,12 @@ jobs:
         with:
           repo-token: ${{ secrets.GITHUB_TOKEN }}
       - uses: actions-rust-lang/setup-rust-toolchain@v1
-      - name: Rust Cache
-        uses: Swatinem/rust-cache@v2
-        with:
-          # Shares across multiple jobs
-          # Shares with `Clippy` job
-          shared-key: "check-lint"
+      # - name: Rust Cache
+      #   uses: Swatinem/rust-cache@v2
+      #   with:
+      #     # Shares across multiple jobs
+      #     # Shares with `Clippy` job
+      #     shared-key: "check-lint"
       - name: Run cargo check
         run: cargo check --locked --workspace --all-targets
 
@@ -67,11 +67,11 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - uses: actions-rust-lang/setup-rust-toolchain@v1
-      - name: Rust Cache
-        uses: Swatinem/rust-cache@v2
-        with:
-          # Shares across multiple jobs
-          shared-key: "check-toml"
+      # - name: Rust Cache
+      #   uses: Swatinem/rust-cache@v2
+      #   with:
+      #     # Shares across multiple jobs
+      #     shared-key: "check-toml"
       - name: Install taplo
         run: cargo +stable install taplo-cli --version ^0.9 --locked --force
       - name: Run taplo
@@ -100,7 +100,7 @@ jobs:
       - name: Build greptime binaries
         shell: bash
         # `cargo gc` will invoke `cargo build` with specified args
-        run: cargo gc -- --bin greptime --bin sqlness-runner
+        run: cargo gc -- --bin greptime --bin sqlness-runner --features pg_kvbackend
       - name: Pack greptime binaries
         shell: bash
         run: |
@@ -142,11 +142,11 @@ jobs:
         with:
           repo-token: ${{ secrets.GITHUB_TOKEN }}
       - uses: actions-rust-lang/setup-rust-toolchain@v1
-      - name: Rust Cache
-        uses: Swatinem/rust-cache@v2
-        with:
-          # Shares across multiple jobs
-          shared-key: "fuzz-test-targets"
+      # - name: Rust Cache
+      #   uses: Swatinem/rust-cache@v2
+      #   with:
+      #     # Shares across multiple jobs
+      #     shared-key: "fuzz-test-targets"
       - name: Set Rust Fuzz
         shell: bash
         run: |
@@ -200,11 +200,11 @@ jobs:
         with:
           repo-token: ${{ secrets.GITHUB_TOKEN }}
       - uses: actions-rust-lang/setup-rust-toolchain@v1
-      - name: Rust Cache
-        uses: Swatinem/rust-cache@v2
-        with:
-          # Shares across multiple jobs
-          shared-key: "fuzz-test-targets"
+      # - name: Rust Cache
+      #   uses: Swatinem/rust-cache@v2
+      #   with:
+      #     # Shares across multiple jobs
+      #     shared-key: "fuzz-test-targets"
       - name: Set Rust Fuzz
         shell: bash
         run: |
@@ -261,7 +261,7 @@ jobs:
       - name: Build greptime bianry
         shell: bash
         # `cargo gc` will invoke `cargo build` with specified args
-        run: cargo gc --profile ci -- --bin greptime
+        run: cargo gc --profile ci -- --bin greptime --features pg_kvbackend
       - name: Pack greptime binary
         shell: bash
         run: |
@@ -317,11 +317,11 @@ jobs:
         with:
           repo-token: ${{ secrets.GITHUB_TOKEN }}
       - uses: actions-rust-lang/setup-rust-toolchain@v1
-      - name: Rust Cache
-        uses: Swatinem/rust-cache@v2
-        with:
-          # Shares across multiple jobs
-          shared-key: "fuzz-test-targets"
+      # - name: Rust Cache
+      #   uses: Swatinem/rust-cache@v2
+      #   with:
+      #     # Shares across multiple jobs
+      #     shared-key: "fuzz-test-targets"
       - name: Set Rust Fuzz
         shell: bash
         run: |
@@ -466,11 +466,11 @@ jobs:
         with:
           repo-token: ${{ secrets.GITHUB_TOKEN }}
       - uses: actions-rust-lang/setup-rust-toolchain@v1
-      - name: Rust Cache
-        uses: Swatinem/rust-cache@v2
-        with:
-          # Shares across multiple jobs
-          shared-key: "fuzz-test-targets"
+      # - name: Rust Cache
+      #   uses: Swatinem/rust-cache@v2
+      #   with:
+      #     # Shares across multiple jobs
+      #     shared-key: "fuzz-test-targets"
       - name: Set Rust Fuzz
         shell: bash
         run: |
@@ -573,13 +573,16 @@ jobs:
           - name: "Remote WAL"
             opts: "-w kafka -k 127.0.0.1:9092"
             kafka: true
+          - name: "Pg Kvbackend"
+            opts: "--setup-pg"
+            kafka: false
     timeout-minutes: 60
     steps:
       - uses: actions/checkout@v4
       - if: matrix.mode.kafka
         name: Setup kafka server
-        working-directory: tests-integration/fixtures/kafka
-        run: docker compose -f docker-compose-standalone.yml up -d --wait
+        working-directory: tests-integration/fixtures
+        run: docker compose up -d --wait kafka
       - name: Download pre-built binaries
         uses: actions/download-artifact@v4
         with:
@@ -609,11 +612,11 @@ jobs:
       - uses: actions-rust-lang/setup-rust-toolchain@v1
         with:
           components: rustfmt
-      - name: Rust Cache
-        uses: Swatinem/rust-cache@v2
-        with:
-          # Shares across multiple jobs
-          shared-key: "check-rust-fmt"
+      # - name: Rust Cache
+      #   uses: Swatinem/rust-cache@v2
+      #   with:
+      #     # Shares across multiple jobs
+      #     shared-key: "check-rust-fmt"
       - name: Check format
         run: make fmt-check
 
@@ -638,52 +641,52 @@ jobs:
       - name: Run cargo clippy
         run: make clippy
 
+  conflict-check:
+    name: Check for conflict
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Merge Conflict Finder
+        uses: olivernybroe/action-conflict-finder@v4.0
+
   coverage:
     if: github.event.pull_request.draft == false
     runs-on: ubuntu-20.04-8-cores
     timeout-minutes: 60
-    needs:  [clippy, fmt]
+    needs:  [conflict-check, clippy, fmt]
     steps:
       - uses: actions/checkout@v4
       - uses: arduino/setup-protoc@v3
         with:
           repo-token: ${{ secrets.GITHUB_TOKEN }}
-      - uses: KyleMayes/install-llvm-action@v1
-        with:
-          version: "14.0"
+      - uses: rui314/setup-mold@v1
       - name: Install toolchain
         uses: actions-rust-lang/setup-rust-toolchain@v1
         with:
-          components: llvm-tools-preview
+          components: llvm-tools
+          cache: false
       - name: Rust Cache
         uses: Swatinem/rust-cache@v2
         with:
           # Shares cross multiple jobs
           shared-key: "coverage-test"
-      - name: Docker Cache
-        uses: ScribeMD/docker-cache@0.3.7
-        with:
-          key: docker-${{ runner.os }}-coverage
+          save-if: ${{ github.ref == 'refs/heads/main' }}
+      # Disabled temporarily to see performance
+      # - name: Docker Cache
+      #   uses: ScribeMD/docker-cache@0.5.0
+      #   with:
+      #     key: docker-${{ runner.os }}-coverage
       - name: Install latest nextest release
         uses: taiki-e/install-action@nextest
       - name: Install cargo-llvm-cov
         uses: taiki-e/install-action@cargo-llvm-cov
-      - name: Setup etcd server
-        working-directory: tests-integration/fixtures/etcd
-        run: docker compose -f docker-compose-standalone.yml up -d --wait
-      - name: Setup kafka server
-        working-directory: tests-integration/fixtures/kafka
-        run: docker compose -f docker-compose-standalone.yml up -d --wait
-      - name: Setup minio
-        working-directory: tests-integration/fixtures/minio
-        run: docker compose -f docker-compose-standalone.yml up -d --wait
-      - name: Setup postgres server
-        working-directory: tests-integration/fixtures/postgres
-        run: docker compose -f docker-compose-standalone.yml up -d --wait
+      - name: Setup external services
+        working-directory: tests-integration/fixtures
+        run: docker compose up -d --wait
       - name: Run nextest cases
         run: cargo llvm-cov nextest --workspace --lcov --output-path lcov.info -F dashboard -F pg_kvbackend
         env:
-          CARGO_BUILD_RUSTFLAGS: "-C link-arg=-fuse-ld=lld"
+          CARGO_BUILD_RUSTFLAGS: "-C link-arg=-fuse-ld=mold"
           RUST_BACKTRACE: 1
           CARGO_INCREMENTAL: 0
           GT_S3_BUCKET: ${{ vars.AWS_CI_TEST_BUCKET }}

.github/workflows/nightly-ci.yml

@@ -108,7 +108,53 @@ jobs:
           GT_S3_REGION: ${{ vars.AWS_CI_TEST_BUCKET_REGION }}
           UNITTEST_LOG_DIR: "__unittest_logs"
 
+  ## this is designed for generating cache that usable for pull requests
+  test-on-linux:
+    name: Run tests on Linux
+    if: ${{ github.repository == 'GreptimeTeam/greptimedb' }}
+    runs-on: ubuntu-20.04-8-cores
+    timeout-minutes: 60
+    steps:
+      - uses: actions/checkout@v4
+      - uses: arduino/setup-protoc@v3
+        with:
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
+      - uses: rui314/setup-mold@v1
+      - name: Install Rust toolchain
+        uses: actions-rust-lang/setup-rust-toolchain@v1
+      - name: Rust Cache
+        uses: Swatinem/rust-cache@v2
+        with:
+          # Shares cross multiple jobs
+          shared-key: "coverage-test"
+      - name: Install Cargo Nextest
+        uses: taiki-e/install-action@nextest
+      - name: Setup external services
+        working-directory: tests-integration/fixtures
+        run: docker compose up -d --wait
+      - name: Running tests
+        run: cargo nextest run -F dashboard -F pg_kvbackend
+        env:
+          CARGO_BUILD_RUSTFLAGS: "-C link-arg=-fuse-ld=mold"
+          RUST_BACKTRACE: 1
+          CARGO_INCREMENTAL: 0
+          GT_S3_BUCKET: ${{ vars.AWS_CI_TEST_BUCKET }}
+          GT_S3_ACCESS_KEY_ID: ${{ secrets.AWS_CI_TEST_ACCESS_KEY_ID }}
+          GT_S3_ACCESS_KEY: ${{ secrets.AWS_CI_TEST_SECRET_ACCESS_KEY }}
+          GT_S3_REGION: ${{ vars.AWS_CI_TEST_BUCKET_REGION }}
+          GT_MINIO_BUCKET: greptime
+          GT_MINIO_ACCESS_KEY_ID: superpower_ci_user
+          GT_MINIO_ACCESS_KEY: superpower_password
+          GT_MINIO_REGION: us-west-2
+          GT_MINIO_ENDPOINT_URL: http://127.0.0.1:9000
+          GT_ETCD_ENDPOINTS: http://127.0.0.1:2379
+          GT_POSTGRES_ENDPOINTS: postgres://greptimedb:admin@127.0.0.1:5432/postgres
+          GT_KAFKA_ENDPOINTS: 127.0.0.1:9092
+          GT_KAFKA_SASL_ENDPOINTS: 127.0.0.1:9093
+          UNITTEST_LOG_DIR: "__unittest_logs"
+
   cleanbuild-linux-nix:
+    name: Run clean build on Linux
     runs-on: ubuntu-latest-8-cores
     timeout-minutes: 60
     steps:

Cargo.toml

@@ -55,7 +55,6 @@ members = [
     "src/promql",
     "src/puffin",
     "src/query",
-    "src/script",
     "src/servers",
     "src/session",
     "src/sql",
@@ -79,8 +78,6 @@ clippy.dbg_macro = "warn"
 clippy.implicit_clone = "warn"
 clippy.readonly_write_lock = "allow"
 rust.unknown_lints = "deny"
-# Remove this after https://github.com/PyO3/pyo3/issues/4094
-rust.non_local_definitions = "allow"
 rust.unexpected_cfgs = { level = "warn", check-cfg = ['cfg(tokio_unstable)'] }
 
 [workspace.dependencies]
@@ -99,6 +96,7 @@ arrow-schema = { version = "51.0", features = ["serde"] }
 async-stream = "0.3"
 async-trait = "0.1"
 axum = { version = "0.6", features = ["headers"] }
+backon = "1"
 base64 = "0.21"
 bigdecimal = "0.4.2"
 bitflags = "2.4.1"
@@ -118,13 +116,15 @@ datafusion-physical-expr = { git = "https://github.com/waynexia/arrow-datafusion
 datafusion-physical-plan = { git = "https://github.com/waynexia/arrow-datafusion.git", rev = "7823ef2f63663907edab46af0d51359900f608d6" }
 datafusion-sql = { git = "https://github.com/waynexia/arrow-datafusion.git", rev = "7823ef2f63663907edab46af0d51359900f608d6" }
 datafusion-substrait = { git = "https://github.com/waynexia/arrow-datafusion.git", rev = "7823ef2f63663907edab46af0d51359900f608d6" }
+deadpool = "0.10"
+deadpool-postgres = "0.12"
 derive_builder = "0.12"
 dotenv = "0.15"
 etcd-client = "0.13"
 fst = "0.4.7"
 futures = "0.3"
 futures-util = "0.3"
-greptime-proto = { git = "https://github.com/GreptimeTeam/greptime-proto.git", rev = "a875e976441188028353f7274a46a7e6e065c5d4" }
+greptime-proto = { git = "https://github.com/GreptimeTeam/greptime-proto.git", rev = "43ddd8dea69f4df0fe2e8b5cdc0044d2cfa35908" }
 hex = "0.4"
 http = "0.2"
 humantime = "2.1"
@@ -132,6 +132,7 @@ humantime-serde = "1.1"
 itertools = "0.10"
 jsonb = { git = "https://github.com/databendlabs/jsonb.git", rev = "8c8d2fc294a39f3ff08909d60f718639cfba3875", default-features = false }
 lazy_static = "1.4"
+local-ip-address = "0.6"
 meter-core = { git = "https://github.com/GreptimeTeam/greptime-meter.git", rev = "a10facb353b41460eeb98578868ebf19c2084fac" }
 mockall = "0.11.4"
 moka = "0.12"
@@ -179,15 +180,17 @@ similar-asserts = "1.6.0"
 smallvec = { version = "1", features = ["serde"] }
 snafu = "0.8"
 sysinfo = "0.30"
-# on branch v0.44.x
+
+rustls = { version = "0.23.20", default-features = false } # override by patch, see [patch.crates-io]
 sqlparser = { git = "https://github.com/GreptimeTeam/sqlparser-rs.git", rev = "54a267ac89c09b11c0c88934690530807185d3e7", features = [
     "visitor",
     "serde",
-] }
+] } # on branch v0.44.x
 strum = { version = "0.25", features = ["derive"] }
 tempfile = "3"
 tokio = { version = "1.40", features = ["full"] }
 tokio-postgres = "0.7"
+tokio-rustls = { version = "0.26.0", default-features = false } # override by patch, see [patch.crates-io]
 tokio-stream = "0.1"
 tokio-util = { version = "0.7", features = ["io-util", "compat"] }
 toml = "0.8.8"
@@ -254,7 +257,6 @@ plugins = { path = "src/plugins" }
 promql = { path = "src/promql" }
 puffin = { path = "src/puffin" }
 query = { path = "src/query" }
-script = { path = "src/script" }
 servers = { path = "src/servers" }
 session = { path = "src/session" }
 sql = { path = "src/sql" }
@@ -264,9 +266,9 @@ table = { path = "src/table" }
 
 [patch.crates-io]
 # change all rustls dependencies to use our fork to default to `ring` to make it "just work"
-hyper-rustls = { git = "https://github.com/GreptimeTeam/hyper-rustls" }
-rustls = { git = "https://github.com/GreptimeTeam/rustls" }
-tokio-rustls = { git = "https://github.com/GreptimeTeam/tokio-rustls" }
+hyper-rustls = { git = "https://github.com/GreptimeTeam/hyper-rustls", rev = "a951e03" } # version = "0.27.5" with ring patch
+rustls = { git = "https://github.com/GreptimeTeam/rustls", rev = "34fd0c6" }             # version = "0.23.20" with ring patch
+tokio-rustls = { git = "https://github.com/GreptimeTeam/tokio-rustls", rev = "4604ca6" } # version = "0.26.0" with ring patch
 # This is commented, since we are not using aws-lc-sys, if we need to use it, we need to uncomment this line or use a release after this commit, or it wouldn't compile with gcc < 8.1
 # see https://github.com/aws/aws-lc-rs/pull/526
 # aws-lc-sys = { git ="https://github.com/aws/aws-lc-rs", rev = "556558441e3494af4b156ae95ebc07ebc2fd38aa" }

README.md

@@ -138,7 +138,8 @@ Check the prerequisite:
 
 * [Rust toolchain](https://www.rust-lang.org/tools/install) (nightly)
 * [Protobuf compiler](https://grpc.io/docs/protoc-installation/) (>= 3.15)
-* Python toolchain (optional): Required only if built with PyO3 backend. More details for compiling with PyO3 can be found in its [documentation](https://pyo3.rs/v0.18.1/building_and_distribution#configuring-the-python-version).
+* C/C++ building essentials, including `gcc`/`g++`/`autoconf` and glibc library (eg. `libc6-dev` on Ubuntu and `glibc-devel` on Fedora)
+* Python toolchain (optional): Required only if using some test scripts.
 
 Build GreptimeDB binary:
 
@@ -228,4 +229,3 @@ Special thanks to all the contributors who have propelled GreptimeDB forward. Fo
 - GreptimeDB's query engine is powered by [Apache Arrow DataFusion™](https://arrow.apache.org/datafusion/).
 - [Apache OpenDAL™](https://opendal.apache.org) gives GreptimeDB a very general and elegant data access abstraction layer.
 - GreptimeDB's meta service is based on [etcd](https://etcd.io/).
-- GreptimeDB uses [RustPython](https://github.com/RustPython/RustPython) for experimental embedded python scripting.

config/config.md

@@ -94,7 +94,7 @@
 | `storage` | -- | -- | The data storage options. |
 | `storage.data_home` | String | `/tmp/greptimedb/` | The working home directory. |
 | `storage.type` | String | `File` | The storage type used to store the data.<br/>- `File`: the data is stored in the local file system.<br/>- `S3`: the data is stored in the S3 object storage.<br/>- `Gcs`: the data is stored in the Google Cloud Storage.<br/>- `Azblob`: the data is stored in the Azure Blob Storage.<br/>- `Oss`: the data is stored in the Aliyun OSS. |
-| `storage.cache_path` | String | Unset | Read cache configuration for object storage such as 'S3' etc, it's configured by default when using object storage. It is recommended to configure it when using object storage for better performance.<br/>A local file directory, defaults to `{data_home}/object_cache/read`. An empty string means disabling. |
+| `storage.cache_path` | String | Unset | Read cache configuration for object storage such as 'S3' etc, it's configured by default when using object storage. It is recommended to configure it when using object storage for better performance.<br/>A local file directory, defaults to `{data_home}`. An empty string means disabling. |
 | `storage.cache_capacity` | String | Unset | The local file cache capacity in bytes. If your disk space is sufficient, it is recommended to set it larger. |
 | `storage.bucket` | String | Unset | The S3 bucket name.<br/>**It's only used when the storage type is `S3`, `Oss` and `Gcs`**. |
 | `storage.root` | String | Unset | The S3 data will be stored in the specified prefix, for example, `s3://${bucket}/${root}`.<br/>**It's only used when the storage type is `S3`, `Oss` and `Azblob`**. |
@@ -132,10 +132,10 @@
 | `region_engine.mito.vector_cache_size` | String | Auto | Cache size for vectors and arrow arrays. Setting it to 0 to disable the cache.<br/>If not set, it's default to 1/16 of OS memory with a max limitation of 512MB. |
 | `region_engine.mito.page_cache_size` | String | Auto | Cache size for pages of SST row groups. Setting it to 0 to disable the cache.<br/>If not set, it's default to 1/8 of OS memory. |
 | `region_engine.mito.selector_result_cache_size` | String | Auto | Cache size for time series selector (e.g. `last_value()`). Setting it to 0 to disable the cache.<br/>If not set, it's default to 1/16 of OS memory with a max limitation of 512MB. |
-| `region_engine.mito.enable_experimental_write_cache` | Bool | `false` | Whether to enable the experimental write cache, it's enabled by default when using object storage. It is recommended to enable it when using object storage for better performance. |
-| `region_engine.mito.experimental_write_cache_path` | String | `""` | File system path for write cache, defaults to `{data_home}/object_cache/write`. |
-| `region_engine.mito.experimental_write_cache_size` | String | `5GiB` | Capacity for write cache. If your disk space is sufficient, it is recommended to set it larger. |
-| `region_engine.mito.experimental_write_cache_ttl` | String | Unset | TTL for write cache. |
+| `region_engine.mito.enable_write_cache` | Bool | `false` | Whether to enable the write cache, it's enabled by default when using object storage. It is recommended to enable it when using object storage for better performance. |
+| `region_engine.mito.write_cache_path` | String | `""` | File system path for write cache, defaults to `{data_home}`. |
+| `region_engine.mito.write_cache_size` | String | `5GiB` | Capacity for write cache. If your disk space is sufficient, it is recommended to set it larger. |
+| `region_engine.mito.write_cache_ttl` | String | Unset | TTL for write cache. |
 | `region_engine.mito.sst_write_buffer_size` | String | `8MB` | Buffer size for SST writing. |
 | `region_engine.mito.parallel_scan_channel_size` | Integer | `32` | Capacity of the channel to send data from parallel scan tasks to the main task. |
 | `region_engine.mito.allow_stale_entries` | Bool | `false` | Whether to allow stale WAL entries read during replay. |
@@ -151,7 +151,7 @@
 | `region_engine.mito.inverted_index.intermediate_path` | String | `""` | Deprecated, use `region_engine.mito.index.aux_path` instead. |
 | `region_engine.mito.inverted_index.metadata_cache_size` | String | `64MiB` | Cache size for inverted index metadata. |
 | `region_engine.mito.inverted_index.content_cache_size` | String | `128MiB` | Cache size for inverted index content. |
-| `region_engine.mito.inverted_index.content_cache_page_size` | String | `8MiB` | Page size for inverted index content cache. |
+| `region_engine.mito.inverted_index.content_cache_page_size` | String | `64KiB` | Page size for inverted index content cache. |
 | `region_engine.mito.fulltext_index` | -- | -- | The options for full-text index in Mito engine. |
 | `region_engine.mito.fulltext_index.create_on_flush` | String | `auto` | Whether to create the index on flush.<br/>- `auto`: automatically (default)<br/>- `disable`: never |
 | `region_engine.mito.fulltext_index.create_on_compaction` | String | `auto` | Whether to create the index on compaction.<br/>- `auto`: automatically (default)<br/>- `disable`: never |
@@ -214,7 +214,7 @@
 | `http.body_limit` | String | `64MB` | HTTP request body limit.<br/>The following units are supported: `B`, `KB`, `KiB`, `MB`, `MiB`, `GB`, `GiB`, `TB`, `TiB`, `PB`, `PiB`.<br/>Set to 0 to disable limit. |
 | `grpc` | -- | -- | The gRPC server options. |
 | `grpc.addr` | String | `127.0.0.1:4001` | The address to bind the gRPC server. |
-| `grpc.hostname` | String | `127.0.0.1` | The hostname advertised to the metasrv,<br/>and used for connections from outside the host |
+| `grpc.hostname` | String | `127.0.0.1:4001` | The hostname advertised to the metasrv,<br/>and used for connections from outside the host |
 | `grpc.runtime_size` | Integer | `8` | The number of server worker threads. |
 | `grpc.tls` | -- | -- | gRPC server TLS options, see `mysql.tls` section. |
 | `grpc.tls.mode` | String | `disable` | TLS mode. |
@@ -293,9 +293,11 @@
 | `data_home` | String | `/tmp/metasrv/` | The working home directory. |
 | `bind_addr` | String | `127.0.0.1:3002` | The bind address of metasrv. |
 | `server_addr` | String | `127.0.0.1:3002` | The communication server address for frontend and datanode to connect to metasrv,  "127.0.0.1:3002" by default for localhost. |
-| `store_addrs` | Array | -- | Store server address default to etcd store. |
+| `store_addrs` | Array | -- | Store server address default to etcd store.<br/>For postgres store, the format is:<br/>"password=password dbname=postgres user=postgres host=localhost port=5432"<br/>For etcd store, the format is:<br/>"127.0.0.1:2379" |
 | `store_key_prefix` | String | `""` | If it's not empty, the metasrv will store all data with this key prefix. |
-| `backend` | String | `EtcdStore` | The datastore for meta server. |
+| `backend` | String | `etcd_store` | The datastore for meta server.<br/>Available values:<br/>- `etcd_store` (default value)<br/>- `memory_store`<br/>- `postgres_store` |
+| `meta_table_name` | String | `greptime_metakv` | Table name in RDS to store metadata. Effect when using a RDS kvbackend.<br/>**Only used when backend is `postgres_store`.** |
+| `meta_election_lock_id` | Integer | `1` | Advisory lock id in PostgreSQL for election. Effect when using PostgreSQL as kvbackend<br/>Only used when backend is `postgres_store`. |
 | `selector` | String | `round_robin` | Datanode selector type.<br/>- `round_robin` (default value)<br/>- `lease_based`<br/>- `load_based`<br/>For details, please see "https://docs.greptime.com/developer-guide/metasrv/selector". |
 | `use_memory_store` | Bool | `false` | Store data in memory. |
 | `enable_region_failover` | Bool | `false` | Whether to enable region failover.<br/>This feature is only available on GreptimeDB running on cluster mode and<br/>- Using Remote WAL<br/>- Using shared storage (e.g., s3). |
@@ -378,7 +380,7 @@
 | `http.body_limit` | String | `64MB` | HTTP request body limit.<br/>The following units are supported: `B`, `KB`, `KiB`, `MB`, `MiB`, `GB`, `GiB`, `TB`, `TiB`, `PB`, `PiB`.<br/>Set to 0 to disable limit. |
 | `grpc` | -- | -- | The gRPC server options. |
 | `grpc.addr` | String | `127.0.0.1:3001` | The address to bind the gRPC server. |
-| `grpc.hostname` | String | `127.0.0.1` | The hostname advertised to the metasrv,<br/>and used for connections from outside the host |
+| `grpc.hostname` | String | `127.0.0.1:3001` | The hostname advertised to the metasrv,<br/>and used for connections from outside the host |
 | `grpc.runtime_size` | Integer | `8` | The number of server worker threads. |
 | `grpc.max_recv_message_size` | String | `512MB` | The maximum receive message size for gRPC server. |
 | `grpc.max_send_message_size` | String | `512MB` | The maximum send message size for gRPC server. |
@@ -466,10 +468,10 @@
 | `region_engine.mito.vector_cache_size` | String | Auto | Cache size for vectors and arrow arrays. Setting it to 0 to disable the cache.<br/>If not set, it's default to 1/16 of OS memory with a max limitation of 512MB. |
 | `region_engine.mito.page_cache_size` | String | Auto | Cache size for pages of SST row groups. Setting it to 0 to disable the cache.<br/>If not set, it's default to 1/8 of OS memory. |
 | `region_engine.mito.selector_result_cache_size` | String | Auto | Cache size for time series selector (e.g. `last_value()`). Setting it to 0 to disable the cache.<br/>If not set, it's default to 1/16 of OS memory with a max limitation of 512MB. |
-| `region_engine.mito.enable_experimental_write_cache` | Bool | `false` | Whether to enable the experimental write cache, it's enabled by default when using object storage. It is recommended to enable it when using object storage for better performance. |
-| `region_engine.mito.experimental_write_cache_path` | String | `""` | File system path for write cache, defaults to `{data_home}`. |
-| `region_engine.mito.experimental_write_cache_size` | String | `5GiB` | Capacity for write cache. If your disk space is sufficient, it is recommended to set it larger. |
-| `region_engine.mito.experimental_write_cache_ttl` | String | Unset | TTL for write cache. |
+| `region_engine.mito.enable_write_cache` | Bool | `false` | Whether to enable the write cache, it's enabled by default when using object storage. It is recommended to enable it when using object storage for better performance. |
+| `region_engine.mito.write_cache_path` | String | `""` | File system path for write cache, defaults to `{data_home}`. |
+| `region_engine.mito.write_cache_size` | String | `5GiB` | Capacity for write cache. If your disk space is sufficient, it is recommended to set it larger. |
+| `region_engine.mito.write_cache_ttl` | String | Unset | TTL for write cache. |
 | `region_engine.mito.sst_write_buffer_size` | String | `8MB` | Buffer size for SST writing. |
 | `region_engine.mito.parallel_scan_channel_size` | Integer | `32` | Capacity of the channel to send data from parallel scan tasks to the main task. |
 | `region_engine.mito.allow_stale_entries` | Bool | `false` | Whether to allow stale WAL entries read during replay. |
@@ -485,7 +487,7 @@
 | `region_engine.mito.inverted_index.intermediate_path` | String | `""` | Deprecated, use `region_engine.mito.index.aux_path` instead. |
 | `region_engine.mito.inverted_index.metadata_cache_size` | String | `64MiB` | Cache size for inverted index metadata. |
 | `region_engine.mito.inverted_index.content_cache_size` | String | `128MiB` | Cache size for inverted index content. |
-| `region_engine.mito.inverted_index.content_cache_page_size` | String | `8MiB` | Page size for inverted index content cache. |
+| `region_engine.mito.inverted_index.content_cache_page_size` | String | `64KiB` | Page size for inverted index content cache. |
 | `region_engine.mito.fulltext_index` | -- | -- | The options for full-text index in Mito engine. |
 | `region_engine.mito.fulltext_index.create_on_flush` | String | `auto` | Whether to create the index on flush.<br/>- `auto`: automatically (default)<br/>- `disable`: never |
 | `region_engine.mito.fulltext_index.create_on_compaction` | String | `auto` | Whether to create the index on compaction.<br/>- `auto`: automatically (default)<br/>- `disable`: never |

config/datanode.example.toml

@@ -59,7 +59,7 @@ body_limit = "64MB"
 addr = "127.0.0.1:3001"
 ## The hostname advertised to the metasrv,
 ## and used for connections from outside the host
-hostname = "127.0.0.1"
+hostname = "127.0.0.1:3001"
 ## The number of server worker threads.
 runtime_size = 8
 ## The maximum receive message size for gRPC server.
@@ -475,18 +475,18 @@ auto_flush_interval = "1h"
 ## @toml2docs:none-default="Auto"
 #+ selector_result_cache_size = "512MB"
 
-## Whether to enable the experimental write cache, it's enabled by default when using object storage. It is recommended to enable it when using object storage for better performance.
-enable_experimental_write_cache = false
+## Whether to enable the write cache, it's enabled by default when using object storage. It is recommended to enable it when using object storage for better performance.
+enable_write_cache = false
 
 ## File system path for write cache, defaults to `{data_home}`.
-experimental_write_cache_path = ""
+write_cache_path = ""
 
 ## Capacity for write cache. If your disk space is sufficient, it is recommended to set it larger.
-experimental_write_cache_size = "5GiB"
+write_cache_size = "5GiB"
 
 ## TTL for write cache.
 ## @toml2docs:none-default
-experimental_write_cache_ttl = "8h"
+write_cache_ttl = "8h"
 
 ## Buffer size for SST writing.
 sst_write_buffer_size = "8MB"
@@ -550,7 +550,7 @@ metadata_cache_size = "64MiB"
 content_cache_size = "128MiB"
 
 ## Page size for inverted index content cache.
-content_cache_page_size = "8MiB"
+content_cache_page_size = "64KiB"
 
 ## The options for full-text index in Mito engine.
 [region_engine.mito.fulltext_index]

config/frontend.example.toml

@@ -38,7 +38,7 @@ body_limit = "64MB"
 addr = "127.0.0.1:4001"
 ## The hostname advertised to the metasrv,
 ## and used for connections from outside the host
-hostname = "127.0.0.1"
+hostname = "127.0.0.1:4001"
 ## The number of server worker threads.
 runtime_size = 8
 

config/metasrv.example.toml

@@ -8,13 +8,29 @@ bind_addr = "127.0.0.1:3002"
 server_addr = "127.0.0.1:3002"
 
 ## Store server address default to etcd store.
+## For postgres store, the format is:
+## "password=password dbname=postgres user=postgres host=localhost port=5432"
+## For etcd store, the format is:
+## "127.0.0.1:2379"
 store_addrs = ["127.0.0.1:2379"]
 
 ## If it's not empty, the metasrv will store all data with this key prefix.
 store_key_prefix = ""
 
 ## The datastore for meta server.
-backend = "EtcdStore"
+## Available values:
+## - `etcd_store` (default value)
+## - `memory_store`
+## - `postgres_store`
+backend = "etcd_store"
+
+## Table name in RDS to store metadata. Effect when using a RDS kvbackend.
+## **Only used when backend is `postgres_store`.**
+meta_table_name = "greptime_metakv"
+
+## Advisory lock id in PostgreSQL for election. Effect when using PostgreSQL as kvbackend
+## Only used when backend is `postgres_store`.
+meta_election_lock_id = 1
 
 ## Datanode selector type.
 ## - `round_robin` (default value)

config/standalone.example.toml

@@ -337,7 +337,7 @@ data_home = "/tmp/greptimedb/"
 type = "File"
 
 ## Read cache configuration for object storage such as 'S3' etc, it's configured by default when using object storage. It is recommended to configure it when using object storage for better performance.
-## A local file directory, defaults to `{data_home}/object_cache/read`. An empty string means disabling.
+## A local file directory, defaults to `{data_home}`. An empty string means disabling.
 ## @toml2docs:none-default
 #+ cache_path = ""
 
@@ -518,18 +518,18 @@ auto_flush_interval = "1h"
 ## @toml2docs:none-default="Auto"
 #+ selector_result_cache_size = "512MB"
 
-## Whether to enable the experimental write cache, it's enabled by default when using object storage. It is recommended to enable it when using object storage for better performance.
-enable_experimental_write_cache = false
+## Whether to enable the write cache, it's enabled by default when using object storage. It is recommended to enable it when using object storage for better performance.
+enable_write_cache = false
 
-## File system path for write cache, defaults to `{data_home}/object_cache/write`.
-experimental_write_cache_path = ""
+## File system path for write cache, defaults to `{data_home}`.
+write_cache_path = ""
 
 ## Capacity for write cache. If your disk space is sufficient, it is recommended to set it larger.
-experimental_write_cache_size = "5GiB"
+write_cache_size = "5GiB"
 
 ## TTL for write cache.
 ## @toml2docs:none-default
-experimental_write_cache_ttl = "8h"
+write_cache_ttl = "8h"
 
 ## Buffer size for SST writing.
 sst_write_buffer_size = "8MB"
@@ -593,7 +593,7 @@ metadata_cache_size = "64MiB"
 content_cache_size = "128MiB"
 
 ## Page size for inverted index content cache.
-content_cache_page_size = "8MiB"
+content_cache_page_size = "64KiB"
 
 ## The options for full-text index in Mito engine.
 [region_engine.mito.fulltext_index]

docker/buildx/centos/Dockerfile

@@ -22,7 +22,7 @@ RUN unzip protoc-3.15.8-linux-x86_64.zip -d /usr/local/
 # Install Rust
 SHELL ["/bin/bash", "-c"]
 RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- --no-modify-path --default-toolchain none -y
-ENV PATH /opt/rh/rh-python38/root/usr/bin:/usr/local/bin:/root/.cargo/bin/:$PATH
+ENV PATH /usr/local/bin:/root/.cargo/bin/:$PATH
 
 # Build the project in release mode.
 RUN --mount=target=.,rw \

docker/buildx/ubuntu/Dockerfile

@@ -7,10 +7,8 @@ ARG OUTPUT_DIR
 ENV LANG en_US.utf8
 WORKDIR /greptimedb
 
-# Add PPA for Python 3.10.
 RUN apt-get update && \
-    DEBIAN_FRONTEND=noninteractive apt-get install -y software-properties-common && \
-    add-apt-repository ppa:deadsnakes/ppa -y
+    DEBIAN_FRONTEND=noninteractive apt-get install -y software-properties-common
 
 # Install dependencies.
 RUN --mount=type=cache,target=/var/cache/apt \

docker/dev-builder/android/Dockerfile

@@ -13,12 +13,7 @@ RUN apt-get update && apt-get install -y \
     curl \
     git \
     build-essential \
-    pkg-config \
-    python3 \
-    python3-dev \
-    python3-pip \
-    && pip3 install --upgrade pip \
-    && pip3 install pyarrow
+    pkg-config
 
 # Trust workdir
 RUN git config --global --add safe.directory /greptimedb

docker/dev-builder/centos/Dockerfile

@@ -12,8 +12,6 @@ RUN yum install -y epel-release  \
     openssl \
     openssl-devel  \
     centos-release-scl  \
-    rh-python38  \
-    rh-python38-python-devel \
     which
 
 # Install protoc
@@ -23,7 +21,7 @@ RUN unzip protoc-3.15.8-linux-x86_64.zip -d /usr/local/
 # Install Rust
 SHELL ["/bin/bash", "-c"]
 RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- --no-modify-path --default-toolchain none -y
-ENV PATH /opt/rh/rh-python38/root/usr/bin:/usr/local/bin:/root/.cargo/bin/:$PATH
+ENV PATH /usr/local/bin:/root/.cargo/bin/:$PATH
 
 # Install Rust toolchains.
 ARG RUST_TOOLCHAIN

docker/dev-builder/ubuntu/Dockerfile

@@ -6,11 +6,8 @@ ARG DOCKER_BUILD_ROOT=.
 ENV LANG en_US.utf8
 WORKDIR /greptimedb
 
-# Add PPA for Python 3.10.
 RUN apt-get update && \
-    DEBIAN_FRONTEND=noninteractive apt-get install -y software-properties-common && \
-    add-apt-repository ppa:deadsnakes/ppa -y
-
+    DEBIAN_FRONTEND=noninteractive apt-get install -y software-properties-common
 # Install dependencies.
 RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \
     libssl-dev \
@@ -20,9 +17,7 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \
     ca-certificates \
     git \
     build-essential \
-    pkg-config \
-    python3.10 \
-    python3.10-dev
+    pkg-config
 
 ARG TARGETPLATFORM
 RUN echo "target platform: $TARGETPLATFORM"
@@ -38,21 +33,6 @@ fi
 RUN mv protoc3/bin/* /usr/local/bin/
 RUN mv protoc3/include/* /usr/local/include/
 
-# https://github.com/GreptimeTeam/greptimedb/actions/runs/10935485852/job/30357457188#step:3:7106
-# `aws-lc-sys` require gcc >= 10.3.0 to work, hence alias to use gcc-10
-RUN apt-get remove -y gcc-9 g++-9 cpp-9 && \
-    apt-get install -y gcc-10 g++-10 cpp-10 make cmake && \
-    ln -sf /usr/bin/gcc-10 /usr/bin/gcc && ln -sf /usr/bin/g++-10 /usr/bin/g++ && \
-    ln -sf /usr/bin/gcc-10 /usr/bin/cc && \
-    ln -sf /usr/bin/g++-10 /usr/bin/cpp && ln -sf /usr/bin/g++-10 /usr/bin/c++ && \
-    cc --version && gcc --version && g++ --version && cpp --version && c++ --version
-
-# Remove Python 3.8 and install pip.
-RUN apt-get -y purge python3.8 && \
-    apt-get -y autoremove && \
-    ln -s /usr/bin/python3.10 /usr/bin/python3 && \
-    curl -sS https://bootstrap.pypa.io/get-pip.py | python3.10
-
 # Silence all `safe.directory` warnings, to avoid the "detect dubious repository" error when building with submodules.
 # Disabling the safe directory check here won't pose extra security issues, because in our usage for this dev build
 # image, we use it solely on our own environment (that github action's VM, or ECS created dynamically by ourselves),
@@ -65,10 +45,6 @@ RUN apt-get -y purge python3.8 && \
 # it can be a different user that have prepared the submodules.
 RUN git config --global --add safe.directory '*'
 
-# Install Python dependencies.
-COPY $DOCKER_BUILD_ROOT/docker/python/requirements.txt /etc/greptime/requirements.txt
-RUN python3 -m pip install -r /etc/greptime/requirements.txt
-
 # Install Rust.
 SHELL ["/bin/bash", "-c"]
 RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- --no-modify-path --default-toolchain none -y

docker/python/requirements.txt

@@ -1,5 +0,0 @@
-numpy>=1.24.2
-pandas>=1.5.3
-pyarrow>=11.0.0
-requests>=2.28.2
-scipy>=1.10.1

grafana/greptimedb-cluster.json

@@ -5296,7 +5296,7 @@
             "uid": "${metrics}"
           },
           "editorMode": "code",
-          "expr": "sum by(pod, scheme, operation) (rate(opendal_requests_total{pod=~\"$datanode\"}[$__rate_interval]))",
+          "expr": "sum by(pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{pod=~\"$datanode\"}[$__rate_interval]))",
           "instant": false,
           "legendFormat": "[{{pod}}]-[{{scheme}}]-[{{operation}}]-qps",
           "range": true,
@@ -5392,7 +5392,7 @@
             "uid": "${metrics}"
           },
           "editorMode": "code",
-          "expr": "sum by(pod, scheme) (rate(opendal_requests_duration_seconds_count{pod=~\"$datanode\", operation=\"read\"}[$__rate_interval]))",
+          "expr": "sum by(pod, scheme) (rate(opendal_operation_duration_seconds_count{pod=~\"$datanode\", operation=\"read\"}[$__rate_interval]))",
           "instant": false,
           "legendFormat": "[{{pod}}]-[{{scheme}}]-qps",
           "range": true,
@@ -5488,7 +5488,7 @@
             "uid": "${metrics}"
           },
           "editorMode": "code",
-          "expr": "histogram_quantile(0.99, sum by(pod, le, scheme) (rate(opendal_requests_duration_seconds_bucket{pod=~\"$datanode\",operation=\"read\"}[$__rate_interval])))",
+          "expr": "histogram_quantile(0.99, sum by(pod, le, scheme) (rate(opendal_operation_duration_seconds_bucket{pod=~\"$datanode\",operation=\"read\"}[$__rate_interval])))",
           "instant": false,
           "legendFormat": "[{{pod}}]-{{scheme}}-p99",
           "range": true,
@@ -5584,7 +5584,7 @@
             "uid": "${metrics}"
           },
           "editorMode": "code",
-          "expr": "sum by(pod, scheme) (rate(opendal_requests_duration_seconds_count{pod=~\"$datanode\", operation=\"write\"}[$__rate_interval]))",
+          "expr": "sum by(pod, scheme) (rate(opendal_operation_duration_seconds_count{pod=~\"$datanode\", operation=\"write\"}[$__rate_interval]))",
           "instant": false,
           "legendFormat": "[{{pod}}]-[{{scheme}}]-qps",
           "range": true,
@@ -5680,7 +5680,7 @@
             "uid": "${metrics}"
           },
           "editorMode": "code",
-          "expr": "histogram_quantile(0.99, sum by(pod, le, scheme) (rate(opendal_requests_duration_seconds_bucket{pod=~\"$datanode\", operation=\"write\"}[$__rate_interval])))",
+          "expr": "histogram_quantile(0.99, sum by(pod, le, scheme) (rate(opendal_operation_duration_seconds_bucket{pod=~\"$datanode\", operation=\"write\"}[$__rate_interval])))",
           "instant": false,
           "legendFormat": "[{{pod}}]-[{{scheme}}]-p99",
           "range": true,
@@ -5776,7 +5776,7 @@
             "uid": "${metrics}"
           },
           "editorMode": "code",
-          "expr": "sum by(pod, scheme) (rate(opendal_requests_duration_seconds_count{pod=~\"$datanode\", operation=\"list\"}[$__rate_interval]))",
+          "expr": "sum by(pod, scheme) (rate(opendal_operation_duration_seconds_count{pod=~\"$datanode\", operation=\"list\"}[$__rate_interval]))",
           "instant": false,
           "interval": "",
           "legendFormat": "[{{pod}}]-[{{scheme}}]-qps",
@@ -5873,7 +5873,7 @@
             "uid": "${metrics}"
           },
           "editorMode": "code",
-          "expr": "histogram_quantile(0.99, sum by(pod, le, scheme) (rate(opendal_requests_duration_seconds_bucket{pod=~\"$datanode\", operation=\"list\"}[$__rate_interval])))",
+          "expr": "histogram_quantile(0.99, sum by(pod, le, scheme) (rate(opendal_operation_duration_seconds_bucket{pod=~\"$datanode\", operation=\"list\"}[$__rate_interval])))",
           "instant": false,
           "interval": "",
           "legendFormat": "[{{pod}}]-[{{scheme}}]-p99",
@@ -5970,7 +5970,7 @@
             "uid": "${metrics}"
           },
           "editorMode": "code",
-          "expr": "sum by(pod, scheme, operation) (rate(opendal_requests_duration_seconds_count{pod=~\"$datanode\",operation!~\"read|write|list|stat\"}[$__rate_interval]))",
+          "expr": "sum by(pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{pod=~\"$datanode\",operation!~\"read|write|list|stat\"}[$__rate_interval]))",
           "instant": false,
           "legendFormat": "[{{pod}}]-[{{scheme}}]-[{{operation}}]-qps",
           "range": true,
@@ -6066,7 +6066,7 @@
             "uid": "${metrics}"
           },
           "editorMode": "code",
-          "expr": "histogram_quantile(0.99, sum by(pod, le, scheme, operation) (rate(opendal_requests_duration_seconds_bucket{pod=~\"$datanode\", operation!~\"read|write|list\"}[$__rate_interval])))",
+          "expr": "histogram_quantile(0.99, sum by(pod, le, scheme, operation) (rate(opendal_operation_duration_seconds_bucket{pod=~\"$datanode\", operation!~\"read|write|list\"}[$__rate_interval])))",
           "instant": false,
           "legendFormat": "[{{pod}}]-[{{scheme}}]-[{{operation}}]-p99",
           "range": true,
@@ -6298,6 +6298,6 @@
   "timezone": "",
   "title": "GreptimeDB Cluster Metrics",
   "uid": "ce3q6xwn3xa0wa",
-  "version": 1,
+  "version": 2,
   "weekStart": ""
-}
+}

scripts/check-snafu.py

@@ -14,6 +14,7 @@
 
 import os
 import re
+from multiprocessing import Pool
 
 
 def find_rust_files(directory):
@@ -33,13 +34,11 @@ def extract_branch_names(file_content):
     return pattern.findall(file_content)
 
 
-def check_snafu_in_files(branch_name, rust_files):
+def check_snafu_in_files(branch_name, rust_files_content):
     branch_name_snafu = f"{branch_name}Snafu"
-    for rust_file in rust_files:
-        with open(rust_file, "r") as file:
-            content = file.read()
-            if branch_name_snafu in content:
-                return True
+    for content in rust_files_content.values():
+        if branch_name_snafu in content:
+            return True
     return False
 
 
@@ -49,21 +48,24 @@ def main():
 
     for error_file in error_files:
         with open(error_file, "r") as file:
-            content = file.read()
-            branch_names.extend(extract_branch_names(content))
+            branch_names.extend(extract_branch_names(file.read()))
 
-    unused_snafu = [
-        branch_name
-        for branch_name in branch_names
-        if not check_snafu_in_files(branch_name, other_rust_files)
-    ]
+    # Read all rust files into memory once
+    rust_files_content = {}
+    for rust_file in other_rust_files:
+        with open(rust_file, "r") as file:
+            rust_files_content[rust_file] = file.read()
+
+    with Pool() as pool:
+        results = pool.starmap(
+            check_snafu_in_files, [(bn, rust_files_content) for bn in branch_names]
+        )
+    unused_snafu = [bn for bn, found in zip(branch_names, results) if not found]
 
     if unused_snafu:
         print("Unused error variants:")
         for name in unused_snafu:
             print(name)
-
-    if unused_snafu:
         raise SystemExit(1)
 
 

shell.nix

@@ -1,5 +1,5 @@
 let
-  nixpkgs = fetchTarball "https://github.com/NixOS/nixpkgs/tarball/nixos-unstable";
+  nixpkgs = fetchTarball "https://github.com/NixOS/nixpkgs/tarball/nixos-24.11";
   fenix = import (fetchTarball "https://github.com/nix-community/fenix/archive/main.tar.gz") {};
   pkgs = import nixpkgs { config = {}; overlays = []; };
 in
@@ -17,10 +17,12 @@ pkgs.mkShell rec {
     })
     cargo-nextest
     taplo
+    curl
   ];
 
   buildInputs = with pkgs; [
     libgit2
+    libz
   ];
 
   LD_LIBRARY_PATH = pkgs.lib.makeLibraryPath buildInputs;

src/catalog/src/error.rs

@@ -122,13 +122,6 @@ pub enum Error {
         source: BoxedError,
     },
 
-    #[snafu(display("Failed to re-compile script due to internal error"))]
-    CompileScriptInternal {
-        #[snafu(implicit)]
-        location: Location,
-        source: BoxedError,
-    },
-
     #[snafu(display("Failed to create table, table info: {}", table_info))]
     CreateTable {
         table_info: String,
@@ -343,9 +336,7 @@ impl ErrorExt for Error {
             Error::DecodePlan { source, .. } => source.status_code(),
             Error::InvalidTableInfoInCatalog { source, .. } => source.status_code(),
 
-            Error::CompileScriptInternal { source, .. } | Error::Internal { source, .. } => {
-                source.status_code()
-            }
+            Error::Internal { source, .. } => source.status_code(),
 
             Error::QueryAccessDenied { .. } => StatusCode::AccessDenied,
             Error::Datafusion { error, .. } => datafusion_status_code::<Self>(error, None),

src/catalog/src/system_schema/information_schema/key_column_usage.rs

@@ -58,6 +58,8 @@ pub(crate) const TIME_INDEX_CONSTRAINT_NAME: &str = "TIME INDEX";
 pub(crate) const INVERTED_INDEX_CONSTRAINT_NAME: &str = "INVERTED INDEX";
 /// Fulltext index constraint name
 pub(crate) const FULLTEXT_INDEX_CONSTRAINT_NAME: &str = "FULLTEXT INDEX";
+/// Skipping index constraint name
+pub(crate) const SKIPPING_INDEX_CONSTRAINT_NAME: &str = "SKIPPING INDEX";
 
 /// The virtual table implementation for `information_schema.KEY_COLUMN_USAGE`.
 pub(super) struct InformationSchemaKeyColumnUsage {
@@ -225,6 +227,12 @@ impl InformationSchemaKeyColumnUsageBuilder {
                 let keys = &table_info.meta.primary_key_indices;
                 let schema = table.schema();
 
+                // For compatibility, use primary key columns as inverted index columns.
+                let pk_as_inverted_index = !schema
+                    .column_schemas()
+                    .iter()
+                    .any(|c| c.has_inverted_index_key());
+
                 for (idx, column) in schema.column_schemas().iter().enumerate() {
                     let mut constraints = vec![];
                     if column.is_time_index() {
@@ -242,14 +250,20 @@ impl InformationSchemaKeyColumnUsageBuilder {
                     // TODO(dimbtp): foreign key constraint not supported yet
                     if keys.contains(&idx) {
                         constraints.push(PRI_CONSTRAINT_NAME);
+
+                        if pk_as_inverted_index {
+                            constraints.push(INVERTED_INDEX_CONSTRAINT_NAME);
+                        }
                     }
                     if column.is_inverted_indexed() {
                         constraints.push(INVERTED_INDEX_CONSTRAINT_NAME);
                     }
-
-                    if column.has_fulltext_index_key() {
+                    if column.is_fulltext_indexed() {
                         constraints.push(FULLTEXT_INDEX_CONSTRAINT_NAME);
                     }
+                    if column.is_skipping_indexed() {
+                        constraints.push(SKIPPING_INDEX_CONSTRAINT_NAME);
+                    }
 
                     if !constraints.is_empty() {
                         let aggregated_constraints = constraints.join(", ");

src/cli/Cargo.toml

@@ -4,6 +4,9 @@ version.workspace = true
 edition.workspace = true
 license.workspace = true
 
+[features]
+pg_kvbackend = ["common-meta/pg_kvbackend"]
+
 [lints]
 workspace = true
 
@@ -56,7 +59,6 @@ tokio.workspace = true
 tracing-appender.workspace = true
 
 [dev-dependencies]
-common-test-util.workspace = true
 common-version.workspace = true
 serde.workspace = true
 tempfile.workspace = true

src/cli/src/bench.rs

@@ -22,6 +22,9 @@ use clap::Parser;
 use common_error::ext::BoxedError;
 use common_meta::key::{TableMetadataManager, TableMetadataManagerRef};
 use common_meta::kv_backend::etcd::EtcdStore;
+use common_meta::kv_backend::memory::MemoryKvBackend;
+#[cfg(feature = "pg_kvbackend")]
+use common_meta::kv_backend::postgres::PgStore;
 use common_meta::peer::Peer;
 use common_meta::rpc::router::{Region, RegionRoute};
 use common_telemetry::info;
@@ -55,18 +58,34 @@ where
 #[derive(Debug, Default, Parser)]
 pub struct BenchTableMetadataCommand {
     #[clap(long)]
-    etcd_addr: String,
+    etcd_addr: Option<String>,
+    #[cfg(feature = "pg_kvbackend")]
+    #[clap(long)]
+    postgres_addr: Option<String>,
     #[clap(long)]
     count: u32,
 }
 
 impl BenchTableMetadataCommand {
     pub async fn build(&self) -> std::result::Result<Box<dyn Tool>, BoxedError> {
-        let etcd_store = EtcdStore::with_endpoints([&self.etcd_addr], 128)
-            .await
-            .unwrap();
+        let kv_backend = if let Some(etcd_addr) = &self.etcd_addr {
+            info!("Using etcd as kv backend");
+            EtcdStore::with_endpoints([etcd_addr], 128).await.unwrap()
+        } else {
+            Arc::new(MemoryKvBackend::new())
+        };
 
-        let table_metadata_manager = Arc::new(TableMetadataManager::new(etcd_store));
+        #[cfg(feature = "pg_kvbackend")]
+        let kv_backend = if let Some(postgres_addr) = &self.postgres_addr {
+            info!("Using postgres as kv backend");
+            PgStore::with_url(postgres_addr, "greptime_metakv", 128)
+                .await
+                .unwrap()
+        } else {
+            kv_backend
+        };
+
+        let table_metadata_manager = Arc::new(TableMetadataManager::new(kv_backend));
 
         let tool = BenchTableMetadata {
             table_metadata_manager,

src/cmd/Cargo.toml

@@ -10,9 +10,8 @@ name = "greptime"
 path = "src/bin/greptime.rs"
 
 [features]
-default = ["python", "servers/pprof", "servers/mem-prof"]
+default = ["servers/pprof", "servers/mem-prof"]
 tokio-console = ["common-telemetry/tokio-console"]
-python = ["frontend/python"]
 
 [lints]
 workspace = true

src/cmd/src/datanode.rs

@@ -62,6 +62,11 @@ impl Instance {
     pub fn datanode(&self) -> &Datanode {
         &self.datanode
     }
+
+    /// allow customizing datanode for downstream projects
+    pub fn datanode_mut(&mut self) -> &mut Datanode {
+        &mut self.datanode
+    }
 }
 
 #[async_trait]
@@ -271,7 +276,8 @@ impl StartCommand {
         info!("Datanode options: {:#?}", opts);
 
         let plugin_opts = opts.plugins;
-        let opts = opts.component;
+        let mut opts = opts.component;
+        opts.grpc.detect_hostname();
         let mut plugins = Plugins::new();
         plugins::setup_datanode_plugins(&mut plugins, &plugin_opts, &opts)
             .await

src/cmd/src/flownode.rs

@@ -66,6 +66,11 @@ impl Instance {
     pub fn flownode(&self) -> &FlownodeInstance {
         &self.flownode
     }
+
+    /// allow customizing flownode for downstream projects
+    pub fn flownode_mut(&mut self) -> &mut FlownodeInstance {
+        &mut self.flownode
+    }
 }
 
 #[async_trait::async_trait]
@@ -217,7 +222,8 @@ impl StartCommand {
         info!("Flownode start command: {:#?}", self);
         info!("Flownode options: {:#?}", opts);
 
-        let opts = opts.component;
+        let mut opts = opts.component;
+        opts.grpc.detect_hostname();
 
         // TODO(discord9): make it not optionale after cluster id is required
         let cluster_id = opts.cluster_id.unwrap_or(0);

src/cmd/src/frontend.rs

@@ -268,7 +268,8 @@ impl StartCommand {
         info!("Frontend options: {:#?}", opts);
 
         let plugin_opts = opts.plugins;
-        let opts = opts.component;
+        let mut opts = opts.component;
+        opts.grpc.detect_hostname();
         let mut plugins = Plugins::new();
         plugins::setup_frontend_plugins(&mut plugins, &plugin_opts, &opts)
             .await

src/cmd/src/metasrv.rs

@@ -249,8 +249,6 @@ impl StartCommand {
 
         if let Some(backend) = &self.backend {
             opts.backend.clone_from(backend);
-        } else {
-            opts.backend = BackendImpl::default()
         }
 
         // Disable dashboard in metasrv.
@@ -274,7 +272,8 @@ impl StartCommand {
         info!("Metasrv options: {:#?}", opts);
 
         let plugin_opts = opts.plugins;
-        let opts = opts.component;
+        let mut opts = opts.component;
+        opts.detect_server_addr();
         let mut plugins = Plugins::new();
         plugins::setup_metasrv_plugins(&mut plugins, &plugin_opts, &opts)
             .await

src/cmd/src/standalone.rs

@@ -461,7 +461,8 @@ impl StartCommand {
 
         let mut plugins = Plugins::new();
         let plugin_opts = opts.plugins;
-        let opts = opts.component;
+        let mut opts = opts.component;
+        opts.grpc.detect_hostname();
         let fe_opts = opts.frontend_options();
         let dn_opts = opts.datanode_options();
 

src/cmd/tests/load_config_test.rs

@@ -69,7 +69,7 @@ fn test_load_datanode_example_config() {
             region_engine: vec![
                 RegionEngineConfig::Mito(MitoConfig {
                     auto_flush_interval: Duration::from_secs(3600),
-                    experimental_write_cache_ttl: Some(Duration::from_secs(60 * 60 * 8)),
+                    write_cache_ttl: Some(Duration::from_secs(60 * 60 * 8)),
                     ..Default::default()
                 }),
                 RegionEngineConfig::File(EngineConfig {}),
@@ -85,7 +85,9 @@ fn test_load_datanode_example_config() {
                 remote_write: Some(Default::default()),
                 ..Default::default()
             },
-            grpc: GrpcOptions::default().with_addr("127.0.0.1:3001"),
+            grpc: GrpcOptions::default()
+                .with_addr("127.0.0.1:3001")
+                .with_hostname("127.0.0.1:3001"),
             rpc_addr: Some("127.0.0.1:3001".to_string()),
             rpc_hostname: Some("127.0.0.1".to_string()),
             rpc_runtime_size: Some(8),
@@ -137,6 +139,7 @@ fn test_load_frontend_example_config() {
                 remote_write: Some(Default::default()),
                 ..Default::default()
             },
+            grpc: GrpcOptions::default().with_hostname("127.0.0.1:4001"),
             ..Default::default()
         },
         ..Default::default()
@@ -154,6 +157,7 @@ fn test_load_metasrv_example_config() {
         component: MetasrvOptions {
             selector: SelectorType::default(),
             data_home: "/tmp/metasrv/".to_string(),
+            server_addr: "127.0.0.1:3002".to_string(),
             logging: LoggingOptions {
                 dir: "/tmp/greptimedb/logs".to_string(),
                 level: Some("info".to_string()),
@@ -203,7 +207,7 @@ fn test_load_standalone_example_config() {
             region_engine: vec![
                 RegionEngineConfig::Mito(MitoConfig {
                     auto_flush_interval: Duration::from_secs(3600),
-                    experimental_write_cache_ttl: Some(Duration::from_secs(60 * 60 * 8)),
+                    write_cache_ttl: Some(Duration::from_secs(60 * 60 * 8)),
                     ..Default::default()
                 }),
                 RegionEngineConfig::File(EngineConfig {}),

src/common/base/Cargo.toml

@@ -4,6 +4,9 @@ version.workspace = true
 edition.workspace = true
 license.workspace = true
 
+[features]
+testing = []
+
 [lints]
 workspace = true
 

src/common/base/src/range_read.rs

@@ -17,6 +17,7 @@ use std::io;
 use std::ops::Range;
 use std::path::Path;
 use std::pin::Pin;
+use std::sync::atomic::{AtomicU64, Ordering};
 use std::sync::Arc;
 use std::task::{Context, Poll};
 
@@ -33,19 +34,22 @@ pub struct Metadata {
     pub content_length: u64,
 }
 
-/// `RangeReader` reads a range of bytes from a source.
-#[async_trait]
-pub trait RangeReader: Send + Unpin {
+/// `SizeAwareRangeReader` is a `RangeReader` that supports setting a file size hint.
+pub trait SizeAwareRangeReader: RangeReader {
     /// Sets the file size hint for the reader.
     ///
     /// It's used to optimize the reading process by reducing the number of remote requests.
     fn with_file_size_hint(&mut self, file_size_hint: u64);
+}
 
+/// `RangeReader` reads a range of bytes from a source.
+#[async_trait]
+pub trait RangeReader: Sync + Send + Unpin {
     /// Returns the metadata of the source.
-    async fn metadata(&mut self) -> io::Result<Metadata>;
+    async fn metadata(&self) -> io::Result<Metadata>;
 
     /// Reads the bytes in the given range.
-    async fn read(&mut self, range: Range<u64>) -> io::Result<Bytes>;
+    async fn read(&self, range: Range<u64>) -> io::Result<Bytes>;
 
     /// Reads the bytes in the given range into the buffer.
     ///
@@ -53,18 +57,14 @@ pub trait RangeReader: Send + Unpin {
     /// - If the buffer is insufficient to hold the bytes, it will either:
     ///   - Allocate additional space (e.g., for `Vec<u8>`)
     ///   - Panic (e.g., for `&mut [u8]`)
-    async fn read_into(
-        &mut self,
-        range: Range<u64>,
-        buf: &mut (impl BufMut + Send),
-    ) -> io::Result<()> {
+    async fn read_into(&self, range: Range<u64>, buf: &mut (impl BufMut + Send)) -> io::Result<()> {
         let bytes = self.read(range).await?;
         buf.put_slice(&bytes);
         Ok(())
     }
 
     /// Reads the bytes in the given ranges.
-    async fn read_vec(&mut self, ranges: &[Range<u64>]) -> io::Result<Vec<Bytes>> {
+    async fn read_vec(&self, ranges: &[Range<u64>]) -> io::Result<Vec<Bytes>> {
         let mut result = Vec::with_capacity(ranges.len());
         for range in ranges {
             result.push(self.read(range.clone()).await?);
@@ -74,25 +74,20 @@ pub trait RangeReader: Send + Unpin {
 }
 
 #[async_trait]
-impl<R: ?Sized + RangeReader> RangeReader for &mut R {
-    fn with_file_size_hint(&mut self, file_size_hint: u64) {
-        (*self).with_file_size_hint(file_size_hint)
-    }
-
-    async fn metadata(&mut self) -> io::Result<Metadata> {
+impl<R: ?Sized + RangeReader> RangeReader for &R {
+    async fn metadata(&self) -> io::Result<Metadata> {
         (*self).metadata().await
     }
-    async fn read(&mut self, range: Range<u64>) -> io::Result<Bytes> {
+
+    async fn read(&self, range: Range<u64>) -> io::Result<Bytes> {
         (*self).read(range).await
     }
-    async fn read_into(
-        &mut self,
-        range: Range<u64>,
-        buf: &mut (impl BufMut + Send),
-    ) -> io::Result<()> {
+
+    async fn read_into(&self, range: Range<u64>, buf: &mut (impl BufMut + Send)) -> io::Result<()> {
         (*self).read_into(range, buf).await
     }
-    async fn read_vec(&mut self, ranges: &[Range<u64>]) -> io::Result<Vec<Bytes>> {
+
+    async fn read_vec(&self, ranges: &[Range<u64>]) -> io::Result<Vec<Bytes>> {
         (*self).read_vec(ranges).await
     }
 }
@@ -120,7 +115,7 @@ pub struct AsyncReadAdapter<R> {
 
 impl<R: RangeReader + 'static> AsyncReadAdapter<R> {
     pub async fn new(inner: R) -> io::Result<Self> {
-        let mut inner = inner;
+        let inner = inner;
         let metadata = inner.metadata().await?;
         Ok(AsyncReadAdapter {
             inner: Arc::new(Mutex::new(inner)),
@@ -160,7 +155,7 @@ impl<R: RangeReader + 'static> AsyncRead for AsyncReadAdapter<R> {
             let range = *this.position..(*this.position + size);
             let inner = this.inner.clone();
             let fut = async move {
-                let mut inner = inner.lock().await;
+                let inner = inner.lock().await;
                 inner.read(range).await
             };
 
@@ -195,27 +190,24 @@ impl<R: RangeReader + 'static> AsyncRead for AsyncReadAdapter<R> {
 
 #[async_trait]
 impl RangeReader for Vec<u8> {
-    fn with_file_size_hint(&mut self, _file_size_hint: u64) {
-        // do nothing
-    }
-
-    async fn metadata(&mut self) -> io::Result<Metadata> {
+    async fn metadata(&self) -> io::Result<Metadata> {
         Ok(Metadata {
             content_length: self.len() as u64,
         })
     }
 
-    async fn read(&mut self, range: Range<u64>) -> io::Result<Bytes> {
+    async fn read(&self, range: Range<u64>) -> io::Result<Bytes> {
         let bytes = Bytes::copy_from_slice(&self[range.start as usize..range.end as usize]);
         Ok(bytes)
     }
 }
 
+// TODO(weny): considers replacing `tokio::fs::File` with opendal reader.
 /// `FileReader` is a `RangeReader` for reading a file.
 pub struct FileReader {
     content_length: u64,
-    position: u64,
-    file: tokio::fs::File,
+    position: AtomicU64,
+    file: Mutex<tokio::fs::File>,
 }
 
 impl FileReader {
@@ -225,32 +217,37 @@ impl FileReader {
         let metadata = file.metadata().await?;
         Ok(FileReader {
             content_length: metadata.len(),
-            position: 0,
-            file,
+            position: AtomicU64::new(0),
+            file: Mutex::new(file),
         })
     }
 }
 
+#[cfg(any(test, feature = "testing"))]
+impl SizeAwareRangeReader for FileReader {
+    fn with_file_size_hint(&mut self, _file_size_hint: u64) {
+        // do nothing
+    }
+}
+
 #[async_trait]
 impl RangeReader for FileReader {
-    fn with_file_size_hint(&mut self, _file_size_hint: u64) {
-        // do nothing
-    }
-
-    async fn metadata(&mut self) -> io::Result<Metadata> {
+    async fn metadata(&self) -> io::Result<Metadata> {
         Ok(Metadata {
             content_length: self.content_length,
         })
     }
 
-    async fn read(&mut self, mut range: Range<u64>) -> io::Result<Bytes> {
-        if range.start != self.position {
-            self.file.seek(io::SeekFrom::Start(range.start)).await?;
-            self.position = range.start;
+    async fn read(&self, mut range: Range<u64>) -> io::Result<Bytes> {
+        let mut file = self.file.lock().await;
+
+        if range.start != self.position.load(Ordering::Relaxed) {
+            file.seek(io::SeekFrom::Start(range.start)).await?;
+            self.position.store(range.start, Ordering::Relaxed);
         }
 
         range.end = range.end.min(self.content_length);
-        if range.end <= self.position {
+        if range.end <= self.position.load(Ordering::Relaxed) {
             return Err(io::Error::new(
                 io::ErrorKind::UnexpectedEof,
                 "Start of range is out of bounds",
@@ -259,8 +256,8 @@ impl RangeReader for FileReader {
 
         let mut buf = vec![0; (range.end - range.start) as usize];
 
-        self.file.read_exact(&mut buf).await?;
-        self.position = range.end;
+        file.read_exact(&mut buf).await?;
+        self.position.store(range.end, Ordering::Relaxed);
 
         Ok(Bytes::from(buf))
     }
@@ -301,7 +298,7 @@ mod tests {
         let data = b"hello world";
         tokio::fs::write(path, data).await.unwrap();
 
-        let mut reader = FileReader::new(path).await.unwrap();
+        let reader = FileReader::new(path).await.unwrap();
         let metadata = reader.metadata().await.unwrap();
         assert_eq!(metadata.content_length, data.len() as u64);
 

src/common/datasource/tests/orc/write.py

@@ -35,10 +35,23 @@ data = {
     "bigint_other": [5, -5, 1, 5, 5],
     "utf8_increase": ["a", "bb", "ccc", "dddd", "eeeee"],
     "utf8_decrease": ["eeeee", "dddd", "ccc", "bb", "a"],
-    "timestamp_simple": [datetime.datetime(2023, 4, 1, 20, 15, 30, 2000), datetime.datetime.fromtimestamp(int('1629617204525777000')/1000000000), datetime.datetime(2023, 1, 1), datetime.datetime(2023, 2, 1), datetime.datetime(2023, 3, 1)],
-    "date_simple": [datetime.date(2023, 4, 1), datetime.date(2023, 3, 1), datetime.date(2023, 1, 1), datetime.date(2023, 2, 1), datetime.date(2023, 3, 1)]
+    "timestamp_simple": [
+        datetime.datetime(2023, 4, 1, 20, 15, 30, 2000),
+        datetime.datetime.fromtimestamp(int("1629617204525777000") / 1000000000),
+        datetime.datetime(2023, 1, 1),
+        datetime.datetime(2023, 2, 1),
+        datetime.datetime(2023, 3, 1),
+    ],
+    "date_simple": [
+        datetime.date(2023, 4, 1),
+        datetime.date(2023, 3, 1),
+        datetime.date(2023, 1, 1),
+        datetime.date(2023, 2, 1),
+        datetime.date(2023, 3, 1),
+    ],
 }
 
+
 def infer_schema(data):
     schema = "struct<"
     for key, value in data.items():
@@ -56,7 +69,7 @@ def infer_schema(data):
         elif key.startswith("date"):
             dt = "date"
         else:
-            print(key,value,dt)
+            print(key, value, dt)
             raise NotImplementedError
         if key.startswith("double"):
             dt = "double"
@@ -68,7 +81,6 @@ def infer_schema(data):
     return schema
 
 
-
 def _write(
     schema: str,
     data,

src/common/function/src/scalars/aggregate.rs

@@ -32,6 +32,7 @@ pub use scipy_stats_norm_cdf::ScipyStatsNormCdfAccumulatorCreator;
 pub use scipy_stats_norm_pdf::ScipyStatsNormPdfAccumulatorCreator;
 
 use crate::function_registry::FunctionRegistry;
+use crate::scalars::vector::product::VectorProductCreator;
 use crate::scalars::vector::sum::VectorSumCreator;
 
 /// A function creates `AggregateFunctionCreator`.
@@ -93,6 +94,7 @@ impl AggregateFunctions {
         register_aggr_func!("scipystatsnormcdf", 2, ScipyStatsNormCdfAccumulatorCreator);
         register_aggr_func!("scipystatsnormpdf", 2, ScipyStatsNormPdfAccumulatorCreator);
         register_aggr_func!("vec_sum", 1, VectorSumCreator);
+        register_aggr_func!("vec_product", 1, VectorProductCreator);
 
         #[cfg(feature = "geo")]
         register_aggr_func!(

src/common/function/src/scalars/vector.rs

@@ -14,14 +14,17 @@
 
 mod convert;
 mod distance;
+mod elem_product;
 mod elem_sum;
 pub mod impl_conv;
+pub(crate) mod product;
 mod scalar_add;
 mod scalar_mul;
 mod sub;
 pub(crate) mod sum;
 mod vector_div;
 mod vector_mul;
+mod vector_norm;
 
 use std::sync::Arc;
 
@@ -46,8 +49,10 @@ impl VectorFunction {
 
         // vector calculation
         registry.register(Arc::new(vector_mul::VectorMulFunction));
+        registry.register(Arc::new(vector_norm::VectorNormFunction));
         registry.register(Arc::new(vector_div::VectorDivFunction));
         registry.register(Arc::new(sub::SubFunction));
         registry.register(Arc::new(elem_sum::ElemSumFunction));
+        registry.register(Arc::new(elem_product::ElemProductFunction));
     }
 }

src/common/function/src/scalars/vector/elem_product.rs

@@ -0,0 +1,142 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::borrow::Cow;
+use std::fmt::Display;
+
+use common_query::error::InvalidFuncArgsSnafu;
+use common_query::prelude::{Signature, TypeSignature, Volatility};
+use datatypes::prelude::ConcreteDataType;
+use datatypes::scalars::ScalarVectorBuilder;
+use datatypes::vectors::{Float32VectorBuilder, MutableVector, VectorRef};
+use nalgebra::DVectorView;
+use snafu::ensure;
+
+use crate::function::{Function, FunctionContext};
+use crate::scalars::vector::impl_conv::{as_veclit, as_veclit_if_const};
+
+const NAME: &str = "vec_elem_product";
+
+/// Multiplies all elements of the vector, returns a scalar.
+///
+/// # Example
+///
+/// ```sql
+/// SELECT vec_elem_product(parse_vec('[1.0, 2.0, 3.0, 4.0]'));
+///
+// +-----------------------------------------------------------+
+// | vec_elem_product(parse_vec(Utf8("[1.0, 2.0, 3.0, 4.0]"))) |
+// +-----------------------------------------------------------+
+// | 24.0                                                      |
+// +-----------------------------------------------------------+
+/// ``````
+#[derive(Debug, Clone, Default)]
+pub struct ElemProductFunction;
+
+impl Function for ElemProductFunction {
+    fn name(&self) -> &str {
+        NAME
+    }
+
+    fn return_type(
+        &self,
+        _input_types: &[ConcreteDataType],
+    ) -> common_query::error::Result<ConcreteDataType> {
+        Ok(ConcreteDataType::float32_datatype())
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::one_of(
+            vec![
+                TypeSignature::Exact(vec![ConcreteDataType::string_datatype()]),
+                TypeSignature::Exact(vec![ConcreteDataType::binary_datatype()]),
+            ],
+            Volatility::Immutable,
+        )
+    }
+
+    fn eval(
+        &self,
+        _func_ctx: FunctionContext,
+        columns: &[VectorRef],
+    ) -> common_query::error::Result<VectorRef> {
+        ensure!(
+            columns.len() == 1,
+            InvalidFuncArgsSnafu {
+                err_msg: format!(
+                    "The length of the args is not correct, expect exactly one, have: {}",
+                    columns.len()
+                )
+            }
+        );
+        let arg0 = &columns[0];
+
+        let len = arg0.len();
+        let mut result = Float32VectorBuilder::with_capacity(len);
+        if len == 0 {
+            return Ok(result.to_vector());
+        }
+
+        let arg0_const = as_veclit_if_const(arg0)?;
+
+        for i in 0..len {
+            let arg0 = match arg0_const.as_ref() {
+                Some(arg0) => Some(Cow::Borrowed(arg0.as_ref())),
+                None => as_veclit(arg0.get_ref(i))?,
+            };
+            let Some(arg0) = arg0 else {
+                result.push_null();
+                continue;
+            };
+            result.push(Some(DVectorView::from_slice(&arg0, arg0.len()).product()));
+        }
+
+        Ok(result.to_vector())
+    }
+}
+
+impl Display for ElemProductFunction {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "{}", NAME.to_ascii_uppercase())
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use std::sync::Arc;
+
+    use datatypes::vectors::StringVector;
+
+    use super::*;
+    use crate::function::FunctionContext;
+
+    #[test]
+    fn test_elem_product() {
+        let func = ElemProductFunction;
+
+        let input0 = Arc::new(StringVector::from(vec![
+            Some("[1.0,2.0,3.0]".to_string()),
+            Some("[4.0,5.0,6.0]".to_string()),
+            None,
+        ]));
+
+        let result = func.eval(FunctionContext::default(), &[input0]).unwrap();
+
+        let result = result.as_ref();
+        assert_eq!(result.len(), 3);
+        assert_eq!(result.get_ref(0).as_f32().unwrap(), Some(6.0));
+        assert_eq!(result.get_ref(1).as_f32().unwrap(), Some(120.0));
+        assert_eq!(result.get_ref(2).as_f32().unwrap(), None);
+    }
+}

src/common/function/src/scalars/vector/product.rs

@@ -0,0 +1,211 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::sync::Arc;
+
+use common_macro::{as_aggr_func_creator, AggrFuncTypeStore};
+use common_query::error::{CreateAccumulatorSnafu, Error, InvalidFuncArgsSnafu};
+use common_query::logical_plan::{Accumulator, AggregateFunctionCreator};
+use common_query::prelude::AccumulatorCreatorFunction;
+use datatypes::prelude::{ConcreteDataType, Value, *};
+use datatypes::vectors::VectorRef;
+use nalgebra::{Const, DVectorView, Dyn, OVector};
+use snafu::ensure;
+
+use crate::scalars::vector::impl_conv::{as_veclit, as_veclit_if_const, veclit_to_binlit};
+
+/// Aggregates by multiplying elements across the same dimension, returns a vector.
+#[derive(Debug, Default)]
+pub struct VectorProduct {
+    product: Option<OVector<f32, Dyn>>,
+    has_null: bool,
+}
+
+#[as_aggr_func_creator]
+#[derive(Debug, Default, AggrFuncTypeStore)]
+pub struct VectorProductCreator {}
+
+impl AggregateFunctionCreator for VectorProductCreator {
+    fn creator(&self) -> AccumulatorCreatorFunction {
+        let creator: AccumulatorCreatorFunction = Arc::new(move |types: &[ConcreteDataType]| {
+            ensure!(
+                types.len() == 1,
+                InvalidFuncArgsSnafu {
+                    err_msg: format!(
+                        "The length of the args is not correct, expect exactly one, have: {}",
+                        types.len()
+                    )
+                }
+            );
+            let input_type = &types[0];
+            match input_type {
+                ConcreteDataType::String(_) | ConcreteDataType::Binary(_) => {
+                    Ok(Box::new(VectorProduct::default()))
+                }
+                _ => {
+                    let err_msg = format!(
+                        "\"VEC_PRODUCT\" aggregate function not support data type {:?}",
+                        input_type.logical_type_id(),
+                    );
+                    CreateAccumulatorSnafu { err_msg }.fail()?
+                }
+            }
+        });
+        creator
+    }
+
+    fn output_type(&self) -> common_query::error::Result<ConcreteDataType> {
+        Ok(ConcreteDataType::binary_datatype())
+    }
+
+    fn state_types(&self) -> common_query::error::Result<Vec<ConcreteDataType>> {
+        Ok(vec![self.output_type()?])
+    }
+}
+
+impl VectorProduct {
+    fn inner(&mut self, len: usize) -> &mut OVector<f32, Dyn> {
+        self.product.get_or_insert_with(|| {
+            OVector::from_iterator_generic(Dyn(len), Const::<1>, (0..len).map(|_| 1.0))
+        })
+    }
+
+    fn update(&mut self, values: &[VectorRef], is_update: bool) -> Result<(), Error> {
+        if values.is_empty() || self.has_null {
+            return Ok(());
+        };
+        let column = &values[0];
+        let len = column.len();
+
+        match as_veclit_if_const(column)? {
+            Some(column) => {
+                let vec_column = DVectorView::from_slice(&column, column.len()).scale(len as f32);
+                *self.inner(vec_column.len()) =
+                    (*self.inner(vec_column.len())).component_mul(&vec_column);
+            }
+            None => {
+                for i in 0..len {
+                    let Some(arg0) = as_veclit(column.get_ref(i))? else {
+                        if is_update {
+                            self.has_null = true;
+                            self.product = None;
+                        }
+                        return Ok(());
+                    };
+                    let vec_column = DVectorView::from_slice(&arg0, arg0.len());
+                    *self.inner(vec_column.len()) =
+                        (*self.inner(vec_column.len())).component_mul(&vec_column);
+                }
+            }
+        }
+        Ok(())
+    }
+}
+
+impl Accumulator for VectorProduct {
+    fn state(&self) -> common_query::error::Result<Vec<Value>> {
+        self.evaluate().map(|v| vec![v])
+    }
+
+    fn update_batch(&mut self, values: &[VectorRef]) -> common_query::error::Result<()> {
+        self.update(values, true)
+    }
+
+    fn merge_batch(&mut self, states: &[VectorRef]) -> common_query::error::Result<()> {
+        self.update(states, false)
+    }
+
+    fn evaluate(&self) -> common_query::error::Result<Value> {
+        match &self.product {
+            None => Ok(Value::Null),
+            Some(vector) => {
+                let v = vector.as_slice();
+                Ok(Value::from(veclit_to_binlit(v)))
+            }
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use std::sync::Arc;
+
+    use datatypes::vectors::{ConstantVector, StringVector};
+
+    use super::*;
+
+    #[test]
+    fn test_update_batch() {
+        // test update empty batch, expect not updating anything
+        let mut vec_product = VectorProduct::default();
+        vec_product.update_batch(&[]).unwrap();
+        assert!(vec_product.product.is_none());
+        assert!(!vec_product.has_null);
+        assert_eq!(Value::Null, vec_product.evaluate().unwrap());
+
+        // test update one not-null value
+        let mut vec_product = VectorProduct::default();
+        let v: Vec<VectorRef> = vec![Arc::new(StringVector::from(vec![Some(
+            "[1.0,2.0,3.0]".to_string(),
+        )]))];
+        vec_product.update_batch(&v).unwrap();
+        assert_eq!(
+            Value::from(veclit_to_binlit(&[1.0, 2.0, 3.0])),
+            vec_product.evaluate().unwrap()
+        );
+
+        // test update one null value
+        let mut vec_product = VectorProduct::default();
+        let v: Vec<VectorRef> = vec![Arc::new(StringVector::from(vec![Option::<String>::None]))];
+        vec_product.update_batch(&v).unwrap();
+        assert_eq!(Value::Null, vec_product.evaluate().unwrap());
+
+        // test update no null-value batch
+        let mut vec_product = VectorProduct::default();
+        let v: Vec<VectorRef> = vec![Arc::new(StringVector::from(vec![
+            Some("[1.0,2.0,3.0]".to_string()),
+            Some("[4.0,5.0,6.0]".to_string()),
+            Some("[7.0,8.0,9.0]".to_string()),
+        ]))];
+        vec_product.update_batch(&v).unwrap();
+        assert_eq!(
+            Value::from(veclit_to_binlit(&[28.0, 80.0, 162.0])),
+            vec_product.evaluate().unwrap()
+        );
+
+        // test update null-value batch
+        let mut vec_product = VectorProduct::default();
+        let v: Vec<VectorRef> = vec![Arc::new(StringVector::from(vec![
+            Some("[1.0,2.0,3.0]".to_string()),
+            None,
+            Some("[7.0,8.0,9.0]".to_string()),
+        ]))];
+        vec_product.update_batch(&v).unwrap();
+        assert_eq!(Value::Null, vec_product.evaluate().unwrap());
+
+        // test update with constant vector
+        let mut vec_product = VectorProduct::default();
+        let v: Vec<VectorRef> = vec![Arc::new(ConstantVector::new(
+            Arc::new(StringVector::from_vec(vec!["[1.0,2.0,3.0]".to_string()])),
+            4,
+        ))];
+
+        vec_product.update_batch(&v).unwrap();
+
+        assert_eq!(
+            Value::from(veclit_to_binlit(&[4.0, 8.0, 12.0])),
+            vec_product.evaluate().unwrap()
+        );
+    }
+}

src/common/function/src/scalars/vector/vector_norm.rs

@@ -0,0 +1,168 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::borrow::Cow;
+use std::fmt::Display;
+
+use common_query::error::{InvalidFuncArgsSnafu, Result};
+use common_query::prelude::{Signature, TypeSignature, Volatility};
+use datatypes::prelude::ConcreteDataType;
+use datatypes::scalars::ScalarVectorBuilder;
+use datatypes::vectors::{BinaryVectorBuilder, MutableVector, VectorRef};
+use nalgebra::DVectorView;
+use snafu::ensure;
+
+use crate::function::{Function, FunctionContext};
+use crate::scalars::vector::impl_conv::{as_veclit, as_veclit_if_const, veclit_to_binlit};
+
+const NAME: &str = "vec_norm";
+
+/// Normalizes the vector to length 1, returns a vector.
+/// This's equivalent to `VECTOR_SCALAR_MUL(1/SQRT(VECTOR_ELEM_SUM(VECTOR_MUL(v, v))), v)`.
+///
+/// # Example
+///
+/// ```sql
+/// SELECT vec_to_string(vec_norm('[7.0, 8.0, 9.0]'));
+///
+/// +--------------------------------------------------+
+/// | vec_to_string(vec_norm(Utf8("[7.0, 8.0, 9.0]"))) |
+/// +--------------------------------------------------+
+/// | [0.013888889,0.015873017,0.017857144]            |
+/// +--------------------------------------------------+
+///
+/// ```
+#[derive(Debug, Clone, Default)]
+pub struct VectorNormFunction;
+
+impl Function for VectorNormFunction {
+    fn name(&self) -> &str {
+        NAME
+    }
+
+    fn return_type(&self, _input_types: &[ConcreteDataType]) -> Result<ConcreteDataType> {
+        Ok(ConcreteDataType::binary_datatype())
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::one_of(
+            vec![
+                TypeSignature::Exact(vec![ConcreteDataType::string_datatype()]),
+                TypeSignature::Exact(vec![ConcreteDataType::binary_datatype()]),
+            ],
+            Volatility::Immutable,
+        )
+    }
+
+    fn eval(
+        &self,
+        _func_ctx: FunctionContext,
+        columns: &[VectorRef],
+    ) -> common_query::error::Result<VectorRef> {
+        ensure!(
+            columns.len() == 1,
+            InvalidFuncArgsSnafu {
+                err_msg: format!(
+                    "The length of the args is not correct, expect exactly one, have: {}",
+                    columns.len()
+                )
+            }
+        );
+        let arg0 = &columns[0];
+
+        let len = arg0.len();
+        let mut result = BinaryVectorBuilder::with_capacity(len);
+        if len == 0 {
+            return Ok(result.to_vector());
+        }
+
+        let arg0_const = as_veclit_if_const(arg0)?;
+
+        for i in 0..len {
+            let arg0 = match arg0_const.as_ref() {
+                Some(arg0) => Some(Cow::Borrowed(arg0.as_ref())),
+                None => as_veclit(arg0.get_ref(i))?,
+            };
+            let Some(arg0) = arg0 else {
+                result.push_null();
+                continue;
+            };
+
+            let vec0 = DVectorView::from_slice(&arg0, arg0.len());
+            let vec1 = DVectorView::from_slice(&arg0, arg0.len());
+            let vec2scalar = vec1.component_mul(&vec0);
+            let scalar_var = vec2scalar.sum().sqrt();
+
+            let vec = DVectorView::from_slice(&arg0, arg0.len());
+            // Use unscale to avoid division by zero and keep more precision as possible
+            let vec_res = vec.unscale(scalar_var);
+
+            let veclit = vec_res.as_slice();
+            let binlit = veclit_to_binlit(veclit);
+            result.push(Some(&binlit));
+        }
+
+        Ok(result.to_vector())
+    }
+}
+
+impl Display for VectorNormFunction {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "{}", NAME.to_ascii_uppercase())
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use std::sync::Arc;
+
+    use datatypes::vectors::StringVector;
+
+    use super::*;
+
+    #[test]
+    fn test_vec_norm() {
+        let func = VectorNormFunction;
+
+        let input0 = Arc::new(StringVector::from(vec![
+            Some("[0.0,2.0,3.0]".to_string()),
+            Some("[1.0,2.0,3.0]".to_string()),
+            Some("[7.0,8.0,9.0]".to_string()),
+            Some("[7.0,-8.0,9.0]".to_string()),
+            None,
+        ]));
+
+        let result = func.eval(FunctionContext::default(), &[input0]).unwrap();
+
+        let result = result.as_ref();
+        assert_eq!(result.len(), 5);
+        assert_eq!(
+            result.get_ref(0).as_binary().unwrap(),
+            Some(veclit_to_binlit(&[0.0, 0.5547002, 0.8320503]).as_slice())
+        );
+        assert_eq!(
+            result.get_ref(1).as_binary().unwrap(),
+            Some(veclit_to_binlit(&[0.26726124, 0.5345225, 0.8017837]).as_slice())
+        );
+        assert_eq!(
+            result.get_ref(2).as_binary().unwrap(),
+            Some(veclit_to_binlit(&[0.5025707, 0.5743665, 0.64616233]).as_slice())
+        );
+        assert_eq!(
+            result.get_ref(3).as_binary().unwrap(),
+            Some(veclit_to_binlit(&[0.5025707, -0.5743665, 0.64616233]).as_slice())
+        );
+        assert!(result.get_ref(4).is_null());
+    }
+}

src/common/function/src/system.rs

@@ -22,7 +22,7 @@ mod version;
 use std::sync::Arc;
 
 use build::BuildFunction;
-use database::{CurrentSchemaFunction, DatabaseFunction};
+use database::{CurrentSchemaFunction, DatabaseFunction, SessionUserFunction};
 use pg_catalog::PGCatalogFunction;
 use procedure_state::ProcedureStateFunction;
 use timezone::TimezoneFunction;
@@ -36,8 +36,9 @@ impl SystemFunction {
     pub fn register(registry: &FunctionRegistry) {
         registry.register(Arc::new(BuildFunction));
         registry.register(Arc::new(VersionFunction));
-        registry.register(Arc::new(DatabaseFunction));
         registry.register(Arc::new(CurrentSchemaFunction));
+        registry.register(Arc::new(DatabaseFunction));
+        registry.register(Arc::new(SessionUserFunction));
         registry.register(Arc::new(TimezoneFunction));
         registry.register_async(Arc::new(ProcedureStateFunction));
         PGCatalogFunction::register(registry);

src/common/function/src/system/database.rs

@@ -28,9 +28,11 @@ pub struct DatabaseFunction;
 
 #[derive(Clone, Debug, Default)]
 pub struct CurrentSchemaFunction;
+pub struct SessionUserFunction;
 
 const DATABASE_FUNCTION_NAME: &str = "database";
 const CURRENT_SCHEMA_FUNCTION_NAME: &str = "current_schema";
+const SESSION_USER_FUNCTION_NAME: &str = "session_user";
 
 impl Function for DatabaseFunction {
     fn name(&self) -> &str {
@@ -72,6 +74,26 @@ impl Function for CurrentSchemaFunction {
     }
 }
 
+impl Function for SessionUserFunction {
+    fn name(&self) -> &str {
+        SESSION_USER_FUNCTION_NAME
+    }
+
+    fn return_type(&self, _input_types: &[ConcreteDataType]) -> Result<ConcreteDataType> {
+        Ok(ConcreteDataType::string_datatype())
+    }
+
+    fn signature(&self) -> Signature {
+        Signature::uniform(0, vec![], Volatility::Immutable)
+    }
+
+    fn eval(&self, func_ctx: FunctionContext, _columns: &[VectorRef]) -> Result<VectorRef> {
+        let user = func_ctx.query_ctx.current_user();
+
+        Ok(Arc::new(StringVector::from_slice(&[user.username()])) as _)
+    }
+}
+
 impl fmt::Display for DatabaseFunction {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         write!(f, "DATABASE")
@@ -84,6 +106,12 @@ impl fmt::Display for CurrentSchemaFunction {
     }
 }
 
+impl fmt::Display for SessionUserFunction {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "SESSION_USER")
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use std::sync::Arc;

src/common/grpc-expr/src/alter.rs

@@ -60,6 +60,7 @@ pub fn alter_expr_to_request(table_id: TableId, expr: AlterTableExpr) -> Result<
                         column_schema: schema,
                         is_key: column_def.semantic_type == SemanticType::Tag as i32,
                         location: parse_location(ac.location)?,
+                        add_if_not_exists: ac.add_if_not_exists,
                     })
                 })
                 .collect::<Result<Vec<_>>>()?;
@@ -220,6 +221,7 @@ mod tests {
                         ..Default::default()
                     }),
                     location: None,
+                    add_if_not_exists: true,
                 }],
             })),
         };
@@ -240,6 +242,7 @@ mod tests {
             add_column.column_schema.data_type
         );
         assert_eq!(None, add_column.location);
+        assert!(add_column.add_if_not_exists);
     }
 
     #[test]
@@ -265,6 +268,7 @@ mod tests {
                             location_type: LocationType::First.into(),
                             after_column_name: String::default(),
                         }),
+                        add_if_not_exists: false,
                     },
                     AddColumn {
                         column_def: Some(ColumnDef {
@@ -280,6 +284,7 @@ mod tests {
                             location_type: LocationType::After.into(),
                             after_column_name: "ts".to_string(),
                         }),
+                        add_if_not_exists: true,
                     },
                 ],
             })),
@@ -308,6 +313,7 @@ mod tests {
             }),
             add_column.location
         );
+        assert!(add_column.add_if_not_exists);
 
         let add_column = add_columns.pop().unwrap();
         assert!(!add_column.is_key);
@@ -317,6 +323,7 @@ mod tests {
             add_column.column_schema.data_type
         );
         assert_eq!(Some(AddColumnLocation::First), add_column.location);
+        assert!(!add_column.add_if_not_exists);
     }
 
     #[test]

src/common/grpc-expr/src/insert.rs

@@ -299,6 +299,7 @@ mod tests {
                 .unwrap()
             )
         );
+        assert!(host_column.add_if_not_exists);
 
         let memory_column = &add_columns.add_columns[1];
         assert_eq!(
@@ -311,6 +312,7 @@ mod tests {
                 .unwrap()
             )
         );
+        assert!(host_column.add_if_not_exists);
 
         let time_column = &add_columns.add_columns[2];
         assert_eq!(
@@ -323,6 +325,7 @@ mod tests {
                 .unwrap()
             )
         );
+        assert!(host_column.add_if_not_exists);
 
         let interval_column = &add_columns.add_columns[3];
         assert_eq!(
@@ -335,6 +338,7 @@ mod tests {
                 .unwrap()
             )
         );
+        assert!(host_column.add_if_not_exists);
 
         let decimal_column = &add_columns.add_columns[4];
         assert_eq!(
@@ -352,6 +356,7 @@ mod tests {
                 .unwrap()
             )
         );
+        assert!(host_column.add_if_not_exists);
     }
 
     #[test]

src/common/grpc-expr/src/util.rs

@@ -192,6 +192,9 @@ pub fn build_create_table_expr(
     Ok(expr)
 }
 
+/// Find columns that are not present in the schema and return them as `AddColumns`
+/// for adding columns automatically.
+/// It always sets `add_if_not_exists` to `true` for now.
 pub fn extract_new_columns(
     schema: &Schema,
     column_exprs: Vec<ColumnExpr>,
@@ -213,6 +216,7 @@ pub fn extract_new_columns(
             AddColumn {
                 column_def,
                 location: None,
+                add_if_not_exists: true,
             }
         })
         .collect::<Vec<_>>();

src/common/meta/Cargo.toml

@@ -6,7 +6,7 @@ license.workspace = true
 
 [features]
 testing = []
-pg_kvbackend = ["dep:tokio-postgres"]
+pg_kvbackend = ["dep:tokio-postgres", "dep:backon"]
 
 [lints]
 workspace = true
@@ -17,6 +17,7 @@ api.workspace = true
 async-recursion = "1.0"
 async-stream = "0.3"
 async-trait.workspace = true
+backon = { workspace = true, optional = true }
 base64.workspace = true
 bytes.workspace = true
 chrono.workspace = true
@@ -35,6 +36,8 @@ common-wal.workspace = true
 datafusion-common.workspace = true
 datafusion-expr.workspace = true
 datatypes.workspace = true
+deadpool.workspace = true
+deadpool-postgres.workspace = true
 derive_builder.workspace = true
 etcd-client.workspace = true
 futures.workspace = true

src/common/meta/src/ddl/alter_logical_tables/update_metadata.rs

@@ -105,7 +105,7 @@ impl AlterLogicalTablesProcedure {
                 .context(ConvertAlterTableRequestSnafu)?;
         let new_meta = table_info
             .meta
-            .builder_with_alter_kind(table_ref.table, &request.alter_kind, true)
+            .builder_with_alter_kind(table_ref.table, &request.alter_kind)
             .context(error::TableSnafu)?
             .build()
             .with_context(|_| error::BuildTableMetaSnafu {

src/common/meta/src/ddl/alter_table.rs

@@ -28,13 +28,13 @@ use common_procedure::error::{FromJsonSnafu, Result as ProcedureResult, ToJsonSn
 use common_procedure::{
     Context as ProcedureContext, Error as ProcedureError, LockKey, Procedure, Status, StringKey,
 };
-use common_telemetry::{debug, info};
+use common_telemetry::{debug, error, info};
 use futures::future;
 use serde::{Deserialize, Serialize};
 use snafu::ResultExt;
 use store_api::storage::RegionId;
 use strum::AsRefStr;
-use table::metadata::{RawTableInfo, TableId};
+use table::metadata::{RawTableInfo, TableId, TableInfo};
 use table::table_reference::TableReference;
 
 use crate::cache_invalidator::Context;
@@ -51,10 +51,14 @@ use crate::{metrics, ClusterId};
 
 /// The alter table procedure
 pub struct AlterTableProcedure {
-    // The runtime context.
+    /// The runtime context.
     context: DdlContext,
-    // The serialized data.
+    /// The serialized data.
     data: AlterTableData,
+    /// Cached new table metadata in the prepare step.
+    /// If we recover the procedure from json, then the table info value is not cached.
+    /// But we already validated it in the prepare step.
+    new_table_info: Option<TableInfo>,
 }
 
 impl AlterTableProcedure {
@@ -70,18 +74,31 @@ impl AlterTableProcedure {
         Ok(Self {
             context,
             data: AlterTableData::new(task, table_id, cluster_id),
+            new_table_info: None,
         })
     }
 
     pub fn from_json(json: &str, context: DdlContext) -> ProcedureResult<Self> {
         let data: AlterTableData = serde_json::from_str(json).context(FromJsonSnafu)?;
-        Ok(AlterTableProcedure { context, data })
+        Ok(AlterTableProcedure {
+            context,
+            data,
+            new_table_info: None,
+        })
     }
 
     // Checks whether the table exists.
     pub(crate) async fn on_prepare(&mut self) -> Result<Status> {
         self.check_alter().await?;
         self.fill_table_info().await?;
+
+        // Validates the request and builds the new table info.
+        // We need to build the new table info here because we should ensure the alteration
+        // is valid in `UpdateMeta` state as we already altered the region.
+        // Safety: `fill_table_info()` already set it.
+        let table_info_value = self.data.table_info_value.as_ref().unwrap();
+        self.new_table_info = Some(self.build_new_table_info(&table_info_value.table_info)?);
+
         // Safety: Checked in `AlterTableProcedure::new`.
         let alter_kind = self.data.task.alter_table.kind.as_ref().unwrap();
         if matches!(alter_kind, Kind::RenameTable { .. }) {
@@ -106,6 +123,14 @@ impl AlterTableProcedure {
 
         let leaders = find_leaders(&physical_table_route.region_routes);
         let mut alter_region_tasks = Vec::with_capacity(leaders.len());
+        let alter_kind = self.make_region_alter_kind()?;
+
+        info!(
+            "Submitting alter region requests for table {}, table_id: {}, alter_kind: {:?}",
+            self.data.table_ref(),
+            table_id,
+            alter_kind,
+        );
 
         for datanode in leaders {
             let requester = self.context.node_manager.datanode(&datanode).await;
@@ -113,7 +138,7 @@ impl AlterTableProcedure {
 
             for region in regions {
                 let region_id = RegionId::new(table_id, region);
-                let request = self.make_alter_region_request(region_id)?;
+                let request = self.make_alter_region_request(region_id, alter_kind.clone())?;
                 debug!("Submitting {request:?} to {datanode}");
 
                 let datanode = datanode.clone();
@@ -150,7 +175,15 @@ impl AlterTableProcedure {
         let table_ref = self.data.table_ref();
         // Safety: checked before.
         let table_info_value = self.data.table_info_value.as_ref().unwrap();
-        let new_info = self.build_new_table_info(&table_info_value.table_info)?;
+        // Gets the table info from the cache or builds it.
+        let new_info = match &self.new_table_info {
+            Some(cached) => cached.clone(),
+            None => self.build_new_table_info(&table_info_value.table_info)
+                .inspect_err(|e| {
+                    // We already check the table info in the prepare step so this should not happen.
+                    error!(e; "Unable to build info for table {} in update metadata step, table_id: {}", table_ref, table_id);
+                })?,
+        };
 
         debug!(
             "Starting update table: {} metadata, new table info {:?}",
@@ -174,7 +207,7 @@ impl AlterTableProcedure {
             .await?;
         }
 
-        info!("Updated table metadata for table {table_ref}, table_id: {table_id}");
+        info!("Updated table metadata for table {table_ref}, table_id: {table_id}, kind: {alter_kind:?}");
         self.data.state = AlterTableState::InvalidateTableCache;
         Ok(Status::executing(true))
     }

src/common/meta/src/ddl/alter_table/region_request.rs

@@ -12,6 +12,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use std::collections::HashSet;
+
 use api::v1::alter_table_expr::Kind;
 use api::v1::region::region_request::Body;
 use api::v1::region::{
@@ -27,13 +29,15 @@ use crate::ddl::alter_table::AlterTableProcedure;
 use crate::error::{InvalidProtoMsgSnafu, Result};
 
 impl AlterTableProcedure {
-    /// Makes alter region request.
-    pub(crate) fn make_alter_region_request(&self, region_id: RegionId) -> Result<RegionRequest> {
-        // Safety: Checked in `AlterTableProcedure::new`.
-        let alter_kind = self.data.task.alter_table.kind.as_ref().unwrap();
+    /// Makes alter region request from existing an alter kind.
+    /// Region alter request always add columns if not exist.
+    pub(crate) fn make_alter_region_request(
+        &self,
+        region_id: RegionId,
+        kind: Option<alter_request::Kind>,
+    ) -> Result<RegionRequest> {
         // Safety: checked
         let table_info = self.data.table_info().unwrap();
-        let kind = create_proto_alter_kind(table_info, alter_kind)?;
 
         Ok(RegionRequest {
             header: Some(RegionRequestHeader {
@@ -47,45 +51,66 @@ impl AlterTableProcedure {
             })),
         })
     }
+
+    /// Makes alter kind proto that all regions can reuse.
+    /// Region alter request always add columns if not exist.
+    pub(crate) fn make_region_alter_kind(&self) -> Result<Option<alter_request::Kind>> {
+        // Safety: Checked in `AlterTableProcedure::new`.
+        let alter_kind = self.data.task.alter_table.kind.as_ref().unwrap();
+        // Safety: checked
+        let table_info = self.data.table_info().unwrap();
+        let kind = create_proto_alter_kind(table_info, alter_kind)?;
+
+        Ok(kind)
+    }
 }
 
 /// Creates region proto alter kind from `table_info` and `alter_kind`.
 ///
-/// Returns the kind and next column id if it adds new columns.
+/// It always adds column if not exists and drops column if exists.
+/// It skips the column if it already exists in the table.
 fn create_proto_alter_kind(
     table_info: &RawTableInfo,
     alter_kind: &Kind,
 ) -> Result<Option<alter_request::Kind>> {
     match alter_kind {
         Kind::AddColumns(x) => {
+            // Construct a set of existing columns in the table.
+            let existing_columns: HashSet<_> = table_info
+                .meta
+                .schema
+                .column_schemas
+                .iter()
+                .map(|col| &col.name)
+                .collect();
             let mut next_column_id = table_info.meta.next_column_id;
 
-            let add_columns = x
-                .add_columns
-                .iter()
-                .map(|add_column| {
-                    let column_def =
-                        add_column
-                            .column_def
-                            .as_ref()
-                            .context(InvalidProtoMsgSnafu {
-                                err_msg: "'column_def' is absent",
-                            })?;
+            let mut add_columns = Vec::with_capacity(x.add_columns.len());
+            for add_column in &x.add_columns {
+                let column_def = add_column
+                    .column_def
+                    .as_ref()
+                    .context(InvalidProtoMsgSnafu {
+                        err_msg: "'column_def' is absent",
+                    })?;
 
-                    let column_id = next_column_id;
-                    next_column_id += 1;
+                // Skips existing columns.
+                if existing_columns.contains(&column_def.name) {
+                    continue;
+                }
 
-                    let column_def = RegionColumnDef {
-                        column_def: Some(column_def.clone()),
-                        column_id,
-                    };
+                let column_id = next_column_id;
+                next_column_id += 1;
+                let column_def = RegionColumnDef {
+                    column_def: Some(column_def.clone()),
+                    column_id,
+                };
 
-                    Ok(AddColumn {
-                        column_def: Some(column_def),
-                        location: add_column.location.clone(),
-                    })
-                })
-                .collect::<Result<Vec<_>>>()?;
+                add_columns.push(AddColumn {
+                    column_def: Some(column_def),
+                    location: add_column.location.clone(),
+                });
+            }
 
             Ok(Some(alter_request::Kind::AddColumns(AddColumns {
                 add_columns,
@@ -143,6 +168,7 @@ mod tests {
     use crate::rpc::router::{Region, RegionRoute};
     use crate::test_util::{new_ddl_context, MockDatanodeManager};
 
+    /// Prepares a region with schema `[ts: Timestamp, host: Tag, cpu: Field]`.
     async fn prepare_ddl_context() -> (DdlContext, u64, TableId, RegionId, String) {
         let datanode_manager = Arc::new(MockDatanodeManager::new(()));
         let ddl_context = new_ddl_context(datanode_manager);
@@ -171,6 +197,7 @@ mod tests {
                     .name("cpu")
                     .data_type(ColumnDataType::Float64)
                     .semantic_type(SemanticType::Field)
+                    .is_nullable(true)
                     .build()
                     .unwrap()
                     .into(),
@@ -225,15 +252,16 @@ mod tests {
                             name: "my_tag3".to_string(),
                             data_type: ColumnDataType::String as i32,
                             is_nullable: true,
-                            default_constraint: b"hello".to_vec(),
+                            default_constraint: Vec::new(),
                             semantic_type: SemanticType::Tag as i32,
                             comment: String::new(),
                             ..Default::default()
                         }),
                         location: Some(AddColumnLocation {
                             location_type: LocationType::After as i32,
-                            after_column_name: "my_tag2".to_string(),
+                            after_column_name: "host".to_string(),
                         }),
+                        add_if_not_exists: false,
                     }],
                 })),
             },
@@ -242,8 +270,11 @@ mod tests {
         let mut procedure =
             AlterTableProcedure::new(cluster_id, table_id, task, ddl_context).unwrap();
         procedure.on_prepare().await.unwrap();
-        let Some(Body::Alter(alter_region_request)) =
-            procedure.make_alter_region_request(region_id).unwrap().body
+        let alter_kind = procedure.make_region_alter_kind().unwrap();
+        let Some(Body::Alter(alter_region_request)) = procedure
+            .make_alter_region_request(region_id, alter_kind)
+            .unwrap()
+            .body
         else {
             unreachable!()
         };
@@ -259,7 +290,7 @@ mod tests {
                                 name: "my_tag3".to_string(),
                                 data_type: ColumnDataType::String as i32,
                                 is_nullable: true,
-                                default_constraint: b"hello".to_vec(),
+                                default_constraint: Vec::new(),
                                 semantic_type: SemanticType::Tag as i32,
                                 comment: String::new(),
                                 ..Default::default()
@@ -268,7 +299,7 @@ mod tests {
                         }),
                         location: Some(AddColumnLocation {
                             location_type: LocationType::After as i32,
-                            after_column_name: "my_tag2".to_string(),
+                            after_column_name: "host".to_string(),
                         }),
                     }]
                 }
@@ -299,8 +330,11 @@ mod tests {
         let mut procedure =
             AlterTableProcedure::new(cluster_id, table_id, task, ddl_context).unwrap();
         procedure.on_prepare().await.unwrap();
-        let Some(Body::Alter(alter_region_request)) =
-            procedure.make_alter_region_request(region_id).unwrap().body
+        let alter_kind = procedure.make_region_alter_kind().unwrap();
+        let Some(Body::Alter(alter_region_request)) = procedure
+            .make_alter_region_request(region_id, alter_kind)
+            .unwrap()
+            .body
         else {
             unreachable!()
         };

src/common/meta/src/ddl/alter_table/update_metadata.rs

@@ -23,7 +23,9 @@ use crate::key::table_info::TableInfoValue;
 use crate::key::{DeserializedValueWithBytes, RegionDistribution};
 
 impl AlterTableProcedure {
-    /// Builds new_meta
+    /// Builds new table info after alteration.
+    /// It bumps the column id of the table by the number of the add column requests.
+    /// So there may be holes in the column id sequence.
     pub(crate) fn build_new_table_info(&self, table_info: &RawTableInfo) -> Result<TableInfo> {
         let table_info =
             TableInfo::try_from(table_info.clone()).context(error::ConvertRawTableInfoSnafu)?;
@@ -34,7 +36,7 @@ impl AlterTableProcedure {
 
         let new_meta = table_info
             .meta
-            .builder_with_alter_kind(table_ref.table, &request.alter_kind, false)
+            .builder_with_alter_kind(table_ref.table, &request.alter_kind)
             .context(error::TableSnafu)?
             .build()
             .with_context(|_| error::BuildTableMetaSnafu {
@@ -46,6 +48,9 @@ impl AlterTableProcedure {
         new_info.ident.version = table_info.ident.version + 1;
         match request.alter_kind {
             AlterKind::AddColumns { columns } => {
+                // Bumps the column id for the new columns.
+                // It may bump more than the actual number of columns added if there are
+                // existing columns, but it's fine.
                 new_info.meta.next_column_id += columns.len() as u32;
             }
             AlterKind::RenameTable { new_table_name } => {

src/common/meta/src/ddl/test_util/alter_table.rs

@@ -30,6 +30,8 @@ pub struct TestAlterTableExpr {
     add_columns: Vec<ColumnDef>,
     #[builder(setter(into, strip_option))]
     new_table_name: Option<String>,
+    #[builder(setter)]
+    add_if_not_exists: bool,
 }
 
 impl From<TestAlterTableExpr> for AlterTableExpr {
@@ -53,6 +55,7 @@ impl From<TestAlterTableExpr> for AlterTableExpr {
                         .map(|col| AddColumn {
                             column_def: Some(col),
                             location: None,
+                            add_if_not_exists: value.add_if_not_exists,
                         })
                         .collect(),
                 })),

src/common/meta/src/ddl/tests/alter_logical_tables.rs

@@ -56,6 +56,7 @@ fn make_alter_logical_table_add_column_task(
     let alter_table = alter_table
         .table_name(table.to_string())
         .add_columns(add_columns)
+        .add_if_not_exists(true)
         .build()
         .unwrap();
 

src/common/meta/src/ddl/tests/alter_table.rs

@@ -139,7 +139,7 @@ async fn test_on_submit_alter_request() {
             table_name: table_name.to_string(),
             kind: Some(Kind::DropColumns(DropColumns {
                 drop_columns: vec![DropColumn {
-                    name: "my_field_column".to_string(),
+                    name: "cpu".to_string(),
                 }],
             })),
         },
@@ -225,7 +225,7 @@ async fn test_on_submit_alter_request_with_outdated_request() {
             table_name: table_name.to_string(),
             kind: Some(Kind::DropColumns(DropColumns {
                 drop_columns: vec![DropColumn {
-                    name: "my_field_column".to_string(),
+                    name: "cpu".to_string(),
                 }],
             })),
         },
@@ -330,6 +330,7 @@ async fn test_on_update_metadata_add_columns() {
                         ..Default::default()
                     }),
                     location: None,
+                    add_if_not_exists: false,
                 }],
             })),
         },

src/common/meta/src/error.rs

@@ -639,15 +639,6 @@ pub enum Error {
         location: Location,
     },
 
-    #[snafu(display("Failed to parse {} from str to utf8", name))]
-    StrFromUtf8 {
-        name: String,
-        #[snafu(source)]
-        error: std::str::Utf8Error,
-        #[snafu(implicit)]
-        location: Location,
-    },
-
     #[snafu(display("Value not exists"))]
     ValueNotExist {
         #[snafu(implicit)]
@@ -658,8 +649,9 @@ pub enum Error {
     GetCache { source: Arc<Error> },
 
     #[cfg(feature = "pg_kvbackend")]
-    #[snafu(display("Failed to execute via Postgres"))]
+    #[snafu(display("Failed to execute via Postgres, sql: {}", sql))]
     PostgresExecution {
+        sql: String,
         #[snafu(source)]
         error: tokio_postgres::Error,
         #[snafu(implicit)]
@@ -667,12 +659,37 @@ pub enum Error {
     },
 
     #[cfg(feature = "pg_kvbackend")]
-    #[snafu(display("Failed to connect to Postgres"))]
-    ConnectPostgres {
+    #[snafu(display("Failed to create connection pool for Postgres"))]
+    CreatePostgresPool {
+        #[snafu(source)]
+        error: deadpool_postgres::CreatePoolError,
+        #[snafu(implicit)]
+        location: Location,
+    },
+
+    #[cfg(feature = "pg_kvbackend")]
+    #[snafu(display("Failed to get Postgres connection from pool: {}", reason))]
+    GetPostgresConnection {
+        reason: String,
+        #[snafu(implicit)]
+        location: Location,
+    },
+
+    #[cfg(feature = "pg_kvbackend")]
+    #[snafu(display("Failed to {} Postgres transaction", operation))]
+    PostgresTransaction {
         #[snafu(source)]
         error: tokio_postgres::Error,
         #[snafu(implicit)]
         location: Location,
+        operation: String,
+    },
+
+    #[cfg(feature = "pg_kvbackend")]
+    #[snafu(display("Postgres transaction retry failed"))]
+    PostgresTransactionRetryFailed {
+        #[snafu(implicit)]
+        location: Location,
     },
 
     #[snafu(display(
@@ -738,8 +755,7 @@ impl ErrorExt for Error {
             | UnexpectedLogicalRouteTable { .. }
             | ProcedureOutput { .. }
             | FromUtf8 { .. }
-            | MetadataCorruption { .. }
-            | StrFromUtf8 { .. } => StatusCode::Unexpected,
+            | MetadataCorruption { .. } => StatusCode::Unexpected,
 
             SendMessage { .. } | GetKvCache { .. } | CacheNotGet { .. } => StatusCode::Internal,
 
@@ -786,9 +802,11 @@ impl ErrorExt for Error {
             | EmptyDdlTasks { .. } => StatusCode::InvalidArguments,
 
             #[cfg(feature = "pg_kvbackend")]
-            PostgresExecution { .. } => StatusCode::Internal,
-            #[cfg(feature = "pg_kvbackend")]
-            ConnectPostgres { .. } => StatusCode::Internal,
+            PostgresExecution { .. }
+            | CreatePostgresPool { .. }
+            | GetPostgresConnection { .. }
+            | PostgresTransaction { .. }
+            | PostgresTransactionRetryFailed { .. } => StatusCode::Internal,
             Error::DatanodeTableInfoNotFound { .. } => StatusCode::Internal,
         }
     }
@@ -799,6 +817,20 @@ impl ErrorExt for Error {
 }
 
 impl Error {
+    #[cfg(feature = "pg_kvbackend")]
+    /// Check if the error is a serialization error.
+    pub fn is_serialization_error(&self) -> bool {
+        match self {
+            Error::PostgresTransaction { error, .. } => {
+                error.code() == Some(&tokio_postgres::error::SqlState::T_R_SERIALIZATION_FAILURE)
+            }
+            Error::PostgresExecution { error, .. } => {
+                error.code() == Some(&tokio_postgres::error::SqlState::T_R_SERIALIZATION_FAILURE)
+            }
+            _ => false,
+        }
+    }
+
     /// Creates a new [Error::RetryLater] error from source `err`.
     pub fn retry_later<E: ErrorExt + Send + Sync + 'static>(err: E) -> Error {
         Error::RetryLater {

src/common/meta/src/key/table_info.rs

@@ -190,6 +190,13 @@ impl TableInfoManager {
         ))
     }
 
+    /// Checks if the table exists.
+    pub async fn exists(&self, table_id: TableId) -> Result<bool> {
+        let key = TableInfoKey::new(table_id);
+        let raw_key = key.to_bytes();
+        self.kv_backend.exists(&raw_key).await
+    }
+
     pub async fn get(
         &self,
         table_id: TableId,

src/common/meta/src/kv_backend/etcd.rs

@@ -542,6 +542,8 @@ mod tests {
         prepare_kv_with_prefix, test_kv_batch_delete_with_prefix, test_kv_batch_get_with_prefix,
         test_kv_compare_and_put_with_prefix, test_kv_delete_range_with_prefix,
         test_kv_put_with_prefix, test_kv_range_2_with_prefix, test_kv_range_with_prefix,
+        test_txn_compare_equal, test_txn_compare_greater, test_txn_compare_less,
+        test_txn_compare_not_equal, test_txn_one_compare_op, text_txn_multi_compare_op,
         unprepare_kv,
     };
 
@@ -589,7 +591,7 @@ mod tests {
     #[tokio::test]
     async fn test_range_2() {
         if let Some(kv_backend) = build_kv_backend().await {
-            test_kv_range_2_with_prefix(kv_backend, b"range2/".to_vec()).await;
+            test_kv_range_2_with_prefix(&kv_backend, b"range2/".to_vec()).await;
         }
     }
 
@@ -616,7 +618,8 @@ mod tests {
         if let Some(kv_backend) = build_kv_backend().await {
             let prefix = b"deleteRange/";
             prepare_kv_with_prefix(&kv_backend, prefix.to_vec()).await;
-            test_kv_delete_range_with_prefix(kv_backend, prefix.to_vec()).await;
+            test_kv_delete_range_with_prefix(&kv_backend, prefix.to_vec()).await;
+            unprepare_kv(&kv_backend, prefix).await;
         }
     }
 
@@ -625,7 +628,20 @@ mod tests {
         if let Some(kv_backend) = build_kv_backend().await {
             let prefix = b"batchDelete/";
             prepare_kv_with_prefix(&kv_backend, prefix.to_vec()).await;
-            test_kv_batch_delete_with_prefix(kv_backend, prefix.to_vec()).await;
+            test_kv_batch_delete_with_prefix(&kv_backend, prefix.to_vec()).await;
+            unprepare_kv(&kv_backend, prefix).await;
+        }
+    }
+
+    #[tokio::test]
+    async fn test_etcd_txn() {
+        if let Some(kv_backend) = build_kv_backend().await {
+            test_txn_one_compare_op(&kv_backend).await;
+            text_txn_multi_compare_op(&kv_backend).await;
+            test_txn_compare_equal(&kv_backend).await;
+            test_txn_compare_greater(&kv_backend).await;
+            test_txn_compare_less(&kv_backend).await;
+            test_txn_compare_not_equal(&kv_backend).await;
         }
     }
 }

src/common/meta/src/kv_backend/memory.rs

@@ -325,7 +325,9 @@ mod tests {
     use crate::error::Error;
     use crate::kv_backend::test::{
         prepare_kv, test_kv_batch_delete, test_kv_batch_get, test_kv_compare_and_put,
-        test_kv_delete_range, test_kv_put, test_kv_range, test_kv_range_2,
+        test_kv_delete_range, test_kv_put, test_kv_range, test_kv_range_2, test_txn_compare_equal,
+        test_txn_compare_greater, test_txn_compare_less, test_txn_compare_not_equal,
+        test_txn_one_compare_op, text_txn_multi_compare_op,
     };
 
     async fn mock_mem_store_with_data() -> MemoryKvBackend<Error> {
@@ -353,7 +355,7 @@ mod tests {
     async fn test_range_2() {
         let kv = MemoryKvBackend::<Error>::new();
 
-        test_kv_range_2(kv).await;
+        test_kv_range_2(&kv).await;
     }
 
     #[tokio::test]
@@ -374,13 +376,24 @@ mod tests {
     async fn test_delete_range() {
         let kv_backend = mock_mem_store_with_data().await;
 
-        test_kv_delete_range(kv_backend).await;
+        test_kv_delete_range(&kv_backend).await;
     }
 
     #[tokio::test]
     async fn test_batch_delete() {
         let kv_backend = mock_mem_store_with_data().await;
 
-        test_kv_batch_delete(kv_backend).await;
+        test_kv_batch_delete(&kv_backend).await;
+    }
+
+    #[tokio::test]
+    async fn test_memory_txn() {
+        let kv_backend = MemoryKvBackend::<Error>::new();
+        test_txn_one_compare_op(&kv_backend).await;
+        text_txn_multi_compare_op(&kv_backend).await;
+        test_txn_compare_equal(&kv_backend).await;
+        test_txn_compare_greater(&kv_backend).await;
+        test_txn_compare_less(&kv_backend).await;
+        test_txn_compare_not_equal(&kv_backend).await;
     }
 }

src/common/meta/src/kv_backend/test.rs

@@ -15,6 +15,8 @@
 use std::sync::atomic::{AtomicU8, Ordering};
 use std::sync::Arc;
 
+use txn::{Compare, CompareOp, TxnOp};
+
 use super::{KvBackend, *};
 use crate::error::Error;
 use crate::rpc::store::{BatchGetRequest, PutRequest};
@@ -59,14 +61,18 @@ pub async fn prepare_kv_with_prefix(kv_backend: &impl KvBackend, prefix: Vec<u8>
 
 pub async fn unprepare_kv(kv_backend: &impl KvBackend, prefix: &[u8]) {
     let range_end = util::get_prefix_end_key(prefix);
-    assert!(kv_backend
-        .delete_range(DeleteRangeRequest {
-            key: prefix.to_vec(),
-            range_end,
-            ..Default::default()
-        })
-        .await
-        .is_ok());
+    assert!(
+        kv_backend
+            .delete_range(DeleteRangeRequest {
+                key: prefix.to_vec(),
+                range_end,
+                ..Default::default()
+            })
+            .await
+            .is_ok(),
+        "prefix: {:?}",
+        std::str::from_utf8(prefix).unwrap()
+    );
 }
 
 pub async fn test_kv_put(kv_backend: &impl KvBackend) {
@@ -168,11 +174,11 @@ pub async fn test_kv_range_with_prefix(kv_backend: &impl KvBackend, prefix: Vec<
     assert_eq!(b"val1", resp.kvs[0].value());
 }
 
-pub async fn test_kv_range_2(kv_backend: impl KvBackend) {
+pub async fn test_kv_range_2(kv_backend: &impl KvBackend) {
     test_kv_range_2_with_prefix(kv_backend, vec![]).await;
 }
 
-pub async fn test_kv_range_2_with_prefix(kv_backend: impl KvBackend, prefix: Vec<u8>) {
+pub async fn test_kv_range_2_with_prefix(kv_backend: &impl KvBackend, prefix: Vec<u8>) {
     let atest = [prefix.clone(), b"atest".to_vec()].concat();
     let test = [prefix.clone(), b"test".to_vec()].concat();
 
@@ -346,11 +352,11 @@ pub async fn test_kv_compare_and_put_with_prefix(
     assert!(resp.is_none());
 }
 
-pub async fn test_kv_delete_range(kv_backend: impl KvBackend) {
+pub async fn test_kv_delete_range(kv_backend: &impl KvBackend) {
     test_kv_delete_range_with_prefix(kv_backend, vec![]).await;
 }
 
-pub async fn test_kv_delete_range_with_prefix(kv_backend: impl KvBackend, prefix: Vec<u8>) {
+pub async fn test_kv_delete_range_with_prefix(kv_backend: &impl KvBackend, prefix: Vec<u8>) {
     let key3 = [prefix.clone(), b"key3".to_vec()].concat();
     let req = DeleteRangeRequest {
         key: key3.clone(),
@@ -401,11 +407,11 @@ pub async fn test_kv_delete_range_with_prefix(kv_backend: impl KvBackend, prefix
     assert!(resp.kvs.is_empty());
 }
 
-pub async fn test_kv_batch_delete(kv_backend: impl KvBackend) {
+pub async fn test_kv_batch_delete(kv_backend: &impl KvBackend) {
     test_kv_batch_delete_with_prefix(kv_backend, vec![]).await;
 }
 
-pub async fn test_kv_batch_delete_with_prefix(kv_backend: impl KvBackend, prefix: Vec<u8>) {
+pub async fn test_kv_batch_delete_with_prefix(kv_backend: &impl KvBackend, prefix: Vec<u8>) {
     let key1 = [prefix.clone(), b"key1".to_vec()].concat();
     let key100 = [prefix.clone(), b"key100".to_vec()].concat();
     assert!(kv_backend.get(&key1).await.unwrap().is_some());
@@ -444,3 +450,207 @@ pub async fn test_kv_batch_delete_with_prefix(kv_backend: impl KvBackend, prefix
     assert!(kv_backend.get(&key3).await.unwrap().is_none());
     assert!(kv_backend.get(&key11).await.unwrap().is_none());
 }
+
+pub async fn test_txn_one_compare_op(kv_backend: &impl KvBackend) {
+    let _ = kv_backend
+        .put(PutRequest {
+            key: vec![11],
+            value: vec![3],
+            ..Default::default()
+        })
+        .await
+        .unwrap();
+
+    let txn = Txn::new()
+        .when(vec![Compare::with_value(
+            vec![11],
+            CompareOp::Greater,
+            vec![1],
+        )])
+        .and_then(vec![TxnOp::Put(vec![11], vec![1])])
+        .or_else(vec![TxnOp::Put(vec![11], vec![2])]);
+
+    let txn_response = kv_backend.txn(txn).await.unwrap();
+
+    assert!(txn_response.succeeded);
+    assert_eq!(txn_response.responses.len(), 1);
+}
+
+pub async fn text_txn_multi_compare_op(kv_backend: &impl KvBackend) {
+    for i in 1..3 {
+        let _ = kv_backend
+            .put(PutRequest {
+                key: vec![i],
+                value: vec![i],
+                ..Default::default()
+            })
+            .await
+            .unwrap();
+    }
+
+    let when: Vec<_> = (1..3u8)
+        .map(|i| Compare::with_value(vec![i], CompareOp::Equal, vec![i]))
+        .collect();
+
+    let txn = Txn::new()
+        .when(when)
+        .and_then(vec![
+            TxnOp::Put(vec![1], vec![10]),
+            TxnOp::Put(vec![2], vec![20]),
+        ])
+        .or_else(vec![TxnOp::Put(vec![1], vec![11])]);
+
+    let txn_response = kv_backend.txn(txn).await.unwrap();
+
+    assert!(txn_response.succeeded);
+    assert_eq!(txn_response.responses.len(), 2);
+}
+
+pub async fn test_txn_compare_equal(kv_backend: &impl KvBackend) {
+    let key = vec![101u8];
+    kv_backend.delete(&key, false).await.unwrap();
+
+    let txn = Txn::new()
+        .when(vec![Compare::with_value_not_exists(
+            key.clone(),
+            CompareOp::Equal,
+        )])
+        .and_then(vec![TxnOp::Put(key.clone(), vec![1])])
+        .or_else(vec![TxnOp::Put(key.clone(), vec![2])]);
+    let txn_response = kv_backend.txn(txn.clone()).await.unwrap();
+    assert!(txn_response.succeeded);
+
+    let txn_response = kv_backend.txn(txn).await.unwrap();
+    assert!(!txn_response.succeeded);
+
+    let txn = Txn::new()
+        .when(vec![Compare::with_value(
+            key.clone(),
+            CompareOp::Equal,
+            vec![2],
+        )])
+        .and_then(vec![TxnOp::Put(key.clone(), vec![3])])
+        .or_else(vec![TxnOp::Put(key, vec![4])]);
+    let txn_response = kv_backend.txn(txn).await.unwrap();
+    assert!(txn_response.succeeded);
+}
+
+pub async fn test_txn_compare_greater(kv_backend: &impl KvBackend) {
+    let key = vec![102u8];
+    kv_backend.delete(&key, false).await.unwrap();
+
+    let txn = Txn::new()
+        .when(vec![Compare::with_value_not_exists(
+            key.clone(),
+            CompareOp::Greater,
+        )])
+        .and_then(vec![TxnOp::Put(key.clone(), vec![1])])
+        .or_else(vec![TxnOp::Put(key.clone(), vec![2])]);
+    let txn_response = kv_backend.txn(txn.clone()).await.unwrap();
+    assert!(!txn_response.succeeded);
+
+    let txn_response = kv_backend.txn(txn).await.unwrap();
+    assert!(txn_response.succeeded);
+
+    let txn = Txn::new()
+        .when(vec![Compare::with_value(
+            key.clone(),
+            CompareOp::Greater,
+            vec![1],
+        )])
+        .and_then(vec![TxnOp::Put(key.clone(), vec![3])])
+        .or_else(vec![TxnOp::Get(key.clone())]);
+    let mut txn_response = kv_backend.txn(txn).await.unwrap();
+    assert!(!txn_response.succeeded);
+    let res = txn_response.responses.pop().unwrap();
+    assert_eq!(
+        res,
+        TxnOpResponse::ResponseGet(RangeResponse {
+            kvs: vec![KeyValue {
+                key,
+                value: vec![1]
+            }],
+            more: false,
+        })
+    );
+}
+
+pub async fn test_txn_compare_less(kv_backend: &impl KvBackend) {
+    let key = vec![103u8];
+    kv_backend.delete(&[3], false).await.unwrap();
+
+    let txn = Txn::new()
+        .when(vec![Compare::with_value_not_exists(
+            key.clone(),
+            CompareOp::Less,
+        )])
+        .and_then(vec![TxnOp::Put(key.clone(), vec![1])])
+        .or_else(vec![TxnOp::Put(key.clone(), vec![2])]);
+    let txn_response = kv_backend.txn(txn.clone()).await.unwrap();
+    assert!(!txn_response.succeeded);
+
+    let txn_response = kv_backend.txn(txn).await.unwrap();
+    assert!(!txn_response.succeeded);
+
+    let txn = Txn::new()
+        .when(vec![Compare::with_value(
+            key.clone(),
+            CompareOp::Less,
+            vec![2],
+        )])
+        .and_then(vec![TxnOp::Put(key.clone(), vec![3])])
+        .or_else(vec![TxnOp::Get(key.clone())]);
+    let mut txn_response = kv_backend.txn(txn).await.unwrap();
+    assert!(!txn_response.succeeded);
+    let res = txn_response.responses.pop().unwrap();
+    assert_eq!(
+        res,
+        TxnOpResponse::ResponseGet(RangeResponse {
+            kvs: vec![KeyValue {
+                key,
+                value: vec![2]
+            }],
+            more: false,
+        })
+    );
+}
+
+pub async fn test_txn_compare_not_equal(kv_backend: &impl KvBackend) {
+    let key = vec![104u8];
+    kv_backend.delete(&key, false).await.unwrap();
+
+    let txn = Txn::new()
+        .when(vec![Compare::with_value_not_exists(
+            key.clone(),
+            CompareOp::NotEqual,
+        )])
+        .and_then(vec![TxnOp::Put(key.clone(), vec![1])])
+        .or_else(vec![TxnOp::Put(key.clone(), vec![2])]);
+    let txn_response = kv_backend.txn(txn.clone()).await.unwrap();
+    assert!(!txn_response.succeeded);
+
+    let txn_response = kv_backend.txn(txn).await.unwrap();
+    assert!(txn_response.succeeded);
+
+    let txn = Txn::new()
+        .when(vec![Compare::with_value(
+            key.clone(),
+            CompareOp::Equal,
+            vec![2],
+        )])
+        .and_then(vec![TxnOp::Put(key.clone(), vec![3])])
+        .or_else(vec![TxnOp::Get(key.clone())]);
+    let mut txn_response = kv_backend.txn(txn).await.unwrap();
+    assert!(!txn_response.succeeded);
+    let res = txn_response.responses.pop().unwrap();
+    assert_eq!(
+        res,
+        TxnOpResponse::ResponseGet(RangeResponse {
+            kvs: vec![KeyValue {
+                key,
+                value: vec![1]
+            }],
+            more: false,
+        })
+    );
+}

src/common/meta/src/kv_backend/txn.rs

@@ -131,9 +131,9 @@ pub struct TxnResponse {
 pub struct Txn {
     // HACK - chroot would modify this field
     pub(super) req: TxnRequest,
-    c_when: bool,
-    c_then: bool,
-    c_else: bool,
+    pub(super) c_when: bool,
+    pub(super) c_then: bool,
+    pub(super) c_else: bool,
 }
 
 #[cfg(any(test, feature = "testing"))]
@@ -241,14 +241,7 @@ impl From<Txn> for TxnRequest {
 
 #[cfg(test)]
 mod tests {
-    use std::sync::Arc;
-
     use super::*;
-    use crate::error::Error;
-    use crate::kv_backend::memory::MemoryKvBackend;
-    use crate::kv_backend::KvBackendRef;
-    use crate::rpc::store::PutRequest;
-    use crate::rpc::KeyValue;
 
     #[test]
     fn test_compare() {
@@ -310,232 +303,4 @@ mod tests {
             }
         );
     }
-
-    #[tokio::test]
-    async fn test_txn_one_compare_op() {
-        let kv_backend = create_kv_backend().await;
-
-        let _ = kv_backend
-            .put(PutRequest {
-                key: vec![11],
-                value: vec![3],
-                ..Default::default()
-            })
-            .await
-            .unwrap();
-
-        let txn = Txn::new()
-            .when(vec![Compare::with_value(
-                vec![11],
-                CompareOp::Greater,
-                vec![1],
-            )])
-            .and_then(vec![TxnOp::Put(vec![11], vec![1])])
-            .or_else(vec![TxnOp::Put(vec![11], vec![2])]);
-
-        let txn_response = kv_backend.txn(txn).await.unwrap();
-
-        assert!(txn_response.succeeded);
-        assert_eq!(txn_response.responses.len(), 1);
-    }
-
-    #[tokio::test]
-    async fn test_txn_multi_compare_op() {
-        let kv_backend = create_kv_backend().await;
-
-        for i in 1..3 {
-            let _ = kv_backend
-                .put(PutRequest {
-                    key: vec![i],
-                    value: vec![i],
-                    ..Default::default()
-                })
-                .await
-                .unwrap();
-        }
-
-        let when: Vec<_> = (1..3u8)
-            .map(|i| Compare::with_value(vec![i], CompareOp::Equal, vec![i]))
-            .collect();
-
-        let txn = Txn::new()
-            .when(when)
-            .and_then(vec![
-                TxnOp::Put(vec![1], vec![10]),
-                TxnOp::Put(vec![2], vec![20]),
-            ])
-            .or_else(vec![TxnOp::Put(vec![1], vec![11])]);
-
-        let txn_response = kv_backend.txn(txn).await.unwrap();
-
-        assert!(txn_response.succeeded);
-        assert_eq!(txn_response.responses.len(), 2);
-    }
-
-    #[tokio::test]
-    async fn test_txn_compare_equal() {
-        let kv_backend = create_kv_backend().await;
-        let key = vec![101u8];
-        kv_backend.delete(&key, false).await.unwrap();
-
-        let txn = Txn::new()
-            .when(vec![Compare::with_value_not_exists(
-                key.clone(),
-                CompareOp::Equal,
-            )])
-            .and_then(vec![TxnOp::Put(key.clone(), vec![1])])
-            .or_else(vec![TxnOp::Put(key.clone(), vec![2])]);
-        let txn_response = kv_backend.txn(txn.clone()).await.unwrap();
-        assert!(txn_response.succeeded);
-
-        let txn_response = kv_backend.txn(txn).await.unwrap();
-        assert!(!txn_response.succeeded);
-
-        let txn = Txn::new()
-            .when(vec![Compare::with_value(
-                key.clone(),
-                CompareOp::Equal,
-                vec![2],
-            )])
-            .and_then(vec![TxnOp::Put(key.clone(), vec![3])])
-            .or_else(vec![TxnOp::Put(key, vec![4])]);
-        let txn_response = kv_backend.txn(txn).await.unwrap();
-        assert!(txn_response.succeeded);
-    }
-
-    #[tokio::test]
-    async fn test_txn_compare_greater() {
-        let kv_backend = create_kv_backend().await;
-        let key = vec![102u8];
-        kv_backend.delete(&key, false).await.unwrap();
-
-        let txn = Txn::new()
-            .when(vec![Compare::with_value_not_exists(
-                key.clone(),
-                CompareOp::Greater,
-            )])
-            .and_then(vec![TxnOp::Put(key.clone(), vec![1])])
-            .or_else(vec![TxnOp::Put(key.clone(), vec![2])]);
-        let txn_response = kv_backend.txn(txn.clone()).await.unwrap();
-        assert!(!txn_response.succeeded);
-
-        let txn_response = kv_backend.txn(txn).await.unwrap();
-        assert!(txn_response.succeeded);
-
-        let txn = Txn::new()
-            .when(vec![Compare::with_value(
-                key.clone(),
-                CompareOp::Greater,
-                vec![1],
-            )])
-            .and_then(vec![TxnOp::Put(key.clone(), vec![3])])
-            .or_else(vec![TxnOp::Get(key.clone())]);
-        let mut txn_response = kv_backend.txn(txn).await.unwrap();
-        assert!(!txn_response.succeeded);
-        let res = txn_response.responses.pop().unwrap();
-        assert_eq!(
-            res,
-            TxnOpResponse::ResponseGet(RangeResponse {
-                kvs: vec![KeyValue {
-                    key,
-                    value: vec![1]
-                }],
-                more: false,
-            })
-        );
-    }
-
-    #[tokio::test]
-    async fn test_txn_compare_less() {
-        let kv_backend = create_kv_backend().await;
-        let key = vec![103u8];
-        kv_backend.delete(&[3], false).await.unwrap();
-
-        let txn = Txn::new()
-            .when(vec![Compare::with_value_not_exists(
-                key.clone(),
-                CompareOp::Less,
-            )])
-            .and_then(vec![TxnOp::Put(key.clone(), vec![1])])
-            .or_else(vec![TxnOp::Put(key.clone(), vec![2])]);
-        let txn_response = kv_backend.txn(txn.clone()).await.unwrap();
-        assert!(!txn_response.succeeded);
-
-        let txn_response = kv_backend.txn(txn).await.unwrap();
-        assert!(!txn_response.succeeded);
-
-        let txn = Txn::new()
-            .when(vec![Compare::with_value(
-                key.clone(),
-                CompareOp::Less,
-                vec![2],
-            )])
-            .and_then(vec![TxnOp::Put(key.clone(), vec![3])])
-            .or_else(vec![TxnOp::Get(key.clone())]);
-        let mut txn_response = kv_backend.txn(txn).await.unwrap();
-        assert!(!txn_response.succeeded);
-        let res = txn_response.responses.pop().unwrap();
-        assert_eq!(
-            res,
-            TxnOpResponse::ResponseGet(RangeResponse {
-                kvs: vec![KeyValue {
-                    key,
-                    value: vec![2]
-                }],
-                more: false,
-            })
-        );
-    }
-
-    #[tokio::test]
-    async fn test_txn_compare_not_equal() {
-        let kv_backend = create_kv_backend().await;
-        let key = vec![104u8];
-        kv_backend.delete(&key, false).await.unwrap();
-
-        let txn = Txn::new()
-            .when(vec![Compare::with_value_not_exists(
-                key.clone(),
-                CompareOp::NotEqual,
-            )])
-            .and_then(vec![TxnOp::Put(key.clone(), vec![1])])
-            .or_else(vec![TxnOp::Put(key.clone(), vec![2])]);
-        let txn_response = kv_backend.txn(txn.clone()).await.unwrap();
-        assert!(!txn_response.succeeded);
-
-        let txn_response = kv_backend.txn(txn).await.unwrap();
-        assert!(txn_response.succeeded);
-
-        let txn = Txn::new()
-            .when(vec![Compare::with_value(
-                key.clone(),
-                CompareOp::Equal,
-                vec![2],
-            )])
-            .and_then(vec![TxnOp::Put(key.clone(), vec![3])])
-            .or_else(vec![TxnOp::Get(key.clone())]);
-        let mut txn_response = kv_backend.txn(txn).await.unwrap();
-        assert!(!txn_response.succeeded);
-        let res = txn_response.responses.pop().unwrap();
-        assert_eq!(
-            res,
-            TxnOpResponse::ResponseGet(RangeResponse {
-                kvs: vec![KeyValue {
-                    key,
-                    value: vec![1]
-                }],
-                more: false,
-            })
-        );
-    }
-
-    async fn create_kv_backend() -> KvBackendRef {
-        Arc::new(MemoryKvBackend::<Error>::new())
-        // TODO(jiachun): Add a feature to test against etcd in github CI
-        //
-        // The same test can be run against etcd by uncommenting the following line
-        // crate::service::store::etcd::EtcdStore::with_endpoints(["127.0.0.1:2379"])
-        //     .await
-        //     .unwrap()
-    }
 }

src/common/meta/src/rpc/store.rs

@@ -266,7 +266,7 @@ impl PutRequest {
     }
 }
 
-#[derive(Debug, Clone, PartialEq)]
+#[derive(Debug, Clone, PartialEq, Default)]
 pub struct PutResponse {
     pub prev_kv: Option<KeyValue>,
 }
@@ -425,7 +425,7 @@ impl BatchPutRequest {
     }
 }
 
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, Default)]
 pub struct BatchPutResponse {
     pub prev_kvs: Vec<KeyValue>,
 }
@@ -509,7 +509,7 @@ impl BatchDeleteRequest {
     }
 }
 
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, Default)]
 pub struct BatchDeleteResponse {
     pub prev_kvs: Vec<KeyValue>,
 }
@@ -754,6 +754,19 @@ impl TryFrom<PbDeleteRangeResponse> for DeleteRangeResponse {
 }
 
 impl DeleteRangeResponse {
+    /// Creates a new [`DeleteRangeResponse`] with the given deleted count.
+    pub fn new(deleted: i64) -> Self {
+        Self {
+            deleted,
+            prev_kvs: vec![],
+        }
+    }
+
+    /// Creates a new [`DeleteRangeResponse`] with the given deleted count and previous key-value pairs.
+    pub fn with_prev_kvs(&mut self, prev_kvs: Vec<KeyValue>) {
+        self.prev_kvs = prev_kvs;
+    }
+
     pub fn to_proto_resp(self, header: PbResponseHeader) -> PbDeleteRangeResponse {
         PbDeleteRangeResponse {
             header: Some(header),

src/common/procedure/Cargo.toml

@@ -13,7 +13,7 @@ workspace = true
 [dependencies]
 async-stream.workspace = true
 async-trait.workspace = true
-backon = "1"
+backon.workspace = true
 common-base.workspace = true
 common-error.workspace = true
 common-macro.workspace = true

src/common/query/src/error.rs

@@ -18,7 +18,6 @@ use arrow::error::ArrowError;
 use common_error::ext::{BoxedError, ErrorExt};
 use common_error::status_code::StatusCode;
 use common_macro::stack_trace_debug;
-use common_recordbatch::error::Error as RecordbatchError;
 use datafusion_common::DataFusionError;
 use datatypes::arrow;
 use datatypes::arrow::datatypes::DataType as ArrowDatatype;
@@ -31,21 +30,6 @@ use statrs::StatsError;
 #[snafu(visibility(pub))]
 #[stack_trace_debug]
 pub enum Error {
-    #[snafu(display("Failed to execute Python UDF: {}", msg))]
-    PyUdf {
-        // TODO(discord9): find a way that prevent circle depend(query<-script<-query) and can use script's error type
-        msg: String,
-        #[snafu(implicit)]
-        location: Location,
-    },
-
-    #[snafu(display("Failed to create temporary recordbatch when eval Python UDF"))]
-    UdfTempRecordBatch {
-        #[snafu(implicit)]
-        location: Location,
-        source: RecordbatchError,
-    },
-
     #[snafu(display("Failed to execute function"))]
     ExecuteFunction {
         #[snafu(source)]
@@ -260,9 +244,7 @@ pub type Result<T> = std::result::Result<T, Error>;
 impl ErrorExt for Error {
     fn status_code(&self) -> StatusCode {
         match self {
-            Error::UdfTempRecordBatch { .. }
-            | Error::PyUdf { .. }
-            | Error::CreateAccumulator { .. }
+            Error::CreateAccumulator { .. }
             | Error::DowncastVector { .. }
             | Error::InvalidInputState { .. }
             | Error::InvalidInputCol { .. }

src/common/recordbatch/src/recordbatch.rs

@@ -35,7 +35,7 @@ use crate::DfRecordBatch;
 #[derive(Clone, Debug, PartialEq)]
 pub struct RecordBatch {
     pub schema: SchemaRef,
-    columns: Vec<VectorRef>,
+    pub columns: Vec<VectorRef>,
     df_record_batch: DfRecordBatch,
 }
 

src/common/wal/Cargo.toml

@@ -19,7 +19,7 @@ futures-util.workspace = true
 humantime-serde.workspace = true
 num_cpus.workspace = true
 rskafka.workspace = true
-rustls = { version = "0.23", default-features = false, features = ["ring", "logging", "std", "tls12"] }
+rustls = { workspace = true, default-features = false, features = ["ring", "logging", "std", "tls12"] }
 rustls-native-certs = "0.7"
 rustls-pemfile = "2.1"
 serde.workspace = true

src/datanode/src/datanode.rs

@@ -433,8 +433,8 @@ impl DatanodeBuilder {
     ) -> Result<MitoEngine> {
         if opts.storage.is_object_storage() {
             // Enable the write cache when setting object storage
-            config.enable_experimental_write_cache = true;
-            info!("Configured 'enable_experimental_write_cache=true' for mito engine.");
+            config.enable_write_cache = true;
+            info!("Configured 'enable_write_cache=true' for mito engine.");
         }
 
         let mito_engine = match &opts.wal {

src/datatypes/src/schema/column_schema.rs

@@ -164,8 +164,15 @@ impl ColumnSchema {
             .unwrap_or(false)
     }
 
-    pub fn has_fulltext_index_key(&self) -> bool {
-        self.metadata.contains_key(FULLTEXT_KEY)
+    pub fn is_fulltext_indexed(&self) -> bool {
+        self.fulltext_options()
+            .unwrap_or_default()
+            .map(|option| option.enable)
+            .unwrap_or_default()
+    }
+
+    pub fn is_skipping_indexed(&self) -> bool {
+        self.skipping_index_options().unwrap_or_default().is_some()
     }
 
     pub fn has_inverted_index_key(&self) -> bool {

src/flow/Cargo.toml

@@ -40,7 +40,6 @@ datatypes.workspace = true
 enum-as-inner = "0.6.0"
 enum_dispatch = "0.3"
 futures = "0.3"
-get-size-derive2 = "0.1.2"
 get-size2 = "0.1.2"
 greptime-proto.workspace = true
 # This fork of hydroflow is simply for keeping our dependency in our org, and pin the version

src/flow/src/adapter.rs

@@ -45,17 +45,12 @@ use tokio::sync::broadcast::error::TryRecvError;
 use tokio::sync::{broadcast, watch, Mutex, RwLock};
 
 pub(crate) use crate::adapter::node_context::FlownodeContext;
-use crate::adapter::table_source::TableSource;
-use crate::adapter::util::{
-    relation_desc_to_column_schemas_with_fallback, table_info_value_to_relation_desc,
-};
+use crate::adapter::table_source::ManagedTableSource;
+use crate::adapter::util::relation_desc_to_column_schemas_with_fallback;
 use crate::adapter::worker::{create_worker, Worker, WorkerHandle};
 use crate::compute::ErrCollector;
 use crate::df_optimizer::sql_to_flow_plan;
-use crate::error::{
-    EvalSnafu, ExternalSnafu, FlowAlreadyExistSnafu, InternalSnafu, InvalidQuerySnafu,
-    UnexpectedSnafu,
-};
+use crate::error::{EvalSnafu, ExternalSnafu, InternalSnafu, InvalidQuerySnafu, UnexpectedSnafu};
 use crate::expr::Batch;
 use crate::metrics::{METRIC_FLOW_INSERT_ELAPSED, METRIC_FLOW_ROWS, METRIC_FLOW_RUN_INTERVAL_MS};
 use crate::repr::{self, DiffRow, RelationDesc, Row, BATCH_SIZE};
@@ -69,7 +64,7 @@ mod util;
 mod worker;
 
 pub(crate) mod node_context;
-mod table_source;
+pub(crate) mod table_source;
 
 use crate::error::Error;
 use crate::utils::StateReportHandler;
@@ -129,7 +124,7 @@ pub struct FlowWorkerManager {
     /// The query engine that will be used to parse the query and convert it to a dataflow plan
     pub query_engine: Arc<dyn QueryEngine>,
     /// Getting table name and table schema from table info manager
-    table_info_source: TableSource,
+    table_info_source: ManagedTableSource,
     frontend_invoker: RwLock<Option<FrontendInvoker>>,
     /// contains mapping from table name to global id, and table schema
     node_context: RwLock<FlownodeContext>,
@@ -158,11 +153,11 @@ impl FlowWorkerManager {
         query_engine: Arc<dyn QueryEngine>,
         table_meta: TableMetadataManagerRef,
     ) -> Self {
-        let srv_map = TableSource::new(
+        let srv_map = ManagedTableSource::new(
             table_meta.table_info_manager().clone(),
             table_meta.table_name_manager().clone(),
         );
-        let node_context = FlownodeContext::default();
+        let node_context = FlownodeContext::new(Box::new(srv_map.clone()) as _);
         let tick_manager = FlowTickManager::new();
         let worker_handles = Vec::new();
         FlowWorkerManager {
@@ -409,7 +404,7 @@ impl FlowWorkerManager {
     ) -> Result<Option<(Vec<String>, Option<usize>, Vec<ColumnSchema>)>, Error> {
         if let Some(table_id) = self
             .table_info_source
-            .get_table_id_from_name(table_name)
+            .get_opt_table_id_from_name(table_name)
             .await?
         {
             let table_info = self
@@ -729,43 +724,6 @@ impl FlowWorkerManager {
             query_ctx,
         } = args;
 
-        let already_exist = {
-            let mut flag = false;
-
-            // check if the task already exists
-            for handle in self.worker_handles.iter() {
-                if handle.lock().await.contains_flow(flow_id).await? {
-                    flag = true;
-                    break;
-                }
-            }
-            flag
-        };
-        match (create_if_not_exists, or_replace, already_exist) {
-            // do replace
-            (_, true, true) => {
-                info!("Replacing flow with id={}", flow_id);
-                self.remove_flow(flow_id).await?;
-            }
-            (false, false, true) => FlowAlreadyExistSnafu { id: flow_id }.fail()?,
-            // do nothing if exists
-            (true, false, true) => {
-                info!("Flow with id={} already exists, do nothing", flow_id);
-                return Ok(None);
-            }
-            // create if not exists
-            (_, _, false) => (),
-        }
-
-        if create_if_not_exists {
-            // check if the task already exists
-            for handle in self.worker_handles.iter() {
-                if handle.lock().await.contains_flow(flow_id).await? {
-                    return Ok(None);
-                }
-            }
-        }
-
         let mut node_ctx = self.node_context.write().await;
         // assign global id to source and sink table
         for source in &source_table_ids {
@@ -828,27 +786,9 @@ impl FlowWorkerManager {
                     .fail()?,
                 }
             }
-
-            let table_id = self
-                .table_info_source
-                .get_table_id_from_name(&sink_table_name)
-                .await?
-                .context(UnexpectedSnafu {
-                    reason: format!("Can't get table id for table name {:?}", sink_table_name),
-                })?;
-            let table_info_value = self
-                .table_info_source
-                .get_table_info_value(&table_id)
-                .await?
-                .context(UnexpectedSnafu {
-                    reason: format!("Can't get table info value for table id {:?}", table_id),
-                })?;
-            let real_schema = table_info_value_to_relation_desc(table_info_value)?;
-            node_ctx.assign_table_schema(&sink_table_name, real_schema.clone())?;
         } else {
             // assign inferred schema to sink table
             // create sink table
-            node_ctx.assign_table_schema(&sink_table_name, flow_plan.schema.clone())?;
             let did_create = self
                 .create_table_from_relation(
                     &format!("flow-id={flow_id}"),
@@ -864,6 +804,8 @@ impl FlowWorkerManager {
             }
         }
 
+        node_ctx.add_flow_plan(flow_id, flow_plan.clone());
+
         let _ = comment;
         let _ = flow_options;
 
@@ -897,9 +839,11 @@ impl FlowWorkerManager {
             source_ids,
             src_recvs: source_receivers,
             expire_after,
+            or_replace,
             create_if_not_exists,
             err_collector,
         };
+
         handle.create_flow(create_request).await?;
         info!("Successfully create flow with id={}", flow_id);
         Ok(Some(flow_id))

src/flow/src/adapter/flownode_impl.rs

@@ -25,20 +25,24 @@ use common_meta::error::{ExternalSnafu, Result, UnexpectedSnafu};
 use common_meta::node_manager::Flownode;
 use common_telemetry::{debug, trace};
 use itertools::Itertools;
-use snafu::{OptionExt, ResultExt};
+use snafu::{IntoError, OptionExt, ResultExt};
 use store_api::storage::RegionId;
 
-use super::util::from_proto_to_data_type;
 use crate::adapter::{CreateFlowArgs, FlowWorkerManager};
-use crate::error::InternalSnafu;
+use crate::error::{CreateFlowSnafu, InsertIntoFlowSnafu, InternalSnafu};
 use crate::metrics::METRIC_FLOW_TASK_COUNT;
 use crate::repr::{self, DiffRow};
 
-fn to_meta_err(err: crate::error::Error) -> common_meta::error::Error {
-    // TODO(discord9): refactor this
-    Err::<(), _>(BoxedError::new(err))
-        .with_context(|_| ExternalSnafu)
-        .unwrap_err()
+/// return a function to convert `crate::error::Error` to `common_meta::error::Error`
+fn to_meta_err(
+    location: snafu::Location,
+) -> impl FnOnce(crate::error::Error) -> common_meta::error::Error {
+    move |err: crate::error::Error| -> common_meta::error::Error {
+        common_meta::error::Error::External {
+            location,
+            source: BoxedError::new(err),
+        }
+    }
 }
 
 #[async_trait::async_trait]
@@ -75,11 +79,16 @@ impl Flownode for FlowWorkerManager {
                     or_replace,
                     expire_after,
                     comment: Some(comment),
-                    sql,
+                    sql: sql.clone(),
                     flow_options,
                     query_ctx,
                 };
-                let ret = self.create_flow(args).await.map_err(to_meta_err)?;
+                let ret = self
+                    .create_flow(args)
+                    .await
+                    .map_err(BoxedError::new)
+                    .with_context(|_| CreateFlowSnafu { sql: sql.clone() })
+                    .map_err(to_meta_err(snafu::location!()))?;
                 METRIC_FLOW_TASK_COUNT.inc();
                 Ok(FlowResponse {
                     affected_flows: ret
@@ -94,7 +103,7 @@ impl Flownode for FlowWorkerManager {
             })) => {
                 self.remove_flow(flow_id.id as u64)
                     .await
-                    .map_err(to_meta_err)?;
+                    .map_err(to_meta_err(snafu::location!()))?;
                 METRIC_FLOW_TASK_COUNT.dec();
                 Ok(Default::default())
             }
@@ -112,9 +121,15 @@ impl Flownode for FlowWorkerManager {
                     .await
                     .flush_all_sender()
                     .await
-                    .map_err(to_meta_err)?;
-                let rows_send = self.run_available(true).await.map_err(to_meta_err)?;
-                let row = self.send_writeback_requests().await.map_err(to_meta_err)?;
+                    .map_err(to_meta_err(snafu::location!()))?;
+                let rows_send = self
+                    .run_available(true)
+                    .await
+                    .map_err(to_meta_err(snafu::location!()))?;
+                let row = self
+                    .send_writeback_requests()
+                    .await
+                    .map_err(to_meta_err(snafu::location!()))?;
 
                 debug!(
                     "Done to flush flow_id={:?} with {} input rows flushed, {} rows sended and {} output rows flushed",
@@ -154,17 +169,23 @@ impl Flownode for FlowWorkerManager {
             // TODO(discord9): reconsider time assignment mechanism
             let now = self.tick_manager.tick();
 
-            let fetch_order = {
+            let (table_types, fetch_order) = {
                 let ctx = self.node_context.read().await;
-                let table_col_names = ctx
-                    .table_repr
-                    .get_by_table_id(&table_id)
-                    .map(|r| r.1)
-                    .and_then(|id| ctx.schema.get(&id))
-                    .map(|desc| &desc.names)
-                    .context(UnexpectedSnafu {
-                        err_msg: format!("Table not found: {}", table_id),
-                    })?;
+
+                // TODO(discord9): also check schema version so that altered table can be reported
+                let table_schema = ctx
+                    .table_source
+                    .table_from_id(&table_id)
+                    .await
+                    .map_err(to_meta_err(snafu::location!()))?;
+                let table_types = table_schema
+                    .typ()
+                    .column_types
+                    .clone()
+                    .into_iter()
+                    .map(|t| t.scalar_type)
+                    .collect_vec();
+                let table_col_names = table_schema.names;
                 let table_col_names = table_col_names
                     .iter().enumerate()
                     .map(|(idx,name)| match name {
@@ -183,16 +204,19 @@ impl Flownode for FlowWorkerManager {
                 );
                 let fetch_order: Vec<usize> = table_col_names
                     .iter()
-                    .map(|names| {
-                        name_to_col.get(names).copied().context(UnexpectedSnafu {
-                            err_msg: format!("Column not found: {}", names),
-                        })
+                    .map(|col_name| {
+                        name_to_col
+                            .get(col_name)
+                            .copied()
+                            .with_context(|| UnexpectedSnafu {
+                                err_msg: format!("Column not found: {}", col_name),
+                            })
                     })
                     .try_collect()?;
                 if !fetch_order.iter().enumerate().all(|(i, &v)| i == v) {
                     trace!("Reordering columns: {:?}", fetch_order)
                 }
-                fetch_order
+                (table_types, fetch_order)
             };
 
             let rows: Vec<DiffRow> = rows_proto
@@ -207,17 +231,29 @@ impl Flownode for FlowWorkerManager {
                 })
                 .map(|r| (r, now, 1))
                 .collect_vec();
-            let batch_datatypes = insert_schema
-                .iter()
-                .map(from_proto_to_data_type)
-                .collect::<std::result::Result<Vec<_>, _>>()
-                .map_err(to_meta_err)?;
-            self.handle_write_request(region_id.into(), rows, &batch_datatypes)
+            if let Err(err) = self
+                .handle_write_request(region_id.into(), rows, &table_types)
                 .await
-                .map_err(|err| {
-                    common_telemetry::error!(err;"Failed to handle write request");
-                    to_meta_err(err)
-                })?;
+            {
+                let err = BoxedError::new(err);
+                let flow_ids = self
+                    .node_context
+                    .read()
+                    .await
+                    .get_flow_ids(table_id)
+                    .into_iter()
+                    .flatten()
+                    .cloned()
+                    .collect_vec();
+                let err = InsertIntoFlowSnafu {
+                    region_id,
+                    flow_ids,
+                }
+                .into_error(err);
+                common_telemetry::error!(err; "Failed to handle write request");
+                let err = to_meta_err(snafu::location!())(err);
+                return Err(err);
+            }
         }
         Ok(Default::default())
     }

src/flow/src/adapter/node_context.rs

@@ -18,6 +18,7 @@ use std::collections::{BTreeMap, BTreeSet, HashMap};
 use std::sync::atomic::{AtomicUsize, Ordering};
 use std::sync::Arc;
 
+use common_recordbatch::RecordBatch;
 use common_telemetry::trace;
 use datatypes::prelude::ConcreteDataType;
 use session::context::QueryContext;
@@ -25,20 +26,23 @@ use snafu::{OptionExt, ResultExt};
 use table::metadata::TableId;
 use tokio::sync::{broadcast, mpsc, RwLock};
 
-use crate::adapter::{FlowId, TableName, TableSource};
+use crate::adapter::table_source::FlowTableSource;
+use crate::adapter::{FlowId, ManagedTableSource, TableName};
 use crate::error::{Error, EvalSnafu, TableNotFoundSnafu};
 use crate::expr::error::InternalSnafu;
 use crate::expr::{Batch, GlobalId};
 use crate::metrics::METRIC_FLOW_INPUT_BUF_SIZE;
+use crate::plan::TypedPlan;
 use crate::repr::{DiffRow, RelationDesc, BATCH_SIZE, BROADCAST_CAP, SEND_BUF_CAP};
 
 /// A context that holds the information of the dataflow
-#[derive(Default, Debug)]
+#[derive(Debug)]
 pub struct FlownodeContext {
     /// mapping from source table to tasks, useful for schedule which task to run when a source table is updated
     pub source_to_tasks: BTreeMap<TableId, BTreeSet<FlowId>>,
     /// mapping from task to sink table, useful for sending data back to the client when a task is done running
     pub flow_to_sink: BTreeMap<FlowId, TableName>,
+    pub flow_plans: BTreeMap<FlowId, TypedPlan>,
     pub sink_to_flow: BTreeMap<TableName, FlowId>,
     /// broadcast sender for source table, any incoming write request will be sent to the source table's corresponding sender
     ///
@@ -50,13 +54,33 @@ pub struct FlownodeContext {
     /// note that the sink receiver should only have one, and we are using broadcast as mpsc channel here
     pub sink_receiver:
         BTreeMap<TableName, (mpsc::UnboundedSender<Batch>, mpsc::UnboundedReceiver<Batch>)>,
-    /// the schema of the table, query from metasrv or inferred from TypedPlan
-    pub schema: HashMap<GlobalId, RelationDesc>,
+    /// can query the schema of the table source, from metasrv with local cache
+    pub table_source: Box<dyn FlowTableSource>,
     /// All the tables that have been registered in the worker
     pub table_repr: IdToNameMap,
     pub query_context: Option<Arc<QueryContext>>,
 }
 
+impl FlownodeContext {
+    pub fn new(table_source: Box<dyn FlowTableSource>) -> Self {
+        Self {
+            source_to_tasks: Default::default(),
+            flow_to_sink: Default::default(),
+            flow_plans: Default::default(),
+            sink_to_flow: Default::default(),
+            source_sender: Default::default(),
+            sink_receiver: Default::default(),
+            table_source,
+            table_repr: Default::default(),
+            query_context: Default::default(),
+        }
+    }
+
+    pub fn get_flow_ids(&self, table_id: TableId) -> Option<&BTreeSet<FlowId>> {
+        self.source_to_tasks.get(&table_id)
+    }
+}
+
 /// a simple broadcast sender with backpressure, bounded capacity and blocking on send when send buf is full
 /// note that it wouldn't evict old data, so it's possible to block forever if the receiver is slow
 ///
@@ -159,6 +183,22 @@ impl SourceSender {
 
         Ok(0)
     }
+
+    /// send record batch
+    pub async fn send_record_batch(&self, batch: RecordBatch) -> Result<usize, Error> {
+        let row_cnt = batch.num_rows();
+        let batch = Batch::from(batch);
+
+        self.send_buf_row_cnt.fetch_add(row_cnt, Ordering::SeqCst);
+
+        self.send_buf_tx.send(batch).await.map_err(|e| {
+            crate::error::InternalSnafu {
+                reason: format!("Failed to send batch, error = {:?}", e),
+            }
+            .build()
+        })?;
+        Ok(row_cnt)
+    }
 }
 
 impl FlownodeContext {
@@ -180,6 +220,16 @@ impl FlownodeContext {
         sender.send_rows(rows, batch_datatypes).await
     }
 
+    pub async fn send_rb(&self, table_id: TableId, batch: RecordBatch) -> Result<usize, Error> {
+        let sender = self
+            .source_sender
+            .get(&table_id)
+            .with_context(|| TableNotFoundSnafu {
+                name: table_id.to_string(),
+            })?;
+        sender.send_record_batch(batch).await
+    }
+
     /// flush all sender's buf
     ///
     /// return numbers being sent
@@ -215,6 +265,15 @@ impl FlownodeContext {
         self.sink_to_flow.insert(sink_table_name, task_id);
     }
 
+    /// add flow plan to worker context
+    pub fn add_flow_plan(&mut self, task_id: FlowId, plan: TypedPlan) {
+        self.flow_plans.insert(task_id, plan);
+    }
+
+    pub fn get_flow_plan(&self, task_id: &FlowId) -> Option<TypedPlan> {
+        self.flow_plans.get(task_id).cloned()
+    }
+
     /// remove flow from worker context
     pub fn remove_flow(&mut self, task_id: FlowId) {
         if let Some(sink_table_name) = self.flow_to_sink.remove(&task_id) {
@@ -226,6 +285,7 @@ impl FlownodeContext {
                 self.source_sender.remove(source_table_id);
             }
         }
+        self.flow_plans.remove(&task_id);
     }
 
     /// try add source sender, if already exist, do nothing
@@ -284,7 +344,7 @@ impl FlownodeContext {
     /// Retrieves a GlobalId and table schema representing a table previously registered by calling the [register_table] function.
     ///
     /// Returns an error if no table has been registered with the provided names
-    pub fn table(&self, name: &TableName) -> Result<(GlobalId, RelationDesc), Error> {
+    pub async fn table(&self, name: &TableName) -> Result<(GlobalId, RelationDesc), Error> {
         let id = self
             .table_repr
             .get_by_name(name)
@@ -292,13 +352,7 @@ impl FlownodeContext {
             .with_context(|| TableNotFoundSnafu {
                 name: name.join("."),
             })?;
-        let schema = self
-            .schema
-            .get(&id)
-            .cloned()
-            .with_context(|| TableNotFoundSnafu {
-                name: name.join("."),
-            })?;
+        let schema = self.table_source.table(name).await?;
         Ok((id, schema))
     }
 
@@ -312,7 +366,7 @@ impl FlownodeContext {
     /// merely creating a mapping from table id to global id
     pub async fn assign_global_id_to_table(
         &mut self,
-        srv_map: &TableSource,
+        srv_map: &ManagedTableSource,
         mut table_name: Option<TableName>,
         table_id: Option<TableId>,
     ) -> Result<GlobalId, Error> {
@@ -333,9 +387,8 @@ impl FlownodeContext {
 
             // table id is Some meaning db must have created the table
             if let Some(table_id) = table_id {
-                let (known_table_name, schema) = srv_map.get_table_name_schema(&table_id).await?;
+                let known_table_name = srv_map.get_table_name(&table_id).await?;
                 table_name = table_name.or(Some(known_table_name));
-                self.schema.insert(global_id, schema);
             } // if we don't have table id, it means database haven't assign one yet or we don't need it
 
             // still update the mapping with new global id
@@ -344,26 +397,6 @@ impl FlownodeContext {
         }
     }
 
-    /// Assign a schema to a table
-    ///
-    pub fn assign_table_schema(
-        &mut self,
-        table_name: &TableName,
-        schema: RelationDesc,
-    ) -> Result<(), Error> {
-        let gid = self
-            .table_repr
-            .get_by_name(table_name)
-            .map(|(_, gid)| gid)
-            .context(TableNotFoundSnafu {
-                name: format!("Table not found: {:?} in flownode cache", table_name),
-            })?;
-
-        self.schema.insert(gid, schema);
-
-        Ok(())
-    }
-
     /// Get a new global id
     pub fn new_global_id(&self) -> GlobalId {
         GlobalId::User(self.table_repr.global_id_to_name_id.len() as u64)

src/flow/src/adapter/table_source.rs

@@ -27,21 +27,86 @@ use crate::error::{
 };
 use crate::repr::RelationDesc;
 
-/// mapping of table name <-> table id should be query from tableinfo manager
-pub struct TableSource {
+/// Table source but for flow, provide table schema by table name/id
+#[async_trait::async_trait]
+pub trait FlowTableSource: Send + Sync + std::fmt::Debug {
+    async fn table_name_from_id(&self, table_id: &TableId) -> Result<TableName, Error>;
+    async fn table_id_from_name(&self, name: &TableName) -> Result<TableId, Error>;
+
+    /// Get the table schema by table name
+    async fn table(&self, name: &TableName) -> Result<RelationDesc, Error> {
+        let id = self.table_id_from_name(name).await?;
+        self.table_from_id(&id).await
+    }
+    async fn table_from_id(&self, table_id: &TableId) -> Result<RelationDesc, Error>;
+}
+
+/// managed table source information, query from table info manager and table name manager
+#[derive(Clone)]
+pub struct ManagedTableSource {
     /// for query `TableId -> TableName` mapping
     table_info_manager: TableInfoManager,
     table_name_manager: TableNameManager,
 }
 
-impl TableSource {
+#[async_trait::async_trait]
+impl FlowTableSource for ManagedTableSource {
+    async fn table_from_id(&self, table_id: &TableId) -> Result<RelationDesc, Error> {
+        let table_info_value = self
+            .get_table_info_value(table_id)
+            .await?
+            .with_context(|| TableNotFoundSnafu {
+                name: format!("TableId = {:?}, Can't found table info", table_id),
+            })?;
+        let desc = table_info_value_to_relation_desc(table_info_value)?;
+
+        Ok(desc)
+    }
+    async fn table_name_from_id(&self, table_id: &TableId) -> Result<TableName, Error> {
+        self.get_table_name(table_id).await
+    }
+    async fn table_id_from_name(&self, name: &TableName) -> Result<TableId, Error> {
+        self.get_opt_table_id_from_name(name)
+            .await?
+            .with_context(|| TableNotFoundSnafu {
+                name: name.join("."),
+            })
+    }
+}
+
+impl ManagedTableSource {
     pub fn new(table_info_manager: TableInfoManager, table_name_manager: TableNameManager) -> Self {
-        TableSource {
+        ManagedTableSource {
             table_info_manager,
             table_name_manager,
         }
     }
 
+    /// Get the time index column from table id
+    pub async fn get_time_index_column_from_table_id(
+        &self,
+        table_id: TableId,
+    ) -> Result<(usize, datatypes::schema::ColumnSchema), Error> {
+        let info = self
+            .table_info_manager
+            .get(table_id)
+            .await
+            .map_err(BoxedError::new)
+            .context(ExternalSnafu)?
+            .context(UnexpectedSnafu {
+                reason: format!("Table id = {:?}, couldn't found table info", table_id),
+            })?;
+        let raw_schema = &info.table_info.meta.schema;
+        let Some(ts_index) = raw_schema.timestamp_index else {
+            UnexpectedSnafu {
+                reason: format!("Table id = {:?}, couldn't found timestamp index", table_id),
+            }
+            .fail()?
+        };
+        let col_schema = raw_schema.column_schemas[ts_index].clone();
+        Ok((ts_index, col_schema))
+    }
+
     pub async fn get_table_id_from_proto_name(
         &self,
         name: &greptime_proto::v1::TableName,
@@ -63,7 +128,10 @@ impl TableSource {
     }
 
     /// If the table haven't been created in database, the tableId returned would be null
-    pub async fn get_table_id_from_name(&self, name: &TableName) -> Result<Option<TableId>, Error> {
+    pub async fn get_opt_table_id_from_name(
+        &self,
+        name: &TableName,
+    ) -> Result<Option<TableId>, Error> {
         let ret = self
             .table_name_manager
             .get(TableNameKey::new(&name[0], &name[1], &name[2]))
@@ -125,4 +193,126 @@ impl TableSource {
         let desc = table_info_value_to_relation_desc(table_info_value)?;
         Ok((table_name, desc))
     }
+
+    pub async fn check_table_exist(&self, table_id: &TableId) -> Result<bool, Error> {
+        self.table_info_manager
+            .exists(*table_id)
+            .await
+            .map_err(BoxedError::new)
+            .context(ExternalSnafu)
+    }
+}
+
+impl std::fmt::Debug for ManagedTableSource {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("KvBackendTableSource").finish()
+    }
+}
+
+#[cfg(test)]
+pub(crate) mod test {
+    use std::collections::HashMap;
+
+    use datatypes::data_type::ConcreteDataType as CDT;
+
+    use super::*;
+    use crate::repr::{ColumnType, RelationType};
+
+    pub struct FlowDummyTableSource {
+        pub id_names_to_desc: Vec<(TableId, TableName, RelationDesc)>,
+        id_to_idx: HashMap<TableId, usize>,
+        name_to_idx: HashMap<TableName, usize>,
+    }
+
+    impl Default for FlowDummyTableSource {
+        fn default() -> Self {
+            let id_names_to_desc = vec![
+                (
+                    1024,
+                    [
+                        "greptime".to_string(),
+                        "public".to_string(),
+                        "numbers".to_string(),
+                    ],
+                    RelationType::new(vec![ColumnType::new(CDT::uint32_datatype(), false)])
+                        .into_named(vec![Some("number".to_string())]),
+                ),
+                (
+                    1025,
+                    [
+                        "greptime".to_string(),
+                        "public".to_string(),
+                        "numbers_with_ts".to_string(),
+                    ],
+                    RelationType::new(vec![
+                        ColumnType::new(CDT::uint32_datatype(), false),
+                        ColumnType::new(CDT::timestamp_millisecond_datatype(), false),
+                    ])
+                    .into_named(vec![Some("number".to_string()), Some("ts".to_string())]),
+                ),
+            ];
+            let id_to_idx = id_names_to_desc
+                .iter()
+                .enumerate()
+                .map(|(idx, (id, _name, _desc))| (*id, idx))
+                .collect();
+            let name_to_idx = id_names_to_desc
+                .iter()
+                .enumerate()
+                .map(|(idx, (_id, name, _desc))| (name.clone(), idx))
+                .collect();
+            Self {
+                id_names_to_desc,
+                id_to_idx,
+                name_to_idx,
+            }
+        }
+    }
+
+    #[async_trait::async_trait]
+    impl FlowTableSource for FlowDummyTableSource {
+        async fn table_from_id(&self, table_id: &TableId) -> Result<RelationDesc, Error> {
+            let idx = self.id_to_idx.get(table_id).context(TableNotFoundSnafu {
+                name: format!("Table id = {:?}, couldn't found table desc", table_id),
+            })?;
+            let desc = self
+                .id_names_to_desc
+                .get(*idx)
+                .map(|x| x.2.clone())
+                .context(TableNotFoundSnafu {
+                    name: format!("Table id = {:?}, couldn't found table desc", table_id),
+                })?;
+            Ok(desc)
+        }
+
+        async fn table_name_from_id(&self, table_id: &TableId) -> Result<TableName, Error> {
+            let idx = self.id_to_idx.get(table_id).context(TableNotFoundSnafu {
+                name: format!("Table id = {:?}, couldn't found table desc", table_id),
+            })?;
+            self.id_names_to_desc
+                .get(*idx)
+                .map(|x| x.1.clone())
+                .context(TableNotFoundSnafu {
+                    name: format!("Table id = {:?}, couldn't found table desc", table_id),
+                })
+        }
+
+        async fn table_id_from_name(&self, name: &TableName) -> Result<TableId, Error> {
+            for (id, table_name, _desc) in &self.id_names_to_desc {
+                if name == table_name {
+                    return Ok(*id);
+                }
+            }
+            TableNotFoundSnafu {
+                name: format!("Table name = {:?}, couldn't found table id", name),
+            }
+            .fail()?
+        }
+    }
+
+    impl std::fmt::Debug for FlowDummyTableSource {
+        fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+            f.debug_struct("DummyTableSource").finish()
+        }
+    }
 }

src/flow/src/adapter/worker.rs

@@ -247,15 +247,25 @@ impl<'s> Worker<'s> {
         src_recvs: Vec<broadcast::Receiver<Batch>>,
         // TODO(discord9): set expire duration for all arrangement and compare to sys timestamp instead
         expire_after: Option<repr::Duration>,
+        or_replace: bool,
         create_if_not_exists: bool,
         err_collector: ErrCollector,
     ) -> Result<Option<FlowId>, Error> {
-        let already_exists = self.task_states.contains_key(&flow_id);
-        match (already_exists, create_if_not_exists) {
-            (true, true) => return Ok(None),
-            (true, false) => FlowAlreadyExistSnafu { id: flow_id }.fail()?,
-            (false, _) => (),
-        };
+        let already_exist = self.task_states.contains_key(&flow_id);
+        match (create_if_not_exists, or_replace, already_exist) {
+            // if replace, ignore that old flow exists
+            (_, true, true) => {
+                info!("Replacing flow with id={}", flow_id);
+            }
+            (false, false, true) => FlowAlreadyExistSnafu { id: flow_id }.fail()?,
+            // already exists, and not replace, return None
+            (true, false, true) => {
+                info!("Flow with id={} already exists, do nothing", flow_id);
+                return Ok(None);
+            }
+            // continue as normal
+            (_, _, false) => (),
+        }
 
         let mut cur_task_state = ActiveDataflowState::<'s> {
             err_collector,
@@ -341,6 +351,7 @@ impl<'s> Worker<'s> {
                 source_ids,
                 src_recvs,
                 expire_after,
+                or_replace,
                 create_if_not_exists,
                 err_collector,
             } => {
@@ -352,6 +363,7 @@ impl<'s> Worker<'s> {
                     &source_ids,
                     src_recvs,
                     expire_after,
+                    or_replace,
                     create_if_not_exists,
                     err_collector,
                 );
@@ -398,6 +410,7 @@ pub enum Request {
         source_ids: Vec<GlobalId>,
         src_recvs: Vec<broadcast::Receiver<Batch>>,
         expire_after: Option<repr::Duration>,
+        or_replace: bool,
         create_if_not_exists: bool,
         err_collector: ErrCollector,
     },
@@ -547,6 +560,7 @@ mod test {
             source_ids: src_ids,
             src_recvs: vec![rx],
             expire_after: None,
+            or_replace: false,
             create_if_not_exists: true,
             err_collector: ErrCollector::default(),
         };

src/flow/src/error.rs

@@ -21,7 +21,7 @@ use common_error::{define_into_tonic_status, from_err_code_msg_to_header};
 use common_macro::stack_trace_debug;
 use common_telemetry::common_error::ext::ErrorExt;
 use common_telemetry::common_error::status_code::StatusCode;
-use snafu::{Location, Snafu};
+use snafu::{Location, ResultExt, Snafu};
 use tonic::metadata::MetadataMap;
 
 use crate::adapter::FlowId;
@@ -32,6 +32,27 @@ use crate::expr::EvalError;
 #[snafu(visibility(pub))]
 #[stack_trace_debug]
 pub enum Error {
+    #[snafu(display(
+        "Failed to insert into flow: region_id={}, flow_ids={:?}",
+        region_id,
+        flow_ids
+    ))]
+    InsertIntoFlow {
+        region_id: u64,
+        flow_ids: Vec<u64>,
+        source: BoxedError,
+        #[snafu(implicit)]
+        location: Location,
+    },
+
+    #[snafu(display("Error encountered while creating flow: {sql}"))]
+    CreateFlow {
+        sql: String,
+        source: BoxedError,
+        #[snafu(implicit)]
+        location: Location,
+    },
+
     #[snafu(display("External error"))]
     External {
         source: BoxedError,
@@ -207,16 +228,17 @@ pub type Result<T> = std::result::Result<T, Error>;
 impl ErrorExt for Error {
     fn status_code(&self) -> StatusCode {
         match self {
-            Self::Eval { .. } | Self::JoinTask { .. } | Self::Datafusion { .. } => {
-                StatusCode::Internal
-            }
+            Self::Eval { .. }
+            | Self::JoinTask { .. }
+            | Self::Datafusion { .. }
+            | Self::InsertIntoFlow { .. } => StatusCode::Internal,
             Self::FlowAlreadyExist { .. } => StatusCode::TableAlreadyExists,
             Self::TableNotFound { .. }
             | Self::TableNotFoundMeta { .. }
             | Self::FlowNotFound { .. }
             | Self::ListFlows { .. } => StatusCode::TableNotFound,
             Self::Plan { .. } | Self::Datatypes { .. } => StatusCode::PlanQuery,
-            Self::InvalidQuery { .. } => StatusCode::EngineExecuteQuery,
+            Self::InvalidQuery { .. } | Self::CreateFlow { .. } => StatusCode::EngineExecuteQuery,
             Self::Unexpected { .. } => StatusCode::Unexpected,
             Self::NotImplemented { .. } | Self::UnsupportedTemporalFilter { .. } => {
                 StatusCode::Unsupported
@@ -237,3 +259,9 @@ impl ErrorExt for Error {
 }
 
 define_into_tonic_status!(Error);
+
+impl From<EvalError> for Error {
+    fn from(e: EvalError) -> Self {
+        Err::<(), _>(e).context(EvalSnafu).unwrap_err()
+    }
+}

src/flow/src/expr.rs

@@ -22,6 +22,7 @@ mod linear;
 pub(crate) mod relation;
 mod scalar;
 mod signature;
+pub(crate) mod utils;
 
 use arrow::compute::FilterBuilder;
 use datatypes::prelude::{ConcreteDataType, DataType};
@@ -54,6 +55,16 @@ pub struct Batch {
     diffs: Option<VectorRef>,
 }
 
+impl From<common_recordbatch::RecordBatch> for Batch {
+    fn from(value: common_recordbatch::RecordBatch) -> Self {
+        Self {
+            row_count: value.num_rows(),
+            batch: value.columns,
+            diffs: None,
+        }
+    }
+}
+
 impl PartialEq for Batch {
     fn eq(&self, other: &Self) -> bool {
         let mut batch_eq = true;

src/flow/src/expr/linear.rs

@@ -94,6 +94,30 @@ impl MapFilterProject {
         }
     }
 
+    pub fn get_nth_expr(&self, n: usize) -> Option<ScalarExpr> {
+        let idx = *self.projection.get(n)?;
+        if idx < self.input_arity {
+            Some(ScalarExpr::Column(idx))
+        } else {
+            // find direct ref to input's expr
+
+            let mut expr = self.expressions.get(idx - self.input_arity)?;
+            loop {
+                match expr {
+                    ScalarExpr::Column(prev) => {
+                        if *prev < self.input_arity {
+                            return Some(ScalarExpr::Column(*prev));
+                        } else {
+                            expr = self.expressions.get(*prev - self.input_arity)?;
+                            continue;
+                        }
+                    }
+                    _ => return Some(expr.clone()),
+                }
+            }
+        }
+    }
+
     /// The number of columns expected in the output row.
     pub fn output_arity(&self) -> usize {
         self.projection.len()

src/flow/src/expr/scalar.rs

@@ -311,6 +311,9 @@ impl ScalarExpr {
     }
 
     /// Eval this expression with the given values.
+    ///
+    /// TODO(discord9): add tests to make sure `eval_batch` is the same as `eval` in
+    /// most cases
     pub fn eval(&self, values: &[Value]) -> Result<Value, EvalError> {
         match self {
             ScalarExpr::Column(index) => Ok(values[*index].clone()),

src/flow/src/expr/utils.rs

@@ -0,0 +1,340 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! This module contains utility functions for expressions.
+
+use std::cmp::Ordering;
+use std::collections::BTreeMap;
+
+use datatypes::value::Value;
+use snafu::{ensure, OptionExt};
+
+use crate::error::UnexpectedSnafu;
+use crate::expr::ScalarExpr;
+use crate::plan::TypedPlan;
+use crate::Result;
+
+/// Find lower bound for time `current` in given `plan` for the time window expr.
+///
+/// i.e. for time window expr being `date_bin(INTERVAL '5 minutes', ts) as time_window` and `current="2021-07-01 00:01:01.000"`,
+/// return `Some("2021-07-01 00:00:00.000")`
+///
+/// if `plan` doesn't contain a `TIME INDEX` column, return `None`
+pub fn find_plan_time_window_expr_lower_bound(
+    plan: &TypedPlan,
+    current: common_time::Timestamp,
+) -> Result<Option<common_time::Timestamp>> {
+    let typ = plan.schema.typ();
+    let Some(mut time_index) = typ.time_index else {
+        return Ok(None);
+    };
+
+    let mut cur_plan = plan;
+    let mut expr_time_index;
+
+    loop {
+        // follow upward and find deepest time index expr that is not a column ref
+        expr_time_index = Some(cur_plan.plan.get_nth_expr(time_index).context(
+            UnexpectedSnafu {
+                reason: "Failed to find time index expr",
+            },
+        )?);
+
+        if let Some(ScalarExpr::Column(i)) = expr_time_index {
+            time_index = i;
+        } else {
+            break;
+        }
+        if let Some(input) = cur_plan.plan.get_first_input_plan() {
+            cur_plan = input;
+        } else {
+            break;
+        }
+    }
+
+    let expr_time_index = expr_time_index.context(UnexpectedSnafu {
+        reason: "Failed to find time index expr",
+    })?;
+
+    let ts_col = expr_time_index
+        .get_all_ref_columns()
+        .first()
+        .cloned()
+        .context(UnexpectedSnafu {
+            reason: "Failed to find time index column",
+        })?;
+
+    find_time_window_lower_bound(&expr_time_index, ts_col, current)
+}
+
+/// Find the lower bound of time window in given `expr` and `current` timestamp.
+///
+/// i.e. for `current="2021-07-01 00:01:01.000"` and `expr=date_bin(INTERVAL '5 minutes', ts) as time_window` and `ts_col=ts`,
+/// return `Some("2021-07-01 00:00:00.000")` since it's the lower bound
+/// of current time window given the current timestamp
+///
+/// if return None, meaning this time window have no lower bound
+pub fn find_time_window_lower_bound(
+    expr: &ScalarExpr,
+    ts_col_idx: usize,
+    current: common_time::Timestamp,
+) -> Result<Option<common_time::Timestamp>> {
+    let all_ref_columns = expr.get_all_ref_columns();
+
+    ensure!(
+        all_ref_columns.contains(&ts_col_idx),
+        UnexpectedSnafu {
+            reason: format!(
+                "Expected column {} to be referenced in expression {expr:?}",
+                ts_col_idx
+            ),
+        }
+    );
+
+    ensure!(all_ref_columns.len() == 1, UnexpectedSnafu {
+        reason: format!(
+            "Expect only one column to be referenced in expression {expr:?}, found {all_ref_columns:?}"
+        ),
+    });
+
+    let permute_map = BTreeMap::from([(ts_col_idx, 0usize)]);
+
+    let mut rewrote_expr = expr.clone();
+
+    rewrote_expr.permute_map(&permute_map)?;
+
+    fn eval_to_timestamp(expr: &ScalarExpr, values: &[Value]) -> Result<common_time::Timestamp> {
+        let val = expr.eval(values)?;
+        if let Value::Timestamp(ts) = val {
+            Ok(ts)
+        } else {
+            UnexpectedSnafu {
+                reason: format!("Expected timestamp in expression {expr:?} but got {val:?}"),
+            }
+            .fail()?
+        }
+    }
+
+    let cur_time_window = eval_to_timestamp(&rewrote_expr, &[current.into()])?;
+
+    // search to find the lower bound
+    let mut offset: i64 = 1;
+    let lower_bound;
+    let mut upper_bound = Some(current);
+    // first expontial probe to found a range for binary search
+    loop {
+        let Some(next_val) = current.value().checked_sub(offset) else {
+            // no lower bound
+            return Ok(None);
+        };
+
+        let prev_time_probe = common_time::Timestamp::new(next_val, current.unit());
+
+        let prev_time_window = eval_to_timestamp(&rewrote_expr, &[prev_time_probe.into()])?;
+
+        match prev_time_window.cmp(&cur_time_window) {
+            Ordering::Less => {
+                lower_bound = Some(prev_time_probe);
+                break;
+            }
+            Ordering::Equal => {
+                upper_bound = Some(prev_time_probe);
+            }
+            Ordering::Greater => {
+                UnexpectedSnafu {
+                    reason: format!(
+                        "Unsupported time window expression {rewrote_expr:?}, expect monotonic increasing for time window expression {expr:?}"
+                    ),
+                }
+                .fail()?
+            }
+        }
+
+        let Some(new_offset) = offset.checked_mul(2) else {
+            // no lower bound
+            return Ok(None);
+        };
+        offset = new_offset;
+    }
+
+    // binary search for the lower bound
+
+    ensure!(lower_bound.map(|v|v.unit())==upper_bound.map(|v|v.unit()), UnexpectedSnafu{
+        reason: format!(" unit mismatch for time window expression {expr:?}, found {lower_bound:?} and {upper_bound:?}"),
+    });
+
+    let output_unit = lower_bound.expect("should have lower bound").unit();
+
+    let mut low = lower_bound.expect("should have lower bound").value();
+    let mut high = upper_bound.expect("should have upper bound").value();
+    while low < high {
+        let mid = (low + high) / 2;
+        let mid_probe = common_time::Timestamp::new(mid, output_unit);
+        let mid_time_window = eval_to_timestamp(&rewrote_expr, &[mid_probe.into()])?;
+
+        match mid_time_window.cmp(&cur_time_window) {
+            Ordering::Less => low = mid + 1,
+            Ordering::Equal => high = mid,
+            Ordering::Greater => UnexpectedSnafu {
+                reason: format!("Binary search failed for time window expression {expr:?}"),
+            }
+            .fail()?,
+        }
+    }
+
+    let final_lower_bound_for_time_window = common_time::Timestamp::new(low, output_unit);
+
+    Ok(Some(final_lower_bound_for_time_window))
+}
+
+#[cfg(test)]
+mod test {
+    use pretty_assertions::assert_eq;
+
+    use super::*;
+    use crate::plan::{Plan, TypedPlan};
+    use crate::test_utils::{create_test_ctx, create_test_query_engine, sql_to_substrait};
+
+    #[tokio::test]
+    async fn test_plan_time_window_lower_bound() {
+        let testcases = [
+            // no time index
+            (
+                "SELECT date_bin('5 minutes', ts) FROM numbers_with_ts;",
+                "2021-07-01 00:01:01.000",
+                None,
+            ),
+            // time index
+            (
+                "SELECT date_bin('5 minutes', ts) as time_window FROM numbers_with_ts GROUP BY time_window;",
+                "2021-07-01 00:01:01.000",
+                Some("2021-07-01 00:00:00.000"),
+            ),
+            // time index with other fields
+            (
+                "SELECT sum(number) as sum_up, date_bin('5 minutes', ts) as time_window FROM numbers_with_ts GROUP BY time_window;",
+                "2021-07-01 00:01:01.000",
+                Some("2021-07-01 00:00:00.000"),
+            ),
+            // time index with other pks
+            (
+                "SELECT number, date_bin('5 minutes', ts) as time_window FROM numbers_with_ts GROUP BY time_window, number;",
+                "2021-07-01 00:01:01.000",
+                Some("2021-07-01 00:00:00.000"),
+            ),
+        ];
+        let engine = create_test_query_engine();
+
+        for (sql, current, expected) in &testcases {
+            let plan = sql_to_substrait(engine.clone(), sql).await;
+            let mut ctx = create_test_ctx();
+            let flow_plan = TypedPlan::from_substrait_plan(&mut ctx, &plan)
+                .await
+                .unwrap();
+
+            let current = common_time::Timestamp::from_str(current, None).unwrap();
+
+            let expected =
+                expected.map(|expected| common_time::Timestamp::from_str(expected, None).unwrap());
+
+            assert_eq!(
+                find_plan_time_window_expr_lower_bound(&flow_plan, current).unwrap(),
+                expected
+            );
+        }
+    }
+
+    #[tokio::test]
+    async fn test_timewindow_lower_bound() {
+        let testcases = [
+            (
+                ("'5 minutes'", "ts", Some("2021-07-01 00:00:00.000")),
+                "2021-07-01 00:01:01.000",
+                "2021-07-01 00:00:00.000",
+            ),
+            (
+                ("'5 minutes'", "ts", None),
+                "2021-07-01 00:01:01.000",
+                "2021-07-01 00:00:00.000",
+            ),
+            (
+                ("'5 minutes'", "ts", None),
+                "2021-07-01 00:00:00.000",
+                "2021-07-01 00:00:00.000",
+            ),
+            // test edge cases
+            (
+                ("'5 minutes'", "ts", None),
+                "2021-07-01 00:05:00.000",
+                "2021-07-01 00:05:00.000",
+            ),
+            (
+                ("'5 minutes'", "ts", None),
+                "2021-07-01 00:04:59.999",
+                "2021-07-01 00:00:00.000",
+            ),
+            (
+                ("'5 minutes'", "ts", None),
+                "2021-07-01 00:04:59.999999999",
+                "2021-07-01 00:00:00.000",
+            ),
+            (
+                ("'5 minutes'", "ts", None),
+                "2021-07-01 00:04:59.999999999999",
+                "2021-07-01 00:00:00.000",
+            ),
+            (
+                ("'5 minutes'", "ts", None),
+                "2021-07-01 00:04:59.999999999999999",
+                "2021-07-01 00:00:00.000",
+            ),
+        ];
+        let engine = create_test_query_engine();
+
+        for (args, current, expected) in testcases {
+            let sql = if let Some(origin) = args.2 {
+                format!(
+                    "SELECT date_bin({}, {}, '{origin}') FROM numbers_with_ts;",
+                    args.0, args.1
+                )
+            } else {
+                format!(
+                    "SELECT date_bin({}, {}) FROM numbers_with_ts;",
+                    args.0, args.1
+                )
+            };
+            let plan = sql_to_substrait(engine.clone(), &sql).await;
+            let mut ctx = create_test_ctx();
+            let flow_plan = TypedPlan::from_substrait_plan(&mut ctx, &plan)
+                .await
+                .unwrap();
+
+            let expr = {
+                let mfp = flow_plan.plan;
+                let Plan::Mfp { mfp, .. } = mfp else {
+                    unreachable!()
+                };
+                mfp.expressions[0].clone()
+            };
+
+            let current = common_time::Timestamp::from_str(current, None).unwrap();
+
+            let res = find_time_window_lower_bound(&expr, 1, current).unwrap();
+
+            let expected = Some(common_time::Timestamp::from_str(expected, None).unwrap());
+
+            assert_eq!(res, expected);
+        }
+    }
+}

src/flow/src/lib.rs

@@ -38,6 +38,9 @@ mod server;
 mod transform;
 mod utils;
 
+#[cfg(test)]
+mod test_utils;
+
 pub use adapter::{FlowWorkerManager, FlowWorkerManagerRef, FlownodeOptions};
 pub use error::{Error, Result};
 pub use server::{FlownodeBuilder, FlownodeInstance, FlownodeServer, FrontendInvoker};

src/flow/src/plan.rs

@@ -18,8 +18,10 @@
 mod join;
 mod reduce;
 
+use std::collections::BTreeSet;
+
 use crate::error::Error;
-use crate::expr::{Id, LocalId, MapFilterProject, SafeMfpPlan, TypedExpr};
+use crate::expr::{GlobalId, Id, LocalId, MapFilterProject, SafeMfpPlan, ScalarExpr, TypedExpr};
 use crate::plan::join::JoinPlan;
 pub(crate) use crate::plan::reduce::{AccumulablePlan, AggrWithIndex, KeyValPlan, ReducePlan};
 use crate::repr::{DiffRow, RelationDesc};
@@ -189,3 +191,78 @@ impl Plan {
         TypedPlan { schema, plan: self }
     }
 }
+
+impl Plan {
+    /// Get nth expr using column ref
+    pub fn get_nth_expr(&self, n: usize) -> Option<ScalarExpr> {
+        match self {
+            Self::Mfp { mfp, .. } => mfp.get_nth_expr(n),
+            Self::Reduce { key_val_plan, .. } => key_val_plan.get_nth_expr(n),
+            _ => None,
+        }
+    }
+
+    /// Get the first input plan if exists
+    pub fn get_first_input_plan(&self) -> Option<&TypedPlan> {
+        match self {
+            Plan::Let { value, .. } => Some(value),
+            Plan::Mfp { input, .. } => Some(input),
+            Plan::Reduce { input, .. } => Some(input),
+            Plan::Join { inputs, .. } => inputs.first(),
+            Plan::Union { inputs, .. } => inputs.first(),
+            _ => None,
+        }
+    }
+
+    /// Get mutable ref to the first input plan if exists
+    pub fn get_mut_first_input_plan(&mut self) -> Option<&mut TypedPlan> {
+        match self {
+            Plan::Let { value, .. } => Some(value),
+            Plan::Mfp { input, .. } => Some(input),
+            Plan::Reduce { input, .. } => Some(input),
+            Plan::Join { inputs, .. } => inputs.first_mut(),
+            Plan::Union { inputs, .. } => inputs.first_mut(),
+            _ => None,
+        }
+    }
+
+    /// Find all the used collection in the plan
+    pub fn find_used_collection(&self) -> BTreeSet<GlobalId> {
+        fn recur_find_use(plan: &Plan, used: &mut BTreeSet<GlobalId>) {
+            match plan {
+                Plan::Get { id } => {
+                    match id {
+                        Id::Local(_) => (),
+                        Id::Global(g) => {
+                            used.insert(*g);
+                        }
+                    };
+                }
+                Plan::Let { value, body, .. } => {
+                    recur_find_use(&value.plan, used);
+                    recur_find_use(&body.plan, used);
+                }
+                Plan::Mfp { input, .. } => {
+                    recur_find_use(&input.plan, used);
+                }
+                Plan::Reduce { input, .. } => {
+                    recur_find_use(&input.plan, used);
+                }
+                Plan::Join { inputs, .. } => {
+                    for input in inputs {
+                        recur_find_use(&input.plan, used);
+                    }
+                }
+                Plan::Union { inputs, .. } => {
+                    for input in inputs {
+                        recur_find_use(&input.plan, used);
+                    }
+                }
+                _ => {}
+            }
+        }
+        let mut ret = Default::default();
+        recur_find_use(self, &mut ret);
+        ret
+    }
+}

src/flow/src/plan/reduce.rs

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use crate::expr::{AggregateExpr, SafeMfpPlan};
+use crate::expr::{AggregateExpr, SafeMfpPlan, ScalarExpr};
 
 /// Describe how to extract key-value pair from a `Row`
 #[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd)]
@@ -23,6 +23,16 @@ pub struct KeyValPlan {
     pub val_plan: SafeMfpPlan,
 }
 
+impl KeyValPlan {
+    /// Get nth expr using column ref
+    pub fn get_nth_expr(&self, n: usize) -> Option<ScalarExpr> {
+        self.key_plan.get_nth_expr(n).or_else(|| {
+            self.val_plan
+                .get_nth_expr(n - self.key_plan.projection.len())
+        })
+    }
+}
+
 /// TODO(discord9): def&impl of Hierarchical aggregates(for min/max with support to deletion) and
 /// basic aggregates(for other aggregate functions) and mixed aggregate
 #[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd)]

src/flow/src/server.rs

@@ -50,8 +50,8 @@ use tonic::{Request, Response, Status};
 
 use crate::adapter::{CreateFlowArgs, FlowWorkerManagerRef};
 use crate::error::{
-    to_status_with_last_err, CacheRequiredSnafu, ExternalSnafu, FlowNotFoundSnafu, ListFlowsSnafu,
-    ParseAddrSnafu, ShutdownServerSnafu, StartServerSnafu, UnexpectedSnafu,
+    to_status_with_last_err, CacheRequiredSnafu, CreateFlowSnafu, ExternalSnafu, FlowNotFoundSnafu,
+    ListFlowsSnafu, ParseAddrSnafu, ShutdownServerSnafu, StartServerSnafu, UnexpectedSnafu,
 };
 use crate::heartbeat::HeartbeatTask;
 use crate::metrics::{METRIC_FLOW_PROCESSING_TIME, METRIC_FLOW_ROWS};
@@ -392,7 +392,13 @@ impl FlownodeBuilder {
                         .build(),
                 ),
             };
-            manager.create_flow(args).await?;
+            manager
+                .create_flow(args)
+                .await
+                .map_err(BoxedError::new)
+                .with_context(|_| CreateFlowSnafu {
+                    sql: info.raw_sql().clone(),
+                })?;
         }
 
         Ok(cnt)
@@ -434,6 +440,7 @@ impl FlownodeBuilder {
     }
 }
 
+#[derive(Clone)]
 pub struct FrontendInvoker {
     inserter: Arc<Inserter>,
     deleter: Arc<Deleter>,
@@ -543,3 +550,42 @@ impl FrontendInvoker {
         self.statement_executor.clone()
     }
 }
+
+/// get all flow ids in this flownode
+pub(crate) async fn get_all_flow_ids(
+    flow_metadata_manager: &FlowMetadataManagerRef,
+    catalog_manager: &CatalogManagerRef,
+    nodeid: Option<u64>,
+) -> Result<Vec<u32>, Error> {
+    let ret = if let Some(nodeid) = nodeid {
+        let flow_ids_one_node = flow_metadata_manager
+            .flownode_flow_manager()
+            .flows(nodeid)
+            .try_collect::<Vec<_>>()
+            .await
+            .context(ListFlowsSnafu { id: Some(nodeid) })?;
+        flow_ids_one_node.into_iter().map(|(id, _)| id).collect()
+    } else {
+        let all_catalogs = catalog_manager
+            .catalog_names()
+            .await
+            .map_err(BoxedError::new)
+            .context(ExternalSnafu)?;
+        let mut all_flow_ids = vec![];
+        for catalog in all_catalogs {
+            let flows = flow_metadata_manager
+                .flow_name_manager()
+                .flow_names(&catalog)
+                .await
+                .try_collect::<Vec<_>>()
+                .await
+                .map_err(BoxedError::new)
+                .context(ExternalSnafu)?;
+
+            all_flow_ids.extend(flows.into_iter().map(|(_, id)| id.flow_id()));
+        }
+        all_flow_ids
+    };
+
+    Ok(ret)
+}

src/flow/src/test_utils.rs

@@ -0,0 +1,142 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::sync::Arc;
+
+use catalog::RegisterTableRequest;
+use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, NUMBERS_TABLE_ID};
+use datatypes::data_type::ConcreteDataType as CDT;
+use datatypes::prelude::*;
+use datatypes::schema::Schema;
+use datatypes::timestamp::TimestampMillisecond;
+use datatypes::vectors::{TimestampMillisecondVectorBuilder, VectorRef};
+use itertools::Itertools;
+use prost::Message;
+use query::parser::QueryLanguageParser;
+use query::query_engine::DefaultSerializer;
+use query::QueryEngine;
+use session::context::QueryContext;
+/// note here we are using the `substrait_proto_df` crate from the `substrait` module and
+/// rename it to `substrait_proto`
+use substrait::substrait_proto_df as substrait_proto;
+use substrait::{DFLogicalSubstraitConvertor, SubstraitPlan};
+use substrait_proto::proto;
+use table::table::numbers::{NumbersTable, NUMBERS_TABLE_NAME};
+use table::test_util::MemTable;
+
+use crate::adapter::node_context::IdToNameMap;
+use crate::adapter::table_source::test::FlowDummyTableSource;
+use crate::adapter::FlownodeContext;
+use crate::df_optimizer::apply_df_optimizer;
+use crate::expr::GlobalId;
+use crate::transform::register_function_to_query_engine;
+
+pub fn create_test_ctx() -> FlownodeContext {
+    let mut tri_map = IdToNameMap::new();
+    {
+        let gid = GlobalId::User(0);
+        let name = [
+            "greptime".to_string(),
+            "public".to_string(),
+            "numbers".to_string(),
+        ];
+        tri_map.insert(Some(name.clone()), Some(1024), gid);
+    }
+
+    {
+        let gid = GlobalId::User(1);
+        let name = [
+            "greptime".to_string(),
+            "public".to_string(),
+            "numbers_with_ts".to_string(),
+        ];
+        tri_map.insert(Some(name.clone()), Some(1025), gid);
+    }
+
+    let dummy_source = FlowDummyTableSource::default();
+
+    let mut ctx = FlownodeContext::new(Box::new(dummy_source));
+    ctx.table_repr = tri_map;
+    ctx.query_context = Some(Arc::new(QueryContext::with("greptime", "public")));
+
+    ctx
+}
+
+pub fn create_test_query_engine() -> Arc<dyn QueryEngine> {
+    let catalog_list = catalog::memory::new_memory_catalog_manager().unwrap();
+    let req = RegisterTableRequest {
+        catalog: DEFAULT_CATALOG_NAME.to_string(),
+        schema: DEFAULT_SCHEMA_NAME.to_string(),
+        table_name: NUMBERS_TABLE_NAME.to_string(),
+        table_id: NUMBERS_TABLE_ID,
+        table: NumbersTable::table(NUMBERS_TABLE_ID),
+    };
+    catalog_list.register_table_sync(req).unwrap();
+
+    let schema = vec![
+        datatypes::schema::ColumnSchema::new("number", CDT::uint32_datatype(), false),
+        datatypes::schema::ColumnSchema::new("ts", CDT::timestamp_millisecond_datatype(), false),
+    ];
+    let mut columns = vec![];
+    let numbers = (1..=10).collect_vec();
+    let column: VectorRef = Arc::new(<u32 as Scalar>::VectorType::from_vec(numbers));
+    columns.push(column);
+
+    let ts = (1..=10).collect_vec();
+    let mut builder = TimestampMillisecondVectorBuilder::with_capacity(10);
+    ts.into_iter()
+        .map(|v| builder.push(Some(TimestampMillisecond::new(v))))
+        .count();
+    let column: VectorRef = builder.to_vector_cloned();
+    columns.push(column);
+
+    let schema = Arc::new(Schema::new(schema));
+    let recordbatch = common_recordbatch::RecordBatch::new(schema, columns).unwrap();
+    let table = MemTable::table("numbers_with_ts", recordbatch);
+
+    let req_with_ts = RegisterTableRequest {
+        catalog: DEFAULT_CATALOG_NAME.to_string(),
+        schema: DEFAULT_SCHEMA_NAME.to_string(),
+        table_name: "numbers_with_ts".to_string(),
+        table_id: 1024,
+        table,
+    };
+    catalog_list.register_table_sync(req_with_ts).unwrap();
+
+    let factory = query::QueryEngineFactory::new(catalog_list, None, None, None, None, false);
+
+    let engine = factory.query_engine();
+    register_function_to_query_engine(&engine);
+
+    assert_eq!("datafusion", engine.name());
+    engine
+}
+
+pub async fn sql_to_substrait(engine: Arc<dyn QueryEngine>, sql: &str) -> proto::Plan {
+    // let engine = create_test_query_engine();
+    let stmt = QueryLanguageParser::parse_sql(sql, &QueryContext::arc()).unwrap();
+    let plan = engine
+        .planner()
+        .plan(&stmt, QueryContext::arc())
+        .await
+        .unwrap();
+    let plan = apply_df_optimizer(plan).await.unwrap();
+
+    // encode then decode so to rely on the impl of conversion from logical plan to substrait plan
+    let bytes = DFLogicalSubstraitConvertor {}
+        .encode(&plan, DefaultSerializer)
+        .unwrap();
+
+    proto::Plan::decode(bytes).unwrap()
+}

src/flow/src/transform.rs

@@ -173,12 +173,11 @@ mod test {
 
     use super::*;
     use crate::adapter::node_context::IdToNameMap;
+    use crate::adapter::table_source::test::FlowDummyTableSource;
     use crate::df_optimizer::apply_df_optimizer;
     use crate::expr::GlobalId;
-    use crate::repr::{ColumnType, RelationType};
 
     pub fn create_test_ctx() -> FlownodeContext {
-        let mut schemas = HashMap::new();
         let mut tri_map = IdToNameMap::new();
         {
             let gid = GlobalId::User(0);
@@ -187,10 +186,7 @@ mod test {
                 "public".to_string(),
                 "numbers".to_string(),
             ];
-            let schema = RelationType::new(vec![ColumnType::new(CDT::uint32_datatype(), false)]);
-
             tri_map.insert(Some(name.clone()), Some(1024), gid);
-            schemas.insert(gid, schema.into_named(vec![Some("number".to_string())]));
         }
 
         {
@@ -200,23 +196,16 @@ mod test {
                 "public".to_string(),
                 "numbers_with_ts".to_string(),
             ];
-            let schema = RelationType::new(vec![
-                ColumnType::new(CDT::uint32_datatype(), false),
-                ColumnType::new(CDT::timestamp_millisecond_datatype(), false),
-            ]);
-            schemas.insert(
-                gid,
-                schema.into_named(vec![Some("number".to_string()), Some("ts".to_string())]),
-            );
             tri_map.insert(Some(name.clone()), Some(1025), gid);
         }
 
-        FlownodeContext {
-            schema: schemas,
-            table_repr: tri_map,
-            query_context: Some(Arc::new(QueryContext::with("greptime", "public"))),
-            ..Default::default()
-        }
+        let dummy_source = FlowDummyTableSource::default();
+
+        let mut ctx = FlownodeContext::new(Box::new(dummy_source));
+        ctx.table_repr = tri_map;
+        ctx.query_context = Some(Arc::new(QueryContext::with("greptime", "public")));
+
+        ctx
     }
 
     pub fn create_test_query_engine() -> Arc<dyn QueryEngine> {

src/flow/src/transform/aggr.rs

@@ -128,7 +128,11 @@ impl AggregateExpr {
         }
 
         if args.len() != 1 {
-            return not_impl_err!("Aggregated function with multiple arguments is not supported");
+            let fn_name = extensions.get(&f.function_reference).cloned();
+            return not_impl_err!(
+                "Aggregated function (name={:?}) with multiple arguments is not supported",
+                fn_name
+            );
         }
 
         let arg = if let Some(first) = args.first() {

src/flow/src/transform/plan.rs

@@ -176,7 +176,7 @@ impl TypedPlan {
                 }
                 .fail()?,
             };
-            let table = ctx.table(&table_reference)?;
+            let table = ctx.table(&table_reference).await?;
             let get_table = Plan::Get {
                 id: crate::expr::Id::Global(table.0),
             };

src/frontend/Cargo.toml

@@ -5,8 +5,6 @@ edition.workspace = true
 license.workspace = true
 
 [features]
-default = ["python"]
-python = ["dep:script"]
 testing = []
 
 [lints]
@@ -52,7 +50,6 @@ prometheus.workspace = true
 prost.workspace = true
 query.workspace = true
 raft-engine.workspace = true
-script = { workspace = true, features = ["python"], optional = true }
 serde.workspace = true
 servers.workspace = true
 session.workspace = true

src/frontend/src/error.rs

@@ -238,14 +238,6 @@ pub enum Error {
         location: Location,
     },
 
-    #[cfg(feature = "python")]
-    #[snafu(display("Failed to start script manager"))]
-    StartScriptManager {
-        #[snafu(implicit)]
-        location: Location,
-        source: script::error::Error,
-    },
-
     #[snafu(display("Failed to insert value into table: {}", table_name))]
     Insert {
         table_name: String,
@@ -394,9 +386,6 @@ impl ErrorExt for Error {
             }
             Error::FindTableRoute { source, .. } => source.status_code(),
 
-            #[cfg(feature = "python")]
-            Error::StartScriptManager { source, .. } => source.status_code(),
-
             Error::TableOperation { source, .. } => source.status_code(),
 
             Error::InFlightWriteBytesExceeded { .. } => StatusCode::RateLimited,

src/frontend/src/instance.rs

@@ -21,7 +21,6 @@ mod opentsdb;
 mod otlp;
 mod prom_store;
 mod region_query;
-mod script;
 pub mod standalone;
 
 use std::sync::Arc;
@@ -66,7 +65,7 @@ use servers::query_handler::grpc::GrpcQueryHandler;
 use servers::query_handler::sql::SqlQueryHandler;
 use servers::query_handler::{
     InfluxdbLineProtocolHandler, LogQueryHandler, OpenTelemetryProtocolHandler,
-    OpentsdbProtocolHandler, PipelineHandler, PromStoreProtocolHandler, ScriptHandler,
+    OpentsdbProtocolHandler, PipelineHandler, PromStoreProtocolHandler,
 };
 use servers::server::ServerHandlers;
 use session::context::QueryContextRef;
@@ -88,7 +87,6 @@ use crate::error::{
 use crate::frontend::FrontendOptions;
 use crate::heartbeat::HeartbeatTask;
 use crate::limiter::LimiterRef;
-use crate::script::ScriptExecutor;
 
 #[async_trait]
 pub trait FrontendInstance:
@@ -98,7 +96,6 @@ pub trait FrontendInstance:
     + InfluxdbLineProtocolHandler
     + PromStoreProtocolHandler
     + OpenTelemetryProtocolHandler
-    + ScriptHandler
     + PrometheusHandler
     + PipelineHandler
     + LogQueryHandler
@@ -115,7 +112,6 @@ pub type FrontendInstanceRef = Arc<dyn FrontendInstance>;
 pub struct Instance {
     options: FrontendOptions,
     catalog_manager: CatalogManagerRef,
-    script_executor: Arc<ScriptExecutor>,
     pipeline_operator: Arc<PipelineOperator>,
     statement_executor: Arc<StatementExecutor>,
     query_engine: QueryEngineRef,
@@ -205,8 +201,6 @@ impl FrontendInstance for Instance {
             heartbeat_task.start().await?;
         }
 
-        self.script_executor.start(self)?;
-
         if let Some(t) = self.export_metrics_task.as_ref() {
             if t.send_by_handler {
                 let handler = ExportMetricHandler::new_handler(
@@ -495,7 +489,8 @@ pub fn check_permission(
         | Statement::Explain(_)
         | Statement::Tql(_)
         | Statement::Delete(_)
-        | Statement::DeclareCursor(_) => {}
+        | Statement::DeclareCursor(_)
+        | Statement::Copy(sql::statements::copy::Copy::CopyQueryTo(_)) => {}
         // database ops won't be checked
         Statement::CreateDatabase(_)
         | Statement::ShowDatabases(_)
@@ -570,6 +565,7 @@ pub fn check_permission(
             validate_db_permission!(stmt, query_ctx);
         }
         Statement::ShowStatus(_stmt) => {}
+        Statement::ShowSearchPath(_stmt) => {}
         Statement::DescribeTable(stmt) => {
             validate_param(stmt.name(), query_ctx)?;
         }

src/frontend/src/instance/builder.rs

@@ -44,7 +44,6 @@ use crate::heartbeat::HeartbeatTask;
 use crate::instance::region_query::FrontendRegionQueryHandler;
 use crate::instance::Instance;
 use crate::limiter::Limiter;
-use crate::script::ScriptExecutor;
 
 /// The frontend [`Instance`] builder.
 pub struct FrontendBuilder {
@@ -174,10 +173,6 @@ impl FrontendBuilder {
         )
         .query_engine();
 
-        let script_executor = Arc::new(
-            ScriptExecutor::new(self.catalog_manager.clone(), query_engine.clone()).await?,
-        );
-
         let statement_executor = Arc::new(StatementExecutor::new(
             self.catalog_manager.clone(),
             query_engine.clone(),
@@ -208,7 +203,6 @@ impl FrontendBuilder {
         Ok(Instance {
             options: self.options,
             catalog_manager: self.catalog_manager,
-            script_executor,
             pipeline_operator,
             statement_executor,
             query_engine,

src/frontend/src/instance/script.rs

@@ -1,58 +0,0 @@
-// Copyright 2023 Greptime Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use std::collections::HashMap;
-
-use async_trait::async_trait;
-use common_query::Output;
-use servers::error::Error;
-use servers::interceptor::{ScriptInterceptor, ScriptInterceptorRef};
-use servers::query_handler::ScriptHandler;
-use session::context::QueryContextRef;
-
-use crate::instance::Instance;
-use crate::metrics;
-
-#[async_trait]
-impl ScriptHandler for Instance {
-    async fn insert_script(
-        &self,
-        query_ctx: QueryContextRef,
-        name: &str,
-        script: &str,
-    ) -> servers::error::Result<()> {
-        let interceptor_ref = self.plugins.get::<ScriptInterceptorRef<Error>>();
-        interceptor_ref.pre_execute(name, query_ctx.clone())?;
-
-        let _timer = metrics::INSERT_SCRIPTS_ELAPSED.start_timer();
-        self.script_executor
-            .insert_script(query_ctx, name, script)
-            .await
-    }
-
-    async fn execute_script(
-        &self,
-        query_ctx: QueryContextRef,
-        name: &str,
-        params: HashMap<String, String>,
-    ) -> servers::error::Result<Output> {
-        let interceptor_ref = self.plugins.get::<ScriptInterceptorRef<Error>>();
-        interceptor_ref.pre_execute(name, query_ctx.clone())?;
-
-        let _timer = metrics::EXECUTE_SCRIPT_ELAPSED.start_timer();
-        self.script_executor
-            .execute_script(query_ctx, name, params)
-            .await
-    }
-}

src/frontend/src/lib.rs

@@ -20,6 +20,5 @@ pub mod heartbeat;
 pub mod instance;
 pub(crate) mod limiter;
 pub(crate) mod metrics;
-mod script;
 pub mod server;
 pub mod service_config;

src/frontend/src/metrics.rs

@@ -29,19 +29,6 @@ lazy_static! {
     pub static ref GRPC_HANDLE_PROMQL_ELAPSED: Histogram = GRPC_HANDLE_QUERY_ELAPSED
         .with_label_values(&["promql"]);
 
-    /// Timer of handling scripts in the script handler.
-    pub static ref HANDLE_SCRIPT_ELAPSED: HistogramVec = register_histogram_vec!(
-        "greptime_frontend_handle_script_elapsed",
-        "Elapsed time of handling scripts in the script handler",
-        &["type"],
-        vec![0.005, 0.01, 0.05, 0.1, 0.5, 1.0, 5.0, 10.0, 60.0, 300.0]
-    )
-    .unwrap();
-    pub static ref INSERT_SCRIPTS_ELAPSED: Histogram = HANDLE_SCRIPT_ELAPSED
-        .with_label_values(&["insert"]);
-    pub static ref EXECUTE_SCRIPT_ELAPSED: Histogram = HANDLE_SCRIPT_ELAPSED
-        .with_label_values(&["execute"]);
-
     /// The number of OpenTelemetry metrics send by frontend node.
     pub static ref OTLP_METRICS_ROWS: IntCounter = register_int_counter!(
         "greptime_frontend_otlp_metrics_rows",

src/frontend/src/script.rs

@@ -1,294 +0,0 @@
-// Copyright 2023 Greptime Team
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use std::collections::HashMap;
-
-use catalog::CatalogManagerRef;
-use common_query::Output;
-use query::QueryEngineRef;
-use session::context::QueryContextRef;
-
-use crate::error::Result;
-use crate::instance::Instance;
-
-#[cfg(not(feature = "python"))]
-mod dummy {
-    use super::*;
-
-    pub struct ScriptExecutor;
-
-    impl ScriptExecutor {
-        pub async fn new(
-            _catalog_manager: CatalogManagerRef,
-            _query_engine: QueryEngineRef,
-        ) -> Result<Self> {
-            Ok(Self {})
-        }
-
-        pub fn start(&self, _instance: &Instance) -> Result<()> {
-            Ok(())
-        }
-
-        pub async fn insert_script(
-            &self,
-            _query_ctx: QueryContextRef,
-            _name: &str,
-            _script: &str,
-        ) -> servers::error::Result<()> {
-            servers::error::NotSupportedSnafu { feat: "script" }.fail()
-        }
-
-        pub async fn execute_script(
-            &self,
-            _query_ctx: QueryContextRef,
-            _name: &str,
-            _params: HashMap<String, String>,
-        ) -> servers::error::Result<Output> {
-            servers::error::NotSupportedSnafu { feat: "script" }.fail()
-        }
-    }
-}
-
-#[cfg(feature = "python")]
-mod python {
-    use std::sync::Arc;
-
-    use api::v1::ddl_request::Expr;
-    use api::v1::greptime_request::Request;
-    use api::v1::DdlRequest;
-    use arc_swap::ArcSwap;
-    use catalog::RegisterSystemTableRequest;
-    use common_error::ext::{BoxedError, ErrorExt};
-    use common_telemetry::{error, info};
-    use script::manager::ScriptManager;
-    use servers::query_handler::grpc::GrpcQueryHandler;
-    use session::context::QueryContext;
-    use snafu::{OptionExt, ResultExt};
-    use table::table_name::TableName;
-
-    use super::*;
-    use crate::error::{CatalogSnafu, Error, TableNotFoundSnafu};
-
-    type FrontendGrpcQueryHandlerRef = Arc<dyn GrpcQueryHandler<Error = Error> + Send + Sync>;
-
-    /// A placeholder for the real gRPC handler.
-    /// It is temporary and will be replaced soon.
-    struct DummyHandler;
-
-    impl DummyHandler {
-        fn arc() -> Arc<Self> {
-            Arc::new(Self {})
-        }
-    }
-
-    #[async_trait::async_trait]
-    impl GrpcQueryHandler for DummyHandler {
-        type Error = Error;
-
-        async fn do_query(
-            &self,
-            _query: Request,
-            _ctx: QueryContextRef,
-        ) -> std::result::Result<Output, Self::Error> {
-            unreachable!();
-        }
-    }
-
-    pub struct ScriptExecutor {
-        script_manager: ScriptManager<Error>,
-        grpc_handler: ArcSwap<FrontendGrpcQueryHandlerRef>,
-        catalog_manager: CatalogManagerRef,
-    }
-
-    impl ScriptExecutor {
-        pub async fn new(
-            catalog_manager: CatalogManagerRef,
-            query_engine: QueryEngineRef,
-        ) -> Result<Self> {
-            let grpc_handler = DummyHandler::arc();
-            Ok(Self {
-                grpc_handler: ArcSwap::new(Arc::new(grpc_handler.clone() as _)),
-                script_manager: ScriptManager::new(grpc_handler as _, query_engine)
-                    .await
-                    .context(crate::error::StartScriptManagerSnafu)?,
-                catalog_manager,
-            })
-        }
-
-        pub fn start(&self, instance: &Instance) -> Result<()> {
-            let handler = Arc::new(instance.clone());
-            self.grpc_handler.store(Arc::new(handler.clone() as _));
-            self.script_manager
-                .start(handler)
-                .context(crate::error::StartScriptManagerSnafu)?;
-
-            Ok(())
-        }
-
-        /// Create scripts table for the specific catalog if it's not exists.
-        /// The function is idempotent and safe to be called more than once for the same catalog
-        async fn create_scripts_table_if_need(&self, catalog: &str) -> Result<()> {
-            let scripts_table = self.script_manager.get_scripts_table(catalog);
-
-            if scripts_table.is_some() {
-                return Ok(());
-            }
-
-            let RegisterSystemTableRequest {
-                create_table_expr: expr,
-                open_hook,
-            } = self.script_manager.create_table_request(catalog);
-
-            if let Some(table) = self
-                .catalog_manager
-                .table(
-                    &expr.catalog_name,
-                    &expr.schema_name,
-                    &expr.table_name,
-                    None,
-                )
-                .await
-                .context(CatalogSnafu)?
-            {
-                if let Some(open_hook) = open_hook {
-                    (open_hook)(table.clone()).await.context(CatalogSnafu)?;
-                }
-
-                self.script_manager.insert_scripts_table(catalog, table);
-
-                return Ok(());
-            }
-
-            let table_name =
-                TableName::new(&expr.catalog_name, &expr.schema_name, &expr.table_name);
-
-            let _ = self
-                .grpc_handler
-                .load()
-                .do_query(
-                    Request::Ddl(DdlRequest {
-                        expr: Some(Expr::CreateTable(expr)),
-                    }),
-                    QueryContext::arc(),
-                )
-                .await?;
-
-            let table = self
-                .catalog_manager
-                .table(
-                    &table_name.catalog_name,
-                    &table_name.schema_name,
-                    &table_name.table_name,
-                    None,
-                )
-                .await
-                .context(CatalogSnafu)?
-                .with_context(|| TableNotFoundSnafu {
-                    table_name: table_name.to_string(),
-                })?;
-
-            if let Some(open_hook) = open_hook {
-                (open_hook)(table.clone()).await.context(CatalogSnafu)?;
-            }
-
-            info!(
-                "Created scripts table {}.",
-                table.table_info().full_table_name()
-            );
-
-            self.script_manager.insert_scripts_table(catalog, table);
-
-            Ok(())
-        }
-
-        pub async fn insert_script(
-            &self,
-            query_ctx: QueryContextRef,
-            name: &str,
-            script: &str,
-        ) -> servers::error::Result<()> {
-            self.create_scripts_table_if_need(query_ctx.current_catalog())
-                .await
-                .map_err(|e| {
-                    if e.status_code().should_log_error() {
-                        error!(e; "Failed to create scripts table");
-                    }
-
-                    servers::error::InternalSnafu {
-                        err_msg: e.to_string(),
-                    }
-                    .build()
-                })?;
-
-            let _s = self
-                .script_manager
-                .insert_and_compile(
-                    query_ctx.current_catalog(),
-                    &query_ctx.current_schema(),
-                    name,
-                    script,
-                )
-                .await
-                .map_err(|e| {
-                    if e.status_code().should_log_error() {
-                        error!(e; "Failed to insert script");
-                    }
-
-                    BoxedError::new(e)
-                })
-                .context(servers::error::InsertScriptSnafu { name })?;
-
-            Ok(())
-        }
-
-        pub async fn execute_script(
-            &self,
-            query_ctx: QueryContextRef,
-            name: &str,
-            params: HashMap<String, String>,
-        ) -> servers::error::Result<Output> {
-            self.create_scripts_table_if_need(query_ctx.current_catalog())
-                .await
-                .map_err(|e| {
-                    error!(e; "Failed to create scripts table");
-                    servers::error::InternalSnafu {
-                        err_msg: e.to_string(),
-                    }
-                    .build()
-                })?;
-
-            self.script_manager
-                .execute(
-                    query_ctx.current_catalog(),
-                    &query_ctx.current_schema(),
-                    name,
-                    params,
-                )
-                .await
-                .map_err(|e| {
-                    if e.status_code().should_log_error() {
-                        error!(e; "Failed to execute script");
-                    }
-
-                    BoxedError::new(e)
-                })
-                .context(servers::error::ExecuteScriptSnafu { name })
-        }
-    }
-}
-
-#[cfg(not(feature = "python"))]
-pub use self::dummy::*;
-#[cfg(feature = "python")]
-pub use self::python::*;

src/frontend/src/server.rs

@@ -78,10 +78,8 @@ where
     }
 
     pub fn http_server_builder(&self, opts: &FrontendOptions) -> HttpServerBuilder {
-        let mut builder = HttpServerBuilder::new(opts.http.clone()).with_sql_handler(
-            ServerSqlQueryHandlerAdapter::arc(self.instance.clone()),
-            Some(self.instance.clone()),
-        );
+        let mut builder = HttpServerBuilder::new(opts.http.clone())
+            .with_sql_handler(ServerSqlQueryHandlerAdapter::arc(self.instance.clone()));
 
         let validator = self.plugins.get::<LogValidatorRef>();
         let ingest_interceptor = self.plugins.get::<LogIngestInterceptorRef<ServerError>>();

src/index/Cargo.toml

@@ -22,7 +22,6 @@ fst.workspace = true
 futures.workspace = true
 greptime-proto.workspace = true
 mockall.workspace = true
-parquet.workspace = true
 pin-project.workspace = true
 prost.workspace = true
 regex.workspace = true

src/index/src/bloom_filter.rs

@@ -45,7 +45,7 @@ pub struct BloomFilterMeta {
 }
 
 /// The location of the bloom filter segment in the file.
-#[derive(Debug, Serialize, Deserialize, Clone, Hash, PartialEq, Eq)]
+#[derive(Debug, Serialize, Deserialize, Clone, Copy, Hash, PartialEq, Eq)]
 pub struct BloomFilterSegmentLocation {
     /// The offset of the bloom filter segment in the file.
     pub offset: u64,