rust/greptimedb
1
0
Fork 0
mirror of https://github.com/GreptimeTeam/greptimedb.git synced 2026-11-04 09:57:02 +00:00
Code Issues Packages Projects Releases Wiki Activity

merge: main into fix/metric-engine-time-index-unit

Browse Source 
Merge latest main (logical table batching for OTLP/prom remote write,
#9288) into the time index unit branch, with two compatibility fixes:

- OTLP metric path: align the requests' time index unit to the physical
  table's before the logical batcher eligibility check
  (Inserter::align_metric_row_inserts_time_unit, new pub method; a
  missing physical table aligns to the auto-created millisecond
  default). Without this, the nanosecond encoding on the metric engine
  path made can_batch_metric_rows reject every OTLP request, silently
  disabling logical batching (test_otlp_logical_batcher_alignment).
  Non-millisecond physical tables still fall back to the ordinary
  insert path, which converts the unit.
- Prometheus remote write v1/v2: only take the bulk batcher path when
  the physical metric table's time index is millisecond
  (LogicalTablePendingRowsBatcher::accepts_physical_table_time_index);
  otherwise fall back to the ordinary insert path, which converts the
  requests to the physical table's unit.

Signed-off-by: Ning Sun <sunning@greptime.com>
This commit is contained in:
Ning Sun
2026-09-23 14:10:58 +08:00
parent 03cff447be 0f118bc5ba
commit 2b2d456749
362 changed files with 32005 additions and 7429 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/actions/build-linux-artifacts/action.yml
+23
View File
@@ -31,6 +31,22 @@ inputs:
description: Build GreptimeDB with large page size (65536).
required: false
default: 'false'
aws-ci-test-bucket:
description: Bucket used by the object-store integration tests
required: false
default: ''
aws-region:
description: Region of the test bucket
required: false
default: ''
aws-access-key-id:
description: Access key id for the test bucket
required: false
default: ''
aws-secret-access-key:
description: Secret access key for the test bucket
required: false
default: ''
runs:
using: composite
steps:
@@ -38,6 +54,13 @@ runs:
if: ${{ inputs.disable-run-tests == 'false' }}
shell: bash
# NOTE: If the BUILD_JOBS > 4, it's always OOM in EC2 instance.
# Without GT_S3_*, every remote object-store case is skipped and the
# release only ever exercises the local file backend.
env:
GT_S3_BUCKET: ${{ inputs.aws-ci-test-bucket }}
GT_S3_REGION: ${{ inputs.aws-region }}
GT_S3_ACCESS_KEY_ID: ${{ inputs.aws-access-key-id }}
GT_S3_ACCESS_KEY: ${{ inputs.aws-secret-access-key }}
run: |
cd ${{ inputs.working-dir }} && \
make run-it-in-container BUILD_JOBS=4 \
.github/scripts/run-fuzz-targets-test.sh
+18 -3
View File
@@ -31,7 +31,7 @@ new_fixture() {
#!/usr/bin/env bash
set -euo pipefail
printf '%s\t%s\t%s\n' "${GT_FUZZ_DUMP_DIR}" "$*" "${MOCK_CARGO_MARKER:-}" >>"${MOCK_CARGO_LOG}"
target="$3"
target="$4"
case " ${MOCK_FAIL_TARGETS:-} " in
*" ${target} "*) exit 17 ;;
esac
@@ -117,6 +117,20 @@ EOF
set -e
}
test_rustup_toolchain_env_is_honored() {
new_fixture
export RUSTUP_TOOLCHAIN=nightly-2026-03-21
run_fixture $'fuzz_create_table' true true "fuzz_create_table"
unset RUSTUP_TOOLCHAIN
assert_eq 1 "${fixture_status}" "pinned toolchain run status"
grep -q -- '+nightly-2026-03-21 fuzz run fuzz_create_table' "${fixture}/cargo.log" || \
fail "pinned toolchain missing from cargo invocation"
grep -q 'cargo +nightly-2026-03-21 fuzz run fuzz_create_table' "${fixture}/artifacts/summary.md" || \
fail "pinned toolchain missing from reproduce command"
cleanup_fixture
}
test_successful_targets_run_in_order() {
new_fixture
run_fixture $'fuzz_create_table\nfuzz_insert' false true ""
@@ -125,7 +139,7 @@ test_successful_targets_run_in_order() {
assert_eq 2 "$(wc -l <"${fixture}/cargo.log" | tr -d ' ')" "cargo invocation count"
assert_eq \
$'fuzz_create_table\nfuzz_insert' \
"$(awk -F '\t' '{print $2}' "${fixture}/cargo.log" | sed -E 's/^fuzz run ([^ ]+).*/\1/')" \
"$(awk -F '\t' '{print $2}' "${fixture}/cargo.log" | sed -E 's/^\+nightly fuzz run ([^ ]+).*/\1/')" \
"target order"
grep -q -- '--features=unstable' "${fixture}/cargo.log" || fail "unstable feature missing"
grep -q -- '-max_total_time=120' "${fixture}/cargo.log" || fail "fuzz time missing"
@@ -175,7 +189,7 @@ test_fail_fast_stops_after_first_failure() {
fail "skipped target annotation missing"
grep -q '### Reproduce failed targets' "${fixture}/artifacts/summary.md" || \
fail "reproduction section missing"
grep -q 'cargo fuzz run fuzz_insert' "${fixture}/artifacts/summary.md" || \
grep -q 'cargo +nightly fuzz run fuzz_insert' "${fixture}/artifacts/summary.md" || \
fail "reproduction command missing"
cleanup_fixture
}
@@ -367,5 +381,6 @@ test_collector_keeps_target_scopes_separate
test_cluster_collector_honors_target_scope_and_namespace
test_setup_failure_writes_artifact_contract
test_setup_failure_keeps_manifest_when_collection_fails
test_rustup_toolchain_env_is_honored
printf 'All fuzz orchestration script tests passed.\n'
.github/scripts/run-fuzz-targets.sh
+2 -2
View File
@@ -104,7 +104,7 @@ write_summary() {
reproduce_args+=("GT_FUZZ_INSTANCE_ROOT_DIR=${GT_FUZZ_INSTANCE_ROOT_DIR}")
fi
fi
reproduce_args+=(cargo fuzz run "${target}" --fuzz-dir tests-fuzz -D -s none)
reproduce_args+=(cargo +"${RUSTUP_TOOLCHAIN:-nightly}" fuzz run "${target}" --fuzz-dir tests-fuzz -D -s none)
if [[ "${fuzz_unstable}" == true ]]; then
reproduce_args+=(--features=unstable)
fi
@@ -195,7 +195,7 @@ for ((index = 0; index < ${#targets[@]}; index++)); do
fi
run_args=("${fuzz_binary}" "-max_total_time=${FUZZ_MAX_TOTAL_TIME}" "-artifact_prefix=${target_dir}/libfuzzer/")
else
run_args=(cargo fuzz run "${target}" --fuzz-dir tests-fuzz -D -s none)
run_args=(cargo +"${RUSTUP_TOOLCHAIN:-nightly}" fuzz run "${target}" --fuzz-dir tests-fuzz -D -s none)
if [[ "${fuzz_unstable}" == true ]]; then
run_args+=(--features=unstable)
fi
.github/workflows/checks.yml
+1 -1
View File
@@ -39,7 +39,7 @@ jobs:
- uses: actions/checkout@v4
with:
persist-credentials: false
- uses: crate-ci/typos@master
- uses: crate-ci/typos@v1.50.2
- name: Check the config docs
run: |
make config-docs && \
.github/workflows/dev-build.yml
+8
View File
@@ -183,6 +183,10 @@ jobs:
image-registry: ${{ vars.ECR_IMAGE_REGISTRY }}
image-namespace: ${{ vars.ECR_IMAGE_NAMESPACE }}
large-page-size: ${{ inputs.large-page-size }}
aws-ci-test-bucket: ${{ vars.AWS_CI_TEST_BUCKET }}
aws-region: ${{ vars.AWS_CI_TEST_BUCKET_REGION }}
aws-access-key-id: ${{ secrets.AWS_CI_TEST_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.AWS_CI_TEST_SECRET_ACCESS_KEY }}
build-linux-arm64-artifacts:
name: Build linux-arm64 artifacts
@@ -217,6 +221,10 @@ jobs:
image-registry: ${{ vars.ECR_IMAGE_REGISTRY }}
image-namespace: ${{ vars.ECR_IMAGE_NAMESPACE }}
large-page-size: ${{ inputs.large-page-size }}
aws-ci-test-bucket: ${{ vars.AWS_CI_TEST_BUCKET }}
aws-region: ${{ vars.AWS_CI_TEST_BUCKET_REGION }}
aws-access-key-id: ${{ secrets.AWS_CI_TEST_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.AWS_CI_TEST_SECRET_ACCESS_KEY }}
release-images-to-dockerhub:
name: Build and push images to DockerHub
.github/workflows/docbot.yml
+3 -2
View File
@@ -1,10 +1,10 @@
name: Follow Up Docs
on:
pull_request_target:
types: [opened, edited]
types: [opened, edited, closed]
concurrency:
group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
group: ${{ github.workflow }}-${{ github.event.action }}-${{ github.head_ref || github.run_id }}
cancel-in-progress: true
jobs:
@@ -13,6 +13,7 @@ jobs:
permissions:
pull-requests: write
contents: read
actions: read
timeout-minutes: 10
steps:
- uses: actions/checkout@v4
.github/workflows/docs.yml
+1 -1
View File
@@ -45,7 +45,7 @@ jobs:
- uses: actions/checkout@v4
with:
persist-credentials: false
- uses: crate-ci/typos@master
- uses: crate-ci/typos@v1.50.2
license-header-check:
if: ${{ github.event_name != 'pull_request' || github.event.pull_request.draft == false }}
.github/workflows/integration.yml
+16 -3
View File
@@ -31,6 +31,11 @@ name: Integration CI
env:
CARGO_FUZZ_VERSION: "0.13.2"
# Fuzz targets require a nightly toolchain (cargo-fuzz needs
# `-Zsanitizer=fuzzer`). This pin decouples the fuzz workflow from the
# workspace default in rust-toolchain.toml, so it keeps working when the
# workspace moves to a stable toolchain.
FUZZ_RUST_TOOLCHAIN: "nightly-2026-03-21"
concurrency:
group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
@@ -81,7 +86,7 @@ jobs:
# `cargo gc` will invoke `cargo build` with specified args
run: |
started_epoch="$(date +%s)"
cargo gc -- --bin greptime --bin sqlness-runner --features "pg_kvbackend,mysql_kvbackend,vector_index"
cargo gc -- --bin greptime --bin sqlness-runner --features "pg_kvbackend,mysql_kvbackend,vector_index,ai_functions"
elapsed_secs=$(( $(date +%s) - started_epoch ))
printf -- '- Binary build: **%ss**\n' "${elapsed_secs}" >>"${GITHUB_STEP_SUMMARY}"
- name: Pack greptime binaries
@@ -328,6 +333,7 @@ jobs:
- uses: actions-rust-lang/setup-rust-toolchain@v1
with:
cache: false
toolchain: ${{ env.FUZZ_RUST_TOOLCHAIN }}
- name: Fuzz Dependency Cache
id: fuzz-dependencies-cache
uses: Swatinem/rust-cache@v2
@@ -343,7 +349,7 @@ jobs:
uses: actions/cache/restore@v4
with:
path: ${{ runner.temp }}/greptime-fuzz-binaries
key: v1-fuzz-binaries-${{ env.CARGO_FUZZ_VERSION }}-all-${{ hashFiles('Cargo.toml', 'Cargo.lock', 'rust-toolchain.toml', '.cargo/**', 'tests-fuzz/**', 'src/**') }}
key: v1-fuzz-binaries-${{ env.CARGO_FUZZ_VERSION }}-${{ env.FUZZ_RUST_TOOLCHAIN }}-all-${{ hashFiles('Cargo.toml', 'Cargo.lock', '.cargo/**', 'tests-fuzz/**', 'src/**') }}
- name: Report fuzz cache lookup
shell: bash
run: |
@@ -355,6 +361,8 @@ jobs:
- name: Set Rust Fuzz
if: steps.fuzz-binaries.outputs.cache-hit != 'true'
shell: bash
env:
RUSTUP_TOOLCHAIN: ${{ env.FUZZ_RUST_TOOLCHAIN }}
run: |
sudo apt-get install -y libfuzzer-14-dev
cargo install cargo-fuzz --version "${CARGO_FUZZ_VERSION}" --locked
@@ -363,6 +371,7 @@ jobs:
shell: bash
env:
CUSTOM_LIBFUZZER_PATH: /usr/lib/llvm-14/lib/libFuzzer.a
RUSTUP_TOOLCHAIN: ${{ env.FUZZ_RUST_TOOLCHAIN }}
FUZZ_TARGETS: |
fuzz_create_database
fuzz_create_table
@@ -410,7 +419,7 @@ jobs:
uses: actions/cache/save@v4
with:
path: ${{ runner.temp }}/greptime-fuzz-binaries
key: v1-fuzz-binaries-${{ env.CARGO_FUZZ_VERSION }}-all-${{ hashFiles('Cargo.toml', 'Cargo.lock', 'rust-toolchain.toml', '.cargo/**', 'tests-fuzz/**', 'src/**') }}
key: v1-fuzz-binaries-${{ env.CARGO_FUZZ_VERSION }}-${{ env.FUZZ_RUST_TOOLCHAIN }}-all-${{ hashFiles('Cargo.toml', 'Cargo.lock', '.cargo/**', 'tests-fuzz/**', 'src/**') }}
- name: Pack prebuilt fuzz binaries
shell: bash
run: |
@@ -479,6 +488,7 @@ jobs:
- name: Fuzz Test
uses: ./.github/actions/fuzz-test
env:
RUSTUP_TOOLCHAIN: ${{ env.FUZZ_RUST_TOOLCHAIN }}
GT_MYSQL_ADDR: 127.0.0.1:4002
FUZZ_SERVICE_LOG: /tmp/greptime-fuzz-runtime/greptime.log
with:
@@ -537,6 +547,7 @@ jobs:
- name: Run Fuzz Test
uses: ./.github/actions/fuzz-test
env:
RUSTUP_TOOLCHAIN: ${{ env.FUZZ_RUST_TOOLCHAIN }}
GT_MYSQL_ADDR: 127.0.0.1:4002
GT_FUZZ_BINARY_PATH: ./bin/greptime
GT_FUZZ_INSTANCE_ROOT_DIR: /tmp/greptime-fuzz-artifacts/targets/unstable_fuzz_create_table_standalone/service/instance/
@@ -820,6 +831,7 @@ jobs:
- name: Fuzz Test
uses: ./.github/actions/fuzz-test
env:
RUSTUP_TOOLCHAIN: ${{ env.FUZZ_RUST_TOOLCHAIN }}
GT_MYSQL_ADDR: 127.0.0.1:4002
GT_KAFKA_WAL_HELPER_URL: http://127.0.0.1:8080
GT_KAFKA_ADDR: kafka.kafka-cluster.svc.cluster.local:9092
@@ -994,6 +1006,7 @@ jobs:
- name: Fuzz Test
uses: ./.github/actions/fuzz-test
env:
RUSTUP_TOOLCHAIN: ${{ env.FUZZ_RUST_TOOLCHAIN }}
GT_MYSQL_ADDR: 127.0.0.1:4002
with:
targets: ${{ matrix.group.targets }}
.github/workflows/nightly-build.yml
+8
View File
@@ -151,6 +151,10 @@ jobs:
disable-run-tests: ${{ env.DISABLE_RUN_TESTS }}
image-registry: ${{ vars.ECR_IMAGE_REGISTRY }}
image-namespace: ${{ vars.ECR_IMAGE_NAMESPACE }}
aws-ci-test-bucket: ${{ vars.AWS_CI_TEST_BUCKET }}
aws-region: ${{ vars.AWS_CI_TEST_BUCKET_REGION }}
aws-access-key-id: ${{ secrets.AWS_CI_TEST_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.AWS_CI_TEST_SECRET_ACCESS_KEY }}
build-linux-arm64-artifacts:
name: Build linux-arm64 artifacts
@@ -173,6 +177,10 @@ jobs:
disable-run-tests: ${{ env.DISABLE_RUN_TESTS }}
image-registry: ${{ vars.ECR_IMAGE_REGISTRY }}
image-namespace: ${{ vars.ECR_IMAGE_NAMESPACE }}
aws-ci-test-bucket: ${{ vars.AWS_CI_TEST_BUCKET }}
aws-region: ${{ vars.AWS_CI_TEST_BUCKET_REGION }}
aws-access-key-id: ${{ secrets.AWS_CI_TEST_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.AWS_CI_TEST_SECRET_ACCESS_KEY }}
release-images-to-dockerhub:
name: Build and push images to DockerHub
.github/workflows/nightly-ci.yml
+52 -1
View File
@@ -71,6 +71,57 @@ jobs:
path: C:\Users\RUNNER~1\AppData\Local\Temp\sqlness*
retention-days: 3
# PR CI runs the object-store matrix against a local MinIO. This job is where
# the same matrix meets real AWS S3.
test-on-linux-real-s3:
name: Run tests on Linux against real S3
if: ${{ github.repository == 'GreptimeTeam/greptimedb' }}
runs-on: ubuntu-22.04-arm
timeout-minutes: 90
steps:
- uses: actions/checkout@v4
with:
persist-credentials: false
- uses: arduino/setup-protoc@v3
with:
repo-token: ${{ secrets.GITHUB_TOKEN }}
- uses: rui314/setup-mold@v1
- name: Install toolchain
uses: actions-rust-lang/setup-rust-toolchain@v1
with:
cache: false
- name: Rust Cache
uses: Swatinem/rust-cache@v2
with:
shared-key: "coverage-test"
cache-all-crates: "true"
save-if: false
- name: Install latest nextest release
uses: taiki-e/install-action@nextest
- name: Setup external services
working-directory: tests-integration/fixtures
run: ../../.github/scripts/pull-test-deps-images.sh && docker compose up -d --wait
- name: Run nextest cases
run: cargo nextest run --workspace -F dashboard -F pg_kvbackend -F mysql_kvbackend -F vector_index -F ai_functions
env:
CARGO_BUILD_RUSTFLAGS: "-C link-arg=-fuse-ld=mold"
RUST_BACKTRACE: 1
RUST_MIN_STACK: 8388608 # 8MB
CARGO_INCREMENTAL: 0
GT_S3_BUCKET: ${{ vars.AWS_CI_TEST_BUCKET }}
GT_S3_ACCESS_KEY_ID: ${{ secrets.AWS_CI_TEST_ACCESS_KEY_ID }}
GT_S3_ACCESS_KEY: ${{ secrets.AWS_CI_TEST_SECRET_ACCESS_KEY }}
GT_S3_REGION: ${{ vars.AWS_CI_TEST_BUCKET_REGION }}
GT_ETCD_TLS_ENDPOINTS: https://127.0.0.1:2378
GT_ETCD_ENDPOINTS: http://127.0.0.1:2379
GT_POSTGRES_ENDPOINTS: postgres://greptimedb:admin@127.0.0.1:5432/postgres
GT_POSTGRES15_ENDPOINTS: postgres://test_user:test_password@127.0.0.1:5433/postgres
GT_POSTGRES15_SCHEMA: test_schema
GT_MYSQL_ENDPOINTS: mysql://greptimedb:admin@127.0.0.1:3306/mysql
GT_KAFKA_ENDPOINTS: 127.0.0.1:9092
GT_KAFKA_SASL_ENDPOINTS: 127.0.0.1:9093
UNITTEST_LOG_DIR: "__unittest_logs"
test-on-windows:
name: Run tests on Windows
if: ${{ github.repository == 'GreptimeTeam/greptimedb' }}
@@ -130,7 +181,7 @@ jobs:
check-status:
name: Check status
needs: [sqlness-test, sqlness-windows, test-on-windows]
needs: [sqlness-test, sqlness-windows, test-on-windows, test-on-linux-real-s3]
if: ${{ github.repository == 'GreptimeTeam/greptimedb' }}
runs-on: ubuntu-latest
outputs:
.github/workflows/release.yml
+8
View File
@@ -204,6 +204,10 @@ jobs:
disable-run-tests: ${{ env.DISABLE_RUN_TESTS }}
image-registry: ${{ vars.ECR_IMAGE_REGISTRY }}
image-namespace: ${{ vars.ECR_IMAGE_NAMESPACE }}
aws-ci-test-bucket: ${{ vars.AWS_CI_TEST_BUCKET }}
aws-region: ${{ vars.AWS_CI_TEST_BUCKET_REGION }}
aws-access-key-id: ${{ secrets.AWS_CI_TEST_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.AWS_CI_TEST_SECRET_ACCESS_KEY }}
build-linux-arm64-artifacts:
name: Build linux-arm64 artifacts
@@ -227,6 +231,10 @@ jobs:
image-registry: ${{ vars.ECR_IMAGE_REGISTRY }}
image-namespace: ${{ vars.ECR_IMAGE_NAMESPACE }}
large-page-size: true
aws-ci-test-bucket: ${{ vars.AWS_CI_TEST_BUCKET }}
aws-region: ${{ vars.AWS_CI_TEST_BUCKET_REGION }}
aws-access-key-id: ${{ secrets.AWS_CI_TEST_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.AWS_CI_TEST_SECRET_ACCESS_KEY }}
build-linux-riscv64-artifacts:
name: Build linux-riscv64 artifacts
.github/workflows/rust.yml
+18 -20
View File
@@ -200,21 +200,20 @@ jobs:
run: ../../.github/scripts/pull-test-deps-images.sh && docker compose up -d --wait
- name: Run nextest cases
run: cargo nextest run --workspace -F dashboard -F pg_kvbackend -F mysql_kvbackend -F vector_index
run: cargo nextest run --workspace -F dashboard -F pg_kvbackend -F mysql_kvbackend -F vector_index -F ai_functions
env:
CARGO_BUILD_RUSTFLAGS: "-C link-arg=-fuse-ld=mold"
RUST_BACKTRACE: 1
RUST_MIN_STACK: 8388608 # 8MB
CARGO_INCREMENTAL: 0
GT_S3_BUCKET: ${{ vars.AWS_CI_TEST_BUCKET }}
GT_S3_ACCESS_KEY_ID: ${{ secrets.AWS_CI_TEST_ACCESS_KEY_ID }}
GT_S3_ACCESS_KEY: ${{ secrets.AWS_CI_TEST_SECRET_ACCESS_KEY }}
GT_S3_REGION: ${{ vars.AWS_CI_TEST_BUCKET_REGION }}
GT_MINIO_BUCKET: greptime
GT_MINIO_ACCESS_KEY_ID: superpower_ci_user
GT_MINIO_ACCESS_KEY: superpower_password
GT_MINIO_REGION: us-west-2
GT_MINIO_ENDPOINT_URL: http://127.0.0.1:9000
# Runs the object-store matrix against the local MinIO from
# tests-integration/fixtures. Real AWS S3 is covered by the nightly
# jobs and by the release build.
GT_S3_BUCKET: greptime
GT_S3_ACCESS_KEY_ID: superpower_ci_user
GT_S3_ACCESS_KEY: superpower_password
GT_S3_REGION: us-west-2
GT_S3_ENDPOINT_URL: http://127.0.0.1:9000
GT_ETCD_TLS_ENDPOINTS: https://127.0.0.1:2378
GT_ETCD_ENDPOINTS: http://127.0.0.1:2379
GT_POSTGRES_ENDPOINTS: postgres://greptimedb:admin@127.0.0.1:5432/postgres
@@ -258,20 +257,19 @@ jobs:
run: ../../.github/scripts/pull-test-deps-images.sh && docker compose up -d --wait
- name: Run nextest cases
run: cargo llvm-cov nextest --workspace --lcov --output-path lcov.info -F dashboard -F pg_kvbackend -F mysql_kvbackend -F vector_index
run: cargo llvm-cov nextest --workspace --lcov --output-path lcov.info -F dashboard -F pg_kvbackend -F mysql_kvbackend -F vector_index -F ai_functions
env:
CARGO_BUILD_RUSTFLAGS: "-C link-arg=-fuse-ld=mold"
RUST_BACKTRACE: 1
CARGO_INCREMENTAL: 0
GT_S3_BUCKET: ${{ vars.AWS_CI_TEST_BUCKET }}
GT_S3_ACCESS_KEY_ID: ${{ secrets.AWS_CI_TEST_ACCESS_KEY_ID }}
GT_S3_ACCESS_KEY: ${{ secrets.AWS_CI_TEST_SECRET_ACCESS_KEY }}
GT_S3_REGION: ${{ vars.AWS_CI_TEST_BUCKET_REGION }}
GT_MINIO_BUCKET: greptime
GT_MINIO_ACCESS_KEY_ID: superpower_ci_user
GT_MINIO_ACCESS_KEY: superpower_password
GT_MINIO_REGION: us-west-2
GT_MINIO_ENDPOINT_URL: http://127.0.0.1:9000
# Runs the object-store matrix against the local MinIO from
# tests-integration/fixtures. Real AWS S3 is covered by the nightly
# jobs and by the release build.
GT_S3_BUCKET: greptime
GT_S3_ACCESS_KEY_ID: superpower_ci_user
GT_S3_ACCESS_KEY: superpower_password
GT_S3_REGION: us-west-2
GT_S3_ENDPOINT_URL: http://127.0.0.1:9000
GT_ETCD_TLS_ENDPOINTS: https://127.0.0.1:2378
GT_ETCD_ENDPOINTS: http://127.0.0.1:2379
GT_POSTGRES_ENDPOINTS: postgres://greptimedb:admin@127.0.0.1:5432/postgres
.github/workflows/vmbench-long-range.yml
+33 -11
View File
@@ -33,13 +33,18 @@ on:
runtime_image:
description: Aliyun runtime image (tag or digest reference)
type: string
default: greptime-registry.cn-hangzhou.cr.aliyuncs.com/tools/o11ybench-runtime:20260917-65906ca9@sha256:5af45e298019474d4c5089d63031a15f9bceb663890dd87fa0a56876c99c8597
default: greptime-registry.cn-hangzhou.cr.aliyuncs.com/tools/o11ybench-runtime:20260918-545dc570@sha256:0ff03e7e690575e875bb85b3054a03486dd9359073546dc715da177153cf7a29
required: true
ecs_instance_type:
description: ECS instance type (independent of Query Regression)
type: string
default: ecs.c9i.2xlarge
required: true
benchmark_root:
description: Absolute disk-backed work directory (outside the o11ybench checkout)
type: string
default: /home/runner/benchmark-data
required: true
system_disk_gib:
description: System disk GiB (20..2048)
type: number
@@ -68,6 +73,7 @@ on:
permissions:
contents: read
env:
BENCHMARK_ROOT: ${{ inputs.benchmark_root }}
RANGE_PRESET: ${{ inputs.range_preset }}
GREPTIMEDB_TAG: ${{ inputs.greptimedb_tag }}
VICTORIAMETRICS_TAG: ${{ inputs.victoriametrics_tag }}
@@ -103,6 +109,10 @@ jobs:
run: |
set -euo pipefail
[[ -n "$ECS_INSTANCE_TYPE" ]]
[[ "$BENCHMARK_ROOT" == /* && "$BENCHMARK_ROOT" != / && "$BENCHMARK_ROOT" != */ ]]
[[ "$BENCHMARK_ROOT" != /tmp && "$BENCHMARK_ROOT" != /tmp/* ]]
[[ "$BENCHMARK_ROOT" != *$'\n'* && "$BENCHMARK_ROOT" != *$'\r'* ]]
[[ "/${BENCHMARK_ROOT#/}/" != */../* && "/${BENCHMARK_ROOT#/}/" != */./* ]]
for value in "$SYSTEM_DISK_GIB" "$BENCHMARK_TIMEOUT_MINUTES" "$JANITOR_TTL_HOURS"; do
[[ "$value" =~ ^[1-9][0-9]{0,3}$ ]] || { echo 'Resource budgets must be positive integers'; exit 1; }
done
@@ -136,7 +146,7 @@ jobs:
# Keep the original warm defaults: serial 5/1 and c1/c4 30s/5s.
docker run --rm "$resolved" scripts/o11ybench_plan.py "${args[@]}" \
--run-id "long-range-$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT-warm" \
--output-root "/tmp/o11ybench-long-range-$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT-warm" \
--output-root "$BENCHMARK_ROOT/o11ybench/o11ybench-long-range-$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT-warm" \
--cache-policy warm > "$RUNNER_TEMP/vmbench-plans/warm.json"
- uses: actions/upload-artifact@v4
with:
@@ -185,10 +195,12 @@ jobs:
RESOLVED_RUNTIME_IMAGE: ${{ needs.validate.outputs.runtime_image }}
RESOLVED_O11YBENCH_REF: ${{ needs.validate.outputs.o11ybench_ref }}
RUN_ID: ${{ needs.validate.outputs.run_id }}
ARTIFACT_ROOT: /tmp/o11ybench-${{ needs.validate.outputs.run_id }}
ARTIFACT_ROOT: ${{ inputs.benchmark_root }}/o11ybench/o11ybench-${{ needs.validate.outputs.run_id }}
METADATA_ROOT: ${{ github.workspace }}/vmbench-metadata
O11YBENCH_DIR: ${{ github.workspace }}/o11ybench
TSDG_PARALLELISM: '8'
TSDG_PARALLELISM: '4'
TSDG_BATCH_SIZE: '100000'
O11YBENCH_GENERATOR_CPUS: '4'
steps:
- name: Checkout matching benchmark tools
uses: actions/checkout@v4
@@ -209,9 +221,11 @@ jobs:
test "$(git -C "$O11YBENCH_DIR" rev-parse HEAD)" = "$RESOLVED_O11YBENCH_REF"
jq --version
docker info
df -h /tmp /var/lib/docker
mkdir -p "$BENCHMARK_ROOT/o11ybench"
test "$(stat -f -c %T "$BENCHMARK_ROOT")" != tmpfs || { echo 'Benchmark root must be disk-backed'; exit 1; }
df -h "$BENCHMARK_ROOT" /var/lib/docker
# Fail before generation if either dataset space or Docker storage is too small.
for path in /tmp /var/lib/docker; do
for path in "$BENCHMARK_ROOT" /var/lib/docker; do
available=$(df -Pk "$path" | awk 'NR == 2 {print $4}')
((available >= 10 * 1024 * 1024)) || { echo "Need at least 10 GiB free on $path"; exit 1; }
done
@@ -245,16 +259,24 @@ jobs:
if: always()
run: |
docker ps -aq --filter "label=o11ybench.run-id=$RUN_ID-warm" | xargs -r docker rm -fv
- name: Stage benchmark artifacts
if: always()
run: |
mkdir -p "$RUNNER_TEMP/vmbench-artifacts"
if [[ -d "$METADATA_ROOT" ]]; then
rsync -a --no-owner --no-group "$METADATA_ROOT" "$RUNNER_TEMP/vmbench-artifacts/"
fi
if [[ -d "$ARTIFACT_ROOT-warm" ]]; then
# Exclude DB directories before traversal; they may be unreadable by the runner.
rsync -a --no-owner --no-group --exclude='db-data/' --exclude='/data/*.bin' \
"$ARTIFACT_ROOT-warm/" "$RUNNER_TEMP/vmbench-artifacts/$(basename "$ARTIFACT_ROOT-warm")/"
fi
- name: Upload HTML and results (no dataset or DB files)
if: always()
uses: actions/upload-artifact@v4
with:
name: vmbench-long-range-${{ github.run_id }}-${{ github.run_attempt }}
path: |
${{ env.METADATA_ROOT }}
${{ env.ARTIFACT_ROOT }}-warm
!${{ env.ARTIFACT_ROOT }}-warm/data/*.bin
!${{ env.ARTIFACT_ROOT }}-warm/**/db-data/**
path: ${{ runner.temp }}/vmbench-artifacts
if-no-files-found: warn
retention-days: 14
teardown:
AGENTS.md
+11 -1
View File
@@ -66,6 +66,7 @@ a module map, read/write paths, change-coupling points, and gotchas:
- [`src/flow/AGENTS.md`](src/flow/AGENTS.md)
- [`src/frontend/AGENTS.md`](src/frontend/AGENTS.md)
- [`src/meta-srv/AGENTS.md`](src/meta-srv/AGENTS.md)
- [`tests-integration/AGENTS.md`](tests-integration/AGENTS.md)
- [`tests/compatibility/AGENTS.md`](tests/compatibility/AGENTS.md)
- [`tests/perf/AGENTS.md`](tests/perf/AGENTS.md)
@@ -117,6 +118,10 @@ blast radius requires it.
| Query regression harness or DSL | Follow `tests/perf/AGENTS.md` |
| Enterprise-gated code | Build/test with `--features enterprise` where applicable and run `make check-enterprise-license` |
For import/export, COPY, or snapshot-storage changes, follow the Windows
portability and validation guidance in
[`tests-integration/AGENTS.md`](tests-integration/AGENTS.md).
## Before opening a PR
1. If you added a `.rs`, `.py`, or `.ts` file, apply and verify its license
@@ -140,7 +145,12 @@ blast radius requires it.
10. When creating or updating a pull request, follow
[`.github/pull_request_template.md`](.github/pull_request_template.md): include
the CLA statement, fill the change-intention section with enough detail, and
update checklist items accurately.
update checklist items accurately. "This PR requires documentation updates"
refers to the official docs site repository
([`GreptimeTeam/docs`](https://github.com/GreptimeTeam/docs)), not to rustdoc
or comments in this repo. Check it only when that site would be wrong or
incomplete without a matching change; a change users cannot observe does not
need it.
11. If the change must also land on release branches, add the matching
`backport-<target>` labels (e.g. `backport-v1.3` targets `release/v1.3`). The
Backport workflow (`.github/workflows/backport.yml`) cherry-picks a merged PR
Cargo.lock
Generated
+168 -153
View File
@@ -653,7 +653,7 @@ dependencies = [
[[package]]
name = "arrow-pg"
version = "0.15.0"
source = "git+https://github.com/GreptimeTeam/datafusion-postgres.git?rev=3c77e6c32b8db80635a0d2f4b318a36b31170bc3#3c77e6c32b8db80635a0d2f4b318a36b31170bc3"
source = "git+https://github.com/datafusion-contrib/datafusion-postgres.git?rev=ffb14a52b3d7c7489559812e1ec38096f225ce93#ffb14a52b3d7c7489559812e1ec38096f225ce93"
dependencies = [
"arrow 59.2.0",
"arrow-schema 59.2.0",
@@ -912,6 +912,12 @@ dependencies = [
"thiserror 2.0.17",
]
[[package]]
name = "asyncband"
version = "0.7.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f2d85fd3d291fabcc40c7232c92c280ec1754fd7b5d7ea769f143222143e179a"
[[package]]
name = "asynchronous-codec"
version = "0.7.0"
@@ -2010,6 +2016,7 @@ dependencies = [
"common-base",
"common-catalog",
"common-config",
"common-datasource",
"common-error",
"common-grpc",
"common-macro",
@@ -2043,6 +2050,7 @@ dependencies = [
"servers",
"session",
"snafu 0.8.6",
"sql",
"standalone",
"store-api",
"table",
@@ -2239,7 +2247,7 @@ checksum = "fe6d2e5af09e8c8ad56c969f2157a3d4238cebc7c55f0a517728c38f7b200f81"
dependencies = [
"serde",
"termcolor",
"unicode-width 0.2.1",
"unicode-width 0.1.14",
]
[[package]]
@@ -2250,7 +2258,7 @@ checksum = "af491d569909a7e4dee0ad7db7f5341fef5c614d5b8ec8cf765732aba3cff681"
dependencies = [
"serde",
"termcolor",
"unicode-width 0.2.1",
"unicode-width 0.1.14",
]
[[package]]
@@ -2266,7 +2274,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "117725a109d387c937a1533ce01b450cbde6b88abceea8473c4d7a85853cda3c"
dependencies = [
"lazy_static",
"windows-sys 0.48.0",
"windows-sys 0.59.0",
]
[[package]]
@@ -2471,6 +2479,7 @@ dependencies = [
"arrow-cast 59.2.0",
"arrow-schema 59.2.0",
"async-trait",
"axum 0.8.4",
"bincode",
"catalog",
"chrono",
@@ -2510,6 +2519,7 @@ dependencies = [
"paste",
"pretty_assertions",
"regex",
"reqwest 0.13.4",
"s2",
"serde",
"serde_json",
@@ -3771,8 +3781,8 @@ checksum = "a4ae5f15dda3c708c0ade84bfee31ccab44a3da4f88015ed22f63732abe300c8"
[[package]]
name = "datafusion"
version = "55.0.0"
source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=b66dca3260c8e04f314c8036cdfbb08f1a564d95#b66dca3260c8e04f314c8036cdfbb08f1a564d95"
version = "55.1.0"
source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=2aa87d52cdce7006af492330064738f33ed294c1#2aa87d52cdce7006af492330064738f33ed294c1"
dependencies = [
"arrow 59.2.0",
"arrow-schema 59.2.0",
@@ -3823,8 +3833,8 @@ dependencies = [
[[package]]
name = "datafusion-catalog"
version = "55.0.0"
source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=b66dca3260c8e04f314c8036cdfbb08f1a564d95#b66dca3260c8e04f314c8036cdfbb08f1a564d95"
version = "55.1.0"
source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=2aa87d52cdce7006af492330064738f33ed294c1#2aa87d52cdce7006af492330064738f33ed294c1"
dependencies = [
"arrow 59.2.0",
"async-trait",
@@ -3847,8 +3857,8 @@ dependencies = [
[[package]]
name = "datafusion-catalog-listing"
version = "55.0.0"
source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=b66dca3260c8e04f314c8036cdfbb08f1a564d95#b66dca3260c8e04f314c8036cdfbb08f1a564d95"
version = "55.1.0"
source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=2aa87d52cdce7006af492330064738f33ed294c1#2aa87d52cdce7006af492330064738f33ed294c1"
dependencies = [
"arrow 59.2.0",
"async-trait",
@@ -3870,8 +3880,8 @@ dependencies = [
[[package]]
name = "datafusion-common"
version = "55.0.0"
source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=b66dca3260c8e04f314c8036cdfbb08f1a564d95#b66dca3260c8e04f314c8036cdfbb08f1a564d95"
version = "55.1.0"
source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=2aa87d52cdce7006af492330064738f33ed294c1#2aa87d52cdce7006af492330064738f33ed294c1"
dependencies = [
"arrow 59.2.0",
"arrow-ipc 59.2.0",
@@ -3896,8 +3906,8 @@ dependencies = [
[[package]]
name = "datafusion-common-runtime"
version = "55.0.0"
source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=b66dca3260c8e04f314c8036cdfbb08f1a564d95#b66dca3260c8e04f314c8036cdfbb08f1a564d95"
version = "55.1.0"
source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=2aa87d52cdce7006af492330064738f33ed294c1#2aa87d52cdce7006af492330064738f33ed294c1"
dependencies = [
"futures",
"log",
@@ -3906,8 +3916,8 @@ dependencies = [
[[package]]
name = "datafusion-datasource"
version = "55.0.0"
source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=b66dca3260c8e04f314c8036cdfbb08f1a564d95#b66dca3260c8e04f314c8036cdfbb08f1a564d95"
version = "55.1.0"
source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=2aa87d52cdce7006af492330064738f33ed294c1#2aa87d52cdce7006af492330064738f33ed294c1"
dependencies = [
"arrow 59.2.0",
"async-compression",
@@ -3942,8 +3952,8 @@ dependencies = [
[[package]]
name = "datafusion-datasource-arrow"
version = "55.0.0"
source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=b66dca3260c8e04f314c8036cdfbb08f1a564d95#b66dca3260c8e04f314c8036cdfbb08f1a564d95"
version = "55.1.0"
source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=2aa87d52cdce7006af492330064738f33ed294c1#2aa87d52cdce7006af492330064738f33ed294c1"
dependencies = [
"arrow 59.2.0",
"arrow-ipc 59.2.0",
@@ -3966,8 +3976,8 @@ dependencies = [
[[package]]
name = "datafusion-datasource-csv"
version = "55.0.0"
source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=b66dca3260c8e04f314c8036cdfbb08f1a564d95#b66dca3260c8e04f314c8036cdfbb08f1a564d95"
version = "55.1.0"
source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=2aa87d52cdce7006af492330064738f33ed294c1#2aa87d52cdce7006af492330064738f33ed294c1"
dependencies = [
"arrow 59.2.0",
"async-trait",
@@ -3989,8 +3999,8 @@ dependencies = [
[[package]]
name = "datafusion-datasource-json"
version = "55.0.0"
source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=b66dca3260c8e04f314c8036cdfbb08f1a564d95#b66dca3260c8e04f314c8036cdfbb08f1a564d95"
version = "55.1.0"
source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=2aa87d52cdce7006af492330064738f33ed294c1#2aa87d52cdce7006af492330064738f33ed294c1"
dependencies = [
"arrow 59.2.0",
"async-trait",
@@ -4012,8 +4022,8 @@ dependencies = [
[[package]]
name = "datafusion-datasource-parquet"
version = "55.0.0"
source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=b66dca3260c8e04f314c8036cdfbb08f1a564d95#b66dca3260c8e04f314c8036cdfbb08f1a564d95"
version = "55.1.0"
source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=2aa87d52cdce7006af492330064738f33ed294c1#2aa87d52cdce7006af492330064738f33ed294c1"
dependencies = [
"arrow 59.2.0",
"arrow-schema 59.2.0",
@@ -4044,13 +4054,13 @@ dependencies = [
[[package]]
name = "datafusion-doc"
version = "55.0.0"
source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=b66dca3260c8e04f314c8036cdfbb08f1a564d95#b66dca3260c8e04f314c8036cdfbb08f1a564d95"
version = "55.1.0"
source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=2aa87d52cdce7006af492330064738f33ed294c1#2aa87d52cdce7006af492330064738f33ed294c1"
[[package]]
name = "datafusion-execution"
version = "55.0.0"
source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=b66dca3260c8e04f314c8036cdfbb08f1a564d95#b66dca3260c8e04f314c8036cdfbb08f1a564d95"
version = "55.1.0"
source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=2aa87d52cdce7006af492330064738f33ed294c1#2aa87d52cdce7006af492330064738f33ed294c1"
dependencies = [
"arrow 59.2.0",
"arrow-buffer 59.2.0",
@@ -4074,8 +4084,8 @@ dependencies = [
[[package]]
name = "datafusion-expr"
version = "55.0.0"
source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=b66dca3260c8e04f314c8036cdfbb08f1a564d95#b66dca3260c8e04f314c8036cdfbb08f1a564d95"
version = "55.1.0"
source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=2aa87d52cdce7006af492330064738f33ed294c1#2aa87d52cdce7006af492330064738f33ed294c1"
dependencies = [
"arrow 59.2.0",
"arrow-schema 59.2.0",
@@ -4098,8 +4108,8 @@ dependencies = [
[[package]]
name = "datafusion-expr-common"
version = "55.0.0"
source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=b66dca3260c8e04f314c8036cdfbb08f1a564d95#b66dca3260c8e04f314c8036cdfbb08f1a564d95"
version = "55.1.0"
source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=2aa87d52cdce7006af492330064738f33ed294c1#2aa87d52cdce7006af492330064738f33ed294c1"
dependencies = [
"arrow 59.2.0",
"datafusion-common",
@@ -4109,8 +4119,8 @@ dependencies = [
[[package]]
name = "datafusion-functions"
version = "55.0.0"
source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=b66dca3260c8e04f314c8036cdfbb08f1a564d95#b66dca3260c8e04f314c8036cdfbb08f1a564d95"
version = "55.1.0"
source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=2aa87d52cdce7006af492330064738f33ed294c1#2aa87d52cdce7006af492330064738f33ed294c1"
dependencies = [
"arrow 59.2.0",
"arrow-buffer 59.2.0",
@@ -4140,8 +4150,8 @@ dependencies = [
[[package]]
name = "datafusion-functions-aggregate"
version = "55.0.0"
source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=b66dca3260c8e04f314c8036cdfbb08f1a564d95#b66dca3260c8e04f314c8036cdfbb08f1a564d95"
version = "55.1.0"
source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=2aa87d52cdce7006af492330064738f33ed294c1#2aa87d52cdce7006af492330064738f33ed294c1"
dependencies = [
"arrow 59.2.0",
"datafusion-common",
@@ -4160,8 +4170,8 @@ dependencies = [
[[package]]
name = "datafusion-functions-aggregate-common"
version = "55.0.0"
source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=b66dca3260c8e04f314c8036cdfbb08f1a564d95#b66dca3260c8e04f314c8036cdfbb08f1a564d95"
version = "55.1.0"
source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=2aa87d52cdce7006af492330064738f33ed294c1#2aa87d52cdce7006af492330064738f33ed294c1"
dependencies = [
"arrow 59.2.0",
"datafusion-common",
@@ -4171,8 +4181,8 @@ dependencies = [
[[package]]
name = "datafusion-functions-nested"
version = "55.0.0"
source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=b66dca3260c8e04f314c8036cdfbb08f1a564d95#b66dca3260c8e04f314c8036cdfbb08f1a564d95"
version = "55.1.0"
source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=2aa87d52cdce7006af492330064738f33ed294c1#2aa87d52cdce7006af492330064738f33ed294c1"
dependencies = [
"arrow 59.2.0",
"arrow-ord 59.2.0",
@@ -4195,8 +4205,8 @@ dependencies = [
[[package]]
name = "datafusion-functions-table"
version = "55.0.0"
source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=b66dca3260c8e04f314c8036cdfbb08f1a564d95#b66dca3260c8e04f314c8036cdfbb08f1a564d95"
version = "55.1.0"
source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=2aa87d52cdce7006af492330064738f33ed294c1#2aa87d52cdce7006af492330064738f33ed294c1"
dependencies = [
"arrow 59.2.0",
"async-trait",
@@ -4210,8 +4220,8 @@ dependencies = [
[[package]]
name = "datafusion-functions-window"
version = "55.0.0"
source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=b66dca3260c8e04f314c8036cdfbb08f1a564d95#b66dca3260c8e04f314c8036cdfbb08f1a564d95"
version = "55.1.0"
source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=2aa87d52cdce7006af492330064738f33ed294c1#2aa87d52cdce7006af492330064738f33ed294c1"
dependencies = [
"arrow 59.2.0",
"datafusion-common",
@@ -4226,8 +4236,8 @@ dependencies = [
[[package]]
name = "datafusion-functions-window-common"
version = "55.0.0"
source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=b66dca3260c8e04f314c8036cdfbb08f1a564d95#b66dca3260c8e04f314c8036cdfbb08f1a564d95"
version = "55.1.0"
source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=2aa87d52cdce7006af492330064738f33ed294c1#2aa87d52cdce7006af492330064738f33ed294c1"
dependencies = [
"datafusion-common",
"datafusion-physical-expr-common",
@@ -4235,8 +4245,8 @@ dependencies = [
[[package]]
name = "datafusion-macros"
version = "55.0.0"
source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=b66dca3260c8e04f314c8036cdfbb08f1a564d95#b66dca3260c8e04f314c8036cdfbb08f1a564d95"
version = "55.1.0"
source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=2aa87d52cdce7006af492330064738f33ed294c1#2aa87d52cdce7006af492330064738f33ed294c1"
dependencies = [
"datafusion-doc",
"quote",
@@ -4245,8 +4255,8 @@ dependencies = [
[[package]]
name = "datafusion-optimizer"
version = "55.0.0"
source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=b66dca3260c8e04f314c8036cdfbb08f1a564d95#b66dca3260c8e04f314c8036cdfbb08f1a564d95"
version = "55.1.0"
source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=2aa87d52cdce7006af492330064738f33ed294c1#2aa87d52cdce7006af492330064738f33ed294c1"
dependencies = [
"arrow 59.2.0",
"chrono",
@@ -4280,7 +4290,7 @@ dependencies = [
[[package]]
name = "datafusion-pg-catalog"
version = "0.18.3"
source = "git+https://github.com/GreptimeTeam/datafusion-postgres.git?rev=3c77e6c32b8db80635a0d2f4b318a36b31170bc3#3c77e6c32b8db80635a0d2f4b318a36b31170bc3"
source = "git+https://github.com/datafusion-contrib/datafusion-postgres.git?rev=ffb14a52b3d7c7489559812e1ec38096f225ce93#ffb14a52b3d7c7489559812e1ec38096f225ce93"
dependencies = [
"arrow-pg",
"async-trait",
@@ -4293,10 +4303,11 @@ dependencies = [
[[package]]
name = "datafusion-physical-expr"
version = "55.0.0"
source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=b66dca3260c8e04f314c8036cdfbb08f1a564d95#b66dca3260c8e04f314c8036cdfbb08f1a564d95"
version = "55.1.0"
source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=2aa87d52cdce7006af492330064738f33ed294c1#2aa87d52cdce7006af492330064738f33ed294c1"
dependencies = [
"arrow 59.2.0",
"arrow-schema 59.2.0",
"datafusion-common",
"datafusion-expr",
"datafusion-expr-common",
@@ -4315,8 +4326,8 @@ dependencies = [
[[package]]
name = "datafusion-physical-expr-adapter"
version = "55.0.0"
source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=b66dca3260c8e04f314c8036cdfbb08f1a564d95#b66dca3260c8e04f314c8036cdfbb08f1a564d95"
version = "55.1.0"
source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=2aa87d52cdce7006af492330064738f33ed294c1#2aa87d52cdce7006af492330064738f33ed294c1"
dependencies = [
"arrow 59.2.0",
"datafusion-common",
@@ -4329,8 +4340,8 @@ dependencies = [
[[package]]
name = "datafusion-physical-expr-common"
version = "55.0.0"
source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=b66dca3260c8e04f314c8036cdfbb08f1a564d95#b66dca3260c8e04f314c8036cdfbb08f1a564d95"
version = "55.1.0"
source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=2aa87d52cdce7006af492330064738f33ed294c1#2aa87d52cdce7006af492330064738f33ed294c1"
dependencies = [
"arrow 59.2.0",
"chrono",
@@ -4346,8 +4357,8 @@ dependencies = [
[[package]]
name = "datafusion-physical-optimizer"
version = "55.0.0"
source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=b66dca3260c8e04f314c8036cdfbb08f1a564d95#b66dca3260c8e04f314c8036cdfbb08f1a564d95"
version = "55.1.0"
source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=2aa87d52cdce7006af492330064738f33ed294c1#2aa87d52cdce7006af492330064738f33ed294c1"
dependencies = [
"arrow 59.2.0",
"datafusion-common",
@@ -4365,8 +4376,8 @@ dependencies = [
[[package]]
name = "datafusion-physical-plan"
version = "55.0.0"
source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=b66dca3260c8e04f314c8036cdfbb08f1a564d95#b66dca3260c8e04f314c8036cdfbb08f1a564d95"
version = "55.1.0"
source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=2aa87d52cdce7006af492330064738f33ed294c1#2aa87d52cdce7006af492330064738f33ed294c1"
dependencies = [
"arrow 59.2.0",
"arrow-data 59.2.0",
@@ -4402,8 +4413,8 @@ dependencies = [
[[package]]
name = "datafusion-proto"
version = "55.0.0"
source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=b66dca3260c8e04f314c8036cdfbb08f1a564d95#b66dca3260c8e04f314c8036cdfbb08f1a564d95"
version = "55.1.0"
source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=2aa87d52cdce7006af492330064738f33ed294c1#2aa87d52cdce7006af492330064738f33ed294c1"
dependencies = [
"arrow 59.2.0",
"datafusion-catalog",
@@ -4428,8 +4439,8 @@ dependencies = [
[[package]]
name = "datafusion-proto-common"
version = "55.0.0"
source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=b66dca3260c8e04f314c8036cdfbb08f1a564d95#b66dca3260c8e04f314c8036cdfbb08f1a564d95"
version = "55.1.0"
source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=2aa87d52cdce7006af492330064738f33ed294c1#2aa87d52cdce7006af492330064738f33ed294c1"
dependencies = [
"arrow 59.2.0",
"datafusion-common",
@@ -4438,8 +4449,8 @@ dependencies = [
[[package]]
name = "datafusion-proto-models"
version = "55.0.0"
source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=b66dca3260c8e04f314c8036cdfbb08f1a564d95#b66dca3260c8e04f314c8036cdfbb08f1a564d95"
version = "55.1.0"
source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=2aa87d52cdce7006af492330064738f33ed294c1#2aa87d52cdce7006af492330064738f33ed294c1"
dependencies = [
"datafusion-common",
"datafusion-proto-common",
@@ -4448,8 +4459,8 @@ dependencies = [
[[package]]
name = "datafusion-pruning"
version = "55.0.0"
source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=b66dca3260c8e04f314c8036cdfbb08f1a564d95#b66dca3260c8e04f314c8036cdfbb08f1a564d95"
version = "55.1.0"
source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=2aa87d52cdce7006af492330064738f33ed294c1#2aa87d52cdce7006af492330064738f33ed294c1"
dependencies = [
"arrow 59.2.0",
"datafusion-common",
@@ -4463,8 +4474,8 @@ dependencies = [
[[package]]
name = "datafusion-session"
version = "55.0.0"
source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=b66dca3260c8e04f314c8036cdfbb08f1a564d95#b66dca3260c8e04f314c8036cdfbb08f1a564d95"
version = "55.1.0"
source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=2aa87d52cdce7006af492330064738f33ed294c1#2aa87d52cdce7006af492330064738f33ed294c1"
dependencies = [
"arrow-schema 59.2.0",
"async-trait",
@@ -4477,8 +4488,8 @@ dependencies = [
[[package]]
name = "datafusion-sql"
version = "55.0.0"
source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=b66dca3260c8e04f314c8036cdfbb08f1a564d95#b66dca3260c8e04f314c8036cdfbb08f1a564d95"
version = "55.1.0"
source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=2aa87d52cdce7006af492330064738f33ed294c1#2aa87d52cdce7006af492330064738f33ed294c1"
dependencies = [
"arrow 59.2.0",
"bigdecimal 0.4.8",
@@ -4496,8 +4507,8 @@ dependencies = [
[[package]]
name = "datafusion-substrait"
version = "55.0.0"
source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=b66dca3260c8e04f314c8036cdfbb08f1a564d95#b66dca3260c8e04f314c8036cdfbb08f1a564d95"
version = "55.1.0"
source = "git+https://github.com/GreptimeTeam/datafusion.git?rev=2aa87d52cdce7006af492330064738f33ed294c1#2aa87d52cdce7006af492330064738f33ed294c1"
dependencies = [
"async-recursion",
"async-trait",
@@ -5164,7 +5175,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "778e2ac28f6c47af28e4907f13ffd1e1ddbd400980a9abd7c8df189bf578a5ad"
dependencies = [
"libc",
"windows-sys 0.52.0",
"windows-sys 0.60.2",
]
[[package]]
@@ -5764,7 +5775,7 @@ checksum = "94e7099f6313ecacbe1256e8ff9d617b75d1bcb16a6fddef94866d225a01a14a"
dependencies = [
"io-lifetimes 2.0.4",
"rustix 1.0.7",
"windows-sys 0.52.0",
"windows-sys 0.59.0",
]
[[package]]
@@ -6119,7 +6130,7 @@ dependencies = [
[[package]]
name = "greptime-proto"
version = "0.1.0"
source = "git+https://github.com/GreptimeTeam/greptime-proto.git?rev=0c4beffaa2b3968034915867593a717c6ab0c6b4#0c4beffaa2b3968034915867593a717c6ab0c6b4"
source = "git+https://github.com/GreptimeTeam/greptime-proto.git?rev=3d232f406fad574e8c630fb1c304fe2ebb013c9c#3d232f406fad574e8c630fb1c304fe2ebb013c9c"
dependencies = [
"prost 0.14.1",
"prost-types 0.14.1",
@@ -6672,7 +6683,7 @@ dependencies = [
"libc",
"percent-encoding",
"pin-project-lite",
"socket2 0.6.4",
"socket2 0.5.10",
"tokio",
"tower-service",
"tracing",
@@ -6743,7 +6754,7 @@ dependencies = [
"js-sys",
"log",
"wasm-bindgen",
"windows-core 0.61.2",
"windows-core 0.57.0",
]
[[package]]
@@ -7165,7 +7176,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "20fd6de4ccfcc187e38bc21cfa543cb5a302cb86a8b114eb7f0bf0dc9f8ac00f"
dependencies = [
"io-lifetimes 3.0.1",
"windows-sys 0.52.0",
"windows-sys 0.60.2",
]
[[package]]
@@ -7203,7 +7214,7 @@ checksum = "e04d7f318608d35d4b61ddd75cbdaee86b023ebe2bd5a66ee0915f0bf93095a9"
dependencies = [
"hermit-abi",
"libc",
"windows-sys 0.52.0",
"windows-sys 0.59.0",
]
[[package]]
@@ -7932,7 +7943,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "07033963ba89ebaf1584d767badaa2e8fcec21aedea6b8c0346d487d49c28667"
dependencies = [
"cfg-if",
"windows-targets 0.48.5",
"windows-targets 0.53.5",
]
[[package]]
@@ -8555,7 +8566,7 @@ dependencies = [
[[package]]
name = "meter-core"
version = "0.1.0"
source = "git+https://github.com/GreptimeTeam/greptime-meter.git?rev=5618e779cf2bb4755b499c630fba4c35e91898cb#5618e779cf2bb4755b499c630fba4c35e91898cb"
source = "git+https://github.com/GreptimeTeam/greptime-meter.git?rev=f743c3a5fe8c1f57363eac5866c356b7d91f9b39#f743c3a5fe8c1f57363eac5866c356b7d91f9b39"
dependencies = [
"anymap2",
"once_cell",
@@ -8566,7 +8577,7 @@ dependencies = [
[[package]]
name = "meter-macros"
version = "0.1.0"
source = "git+https://github.com/GreptimeTeam/greptime-meter.git?rev=5618e779cf2bb4755b499c630fba4c35e91898cb#5618e779cf2bb4755b499c630fba4c35e91898cb"
source = "git+https://github.com/GreptimeTeam/greptime-meter.git?rev=f743c3a5fe8c1f57363eac5866c356b7d91f9b39#f743c3a5fe8c1f57363eac5866c356b7d91f9b39"
dependencies = [
"meter-core",
]
@@ -9454,7 +9465,7 @@ version = "0.7.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "77e878c846a8abae00dd069496dbe8751b16ac1c3d6bd2a7283a938e8228f90d"
dependencies = [
"proc-macro-crate 3.3.0",
"proc-macro-crate 1.3.1",
"proc-macro2",
"quote",
"syn 2.0.117",
@@ -9609,8 +9620,7 @@ dependencies = [
[[package]]
name = "object_store_opendal"
version = "0.58.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "88f165780495c17aa3ce86846600504198c3fffd99073521552751c2430fa6ac"
source = "git+https://github.com/GreptimeTeam/opendal.git?rev=c03e56802b47de3ccfc8fe028e59a459b278a0d2#c03e56802b47de3ccfc8fe028e59a459b278a0d2"
dependencies = [
"async-trait",
"bytes",
@@ -9697,9 +9707,9 @@ checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381"
[[package]]
name = "opendal"
version = "0.58.1"
version = "0.59.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4f20562cc7447fcc915fc5c23df305a412ea80a733c9f2fd9e2d267e2815be6d"
checksum = "9fe43e16d96bed57937eb7c4a28559bf4c5fc4a3951656505d1fdd4f856349b4"
dependencies = [
"ctor",
"opendal-core",
@@ -9721,11 +9731,12 @@ dependencies = [
[[package]]
name = "opendal-core"
version = "0.58.1"
version = "0.59.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ec75551ff4cf3e57da98979f6a937aaa9ddb3915bf68cc17d03df733be6646ed"
checksum = "de4566a412776e3f65d53dd9fceced5b432a0a9c9a1e8d50ccf58823795e5c5b"
dependencies = [
"anyhow",
"asyncband",
"base64 0.23.1",
"bytes",
"futures",
@@ -9733,7 +9744,6 @@ dependencies = [
"jiff",
"log",
"md-5 0.11.0",
"mea",
"percent-encoding",
"quick-xml 0.41.0",
"reqsign-core",
@@ -9747,9 +9757,9 @@ dependencies = [
[[package]]
name = "opendal-http-transport-reqwest"
version = "0.58.1"
version = "0.59.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ad4d4f19c3ce01126a30611f8e544eaa217104a278c889ac17c9374fe4f9e4ef"
checksum = "f772ce6c137ab43647726116d505a649a88d41928fcd813438eb30b02312bcbb"
dependencies = [
"bytes",
"futures",
@@ -9761,21 +9771,21 @@ dependencies = [
[[package]]
name = "opendal-layer-concurrent-limit"
version = "0.58.1"
version = "0.59.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "249ac5b0aa5a7a6c3737342d10456067937f9c9a6f3f02544271f7908ab91081"
checksum = "3734c648617e5ad724d49175dde8323550624f9614f50e9e81b7aafae30b1734"
dependencies = [
"asyncband",
"futures",
"http 1.5.0",
"mea",
"opendal-core",
]
[[package]]
name = "opendal-layer-logging"
version = "0.58.1"
version = "0.59.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5c75411ab00f77851ff086b686c1e9ca8175ac18c15afa2cb75b9036436cb06c"
checksum = "0a8daab5a91eac76f94516e7c71e677195c9221840ce8f5e71815e3fc8207742"
dependencies = [
"log",
"opendal-core",
@@ -9783,9 +9793,9 @@ dependencies = [
[[package]]
name = "opendal-layer-observe-metrics-common"
version = "0.58.1"
version = "0.59.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "12dbc31b27ee8e5f658ae6c8514b8276059e5235e2f85138a42e6ed172324876"
checksum = "0fbe664be117c293cfa3980a0d2fc238bbb38eae5f6c191f376e88ccb830d26b"
dependencies = [
"futures",
"http 1.5.0",
@@ -9794,9 +9804,9 @@ dependencies = [
[[package]]
name = "opendal-layer-prometheus"
version = "0.58.1"
version = "0.59.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c14e6cb0b3ea8d83bef751613b8bf4512678afe8b49c2c96a633515bc0b34d79"
checksum = "b32078d8248555ee6271332ec480b78306a44f0d902ea4721ccac00d9b866a21"
dependencies = [
"opendal-core",
"opendal-layer-observe-metrics-common",
@@ -9805,9 +9815,9 @@ dependencies = [
[[package]]
name = "opendal-layer-retry"
version = "0.58.1"
version = "0.59.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "80b7738bd5f233ad8da39af9b9316b9b7a4eaddd91e8e32a1e19b7030688121d"
checksum = "eb8e22cb1f86ef9af771a8947d3ccf380c8f5b76236f628314dd41c9ba73331a"
dependencies = [
"backon",
"log",
@@ -9816,9 +9826,9 @@ dependencies = [
[[package]]
name = "opendal-layer-timeout"
version = "0.58.1"
version = "0.59.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a704141924500f3803c05ed871b53305d2a2f11cb5ef20160c3ee688a1857f66"
checksum = "08ac6c77fe0d1b18e0064a02c19b98dbebcab78bf5e7753ad1f3abbc18850ff4"
dependencies = [
"opendal-core",
"tokio",
@@ -9826,9 +9836,9 @@ dependencies = [
[[package]]
name = "opendal-layer-tracing"
version = "0.58.1"
version = "0.59.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4827cf8b2ebd9123b1fb7286c421be079948e2b963a452f1f7e8aee2c12e7997"
checksum = "603807927a799ea981f62a3b0012e17181e39f468793b78ca8bedda66d19813c"
dependencies = [
"futures",
"http 1.5.0",
@@ -9838,9 +9848,9 @@ dependencies = [
[[package]]
name = "opendal-service-azblob"
version = "0.58.1"
version = "0.59.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b3310fbbb48f111c6f590473c2cd15e1b7f8e384444b0d4e328f0464c864d767"
checksum = "c6cf31b02e7b44191d97c07254eea109b5da9c17210a29e478c8d0dc26dffb1f"
dependencies = [
"base64 0.23.1",
"bytes",
@@ -9859,9 +9869,9 @@ dependencies = [
[[package]]
name = "opendal-service-azure-common"
version = "0.58.1"
version = "0.59.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7348c88edf15af435b7be930077746b569fac5e738c1bf6a363b675e7317c9df"
checksum = "8107324c3f3cb9970fe20526c2c341b7c5769acbd4699bab73bd525f5fd63d62"
dependencies = [
"http 1.5.0",
"opendal-core",
@@ -9869,9 +9879,9 @@ dependencies = [
[[package]]
name = "opendal-service-fs"
version = "0.58.1"
version = "0.59.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "826c4e17a30643b888fe983897f9a4b23b07066e1d069727a923cc8fb419a702"
checksum = "a0e7d8974aebd33be899200c9d0d50b2141fd5fa5c462b29e20653e3ddbfe339"
dependencies = [
"bytes",
"log",
@@ -9883,9 +9893,9 @@ dependencies = [
[[package]]
name = "opendal-service-gcs"
version = "0.58.1"
version = "0.59.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "007f3fba63c21e516c956b891e96ff9892d8175662bfb781cdada9d3766a11e6"
checksum = "3d80eca6f9227bdf65d43e5b8d5fe75dca8cfab1da07571442a0420400847bea"
dependencies = [
"async-trait",
"bytes",
@@ -9900,13 +9910,14 @@ dependencies = [
"serde",
"serde_json",
"tokio",
"uuid",
]
[[package]]
name = "opendal-service-http"
version = "0.58.1"
version = "0.59.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d2fd1f0f28bd052e068d9bb4bc5d30a09de22a4f7cea24558dbeef2b91090cc3"
checksum = "9ac1553d6b57ca79e200e5c7ded9ab708f2016dadb28ede0f2da71e02fd0b960"
dependencies = [
"http 1.5.0",
"log",
@@ -9916,11 +9927,11 @@ dependencies = [
[[package]]
name = "opendal-service-mysql"
version = "0.58.1"
version = "0.59.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2d4a25f582068ce30ba24eb3efd4ac16ca833c87b5282d8d79634fdbead74d84"
checksum = "e15592be32b105cd152ab398b3db847e22c8c5f230d9b34ff095ea3d0dc6fd97"
dependencies = [
"mea",
"asyncband",
"opendal-core",
"serde",
"sqlx",
@@ -9928,9 +9939,9 @@ dependencies = [
[[package]]
name = "opendal-service-oss"
version = "0.58.1"
version = "0.59.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cd528ec2d49c5ca69e674ffed7b3e0686fb9cfcfea0596870de381467fda4f1b"
checksum = "a0ebfa355b1d4edb0e5b2a5b86d8de9f811fae1d7baca6d65246b230b2afc0bd"
dependencies = [
"bytes",
"http 1.5.0",
@@ -9945,9 +9956,9 @@ dependencies = [
[[package]]
name = "opendal-service-s3"
version = "0.58.1"
version = "0.59.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "58e80cdf192d7eff05feed747894d64f81905ac4eaf132edf7ea270abdd2d663"
checksum = "2942f2d8d3d4953c0e8d07cd1244879628d084948e48ce6b6d808c5938f7858d"
dependencies = [
"base64 0.23.1",
"bytes",
@@ -10698,9 +10709,9 @@ dependencies = [
[[package]]
name = "pg_interval"
version = "0.5.0"
version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c386dd54fce258fc04e668126ae68589a0d92e03a90ea67881d1300f70fd6170"
checksum = "8408fb1e08ebf2e7676d001caa6482d1b33e7e1f1dad9d3a758a05640eea2026"
dependencies = [
"bytes",
"chrono",
@@ -10709,9 +10720,9 @@ dependencies = [
[[package]]
name = "pgwire"
version = "0.40.7"
version = "0.41.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8265901ede50d0879fe401c6fe282e7e4ff83ae10a48c1f8e4f89b92f7d5f604"
checksum = "c667f8f4046889d52ed11e17aae954b1816d3df48dd23bdebcf199cef332df93"
dependencies = [
"async-trait",
"aws-lc-rs",
@@ -11485,7 +11496,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "22505a5c94da8e3b7c2996394d1c933236c4d743e81a410bcca4e6989fc066a4"
dependencies = [
"bytes",
"heck 0.5.0",
"heck 0.4.1",
"itertools 0.12.1",
"log",
"multimap",
@@ -11505,8 +11516,8 @@ version = "0.14.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ac6c3320f9abac597dcbc668774ef006702672474aad53c6d596b62e487b40b1"
dependencies = [
"heck 0.5.0",
"itertools 0.14.0",
"heck 0.4.1",
"itertools 0.10.5",
"log",
"multimap",
"once_cell",
@@ -11554,7 +11565,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d"
dependencies = [
"anyhow",
"itertools 0.14.0",
"itertools 0.10.5",
"proc-macro2",
"quote",
"syn 2.0.117",
@@ -11567,7 +11578,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9120690fafc389a67ba3803df527d0ec9cbbc9cc45e4cc20b332996dfb672425"
dependencies = [
"anyhow",
"itertools 0.14.0",
"itertools 0.10.5",
"proc-macro2",
"quote",
"syn 2.0.117",
@@ -12016,7 +12027,7 @@ dependencies = [
"once_cell",
"socket2 0.5.10",
"tracing",
"windows-sys 0.52.0",
"windows-sys 0.59.0",
]
[[package]]
@@ -12419,9 +12430,9 @@ dependencies = [
[[package]]
name = "reqsign-aws-core"
version = "3.0.3"
version = "3.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e4af084e1f3cbf3e67e0c972765399bce54ecec804cceba46b39a8331f3c1bff"
checksum = "bac4749b7dfa7bfaccd01eb03e9dc795ed37e3f20d6f0f38e2c67ee85ad6bc86"
dependencies = [
"bytes",
"form_urlencoded",
@@ -12440,9 +12451,9 @@ dependencies = [
[[package]]
name = "reqsign-aws-v4"
version = "3.1.0"
version = "3.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4ac5b3b7cefa28933792b439186459f77f19f9b6edbeab41b8b187150361a206"
checksum = "ff250f0fd0b913fbd565e405acc553da0f13bde30bfb5403178c9d0313cdc15f"
dependencies = [
"bytes",
"http 1.5.0",
@@ -12476,9 +12487,9 @@ dependencies = [
[[package]]
name = "reqsign-core"
version = "3.2.1"
version = "3.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c07dd510b1e1b9b241883e483358147fb2ed2d497a7b39b065ba61eb93deceb0"
checksum = "ff052daffb0599681c50f85c59e7236438976efe991ab864edd9f3b235501a0f"
dependencies = [
"anyhow",
"base64 0.23.1",
@@ -12489,6 +12500,7 @@ dependencies = [
"http 1.5.0",
"jiff",
"log",
"mea",
"percent-encoding",
"rsa",
"serde",
@@ -12500,9 +12512,9 @@ dependencies = [
[[package]]
name = "reqsign-file-read-tokio"
version = "3.0.4"
version = "3.0.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "663d9d55abd0df0830ef0ae43708297cc1371cf4e8ca91f3ac813c309cca8c98"
checksum = "b3235df90a6bca681aa47dd86f2393d122a6d77042aa8a7c81e218cd45c5bfc0"
dependencies = [
"anyhow",
"reqsign-core",
@@ -12981,7 +12993,7 @@ dependencies = [
"errno",
"libc",
"linux-raw-sys 0.4.15",
"windows-sys 0.52.0",
"windows-sys 0.59.0",
]
[[package]]
@@ -12994,7 +13006,7 @@ dependencies = [
"errno",
"libc",
"linux-raw-sys 0.9.4",
"windows-sys 0.52.0",
"windows-sys 0.59.0",
]
[[package]]
@@ -13085,7 +13097,7 @@ dependencies = [
"security-framework 3.7.0",
"security-framework-sys",
"webpki-root-certs",
"windows-sys 0.52.0",
"windows-sys 0.61.2",
]
[[package]]
@@ -13616,6 +13628,8 @@ dependencies = [
"local-ip-address",
"log-query",
"loki-proto",
"meter-core",
"meter-macros",
"metric-engine",
"mime_guess",
"moka",
@@ -13680,7 +13694,6 @@ dependencies = [
"tower 0.5.2",
"tower-http 0.6.11",
"tracing",
"tracing-opentelemetry",
"urlencoding",
"uuid",
"vrl",
@@ -13963,7 +13976,7 @@ version = "0.8.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1961e2ef424c1424204d3a5d6975f934f56b6d50ff5732382d84ebf460e147f7"
dependencies = [
"heck 0.5.0",
"heck 0.4.1",
"proc-macro2",
"quote",
"syn 2.0.117",
@@ -14444,6 +14457,7 @@ version = "1.3.0-alpha.1"
dependencies = [
"api",
"aquamarine",
"arrow-schema 59.2.0",
"async-stream",
"async-trait",
"bytes",
@@ -15014,7 +15028,7 @@ dependencies = [
"getrandom 0.3.4",
"once_cell",
"rustix 1.0.7",
"windows-sys 0.52.0",
"windows-sys 0.61.2",
]
[[package]]
@@ -15116,6 +15130,7 @@ dependencies = [
"catalog",
"chrono",
"clap",
"cli",
"client",
"cmd",
"common-base",
@@ -16794,7 +16809,7 @@ version = "0.1.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb"
dependencies = [
"windows-sys 0.48.0",
"windows-sys 0.59.0",
]
[[package]]
@@ -17244,7 +17259,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3f3fd376f71958b862e7afb20cfe5a22830e1963462f3a17f49d82a6c1d1f42d"
dependencies = [
"bitflags 2.12.1",
"windows-sys 0.52.0",
"windows-sys 0.59.0",
]
[[package]]
Cargo.toml
+39 -35
View File
@@ -130,22 +130,22 @@ crc32fast = "1"
criterion = "0.7"
crossbeam-utils = "0.8"
dashmap = "6.1"
datafusion = "=55.0.0"
datafusion-common = "=55.0.0"
datafusion-datasource = "=55.0.0"
datafusion-expr = "=55.0.0"
datafusion-expr-common = "=55.0.0"
datafusion-functions = "=55.0.0"
datafusion-functions-aggregate-common = "=55.0.0"
datafusion-functions-window-common = "=55.0.0"
datafusion-optimizer = "=55.0.0"
datafusion = "=55.1.0"
datafusion-common = "=55.1.0"
datafusion-datasource = "=55.1.0"
datafusion-expr = "=55.1.0"
datafusion-expr-common = "=55.1.0"
datafusion-functions = "=55.1.0"
datafusion-functions-aggregate-common = "=55.1.0"
datafusion-functions-window-common = "=55.1.0"
datafusion-optimizer = "=55.1.0"
datafusion-orc = "0.10.0"
datafusion-pg-catalog = "0.18.3"
datafusion-physical-expr = "=55.0.0"
datafusion-physical-plan = "=55.0.0"
datafusion-proto = "=55.0.0"
datafusion-sql = "=55.0.0"
datafusion-substrait = "=55.0.0"
datafusion-physical-expr = "=55.1.0"
datafusion-physical-plan = "=55.1.0"
datafusion-proto = "=55.1.0"
datafusion-sql = "=55.1.0"
datafusion-substrait = "=55.1.0"
datafusion_object_store = { package = "object_store", version = "0.13.2" }
deadpool = "0.12"
deadpool-postgres = "0.14"
@@ -161,7 +161,7 @@ fs2 = "0.4"
fst = "0.4.7"
futures = "0.3"
futures-util = "0.3"
greptime-proto = { git = "https://github.com/GreptimeTeam/greptime-proto.git", rev = "0c4beffaa2b3968034915867593a717c6ab0c6b4" }
greptime-proto = { git = "https://github.com/GreptimeTeam/greptime-proto.git", rev = "3d232f406fad574e8c630fb1c304fe2ebb013c9c" }
hex = "0.4"
hostname = "0.4.0"
http = "1"
@@ -178,14 +178,18 @@ jsonb = { git = "https://github.com/GreptimeTeam/jsonb", rev = "dd1e2341e3df35e9
lazy_static = "1.4"
local-ip-address = "0.6"
loki-proto = { git = "https://github.com/GreptimeTeam/loki-proto.git", rev = "f69c8924c4babe516373e26a4118be82d976629c" }
meter-core = { git = "https://github.com/GreptimeTeam/greptime-meter.git", rev = "5618e779cf2bb4755b499c630fba4c35e91898cb" }
meter-core = { git = "https://github.com/GreptimeTeam/greptime-meter.git", rev = "f743c3a5fe8c1f57363eac5866c356b7d91f9b39" }
mockall = "0.13"
moka = "0.12"
nalgebra = "0.33"
nix = { version = "0.30.1", default-features = false, features = ["event", "fs", "process"] }
notify = "8.0"
num_cpus = "1.16"
object_store_opendal = "0.58"
# Keep object_store 0.13 compatibility while using OpenDAL 0.59.
# TODO(#9240): Switch back to the official bridge after the DataFusion/Arrow/Parquet
# upgrade aligns object_store versions. Keep the bridge and OpenDAL releases compatible.
# https://github.com/GreptimeTeam/greptimedb/issues/9240
object_store_opendal = { git = "https://github.com/GreptimeTeam/opendal.git", rev = "c03e56802b47de3ccfc8fe028e59a459b278a0d2" }
once_cell = "1.18"
opentelemetry-proto = { version = "0.32", features = [
"gen-tonic",
@@ -357,27 +361,27 @@ table = { path = "src/table" }
[workspace.dependencies.meter-macros]
git = "https://github.com/GreptimeTeam/greptime-meter.git"
rev = "5618e779cf2bb4755b499c630fba4c35e91898cb"
rev = "f743c3a5fe8c1f57363eac5866c356b7d91f9b39"
[patch.crates-io]
substrait = { git = "https://github.com/GreptimeTeam/substrait-rs.git", rev = "91ec978b0649417ad3da8390e7a515baec723b1b" }
datafusion = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "b66dca3260c8e04f314c8036cdfbb08f1a564d95" }
datafusion-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "b66dca3260c8e04f314c8036cdfbb08f1a564d95" }
datafusion-datasource = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "b66dca3260c8e04f314c8036cdfbb08f1a564d95" }
datafusion-expr = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "b66dca3260c8e04f314c8036cdfbb08f1a564d95" }
datafusion-expr-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "b66dca3260c8e04f314c8036cdfbb08f1a564d95" }
datafusion-functions = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "b66dca3260c8e04f314c8036cdfbb08f1a564d95" }
datafusion-functions-aggregate-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "b66dca3260c8e04f314c8036cdfbb08f1a564d95" }
datafusion-functions-window-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "b66dca3260c8e04f314c8036cdfbb08f1a564d95" }
datafusion-optimizer = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "b66dca3260c8e04f314c8036cdfbb08f1a564d95" }
datafusion-physical-expr = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "b66dca3260c8e04f314c8036cdfbb08f1a564d95" }
datafusion-physical-expr-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "b66dca3260c8e04f314c8036cdfbb08f1a564d95" }
datafusion-physical-plan = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "b66dca3260c8e04f314c8036cdfbb08f1a564d95" }
datafusion-proto = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "b66dca3260c8e04f314c8036cdfbb08f1a564d95" }
datafusion-sql = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "b66dca3260c8e04f314c8036cdfbb08f1a564d95" }
datafusion-substrait = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "b66dca3260c8e04f314c8036cdfbb08f1a564d95" }
datafusion-pg-catalog = { git = "https://github.com/GreptimeTeam/datafusion-postgres.git", rev = "3c77e6c32b8db80635a0d2f4b318a36b31170bc3" }
arrow-pg = { git = "https://github.com/GreptimeTeam/datafusion-postgres.git", rev = "3c77e6c32b8db80635a0d2f4b318a36b31170bc3" }
datafusion = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "2aa87d52cdce7006af492330064738f33ed294c1" }
datafusion-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "2aa87d52cdce7006af492330064738f33ed294c1" }
datafusion-datasource = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "2aa87d52cdce7006af492330064738f33ed294c1" }
datafusion-expr = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "2aa87d52cdce7006af492330064738f33ed294c1" }
datafusion-expr-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "2aa87d52cdce7006af492330064738f33ed294c1" }
datafusion-functions = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "2aa87d52cdce7006af492330064738f33ed294c1" }
datafusion-functions-aggregate-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "2aa87d52cdce7006af492330064738f33ed294c1" }
datafusion-functions-window-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "2aa87d52cdce7006af492330064738f33ed294c1" }
datafusion-optimizer = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "2aa87d52cdce7006af492330064738f33ed294c1" }
datafusion-physical-expr = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "2aa87d52cdce7006af492330064738f33ed294c1" }
datafusion-physical-expr-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "2aa87d52cdce7006af492330064738f33ed294c1" }
datafusion-physical-plan = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "2aa87d52cdce7006af492330064738f33ed294c1" }
datafusion-proto = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "2aa87d52cdce7006af492330064738f33ed294c1" }
datafusion-sql = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "2aa87d52cdce7006af492330064738f33ed294c1" }
datafusion-substrait = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "2aa87d52cdce7006af492330064738f33ed294c1" }
datafusion-pg-catalog = { git = "https://github.com/datafusion-contrib/datafusion-postgres.git", rev = "ffb14a52b3d7c7489559812e1ec38096f225ce93" }
arrow-pg = { git = "https://github.com/datafusion-contrib/datafusion-postgres.git", rev = "ffb14a52b3d7c7489559812e1ec38096f225ce93" }
sqlparser = { git = "https://github.com/GreptimeTeam/sqlparser-rs.git", rev = "9e9019bb1c7040ed956f654e39378dd42ab17884" }
# Temporary: use the GreptimeTeam fork of tikv-jemalloc-sys embedding
Makefile
+6 -5
View File
@@ -10,7 +10,7 @@ CARGO_BUILD_OPTS := --locked
IMAGE_REGISTRY ?= docker.io
IMAGE_NAMESPACE ?= greptime
IMAGE_TAG ?= latest
DEV_BUILDER_IMAGE_TAG ?= 2026-03-21-7fa6f5f9-20260810091725
DEV_BUILDER_IMAGE_TAG ?= 2026-03-21-04f07a91-20260921031255
DEV_BUILDER_RISCV64_IMAGE_TAG ?= $(DEV_BUILDER_IMAGE_TAG)
DEV_BUILDER_RISCV64_BASE_IMAGE ?= ubuntu:22.04
RISCV64_TARGET ?= riscv64gc-unknown-linux-gnu
@@ -212,12 +212,12 @@ sqlness-test: ## Run sqlness test.
RUNS ?= 1
FUZZ_TARGET ?= fuzz_alter_table
.PHONY: fuzz
fuzz: ## Run fuzz test ${FUZZ_TARGET}.
cargo fuzz run ${FUZZ_TARGET} --fuzz-dir tests-fuzz -D -s none -- -runs=${RUNS}
fuzz: ## Run fuzz test ${FUZZ_TARGET} (requires a nightly toolchain).
cargo +nightly fuzz run ${FUZZ_TARGET} --fuzz-dir tests-fuzz -D -s none -- -runs=${RUNS}
.PHONY: fuzz-ls
fuzz-ls: ## List all fuzz targets.
cargo fuzz list --fuzz-dir tests-fuzz
fuzz-ls: ## List all fuzz targets (requires a nightly toolchain).
cargo +nightly fuzz list --fuzz-dir tests-fuzz
.PHONY: check
check: ## Cargo check all the targets.
@@ -264,6 +264,7 @@ stop-etcd: ## Stop single node etcd for testing purpose.
run-it-in-container: start-etcd ## Run integration tests in dev-builder.
docker run --network=host \
-v ${PWD}:/greptimedb -v ${CARGO_REGISTRY_CACHE}:/root/.cargo/registry -v ${CARGO_GIT_CACHE}:/root/.cargo/git -v /tmp:/tmp \
-e GT_S3_BUCKET -e GT_S3_ACCESS_KEY_ID -e GT_S3_ACCESS_KEY -e GT_S3_REGION \
-w /greptimedb ${IMAGE_REGISTRY}/${IMAGE_NAMESPACE}/dev-builder-${BASE_IMAGE}:${DEV_BUILDER_IMAGE_TAG} \
make test sqlness-test BUILD_JOBS=${BUILD_JOBS}
config/config.md
+26 -20
View File
@@ -15,6 +15,7 @@
| `default_timezone` | String | Unset | The default timezone of the server. |
| `default_column_prefix` | String | Unset | The default column prefix for auto-created time index, value, and native histogram columns.<br/>Legacy OTLP summary columns keep their historical `greptime_` prefix. |
| `auto_create_table` | Bool | `true` | Server-side global switch for auto table creation on write.<br/>When `false`, a missing table is never auto-created even if the request sets the `auto_create_table` hint to `true`. Default: `true`. |
| `experimental_metric_export` | Bool | `false` | Enables experimental Parquet COPY DATABASE using shared Metric physical scans.<br/>Resume requires the previous export and storage writes to have ended; HTTP timeout is not confirmation. |
| `user_provider` | String | Unset | The user provider for authentication.<br/>Examples: "static_user_provider:file:/path/to/users", "static_user_provider:cmd:greptime_user=greptime_pwd"<br/>Password verifier formats: "plain:<password>", "pbkdf2_sha256:<iterations>:<hex_salt>:<hex_hash>",<br/>"mysql_native_password:<hex_sha1_sha1_password>",<br/>"pg_scram_sha256:<iterations>:<hex_salt>:<hex_stored_key>:<hex_server_key>"<br/>"pbkdf2_sha256" and "pg_scram_sha256" protect passwords at rest, but cannot authenticate over MySQL's<br/>native password handshake; a MySQL client must send the password in cleartext for such users.<br/>"mysql_native_password" is MySQL-specific and cannot authenticate over PostgreSQL at all.<br/>PostgreSQL SCRAM only covers "plain" and "pg_scram_sha256" users; if any user is "pbkdf2_sha256" or<br/>"mysql_native_password", PostgreSQL falls back to cleartext password auth for every user.<br/>For "pg_scram_sha256" users, keep the default iteration count (4096) and salt length (16): both are<br/>observable in the SCRAM server-first message, and non-default values weaken resistance to username<br/>enumeration. |
| `max_in_flight_write_bytes` | String | Unset | Maximum total memory for all concurrent write request bodies and messages (HTTP, gRPC, Flight).<br/>Set to 0 to disable the limit. Default: "0" (unlimited) |
| `write_bytes_exhausted_policy` | String | Unset | Policy when write bytes quota is exhausted.<br/>Options: "wait" (default, 10s timeout), "wait(<duration>)" (e.g., "wait(30s)"), "fail" |
@@ -33,7 +34,7 @@
| `runtime.experimental_workload_scheduler.sample_every_polls` | Integer | `16` | Number of polls between scheduler fairness samples. Must be greater than zero. |
| `http` | -- | -- | The HTTP server options. |
| `http.addr` | String | `127.0.0.1:4000` | The address to bind the HTTP server. |
| `http.timeout` | String | `0s` | HTTP request timeout. Set to 0 to disable timeout.<br/>When synchronous Prometheus or shared table batching is enabled, a nonzero timeout is<br/>raised to at least the largest active flush interval plus 1 second. The intervals come from<br/>`prom_store.pending_rows_flush_interval` and `pending_rows_batcher.pending_rows_flush_interval`. |
| `http.timeout` | String | `0s` | HTTP request timeout. Set to 0 to disable timeout.<br/>When synchronous Prometheus, OTLP metrics, or ordinary-table batching is enabled, a nonzero timeout is<br/>raised to at least the largest active flush interval plus 1 second. The intervals come from<br/>`pending_rows_batcher.logical_table.pending_rows_flush_interval` and `pending_rows_batcher.pending_rows_flush_interval`. |
| `http.body_limit` | String | `64MB` | HTTP request body limit.<br/>The following units are supported: `B`, `KB`, `KiB`, `MB`, `MiB`, `GB`, `GiB`, `TB`, `TiB`, `PB`, `PiB`.<br/>Set to 0 to disable limit. |
| `http.enable_cors` | Bool | `true` | HTTP CORS support, it's turned on by default<br/>This allows browser to access http APIs without CORS restrictions |
| `http.cors_allowed_origins` | Array | Unset | Customize allowed origins for HTTP CORS. |
@@ -74,13 +75,20 @@
| `influxdb` | -- | -- | InfluxDB protocol options. |
| `influxdb.enable` | Bool | `true` | Whether to enable InfluxDB protocol in HTTP API. |
| `influxdb.default_merge_mode` | String | `last_non_null` | Default merge mode for tables automatically created by InfluxDB protocol.<br/>Available values: "last_non_null", "last_row". |
| `pending_rows_batcher` | -- | -- | Shared experimental ordinary-table batching for opted-in ingestion protocols.<br/>Legacy Prometheus batching settings under prom_store remain supported.<br/>HTTP write protocols sharing this batcher. Omitted or empty disables all entrances.<br/>Supported: influxdb, opentsdb, otlp, logs, loki, splunk, elasticsearch, http_sql, prom.<br/>Prom uses ordinary-table batching without metric engine, otherwise its dedicated batcher.<br/>Effective shared Prom settings take precedence; existing prom_store settings remain compatible. |
| `pending_rows_batcher` | -- | -- | Ordinary-table batching for opted-in HTTP ingestion protocols.<br/>PENDING_ROWS_BATCH_SYNC defaults to true for both batchers. Set it to false to acknowledge<br/>queue admission without waiting for storage; later failures cannot be returned to the client.<br/>Omitted or empty protocols disables batching. Prom without metric engine uses this batcher.<br/>OTLP logs, traces and ordinary metrics use this batcher. |
| `pending_rows_batcher.pending_rows_flush_interval` | String | `0s` | Flush interval measured from the first pending submission. Zero disables batching. |
| `pending_rows_batcher.max_batch_rows` | Integer | `100000` | Flush after a complete submission reaches this row threshold. |
| `pending_rows_batcher.max_concurrent_flushes` | Integer | `256` | Maximum concurrent flushes shared by the frontend batcher. |
| `pending_rows_batcher.worker_channel_capacity` | Integer | `65526` | Maximum queued submissions per table worker. |
| `pending_rows_batcher.worker_channel_capacity` | Integer | `65536` | Maximum queued submissions per table worker. |
| `pending_rows_batcher.max_inflight_requests` | Integer | `3000` | Maximum admitted original requests awaiting completion. |
| `pending_rows_batcher.flow_notification_queue_capacity` | Integer | `1024` | Maximum number of queued table Flow notifications. |
| `pending_rows_batcher.logical_table` | -- | -- | Metric-engine logical-table batching for Prom remote write and non-legacy OTLP metrics.<br/>Requires prom_store.with_metric_engine. Logs, traces and legacy metrics are not eligible.<br/>Enable independently with protocols and a nonzero flush interval.<br/>Omitted fields use independent defaults, not parent settings.<br/>Empty protocols or a zero interval disables logical batching without fallback.<br/>Omitting this entire section preserves legacy Prom batching; it does not enable OTLP batching. |
| `pending_rows_batcher.logical_table.pending_rows_flush_interval` | String | `0s` | Flush interval measured from the first pending submission. Zero disables batching. |
| `pending_rows_batcher.logical_table.max_batch_rows` | Integer | `100000` | Flush after a complete submission reaches this row threshold. |
| `pending_rows_batcher.logical_table.max_concurrent_flushes` | Integer | `256` | Maximum concurrent flushes shared by Prom and OTLP metrics. |
| `pending_rows_batcher.logical_table.worker_channel_capacity` | Integer | `65536` | Maximum queued submissions per physical-table worker. |
| `pending_rows_batcher.logical_table.max_inflight_requests` | Integer | `3000` | Maximum admitted original requests awaiting completion. |
| `pending_rows_batcher.logical_table.flow_notification_queue_capacity` | Integer | `1024` | Maximum number of queued logical-table Flow notifications. |
| `jaeger` | -- | -- | Jaeger protocol options. |
| `jaeger.enable` | Bool | `true` | Whether to enable Jaeger protocol in HTTP API. |
| `otlp` | -- | -- | OpenTelemetry protocol options. |
@@ -93,12 +101,6 @@
| `prom_store.with_metric_engine` | Bool | `true` | Whether to store the data from Prometheus remote write in metric engine. |
| `prom_store.prom_validation_mode` | String | `strict` | Whether to enable validation for Prometheus remote write requests.<br/>Available options:<br/>- strict: deny invalid UTF-8 strings (default).<br/>- lossy: allow invalid UTF-8 strings, replace invalid characters with REPLACEMENT_CHARACTER(U+FFFD).<br/>- unchecked: do not valid strings. |
| `prom_store.experimental_enable_prometheus_native_histogram` | Bool | `false` | Experimental: enable Prometheus remote write v2 native histogram ingestion. |
| `prom_store.pending_rows_flush_interval` | String | `0s` | Interval to flush pending rows batcher.<br/>Set to "0s" to disable batching mode in Prometheus Remote Write endpoint |
| `prom_store.max_batch_rows` | Integer | `100000` | Max rows per pending batch before triggering a flush. |
| `prom_store.max_concurrent_flushes` | Integer | `256` | Max number of concurrent batch flushes. |
| `prom_store.worker_channel_capacity` | Integer | `65526` | Capacity of the pending batch worker channel. |
| `prom_store.max_inflight_requests` | Integer | `3000` | Max inflight write requests before backpressure. |
| `prom_store.flow_notification_queue_capacity` | Integer | `1024` | Maximum number of logical-table flow notifications waiting in the shared queue. |
| `wal` | -- | -- | The WAL options. |
| `wal.provider` | String | `raft_engine` | The provider of the WAL.<br/>- `raft_engine`: the wal is stored in the local file system by raft-engine.<br/>- `kafka`: it's remote wal that data is stored in Kafka.<br/>- `experimental_object_store`: the wal is stored as objects in an object store.<br/>**Notes: experimental and not supported yet.** |
| `wal.dir` | String | Unset | The directory to store the WAL files.<br/>**It's only used when the provider is `raft_engine`**. |
@@ -169,7 +171,7 @@
| `storage.http_client.timeout` | String | `30s` | The total request timeout, applied from when the request starts connecting until the response body has finished.<br/>Also considered a total deadline. |
| `storage.http_client.pool_idle_timeout` | String | `90s` | The timeout for idle sockets being kept-alive. |
| `storage.http_client.skip_ssl_validation` | Bool | `false` | To skip the ssl verification<br/>**Security Notice**: Setting `skip_ssl_validation = true` disables certificate verification, making connections vulnerable to man-in-the-middle attacks. Only use this in development or trusted private networks. |
| `[[region_engine]]` | -- | -- | The region engine options. You can configure multiple region engines. |
| `[[region_engine]]` | -- | -- | The region engine options. You can configure multiple region engines.<br/>Each engine type (mito, file, metric) may appear only once; duplicates cause startup to fail. |
| `region_engine.mito` | -- | -- | The Mito engine options. |
| `region_engine.mito.num_workers` | Integer | `8` | Number of region workers. |
| `region_engine.mito.worker_channel_size` | Integer | `128` | Request channel size of each worker. |
@@ -177,6 +179,7 @@
| `region_engine.mito.manifest_checkpoint_distance` | Integer | `10` | Number of meta action updated to trigger a new checkpoint for the manifest. |
| `region_engine.mito.compress_manifest` | Bool | `false` | Whether to compress manifest and checkpoint file by gzip (default false). |
| `region_engine.mito.experimental_enable_series_index` | Bool | `false` | Under development; do not enable. Whether to enable series indexes.<br/>Indexes are stored on the local filesystem under `{data_home}/series_index`. |
| `region_engine.mito.experimental_enable_range_index` | Bool | `false` | Whether to build and query range indexes when series indexes are enabled.<br/>Obsolete range-index metadata and files are still cleaned up when disabled. |
| `region_engine.mito.experimental_series_index_maintenance_interval` | String | `5m` | Interval between series-index maintenance runs. Zero uses the default of 5 min. |
| `region_engine.mito.experimental_series_index_bucket_width` | String | `5days` | Requested minimum series-index bucket width (default: 5 days), rounded up to<br/>an exact multiple of each region's compaction time window. |
| `region_engine.mito.max_background_flushes` | Integer | Auto | Max number of running background flush jobs (default: 1/2 of cpu cores). |
@@ -277,6 +280,7 @@
| `default_timezone` | String | Unset | The default timezone of the server. |
| `default_column_prefix` | String | Unset | The default column prefix for auto-created time index, value, and native histogram columns.<br/>Legacy OTLP summary columns keep their historical `greptime_` prefix. |
| `auto_create_table` | Bool | `true` | Server-side global switch for auto table creation on write.<br/>When `false`, a missing table is never auto-created even if the request sets the `auto_create_table` hint to `true`. Default: `true`. |
| `experimental_metric_export` | Bool | `false` | Enables experimental Parquet COPY DATABASE using shared Metric physical scans.<br/>Resume requires the previous export and storage writes to have ended; HTTP timeout is not confirmation. |
| `user_provider` | String | Unset | The user provider for authentication.<br/>Examples: "static_user_provider:file:/path/to/users", "static_user_provider:cmd:greptime_user=greptime_pwd"<br/>Password verifier formats: "plain:<password>", "pbkdf2_sha256:<iterations>:<hex_salt>:<hex_hash>",<br/>"mysql_native_password:<hex_sha1_sha1_password>",<br/>"pg_scram_sha256:<iterations>:<hex_salt>:<hex_stored_key>:<hex_server_key>"<br/>"pbkdf2_sha256" and "pg_scram_sha256" protect passwords at rest, but cannot authenticate over MySQL's<br/>native password handshake; a MySQL client must send the password in cleartext for such users.<br/>"mysql_native_password" is MySQL-specific and cannot authenticate over PostgreSQL at all.<br/>PostgreSQL SCRAM only covers "plain" and "pg_scram_sha256" users; if any user is "pbkdf2_sha256" or<br/>"mysql_native_password", PostgreSQL falls back to cleartext password auth for every user.<br/>For "pg_scram_sha256" users, keep the default iteration count (4096) and salt length (16): both are<br/>observable in the SCRAM server-first message, and non-default values weaken resistance to username<br/>enumeration. |
| `max_in_flight_write_bytes` | String | Unset | Maximum total memory for all concurrent write request bodies and messages (HTTP, gRPC, Flight).<br/>Set to 0 to disable the limit. Default: "0" (unlimited) |
| `write_bytes_exhausted_policy` | String | Unset | Policy when write bytes quota is exhausted.<br/>Options: "wait" (default, 10s timeout), "wait(<duration>)" (e.g., "wait(30s)"), "fail" |
@@ -286,7 +290,7 @@
| `runtime.compact_rt_max_blocking_threads` | Integer | `4` | The maximum number of blocking threads for compact operations.<br/>Defaults to max(num_cpus / 2, 2). |
| `http` | -- | -- | The HTTP server options. |
| `http.addr` | String | `127.0.0.1:4000` | The address to bind the HTTP server. |
| `http.timeout` | String | `0s` | HTTP request timeout. Set to 0 to disable timeout.<br/>When synchronous Prometheus or shared table batching is enabled, a nonzero timeout is<br/>raised to at least the largest active flush interval plus 1 second. The intervals come from<br/>`prom_store.pending_rows_flush_interval` and `pending_rows_batcher.pending_rows_flush_interval`. |
| `http.timeout` | String | `0s` | HTTP request timeout. Set to 0 to disable timeout.<br/>When synchronous Prometheus, OTLP metrics, or ordinary-table batching is enabled, a nonzero timeout is<br/>raised to at least the largest active flush interval plus 1 second. The intervals come from<br/>`pending_rows_batcher.logical_table.pending_rows_flush_interval` and `pending_rows_batcher.pending_rows_flush_interval`. |
| `http.body_limit` | String | `64MB` | HTTP request body limit.<br/>The following units are supported: `B`, `KB`, `KiB`, `MB`, `MiB`, `GB`, `GiB`, `TB`, `TiB`, `PB`, `PiB`.<br/>Set to 0 to disable limit. |
| `http.enable_cors` | Bool | `true` | HTTP CORS support, it's turned on by default<br/>This allows browser to access http APIs without CORS restrictions |
| `http.cors_allowed_origins` | Array | Unset | Customize allowed origins for HTTP CORS. |
@@ -339,13 +343,20 @@
| `influxdb` | -- | -- | InfluxDB protocol options. |
| `influxdb.enable` | Bool | `true` | Whether to enable InfluxDB protocol in HTTP API. |
| `influxdb.default_merge_mode` | String | `last_non_null` | Default merge mode for tables automatically created by InfluxDB protocol.<br/>Available values: "last_non_null", "last_row". |
| `pending_rows_batcher` | -- | -- | Shared experimental ordinary-table batching for opted-in ingestion protocols.<br/>Legacy Prometheus batching settings under prom_store remain supported.<br/>HTTP write protocols sharing this batcher. Omitted or empty disables all entrances.<br/>Supported: influxdb, opentsdb, otlp, logs, loki, splunk, elasticsearch, http_sql, prom.<br/>Prom uses ordinary-table batching without metric engine, otherwise its dedicated batcher.<br/>Effective shared Prom settings take precedence; existing prom_store settings remain compatible. |
| `pending_rows_batcher` | -- | -- | Ordinary-table batching for opted-in HTTP ingestion protocols.<br/>PENDING_ROWS_BATCH_SYNC defaults to true for both batchers. Set it to false to acknowledge<br/>queue admission without waiting for storage; later failures cannot be returned to the client.<br/>Omitted or empty protocols disables batching. Prom without metric engine uses this batcher.<br/>OTLP logs, traces and ordinary metrics use this batcher. |
| `pending_rows_batcher.pending_rows_flush_interval` | String | `0s` | Flush interval measured from the first pending submission. Zero disables batching. |
| `pending_rows_batcher.max_batch_rows` | Integer | `100000` | Flush after a complete submission reaches this row threshold. |
| `pending_rows_batcher.max_concurrent_flushes` | Integer | `256` | Maximum concurrent flushes shared by the frontend batcher. |
| `pending_rows_batcher.worker_channel_capacity` | Integer | `65526` | Maximum queued submissions per table worker. |
| `pending_rows_batcher.worker_channel_capacity` | Integer | `65536` | Maximum queued submissions per table worker. |
| `pending_rows_batcher.max_inflight_requests` | Integer | `3000` | Maximum admitted original requests awaiting completion. |
| `pending_rows_batcher.flow_notification_queue_capacity` | Integer | `1024` | Maximum number of queued table Flow notifications. |
| `pending_rows_batcher.logical_table` | -- | -- | Metric-engine logical-table batching for Prom remote write and non-legacy OTLP metrics.<br/>Requires prom_store.with_metric_engine. Logs, traces and legacy metrics are not eligible.<br/>Enable independently with protocols and a nonzero flush interval.<br/>Omitted fields use independent defaults, not parent settings.<br/>Empty protocols or a zero interval disables logical batching without fallback.<br/>Omitting this entire section preserves legacy Prom batching; it does not enable OTLP batching. |
| `pending_rows_batcher.logical_table.pending_rows_flush_interval` | String | `0s` | Flush interval measured from the first pending submission. Zero disables batching. |
| `pending_rows_batcher.logical_table.max_batch_rows` | Integer | `100000` | Flush after a complete submission reaches this row threshold. |
| `pending_rows_batcher.logical_table.max_concurrent_flushes` | Integer | `256` | Maximum concurrent flushes shared by Prom and OTLP metrics. |
| `pending_rows_batcher.logical_table.worker_channel_capacity` | Integer | `65536` | Maximum queued submissions per physical-table worker. |
| `pending_rows_batcher.logical_table.max_inflight_requests` | Integer | `3000` | Maximum admitted original requests awaiting completion. |
| `pending_rows_batcher.logical_table.flow_notification_queue_capacity` | Integer | `1024` | Maximum number of queued logical-table Flow notifications. |
| `jaeger` | -- | -- | Jaeger protocol options. |
| `jaeger.enable` | Bool | `true` | Whether to enable Jaeger protocol in HTTP API. |
| `otlp` | -- | -- | OpenTelemetry protocol options. |
@@ -358,12 +369,6 @@
| `prom_store.with_metric_engine` | Bool | `true` | Whether to store the data from Prometheus remote write in metric engine. |
| `prom_store.prom_validation_mode` | String | `strict` | Whether to enable validation for Prometheus remote write requests.<br/>Available options:<br/>- strict: deny invalid UTF-8 strings (default).<br/>- lossy: allow invalid UTF-8 strings, replace invalid characters with REPLACEMENT_CHARACTER(U+FFFD).<br/>- unchecked: do not valid strings. |
| `prom_store.experimental_enable_prometheus_native_histogram` | Bool | `false` | Experimental: enable Prometheus remote write v2 native histogram ingestion. |
| `prom_store.pending_rows_flush_interval` | String | `0s` | Interval to flush pending rows batcher.<br/>Set to "0s" to disable batching mode in Prometheus Remote Write endpoint |
| `prom_store.max_batch_rows` | Integer | `100000` | Max rows per pending batch before triggering a flush. |
| `prom_store.max_concurrent_flushes` | Integer | `256` | Max number of concurrent batch flushes. |
| `prom_store.worker_channel_capacity` | Integer | `65526` | Capacity of the pending batch worker channel. |
| `prom_store.max_inflight_requests` | Integer | `3000` | Max inflight write requests before backpressure. |
| `prom_store.flow_notification_queue_capacity` | Integer | `1024` | Maximum number of logical-table flow notifications waiting in the shared queue. |
| `meta_client` | -- | -- | The metasrv client options. |
| `meta_client.metasrv_addrs` | Array | -- | The addresses of the metasrv. |
| `meta_client.timeout` | String | `3s` | Operation timeout. |
@@ -628,7 +633,7 @@
| `storage.http_client.timeout` | String | `30s` | The total request timeout, applied from when the request starts connecting until the response body has finished.<br/>Also considered a total deadline. |
| `storage.http_client.pool_idle_timeout` | String | `90s` | The timeout for idle sockets being kept-alive. |
| `storage.http_client.skip_ssl_validation` | Bool | `false` | To skip the ssl verification<br/>**Security Notice**: Setting `skip_ssl_validation = true` disables certificate verification, making connections vulnerable to man-in-the-middle attacks. Only use this in development or trusted private networks. |
| `[[region_engine]]` | -- | -- | The region engine options. You can configure multiple region engines. |
| `[[region_engine]]` | -- | -- | The region engine options. You can configure multiple region engines.<br/>Each engine type (mito, file, metric) may appear only once; duplicates cause startup to fail. |
| `region_engine.mito` | -- | -- | The Mito engine options. |
| `region_engine.mito.num_workers` | Integer | `8` | Number of region workers. |
| `region_engine.mito.worker_channel_size` | Integer | `128` | Request channel size of each worker. |
@@ -638,6 +643,7 @@
| `region_engine.mito.experimental_manifest_keep_removed_file_ttl` | String | `1h` | How long to keep removed files in the `removed_files` field of manifest<br/>after they are removed from manifest.<br/>files will only be removed from `removed_files` field<br/>if both `keep_removed_file_count` and `keep_removed_file_ttl` is reached. |
| `region_engine.mito.compress_manifest` | Bool | `false` | Whether to compress manifest and checkpoint file by gzip (default false). |
| `region_engine.mito.experimental_enable_series_index` | Bool | `false` | Under development; do not enable. Whether to enable series indexes.<br/>Indexes are stored on the local filesystem under `{data_home}/series_index`. |
| `region_engine.mito.experimental_enable_range_index` | Bool | `false` | Whether to build and query range indexes when series indexes are enabled.<br/>Obsolete range-index metadata and files are still cleaned up when disabled. |
| `region_engine.mito.experimental_series_index_maintenance_interval` | String | `5m` | Interval between series-index maintenance runs. Zero uses the default of 5 min. |
| `region_engine.mito.experimental_series_index_bucket_width` | String | `5days` | Requested minimum series-index bucket width (default: 5 days), rounded up to<br/>an exact multiple of each region's compaction time window. |
| `region_engine.mito.max_background_flushes` | Integer | Auto | Max number of running background flush jobs (default: 1/2 of cpu cores). |
config/datanode.example.toml
+5
View File
@@ -464,6 +464,7 @@ skip_ssl_validation = false
# endpoint = "https://storage.googleapis.com"
## The region engine options. You can configure multiple region engines.
## Each engine type (mito, file, metric) may appear only once; duplicates cause startup to fail.
[[region_engine]]
## The Mito engine options.
@@ -501,6 +502,10 @@ compress_manifest = false
## Indexes are stored on the local filesystem under `{data_home}/series_index`.
#+ experimental_enable_series_index = false
## Whether to build and query range indexes when series indexes are enabled.
## Obsolete range-index metadata and files are still cleaned up when disabled.
#+ experimental_enable_range_index = false
## Interval between series-index maintenance runs. Zero uses the default of 5 min.
#+ experimental_series_index_maintenance_interval = "5m"
config/frontend.example.toml
+34 -22
View File
@@ -11,6 +11,10 @@ default_column_prefix = "greptime"
## When `false`, a missing table is never auto-created even if the request sets the `auto_create_table` hint to `true`. Default: `true`.
#+ auto_create_table = true
## Enables experimental Parquet COPY DATABASE using shared Metric physical scans.
## Resume requires the previous export and storage writes to have ended; HTTP timeout is not confirmation.
#+ experimental_metric_export = false
## The user provider for authentication.
## Examples: "static_user_provider:file:/path/to/users", "static_user_provider:cmd:greptime_user=greptime_pwd"
## Password verifier formats: "plain:<password>", "pbkdf2_sha256:<iterations>:<hex_salt>:<hex_hash>",
@@ -52,9 +56,9 @@ default_column_prefix = "greptime"
## The address to bind the HTTP server.
addr = "127.0.0.1:4000"
## HTTP request timeout. Set to 0 to disable timeout.
## When synchronous Prometheus or shared table batching is enabled, a nonzero timeout is
## When synchronous Prometheus, OTLP metrics, or ordinary-table batching is enabled, a nonzero timeout is
## raised to at least the largest active flush interval plus 1 second. The intervals come from
## `prom_store.pending_rows_flush_interval` and `pending_rows_batcher.pending_rows_flush_interval`.
## `pending_rows_batcher.logical_table.pending_rows_flush_interval` and `pending_rows_batcher.pending_rows_flush_interval`.
timeout = "0s"
## HTTP request body limit.
## The following units are supported: `B`, `KB`, `KiB`, `MB`, `MiB`, `GB`, `GiB`, `TB`, `TiB`, `PB`, `PiB`.
@@ -227,12 +231,11 @@ enable = true
## Available values: "last_non_null", "last_row".
default_merge_mode = "last_non_null"
## Shared experimental ordinary-table batching for opted-in ingestion protocols.
## Legacy Prometheus batching settings under prom_store remain supported.
## HTTP write protocols sharing this batcher. Omitted or empty disables all entrances.
## Supported: influxdb, opentsdb, otlp, logs, loki, splunk, elasticsearch, http_sql, prom.
## Prom uses ordinary-table batching without metric engine, otherwise its dedicated batcher.
## Effective shared Prom settings take precedence; existing prom_store settings remain compatible.
## Ordinary-table batching for opted-in HTTP ingestion protocols.
## PENDING_ROWS_BATCH_SYNC defaults to true for both batchers. Set it to false to acknowledge
## queue admission without waiting for storage; later failures cannot be returned to the client.
## Omitted or empty protocols disables batching. Prom without metric engine uses this batcher.
## OTLP logs, traces and ordinary metrics use this batcher.
[pending_rows_batcher]
# protocols = [
# "influxdb",
@@ -252,12 +255,33 @@ max_batch_rows = 100000
## Maximum concurrent flushes shared by the frontend batcher.
max_concurrent_flushes = 256
## Maximum queued submissions per table worker.
worker_channel_capacity = 65526
worker_channel_capacity = 65536
## Maximum admitted original requests awaiting completion.
max_inflight_requests = 3000
## Maximum number of queued table Flow notifications.
flow_notification_queue_capacity = 1024
## Metric-engine logical-table batching for Prom remote write and non-legacy OTLP metrics.
## Requires prom_store.with_metric_engine. Logs, traces and legacy metrics are not eligible.
## Enable independently with protocols and a nonzero flush interval.
## Omitted fields use independent defaults, not parent settings.
## Empty protocols or a zero interval disables logical batching without fallback.
## Omitting this entire section preserves legacy Prom batching; it does not enable OTLP batching.
[pending_rows_batcher.logical_table]
# protocols = ["prom", "otlp"]
## Flush interval measured from the first pending submission. Zero disables batching.
pending_rows_flush_interval = "0s"
## Flush after a complete submission reaches this row threshold.
max_batch_rows = 100000
## Maximum concurrent flushes shared by Prom and OTLP metrics.
max_concurrent_flushes = 256
## Maximum queued submissions per physical-table worker.
worker_channel_capacity = 65536
## Maximum admitted original requests awaiting completion.
max_inflight_requests = 3000
## Maximum number of queued logical-table Flow notifications.
flow_notification_queue_capacity = 1024
## Jaeger protocol options.
[jaeger]
## Whether to enable Jaeger protocol in HTTP API.
@@ -289,19 +313,7 @@ with_metric_engine = true
prom_validation_mode = "strict"
## Experimental: enable Prometheus remote write v2 native histogram ingestion.
experimental_enable_prometheus_native_histogram = false
## Interval to flush pending rows batcher.
## Set to "0s" to disable batching mode in Prometheus Remote Write endpoint
#+pending_rows_flush_interval = "0s"
## Max rows per pending batch before triggering a flush.
#+max_batch_rows = 100000
## Max number of concurrent batch flushes.
#+max_concurrent_flushes = 256
## Capacity of the pending batch worker channel.
#+worker_channel_capacity = 65526
## Max inflight write requests before backpressure.
#+max_inflight_requests = 3000
## Maximum number of logical-table flow notifications waiting in the shared queue.
#+flow_notification_queue_capacity = 1024
## The metasrv client options.
[meta_client]
config/standalone.example.toml
+39 -22
View File
@@ -11,6 +11,10 @@ default_column_prefix = "greptime"
## When `false`, a missing table is never auto-created even if the request sets the `auto_create_table` hint to `true`. Default: `true`.
#+ auto_create_table = true
## Enables experimental Parquet COPY DATABASE using shared Metric physical scans.
## Resume requires the previous export and storage writes to have ended; HTTP timeout is not confirmation.
#+ experimental_metric_export = false
## The user provider for authentication.
## Examples: "static_user_provider:file:/path/to/users", "static_user_provider:cmd:greptime_user=greptime_pwd"
## Password verifier formats: "plain:<password>", "pbkdf2_sha256:<iterations>:<hex_salt>:<hex_hash>",
@@ -77,9 +81,9 @@ max_concurrent_queries = 0
## The address to bind the HTTP server.
addr = "127.0.0.1:4000"
## HTTP request timeout. Set to 0 to disable timeout.
## When synchronous Prometheus or shared table batching is enabled, a nonzero timeout is
## When synchronous Prometheus, OTLP metrics, or ordinary-table batching is enabled, a nonzero timeout is
## raised to at least the largest active flush interval plus 1 second. The intervals come from
## `prom_store.pending_rows_flush_interval` and `pending_rows_batcher.pending_rows_flush_interval`.
## `pending_rows_batcher.logical_table.pending_rows_flush_interval` and `pending_rows_batcher.pending_rows_flush_interval`.
timeout = "0s"
## HTTP request body limit.
## The following units are supported: `B`, `KB`, `KiB`, `MB`, `MiB`, `GB`, `GiB`, `TB`, `TiB`, `PB`, `PiB`.
@@ -206,12 +210,11 @@ enable = true
## Available values: "last_non_null", "last_row".
default_merge_mode = "last_non_null"
## Shared experimental ordinary-table batching for opted-in ingestion protocols.
## Legacy Prometheus batching settings under prom_store remain supported.
## HTTP write protocols sharing this batcher. Omitted or empty disables all entrances.
## Supported: influxdb, opentsdb, otlp, logs, loki, splunk, elasticsearch, http_sql, prom.
## Prom uses ordinary-table batching without metric engine, otherwise its dedicated batcher.
## Effective shared Prom settings take precedence; existing prom_store settings remain compatible.
## Ordinary-table batching for opted-in HTTP ingestion protocols.
## PENDING_ROWS_BATCH_SYNC defaults to true for both batchers. Set it to false to acknowledge
## queue admission without waiting for storage; later failures cannot be returned to the client.
## Omitted or empty protocols disables batching. Prom without metric engine uses this batcher.
## OTLP logs, traces and ordinary metrics use this batcher.
[pending_rows_batcher]
# protocols = [
# "influxdb",
@@ -231,12 +234,33 @@ max_batch_rows = 100000
## Maximum concurrent flushes shared by the frontend batcher.
max_concurrent_flushes = 256
## Maximum queued submissions per table worker.
worker_channel_capacity = 65526
worker_channel_capacity = 65536
## Maximum admitted original requests awaiting completion.
max_inflight_requests = 3000
## Maximum number of queued table Flow notifications.
flow_notification_queue_capacity = 1024
## Metric-engine logical-table batching for Prom remote write and non-legacy OTLP metrics.
## Requires prom_store.with_metric_engine. Logs, traces and legacy metrics are not eligible.
## Enable independently with protocols and a nonzero flush interval.
## Omitted fields use independent defaults, not parent settings.
## Empty protocols or a zero interval disables logical batching without fallback.
## Omitting this entire section preserves legacy Prom batching; it does not enable OTLP batching.
[pending_rows_batcher.logical_table]
# protocols = ["prom", "otlp"]
## Flush interval measured from the first pending submission. Zero disables batching.
pending_rows_flush_interval = "0s"
## Flush after a complete submission reaches this row threshold.
max_batch_rows = 100000
## Maximum concurrent flushes shared by Prom and OTLP metrics.
max_concurrent_flushes = 256
## Maximum queued submissions per physical-table worker.
worker_channel_capacity = 65536
## Maximum admitted original requests awaiting completion.
max_inflight_requests = 3000
## Maximum number of queued logical-table Flow notifications.
flow_notification_queue_capacity = 1024
## Jaeger protocol options.
[jaeger]
## Whether to enable Jaeger protocol in HTTP API.
@@ -268,19 +292,7 @@ with_metric_engine = true
prom_validation_mode = "strict"
## Experimental: enable Prometheus remote write v2 native histogram ingestion.
experimental_enable_prometheus_native_histogram = false
## Interval to flush pending rows batcher.
## Set to "0s" to disable batching mode in Prometheus Remote Write endpoint
#+pending_rows_flush_interval = "0s"
## Max rows per pending batch before triggering a flush.
#+max_batch_rows = 100000
## Max number of concurrent batch flushes.
#+max_concurrent_flushes = 256
## Capacity of the pending batch worker channel.
#+worker_channel_capacity = 65526
## Max inflight write requests before backpressure.
#+max_inflight_requests = 3000
## Maximum number of logical-table flow notifications waiting in the shared queue.
#+flow_notification_queue_capacity = 1024
## The WAL options.
[wal]
@@ -670,6 +682,7 @@ skip_ssl_validation = false
# endpoint = "https://storage.googleapis.com"
## The region engine options. You can configure multiple region engines.
## Each engine type (mito, file, metric) may appear only once; duplicates cause startup to fail.
[[region_engine]]
## The Mito engine options.
@@ -694,6 +707,10 @@ compress_manifest = false
## Indexes are stored on the local filesystem under `{data_home}/series_index`.
#+ experimental_enable_series_index = false
## Whether to build and query range indexes when series indexes are enabled.
## Obsolete range-index metadata and files are still cleaned up when disabled.
#+ experimental_enable_range_index = false
## Interval between series-index maintenance runs. Zero uses the default of 5 min.
#+ experimental_series_index_maintenance_interval = "5m"
cyborg/bin/follow-up-docs-issue.ts
+111 -42
View File
@@ -17,7 +17,12 @@
import * as core from '@actions/core'
import {handleError, obtainClient} from "@/common";
import {context} from "@actions/github";
import {PullRequestEditedEvent, PullRequestEvent, PullRequestOpenedEvent} from "@octokit/webhooks-types";
import {
PullRequestClosedEvent,
PullRequestEditedEvent,
PullRequestEvent,
PullRequestOpenedEvent
} from "@octokit/webhooks-types";
// @ts-expect-error moduleResolution:nodenext issue 54523
import {RequestError} from "@octokit/request-error";
@@ -30,18 +35,59 @@ async function main() {
throw new Error(`Only pull request event supported. ${context.eventName} is unsupported.`)
}
const client = obtainClient("GITHUB_TOKEN")
const docsClient = obtainClient("DOCS_REPO_TOKEN")
const payload = context.payload as PullRequestEvent
const { owner, repo, number, actor, title, html_url } = {
const { owner, repo, number, actor } = {
owner: payload.pull_request.base.user.login,
repo: payload.pull_request.base.repo.name,
number: payload.pull_request.number,
title: payload.pull_request.title,
html_url: payload.pull_request.html_url,
actor: payload.pull_request.user.login,
}
const followUpDocs = checkPullRequestEvent(payload)
switch (payload.action) {
case "opened": {
const client = obtainClient("GITHUB_TOKEN")
await updateDocsLabels(client, owner, repo, number, checkPullRequestOpenedEvent(payload as PullRequestOpenedEvent))
break
}
case "edited": {
const followUpDocs = checkPullRequestEditedEvent(payload as PullRequestEditedEvent)
if (followUpDocs === undefined) {
core.info("Docs checkbox state unchanged; leaving labels as-is.")
return
}
const client = obtainClient("GITHUB_TOKEN")
await updateDocsLabels(client, owner, repo, number, followUpDocs)
break
}
case "closed": {
const event = payload as PullRequestClosedEvent
if (!event.pull_request.merged) {
core.info("PR closed without merging; no docs issue needed.")
return
}
// The docs-required label is the single source of truth. Coordinate
// with pending checkbox-driven label updates before deciding: an edit
// run may still be applying or removing the label, and the event
// payload's label snapshot may be stale.
const client = obtainClient("GITHUB_TOKEN")
const { data: pr } = await client.rest.pulls.get({ owner, repo, pull_number: number })
await waitForPendingLabelUpdates(client, owner, repo, pr.head.sha)
const { data: freshPr } = await client.rest.pulls.get({ owner, repo, pull_number: number })
const hasDocsLabel = freshPr.labels.some((label) => label.name === labelDocsRequired)
if (!hasDocsLabel) {
core.info(`Label ${labelDocsRequired} not present; no docs issue needed.`)
return
}
const docsClient = obtainClient("DOCS_REPO_TOKEN")
await createDocsIssue(docsClient, freshPr.title, freshPr.html_url, actor)
break
}
default:
throw new Error(`${payload.action} is unsupported.`)
}
}
async function updateDocsLabels(client: ReturnType<typeof obtainClient>, owner: string, repo: string, number: number, followUpDocs: boolean) {
if (followUpDocs) {
core.info("Follow up docs.")
await client.rest.issues.removeLabel({
@@ -55,28 +101,6 @@ async function main() {
await client.rest.issues.addLabels({
owner, repo, issue_number: number, labels: [labelDocsRequired],
})
// Get available assignees for the docs repo
const assigneesResponse = await docsClient.rest.issues.listAssignees({
owner: 'GreptimeTeam',
repo: 'docs',
})
const validAssignees = assigneesResponse.data.map(assignee => assignee.login)
core.info(`Available assignees: ${validAssignees.join(', ')}`)
// Check if the actor is a valid assignee, otherwise fallback to fengjiachun
const assignee = validAssignees.includes(actor) ? actor : 'fengjiachun'
core.info(`Assigning issue to: ${assignee}`)
await docsClient.rest.issues.create({
owner: 'GreptimeTeam',
repo: 'docs',
title: `Update docs for ${title}`,
body: `A document change request is generated from ${html_url}`,
assignee: assignee,
}).then((res) => {
core.info(`Created issue ${res.data}`)
})
} else {
core.info("No need to follow up docs.")
await client.rest.issues.removeLabel({
@@ -93,15 +117,52 @@ async function main() {
}
}
function checkPullRequestEvent(payload: PullRequestEvent) {
switch (payload.action) {
case "opened":
return checkPullRequestOpenedEvent(payload as PullRequestOpenedEvent)
case "edited":
return checkPullRequestEditedEvent(payload as PullRequestEditedEvent)
default:
throw new Error(`${payload.action} is unsupported.`)
async function createDocsIssue(docsClient: ReturnType<typeof obtainClient>, title: string, html_url: string, actor: string) {
core.info("Creating follow-up docs issue for merged PR.")
// Get available assignees for the docs repo
const assigneesResponse = await docsClient.rest.issues.listAssignees({
owner: 'GreptimeTeam',
repo: 'docs',
})
const validAssignees = assigneesResponse.data.map(assignee => assignee.login)
core.info(`Available assignees: ${validAssignees.join(', ')}`)
// Check if the actor is a valid assignee, otherwise fallback to fengjiachun
const assignee = validAssignees.includes(actor) ? actor : 'fengjiachun'
core.info(`Assigning issue to: ${assignee}`)
await docsClient.rest.issues.create({
owner: 'GreptimeTeam',
repo: 'docs',
title: `Update docs for ${title}`,
body: `A document change request is generated from ${html_url}`,
assignee: assignee,
}).then((res) => {
core.info(`Created issue ${res.data}`)
})
}
// Waits until no other runs of this workflow are pending for the PR head SHA,
// so that any checkbox-driven label update has been applied before the caller
// reads the labels. Bounded by a deadline; on timeout it proceeds with whatever
// the current label state is.
async function waitForPendingLabelUpdates(client: ReturnType<typeof obtainClient>, owner: string, repo: string, headSha: string) {
const deadlineMs = 5 * 60 * 1000
const pollIntervalMs = 5000
const deadline = Date.now() + deadlineMs
while (Date.now() < deadline) {
const { data } = await client.rest.actions.listWorkflowRuns({
owner, repo, workflow_id: "docbot.yml", head_sha: headSha, per_page: 20,
})
const pending = data.workflow_runs.filter((run) => run.id !== context.runId && run.status !== "completed")
if (pending.length === 0) {
return
}
core.info(`Waiting for ${pending.length} pending docbot run(s) to finish label updates...`)
await new Promise((resolve) => setTimeout(resolve, pollIntervalMs))
}
core.warning("Timed out waiting for pending docbot runs; proceeding with the current label state.")
}
function checkPullRequestOpenedEvent(event: PullRequestOpenedEvent): boolean {
@@ -109,11 +170,19 @@ function checkPullRequestOpenedEvent(event: PullRequestOpenedEvent): boolean {
return event.pull_request.body?.includes(needFollowUpDocs)
}
function checkPullRequestEditedEvent(event: PullRequestEditedEvent): boolean {
const previous = event.changes.body?.from.includes(needFollowUpDocs)
const current = event.pull_request.body?.includes(needFollowUpDocs)
// from docs-not-need to docs-required
return (!previous) && current
// Returns undefined when the checkbox state did not change in this edit, so the
// caller leaves the labels untouched (preserving manual label overrides).
function checkPullRequestEditedEvent(event: PullRequestEditedEvent): boolean | undefined {
if (!event.changes.body) {
// The body was not part of this edit (e.g. title-only edit).
return undefined
}
const previous = event.changes.body.from.includes(needFollowUpDocs)
const current = event.pull_request.body?.includes(needFollowUpDocs) ?? false
if (previous === current) {
return undefined
}
return current
}
main().catch(handleError)
docker/dev-builder/riscv64/Dockerfile
+15
View File
@@ -11,6 +11,7 @@ RUN apt-get update && \
# Install dependencies, including the riscv64 cross toolchain.
RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \
libssl-dev \
libatomic1 \
tzdata \
curl \
unzip \
@@ -40,6 +41,20 @@ RUN mv -f protoc3/bin/* /usr/local/bin/
# contains protobuf headers under /usr/local/include.
RUN cp -r protoc3/include/* /usr/local/include/
# Install mold and make it the default linker (the equivalent of
# rui314/setup-mold with its default options on GitHub-hosted runners) for
# lower peak memory usage during linking. The host `ld` symlink covers
# host-side linking (build scripts, proc macros); the riscv64 symlink covers
# the cross target -- the cross gcc driver passes -m elf64lriscv explicitly,
# so mold picks the right emulation.
ARG MOLD_VERSION=2.42.1
RUN arch="$(uname -m)" && \
curl -fsSL -o /tmp/mold.tar.gz https://github.com/rui314/mold/releases/download/v${MOLD_VERSION}/mold-${MOLD_VERSION}-${arch}-linux.tar.gz && \
tar -C /usr/local --strip-components=1 -xzf /tmp/mold.tar.gz && \
rm /tmp/mold.tar.gz && \
ln -sf /usr/local/bin/mold "$(readlink -f /usr/bin/ld)" && \
ln -sf /usr/local/bin/mold "$(readlink -f "$(command -v riscv64-linux-gnu-ld)")"
# Silence all `safe.directory` warnings, to avoid the "detect dubious repository"
# error when building with the repository mounted from a different user.
RUN git config --global --add safe.directory '*'
docker/dev-builder/ubuntu/Dockerfile
+13
View File
@@ -11,6 +11,7 @@ RUN apt-get update && \
# Install dependencies.
RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \
libssl-dev \
libatomic1 \
tzdata \
curl \
unzip \
@@ -35,6 +36,18 @@ fi
RUN mv protoc3/bin/* /usr/local/bin/
RUN mv protoc3/include/* /usr/local/include/
# Install mold and make it the default linker (the equivalent of
# rui314/setup-mold with its default options on GitHub-hosted runners), so
# every link inside this image -- binaries, test executables, build scripts,
# proc macros -- goes through mold for lower peak memory usage. The prebuilt
# binaries require GLIBC 2.18+.
ARG MOLD_VERSION=2.42.1
RUN arch="$(uname -m)" && \
curl -fsSL -o /tmp/mold.tar.gz https://github.com/rui314/mold/releases/download/v${MOLD_VERSION}/mold-${MOLD_VERSION}-${arch}-linux.tar.gz && \
tar -C /usr/local --strip-components=1 -xzf /tmp/mold.tar.gz && \
rm /tmp/mold.tar.gz && \
ln -sf /usr/local/bin/mold "$(readlink -f /usr/bin/ld)"
# Silence all `safe.directory` warnings, to avoid the "detect dubious repository" error when building with submodules.
# Disabling the safe directory check here won't pose extra security issues, because in our usage for this dev build
# image, we use it solely on our own environment (that github action's VM, or ECS created dynamically by ourselves),
docker/dev-builder/ubuntu/Dockerfile-20.04
-66
View File
@@ -1,66 +0,0 @@
FROM ubuntu:20.04
# The root path under which contains all the dependencies to build this Dockerfile.
ARG DOCKER_BUILD_ROOT=.
ENV LANG en_US.utf8
WORKDIR /greptimedb
RUN apt-get update && \
DEBIAN_FRONTEND=noninteractive apt-get install -y software-properties-common
# Install dependencies.
RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \
libssl-dev \
tzdata \
curl \
unzip \
ca-certificates \
git \
build-essential \
pkg-config
ARG TARGETPLATFORM
RUN echo "target platform: $TARGETPLATFORM"
ARG PROTOBUF_VERSION=29.3
# Install protobuf, because the one in the apt is too old (v3.12).
RUN if [ "$TARGETPLATFORM" = "linux/arm64" ]; then \
curl -OL https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOBUF_VERSION}/protoc-${PROTOBUF_VERSION}-linux-aarch_64.zip && \
unzip protoc-${PROTOBUF_VERSION}-linux-aarch_64.zip -d protoc3; \
elif [ "$TARGETPLATFORM" = "linux/amd64" ]; then \
curl -OL https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOBUF_VERSION}/protoc-${PROTOBUF_VERSION}-linux-x86_64.zip && \
unzip protoc-${PROTOBUF_VERSION}-linux-x86_64.zip -d protoc3; \
fi
RUN mv protoc3/bin/* /usr/local/bin/
RUN mv protoc3/include/* /usr/local/include/
# Silence all `safe.directory` warnings, to avoid the "detect dubious repository" error when building with submodules.
# Disabling the safe directory check here won't pose extra security issues, because in our usage for this dev build
# image, we use it solely on our own environment (that github action's VM, or ECS created dynamically by ourselves),
# and the repositories are pulled from trusted sources (still us, of course). Doing so does not violate the intention
# of the Git's addition to the "safe.directory" at the first place (see the commit message here:
# https://github.com/git/git/commit/8959555cee7ec045958f9b6dd62e541affb7e7d9).
# There's also another solution to this, that we add the desired submodules to the safe directory, instead of using
# wildcard here. However, that requires the git's config files and the submodules all owned by the very same user.
# It's troublesome to do this since the dev build runs in Docker, which is under user "root"; while outside the Docker,
# it can be a different user that have prepared the submodules.
RUN git config --global --add safe.directory '*'
# Install Rust.
SHELL ["/bin/bash", "-c"]
RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- --no-modify-path --default-toolchain none -y
ENV PATH /root/.cargo/bin/:$PATH
# Install Rust toolchains.
ARG RUST_TOOLCHAIN
RUN rustup toolchain install ${RUST_TOOLCHAIN}
# Install cargo-binstall with a specific version to adapt the current rust toolchain.
# Note: if we use the latest version, we may encounter the following `use of unstable library feature 'io_error_downcast'` error.
# compile from source take too long, so we use the precompiled binary instead
COPY $DOCKER_BUILD_ROOT/docker/dev-builder/binstall/pull_binstall.sh /usr/local/bin/pull_binstall.sh
RUN chmod +x /usr/local/bin/pull_binstall.sh && /usr/local/bin/pull_binstall.sh
# Install nextest.
RUN cargo binstall cargo-nextest --no-confirm
docs/how-to/ai-functions.md
+247
View File
@@ -0,0 +1,247 @@
# AI functions (experimental MVP)
Three asynchronous SQL scalar functions provide natural-language matching,
classification, and rating. The current backend is TypeSafe's Jev model:
| Function | Mode | Result |
| --- | --- | --- |
| `ai_match(text, prompt)` | Noul | `Float64` probability in **[0, 1]** that the prompt's statement is true |
| `ai_choose(text, prompt, criteria)` | Choice | `String` containing the selected option's name |
| `ai_score(text, prompt, criteria)` | Score | JSONB object containing `score`, `confidence`, and a `probabilities` array |
Use scalar results or extracted JSON fields in `SELECT`, comparisons in `WHERE`,
and `ORDER BY`.
All arguments are SQL strings. Any SQL NULL argument produces SQL `NULL`
without validating that row's criteria or calling the API.
## Start GreptimeDB
The functions are compiled and registered by the **default-enabled Cargo feature
`ai_functions`**. Enable the Jev backend separately in the server process:
```sh
export GREPTIMEDB_EXPERIMENTAL_JEV=true
export JEV_API_KEY='<your TypeSafe API key>'
cargo run -p cmd -- standalone start
```
If your key is already exported in `~/.zshrc`, run `source ~/.zshrc` first.
The optional `JEV_MODEL` defaults to `jev-latest`; `JEV_ENDPOINT` defaults to
`https://api.typesafe.ai/v1/systemone` (the full evaluation endpoint URL).
The MVP uses environment variables rather than TOML configuration.
The Cargo feature includes all three SQL functions in the build; the runtime environment
variables enable and configure its API calls. Default compilation does not turn
on external API calls. To enable the feature explicitly, use
`--features ai_functions`.
## Query
For a new example table, quote `service` and `message` in the DDL because the
current parser treats them as keywords:
```sql
CREATE TABLE events (
occurred_at TIMESTAMP TIME INDEX,
"service" STRING,
"message" STRING,
PRIMARY KEY ("service")
);
```
### Matching probability (Noul)
```sql
SELECT occurred_at, service, message
FROM events
WHERE occurred_at >= '2026-09-19T00:00:00Z'
AND occurred_at < '2026-09-20T00:00:00Z'
AND service = 'payments'
AND ai_match(
(message),
'The event reports that a payment still failed after retries.'
) >= 0.8
ORDER BY occurred_at;
```
To return the scores and rank matching events:
```sql
SELECT occurred_at, message,
ai_match(message, 'The event reports that a payment still failed after retries.') AS score
FROM events
WHERE occurred_at >= '2026-09-19T00:00:00Z'
AND occurred_at < '2026-09-20T00:00:00Z'
AND service = 'payments'
ORDER BY score DESC NULLS LAST;
```
Both arguments are strings; larger scores indicate a stronger match.
`(message)` is an ordinary parenthesized string expression. For several columns,
combine them explicitly, for example `concat(service, ': ', message)`.
Any null argument produces SQL `NULL`, which `WHERE` excludes, without an API call.
### Classification (Choice)
`criteria` is a JSON object with **1 to 255 options**. Each key is an option name;
its value is a description (string, object, or array), or JSON `null` if the name
is sufficient. The function returns the selected key, suitable for filtering
or grouping:
```sql
SELECT occurred_at, message,
ai_choose(message, 'Which team should handle this event?',
'{"billing":"Payments, invoices, refunds","technical":"Bugs and outages","other":null}') AS team
FROM events
WHERE occurred_at >= '2026-09-19T00:00:00Z'
AND occurred_at < '2026-09-20T00:00:00Z';
```
### Rating (Score)
`criteria` is a JSON array of **2 to 10 ordered level descriptions**, from low
to high. Descriptions can be strings, objects, or arrays. Level numbers start
at zero. The function returns a JSONB object from one model evaluation:
```json
{
"score": 1.05,
"confidence": 0.92,
"probabilities": [0.0, 0.95, 0.05]
}
```
- `score` is the provider's probability-weighted mean of the level numbers, in
`[0, N − 1]` for N levels. It may fall between levels.
- `confidence` is the provider's confidence in `[0, 1]`, reflecting how concentrated
the distribution is. It is not a guarantee that the answer is correct.
- `probabilities[i]` is the probability of `criteria[i]`, with exactly N entries in
level order. These are individual level probabilities, not cumulative probabilities.
Different distributions can have the same score. `[0, 1, 0]` and `[0.5, 0, 0.5]`
both have score `1`: the first concentrates on the middle level, while the second
is split between the extremes. Inspect confidence and the distribution before
treating a mean score as a definite severity level.
For example, compute the rating once, extract numeric fields, and rank only rows
meeting a chosen confidence threshold:
```sql
SELECT occurred_at, message,
json_get_float(rating, 'score') AS severity,
json_get_float(rating, 'confidence') AS confidence
FROM (
SELECT occurred_at, message,
ai_score(message, 'How severe is this event?',
'["No impact to functionality","Degraded service with a workaround","Blocking issue with no workaround"]') AS rating
FROM events
WHERE occurred_at >= '2026-09-19T00:00:00Z'
AND occurred_at < '2026-09-20T00:00:00Z'
) AS rated
WHERE json_get_float(rating, 'confidence') >= 0.8
ORDER BY severity DESC NULLS LAST;
```
Use `json_to_string(rating)` to display the full object, or
`json_get_float(rating, 'probabilities[2]')` to read the probability of level 2.
The array positions correspond to the supplied criteria, which provide the level
descriptions.
## Reusing an evaluation
AI functions are marked volatile to prevent HTTP calls during planning. Identical
calls written separately in `SELECT` and `WHERE` are evaluated separately. If the
filter evaluates N non-null rows and M rows pass to the projection, this can cost
N + M requests, reaching 2N when all rows pass.
To return a result and filter by it, compute it once in a subquery and reference
its alias in the outer query:
```sql
SELECT occurred_at, message, score
FROM (
SELECT occurred_at, message,
ai_match(message, 'The event reports that a payment still failed after retries.') AS score
FROM events
WHERE occurred_at >= '2026-09-19T00:00:00Z'
AND occurred_at < '2026-09-20T00:00:00Z'
AND service = 'payments'
) AS scored
WHERE score >= 0.8
ORDER BY score DESC;
```
This also applies to `ai_choose` and `ai_score`. Extract multiple JSON fields from
the aliased `ai_score` result, as in the rating example above, rather than calling
the model again for each field. Filter pushdown preserves the volatile projection
instead of duplicating it into the outer predicate.
## MVP behavior
- Each non-null row makes one HTTP request per function invocation: the text is
`state`, and the prompt is the question's `instructions`. Choice and Score pass
the parsed JSON as `criteria`. Calls to different functions are separate requests.
- Criteria for all non-null rows in a batch are validated before sending that
batch's requests. Malformed JSON, invalid description types, or invalid option/
level counts fail the query locally.
- Answers must have the requested question type. Noul probabilities outside
`[0, 1]`, Choice labels not in the criteria, and Score values outside `[0, N − 1]`
fail the query rather than being clamped or replaced with NULL. Score responses
must also contain a numeric confidence in `[0, 1]` and probabilities for every
level. Each probability must be numeric and in `[0, 1]`; their sum must be within
`1e-6` of 1. Provider values are preserved without renormalizing the distribution
or recomputing the score.
- Up to eight requests run concurrently per expression/batch invocation, not per
query or process. Concurrent partitions and queries can exceed eight requests
in total. Each request has a 30-second timeout. Errors (including rate limits
and invalid responses) fail the query. There is no automatic retry or
cross-query cache.
- Start with small, time-bounded queries. Ordinary SQL predicates can reduce the
candidate set, but SQL does not guarantee left-to-right predicate evaluation or
that `LIMIT` bounds the number of API calls.
- Text passed to the function is sent to the configured TypeSafe endpoint.
The API key remains in the server environment, not in SQL.
- This MVP is intended for standalone use. A distributed deployment would need
the environment on every node evaluating the function and separate validation.
API reference: <https://docs.typesafe.ai/api>
## Before stabilizing
The experimental MVP still needs the following controls before stabilization:
- A process-wide concurrency limit shared by AI function invocations, such as a semaphore.
- Bounded retries with exponential backoff for HTTP `429` and `529`, following
TypeSafe's rate-limit guidance.
- Request budgets and metrics for API calls, latency, retries, and rate-limit errors.
These are follow-up work, not guarantees provided by the current implementation.
## Validation
Check both the default build (where all three functions are registered) and an isolated
`common-function` build without default features. The regular AI function tests use a local
HTTP server and need no API key:
```sh
cargo nextest run -p common-function
cargo nextest run -p common-function --no-default-features
cargo nextest run -p common-function --no-default-features --features ai_functions
```
The sqlness runner explicitly includes `ai_functions` when building its test binary:
```sh
cargo sqlness bare -t ai_functions
```
If using `--bins-dir`, provide a binary built with `ai_functions` (included by
default). CI covers the AI-enabled path through the regular unit tests and
sqlness tests; the feature-off checks above can be run locally when needed.
An opt-in test calls the real service on synthetic payment events:
```sh
source ~/.zshrc
GREPTIMEDB_EXPERIMENTAL_JEV=true cargo nextest run -p common-function \
-E 'test(test_ai_match_live)' --run-ignored ignored-only
```
docs/rfcs/2026-09-11-metric-export-import.md
+384 -89
View File
@@ -2,21 +2,27 @@
Feature Name: Metric Export and Import Optimization
Tracking Issue: https://github.com/GreptimeTeam/greptimedb/issues/9120
Date: 2026-09-11
Updated: 2026-09-21
Status: Draft for discussion
---
# Summary
Group the selected Metric logical tables by physical table. Query each physical
table across all its regions, order the rows by `__table_id`, and project each
logical schema into the existing logical-table Parquet files. Restore the same
V2 snapshot with batched logical-table DDL and the existing data COPY path. A
separate generic COPY fix removes repeated directory scans.
Scan Metric tables by physical group, project each logical schema, and pack
small, complete logical-table Parquet streams into shared objects. An index maps
each table to its object and byte range. Restore through a bounded range reader
and normal target-table insertion, with batched logical-table DDL.
The design preserves `manifest.version = 1` and the current V2 layout. It does
not introduce physical backup files or require importing source table IDs.
Merged data writes remain a first-release scope decision, pending representative
measurements after the COPY fix.
Packing is required for the first release of this optimization. Packed snapshots
use `manifest.version = 2`; new importers read versions 1 and 2, while old
importers reject version 2 before executing DDL. The command names remain
`export-v2` and `import-v2`. Existing version-1 snapshots retain their layout.
Each storage write and multipart data part is at most 8 MiB. Objects can be
larger, and a logical table is not split to meet that write size. Shared physical
scans, bounded writer concurrency, removal of redundant object checks, schema
export batching and batch CREATE together address the complete transfer cost.
Merged database insert requests remain a separate follow-up.
# Motivation and evidence
@@ -25,11 +31,13 @@ includes reports of exports taking days for roughly 100,000 logical tables with
only several GiB of data. Table count, physical schema width and data volume must
be measured separately; total database bytes do not describe the transfer cost.
Local experiments support three independent optimizations: shared physical scans,
batched logical CREATE, and removal of repeated file lookup. The
[experiment results](2026-09-11-metric-export-import/experiment-results.md)
contain the measurements, controls and coverage limits. The COPY improvement
uses per-table writes; merged-write benefits remain unmeasured.
The [experiment conclusions](2026-09-11-metric-export-import/experiment-results.md)
support shared scans, batch DDL, explicit-file COPY and packed-object transfer.
Packing with a bounded range reader substantially reduces storage requests and
improves full export and restore for many small logical tables. Benefits depend
on request latency, and memory use increases. Remaining CREATE work makes batch
DDL part of the first release. These local experiments establish direction;
production V2 recovery and release performance still require acceptance tests.
# Terminology
@@ -41,10 +49,12 @@ uses per-table writes; merged-write benefits remain unmeasured.
| Export unit | The selected logical tables sharing one physical table, within one schema and V2 time chunk. This is an export grouping, not a storage object or completion checkpoint. |
| Query execution partition | A stream of work in the query plan. Frontend `target_partitions` influences the partition count; execution partitions are distinct from storage regions and export units. |
| Logical-table descriptor | Captured catalog metadata: source ID, name, physical association, column names/types/order, time index and primary key. Capturing and reusing it provides no snapshot or locking guarantee. |
| Parquet writer | The writer for one logical table's output file in an export unit. |
| Parquet writer | An encoder for one complete logical-table Parquet stream, including its schema and footer. |
| Pack | An object containing consecutive complete Parquet streams, confined to one schema and time chunk. |
| Pack index | The table-to-object/range mapping and object lengths for one schema and time chunk, including standalone objects. |
| Merged writes | Combining insert requests for multiple target logical tables into a region batch dispatch while retaining each table's identity, schema and routing. |
# Physical export and logical files
# Physical export and logical Parquet streams
## Query and routing
@@ -77,14 +87,18 @@ physical table directly; it does not first write and reread `phy.parquet`.
```text
physical table query: selected column union + __table_id, ordered by ID
→ reject rows whose IDs are absent from the captured descriptor set
→ contiguous rows for cpu → project (ts, host, cpu) → cpu.parquet
→ contiguous rows for requests → project (ts, host, service, requests) → requests.parquet
→ contiguous cpu rows → project (ts, host, cpu) → cpu Parquet
→ contiguous requests rows → project (ts, host, service, requests) → requests Parquet
→ append complete small Parquet streams to a shared pack and record their ranges
```
Only one Parquet writer is active per export unit. At an ID transition, close the
previous writer and open the next. Verify non-null UInt32 IDs and monotonic order
across input batches. Create a valid zero-row file for every selected empty table.
Never put `__table_id`, `__tsid`, or another table's columns into logical files.
The router sends each table's contiguous batches to one encoder. At an ID
transition, finish that encoder's input; earlier encoders may finish concurrently
under a request-wide writer budget. Append only complete Parquet streams, so
completion order need not match source table-ID order. Verify non-null UInt32 IDs
and monotonic input order across batches. Encode a valid zero-row Parquet for
every selected empty table. Never include `__table_id`, `__tsid`, or another
table's columns in a logical Parquet stream.
The captured catalog IDs are the routing whitelist; physical rows are not the
source of table membership. Dropped tables can leave residual rows. Predicate
@@ -109,6 +123,36 @@ with query memory accounting, bounded conversion slices, row-group limits and
cancellation. Reject an oversized row explicitly. These controls do not provide
a hard process RSS guarantee.
Start with a 64 MiB request-wide budget for retained/queued Arrow payload and an
8 MiB encoded small-table buffer per admitted writer. When a stream outgrows that
buffer, flush its prefix into a standalone object and continue streaming the
same Parquet file. Bound encoder and upload concurrency by the request's
parallelism, shared across physical groups and ordinary-table work. Count shared
Arrow backing allocations and buffers retained by in-flight tasks, not only the
visible slice sizes. Codec and query reservations remain separate controls.
Finishers retain their writer permits until their buffers are appended or
released. This prevents completed small files from forming an unbounded queue.
Each upload submission is at most 8 MiB, including standalone files and indexes.
CLI chunk concurrency multiplies per-request budgets and must be included in
resource sizing. Index and catalog memory scale with table count and are measured
separately. These initial budgets require the release resource tests below.
## Schema export and destination checks
Reuse an HTTP client and retrieve SHOW CREATE results in bounded multi-statement
requests, initially at most 128 statements and 128 KiB of SQL. Preserve statement
order, identifier quoting, per-statement errors and authorization. Respect response
limits and reject an oversized single request rather than bypassing the byte cap.
This reduces round trips without changing database/physical/logical/view DDL order.
Reuse destination validation only within the same export request and exclusively
owned chunk directory. Check the destination before admission, and preserve
existing collision handling and local-file access rules for independent COPY
operations. An environment switch that globally skips HEAD is not the production
interface. A preflight check alone does not provide ownership against a live
overlapping exporter.
## Ordering and multiple regions
The first release covers every selected Metric physical table, any region count,
@@ -134,21 +178,195 @@ export to per-logical-table queries. The
# Snapshot and metadata contract
Retain the current implementation's layout:
## Versions and capability checks
```text
manifest.json # version 1
schema/schemas.json # schema index
schema/ddl/public.sql # canonical database/table/view DDL
data/public/1/cpu.parquet
data/public/1/requests.parquet
Packed Metric exports write manifest version 2 with a required data-layout marker
`metric-parquet-packs`. Parquet remains the encoding of each table. Ordinary
exports can keep version 1. A version-2 snapshot indexes all data in its schema
chunks, including ordinary tables stored as standalone Parquet objects.
Before creating or replacing a snapshot, the CLI checks the source server's
explicit packed-export capability. Before target DDL or data writes, the importer
checks the manifest version/layout and the target's packed-import capability.
An unsupported packed layout fails; it must not fall through to listing `.parquet`
files and silently omit packed tables. Batch-DDL capability is checked separately.
| Reader and snapshot | Behavior |
| --- | --- |
| New importer, version 1 | Use the existing file layout and normal restore semantics. |
| New importer, version 2 | Require the packed layout and a target supporting packed import. |
| Old importer, version 2 | Reject the unsupported manifest version before DDL or data writes. |
| Unknown version or layout | Reject explicitly. |
Update create, resume, verify, status and import together. Resume retains the
snapshot's original version and layout; it does not migrate an unfinished
version-1 snapshot. Validate old-reader rejection with supported release binaries.
## Manifest and chunk schema
The following is a complete version-2 manifest example. Field names and enum
values are part of the persisted contract; the timestamps and object names are
illustrative.
```json
{
"version": 2,
"data_layout": "metric-parquet-packs",
"snapshot_id": "123e4567-e89b-42d3-a456-426614174000",
"catalog": "greptime",
"schemas": ["public"],
"time_range": {"start": "2026-09-01T00:00:00Z", "end": "2026-09-02T00:00:00Z"},
"schema_only": false,
"format": "parquet",
"chunks": [{
"id": 1,
"time_range": {"start": "2026-09-01T00:00:00Z", "end": "2026-09-02T00:00:00Z"},
"status": "completed",
"files": [
"data/public/1/pack-000000.bin",
"data/public/1/table-000000.parquet",
"data/public/1/pack-index.json"
]
}],
"created_at": "2026-09-19T00:00:00Z",
"updated_at": "2026-09-19T00:01:00Z"
}
```
Schema path segments retain existing V2 encoding. Logical data filenames retain
COPY's raw table-name semantics and import's `file_stem` matching, including dots.
There is no existing general table-filename escaping contract to reuse. Reject
path separators for the fast path; changing that contract requires a separate
compatibility design.
All top-level and chunk fields shown are required in version 2. `snapshot_id`
is a UUID; timestamps are RFC3339; `time_range` retains the existing half-open interval semantics,
with an absent or null bound meaning unbounded on that side. Schema names are
literal identifiers and must be unique. Chunk IDs are unique positive `u32`
integers. Chunk statuses remain `pending`, `in_progress`, `completed`, `skipped`
and `failed`; a failed chunk may carry an optional string `error`.
`schema_only = true` requires an empty `chunks` array and no data objects;
schema-only exports continue to produce version 1.
For version 1, an absent or null `data_layout` selects the existing layout and
all existing field defaults remain valid. Version 2 requires the exact non-null
`data_layout` above and `format = "parquet"`. Reject a layout marker on version
1, a missing or unknown version-2 layout, an unsupported version, or an
incompatible format before DDL, data writes or resume cleanup. Unknown manifest
and chunk fields may be ignored as optional metadata; a required decoding or
integrity feature must change the manifest version, not rely on such a field.
For a completed version-2 chunk, `files` is the duplicate-free, order-independent
inventory of **all** schema indexes, pack objects and standalone objects in that
time chunk. It contains snapshot-relative paths using `/` on every platform.
For each schema, its subset must equal `pack-index.json` plus every object named
by that index, under `data/<encoded-schema>/<chunk-id>/`. No other files belong
in this list, and DDL stays under `schema/`. Require canonical paths with no
leading slash, backslash or traversal; use the existing schema path encoding.
A filename extension never selects the layout or identifies a table.
Every completed schema chunk has an index, including an empty index when its
DDL selects no data-bearing tables. A selected table with no rows still has a
zero-row Parquet entry. A chunk may be `skipped` with `files = []` only when
none of its schemas selects a data-bearing table, not merely when a time window
contains no rows. Pending, in-progress and failed chunks are never importable;
their files or index presence cannot establish completion.
## Objects and index
```text
manifest.json # version 2, metric-parquet-packs layout
schema/schemas.json
schema/ddl/public.sql
data/public/1/pack-000000.bin
data/public/1/table-000000.parquet # large logical or ordinary table
data/public/1/pack-index.json
```
Schema and chunk paths retain V2 encoding and final-directory placement. Object
names are generated within the chunk; table identity comes from the index, not
from a filename or a source ID. A dotted table name remains a literal identifier.
Version-1 file naming and `file_stem` matching remain unchanged.
The matching `pack-index.json` has this shape; the lengths and row counts below
illustrate the mapping rather than an actual encoded fixture:
```json
{
"version": 1,
"objects": [
{"path": "pack-000000.bin", "kind": "pack", "length": 4096},
{"path": "table-000000.parquet", "kind": "parquet", "length": 8192}
],
"tables": [
{"table_name": "cpu", "object": "pack-000000.bin", "offset": 0, "length": 1536, "row_count": 10},
{"table_name": "requests", "object": "pack-000000.bin", "offset": 1536, "length": 2560, "row_count": 20},
{"table_name": "events", "object": "table-000000.parquet", "offset": 0, "length": 8192, "row_count": 100}
]
}
```
The index version is independent of the manifest version and starts at 1.
All shown fields are required. Reject unknown index versions, fields or object
kinds. `length`, `offset` and `row_count` are unsigned `u64` integers; offsets
are zero-based bytes. Object paths are direct chunk children matching
`pack-[0-9]+.bin` or `table-[0-9]+.parquet` according to kind. The `object` field
references an object by its exact path. Table names are literal identifiers.
A standalone entry covers its entire object. Empty tables still have a nonempty,
valid zero-row Parquet stream. Each selected table has exactly one entry per
schema chunk; an empty index has `objects = []` and `tables = []`.
Validate unique object paths and table names, object kinds/references, unsigned
range arithmetic, bounds, and nonoverlapping complete Parquet ranges. For a pack,
the ranges cover its bytes consecutively; standalone objects have one full-range
entry. Restrict object paths to generated direct children of the chunk directory;
reject absolute paths, separators and traversal. Resolve destination tables only
within the request's authorized catalog/schema. An index is not an authorization
grant. Match index membership to the snapshot table DDL, excluding Metric
physical tables and views, so a missing entry cannot silently omit a table.
Validate the index structure and membership before admitting writes, each stream's
Parquet schema through normal COPY checks, and its decoded row count against
the index before reporting that table complete.
Use one streaming pack writer per schema chunk initially. Multiple physical
groups can feed it, with bounded multipart upload concurrency. Append each small
Parquet intact. A pack may exceed 8 MiB; 8 MiB limits one write/part, not the object.
Roll to a new pack at a table boundary before exhausting a backend object/part
limit. A standalone table is subject to both the backend object-size limit and
its multipart part-count limit. With at most N parts and an 8 MiB part cap, its
effective size ceiling is no greater than min(object-size limit, N × 8 MiB), and
can be lower if emitted parts are smaller. Before submitting an out-of-limit
part or exceeding the object limit, fail explicitly, abort the upload and leave
the chunk incomplete. Do not enlarge parts or split the logical table to bypass
these limits. Exercise this boundary with a reduced-part-limit test backend.
Record object lengths while writing and include all standalone outputs in the
index. Restore need not list the directory or rediscover a pack's size for each
table. Stream index serialization in bounded writes; account for retained table
and index metadata independently of data buffers.
## Validation and checksum coverage
Before target DDL, import validates the manifest contract and, for every selected
schema and completed chunk, loads the required index, matches its inventory to
`files`, validates ranges and DDL membership, and checks object existence and
length once per object. Perform the same preflight on import resume, even when
local state records DDL or data tasks as completed. A malformed snapshot must
not reach the DDL executor or create new completed-task records.
Version 2 retains the existing optional `checksum` fields at manifest and chunk
level, but writers omit both. Readers accept absent or null values and reject
non-null values as unsupported before DDL or resume cleanup. The existing
version-1 exporter does not compute these fields and `verify` does not validate
cryptographic checksums; its behavior remains unchanged. This version-2 contract
therefore provides no checksum coverage for the manifest, DDL, indexes or data
objects. Object-store ETags are not substituted for content checksums. Adding
checksums requires a versioned definition of the algorithm, covered bytes and
aggregation order before readers may claim to verify them.
For version 2, `verify` applies the same metadata/index checks across all schemas,
checks completion and object lengths, and reports missing or unexpected data
files. Its result establishes structural completeness, not content-hash or
full row validation. Import additionally validates each Parquet stream and its
decoded row count through COPY; a same-length payload change can escape the
structural check. Detecting corrupt Parquet data during restore may leave partial
target writes, as described in the retry contract below.
## DDL and target metadata
Keep database, physical-table, logical/ordinary-table and view DDL in dependency
order. Physical DDL describes the table and its options; logical DDL describes
@@ -174,75 +392,109 @@ request size and the importer's DDL buffering separately.
The production CLI needs an authenticated server operation that checks CREATE
authorization for every member and reuses the ordinary DDL path. Transport,
capability discovery and request limits must be agreed before accepting this
interface. An older server uses ordinary DDL; do not retry an ambiguously
completed batch through another path.
interface. Initially bound a batch to 128 statements and 1 MiB of SQL, with an
explicit oversized-statement error; separately bound the importer's parsed DDL
buffer. A target without batch-DDL capability uses ordinary DDL if it supports
the snapshot's data layout. Do not retry an ambiguously completed batch through
another path.
## Data COPY and generic directory fix
## Packed data COPY
Read each logical Parquet file through the current COPY path and write through
normal target logical-table routing. No source-ID remapping belongs in these
files, and batch DDL does not imply batched data insertion.
The CLI retains one `COPY DATABASE FROM` operation per `(chunk, schema)` task.
Add an explicit `metric_data_layout = 'packed'` COPY option, paired with
`FORMAT = 'parquet'`, for the advertised capability. The server validates the
index and selected destination tables, then schedules table streams by object
and offset. Use the same explicit layout option for packed export. The existing
Metric feature gate continues to control the experimental export path.
The generic fix changes explicit-file lookup from `stat → list parent → find`
to `stat → use path`, with a sandboxed file-type check for explicit local
inputs to preserve symlink filtering. Directory COPY retains listing, pattern
matching and file-type filtering. Backend authorization and error handling
remain in place. With N files in one directory, this removes roughly N²
directory-entry work.
COPY bypasses the common Lister's filename branch; the branch itself is unchanged.
This fix can ship independently of the Metric design.
A reader owned by that request translates each Parquet-relative read into a
bounded object range. Start with two shared 8 MiB read windows per request;
account for windows pinned by decoders before fetching replacements. Bound active
decoders and decoded batches separately. Large standalone tables stream through
the existing reader rather than being loaded into a small-file buffer.
Reuse adjacent bytes for multiple tables and coalesce duplicate window fetches.
The cache is scoped to this request, authorized storage backend and snapshot;
it is not a process-global cache keyed only by object name. Use index lengths
and perform any necessary existence/size checks once per object, not per table.
Check short reads and Parquet schema/metadata errors normally. Objects must stay
stable for the duration of restore.
Reuse COPY's schema validation, Parquet decoding and normal target-table insertion.
Preserve per-table authorization for all admitted entries and backend/local-file
access controls. Keep current schema selection. Per-table selection is a separate
future interface. Pack order changes storage access, not target table routing.
## Version-1 COPY
Keep the explicit-file optimization from #9126: `stat → use path`, including
the sandboxed local file-type check. Directory COPY retains listing, matching
and filtering. New importers continue to use this path for version-1 snapshots.
General removal of redundant restore metadata requests benefits that path too
and must be included in the baseline when measuring packing's incremental gain.
## Completion, interruption and retry
Keep V2 completion units: export chunk completion and import `(chunk, schema)`
data tasks, with the existing DDL-completed flag. Finish and close every required
file before recording completion. A closed subset of files is not a completed
chunk. On export retry, replace only the unfinished attempt's owned outputs;
never touch completed chunks. Cancellation must close/abort writers and leave
the chunk incomplete.
Keep export chunk completion, import `(chunk, schema)` data tasks and the existing
DDL-completed flag. Write data directly to final chunk paths. Close every pack
and standalone object, then finish the schema chunk's index. Only after all
required schema outputs and indexes succeed may V2 mark the chunk Completed.
Index presence alone does not establish completion; a failed manifest update
leaves the chunk incomplete.
Batch DDL is not a transaction across the restore. A later failure can leave
earlier batches created. Keep DDL completion false until all statements succeed,
then use the current durable state update. Data retries retain existing COPY
semantics and may replay an incomplete task; there is no new exactly-once
guarantee or per-file checkpoint in this proposal.
On failure, stop admission, cancel and drain started work, and abort unfinished
multipart uploads. Explicit retry cleans only owned outputs of an incomplete
chunk, including pack/index files, and reruns that chunk. Preserve completed
chunks and unrelated objects. A transport timeout does not establish that the
server stopped: require the existing stable-source and stopped-writer conditions
before cleanup/retry. Handle normal cancellation's multipart abort separately
from abandoned multipart uploads after a process crash; listing final objects
does not find the latter.
The importer records a data task complete only after every indexed table succeeds.
Disable continue-on-error for this path: skipped failed tables must not produce a
successful task result. Partial target writes may remain after failure. Retry
retains existing COPY replay semantics, without per-table checkpoints or an
exactly-once guarantee. Cancel and drain active readers/inserts before returning
failure, so a later attempt cannot overlap leftover work from that request.
Batch DDL is not transactional across restore. Earlier batches can remain after
a later failure. Keep DDL completion false until all statements succeed, then use
the existing durable state update.
# Rationale and alternatives
| Choice | Reason and trade-off |
| --- | --- |
| Keep logical-table Parquet files | Reuses V2 DDL, file lookup and normal target CREATE/routing with fresh IDs. Physical backup files would need a new restore contract for mixed schemas and source metadata. The retained format still has one file per logical table per time chunk. |
| Scan by physical table | Shares query and scan work among logical tables. Per-logical-table COPY is simpler but retains the table-count overhead; physical scans instead pay for the selected column union and ID routing. |
| Global ID ordering and one Parquet writer per export unit | Contiguous IDs allow a file to be closed at each table transition. Unordered routing would require many open writers or intermediate buffering/files. Ordering bounds writer state, but query sorting and merging need separate resource controls. |
| Use query-engine ordering across regions | Reuses distributed planning and SQL semantics. An exporter-owned region scheduler would add merge, cancellation and retry coordination. The chosen approach must handle both streaming merge and additional-sort plans. |
| Batch DDL while retaining per-table data COPY | Reuses logical CREATE procedures and current data-task replay semantics. Merged writes may reduce dispatch/WAL overhead, but introduce partial-success and checkpoint questions and require a separate benefit comparison. |
| Pack complete logical Parquet streams | Reduces per-object requests while preserving different logical schemas, Parquet metadata and normal insertion. Requires a new snapshot version, index and reader. |
| Scan by physical table | Shares query/scan work; the selected column union and global ordering still need resource controls. |
| Ordered routing with bounded concurrent encoders | Keeps each table contiguous while overlapping encoding and output. Shared byte and task budgets bound downstream work. |
| 8 MiB write/part bound | Limits submission size and in-flight buffering while allowing larger objects. It is an initial balance, not a measured universal optimum. |
| Request-owned range windows | Shares object reads across tables without cross-request or cross-credential cache reuse. Decoder pinning must remain inside the byte budget. |
| Batch DDL and ordinary target insertion | Addresses remaining CREATE overhead while retaining target routing and existing replay semantics. Database merged writes remain a separate follow-up. |
# Unresolved questions
# Decisions still requiring implementation review
- Which authenticated batch-DDL operation and capability-discovery mechanism
should the CLI use? Agree request byte/table limits and DDL buffering bounds.
- What are the query memory, spill-space, writer and export-unit concurrency
limits? Agree the test workloads and resource/performance thresholds before
running the ordering acceptance benchmark.
- Which released import-v2 versions and object-store backends are compatibility
targets? Test those readers against the retained V2 format.
- What name and defaults should the `experimental_` export option use?
- Should merged writes ship in the first release? Compare a bounded merged-write
PoC against corrected per-table COPY with the same batch DDL and concurrency.
Bound decoded rows/bytes and in-flight requests, preserving normal logical
validation and target routing through the existing region batch dispatch.
Cover small/larger files, wide schemas and distributed/multi-region targets;
report repeated total/data times, rows/s, bytes/s, request counts and memory.
Agree a minimum useful gain and memory ceiling before the comparison. Validate
partial success, ambiguous completion and checkpoint advancement, then record
an explicit include/defer decision.
- Bind the authenticated batch-DDL operation to its production transport and
capability response before PR05; preserve ordinary CREATE validation,
per-table authorization and the batch/byte bounds above.
- Validate the initial writer and reader budgets against codec/query memory,
wide physical schemas, concurrent chunks and distributed sorting. Agree
deployment-level memory/spill ceilings before performance runs.
- Pin supported old-reader binaries for version-2 rejection and version-1
compatibility tests. The first-release storage targets are local files and
S3-compatible storage exercised through MinIO.
- Decide when release acceptance permits removing the existing experimental
gate. Packed export/restore capability is required while mixed versions exist.
# Integration and release acceptance
Use existing COPY DATABASE orchestration to form Metric export units. Retain
ordinary COPY for non-Metric tables and other formats, and direct COPY TABLE
behavior. Gate the unfinished Metric fast path behind an `experimental_`
configuration option; disabling it selects the existing path.
configuration option; disabling it selects the existing path. Add packed-import
support before enabling packed snapshot creation. The writer and reader form one
first-release feature, even though they land in separate PRs.
The [tracking issue](https://github.com/GreptimeTeam/greptimedb/issues/9120) owns
the PR breakdown and implementation dependencies. Each implementation slice
@@ -252,9 +504,52 @@ must verify its own correctness and failure behavior. Release acceptance require
| --- | --- |
| Export | Actual CLI export with authorization, multiple physical tables and regions, dense/sparse keys, empty/residual tables, supported Metric Parquet types, and logical schema/value equivalence. |
| Restore | Authenticated remote batch DDL, per-table authorization, fresh target IDs and reconstructed metadata, dependency ordering, older-server behavior and batch failure/retry. |
| V2 lifecycle | Supported released readers; local and object-store interruption, owned-output cleanup, durable completion state and import replay. |
| V2 lifecycle | New-reader version-1/2 coverage, old-reader version-2 rejection, target capability preflight; local/object-store interruption, multipart abort, owned-output cleanup, index/manifest failure and import replay. |
| Scale | Repeated release-build measurements varying table count up to approximately 100,000, rows/file, physical union width, regions and backend. Report median/spread, control execution order/cache, and include schema export and metadata discovery in end-to-end timing. |
| Open decisions | Agreed batch-DDL interface, resource limits, compatibility targets and experimental option; documented merged-write include/defer decision. |
| Open decisions | Agreed batch-DDL interface, resource limits, compatibility targets and experimental gate. |
## Snapshot format acceptance
Run these cases against actual CLI entry points on local files and S3-compatible
storage. Shared serialization tests alone do not establish lifecycle behavior.
| Case | Required result |
| --- | --- |
| Version-1 compatibility | Existing Parquet, CSV, JSON and schema-only fixtures retain import, resume and verify behavior, including omitted optional fields. Resuming export preserves version 1. |
| Valid version 2 | Round-trip multiple schemas/chunks, shared packs with different logical schemas, standalone ordinary/large tables, zero-row tables and empty schemas. Check index membership, typed rows and `/`-separated manifest paths on Windows too. |
| Invalid format before DDL | Missing required fields, unsupported manifest/index version or layout, version/layout/format mismatch, non-null checksums, duplicate or unsafe paths, inventory mismatch, invalid ranges, missing/wrong-length objects, and DDL/index membership mismatch fail before any target DDL or data task, including import resume. Ignorable manifest metadata remains accepted. |
| Old reader | Supported release binaries reject a valid version-2 snapshot before target DDL or writes. |
| Export resume | Interrupt object upload, index finalization and manifest completion separately; resume preserves version/layout and completed chunks, cleans only owned incomplete outputs, then produces the same complete inventory. Reject incompatible metadata before cleanup. |
| Import resume | Revalidate metadata before honoring saved state; skip successfully checkpointed tasks, retain DDL completion and replay semantics, and never mark a partially restored task complete. |
| Verify | Valid completed snapshots pass; incomplete chunks, malformed indexes, inventory mismatch, missing/extra files and object-length mismatches fail. A same-length data mutation is not required to fail a structural-only check. |
## Performance acceptance
Compare four controls with the same data and concurrency: (A) the existing
per-logical-table export and restore with the explicit-file COPY fix; (B) shared
scans, concurrent writers, redundant export HEAD removal and schema-export
batching, still writing standalone table objects; (C) B plus packing and its
reader, with identical DDL handling; (D) C plus batch DDL. If general restore
request deduplication is available, include it in B as well.
The proposed release target is at least 3x faster full export and full restore
for D versus A on 10,000 and 100,000 small logical tables with MinIO request
latency increased by 5 ms. Fix this target during RFC review, before running
release measurements. The independent local C-versus-B experiment establishes
neither this comparison nor proof of that target.
Use Linux release builds and at least three alternating rounds for primary
comparisons. Report medians/spread, schema/data/full durations, CPU, RSS, bytes,
objects, request counts and queue/cache peaks. Include zero-added-latency,
local-file, mixed small/large-table and wide-schema controls. Investigate and
remeasure full-duration regressions exceeding 5% versus A; do not generalize the
latency-injected speedup to these cases. Check values and schemas outside timing.
Validate actual write/part sizes at or below 8 MiB, aggregate concurrency across
groups/chunks, and resource release after success, cancellation and failure.
Metadata growth and query/codec memory remain visible separately from data
buffer budgets. Report C versus B and D versus C independently; do not multiply
speedups from different workloads or attribute all reader savings to packing.
## Ordering acceptance gate
@@ -268,13 +563,13 @@ Exercise both streaming-merge and additional-sort plans.
For each case, report frontend and datanode execution plans, peak memory,
query memory-accounting metrics, spill volume and elapsed time. Run with the
agreed query memory, spill-space, writer and concurrency limits. Verify global
ID order and exactly one complete file per selected logical table in each export
unit, checking schemas and typed rows outside the timed interval.
ID order and exactly one complete indexed Parquet stream per selected logical
table in each schema chunk, checking schemas and typed rows outside timing.
Passing requires correct exports within the agreed resource/performance
thresholds on both plan paths. Also force resource exhaustion: cancellation
must leave the chunk incomplete, and retry with sufficient resources must
produce complete files without altering completed chunks. Writer-only memory
measurements and single-region runs cannot satisfy this gate. Until both the
produce complete indexed data without altering completed chunks. Writer-only
memory measurements and single-region runs cannot satisfy this gate. Until both the
normal and failure/retry cases pass, multi-region resource acceptance remains
open.
docs/rfcs/2026-09-11-metric-export-import/experiment-results.md
+30 -105
View File
@@ -1,109 +1,34 @@
# Metric export/import experiment results
# Metric export/import experiment conclusions
These experiments support three changes: physical-table export, batched logical
DDL, and removal of repeated directory listing during COPY. They measure separate
parts of export/import; their speedups should not be multiplied.
The experiments support shared physical scans, explicit-file COPY, batched
logical DDL, and packing small logical-table Parquet streams into shared objects.
Measurements were taken on September 10–11, 2026, on an ARM64 machine with 16 GiB
RAM, using local standalone instances, local files, warm caches and unoptimized
test builds. Each table entry represents one run. The results establish useful
optimization directions, not production throughput estimates.
- **Shared scans reduce repeated query work.** Projecting each logical schema
before dictionary expansion preserves its columns and types, but does not
remove the memory cost of a wide physical schema or distributed ordering.
- **Explicit-file COPY removes repeated directory traversal.** This improvement
uses ordinary per-table insertion; it is independent of database merged writes.
- **Packing and its range reader reduce small-object overhead.** They improve
complete export and restore and sharply reduce object-store requests. Benefits
grow with table count and request latency; low-latency export gains are smaller.
Some reader request savings can also benefit standalone files.
- **Batch DDL remains necessary.** Once data restore is faster, logical-table
creation becomes the main remaining restore cost. The batch-DDL experiment used
an internal interface; production transport and retry integration remain work.
- **Bounded writes allow larger objects.** The packing experiment kept writes
and upload parts within 8 MiB without imposing that limit on whole objects or
splitting logical tables. This was the chosen size, not a measured optimum.
- **Packing has a memory trade-off.** CPU and request overhead fell, while peak
process memory increased. Encoding buffers, upload concurrency, range windows
and table/index metadata need separate accounting.
## Physical-table export
The exercised fixtures preserved schemas, DDL and typed values after restore.
Earlier standalone-file experiments used the existing importer; the packing
experiment used an independent driver and prototype range reader. The latter
does not establish production packed-format compatibility or V2 resume safety.
The comparison uses 64 populated logical tables with 1,000 source rows each,
plus an empty table. A time filter selects 63,744 rows, written into 65 logical
Parquet files. Ordinary COPY DATABASE uses parallelism one. Timings cover data
export and exclude schema export and fixture preparation. Both execution orders
were measured at narrow and wide physical schemas.
| Physical tag columns | Execution order | Ordinary COPY (ms) | Physical-table export (ms) | Speedup |
| ---: | --- | ---: | ---: | ---: |
| 4 | Ordinary COPY first | 889.4 | 262.7 | 3.39x |
| 4 | Physical export first | 893.7 | 309.5 | 2.89x |
| 67 | Ordinary COPY first | 1062.0 | 316.0 | 3.36x |
| 67 | Physical export first | 869.0 | 308.3 | 2.82x |
Physical-table export was **2.82–3.39x faster** in these runs. Each populated
logical file still contains only its own four columns. Projecting the logical
schema before dictionary expansion kept the expanded-array peak at 0.148 MiB
and the Parquet row-group estimate at 1.112 MiB for both widths. Incoming array
estimates grew from 0.267 MiB to 2.291–2.408 MiB: logical projection does not remove
the memory cost of a wide physical schema. These estimates are not process RSS
limits, and a writer flush threshold does not cap codec allocations.
The measured single-region query used `SeriesScan` and `SortPreservingMergeExec`
without a separate `SortExec`; `__table_id` matches a physical primary-key prefix.
Only Float64 fields, string tags and timestamps were exercised. These runs do not
establish sorting memory, spill or performance when region count exceeds
frontend `target_partitions`; see the RFC's
[ordering acceptance gate](../2026-09-11-metric-export-import.md#ordering-acceptance-gate).
The timestamp encoder reserved about 1 MiB even for a small batch, illustrating
why a writer flush threshold is not an allocation ceiling.
## Batched logical DDL
The restore fixture has logical tables sharing one physical table, one file per
logical table, two rows per populated table and one empty table. The 10,000-table
case contains 19,998 rows. The three paths use SQL HTTP per table, an internal call per table, and
internal batches of up to 128 tables and 1 MiB of rendered SQL, submitted
sequentially. These request bounds do not bound the whole parsed DDL list in
memory. The internal single-call control separates transport savings from
procedure batching.
| Logical tables | SQL HTTP CREATE (s) | Internal single CREATE (s) | Batched CREATE (s) | Logical calls: single → batch |
| ---: | ---: | ---: | ---: | ---: |
| 100 | 0.318 | 0.177 | 0.056 | 100 → 1 |
| 1,000 | 3.091 | 1.601 | 0.544 | 1,000 → 8 |
| 10,000 | 29.181 | 19.524 | 5.776 | 10,000 → 79 |
These are logical CREATE call durations. At 10,000 tables, the complete DDL phase
fell from 30.752 s with SQL HTTP to 7.019 s with batching. Total restore fell from
187.279 s to 157.492 s; unchanged data COPY still took about 150 s. Data-phase
variation is not a DDL benefit. Batching improves CREATE itself, but the original
COPY bottleneck limits its effect on total restore.
The batch executor was in-process. A production authenticated server interface
is still needed; these times do not predict batch HTTP latency.
## COPY file lookup
Both variants below use batch DDL and the same COPY concurrency. Previously,
COPY listed a file's parent directory even when it already knew the filename.
With N files in one directory, repeating this for each file causes roughly N²
directory-entry work. The fix checks the known path directly; directory COPY
still lists files normally.
| Logical tables / files | COPY before (s) | COPY after (s) | Total restore before (s) | Total restore after (s) |
| ---: | ---: | ---: | ---: | ---: |
| 100 | 0.094 | 0.071 | 0.183 | 0.164 |
| 1,000 | 2.135 | 0.612 | 2.849 | 1.289 |
| 10,000 | 146.496 | 5.481 | 153.647 | 12.300 |
At 10,000 files, COPY improved **26.73x** and total restore improved **12.49x**.
Before the fix, backend/file lookup accounted for about 92% of cumulative active
file-COPY time. A control with profiling disabled completed COPY in 5.449 s.
The **5.481 s result still uses per-table writes**, with only 19,998 rows. It is
not a merged-write result. Larger files, distributed targets and object stores
are needed to decide whether merged writes provide enough additional benefit.
## Correctness and remaining coverage
The experiments compared exported/restored schemas and typed rows with the
source. Checks covered dense/sparse keys, different logical schemas, NULL and
empty values, empty tables, dropped-table residue and fresh target IDs. The
unmodified import-v2 command used in the experiment restored the exported files
into a fresh target with different IDs. This does not establish compatibility
with all released readers. Batch-DDL checks covered two schemas, ordinary
tables/views, failure after a successful 128-table batch and retry of the original
DDL. Existing standalone COPY cases also passed.
The internal exporter assumed a trusted caller. It could leave a partial output
directory after failure and did not implement completion publication or resume.
These limitations require production integration work.
Production acceptance still requires release-build measurements with repeated
runs, larger workloads, distributed/multi-region execution, object stores,
released-reader compatibility and export cancellation/recovery. These are tracked
in the [tracking issue](https://github.com/GreptimeTeam/greptimedb/issues/9120).
These were local standalone experiments using development builds, local files
and Docker MinIO. Release acceptance still needs the actual packed V2 path,
request-owned reader isolation, old-reader rejection, large-table throughput,
distributed ordering, cancellation and recovery. See the [RFC](../2026-09-11-metric-export-import.md)
and [tracking issue](https://github.com/GreptimeTeam/greptimedb/issues/9120).
src/catalog/src/system_schema/information_schema/region_statistics.rs
+38 -1
View File
@@ -20,13 +20,18 @@ use common_error::ext::BoxedError;
use common_meta::datanode::RegionStat;
use common_recordbatch::adapter::RecordBatchStreamAdapter;
use common_recordbatch::{DfSendableRecordBatchStream, RecordBatch, SendableRecordBatchStream};
use common_time::timestamp::TimeUnit;
use datafusion::execution::TaskContext;
use datafusion::physical_plan::stream::RecordBatchStreamAdapter as DfRecordBatchStreamAdapter;
use datafusion::physical_plan::streaming::PartitionStream as DfPartitionStream;
use datatypes::prelude::{ConcreteDataType, ScalarVectorBuilder, VectorRef};
use datatypes::schema::{ColumnSchema, Schema, SchemaRef};
use datatypes::timestamp::TimestampMillisecond;
use datatypes::value::Value;
use datatypes::vectors::{StringVectorBuilder, UInt32VectorBuilder, UInt64VectorBuilder};
use datatypes::vectors::{
StringVectorBuilder, TimestampMillisecondVectorBuilder, UInt32VectorBuilder,
UInt64VectorBuilder,
};
use snafu::ResultExt;
use store_api::storage::{ScanRequest, TableId};
@@ -51,6 +56,8 @@ const SST_NUM: &str = "sst_num";
const INDEX_SIZE: &str = "index_size";
const ENGINE: &str = "engine";
const REGION_ROLE: &str = "region_role";
const MIN_TIMESTAMP: &str = "min_timestamp";
const MAX_TIMESTAMP: &str = "max_timestamp";
const INIT_CAPACITY: usize = 42;
@@ -109,6 +116,16 @@ impl InformationSchemaRegionStatistics {
ColumnSchema::new(INDEX_SIZE, ConcreteDataType::uint64_datatype(), true),
ColumnSchema::new(ENGINE, ConcreteDataType::string_datatype(), true),
ColumnSchema::new(REGION_ROLE, ConcreteDataType::string_datatype(), true),
ColumnSchema::new(
MIN_TIMESTAMP,
ConcreteDataType::timestamp_millisecond_datatype(),
true,
),
ColumnSchema::new(
MAX_TIMESTAMP,
ConcreteDataType::timestamp_millisecond_datatype(),
true,
),
]))
}
@@ -175,6 +192,8 @@ struct InformationSchemaRegionStatisticsBuilder {
index_sizes: UInt64VectorBuilder,
engines: StringVectorBuilder,
region_roles: StringVectorBuilder,
min_timestamps: TimestampMillisecondVectorBuilder,
max_timestamps: TimestampMillisecondVectorBuilder,
}
impl InformationSchemaRegionStatisticsBuilder {
@@ -197,6 +216,8 @@ impl InformationSchemaRegionStatisticsBuilder {
index_sizes: UInt64VectorBuilder::with_capacity(INIT_CAPACITY),
engines: StringVectorBuilder::with_capacity(INIT_CAPACITY),
region_roles: StringVectorBuilder::with_capacity(INIT_CAPACITY),
min_timestamps: TimestampMillisecondVectorBuilder::with_capacity(INIT_CAPACITY),
max_timestamps: TimestampMillisecondVectorBuilder::with_capacity(INIT_CAPACITY),
}
}
@@ -261,6 +282,20 @@ impl InformationSchemaRegionStatisticsBuilder {
self.index_sizes.push(Some(region_stat.index_size));
self.engines.push(Some(&region_stat.engine));
self.region_roles.push(Some(&region_stat.role.to_string()));
// Floor the min and ceil the max so the window stays a superset: a narrower
// one would hide regions from a time-bounded lookup.
self.min_timestamps.push(
region_stat
.min_timestamp
.and_then(|ts| ts.convert_to(TimeUnit::Millisecond))
.map(TimestampMillisecond),
);
self.max_timestamps.push(
region_stat
.max_timestamp
.and_then(|ts| ts.convert_to_ceil(TimeUnit::Millisecond))
.map(TimestampMillisecond),
);
}
fn finish(&mut self) -> Result<RecordBatch> {
@@ -280,6 +315,8 @@ impl InformationSchemaRegionStatisticsBuilder {
Arc::new(self.index_sizes.finish()),
Arc::new(self.engines.finish()),
Arc::new(self.region_roles.finish()),
Arc::new(self.min_timestamps.finish()),
Arc::new(self.max_timestamps.finish()),
];
RecordBatch::new(self.schema.clone(), columns).context(CreateRecordBatchSnafu)
src/cli/Cargo.toml
+3
View File
@@ -28,6 +28,7 @@ client = { workspace = true, features = ["testing"] }
common-base.workspace = true
common-catalog.workspace = true
common-config.workspace = true
common-datasource.workspace = true
common-error.workspace = true
common-grpc.workspace = true
common-macro.workspace = true
@@ -62,6 +63,7 @@ serde_json.workspace = true
servers.workspace = true
session.workspace = true
snafu.workspace = true
sql.workspace = true
standalone.workspace = true
store-api.workspace = true
table.workspace = true
@@ -74,5 +76,6 @@ uuid.workspace = true
common-meta = { workspace = true, features = ["testing"] }
common-test-util.workspace = true
common-version.workspace = true
object-store = { workspace = true, features = ["testing"] }
serde.workspace = true
tempfile.workspace = true
src/cli/src/data/export_v2.rs
+9
View File
@@ -37,6 +37,15 @@
//! --start-time 2025-01-01T00:00:00Z \
//! --end-time 2025-01-31T23:59:59Z
//! ```
//!
//! `--experimental-metric-export` enables shared Metric physical scans for Parquet.
//! Enable `experimental_metric_export = true` on the frontend or standalone server
//! and use an endpoint whose frontends all support and enable this option.
//! The command checks the server capability before modifying the snapshot.
//! Each snapshot path belongs to one export task. Resume requires stable source
//! schemas/data and confirmation that the previous export and storage writes ended;
//! an HTTP timeout does not establish that. Only unfinished chunks are cleaned and
//! rerun. Completed chunks and the V2 snapshot/import format remain unchanged.
mod chunker;
mod command;
src/cli/src/data/export_v2/command.rs
+72 -11
View File
@@ -20,9 +20,11 @@ use std::time::Duration;
use async_trait::async_trait;
use clap::{Parser, Subcommand};
use common_catalog::consts::DEFAULT_SCHEMA_NAME;
use common_error::ext::BoxedError;
use common_telemetry::info;
use serde_json::Value;
use servers::http::{ColumnSchema, GreptimeQueryOutput, OutputSchema};
use snafu::{OptionExt, ResultExt};
use crate::Tool;
@@ -289,6 +291,11 @@ pub struct ExportCreateCommand {
#[clap(long, value_enum, default_value = "parquet")]
format: DataFormat,
/// Use shared Metric physical scans (Parquet only). Resume requires that the previous
/// export and storage writes have ended; an HTTP timeout does not establish this.
#[clap(long)]
experimental_metric_export: bool,
/// Delete existing snapshot and recreate.
#[clap(long)]
force: bool,
@@ -379,9 +386,6 @@ impl ExportCreateCommand {
Some(self.schemas.clone())
};
// Build storage
let storage = OpenDalStorage::from_uri(&self.to, &self.storage).map_err(BoxedError::new)?;
// Build database client
let proxy = parse_proxy_opts(self.proxy.clone(), self.no_proxy)?;
let database_client = DatabaseClient::new(
@@ -393,12 +397,43 @@ impl ExportCreateCommand {
self.no_proxy,
);
// The filesystem storage constructor can create the snapshot root.
if self.experimental_metric_export {
if self.format != DataFormat::Parquet {
return crate::data::export_v2::error::MetricExportUnavailableSnafu
.fail()
.map_err(BoxedError::new);
}
let capability = database_client
.sql_response(
"SHOW VARIABLES experimental_metric_export",
DEFAULT_SCHEMA_NAME,
)
.await
.context(DatabaseSnafu)
.map_err(BoxedError::new)?;
let expected_schema = OutputSchema::new(vec![ColumnSchema::new(
"EXPERIMENTAL_METRIC_EXPORT".to_string(),
"String".to_string(),
)]);
if !matches!(capability.output(), [GreptimeQueryOutput::Records(records)]
if records.schema() == &expected_schema
&& records.rows() == &vec![vec![Value::String("true".to_string())]])
{
return crate::data::export_v2::error::MetricExportUnavailableSnafu
.fail()
.map_err(BoxedError::new);
}
}
let storage = OpenDalStorage::from_uri(&self.to, &self.storage).map_err(BoxedError::new)?;
Ok(Box::new(ExportCreate {
config: ExportConfig {
catalog: self.catalog.clone(),
schemas,
schema_only: self.schema_only,
format: self.format,
experimental_metric_export: self.experimental_metric_export,
force: self.force,
time_range,
chunk_time_window: self.chunk_time_window,
@@ -426,6 +461,7 @@ struct ExportConfig {
schemas: Option<Vec<String>>,
schema_only: bool,
format: DataFormat,
experimental_metric_export: bool,
force: bool,
time_range: TimeRange,
chunk_time_window: Option<Duration>,
@@ -468,7 +504,7 @@ impl ExportCreate {
let mut manifest = self.storage.read_manifest().await?;
// Check version compatibility
if manifest.version != MANIFEST_VERSION {
if manifest.version != MANIFEST_VERSION || manifest.data_layout.is_some() {
return ManifestVersionMismatchSnafu {
expected: MANIFEST_VERSION,
found: manifest.version,
@@ -504,6 +540,8 @@ impl ExportCreate {
storage_config: &self.config.storage_config,
parallelism: self.config.parallelism,
chunk_parallelism: self.config.chunk_parallelism,
experimental_metric_export: self.config.experimental_metric_export,
resume: true,
},
progress.as_ref(),
)
@@ -533,6 +571,14 @@ impl ExportCreate {
self.config.chunk_time_window,
)?;
if self.config.experimental_metric_export {
for chunk in &manifest.chunks {
self.storage
.prepare_export_chunk(&schema_names, chunk.id, false)
.await?;
}
}
// 4. Write schema files
self.storage.write_schema(&schema_snapshot).await?;
info!("Exported {} schemas", schema_snapshot.schemas.len());
@@ -566,6 +612,8 @@ impl ExportCreate {
storage_config: &self.config.storage_config,
parallelism: self.config.parallelism,
chunk_parallelism: self.config.chunk_parallelism,
experimental_metric_export: self.config.experimental_metric_export,
resume: false,
},
progress.as_ref(),
)
@@ -960,7 +1008,9 @@ fn directory_word(count: usize) -> &'static str {
}
fn snapshot_status(manifest: &Manifest) -> &'static str {
if manifest.schema_only {
if manifest.validate_layout().is_err() {
"unsupported"
} else if manifest.schema_only {
"schema-only"
} else if manifest.is_complete() {
"complete"
@@ -1083,11 +1133,17 @@ async fn verify_snapshot(storage: &OpenDalStorage) -> Result<VerifyReport> {
problems: Vec::new(),
};
if report.manifest.version != MANIFEST_VERSION {
report.push_error(format!(
"Manifest version mismatch: expected {}, found {}",
MANIFEST_VERSION, report.manifest.version
));
if let Err(reason) = report.manifest.validate_layout() {
report.push_error(reason);
} else if report.manifest.is_packed()
&& let Err(error) = crate::data::import_v2::packed::validate_snapshot(
storage,
&report.manifest,
&report.manifest.schemas,
)
.await
{
report.push_error(error.to_string());
}
if !report.schema_index_exists {
@@ -1384,7 +1440,7 @@ fn safe_manifest_data_file_path(path: &str) -> Option<&str> {
fn print_verify_report(snapshot: &str, report: &VerifyReport) {
println!("Verifying snapshot: {}", report.manifest.snapshot_id);
println!(" Location: {}", snapshot);
if report.manifest.version == MANIFEST_VERSION {
if report.manifest.validate_layout().is_ok() {
println!(" Manifest: OK (version {})", report.manifest.version);
} else {
println!(
@@ -1744,6 +1800,7 @@ mod tests {
schemas: None,
schema_only: false,
format: DataFormat::Parquet,
experimental_metric_export: false,
force: false,
time_range: TimeRange::unbounded(),
chunk_time_window: None,
@@ -1781,6 +1838,7 @@ mod tests {
]),
schema_only: false,
format: DataFormat::Parquet,
experimental_metric_export: false,
force: false,
time_range: TimeRange::unbounded(),
chunk_time_window: None,
@@ -1813,6 +1871,7 @@ mod tests {
schemas: None,
schema_only: false,
format: DataFormat::Parquet,
experimental_metric_export: false,
force: false,
time_range,
chunk_time_window: Some(Duration::from_secs(3600)),
@@ -1846,6 +1905,7 @@ mod tests {
schemas: None,
schema_only: false,
format: DataFormat::Csv,
experimental_metric_export: false,
force: false,
time_range: TimeRange::unbounded(),
chunk_time_window: None,
@@ -1881,6 +1941,7 @@ mod tests {
schemas: None,
schema_only: false,
format: DataFormat::Parquet,
experimental_metric_export: false,
force: false,
time_range: TimeRange::new(Some(start), Some(start)),
chunk_time_window: None,
src/cli/src/data/export_v2/coordinator.rs
+161 -25
View File
@@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use common_telemetry::info;
use common_telemetry::{error, info};
use futures::StreamExt;
use futures::stream::FuturesUnordered;
@@ -39,6 +39,8 @@ struct ExportContext<'a> {
schemas: Vec<String>,
format: DataFormat,
parallelism: usize,
experimental_metric_export: bool,
resume: bool,
}
pub struct ExportDataOptions<'a> {
@@ -46,6 +48,8 @@ pub struct ExportDataOptions<'a> {
pub storage_config: &'a ObjectStoreConfig,
pub parallelism: usize,
pub chunk_parallelism: usize,
pub experimental_metric_export: bool,
pub resume: bool,
}
pub async fn export_data(
@@ -68,6 +72,8 @@ pub async fn export_data(
schemas: manifest.schemas.clone(),
format: manifest.format,
parallelism: options.parallelism,
experimental_metric_export: options.experimental_metric_export,
resume: options.resume,
};
// One progress unit per chunk. Already completed/skipped chunks from a
@@ -88,11 +94,11 @@ pub async fn export_data(
export_data_serial(&context, storage, manifest, progress).await
} else {
export_data_concurrent(
&context,
storage,
manifest,
options.chunk_parallelism,
progress,
|id, range| export_chunk(&context, id, range),
)
.await
};
@@ -134,11 +140,17 @@ async fn export_data_serial(
};
manifest.touch();
storage.write_manifest(manifest).await?;
// The chunk is finalized (completed, skipped, or failed) and persisted.
progress.inc(1);
result?;
let persistence = storage.write_manifest(manifest).await;
if persistence.is_ok() {
progress.inc(1);
}
if let Err(err) = result {
if let Err(secondary) = persistence {
error!(secondary; "Failed to persist failed export chunk");
}
return Err(err);
}
persistence?;
}
Ok(())
@@ -155,39 +167,44 @@ async fn export_data_serial(
/// On the first chunk failure we stop scheduling new chunks but let already
/// in-flight chunks finish and persist their final status, then return the
/// first error.
async fn export_data_concurrent(
context: &ExportContext<'_>,
async fn export_data_concurrent<F, Fut>(
storage: &dyn SnapshotStorage,
manifest: &mut Manifest,
chunk_parallelism: usize,
progress: &dyn ProgressReporter,
) -> Result<()> {
export: F,
) -> Result<()>
where
F: Fn(u32, TimeRange) -> Fut,
Fut: std::future::Future<Output = Result<Vec<String>>>,
{
let mut pending = FuturesUnordered::new();
let mut next_idx = 0;
let mut first_error: Option<Error> = None;
loop {
let mut scheduled = false;
let mut admitted = Vec::new();
// Schedule eligible chunks in order up to the parallelism limit. Once a
// failure is seen, stop scheduling but keep draining in-flight chunks.
while first_error.is_none() && pending.len() < chunk_parallelism {
while first_error.is_none() && pending.len() + admitted.len() < chunk_parallelism {
let Some(idx) = next_eligible_chunk(manifest, &mut next_idx) else {
break;
};
let (chunk_id, time_range) = mark_chunk_in_progress(manifest, idx);
scheduled = true;
pending.push(async move {
let result = export_chunk(context, chunk_id, time_range).await;
(idx, result)
});
admitted.push((idx, chunk_id, time_range));
}
if scheduled {
if !admitted.is_empty() {
manifest.touch();
storage.write_manifest(manifest).await?;
match storage.write_manifest(manifest).await {
Ok(()) => {
for (idx, chunk_id, time_range) in admitted {
let export = &export;
pending.push(async move { (idx, export(chunk_id, time_range).await) });
}
}
Err(err) => first_error = Some(err),
}
}
let Some((idx, export_result)) = pending.next().await else {
@@ -204,9 +221,11 @@ async fn export_data_concurrent(
}
}
manifest.touch();
storage.write_manifest(manifest).await?;
// The chunk is finalized (completed, skipped, or failed) and persisted.
progress.inc(1);
match storage.write_manifest(manifest).await {
Ok(()) => progress.inc(1),
Err(err) if first_error.is_none() => first_error = Some(err),
Err(err) => error!(err; "Failed to persist export chunk while draining"),
}
}
match first_error {
@@ -257,12 +276,19 @@ async fn export_chunk(
chunk_id: u32,
time_range: TimeRange,
) -> Result<Vec<String>> {
if context.experimental_metric_export {
context
.storage
.prepare_export_chunk(&context.schemas, chunk_id, context.resume)
.await?;
}
let scheme = StorageScheme::from_uri(context.snapshot_uri)?;
let needs_dir = matches!(scheme, StorageScheme::File);
let copy_options = CopyOptions {
format: context.format,
time_range,
parallelism: context.parallelism,
experimental_metric_export: context.experimental_metric_export,
};
for schema in &context.schemas {
@@ -326,6 +352,116 @@ mod tests {
manifest
}
struct FailingManifest {
fail_at: usize,
writes: std::sync::atomic::AtomicUsize,
release: tokio::sync::Notify,
}
#[async_trait::async_trait]
impl SnapshotStorage for FailingManifest {
async fn write_manifest(&self, manifest: &Manifest) -> Result<()> {
use std::sync::atomic::Ordering;
let write = self.writes.fetch_add(1, Ordering::SeqCst) + 1;
if write == 1 {
assert_eq!(manifest.in_progress_count(), 2);
}
if write == self.fail_at {
self.release.notify_one();
return crate::data::export_v2::error::IoSnafu {
operation: "injected manifest failure",
error: std::io::Error::other("manifest failure"),
}
.fail();
}
Ok(())
}
async fn exists(&self) -> Result<bool> {
unreachable!()
}
async fn read_manifest(&self) -> Result<Manifest> {
unreachable!()
}
async fn file_size(&self, _: &str) -> Result<Option<u64>> {
unreachable!()
}
async fn write_schema(
&self,
_: &crate::data::export_v2::schema::SchemaSnapshot,
) -> Result<()> {
unreachable!()
}
async fn write_text(&self, _: &str, _: &str) -> Result<()> {
unreachable!()
}
async fn read_text(&self, _: &str) -> Result<String> {
unreachable!()
}
async fn create_dir_all(&self, _: &str) -> Result<()> {
unreachable!()
}
async fn list_files_recursive(&self, _: &str) -> Result<Vec<String>> {
unreachable!()
}
async fn delete_snapshot(&self) -> Result<()> {
unreachable!()
}
}
#[tokio::test]
async fn test_manifest_failure_stops_admission_and_drains_started_chunks() {
use std::sync::atomic::{AtomicUsize, Ordering};
for fail_at in [1, 2, 3] {
let storage = FailingManifest {
fail_at,
writes: AtomicUsize::new(0),
release: tokio::sync::Notify::new(),
};
let started = AtomicUsize::new(0);
let finished = AtomicUsize::new(0);
let second_started = tokio::sync::Notify::new();
let mut manifest = pending_manifest(4);
let result = tokio::time::timeout(
std::time::Duration::from_secs(5),
export_data_concurrent(
&storage,
&mut manifest,
2,
&crate::data::progress::NoopProgress,
|id, _| {
let (started, finished, second_started, storage) =
(&started, &finished, &second_started, &storage);
async move {
started.fetch_add(1, Ordering::SeqCst);
match id {
1 => second_started.notified().await,
2 => {
second_started.notify_one();
storage.release.notified().await;
}
_ => panic!("new chunk started after manifest persistence failed"),
}
finished.fetch_add(1, Ordering::SeqCst);
Ok(vec![format!("data/public/{id}/a.parquet")])
}
},
),
)
.await
.unwrap();
assert!(
result
.unwrap_err()
.to_string()
.contains("injected manifest failure")
);
let expected = if fail_at == 1 { 0 } else { 2 };
assert_eq!(started.load(Ordering::SeqCst), expected);
assert_eq!(finished.load(Ordering::SeqCst), expected);
}
}
#[test]
fn test_next_eligible_chunk_scans_in_order() {
let manifest = pending_manifest(3);
src/cli/src/data/export_v2/data.rs
+11 -1
View File
@@ -30,6 +30,7 @@ pub(super) struct CopyOptions {
pub(super) format: DataFormat,
pub(super) time_range: TimeRange,
pub(super) parallelism: usize,
pub(crate) experimental_metric_export: bool,
}
pub(super) struct CopyTarget {
@@ -190,13 +191,19 @@ pub(crate) async fn execute_copy_database_from(
schema: &str,
source: &CopySource,
format: DataFormat,
packed: bool,
) -> Result<()> {
let sql = format!(
r#"COPY DATABASE "{}"."{}" FROM '{}' WITH (FORMAT='{}'){};"#,
r#"COPY DATABASE "{}"."{}" FROM '{}' WITH (FORMAT='{}'{}){};"#,
escape_sql_identifier(catalog),
escape_sql_identifier(schema),
escape_sql_literal(&source.location),
format,
if packed {
", metric_data_layout='packed'"
} else {
""
},
source.connection
);
let safe_sql = source.mask_sql(&sql);
@@ -210,6 +217,9 @@ pub(crate) async fn execute_copy_database_from(
fn build_with_options(options: &CopyOptions) -> String {
let mut parts = vec![format!("FORMAT='{}'", options.format)];
if options.experimental_metric_export {
parts.push("experimental_metric_export='true'".to_string());
}
if let Some(start) = options.time_range.start {
parts.push(format!(
"START_TIME='{}'",
src/cli/src/data/export_v2/error.rs
+10 -1
View File
@@ -24,6 +24,14 @@ use snafu::{Location, Snafu};
#[snafu(visibility(pub))]
#[stack_trace_debug]
pub enum Error {
#[snafu(display(
"Experimental Metric export requires Parquet and a frontend that explicitly reports experimental_metric_export=true. Use an upgraded, consistently configured frontend entry point."
))]
MetricExportUnavailable {
#[snafu(implicit)]
location: Location,
},
#[snafu(display("Invalid URI '{}': {}", uri, reason))]
InvalidUri {
uri: String,
@@ -211,7 +219,8 @@ pub type Result<T> = std::result::Result<T, Error>;
impl ErrorExt for Error {
fn status_code(&self) -> StatusCode {
match self {
Error::InvalidUri { .. }
Error::MetricExportUnavailable { .. }
| Error::InvalidUri { .. }
| Error::UnsupportedScheme { .. }
| Error::SchemaOnlyModeMismatch { .. }
| Error::ResumeConfigMismatch { .. }
src/cli/src/data/export_v2/manifest.rs
+169 -1
View File
@@ -14,6 +14,7 @@
//! Manifest data structures for Export/Import V2.
use std::collections::HashSet;
use std::time::Duration;
use std::{fmt, str};
@@ -27,7 +28,7 @@ use crate::data::export_v2::error::{
TimeParseInvalidFormatSnafu,
};
/// Current manifest format version.
/// Manifest format version produced by the current exporter.
pub const MANIFEST_VERSION: u32 = 1;
/// Manifest file name within snapshot directory.
@@ -234,6 +235,8 @@ impl str::FromStr for DataFormat {
pub struct Manifest {
/// Manifest format version for compatibility checking.
pub version: u32,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub data_layout: Option<String>,
/// Unique snapshot identifier.
pub snapshot_id: Uuid,
/// Catalog name.
@@ -259,6 +262,81 @@ pub struct Manifest {
}
impl Manifest {
/// Decodes version-specific required fields without changing version-1 defaults.
pub fn from_json(data: &[u8]) -> serde_json::Result<Self> {
use serde::de::Error;
let value: serde_json::Value = serde_json::from_slice(data)?;
if value["version"] == 2 {
let chunks = value["chunks"].as_array().ok_or_else(|| {
serde_json::Error::custom("version-2 manifest requires a chunks array")
})?;
for chunk in chunks {
if !chunk["files"].is_array() {
return Err(serde_json::Error::custom(
"version-2 chunk requires a files array",
));
}
}
}
serde_json::from_slice(data)
}
/// Validates the encoding contract before any snapshot operation.
pub fn validate_layout(&self) -> std::result::Result<(), String> {
match (self.version, self.data_layout.as_deref(), self.format) {
(1, None, _) => Ok(()),
(2, Some(common_datasource::packed_snapshot::PACKED_LAYOUT), DataFormat::Parquet) => {
if self.checksum.is_some() || self.chunks.iter().any(|c| c.checksum.is_some()) {
return Err("version-2 checksums are unsupported".into());
}
if self.schema_only != self.chunks.is_empty() {
return Err("version-2 schema_only requires empty chunks; data snapshots require chunks".into());
}
let schemas: HashSet<_> = self.schemas.iter().collect();
if schemas.len() != self.schemas.len() {
return Err("duplicate version-2 schemas".into());
}
let mut ids = HashSet::new();
for chunk in &self.chunks {
if chunk.id == 0 || !ids.insert(chunk.id) {
return Err("version-2 chunk IDs must be unique and positive".into());
}
let files: HashSet<_> = chunk.files.iter().collect();
if files.len() != chunk.files.len() {
return Err("duplicate version-2 chunk files".into());
}
if chunk.status == ChunkStatus::Skipped && !chunk.files.is_empty() {
return Err("skipped version-2 chunks must have no files".into());
}
for file in &chunk.files {
if file.contains('\\')
|| file
.split('/')
.any(|part| part.is_empty() || part == "." || part == "..")
|| !self.schemas.iter().any(|schema| {
file.starts_with(&crate::data::path::data_dir_for_schema_chunk(
schema, chunk.id,
))
})
{
return Err(format!("invalid version-2 chunk file: {file}"));
}
}
}
Ok(())
}
_ => Err(format!(
"Manifest version mismatch or unsupported layout: version {}, layout {:?}, format {}",
self.version, self.data_layout, self.format
)),
}
}
pub fn is_packed(&self) -> bool {
self.version == 2
}
pub fn new_for_export(
catalog: String,
schemas: Vec<String>,
@@ -293,6 +371,7 @@ impl Manifest {
let now = Utc::now();
Self {
version: MANIFEST_VERSION,
data_layout: None,
snapshot_id: Uuid::new_v4(),
catalog,
schemas,
@@ -316,6 +395,7 @@ impl Manifest {
let now = Utc::now();
Self {
version: MANIFEST_VERSION,
data_layout: None,
snapshot_id: Uuid::new_v4(),
catalog,
schemas,
@@ -413,6 +493,94 @@ fn generate_single_chunk(time_range: &TimeRange) -> Vec<ChunkMeta> {
#[cfg(test)]
mod tests {
#[test]
fn packed_manifest_read_contract_preserves_legacy_writer() {
let mut manifest =
super::Manifest::new_schema_only("greptime".into(), vec!["public".into()]);
assert!(manifest.validate_layout().is_ok());
manifest.version = 2;
assert!(manifest.validate_layout().is_err());
manifest.data_layout = Some("metric-parquet-packs".into());
assert!(manifest.validate_layout().is_ok());
manifest.format = super::DataFormat::Csv;
assert!(manifest.validate_layout().is_err());
manifest.format = super::DataFormat::Parquet;
manifest.version = 3;
assert!(manifest.validate_layout().is_err());
manifest.version = 1;
assert!(manifest.validate_layout().is_err());
}
#[test]
fn packed_required_fields_and_checksums_preserve_v1_defaults() {
let mut manifest =
super::Manifest::new_schema_only("greptime".into(), vec!["public".into()]);
manifest
.chunks
.push(super::ChunkMeta::new(1, super::TimeRange::unbounded()));
let legacy = serde_json::to_value(manifest).unwrap();
for version in [1, 2] {
let mut value = legacy.clone();
value["version"] = version.into();
if version == 2 {
value["data_layout"] = "metric-parquet-packs".into();
value["schema_only"] = false.into();
}
let raw = serde_json::to_string(&value).unwrap();
for duplicate in [
format!(
"{{\"checksum\":\"unsupported\",\"checksum\":null,{}",
&raw[1..]
),
format!("{{\"chunks\":[],{}", &raw[1..]),
raw.replacen("\"files\":[]", "\"files\":[],\"files\":[]", 1),
raw.replacen(
"\"files\":[]",
"\"checksum\":\"unsupported\",\"checksum\":null,\"files\":[]",
1,
),
] {
assert!(super::Manifest::from_json(duplicate.as_bytes()).is_err());
}
for field in ["chunks", "files"] {
let mut missing = value.clone();
if field == "chunks" {
missing.as_object_mut().unwrap().remove(field);
} else {
missing["chunks"][0].as_object_mut().unwrap().remove(field);
}
assert_eq!(
super::Manifest::from_json(&serde_json::to_vec(&missing).unwrap()).is_ok(),
version == 1
);
}
for checksum in [
None,
Some(serde_json::Value::Null),
Some("".into()),
Some("sha256:abc".into()),
] {
for chunk_level in [false, true] {
let mut candidate = value.clone();
if let Some(checksum) = &checksum {
if chunk_level {
candidate["chunks"][0]["checksum"] = checksum.clone();
} else {
candidate["checksum"] = checksum.clone();
}
}
let decoded =
super::Manifest::from_json(&serde_json::to_vec(&candidate).unwrap())
.unwrap();
assert_eq!(
decoded.validate_layout().is_ok(),
version == 1 || checksum.as_ref().is_none_or(serde_json::Value::is_null)
);
}
}
}
}
use std::time::Duration;
use chrono::{TimeZone, Utc};
src/cli/src/data/import_v2.rs
+1
View File
@@ -38,6 +38,7 @@ mod command;
pub(crate) mod coordinator;
pub mod error;
pub mod executor;
pub(crate) mod packed;
pub(crate) mod state;
pub use command::ImportV2Command;
src/cli/src/data/import_v2/command.rs
+79 -10
View File
@@ -27,15 +27,15 @@ use snafu::{OptionExt, ResultExt};
use crate::Tool;
use crate::common::ObjectStoreConfig;
use crate::data::export_v2::data::{build_copy_source, execute_copy_database_from};
use crate::data::export_v2::manifest::{ChunkMeta, ChunkStatus, DataFormat, MANIFEST_VERSION};
use crate::data::export_v2::manifest::{ChunkMeta, ChunkStatus, DataFormat};
use crate::data::import_v2::coordinator::{
ImportResumeConfig, ImportTaskExecutor, build_import_tasks, chunk_has_schema_files,
import_with_resume_session_with_progress, prepare_import_resume,
};
use crate::data::import_v2::error::{
ChunkImportFailedSnafu, EmptyChunkManifestSnafu, ImportStatePathUnavailableSnafu,
IncompleteSnapshotSnafu, ManifestVersionMismatchSnafu, MissingChunkDataSnafu, Result,
SchemaNotInSnapshotSnafu, SnapshotStorageSnafu,
IncompleteSnapshotSnafu, MissingChunkDataSnafu, Result, SchemaNotInSnapshotSnafu,
SnapshotStorageSnafu,
};
use crate::data::import_v2::executor::{DdlExecutor, DdlStatement};
use crate::data::import_v2::state::{ImportTaskKey, default_state_path};
@@ -219,13 +219,14 @@ impl Import {
manifest.snapshot_id, manifest.version, manifest.schema_only
);
// Check version compatibility
if manifest.version != MANIFEST_VERSION {
return ManifestVersionMismatchSnafu {
expected: MANIFEST_VERSION,
found: manifest.version,
}
.fail();
manifest.validate_layout().map_err(|reason| {
crate::data::import_v2::error::InvalidPackedSnapshotSnafu { reason }.build()
})?;
if manifest.is_packed() {
self.database_client
.require_packed_import()
.await
.context(crate::data::import_v2::error::DatabaseSnafu)?;
}
info!("Snapshot contains {} schema(s)", manifest.schemas.len());
@@ -251,6 +252,15 @@ impl Import {
Vec::new()
};
if manifest.is_packed() {
crate::data::import_v2::packed::validate_snapshot(
self.storage.as_ref(),
&manifest,
&schemas_to_import,
)
.await?;
}
// 4. Dry-run mode: print DDL and exit
if self.dry_run {
info!("Dry-run mode - DDL statements to execute:");
@@ -317,6 +327,7 @@ impl Import {
let executor = CopyDatabaseImportTaskExecutor {
import: self,
format: manifest.format,
packed: manifest.is_packed(),
};
let progress = build_progress_reporter(self.progress);
import_with_resume_session_with_progress(resume_session, &executor, progress.as_ref())
@@ -358,6 +369,7 @@ impl Import {
struct CopyDatabaseImportTaskExecutor<'a> {
import: &'a Import,
format: DataFormat,
packed: bool,
}
#[async_trait]
@@ -380,6 +392,7 @@ impl ImportTaskExecutor for CopyDatabaseImportTaskExecutor<'_> {
&task.schema,
&source,
self.format,
self.packed,
)
.await
.context(ChunkImportFailedSnafu {
@@ -684,6 +697,13 @@ mod tests {
Ok(self.manifest.clone())
}
async fn file_size(
&self,
_path: &str,
) -> crate::data::export_v2::error::Result<Option<u64>> {
unimplemented!("not needed in import_v2::command tests")
}
async fn write_manifest(
&self,
_manifest: &Manifest,
@@ -731,6 +751,55 @@ mod tests {
}
}
#[tokio::test]
async fn packed_capability_failure_precedes_ddl_reads_and_state_writes() {
use tokio::io::{AsyncReadExt, AsyncWriteExt};
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let address = listener.local_addr().unwrap();
let server = tokio::spawn(async move {
let (mut socket, _) = listener.accept().await.unwrap();
let mut request = vec![0; 4096];
let n = socket.read(&mut request).await.unwrap();
assert!(String::from_utf8_lossy(&request[..n]).starts_with("GET /v1/capabilities "));
socket
.write_all(
b"HTTP/1.1 404 Not Found\r\nContent-Length: 2\r\nConnection: close\r\n\r\n{}",
)
.await
.unwrap();
});
let dir = tempfile::tempdir().unwrap();
let state_path = dir.path().join("state.json");
let mut manifest = Manifest::new_schema_only("greptime".into(), vec!["public".into()]);
manifest.version = 2;
manifest.data_layout = Some("metric-parquet-packs".into());
let import = Import {
catalog: "greptime".into(),
schemas: None,
dry_run: false,
progress: ProgressMode::default(),
task_parallelism: 1,
state_path: Some(state_path.clone()),
snapshot_uri: "file:///unused".into(),
storage_config: Default::default(),
storage: Box::new(StubStorage {
manifest,
files_by_prefix: Default::default(),
}),
database_client: DatabaseClient::new(
address.to_string(),
"greptime".into(),
None,
std::time::Duration::from_secs(5),
None,
true,
),
};
assert!(import.run().await.is_err());
assert!(!state_path.exists());
server.await.unwrap();
}
fn parse_command(extra: &[&str]) -> ImportV2Command {
let mut args = vec![
"import-v2",
src/cli/src/data/import_v2/error.rs
+4 -9
View File
@@ -25,6 +25,9 @@ use crate::data::export_v2::manifest::ChunkStatus;
#[snafu(visibility(pub))]
#[stack_trace_debug]
pub enum Error {
#[snafu(display("Invalid packed snapshot: {reason}"))]
InvalidPackedSnapshot { reason: String },
#[snafu(display("Snapshot not found at '{}'", uri))]
SnapshotNotFound {
uri: String,
@@ -32,14 +35,6 @@ pub enum Error {
location: Location,
},
#[snafu(display("Manifest version mismatch: expected {}, found {}", expected, found))]
ManifestVersionMismatch {
expected: u32,
found: u32,
#[snafu(implicit)]
location: Location,
},
#[snafu(display("Schema '{}' not found in snapshot", schema))]
SchemaNotInSnapshot {
schema: String,
@@ -190,9 +185,9 @@ impl ErrorExt for Error {
match self {
Error::SnapshotNotFound { .. }
| Error::SchemaNotInSnapshot { .. }
| Error::ManifestVersionMismatch { .. }
| Error::IncompleteSnapshot { .. }
| Error::EmptyChunkManifest { .. }
| Error::InvalidPackedSnapshot { .. }
| Error::MissingChunkData { .. } => StatusCode::InvalidArguments,
Error::ImportStatePathUnavailable { .. }
| Error::ImportStateUnknownTask { .. }
src/cli/src/data/import_v2/packed.rs
+225
View File
@@ -0,0 +1,225 @@
// Copyright 2023 Greptime Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use std::collections::HashSet;
use common_datasource::packed_snapshot::{PACK_INDEX_FILE, PackIndex};
use snafu::ResultExt;
use sql::dialect::GreptimeDbDialect;
use sql::parser::{ParseOptions, ParserContext};
use sql::statements::statement::Statement;
use crate::data::export_v2::manifest::{ChunkStatus, Manifest};
use crate::data::import_v2::error::{InvalidPackedSnapshotSnafu, Result, SnapshotStorageSnafu};
use crate::data::path::{data_dir_for_schema_chunk, ddl_path_for_schema};
use crate::data::snapshot_storage::SnapshotStorage;
/// Matches every completed schema chunk to its source DDL before target writes.
pub(crate) async fn validate_snapshot(
storage: &dyn SnapshotStorage,
manifest: &Manifest,
schemas: &[String],
) -> Result<()> {
manifest
.validate_layout()
.map_err(|reason| InvalidPackedSnapshotSnafu { reason }.build())?;
if manifest.schema_only
&& !storage
.list_files_recursive("data/")
.await
.context(SnapshotStorageSnafu)?
.is_empty()
{
return InvalidPackedSnapshotSnafu {
reason: "schema-only snapshot contains data objects",
}
.fail();
}
for schema in schemas {
let ddl = storage
.read_text(&ddl_path_for_schema(schema))
.await
.context(SnapshotStorageSnafu)?;
let statements = ParserContext::create_with_dialect(
&ddl,
&GreptimeDbDialect {},
ParseOptions::default(),
)
.map_err(|e| {
InvalidPackedSnapshotSnafu {
reason: e.to_string(),
}
.build()
})?;
let mut names = HashSet::new();
for statement in statements {
if let Statement::CreateTable(create) = statement {
if create.engine.eq_ignore_ascii_case("metric")
&& create.options.get("on_physical_table").is_none()
{
continue;
}
let parts: Vec<_> = create
.name
.0
.iter()
.map(|p| p.as_ident().map(|i| i.value.as_str()))
.collect();
let allowed = match parts.as_slice() {
[Some(_)] => true,
[Some(s), Some(_)] => *s == schema,
[Some(c), Some(s), Some(_)] => *c == manifest.catalog && *s == schema,
_ => false,
};
if !allowed {
return InvalidPackedSnapshotSnafu {
reason: "snapshot table DDL escapes its catalog/schema",
}
.fail();
}
let name = parts.last().and_then(|p| *p).unwrap_or_default();
if !names.insert(name.to_string()) {
return InvalidPackedSnapshotSnafu {
reason: "duplicate snapshot table DDL",
}
.fail();
}
}
}
for chunk in &manifest.chunks {
if chunk.status == ChunkStatus::Skipped && !names.is_empty() {
return InvalidPackedSnapshotSnafu {
reason: "packed chunks require zero-row streams for empty tables",
}
.fail();
}
if chunk.status != ChunkStatus::Completed {
continue;
}
let path = format!(
"{}{PACK_INDEX_FILE}",
data_dir_for_schema_chunk(schema, chunk.id)
);
let text = storage
.read_text(&path)
.await
.context(SnapshotStorageSnafu)?;
let index: PackIndex = serde_json::from_str(&text).map_err(|e| {
InvalidPackedSnapshotSnafu {
reason: e.to_string(),
}
.build()
})?;
let prefix = data_dir_for_schema_chunk(schema, chunk.id);
let expected: HashSet<_> = std::iter::once(path)
.chain(index.objects.iter().map(|o| format!("{prefix}{}", o.path)))
.collect();
let actual: HashSet<_> = chunk
.files
.iter()
.filter(|file| file.starts_with(&prefix))
.cloned()
.collect();
if actual != expected {
return InvalidPackedSnapshotSnafu {
reason: "chunk manifest inventory differs from indexed objects",
}
.fail();
}
index
.validate_membership(names.iter().map(String::as_str))
.map_err(|e| {
InvalidPackedSnapshotSnafu {
reason: e.to_string(),
}
.build()
})?;
for object in &index.objects {
let path = format!("{prefix}{}", object.path);
let length = storage
.file_size(&path)
.await
.context(SnapshotStorageSnafu)?;
if length != Some(object.length) {
return InvalidPackedSnapshotSnafu {
reason: format!(
"object {path}: expected length {}, actual {length:?}",
object.length
),
}
.fail();
}
}
}
}
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
use crate::data::export_v2::manifest::{ChunkMeta, TimeRange};
use crate::data::snapshot_storage::OpenDalStorage;
#[tokio::test]
async fn packed_membership_uses_literal_names_and_excludes_physical_and_views() {
let dir = tempfile::Builder::new()
.prefix("packed snapshot # ")
.tempdir()
.unwrap();
let uri = url::Url::from_file_path(dir.path()).unwrap();
let storage = OpenDalStorage::from_uri(uri.as_str(), &Default::default()).unwrap();
storage.write_text("schema/ddl/public.sql", "CREATE TABLE p (ts TIMESTAMP TIME INDEX) ENGINE=metric WITH(physical_metric_table=''); CREATE TABLE \"logical.name\" (ts TIMESTAMP TIME INDEX) ENGINE=metric WITH(on_physical_table='p'); CREATE VIEW v AS SELECT * FROM \"logical.name\";").await.unwrap();
let path = "data/public/1/pack-index.json";
let index = serde_json::json!({"version":1,"objects":[{"path":"pack-0.bin","kind":"pack","length":12}],"tables":[{"table_name":"logical.name","object":"pack-0.bin","offset":0,"length":12,"row_count":0}]});
storage.write_text(path, &index.to_string()).await.unwrap();
assert!(dir.path().join(path).is_file());
storage
.write_text("data/public/1/pack-0.bin", "abcdefghijkl")
.await
.unwrap();
let mut manifest = Manifest::new_schema_only("greptime".into(), vec!["public".into()]);
let mut chunk = ChunkMeta::new(1, TimeRange::unbounded());
chunk.mark_completed(vec![path.into(), "data/public/1/pack-0.bin".into()], None);
manifest.version = 2;
manifest.data_layout = Some("metric-parquet-packs".into());
manifest.schema_only = false;
manifest.chunks.push(chunk);
validate_snapshot(&storage, &manifest, &manifest.schemas)
.await
.unwrap();
let mut missing = index.clone();
missing["tables"] = serde_json::json!([]);
storage
.write_text(path, &missing.to_string())
.await
.unwrap();
assert!(
validate_snapshot(&storage, &manifest, &manifest.schemas)
.await
.is_err()
);
let mut escaped = index;
escaped["objects"][0]["path"] = "../pack-0.bin".into();
storage
.write_text(path, &escaped.to_string())
.await
.unwrap();
assert!(
validate_snapshot(&storage, &manifest, &manifest.schemas)
.await
.is_err()
);
}
}
src/cli/src/data/snapshot_storage.rs
+231 -1
View File
@@ -40,6 +40,7 @@ use crate::data::export_v2::manifest::{MANIFEST_FILE, Manifest};
#[cfg(test)]
use crate::data::export_v2::schema::SchemaDefinition;
use crate::data::export_v2::schema::{SCHEMA_DIR, SCHEMAS_FILE, SchemaSnapshot};
use crate::data::path::data_dir_for_schema_chunk;
struct RemoteLocation {
bucket_or_container: String,
@@ -267,6 +268,9 @@ pub trait SnapshotStorage: Send + Sync {
/// Reads the manifest file.
async fn read_manifest(&self) -> Result<Manifest>;
/// Returns file length, or None for a missing path or directory.
async fn file_size(&self, path: &str) -> Result<Option<u64>>;
/// Writes the manifest file.
async fn write_manifest(&self, manifest: &Manifest) -> Result<()>;
@@ -285,6 +289,21 @@ pub trait SnapshotStorage: Send + Sync {
/// Lists files recursively under a relative prefix.
async fn list_files_recursive(&self, prefix: &str) -> Result<Vec<String>>;
/// Checks fresh output directories or removes a terminated, unfinished chunk's files.
async fn prepare_export_chunk(
&self,
schemas: &[String],
chunk_id: u32,
resume: bool,
) -> Result<()> {
let _ = (schemas, chunk_id, resume);
InvalidUriSnafu {
uri: "snapshot",
reason: "storage does not support preparing export chunks",
}
.fail()
}
/// Deletes the entire snapshot (for --force).
async fn delete_snapshot(&self) -> Result<()>;
}
@@ -653,7 +672,17 @@ impl SnapshotStorage for OpenDalStorage {
ensure_snapshot_exists(self).await?;
let data = self.read_file(MANIFEST_FILE).await?;
serde_json::from_slice(&data).context(ManifestParseSnafu)
Manifest::from_json(&data).context(ManifestParseSnafu)
}
async fn file_size(&self, path: &str) -> Result<Option<u64>> {
match self.object_store.stat(path).await {
Ok(metadata) => Ok(metadata.is_file().then(|| metadata.content_length())),
Err(error) if error.kind() == ErrorKind::NotFound => Ok(None),
Err(error) => Err(error).with_context(|_| StorageOperationSnafu {
operation: format!("stat {path}"),
}),
}
}
async fn write_manifest(&self, manifest: &Manifest) -> Result<()> {
@@ -696,6 +725,56 @@ impl SnapshotStorage for OpenDalStorage {
Ok(files)
}
async fn prepare_export_chunk(
&self,
schemas: &[String],
chunk_id: u32,
resume: bool,
) -> Result<()> {
let mut files = Vec::new();
for schema in schemas {
let prefix = data_dir_for_schema_chunk(schema, chunk_id);
let mut entries = match self.object_store.lister_with(&prefix).recursive(true).await {
Ok(entries) => entries,
Err(error) if error.kind() == ErrorKind::NotFound => continue,
Err(error) => {
return Err(error).context(StorageOperationSnafu {
operation: format!("list {prefix}"),
});
}
};
while let Some(entry) = entries.try_next().await.context(StorageOperationSnafu {
operation: format!("list {prefix}"),
})? {
let path = entry.path();
if path == prefix && entry.metadata().is_dir() {
continue;
}
let name = path.strip_prefix(&prefix).unwrap_or("");
if !resume || entry.metadata().is_dir() || !valid_chunk_filename(name) {
return InvalidUriSnafu {
uri: path,
reason: "expected an empty new chunk or direct Parquet files in an owned unfinished chunk",
}.fail();
}
files.push(path.to_string());
}
}
// Validate every schema before deleting any files; COPY starts only after this returns.
for path in files {
match self.object_store.delete(&path).await {
Ok(()) => {}
Err(error) if error.kind() == ErrorKind::NotFound => {}
Err(error) => {
return Err(error).context(StorageOperationSnafu {
operation: format!("delete unfinished chunk file {path}"),
});
}
}
}
Ok(())
}
async fn delete_snapshot(&self) -> Result<()> {
self.object_store
.delete_with("/")
@@ -707,17 +786,26 @@ impl SnapshotStorage for OpenDalStorage {
}
}
fn valid_chunk_filename(name: &str) -> bool {
name.strip_suffix(".parquet")
.is_some_and(|stem| !stem.is_empty())
&& !name.contains(['/', '\\', '\0'])
}
#[cfg(test)]
mod tests {
use std::collections::HashMap;
use std::path::Path;
use std::sync::Arc;
use object_store::ObjectStore;
use object_store::layers::mock::MockLayerBuilder;
use object_store::services::Fs;
use tempfile::tempdir;
use url::Url;
use super::*;
use crate::data::export_v2::error::Error;
use crate::data::export_v2::manifest::{DataFormat, TimeRange};
use crate::data::export_v2::schema::SchemaDefinition;
@@ -971,6 +1059,148 @@ mod tests {
assert!(storage.exists().await.unwrap());
}
#[tokio::test]
async fn test_prepare_export_chunk_preserves_other_chunks_and_schema_artifacts() {
let dir = tempdir().unwrap();
let storage = make_storage_with_rooted_fs(dir.path());
for path in [
"data/public/1/keep.parquet",
"data/public/2/cpu.v1.parquet",
"data/other/2/a.parquet",
"schema/ddl/public.sql",
] {
storage.write_text(path, "original").await.unwrap();
}
let schemas = vec!["public".to_string(), "other".to_string()];
assert!(
storage
.prepare_export_chunk(&schemas, 2, false)
.await
.is_err()
);
assert!(
storage
.file_exists("data/public/2/cpu.v1.parquet")
.await
.unwrap()
);
storage
.prepare_export_chunk(&schemas, 2, true)
.await
.unwrap();
storage
.prepare_export_chunk(&schemas, 2, true)
.await
.unwrap();
assert!(
!storage
.file_exists("data/public/2/cpu.v1.parquet")
.await
.unwrap()
);
assert!(!storage.file_exists("data/other/2/a.parquet").await.unwrap());
assert_eq!(
storage
.read_text("data/public/1/keep.parquet")
.await
.unwrap(),
"original"
);
assert_eq!(
storage.read_text("schema/ddl/public.sql").await.unwrap(),
"original"
);
}
#[tokio::test]
async fn test_prepare_export_chunk_rejects_foreign_or_nested_entries_before_deletion() {
for foreign in ["notes.txt", "nested/a.parquet", "empty/"] {
let dir = tempdir().unwrap();
let storage = make_storage_with_rooted_fs(dir.path());
storage
.write_text("data/public/2/cpu.parquet", "keep")
.await
.unwrap();
let path = format!("data/other/2/{foreign}");
if foreign.ends_with('/') {
storage.create_dir_all(&path).await.unwrap();
} else {
storage.write_text(&path, "foreign").await.unwrap();
}
assert!(
storage
.prepare_export_chunk(&["public".into(), "other".into()], 2, true)
.await
.is_err()
);
assert_eq!(
storage
.read_text("data/public/2/cpu.parquet")
.await
.unwrap(),
"keep"
);
}
}
#[tokio::test]
async fn test_prepare_export_chunk_reports_delete_failure() {
let dir = tempdir().unwrap();
let mut storage = make_storage_with_rooted_fs(dir.path());
storage.object_store = storage.object_store.layer(
MockLayerBuilder::default()
// OpenDAL's unit deleter returns Unsupported for every deletion.
.deleter_factory(Arc::new(|_| Box::new(())))
.build()
.unwrap(),
);
storage
.write_text("data/public/2/a.parquet", "keep")
.await
.unwrap();
let error = storage
.prepare_export_chunk(&["public".into()], 2, true)
.await
.unwrap_err();
let Error::StorageOperation {
operation, error, ..
} = error
else {
panic!("expected storage operation error, got {error:?}");
};
assert_eq!(
operation,
"delete unfinished chunk file data/public/2/a.parquet"
);
assert_eq!(error.kind(), ErrorKind::Unsupported);
assert_eq!(
storage.read_text("data/public/2/a.parquet").await.unwrap(),
"keep"
);
}
#[tokio::test]
async fn test_prepare_export_chunk_encodes_schema_boundary() {
let dir = tempdir().unwrap();
let storage = make_storage_with_rooted_fs(dir.path());
let schema = "../other".to_string();
let owned = format!("{}a.parquet", data_dir_for_schema_chunk(&schema, 2));
storage.write_text(&owned, "remove").await.unwrap();
storage
.write_text("data/other/2/a.parquet", "keep")
.await
.unwrap();
storage
.prepare_export_chunk(&[schema], 2, true)
.await
.unwrap();
assert!(!storage.file_exists(&owned).await.unwrap());
assert_eq!(
storage.read_text("data/other/2/a.parquet").await.unwrap(),
"keep"
);
}
#[tokio::test]
async fn test_delete_snapshot_only_removes_rooted_contents() {
let parent = tempdir().unwrap();
src/cli/src/database.rs
+102 -5
View File
@@ -97,8 +97,68 @@ impl DatabaseClient {
self.sql(sql, DEFAULT_SCHEMA_NAME).await
}
/// Requires the explicit packed-import protocol before any restore mutation.
pub async fn require_packed_import(&self) -> Result<()> {
let url = format!("http://{}/v1/capabilities", self.addr);
let mut builder = reqwest::Client::builder().timeout(self.timeout);
if let Some(proxy) = self.proxy.clone() {
builder = builder.proxy(proxy);
}
if self.no_proxy {
builder = builder.no_proxy();
}
let client = builder.build().context(BuildClientSnafu)?;
let mut request = client.get(&url);
if let Some(auth) = &self.auth_header {
request = request.header("Authorization", auth);
}
let response = request.send().await.with_context(|_| HttpQuerySqlSnafu {
reason: "packed import capability request failed",
})?;
if response.status() == reqwest::StatusCode::NOT_FOUND {
return crate::error::InvalidArgumentsSnafu {
msg: "target does not support packed import",
}
.fail();
}
let response = response
.error_for_status()
.with_context(|_| HttpQuerySqlSnafu {
reason: "packed import capability request rejected",
})?;
let body = response.text().await.with_context(|_| HttpQuerySqlSnafu {
reason: "cannot read capability response",
})?;
let value: Value = serde_json::from_str(&body).context(SerdeJsonSnafu)?;
if !value.is_object() {
return crate::error::UnexpectedSnafu {
msg: "invalid capability response: expected JSON object",
}
.fail();
}
if value.get("metric_packed_import").and_then(Value::as_u64) != Some(1) {
return crate::error::InvalidArgumentsSnafu {
msg: "target does not support packed import version 1",
}
.fail();
}
Ok(())
}
/// Execute sql query.
pub async fn sql(&self, sql: &str, schema: &str) -> Result<Option<Vec<Vec<Value>>>> {
let body = self.sql_response(sql, schema).await?;
Ok(body.output().first().and_then(|output| match output {
GreptimeQueryOutput::Records(records) => Some(records.rows().clone()),
GreptimeQueryOutput::AffectedRows(_) => None,
}))
}
pub(crate) async fn sql_response(
&self,
sql: &str,
schema: &str,
) -> Result<GreptimedbV1Response> {
let url = format!("http://{}/v1/sql", self.addr);
let params = [
("db", format!("{}-{}", self.catalog, schema)),
@@ -138,11 +198,7 @@ impl DatabaseClient {
reason: "cannot get response text".to_string(),
})?;
let body = serde_json::from_str::<GreptimedbV1Response>(&text).context(SerdeJsonSnafu)?;
Ok(body.output().first().and_then(|output| match output {
GreptimeQueryOutput::Records(records) => Some(records.rows().clone()),
GreptimeQueryOutput::AffectedRows(_) => None,
}))
serde_json::from_str::<GreptimedbV1Response>(&text).context(SerdeJsonSnafu)
}
}
@@ -162,6 +218,47 @@ pub(crate) fn split_database(database: &str) -> Result<(String, Option<String>)>
#[cfg(test)]
mod tests {
#[tokio::test]
async fn packed_capability_probe_checks_auth_and_protocol_version() {
use tokio::io::{AsyncReadExt, AsyncWriteExt};
for (status, body, supported) in [
(200, r#"{"metric_packed_import":1,"future":2}"#, true),
(200, "{}", false),
(200, r#"{"metric_packed_import":2}"#, false),
(404, "{}", false),
(401, "{}", false),
(403, "{}", false),
(200, "[]", false),
(200, "invalid-json", false),
] {
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let address = listener.local_addr().unwrap();
let server = tokio::spawn(async move {
let (mut socket, _) = listener.accept().await.unwrap();
let mut request = vec![0; 4096];
let n = socket.read(&mut request).await.unwrap();
let request = String::from_utf8_lossy(&request[..n]).to_lowercase();
assert!(request.starts_with("get /v1/capabilities "));
assert!(request.contains("authorization: basic dxnlcjpwyxnzd29yza=="));
socket.write_all(format!("HTTP/1.1 {status} Response\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", body.len()).as_bytes()).await.unwrap();
});
let client = super::DatabaseClient::new(
address.to_string(),
"greptime".into(),
Some("user:password".into()),
std::time::Duration::from_secs(5),
None,
true,
);
assert_eq!(
client.require_packed_import().await.is_ok(),
supported,
"{status}: {body}"
);
server.await.unwrap();
}
}
use super::*;
#[test]
src/cmd/Cargo.toml
+2
View File
@@ -21,11 +21,13 @@ required-features = ["dev-tools"]
[features]
default = [
"ai_functions",
"servers/pprof",
"servers/mem-prof",
"meta-srv/pg_kvbackend",
"meta-srv/mysql_kvbackend",
]
ai_functions = ["query/ai_functions"]
enterprise = ["common-meta/enterprise", "frontend/enterprise", "meta-srv/enterprise"]
# Developer-only helper binaries and diagnostic datanode commands.
# Kept out of `default` so normal/release builds don't compile them.
src/cmd/src/bin/query_perf_fixture/case.rs
+1 -1
View File
@@ -185,7 +185,7 @@ pub(super) fn default_max_concurrent_flushes() -> u64 {
256
}
pub(super) fn default_worker_channel_capacity() -> u64 {
65526
65_536
}
pub(super) fn default_max_inflight_requests() -> u64 {
3000
src/cmd/tests/load_config_test.rs
+9
View File
@@ -30,6 +30,7 @@ use datanode::config::{DatanodeOptions, RegionEngineConfig, StorageConfig};
use file_engine::config::EngineConfig as FileEngineConfig;
use flow::FlownodeOptions;
use frontend::frontend::FrontendOptions;
use frontend::service_config::PendingRowsBatcherOptions;
use meta_client::MetaClientOptions;
use meta_srv::metasrv::MetasrvOptions;
use meta_srv::selector::SelectorType;
@@ -206,6 +207,10 @@ fn test_load_frontend_example_config() {
);
let expected = GreptimeOptions::<FrontendOptions> {
component: FrontendOptions {
pending_rows_batcher: PendingRowsBatcherOptions {
logical_table: Some(Default::default()),
..Default::default()
},
default_timezone: Some("UTC".to_string()),
default_column_prefix: Some("greptime".to_string()),
auto_create_table: true,
@@ -389,6 +394,10 @@ fn test_load_standalone_example_config() {
);
let expected = GreptimeOptions::<StandaloneOptions> {
component: StandaloneOptions {
pending_rows_batcher: PendingRowsBatcherOptions {
logical_table: Some(Default::default()),
..Default::default()
},
default_timezone: Some("UTC".to_string()),
default_column_prefix: Some("greptime".to_string()),
auto_create_table: true,
src/common/base/src/protocol.rs
+6 -1
View File
@@ -14,7 +14,7 @@
use std::fmt::{Display, Formatter};
/// The protocol through which a query is received.
/// The protocol or internal subsystem through which a query is received.
#[derive(Debug, PartialEq, Default, Clone, Copy, strum::FromRepr)]
#[repr(u8)]
pub enum Channel {
@@ -35,6 +35,8 @@ pub enum Channel {
Log = 12,
Promql = 13,
Splunk = 14,
/// Trusted internal requests and local subsystem execution.
Internal = 255,
}
impl From<u32> for Channel {
@@ -70,6 +72,7 @@ impl AsRef<str> for Channel {
Self::Log => "log",
Self::Promql => "promql",
Self::Splunk => "splunk",
Self::Internal => "internal",
}
}
}
@@ -95,6 +98,7 @@ mod tests {
(12, "log"),
(13, "promql"),
(14, "splunk"),
(255, "internal"),
];
for (value, name) in expected {
@@ -102,5 +106,6 @@ mod tests {
}
assert_eq!("unknown", Channel::from(0).as_ref());
assert_eq!("unknown", Channel::from(15).as_ref());
assert_eq!("unknown", Channel::from(256).as_ref());
}
}
src/common/catalog/src/consts.rs
+6
View File
@@ -202,6 +202,12 @@ pub fn is_ddl_reserved_table(schema: &str, table: &str) -> bool {
pub const TRACE_ID_COLUMN: &str = "trace_id";
pub const SPAN_ID_COLUMN: &str = "span_id";
pub const SPAN_NAME_COLUMN: &str = "span_name";
/// Column containing span attributes as JSON/JSON2.
pub const SPAN_ATTRIBUTES_COLUMN: &str = "span_attributes";
/// Column containing instrumentation scope attributes as JSON2.
pub const SCOPE_ATTRIBUTES_COLUMN: &str = "scope_attributes";
/// Column containing resource attributes as JSON2.
pub const RESOURCE_ATTRIBUTES_COLUMN: &str = "resource_attributes";
pub const SERVICE_NAME_COLUMN: &str = "service_name";
pub const PARENT_SPAN_ID_COLUMN: &str = "parent_span_id";
// More fixed columns/values of the `greptime_trace_v1` data model, shared by
src/common/datasource/src/error.rs
+4
View File
@@ -27,6 +27,9 @@ use url::ParseError;
#[snafu(visibility(pub))]
#[stack_trace_debug]
pub enum Error {
#[snafu(display("Invalid packed snapshot: {reason}"))]
InvalidPackedSnapshot { reason: String },
#[snafu(display("Parquet write cancelled"))]
ParquetWriteCancelled {},
@@ -285,6 +288,7 @@ impl ErrorExt for Error {
use Error::*;
match self {
ParquetWriteCancelled {} => StatusCode::Cancelled,
InvalidPackedSnapshot { .. } => StatusCode::InvalidArguments,
InvalidParquetWriterLimits {} => StatusCode::InvalidArguments,
ParquetWriterResource { .. } => StatusCode::Suspended,
BuildBackend { .. }
src/common/datasource/src/file_format/parquet.rs
+2
View File
@@ -12,6 +12,8 @@
// See the License for the specific language governing permissions and
// limitations under the License.
pub mod packed_reader;
use std::result;
use std::sync::Arc;
src/common/datasource/src/file_format/parquet/packed_reader.rs
+351
View File
@@ -0,0 +1,351 @@
// Copyright 2023 Greptime Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Request-owned, bounded backing buffers for independent Parquet streams.
use std::ops::Range;
use std::sync::Arc;
use bytes::Bytes;
use futures::future::BoxFuture;
use object_store::ObjectStore;
use parquet::arrow::arrow_reader::ArrowReaderOptions;
use parquet::arrow::async_reader::AsyncFileReader;
use parquet::errors::{ParquetError, Result};
use parquet::file::metadata::{ParquetMetaData, ParquetMetaDataReader};
use tokio::sync::Mutex;
pub const WINDOW_SIZE: usize = 8 * 1024 * 1024;
const WINDOW_BUDGET: usize = 2 * WINDOW_SIZE;
struct Window {
path: String,
offset: u64,
bytes: Bytes,
}
/// Shared only by table decoders belonging to one authorized COPY request.
/// The lock also coalesces concurrent fetches for the same bytes.
pub struct PackReadWindows {
store: ObjectStore,
windows: Mutex<Vec<Window>>,
}
impl PackReadWindows {
pub fn new(store: ObjectStore) -> Arc<Self> {
Arc::new(Self {
store,
windows: Mutex::new(Vec::new()),
})
}
async fn read(&self, path: &str, object_length: u64, range: Range<u64>) -> Result<Bytes> {
if range.start > range.end || range.end > object_length {
return Err(ParquetError::General("packed read outside object".into()));
}
if range.is_empty() {
return Ok(Bytes::new());
}
let mut windows = self.windows.lock().await;
for window in windows.iter() {
if window.path == path
&& range.start >= window.offset
&& range.end <= window.offset + window.bytes.len() as u64
{
return Ok(window.bytes.slice(
(range.start - window.offset) as usize..(range.end - window.offset) as usize,
));
}
}
let required = usize::try_from(range.end - range.start)
.map_err(|_| ParquetError::General("packed read length overflow".into()))?;
if required > WINDOW_SIZE {
return Err(ParquetError::General(
"packed cache range exceeds window".into(),
));
}
let fetch_len = (object_length - range.start).min(WINDOW_SIZE as u64) as usize;
// Evict only unpinned allocations. Bytes slices held by a decoder still
// count against the request budget, including their entire backing window.
let mut retained: usize = windows.iter().map(|w| w.bytes.len()).sum();
while retained.saturating_add(fetch_len) > WINDOW_BUDGET || windows.len() >= 2 {
let Some(i) = windows.iter().position(|w| w.bytes.is_unique()) else {
return Err(ParquetError::General(
"packed read backing-buffer budget exhausted".into(),
));
};
retained -= windows.remove(i).bytes.len();
}
let bytes = self
.store
.read_with(path)
.range(range.start..range.start + fetch_len as u64)
.await
.map_err(|e| ParquetError::External(Box::new(e)))?
.to_bytes();
if bytes.len() != fetch_len {
return Err(ParquetError::General("short packed object read".into()));
}
// Backend buffers can be slices of larger allocations. Retain an exact
// window allocation; transport staging is separate from cached backing.
let bytes = Bytes::copy_from_slice(&bytes);
common_telemetry::debug!(
path,
offset = range.start,
length = fetch_len,
"Fetched packed read window"
);
let result = bytes.slice(..required);
windows.push(Window {
path: path.into(),
offset: range.start,
bytes,
});
Ok(result)
}
}
/// Pins one complete small stream; larger streams use the streaming reader.
pub struct PackedParquetReader {
windows: Arc<PackReadWindows>,
path: String,
object_length: u64,
offset: u64,
length: u64,
bytes: Option<Bytes>,
}
impl PackedParquetReader {
pub fn new(
windows: Arc<PackReadWindows>,
path: String,
object_length: u64,
offset: u64,
length: u64,
) -> Result<Self> {
if length < 12
|| length > WINDOW_SIZE as u64
|| offset
.checked_add(length)
.is_none_or(|end| end > object_length)
{
return Err(ParquetError::General("invalid packed Parquet range".into()));
}
Ok(Self {
windows,
path,
object_length,
offset,
length,
bytes: None,
})
}
}
impl AsyncFileReader for PackedParquetReader {
fn get_bytes(&mut self, range: Range<u64>) -> BoxFuture<'_, Result<Bytes>> {
Box::pin(async move {
if range.start > range.end || range.end > self.length {
return Err(ParquetError::General(
"read outside indexed Parquet stream".into(),
));
}
if self.bytes.is_none() {
self.bytes = Some(
self.windows
.read(
&self.path,
self.object_length,
self.offset..self.offset + self.length,
)
.await?,
);
}
Ok(self
.bytes
.as_ref()
.unwrap()
.slice(range.start as usize..range.end as usize))
})
}
fn get_metadata<'a>(
&'a mut self,
options: Option<&'a ArrowReaderOptions>,
) -> BoxFuture<'a, Result<Arc<ParquetMetaData>>> {
Box::pin(async move {
let length = self.length;
let metadata = ParquetMetaDataReader::new()
.with_metadata_options(options.map(|o| o.metadata_options().clone()))
.load_and_finish(self, length)
.await?;
Ok(Arc::new(metadata))
})
}
}
#[cfg(test)]
mod tests {
use arrow::array::{ArrayRef, Int64Array, StringArray};
use arrow::record_batch::RecordBatch;
use arrow_schema::{Field, Schema};
use futures::TryStreamExt;
use parquet::arrow::{ArrowWriter, ParquetRecordBatchStreamBuilder};
use super::*;
fn parquet(array: ArrayRef) -> (Vec<u8>, RecordBatch) {
let schema = Arc::new(Schema::new(vec![Field::new(
"value",
array.data_type().clone(),
false,
)]));
let batch = RecordBatch::try_new(schema.clone(), vec![array]).unwrap();
let mut writer = ArrowWriter::try_new(Vec::new(), schema, None).unwrap();
writer.write(&batch).unwrap();
(writer.into_inner().unwrap(), batch)
}
#[tokio::test]
async fn independent_schemas_empty_streams_and_shared_ranges() {
let dir = common_test_util::temp_dir::create_temp_dir("packed-reader");
let store = crate::test_util::test_store(dir.path().to_str().unwrap());
let fixtures = [
parquet(Arc::new(Int64Array::from(vec![42]))),
parquet(Arc::new(StringArray::from(vec!["different type"]))),
parquet(Arc::new(Int64Array::from(Vec::<i64>::new()))),
];
let packed: Vec<u8> = fixtures
.iter()
.flat_map(|(b, _)| b.iter().copied())
.collect();
store.write("pack-0.bin", packed.clone()).await.unwrap();
let windows = PackReadWindows::new(store.clone());
// A pinned shared window serves all the independent stream readers.
let pinned = windows
.read("pack-0.bin", packed.len() as u64, 0..packed.len() as u64)
.await
.unwrap();
store.delete("pack-0.bin").await.unwrap();
let mut offset = 0;
for (bytes, expected) in fixtures {
let reader = PackedParquetReader::new(
windows.clone(),
"pack-0.bin".into(),
packed.len() as u64,
offset,
bytes.len() as u64,
)
.unwrap();
let builder = ParquetRecordBatchStreamBuilder::new(reader).await.unwrap();
assert_eq!(builder.schema(), &expected.schema());
let batches: Vec<_> = builder.build().unwrap().try_collect().await.unwrap();
assert_eq!(
batches.iter().map(|b| b.num_rows()).sum::<usize>(),
expected.num_rows()
);
if expected.num_rows() > 0 {
assert_eq!(batches[0], expected);
}
offset += bytes.len() as u64;
}
assert_eq!(windows.windows.lock().await.len(), 1);
assert_eq!(pinned.len(), packed.len());
let isolated = PackReadWindows::new(store);
assert!(
isolated
.read("pack-0.bin", packed.len() as u64, 0..1)
.await
.is_err()
);
}
#[tokio::test]
async fn small_stream_pins_one_window_for_multiple_columns() {
use parquet::file::properties::WriterProperties;
let schema = Arc::new(Schema::new(
(0..3)
.map(|i| Field::new(format!("v{i}"), arrow_schema::DataType::Utf8, false))
.collect::<Vec<_>>(),
));
let value = "x".repeat(1024 * 1024);
let arrays = (0..3)
.map(|_| Arc::new(StringArray::from(vec![value.as_str()])) as ArrayRef)
.collect();
let batch = RecordBatch::try_new(schema.clone(), arrays).unwrap();
let mut writer = ArrowWriter::try_new(
Vec::new(),
schema,
Some(
WriterProperties::builder()
.set_dictionary_enabled(false)
.build(),
),
)
.unwrap();
writer.write(&batch).unwrap();
let bytes = writer.into_inner().unwrap();
assert!(bytes.len() < WINDOW_SIZE);
let dir = common_test_util::temp_dir::create_temp_dir("packed-columns");
let store = crate::test_util::test_store(dir.path().to_str().unwrap());
let mut object = vec![0; WINDOW_SIZE - 1024];
let offset = object.len() as u64;
object.extend_from_slice(&bytes);
let length = object.len() as u64;
store.write("pack-0.bin", object).await.unwrap();
let windows = PackReadWindows::new(store);
let pinned = windows.read("pack-0.bin", length, 0..1).await.unwrap();
let mut reader = PackedParquetReader::new(
windows.clone(),
"pack-0.bin".into(),
length,
offset,
bytes.len() as u64,
)
.unwrap();
let metadata = reader.get_metadata(None).await.unwrap();
let mut columns = Vec::new();
for column in metadata.row_group(0).columns() {
let (start, len) = column.byte_range();
columns.push(reader.get_bytes(start..start + len).await.unwrap());
}
let batches: Vec<_> = ParquetRecordBatchStreamBuilder::new(reader)
.await
.unwrap()
.build()
.unwrap()
.try_collect()
.await
.unwrap();
assert_eq!(batches, vec![batch]);
let retained = windows.windows.lock().await;
assert_eq!(retained.len(), 2);
assert!(retained.iter().all(|w| w.bytes.len() <= WINDOW_SIZE));
assert!(retained.iter().map(|w| w.bytes.len()).sum::<usize>() <= WINDOW_BUDGET);
assert_eq!(columns.len(), 3);
assert_eq!(pinned[0], 0);
}
#[tokio::test]
async fn corrupt_metadata_short_reads_and_relative_bounds() {
let dir = common_test_util::temp_dir::create_temp_dir("packed-corrupt");
let store = crate::test_util::test_store(dir.path().to_str().unwrap());
store.write("pack-0.bin", vec![0u8; 12]).await.unwrap();
let windows = PackReadWindows::new(store);
assert!(windows.read("pack-0.bin", 20, 0..20).await.is_err());
let mut reader = PackedParquetReader::new(windows, "pack-0.bin".into(), 12, 0, 12).unwrap();
assert!(reader.get_bytes(0..13).await.is_err());
assert!(reader.get_metadata(None).await.is_err());
}
}
src/common/datasource/src/lib.rs
+1
View File
@@ -19,6 +19,7 @@ pub mod error;
pub mod file_format;
pub mod lister;
pub mod object_store;
pub mod packed_snapshot;
pub mod parquet_writer;
pub mod share_buffer;
#[cfg(test)]
src/common/datasource/src/object_store.rs
+11 -2
View File
@@ -56,8 +56,17 @@ pub struct BuiltBackend {
impl BuiltBackend {
/// Checks an explicit input using the same file-type filter as directory listing.
pub async fn is_file(&self, path: &str) -> object_store::Result<bool> {
// Stat first preserves errors for dangling or inaccessible symlink targets.
if self.object_store.stat(path).await?.mode() != object_store::EntryMode::FILE {
let meta = self.object_store.stat(path).await?;
self.is_file_with_mode(path, meta.mode()).await
}
/// Applies local file-type protections to a previously fetched object stat.
pub async fn is_file_with_mode(
&self,
path: &str,
mode: object_store::EntryMode,
) -> object_store::Result<bool> {
if mode != object_store::EntryMode::FILE {
return Ok(false);
}
if let Some(root) = &self.local_root {
src/common/datasource/src/packed_snapshot.rs
+188
View File
@@ -0,0 +1,188 @@
// Copyright 2023 Greptime Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Shared on-disk index for one packed snapshot schema chunk.
use std::collections::{HashMap, HashSet};
use serde::{Deserialize, Serialize};
use crate::error::{InvalidPackedSnapshotSnafu, Result};
pub const PACK_INDEX_FILE: &str = "pack-index.json";
pub const PACKED_LAYOUT: &str = "metric-parquet-packs";
/// Versioned index. Object paths are generated direct children of the chunk.
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct PackIndex {
pub version: u32,
pub objects: Vec<PackObject>,
pub tables: Vec<PackTable>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct PackObject {
pub path: String,
pub kind: ObjectKind,
pub length: u64,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "lowercase")]
pub enum ObjectKind {
Pack,
Parquet,
}
/// One complete independent Parquet stream, including its footer.
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct PackTable {
pub table_name: String,
pub object: String,
pub offset: u64,
pub length: u64,
pub row_count: u64,
}
impl PackIndex {
/// Validates all references and complete, nonoverlapping object coverage.
pub fn validate(&self) -> Result<()> {
let invalid = |reason: &str| InvalidPackedSnapshotSnafu { reason }.build();
if self.version != 1 {
return Err(invalid("unsupported index version"));
}
let mut objects = HashMap::new();
for object in &self.objects {
let (prefix, suffix) = match object.kind {
ObjectKind::Pack => ("pack-", ".bin"),
ObjectKind::Parquet => ("table-", ".parquet"),
};
let generated = object
.path
.strip_prefix(prefix)
.and_then(|s| s.strip_suffix(suffix))
.is_some_and(|s| !s.is_empty() && s.bytes().all(|b| b.is_ascii_digit()));
if !generated
|| object.length == 0
|| objects.insert(object.path.as_str(), object).is_some()
{
return Err(invalid("invalid or duplicate object path/length"));
}
}
let mut names = HashSet::new();
let mut ranges: HashMap<&str, Vec<&PackTable>> = HashMap::new();
for table in &self.tables {
if table.table_name.is_empty() || !names.insert(table.table_name.as_str()) {
return Err(invalid("empty or duplicate table name"));
}
let object = objects
.get(table.object.as_str())
.ok_or_else(|| invalid("unknown object reference"))?;
let end = table
.offset
.checked_add(table.length)
.ok_or_else(|| invalid("table range overflow"))?;
if table.length < 12 || end > object.length {
return Err(invalid("invalid Parquet stream range"));
}
ranges.entry(&table.object).or_default().push(table);
}
for object in &self.objects {
let entries = ranges
.get_mut(object.path.as_str())
.ok_or_else(|| invalid("unreferenced object"))?;
entries.sort_unstable_by_key(|t| t.offset);
if object.kind == ObjectKind::Parquet && entries.len() != 1 {
return Err(invalid("standalone object must contain one table"));
}
let mut end = 0;
for entry in entries {
if entry.offset != end {
return Err(invalid("object ranges have a gap or overlap"));
}
end += entry.length; // Checked against object length above.
}
if end != object.length {
return Err(invalid("object ranges do not cover the full object"));
}
}
Ok(())
}
/// Requires exactly the data-bearing tables selected by snapshot DDL.
pub fn validate_membership<'a>(&self, tables: impl IntoIterator<Item = &'a str>) -> Result<()> {
self.validate()?;
let expected: HashSet<_> = tables.into_iter().collect();
let actual: HashSet<_> = self.tables.iter().map(|t| t.table_name.as_str()).collect();
if expected != actual {
return InvalidPackedSnapshotSnafu {
reason: "index table membership differs from snapshot DDL",
}
.fail();
}
Ok(())
}
}
#[cfg(test)]
mod tests {
use super::*;
fn index() -> PackIndex {
PackIndex {
version: 1,
objects: vec![PackObject {
path: "pack-000000.bin".into(),
kind: ObjectKind::Pack,
length: 24,
}],
tables: ["literal.name", "other"]
.into_iter()
.enumerate()
.map(|(i, name)| PackTable {
table_name: name.into(),
object: "pack-000000.bin".into(),
offset: i as u64 * 12,
length: 12,
row_count: 0,
})
.collect(),
}
}
#[test]
fn validates_complete_membership_and_ranges() {
index()
.validate_membership(["other", "literal.name"])
.unwrap();
assert!(index().validate_membership(["other"]).is_err());
for mutate in [
|i: &mut PackIndex| i.version = 2,
|i: &mut PackIndex| i.objects[0].path = "../pack-0.bin".into(),
|i: &mut PackIndex| i.objects.push(i.objects[0].clone()),
|i: &mut PackIndex| i.tables[1].table_name = "literal.name".into(),
|i: &mut PackIndex| i.tables[1].object = "missing".into(),
|i: &mut PackIndex| i.tables[1].offset = 11,
|i: &mut PackIndex| i.tables[1].length = u64::MAX,
|i: &mut PackIndex| i.tables.pop().map(|_| ()).unwrap(),
] {
let mut bad = index();
mutate(&mut bad);
assert!(bad.validate().is_err(), "{bad:?}");
}
}
}
src/common/event-recorder/src/recorder.rs
+22 -8
View File
@@ -291,17 +291,29 @@ pub struct EventRecorderOptions {
skip_serializing_if = "event_type_filter_is_all"
)]
pub event_types: EventTypeFilterRef,
/// How long buffered events may wait before reaching the event handler.
///
/// Not a configuration option: it is skipped by serde so it never appears in
/// config files or `config/config.md`. Integration tests shorten it so they
/// do not pay a flush window per DDL they assert on.
#[serde(skip, default = "default_flush_interval")]
pub flush_interval: Duration,
}
fn default_events_table_ttl() -> Duration {
DEFAULT_EVENTS_TABLE_TTL
}
fn default_flush_interval() -> Duration {
DEFAULT_FLUSH_INTERVAL_SECONDS
}
impl Default for EventRecorderOptions {
fn default() -> Self {
Self {
ttl: DEFAULT_EVENTS_TABLE_TTL,
event_types: Arc::new(EventTypeFilter::All),
flush_interval: DEFAULT_FLUSH_INTERVAL_SECONDS,
}
}
}
@@ -321,13 +333,18 @@ pub struct EventRecorderImpl {
impl EventRecorderImpl {
pub fn new(event_handler: Box<dyn EventHandler>) -> Self {
Self::with_event_type_filter(event_handler, Arc::new(EventTypeFilter::All))
Self::with_event_type_filter(
event_handler,
Arc::new(EventTypeFilter::All),
DEFAULT_FLUSH_INTERVAL_SECONDS,
)
}
/// Creates an event recorder with an event-type filter.
pub fn with_event_type_filter(
event_handler: Box<dyn EventHandler>,
event_types: EventTypeFilterRef,
flush_interval: Duration,
) -> Self {
let (tx, rx) = channel(DEFAULT_CHANNEL_SIZE);
let cancel_token = CancellationToken::new();
@@ -339,13 +356,9 @@ impl EventRecorderImpl {
cancel_token: cancel_token.clone(),
};
let processor = EventProcessor::new(
rx,
event_handler,
DEFAULT_FLUSH_INTERVAL_SECONDS,
DEFAULT_MAX_RETRY_TIMES,
)
.with_cancel_token(cancel_token);
let processor =
EventProcessor::new(rx, event_handler, flush_interval, DEFAULT_MAX_RETRY_TIMES)
.with_cancel_token(cancel_token);
// Spawn a background task to process the events.
let handle = tokio::spawn(async move {
@@ -611,6 +624,7 @@ mod tests {
count: count.clone(),
}),
event_type_filter.clone(),
DEFAULT_FLUSH_INTERVAL_SECONDS,
);
assert!(Arc::ptr_eq(
src/common/function/Cargo.toml
+9 -1
View File
@@ -10,8 +10,9 @@ workspace = true
[features]
testing = []
enterprise = []
default = ["geo"]
default = ["geo", "ai_functions"]
geo = ["geohash", "h3o", "s2", "wkt", "geo-types", "dep:geo"]
ai_functions = ["dep:futures", "dep:reqwest"]
[dependencies]
ahash.workspace = true
@@ -43,6 +44,7 @@ datafusion-pg-catalog.workspace = true
datafusion-physical-expr.workspace = true
datatypes.workspace = true
derive_more.workspace = true
futures = { workspace = true, optional = true }
geo = { version = "0.29", optional = true }
geo-types = { version = "0.7", optional = true }
geohash = { version = "0.13", optional = true }
@@ -57,6 +59,7 @@ num = "0.4"
num-traits = "0.2"
paste.workspace = true
regex.workspace = true
reqwest = { workspace = true, optional = true }
s2 = { version = "0.0.12", optional = true }
serde.workspace = true
serde_json.workspace = true
@@ -71,6 +74,7 @@ wkt = { version = "0.11", optional = true }
[dev-dependencies]
approx = "0.5"
axum.workspace = true
criterion.workspace = true
futures.workspace = true
pretty_assertions.workspace = true
@@ -80,3 +84,7 @@ tokio.workspace = true
[[bench]]
name = "uddsketch"
harness = false
[package.metadata.cargo-udeps.ignore]
# Used only by the feature-gated Jev HTTP tests.
development = ["axum"]
src/common/function/src/aggrs/approximate.rs
+11
View File
@@ -18,6 +18,7 @@ use datatypes::arrow::datatypes::DataType;
use crate::aggrs::aggr_wrapper::DeltaMergeWrapper;
use crate::function_registry::FunctionRegistry;
pub mod avg;
pub mod hll;
pub mod uddsketch;
pub mod welford;
@@ -26,6 +27,16 @@ pub(crate) struct ApproximateFunction;
impl ApproximateFunction {
pub fn register(registry: &FunctionRegistry) {
let avg_merge = avg::AvgAccumulator::merge_udf_impl();
registry.register_aggr(avg::AvgAccumulator::state_udf_impl());
registry.register_aggr(avg_merge.clone());
registry.register_aggr(AggregateUDF::new_from_impl(DeltaMergeWrapper::new(
avg_merge.clone(),
avg::AVG_STATE_NAME,
vec![DataType::Binary],
DataType::Binary,
)));
let uddsketch_state = uddsketch::UddSketchState::state_udf_impl();
let uddsketch_merge = uddsketch::UddSketchState::merge_udf_impl();
let uddsketch_delta = AggregateUDF::new_from_impl(DeltaMergeWrapper::new(
src/common/function/src/aggrs/approximate/avg.rs
+594
View File
@@ -0,0 +1,594 @@
// Copyright 2023 Greptime Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use datafusion::arrow::array::{ArrayRef, Float64Array};
use datafusion::arrow::compute::sum;
use datafusion::common::cast::{as_binary_array, as_primitive_array};
use datafusion::common::not_impl_err;
use datafusion::error::{DataFusionError, Result as DfResult};
use datafusion::logical_expr::function::AccumulatorArgs;
use datafusion::logical_expr::{
Accumulator as DfAccumulator, AggregateUDF, AggregateUDFImpl, Signature,
};
use datafusion_common::ScalarValue;
use datatypes::arrow::datatypes::{DataType, Float64Type};
pub const AVG_STATE_NAME: &str = "avg_state";
pub const AVG_MERGE_NAME: &str = "avg_merge";
const ENCODED_LEN: usize = 20;
const MAGIC: &[u8; 4] = b"AVG1";
/// The portable state used by the Float64 average aggregate functions.
#[derive(Debug, Clone, Copy, PartialEq)]
pub struct AvgState {
count: u64,
sum: f64,
}
impl Default for AvgState {
fn default() -> Self {
Self { count: 0, sum: 0.0 }
}
}
impl AvgState {
/// Returns the exact AVG1 representation of this state.
pub(crate) fn encode(&self) -> [u8; ENCODED_LEN] {
let mut encoded = [0; ENCODED_LEN];
encoded[..4].copy_from_slice(MAGIC);
encoded[4..12].copy_from_slice(&self.count.to_le_bytes());
encoded[12..20].copy_from_slice(&self.sum.to_bits().to_le_bytes());
encoded
}
/// Decodes and validates an AVG1 state.
pub fn decode(encoded: &[u8]) -> DfResult<Self> {
if encoded.len() != ENCODED_LEN || &encoded[..4] != MAGIC {
return Err(invalid_state());
}
let count = decode_u64(encoded, 4);
let sum = f64::from_bits(decode_u64(encoded, 12));
if count == 0 && sum.to_bits() != 0 {
return Err(invalid_state());
}
Ok(Self { count, sum })
}
/// Returns the number of non-null input values in this state.
pub(crate) fn count(&self) -> u64 {
self.count
}
/// Returns the average, or `None` for the canonical empty state.
pub fn average(&self) -> Option<f64> {
(self.count() != 0).then(|| self.sum / self.count() as f64)
}
}
fn decode_u64(encoded: &[u8], offset: usize) -> u64 {
let mut bytes = [0; 8];
bytes.copy_from_slice(&encoded[offset..offset + 8]);
u64::from_le_bytes(bytes)
}
fn invalid_state() -> DataFusionError {
DataFusionError::Execution("Invalid AVG1 state".to_string())
}
fn count_overflow() -> DataFusionError {
DataFusionError::Execution("AVG count overflow".to_string())
}
/// The `avg_state` / `avg_merge` aggregate UDF.
///
/// Declares a canonical AVG1 empty state as `default_value` so window frames
/// without rows observe the same contract as the accumulator's `evaluate`.
#[derive(Debug, Clone, Eq, PartialEq, Hash)]
struct AvgUdaf {
name: &'static str,
signature: Signature,
input: InputKind,
}
impl AggregateUDFImpl for AvgUdaf {
fn name(&self) -> &str {
self.name
}
fn signature(&self) -> &Signature {
&self.signature
}
fn return_type(&self, _arg_types: &[DataType]) -> DfResult<DataType> {
Ok(DataType::Binary)
}
fn accumulator(&self, acc_args: AccumulatorArgs) -> DfResult<Box<dyn DfAccumulator>> {
if acc_args.is_distinct {
return not_impl_err!("AVG DISTINCT aggregations are not available");
}
let input = match acc_args.exprs[0].data_type(acc_args.schema)? {
DataType::Float64 => InputKind::Float64,
DataType::Binary => InputKind::Binary,
data_type => return not_impl_err!("AVG functions do not support {data_type:?}"),
};
Ok(Box::new(AvgAccumulator {
state: AvgState::default(),
input,
}))
}
fn default_value(&self, _data_type: &DataType) -> DfResult<ScalarValue> {
Ok(ScalarValue::Binary(Some(
AvgState::default().encode().to_vec(),
)))
}
}
#[derive(Debug, Clone, Copy, Eq, PartialEq, Hash)]
enum InputKind {
Float64,
Binary,
}
/// Accumulates and merges AVG1 states.
#[derive(Debug)]
pub(crate) struct AvgAccumulator {
state: AvgState,
input: InputKind,
}
impl Default for AvgAccumulator {
fn default() -> Self {
Self {
state: AvgState::default(),
input: InputKind::Float64,
}
}
}
impl AvgAccumulator {
pub fn state_udf_impl() -> AggregateUDF {
AggregateUDF::new_from_impl(AvgUdaf {
name: AVG_STATE_NAME,
signature: Signature::exact(
vec![DataType::Float64],
datafusion::logical_expr::Volatility::Immutable,
),
input: InputKind::Float64,
})
}
pub fn merge_udf_impl() -> AggregateUDF {
AggregateUDF::new_from_impl(AvgUdaf {
name: AVG_MERGE_NAME,
signature: Signature::exact(
vec![DataType::Binary],
datafusion::logical_expr::Volatility::Immutable,
),
input: InputKind::Binary,
})
}
fn update_float64(&mut self, array: &ArrayRef) -> DfResult<()> {
let array = as_primitive_array::<Float64Type>(array)?;
let mut count = self.state.count;
for _ in array.iter().flatten() {
count = count.checked_add(1).ok_or_else(count_overflow)?;
}
let sum = sum(array)
.map(|batch_sum| self.state.sum + batch_sum)
.unwrap_or(self.state.sum);
self.state = AvgState { count, sum };
Ok(())
}
fn merge_states(&mut self, array: &ArrayRef) -> DfResult<()> {
let array = as_binary_array(array)?;
let states = array
.iter()
.flatten()
.map(AvgState::decode)
.collect::<DfResult<Vec<_>>>()?;
let count = states.iter().try_fold(self.state.count, |count, state| {
count.checked_add(state.count).ok_or_else(count_overflow)
})?;
let sums = states
.iter()
.filter(|state| state.count != 0)
.map(|state| Some(state.sum))
.collect::<Vec<_>>();
let sum = sum(&Float64Array::from(sums))
.map(|batch_sum| self.state.sum + batch_sum)
.unwrap_or(self.state.sum);
self.state = AvgState { count, sum };
Ok(())
}
}
impl DfAccumulator for AvgAccumulator {
fn update_batch(&mut self, values: &[ArrayRef]) -> DfResult<()> {
let array = &values[0];
match (self.input, array.data_type()) {
(InputKind::Float64, DataType::Float64) => self.update_float64(array),
(InputKind::Binary, DataType::Binary) => self.merge_states(array),
(_, data_type) => not_impl_err!("AVG input type does not match: {data_type:?}"),
}
}
fn evaluate(&mut self) -> DfResult<ScalarValue> {
Ok(ScalarValue::Binary(Some(self.state.encode().to_vec())))
}
fn size(&self) -> usize {
std::mem::size_of::<Self>()
}
fn state(&mut self) -> DfResult<Vec<ScalarValue>> {
Ok(vec![ScalarValue::Binary(Some(
self.state.encode().to_vec(),
))])
}
fn merge_batch(&mut self, states: &[ArrayRef]) -> DfResult<()> {
self.merge_states(&states[0])
}
}
#[cfg(test)]
mod tests {
use std::sync::Arc;
use arrow::array::{BinaryArray, Float64Array};
use datafusion_common::ScalarValue;
use datafusion_common::arrow::datatypes::DataType;
use datafusion_expr::TypeSignature;
use datafusion_physical_expr::aggregate::AggregateExprBuilder;
use datafusion_physical_expr::expressions::{Column, lit as physical_lit};
use super::*;
use crate::aggrs::aggr_wrapper::{aggr_delta_merge_func_name, aggr_state_func_name};
use crate::function_registry::FUNCTION_REGISTRY;
fn state(count: u64, sum: f64) -> Vec<u8> {
AvgState { count, sum }.encode().to_vec()
}
#[test]
fn codec_golden_and_roundtrip() {
let empty = AvgState::default().encode();
assert_eq!(empty.len(), ENCODED_LEN);
assert_eq!(empty.as_slice(), b"AVG1\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0");
let mut accumulator = AvgAccumulator::default();
accumulator
.update_batch(&[Arc::new(Float64Array::from(vec![Some(1.5)]))])
.unwrap();
let one = accumulator.state.encode();
assert_eq!(one.len(), ENCODED_LEN);
assert_eq!(
one.as_slice(),
&[
b'A', b'V', b'G', b'1', 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0xf8, 0x3f,
]
);
assert_eq!(&one[12..20], &1.5f64.to_bits().to_le_bytes());
assert_eq!(AvgState::decode(&empty).unwrap().encode(), empty);
assert_eq!(AvgState::decode(&one).unwrap().encode(), one);
}
#[test]
fn codec_rejects_malformed_states() {
assert!(AvgState::decode(b"").is_err());
assert!(AvgState::decode(&[0; 19]).is_err());
assert!(AvgState::decode(&[0; 21]).is_err());
let mut avg2 = AvgState::default().encode();
avg2[..4].copy_from_slice(b"AVG2");
assert!(AvgState::decode(&avg2).is_err());
let mut wrong_magic = AvgState::default().encode();
wrong_magic[0] = b'X';
assert!(AvgState::decode(&wrong_magic).is_err());
for sum in [1.0, -0.0] {
assert!(AvgState::decode(&state(0, sum)).is_err());
}
let mut count = AvgState {
count: 0x0102_0304_0506_0708,
sum: 0.0,
}
.encode();
assert_eq!(&count[4..12], &0x0102_0304_0506_0708u64.to_le_bytes());
count[4..12].reverse();
assert_ne!(
AvgState::decode(&count).unwrap().count(),
0x0102_0304_0506_0708
);
let mut sum = AvgState { count: 1, sum: 1.5 }.encode();
assert_eq!(&sum[12..20], &1.5f64.to_bits().to_le_bytes());
sum[12..20].reverse();
assert_ne!(AvgState::decode(&sum).unwrap().average(), Some(1.5));
}
#[test]
fn codec_preserves_populated_float_bits() {
for bits in [
0.0f64.to_bits(),
(-0.0f64).to_bits(),
f64::INFINITY.to_bits(),
f64::NEG_INFINITY.to_bits(),
0x7ff8_0000_0000_0001,
0x7ff0_0000_0000_0001,
] {
let encoded = state(1, f64::from_bits(bits));
assert_eq!(
AvgState::decode(&encoded).unwrap().encode().as_slice(),
encoded
);
}
}
#[test]
fn distinct_is_rejected() {
let udf = AvgAccumulator::state_udf_impl();
let schema = arrow_schema::Schema::empty();
let expr = physical_lit(1.0f64);
let field = Arc::new(arrow_schema::Field::new("in", DataType::Float64, true));
let args = AccumulatorArgs {
return_field: Arc::new(arrow_schema::Field::new("out", DataType::Binary, true)),
schema: &schema,
ignore_nulls: false,
order_bys: &[],
is_reversed: false,
name: AVG_STATE_NAME,
is_distinct: true,
exprs: std::slice::from_ref(&expr),
expr_fields: std::slice::from_ref(&field),
};
assert!(udf.accumulator(args).is_err());
}
#[test]
fn state_counts_nulls_and_empty_is_canonical() {
let mut accumulator = AvgAccumulator::default();
accumulator
.update_batch(&[Arc::new(Float64Array::from(vec![None, None]))])
.unwrap();
assert_eq!(accumulator.state.encode(), AvgState::default().encode());
accumulator
.update_batch(&[Arc::new(Float64Array::from(vec![
Some(1.0),
None,
Some(3.0),
Some(8.0),
]))])
.unwrap();
assert_eq!(accumulator.state.count(), 3);
assert_eq!(accumulator.state.average(), Some(4.0));
}
#[test]
fn default_value_matches_empty_accumulator_evaluate() {
for udf in [
AvgAccumulator::state_udf_impl(),
AvgAccumulator::merge_udf_impl(),
] {
let default = udf.default_value(&DataType::Binary).unwrap();
let expected = ScalarValue::Binary(Some(AvgState::default().encode().to_vec()));
assert_eq!(default, expected);
let mut empty = AvgAccumulator::default();
assert_eq!(
default,
empty.evaluate().unwrap(),
"{} default_value must equal the empty accumulator evaluate",
udf.name()
);
}
}
#[test]
fn merge_preserves_populated_negative_zero_for_empty_input() {
let mut accumulator = AvgAccumulator {
state: AvgState {
count: 1,
sum: -0.0,
},
input: InputKind::Binary,
};
let expected = accumulator.state.encode();
accumulator
.update_batch(&[Arc::new(BinaryArray::from(vec![
None,
Some(AvgState::default().encode().as_slice()),
]))])
.unwrap();
assert_eq!(accumulator.state.encode(), expected);
}
#[test]
fn merge_ignores_nulls_and_merges_weighted_states() {
let mut accumulator = AvgAccumulator {
state: AvgState::default(),
input: InputKind::Binary,
};
accumulator
.update_batch(&[Arc::new(BinaryArray::from(vec![
Some(state(2, 4.0).as_slice()),
None,
Some(state(3, 15.0).as_slice()),
]))])
.unwrap();
assert_eq!(accumulator.state.count(), 5);
assert_eq!(accumulator.state.average(), Some(19.0 / 5.0));
let before = accumulator.state;
assert!(
accumulator
.update_batch(&[Arc::new(BinaryArray::from(vec![Some(&[][..])]))])
.is_err()
);
assert_eq!(accumulator.state, before);
}
#[test]
fn overflow_does_not_mutate_update_or_merge() {
let mut update = AvgAccumulator {
state: AvgState {
count: u64::MAX,
sum: 1.0,
},
input: InputKind::Float64,
};
let before = update.state;
assert!(
update
.update_batch(&[Arc::new(Float64Array::from(vec![Some(2.0)]))])
.is_err()
);
assert_eq!(update.state, before);
let mut merge = AvgAccumulator {
state: AvgState {
count: u64::MAX,
sum: 1.0,
},
input: InputKind::Binary,
};
let before = merge.state;
assert!(
merge
.update_batch(&[Arc::new(BinaryArray::from(vec![Some(
state(1, 2.0).as_slice()
)]))])
.is_err()
);
assert_eq!(merge.state, before);
}
#[test]
fn registered_delta_merge_has_four_way_and_malformed_behavior() {
let udf = FUNCTION_REGISTRY
.get_aggr_func(&aggr_delta_merge_func_name(AVG_STATE_NAME))
.unwrap();
assert_eq!(udf.name(), "__avg_state_delta_merge");
assert_eq!(
udf.signature().type_signature,
TypeSignature::Exact(vec![DataType::Binary, DataType::Binary])
);
let schema = Arc::new(arrow_schema::Schema::new(vec![
arrow_schema::Field::new("delta", DataType::Binary, true),
arrow_schema::Field::new("persisted", DataType::Binary, true),
]));
let expr = AggregateExprBuilder::new(
Arc::new(udf),
vec![
Arc::new(Column::new("delta", 0)),
Arc::new(Column::new("persisted", 1)),
],
)
.schema(schema)
.alias("avg_delta_merge")
.build()
.unwrap();
let delta = state(2, 3.0);
let persisted = state(2, 7.0);
for (left, right, expected) in [
(
Some(delta.as_slice()),
None,
AvgState { count: 2, sum: 3.0 }.encode(),
),
(
None,
Some(persisted.as_slice()),
AvgState { count: 2, sum: 7.0 }.encode(),
),
(None, None, AvgState::default().encode()),
(
Some(delta.as_slice()),
Some(persisted.as_slice()),
AvgState {
count: 4,
sum: 10.0,
}
.encode(),
),
] {
let mut accumulator = expr.create_accumulator().unwrap();
accumulator
.update_batch(&[
Arc::new(BinaryArray::from(vec![left])),
Arc::new(BinaryArray::from(vec![right])),
])
.unwrap();
let ScalarValue::Binary(Some(actual)) = accumulator.evaluate().unwrap() else {
panic!("AVG delta merge state must be binary");
};
assert_eq!(actual.as_slice(), expected.as_slice());
}
let mut accumulator = expr.create_accumulator().unwrap();
assert!(
accumulator
.update_batch(&[
Arc::new(BinaryArray::from(vec![Some(&[][..])])),
Arc::new(BinaryArray::from(vec![None])),
])
.is_err()
);
let mut accumulator = expr.create_accumulator().unwrap();
assert!(
accumulator
.update_batch(&[
Arc::new(BinaryArray::from(vec![None])),
Arc::new(BinaryArray::from(vec![Some(&[][..])])),
])
.is_err()
);
let mut accumulator = expr.create_accumulator().unwrap();
accumulator
.update_batch(&[
Arc::new(BinaryArray::from(vec![Some(delta.as_slice())])),
Arc::new(BinaryArray::from(vec![None])),
])
.unwrap();
assert!(
accumulator
.update_batch(&[
Arc::new(BinaryArray::from(vec![Some(delta.as_slice())])),
Arc::new(BinaryArray::from(vec![Some(
AvgState {
count: u64::MAX,
sum: 1.0
}
.encode()
.as_slice()
)])),
])
.is_err()
);
}
#[test]
fn avg_registry_does_not_replace_native_state_registry() {
let avg = FUNCTION_REGISTRY.get_aggr_func(AVG_STATE_NAME).unwrap();
let native = FUNCTION_REGISTRY
.get_aggr_func(&aggr_state_func_name("avg"))
.unwrap();
assert_eq!(
avg.return_type(&[DataType::Float64]).unwrap(),
DataType::Binary
);
assert!(matches!(
native.return_type(&[DataType::Float64]).unwrap(),
DataType::Struct(_)
));
}
}
src/common/function/src/function_registry.rs
+20
View File
@@ -28,7 +28,10 @@ use crate::aggrs::count_hash::CountHash;
use crate::aggrs::vector::VectorFunction as VectorAggrFunction;
use crate::function::{Function, FunctionRef};
use crate::function_factory::ScalarFunctionFactory;
#[cfg(feature = "ai_functions")]
use crate::scalars::ai;
use crate::scalars::anomaly::AnomalyFunction;
use crate::scalars::avg_calc::AvgCalcFunction;
use crate::scalars::date::DateFunction;
use crate::scalars::expression::ExpressionFunction;
use crate::scalars::hll_count::HllCalcFunction;
@@ -215,6 +218,7 @@ pub static FUNCTION_REGISTRY: LazyLock<Arc<FunctionRegistry>> = LazyLock::new(||
TimestampFunction::register(&function_registry);
DateFunction::register(&function_registry);
ExpressionFunction::register(&function_registry);
AvgCalcFunction::register(&function_registry);
UddSketchCalcFunction::register(&function_registry);
UddSketchRankFunction::register(&function_registry);
HllCalcFunction::register(&function_registry);
@@ -224,6 +228,8 @@ pub static FUNCTION_REGISTRY: LazyLock<Arc<FunctionRegistry>> = LazyLock::new(||
// Full text search function
MatchesFunction::register(&function_registry);
MatchesTermFunction::register(&function_registry);
#[cfg(feature = "ai_functions")]
ai::register(&function_registry);
// System and administration functions
SystemFunction::register(&function_registry);
@@ -377,6 +383,20 @@ mod tests {
assert!(FUNCTION_REGISTRY.get_function("uddsketch_rank").is_some());
}
#[test]
fn test_ai_registration_matches_feature() {
for name in ["ai_match", "ai_choose", "ai_score"] {
assert_eq!(
FUNCTION_REGISTRY.get_function(name).is_some(),
cfg!(feature = "ai_functions"),
"{name}"
);
}
for name in ["jev", "jev_choice", "jev_score"] {
assert!(FUNCTION_REGISTRY.get_function(name).is_none(), "{name}");
}
}
#[test]
fn test_register_if_absent_registers_new_function() {
let registry = FunctionRegistry::default();
src/common/function/src/scalars.rs
+3
View File
@@ -12,6 +12,8 @@
// See the License for the specific language governing permissions and
// limitations under the License.
#[cfg(feature = "ai_functions")]
pub(crate) mod ai;
pub mod anomaly;
pub(crate) mod date;
pub mod expression;
@@ -25,6 +27,7 @@ pub mod primary_key;
pub(crate) mod string;
pub mod vector;
pub(crate) mod avg_calc;
pub(crate) mod hll_count;
pub mod ip;
#[cfg(test)]
src/common/function/src/scalars/ai.rs
+454
View File
@@ -0,0 +1,454 @@
// Copyright 2023 Greptime Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#[cfg(test)]
mod tests;
use std::fmt;
use std::hash::Hash;
use std::marker::PhantomData;
use std::sync::Arc;
use std::time::Duration;
use arrow::array::{Array, StringViewArray, new_null_array};
use arrow::datatypes::DataType;
use async_trait::async_trait;
use datafusion_common::cast::as_string_view_array;
use datafusion_common::{Result, ScalarValue, exec_datafusion_err, exec_err, not_impl_err};
use datafusion_expr::async_udf::{AsyncScalarUDF, AsyncScalarUDFImpl};
use datafusion_expr::{ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility};
use futures::{StreamExt, TryStreamExt, stream};
use reqwest::Client;
use serde::Serialize;
use serde_json::{Map, Value, json};
use crate::function_factory::ScalarFunctionFactory;
use crate::function_registry::FunctionRegistry;
/// Registers the experimental AI matching, classification, and rating functions.
pub(crate) fn register(registry: &FunctionRegistry) {
AiFunction::<Noul>::register(registry);
AiFunction::<Choice>::register(registry);
AiFunction::<Score>::register(registry);
}
/// Shared asynchronous execution for AI functions using the Jev backend.
#[derive(PartialEq, Eq, Hash)]
struct AiFunction<Q> {
signature: Signature,
question: PhantomData<Q>,
enabled: bool,
api_key: Option<String>,
endpoint: String,
model: String,
}
struct AiRequest<'a, Q: AiQuestion> {
text: &'a str,
prompt: &'a str,
criteria: Arc<Q::Criteria>,
}
/// Borrows string arguments without expanding constants to the batch size.
enum StringArgument<'a> {
Scalar(Option<&'a str>),
Array(&'a StringViewArray),
}
impl<'a> StringArgument<'a> {
fn try_new(arg: &'a ColumnarValue) -> Result<Self> {
match arg {
ColumnarValue::Scalar(ScalarValue::Utf8View(value)) => {
Ok(Self::Scalar(value.as_deref()))
}
ColumnarValue::Array(array) => Ok(Self::Array(as_string_view_array(array)?)),
_ => exec_err!("AI functions expect Utf8View arguments"),
}
}
fn value(&self, row: usize) -> Option<&'a str> {
match self {
Self::Scalar(value) => *value,
Self::Array(array) => {
let array = *array;
array.is_valid(row).then(|| array.value(row))
}
}
}
}
impl<Q: AiQuestion> AiFunction<Q> {
fn register(registry: &FunctionRegistry) {
registry.register(ScalarFunctionFactory {
name: Q::NAME.to_string(),
factory: Arc::new(|_| AsyncScalarUDF::new(Arc::new(Self::default())).into_scalar_udf()),
});
}
async fn evaluate(&self, client: &Client, request: AiRequest<'_, Q>) -> Result<ScalarValue> {
let mut question = json!({ "type": Q::TYPE, "instructions": request.prompt });
if Q::ARG_COUNT == 3 {
question["criteria"] = json!(request.criteria.as_ref());
}
let response: Value = client
.post(&self.endpoint)
.json(&json!({
"model": self.model,
"state": request.text,
"questions": { "matches": question }
}))
.send()
.await
.and_then(|response| response.error_for_status())
.map_err(|e| exec_datafusion_err!("{} request failed: {e}", Q::NAME))?
.json()
.await
.map_err(|e| exec_datafusion_err!("{} response is not valid JSON: {e}", Q::NAME))?;
let answer = &response["answers"]["matches"];
if answer["type"].as_str() != Some(Q::TYPE) {
return exec_err!(
"{} response must contain answers.matches with type {}",
Q::NAME,
Q::TYPE
);
}
Q::parse_answer(answer, request.criteria.as_ref())
}
fn prepare_requests<'a>(
&self,
args: &'a [ColumnarValue],
number_rows: usize,
) -> Result<Vec<Option<AiRequest<'a, Q>>>> {
if args.len() != Q::ARG_COUNT {
return exec_err!("{} requires {} arguments", Q::NAME, Q::ARG_COUNT);
}
let args = args
.iter()
.map(StringArgument::try_new)
.collect::<Result<Vec<_>>>()?;
let scalar_criteria = args[2..]
.iter()
.all(|arg| matches!(arg, StringArgument::Scalar(_)));
let mut shared_criteria: Option<Arc<Q::Criteria>> = None;
// Validate all non-null rows before making any billable requests.
(0..number_rows)
.map(|row| {
let values: Option<Vec<_>> = args.iter().map(|arg| arg.value(row)).collect();
values
.map(|values| {
// Initialize lazily so NULL rows never validate otherwise invalid criteria.
let criteria = match &shared_criteria {
Some(criteria) => Arc::clone(criteria),
None => {
let criteria = Arc::new(Q::parse_criteria(&values[2..])?);
if scalar_criteria {
shared_criteria = Some(Arc::clone(&criteria));
}
criteria
}
};
Ok(AiRequest {
text: values[0],
prompt: values[1],
criteria,
})
})
.transpose()
})
.collect()
}
fn client(&self) -> Result<Client> {
if !self.enabled {
return exec_err!(
"{} is experimental; set GREPTIMEDB_EXPERIMENTAL_JEV=true to enable it",
Q::NAME
);
}
let key = self
.api_key
.as_deref()
.filter(|key| !key.trim().is_empty())
.ok_or_else(|| {
exec_datafusion_err!("{} requires the JEV_API_KEY environment variable", Q::NAME)
})?;
let mut authorization = reqwest::header::HeaderValue::from_str(&format!("Bearer {key}"))
.map_err(|_| exec_datafusion_err!("JEV_API_KEY is not a valid HTTP header value"))?;
authorization.set_sensitive(true);
let mut headers = reqwest::header::HeaderMap::new();
headers.insert(reqwest::header::AUTHORIZATION, authorization);
Client::builder()
.default_headers(headers)
.timeout(Duration::from_secs(30))
.build()
.map_err(|e| exec_datafusion_err!("failed to create {} HTTP client: {e}", Q::NAME))
}
}
impl<Q: AiQuestion> Default for AiFunction<Q> {
fn default() -> Self {
Self {
// External model evaluations must not be constant-folded during planning.
signature: Signature::exact(
vec![DataType::Utf8View; Q::ARG_COUNT],
Volatility::Volatile,
),
question: PhantomData,
enabled: std::env::var("GREPTIMEDB_EXPERIMENTAL_JEV").as_deref() == Ok("true"),
api_key: std::env::var("JEV_API_KEY").ok(),
endpoint: std::env::var("JEV_ENDPOINT")
.unwrap_or_else(|_| "https://api.typesafe.ai/v1/systemone".to_string()),
model: std::env::var("JEV_MODEL").unwrap_or_else(|_| "jev-latest".to_string()),
}
}
}
impl<Q: AiQuestion> fmt::Debug for AiFunction<Q> {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
// Credentials must never appear in plans or diagnostic output.
f.debug_struct("AiFunction")
.field("name", &Q::NAME)
.field("signature", &self.signature)
.field("enabled", &self.enabled)
.field("model", &self.model)
.finish_non_exhaustive()
}
}
impl<Q: AiQuestion> ScalarUDFImpl for AiFunction<Q> {
fn name(&self) -> &str {
Q::NAME
}
fn signature(&self) -> &Signature {
&self.signature
}
fn return_type(&self, _arg_types: &[DataType]) -> Result<DataType> {
Ok(Q::return_type())
}
fn invoke_with_args(&self, _args: ScalarFunctionArgs) -> Result<ColumnarValue> {
not_impl_err!("{} can only be called from async contexts", Q::NAME)
}
}
#[async_trait]
impl<Q: AiQuestion> AsyncScalarUDFImpl for AiFunction<Q> {
async fn invoke_async_with_args(&self, args: ScalarFunctionArgs) -> Result<ColumnarValue> {
let rows = self.prepare_requests(&args.args, args.number_rows)?;
if rows.iter().all(Option::is_none) {
return Ok(ColumnarValue::Array(new_null_array(
&Q::return_type(),
args.number_rows,
)));
}
// Reuse connections within the batch; bound in-flight requests and preserve row order.
let client = self.client()?;
// Collect futures to avoid async_trait's higher-ranked lifetime inference issue.
let requests: Vec<_> = rows
.into_iter()
.map(|row| {
let client = &client;
async move {
match row {
Some(request) => self.evaluate(client, request).await,
None => ScalarValue::try_from(&Q::return_type()),
}
}
})
.collect();
// This limit is per expression/batch invocation, not per query or process.
// Concurrent partitions and queries can each have their own in-flight requests.
let answers: Vec<ScalarValue> = stream::iter(requests).buffered(8).try_collect().await?;
Ok(ColumnarValue::Array(ScalarValue::iter_to_array(answers)?))
}
}
/// The request criteria and scalar answer contract for an AI question type.
trait AiQuestion: fmt::Debug + Eq + Hash + Send + Sync + 'static {
type Criteria: Serialize + Send + Sync;
const NAME: &'static str;
const TYPE: &'static str;
const ARG_COUNT: usize;
fn return_type() -> DataType;
fn parse_criteria(args: &[&str]) -> Result<Self::Criteria>;
fn parse_answer(answer: &Value, criteria: &Self::Criteria) -> Result<ScalarValue>;
}
#[derive(Debug, PartialEq, Eq, Hash)]
struct Noul;
impl AiQuestion for Noul {
type Criteria = ();
const NAME: &'static str = "ai_match";
const TYPE: &'static str = "noul";
const ARG_COUNT: usize = 2;
fn return_type() -> DataType {
DataType::Float64
}
fn parse_criteria(_args: &[&str]) -> Result<()> {
Ok(())
}
fn parse_answer(answer: &Value, _criteria: &()) -> Result<ScalarValue> {
numeric_answer(Self::NAME, &answer["noul"], "noul", 1.0)
.map(|probability| ScalarValue::Float64(Some(probability)))
}
}
#[derive(Debug, PartialEq, Eq, Hash)]
struct Choice;
impl AiQuestion for Choice {
type Criteria = Map<String, Value>;
const NAME: &'static str = "ai_choose";
const TYPE: &'static str = "choice";
const ARG_COUNT: usize = 3;
fn return_type() -> DataType {
DataType::Utf8
}
fn parse_criteria(args: &[&str]) -> Result<Self::Criteria> {
let criteria: Value = serde_json::from_str(args[0])
.map_err(|e| exec_datafusion_err!("ai_choose criteria is not valid JSON: {e}"))?;
match criteria {
Value::Object(options)
if (1..=255).contains(&options.len())
&& options.values().all(|v| v.is_null() || is_description(v)) =>
{
Ok(options)
}
_ => exec_err!(
"ai_choose criteria must be a JSON object with 1 to 255 options; descriptions must be strings, objects, arrays, or null"
),
}
}
fn parse_answer(answer: &Value, criteria: &Self::Criteria) -> Result<ScalarValue> {
let choice = answer["choice"]
.as_str()
.filter(|choice| criteria.contains_key(*choice))
.ok_or_else(|| {
exec_datafusion_err!("ai_choose response must contain a choice from the criteria")
})?;
Ok(ScalarValue::Utf8(Some(choice.to_string())))
}
}
#[derive(Debug, PartialEq, Eq, Hash)]
struct Score;
impl AiQuestion for Score {
type Criteria = Vec<Value>;
const NAME: &'static str = "ai_score";
const TYPE: &'static str = "score";
const ARG_COUNT: usize = 3;
fn return_type() -> DataType {
DataType::BinaryView
}
fn parse_criteria(args: &[&str]) -> Result<Self::Criteria> {
let criteria: Value = serde_json::from_str(args[0])
.map_err(|e| exec_datafusion_err!("ai_score criteria is not valid JSON: {e}"))?;
match criteria {
Value::Array(levels)
if (2..=10).contains(&levels.len()) && levels.iter().all(is_description) =>
{
Ok(levels)
}
_ => exec_err!(
"ai_score criteria must be a JSON array with 2 to 10 levels; descriptions must be strings, objects, or arrays"
),
}
}
fn parse_answer(answer: &Value, criteria: &Self::Criteria) -> Result<ScalarValue> {
let score = numeric_answer(
Self::NAME,
&answer["score"],
"score",
(criteria.len() - 1) as f64,
)?;
let confidence = numeric_answer(Self::NAME, &answer["confidence"], "confidence", 1.0)?;
let probabilities = score_probabilities(&answer["probabilities"], criteria.len())?;
let object = jsonb::Object::from([
("score".to_string(), jsonb::Value::from(score)),
("confidence".to_string(), jsonb::Value::from(confidence)),
(
"probabilities".to_string(),
jsonb::Value::Array(probabilities.into_iter().map(jsonb::Value::from).collect()),
),
]);
Ok(ScalarValue::BinaryView(Some(
jsonb::Value::Object(object).to_vec(),
)))
}
}
fn score_probabilities(distribution: &Value, level_count: usize) -> Result<Vec<f64>> {
if distribution
.as_object()
.is_none_or(|probabilities| probabilities.len() != level_count)
{
return exec_err!(
"ai_score response must contain probabilities for all {level_count} levels"
);
}
let probabilities = (0..level_count)
.map(|level| {
numeric_answer(
Score::NAME,
&distribution[level.to_string()],
"probability",
1.0,
)
})
.collect::<Result<Vec<_>>>()?;
// Allow small rounding differences without renormalizing the provider's distribution.
if (probabilities.iter().sum::<f64>() - 1.0).abs() > 1e-6 {
return exec_err!("ai_score response probabilities must sum to 1 within 1e-6");
}
Ok(probabilities)
}
fn is_description(description: &Value) -> bool {
matches!(
description,
Value::String(_) | Value::Object(_) | Value::Array(_)
)
}
fn numeric_answer(name: &str, answer: &Value, field: &str, max: f64) -> Result<f64> {
answer
.as_f64()
.filter(|score| (0.0..=max).contains(score))
.ok_or_else(|| {
exec_datafusion_err!("{name} response must contain a finite {field} in [0, {max}]")
})
}
src/common/function/src/scalars/ai/tests.rs
+766
View File
@@ -0,0 +1,766 @@
// Copyright 2023 Greptime Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use std::sync::atomic::{AtomicUsize, Ordering};
use axum::http::{HeaderMap, StatusCode};
use axum::routing::post;
use axum::{Json, Router};
use datafusion::assert_batches_eq;
use datafusion::prelude::{SessionConfig, SessionContext};
use tokio::net::TcpListener;
use tokio::task::JoinHandle;
use super::*;
use crate::function::FunctionContext;
use crate::function_registry::FUNCTION_REGISTRY;
struct MockServer {
endpoint: String,
task: JoinHandle<()>,
}
impl MockServer {
async fn start(router: Router) -> Self {
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
let endpoint = format!("http://{}/v1/systemone", listener.local_addr().unwrap());
let task = tokio::spawn(async move { axum::serve(listener, router).await.unwrap() });
Self { endpoint, task }
}
fn function<Q: AiQuestion>(&self) -> AiFunction<Q> {
AiFunction {
enabled: true,
api_key: Some("test-key".to_string()),
endpoint: self.endpoint.clone(),
model: "test-model".to_string(),
..Default::default()
}
}
}
impl Drop for MockServer {
fn drop(&mut self) {
self.task.abort();
}
}
fn context<Q: AiQuestion>(function: AiFunction<Q>) -> SessionContext {
let ctx = SessionContext::new_with_config(SessionConfig::new().with_target_partitions(1));
ctx.register_udf(AsyncScalarUDF::new(Arc::new(function)).into_scalar_udf());
let json_get_float = FUNCTION_REGISTRY.get_function("json_get_float").unwrap();
ctx.register_udf(json_get_float.provide(FunctionContext::default()));
ctx
}
#[test]
fn test_ai_scalar_criteria_share_allocation_and_skip_null_rows() {
fn assert_shared_criteria<Q: AiQuestion>(criteria: &str) {
let function = AiFunction::<Q>::default();
for (texts, criteria) in [
(vec![None, Some("first"), Some("second")], criteria),
(vec![None, None], "not json"),
(vec![], "not json"),
] {
let number_rows = texts.len();
let expected_requests = texts.iter().flatten().count();
let args = vec![
ColumnarValue::Array(Arc::new(StringViewArray::from(texts))),
ColumnarValue::Scalar(ScalarValue::Utf8View(Some("prompt".to_string()))),
ColumnarValue::Scalar(ScalarValue::Utf8View(Some(criteria.to_string()))),
];
let requests = function.prepare_requests(&args, number_rows).unwrap();
assert_eq!(requests.len(), number_rows);
assert_eq!(requests.iter().flatten().count(), expected_requests);
// Regression: parsed constant criteria must not be duplicated for each row.
let mut criteria = requests.iter().flatten().map(|request| &request.criteria);
if let Some(first) = criteria.next() {
assert!(criteria.all(|other| Arc::ptr_eq(first, other)));
}
}
}
assert_shared_criteria::<Choice>(r#"{"billing":null,"technical":"errors"}"#);
assert_shared_criteria::<Score>(r#"["low","high"]"#);
}
#[tokio::test]
async fn test_ai_match_sql_returns_and_filters_noul_probability() {
let server = MockServer::start(Router::new().route(
"/v1/systemone",
post(
|headers: HeaderMap, Json(request): Json<Value>| async move {
assert_eq!(headers["authorization"], "Bearer test-key");
assert_eq!(request["model"], "test-model");
assert_eq!(request["questions"]["matches"]["type"], "noul");
// The prompt is passed through verbatim.
assert_eq!(
request["questions"]["matches"]["instructions"],
"payment failed"
);
let probability = match request["state"].as_str().unwrap() {
"failed" => {
// Complete this row after later rows to exercise result alignment.
tokio::time::sleep(Duration::from_millis(20)).await;
0.95
}
"boundary" => 0.8,
"recovered" => 0.1,
"impossible" => 0.0,
"certain" => 1.0,
unexpected => panic!("unexpected state: {unexpected}"),
};
Json(json!({"answers": {"matches": {"type": "noul", "noul": probability}}}))
},
),
))
.await;
let ctx = context(server.function::<Noul>());
let events = "(VALUES (1, 'failed'), (2, 'boundary'), (3, 'recovered'), (4, NULL), (5, 'impossible'), (6, 'certain')) AS events(id, message)";
let batches = ctx
.sql(&format!(
"SELECT id FROM {events} WHERE ai_match((message), 'payment failed') >= 0.8 ORDER BY id"
))
.await
.unwrap()
.collect()
.await
.unwrap();
assert_batches_eq!(
[
"+----+", "| id |", "+----+", "| 1 |", "| 2 |", "| 6 |", "+----+"
],
&batches
);
let batches = ctx
.sql(&format!(
"SELECT id, ai_match(message, 'payment failed') AS score FROM {events} ORDER BY score DESC NULLS LAST"
))
.await
.unwrap()
.collect()
.await
.unwrap();
assert_eq!(batches[0].schema().field(1).data_type(), &DataType::Float64);
assert_batches_eq!(
[
"+----+-------+",
"| id | score |",
"+----+-------+",
"| 6 | 1.0 |",
"| 1 | 0.95 |",
"| 2 | 0.8 |",
"| 3 | 0.1 |",
"| 5 | 0.0 |",
"| 4 | |",
"+----+-------+"
],
&batches
);
}
#[tokio::test]
async fn test_ai_match_nulls_need_no_api_key() {
let ctx = context(AiFunction::<Noul> {
enabled: true,
api_key: None,
..Default::default()
});
let batches = ctx
.sql("SELECT ai_match(NULL, 'condition') AS a, ai_match('text', NULL) AS b")
.await
.unwrap()
.collect()
.await
.unwrap();
assert_eq!(batches[0].schema().field(0).data_type(), &DataType::Float64);
assert_batches_eq!(
[
"+---+---+",
"| a | b |",
"+---+---+",
"| | |",
"+---+---+"
],
&batches
);
let error = ctx
.sql("SELECT ai_match('text', 'condition')")
.await
.unwrap()
.collect()
.await
.unwrap_err();
assert!(error.to_string().contains("JEV_API_KEY"), "{error}");
let ctx = context(AiFunction::<Noul> {
enabled: false,
api_key: None,
..Default::default()
});
let error = ctx
.sql("SELECT ai_match('text', 'condition')")
.await
.unwrap()
.collect()
.await
.unwrap_err();
assert!(
error
.to_string()
.contains("GREPTIMEDB_EXPERIMENTAL_JEV=true"),
"{error}"
);
}
#[tokio::test]
async fn test_ai_match_bad_api_responses_fail_the_query() {
for (status, body, expected) in [
(
StatusCode::TOO_MANY_REQUESTS,
"{}",
"ai_match request failed",
),
(
StatusCode::OK,
"not json",
"ai_match response is not valid JSON",
),
(
StatusCode::OK,
r#"{"answers":{}}"#,
"ai_match response must contain",
),
(
StatusCode::OK,
r#"{"answers":{"matches":{"type":"noul","noul":1.1}}}"#,
"ai_match response must contain",
),
(
StatusCode::OK,
r#"{"answers":{"matches":{"type":"noul","noul":-0.1}}}"#,
"ai_match response must contain",
),
(
StatusCode::OK,
r#"{"answers":{"matches":{"type":"score","noul":0.9}}}"#,
"ai_match response must contain",
),
] {
let server = MockServer::start(
Router::new().route("/v1/systemone", post(move || async move { (status, body) })),
)
.await;
let ctx = context(server.function::<Noul>());
let error = ctx
.sql("SELECT ai_match('text', 'condition')")
.await
.unwrap()
.collect()
.await
.unwrap_err();
assert!(error.to_string().contains(expected), "{error}");
}
}
#[tokio::test]
async fn test_ai_choose_returns_and_filters_supplied_labels() {
let server = MockServer::start(Router::new().route(
"/v1/systemone",
post(|Json(request): Json<Value>| async move {
let question = &request["questions"]["matches"];
assert_eq!(question["type"], "choice");
assert_eq!(question["instructions"], "Route the ticket");
let choice = match request["state"].as_str().unwrap() {
"refund" => {
assert_eq!(question["criteria"], json!({"billing": null, "technical": {"examples": ["crash"]}}));
tokio::time::sleep(Duration::from_millis(20)).await;
"billing"
}
"crash" => {
assert_eq!(question["criteria"], json!({"billing": ["refunds"], "technical": "errors"}));
"technical"
}
unexpected => panic!("unexpected state: {unexpected}"),
};
Json(json!({"answers": {"matches": {"type": "choice", "choice": choice, "confidence": 0.7}}}))
}),
))
.await;
let ctx = context(server.function::<Choice>());
let events = r#"(VALUES
(1, 'refund', 'Route the ticket', '{"billing":null,"technical":{"examples":["crash"]}}'),
(2, 'crash', 'Route the ticket', '{"billing":["refunds"],"technical":"errors"}'),
(3, NULL, 'Route the ticket', 'invalid JSON'),
(4, 'refund', NULL, 'invalid JSON'),
(5, 'refund', 'Route the ticket', NULL)
) AS events(id, message, prompt, criteria)"#;
let batches = ctx
.sql(&format!(
"SELECT id, ai_choose(message, prompt, criteria) AS team FROM {events} ORDER BY id"
))
.await
.unwrap()
.collect()
.await
.unwrap();
assert_eq!(batches[0].schema().field(1).data_type(), &DataType::Utf8);
assert_batches_eq!(
[
"+----+-----------+",
"| id | team |",
"+----+-----------+",
"| 1 | billing |",
"| 2 | technical |",
"| 3 | |",
"| 4 | |",
"| 5 | |",
"+----+-----------+"
],
&batches
);
let batches = ctx
.sql(&format!(
"SELECT id FROM {events} WHERE ai_choose(message, prompt, criteria) = 'billing'"
))
.await
.unwrap()
.collect()
.await
.unwrap();
assert_batches_eq!(["+----+", "| id |", "+----+", "| 1 |", "+----+"], &batches);
let batches = ctx
.sql(
r#"SELECT id, ai_choose(message, 'Route the ticket',
'{"billing":null,"technical":{"examples":["crash"]}}') AS team
FROM (VALUES (1, 'refund'), (2, NULL), (3, 'refund')) AS events(id, message)
ORDER BY id"#,
)
.await
.unwrap()
.collect()
.await
.unwrap();
assert_batches_eq!(
[
"+----+---------+",
"| id | team |",
"+----+---------+",
"| 1 | billing |",
"| 2 | |",
"| 3 | billing |",
"+----+---------+"
],
&batches
);
}
#[tokio::test]
async fn test_ai_score_returns_fractional_scores_on_each_rows_scale() {
let server = MockServer::start(Router::new().route(
"/v1/systemone",
post(|Json(request): Json<Value>| async move {
let question = &request["questions"]["matches"];
assert_eq!(question["type"], "score");
assert_eq!(question["instructions"], "Rate severity");
let (score, probabilities) = match request["state"].as_str().unwrap() {
"low" => {
assert_eq!(question["criteria"], json!(["low", {"description": "medium"}, ["high"]]));
tokio::time::sleep(Duration::from_millis(20)).await;
(0.0, json!({"0":1.0,"1":0.0,"2":0.0}))
}
"fractional" => (1.25, json!({"2":0.25,"0":0.0,"1":0.75})),
"high" => (2.0, json!({"0":0.0,"1":0.0,"2":1.0})),
"binary" => {
assert_eq!(question["criteria"], json!(["low", "high"]));
(1.0, json!({"0":0.0,"1":1.0}))
}
"ten levels" => {
assert_eq!(question["criteria"].as_array().unwrap().len(), 10);
let probabilities: Map<String, Value> = (0..10)
.map(|level| (level.to_string(), json!(if level == 9 { 1.0 } else { 0.0 })))
.collect();
(9.0, Value::Object(probabilities))
}
unexpected => panic!("unexpected state: {unexpected}"),
};
Json(json!({"answers": {"matches": {"type": "score", "score": score, "confidence": 0.4, "probabilities": probabilities}}}))
}),
))
.await;
let ctx = context(server.function::<Score>());
let batches = ctx
.sql(
r#"
SELECT id, json_get_float(ai_score(message, 'Rate severity', criteria), 'score') AS severity
FROM (VALUES
(1, 'low', '["low", {"description":"medium"}, ["high"]]'),
(2, 'fractional', '["low", "medium", "high"]'),
(3, 'high', '["low", "medium", "high"]'),
(4, 'binary', '["low", "high"]'),
(5, 'ten levels', '["a","b","c","d","e","f","g","h","i","j"]'),
(6, NULL, 'invalid JSON'),
(7, 'high', NULL)
) AS events(id, message, criteria)
ORDER BY severity DESC NULLS LAST, id
"#,
)
.await
.unwrap()
.collect()
.await
.unwrap();
assert_eq!(batches[0].schema().field(1).data_type(), &DataType::Float64);
assert_batches_eq!(
[
"+----+----------+",
"| id | severity |",
"+----+----------+",
"| 5 | 9.0 |",
"| 3 | 2.0 |",
"| 2 | 1.25 |",
"| 4 | 1.0 |",
"| 1 | 0.0 |",
"| 6 | |",
"| 7 | |",
"+----+----------+"
],
&batches
);
let batches = ctx
.sql(
r#"SELECT id, json_get_float(ai_score(message, 'Rate severity',
'["low",{"description":"medium"},["high"]]'), 'score') AS severity
FROM (VALUES (1, 'low'), (2, 'fractional'), (3, 'high'), (4, NULL)) AS events(id, message)
ORDER BY id"#,
)
.await
.unwrap()
.collect()
.await
.unwrap();
assert_batches_eq!(
[
"+----+----------+",
"| id | severity |",
"+----+----------+",
"| 1 | 0.0 |",
"| 2 | 1.25 |",
"| 3 | 2.0 |",
"| 4 | |",
"+----+----------+"
],
&batches
);
}
#[tokio::test]
async fn test_ai_score_preserves_uncertainty_in_one_evaluation() {
let calls = Arc::new(AtomicUsize::new(0));
let request_calls = calls.clone();
let server = MockServer::start(Router::new().route(
"/v1/systemone",
post(move |Json(request): Json<Value>| {
request_calls.fetch_add(1, Ordering::Relaxed);
async move {
let (confidence, probabilities) = match request["state"].as_str().unwrap() {
"medium" => (1.0, json!({"2":0.0,"0":0.0,"1":1.0})),
"split" => (0.25, json!({"2":0.5,"0":0.5,"1":0.0})),
unexpected => panic!("unexpected state: {unexpected}"),
};
Json(json!({"answers":{"matches":{
"type":"score", "score":1.0, "confidence":confidence,
"probabilities":probabilities
}}}))
}
}),
))
.await;
let ctx = context(server.function::<Score>());
let rated = r#"(SELECT id, ai_score(message, 'Rate severity', '["low","medium","high"]') AS rating
FROM (VALUES (1, 'medium'), (2, 'split'), (3, NULL)) AS events(id, message)) AS rated"#;
let batches = ctx
.sql(&format!(
"SELECT id, json_get_float(rating, 'score') AS score,
json_get_float(rating, 'confidence') AS confidence,
json_get_float(rating, 'probabilities[0]') AS p0,
json_get_float(rating, 'probabilities[1]') AS p1,
json_get_float(rating, 'probabilities[2]') AS p2
FROM {rated} ORDER BY id"
))
.await
.unwrap()
.collect()
.await
.unwrap();
assert_batches_eq!(
[
"+----+-------+------------+-----+-----+-----+",
"| id | score | confidence | p0 | p1 | p2 |",
"+----+-------+------------+-----+-----+-----+",
"| 1 | 1.0 | 1.0 | 0.0 | 1.0 | 0.0 |",
"| 2 | 1.0 | 0.25 | 0.5 | 0.0 | 0.5 |",
"| 3 | | | | | |",
"+----+-------+------------+-----+-----+-----+"
],
&batches
);
assert_eq!(calls.load(Ordering::Relaxed), 2);
let batches = ctx
.sql(&format!(
"SELECT id FROM {rated} WHERE json_get_float(rating, 'confidence') >= 0.8
ORDER BY json_get_float(rating, 'score') DESC"
))
.await
.unwrap()
.collect()
.await
.unwrap();
assert_batches_eq!(["+----+", "| id |", "+----+", "| 1 |", "+----+"], &batches);
assert_eq!(calls.load(Ordering::Relaxed), 4);
}
#[test]
fn test_ai_score_preserves_provider_precision() {
let answer = json!({
"score":1.6, "confidence":0.92,
"probabilities":{"2":0.6999995,"0":0.1,"1":0.2}
});
let criteria = vec![json!("low"), json!("medium"), json!("high")];
let ScalarValue::BinaryView(Some(bytes)) = Score::parse_answer(&answer, &criteria).unwrap()
else {
panic!("expected a JSONB object");
};
let object: Value =
serde_json::from_str(&jsonb::from_slice(&bytes).unwrap().to_string()).unwrap();
assert_eq!(object.as_object().unwrap().len(), 3);
assert_eq!(object["score"].as_f64(), Some(1.6));
assert_eq!(object["confidence"].as_f64(), Some(0.92));
let probabilities: Vec<f64> = object["probabilities"]
.as_array()
.unwrap()
.iter()
.map(|probability| probability.as_f64().unwrap())
.collect();
assert_eq!(probabilities, vec![0.1, 0.2, 0.6999995]);
}
#[tokio::test]
async fn test_ai_score_rejects_invalid_uncertainty() {
for (field, value) in [
("confidence", Value::Null),
("confidence", json!(-0.1)),
("confidence", json!(1.1)),
("confidence", json!("0.9")),
("probabilities", Value::Null),
("probabilities", json!([0.0, 1.0, 0.0])),
("probabilities", json!({"0":1.0})),
("probabilities", json!({"0":0.5,"2":0.5,"3":0.0})),
("probabilities", json!({"0":0.0,"1":1.0,"2":0.0,"3":0.0})),
("probabilities", json!({"0":-0.1,"1":0.6,"2":0.5})),
("probabilities", json!({"0":0.0,"1":1.1,"2":0.0})),
("probabilities", json!({"0":0.0,"1":0.95,"2":"0.05"})),
("probabilities", json!({"0":0.0,"1":0.0,"2":0.0})),
("probabilities", json!({"0":0.5,"1":0.5,"2":0.5})),
] {
let mut answer = json!({
"type":"score", "score":1.0, "confidence":0.9,
"probabilities":{"0":0.0,"1":1.0,"2":0.0}
});
answer[field] = value;
let server = MockServer::start(Router::new().route(
"/v1/systemone",
post(move || {
let answer = answer.clone();
async move { Json(json!({"answers":{"matches":answer}})) }
}),
))
.await;
let ctx = context(server.function::<Score>());
let error = ctx
.sql(r#"SELECT ai_score('text', 'prompt', '["low","medium","high"]')"#)
.await
.unwrap()
.collect()
.await
.unwrap_err();
assert!(error.to_string().contains("ai_score response"), "{error}");
}
}
#[tokio::test]
async fn test_ai_invalid_criteria_fail_before_requests() {
let choice_ctx = context(AiFunction::<Choice> {
enabled: true,
api_key: None,
..Default::default()
});
let score_ctx = context(AiFunction::<Score> {
enabled: true,
api_key: None,
..Default::default()
});
let too_many_choices: Map<String, Value> = (0..256)
.map(|index| (index.to_string(), Value::Null))
.collect();
for (ctx, name, valid, invalid) in [
(
&choice_ctx,
"ai_choose",
r#"{"billing":null}"#,
vec![
"not json".to_string(),
"null".to_string(),
"{}".to_string(),
"[]".to_string(),
r#"{"billing":1}"#.to_string(),
r#"{"billing":true}"#.to_string(),
Value::Object(too_many_choices).to_string(),
],
),
(
&score_ctx,
"ai_score",
r#"["low","high"]"#,
vec![
"not json".to_string(),
"null".to_string(),
"{}".to_string(),
"[]".to_string(),
r#"["low"]"#.to_string(),
r#"["low",null]"#.to_string(),
r#"["low",1]"#.to_string(),
r#"["low",true]"#.to_string(),
json!(vec!["level"; 11]).to_string(),
],
),
] {
let error = ctx
.sql(&format!(
"SELECT {name}(message, 'prompt', 'not json') FROM (VALUES (NULL), ('text')) AS input(message)"
))
.await
.unwrap()
.collect()
.await
.unwrap_err();
assert!(
error.to_string().contains(&format!("{name} criteria")),
"{error}"
);
for criteria in invalid {
// A later invalid row must fail before even creating a client for the first row.
let error = ctx.sql(&format!(
"SELECT {name}('text', 'prompt', criteria) FROM (VALUES ('{valid}'), ('{criteria}')) AS input(criteria)"
)).await.unwrap().collect().await.unwrap_err();
assert!(
error.to_string().contains(&format!("{name} criteria")),
"{error}"
);
}
}
}
#[tokio::test]
async fn test_ai_choose_and_score_reject_invalid_answers() {
for (name, criteria, answer) in [
(
"ai_choose",
r#"{"billing":null}"#,
json!({"type":"score","choice":"billing"}),
),
(
"ai_choose",
r#"{"billing":null}"#,
json!({"type":"choice","choice":"unknown"}),
),
(
"ai_choose",
r#"{"billing":null}"#,
json!({"type":"choice","choice":1}),
),
("ai_choose", r#"{"billing":null}"#, json!({"type":"choice"})),
(
"ai_score",
r#"["low","medium","high"]"#,
json!({"type":"noul","score":0.5}),
),
(
"ai_score",
r#"["low","medium","high"]"#,
json!({"type":"score","score":2.1}),
),
(
"ai_score",
r#"["low","high"]"#,
json!({"type":"score","score":-0.1}),
),
(
"ai_score",
r#"["low","high"]"#,
json!({"type":"score","score":"NaN"}),
),
("ai_score", r#"["low","high"]"#, json!({"type":"score"})),
] {
let server = MockServer::start(Router::new().route(
"/v1/systemone",
post(move || {
let answer = answer.clone();
async move { Json(json!({"answers":{"matches":answer}})) }
}),
))
.await;
let ctx = context(server.function::<Choice>());
ctx.register_udf(
AsyncScalarUDF::new(Arc::new(server.function::<Score>())).into_scalar_udf(),
);
let error = ctx
.sql(&format!("SELECT {name}('text', 'prompt', '{criteria}')"))
.await
.unwrap()
.collect()
.await
.unwrap_err();
assert!(
error
.to_string()
.contains(&format!("{name} response must contain")),
"{error}"
);
}
}
#[tokio::test]
#[ignore = "calls the real Jev API; requires JEV_API_KEY and GREPTIMEDB_EXPERIMENTAL_JEV=true"]
async fn test_ai_match_live() {
let ctx = SessionContext::new();
let function = FUNCTION_REGISTRY.get_function("ai_match").unwrap();
ctx.register_udf(function.provide(FunctionContext::default()));
let batches = ctx.sql(
"SELECT id FROM (VALUES
(1, 'Payment failed permanently: all three retries exhausted; the payment is still unsuccessful.'),
(2, 'Payment succeeded on the second retry; the payment is complete.'),
(3, 'User logged in successfully.')
) AS events(id, message)
WHERE ai_match(message, 'The event reports that a payment still failed after retries.') >= 0.8
ORDER BY id",
).await.unwrap().collect().await.unwrap();
assert_batches_eq!(["+----+", "| id |", "+----+", "| 1 |", "+----+"], &batches);
}
src/common/function/src/scalars/avg_calc.rs
+266
View File
@@ -0,0 +1,266 @@
// Copyright 2023 Greptime Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Implementation of the scalar function `avg_calc`.
use std::fmt;
use std::fmt::Display;
use std::sync::Arc;
use datafusion_common::arrow::array::{Array, AsArray, Float64Builder};
use datafusion_common::{DataFusionError, ScalarValue};
use datafusion_expr::{ColumnarValue, ScalarFunctionArgs, Signature, Volatility};
use datatypes::arrow::datatypes::DataType;
use crate::aggrs::approximate::avg::AvgState;
use crate::function::Function;
use crate::function_registry::FunctionRegistry;
const NAME: &str = "avg_calc";
/// Calculates an average from a serialized AVG1 state.
#[derive(Debug)]
pub(crate) struct AvgCalcFunction {
signature: Signature,
}
impl AvgCalcFunction {
pub fn register(registry: &FunctionRegistry) {
registry.register_scalar(Self::default());
}
}
impl Default for AvgCalcFunction {
fn default() -> Self {
Self {
signature: Signature::exact(vec![DataType::Binary], Volatility::Immutable),
}
}
}
impl Display for AvgCalcFunction {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
write!(f, "{}", NAME.to_ascii_uppercase())
}
}
impl Function for AvgCalcFunction {
fn name(&self) -> &str {
NAME
}
fn return_type(&self, _: &[DataType]) -> datafusion_common::Result<DataType> {
Ok(DataType::Float64)
}
fn signature(&self) -> &Signature {
&self.signature
}
fn invoke_with_args(
&self,
args: ScalarFunctionArgs,
) -> datafusion_common::Result<ColumnarValue> {
let [arg] = datafusion_common::utils::take_function_args(self.name(), &args.args)?;
match arg {
ColumnarValue::Scalar(ScalarValue::Binary(state)) => {
Ok(ColumnarValue::Scalar(ScalarValue::Float64(
state
.as_deref()
.map(AvgState::decode)
.transpose()?
.and_then(|state| state.average()),
)))
}
ColumnarValue::Scalar(ScalarValue::Null) => {
Ok(ColumnarValue::Scalar(ScalarValue::Float64(None)))
}
ColumnarValue::Array(states) => {
let Some(states) = states.as_binary_opt::<i32>() else {
return Err(invalid_type(self.name(), states.data_type()));
};
let mut builder = Float64Builder::with_capacity(states.len());
for state in states.iter() {
builder.append_option(match state {
Some(state) => AvgState::decode(state)?.average(),
None => None,
});
}
Ok(ColumnarValue::Array(Arc::new(builder.finish())))
}
_ => Err(invalid_type(self.name(), &arg.data_type())),
}
}
}
fn invalid_type(name: &str, data_type: &DataType) -> DataFusionError {
DataFusionError::Execution(format!(
"'{name}' expects argument to be Binary datatype, got {data_type}"
))
}
#[cfg(test)]
mod tests {
use std::sync::Arc;
use arrow_schema::Field;
use datafusion::arrow::array::{Array, AsArray, BinaryArray, Float64Array};
use datafusion::logical_expr::Accumulator;
use datafusion::prelude::SessionContext;
use datafusion_common::arrow::datatypes::Float64Type;
use datafusion_expr::{ColumnarValue, ScalarFunctionArgs};
use super::*;
use crate::aggrs::approximate::avg::AvgAccumulator;
use crate::function::{Function, FunctionContext};
use crate::function_registry::FUNCTION_REGISTRY;
fn produce_state(values: Vec<Option<f64>>) -> Vec<u8> {
let mut accumulator = AvgAccumulator::default();
accumulator
.update_batch(&[Arc::new(Float64Array::from(values))])
.unwrap();
let ScalarValue::Binary(Some(state)) = accumulator.evaluate().unwrap() else {
panic!("AVG state must be binary");
};
state
}
fn invoke(arg: ColumnarValue, number_rows: usize) -> datafusion_common::Result<ColumnarValue> {
AvgCalcFunction::default().invoke_with_args(ScalarFunctionArgs {
args: vec![arg],
arg_fields: vec![],
number_rows,
return_field: Arc::new(Field::new("x", DataType::Float64, true)),
config_options: Arc::new(Default::default()),
})
}
#[test]
fn scalar_and_array_states_decode_to_averages() {
let state = produce_state(vec![Some(1.0), Some(2.0), Some(6.0)]);
let ColumnarValue::Scalar(ScalarValue::Float64(Some(value))) =
invoke(ColumnarValue::Scalar(ScalarValue::Binary(Some(state))), 1).unwrap()
else {
panic!("Expected Float64 scalar");
};
assert_eq!(value, 3.0);
let ColumnarValue::Scalar(ScalarValue::Float64(None)) =
invoke(ColumnarValue::Scalar(ScalarValue::Binary(None)), 1).unwrap()
else {
panic!("Expected NULL Float64 scalar");
};
let empty = produce_state(vec![None]);
let ColumnarValue::Scalar(ScalarValue::Float64(None)) = invoke(
ColumnarValue::Scalar(ScalarValue::Binary(Some(empty.clone()))),
1,
)
.unwrap() else {
panic!("Expected NULL Float64 scalar");
};
let infinity = produce_state(vec![Some(f64::INFINITY)]);
let nan = produce_state(vec![Some(f64::NAN)]);
let ColumnarValue::Array(result) = invoke(
ColumnarValue::Array(Arc::new(BinaryArray::from(vec![
Some(empty.as_slice()),
None,
Some(infinity.as_slice()),
Some(nan.as_slice()),
]))),
4,
)
.unwrap() else {
panic!("Expected Float64 array");
};
let result = result.as_primitive::<Float64Type>();
assert!(result.is_null(0));
assert!(result.is_null(1));
assert_eq!(result.value(2), f64::INFINITY);
assert!(result.value(3).is_nan());
}
#[test]
fn malformed_and_unknown_version_states_fail_the_whole_batch() {
let valid = produce_state(vec![Some(3.0)]);
let mut unknown_version = valid.clone();
unknown_version[..4].copy_from_slice(b"AVG2");
assert!(
invoke(
ColumnarValue::Scalar(ScalarValue::Binary(Some(unknown_version.clone()))),
1,
)
.is_err()
);
assert!(
invoke(
ColumnarValue::Array(Arc::new(BinaryArray::from(vec![
Some(valid.as_slice()),
Some(b"malformed".as_slice()),
Some(unknown_version.as_slice()),
]))),
3,
)
.is_err()
);
}
#[tokio::test]
async fn registry_query_decodes_avg_state_and_weighted_avg_merge() {
let ctx = SessionContext::new();
let avg_calc = FUNCTION_REGISTRY
.get_function(NAME)
.expect("avg_calc must be registered")
.provide(FunctionContext::default());
ctx.register_udf(avg_calc);
for name in ["avg_state", "avg_merge"] {
ctx.register_udaf(
FUNCTION_REGISTRY
.get_aggr_func(name)
.expect("AVG aggregate must be registered"),
);
}
let batches = ctx
.sql(
"SELECT avg_calc(avg_state(CAST(value AS DOUBLE))) FROM \
(VALUES (1.0), (2.0), (6.0)) AS values_table(value)",
)
.await
.unwrap()
.collect()
.await
.unwrap();
let result = batches[0].column(0).as_primitive::<Float64Type>();
assert_eq!(result.value(0), 3.0);
let batches = ctx
.sql(
"WITH states AS (\
SELECT avg_state(CAST(value AS DOUBLE)) AS state FROM (VALUES (1.0), (3.0)) AS left_values(value) \
UNION ALL \
SELECT avg_state(CAST(value AS DOUBLE)) AS state FROM (VALUES (6.0)) AS right_values(value)\
) SELECT avg_calc(avg_merge(state)) FROM states",
)
.await
.unwrap()
.collect()
.await
.unwrap();
let result = batches[0].column(0).as_primitive::<Float64Type>();
assert_eq!(result.value(0), 10.0 / 3.0);
}
}
src/common/grpc-expr/src/alter.rs
+46 -5
View File
@@ -24,21 +24,23 @@ use api::v1::{
SkippingIndexType as PbSkippingIndexType, column_def,
};
use common_query::AddColumnLocation;
use datatypes::json::{JsonSettings, JsonTypeHint};
use datatypes::prelude::ConcreteDataType;
use datatypes::schema::{ColumnSchema, FulltextOptions, Schema, SkippingIndexOptions};
use snafu::{OptionExt, ResultExt, ensure};
use store_api::region_request::{SetRegionOption, UnsetRegionOption};
use table::metadata::{TableId, TableMeta};
use table::requests::{
AddColumnRequest, AlterKind, AlterTableRequest, AnnotationFamily, ModifyColumnTypeRequest,
SetDefaultRequest, SetIndexOption, UnsetIndexOption,
SetDefaultRequest, SetIndexOption, SetJsonSettingsRequest, UnsetIndexOption,
};
use crate::error::{
self, ColumnNotFoundSnafu, InvalidColumnDefSnafu, InvalidIndexOptionSnafu,
InvalidSetFulltextOptionRequestSnafu, InvalidSetSkippingIndexOptionRequestSnafu,
InvalidSetTableOptionRequestSnafu, InvalidUnsetTableOptionRequestSnafu,
MissingAlterIndexOptionSnafu, MissingFieldSnafu, MissingTableMetaSnafu,
MissingTimestampColumnSnafu, Result, UnknownLocationTypeSnafu,
InvalidJsonSettingsSnafu, InvalidSetFulltextOptionRequestSnafu,
InvalidSetSkippingIndexOptionRequestSnafu, InvalidSetTableOptionRequestSnafu,
InvalidUnsetTableOptionRequestSnafu, MissingAlterIndexOptionSnafu, MissingFieldSnafu,
MissingTableMetaSnafu, MissingTimestampColumnSnafu, Result, UnknownLocationTypeSnafu,
};
const LOCATION_TYPE_FIRST: i32 = LocationType::First as i32;
@@ -59,6 +61,34 @@ fn annotation_family_of_keys<'a>(
})
}
fn json_settings_from_proto(settings: api::v1::JsonSettings) -> Result<JsonSettings> {
let type_hints = settings
.type_hints
.into_iter()
.map(|hint| {
let data_type = ConcreteDataType::from(
ColumnDataTypeWrapper::try_new(hint.data_type, hint.datatype_extension)
.context(error::ColumnDataTypeSnafu)?,
);
Ok(JsonTypeHint {
path: hint.path,
data_type,
// Index configuration is not supported yet, so this is temporarily
// hardcoded to false.
inverted_index: false,
})
})
.collect::<Result<Vec<_>>>()?;
JsonSettings::try_new(type_hints, settings.max_auto_expanded_paths).map_err(|err| {
InvalidJsonSettingsSnafu {
err: err.to_string(),
}
.build()
})
}
/// Returns the annotation family when `kind` is a SET/UNSET whose keys all
/// belong to one family — the alters that only rewrite table metadata and skip
/// region dispatch. A mixed batch is an error; never interpret it as "not an
@@ -189,6 +219,17 @@ pub fn alter_expr_to_request(
columns: modify_column_type_requests,
}
}
Kind::SetJsonSettings(set_json_settings) => {
let settings = set_json_settings
.settings
.context(MissingFieldSnafu { field: "settings" })?;
AlterKind::SetJsonSettings {
request: SetJsonSettingsRequest {
column_name: set_json_settings.column_name,
settings: json_settings_from_proto(settings)?,
},
}
}
Kind::DropColumns(DropColumns { drop_columns }) => AlterKind::DropColumns {
names: drop_columns.into_iter().map(|c| c.name).collect(),
},
src/common/grpc-expr/src/error.rs
+8
View File
@@ -73,6 +73,13 @@ pub enum Error {
source: api::error::Error,
},
#[snafu(display("Invalid JSON settings: {}", err))]
InvalidJsonSettings {
err: String,
#[snafu(implicit)]
location: Location,
},
#[snafu(display("Unknown location type: {}", location_type))]
UnknownLocationType {
location_type: i32,
@@ -188,6 +195,7 @@ impl ErrorExt for Error {
| Error::MissingTimestampColumn { .. } => StatusCode::InvalidArguments,
Error::MissingField { .. } => StatusCode::InvalidArguments,
Error::InvalidColumnDef { source, .. } => source.status_code(),
Error::InvalidJsonSettings { .. } => StatusCode::InvalidArguments,
Error::UnknownLocationType { .. } => StatusCode::InvalidArguments,
Error::UnknownColumnDataType { .. } | Error::InvalidFulltextIndexColumnType { .. } => {
src/common/meta/src/datanode.rs
+11 -1
View File
@@ -16,7 +16,7 @@ use std::collections::{HashMap, HashSet};
use std::str::FromStr;
use api::v1::meta::{DatanodeWorkloads, HeartbeatRequest, RequestHeader};
use common_time::util as time_util;
use common_time::{Timestamp, util as time_util};
use lazy_static::lazy_static;
use regex::Regex;
use serde::{Deserialize, Serialize};
@@ -117,6 +117,12 @@ pub struct RegionStat {
/// **Only used by remote WAL prune.**
/// In mito engine, this is the same as `data_topic_latest_entry_id`.
pub metadata_topic_latest_entry_id: u64,
/// The earliest timestamp of the region's current data, if it holds any.
#[serde(default)]
pub min_timestamp: Option<Timestamp>,
/// The latest timestamp of the region's current data, if it holds any.
#[serde(default)]
pub max_timestamp: Option<Timestamp>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
@@ -331,6 +337,8 @@ impl From<&api::v1::meta::RegionStat> for RegionStat {
query_scanned_bytes: region_stat.query_scanned_bytes,
data_topic_latest_entry_id: region_stat.data_topic_latest_entry_id,
metadata_topic_latest_entry_id: region_stat.metadata_topic_latest_entry_id,
min_timestamp: region_stat.min_timestamp,
max_timestamp: region_stat.max_timestamp,
}
}
}
@@ -593,6 +601,8 @@ mod tests {
query_scanned_bytes: 20,
data_topic_latest_entry_id: 0,
metadata_topic_latest_entry_id: 0,
min_timestamp: None,
max_timestamp: None,
}],
..Default::default()
};
src/common/meta/src/ddl/alter_table/executor.rs
+1
View File
@@ -333,6 +333,7 @@ fn build_new_table_info(
}
AlterKind::DropColumns { .. }
| AlterKind::ModifyColumnTypes { .. }
| AlterKind::SetJsonSettings { .. }
| AlterKind::SetTableOptions { .. }
| AlterKind::UnsetTableOptions { .. }
| AlterKind::SetAnnotations { .. }
src/common/meta/src/ddl/alter_table/region_request.rs
+1
View File
@@ -90,6 +90,7 @@ fn create_proto_alter_kind(
})))
}
Kind::ModifyColumnTypes(x) => Ok(Some(alter_request::Kind::ModifyColumnTypes(x.clone()))),
Kind::SetJsonSettings(x) => Ok(Some(alter_request::Kind::SetJsonSettings(x.clone()))),
Kind::DropColumns(x) => {
let drop_columns = x
.drop_columns
src/common/meta/src/ddl/create_table/executor.rs
+13 -1
View File
@@ -182,7 +182,19 @@ impl CreateTableExecutor {
region_wal_options: RegionWalOptions,
) -> Result<()> {
if !column_metadatas.is_empty() {
update_table_info_column_ids(&mut table_info, column_metadatas);
// Region metadata can include engine-internal columns that are not visible in
// the table schema. Only visible columns belong in the table's column IDs.
let visible_column_metadatas = column_metadatas
.iter()
.filter(|column| {
table_info
.meta
.schema
.contains_column(&column.column_schema.name)
})
.cloned()
.collect::<Vec<_>>();
update_table_info_column_ids(&mut table_info, &visible_column_metadatas);
}
let detecting_regions =
convert_region_routes_to_detecting_regions(&table_route.region_routes);
src/common/meta/src/ddl/event/table.rs
+1
View File
@@ -193,6 +193,7 @@ pub(crate) fn alter_table_kind_name(kind: &AlterTableKind) -> Option<&'static st
AlterTableKind::DropColumns(_) => Some("drop_columns"),
AlterTableKind::RenameTable(_) => Some("rename_table"),
AlterTableKind::ModifyColumnTypes(_) => Some("modify_column_types"),
AlterTableKind::SetJsonSettings(_) => Some("set_json_settings"),
AlterTableKind::SetTableOptions(_) => Some("set_table_options"),
AlterTableKind::UnsetTableOptions(_) => Some("unset_table_options"),
AlterTableKind::SetIndex(_) => Some("set_index"),
src/common/query/src/request.rs
+130 -7
View File
@@ -15,20 +15,26 @@
mod base64_serde;
mod initial_remote_dyn_filter_reg;
use std::cell::Cell;
use std::sync::Arc;
use api::v1::region::RegionRequestHeader;
use datafusion::execution::TaskContext;
use datafusion::physical_expr::expressions::{Column, InListExpr, lit};
use datafusion::physical_plan::PhysicalExpr;
use datafusion::physical_plan::joins::HashTableLookupExpr;
use datafusion::physical_plan::{ExecutionPlan, PhysicalExpr};
use datafusion_common::tree_node::{Transformed, TreeNode, TreeNodeRecursion};
use datafusion_common::{DataFusionError, Result as DataFusionResult};
use datafusion_expr::LogicalPlan;
use datafusion_proto::physical_plan::DefaultPhysicalExtensionCodec;
use datafusion_proto::physical_plan::from_proto::parse_physical_expr;
use datafusion_proto::physical_plan::to_proto::serialize_physical_expr;
use datafusion_proto::protobuf::PhysicalExprNode;
use datafusion_proto::physical_plan::to_proto::{
serialize_physical_expr, serialize_physical_expr_with_converter,
};
use datafusion_proto::physical_plan::{
DefaultPhysicalExtensionCodec, DefaultPhysicalProtoConverter, PhysicalExtensionCodec,
PhysicalPlanDecodeContext, PhysicalProtoConverterExtension,
};
use datafusion_proto::protobuf::{PhysicalExprNode, PhysicalPlanNode};
use prost::Message;
use serde::{Deserialize, Serialize};
use snafu::ensure;
@@ -120,6 +126,69 @@ fn encode_physical_expr_to_bytes(expr: &Arc<dyn PhysicalExpr>) -> DataFusionResu
Ok(bytes)
}
struct BoundedPhysicalExprEncoder {
max_payload_bytes: usize,
encoded_bytes: Cell<usize>,
}
impl PhysicalProtoConverterExtension for BoundedPhysicalExprEncoder {
fn proto_to_execution_plan(
&self,
proto: &PhysicalPlanNode,
ctx: &PhysicalPlanDecodeContext<'_>,
) -> DataFusionResult<Arc<dyn ExecutionPlan>> {
DefaultPhysicalProtoConverter {}.proto_to_execution_plan(proto, ctx)
}
fn execution_plan_to_proto(
&self,
plan: &Arc<dyn ExecutionPlan>,
codec: &dyn PhysicalExtensionCodec,
) -> DataFusionResult<PhysicalPlanNode> {
DefaultPhysicalProtoConverter {}.execution_plan_to_proto(plan, codec)
}
fn proto_to_physical_expr(
&self,
proto: &PhysicalExprNode,
input_schema: &datafusion::arrow::datatypes::Schema,
ctx: &PhysicalPlanDecodeContext<'_>,
) -> DataFusionResult<Arc<dyn PhysicalExpr>> {
DefaultPhysicalProtoConverter {}.proto_to_physical_expr(proto, input_schema, ctx)
}
fn physical_expr_to_proto(
&self,
expr: &Arc<dyn PhysicalExpr>,
codec: &dyn PhysicalExtensionCodec,
) -> DataFusionResult<PhysicalExprNode> {
let before_children = self.encoded_bytes.get();
let proto = serialize_physical_expr_with_converter(expr, codec, self)?;
// Replace the children's contribution with the whole subtree, including protobuf framing.
// Checking each child stops large IN lists before materializing all their Arrow IPC literals.
let encoded_bytes = before_children.saturating_add(proto.encoded_len());
self.encoded_bytes.set(encoded_bytes);
validate_payload_size(encoded_bytes, self.max_payload_bytes)
.map_err(DataFusionError::from)?;
Ok(proto)
}
}
fn encode_bounded_physical_expr_to_bytes(
expr: &Arc<dyn PhysicalExpr>,
max_payload_bytes: usize,
) -> Result<Vec<u8>, CommonQueryError> {
let codec = DefaultPhysicalExtensionCodec {};
let encoder = BoundedPhysicalExprEncoder {
max_payload_bytes,
encoded_bytes: Cell::new(0),
};
let proto = encoder.physical_expr_to_proto(expr, &codec);
// Preserve the size error so the caller can retry with bounds only.
validate_payload_size(encoder.encoded_bytes.get(), max_payload_bytes)?;
Ok(proto.map_err(CommonQueryError::from)?.encode_to_vec())
}
fn encode_remote_dyn_filter_expr(
expr: &Arc<dyn PhysicalExpr>,
max_payload_bytes: usize,
@@ -127,9 +196,7 @@ fn encode_remote_dyn_filter_expr(
) -> Result<Vec<u8>, CommonQueryError> {
let expr = portable_remote_dyn_filter_expr(Arc::clone(expr), bounds_only)
.map_err(CommonQueryError::from)?;
let bytes = encode_physical_expr_to_bytes(&expr).map_err(CommonQueryError::from)?;
validate_payload_size(bytes.len(), max_payload_bytes)?;
Ok(bytes)
encode_bounded_physical_expr_to_bytes(&expr, max_payload_bytes)
}
fn portable_remote_dyn_filter_expr(
@@ -522,6 +589,62 @@ mod tests {
assert!(decoded_display.contains("<="));
}
#[test]
fn bounded_encoding_preserves_bytes_and_exact_limit() {
let schema = Schema::new(vec![Field::new("device_id", DataType::Int32, false)]);
let device_id = Arc::new(Column::new("device_id", 0)) as Arc<dyn PhysicalExpr>;
let in_list = Arc::new(
InListExpr::try_new(
Arc::clone(&device_id),
(0..64).map(lit).collect(),
false,
&schema,
)
.unwrap(),
) as Arc<dyn PhysicalExpr>;
let lower_bound = Arc::new(BinaryExpr::new(device_id, Operator::GtEq, lit(10i32)))
as Arc<dyn PhysicalExpr>;
let expr =
Arc::new(BinaryExpr::new(lower_bound, Operator::And, in_list)) as Arc<dyn PhysicalExpr>;
let expected = serialize_physical_expr(&expr, &DefaultPhysicalExtensionCodec {})
.unwrap()
.encode_to_vec();
assert_eq!(
encode_bounded_physical_expr_to_bytes(&expr, expected.len()).unwrap(),
expected
);
assert!(matches!(
encode_bounded_physical_expr_to_bytes(&expr, expected.len() - 1),
Err(CommonQueryError::DynFilterPayloadTooLarge { .. })
));
}
#[test]
fn bounded_encoding_stops_before_later_children() {
let schema = Schema::new(vec![Field::new("host", DataType::Utf8, false)]);
let expr = Arc::new(
InListExpr::try_new(
Arc::new(Column::new("host", 0)),
(0..8).map(|i| lit(i.to_string().repeat(1024))).collect(),
false,
&schema,
)
.unwrap(),
) as Arc<dyn PhysicalExpr>;
let full_size = encode_physical_expr_to_bytes(&expr).unwrap().len();
assert!(full_size > 8192);
let error = encode_bounded_physical_expr_to_bytes(&expr, 2048).unwrap_err();
let CommonQueryError::DynFilterPayloadTooLarge {
payload_size_bytes, ..
} = error
else {
panic!("expected payload size error, got {error:?}");
};
assert!((2048..4096).contains(&payload_size_bytes));
}
#[test]
fn dyn_filter_payload_rejects_oversized_payload() {
let expr: Arc<dyn PhysicalExpr> = Arc::new(Column::new("host", 0));
src/common/telemetry/src/logging.rs
+281 -34
View File
@@ -18,7 +18,8 @@ mod file_retention;
use std::collections::HashMap;
use std::env;
use std::io::IsTerminal;
use std::sync::{Arc, Mutex, Once, RwLock};
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::{Arc, Mutex, Once};
use std::time::Duration;
use common_base::readable_size::ReadableSize;
@@ -60,35 +61,55 @@ pub static LOG_RELOAD_HANDLE: OnceCell<tracing_subscriber::reload::Handle<Target
type DynSubscriber = Layered<tracing_subscriber::reload::Layer<Targets, Registry>, Registry>;
type OtelTraceLayer = tracing_opentelemetry::OpenTelemetryLayer<DynSubscriber, Tracer>;
struct TraceLayerState {
enabled: AtomicBool,
layer: OnceCell<OtelTraceLayer>,
}
#[derive(Clone)]
pub struct TraceReloadHandle {
inner: Arc<RwLock<Option<OtelTraceLayer>>>,
inner: Arc<TraceLayerState>,
}
impl TraceReloadHandle {
fn new(inner: Arc<RwLock<Option<OtelTraceLayer>>>) -> Self {
fn new(inner: Arc<TraceLayerState>) -> Self {
Self { inner }
}
pub fn reload(&self, new_layer: Option<OtelTraceLayer>) {
let mut guard = self.inner.write().unwrap();
*guard = new_layer;
drop(guard);
/// Enables or disables OTLP data collection, initializing it on first enable.
/// Disabling stops new spans, events, fields and links. Existing spans still
/// finish their lifecycle and export on close.
pub fn set_enabled(&self, enabled: bool) -> Result<(), &'static str> {
self.set_enabled_with(enabled, || {
get_or_init_tracer().map(|tracer| tracing_opentelemetry::layer().with_tracer(tracer))
})
}
fn set_enabled_with(
&self,
enabled: bool,
init: impl FnOnce() -> Result<OtelTraceLayer, &'static str>,
) -> Result<(), &'static str> {
if enabled {
self.inner.layer.get_or_try_init(init)?;
}
self.inner.enabled.store(enabled, Ordering::Release);
callsite::rebuild_interest_cache();
Ok(())
}
}
/// A tracing layer that can be dynamically reloaded.
///
/// Mostly copied from [`tracing_subscriber::reload::Layer`].
/// An OTLP layer with a runtime switch and a stable address for downcasts.
struct TraceLayer {
inner: Arc<RwLock<Option<OtelTraceLayer>>>,
inner: Arc<TraceLayerState>,
}
impl TraceLayer {
fn new(initial: Option<OtelTraceLayer>) -> (Self, TraceReloadHandle) {
let inner = Arc::new(RwLock::new(initial));
let inner = Arc::new(TraceLayerState {
enabled: AtomicBool::new(initial.is_some()),
layer: initial.map(OnceCell::with_value).unwrap_or_default(),
});
(
Self {
inner: inner.clone(),
@@ -98,17 +119,11 @@ impl TraceLayer {
}
fn with_layer<R>(&self, f: impl FnOnce(&OtelTraceLayer) -> R) -> Option<R> {
self.inner
.read()
.ok()
.and_then(|guard| guard.as_ref().map(f))
self.inner.layer.get().map(f)
}
fn with_layer_mut<R>(&self, f: impl FnOnce(&mut OtelTraceLayer) -> R) -> Option<R> {
self.inner
.write()
.ok()
.and_then(|mut guard| guard.as_mut().map(f))
fn is_enabled(&self) -> bool {
self.inner.enabled.load(Ordering::Acquire)
}
}
@@ -117,10 +132,6 @@ impl tracing_subscriber::Layer<DynSubscriber> for TraceLayer {
let _ = self.with_layer(|layer| layer.on_register_dispatch(subscriber));
}
fn on_layer(&mut self, subscriber: &mut DynSubscriber) {
let _ = self.with_layer_mut(|layer| layer.on_layer(subscriber));
}
fn register_callsite(
&self,
metadata: &'static tracing::Metadata<'static>,
@@ -144,7 +155,9 @@ impl tracing_subscriber::Layer<DynSubscriber> for TraceLayer {
id: &tracing::span::Id,
ctx: tracing_subscriber::layer::Context<'_, DynSubscriber>,
) {
let _ = self.with_layer(|layer| layer.on_new_span(attrs, id, ctx));
if self.is_enabled() {
let _ = self.with_layer(|layer| layer.on_new_span(attrs, id, ctx));
}
}
fn max_level_hint(&self) -> Option<LevelFilter> {
@@ -157,7 +170,9 @@ impl tracing_subscriber::Layer<DynSubscriber> for TraceLayer {
values: &tracing::span::Record<'_>,
ctx: tracing_subscriber::layer::Context<'_, DynSubscriber>,
) {
let _ = self.with_layer(|layer| layer.on_record(span, values, ctx));
if self.is_enabled() {
let _ = self.with_layer(|layer| layer.on_record(span, values, ctx));
}
}
fn on_follows_from(
@@ -166,7 +181,9 @@ impl tracing_subscriber::Layer<DynSubscriber> for TraceLayer {
follows: &tracing::span::Id,
ctx: tracing_subscriber::layer::Context<'_, DynSubscriber>,
) {
let _ = self.with_layer(|layer| layer.on_follows_from(span, follows, ctx));
if self.is_enabled() {
let _ = self.with_layer(|layer| layer.on_follows_from(span, follows, ctx));
}
}
fn event_enabled(
@@ -183,7 +200,9 @@ impl tracing_subscriber::Layer<DynSubscriber> for TraceLayer {
event: &tracing::Event<'_>,
ctx: tracing_subscriber::layer::Context<'_, DynSubscriber>,
) {
let _ = self.with_layer(|layer| layer.on_event(event, ctx));
if self.is_enabled() {
let _ = self.with_layer(|layer| layer.on_event(event, ctx));
}
}
fn on_enter(
@@ -220,11 +239,13 @@ impl tracing_subscriber::Layer<DynSubscriber> for TraceLayer {
}
unsafe fn downcast_raw(&self, id: std::any::TypeId) -> Option<*const ()> {
self.inner.read().ok().and_then(|guard| {
guard
.as_ref()
.and_then(|layer| unsafe { layer.downcast_raw(id) })
})
// Keep downcasts available while disabled: an in-flight WithContext
// callback may still need the layer. OnceCell keeps both addresses valid
// for the subscriber's lifetime, even across concurrent toggles.
self.inner
.layer
.get()
.and_then(|layer| unsafe { layer.downcast_raw(id) })
}
}
@@ -764,8 +785,234 @@ where
#[cfg(test)]
mod tests {
use std::sync::Barrier;
use std::sync::atomic::AtomicUsize;
use opentelemetry::trace::TraceContextExt;
use opentelemetry_sdk::trace::SdkTracerProvider;
use tracing_opentelemetry::OpenTelemetrySpanExt;
use super::*;
#[test]
fn test_trace_switch_preserves_active_spans() {
let provider = SdkTracerProvider::builder()
.with_sampler(Sampler::AlwaysOn)
.build();
for initially_enabled in [false, true] {
let new_layer =
|| tracing_opentelemetry::layer().with_tracer(provider.tracer("switch"));
let (filter, _) = tracing_subscriber::reload::Layer::new(
Targets::new().with_default(tracing::Level::INFO),
);
let (layer, handle) = TraceLayer::new(initially_enabled.then(new_layer));
let dispatch = tracing::Dispatch::new(Registry::default().with(filter).with(layer));
tracing::dispatcher::with_default(&dispatch, || {
let initial = tracing::info_span!("initial");
assert!(!initial.is_disabled());
assert_eq!(
initial.context().span().span_context().is_valid(),
initially_enabled
);
handle.set_enabled_with(true, || Ok(new_layer())).unwrap();
let parent = tracing::info_span!("parent");
let parent_context = parent.context();
assert!(parent_context.span().span_context().is_valid());
let previous_context = opentelemetry::Context::current();
let entered = parent.enter();
handle.set_enabled(false).unwrap();
let disabled = tracing::info_span!("disabled");
assert!(!disabled.is_disabled());
assert!(!disabled.context().span().span_context().is_valid());
assert_eq!(
parent.context().span().span_context(),
parent_context.span().span_context()
);
assert!(dispatch.downcast_ref::<OtelTraceLayer>().is_some());
drop(entered);
assert_eq!(
opentelemetry::Context::current().span().span_context(),
previous_context.span().span_context()
);
let disabled_entered = disabled.enter();
handle
.set_enabled_with(true, || panic!("layer must not be replaced"))
.unwrap();
drop(disabled_entered);
let child = tracing::info_span!(parent: &parent, "child");
assert_eq!(
child.context().span().span_context().trace_id(),
parent_context.span().span_context().trace_id()
);
assert!(!disabled.context().span().span_context().is_valid());
});
}
}
#[test]
fn test_trace_switch_retries_initialization() {
let (filter, _) = tracing_subscriber::reload::Layer::new(
Targets::new().with_default(tracing::Level::INFO),
);
let (layer, handle) = TraceLayer::new(None);
let subscriber = Registry::default().with(filter).with(layer);
tracing::subscriber::with_default(subscriber, || {
handle
.set_enabled_with(false, || panic!("disabling must not initialize OTLP"))
.unwrap();
assert_eq!(
handle.set_enabled_with(true, || Err("initialization failed")),
Err("initialization failed")
);
let failed = tracing::info_span!("after_failed_enable");
assert!(!failed.context().span().span_context().is_valid());
let provider = SdkTracerProvider::builder()
.with_sampler(Sampler::AlwaysOn)
.build();
handle
.set_enabled_with(true, || {
Ok(tracing_opentelemetry::layer().with_tracer(provider.tracer("retry")))
})
.unwrap();
let enabled = tracing::info_span!("after_successful_enable");
assert!(enabled.context().span().span_context().is_valid());
});
}
#[test]
fn test_trace_switch_stops_collecting_fields_when_disabled() {
struct CountFormatting(AtomicUsize);
impl std::fmt::Debug for CountFormatting {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
self.0.fetch_add(1, Ordering::Relaxed);
f.write_str("value")
}
}
let value = CountFormatting(AtomicUsize::new(0));
let provider = SdkTracerProvider::builder()
.with_sampler(Sampler::AlwaysOn)
.build();
let (filter, _) = tracing_subscriber::reload::Layer::new(
Targets::new().with_default(tracing::Level::INFO),
);
let (layer, handle) = TraceLayer::new(Some(
tracing_opentelemetry::layer().with_tracer(provider.tracer("fields")),
));
tracing::subscriber::with_default(Registry::default().with(filter).with(layer), || {
let admitted = tracing::info_span!("admitted", field = tracing::field::Empty);
admitted.record("field", tracing::field::debug(&value));
admitted.in_scope(|| tracing::info!(value = ?value));
assert_eq!(value.0.load(Ordering::Relaxed), 2);
handle.set_enabled(false).unwrap();
let unadmitted = tracing::info_span!("unadmitted", field = tracing::field::Empty);
for span in [&admitted, &unadmitted] {
span.record("field", tracing::field::debug(&value));
span.in_scope(|| tracing::info!(value = ?value));
}
assert_eq!(value.0.load(Ordering::Relaxed), 2);
handle
.set_enabled_with(true, || panic!("layer must not be replaced"))
.unwrap();
admitted.record("field", tracing::field::debug(&value));
admitted.in_scope(|| tracing::info!(value = ?value));
assert_eq!(value.0.load(Ordering::Relaxed), 4);
});
}
#[test]
fn test_trace_switch_finishes_admitted_spans() {
#[derive(Debug)]
struct Exporter(std::sync::mpsc::Sender<String>);
impl opentelemetry_sdk::trace::SpanExporter for Exporter {
async fn export(
&self,
batch: Vec<opentelemetry_sdk::trace::SpanData>,
) -> opentelemetry_sdk::error::OTelSdkResult {
for span in batch {
self.0.send(span.name.into_owned()).unwrap();
}
Ok(())
}
}
let (sender, receiver) = std::sync::mpsc::channel();
let provider = SdkTracerProvider::builder()
.with_sampler(Sampler::AlwaysOn)
.with_simple_exporter(Exporter(sender))
.build();
let (filter, _) = tracing_subscriber::reload::Layer::new(
Targets::new().with_default(tracing::Level::INFO),
);
let (layer, handle) = TraceLayer::new(Some(
tracing_opentelemetry::layer().with_tracer(provider.tracer("export")),
));
tracing::subscriber::with_default(Registry::default().with(filter).with(layer), || {
let active = tracing::info_span!("admitted");
handle.set_enabled(false).unwrap();
let disabled = tracing::info_span!("not_admitted");
drop(active);
drop(disabled);
});
provider.force_flush().unwrap();
assert_eq!(receiver.try_iter().collect::<Vec<_>>(), ["admitted"]);
}
#[test]
fn test_trace_switch_concurrent_context_access() {
let provider = SdkTracerProvider::builder()
.with_sampler(Sampler::AlwaysOn)
.build();
let (filter, _) = tracing_subscriber::reload::Layer::new(
Targets::new().with_default(tracing::Level::INFO),
);
let (layer, handle) = TraceLayer::new(Some(
tracing_opentelemetry::layer().with_tracer(provider.tracer("concurrent")),
));
let dispatch = tracing::Dispatch::new(Registry::default().with(filter).with(layer));
let parent = tracing::dispatcher::with_default(&dispatch, || {
tracing::info_span!("concurrent_parent")
});
let context = parent.context();
let barrier = Barrier::new(4);
std::thread::scope(|scope| {
for _ in 0..3 {
scope.spawn(|| {
tracing::dispatcher::with_default(&dispatch, || {
let _entered = parent.enter();
barrier.wait();
for _ in 0..500 {
assert_eq!(
parent.context().span().span_context(),
context.span().span_context()
);
{
let child = tracing::info_span!("concurrent_child");
let _entered = child.enter();
let _ = child.context();
tracing::info!("concurrent event");
}
assert_eq!(
opentelemetry::Context::current().span().span_context(),
context.span().span_context()
);
}
});
});
}
barrier.wait();
for i in 0..1000 {
handle
.set_enabled_with(i % 2 == 0, || panic!("layer must not be replaced"))
.unwrap();
}
});
}
#[test]
fn test_logging_options_deserialization_default() {
let json = r#"{}"#;
src/datanode/src/datanode.rs
+103 -6
View File
@@ -14,11 +14,13 @@
//! Datanode implementation.
use std::collections::HashMap;
use std::path::Path;
use std::sync::Arc;
use std::time::{Duration, Instant};
use common_base::Plugins;
use common_catalog::consts::{FILE_ENGINE, METRIC_ENGINE, MITO_ENGINE};
use common_datasource::object_store::LocalFileAccess;
use common_error::ext::BoxedError;
use common_greptimedb_telemetry::GreptimeDBTelemetryTask;
@@ -63,9 +65,10 @@ use tokio::sync::Notify;
use crate::config::{DatanodeOptions, RegionEngineConfig, StorageConfig};
use crate::error::{
self, BuildDatanodeSnafu, BuildMetricEngineSnafu, BuildMitoEngineSnafu, CreateDirSnafu,
DataFusionSnafu, GetMetadataSnafu, InvalidObjectStoreWalConfigSnafu, MissingCacheSnafu,
MissingNodeIdSnafu, ObjectStoreWalNotSupportedSnafu, OpenLogStoreSnafu, Result,
ShutdownInstanceSnafu, ShutdownServerSnafu, StartServerSnafu,
DataFusionSnafu, DuplicateRegionEngineConfigSnafu, GetMetadataSnafu,
InvalidObjectStoreWalConfigSnafu, MissingCacheSnafu, MissingNodeIdSnafu,
ObjectStoreWalNotSupportedSnafu, OpenLogStoreSnafu, Result, ShutdownInstanceSnafu,
ShutdownServerSnafu, StartServerSnafu,
};
use crate::event_listener::{
NoopRegionServerEventListener, RegionServerEventListenerRef, RegionServerEventReceiver,
@@ -253,6 +256,7 @@ impl DatanodeBuilder {
}
pub async fn build(mut self) -> Result<Datanode> {
validate_region_engine_config(&self.opts.region_engine)?;
let node_id = self.opts.node_id.context(MissingNodeIdSnafu)?;
set_default_prefix(self.opts.default_column_prefix.as_deref())
.map_err(BoxedError::new)
@@ -710,6 +714,27 @@ impl DatanodeBuilder {
}
}
/// Rejects repeated engine types before their configs can silently overwrite each other.
fn validate_region_engine_config(configs: &[RegionEngineConfig]) -> Result<()> {
let mut engine_indices = HashMap::new();
for (index, config) in configs.iter().enumerate() {
let engine = match config {
RegionEngineConfig::Mito(_) => MITO_ENGINE,
RegionEngineConfig::File(_) => FILE_ENGINE,
RegionEngineConfig::Metric(_) => METRIC_ENGINE,
};
if let Some(first_index) = engine_indices.insert(engine, index) {
return DuplicateRegionEngineConfigSnafu {
engine,
first_index,
duplicate_index: index,
}
.fail();
}
}
Ok(())
}
/// Rejects an object store WAL config the log store cannot run on.
fn validate_object_store_wal_config(config: &ObjectStoreWalConfig) -> Result<()> {
let prefix = config.prefix.trim();
@@ -879,6 +904,7 @@ async fn open_all_regions(
mod tests {
use std::assert_matches;
use std::collections::{BTreeMap, HashMap};
use std::io::Write;
use std::path::Path;
use std::sync::Arc;
use std::time::Duration;
@@ -886,6 +912,7 @@ mod tests {
use cache::build_datanode_cache_registry;
use common_base::Plugins;
use common_base::readable_size::ReadableSize;
use common_config::Configurable;
use common_error::ext::ErrorExt;
use common_error::status_code::StatusCode;
use common_meta::cache::LayeredCacheRegistryBuilder;
@@ -893,15 +920,17 @@ mod tests {
use common_meta::key::datanode_table::DatanodeTableManager;
use common_meta::kv_backend::KvBackendRef;
use common_meta::kv_backend::memory::MemoryKvBackend;
use common_test_util::temp_dir::create_temp_dir;
use common_test_util::temp_dir::{create_named_temp_file, create_temp_dir};
use common_wal::config::DatanodeWalConfig;
use common_wal::config::object_store::{ObjectStoreWalConfig, STANDALONE_GENERATION};
use mito2::engine::MITO_ENGINE_NAME;
use store_api::region_request::RegionRequest;
use store_api::storage::RegionId;
use crate::config::DatanodeOptions;
use crate::datanode::{DatanodeBuilder, validate_object_store_wal_config};
use crate::config::{DatanodeOptions, RegionEngineConfig};
use crate::datanode::{
DatanodeBuilder, validate_object_store_wal_config, validate_region_engine_config,
};
use crate::error::Error;
use crate::tests::{MockRegionEngine, mock_region_server};
@@ -1014,6 +1043,74 @@ mod tests {
std::fs::read_dir(dir).unwrap().next().is_none()
}
#[test]
fn test_validate_unique_region_engine_config() {
let mut opts = DatanodeOptions::default();
validate_region_engine_config(&opts.region_engine).unwrap();
opts.region_engine
.push(RegionEngineConfig::Metric(Default::default()));
validate_region_engine_config(&opts.region_engine).unwrap();
// Omitted engines still use their defaults during engine construction.
validate_region_engine_config(&[]).unwrap();
}
#[tokio::test]
async fn test_build_rejects_duplicate_region_engine_config() {
for (engine, first_index) in [("mito", 0), ("file", 1), ("metric", 2)] {
let mut file = create_named_temp_file();
write!(
file,
r#"
[[region_engine]]
[region_engine.mito]
global_write_buffer_size = "4GiB"
min_compaction_interval = "600s"
[[region_engine]]
[region_engine.file]
[[region_engine]]
[region_engine.metric]
[[region_engine]]
[region_engine.{engine}]
"#
)
.unwrap();
let opts = DatanodeOptions::load_layered_options(
Some(file.path().to_str().unwrap()),
"DATANODE_DUPLICATE_REGION_ENGINE_UT",
)
.unwrap();
let data_home = create_temp_dir("duplicate-region-engine-config");
let mut builder = datanode_builder(
data_home.path().to_str().unwrap(),
DatanodeWalConfig::default(),
Arc::new(MemoryKvBackend::new()),
);
builder.opts.region_engine = opts.region_engine;
let err = build_err(builder).await;
let Error::DuplicateRegionEngineConfig {
engine: actual_engine,
first_index: actual_first_index,
duplicate_index,
..
} = &err
else {
panic!("unexpected error for {engine}: {err:?}");
};
assert_eq!(*actual_engine, engine);
assert_eq!(*actual_first_index, first_index);
assert_eq!(*duplicate_index, 3);
assert_eq!(err.status_code(), StatusCode::InvalidArguments);
// Rejected before the builder creates any storage or log store.
assert!(is_empty_dir(data_home.path()));
}
}
#[tokio::test]
async fn test_build_rejects_invalid_object_store_wal_config() {
let cases = [
src/datanode/src/error.rs
+12
View File
@@ -299,6 +299,17 @@ pub enum Error {
location: Location,
},
#[snafu(display(
"Duplicate region engine config '{engine}' at region_engine[{first_index}] and region_engine[{duplicate_index}] (indices are zero-based); each engine type may be configured only once"
))]
DuplicateRegionEngineConfig {
engine: &'static str,
first_index: usize,
duplicate_index: usize,
#[snafu(implicit)]
location: Location,
},
#[snafu(display("Invalid object store WAL config, {} = {:?}: {}", field, value, reason))]
InvalidObjectStoreWalConfig {
field: &'static str,
@@ -486,6 +497,7 @@ impl ErrorExt for Error {
| GcConfigMismatch { .. }
| ParseAddr { .. }
| TomlFormat { .. }
| DuplicateRegionEngineConfig { .. }
| InvalidObjectStoreWalConfig { .. }
| BuildDatanode { .. } => StatusCode::InvalidArguments,
src/datanode/src/metrics.rs
+4 -2
View File
@@ -17,6 +17,7 @@ use prometheus::*;
/// Region request type label.
pub const REGION_REQUEST_TYPE: &str = "datanode_region_request_type";
const REGION_REQUEST_CODE: &str = "code";
pub const REGION_ROLE: &str = "region_role";
pub const REGION_ID: &str = "region_id";
@@ -84,11 +85,12 @@ lazy_static! {
)
.unwrap();
/// Total count of failed insert requests to region server.
/// Total count of failed insert requests to region server, labeled with request
/// type and status code.
pub static ref REGION_SERVER_INSERT_FAIL_COUNT: IntCounterVec = register_int_counter_vec!(
"greptime_datanode_region_failed_insert_count",
"failed region server insert requests count",
&[REGION_REQUEST_TYPE]
&[REGION_REQUEST_TYPE, REGION_REQUEST_CODE]
)
.unwrap();
src/datanode/src/region_server.rs
+2 -2
View File
@@ -687,7 +687,7 @@ impl RegionServer {
}
Err(err) => {
crate::metrics::REGION_SERVER_INSERT_FAIL_COUNT
.with_label_values(&[request_type])
.with_label_values(&[request_type, err.status_code().as_ref()])
.inc_by(batch_size as u64);
Err(err)
}
@@ -1640,7 +1640,7 @@ impl RegionServerInner {
Err(err) => {
if matches!(region_change, RegionChange::Ingest) {
crate::metrics::REGION_SERVER_INSERT_FAIL_COUNT
.with_label_values(&[request_type])
.with_label_values(&[request_type, err.status_code().as_ref()])
.inc();
}
// Removes the region status if the operation fails.
src/datatypes/src/data_type.rs
+3 -1
View File
@@ -492,7 +492,9 @@ impl TryFrom<&ArrowDataType> for ConcreteDataType {
ArrowDataType::Decimal128(precision, scale) => {
ConcreteDataType::decimal128_datatype(*precision, *scale)
}
ArrowDataType::Struct(fields) => ConcreteDataType::Struct(StructType::from(fields)),
ArrowDataType::Struct(fields) => {
ConcreteDataType::Struct(StructType::try_from_arrow_fields(fields)?)
}
ArrowDataType::Float16
| ArrowDataType::Date64
| ArrowDataType::FixedSizeBinary(_)
src/datatypes/src/extension/json.rs
+53 -2
View File
@@ -24,9 +24,10 @@ use parquet_variant_compute::VariantType;
use serde::{Deserialize, Serialize};
use snafu::{ResultExt, ensure};
use crate::error::InvalidJson2LayoutSnafu;
use crate::error::{InvalidJson2LayoutSnafu, SerializeSnafu};
pub use crate::json::JSON2_REMAINDER_FIELD_NAME;
use crate::json::JsonSettings;
use crate::schema::Metadata;
/// Aligns JSON2 field types with their built arrays while preserving field and schema metadata.
pub fn align_schema_with_json_array(schema: SchemaRef, columns: &[ArrayRef]) -> SchemaRef {
@@ -151,7 +152,7 @@ pub struct JsonMetadata {
}
impl JsonMetadata {
/// Creates metadata for the JSON2 layout version 2.
/// Creates metadata for the latest JSON2 layout (currently V2).
pub fn new(json_settings: JsonSettings) -> Self {
Self {
json_settings,
@@ -300,6 +301,27 @@ impl ExtensionType for Json2ExtensionType {
}
}
/// Returns JSON2 column metadata with updated settings and the latest layout.
///
/// Existing SSTs retain their own physical layout metadata.
pub fn json2_metadata_with_updated_settings(
current_metadata: &Metadata,
settings: JsonSettings,
) -> crate::error::Result<Metadata> {
let json_metadata = JsonMetadata::new(settings);
let mut metadata = current_metadata.clone();
metadata.insert(
EXTENSION_TYPE_NAME_KEY.to_string(),
Json2ExtensionType::NAME.to_string(),
);
metadata.insert(
EXTENSION_TYPE_METADATA_KEY.to_string(),
serde_json::to_string(&json_metadata).context(SerializeSnafu)?,
);
Ok(metadata)
}
/// Checks whether this field is either a legacy JSONB or JSON2 extension type.
pub fn is_any_json_extension_type<T: AsRef<Field>>(field: T) -> bool {
let name = field.as_ref().extension_type_name();
@@ -442,6 +464,35 @@ mod tests {
Ok(())
}
#[test]
fn test_json2_metadata_with_updated_settings_upgrades_layout() -> crate::error::Result<()> {
for (name, json) in [
(JsonExtensionType::NAME, r#"{"json_settings":{}}"#),
(Json2ExtensionType::NAME, r#"{"json_settings":{}}"#),
(
Json2ExtensionType::NAME,
r#"{"json_settings":{},"layout_version":2}"#,
),
] {
let metadata = HashMap::from([
(EXTENSION_TYPE_NAME_KEY.to_string(), name.to_string()),
(EXTENSION_TYPE_METADATA_KEY.to_string(), json.to_string()),
("other".to_string(), "kept".to_string()),
]);
let settings = JsonSettings::try_new(vec![], Some(10))?;
let updated = json2_metadata_with_updated_settings(&metadata, settings.clone())?;
assert_eq!(Some("kept"), updated.get("other").map(String::as_str));
assert_eq!(
Some(Json2ExtensionType::NAME),
updated.get(EXTENSION_TYPE_NAME_KEY).map(String::as_str)
);
let json_metadata: JsonMetadata =
serde_json::from_str(updated.get(EXTENSION_TYPE_METADATA_KEY).unwrap()).unwrap();
assert_eq!(JsonMetadata::new(settings), json_metadata);
}
Ok(())
}
#[test]
fn test_parse_json2_physical_layout() -> crate::error::Result<()> {
let legacy = Field::new("data", DataType::Struct(Fields::empty()), true)
src/datatypes/src/json.rs
+346 -143
View File
@@ -23,6 +23,7 @@ pub mod value;
use std::collections::BTreeMap;
use std::collections::btree_map::Entry;
use std::sync::Arc;
use serde::{Deserialize, Serialize};
use serde_json::{Map, Value as Json};
@@ -31,7 +32,7 @@ use snafu::ResultExt;
use crate::data_type::ConcreteDataType;
use crate::error::{self, InvalidJson2SettingsSnafu, Result, UnsupportedJsonTypeSnafu};
use crate::json::value::{JsonValue, JsonVariant, encode_serde_json_as_jsonb};
use crate::schema::ColumnDefaultConstraint;
use crate::prelude::DataType as _;
use crate::types::json_type::{JsonNativeType, JsonObjectType};
use crate::value::{ListValue, StructValue, Value};
@@ -91,12 +92,18 @@ pub struct JsonTypeHint {
pub path: Vec<String>,
#[serde(rename = "type")]
pub data_type: ConcreteDataType,
pub nullable: bool,
#[serde(skip_serializing_if = "Option::is_none")]
pub default_constraint: Option<ColumnDefaultConstraint>,
pub inverted_index: bool,
}
/// Specifies how encoding handles a value that does not match its JSON2 type hint.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum TypeHintMismatchPolicy {
/// Returns an error for a type hint mismatch.
Reject,
/// Coerces to the hint type, encoding unconvertible values as null.
CoerceOrNull,
}
/// Context for JSON encoding/decoding that tracks the current key path.
#[derive(Clone, Debug)]
pub struct JsonContext<'a> {
@@ -139,6 +146,30 @@ impl JsonSettings {
(self.type_hints, self.max_auto_expanded_paths)
}
/// Returns whether the settings are equivalent, ignoring type hint order.
pub fn equivalent(&self, other: &Self) -> bool {
let Self {
type_hints,
max_auto_expanded_paths,
} = self;
let Self {
type_hints: other_hints,
max_auto_expanded_paths: other_max_auto_expanded_paths,
} = other;
if max_auto_expanded_paths != other_max_auto_expanded_paths
|| type_hints.len() != other_hints.len()
{
return false;
}
let mut hints = type_hints.iter().collect::<Vec<_>>();
let mut other_hints = other_hints.iter().collect::<Vec<_>>();
hints.sort_unstable_by(|a, b| a.path.cmp(&b.path));
other_hints.sort_unstable_by(|a, b| a.path.cmp(&b.path));
hints == other_hints
}
/// Decode an encoded StructValue back into a serde_json::Value.
pub fn decode(&self, value: Value) -> Result<Json> {
let mut context = JsonContext {
@@ -150,6 +181,15 @@ impl JsonSettings {
/// Encode a serde_json::Value into a Value::Json using current settings.
pub fn encode(&self, json: Json) -> Result<Value> {
self.encode_with_type_hint_mismatch_policy(json, TypeHintMismatchPolicy::Reject)
}
/// Encodes a serde_json::Value using the given type hint mismatch policy.
pub fn encode_with_type_hint_mismatch_policy(
&self,
json: Json,
policy: TypeHintMismatchPolicy,
) -> Result<Value> {
if let Json::Object(object) = &json
&& object.contains_key(JSON2_REMAINDER_FIELD_NAME)
{
@@ -164,7 +204,7 @@ impl JsonSettings {
path: Vec::new(),
settings: self,
};
encode_json_with_context(json, &mut context).map(|v| Value::Json(Box::new(v)))
encode_json_with_context(json, &mut context, policy).map(|v| Value::Json(Box::new(v)))
}
}
@@ -217,20 +257,6 @@ fn validate_type_hints(type_hints: &[JsonTypeHint]) -> Result<()> {
.fail();
}
};
let non_finite_default = match &hint.default_constraint {
Some(ColumnDefaultConstraint::Value(Value::Float32(value))) => !value.0.is_finite(),
Some(ColumnDefaultConstraint::Value(Value::Float64(value))) => !value.0.is_finite(),
_ => false,
};
if non_finite_default {
return InvalidJson2SettingsSnafu {
reason: format!(
"JSON2 type hint default for '{}' must be finite",
hint.path.join(".")
),
}
.fail();
}
validate_type_hint(&mut object, &hint.path, data_type)?;
}
Ok(())
@@ -290,50 +316,56 @@ fn with_key_context<T>(
}
/// Main encoding function with key path tracking
fn encode_json_with_context(json: Json, context: &mut JsonContext) -> Result<JsonValue> {
fn encode_json_with_context(
json: Json,
context: &mut JsonContext,
policy: TypeHintMismatchPolicy,
) -> Result<JsonValue> {
if context.path.is_empty() && !matches!(json, Json::Object(_)) {
return UnsupportedJsonTypeSnafu.fail();
}
match json {
Json::Object(json_object) => encode_json_object_with_context(json_object, context),
Json::Array(json_array) => encode_json_array_with_context(json_array, context),
_ => encode_json_value_with_context(json, context),
Json::Object(json_object) => encode_json_object_with_context(json_object, context, policy),
Json::Array(json_array) => encode_json_array_with_context(json_array, context, policy),
_ => encode_json_value_with_context(json, context, policy),
}
}
fn encode_json_object_with_context<'a>(
json_object: Map<String, Json>,
context: &mut JsonContext<'a>,
policy: TypeHintMismatchPolicy,
) -> Result<JsonValue> {
let mut object = BTreeMap::new();
for (key, value) in json_object {
let value = with_key_context(context, &key, |context| {
if let Some(hint) = context.type_hint() {
encode_json_value_with_hint(value, hint, context)
encode_json_value_with_hint(value, hint, context, policy)
} else {
encode_json_value_with_context(value, context)
encode_json_value_with_context(value, context, policy)
}
})?;
object.insert(key, value.into_variant());
}
apply_missing_type_hints(&mut object, context)?;
fill_missing_type_hints(&mut object, context, policy)?;
Ok(JsonValue::new(JsonVariant::Object(object)))
}
fn apply_missing_type_hints(
fn fill_missing_type_hints(
object: &mut BTreeMap<String, JsonVariant>,
context: &mut JsonContext,
policy: TypeHintMismatchPolicy,
) -> Result<()> {
for hint in &context.settings.type_hints {
if hint.path.len() > context.path.len() && hint.path.starts_with(&context.path) {
let depth = context.path.len();
let key = &hint.path[depth];
with_key_context(context, key, |context| {
insert_missing_type_hint(object, context, hint, depth)
insert_missing_type_hint(object, context, hint, depth, policy)
})?;
}
}
@@ -345,14 +377,14 @@ fn insert_missing_type_hint(
field_context: &mut JsonContext,
hint: &JsonTypeHint,
depth: usize,
policy: TypeHintMismatchPolicy,
) -> Result<()> {
let key = &hint.path[depth];
let is_leaf = depth + 1 == hint.path.len();
if is_leaf {
if !object.contains_key(key) {
let value = encode_missing_type_hint_value(hint, field_context)?;
object.insert(key.clone(), value.into_variant());
object.insert(key.clone(), JsonValue::null().into_variant());
}
return Ok(());
}
@@ -360,7 +392,11 @@ fn insert_missing_type_hint(
match object.entry(key.clone()) {
Entry::Occupied(mut entry) => match entry.get_mut() {
JsonVariant::Object(child) => {
insert_missing_type_hint(child, field_context, hint, depth + 1)
insert_missing_type_hint(child, field_context, hint, depth + 1, policy)
}
_ if policy == TypeHintMismatchPolicy::CoerceOrNull => {
entry.insert(JsonValue::null().into_variant());
Ok(())
}
_ => error::InvalidJsonSnafu {
value: format!(
@@ -373,109 +409,163 @@ fn insert_missing_type_hint(
},
Entry::Vacant(entry) => {
let mut child = BTreeMap::new();
insert_missing_type_hint(&mut child, field_context, hint, depth + 1)?;
insert_missing_type_hint(&mut child, field_context, hint, depth + 1, policy)?;
entry.insert(JsonVariant::Object(child));
Ok(())
}
}
}
fn encode_missing_type_hint_value(
hint: &JsonTypeHint,
context: &mut JsonContext,
) -> Result<JsonValue> {
if let Some(default_constraint) = &hint.default_constraint {
let value = default_constraint.create_default(&hint.data_type, hint.nullable)?;
let json = decode_primitive_value(value)?;
return encode_json_value_with_hint(json, hint, context);
}
if hint.nullable {
Ok(JsonValue::null())
} else {
error::InvalidJsonSnafu {
value: format!(
"missing non-null JSON2 type hint path {}",
hint.path.join(".")
),
}
.fail()
}
}
fn encode_json_value_with_hint(
json: Json,
hint: &JsonTypeHint,
context: &mut JsonContext,
policy: TypeHintMismatchPolicy,
) -> Result<JsonValue> {
if json.is_null() {
return if hint.nullable {
Ok(JsonValue::null())
} else {
error::InvalidJsonSnafu {
value: format!(
"JSON2 type hint path {} is not nullable",
context.path.join(".")
),
}
.fail()
};
return Ok(JsonValue::null());
}
let invalid_type = || {
error::InvalidJsonSnafu {
let json = match (&hint.data_type, json) {
(ConcreteDataType::String(_), Json::String(value)) => {
return Ok(value.into());
}
(
ConcreteDataType::Int8(_)
| ConcreteDataType::Int16(_)
| ConcreteDataType::Int32(_)
| ConcreteDataType::Int64(_),
Json::Number(value),
) => {
if let Some(value) = value.as_i64() {
return Ok(value.into());
}
Json::Number(value)
}
(
ConcreteDataType::UInt8(_)
| ConcreteDataType::UInt16(_)
| ConcreteDataType::UInt32(_)
| ConcreteDataType::UInt64(_),
Json::Number(value),
) => {
if let Some(value) = value.as_u64() {
return Ok(value.into());
}
Json::Number(value)
}
(ConcreteDataType::Float32(_) | ConcreteDataType::Float64(_), Json::Number(value)) => {
if let Some(value) = value.as_f64() {
return Ok(value.into());
}
Json::Number(value)
}
(ConcreteDataType::Boolean(_), Json::Bool(value)) => {
return Ok(value.into());
}
(_, json) => json,
};
match policy {
TypeHintMismatchPolicy::Reject => error::InvalidJsonSnafu {
value: format!(
"JSON value at {} does not match JSON2 type hint {}",
context.path.join("."),
hint.data_type
),
}
.fail()
};
.fail(),
TypeHintMismatchPolicy::CoerceOrNull => {
let value = coerce_json_value_to_type(json, &hint.data_type);
let value = Json::try_from(value).map_err(|error| {
error::InvalidJsonSnafu {
value: error.to_string(),
}
.build()
})?;
Ok(JsonValue::new(value.into()))
}
}
}
match (&hint.data_type, json) {
(ConcreteDataType::String(_), Json::String(v)) => Ok(v.into()),
(
ConcreteDataType::Int8(_)
| ConcreteDataType::Int16(_)
| ConcreteDataType::Int32(_)
| ConcreteDataType::Int64(_),
Json::Number(v),
) => match v.as_i64() {
Some(v) => Ok(v.into()),
None => invalid_type(),
},
(
ConcreteDataType::UInt8(_)
| ConcreteDataType::UInt16(_)
| ConcreteDataType::UInt32(_)
| ConcreteDataType::UInt64(_),
Json::Number(v),
) => match v.as_u64() {
Some(v) => Ok(v.into()),
None => invalid_type(),
},
(ConcreteDataType::Float32(_) | ConcreteDataType::Float64(_), Json::Number(v)) => {
match v.as_f64() {
Some(v) => Ok(v.into()),
None => invalid_type(),
/// Coerces a JSON value using the same semantics as JSON2 query projection.
/// Values that cannot be converted become null.
pub(crate) fn coerce_json_value_to_type(value: Json, to_type: &ConcreteDataType) -> Value {
if value.is_null() {
return Value::Null;
}
if to_type.is_string() {
let value = match value {
Json::String(value) => value,
value => value.to_string(),
};
return Value::String(value.into());
}
if matches!(to_type, ConcreteDataType::Binary(_)) {
return Value::Binary(encode_serde_json_as_jsonb(value).into());
}
if let Some(struct_type) = to_type.as_struct() {
let Json::Object(mut object) = value else {
return Value::Null;
};
let values = struct_type
.fields()
.iter()
.map(|field| {
object
.remove(field.name())
.map(|value| coerce_json_value_to_type(value, field.data_type()))
.unwrap_or(Value::Null)
})
.collect::<Vec<_>>();
return Value::Struct(StructValue::new(values, struct_type.clone()));
}
if let Some(list_type) = to_type.as_list() {
let Json::Array(values) = value else {
return Value::Null;
};
let item_type = list_type.item_type().clone();
let values = values
.into_iter()
.map(|value| coerce_json_value_to_type(value, &item_type))
.collect::<Vec<_>>();
return Value::List(ListValue::new(values, Arc::new(item_type)));
}
let value = match value {
Json::Bool(value) => Value::Boolean(value),
Json::Number(value) => {
if let Some(value) = value.as_i64() {
Value::Int64(value)
} else if let Some(value) = value.as_u64() {
Value::UInt64(value)
} else if let Some(value) = value.as_f64() {
Value::Float64(value.into())
} else {
Value::Null
}
}
(ConcreteDataType::Boolean(_), Json::Bool(v)) => Ok(v.into()),
_ => invalid_type(),
}
Json::String(value) => Value::String(value.into()),
Json::Array(_) | Json::Object(_) | Json::Null => Value::Null,
};
to_type.try_cast(value).unwrap_or(Value::Null)
}
fn encode_json_array_with_context<'a>(
json_array: Vec<Json>,
context: &mut JsonContext<'a>,
policy: TypeHintMismatchPolicy,
) -> Result<JsonValue> {
let json_array_len = json_array.len();
let mut items = Vec::with_capacity(json_array_len);
for (index, value) in json_array.into_iter().enumerate() {
let item_value = with_key_context(context, &index.to_string(), |context| {
encode_json_value_with_context(value, context)
encode_json_value_with_context(value, context, policy)
})?;
items.push(item_value);
}
@@ -510,7 +600,11 @@ fn encode_json_array_with_context<'a>(
}
/// Helper function to encode a JSON value to a Value and determine its ConcreteDataType with context
fn encode_json_value_with_context(json: Json, context: &mut JsonContext) -> Result<JsonValue> {
fn encode_json_value_with_context(
json: Json,
context: &mut JsonContext,
policy: TypeHintMismatchPolicy,
) -> Result<JsonValue> {
if context.path.len() >= JSON2_MAX_STRUCTURED_DEPTH
&& matches!(&json, Json::Object(_) | Json::Array(_))
{
@@ -539,8 +633,8 @@ fn encode_json_value_with_context(json: Json, context: &mut JsonContext) -> Resu
}
}
Json::String(s) => Ok(s.into()),
Json::Array(arr) => encode_json_array_with_context(arr, context),
Json::Object(obj) => encode_json_object_with_context(obj, context),
Json::Array(arr) => encode_json_array_with_context(arr, context, policy),
Json::Object(obj) => encode_json_object_with_context(obj, context, policy),
}
}
@@ -650,7 +744,51 @@ mod tests {
}
#[test]
fn test_json_settings_forward_compatibility()
fn test_json_settings_equivalent() -> Result<()> {
let hints = vec![
JsonTypeHint {
path: vec!["user".to_string(), "name".to_string()],
data_type: ConcreteDataType::string_datatype(),
inverted_index: false,
},
JsonTypeHint {
path: vec!["count".to_string()],
data_type: ConcreteDataType::int64_datatype(),
inverted_index: false,
},
];
let settings = JsonSettings::try_new(hints.clone(), Some(10))?;
let mut reversed = hints.clone();
reversed.reverse();
let reordered = JsonSettings::try_new(reversed, Some(10))?;
assert_ne!(settings, reordered);
assert!(settings.equivalent(&reordered));
assert!(reordered.equivalent(&settings));
assert!(settings.equivalent(&settings));
assert_eq!(hints, settings.type_hints());
assert!(JsonSettings::default().equivalent(&JsonSettings::default()));
for limit in [None, Some(0), Some(11)] {
assert!(!settings.equivalent(&JsonSettings::try_new(hints.clone(), limit)?));
}
assert!(!settings.equivalent(&JsonSettings::try_new(vec![hints[0].clone()], Some(10))?));
let mut changed = hints.clone();
changed[0].path = vec!["user".to_string(), "id".to_string()];
assert!(!settings.equivalent(&JsonSettings::try_new(changed, Some(10))?));
let mut changed = hints.clone();
changed[0].data_type = ConcreteDataType::int64_datatype();
assert!(!settings.equivalent(&JsonSettings::try_new(changed, Some(10))?));
let mut changed = hints;
changed[0].inverted_index = true;
assert!(!settings.equivalent(&JsonSettings::try_new(changed, Some(10))?));
Ok(())
}
#[test]
fn test_json_settings_deserializes_legacy_type_hint_constraints()
-> std::result::Result<(), Box<dyn std::error::Error>> {
let json_str = r#"{
"type_hints": [
@@ -687,15 +825,11 @@ mod tests {
JsonTypeHint {
path: vec!["user".to_string(), "age".to_string()],
data_type: ConcreteDataType::int64_datatype(),
nullable: false,
default_constraint: Some(ColumnDefaultConstraint::Value(Value::Int64(18))),
inverted_index: true,
},
JsonTypeHint {
path: vec!["user".to_string(), "name".to_string()],
data_type: ConcreteDataType::string_datatype(),
nullable: true,
default_constraint: None,
inverted_index: false,
},
],
@@ -711,15 +845,11 @@ mod tests {
JsonTypeHint {
path: vec!["user".to_string(), "age".to_string()],
data_type: ConcreteDataType::int64_datatype(),
nullable: false,
default_constraint: Some(ColumnDefaultConstraint::Value(Value::Int64(18))),
inverted_index: true,
},
JsonTypeHint {
path: vec!["user".to_string(), "name".to_string()],
data_type: ConcreteDataType::string_datatype(),
nullable: true,
default_constraint: None,
inverted_index: false,
},
];
@@ -758,8 +888,6 @@ mod tests {
let hint = |path, data_type| JsonTypeHint {
path,
data_type,
nullable: true,
default_constraint: None,
inverted_index: false,
};
assert!(
@@ -817,6 +945,104 @@ mod tests {
}
}
#[test]
fn test_coerce_or_null_on_type_hint_mismatch() -> Result<()> {
let settings = JsonSettings::try_new(
vec![JsonTypeHint {
path: vec!["kind".to_string()],
data_type: ConcreteDataType::int64_datatype(),
inverted_index: false,
}],
None,
)?;
let result = settings
.encode_with_type_hint_mismatch_policy(
json!({"kind": "invalid", "message": "kept"}),
TypeHintMismatchPolicy::CoerceOrNull,
)?
.into_json_inner()
.unwrap();
let Value::Struct(root) = result else {
panic!("Expected Struct value");
};
assert_eq!(struct_field_value(&root, "kind"), &Value::Null);
assert_eq!(
struct_field_value(&root, "message"),
&Value::String("kept".into())
);
Ok(())
}
#[test]
fn test_coerce_or_null_converts_type_hint_values() -> Result<()> {
let settings = JsonSettings::try_new(
vec![
JsonTypeHint {
path: vec!["to_string".to_string()],
data_type: ConcreteDataType::string_datatype(),
inverted_index: false,
},
JsonTypeHint {
path: vec!["to_int".to_string()],
data_type: ConcreteDataType::int64_datatype(),
inverted_index: false,
},
],
None,
)?;
let result = settings
.encode_with_type_hint_mismatch_policy(
json!({"to_string": 123, "to_int": "456", "invalid": "value"}),
TypeHintMismatchPolicy::CoerceOrNull,
)?
.into_json_inner()
.unwrap();
let Value::Struct(root) = result else {
panic!("Expected Struct value");
};
assert_eq!(
struct_field_value(&root, "to_string"),
&Value::String("123".into())
);
assert_eq!(struct_field_value(&root, "to_int"), &Value::Int64(456));
assert_eq!(
struct_field_value(&root, "invalid"),
&Value::String("value".into())
);
Ok(())
}
#[test]
fn test_coerce_or_null_on_nested_type_hint_structure_mismatch() -> Result<()> {
let settings = JsonSettings::try_new(
vec![JsonTypeHint {
path: vec!["user".to_string(), "age".to_string()],
data_type: ConcreteDataType::int64_datatype(),
inverted_index: false,
}],
None,
)?;
let result = settings
.encode_with_type_hint_mismatch_policy(
json!({"user": "invalid", "message": "kept"}),
TypeHintMismatchPolicy::CoerceOrNull,
)?
.into_json_inner()
.unwrap();
let Value::Struct(root) = result else {
panic!("Expected Struct value");
};
assert_eq!(struct_field_value(&root, "user"), &Value::Null);
assert_eq!(
struct_field_value(&root, "message"),
&Value::String("kept".into())
);
Ok(())
}
#[test]
fn test_encode_rejects_reserved_remainder_field() -> Result<()> {
let settings = JsonSettings::default();
@@ -955,8 +1181,6 @@ mod tests {
let type_hints = vec![JsonTypeHint {
path: vec!["age".to_string()],
data_type: ConcreteDataType::int64_datatype(),
nullable: false,
default_constraint: None,
inverted_index: false,
}];
let settings = JsonSettings::try_new(type_hints, None)?;
@@ -989,8 +1213,6 @@ mod tests {
let type_hints = vec![JsonTypeHint {
path: vec!["count".to_string()],
data_type: ConcreteDataType::uint64_datatype(),
nullable: false,
default_constraint: None,
inverted_index: false,
}];
let settings = JsonSettings::try_new(type_hints, None)?;
@@ -1021,12 +1243,10 @@ mod tests {
}
#[test]
fn test_encode_json_fills_missing_type_hint_with_default() -> Result<()> {
fn test_encode_json_fills_missing_type_hint_with_null() -> Result<()> {
let type_hints = vec![JsonTypeHint {
path: vec!["user".to_string(), "age".to_string()],
data_type: ConcreteDataType::int64_datatype(),
nullable: false,
default_constraint: Some(ColumnDefaultConstraint::Value(Value::Int64(7))),
inverted_index: false,
}];
let settings = JsonSettings::try_new(type_hints, None)?;
@@ -1043,7 +1263,7 @@ mod tests {
let Value::Struct(user) = struct_field_value(&root, "user") else {
panic!("Expected user Struct value");
};
assert_eq!(struct_field_value(user, "age"), &Value::Int64(7));
assert_eq!(struct_field_value(user, "age"), &Value::Null);
Ok(())
}
@@ -1052,8 +1272,6 @@ mod tests {
let type_hints = vec![JsonTypeHint {
path: vec!["user".to_string(), "name".to_string()],
data_type: ConcreteDataType::string_datatype(),
nullable: true,
default_constraint: None,
inverted_index: false,
}];
let settings = JsonSettings::try_new(type_hints, None)?;
@@ -1075,21 +1293,15 @@ mod tests {
}
#[test]
fn test_encode_json_rejects_missing_non_null_type_hint() -> Result<()> {
fn test_encode_json_allows_missing_type_hint() -> Result<()> {
let type_hints = vec![JsonTypeHint {
path: vec!["user".to_string(), "age".to_string()],
data_type: ConcreteDataType::int64_datatype(),
nullable: false,
default_constraint: None,
inverted_index: false,
}];
let settings = JsonSettings::try_new(type_hints, None)?;
let err = settings.encode(json!({})).unwrap_err();
assert!(
err.to_string()
.contains("missing non-null JSON2 type hint path user.age")
);
settings.encode(json!({}))?;
Ok(())
}
@@ -1099,17 +1311,11 @@ mod tests {
JsonTypeHint {
path: vec!["user".to_string(), "age".to_string()],
data_type: ConcreteDataType::int64_datatype(),
nullable: false,
default_constraint: Some(ColumnDefaultConstraint::Value(Value::Int64(7))),
inverted_index: false,
},
JsonTypeHint {
path: vec!["user".to_string(), "name".to_string()],
data_type: ConcreteDataType::string_datatype(),
nullable: false,
default_constraint: Some(ColumnDefaultConstraint::Value(Value::String(
"unknown".into(),
))),
inverted_index: false,
},
];
@@ -1123,11 +1329,8 @@ mod tests {
let Value::Struct(user) = struct_field_value(&root, "user") else {
panic!("Expected user Struct value");
};
assert_eq!(struct_field_value(user, "age"), &Value::Int64(7));
assert_eq!(
struct_field_value(user, "name"),
&Value::String("unknown".into())
);
assert_eq!(struct_field_value(user, "age"), &Value::Null);
assert_eq!(struct_field_value(user, "name"), &Value::Null);
Ok(())
}
src/datatypes/src/types/struct_type.rs
+19
View File
@@ -19,6 +19,7 @@ use arrow::datatypes::{DataType as ArrowDataType, Field};
use arrow_schema::Fields;
use serde::{Deserialize, Serialize};
use crate::error::Result;
use crate::prelude::{ConcreteDataType, DataType, LogicalTypeId};
use crate::value::Value;
use crate::vectors::StructVectorBuilder;
@@ -100,6 +101,24 @@ impl StructType {
}
}
/// Like the `From<&Fields>` impl, but returns an error for arrow field
/// types greptimedb cannot represent instead of panicking.
pub fn try_from_arrow_fields(fields: &Fields) -> Result<Self> {
let struct_fields = fields
.iter()
.map(|field| {
Ok(StructField::new(
field.name().clone(),
ConcreteDataType::try_from(field.data_type())?,
field.is_nullable(),
))
})
.collect::<Result<Vec<_>>>()?;
Ok(StructType {
fields: Arc::new(struct_fields),
})
}
pub fn fields(&self) -> Arc<Vec<StructField>> {
self.fields.clone()
}
src/datatypes/src/value.rs
+94 -5
View File
@@ -1152,12 +1152,13 @@ impl TryFrom<ScalarValue> for Value {
ScalarValue::UInt16(u) => Value::from(u),
ScalarValue::UInt32(u) => Value::from(u),
ScalarValue::UInt64(u) => Value::from(u),
ScalarValue::Utf8(s) | ScalarValue::LargeUtf8(s) => {
ScalarValue::Utf8(s) | ScalarValue::LargeUtf8(s) | ScalarValue::Utf8View(s) => {
Value::from(s.map(StringBytes::from))
}
ScalarValue::Binary(b)
| ScalarValue::LargeBinary(b)
| ScalarValue::FixedSizeBinary(_, b) => Value::from(b.map(Bytes::from)),
| ScalarValue::FixedSizeBinary(_, b)
| ScalarValue::BinaryView(b) => Value::from(b.map(Bytes::from)),
ScalarValue::List(array) => {
// this is for item type
let datatype = ConcreteDataType::try_from(&array.value_type())?;
@@ -1221,7 +1222,15 @@ impl TryFrom<ScalarValue> for Value {
.map(|v| Value::Decimal128(Decimal128::new(v, p, s)))
.unwrap_or(Value::Null),
ScalarValue::Struct(struct_array) => {
let struct_type = StructType::from(struct_array.fields());
// A struct scalar carries a single element; a null (or empty)
// element means the struct itself is null, for example a null
// struct inside a list.
if struct_array.is_empty() || struct_array.is_null(0) {
return Ok(Value::Null);
}
// Build the struct type fallibly: an unrepresentable arrow
// field type must surface as an error, not a panic.
let struct_type = StructType::try_from_arrow_fields(struct_array.fields())?;
let items = struct_array
.columns()
.iter()
@@ -1244,8 +1253,6 @@ impl TryFrom<ScalarValue> for Value {
| ScalarValue::LargeListView(_)
| ScalarValue::Union(_, _, _)
| ScalarValue::Float16(_)
| ScalarValue::Utf8View(_)
| ScalarValue::BinaryView(_)
| ScalarValue::Map(_)
| ScalarValue::Date64(_)
| ScalarValue::RunEndEncoded(_, _, _) => {
@@ -1259,6 +1266,21 @@ impl TryFrom<ScalarValue> for Value {
}
}
/// Converts the value at `index` of an arrow array into a [`Value`].
///
/// Unlike the unchecked conversion in [`crate::vectors::StructVector`]'s `get`,
/// this propagates conversion errors (for example arrow types greptimedb
/// cannot represent, such as `Decimal256`) instead of panicking, so callers
/// can surface a proper error to the user.
pub fn try_value_from_array(array: &dyn Array, index: usize) -> Result<Value> {
if array.is_null(index) {
return Ok(Value::Null);
}
let scalar =
ScalarValue::try_from_array(array, index).context(ConvertArrowArrayToScalarsSnafu)?;
Value::try_from(scalar)
}
impl From<ValueRef<'_>> for Value {
fn from(value: ValueRef<'_>) -> Self {
match value {
@@ -2157,6 +2179,73 @@ pub(crate) mod tests {
Value::Struct(struct_value),
scalar_struct_value.try_into().unwrap()
);
// view-typed strings and binaries convert like their non-view forms
assert_eq!(
Value::String("abc".into()),
ScalarValue::Utf8View(Some("abc".into()))
.try_into()
.unwrap()
);
assert_eq!(Value::Null, ScalarValue::Utf8View(None).try_into().unwrap());
assert_eq!(
Value::Binary(Bytes::from(vec![1, 2])),
ScalarValue::BinaryView(Some(vec![1, 2]))
.try_into()
.unwrap()
);
assert_eq!(
Value::Null,
ScalarValue::BinaryView(None).try_into().unwrap()
);
// a null struct scalar is a null value, not a struct of null fields
let null_struct = ScalarStructBuilder::new_null(build_struct_type().as_arrow_fields());
assert_eq!(Value::Null, null_struct.try_into().unwrap());
}
#[test]
fn test_try_value_from_array() {
use arrow_array::Int32Array;
use datafusion_common::arrow::datatypes::i256;
let array = Int32Array::from(vec![Some(1), None]);
assert_eq!(Value::Int32(1), try_value_from_array(&array, 0).unwrap());
assert_eq!(Value::Null, try_value_from_array(&array, 1).unwrap());
// a struct with representable fields converts
let supported = ScalarStructBuilder::new()
.with_name_and_scalar("i", ScalarValue::Int32(Some(7)))
.build()
.unwrap();
let ScalarValue::Struct(array) = supported else {
unreachable!();
};
assert_eq!(
Value::Struct(StructValue::new(
vec![Value::Int32(7)],
StructType::new(std::sync::Arc::new(vec![StructField::new(
"i",
ConcreteDataType::int32_datatype(),
false,
)]))
)),
try_value_from_array(array.as_ref(), 0).unwrap()
);
// an unsupported arrow field type errors instead of panicking
let unsupported = ScalarStructBuilder::new()
.with_name_and_scalar(
"d",
ScalarValue::Decimal256(Some(i256::from_i128(1)), 38, 10),
)
.build()
.unwrap();
let ScalarValue::Struct(array) = unsupported else {
unreachable!();
};
let error = try_value_from_array(array.as_ref(), 0).unwrap_err();
assert!(error.to_string().contains("Unsupported arrow data type"));
}
#[test]
src/datatypes/src/vectors/json/array.rs
+23 -84
View File
@@ -26,15 +26,13 @@ use serde_json::Value;
use snafu::{OptionExt, ResultExt};
use crate::arrow_array::{binary_array_value, string_array_value};
use crate::data_type::ConcreteDataType;
use crate::data_type::{ConcreteDataType, DataType as _};
use crate::error::{
AlignJsonArraySnafu, ArrowComputeSnafu, InvalidJsonSnafu, InvalidJsonbSnafu, Result,
};
use crate::extension::json::{JSON2_REMAINDER_FIELD_NAME, json2_remainder_field};
use crate::json::JsonSettings;
use crate::json::value::{decode_json_variant, encode_serde_json_as_jsonb};
use crate::prelude::{DataType as _, Value as GreptimeValue};
use crate::value::{ListValue, StructValue};
use crate::json::value::decode_json_variant;
use crate::json::{JsonSettings, TypeHintMismatchPolicy, coerce_json_value_to_type};
use crate::vectors::MutableVector;
use crate::vectors::json::builder::{JsonVectorBuilder, json2_physical_data_type};
use crate::vectors::json::variant::variant_to_json_values;
@@ -123,6 +121,23 @@ impl JsonArray<'_> {
field: &Field,
logical_settings: &JsonSettings,
target_layout: &JsonSettings,
) -> Result<ArrayRef> {
self.rewrite_to_v2_with_type_hint_mismatch_policy(
field,
logical_settings,
target_layout,
TypeHintMismatchPolicy::Reject,
)
}
/// Rewrites a JSON2 array to the specified v2 physical layout using the
/// given type hint mismatch policy.
pub fn rewrite_to_v2_with_type_hint_mismatch_policy(
&self,
field: &Field,
logical_settings: &JsonSettings,
target_layout: &JsonSettings,
policy: TypeHintMismatchPolicy,
) -> Result<ArrayRef> {
let is_v2 = json2_remainder_field(field)?.is_some();
if is_v2 && self.inner.data_type() == &json2_physical_data_type(target_layout) {
@@ -141,7 +156,8 @@ impl JsonArray<'_> {
if value.is_null() {
builder.push_null();
} else {
let value = logical_settings.encode(value)?;
let value =
logical_settings.encode_with_type_hint_mismatch_policy(value, policy)?;
builder.try_push_value_ref(&value.as_value_ref())?;
}
}
@@ -319,87 +335,12 @@ fn project_json_values(values: Vec<Value>, to_type: &DataType) -> Result<ArrayRe
let concrete_type = ConcreteDataType::from_arrow_type(to_type);
let mut builder = concrete_type.create_mutable_vector(values.len());
for value in values {
let value = project_json_value_to_type(value, &concrete_type)?;
let value = coerce_json_value_to_type(value, &concrete_type);
builder.try_push_value_ref(&value.as_value_ref())?;
}
Ok(builder.to_vector().to_arrow_array())
}
fn project_json_value_to_type(value: Value, to_type: &ConcreteDataType) -> Result<GreptimeValue> {
if value.is_null() {
return Ok(GreptimeValue::Null);
}
if to_type.is_string() {
let value = match value {
Value::String(value) => value,
value => value.to_string(),
};
return Ok(GreptimeValue::String(value.into()));
}
if matches!(to_type, ConcreteDataType::Binary(_)) {
return Ok(GreptimeValue::Binary(
encode_serde_json_as_jsonb(value).into(),
));
}
if let Some(struct_type) = to_type.as_struct() {
let Value::Object(mut object) = value else {
return Ok(GreptimeValue::Null);
};
let values = struct_type
.fields()
.iter()
.map(|field| {
object
.remove(field.name())
.map(|value| project_json_value_to_type(value, field.data_type()))
.transpose()
.map(|value| value.unwrap_or(GreptimeValue::Null))
})
.collect::<Result<Vec<_>>>()?;
return Ok(GreptimeValue::Struct(StructValue::new(
values,
struct_type.clone(),
)));
}
if let Some(list_type) = to_type.as_list() {
let Value::Array(values) = value else {
return Ok(GreptimeValue::Null);
};
let item_type = list_type.item_type().clone();
let values = values
.into_iter()
.map(|value| project_json_value_to_type(value, &item_type))
.collect::<Result<Vec<_>>>()?;
return Ok(GreptimeValue::List(ListValue::new(
values,
Arc::new(item_type),
)));
}
let value = match value {
Value::Bool(value) => GreptimeValue::Boolean(value),
Value::Number(value) => {
if let Some(value) = value.as_i64() {
GreptimeValue::Int64(value)
} else if let Some(value) = value.as_u64() {
GreptimeValue::UInt64(value)
} else if let Some(value) = value.as_f64() {
GreptimeValue::Float64(value.into())
} else {
GreptimeValue::Null
}
}
Value::String(value) => GreptimeValue::String(value.into()),
Value::Array(_) | Value::Object(_) => GreptimeValue::Null,
Value::Null => GreptimeValue::Null,
};
Ok(to_type.try_cast(value).unwrap_or(GreptimeValue::Null))
}
impl<'a> From<&'a ArrayRef> for JsonArray<'a> {
fn from(inner: &'a ArrayRef) -> Self {
Self { inner }
@@ -701,8 +642,6 @@ mod test {
vec![JsonTypeHint {
path: vec!["kind".to_string()],
data_type: ConcreteDataType::string_datatype(),
nullable: true,
default_constraint: None,
inverted_index: false,
}],
Some(0),
src/datatypes/src/vectors/json/builder.rs
-10
View File
@@ -1060,22 +1060,16 @@ mod tests {
JsonTypeHint {
path: vec!["kind".to_string()],
data_type: ConcreteDataType::string_datatype(),
nullable: true,
default_constraint: None,
inverted_index: false,
},
JsonTypeHint {
path: vec!["commit".to_string(), "operation".to_string()],
data_type: ConcreteDataType::string_datatype(),
nullable: true,
default_constraint: None,
inverted_index: false,
},
JsonTypeHint {
path: vec!["time_us".to_string()],
data_type: ConcreteDataType::int64_datatype(),
nullable: true,
default_constraint: None,
inverted_index: false,
},
],
@@ -1171,8 +1165,6 @@ mod tests {
vec![JsonTypeHint {
path: vec!["hint".to_string()],
data_type: ConcreteDataType::string_datatype(),
nullable: true,
default_constraint: None,
inverted_index: false,
}],
Some(2),
@@ -1338,8 +1330,6 @@ mod tests {
vec![JsonTypeHint {
path: vec!["nested".to_string(), "hinted".to_string()],
data_type: ConcreteDataType::string_datatype(),
nullable: true,
default_constraint: None,
inverted_index: false,
}],
Some(8),
src/flow/AGENTS.md
+2
View File
@@ -54,6 +54,8 @@ default, and sink-creation helpers.
## Public surface
- gRPC `Flow` service in `src/flow/src/server.rs`.
- Local streaming sink writes and batching query execution use `Channel::Internal`
so meter collectors can account for sink writes without charging ingestion quota.
- `FlowEngine` trait in `src/flow/src/engine.rs`.
- Started from the `cmd` crate via `FlownodeBuilder` / `FlownodeInstance`.
src/flow/src/batching_mode/frontend_client.rs
+6 -2
View File
@@ -34,7 +34,7 @@ use query::options::{FlowQueryExtensions, QueryOptions};
use rand::rng;
use rand::seq::SliceRandom;
use servers::query_handler::grpc::GrpcQueryHandler;
use session::context::{QueryContextBuilder, QueryContextRef};
use session::context::{Channel, QueryContextBuilder, QueryContextRef};
use session::hints::READ_PREFERENCE_HINT;
use snafu::{OptionExt, ResultExt};
use tokio::sync::SetOnce;
@@ -521,6 +521,7 @@ impl FrontendClient {
.current_catalog(catalog.to_string())
.current_schema(schema.to_string())
.extensions(extensions_map)
.channel(Channel::Internal)
.snapshot_seqs(Arc::new(RwLock::new(snapshot_seqs.clone())))
.build();
let ctx = Arc::new(ctx);
@@ -589,6 +590,7 @@ impl FrontendClient {
let ctx = QueryContextBuilder::default()
.current_catalog(catalog.to_string())
.current_schema(schema.to_string())
.channel(Channel::Internal)
.extensions(HashMap::from([(
QUERY_PARALLELISM_HINT.to_string(),
query.parallelism.to_string(),
@@ -894,8 +896,9 @@ mod tests {
async fn do_query(
&self,
_query: Request,
_ctx: QueryContextRef,
ctx: QueryContextRef,
) -> std::result::Result<Output, BoxedError> {
assert_eq!(ctx.channel(), Channel::Internal);
self.calls.fetch_add(1, Ordering::SeqCst);
Ok(Output::new_with_affected_rows(1))
}
@@ -941,6 +944,7 @@ mod tests {
ctx: QueryContextRef,
) -> std::result::Result<Output, BoxedError> {
assert_eq!(ctx.extension("flow.return_region_seq"), Some("true"));
assert_eq!(ctx.channel(), Channel::Internal);
Ok(Output::new_with_affected_rows(1))
}
}
src/flow/src/batching_mode/utils.rs
+18 -2
View File
@@ -21,6 +21,7 @@ use catalog::CatalogManagerRef;
use common_error::ext::BoxedError;
use common_function::aggrs::aggr_wrapper::get_aggr_func;
use common_telemetry::debug;
use datafusion::arrow::datatypes::DataType as ArrowDataType;
use datafusion::datasource::DefaultTableSource;
use datafusion::error::Result as DfResult;
use datafusion::logical_expr::Expr;
@@ -325,7 +326,10 @@ fn is_literal_or_cast_literal(expr: &Expr) -> bool {
}
}
fn merge_op_for_aggregate_expr(aggr_expr: &Expr) -> Result<IncrementalAggregateMergeOp, String> {
fn merge_op_for_aggregate_expr(
aggr_expr: &Expr,
input_schema: &DFSchema,
) -> Result<IncrementalAggregateMergeOp, String> {
let Some(aggr_func) = get_aggr_func(aggr_expr) else {
return Err(aggr_expr.to_string());
};
@@ -355,6 +359,9 @@ fn merge_op_for_aggregate_expr(aggr_expr: &Expr) -> Result<IncrementalAggregateM
"bit_or" => Ok(IncrementalAggregateMergeOp::BitOr),
"bit_xor" => Ok(IncrementalAggregateMergeOp::BitXor),
// Preserve state-family parameters; value coercion is handled by the aggregate.
"avg_state" if aggr_func.params.args.len() == 1 => {
state_delta_merge("__avg_state_delta_merge", vec![])
}
"hll" if aggr_func.params.args.len() == 1 => state_delta_merge("__hll_delta_merge", vec![]),
"stddev_pop_state" if aggr_func.params.args.len() == 1 => {
state_delta_merge("__stddev_pop_state_delta_merge", vec![])
@@ -373,6 +380,15 @@ fn merge_op_for_aggregate_expr(aggr_expr: &Expr) -> Result<IncrementalAggregateM
vec![bucket_size.clone(), error_rate.clone()],
)
}
// AVG's binary merge form is admitted because its state argument is
// already the aggregate result stored by the sink.
"avg_merge"
if aggr_func.params.args.len() == 1
&& aggr_func.params.args[0].get_type(input_schema).ok()
== Some(ArrowDataType::Binary) =>
{
state_delta_merge("__avg_state_delta_merge", vec![])
}
_ => Err(aggr_expr.to_string()),
}
}
@@ -519,7 +535,7 @@ pub fn analyze_incremental_aggregate_plan(
&group_key_names,
));
for aggr_expr in aggr_exprs {
let merge_op = match merge_op_for_aggregate_expr(&aggr_expr) {
let merge_op = match merge_op_for_aggregate_expr(&aggr_expr, aggregate.input.schema()) {
Ok(merge_op) => merge_op,
Err(reason) => {
unsupported_exprs.push(reason);
src/flow/src/batching_mode/utils/test.rs
+168 -2
View File
@@ -16,6 +16,7 @@ use std::collections::BTreeMap;
use std::sync::Arc;
use catalog::RegisterTableRequest;
use common_function::aggrs::approximate::avg::AvgState;
use common_query::OutputData;
use common_recordbatch::recordbatch::merge_record_batches;
use common_recordbatch::{RecordBatch, util};
@@ -1706,11 +1707,154 @@ async fn test_analyze_incremental_aggregate_plan_rejects_avg() {
assert!(!analysis.unsupported_exprs.is_empty());
}
#[tokio::test]
async fn test_analyze_incremental_aggregate_plan_supports_avg_state() {
let query_engine = create_test_query_engine();
let ctx = QueryContext::arc();
let sql = "SELECT avg_state(number) AS avg_num, ts FROM numbers_with_ts GROUP BY ts";
let plan = sql_to_df_plan(ctx, query_engine, sql, false).await.unwrap();
let analysis = analyze_incremental_aggregate_plan(&plan).unwrap().unwrap();
assert!(
analysis.unsupported_exprs.is_empty(),
"avg_state should be supported: {:?}",
analysis.unsupported_exprs
);
assert_eq!(analysis.merge_columns.len(), 1);
assert_eq!(analysis.merge_columns[0].output_field_name, "avg_num");
assert!(matches!(
&analysis.merge_columns[0].merge_op,
IncrementalAggregateMergeOp::StateDeltaMerge {
function_name: "__avg_state_delta_merge",
params,
} if params.is_empty()
));
}
#[tokio::test]
async fn test_analyze_incremental_aggregate_plan_supports_avg_merge() {
let query_engine = create_test_query_engine();
let producer_sql = "SELECT avg_state(number) AS avg_state, CASE WHEN number <= 5 THEN 1 ELSE 2 END AS grp FROM numbers_with_ts GROUP BY grp";
let producer_plan = sql_to_df_plan(
QueryContext::arc(),
query_engine.clone(),
producer_sql,
false,
)
.await
.unwrap();
let producer_output = query_engine
.execute(producer_plan, QueryContext::arc())
.await
.unwrap();
let OutputData::Stream(producer_stream) = producer_output.data else {
panic!("expected AVG state producer execution to be a stream");
};
let producer_batches = util::collect(producer_stream).await.unwrap();
let producer_schema = producer_batches.first().unwrap().schema.clone();
let avg_states = merge_record_batches(producer_schema, &producer_batches).unwrap();
assert_eq!(avg_states.num_rows(), 2);
let avg_states_table = MemTable::table("avg_states", avg_states);
query_engine
.engine_state()
.catalog_manager()
.as_any()
.downcast_ref::<catalog::memory::MemoryCatalogManager>()
.unwrap()
.register_table_sync(RegisterTableRequest {
catalog: "greptime".to_string(),
schema: "public".to_string(),
table_name: "avg_states".to_string(),
table_id: 4096,
table: avg_states_table,
})
.unwrap();
let sql = "SELECT avg_merge(avg_state) AS avg_num, grp FROM avg_states GROUP BY grp";
let plan = sql_to_df_plan(QueryContext::arc(), query_engine.clone(), sql, false)
.await
.unwrap();
let analysis = analyze_incremental_aggregate_plan(&plan).unwrap().unwrap();
assert!(
analysis.unsupported_exprs.is_empty(),
"avg_merge should be supported: {:?}",
analysis.unsupported_exprs
);
assert_eq!(analysis.merge_columns.len(), 1);
assert!(matches!(
&analysis.merge_columns[0].merge_op,
IncrementalAggregateMergeOp::StateDeltaMerge {
function_name: "__avg_state_delta_merge",
params,
} if params.is_empty()
));
let output = query_engine
.execute(plan, QueryContext::arc())
.await
.unwrap();
let OutputData::Stream(stream) = output.data else {
panic!("expected AVG merge execution to be a stream");
};
let batches = util::collect(stream).await.unwrap();
let schema = batches.first().unwrap().schema.clone();
let batch = merge_record_batches(schema, &batches).unwrap();
let groups = batch
.column_by_name("grp")
.unwrap()
.as_primitive::<Int64Type>();
let averages = batch.column_by_name("avg_num").unwrap().as_binary::<i32>();
let mut values = (0..batch.num_rows())
.map(|index| {
(
groups.value(index),
AvgState::decode(averages.value(index)).unwrap().average(),
)
})
.collect::<Vec<_>>();
values.sort_unstable_by_key(|(group, _)| *group);
assert_eq!(values, [(1, Some(3.0)), (2, Some(8.0))]);
}
#[tokio::test]
async fn test_analyze_incremental_aggregate_plan_supports_duplicate_avg_projections() {
let analysis = analyze_test_sql(
"SELECT avg_state(number) AS avg_a, avg_state(number) AS avg_b, avg_state(number + 1) AS avg_num_plus, ts FROM numbers_with_ts GROUP BY ts",
)
.await;
assert!(analysis.unsupported_exprs.is_empty());
assert_eq!(analysis.merge_columns.len(), 3);
assert!(analysis.merge_columns.iter().all(|column| {
matches!(
&column.merge_op,
IncrementalAggregateMergeOp::StateDeltaMerge {
function_name: "__avg_state_delta_merge",
params,
} if params.is_empty()
)
}));
assert_eq!(analysis.merge_columns[0].output_field_name, "avg_a");
assert_eq!(analysis.merge_columns[0].input_field_name, "avg_a");
let avg_b = analysis
.merge_columns
.iter()
.find(|column| column.output_field_name == "avg_b")
.unwrap();
assert_eq!(avg_b.input_field_name, "avg_a");
assert!(
analysis
.merge_columns
.iter()
.any(|column| column.output_field_name == "avg_num_plus")
);
}
#[tokio::test]
async fn test_rewrite_incremental_aggregate_merges_populated_mixed_state_families() {
let query_engine = create_test_query_engine();
let old_sql = "SELECT hll(CAST(number AS VARCHAR)) AS hll_a, hll(CAST(number AS VARCHAR)) AS hll_b, uddsketch_state(128, 0.000001, number) AS percentile_a, uddsketch_state(256, 0.02, number) AS percentile_b, stddev_pop_state(number) AS stddev_state, sum(number) AS total, CASE WHEN number <= 5 THEN CAST(NULL AS BIGINT) ELSE 2 END AS grp FROM numbers_with_ts WHERE number <= 3 OR number = 6 GROUP BY grp";
let new_sql = "SELECT hll(CAST(number AS VARCHAR)) AS hll_a, hll(CAST(number AS VARCHAR)) AS hll_b, uddsketch_state(128, 0.000001, number) AS percentile_a, uddsketch_state(256, 0.02, number) AS percentile_b, stddev_pop_state(number) AS stddev_state, sum(number) AS total, CASE WHEN number <= 5 THEN CAST(NULL AS BIGINT) WHEN number <= 8 THEN 2 ELSE 3 END AS grp FROM numbers_with_ts WHERE number >= 4 AND number != 6 GROUP BY grp";
let old_sql = "SELECT avg_state(number) AS avg_num, hll(CAST(number AS VARCHAR)) AS hll_a, hll(CAST(number AS VARCHAR)) AS hll_b, uddsketch_state(128, 0.000001, number) AS percentile_a, uddsketch_state(256, 0.02, number) AS percentile_b, stddev_pop_state(number) AS stddev_state, sum(number) AS total, CASE WHEN number <= 5 THEN CAST(NULL AS BIGINT) ELSE 2 END AS grp FROM numbers_with_ts WHERE number <= 3 OR number = 6 GROUP BY grp";
let new_sql = "SELECT avg_state(number) AS avg_num, hll(CAST(number AS VARCHAR)) AS hll_a, hll(CAST(number AS VARCHAR)) AS hll_b, uddsketch_state(128, 0.000001, number) AS percentile_a, uddsketch_state(256, 0.02, number) AS percentile_b, stddev_pop_state(number) AS stddev_state, sum(number) AS total, CASE WHEN number <= 5 THEN CAST(NULL AS BIGINT) WHEN number <= 8 THEN 2 ELSE 3 END AS grp FROM numbers_with_ts WHERE number >= 4 AND number != 6 GROUP BY grp";
let old_plan = sql_to_df_plan(QueryContext::arc(), query_engine.clone(), old_sql, false)
.await
.unwrap();
@@ -1757,6 +1901,7 @@ async fn test_rewrite_incremental_aggregate_merges_populated_mixed_state_familie
assert_eq!(
analysis.output_field_names,
vec![
"avg_num",
"hll_a",
"hll_b",
"percentile_a",
@@ -1783,6 +1928,27 @@ async fn test_rewrite_incremental_aggregate_merges_populated_mixed_state_familie
3,
"rewrite must produce one row per group"
);
let avg_states = merged_batch
.column_by_name("avg_num")
.unwrap()
.as_binary::<i32>();
let merged_groups = merged_batch
.column_by_name("grp")
.unwrap()
.as_primitive::<Int64Type>();
assert_eq!(avg_states.null_count(), 0);
for index in 0..merged_batch.num_rows() {
let expected = match (!merged_groups.is_null(index)).then(|| merged_groups.value(index)) {
None => Some(3.0),
Some(2) => Some(7.0),
Some(3) => Some(9.5),
group => panic!("unexpected group: {group:?}"),
};
assert_eq!(
AvgState::decode(avg_states.value(index)).unwrap().average(),
expected
);
}
let merged_table = MemTable::table("merged_states", merged_batch);
query_engine
.engine_state()
src/frontend/AGENTS.md
+14 -2
View File
@@ -41,8 +41,20 @@ remote datanodes via `operator`/`client`.
`StatementExecutor`. Distributed scans enter through `region_query.rs`.
- **Insert** (`instance/grpc.rs`): `handle_inserts` / `handle_row_inserts` →
`check_permission` → `operator`'s `Inserter` (schema validation, optional
auto-create, partition routing) → local `RegionServer` (standalone) or RPC to
datanodes (distributed).
auto-create, partition routing, meter admission) → local `RegionServer`
(standalone) or RPC to datanodes (distributed). Arrow bulk inserts pass the
request channel to `Inserter` and check meter admission for each nonempty batch.
- Finite ingestion requests split internally admit their total rows per database
before dispatch (`operator::insert::admit_write` / `admit_row_insert_batches`).
The returned context covers chunks and derived writes while preserving WCU
accounting and the original protocol channel.
- Internal gRPC listeners mark requests with `Channel::Internal` in middleware
(`server.rs`), including requests handled by Enterprise Flight wrappers.
- **Logical-table batching** (`instance/logical_batcher.rs`): `Services` initializes
one shared batcher for opted-in HTTP Prom and nonlegacy OTLP metric-engine
writes. OTLP checks operator eligibility and falls back for incompatible tables.
The schema adapter holds a weak instance reference to avoid an ownership cycle.
- **Table batching** (`instance/builder.rs`): protocol entry points opt in through
`QueryContext`. The primary inserter prepares eligible ordinary-table writes
src/frontend/src/error.rs
+12 -1
View File
@@ -174,6 +174,15 @@ pub enum Error {
location: Location,
},
#[snafu(display("Unexpected type {data_type} for column '{column}' of table '{table_name}'"))]
UnexpectedColumnType {
table_name: String,
column: String,
data_type: String,
#[snafu(implicit)]
location: Location,
},
#[snafu(display("Failed to collect recordbatch"))]
CollectRecordbatch {
#[snafu(implicit)]
@@ -414,7 +423,9 @@ impl ErrorExt for Error {
Error::RequestQuery { source, .. } => source.status_code(),
Error::CacheRequired { .. } => StatusCode::Internal,
Error::CacheRequired { .. } | Error::UnexpectedColumnType { .. } => {
StatusCode::Internal
}
Error::TableNotFound { .. } => StatusCode::TableNotFound,
src/frontend/src/frontend.rs
+50 -5
View File
@@ -50,6 +50,8 @@ pub struct FrontendOptions {
/// even if a request sets the `auto_create_table` hint to `true`. When `true`
/// (default), the per-request hint still applies. Default: `true`.
pub auto_create_table: bool,
/// Enables experimental Parquet database exports using shared Metric scans.
pub experimental_metric_export: bool,
/// Maximum total memory for all concurrent write request bodies and messages (HTTP, gRPC, Flight).
/// Set to 0 to disable the limit. Default: "0" (unlimited)
pub max_in_flight_write_bytes: ReadableSize,
@@ -65,7 +67,7 @@ pub struct FrontendOptions {
pub postgres: PostgresOptions,
pub opentsdb: OpentsdbOptions,
pub influxdb: InfluxdbOptions,
/// Shared experimental ordinary-table batching; independent of Prom batching.
/// Ordinary-table batching with independent logical-table controls.
pub pending_rows_batcher: PendingRowsBatcherOptions,
pub prom_store: PromStoreOptions,
pub jaeger: JaegerOptions,
@@ -93,6 +95,7 @@ impl Default for FrontendOptions {
default_timezone: None,
default_column_prefix: None,
auto_create_table: true,
experimental_metric_export: false,
max_in_flight_write_bytes: ReadableSize(0),
write_bytes_exhausted_policy: OnExhaustedPolicy::default(),
http: HttpOptions::default(),
@@ -128,6 +131,7 @@ impl Configurable for FrontendOptions {
"meta_client.metasrv_addrs",
"event_recorder.event_types",
"pending_rows_batcher.protocols",
"pending_rows_batcher.logical_table.protocols",
])
}
}
@@ -222,6 +226,38 @@ mod tests {
type GrpcStream<T> =
Pin<Box<dyn Stream<Item = std::result::Result<T, Status>> + Send + Sync + 'static>>;
#[test]
fn test_logical_batcher_protocols_from_env() {
temp_env::with_vars(
[
(
"FRONTEND_LOGICAL_TEST__PENDING_ROWS_BATCHER__PROTOCOLS",
Some("otlp,influxdb"),
),
(
"FRONTEND_LOGICAL_TEST__PENDING_ROWS_BATCHER__LOGICAL_TABLE__PROTOCOLS",
Some("prom,otlp"),
),
],
|| {
let options =
FrontendOptions::load_layered_options(None, "FRONTEND_LOGICAL_TEST").unwrap();
assert_eq!(options.pending_rows_batcher.table.protocols.len(), 2);
assert_eq!(
options
.pending_rows_batcher
.logical_table
.unwrap()
.protocols,
vec![
servers::http::BatchingProtocol::Prom,
servers::http::BatchingProtocol::Otlp
]
);
},
);
}
#[test]
fn test_batcher_protocols_from_env() {
temp_env::with_vars(
@@ -233,7 +269,7 @@ mod tests {
let options =
FrontendOptions::load_layered_options(None, "FRONTEND_BATCHER_TEST").unwrap();
assert_eq!(
options.pending_rows_batcher.protocols,
options.pending_rows_batcher.table.protocols,
vec![
servers::http::BatchingProtocol::Influxdb,
servers::http::BatchingProtocol::HttpSql
@@ -249,6 +285,7 @@ mod tests {
assert!(
!defaults
.pending_rows_batcher
.table
.pending_rows_batching_enabled()
);
let options: FrontendOptions = toml::from_str(
@@ -260,9 +297,14 @@ max_batch_rows = 25
"#,
)
.unwrap();
assert_eq!(options.pending_rows_batcher.max_batch_rows, 25);
assert_eq!(options.pending_rows_batcher.protocols.len(), 2);
assert!(options.pending_rows_batcher.pending_rows_batching_enabled());
assert_eq!(options.pending_rows_batcher.table.max_batch_rows, 25);
assert_eq!(options.pending_rows_batcher.table.protocols.len(), 2);
assert!(
options
.pending_rows_batcher
.table
.pending_rows_batching_enabled()
);
let serialized = toml::to_string(&options).unwrap();
let parsed: FrontendOptions = toml::from_str(&serialized).unwrap();
assert_eq!(options.influxdb, parsed.influxdb);
@@ -273,6 +315,9 @@ max_batch_rows = 25
#[test]
fn test_toml() {
let opts = FrontendOptions::default();
assert!(!opts.experimental_metric_export);
let enabled: FrontendOptions = toml::from_str("experimental_metric_export = true").unwrap();
assert!(enabled.experimental_metric_export);
let toml_string = toml::to_string(&opts).unwrap();
assert!(toml_string.contains("experimental_enable_exponential_histogram = false"));
let parsed: FrontendOptions = toml::from_str(&toml_string).unwrap();
src/frontend/src/instance.rs
+154 -1
View File
@@ -17,9 +17,11 @@ mod dashboard;
mod entity_graph;
mod export_database;
mod grpc;
mod import_packed;
mod influxdb;
mod jaeger;
mod log_handler;
mod logical_batcher;
mod logs;
mod opentsdb;
mod otlp;
@@ -30,7 +32,7 @@ mod region_query;
use std::collections::HashSet;
use std::pin::Pin;
use std::sync::atomic::AtomicBool;
use std::sync::{Arc, atomic};
use std::sync::{Arc, OnceLock, atomic};
use std::time::{Duration, SystemTime};
use async_stream::stream;
@@ -77,6 +79,7 @@ use query::metrics::OnDone;
use query::parser::{PromQuery, QueryStatement};
use query::query_engine::DescribeResult;
use query::query_engine::options::{QueryOptions, validate_catalog_and_schema};
use servers::batcher::logical_table::LogicalTablePendingRowsBatcher;
use servers::error::{
self as server_error, AuthSnafu, CommonMetaSnafu, ExecuteQuerySnafu,
OtlpMetricModeIncompatibleSnafu, UnexpectedResultSnafu,
@@ -122,12 +125,14 @@ lazy_static! {
#[derive(Clone)]
pub struct Instance {
frontend_peer_addr: String,
experimental_metric_export: bool,
catalog_manager: CatalogManagerRef,
pipeline_operator: Arc<PipelineOperator>,
statement_executor: Arc<StatementExecutor>,
query_engine: QueryEngineRef,
plugins: Plugins,
inserter: InserterRef,
logical_batcher: Arc<OnceLock<Option<Arc<LogicalTablePendingRowsBatcher>>>>,
deleter: DeleterRef,
table_metadata_manager: TableMetadataManagerRef,
event_recorder: EventRecorderRef,
@@ -391,6 +396,30 @@ impl Instance {
}
_ => {
query_interceptor.pre_execute(Some(&stmt), None, query_ctx.clone())?;
if let Statement::Copy(sql::statements::copy::Copy::CopyDatabase(
CopyDatabase::From(arg),
)) = &stmt
&& arg.with.get("metric_data_layout").is_some()
{
return self
.copy_packed_database(arg.clone(), &stmt, query_ctx)
.await;
}
if let Statement::ShowVariables(show) = &stmt
&& show
.variable
.to_string()
.eq_ignore_ascii_case("experimental_metric_export")
{
return self.show_metric_export_capability();
}
if let Statement::Copy(sql::statements::copy::Copy::CopyDatabase(CopyDatabase::To(
arg,
))) = &stmt
&& export_database::parse_metric_export_requested(&arg.with)?
{
return self.copy_metric_database(arg.clone(), query_ctx).await;
}
self.statement_executor
.execute_sql(stmt, query_ctx)
.await
@@ -1510,6 +1539,20 @@ impl PrometheusHandler for Instance {
.context(ExecuteQuerySnafu)
}
async fn query_metric_names_by_labels(
&self,
matchers: Vec<Matcher>,
schema: &str,
start: SystemTime,
end: SystemTime,
ctx: &QueryContextRef,
) -> server_error::Result<Vec<String>> {
self.handle_query_metric_names_by_labels(matchers, schema, start, end, ctx)
.await
.map_err(BoxedError::new)
.context(ExecuteQuerySnafu)
}
async fn query_label_values(
&self,
metric: String,
@@ -2720,6 +2763,116 @@ mod tests {
)
}
struct ReleasedExportSource {
schema: GtSchemaRef,
channels: std::sync::Mutex<Option<(oneshot::Sender<()>, oneshot::Receiver<()>)>>,
}
impl DataSource for ReleasedExportSource {
fn get_stream(
&self,
_request: ScanRequest,
) -> std::result::Result<SendableRecordBatchStream, BoxedError> {
let (started, release) = self.channels.lock().unwrap().take().unwrap();
let schema = self.schema.clone();
let stream = futures::stream::once(async move {
started.send(()).unwrap();
release.await.unwrap();
Ok(RecordBatch::new_empty(schema))
});
Ok(Box::pin(RecordBatchStreamWrapper::new(
self.schema.clone(),
Box::pin(stream),
)))
}
}
#[tokio::test]
async fn test_metric_export_http_timeout_drains_ordinary_writer() {
let destination = common_test_util::temp_dir::create_temp_dir("metric_export_timeout");
let (started_tx, started_rx) = oneshot::channel();
let (release_tx, release_rx) = oneshot::channel();
let info = test_table_info(1024, "source").unwrap();
let source = Arc::new(Table::new(
Arc::new(info.clone()),
FilterPushDownType::Unsupported,
Arc::new(ReleasedExportSource {
schema: info.meta.schema.clone(),
channels: std::sync::Mutex::new(Some((started_tx, release_rx))),
}),
));
let catalog = catalog::memory::MemoryCatalogManager::new_with_table(source);
let kv = Arc::new(MemoryKvBackend::new());
let instance = FrontendBuilder::new(
FrontendOptions {
experimental_metric_export: true,
..Default::default()
},
kv.clone(),
test_cache_registry(kv).unwrap(),
catalog,
Arc::new(client::client_manager::NodeClients::default()),
Arc::new(NoopProcedureExecutor),
Arc::new(ProcessManager::new("export-timeout".into(), None)),
)
.with_local_file_access(
common_datasource::object_store::LocalFileAccess::sandboxed(destination.path())
.unwrap(),
)
.try_build()
.await
.unwrap();
let server = servers::http::HttpServerBuilder::new(servers::http::HttpOptions {
timeout: Duration::from_secs(2),
..Default::default()
})
.with_sql_handler(Arc::new(instance))
.build();
let app = server.build(server.make_app()).unwrap();
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
let server_task = tokio::spawn(async move {
axum::serve(listener, app).await.unwrap();
});
let sql = format!(
"COPY DATABASE greptime.public TO '{}/' WITH (experimental_metric_export='true')",
destination.path().display()
);
let request = tokio::spawn(async move {
reqwest::Client::builder()
.no_proxy()
.build()
.unwrap()
.post(format!("http://{addr}/v1/sql"))
.form(&[("sql", sql)])
.send()
.await
.unwrap()
});
tokio::time::timeout(Duration::from_secs(5), started_rx)
.await
.unwrap()
.unwrap();
let response = request.await.unwrap();
assert_eq!(response.status(), reqwest::StatusCode::REQUEST_TIMEOUT);
// A dropped ordinary stream would close this receiver before release.
release_tx.send(()).unwrap();
let file = destination.path().join("source.parquet");
tokio::time::timeout(Duration::from_secs(5), async {
loop {
if std::fs::read(&file)
.is_ok_and(|bytes| bytes.len() > 8 && bytes.ends_with(b"PAR1"))
{
break;
}
tokio::time::sleep(Duration::from_millis(10)).await;
}
})
.await
.unwrap();
server_task.abort();
}
fn pending_table(
table_id: u32,
table_name: &str,
src/frontend/src/instance/builder.rs
+26 -24
View File
@@ -61,7 +61,7 @@ use crate::heartbeat::frontend_peer_addr;
use crate::instance::Instance;
use crate::instance::entity_graph::EntityGraphProviderImpl;
use crate::instance::region_query::FrontendRegionQueryHandler;
use crate::service_config::PendingRowsBatcherOptions;
use crate::service_config::BatcherOptions;
/// The frontend [`Instance`] builder.
pub struct FrontendBuilder {
@@ -237,31 +237,30 @@ impl FrontendBuilder {
};
// The execution-only inserter owns no batchers, avoiding an Arc cycle.
let bulk_inserter = Arc::new(create_inserter());
let build_batcher =
|options: &PendingRowsBatcherOptions| -> Option<Arc<dyn PendingRowsBatcher>> {
if !options.pending_rows_batching_enabled()
|| (self.options.prom_store.with_metric_engine
&& options
.protocols
.iter()
.all(|protocol| *protocol == BatchingProtocol::Prom))
{
return None;
}
TablePendingRowsBatcher::try_new(
options.pending_rows_flush_interval,
options.max_batch_rows,
options.max_concurrent_flushes,
options.worker_channel_capacity,
options.max_inflight_requests,
options.flow_notification_queue_capacity,
bulk_inserter.clone(),
)
.map(|batcher| batcher as Arc<dyn PendingRowsBatcher>)
};
let build_batcher = |options: &BatcherOptions| -> Option<Arc<dyn PendingRowsBatcher>> {
if !options.pending_rows_batching_enabled()
|| (self.options.prom_store.with_metric_engine
&& options
.protocols
.iter()
.all(|protocol| *protocol == BatchingProtocol::Prom))
{
return None;
}
TablePendingRowsBatcher::try_new(
options.pending_rows_flush_interval,
options.max_batch_rows,
options.max_concurrent_flushes,
options.worker_channel_capacity,
options.max_inflight_requests,
options.flow_notification_queue_capacity,
bulk_inserter.clone(),
)
.map(|batcher| batcher as Arc<dyn PendingRowsBatcher>)
};
let inserter = Arc::new(
create_inserter()
.with_pending_rows_batcher(build_batcher(&self.options.pending_rows_batcher)),
.with_pending_rows_batcher(build_batcher(self.options.table_batcher_options())),
);
let deleter = Arc::new(Deleter::new(
self.catalog_manager.clone(),
@@ -382,11 +381,14 @@ impl FrontendBuilder {
self.options.event_recorder.ttl,
)),
self.options.event_recorder.event_types.clone(),
self.options.event_recorder.flush_interval,
));
admin_event_recorder.install(&event_recorder);
Ok(Instance {
logical_batcher: Default::default(),
frontend_peer_addr,
experimental_metric_export: self.options.experimental_metric_export,
catalog_manager: self.catalog_manager,
pipeline_operator,
statement_executor,
src/frontend/src/instance/entity_graph.rs
+84 -21
View File
@@ -23,8 +23,8 @@
//! the query engine. Injected into the catalog manager after the engine is built,
//! breaking the `catalog -> query` cycle.
use std::collections::{BTreeMap, HashMap};
use std::sync::Weak;
use std::collections::{BTreeMap, BTreeSet, HashMap};
use std::sync::{Arc, Weak};
use async_trait::async_trait;
use auth::{
@@ -41,13 +41,16 @@ use common_catalog::consts::{
};
use common_error::ext::{BoxedError, ErrorExt};
use common_error::status_code::StatusCode;
use common_function::scalars::json::json_get::JsonGetWithType;
use common_function::scalars::udf::create_udf;
use common_query::OutputData;
use common_query::prelude::OTLP_AGGREGATION_TEMPORALITY_LABEL;
use common_recordbatch::SendableRecordBatchStream;
use common_telemetry::{debug, warn};
use common_time::timestamp::TimeUnit;
use datafusion::common::{Column, ScalarValue};
use datafusion::dataframe::DataFrame;
use datafusion_expr::LogicalPlan;
use datafusion_expr::{Expr, LogicalPlan, lit};
use futures::TryStreamExt;
use operator::statement::semantic_graph::{
CallsSource, CoDeclaredSource, Conventions, DeclaredSource, ENTITY_TYPE_GEN_AI_AGENT,
@@ -64,8 +67,8 @@ use table::metadata::TableInfo;
use table::predicate::{TimeRangeExtraction, extract_time_range_strict};
use table::requests::{
EntityRole, SEMANTIC_METRIC_TYPE, SEMANTIC_SIGNAL_TYPE, SEMANTIC_SOURCE, SIGNAL_TYPE_METRIC,
SOURCE_OPENTELEMETRY, SOURCE_PROMETHEUS, is_trace_v1_table, parse_entity_columns,
parse_entity_option_key,
SOURCE_OPENTELEMETRY, SOURCE_PROMETHEUS, TABLE_DATA_MODEL_TRACE_V2, is_trace_table,
parse_entity_columns, parse_entity_option_key, trace_v2_attribute,
};
use crate::error;
@@ -164,12 +167,11 @@ impl EntityGraphProviderImpl {
|(entity_type, (id_columns, descriptive_columns, scope_columns))| {
// A stale declaration (e.g. its column was dropped later)
// must not poison every graph scan; skip it.
let schema = &table_info.meta.schema;
if let Some(missing) = id_columns
.iter()
.chain(&descriptive_columns)
.chain(&scope_columns)
.find(|c| schema.column_schema_by_name(c).is_none())
.find(|c| !Self::can_resolve_entity_column(table_info, c))
{
warn!(
"Skipping entity declaration `{}` of table `{}`: column `{}` not found",
@@ -194,7 +196,7 @@ impl EntityGraphProviderImpl {
}
/// All entity declarations of one table: the explicit options plus the
/// zero-configuration conventions (`otlp_trace_entities` for trace-v1
/// zero-configuration conventions (`otlp_trace_entities` for trace-v1/v2
/// tables — including the `service` identity of tables created before the
/// ingest-side auto-stamp — and the Prometheus/OTel descriptor
/// whitelists). An explicit declaration of a type always suppresses the
@@ -207,7 +209,7 @@ impl EntityGraphProviderImpl {
) -> Vec<EntityDeclaration> {
let mut declarations = Self::parse_declarations(table_info);
let mut supersessions = Vec::new();
if is_trace_v1_table(table_info) {
if is_trace_table(table_info) {
Self::extend_with_implicit_entities(
table_info,
&conventions.otlp_trace_entities,
@@ -303,8 +305,8 @@ impl EntityGraphProviderImpl {
}
/// Synthesizes the applicable subset of `entities` on `table_info`:
/// explicit declarations win, every id column must exist (no guessing),
/// descriptive columns are filtered to those present.
/// explicit declarations win, every id reference must be resolvable,
/// and descriptive references are filtered by the same rule.
fn extend_with_implicit_entities(
table_info: &TableInfo,
entities: &[ImplicitEntity],
@@ -330,7 +332,7 @@ impl EntityGraphProviderImpl {
if let Some(missing) = implicit
.id
.iter()
.find(|c| schema.column_schema_by_name(c).is_none())
.find(|c| !Self::can_resolve_entity_column(table_info, c))
{
debug!(
"Table `{}` lacks the id column `{}`; skipping the implicit `{}` declaration",
@@ -350,7 +352,7 @@ impl EntityGraphProviderImpl {
implicit
.descriptive
.iter()
.filter(|c| schema.column_schema_by_name(c).is_some())
.filter(|c| Self::can_resolve_entity_column(table_info, c))
.cloned()
.collect()
};
@@ -359,7 +361,7 @@ impl EntityGraphProviderImpl {
let id_qualifier = implicit
.qualified_by
.clone()
.filter(|c| schema.column_schema_by_name(c).is_some());
.filter(|c| Self::can_resolve_entity_column(table_info, c));
if let Some(entity_type) = &implicit.superseded_by {
supersessions.push((declarations.len(), entity_type.clone()));
}
@@ -434,7 +436,7 @@ impl EntityGraphProviderImpl {
while let Some(table) = tables.try_next().await.map_err(BoxedError::new)? {
let table_info = table.table_info();
let table_declarations = Self::declarations_for(&table_info, conventions);
let is_trace = is_trace_v1_table(&table_info);
let is_trace = is_trace_table(&table_info);
// Authorize only tables that would contribute rows.
if per_table_auth
&& (is_trace || !table_declarations.is_empty())
@@ -532,8 +534,63 @@ impl EntityGraphProviderImpl {
}
}
fn read_table(&self, table: TableRef) -> Result<DataFrame, BoxedError> {
self.query_engine.read_table(table).map_err(BoxedError::new)
fn can_resolve_entity_column(table_info: &TableInfo, column: &str) -> bool {
let table_schema = &table_info.meta.schema;
if table_schema.column_schema_by_name(column).is_some() {
return true;
}
let data_model = table_info.meta.options.data_model();
trace_v2_attribute(table_schema, data_model, column).is_some()
}
/// Projects referenced V2 attributes under their declaration names for
/// the shared entity and relationship plans. Other models scan unchanged.
fn read_table(
&self,
table: TableRef,
declarations: &[EntityDeclaration],
) -> Result<DataFrame, BoxedError> {
let info = table.table_info();
let mut scan = self
.query_engine
.read_table(table)
.map_err(BoxedError::new)?;
let data_model = info.meta.options.data_model();
if data_model != Some(TABLE_DATA_MODEL_TRACE_V2) {
return Ok(scan);
}
// Read Trace V2 table:
let mut columns = BTreeSet::new();
for declaration in declarations {
columns.extend(declaration.id_columns.iter());
columns.extend(declaration.descriptive_columns.iter());
columns.extend(declaration.scope_columns.iter());
columns.extend(declaration.id_qualifier.iter());
columns.extend(declaration.superseded_by_columns.iter());
}
let conventions = conventions()
.map_err(datafusion::error::DataFusionError::Internal)
.context(error::DataFusionSnafu)
.map_err(BoxedError::new)?;
let virtual_candidates = &conventions.virtual_dst_candidates;
columns.extend(virtual_candidates.iter().map(|candidate| &candidate.column));
let get = create_udf(Arc::new(JsonGetWithType::default()));
for column in columns {
if let Some((root, key)) = trace_v2_attribute(&info.meta.schema, data_model, column) {
let path = format!("$.{}", serde_json::Value::String(key.to_string()));
let value = get.call(vec![
Expr::Column(Column::from_name(root)),
lit(path),
lit(ScalarValue::Utf8View(None)),
]);
scan = scan
.with_column(column, value)
.context(error::DataFusionSnafu)
.map_err(BoxedError::new)?;
}
}
Ok(scan)
}
/// The declared-edge branch source, when the physical table exists, the
@@ -587,7 +644,7 @@ impl EntityGraphProviderImpl {
));
}
Ok(Some(DeclaredSource {
scan: self.read_table(table)?,
scan: self.read_table(table, &[])?,
}))
}
@@ -633,8 +690,8 @@ impl EntityGraphProvider for EntityGraphProviderImpl {
let mut plans = Vec::with_capacity(sources.len());
for source in sources {
plans.push(RegistrySource {
scan: self.read_table(source.table, &source.declarations)?,
declarations: source.declarations,
scan: self.read_table(source.table)?,
});
}
let Some(window) = Self::query_window(&request)? else {
@@ -658,18 +715,24 @@ impl EntityGraphProvider for EntityGraphProviderImpl {
let (sources, traces) = self.enumerate(catalog, query_ctx.as_deref()).await?;
let mut calls = Vec::with_capacity(traces.len());
for trace in traces {
let declarations = trace
.service
.iter()
.chain(trace.agent.iter())
.cloned()
.collect::<Vec<_>>();
calls.push(CallsSource {
scan: self.read_table(trace.table, &declarations)?,
service: trace.service,
agent: trace.agent,
scan: self.read_table(trace.table)?,
});
}
let mut co_declared = Vec::with_capacity(sources.len());
for source in sources {
co_declared.push(CoDeclaredSource {
scan: self.read_table(source.table, &source.declarations)?,
declarations: source.declarations,
is_trace: source.is_trace,
scan: self.read_table(source.table)?,
});
}
let declared = self.declared_source(catalog, query_ctx.as_deref()).await?;
src/frontend/src/instance/export_database.rs
+68
View File
@@ -12,14 +12,24 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use std::sync::Arc;
use auth::{
PermissionAction, PermissionChecker, PermissionCheckerRef, PermissionReq,
PermissionTableTarget, PermissionTableTargets,
};
use common_error::ext::BoxedError;
use common_query::Output;
use common_recordbatch::RecordBatches;
use common_telemetry::error;
use datatypes::prelude::ConcreteDataType;
use datatypes::schema::{ColumnSchema, Schema};
use datatypes::vectors::StringVector;
use operator::statement::export_database::{DatabaseExportSummary, PreparedDatabaseExport};
use session::context::QueryContextRef;
use snafu::ResultExt;
use sql::ast::{Ident, ObjectName};
use sql::statements::OptionMap;
use sql::statements::copy::{Copy, CopyDatabase, CopyDatabaseArgument};
use sql::statements::statement::Statement;
use table::requests::CopyDatabaseRequest;
@@ -28,7 +38,65 @@ use tokio_util::sync::CancellationToken;
use crate::error::{PermissionSnafu, Result};
use crate::instance::Instance;
pub(crate) fn parse_metric_export_requested(options: &OptionMap) -> Result<bool> {
match options.get("experimental_metric_export") {
None | Some("false") => Ok(false),
Some("true") => Ok(true),
Some(_) => Err(operator::error::InvalidDatabaseExportSnafu {
reason: "experimental_metric_export must be true or false",
}
.build()
.into()),
}
}
impl Instance {
pub(crate) fn show_metric_export_capability(&self) -> Result<Output> {
let schema = Arc::new(Schema::new(vec![ColumnSchema::new(
"EXPERIMENTAL_METRIC_EXPORT",
ConcreteDataType::string_datatype(),
false,
)]));
let batches = RecordBatches::try_from_columns(
schema,
vec![Arc::new(StringVector::from(vec![
self.experimental_metric_export.to_string(),
])) as datatypes::vectors::VectorRef],
)
.map_err(BoxedError::new)
.context(crate::error::ExternalSnafu)?;
Ok(Output::new_with_record_batches(batches))
}
pub(crate) async fn copy_metric_database(
&self,
arg: CopyDatabaseArgument,
ctx: QueryContextRef,
) -> Result<Output> {
if !self.experimental_metric_export {
return Err(operator::error::InvalidDatabaseExportSnafu {
reason: "experimental_metric_export is disabled on this frontend",
}
.build()
.into());
}
let req = operator::statement::to_copy_database_request(arg, &ctx)?;
let plan = self.prepare_database_export(req, None, &ctx).await?;
let cancellation = CancellationToken::new();
let _guard = cancellation.clone().drop_guard();
let executor = self.statement_executor.clone();
// Dropping the request only cancels; the runtime task retains and drains started I/O.
let task = common_runtime::spawn_query(async move {
let result = executor.export_database(plan, &cancellation, ctx).await;
if let Err(err) = &result {
error!(err; "Experimental database export failed after draining started work");
}
result
});
let summary = task.await.context(operator::error::JoinTaskSnafu)??;
Ok(Output::new_with_affected_rows(summary.rows))
}
#[allow(dead_code)]
async fn export_database(
&self,
src/frontend/src/instance/grpc.rs
+1
View File
@@ -454,6 +454,7 @@ impl Instance {
request.record_batch,
request.schema_bytes,
ctx.skip_wal(),
ctx.channel(),
)
.await
.context(TableOperationSnafu)?;
src/frontend/src/instance/import_packed.rs
+114
View File
@@ -0,0 +1,114 @@
// Copyright 2023 Greptime Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use auth::{PermissionReq, PermissionTableTarget, PermissionTableTargets};
use common_query::Output;
use session::context::QueryContextRef;
use snafu::ResultExt;
use sql::statements::copy::CopyDatabaseArgument;
use sql::statements::statement::Statement;
use tokio_util::sync::CancellationToken;
use crate::error::{PermissionSnafu, Result};
use crate::instance::Instance;
impl Instance {
pub(crate) async fn copy_packed_database(
&self,
arg: CopyDatabaseArgument,
stmt: &Statement,
ctx: QueryContextRef,
) -> Result<Output> {
let req = operator::statement::to_copy_database_request(arg, &ctx)?;
let plan = self
.statement_executor
.prepare_packed_import(req, &ctx)
.await?;
let targets = PermissionTableTargets::resolved(
plan.tables
.iter()
.map(|t| {
let info = t.table_info();
PermissionTableTarget::new(&info.catalog_name, &info.schema_name, &info.name)
})
.collect(),
);
self.check_table_permission(&ctx, PermissionReq::SqlStatement(stmt), targets)
.context(PermissionSnafu)?;
let executor = self.statement_executor.clone();
run_packed_import(move |cancellation| async move {
executor.import_packed(plan, &cancellation, ctx).await
})
.await
}
}
async fn run_packed_import<F, Fut>(worker: F) -> Result<Output>
where
F: FnOnce(CancellationToken) -> Fut,
Fut: std::future::Future<Output = operator::error::Result<Output>> + Send + 'static,
{
let cancellation = CancellationToken::new();
let _guard = cancellation.clone().drop_guard();
// Transport timeout/drop signals cancellation; the worker retains ownership
// until its started inserts finish, even after the response is gone.
common_runtime::spawn_query(worker(cancellation))
.await
.context(operator::error::JoinTaskSnafu)?
.map_err(Into::into)
}
#[cfg(test)]
mod tests {
use std::time::Duration;
use tokio::sync::oneshot;
use super::*;
#[tokio::test]
async fn packed_request_drop_keeps_worker_alive_until_cleanup() {
let (started_tx, started_rx) = oneshot::channel();
let (cancelled_tx, cancelled_rx) = oneshot::channel();
let (drain_tx, drain_rx) = oneshot::channel();
let (finished_tx, mut finished_rx) = oneshot::channel();
let mut request = Box::pin(run_packed_import(move |cancellation| async move {
started_tx.send(()).unwrap();
cancellation.cancelled().await;
cancelled_tx.send(()).unwrap();
// Model an already-started insert that must finish before worker exit.
drain_rx.await.unwrap();
finished_tx.send(()).unwrap();
Ok(Output::new_with_affected_rows(1))
}));
tokio::select! {
_ = started_rx => {}
_ = &mut request => panic!("worker must wait for cancellation"),
}
drop(request);
tokio::time::timeout(Duration::from_secs(5), cancelled_rx)
.await
.unwrap()
.unwrap();
assert!(matches!(
finished_rx.try_recv(),
Err(oneshot::error::TryRecvError::Empty)
));
drain_tx.send(()).unwrap();
tokio::time::timeout(Duration::from_secs(5), finished_rx)
.await
.unwrap()
.unwrap();
}
}
src/frontend/src/instance/jaeger.rs
+76 -14
View File
@@ -23,16 +23,18 @@ use common_catalog::consts::{
};
use common_function::function::FunctionRef;
use common_function::scalars::json::json_get::{
JsonGetBool, JsonGetFloat, JsonGetInt, JsonGetString,
JsonGetBool, JsonGetFloat, JsonGetInt, JsonGetString, JsonGetWithType,
};
use common_function::scalars::udf::create_udf;
use common_query::{Output, OutputData};
use common_recordbatch::adapter::RecordBatchStreamAdapter;
use common_recordbatch::util;
use common_telemetry::warn;
use datafusion::common::ScalarValue;
use datafusion::dataframe::DataFrame;
use datafusion::execution::SessionStateBuilder;
use datafusion::execution::context::SessionContext;
use datafusion::functions::core::expr_fn::coalesce;
use datafusion::functions_window::expr_fn::row_number;
use datafusion_expr::select_expr::SelectExpr;
use datafusion_expr::{Expr, ExprFunctionExt, SortExpr, col, lit, lit_timestamp_nano, wildcard};
@@ -43,9 +45,9 @@ use servers::error::{
};
use servers::http::jaeger::{JAEGER_QUERY_TABLE_NAME_KEY, QueryTraceParams, TraceUserAgent};
use servers::otlp::trace::{
DURATION_NANO_COLUMN, KEY_OTEL_STATUS_ERROR_KEY, SERVICE_NAME_COLUMN, SPAN_ATTRIBUTES_COLUMN,
SPAN_KIND_COLUMN, SPAN_KIND_PREFIX, SPAN_NAME_COLUMN, SPAN_STATUS_CODE, SPAN_STATUS_ERROR,
TIMESTAMP_COLUMN, TRACE_ID_COLUMN,
DURATION_NANO_COLUMN, KEY_OTEL_STATUS_ERROR_KEY, RESOURCE_ATTRIBUTES_COLUMN,
SERVICE_NAME_COLUMN, SPAN_ATTRIBUTES_COLUMN, SPAN_KIND_COLUMN, SPAN_KIND_PREFIX,
SPAN_NAME_COLUMN, SPAN_STATUS_CODE, SPAN_STATUS_ERROR, TIMESTAMP_COLUMN, TRACE_ID_COLUMN,
};
use servers::query_handler::JaegerQueryHandler;
use session::context::QueryContextRef;
@@ -367,11 +369,11 @@ async fn query_trace_table(
schema: ctx.current_schema(),
})?;
let is_data_model_v1 = table::requests::is_trace_v1_table(&table.table_info());
let table_info = table.table_info();
let data_model = table_info.meta.options.data_model();
// collect to set
let col_names = table
.table_info()
let col_names = table_info
.meta
.field_column_names()
.map(|s| format!("\"{}\"", s))
@@ -389,7 +391,7 @@ async fn query_trace_table(
let dataframe = filters
.into_iter()
.chain(tags.map_or(Ok(vec![]), |t| {
tags_filters(&dataframe, t, is_data_model_v1, &col_names)
tags_filters(&dataframe, t, data_model, &col_names)
})?)
.try_fold(dataframe, |df, expr| {
df.filter(expr).context(DataFusionSnafu)
@@ -518,8 +520,9 @@ fn create_df_context(query_engine: &QueryEngineRef) -> ServerResult<SessionConte
SessionStateBuilder::new_from_existing(query_engine.engine_state().session_state()).build(),
);
// The following JSON UDFs will be used for tags filters on v0 data model.
// JSON UDFs used by the v0 and v2 tag filters.
let udfs: Vec<FunctionRef> = vec![
Arc::new(JsonGetWithType::default()),
Arc::new(JsonGetInt::default()),
Arc::new(JsonGetFloat::default()),
Arc::new(JsonGetBool::default()),
@@ -600,6 +603,65 @@ fn json_tag_filters(
Ok(filters)
}
/// Resolves each tag per row, falling back to Resource when the typed Span value is null.
fn json2_tag_filters(
dataframe: &DataFrame,
tags: HashMap<String, JsonValue>,
) -> ServerResult<Vec<Expr>> {
let get = dataframe
.registry()
.udf(JsonGetWithType::NAME)
.context(DataFusionSnafu)?;
tags.into_iter()
.map(|(key, value)| {
if key == KEY_OTEL_STATUS_ERROR_KEY && value == JsonValue::Bool(true) {
return Ok(col(SPAN_STATUS_CODE).eq(lit(SPAN_STATUS_ERROR)));
}
// JSON quoting preserves literal dots, quotes, and backslashes in attribute keys.
let path = lit(format!("$[{}]", JsonValue::String(key)));
let (value, value_type) = match value {
JsonValue::String(value) => (
ScalarValue::Utf8View(Some(value)),
ScalarValue::Utf8View(None),
),
JsonValue::Bool(value) => (
ScalarValue::Boolean(Some(value)),
ScalarValue::Boolean(None),
),
JsonValue::Number(value) => {
if let Some(value) = value.as_i64() {
(ScalarValue::Int64(Some(value)), ScalarValue::Int64(None))
} else if let Some(value) = value.as_u64() {
(ScalarValue::UInt64(Some(value)), ScalarValue::UInt64(None))
} else if let Some(value) = value.as_f64() {
(
ScalarValue::Float64(Some(value)),
ScalarValue::Float64(None),
)
} else {
return Ok(lit(false));
}
}
JsonValue::Null => (ScalarValue::Utf8View(None), ScalarValue::Utf8View(None)),
JsonValue::Array(_) | JsonValue::Object(_) => return Ok(lit(false)),
};
let attribute = coalesce(vec![
get.call(vec![
col(SPAN_ATTRIBUTES_COLUMN),
path.clone(),
lit(value_type.clone()),
]),
get.call(vec![col(RESOURCE_ATTRIBUTES_COLUMN), path, lit(value_type)]),
]);
Ok(if value.is_null() {
attribute.is_null()
} else {
attribute.eq(lit(value))
})
})
.collect()
}
/// Helper function to check if span_key or resource_key exists in col_names and create an expression.
/// If neither exists, logs a warning and returns None.
#[inline]
@@ -679,13 +741,13 @@ fn flatten_tag_filters(
fn tags_filters(
dataframe: &DataFrame,
tags: HashMap<String, JsonValue>,
is_data_model_v1: bool,
data_model: Option<&str>,
col_names: &HashSet<String>,
) -> ServerResult<Vec<Expr>> {
if is_data_model_v1 {
flatten_tag_filters(tags, col_names)
} else {
json_tag_filters(dataframe, tags)
match data_model {
Some(table::requests::TABLE_DATA_MODEL_TRACE_V1) => flatten_tag_filters(tags, col_names),
Some(table::requests::TABLE_DATA_MODEL_TRACE_V2) => json2_tag_filters(dataframe, tags),
_ => json_tag_filters(dataframe, tags),
}
}
src/frontend/src/instance/log_handler.rs
+5
View File
@@ -80,6 +80,11 @@ impl PipelineHandler for Instance {
prepared.push((Arc::new(ctx.fork()), log));
}
operator::insert::admit_row_insert_batches(&mut prepared)
.await
.map_err(BoxedError::new)
.context(servers::error::ExecuteGrpcQuerySnafu)?;
let mut outputs = Vec::with_capacity(prepared.len());
for (ctx, log) in prepared {
outputs.push(self.handle_log_inserts(log, ctx).await);
src/frontend/src/instance/logical_batcher.rs
+102
View File
@@ -0,0 +1,102 @@
// Copyright 2023 Greptime Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use std::sync::{Arc, Weak};
use api::v1::ColumnSchema;
use async_trait::async_trait;
use servers::batcher::logical_table::{LogicalTablePendingRowsBatcher, PendingRowsSchemaAlterer};
use servers::error::{BatcherChannelClosedSnafu, Result};
use servers::http::BatchingProtocol;
use session::context::QueryContextRef;
use snafu::OptionExt;
use crate::frontend::FrontendOptions;
use crate::instance::Instance;
impl Instance {
pub(crate) fn init_logical_batcher(self: &Arc<Self>, options: &FrontendOptions) {
self.logical_batcher.get_or_init(|| {
let options_batcher = options.logical_batcher_options();
let enabled = options_batcher
.protocols
.iter()
.any(|protocol| match protocol {
BatchingProtocol::Prom => options.prom_store.enable,
BatchingProtocol::Otlp => options.otlp.enable,
_ => false,
});
if !options.prom_store.with_metric_engine
|| !enabled
|| !options_batcher.pending_rows_batching_enabled()
{
return None;
}
LogicalTablePendingRowsBatcher::try_new(
self.partition_manager().clone(),
self.node_manager().clone(),
self.catalog_manager().clone(),
self.table_flownode_set_cache().clone(),
true,
Arc::new(LogicalTables(Arc::downgrade(self))),
options_batcher.pending_rows_flush_interval,
options_batcher.max_batch_rows,
options_batcher.max_concurrent_flushes,
options_batcher.worker_channel_capacity,
options_batcher.max_inflight_requests,
options_batcher.flow_notification_queue_capacity,
)
});
}
pub(crate) fn logical_batcher(&self) -> Option<&Arc<LogicalTablePendingRowsBatcher>> {
self.logical_batcher.get().and_then(Option::as_ref)
}
}
// The instance owns the shared batcher; schema preparation must not keep that
// owner alive through a reference cycle.
struct LogicalTables(Weak<Instance>);
#[async_trait]
impl PendingRowsSchemaAlterer for LogicalTables {
async fn create_tables_if_missing_batch(
&self,
catalog: &str,
schema: &str,
tables: &[(&str, &[ColumnSchema])],
with_metric_engine: bool,
ctx: QueryContextRef,
) -> Result<()> {
self.0
.upgrade()
.context(BatcherChannelClosedSnafu)?
.create_tables_if_missing_batch(catalog, schema, tables, with_metric_engine, ctx)
.await
}
async fn add_missing_prom_tag_columns_batch(
&self,
catalog: &str,
schema: &str,
tables: &[(&str, &[String])],
ctx: QueryContextRef,
) -> Result<()> {
self.0
.upgrade()
.context(BatcherChannelClosedSnafu)?
.add_missing_prom_tag_columns_batch(catalog, schema, tables, ctx)
.await
}
}
src/frontend/src/instance/otlp.rs
+76 -13
View File
@@ -27,9 +27,11 @@ use client::Output;
use common_catalog::consts::{trace_operations_table_name, trace_services_table_name};
use common_error::ext::BoxedError;
use common_query::prelude::GREPTIME_PHYSICAL_TABLE;
use common_query::{OutputData, OutputMeta};
use common_telemetry::{tracing, warn};
use opentelemetry_proto::tonic::collector::logs::v1::ExportLogsServiceRequest;
use opentelemetry_proto::tonic::collector::trace::v1::ExportTraceServiceRequest;
use operator::insert::{Inserter, admit_row_insert_batches, admit_write};
use otel_arrow_rust::proto::opentelemetry::collector::metrics::v1::ExportMetricsServiceRequest;
use pipeline::{GreptimePipelineParams, PipelineWay};
use servers::error::{self, AuthSnafu, Result as ServerResult};
@@ -122,7 +124,7 @@ impl OpenTelemetryProtocolHandler for Instance {
metric_ctx.resource_info = self.otlp_resource_info;
let otlp::metrics::MetricsConversion {
requests,
mut requests,
rows,
semantic_index,
resource_info,
@@ -141,6 +143,10 @@ impl OpenTelemetryProtocolHandler for Instance {
self.check_row_insert_permission(&requests, &ctx, PermissionReq::Action(OTLP_WRITE))
.context(AuthSnafu)?;
let ctx = admit_write(rows as u64, &ctx)
.await
.map_err(BoxedError::new)
.context(error::ExecuteGrpcQuerySnafu)?;
self.cache_otlp_legacy(&input_names, &ctx, is_legacy)?;
OTLP_METRICS_ROWS.inc_by(rows as u64);
@@ -160,27 +166,74 @@ impl OpenTelemetryProtocolHandler for Instance {
Arc::new(c)
};
let physical_table = ctx
.extension(PHYSICAL_TABLE_PARAM)
.unwrap_or(GREPTIME_PHYSICAL_TABLE)
.to_string();
let batcher = self.logical_batcher().filter(|_| {
ctx.logical_batching_enabled() && !metric_ctx.is_legacy && metric_ctx.with_metric_engine
});
let batcher = if batcher.is_some() {
// Align the requests' time index unit with the physical table's
// before the bulk eligibility check: the OTLP encoder keeps
// nanosecond precision on the metric engine path, while the bulk
// path assumes the physical table's unit (today millisecond).
// Without this, nanosecond requests would silently skip the
// batcher, and a non-millisecond physical table must not enter
// the bulk path either (the eligibility check rejects its unit).
self.inserter
.align_metric_row_inserts_time_unit(&ctx, &physical_table, &mut requests)
.await
.map_err(BoxedError::new)
.context(error::ExecuteGrpcQuerySnafu)?;
if self
.inserter
.can_batch_metric_rows(&requests, &ctx, &physical_table)
.await
.map_err(BoxedError::new)
.context(error::ExecuteGrpcQuerySnafu)?
{
batcher
} else {
None
}
} else {
None
};
// OTLP tables have one sample field in both the legacy and physical paths.
let output = if metric_ctx.is_legacy || !metric_ctx.with_metric_engine {
let output = if let Some(batcher) = batcher {
let (rows, cost) = batcher
.submit_with(requests, ctx.clone(), |mut requests| {
let ctx = ctx.clone();
async move {
Inserter::meter_row_inserts(&mut requests, &ctx)
.await
.map_err(BoxedError::new)
.context(error::ExecuteGrpcQuerySnafu)
}
})
.await?;
Output::new(
OutputData::AffectedRows(rows as usize),
OutputMeta::new_with_cost(cost as _),
)
} else if metric_ctx.is_legacy || !metric_ctx.with_metric_engine {
self.handle_row_inserts(requests, ctx.clone(), false, true)
.await
.map_err(BoxedError::new)
.context(error::ExecuteGrpcQuerySnafu)
.context(error::ExecuteGrpcQuerySnafu)?
} else {
let physical_table = ctx
.extension(PHYSICAL_TABLE_PARAM)
.unwrap_or(GREPTIME_PHYSICAL_TABLE)
.to_string();
self.handle_metric_row_inserts(requests, ctx.clone(), physical_table)
.await
.map_err(BoxedError::new)
.context(error::ExecuteGrpcQuerySnafu)
}?;
.context(error::ExecuteGrpcQuerySnafu)?
};
outcome.write_cost = output.meta.cost;
// Derived enrichment, written after the metric data is committed:
// failing here would make the client retry data the server already
// accepted, so every failure degrades to a warning instead.
// Derived enrichment follows the accepted metric submission, which may
// still be queued in asynchronous mode. Failures remain warning-only
// to avoid retrying metric data the server already accepted.
if let Some(resource_info) = resource_info {
let written = match self.check_row_insert_permission(
&resource_info,
@@ -239,6 +292,12 @@ impl OpenTelemetryProtocolHandler for Instance {
let targets = trace_permission_targets(&table_name, &spans, &ctx);
self.check_table_permission(&ctx, PermissionReq::Action(OTLP_WRITE), targets)
.context(AuthSnafu)?;
// Count the external spans once, before chunking, retries, or derived tables.
let rows = spans.iter().map(|group| group.spans.len() as u64).sum();
let ctx = admit_write(rows, &ctx)
.await
.map_err(BoxedError::new)
.context(error::ExecuteGrpcQuerySnafu)?;
self.ingest_trace_spans(&pipeline, table_name, spans, &conventions, ctx)
.await
}
@@ -284,11 +343,15 @@ impl OpenTelemetryProtocolHandler for Instance {
)
.await?;
let batches = opt_req.as_req_iter(ctx).collect::<Vec<_>>();
let mut batches = opt_req.as_req_iter(ctx).collect::<Vec<_>>();
for (temp_ctx, requests) in &batches {
self.check_row_insert_permission(requests, temp_ctx, PermissionReq::Action(OTLP_WRITE))
.context(AuthSnafu)?;
}
admit_row_insert_batches(&mut batches)
.await
.map_err(BoxedError::new)
.context(error::ExecuteGrpcQuerySnafu)?;
let mut outputs = Vec::with_capacity(batches.len());
for (temp_ctx, requests) in batches {
src/frontend/src/instance/otlp/README.md
+4
View File
@@ -68,6 +68,10 @@ The parser in
[`servers/src/otlp/trace/span.rs`](../../../../servers/src/otlp/trace/span.rs)
produces one `TraceSpanGroup` per resource/scope pair.
After permission checks, admission counts all spans in the request once per
database. Chunks, schema retries, and derived lookup-table writes retain that
admission; their actual writes still contribute WCU accounting.
`ingest_trace_spans` in [`trace_ingest.rs`](trace_ingest.rs) splits the spans in
each group into owned chunks. `trace_ingest_chunk_size` defaults to 512; setting
it to 0 disables splitting. The option is defined in

Some files were not shown because too many files have changed in this diff Show More