mirror of
https://github.com/GreptimeTeam/greptimedb.git
synced 2026-11-17 00:07:00 +00:00
Merge remote-tracking branch 'upper/main' into pr9099
This commit is contained in:
@@ -108,7 +108,7 @@ agent, which Aliyun public Ubuntu images include.
|
||||
|
||||
An allowlisted repository admin commenting `/query-regression` on the PR is
|
||||
**trust admission for that exact PR revision**. `/query-regression` runs the
|
||||
six routine default cases; `/query-regression heavy` runs only the
|
||||
nine routine default cases; `/query-regression heavy` runs only the
|
||||
high-cardinality `prom_remote_write_7913` remote-write case.
|
||||
`slash-command-dispatch.yml` (`issue_comment` on the default branch) parses
|
||||
the command and `repository_dispatch`es; `query-regression-slash.yml` admits
|
||||
|
||||
@@ -33,6 +33,7 @@ from typing import Any
|
||||
|
||||
DEFAULT_CASES = [
|
||||
"tests/perf/query_cases/smoke_direct_sst/case.toml",
|
||||
"tests/perf/query_cases/sst_float_bss/case.toml",
|
||||
"tests/perf/query_cases/prom_remote_write_seeded_random/case.toml",
|
||||
"tests/perf/query_cases/prom_remote_write_run_heavy/case.toml",
|
||||
"tests/perf/query_cases/prom_remote_write_mixed_every/case.toml",
|
||||
|
||||
@@ -45,6 +45,7 @@ jobs:
|
||||
PR_TITLE: ${{ github.event.pull_request.title }}
|
||||
PR_BODY: ${{ github.event.pull_request.body }}
|
||||
AUTHOR: ${{ github.event.pull_request.merged_by.login }}
|
||||
PR_AUTHOR: ${{ github.event.pull_request.user.login }}
|
||||
BASE_BRANCH: ${{ github.event.pull_request.base.ref }}
|
||||
run: |
|
||||
git config user.name "github-actions[bot]"
|
||||
@@ -76,6 +77,7 @@ jobs:
|
||||
git branch -D "${head_branch}"
|
||||
gh issue create \
|
||||
--title "Backport failure: PR #${PR_NUMBER} to ${release_branch}" \
|
||||
--assignee "${PR_AUTHOR}" \
|
||||
--body "Cherry-pick of ${MERGE_COMMIT} (PR #${PR_NUMBER}) failed on ${release_branch}. Manual backport required." \
|
||||
|| true
|
||||
continue
|
||||
|
||||
@@ -781,10 +781,9 @@ jobs:
|
||||
kubectl -n kafka-cluster apply \
|
||||
-f .github/actions/setup-greptimedb-cluster/kafka-wal-helper.yaml
|
||||
|
||||
kubectl -n kafka-cluster wait \
|
||||
--for=condition=Ready \
|
||||
pod -l app=kafka-wal-helper \
|
||||
--timeout=120s
|
||||
kubectl rollout status deployment/kafka-wal-helper \
|
||||
--timeout=120s \
|
||||
-n kafka-cluster
|
||||
- name: Print etcd info
|
||||
shell: bash
|
||||
run: kubectl get all --show-labels -n etcd-cluster
|
||||
|
||||
@@ -6,9 +6,13 @@ on:
|
||||
- opened
|
||||
- reopened
|
||||
- edited
|
||||
- synchronize
|
||||
- ready_for_review
|
||||
merge_group:
|
||||
types: [checks_requested]
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
@@ -18,6 +22,9 @@ permissions:
|
||||
|
||||
jobs:
|
||||
check:
|
||||
name: Semantic PR title
|
||||
# Validate before queue admission; report a skipped check for merge groups.
|
||||
if: github.event_name == 'pull_request_target'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
steps:
|
||||
|
||||
Generated
+473
-415
File diff suppressed because it is too large
Load Diff
+50
-45
@@ -8,6 +8,7 @@ members = [
|
||||
"src/client",
|
||||
"src/cmd",
|
||||
"src/common/base",
|
||||
"src/common/batcher",
|
||||
"src/common/catalog",
|
||||
"src/common/config",
|
||||
"src/common/datasource",
|
||||
@@ -100,13 +101,13 @@ rust.unexpected_cfgs = { level = "warn", check-cfg = ['cfg(tokio_unstable)'] }
|
||||
# See for more detaiils: https://github.com/rust-lang/cargo/issues/11329
|
||||
ahash = { version = "0.8", features = ["compile-time-rng"] }
|
||||
aquamarine = "0.6"
|
||||
arrow = { version = "58.3", features = ["prettyprint"] }
|
||||
arrow-array = { version = "58.3", default-features = false, features = ["chrono-tz"] }
|
||||
arrow-buffer = "58.3"
|
||||
arrow-cast = "58.3"
|
||||
arrow-flight = "58.3"
|
||||
arrow-ipc = { version = "58.3", default-features = false, features = ["lz4", "zstd"] }
|
||||
arrow-schema = { version = "58.3", features = ["serde"] }
|
||||
arrow = { version = "=59.2.0", features = ["prettyprint"] }
|
||||
arrow-array = { version = "=59.2.0", default-features = false, features = ["chrono-tz"] }
|
||||
arrow-buffer = "=59.2.0"
|
||||
arrow-cast = "=59.2.0"
|
||||
arrow-flight = "=59.2.0"
|
||||
arrow-ipc = { version = "=59.2.0", default-features = false, features = ["lz4", "zstd"] }
|
||||
arrow-schema = { version = "=59.2.0", features = ["serde"] }
|
||||
async-stream = "0.3"
|
||||
async-trait = "0.1"
|
||||
# Remember to update axum-extra, axum-macros when updating axum
|
||||
@@ -128,22 +129,22 @@ const_format = "0.2"
|
||||
criterion = "0.7"
|
||||
crossbeam-utils = "0.8"
|
||||
dashmap = "6.1"
|
||||
datafusion = "=53.1.0"
|
||||
datafusion-common = "=53.1.0"
|
||||
datafusion-datasource = "=53.1.0"
|
||||
datafusion-expr = "=53.1.0"
|
||||
datafusion-expr-common = "=53.1.0"
|
||||
datafusion-functions = "=53.1.0"
|
||||
datafusion-functions-aggregate-common = "=53.1.0"
|
||||
datafusion-functions-window-common = "=53.1.0"
|
||||
datafusion-optimizer = "=53.1.0"
|
||||
datafusion-orc = { git = "https://github.com/datafusion-contrib/datafusion-orc.git", rev = "6c07fa282dc8d62db2aa4ded06ab55485efc811a" }
|
||||
datafusion-pg-catalog = "0.17.3"
|
||||
datafusion-physical-expr = "=53.1.0"
|
||||
datafusion-physical-plan = "=53.1.0"
|
||||
datafusion-proto = "=53.1.0"
|
||||
datafusion-sql = "=53.1.0"
|
||||
datafusion-substrait = "=53.1.0"
|
||||
datafusion = "=55.0.0"
|
||||
datafusion-common = "=55.0.0"
|
||||
datafusion-datasource = "=55.0.0"
|
||||
datafusion-expr = "=55.0.0"
|
||||
datafusion-expr-common = "=55.0.0"
|
||||
datafusion-functions = "=55.0.0"
|
||||
datafusion-functions-aggregate-common = "=55.0.0"
|
||||
datafusion-functions-window-common = "=55.0.0"
|
||||
datafusion-optimizer = "=55.0.0"
|
||||
datafusion-orc = "0.10.0"
|
||||
datafusion-pg-catalog = "0.18.3"
|
||||
datafusion-physical-expr = "=55.0.0"
|
||||
datafusion-physical-plan = "=55.0.0"
|
||||
datafusion-proto = "=55.0.0"
|
||||
datafusion-sql = "=55.0.0"
|
||||
datafusion-substrait = "=55.0.0"
|
||||
datafusion_object_store = { package = "object_store", version = "0.13.2" }
|
||||
deadpool = "0.12"
|
||||
deadpool-postgres = "0.14"
|
||||
@@ -159,7 +160,7 @@ fs2 = "0.4"
|
||||
fst = "0.4.7"
|
||||
futures = "0.3"
|
||||
futures-util = "0.3"
|
||||
greptime-proto = { git = "https://github.com/GreptimeTeam/greptime-proto.git", rev = "b74667d17827481dbb6222fa0326163627211557" }
|
||||
greptime-proto = { git = "https://github.com/GreptimeTeam/greptime-proto.git", rev = "0c4beffaa2b3968034915867593a717c6ab0c6b4" }
|
||||
hex = "0.4"
|
||||
hostname = "0.4.0"
|
||||
http = "1"
|
||||
@@ -195,10 +196,10 @@ otel-arrow-rust = { git = "https://github.com/GreptimeTeam/otel-arrow", rev = "5
|
||||
"server",
|
||||
] }
|
||||
parking_lot = "0.12"
|
||||
parquet = { version = "58.3", default-features = false, features = ["arrow", "async", "object_store"] }
|
||||
parquet-variant = "58.3"
|
||||
parquet-variant-compute = "58.3"
|
||||
parquet-variant-json = "58.3"
|
||||
parquet = { version = "=59.2.0", default-features = false, features = ["arrow", "async", "object_store"] }
|
||||
parquet-variant = "=59.2.0"
|
||||
parquet-variant-compute = "=59.2.0"
|
||||
parquet-variant-json = "=59.2.0"
|
||||
paste = "1.0"
|
||||
pin-project = "1.0"
|
||||
pretty_assertions = "1.4.0"
|
||||
@@ -239,7 +240,7 @@ simd-json = "0.15"
|
||||
similar-asserts = "1.6.0"
|
||||
smallvec = { version = "1", features = ["serde"] }
|
||||
snafu = "0.8"
|
||||
sqlparser = { version = "0.61.0", default-features = false, features = ["std", "visitor", "serde"] }
|
||||
sqlparser = { version = "0.62.0", default-features = false, features = ["std", "visitor", "serde"] }
|
||||
sqlx = { version = "0.8", default-features = false, features = [
|
||||
"any",
|
||||
"macros",
|
||||
@@ -287,6 +288,7 @@ cli = { path = "src/cli" }
|
||||
client = { path = "src/client" }
|
||||
cmd = { path = "src/cmd", default-features = false }
|
||||
common-base = { path = "src/common/base" }
|
||||
common-batcher = { path = "src/common/batcher" }
|
||||
common-catalog = { path = "src/common/catalog" }
|
||||
common-config = { path = "src/common/config" }
|
||||
common-datasource = { path = "src/common/datasource" }
|
||||
@@ -353,22 +355,25 @@ git = "https://github.com/GreptimeTeam/greptime-meter.git"
|
||||
rev = "5618e779cf2bb4755b499c630fba4c35e91898cb"
|
||||
|
||||
[patch.crates-io]
|
||||
datafusion = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "bb531e754b3b36d7a4d3222f35ebc88204dafd2b" }
|
||||
datafusion-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "bb531e754b3b36d7a4d3222f35ebc88204dafd2b" }
|
||||
datafusion-datasource = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "bb531e754b3b36d7a4d3222f35ebc88204dafd2b" }
|
||||
datafusion-expr = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "bb531e754b3b36d7a4d3222f35ebc88204dafd2b" }
|
||||
datafusion-expr-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "bb531e754b3b36d7a4d3222f35ebc88204dafd2b" }
|
||||
datafusion-functions = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "bb531e754b3b36d7a4d3222f35ebc88204dafd2b" }
|
||||
datafusion-functions-aggregate-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "bb531e754b3b36d7a4d3222f35ebc88204dafd2b" }
|
||||
datafusion-functions-window-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "bb531e754b3b36d7a4d3222f35ebc88204dafd2b" }
|
||||
datafusion-optimizer = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "bb531e754b3b36d7a4d3222f35ebc88204dafd2b" }
|
||||
datafusion-physical-expr = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "bb531e754b3b36d7a4d3222f35ebc88204dafd2b" }
|
||||
datafusion-physical-expr-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "bb531e754b3b36d7a4d3222f35ebc88204dafd2b" }
|
||||
datafusion-physical-plan = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "bb531e754b3b36d7a4d3222f35ebc88204dafd2b" }
|
||||
datafusion-proto = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "bb531e754b3b36d7a4d3222f35ebc88204dafd2b" }
|
||||
datafusion-sql = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "bb531e754b3b36d7a4d3222f35ebc88204dafd2b" }
|
||||
datafusion-substrait = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "bb531e754b3b36d7a4d3222f35ebc88204dafd2b" }
|
||||
sqlparser = { git = "https://github.com/GreptimeTeam/sqlparser-rs.git", rev = "2aefa08a8d69c96eec2d6d6703598a009bba6e4c" } # on branch v0.61.x
|
||||
substrait = { git = "https://github.com/GreptimeTeam/substrait-rs.git", rev = "91ec978b0649417ad3da8390e7a515baec723b1b" }
|
||||
datafusion = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "b66dca3260c8e04f314c8036cdfbb08f1a564d95" }
|
||||
datafusion-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "b66dca3260c8e04f314c8036cdfbb08f1a564d95" }
|
||||
datafusion-datasource = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "b66dca3260c8e04f314c8036cdfbb08f1a564d95" }
|
||||
datafusion-expr = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "b66dca3260c8e04f314c8036cdfbb08f1a564d95" }
|
||||
datafusion-expr-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "b66dca3260c8e04f314c8036cdfbb08f1a564d95" }
|
||||
datafusion-functions = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "b66dca3260c8e04f314c8036cdfbb08f1a564d95" }
|
||||
datafusion-functions-aggregate-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "b66dca3260c8e04f314c8036cdfbb08f1a564d95" }
|
||||
datafusion-functions-window-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "b66dca3260c8e04f314c8036cdfbb08f1a564d95" }
|
||||
datafusion-optimizer = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "b66dca3260c8e04f314c8036cdfbb08f1a564d95" }
|
||||
datafusion-physical-expr = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "b66dca3260c8e04f314c8036cdfbb08f1a564d95" }
|
||||
datafusion-physical-expr-common = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "b66dca3260c8e04f314c8036cdfbb08f1a564d95" }
|
||||
datafusion-physical-plan = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "b66dca3260c8e04f314c8036cdfbb08f1a564d95" }
|
||||
datafusion-proto = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "b66dca3260c8e04f314c8036cdfbb08f1a564d95" }
|
||||
datafusion-sql = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "b66dca3260c8e04f314c8036cdfbb08f1a564d95" }
|
||||
datafusion-substrait = { git = "https://github.com/GreptimeTeam/datafusion.git", rev = "b66dca3260c8e04f314c8036cdfbb08f1a564d95" }
|
||||
datafusion-pg-catalog = { git = "https://github.com/GreptimeTeam/datafusion-postgres.git", rev = "3c77e6c32b8db80635a0d2f4b318a36b31170bc3" }
|
||||
arrow-pg = { git = "https://github.com/GreptimeTeam/datafusion-postgres.git", rev = "3c77e6c32b8db80635a0d2f4b318a36b31170bc3" }
|
||||
sqlparser = { git = "https://github.com/GreptimeTeam/sqlparser-rs.git", rev = "9e9019bb1c7040ed956f654e39378dd42ab17884" }
|
||||
|
||||
# Temporary: use the GreptimeTeam fork of tikv-jemalloc-sys embedding
|
||||
# jemalloc 5.3.1 + backport of 54f22c83 ("Initialize TSD tcache before
|
||||
|
||||
@@ -165,6 +165,9 @@
|
||||
| `region_engine.mito.worker_request_batch_size` | Integer | `64` | Max batch size for a worker to handle requests. |
|
||||
| `region_engine.mito.manifest_checkpoint_distance` | Integer | `10` | Number of meta action updated to trigger a new checkpoint for the manifest. |
|
||||
| `region_engine.mito.compress_manifest` | Bool | `false` | Whether to compress manifest and checkpoint file by gzip (default false). |
|
||||
| `region_engine.mito.experimental_enable_series_index` | Bool | `false` | Under development; do not enable. Whether to enable series indexes.<br/>Indexes are stored on the local filesystem under `{data_home}/series_index`. |
|
||||
| `region_engine.mito.experimental_series_index_maintenance_interval` | String | `5m` | Interval between series-index maintenance runs. Zero uses the default of 5 min. |
|
||||
| `region_engine.mito.experimental_series_index_bucket_width` | String | `5days` | Requested minimum series-index bucket width (default: 5 days), rounded up to<br/>an exact multiple of each region's compaction time window. |
|
||||
| `region_engine.mito.max_background_flushes` | Integer | Auto | Max number of running background flush jobs (default: 1/2 of cpu cores). |
|
||||
| `region_engine.mito.max_background_compactions` | Integer | Auto | Max number of running background compaction jobs (default: 1/4 of cpu cores). |
|
||||
| `region_engine.mito.max_background_purges` | Integer | Auto | Max number of running background purge jobs (default: number of cpu cores). |
|
||||
@@ -612,6 +615,9 @@
|
||||
| `region_engine.mito.experimental_manifest_keep_removed_file_count` | Integer | `256` | Number of removed files to keep in manifest's `removed_files` field before also<br/>remove them from `removed_files`. Mostly for debugging purpose.<br/>If set to 0, it will only use `keep_removed_file_ttl` to decide when to remove files<br/>from `removed_files` field. |
|
||||
| `region_engine.mito.experimental_manifest_keep_removed_file_ttl` | String | `1h` | How long to keep removed files in the `removed_files` field of manifest<br/>after they are removed from manifest.<br/>files will only be removed from `removed_files` field<br/>if both `keep_removed_file_count` and `keep_removed_file_ttl` is reached. |
|
||||
| `region_engine.mito.compress_manifest` | Bool | `false` | Whether to compress manifest and checkpoint file by gzip (default false). |
|
||||
| `region_engine.mito.experimental_enable_series_index` | Bool | `false` | Under development; do not enable. Whether to enable series indexes.<br/>Indexes are stored on the local filesystem under `{data_home}/series_index`. |
|
||||
| `region_engine.mito.experimental_series_index_maintenance_interval` | String | `5m` | Interval between series-index maintenance runs. Zero uses the default of 5 min. |
|
||||
| `region_engine.mito.experimental_series_index_bucket_width` | String | `5days` | Requested minimum series-index bucket width (default: 5 days), rounded up to<br/>an exact multiple of each region's compaction time window. |
|
||||
| `region_engine.mito.max_background_flushes` | Integer | Auto | Max number of running background flush jobs (default: 1/2 of cpu cores). |
|
||||
| `region_engine.mito.max_background_compactions` | Integer | Auto | Max number of running background compaction jobs (default: 1/4 of cpu cores). |
|
||||
| `region_engine.mito.max_background_purges` | Integer | Auto | Max number of running background purge jobs (default: number of cpu cores). |
|
||||
|
||||
@@ -474,6 +474,17 @@ experimental_manifest_keep_removed_file_ttl = "1h"
|
||||
## Whether to compress manifest and checkpoint file by gzip (default false).
|
||||
compress_manifest = false
|
||||
|
||||
## Under development; do not enable. Whether to enable series indexes.
|
||||
## Indexes are stored on the local filesystem under `{data_home}/series_index`.
|
||||
#+ experimental_enable_series_index = false
|
||||
|
||||
## Interval between series-index maintenance runs. Zero uses the default of 5 min.
|
||||
#+ experimental_series_index_maintenance_interval = "5m"
|
||||
|
||||
## Requested minimum series-index bucket width (default: 5 days), rounded up to
|
||||
## an exact multiple of each region's compaction time window.
|
||||
#+ experimental_series_index_bucket_width = "5days"
|
||||
|
||||
## Max number of running background flush jobs (default: 1/2 of cpu cores).
|
||||
## @toml2docs:none-default="Auto"
|
||||
#+ max_background_flushes = 4
|
||||
|
||||
@@ -635,6 +635,17 @@ manifest_checkpoint_distance = 10
|
||||
## Whether to compress manifest and checkpoint file by gzip (default false).
|
||||
compress_manifest = false
|
||||
|
||||
## Under development; do not enable. Whether to enable series indexes.
|
||||
## Indexes are stored on the local filesystem under `{data_home}/series_index`.
|
||||
#+ experimental_enable_series_index = false
|
||||
|
||||
## Interval between series-index maintenance runs. Zero uses the default of 5 min.
|
||||
#+ experimental_series_index_maintenance_interval = "5m"
|
||||
|
||||
## Requested minimum series-index bucket width (default: 5 days), rounded up to
|
||||
## an exact multiple of each region's compaction time window.
|
||||
#+ experimental_series_index_bucket_width = "5days"
|
||||
|
||||
## Max number of running background flush jobs (default: 1/2 of cpu cores).
|
||||
## @toml2docs:none-default="Auto"
|
||||
#+ max_background_flushes = 4
|
||||
|
||||
@@ -0,0 +1,280 @@
|
||||
---
|
||||
Feature Name: Metric Export and Import Optimization
|
||||
Tracking Issue: https://github.com/GreptimeTeam/greptimedb/issues/9120
|
||||
Date: 2026-09-11
|
||||
Status: Draft for discussion
|
||||
---
|
||||
|
||||
# Summary
|
||||
|
||||
Group the selected Metric logical tables by physical table. Query each physical
|
||||
table across all its regions, order the rows by `__table_id`, and project each
|
||||
logical schema into the existing logical-table Parquet files. Restore the same
|
||||
V2 snapshot with batched logical-table DDL and the existing data COPY path. A
|
||||
separate generic COPY fix removes repeated directory scans.
|
||||
|
||||
The design preserves `manifest.version = 1` and the current V2 layout. It does
|
||||
not introduce physical backup files or require importing source table IDs.
|
||||
Merged data writes remain a first-release scope decision, pending representative
|
||||
measurements after the COPY fix.
|
||||
|
||||
# Motivation and evidence
|
||||
|
||||
[Discussion #7394](https://github.com/orgs/GreptimeTeam/discussions/7394)
|
||||
includes reports of exports taking days for roughly 100,000 logical tables with
|
||||
only several GiB of data. Table count, physical schema width and data volume must
|
||||
be measured separately; total database bytes do not describe the transfer cost.
|
||||
|
||||
Local experiments support three independent optimizations: shared physical scans,
|
||||
batched logical CREATE, and removal of repeated file lookup. The
|
||||
[experiment results](2026-09-11-metric-export-import/experiment-results.md)
|
||||
contain the measurements, controls and coverage limits. The COPY improvement
|
||||
uses per-table writes; merged-write benefits remain unmeasured.
|
||||
|
||||
# Terminology
|
||||
|
||||
| Term | Meaning in this proposal |
|
||||
| --- | --- |
|
||||
| Physical table | A Metric Engine table whose data regions store rows for multiple logical tables; its schema includes their column union. |
|
||||
| Logical table | A user-facing Metric table with its own schema and an association with a physical table. |
|
||||
| Region | A storage partition of a physical table. A logical table's rows may span regions. |
|
||||
| Export unit | The selected logical tables sharing one physical table, within one schema and V2 time chunk. This is an export grouping, not a storage object or completion checkpoint. |
|
||||
| Query execution partition | A stream of work in the query plan. Frontend `target_partitions` influences the partition count; execution partitions are distinct from storage regions and export units. |
|
||||
| Logical-table descriptor | Captured catalog metadata: source ID, name, physical association, column names/types/order, time index and primary key. Capturing and reusing it provides no snapshot or locking guarantee. |
|
||||
| Parquet writer | The writer for one logical table's output file in an export unit. |
|
||||
| Merged writes | Combining insert requests for multiple target logical tables into a region batch dispatch while retaining each table's identity, schema and routing. |
|
||||
|
||||
# Physical export and logical files
|
||||
|
||||
## Query and routing
|
||||
|
||||
Capture the selected logical-table descriptors and reuse them for schema DDL
|
||||
and data export. For each schema and V2 time chunk, group the selected logical
|
||||
tables by physical table to form export units. Source schemas, selected data and
|
||||
TTL effects must stay stable during export; this proposal does not provide a
|
||||
transactionally consistent online backup.
|
||||
|
||||
The server must authorize every selected logical table and the destination
|
||||
before querying the physical table. Routing filters are not an authorization
|
||||
boundary.
|
||||
|
||||
For example, suppose `cpu` has `(ts, host, cpu)` and `requests` has
|
||||
`(ts, host, service, requests)`, sharing `public.phy`. The physical query is
|
||||
equivalent to:
|
||||
|
||||
```sql
|
||||
SELECT __table_id, ts, host, service, cpu, requests
|
||||
FROM public.phy
|
||||
WHERE ts >= '2026-09-01T00:00:00Z'
|
||||
AND ts < '2026-09-02T00:00:00Z'
|
||||
ORDER BY __table_id;
|
||||
```
|
||||
|
||||
The implementation builds this plan through the existing query engine, using
|
||||
COPY's time-filter semantics and properly resolved columns. It reads the
|
||||
physical table directly; it does not first write and reread `phy.parquet`.
|
||||
|
||||
```text
|
||||
physical table query: selected column union + __table_id, ordered by ID
|
||||
→ reject rows whose IDs are absent from the captured descriptor set
|
||||
→ contiguous rows for cpu → project (ts, host, cpu) → cpu.parquet
|
||||
→ contiguous rows for requests → project (ts, host, service, requests) → requests.parquet
|
||||
```
|
||||
|
||||
Only one Parquet writer is active per export unit. At an ID transition, close the
|
||||
previous writer and open the next. Verify non-null UInt32 IDs and monotonic order
|
||||
across input batches. Create a valid zero-row file for every selected empty table.
|
||||
Never put `__table_id`, `__tsid`, or another table's columns into logical files.
|
||||
|
||||
The captured catalog IDs are the routing whitelist; physical rows are not the
|
||||
source of table membership. Dropped tables can leave residual rows. Predicate
|
||||
pushdown for selected IDs/ranges is an optional optimization to evaluate against
|
||||
residual density and index effectiveness; it does not replace routing checks.
|
||||
|
||||
## Physical column union and memory
|
||||
|
||||
The physical schema can contain the union of thousands of logical schemas.
|
||||
Project the scan to `__table_id` plus the union needed by the selected tables.
|
||||
For each ID run, project again to that logical schema **before** expanding Arrow
|
||||
dictionary columns. Preserve field order, types, NULLs and empty strings.
|
||||
|
||||
Logical projection reduces output conversion but does not eliminate wide scan
|
||||
batches or upstream sorting. Account separately for retained scan and merge
|
||||
buffers, conversion, codec reservations, row-group metadata and object-store
|
||||
buffers. Bound concurrent export units as well as each unit's work.
|
||||
|
||||
Input checks after a batch arrives cannot cap the scan's allocation peak, and a
|
||||
writer flush threshold does not cap codec allocations. Combine schema validation
|
||||
with query memory accounting, bounded conversion slices, row-group limits and
|
||||
cancellation. Reject an oversized row explicitly. These controls do not provide
|
||||
a hard process RSS guarantee.
|
||||
|
||||
## Ordering and multiple regions
|
||||
|
||||
The first release covers every selected Metric physical table, any region count,
|
||||
and the Metric column types supported by existing Parquet COPY. Issue one query
|
||||
per export unit covering all regions of its physical table. The query engine
|
||||
provides globally ordered results through `ORDER BY __table_id`, including any
|
||||
sorting and merging needed across datanodes. The exporter consumes that stream
|
||||
without implementing its own region scheduler.
|
||||
|
||||
In the current implementation, `MergeScanExec` declares the requested ordering
|
||||
only when `output_partition_count >= regions.len()`. When the region count
|
||||
exceeds frontend `target_partitions`, the frontend may need an additional sort
|
||||
to satisfy SQL ordering. That sort operates on the selected column union before
|
||||
per-logical-table projection. Parquet writer limits and downstream conversion
|
||||
slices cannot bound its memory or spill cost.
|
||||
|
||||
SQL ordering is the contract. Keep the runtime ID-monotonicity check, but do not
|
||||
reject a valid plan because it contains `SortExec`. If query resources are
|
||||
exhausted, cancel the query, fail the chunk and retain retry semantics. Region
|
||||
count or resource exhaustion must not silently switch an enabled physical
|
||||
export to per-logical-table queries. The
|
||||
[ordering acceptance gate](#ordering-acceptance-gate) is required before release.
|
||||
|
||||
# Snapshot and metadata contract
|
||||
|
||||
Retain the current implementation's layout:
|
||||
|
||||
```text
|
||||
manifest.json # version 1
|
||||
schema/schemas.json # schema index
|
||||
schema/ddl/public.sql # canonical database/table/view DDL
|
||||
data/public/1/cpu.parquet
|
||||
data/public/1/requests.parquet
|
||||
```
|
||||
|
||||
Schema path segments retain existing V2 encoding. Logical data filenames retain
|
||||
COPY's raw table-name semantics and import's `file_stem` matching, including dots.
|
||||
There is no existing general table-filename escaping contract to reuse. Reject
|
||||
path separators for the fast path; changing that contract requires a separate
|
||||
compatibility design.
|
||||
|
||||
Keep database, physical-table, logical/ordinary-table and view DDL in dependency
|
||||
order. Physical DDL describes the table and its options; logical DDL describes
|
||||
each logical schema and its physical association. Normal target CREATE procedures
|
||||
reconstruct Metric data/metadata regions and logical metadata with newly allocated
|
||||
IDs. Do not copy hidden metadata-region records or source IDs as restore data.
|
||||
Verify effective options, column mappings and physical routes after recreation.
|
||||
|
||||
# Restore path
|
||||
|
||||
## Batched logical DDL
|
||||
|
||||
Parse the existing DDL and batch contiguous logical CREATE statements sharing
|
||||
catalog, schema and the literal `on_physical_table` value. A dot in that option
|
||||
does not make it a qualified identifier. Database, physical, ordinary and view
|
||||
statements are barriers and retain their order.
|
||||
|
||||
Use the existing logical-table batch procedure. Process bounded batches
|
||||
sequentially and validate each batch before submission. Preserve normal CREATE
|
||||
validation, schema-option inheritance and target metadata resolution. Bound the
|
||||
request size and the importer's DDL buffering separately.
|
||||
|
||||
The production CLI needs an authenticated server operation that checks CREATE
|
||||
authorization for every member and reuses the ordinary DDL path. Transport,
|
||||
capability discovery and request limits must be agreed before accepting this
|
||||
interface. An older server uses ordinary DDL; do not retry an ambiguously
|
||||
completed batch through another path.
|
||||
|
||||
## Data COPY and generic directory fix
|
||||
|
||||
Read each logical Parquet file through the current COPY path and write through
|
||||
normal target logical-table routing. No source-ID remapping belongs in these
|
||||
files, and batch DDL does not imply batched data insertion.
|
||||
|
||||
The generic fix changes explicit-file lookup from `stat → list parent → find`
|
||||
to `stat → use path`, with a sandboxed file-type check for explicit local
|
||||
inputs to preserve symlink filtering. Directory COPY retains listing, pattern
|
||||
matching and file-type filtering. Backend authorization and error handling
|
||||
remain in place. With N files in one directory, this removes roughly N²
|
||||
directory-entry work.
|
||||
COPY bypasses the common Lister's filename branch; the branch itself is unchanged.
|
||||
This fix can ship independently of the Metric design.
|
||||
|
||||
## Completion, interruption and retry
|
||||
|
||||
Keep V2 completion units: export chunk completion and import `(chunk, schema)`
|
||||
data tasks, with the existing DDL-completed flag. Finish and close every required
|
||||
file before recording completion. A closed subset of files is not a completed
|
||||
chunk. On export retry, replace only the unfinished attempt's owned outputs;
|
||||
never touch completed chunks. Cancellation must close/abort writers and leave
|
||||
the chunk incomplete.
|
||||
|
||||
Batch DDL is not a transaction across the restore. A later failure can leave
|
||||
earlier batches created. Keep DDL completion false until all statements succeed,
|
||||
then use the current durable state update. Data retries retain existing COPY
|
||||
semantics and may replay an incomplete task; there is no new exactly-once
|
||||
guarantee or per-file checkpoint in this proposal.
|
||||
|
||||
# Rationale and alternatives
|
||||
|
||||
| Choice | Reason and trade-off |
|
||||
| --- | --- |
|
||||
| Keep logical-table Parquet files | Reuses V2 DDL, file lookup and normal target CREATE/routing with fresh IDs. Physical backup files would need a new restore contract for mixed schemas and source metadata. The retained format still has one file per logical table per time chunk. |
|
||||
| Scan by physical table | Shares query and scan work among logical tables. Per-logical-table COPY is simpler but retains the table-count overhead; physical scans instead pay for the selected column union and ID routing. |
|
||||
| Global ID ordering and one Parquet writer per export unit | Contiguous IDs allow a file to be closed at each table transition. Unordered routing would require many open writers or intermediate buffering/files. Ordering bounds writer state, but query sorting and merging need separate resource controls. |
|
||||
| Use query-engine ordering across regions | Reuses distributed planning and SQL semantics. An exporter-owned region scheduler would add merge, cancellation and retry coordination. The chosen approach must handle both streaming merge and additional-sort plans. |
|
||||
| Batch DDL while retaining per-table data COPY | Reuses logical CREATE procedures and current data-task replay semantics. Merged writes may reduce dispatch/WAL overhead, but introduce partial-success and checkpoint questions and require a separate benefit comparison. |
|
||||
|
||||
# Unresolved questions
|
||||
|
||||
- Which authenticated batch-DDL operation and capability-discovery mechanism
|
||||
should the CLI use? Agree request byte/table limits and DDL buffering bounds.
|
||||
- What are the query memory, spill-space, writer and export-unit concurrency
|
||||
limits? Agree the test workloads and resource/performance thresholds before
|
||||
running the ordering acceptance benchmark.
|
||||
- Which released import-v2 versions and object-store backends are compatibility
|
||||
targets? Test those readers against the retained V2 format.
|
||||
- What name and defaults should the `experimental_` export option use?
|
||||
- Should merged writes ship in the first release? Compare a bounded merged-write
|
||||
PoC against corrected per-table COPY with the same batch DDL and concurrency.
|
||||
Bound decoded rows/bytes and in-flight requests, preserving normal logical
|
||||
validation and target routing through the existing region batch dispatch.
|
||||
Cover small/larger files, wide schemas and distributed/multi-region targets;
|
||||
report repeated total/data times, rows/s, bytes/s, request counts and memory.
|
||||
Agree a minimum useful gain and memory ceiling before the comparison. Validate
|
||||
partial success, ambiguous completion and checkpoint advancement, then record
|
||||
an explicit include/defer decision.
|
||||
|
||||
# Integration and release acceptance
|
||||
|
||||
Use existing COPY DATABASE orchestration to form Metric export units. Retain
|
||||
ordinary COPY for non-Metric tables and other formats, and direct COPY TABLE
|
||||
behavior. Gate the unfinished Metric fast path behind an `experimental_`
|
||||
configuration option; disabling it selects the existing path.
|
||||
|
||||
The [tracking issue](https://github.com/GreptimeTeam/greptimedb/issues/9120) owns
|
||||
the PR breakdown and implementation dependencies. Each implementation slice
|
||||
must verify its own correctness and failure behavior. Release acceptance requires:
|
||||
|
||||
| Area | Required evidence |
|
||||
| --- | --- |
|
||||
| Export | Actual CLI export with authorization, multiple physical tables and regions, dense/sparse keys, empty/residual tables, supported Metric Parquet types, and logical schema/value equivalence. |
|
||||
| Restore | Authenticated remote batch DDL, per-table authorization, fresh target IDs and reconstructed metadata, dependency ordering, older-server behavior and batch failure/retry. |
|
||||
| V2 lifecycle | Supported released readers; local and object-store interruption, owned-output cleanup, durable completion state and import replay. |
|
||||
| Scale | Repeated release-build measurements varying table count up to approximately 100,000, rows/file, physical union width, regions and backend. Report median/spread, control execution order/cache, and include schema export and metadata discovery in end-to-end timing. |
|
||||
| Open decisions | Agreed batch-DDL interface, resource limits, compatibility targets and experimental option; documented merged-write include/defer decision. |
|
||||
|
||||
## Ordering acceptance gate
|
||||
|
||||
The first release must pass a distributed export benchmark with region counts
|
||||
**below, equal to and above frontend `target_partitions`**, using representative
|
||||
row counts and narrow/wide selected column unions. Record actual execution
|
||||
partition counts rather than assuming the configured target determines the plan.
|
||||
Include multiple physical tables and a logical table whose rows span regions.
|
||||
Exercise both streaming-merge and additional-sort plans.
|
||||
|
||||
For each case, report frontend and datanode execution plans, peak memory,
|
||||
query memory-accounting metrics, spill volume and elapsed time. Run with the
|
||||
agreed query memory, spill-space, writer and concurrency limits. Verify global
|
||||
ID order and exactly one complete file per selected logical table in each export
|
||||
unit, checking schemas and typed rows outside the timed interval.
|
||||
|
||||
Passing requires correct exports within the agreed resource/performance
|
||||
thresholds on both plan paths. Also force resource exhaustion: cancellation
|
||||
must leave the chunk incomplete, and retry with sufficient resources must
|
||||
produce complete files without altering completed chunks. Writer-only memory
|
||||
measurements and single-region runs cannot satisfy this gate. Until both the
|
||||
normal and failure/retry cases pass, multi-region resource acceptance remains
|
||||
open.
|
||||
@@ -0,0 +1,109 @@
|
||||
# Metric export/import experiment results
|
||||
|
||||
These experiments support three changes: physical-table export, batched logical
|
||||
DDL, and removal of repeated directory listing during COPY. They measure separate
|
||||
parts of export/import; their speedups should not be multiplied.
|
||||
|
||||
Measurements were taken on September 10–11, 2026, on an ARM64 machine with 16 GiB
|
||||
RAM, using local standalone instances, local files, warm caches and unoptimized
|
||||
test builds. Each table entry represents one run. The results establish useful
|
||||
optimization directions, not production throughput estimates.
|
||||
|
||||
## Physical-table export
|
||||
|
||||
The comparison uses 64 populated logical tables with 1,000 source rows each,
|
||||
plus an empty table. A time filter selects 63,744 rows, written into 65 logical
|
||||
Parquet files. Ordinary COPY DATABASE uses parallelism one. Timings cover data
|
||||
export and exclude schema export and fixture preparation. Both execution orders
|
||||
were measured at narrow and wide physical schemas.
|
||||
|
||||
| Physical tag columns | Execution order | Ordinary COPY (ms) | Physical-table export (ms) | Speedup |
|
||||
| ---: | --- | ---: | ---: | ---: |
|
||||
| 4 | Ordinary COPY first | 889.4 | 262.7 | 3.39x |
|
||||
| 4 | Physical export first | 893.7 | 309.5 | 2.89x |
|
||||
| 67 | Ordinary COPY first | 1062.0 | 316.0 | 3.36x |
|
||||
| 67 | Physical export first | 869.0 | 308.3 | 2.82x |
|
||||
|
||||
Physical-table export was **2.82–3.39x faster** in these runs. Each populated
|
||||
logical file still contains only its own four columns. Projecting the logical
|
||||
schema before dictionary expansion kept the expanded-array peak at 0.148 MiB
|
||||
and the Parquet row-group estimate at 1.112 MiB for both widths. Incoming array
|
||||
estimates grew from 0.267 MiB to 2.291–2.408 MiB: logical projection does not remove
|
||||
the memory cost of a wide physical schema. These estimates are not process RSS
|
||||
limits, and a writer flush threshold does not cap codec allocations.
|
||||
|
||||
The measured single-region query used `SeriesScan` and `SortPreservingMergeExec`
|
||||
without a separate `SortExec`; `__table_id` matches a physical primary-key prefix.
|
||||
Only Float64 fields, string tags and timestamps were exercised. These runs do not
|
||||
establish sorting memory, spill or performance when region count exceeds
|
||||
frontend `target_partitions`; see the RFC's
|
||||
[ordering acceptance gate](../2026-09-11-metric-export-import.md#ordering-acceptance-gate).
|
||||
The timestamp encoder reserved about 1 MiB even for a small batch, illustrating
|
||||
why a writer flush threshold is not an allocation ceiling.
|
||||
|
||||
## Batched logical DDL
|
||||
|
||||
The restore fixture has logical tables sharing one physical table, one file per
|
||||
logical table, two rows per populated table and one empty table. The 10,000-table
|
||||
case contains 19,998 rows. The three paths use SQL HTTP per table, an internal call per table, and
|
||||
internal batches of up to 128 tables and 1 MiB of rendered SQL, submitted
|
||||
sequentially. These request bounds do not bound the whole parsed DDL list in
|
||||
memory. The internal single-call control separates transport savings from
|
||||
procedure batching.
|
||||
|
||||
| Logical tables | SQL HTTP CREATE (s) | Internal single CREATE (s) | Batched CREATE (s) | Logical calls: single → batch |
|
||||
| ---: | ---: | ---: | ---: | ---: |
|
||||
| 100 | 0.318 | 0.177 | 0.056 | 100 → 1 |
|
||||
| 1,000 | 3.091 | 1.601 | 0.544 | 1,000 → 8 |
|
||||
| 10,000 | 29.181 | 19.524 | 5.776 | 10,000 → 79 |
|
||||
|
||||
These are logical CREATE call durations. At 10,000 tables, the complete DDL phase
|
||||
fell from 30.752 s with SQL HTTP to 7.019 s with batching. Total restore fell from
|
||||
187.279 s to 157.492 s; unchanged data COPY still took about 150 s. Data-phase
|
||||
variation is not a DDL benefit. Batching improves CREATE itself, but the original
|
||||
COPY bottleneck limits its effect on total restore.
|
||||
|
||||
The batch executor was in-process. A production authenticated server interface
|
||||
is still needed; these times do not predict batch HTTP latency.
|
||||
|
||||
## COPY file lookup
|
||||
|
||||
Both variants below use batch DDL and the same COPY concurrency. Previously,
|
||||
COPY listed a file's parent directory even when it already knew the filename.
|
||||
With N files in one directory, repeating this for each file causes roughly N²
|
||||
directory-entry work. The fix checks the known path directly; directory COPY
|
||||
still lists files normally.
|
||||
|
||||
| Logical tables / files | COPY before (s) | COPY after (s) | Total restore before (s) | Total restore after (s) |
|
||||
| ---: | ---: | ---: | ---: | ---: |
|
||||
| 100 | 0.094 | 0.071 | 0.183 | 0.164 |
|
||||
| 1,000 | 2.135 | 0.612 | 2.849 | 1.289 |
|
||||
| 10,000 | 146.496 | 5.481 | 153.647 | 12.300 |
|
||||
|
||||
At 10,000 files, COPY improved **26.73x** and total restore improved **12.49x**.
|
||||
Before the fix, backend/file lookup accounted for about 92% of cumulative active
|
||||
file-COPY time. A control with profiling disabled completed COPY in 5.449 s.
|
||||
|
||||
The **5.481 s result still uses per-table writes**, with only 19,998 rows. It is
|
||||
not a merged-write result. Larger files, distributed targets and object stores
|
||||
are needed to decide whether merged writes provide enough additional benefit.
|
||||
|
||||
## Correctness and remaining coverage
|
||||
|
||||
The experiments compared exported/restored schemas and typed rows with the
|
||||
source. Checks covered dense/sparse keys, different logical schemas, NULL and
|
||||
empty values, empty tables, dropped-table residue and fresh target IDs. The
|
||||
unmodified import-v2 command used in the experiment restored the exported files
|
||||
into a fresh target with different IDs. This does not establish compatibility
|
||||
with all released readers. Batch-DDL checks covered two schemas, ordinary
|
||||
tables/views, failure after a successful 128-table batch and retry of the original
|
||||
DDL. Existing standalone COPY cases also passed.
|
||||
|
||||
The internal exporter assumed a trusted caller. It could leave a partial output
|
||||
directory after failure and did not implement completion publication or resume.
|
||||
These limitations require production integration work.
|
||||
|
||||
Production acceptance still requires release-build measurements with repeated
|
||||
runs, larger workloads, distributed/multi-region execution, object stores,
|
||||
released-reader compatibility and export cancellation/recovery. These are tracked
|
||||
in the [tracking issue](https://github.com/GreptimeTeam/greptimedb/issues/9120).
|
||||
+34
-1
@@ -449,7 +449,14 @@ impl TryFrom<ConcreteDataType> for ColumnDataTypeWrapper {
|
||||
}),
|
||||
JsonFormat::Json2(native_type) => {
|
||||
if native_type.is_null() {
|
||||
None
|
||||
Some(ColumnDataTypeExtension {
|
||||
type_ext: Some(TypeExt::JsonNativeType(Box::new(
|
||||
JsonNativeTypeExtension {
|
||||
datatype: ColumnDataType::Json as i32,
|
||||
datatype_extension: None,
|
||||
},
|
||||
))),
|
||||
})
|
||||
} else {
|
||||
let concrete_type =
|
||||
ConcreteDataType::from_arrow_type(&native_type.as_arrow_type());
|
||||
@@ -1291,6 +1298,32 @@ mod tests {
|
||||
assert!(values.bool_values.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_json2_unknown_type_encoding() {
|
||||
let datatype = ConcreteDataType::json2(JsonNativeType::Null);
|
||||
let wrapper = ColumnDataTypeWrapper::try_from(datatype.clone()).unwrap();
|
||||
assert_eq!(
|
||||
wrapper.to_parts(),
|
||||
(
|
||||
ColumnDataType::Json,
|
||||
Some(ColumnDataTypeExtension {
|
||||
type_ext: Some(TypeExt::JsonNativeType(Box::new(JsonNativeTypeExtension {
|
||||
datatype: ColumnDataType::Json as i32,
|
||||
datatype_extension: None,
|
||||
}))),
|
||||
}),
|
||||
)
|
||||
);
|
||||
assert_eq!(ConcreteDataType::from(wrapper), datatype);
|
||||
|
||||
for extension in [None, Some(ColumnDataTypeExtension::default())] {
|
||||
assert_eq!(
|
||||
ConcreteDataType::from(ColumnDataTypeWrapper::new(ColumnDataType::Json, extension)),
|
||||
datatype
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_concrete_datatype_from_column_datatype() {
|
||||
assert_eq!(
|
||||
|
||||
@@ -27,8 +27,10 @@ use common_query::request::QueryRequest;
|
||||
use common_recordbatch::adapter::{AsyncRecordBatchStreamAdapter, DfRecordBatchStreamAdapter};
|
||||
use common_recordbatch::util::{ChainedRecordBatchStream, LimitedRecordBatchStream};
|
||||
use common_recordbatch::{DfSendableRecordBatchStream, SendableRecordBatchStream};
|
||||
use datafusion::common::Result;
|
||||
use datafusion::common::tree_node::TreeNodeRecursion;
|
||||
use datafusion::execution::TaskContext;
|
||||
use datafusion::physical_expr::{EquivalenceProperties, Partitioning};
|
||||
use datafusion::physical_expr::{EquivalenceProperties, Partitioning, PhysicalExpr};
|
||||
use datafusion::physical_plan::execution_plan::{Boundedness, EmissionType};
|
||||
use datafusion::physical_plan::{DisplayAs, DisplayFormatType, ExecutionPlan, PlanProperties};
|
||||
use datatypes::arrow::datatypes::SchemaRef as ArrowSchemaRef;
|
||||
@@ -157,10 +159,6 @@ impl ExecutionPlan for DistributedInspectExec {
|
||||
"DistributedInspectExec"
|
||||
}
|
||||
|
||||
fn as_any(&self) -> &dyn std::any::Any {
|
||||
self
|
||||
}
|
||||
|
||||
fn schema(&self) -> ArrowSchemaRef {
|
||||
self.arrow_schema.clone()
|
||||
}
|
||||
@@ -173,6 +171,13 @@ impl ExecutionPlan for DistributedInspectExec {
|
||||
vec![]
|
||||
}
|
||||
|
||||
fn apply_expressions(
|
||||
&self,
|
||||
_f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,
|
||||
) -> Result<TreeNodeRecursion> {
|
||||
Ok(TreeNodeRecursion::Continue)
|
||||
}
|
||||
|
||||
fn with_new_children(
|
||||
self: Arc<Self>,
|
||||
children: Vec<Arc<dyn ExecutionPlan>>,
|
||||
|
||||
@@ -135,6 +135,9 @@ lazy_static! {
|
||||
GLOBAL_STATUS,
|
||||
SESSION_STATUS,
|
||||
PARTITIONS,
|
||||
PLUGINS,
|
||||
USER_PRIVILEGES,
|
||||
PROCESSLIST,
|
||||
];
|
||||
}
|
||||
|
||||
@@ -237,6 +240,9 @@ impl SystemSchemaProviderInner for InformationSchemaProvider {
|
||||
TABLE_PRIVILEGES => setup_memory_table!(TABLE_PRIVILEGES),
|
||||
GLOBAL_STATUS => setup_memory_table!(GLOBAL_STATUS),
|
||||
SESSION_STATUS => setup_memory_table!(SESSION_STATUS),
|
||||
PLUGINS => setup_memory_table!(PLUGINS),
|
||||
USER_PRIVILEGES => setup_memory_table!(USER_PRIVILEGES),
|
||||
PROCESSLIST => setup_memory_table!(PROCESSLIST),
|
||||
KEY_COLUMN_USAGE => Some(Arc::new(InformationSchemaKeyColumnUsage::new(
|
||||
self.catalog_name.clone(),
|
||||
self.catalog_manager.clone(),
|
||||
|
||||
@@ -31,7 +31,7 @@ use datatypes::scalars::ScalarVectorBuilder;
|
||||
use datatypes::schema::{ColumnSchema, Schema, SchemaRef};
|
||||
use datatypes::value::Value;
|
||||
use datatypes::vectors::{
|
||||
ConstantVector, Int64Vector, Int64VectorBuilder, StringVector, StringVectorBuilder, VectorRef,
|
||||
Int64Vector, Int64VectorBuilder, StringVector, StringVectorBuilder, VectorRef,
|
||||
};
|
||||
use futures::TryStreamExt;
|
||||
use snafu::{OptionExt, ResultExt};
|
||||
@@ -408,18 +408,9 @@ impl InformationSchemaColumnsBuilder {
|
||||
fn finish(&mut self) -> Result<RecordBatch> {
|
||||
let rows_num = self.collation_names.len();
|
||||
|
||||
let privileges = Arc::new(ConstantVector::new(
|
||||
Arc::new(StringVector::from(vec![DEFAULT_PRIVILEGES])),
|
||||
rows_num,
|
||||
));
|
||||
let empty_string = Arc::new(ConstantVector::new(
|
||||
Arc::new(StringVector::from(vec![EMPTY_STR])),
|
||||
rows_num,
|
||||
));
|
||||
let srs_ids = Arc::new(ConstantVector::new(
|
||||
Arc::new(Int64Vector::from(vec![None])),
|
||||
rows_num,
|
||||
));
|
||||
let privileges = Arc::new(StringVector::from(vec![DEFAULT_PRIVILEGES; rows_num]));
|
||||
let empty_string = Arc::new(StringVector::from(vec![EMPTY_STR; rows_num]));
|
||||
let srs_ids = Arc::new(Int64Vector::from(vec![None; rows_num]));
|
||||
|
||||
let columns: Vec<VectorRef> = vec![
|
||||
Arc::new(self.catalog_names.finish()),
|
||||
|
||||
@@ -383,6 +383,47 @@ pub(super) fn get_schema_columns(table_name: &str) -> (SchemaRef, Vec<VectorRef>
|
||||
vec![],
|
||||
),
|
||||
|
||||
// GreptimeDB has no pluggable components.
|
||||
PLUGINS => (
|
||||
string_columns(&[
|
||||
"PLUGIN_NAME",
|
||||
"PLUGIN_VERSION",
|
||||
"PLUGIN_STATUS",
|
||||
"PLUGIN_TYPE",
|
||||
"PLUGIN_TYPE_VERSION",
|
||||
"PLUGIN_LIBRARY",
|
||||
"PLUGIN_LIBRARY_VERSION",
|
||||
"PLUGIN_AUTHOR",
|
||||
"PLUGIN_DESCRIPTION",
|
||||
"PLUGIN_LICENSE",
|
||||
"LOAD_OPTION",
|
||||
]),
|
||||
vec![],
|
||||
),
|
||||
|
||||
// Privileges are not exposed through `information_schema`, same as the other
|
||||
// `*_privileges` tables above.
|
||||
USER_PRIVILEGES => (
|
||||
string_columns(&["GRANTEE", "TABLE_CATALOG", "PRIVILEGE_TYPE", "IS_GRANTABLE"]),
|
||||
vec![],
|
||||
),
|
||||
|
||||
// Sessions are reported through `information_schema.process_list` and
|
||||
// `SHOW PROCESSLIST`; this table carries the MySQL column shape only.
|
||||
PROCESSLIST => (
|
||||
vec![
|
||||
bigint_column("ID"),
|
||||
string_column("USER"),
|
||||
string_column("HOST"),
|
||||
string_column("DB"),
|
||||
string_column("COMMAND"),
|
||||
bigint_column("TIME"),
|
||||
string_column("STATE"),
|
||||
string_column("INFO"),
|
||||
],
|
||||
vec![],
|
||||
),
|
||||
|
||||
_ => unreachable!("Unknown table in information_schema: {}", table_name),
|
||||
};
|
||||
|
||||
|
||||
@@ -26,7 +26,7 @@ use datafusion::physical_plan::streaming::PartitionStream as DfPartitionStream;
|
||||
use datatypes::prelude::{ConcreteDataType, MutableVector, ScalarVectorBuilder, VectorRef};
|
||||
use datatypes::schema::{ColumnSchema, Schema, SchemaRef};
|
||||
use datatypes::value::Value;
|
||||
use datatypes::vectors::{ConstantVector, StringVector, StringVectorBuilder, UInt32VectorBuilder};
|
||||
use datatypes::vectors::{StringVector, StringVectorBuilder, UInt32VectorBuilder};
|
||||
use futures_util::TryStreamExt;
|
||||
use snafu::{OptionExt, ResultExt};
|
||||
use store_api::storage::{ScanRequest, TableId};
|
||||
@@ -326,10 +326,7 @@ impl InformationSchemaKeyColumnUsageBuilder {
|
||||
fn finish(&mut self) -> Result<RecordBatch> {
|
||||
let rows_num = self.table_catalog.len();
|
||||
|
||||
let null_string_vector = Arc::new(ConstantVector::new(
|
||||
Arc::new(StringVector::from(vec![None as Option<&str>])),
|
||||
rows_num,
|
||||
));
|
||||
let null_string_vector = Arc::new(StringVector::from(vec![None as Option<&str>; rows_num]));
|
||||
let columns: Vec<VectorRef> = vec![
|
||||
Arc::new(self.constraint_catalog.finish()),
|
||||
Arc::new(self.constraint_schema.finish()),
|
||||
|
||||
@@ -29,8 +29,8 @@ use datatypes::schema::{ColumnSchema, Schema, SchemaRef};
|
||||
use datatypes::timestamp::TimestampSecond;
|
||||
use datatypes::value::Value;
|
||||
use datatypes::vectors::{
|
||||
ConstantVector, Int64Vector, Int64VectorBuilder, MutableVector, StringVector,
|
||||
StringVectorBuilder, TimestampSecondVector, TimestampSecondVectorBuilder, UInt64VectorBuilder,
|
||||
Int64Vector, Int64VectorBuilder, MutableVector, StringVector, StringVectorBuilder,
|
||||
TimestampSecondVector, TimestampSecondVectorBuilder, UInt64VectorBuilder,
|
||||
};
|
||||
use futures::{StreamExt, TryStreamExt};
|
||||
use partition::manager::PartitionInfo;
|
||||
@@ -361,22 +361,11 @@ impl InformationSchemaPartitionsBuilder {
|
||||
fn finish(&mut self) -> Result<RecordBatch> {
|
||||
let rows_num = self.catalog_names.len();
|
||||
|
||||
let null_string_vector = Arc::new(ConstantVector::new(
|
||||
Arc::new(StringVector::from(vec![None as Option<&str>])),
|
||||
rows_num,
|
||||
));
|
||||
let null_i64_vector = Arc::new(ConstantVector::new(
|
||||
Arc::new(Int64Vector::from(vec![None])),
|
||||
rows_num,
|
||||
));
|
||||
let null_timestamp_second_vector = Arc::new(ConstantVector::new(
|
||||
Arc::new(TimestampSecondVector::from(vec![None])),
|
||||
rows_num,
|
||||
));
|
||||
let partition_methods = Arc::new(ConstantVector::new(
|
||||
Arc::new(StringVector::from(vec![Some("RANGE")])),
|
||||
rows_num,
|
||||
));
|
||||
let null_string_vector = Arc::new(StringVector::from(vec![None as Option<&str>; rows_num]));
|
||||
let null_i64_vector = Arc::new(Int64Vector::from(vec![None; rows_num]));
|
||||
let null_timestamp_second_vector =
|
||||
Arc::new(TimestampSecondVector::from(vec![None; rows_num]));
|
||||
let partition_methods = Arc::new(StringVector::from(vec![Some("RANGE"); rows_num]));
|
||||
|
||||
let columns: Vec<VectorRef> = vec![
|
||||
Arc::new(self.catalog_names.finish()),
|
||||
|
||||
@@ -336,7 +336,7 @@ mod tests {
|
||||
};
|
||||
let plan = table.scan_to_plan(request).unwrap().unwrap();
|
||||
|
||||
assert!(plan.as_any().is::<EmptyExec>());
|
||||
assert!(plan.as_ref().is::<EmptyExec>());
|
||||
assert_eq!(1, plan.schema().fields().len());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -27,7 +27,7 @@ use datatypes::prelude::{ConcreteDataType, MutableVector};
|
||||
use datatypes::scalars::ScalarVectorBuilder;
|
||||
use datatypes::schema::{ColumnSchema, Schema, SchemaRef};
|
||||
use datatypes::value::Value;
|
||||
use datatypes::vectors::{ConstantVector, StringVector, StringVectorBuilder, VectorRef};
|
||||
use datatypes::vectors::{StringVector, StringVectorBuilder, VectorRef};
|
||||
use futures::TryStreamExt;
|
||||
use snafu::{OptionExt, ResultExt};
|
||||
use store_api::storage::{ScanRequest, TableId};
|
||||
@@ -242,14 +242,8 @@ impl InformationSchemaTableConstraintsBuilder {
|
||||
fn finish(&mut self) -> Result<RecordBatch> {
|
||||
let rows_num = self.constraint_names.len();
|
||||
|
||||
let constraint_catalogs = Arc::new(ConstantVector::new(
|
||||
Arc::new(StringVector::from(vec!["def"])),
|
||||
rows_num,
|
||||
));
|
||||
let enforceds = Arc::new(ConstantVector::new(
|
||||
Arc::new(StringVector::from(vec!["YES"])),
|
||||
rows_num,
|
||||
));
|
||||
let constraint_catalogs = Arc::new(StringVector::from(vec!["def"; rows_num]));
|
||||
let enforceds = Arc::new(StringVector::from(vec!["YES"; rows_num]));
|
||||
|
||||
let columns: Vec<VectorRef> = vec![
|
||||
constraint_catalogs,
|
||||
|
||||
@@ -55,3 +55,8 @@ pub const SSTS_INDEX_META: &str = "ssts_index_meta";
|
||||
pub const TABLE_SEMANTICS: &str = "table_semantics";
|
||||
pub const STATISTICS: &str = "statistics";
|
||||
pub const RECYCLE_BIN: &str = "recycle_bin";
|
||||
pub const PLUGINS: &str = "plugins";
|
||||
pub const USER_PRIVILEGES: &str = "user_privileges";
|
||||
/// MySQL's session list. GreptimeDB reports its own sessions through [`PROCESS_LIST`];
|
||||
/// this table only exists so MySQL tooling finds the name and column shape it expects.
|
||||
pub const PROCESSLIST: &str = "processlist";
|
||||
|
||||
@@ -14,7 +14,6 @@
|
||||
|
||||
//! Dummy catalog for region server.
|
||||
|
||||
use std::any::Any;
|
||||
use std::fmt;
|
||||
use std::sync::Arc;
|
||||
|
||||
@@ -64,10 +63,6 @@ impl fmt::Debug for DummyCatalogList {
|
||||
}
|
||||
|
||||
impl CatalogProviderList for DummyCatalogList {
|
||||
fn as_any(&self) -> &dyn Any {
|
||||
self
|
||||
}
|
||||
|
||||
fn register_catalog(
|
||||
&self,
|
||||
_name: String,
|
||||
@@ -98,10 +93,6 @@ struct DummyCatalogProvider {
|
||||
}
|
||||
|
||||
impl CatalogProvider for DummyCatalogProvider {
|
||||
fn as_any(&self) -> &dyn Any {
|
||||
self
|
||||
}
|
||||
|
||||
fn schema_names(&self) -> Vec<String> {
|
||||
vec![]
|
||||
}
|
||||
@@ -135,10 +126,6 @@ struct DummySchemaProvider {
|
||||
|
||||
#[async_trait]
|
||||
impl SchemaProvider for DummySchemaProvider {
|
||||
fn as_any(&self) -> &dyn Any {
|
||||
self
|
||||
}
|
||||
|
||||
fn table_names(&self) -> Vec<String> {
|
||||
vec![]
|
||||
}
|
||||
|
||||
@@ -825,7 +825,17 @@ impl Database {
|
||||
/// Ingest a stream of [RecordBatch]es that belong to a table, using Arrow Flight's "`DoPut`"
|
||||
/// method. The return value is also a stream, produces [DoPutResponse]s.
|
||||
pub async fn do_put(&self, stream: FlightDataStream) -> Result<DoPutResponseStream> {
|
||||
self.do_put_with_hints(stream, &[]).await
|
||||
}
|
||||
|
||||
/// Ingest a stream of [RecordBatch]es using Arrow Flight's `DoPut` with request hints.
|
||||
pub async fn do_put_with_hints(
|
||||
&self,
|
||||
stream: FlightDataStream,
|
||||
hints: &[(&str, &str)],
|
||||
) -> Result<DoPutResponseStream> {
|
||||
let mut request = tonic::Request::new(stream);
|
||||
Self::put_hints(request.metadata_mut(), hints)?;
|
||||
|
||||
if let Some(AuthHeader {
|
||||
auth_scheme: Some(AuthScheme::Basic(Basic { username, password })),
|
||||
|
||||
@@ -105,6 +105,10 @@ pub(super) struct PromRemoteWritePlan {
|
||||
#[serde(default = "default_visibility_timeout_seconds")]
|
||||
pub(super) visibility_timeout_seconds: u64,
|
||||
#[serde(default)]
|
||||
pub(super) base_setup_sql: Vec<String>,
|
||||
#[serde(default)]
|
||||
pub(super) candidate_setup_sql: Vec<String>,
|
||||
#[serde(default)]
|
||||
pub(super) prom_store: PromStoreConfig,
|
||||
#[serde(default)]
|
||||
pub(super) value: ValueConfig,
|
||||
@@ -200,6 +204,7 @@ pub(super) enum ValuePattern {
|
||||
QuantizedSignal,
|
||||
SignalWithSporadicStalls,
|
||||
MixedSignalRepeated,
|
||||
BoundedMixed,
|
||||
}
|
||||
|
||||
impl std::fmt::Display for ValuePattern {
|
||||
@@ -436,6 +441,63 @@ pub(super) fn default_parallelism() -> u64 {
|
||||
1
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn remote_write_setup_sql_defaults_to_empty() {
|
||||
let case: CaseFile = toml::from_str(
|
||||
r#"
|
||||
[scenario]
|
||||
kind = "prom_remote_write_then_query"
|
||||
|
||||
[scenario.remote_write]
|
||||
metric = "metric"
|
||||
"#,
|
||||
)
|
||||
.unwrap();
|
||||
let Scenario::PromRemoteWriteThenQuery(scenario) = case.scenario else {
|
||||
panic!("expected prom_remote_write_then_query scenario");
|
||||
};
|
||||
assert!(scenario.remote_write.base_setup_sql.is_empty());
|
||||
assert!(scenario.remote_write.candidate_setup_sql.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn remote_write_setup_sql_roundtrips() {
|
||||
let case: CaseFile = toml::from_str(
|
||||
r#"
|
||||
[scenario]
|
||||
kind = "prom_remote_write_then_query"
|
||||
|
||||
[scenario.remote_write]
|
||||
metric = "metric"
|
||||
base_setup_sql = ["CREATE TABLE base_table"]
|
||||
candidate_setup_sql = ["CREATE TABLE candidate_table", "ALTER TABLE candidate_table SET 'x'='y'"]
|
||||
"#,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let roundtrip: CaseFile =
|
||||
serde_json::from_str(&serde_json::to_string(&case).unwrap()).unwrap();
|
||||
let Scenario::PromRemoteWriteThenQuery(scenario) = roundtrip.scenario else {
|
||||
panic!("expected prom_remote_write_then_query scenario");
|
||||
};
|
||||
assert_eq!(
|
||||
scenario.remote_write.base_setup_sql,
|
||||
["CREATE TABLE base_table"]
|
||||
);
|
||||
assert_eq!(
|
||||
scenario.remote_write.candidate_setup_sql,
|
||||
[
|
||||
"CREATE TABLE candidate_table",
|
||||
"ALTER TABLE candidate_table SET 'x'='y'"
|
||||
]
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
impl Scenario {
|
||||
pub(super) fn kind(&self) -> &'static str {
|
||||
match self {
|
||||
|
||||
@@ -469,6 +469,7 @@ pub(super) async fn run_direct_sst(args: DirectArgs) {
|
||||
write_buffer_size: DEFAULT_WRITE_BUFFER_SIZE,
|
||||
row_group_size: scenario.layout.row_group_size,
|
||||
max_file_size: None,
|
||||
float_field_encoding: Default::default(),
|
||||
};
|
||||
let infos = match format {
|
||||
FormatType::Flat => writer.write_all_flat(source, Some(sequence), &opts).await,
|
||||
|
||||
@@ -14,18 +14,21 @@
|
||||
|
||||
use std::collections::BTreeSet;
|
||||
use std::fs;
|
||||
use std::ops::Range;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::Arc;
|
||||
|
||||
use clap::Args as ClapArgs;
|
||||
use datafusion_object_store::path::Path as StorePath;
|
||||
use datafusion_object_store::{ObjectMeta, ObjectStore};
|
||||
use futures::{StreamExt, TryStreamExt};
|
||||
use datafusion_object_store::{ObjectMeta, ObjectStore, ObjectStoreExt};
|
||||
use futures::future::BoxFuture;
|
||||
use futures::{FutureExt, StreamExt, TryFutureExt, TryStreamExt};
|
||||
use object_store::config::ObjectStoreConfig;
|
||||
use object_store::factory::new_raw_object_store;
|
||||
use object_store::services::Fs;
|
||||
use parquet::arrow::async_reader::ParquetObjectReader;
|
||||
use parquet::file::metadata::ParquetMetaDataReader;
|
||||
use parquet::arrow::async_reader::AsyncFileReader;
|
||||
use parquet::errors::{ParquetError, Result as ParquetResult};
|
||||
use parquet::file::metadata::{ParquetMetaData, ParquetMetaDataReader};
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
/// Same shape as `query_regression_runner::model::DestinationConfig`. The two
|
||||
@@ -100,6 +103,70 @@ struct ListedFile {
|
||||
relative_path: String,
|
||||
}
|
||||
|
||||
/// An asynchronous Parquet reader backed directly by an object store.
|
||||
///
|
||||
/// The file size is retained from the listing so footer reads use bounded
|
||||
/// ranges without an additional stat/head request.
|
||||
#[derive(Clone, Debug)]
|
||||
struct ObjectStoreReader {
|
||||
store: Arc<dyn ObjectStore>,
|
||||
path: StorePath,
|
||||
file_size: u64,
|
||||
}
|
||||
|
||||
impl ObjectStoreReader {
|
||||
fn new(store: Arc<dyn ObjectStore>, path: StorePath, file_size: u64) -> Self {
|
||||
Self {
|
||||
store,
|
||||
path,
|
||||
file_size,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn to_parquet_error(error: datafusion_object_store::Error) -> ParquetError {
|
||||
ParquetError::External(Box::new(error))
|
||||
}
|
||||
|
||||
impl AsyncFileReader for ObjectStoreReader {
|
||||
fn get_bytes(
|
||||
&mut self,
|
||||
range: Range<u64>,
|
||||
) -> BoxFuture<'_, ParquetResult<prost::bytes::Bytes>> {
|
||||
self.store
|
||||
.get_range(&self.path, range)
|
||||
.map_err(to_parquet_error)
|
||||
.boxed()
|
||||
}
|
||||
|
||||
fn get_byte_ranges(
|
||||
&mut self,
|
||||
ranges: Vec<Range<u64>>,
|
||||
) -> BoxFuture<'_, ParquetResult<Vec<prost::bytes::Bytes>>> {
|
||||
async move {
|
||||
self.store
|
||||
.get_ranges(&self.path, &ranges)
|
||||
.await
|
||||
.map_err(to_parquet_error)
|
||||
}
|
||||
.boxed()
|
||||
}
|
||||
|
||||
fn get_metadata<'a>(
|
||||
&'a mut self,
|
||||
_options: Option<&'a parquet::arrow::arrow_reader::ArrowReaderOptions>,
|
||||
) -> BoxFuture<'a, ParquetResult<Arc<ParquetMetaData>>> {
|
||||
let file_size = self.file_size;
|
||||
async move {
|
||||
let metadata = ParquetMetaDataReader::new()
|
||||
.load_and_finish(self, file_size)
|
||||
.await?;
|
||||
Ok(Arc::new(metadata))
|
||||
}
|
||||
.boxed()
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) async fn run_inspect_footer(
|
||||
args: InspectFooterArgs,
|
||||
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
@@ -217,11 +284,8 @@ async fn inspect_file(
|
||||
file: &ListedFile,
|
||||
column: &str,
|
||||
) -> Result<FooterFileReport, Box<dyn std::error::Error + Send + Sync>> {
|
||||
let mut reader =
|
||||
ParquetObjectReader::new(store, file.location.clone()).with_file_size(file.size);
|
||||
let metadata = ParquetMetaDataReader::new()
|
||||
.load_and_finish(&mut reader, file.size)
|
||||
.await?;
|
||||
let mut reader = ObjectStoreReader::new(store, file.location.clone(), file.size);
|
||||
let metadata = reader.get_metadata(None).await?;
|
||||
let file_metadata = metadata.file_metadata();
|
||||
let row_groups = metadata.row_groups();
|
||||
let mut columns = Vec::new();
|
||||
|
||||
@@ -197,8 +197,35 @@ fn deterministic_prom_value(args: &PromRemoteWriteArgs, series_idx: u64, sample_
|
||||
args.value_base + (series_idx % 97) as f64 + local as f64 * args.value_step
|
||||
}
|
||||
}
|
||||
ValuePattern::BoundedMixed => bounded_mixed_value(args, series_idx, local),
|
||||
}
|
||||
}
|
||||
|
||||
fn bounded_mixed_value(args: &PromRemoteWriteArgs, series_idx: u64, local: u64) -> f64 {
|
||||
const SPANS: [i64; 4] = [10, 1_000, 100_000, 10_000_000];
|
||||
let series = splitmix64(series_idx ^ args.value_seed);
|
||||
let span = SPANS[(series >> 62) as usize] * (1 + ((series >> 32) % 10) as i64);
|
||||
let baseline = (series % args.value_cardinality.max(1)) as f64 * span as f64 / 10.0;
|
||||
let run_length = args.value_run_length.max(1);
|
||||
let group = local / run_length;
|
||||
let phase = local % run_length;
|
||||
let anchor = bounded_mixed_anchor(series, group, span);
|
||||
let next_anchor = bounded_mixed_anchor(series, group + 1, span);
|
||||
let signal = baseline + anchor + (next_anchor - anchor) * phase as f64 / run_length as f64;
|
||||
let value = (args.value_base + args.value_step * signal).round();
|
||||
if series_idx % args.value_mixed_every.max(1) == args.value_mixed_every.max(1) - 1 {
|
||||
let numerator = splitmix64(series ^ local) % 999 + 1;
|
||||
value + numerator as f64 / 1_000.0
|
||||
} else {
|
||||
value
|
||||
}
|
||||
}
|
||||
|
||||
fn bounded_mixed_anchor(series: u64, group: u64, span: i64) -> f64 {
|
||||
let limit = span - 1;
|
||||
(splitmix64(series ^ group) % (2 * limit as u64 + 1)) as i64 as f64 - limit as f64
|
||||
}
|
||||
|
||||
fn splitmix64(mut value: u64) -> u64 {
|
||||
value = value.wrapping_add(0x9e3779b97f4a7c15);
|
||||
let mut mixed = value;
|
||||
@@ -307,7 +334,7 @@ mod tests {
|
||||
for chunk_idx in 0..10 {
|
||||
let mut chunk = args(ValuePattern::SeededRandom);
|
||||
chunk.samples_per_series = 1_440;
|
||||
chunk.value_sample_offset = chunk_idx * 1_440;
|
||||
chunk.value_sample_offset = chunk_idx * 1_439;
|
||||
chunk.value_total_samples_per_series = Some(14_400);
|
||||
for sample_idx in 0..chunk.samples_per_series {
|
||||
assert_eq!(
|
||||
@@ -319,6 +346,118 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bounded_mixed_has_bounded_95_5_per_series_values() {
|
||||
let mut args = args(ValuePattern::BoundedMixed);
|
||||
args.series_count = 1_000;
|
||||
args.samples_per_series = 4_320;
|
||||
args.value_cardinality = 1_000;
|
||||
args.value_seed = 42;
|
||||
args.value_run_length = 60;
|
||||
args.value_mixed_every = 20;
|
||||
|
||||
let mut integral_series = 0;
|
||||
let mut fractional_series = 0;
|
||||
let mut spans = HashSet::new();
|
||||
let mut ranges = HashSet::new();
|
||||
for series_idx in 0..args.series_count {
|
||||
let series = splitmix64(series_idx ^ args.value_seed);
|
||||
let span = [10_i64, 1_000, 100_000, 10_000_000][(series >> 62) as usize]
|
||||
* (1 + ((series >> 32) % 10) as i64);
|
||||
let baseline = (series % args.value_cardinality.max(1)) as f64 * span as f64 / 10.0;
|
||||
let values = (0..args.samples_per_series)
|
||||
.map(|sample_idx| deterministic_prom_value(&args, series_idx, sample_idx))
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
assert!(values.iter().all(|value| value.is_finite()));
|
||||
assert!(
|
||||
values
|
||||
.iter()
|
||||
.all(|value| *value >= baseline - span as f64 - 1.0
|
||||
&& *value <= baseline + span as f64 + 1.0)
|
||||
);
|
||||
assert!(
|
||||
values.windows(2).any(|pair| pair[1] > pair[0]),
|
||||
"series {series_idx} never rises"
|
||||
);
|
||||
assert!(
|
||||
values.windows(2).any(|pair| pair[1] < pair[0]),
|
||||
"series {series_idx} never falls"
|
||||
);
|
||||
ranges.insert((
|
||||
values
|
||||
.iter()
|
||||
.copied()
|
||||
.fold(f64::INFINITY, f64::min)
|
||||
.to_bits(),
|
||||
values
|
||||
.iter()
|
||||
.copied()
|
||||
.fold(f64::NEG_INFINITY, f64::max)
|
||||
.to_bits(),
|
||||
));
|
||||
spans.insert(span);
|
||||
if series_idx % args.value_mixed_every == args.value_mixed_every - 1 {
|
||||
fractional_series += 1;
|
||||
assert!(values.iter().all(|value| value.fract() != 0.0));
|
||||
} else {
|
||||
integral_series += 1;
|
||||
assert!(values.iter().all(|value| value.fract() == 0.0));
|
||||
}
|
||||
}
|
||||
|
||||
assert_eq!((integral_series, fractional_series), (950, 50));
|
||||
assert!(spans.len() > 4);
|
||||
assert!(ranges.len() > 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bounded_mixed_is_seeded_and_chunk_bit_identical() {
|
||||
let mut monolithic = args(ValuePattern::BoundedMixed);
|
||||
monolithic.samples_per_series = 14_400;
|
||||
monolithic.value_total_samples_per_series = Some(14_400);
|
||||
monolithic.value_cardinality = 1_000;
|
||||
monolithic.value_seed = 42;
|
||||
monolithic.value_run_length = 60;
|
||||
monolithic.value_mixed_every = 20;
|
||||
|
||||
let original = (0..600)
|
||||
.map(|sample_idx| deterministic_prom_value(&monolithic, 19, sample_idx))
|
||||
.collect::<Vec<_>>();
|
||||
monolithic.value_seed = 43;
|
||||
assert_ne!(
|
||||
original,
|
||||
(0..600)
|
||||
.map(|sample_idx| deterministic_prom_value(&monolithic, 19, sample_idx))
|
||||
.collect::<Vec<_>>()
|
||||
);
|
||||
|
||||
monolithic.value_seed = 42;
|
||||
for series_idx in [0, 19] {
|
||||
for chunk_idx in 0..10 {
|
||||
let mut chunk = args(ValuePattern::BoundedMixed);
|
||||
chunk.samples_per_series = 1_440;
|
||||
chunk.value_sample_offset = chunk_idx * 1_439;
|
||||
chunk.value_total_samples_per_series = Some(14_400);
|
||||
chunk.value_cardinality = 1_000;
|
||||
chunk.value_seed = 42;
|
||||
chunk.value_run_length = 60;
|
||||
chunk.value_mixed_every = 20;
|
||||
for sample_idx in 0..chunk.samples_per_series {
|
||||
assert_eq!(
|
||||
deterministic_prom_value(&chunk, series_idx, sample_idx).to_bits(),
|
||||
deterministic_prom_value(
|
||||
&monolithic,
|
||||
series_idx,
|
||||
chunk.value_sample_offset + sample_idx
|
||||
)
|
||||
.to_bits()
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn values_continue_across_series_boundaries() {
|
||||
let mut args = args(ValuePattern::Unique);
|
||||
|
||||
@@ -116,6 +116,10 @@ pub(super) struct RemoteWrite {
|
||||
pub(super) sample_chunk_size: Option<u64>,
|
||||
pub(super) flush_every_sample_chunks: u64,
|
||||
pub(super) visibility_timeout_seconds: u64,
|
||||
#[serde(default)]
|
||||
pub(super) base_setup_sql: Vec<String>,
|
||||
#[serde(default)]
|
||||
pub(super) candidate_setup_sql: Vec<String>,
|
||||
pub(super) prom_store: PromStore,
|
||||
pub(super) value: RemoteValue,
|
||||
pub(super) storage: Option<StorageConfig>,
|
||||
|
||||
@@ -57,6 +57,7 @@ pub(super) async fn run_prepare_remote(args: PrepareRemoteArgs) -> Result<()> {
|
||||
args.base_http_port,
|
||||
&args.fixture_generator,
|
||||
&remote,
|
||||
&remote.base_setup_sql,
|
||||
&client,
|
||||
)
|
||||
.await?;
|
||||
@@ -65,6 +66,7 @@ pub(super) async fn run_prepare_remote(args: PrepareRemoteArgs) -> Result<()> {
|
||||
args.candidate_http_port,
|
||||
&args.fixture_generator,
|
||||
&remote,
|
||||
&remote.candidate_setup_sql,
|
||||
&client,
|
||||
)
|
||||
.await?;
|
||||
@@ -88,6 +90,7 @@ async fn prepare_remote_target(
|
||||
port: u16,
|
||||
generator: &Path,
|
||||
remote: &RemoteWrite,
|
||||
setup_sql: &[String],
|
||||
client: &Client,
|
||||
) -> Result<Value> {
|
||||
let create_database = http_post_sql(
|
||||
@@ -107,6 +110,7 @@ async fn prepare_remote_target(
|
||||
)
|
||||
.into());
|
||||
}
|
||||
let setup = run_setup_sql(client, port, &remote.database, name, setup_sql).await?;
|
||||
let (remote_write, flushes) = ingest_remote_write(generator, port, remote, client).await?;
|
||||
let expected_rows = remote
|
||||
.series_count
|
||||
@@ -124,6 +128,7 @@ async fn prepare_remote_target(
|
||||
Ok(json!({
|
||||
"name": name,
|
||||
"create_database": create_database,
|
||||
"setup_sql": setup,
|
||||
"remote_write": remote_write,
|
||||
"flushes": flushes,
|
||||
"visibility": visibility,
|
||||
@@ -131,6 +136,28 @@ async fn prepare_remote_target(
|
||||
}))
|
||||
}
|
||||
|
||||
async fn run_setup_sql(
|
||||
client: &Client,
|
||||
port: u16,
|
||||
database: &str,
|
||||
target: &str,
|
||||
setup_sql: &[String],
|
||||
) -> Result<Vec<Value>> {
|
||||
let mut results = Vec::with_capacity(setup_sql.len());
|
||||
for (index, statement) in setup_sql.iter().enumerate() {
|
||||
let result = http_post_sql(client, port, statement, database).await;
|
||||
if !result["ok"].as_bool().unwrap_or(false) {
|
||||
return Err(format!(
|
||||
"setup SQL statement {} failed for {target}: {result}",
|
||||
index + 1
|
||||
)
|
||||
.into());
|
||||
}
|
||||
results.push(result);
|
||||
}
|
||||
Ok(results)
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
struct SampleChunk {
|
||||
index: u64,
|
||||
@@ -434,9 +461,110 @@ fn json_number(value: f64) -> String {
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
||||
use tokio::net::{TcpListener, TcpStream};
|
||||
|
||||
use super::*;
|
||||
use crate::query_regression_runner::model::{PromStore, RemoteValue};
|
||||
|
||||
async fn read_http_request(stream: &mut TcpStream) -> Vec<u8> {
|
||||
let mut request = Vec::new();
|
||||
let mut chunk = [0; 1024];
|
||||
loop {
|
||||
let read = stream.read(&mut chunk).await.unwrap();
|
||||
assert_ne!(
|
||||
read, 0,
|
||||
"HTTP client closed request before sending its body"
|
||||
);
|
||||
request.extend_from_slice(&chunk[..read]);
|
||||
let Some(header_end) = request.windows(4).position(|window| window == b"\r\n\r\n")
|
||||
else {
|
||||
continue;
|
||||
};
|
||||
let headers = std::str::from_utf8(&request[..header_end]).unwrap();
|
||||
let content_length = headers
|
||||
.lines()
|
||||
.find_map(|line| line.strip_prefix("content-length: "))
|
||||
.or_else(|| {
|
||||
headers
|
||||
.lines()
|
||||
.find_map(|line| line.strip_prefix("Content-Length: "))
|
||||
})
|
||||
.unwrap()
|
||||
.parse::<usize>()
|
||||
.unwrap();
|
||||
if request.len() >= header_end + 4 + content_length {
|
||||
return request;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn spawn_sql_server(
|
||||
responses: Vec<String>,
|
||||
) -> (u16, tokio::task::JoinHandle<Vec<String>>) {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let port = listener.local_addr().unwrap().port();
|
||||
let server = tokio::spawn(async move {
|
||||
let mut requests = Vec::with_capacity(responses.len());
|
||||
for body in responses {
|
||||
let (mut stream, _) = listener.accept().await.unwrap();
|
||||
let request = read_http_request(&mut stream).await;
|
||||
requests.push(String::from_utf8(request).unwrap());
|
||||
let response = format!(
|
||||
"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\n\
|
||||
Connection: close\r\nContent-Length: {}\r\n\r\n{body}",
|
||||
body.len()
|
||||
);
|
||||
stream.write_all(response.as_bytes()).await.unwrap();
|
||||
}
|
||||
requests
|
||||
});
|
||||
(port, server)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn setup_sql_runs_in_order_with_configured_database() {
|
||||
let (port, server) = spawn_sql_server(vec!["{\"code\":0}".to_string(); 2]).await;
|
||||
let client = Client::new();
|
||||
let setup = vec![
|
||||
"CREATE TABLE physical (ts TIMESTAMP TIME INDEX) ENGINE=metric".to_string(),
|
||||
"ALTER TABLE physical SET 'compaction.twcs.trigger_file_num'='100'".to_string(),
|
||||
];
|
||||
|
||||
let results = run_setup_sql(&client, port, "perf", "candidate", &setup)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(results.len(), 2);
|
||||
let requests = server.await.unwrap();
|
||||
assert_eq!(requests.len(), 2);
|
||||
assert!(requests[0].contains("db=perf"));
|
||||
assert!(requests[0].contains("sql=CREATE"));
|
||||
assert!(requests[1].contains("db=perf"));
|
||||
assert!(requests[1].contains("sql=ALTER"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn setup_sql_stops_at_the_failing_statement() {
|
||||
let (port, server) =
|
||||
spawn_sql_server(vec!["{\"code\":0}".to_string(), "{\"code\":1}".to_string()]).await;
|
||||
let client = Client::new();
|
||||
let setup = vec![
|
||||
"CREATE TABLE physical (ts TIMESTAMP TIME INDEX) ENGINE=metric".to_string(),
|
||||
"invalid SQL".to_string(),
|
||||
"ALTER TABLE physical SET 'compaction.twcs.trigger_file_num'='100'".to_string(),
|
||||
];
|
||||
|
||||
let error = run_setup_sql(&client, port, "perf", "base", &setup)
|
||||
.await
|
||||
.unwrap_err();
|
||||
|
||||
assert!(error.to_string().contains("statement 2 failed for base"));
|
||||
let requests = server.await.unwrap();
|
||||
assert_eq!(requests.len(), 2);
|
||||
assert!(requests[1].contains("sql=invalid+SQL"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn schedules_remote_sample_chunks_and_flushes() {
|
||||
let remote = RemoteWrite {
|
||||
@@ -452,6 +580,8 @@ mod tests {
|
||||
sample_chunk_size: Some(2),
|
||||
flush_every_sample_chunks: 2,
|
||||
visibility_timeout_seconds: 30,
|
||||
base_setup_sql: Vec::new(),
|
||||
candidate_setup_sql: Vec::new(),
|
||||
prom_store: PromStore {
|
||||
pending_rows_flush_interval: "1s".to_string(),
|
||||
max_batch_rows: 1,
|
||||
|
||||
@@ -306,13 +306,25 @@ fn run_bench_command(command: Vec<String>, mut run: Value) -> Result<Value> {
|
||||
}
|
||||
|
||||
fn parse_average_duration(stdout: &str) -> Option<f64> {
|
||||
Regex::new(r"(?i)Average duration[^0-9]*([0-9.]+)\s*ms")
|
||||
.ok()?
|
||||
.captures(stdout)?
|
||||
.get(1)?
|
||||
.as_str()
|
||||
.parse()
|
||||
.ok()
|
||||
let captures = Regex::new(
|
||||
r"(?i)Average:\s+(?:\x1b\[[0-9;]*m)*\d+(?:\x1b\[[0-9;]*m)*\s+rows(?:,\s+(?:\x1b\[[0-9;]*m)*\d+(?:\x1b\[[0-9;]*m)*\s+record batches)?\s+in\s+([0-9]+(?:\.[0-9]+)?)\s*(ns|µs|us|ms|s)\s+over\s+\d+\s+iterations|Average duration[^0-9]*([0-9]+(?:\.[0-9]+)?)\s*ms",
|
||||
)
|
||||
.ok()?
|
||||
.captures(stdout)?;
|
||||
let (value, unit) = match (captures.get(1), captures.get(2), captures.get(3)) {
|
||||
(Some(value), Some(unit), _) => (value.as_str(), unit.as_str()),
|
||||
(_, _, Some(value)) => (value.as_str(), "ms"),
|
||||
_ => return None,
|
||||
};
|
||||
let value = value.parse::<f64>().ok()?;
|
||||
let milliseconds = match unit.to_ascii_lowercase().as_str() {
|
||||
"ns" => value / 1_000_000.0,
|
||||
"µs" | "us" => value / 1_000.0,
|
||||
"ms" => value,
|
||||
"s" => value * 1_000.0,
|
||||
_ => return None,
|
||||
};
|
||||
milliseconds.is_finite().then_some(milliseconds)
|
||||
}
|
||||
|
||||
fn bench_median(runs: &[Value]) -> Option<f64> {
|
||||
@@ -328,12 +340,51 @@ mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn parses_average_duration_and_groups_scan_paths() {
|
||||
assert_eq!(
|
||||
parse_average_duration("work\nAverage duration: 12.5 ms\n"),
|
||||
Some(12.5)
|
||||
);
|
||||
assert_eq!(parse_average_duration("completed"), None);
|
||||
fn parses_average_durations() {
|
||||
let cases = [
|
||||
(
|
||||
"parquetbench nanoseconds",
|
||||
"ℹ Average: 42 rows, 2 record batches in 500ns over 4 iterations",
|
||||
Some(0.0005),
|
||||
),
|
||||
(
|
||||
"scanbench colored microseconds",
|
||||
"\x1b[1;32mSummary\x1b[0m Average: \x1b[36m12\x1b[0m rows in 250µs over 2 iterations",
|
||||
Some(0.25),
|
||||
),
|
||||
(
|
||||
"scanbench seconds",
|
||||
"Summary Average: 12 rows in 2.5s over 2 iterations",
|
||||
Some(2500.0),
|
||||
),
|
||||
(
|
||||
"legacy milliseconds",
|
||||
"Average duration: 12.5 ms",
|
||||
Some(12.5),
|
||||
),
|
||||
("absent", "Benchmark completed!", None),
|
||||
(
|
||||
"malformed",
|
||||
"Summary Average: 12 rows in unknown over 2 iterations",
|
||||
None,
|
||||
),
|
||||
];
|
||||
|
||||
for (name, stdout, expected) in cases {
|
||||
let actual = parse_average_duration(stdout);
|
||||
match (actual, expected) {
|
||||
(Some(actual), Some(expected)) => assert!(
|
||||
(actual - expected).abs() < f64::EPSILON,
|
||||
"{name}: expected {expected}, got {actual}"
|
||||
),
|
||||
(None, None) => {}
|
||||
(actual, expected) => panic!("{name}: expected {expected:?}, got {actual:?}"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn groups_scan_paths() {
|
||||
let targets = vec![
|
||||
BenchTarget {
|
||||
relative_path: "data/a.parquet".to_string(),
|
||||
|
||||
@@ -0,0 +1,19 @@
|
||||
[package]
|
||||
name = "common-batcher"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
|
||||
[lib]
|
||||
path = "src/lib.rs"
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
|
||||
[dependencies]
|
||||
dashmap.workspace = true
|
||||
futures.workspace = true
|
||||
tokio.workspace = true
|
||||
|
||||
[dev-dependencies]
|
||||
tokio = { workspace = true, features = ["test-util"] }
|
||||
@@ -0,0 +1,73 @@
|
||||
// Copyright 2023 Greptime Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use tokio::sync::{AcquireError, OwnedSemaphorePermit, Semaphore};
|
||||
|
||||
/// Shared execution budget. One permit represents one complete flush, not one RPC.
|
||||
#[derive(Clone)]
|
||||
pub struct FlushLimiter {
|
||||
semaphore: Arc<Semaphore>,
|
||||
}
|
||||
|
||||
impl FlushLimiter {
|
||||
/// Returns `None` if the limit is zero or exceeds Tokio's supported semaphore capacity.
|
||||
pub fn try_new(max_concurrent_flushes: usize) -> Option<Self> {
|
||||
let permits = max_concurrent_flushes;
|
||||
((1..=Semaphore::MAX_PERMITS).contains(&permits)).then(|| Self {
|
||||
semaphore: Arc::new(Semaphore::new(permits)),
|
||||
})
|
||||
}
|
||||
|
||||
/// The caller must retain the returned permit until its flush finishes.
|
||||
pub async fn acquire(&self) -> Result<OwnedSemaphorePermit, AcquireError> {
|
||||
self.semaphore.clone().acquire_owned().await
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::future::Future;
|
||||
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_invalid_capacity() {
|
||||
for capacity in [0, Semaphore::MAX_PERMITS + 1, usize::MAX] {
|
||||
assert!(FlushLimiter::try_new(capacity).is_none());
|
||||
}
|
||||
for capacity in [1, Semaphore::MAX_PERMITS] {
|
||||
assert!(FlushLimiter::try_new(capacity).is_some());
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_clones_share_flush_budget() {
|
||||
let limiter = FlushLimiter::try_new(1).unwrap();
|
||||
let other = limiter.clone();
|
||||
let permit = limiter.acquire().await.unwrap();
|
||||
let waiting = other.acquire();
|
||||
tokio::pin!(waiting);
|
||||
// Poll admission without a sleep or an assumption about scheduler timing.
|
||||
assert!(
|
||||
std::future::poll_fn(|cx| {
|
||||
std::task::Poll::Ready(waiting.as_mut().poll(cx).is_pending())
|
||||
})
|
||||
.await
|
||||
);
|
||||
drop(permit);
|
||||
let _next = waiting.await.unwrap();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
// Copyright 2023 Greptime Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
pub mod timing;
|
||||
|
||||
use tokio::time::Instant;
|
||||
|
||||
use crate::pending_batch::PendingBatch;
|
||||
|
||||
/// The event that caused the worker to reconsider flushing.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum FlushTrigger {
|
||||
/// A complete submission was appended to the batch.
|
||||
Submission,
|
||||
/// The flush timer reached its deadline.
|
||||
Deadline,
|
||||
}
|
||||
|
||||
/// Decides when a pending batch is ready without owning its payloads or execution.
|
||||
///
|
||||
/// A deadline wakeup must either allow flushing or produce a future deadline;
|
||||
/// otherwise the caller would repeatedly poll an expired timer.
|
||||
pub trait FlushPolicy {
|
||||
/// Returns the next timer deadline, or `None` when no timer is needed.
|
||||
fn deadline<T>(&self, batch: &PendingBatch<T>) -> Option<Instant>;
|
||||
|
||||
/// Returns whether the batch should be flushed for this event at `now`.
|
||||
///
|
||||
/// Workers never flush empty batches, regardless of this decision.
|
||||
fn should_flush<T>(&self, batch: &PendingBatch<T>, now: Instant, trigger: FlushTrigger)
|
||||
-> bool;
|
||||
}
|
||||
@@ -0,0 +1,157 @@
|
||||
// Copyright 2023 Greptime Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use std::time::Duration;
|
||||
|
||||
use tokio::time::Instant;
|
||||
|
||||
use crate::flush_policy::{FlushPolicy, FlushTrigger};
|
||||
use crate::pending_batch::PendingBatch;
|
||||
|
||||
/// Decides whether the first-submission deadline or row threshold requires a flush.
|
||||
///
|
||||
/// The row threshold is checked after appending a complete submission. It is not a
|
||||
/// hard batch-size limit and never splits a submission.
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
pub struct TimingFlushPolicy {
|
||||
flush_interval: Duration,
|
||||
max_batch_rows: usize,
|
||||
}
|
||||
|
||||
impl TimingFlushPolicy {
|
||||
/// Creates a policy, rejecting zero limits, zero intervals, and unrepresentable deadlines.
|
||||
pub fn try_new(flush_interval: Duration, max_batch_rows: usize) -> Option<Self> {
|
||||
if max_batch_rows == 0 || !Self::validate(flush_interval) {
|
||||
return None;
|
||||
}
|
||||
Some(Self {
|
||||
flush_interval,
|
||||
max_batch_rows,
|
||||
})
|
||||
}
|
||||
|
||||
/// Checks that the interval is nonzero and its deadline is representable.
|
||||
pub fn validate(flush_interval: Duration) -> bool {
|
||||
!flush_interval.is_zero() && Instant::now().checked_add(flush_interval).is_some()
|
||||
}
|
||||
|
||||
/// Checks only the row trigger when a caller drives deadline events separately.
|
||||
pub fn reached_row_threshold<T>(&self, batch: &PendingBatch<T>) -> bool {
|
||||
!batch.is_empty() && batch.total_rows() >= self.max_batch_rows
|
||||
}
|
||||
}
|
||||
|
||||
impl FlushPolicy for TimingFlushPolicy {
|
||||
/// Returns the deadline anchored to the first submission, or `None` for an empty batch.
|
||||
///
|
||||
/// An unrepresentable deadline falls back to the first submission time so that
|
||||
/// extreme timestamps cannot leave a nonempty batch waiting indefinitely.
|
||||
fn deadline<T>(&self, batch: &PendingBatch<T>) -> Option<Instant> {
|
||||
batch
|
||||
.first_submitted_at()
|
||||
.map(|first| first.checked_add(self.flush_interval).unwrap_or(first))
|
||||
}
|
||||
|
||||
/// Returns whether a nonempty batch has reached either trigger.
|
||||
fn should_flush<T>(
|
||||
&self,
|
||||
batch: &PendingBatch<T>,
|
||||
now: Instant,
|
||||
trigger: FlushTrigger,
|
||||
) -> bool {
|
||||
!batch.is_empty()
|
||||
&& (self.reached_row_threshold(batch)
|
||||
|| (trigger == FlushTrigger::Deadline
|
||||
&& self.deadline(batch).is_some_and(|deadline| now >= deadline)))
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_validate_matches_construction() {
|
||||
for interval in [Duration::ZERO, Duration::from_secs(1), Duration::MAX] {
|
||||
assert_eq!(
|
||||
TimingFlushPolicy::validate(interval),
|
||||
TimingFlushPolicy::try_new(interval, 1).is_some()
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_first_submission_deadline() {
|
||||
let policy = TimingFlushPolicy::try_new(Duration::from_millis(10), 100).unwrap();
|
||||
let mut batch = PendingBatch::new();
|
||||
let first = Instant::now();
|
||||
assert_eq!(policy.deadline(&batch), None);
|
||||
assert!(!policy.should_flush(&batch, first, FlushTrigger::Deadline));
|
||||
batch.push(1, 1, first);
|
||||
batch.push(2, 1, first + Duration::from_millis(9));
|
||||
assert_eq!(
|
||||
policy.deadline(&batch),
|
||||
Some(first + Duration::from_millis(10))
|
||||
);
|
||||
assert!(!policy.should_flush(
|
||||
&batch,
|
||||
first + Duration::from_millis(9),
|
||||
FlushTrigger::Deadline
|
||||
));
|
||||
assert!(policy.should_flush(
|
||||
&batch,
|
||||
first + Duration::from_millis(10),
|
||||
FlushTrigger::Deadline
|
||||
));
|
||||
assert!(!policy.reached_row_threshold(&batch));
|
||||
assert!(!policy.should_flush(
|
||||
&batch,
|
||||
first + Duration::from_millis(10),
|
||||
FlushTrigger::Submission,
|
||||
));
|
||||
assert_eq!(batch.take(), vec![1, 2]);
|
||||
let next = first + Duration::from_millis(20);
|
||||
batch.push(3, 0, next);
|
||||
assert!(!policy.should_flush(&batch, next, FlushTrigger::Deadline));
|
||||
assert!(policy.should_flush(
|
||||
&batch,
|
||||
next + Duration::from_millis(10),
|
||||
FlushTrigger::Deadline
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_row_threshold_preserves_complete_submissions() {
|
||||
let policy = TimingFlushPolicy::try_new(Duration::from_secs(1), 3).unwrap();
|
||||
let now = Instant::now();
|
||||
for rows in [2, 3, 4] {
|
||||
let mut batch = PendingBatch::new();
|
||||
batch.push(vec![0; rows], rows, now);
|
||||
assert_eq!(
|
||||
policy.should_flush(&batch, now, FlushTrigger::Submission),
|
||||
rows >= 3
|
||||
);
|
||||
assert_eq!(policy.reached_row_threshold(&batch), rows >= 3);
|
||||
assert_eq!(batch.take(), vec![vec![0; rows]]);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_invalid_interval() {
|
||||
assert!(TimingFlushPolicy::try_new(Duration::from_secs(1), 0).is_none());
|
||||
let max_batch_rows = 1;
|
||||
assert!(TimingFlushPolicy::try_new(Duration::ZERO, max_batch_rows).is_none());
|
||||
assert!(TimingFlushPolicy::try_new(Duration::MAX, max_batch_rows).is_none());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,141 @@
|
||||
// Copyright 2023 Greptime Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use std::pin::Pin;
|
||||
|
||||
use tokio::time::{Instant, Sleep, sleep_until};
|
||||
|
||||
/// Maintains one reusable timer for the current flush deadline.
|
||||
///
|
||||
/// Creating a timer does not require a Tokio runtime. Arming it requires a Tokio
|
||||
/// runtime with time enabled. Disabling retains the allocation for the next arm.
|
||||
#[derive(Default)]
|
||||
pub struct FlushTimer {
|
||||
deadline: Option<Instant>,
|
||||
sleep: Option<Pin<Box<Sleep>>>,
|
||||
}
|
||||
|
||||
impl FlushTimer {
|
||||
/// Creates a disabled timer without allocating a Tokio sleep.
|
||||
pub fn new() -> Self {
|
||||
Self::default()
|
||||
}
|
||||
|
||||
/// Arms or disables the timer, resetting its sleep only when the deadline changes.
|
||||
///
|
||||
/// # Panics
|
||||
///
|
||||
/// Arming for the first time panics outside a Tokio runtime with time enabled.
|
||||
pub fn set_deadline(&mut self, deadline: Option<Instant>) {
|
||||
if self.deadline == deadline {
|
||||
return;
|
||||
}
|
||||
self.deadline = deadline;
|
||||
if let Some(deadline) = deadline {
|
||||
match &mut self.sleep {
|
||||
Some(sleep) => sleep.as_mut().reset(deadline),
|
||||
None => self.sleep = Some(Box::pin(sleep_until(deadline))),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Waits until the armed deadline, or indefinitely while disabled.
|
||||
///
|
||||
/// Cancelling this wait does not change the deadline. An elapsed timer stays
|
||||
/// ready until the caller changes or disables its deadline.
|
||||
pub async fn wait(&mut self) {
|
||||
if self.deadline.is_some()
|
||||
&& let Some(sleep) = &mut self.sleep
|
||||
{
|
||||
sleep.as_mut().await;
|
||||
} else {
|
||||
std::future::pending::<()>().await;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::future::{Future, poll_fn};
|
||||
use std::task::Poll;
|
||||
use std::time::Duration;
|
||||
|
||||
use super::*;
|
||||
|
||||
async fn is_ready(timer: &mut FlushTimer) -> bool {
|
||||
let wait = timer.wait();
|
||||
tokio::pin!(wait);
|
||||
poll_fn(|cx| Poll::Ready(wait.as_mut().poll(cx).is_ready())).await
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_create_without_runtime() {
|
||||
let mut timer = FlushTimer::new();
|
||||
timer.set_deadline(None);
|
||||
assert!(timer.sleep.is_none());
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn test_deadline_and_cancelled_wait() {
|
||||
let mut timer = FlushTimer::new();
|
||||
assert!(!is_ready(&mut timer).await);
|
||||
let deadline = Instant::now() + Duration::from_millis(10);
|
||||
timer.set_deadline(Some(deadline));
|
||||
assert!(!is_ready(&mut timer).await);
|
||||
tokio::time::advance(Duration::from_millis(9)).await;
|
||||
// Each poll drops its wait future. Repeating the same deadline must not
|
||||
// move the timer forward or replace its sleep allocation.
|
||||
let sleep = timer.sleep.as_ref().unwrap().as_ref().get_ref() as *const Sleep;
|
||||
timer.set_deadline(Some(deadline));
|
||||
assert_eq!(
|
||||
sleep,
|
||||
timer.sleep.as_ref().unwrap().as_ref().get_ref() as *const Sleep
|
||||
);
|
||||
assert!(!is_ready(&mut timer).await);
|
||||
tokio::time::advance(Duration::from_millis(1)).await;
|
||||
assert!(is_ready(&mut timer).await);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn test_disable_and_rearm_reuses_sleep() {
|
||||
let mut timer = FlushTimer::new();
|
||||
timer.set_deadline(Some(Instant::now()));
|
||||
assert!(is_ready(&mut timer).await);
|
||||
let sleep = timer.sleep.as_ref().unwrap().as_ref().get_ref() as *const Sleep;
|
||||
timer.set_deadline(None);
|
||||
assert!(!is_ready(&mut timer).await);
|
||||
assert!(!is_ready(&mut timer).await);
|
||||
timer.set_deadline(Some(Instant::now() + Duration::from_millis(5)));
|
||||
assert_eq!(
|
||||
sleep,
|
||||
timer.sleep.as_ref().unwrap().as_ref().get_ref() as *const Sleep
|
||||
);
|
||||
assert!(!is_ready(&mut timer).await);
|
||||
tokio::time::advance(Duration::from_millis(5)).await;
|
||||
assert!(is_ready(&mut timer).await);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn test_change_armed_deadline() {
|
||||
let mut timer = FlushTimer::new();
|
||||
let now = Instant::now();
|
||||
timer.set_deadline(Some(now + Duration::from_millis(5)));
|
||||
assert!(!is_ready(&mut timer).await);
|
||||
timer.set_deadline(Some(now + Duration::from_millis(10)));
|
||||
tokio::time::advance(Duration::from_millis(5)).await;
|
||||
assert!(!is_ready(&mut timer).await);
|
||||
timer.set_deadline(Some(Instant::now()));
|
||||
assert!(is_ready(&mut timer).await);
|
||||
}
|
||||
}
|
||||
@@ -12,13 +12,16 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use datatypes::arrow::record_batch::RecordBatch;
|
||||
use futures::stream::BoxStream;
|
||||
//! Protocol-independent building blocks for timing-based batching.
|
||||
//!
|
||||
//! Payload preparation, grouping keys, write execution and request completion
|
||||
//! remain the caller's responsibility.
|
||||
|
||||
mod stream;
|
||||
|
||||
pub(crate) use stream::NestedSchemaAligner;
|
||||
|
||||
use crate::error::Result;
|
||||
|
||||
pub(crate) type ProjectedRecordBatchStream = BoxStream<'static, Result<RecordBatch>>;
|
||||
pub mod flush_limiter;
|
||||
pub mod flush_policy;
|
||||
pub mod flush_timer;
|
||||
pub mod notifier;
|
||||
pub mod pending_batch;
|
||||
pub mod pending_worker;
|
||||
pub mod request_limiter;
|
||||
pub mod worker_registry;
|
||||
@@ -0,0 +1,145 @@
|
||||
// Copyright 2023 Greptime Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use std::future::Future;
|
||||
use std::num::NonZeroUsize;
|
||||
|
||||
use futures::{StreamExt, stream};
|
||||
use tokio::sync::Semaphore;
|
||||
use tokio::sync::mpsc::error::TrySendError;
|
||||
use tokio::sync::mpsc::{self, Receiver, Sender};
|
||||
|
||||
/// Best-effort notification admission, independent of the write result.
|
||||
///
|
||||
/// The caller owns the notification payload, delivery handler and failure metrics.
|
||||
pub struct Notifier<T> {
|
||||
sender: Sender<T>,
|
||||
}
|
||||
|
||||
impl<T> Clone for Notifier<T> {
|
||||
fn clone(&self) -> Self {
|
||||
Self {
|
||||
sender: self.sender.clone(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<T> Notifier<T> {
|
||||
/// Returns `None` if the queue capacity is zero or exceeds Tokio's supported capacity.
|
||||
pub fn try_new(capacity: usize) -> Option<(Self, Receiver<T>)> {
|
||||
if capacity == 0 || capacity > Semaphore::MAX_PERMITS {
|
||||
return None;
|
||||
}
|
||||
let (sender, receiver) = mpsc::channel(capacity);
|
||||
Some((Self { sender }, receiver))
|
||||
}
|
||||
|
||||
/// Configured queue capacity, independent of its current occupancy.
|
||||
pub fn max_capacity(&self) -> usize {
|
||||
self.sender.max_capacity()
|
||||
}
|
||||
|
||||
/// Never waits for delivery. Returns the payload on a full or closed queue
|
||||
/// so the caller can record domain-specific diagnostics.
|
||||
pub fn try_notify(&self, notification: T) -> Result<(), TrySendError<T>> {
|
||||
self.sender.try_send(notification)
|
||||
}
|
||||
}
|
||||
|
||||
/// Delivers queued notifications with bounded concurrency.
|
||||
///
|
||||
/// The caller schedules this future on its runtime. Closing all senders drains
|
||||
/// queued and in-flight notifications. Handler failures are handled by the caller;
|
||||
/// this worker does not retry or propagate them into write completion.
|
||||
pub async fn run_notifier<T, F, Fut>(
|
||||
receiver: Receiver<T>,
|
||||
max_concurrent_notifications: NonZeroUsize,
|
||||
handler: F,
|
||||
) where
|
||||
F: FnMut(T) -> Fut,
|
||||
Fut: Future<Output = ()>,
|
||||
{
|
||||
stream::unfold(receiver, |mut receiver| async move {
|
||||
receiver.recv().await.map(|item| (item, receiver))
|
||||
})
|
||||
.for_each_concurrent(max_concurrent_notifications.get(), handler)
|
||||
.await;
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::future::{Future, poll_fn};
|
||||
use std::num::NonZeroUsize;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
use std::task::Poll;
|
||||
|
||||
use tokio::sync::Semaphore;
|
||||
use tokio::sync::mpsc::error::TrySendError;
|
||||
|
||||
use crate::notifier::{Notifier, run_notifier};
|
||||
|
||||
#[test]
|
||||
fn test_admission() {
|
||||
assert!(Notifier::<usize>::try_new(0).is_none());
|
||||
assert!(Notifier::<usize>::try_new(usize::MAX).is_none());
|
||||
let (notifier, receiver) = Notifier::try_new(1).unwrap();
|
||||
let other = notifier.clone();
|
||||
assert_eq!(other.max_capacity(), 1);
|
||||
assert_eq!(notifier.try_notify(1), Ok(()));
|
||||
assert_eq!(other.try_notify(2), Err(TrySendError::Full(2)));
|
||||
drop(receiver);
|
||||
assert_eq!(other.try_notify(3), Err(TrySendError::Closed(3)));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_concurrency_and_drain() {
|
||||
// One-factor comparison: only delivery concurrency changes.
|
||||
for _ in 0..2 {
|
||||
for concurrency in [1, 2] {
|
||||
let (notifier, receiver) = Notifier::try_new(3).unwrap();
|
||||
for item in 0..3 {
|
||||
notifier.try_notify(item).unwrap();
|
||||
}
|
||||
drop(notifier);
|
||||
let started = Arc::new(AtomicUsize::new(0));
|
||||
let completed = Arc::new(AtomicUsize::new(0));
|
||||
let gate = Arc::new(Semaphore::new(0));
|
||||
let worker =
|
||||
run_notifier(receiver, NonZeroUsize::new(concurrency).unwrap(), |_| {
|
||||
let started = started.clone();
|
||||
let completed = completed.clone();
|
||||
let gate = gate.clone();
|
||||
async move {
|
||||
started.fetch_add(1, Ordering::SeqCst);
|
||||
gate.acquire().await.unwrap().forget();
|
||||
completed.fetch_add(1, Ordering::SeqCst);
|
||||
}
|
||||
});
|
||||
tokio::pin!(worker);
|
||||
poll_fn(|cx| {
|
||||
assert!(worker.as_mut().poll(cx).is_pending());
|
||||
Poll::Ready(())
|
||||
})
|
||||
.await;
|
||||
assert_eq!(started.load(Ordering::SeqCst), concurrency);
|
||||
assert_eq!(completed.load(Ordering::SeqCst), 0);
|
||||
gate.add_permits(3);
|
||||
worker.await;
|
||||
assert_eq!(started.load(Ordering::SeqCst), 3);
|
||||
assert_eq!(completed.load(Ordering::SeqCst), 3);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,115 @@
|
||||
// Copyright 2023 Greptime Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use tokio::time::Instant;
|
||||
|
||||
/// Accumulates complete submissions in arrival order, without interpreting their data.
|
||||
#[derive(Debug)]
|
||||
pub struct PendingBatch<T> {
|
||||
items: Vec<T>,
|
||||
total_rows: usize,
|
||||
first_submitted_at: Option<Instant>,
|
||||
}
|
||||
|
||||
impl<T> Default for PendingBatch<T> {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
items: Vec::new(),
|
||||
total_rows: 0,
|
||||
first_submitted_at: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<T> PendingBatch<T> {
|
||||
/// Creates an empty batch.
|
||||
pub fn new() -> Self {
|
||||
Self::default()
|
||||
}
|
||||
|
||||
/// Appends a complete submission without moving the first submission's timestamp.
|
||||
pub fn push(&mut self, item: T, total_rows: usize, now: Instant) {
|
||||
self.first_submitted_at.get_or_insert(now);
|
||||
self.items.push(item);
|
||||
// Saturation preserves threshold decisions even for caller-supplied oversized weights.
|
||||
self.total_rows = self.total_rows.saturating_add(total_rows);
|
||||
}
|
||||
|
||||
/// Returns the accumulated row count, saturated at `usize::MAX`.
|
||||
pub fn total_rows(&self) -> usize {
|
||||
self.total_rows
|
||||
}
|
||||
|
||||
/// Returns the number of complete submissions.
|
||||
pub fn len(&self) -> usize {
|
||||
self.items.len()
|
||||
}
|
||||
|
||||
/// Returns whether there are no submissions, including zero-row submissions.
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.items.is_empty()
|
||||
}
|
||||
|
||||
/// Returns the time the first submission was appended to the current batch.
|
||||
pub fn first_submitted_at(&self) -> Option<Instant> {
|
||||
self.first_submitted_at
|
||||
}
|
||||
|
||||
/// Takes all submissions in arrival order and resets the batch's accounting.
|
||||
pub fn take(&mut self) -> Vec<T> {
|
||||
self.total_rows = 0;
|
||||
self.first_submitted_at = None;
|
||||
std::mem::take(&mut self.items)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::time::Duration;
|
||||
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_complete_submissions_and_reset() {
|
||||
let mut batch = PendingBatch::new();
|
||||
let first = Instant::now();
|
||||
assert!(batch.is_empty());
|
||||
assert_eq!(batch.first_submitted_at(), None);
|
||||
batch.push(vec![1, 2], 2, first);
|
||||
batch.push(vec![3, 4, 5], 3, first + Duration::from_secs(1));
|
||||
assert_eq!(batch.len(), 2);
|
||||
assert_eq!(batch.total_rows(), 5);
|
||||
assert_eq!(batch.first_submitted_at(), Some(first));
|
||||
assert_eq!(batch.take(), vec![vec![1, 2], vec![3, 4, 5]]);
|
||||
assert!(batch.is_empty());
|
||||
assert_eq!(batch.total_rows(), 0);
|
||||
assert_eq!(batch.first_submitted_at(), None);
|
||||
let next = first + Duration::from_secs(2);
|
||||
batch.push(vec![6], 1, next);
|
||||
assert_eq!(batch.first_submitted_at(), Some(next));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_zero_rows_and_saturating_count() {
|
||||
let mut batch = PendingBatch::new();
|
||||
let now = Instant::now();
|
||||
batch.push(1, 0, now);
|
||||
assert!(!batch.is_empty());
|
||||
assert_eq!(batch.total_rows(), 0);
|
||||
batch.push(2, usize::MAX, now);
|
||||
batch.push(3, 1, now);
|
||||
assert_eq!(batch.total_rows(), usize::MAX);
|
||||
assert_eq!(batch.take(), vec![1, 2, 3]);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,260 @@
|
||||
// Copyright 2023 Greptime Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use tokio::time::Instant;
|
||||
|
||||
use crate::flush_policy::{FlushPolicy, FlushTrigger};
|
||||
use crate::flush_timer::FlushTimer;
|
||||
use crate::pending_batch::PendingBatch;
|
||||
|
||||
/// Batch state for one grouping key, independent of the caller's event loop.
|
||||
///
|
||||
/// The caller owns channels, idle/shutdown decisions, execution limits, task
|
||||
/// spawning and business completion. Taking a batch transfers its items to the
|
||||
/// caller; no background task is started by this worker.
|
||||
pub struct PendingWorker<T, P> {
|
||||
batch: PendingBatch<T>,
|
||||
flush_policy: P,
|
||||
flush_timer: FlushTimer,
|
||||
}
|
||||
|
||||
impl<T, P> PendingWorker<T, P> {
|
||||
/// Creates an empty worker without starting a timer or runtime task.
|
||||
pub fn new(flush_policy: P) -> Self {
|
||||
Self {
|
||||
batch: PendingBatch::new(),
|
||||
flush_policy,
|
||||
flush_timer: FlushTimer::new(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns whether the worker has no submissions, including zero-row items.
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.batch.is_empty()
|
||||
}
|
||||
|
||||
/// Returns the pending row count before ownership is transferred to the caller.
|
||||
pub fn total_rows(&self) -> usize {
|
||||
self.batch.total_rows()
|
||||
}
|
||||
|
||||
/// Waits for a timer wakeup without taking or executing the batch.
|
||||
///
|
||||
/// Cancellation is safe: selecting another event does not lose items or
|
||||
/// reset the deadline. After a wakeup, call [`Self::take_ready`] with
|
||||
/// [`FlushTrigger::Deadline`]. An empty or unarmed worker stays pending.
|
||||
pub async fn wait_flush(&mut self) {
|
||||
self.flush_timer.wait().await;
|
||||
}
|
||||
|
||||
/// Takes all pending submissions regardless of policy, for example on shutdown.
|
||||
///
|
||||
/// Returns `None` for an empty batch. The caller decides whether to execute
|
||||
/// inline, acquire a flush permit or discard the returned items.
|
||||
pub fn take_pending(&mut self) -> Option<Vec<T>> {
|
||||
self.flush_timer.set_deadline(None);
|
||||
if self.batch.is_empty() {
|
||||
None
|
||||
} else {
|
||||
Some(self.batch.take())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<T, P: FlushPolicy> PendingWorker<T, P> {
|
||||
/// Appends one complete submission and arms the policy's deadline.
|
||||
///
|
||||
/// Call [`Self::take_ready`] with [`FlushTrigger::Submission`] afterwards to
|
||||
/// check whether this submission triggered a flush. Timer-backed policies
|
||||
/// require a Tokio runtime with time enabled.
|
||||
///
|
||||
/// # Panics
|
||||
///
|
||||
/// Panics if arming a new timer outside a Tokio runtime with time enabled.
|
||||
pub fn submit(&mut self, item: T, total_rows: usize) {
|
||||
self.batch.push(item, total_rows, Instant::now());
|
||||
self.refresh_deadline();
|
||||
}
|
||||
|
||||
/// Takes the batch only when the policy permits flushing for this event.
|
||||
///
|
||||
/// Like [`Self::submit`], this may arm a timer and requires a time-enabled
|
||||
/// Tokio runtime when the policy starts using deadlines.
|
||||
pub fn take_ready(&mut self, trigger: FlushTrigger) -> Option<Vec<T>> {
|
||||
if !self.batch.is_empty()
|
||||
&& self
|
||||
.flush_policy
|
||||
.should_flush(&self.batch, Instant::now(), trigger)
|
||||
{
|
||||
self.take_pending()
|
||||
} else {
|
||||
self.refresh_deadline();
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
fn refresh_deadline(&mut self) {
|
||||
let deadline = if self.batch.is_empty() {
|
||||
None
|
||||
} else {
|
||||
self.flush_policy.deadline(&self.batch)
|
||||
};
|
||||
self.flush_timer.set_deadline(deadline);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::future::Future;
|
||||
use std::task::Poll;
|
||||
use std::time::Duration;
|
||||
|
||||
use super::*;
|
||||
use crate::flush_policy::timing::TimingFlushPolicy;
|
||||
|
||||
fn worker(rows: usize) -> PendingWorker<i32, TimingFlushPolicy> {
|
||||
PendingWorker::new(TimingFlushPolicy::try_new(Duration::from_millis(10), rows).unwrap())
|
||||
}
|
||||
|
||||
async fn assert_wait_pending<T, P>(worker: &mut PendingWorker<T, P>) {
|
||||
let wait = worker.wait_flush();
|
||||
tokio::pin!(wait);
|
||||
assert!(std::future::poll_fn(|cx| Poll::Ready(wait.as_mut().poll(cx).is_pending())).await);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn test_first_deadline_and_cancelled_wait() {
|
||||
let mut worker = worker(100);
|
||||
let start = Instant::now();
|
||||
worker.submit(1, 1);
|
||||
assert_wait_pending(&mut worker).await;
|
||||
tokio::time::advance(Duration::from_millis(5)).await;
|
||||
worker.submit(2, 1);
|
||||
assert!(worker.take_ready(FlushTrigger::Submission).is_none());
|
||||
assert_wait_pending(&mut worker).await;
|
||||
worker.wait_flush().await;
|
||||
assert_eq!(start + Duration::from_millis(10), Instant::now());
|
||||
assert_eq!(Some(vec![1, 2]), worker.take_ready(FlushTrigger::Deadline));
|
||||
assert!(worker.is_empty());
|
||||
assert_eq!(0, worker.total_rows());
|
||||
assert_wait_pending(&mut worker).await;
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn test_submission_does_not_consume_expired_deadline() {
|
||||
let mut worker = worker(100);
|
||||
worker.submit(1, 1);
|
||||
tokio::time::advance(Duration::from_millis(10)).await;
|
||||
worker.submit(2, 1);
|
||||
assert!(worker.take_ready(FlushTrigger::Submission).is_none());
|
||||
worker.wait_flush().await;
|
||||
assert_eq!(Some(vec![1, 2]), worker.take_ready(FlushTrigger::Deadline));
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn test_size_flush_and_rearm() {
|
||||
let mut worker = worker(2);
|
||||
worker.submit(1, 3);
|
||||
assert_eq!(3, worker.total_rows());
|
||||
assert_eq!(Some(vec![1]), worker.take_ready(FlushTrigger::Submission));
|
||||
tokio::time::advance(Duration::from_secs(1)).await;
|
||||
assert_wait_pending(&mut worker).await;
|
||||
let start = Instant::now();
|
||||
worker.submit(2, 1);
|
||||
worker.wait_flush().await;
|
||||
assert_eq!(start + Duration::from_millis(10), Instant::now());
|
||||
assert_eq!(Some(vec![2]), worker.take_ready(FlushTrigger::Deadline));
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn test_take_pending_without_waiting() {
|
||||
let mut worker = worker(100);
|
||||
assert_eq!(None, worker.take_pending());
|
||||
worker.submit(1, 0);
|
||||
assert!(!worker.is_empty());
|
||||
let now = Instant::now();
|
||||
assert_eq!(Some(vec![1]), worker.take_pending());
|
||||
assert_eq!(now, Instant::now());
|
||||
tokio::time::advance(Duration::from_secs(1)).await;
|
||||
assert_wait_pending(&mut worker).await;
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn test_row_threshold_ablation() {
|
||||
for (rows, early) in [(2, true), (100, false)] {
|
||||
let mut worker = worker(rows);
|
||||
let start = Instant::now();
|
||||
worker.submit(1, 1);
|
||||
tokio::time::advance(Duration::from_millis(5)).await;
|
||||
worker.submit(2, 1);
|
||||
let batch = worker.take_ready(FlushTrigger::Submission);
|
||||
if early {
|
||||
assert_eq!(Some(vec![1, 2]), batch);
|
||||
assert_eq!(start + Duration::from_millis(5), Instant::now());
|
||||
} else {
|
||||
assert!(batch.is_none());
|
||||
worker.wait_flush().await;
|
||||
assert_eq!(start + Duration::from_millis(10), Instant::now());
|
||||
assert_eq!(Some(vec![1, 2]), worker.take_ready(FlushTrigger::Deadline));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_non_timing_policy() {
|
||||
struct TwoSubmissionsPolicy;
|
||||
impl FlushPolicy for TwoSubmissionsPolicy {
|
||||
fn deadline<T>(&self, _: &PendingBatch<T>) -> Option<Instant> {
|
||||
None
|
||||
}
|
||||
fn should_flush<T>(
|
||||
&self,
|
||||
batch: &PendingBatch<T>,
|
||||
_: Instant,
|
||||
_: FlushTrigger,
|
||||
) -> bool {
|
||||
batch.len() >= 2
|
||||
}
|
||||
}
|
||||
let mut worker = PendingWorker::new(TwoSubmissionsPolicy);
|
||||
worker.submit(1, 0);
|
||||
assert_wait_pending(&mut worker).await;
|
||||
assert!(worker.take_ready(FlushTrigger::Submission).is_none());
|
||||
worker.submit(2, 0);
|
||||
assert_eq!(
|
||||
Some(vec![1, 2]),
|
||||
worker.take_ready(FlushTrigger::Submission)
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn test_caller_owns_event_loop() {
|
||||
let mut worker = worker(100);
|
||||
let (tx, mut rx) = tokio::sync::mpsc::channel(1);
|
||||
tx.send(1).await.unwrap();
|
||||
tokio::select! {
|
||||
_ = worker.wait_flush() => panic!("empty worker must not wake"),
|
||||
item = rx.recv() => worker.submit(item.unwrap(), 1),
|
||||
}
|
||||
assert!(worker.take_ready(FlushTrigger::Submission).is_none());
|
||||
let start = Instant::now();
|
||||
tokio::select! {
|
||||
_ = worker.wait_flush() => {
|
||||
assert_eq!(Some(vec![1]), worker.take_ready(FlushTrigger::Deadline));
|
||||
}
|
||||
_ = rx.recv() => panic!("sender is still open without another submission"),
|
||||
}
|
||||
assert_eq!(start + Duration::from_millis(10), Instant::now());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,115 @@
|
||||
// Copyright 2023 Greptime Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use tokio::sync::{AcquireError, OwnedSemaphorePermit, Semaphore};
|
||||
|
||||
/// Limits unfinished original requests independently of their number of submissions.
|
||||
///
|
||||
/// Acquire once for each original request, then clone its permit into all of that
|
||||
/// request's submissions. Capacity is released when the last owner drops it.
|
||||
#[derive(Clone)]
|
||||
pub struct RequestLimiter {
|
||||
semaphore: Arc<Semaphore>,
|
||||
}
|
||||
|
||||
impl RequestLimiter {
|
||||
/// Returns `None` if the limit is zero or exceeds Tokio's supported semaphore capacity.
|
||||
pub fn try_new(max_inflight_requests: usize) -> Option<Self> {
|
||||
let permits = max_inflight_requests;
|
||||
((1..=Semaphore::MAX_PERMITS).contains(&permits)).then(|| Self {
|
||||
semaphore: Arc::new(Semaphore::new(permits)),
|
||||
})
|
||||
}
|
||||
|
||||
/// Waits for one request slot and returns a permit shared by its submissions.
|
||||
///
|
||||
/// Cancelling a pending acquisition does not consume a slot. Once acquired,
|
||||
/// callers must retain the permit until every submission has finished, even
|
||||
/// if the original caller stops waiting for its response.
|
||||
pub async fn acquire(&self) -> Result<Arc<OwnedSemaphorePermit>, AcquireError> {
|
||||
self.semaphore.clone().acquire_owned().await.map(Arc::new)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::future::{Future, poll_fn};
|
||||
use std::pin::Pin;
|
||||
use std::task::Poll;
|
||||
|
||||
use tokio::sync::Semaphore;
|
||||
|
||||
use crate::request_limiter::RequestLimiter;
|
||||
|
||||
async fn is_pending<F: Future>(mut future: Pin<&mut F>) -> bool {
|
||||
poll_fn(|cx| Poll::Ready(future.as_mut().poll(cx).is_pending())).await
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_capacity_boundaries() {
|
||||
for capacity in [1, Semaphore::MAX_PERMITS] {
|
||||
assert!(RequestLimiter::try_new(capacity).is_some());
|
||||
}
|
||||
for capacity in [0, Semaphore::MAX_PERMITS + 1, usize::MAX] {
|
||||
assert!(RequestLimiter::try_new(capacity).is_none());
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_last_submission_releases_request_slot() {
|
||||
let limiter = RequestLimiter::try_new(1).unwrap();
|
||||
let other = limiter.clone();
|
||||
let request = limiter.acquire().await.unwrap();
|
||||
let first_submission = request.clone();
|
||||
let second_submission = request.clone();
|
||||
let mut waiting = Box::pin(other.acquire());
|
||||
assert!(is_pending(waiting.as_mut()).await);
|
||||
// The caller and one submission finish, but another submission remains.
|
||||
drop(request);
|
||||
drop(first_submission);
|
||||
assert!(is_pending(waiting.as_mut()).await);
|
||||
drop(second_submission);
|
||||
let next = waiting.await.unwrap();
|
||||
assert_eq!(next.num_permits(), 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_cancelled_acquisition_does_not_leak_capacity() {
|
||||
let limiter = RequestLimiter::try_new(1).unwrap();
|
||||
let permit = limiter.acquire().await.unwrap();
|
||||
let mut cancelled = Box::pin(limiter.acquire());
|
||||
assert!(is_pending(cancelled.as_mut()).await);
|
||||
drop(cancelled);
|
||||
let mut next = Box::pin(limiter.acquire());
|
||||
assert!(is_pending(next.as_mut()).await);
|
||||
drop(permit);
|
||||
let next = next.await.unwrap();
|
||||
drop(next);
|
||||
let _available = limiter.acquire().await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_independent_requests_consume_separate_slots() {
|
||||
let limiter = RequestLimiter::try_new(2).unwrap();
|
||||
let first = limiter.acquire().await.unwrap();
|
||||
let second = limiter.acquire().await.unwrap();
|
||||
let mut third = Box::pin(limiter.acquire());
|
||||
assert!(is_pending(third.as_mut()).await);
|
||||
drop(first);
|
||||
let _third = third.await.unwrap();
|
||||
drop(second);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,265 @@
|
||||
// Copyright 2023 Greptime Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use std::hash::Hash;
|
||||
|
||||
use dashmap::DashMap;
|
||||
use dashmap::mapref::entry::Entry;
|
||||
use tokio::sync::mpsc::{self, Receiver, Sender};
|
||||
|
||||
/// Registers worker senders by key, without owning workers or their execution.
|
||||
///
|
||||
/// Closed senders are replaced on lookup/creation. Cleanup checks channel
|
||||
/// identity under the same lock as replacement, so an old worker cannot remove
|
||||
/// a replacement registered under its key.
|
||||
pub struct WorkerRegistry<K, T> {
|
||||
workers: DashMap<K, Sender<T>>,
|
||||
}
|
||||
|
||||
impl<K: Eq + Hash, T> Default for WorkerRegistry<K, T> {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
workers: DashMap::new(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<K: Eq + Hash, T> WorkerRegistry<K, T> {
|
||||
pub fn new() -> Self {
|
||||
Self::default()
|
||||
}
|
||||
|
||||
/// Returns a live sender, if one is currently registered.
|
||||
/// The receiver can close after this method returns; callers must handle a
|
||||
/// failed send and retry worker lookup without discarding the unsent item.
|
||||
pub async fn get(&self, key: &K) -> Option<Sender<T>> {
|
||||
self.workers
|
||||
.get(key)
|
||||
.filter(|tx| !tx.is_closed())
|
||||
.map(|tx| tx.value().clone())
|
||||
}
|
||||
|
||||
/// Returns the registered sender and, only when created, its receiver.
|
||||
///
|
||||
/// Start the new worker before the next await so cancellation cannot leave
|
||||
/// a registered channel without a consumer. Existing channels retain their
|
||||
/// original capacity. New channel capacity must satisfy `mpsc::channel`.
|
||||
pub async fn get_or_create(&self, key: K, capacity: usize) -> (Sender<T>, Option<Receiver<T>>) {
|
||||
let mut receiver = None;
|
||||
let sender = self
|
||||
.get_or_insert_with(key, || {
|
||||
let (sender, rx) = mpsc::channel(capacity);
|
||||
receiver = Some(rx);
|
||||
sender
|
||||
})
|
||||
.await;
|
||||
(sender, receiver)
|
||||
}
|
||||
|
||||
/// Reuses a live sender or atomically creates its replacement.
|
||||
///
|
||||
/// `create` runs synchronously under the registry shard lock. It should only
|
||||
/// prepare the sender and capture any initialization state (such as the
|
||||
/// receiver) for the caller; start the worker after this method returns.
|
||||
/// Do not block or reenter the registry from `create`.
|
||||
pub async fn get_or_insert_with<F>(&self, key: K, create: F) -> Sender<T>
|
||||
where
|
||||
F: FnOnce() -> Sender<T>,
|
||||
{
|
||||
if let Some(tx) = self.get(&key).await {
|
||||
return tx;
|
||||
}
|
||||
|
||||
match self.workers.entry(key) {
|
||||
Entry::Occupied(mut entry) => {
|
||||
if entry.get().is_closed() {
|
||||
entry.insert(create());
|
||||
}
|
||||
entry.get().clone()
|
||||
}
|
||||
Entry::Vacant(entry) => entry.insert(create()).value().clone(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Removes the key only if it still points to this worker's channel.
|
||||
pub async fn remove_if_same(&self, key: &K, tx: &Sender<T>) -> bool {
|
||||
self.workers
|
||||
.remove_if(key, |_, current| current.same_channel(tx))
|
||||
.is_some()
|
||||
}
|
||||
|
||||
/// Number of registered entries, including senders whose receivers closed.
|
||||
pub async fn len(&self) -> usize {
|
||||
self.workers.len()
|
||||
}
|
||||
|
||||
pub async fn is_empty(&self) -> bool {
|
||||
self.workers.is_empty()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
|
||||
use tokio::sync::{Barrier, mpsc};
|
||||
|
||||
use crate::worker_registry::WorkerRegistry;
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_live_lookup_uses_read_lock() {
|
||||
let registry = Arc::new(WorkerRegistry::<_, ()>::new());
|
||||
let (sender, _receiver) = registry.get_or_create(1, 1).await;
|
||||
let guard = registry.workers.get(&1).unwrap();
|
||||
let (result_tx, result_rx) = std::sync::mpsc::channel();
|
||||
let worker_registry = registry.clone();
|
||||
let runtime = tokio::runtime::Handle::current();
|
||||
let thread = std::thread::spawn(move || {
|
||||
let result = runtime.block_on(worker_registry.get_or_create(1, 1));
|
||||
let _ = result_tx.send(result);
|
||||
});
|
||||
// Release the read guard before joining, even if lookup needs a write lock.
|
||||
let result = result_rx.recv_timeout(std::time::Duration::from_secs(5));
|
||||
drop(guard);
|
||||
thread.join().unwrap();
|
||||
let (reused, receiver) = result.expect("live lookup must not wait for an exclusive lock");
|
||||
assert!(sender.same_channel(&reused));
|
||||
assert!(receiver.is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_reuses_live_sender_and_returns_receiver_to_caller() {
|
||||
let registry = WorkerRegistry::new();
|
||||
assert!(registry.is_empty().await);
|
||||
assert!(registry.get(&1).await.is_none());
|
||||
let mut receiver = None;
|
||||
let first = registry
|
||||
.get_or_insert_with(1, || {
|
||||
let (tx, rx) = mpsc::channel(1);
|
||||
receiver = Some(rx);
|
||||
tx
|
||||
})
|
||||
.await;
|
||||
let second = registry
|
||||
.get_or_insert_with(1, || panic!("live worker must be reused"))
|
||||
.await;
|
||||
assert!(first.same_channel(&second));
|
||||
assert!(registry.get(&1).await.unwrap().same_channel(&first));
|
||||
first.send(7).await.unwrap();
|
||||
assert_eq!(Some(7), receiver.as_mut().unwrap().recv().await);
|
||||
assert_eq!(1, registry.len().await);
|
||||
assert!(registry.remove_if_same(&1, &first).await);
|
||||
assert!(!registry.remove_if_same(&1, &first).await);
|
||||
assert!(registry.is_empty().await);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_old_cleanup_does_not_remove_replacement() {
|
||||
let registry = WorkerRegistry::<_, ()>::new();
|
||||
let (first, mut receiver) = mpsc::channel(1);
|
||||
registry.get_or_insert_with("table", || first.clone()).await;
|
||||
receiver.close();
|
||||
assert!(registry.get(&"table").await.is_none());
|
||||
// Closed entries remain accounted until replacement or explicit removal.
|
||||
assert_eq!(1, registry.len().await);
|
||||
let (second, _receiver) = mpsc::channel(1);
|
||||
registry
|
||||
.get_or_insert_with("table", || second.clone())
|
||||
.await;
|
||||
assert!(!registry.remove_if_same(&"table", &first).await);
|
||||
assert!(registry.get(&"table").await.unwrap().same_channel(&second));
|
||||
assert_eq!(1, registry.len().await);
|
||||
assert!(registry.remove_if_same(&"table", &second).await);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn test_concurrent_lookup_initializes_once() {
|
||||
let registry = Arc::new(WorkerRegistry::<_, ()>::new());
|
||||
let barrier = Arc::new(Barrier::new(3));
|
||||
let count = Arc::new(AtomicUsize::new(0));
|
||||
let mut tasks = Vec::new();
|
||||
for _ in 0..2 {
|
||||
let registry = registry.clone();
|
||||
let barrier = barrier.clone();
|
||||
let count = count.clone();
|
||||
tasks.push(tokio::spawn(async move {
|
||||
barrier.wait().await;
|
||||
let mut receiver = None;
|
||||
let tx = registry
|
||||
.get_or_insert_with(1, || {
|
||||
count.fetch_add(1, Ordering::Relaxed);
|
||||
let (tx, rx) = mpsc::channel(1);
|
||||
receiver = Some(rx);
|
||||
tx
|
||||
})
|
||||
.await;
|
||||
// Keep the newly created receiver alive while both lookups run.
|
||||
barrier.wait().await;
|
||||
(tx, receiver)
|
||||
}));
|
||||
}
|
||||
barrier.wait().await;
|
||||
barrier.wait().await;
|
||||
let first = tasks.remove(0).await.unwrap();
|
||||
let second = tasks.remove(0).await.unwrap();
|
||||
assert!(first.0.same_channel(&second.0));
|
||||
assert_eq!(1, count.load(Ordering::Relaxed));
|
||||
assert_eq!(1, registry.len().await);
|
||||
}
|
||||
#[tokio::test]
|
||||
async fn test_channel_creation_replacement_and_cleanup() {
|
||||
let registry = WorkerRegistry::<_, usize>::new();
|
||||
let (first, receiver) = registry.get_or_create(1, 2).await;
|
||||
let mut receiver = receiver.unwrap();
|
||||
let (reused, absent) = registry.get_or_create(1, 3).await;
|
||||
assert!(absent.is_none());
|
||||
assert!(first.same_channel(&reused));
|
||||
assert_eq!(2, reused.max_capacity());
|
||||
first.send(7).await.unwrap();
|
||||
assert_eq!(Some(7), receiver.recv().await);
|
||||
receiver.close();
|
||||
let (replacement, receiver) = registry.get_or_create(1, 3).await;
|
||||
assert!(receiver.is_some());
|
||||
assert_eq!(3, replacement.max_capacity());
|
||||
assert!(!first.same_channel(&replacement));
|
||||
assert!(!registry.remove_if_same(&1, &first).await);
|
||||
assert!(registry.get(&1).await.unwrap().same_channel(&replacement));
|
||||
assert!(registry.remove_if_same(&1, &replacement).await);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn test_concurrent_channel_creation_returns_one_receiver() {
|
||||
let registry = Arc::new(WorkerRegistry::<_, ()>::new());
|
||||
let barrier = Arc::new(Barrier::new(3));
|
||||
let mut tasks = Vec::new();
|
||||
for _ in 0..2 {
|
||||
let registry = registry.clone();
|
||||
let barrier = barrier.clone();
|
||||
tasks.push(tokio::spawn(async move {
|
||||
barrier.wait().await;
|
||||
let channel = registry.get_or_create(1, 2).await;
|
||||
barrier.wait().await;
|
||||
channel
|
||||
}));
|
||||
}
|
||||
barrier.wait().await;
|
||||
barrier.wait().await;
|
||||
let first = tasks.remove(0).await.unwrap();
|
||||
let second = tasks.remove(0).await.unwrap();
|
||||
assert!(first.0.same_channel(&second.0));
|
||||
assert_ne!(first.1.is_some(), second.1.is_some());
|
||||
assert_eq!(1, registry.len().await);
|
||||
}
|
||||
}
|
||||
@@ -122,6 +122,12 @@ pub const INFORMATION_SCHEMA_STATISTICS_TABLE_ID: u32 = 43;
|
||||
pub const INFORMATION_SCHEMA_RECYCLE_BIN_TABLE_ID: u32 = 44;
|
||||
/// id for information_schema.flow_statistics
|
||||
pub const INFORMATION_SCHEMA_FLOW_STATISTICS_TABLE_ID: u32 = 45;
|
||||
/// id for information_schema.PLUGINS
|
||||
pub const INFORMATION_SCHEMA_PLUGINS_TABLE_ID: u32 = 47;
|
||||
/// id for information_schema.USER_PRIVILEGES
|
||||
pub const INFORMATION_SCHEMA_USER_PRIVILEGES_TABLE_ID: u32 = 48;
|
||||
/// id for information_schema.PROCESSLIST (for mysql)
|
||||
pub const INFORMATION_SCHEMA_PROCESSLIST_TABLE_ID: u32 = 49;
|
||||
|
||||
// ----- End of information_schema tables -----
|
||||
|
||||
|
||||
@@ -34,7 +34,7 @@ futures.workspace = true
|
||||
lazy_static.workspace = true
|
||||
object-store.workspace = true
|
||||
object_store_opendal.workspace = true
|
||||
orc-rust = { version = "0.8", default-features = false, features = ["async"] }
|
||||
orc-rust = { version = "0.9", default-features = false, features = ["async"] }
|
||||
parquet.workspace = true
|
||||
paste.workspace = true
|
||||
regex.workspace = true
|
||||
@@ -47,3 +47,4 @@ url.workspace = true
|
||||
|
||||
[dev-dependencies]
|
||||
common-test-util.workspace = true
|
||||
object-store = { workspace = true, features = ["testing"] }
|
||||
|
||||
@@ -27,6 +27,15 @@ use url::ParseError;
|
||||
#[snafu(visibility(pub))]
|
||||
#[stack_trace_debug]
|
||||
pub enum Error {
|
||||
#[snafu(display("Parquet write cancelled"))]
|
||||
ParquetWriteCancelled {},
|
||||
|
||||
#[snafu(display("Parquet writer limits must be positive"))]
|
||||
InvalidParquetWriterLimits {},
|
||||
|
||||
#[snafu(display("Parquet writer resource limit exceeded: {reason}"))]
|
||||
ParquetWriterResource { reason: String },
|
||||
|
||||
#[snafu(display("Unsupported compression type: {}", compression_type))]
|
||||
UnsupportedCompressionType {
|
||||
compression_type: String,
|
||||
@@ -275,6 +284,9 @@ impl ErrorExt for Error {
|
||||
fn status_code(&self) -> StatusCode {
|
||||
use Error::*;
|
||||
match self {
|
||||
ParquetWriteCancelled {} => StatusCode::Cancelled,
|
||||
InvalidParquetWriterLimits {} => StatusCode::InvalidArguments,
|
||||
ParquetWriterResource { .. } => StatusCode::Suspended,
|
||||
BuildBackend { .. }
|
||||
| ListObjects { .. }
|
||||
| ReadObject { .. }
|
||||
|
||||
@@ -35,7 +35,7 @@ use datafusion::datasource::file_format::file_compression_type::FileCompressionT
|
||||
use datafusion::datasource::listing::PartitionedFile;
|
||||
use datafusion::datasource::object_store::ObjectStoreUrl;
|
||||
use datafusion::datasource::physical_plan::{
|
||||
FileGroup, FileOpenFuture, FileScanConfigBuilder, FileSource, FileStream,
|
||||
FileGroup, FileOpenFuture, FileScanConfigBuilder, FileSource, FileStreamBuilder,
|
||||
};
|
||||
use datafusion::error::{DataFusionError, Result as DataFusionResult};
|
||||
use datafusion::physical_plan::SendableRecordBatchStream;
|
||||
@@ -321,7 +321,11 @@ pub async fn file_to_stream(
|
||||
|
||||
let store = Arc::new(object_store_opendal::OpendalStore::new(store.clone()));
|
||||
let file_opener = config.file_source().create_file_opener(store, &config, 0)?;
|
||||
let stream = FileStream::new(&config, 0, file_opener, &ExecutionPlanMetricsSet::new())?;
|
||||
let stream = FileStreamBuilder::new(&config)
|
||||
.with_partition(0)
|
||||
.with_file_opener(file_opener)
|
||||
.with_metrics(&ExecutionPlanMetricsSet::new())
|
||||
.build()?;
|
||||
|
||||
Ok(Box::pin(stream))
|
||||
}
|
||||
|
||||
@@ -29,22 +29,17 @@ use datafusion::parquet::file::metadata::{
|
||||
use datafusion::physical_plan::SendableRecordBatchStream;
|
||||
use datafusion::physical_plan::metrics::ExecutionPlanMetricsSet;
|
||||
use datafusion_datasource::PartitionedFile;
|
||||
use datatypes::schema::SchemaRef;
|
||||
use futures::StreamExt;
|
||||
use futures::future::BoxFuture;
|
||||
use object_store::{FuturesAsyncReader, ObjectStore};
|
||||
use parquet::arrow::AsyncArrowWriter;
|
||||
use parquet::arrow::arrow_reader::ArrowReaderOptions;
|
||||
use parquet::basic::{Compression, Encoding, ZstdLevel};
|
||||
use parquet::file::properties::{WriterProperties, WriterPropertiesBuilder};
|
||||
use parquet::schema::types::ColumnPath;
|
||||
use snafu::ResultExt;
|
||||
use tokio_util::compat::{Compat, FuturesAsyncReadCompatExt, FuturesAsyncWriteCompatExt};
|
||||
use tokio_util::compat::{Compat, FuturesAsyncReadCompatExt};
|
||||
|
||||
use crate::DEFAULT_WRITE_BUFFER_SIZE;
|
||||
use crate::buffered_writer::{ArrowWriterCloser, DfRecordBatchEncoder};
|
||||
use crate::error::{self, Result, WriteObjectSnafu, WriteParquetSnafu};
|
||||
use crate::error::{self, Result};
|
||||
use crate::file_format::FileFormat;
|
||||
use crate::parquet_writer::ParquetFileWriter;
|
||||
use crate::share_buffer::SharedBuffer;
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
|
||||
@@ -212,61 +207,23 @@ impl ArrowWriterCloser for ArrowWriter<SharedBuffer> {
|
||||
/// Returns number of rows written.
|
||||
pub async fn stream_to_parquet(
|
||||
mut stream: SendableRecordBatchStream,
|
||||
schema: datatypes::schema::SchemaRef,
|
||||
store: ObjectStore,
|
||||
path: &str,
|
||||
concurrency: usize,
|
||||
) -> Result<usize> {
|
||||
let write_props = column_wise_config(
|
||||
WriterProperties::builder()
|
||||
.set_compression(Compression::ZSTD(ZstdLevel::default()))
|
||||
.set_statistics_truncate_length(None)
|
||||
.set_column_index_truncate_length(None),
|
||||
schema,
|
||||
)
|
||||
.build();
|
||||
let inner_writer = store
|
||||
.writer_with(path)
|
||||
.concurrent(concurrency)
|
||||
.chunk(DEFAULT_WRITE_BUFFER_SIZE.as_bytes() as usize)
|
||||
.await
|
||||
.map(|w| w.into_futures_async_write().compat_write())
|
||||
.context(WriteObjectSnafu { path })?;
|
||||
|
||||
let mut writer = AsyncArrowWriter::try_new(inner_writer, stream.schema(), Some(write_props))
|
||||
.context(WriteParquetSnafu { path })?;
|
||||
let mut writer =
|
||||
ParquetFileWriter::open(stream.schema(), store, path, concurrency, None).await?;
|
||||
let mut rows_written = 0;
|
||||
|
||||
while let Some(batch) = stream.next().await {
|
||||
let batch = batch.context(error::ReadRecordBatchSnafu)?;
|
||||
writer
|
||||
.write(&batch)
|
||||
.await
|
||||
.context(WriteParquetSnafu { path })?;
|
||||
rows_written += batch.num_rows();
|
||||
let rows = batch.num_rows();
|
||||
writer.write(batch, None).await?;
|
||||
rows_written += rows;
|
||||
}
|
||||
writer.close().await.context(WriteParquetSnafu { path })?;
|
||||
writer.finish(None).await?;
|
||||
Ok(rows_written)
|
||||
}
|
||||
|
||||
/// Customizes per-column properties.
|
||||
fn column_wise_config(
|
||||
mut props: WriterPropertiesBuilder,
|
||||
schema: SchemaRef,
|
||||
) -> WriterPropertiesBuilder {
|
||||
// Disable dictionary for timestamp column, since for increasing timestamp column,
|
||||
// the dictionary pages will be larger than data pages.
|
||||
for col in schema.column_schemas() {
|
||||
if col.data_type.is_timestamp() {
|
||||
let path = ColumnPath::new(vec![col.name.clone()]);
|
||||
props = props
|
||||
.set_column_dictionary_enabled(path.clone(), false)
|
||||
.set_column_encoding(path, Encoding::DELTA_BINARY_PACKED)
|
||||
}
|
||||
}
|
||||
props
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use common_test_util::find_workspace_path;
|
||||
|
||||
@@ -19,7 +19,7 @@ use std::{assert_matches, vec};
|
||||
use common_test_util::find_workspace_path;
|
||||
use datafusion::assert_batches_eq;
|
||||
use datafusion::datasource::physical_plan::{
|
||||
CsvSource, FileScanConfig, FileSource, FileStream, JsonSource, ParquetSource,
|
||||
CsvSource, FileScanConfig, FileSource, FileStreamBuilder, JsonSource, ParquetSource,
|
||||
};
|
||||
use datafusion::datasource::source::DataSourceExec;
|
||||
use datafusion::execution::context::TaskContext;
|
||||
@@ -50,16 +50,15 @@ impl Test<'_> {
|
||||
.create_file_opener(store, &self.config, 0)
|
||||
.unwrap();
|
||||
|
||||
let result = FileStream::new(
|
||||
&self.config,
|
||||
0,
|
||||
file_opener,
|
||||
&ExecutionPlanMetricsSet::new(),
|
||||
)
|
||||
.unwrap()
|
||||
.map(|b| b.unwrap())
|
||||
.collect::<Vec<_>>()
|
||||
.await;
|
||||
let result = FileStreamBuilder::new(&self.config)
|
||||
.with_partition(0)
|
||||
.with_file_opener(file_opener)
|
||||
.with_metrics(&ExecutionPlanMetricsSet::new())
|
||||
.build()
|
||||
.unwrap()
|
||||
.map(|b| b.unwrap())
|
||||
.collect::<Vec<_>>()
|
||||
.await;
|
||||
|
||||
assert_batches_eq!(self.expected, &result);
|
||||
}
|
||||
|
||||
@@ -50,6 +50,22 @@ pub const AZBLOB_SCHEMA: &str = "AZBLOB";
|
||||
pub struct BuiltBackend {
|
||||
pub object_store: ObjectStore,
|
||||
pub object_path: Option<String>,
|
||||
local_root: Option<SecureFsRoot>,
|
||||
}
|
||||
|
||||
impl BuiltBackend {
|
||||
/// Checks an explicit input using the same file-type filter as directory listing.
|
||||
pub async fn is_file(&self, path: &str) -> object_store::Result<bool> {
|
||||
// Stat first preserves errors for dangling or inaccessible symlink targets.
|
||||
if self.object_store.stat(path).await?.mode() != object_store::EntryMode::FILE {
|
||||
return Ok(false);
|
||||
}
|
||||
if let Some(root) = &self.local_root {
|
||||
root.is_file(path).await
|
||||
} else {
|
||||
Ok(true)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Controls whether SQL paths may access the local filesystem.
|
||||
@@ -379,6 +395,7 @@ async fn build_backend_inner(
|
||||
return Ok(BuiltBackend {
|
||||
object_store: build_fs_backend(&root)?,
|
||||
object_path,
|
||||
local_root: Some(root),
|
||||
});
|
||||
}
|
||||
|
||||
@@ -418,6 +435,7 @@ async fn build_backend_inner(
|
||||
Ok(BuiltBackend {
|
||||
object_store,
|
||||
object_path,
|
||||
local_root: None,
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -12,11 +12,215 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use arrow::datatypes::{DataType, SchemaRef};
|
||||
use arrow::record_batch::RecordBatch;
|
||||
use bytes::Bytes;
|
||||
use futures::future::BoxFuture;
|
||||
use object_store::Writer;
|
||||
use object_store::{ObjectStore, Writer};
|
||||
use parquet::arrow::ArrowWriter;
|
||||
use parquet::arrow::async_writer::AsyncFileWriter;
|
||||
use parquet::basic::{Compression, Encoding, ZstdLevel};
|
||||
use parquet::errors::ParquetError;
|
||||
use parquet::file::properties::WriterProperties;
|
||||
use parquet::schema::types::ColumnPath;
|
||||
use snafu::{IntoError, ResultExt, ensure};
|
||||
use tokio_util::sync::CancellationToken;
|
||||
|
||||
use crate::DEFAULT_WRITE_BUFFER_SIZE;
|
||||
use crate::error::{self, Result};
|
||||
|
||||
/// Limits for one Parquet file. Flush thresholds are not hard memory caps.
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
pub struct ParquetWriterLimits {
|
||||
/// Maximum rows in a row group.
|
||||
pub row_group_rows: usize,
|
||||
/// Flush when encoder memory or encoded size reaches this threshold.
|
||||
pub flush_threshold_bytes: usize,
|
||||
/// Maximum row groups retained in the file footer.
|
||||
pub max_row_groups: usize,
|
||||
}
|
||||
|
||||
/// Encodes batches on a blocking runtime and writes one object-store file.
|
||||
/// Callers must await each operation before aborting; dropping an in-flight
|
||||
/// operation can leave a filesystem worker running after cleanup.
|
||||
pub struct ParquetFileWriter {
|
||||
encoder: Option<ArrowWriter<Vec<u8>>>,
|
||||
sink: Writer,
|
||||
store: ObjectStore,
|
||||
path: String,
|
||||
limits: Option<ParquetWriterLimits>,
|
||||
}
|
||||
|
||||
impl ParquetFileWriter {
|
||||
/// Open a file using COPY's encoding settings. Destination ownership and
|
||||
/// overwrite policy belong to the caller. None preserves Parquet's defaults.
|
||||
pub async fn open(
|
||||
schema: SchemaRef,
|
||||
store: ObjectStore,
|
||||
path: &str,
|
||||
concurrency: usize,
|
||||
limits: Option<ParquetWriterLimits>,
|
||||
) -> Result<Self> {
|
||||
let mut props = WriterProperties::builder()
|
||||
.set_compression(Compression::ZSTD(ZstdLevel::default()))
|
||||
.set_statistics_truncate_length(None)
|
||||
.set_column_index_truncate_length(None);
|
||||
if let Some(limits) = limits {
|
||||
ensure!(
|
||||
limits.row_group_rows > 0
|
||||
&& limits.flush_threshold_bytes > 0
|
||||
&& limits.max_row_groups > 0,
|
||||
error::InvalidParquetWriterLimitsSnafu
|
||||
);
|
||||
props = props
|
||||
.set_max_row_group_row_count(Some(limits.row_group_rows))
|
||||
.set_max_row_group_bytes(None);
|
||||
}
|
||||
for field in schema.fields() {
|
||||
if matches!(field.data_type(), DataType::Timestamp(_, _)) {
|
||||
let column = ColumnPath::new(vec![field.name().clone()]);
|
||||
props = props
|
||||
.set_column_dictionary_enabled(column.clone(), false)
|
||||
.set_column_encoding(column, Encoding::DELTA_BINARY_PACKED);
|
||||
}
|
||||
}
|
||||
let encoder = ArrowWriter::try_new(Vec::new(), schema, Some(props.build()))
|
||||
.context(error::WriteParquetSnafu { path })?;
|
||||
let sink = store
|
||||
.writer_with(path)
|
||||
.concurrent(concurrency)
|
||||
.chunk(DEFAULT_WRITE_BUFFER_SIZE.as_bytes() as usize)
|
||||
.await
|
||||
.context(error::WriteObjectSnafu { path })?;
|
||||
Ok(Self {
|
||||
encoder: Some(encoder),
|
||||
sink,
|
||||
store,
|
||||
path: path.to_owned(),
|
||||
limits,
|
||||
})
|
||||
}
|
||||
|
||||
/// Write a batch, enforcing file limits across batch and row-group boundaries.
|
||||
pub async fn write(
|
||||
&mut self,
|
||||
batch: RecordBatch,
|
||||
cancellation: Option<&CancellationToken>,
|
||||
) -> Result<()> {
|
||||
let mut offset = 0;
|
||||
while offset < batch.num_rows() {
|
||||
check_cancelled(cancellation)?;
|
||||
let mut encoder = self.encoder.take().ok_or_else(|| {
|
||||
error::WriteParquetSnafu { path: &self.path }
|
||||
.into_error(ParquetError::General("Parquet writer is closed".into()))
|
||||
})?;
|
||||
let len = self.limits.map_or(batch.num_rows() - offset, |limits| {
|
||||
(limits.row_group_rows - encoder.in_progress_rows()).min(batch.num_rows() - offset)
|
||||
});
|
||||
let slice = batch.slice(offset, len);
|
||||
let limits = self.limits;
|
||||
let path = self.path.clone();
|
||||
let (encoder, bytes) = common_runtime::spawn_blocking_global(move || {
|
||||
if let Some(limits) = limits {
|
||||
ensure!(
|
||||
encoder.flushed_row_groups().len() < limits.max_row_groups,
|
||||
error::ParquetWriterResourceSnafu {
|
||||
reason: "Parquet row-group metadata budget exceeded"
|
||||
}
|
||||
);
|
||||
}
|
||||
encoder
|
||||
.write(&slice)
|
||||
.context(error::WriteParquetSnafu { path: &path })?;
|
||||
if limits.is_some_and(|limits| {
|
||||
encoder.memory_size() >= limits.flush_threshold_bytes
|
||||
|| encoder.in_progress_size() >= limits.flush_threshold_bytes
|
||||
}) {
|
||||
encoder
|
||||
.flush()
|
||||
.context(error::WriteParquetSnafu { path: &path })?;
|
||||
}
|
||||
// Draining preserves ArrowWriter's cumulative file offsets.
|
||||
let bytes = std::mem::take(encoder.inner_mut());
|
||||
Ok::<_, error::Error>((encoder, bytes))
|
||||
})
|
||||
.await
|
||||
.context(error::JoinHandleSnafu)??;
|
||||
self.encoder = Some(encoder);
|
||||
check_cancelled(cancellation)?;
|
||||
self.write_bytes(bytes).await?;
|
||||
check_cancelled(cancellation)?;
|
||||
offset += len;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn write_bytes(&mut self, bytes: Vec<u8>) -> Result<()> {
|
||||
if !bytes.is_empty() {
|
||||
self.sink
|
||||
.write(bytes)
|
||||
.await
|
||||
.context(error::WriteObjectSnafu { path: &self.path })?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Write the footer and close the file. Retains the handle for cleanup on error.
|
||||
pub async fn finish(&mut self, cancellation: Option<&CancellationToken>) -> Result<()> {
|
||||
check_cancelled(cancellation)?;
|
||||
let mut encoder = self.encoder.take().ok_or_else(|| {
|
||||
error::WriteParquetSnafu { path: &self.path }
|
||||
.into_error(ParquetError::General("Parquet writer is closed".into()))
|
||||
})?;
|
||||
let path = self.path.clone();
|
||||
let bytes = common_runtime::spawn_blocking_global(move || {
|
||||
encoder
|
||||
.finish()
|
||||
.context(error::WriteParquetSnafu { path: &path })?;
|
||||
Ok::<_, error::Error>(std::mem::take(encoder.inner_mut()))
|
||||
})
|
||||
.await
|
||||
.context(error::JoinHandleSnafu)??;
|
||||
self.write_bytes(bytes).await?;
|
||||
check_cancelled(cancellation)?;
|
||||
self.sink
|
||||
.close()
|
||||
.await
|
||||
.context(error::WriteObjectSnafu { path: &self.path })?;
|
||||
check_cancelled(cancellation)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Abort an exclusively owned file after all in-flight operations have completed.
|
||||
/// If the backend cannot abort, deletion assumes this attempt owns the path.
|
||||
pub async fn abort(mut self) -> Result<()> {
|
||||
let result = self.sink.abort().await;
|
||||
if result
|
||||
.as_ref()
|
||||
.is_err_and(|e| e.kind() == object_store::ErrorKind::Unsupported)
|
||||
{
|
||||
let store = self.store.clone();
|
||||
let path = self.path.clone();
|
||||
// Secure filesystem writers require dropping the handle before deletion.
|
||||
drop(self);
|
||||
store
|
||||
.delete(&path)
|
||||
.await
|
||||
.context(error::WriteObjectSnafu { path })?;
|
||||
} else {
|
||||
result.context(error::WriteObjectSnafu { path: &self.path })?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
fn check_cancelled(cancellation: Option<&CancellationToken>) -> Result<()> {
|
||||
ensure!(
|
||||
cancellation.is_none_or(|token| !token.is_cancelled()),
|
||||
error::ParquetWriteCancelledSnafu
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Bridges opendal [Writer] with parquet [AsyncFileWriter].
|
||||
pub struct AsyncWriter {
|
||||
@@ -50,3 +254,244 @@ impl AsyncFileWriter for AsyncWriter {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::sync::Arc;
|
||||
|
||||
use arrow::array::{Int64Array, TimestampMillisecondArray};
|
||||
use common_error::ext::ErrorExt;
|
||||
use common_error::status_code::StatusCode;
|
||||
use datafusion::physical_plan::stream::RecordBatchStreamAdapter;
|
||||
use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder;
|
||||
|
||||
use super::*;
|
||||
use crate::file_format::parquet::stream_to_parquet;
|
||||
|
||||
fn batch() -> RecordBatch {
|
||||
RecordBatch::try_from_iter([
|
||||
(
|
||||
"value",
|
||||
Arc::new(Int64Array::from(vec![Some(1), None, Some(3), Some(4)]))
|
||||
as arrow::array::ArrayRef,
|
||||
),
|
||||
(
|
||||
"ts",
|
||||
Arc::new(TimestampMillisecondArray::from(vec![1, 2, 3, 4])),
|
||||
),
|
||||
])
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
async fn read(store: &ObjectStore, path: &str) -> ParquetRecordBatchReaderBuilder<Bytes> {
|
||||
ParquetRecordBatchReaderBuilder::try_new(store.read(path).await.unwrap().to_bytes())
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn row_and_byte_limits_split_batches() {
|
||||
let store = ObjectStore::new(object_store::services::Memory::default()).unwrap();
|
||||
let batch = batch();
|
||||
for (row_group_rows, flush_threshold_bytes, split) in [(2, usize::MAX, 3), (100, 1, 2)] {
|
||||
let mut writer = ParquetFileWriter::open(
|
||||
batch.schema(),
|
||||
store.clone(),
|
||||
"groups.parquet",
|
||||
1,
|
||||
Some(ParquetWriterLimits {
|
||||
row_group_rows,
|
||||
flush_threshold_bytes,
|
||||
max_row_groups: 2,
|
||||
}),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
writer.write(batch.slice(0, split), None).await.unwrap();
|
||||
writer
|
||||
.write(batch.slice(split, batch.num_rows() - split), None)
|
||||
.await
|
||||
.unwrap();
|
||||
writer.finish(None).await.unwrap();
|
||||
let reader = read(&store, "groups.parquet").await;
|
||||
assert_eq!(reader.metadata().num_row_groups(), 2);
|
||||
for group in reader.metadata().row_groups() {
|
||||
assert_eq!(group.num_rows(), 2);
|
||||
let encodings = group.column(1).encodings().collect::<Vec<_>>();
|
||||
assert!(encodings.contains(&Encoding::DELTA_BINARY_PACKED));
|
||||
assert!(!encodings.contains(&Encoding::RLE_DICTIONARY));
|
||||
assert_eq!(
|
||||
group.column(0).compression(),
|
||||
Compression::ZSTD(ZstdLevel::default())
|
||||
);
|
||||
}
|
||||
let actual = reader
|
||||
.build()
|
||||
.unwrap()
|
||||
.collect::<std::result::Result<Vec<_>, _>>()
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
arrow::compute::concat_batches(&batch.schema(), &actual).unwrap(),
|
||||
batch
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn oversized_batch_stops_at_footer_limit_and_can_abort() {
|
||||
let store = ObjectStore::new(object_store::services::Memory::default()).unwrap();
|
||||
let batch = batch();
|
||||
let mut writer = ParquetFileWriter::open(
|
||||
batch.schema(),
|
||||
store.clone(),
|
||||
"limited.parquet",
|
||||
1,
|
||||
Some(ParquetWriterLimits {
|
||||
row_group_rows: 2,
|
||||
flush_threshold_bytes: usize::MAX,
|
||||
max_row_groups: 1,
|
||||
}),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let err = writer.write(batch, None).await.unwrap_err();
|
||||
assert!(matches!(err, error::Error::ParquetWriterResource { .. }));
|
||||
assert_eq!(err.status_code(), StatusCode::Suspended);
|
||||
writer.abort().await.unwrap();
|
||||
assert!(!store.exists("limited.parquet").await.unwrap());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn copy_stream_preserves_values_and_empty_schema() {
|
||||
let store = ObjectStore::new(object_store::services::Memory::default()).unwrap();
|
||||
let batch = batch();
|
||||
for (path, batches) in [
|
||||
("copy.parquet", vec![batch.clone()]),
|
||||
("empty.parquet", vec![]),
|
||||
] {
|
||||
let stream = RecordBatchStreamAdapter::new(
|
||||
batch.schema(),
|
||||
futures::stream::iter(batches.clone().into_iter().map(Ok)),
|
||||
);
|
||||
assert_eq!(
|
||||
stream_to_parquet(Box::pin(stream), store.clone(), path, 2)
|
||||
.await
|
||||
.unwrap(),
|
||||
batches.iter().map(RecordBatch::num_rows).sum::<usize>()
|
||||
);
|
||||
let reader = read(&store, path).await;
|
||||
assert_eq!(reader.schema().fields(), batch.schema().fields());
|
||||
let actual = reader
|
||||
.build()
|
||||
.unwrap()
|
||||
.collect::<std::result::Result<Vec<_>, _>>()
|
||||
.unwrap();
|
||||
assert_eq!(actual, batches);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn failed_copy_preserves_untouched_securefs_destination() {
|
||||
let directory = common_test_util::temp_dir::create_temp_dir("copy_existing");
|
||||
let path = directory.path().join("existing.parquet");
|
||||
std::fs::write(&path, b"original bytes").unwrap();
|
||||
let access = crate::object_store::LocalFileAccess::sandboxed(directory.path()).unwrap();
|
||||
let store = crate::object_store::build_backend_for_write(
|
||||
&format!("{}/", directory.path().display()),
|
||||
&Default::default(),
|
||||
&access,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let stream = RecordBatchStreamAdapter::new(
|
||||
batch().schema(),
|
||||
futures::stream::iter(vec![Err(datafusion::error::DataFusionError::Execution(
|
||||
"injected input failure".into(),
|
||||
))]),
|
||||
);
|
||||
let err = stream_to_parquet(Box::pin(stream), store, "existing.parquet", 1)
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert!(matches!(err, error::Error::ReadRecordBatch { .. }));
|
||||
assert_eq!(std::fs::read(path).unwrap(), b"original bytes");
|
||||
}
|
||||
|
||||
struct PausedFooterWriter {
|
||||
inner: object_store::layers::mock::oio::Writer,
|
||||
paused: bool,
|
||||
started: Arc<tokio::sync::Notify>,
|
||||
release: Arc<tokio::sync::Notify>,
|
||||
closed: Arc<std::sync::atomic::AtomicBool>,
|
||||
}
|
||||
|
||||
impl object_store::layers::mock::oio::Write for PausedFooterWriter {
|
||||
async fn write(&mut self, bytes: object_store::Buffer) -> object_store::Result<()> {
|
||||
if !self.paused {
|
||||
self.paused = true;
|
||||
self.started.notify_one();
|
||||
self.release.notified().await;
|
||||
}
|
||||
self.inner.write(bytes).await
|
||||
}
|
||||
async fn close(&mut self) -> object_store::Result<object_store::layers::mock::Metadata> {
|
||||
self.closed.store(true, std::sync::atomic::Ordering::SeqCst);
|
||||
self.inner.close().await
|
||||
}
|
||||
async fn abort(&mut self) -> object_store::Result<()> {
|
||||
self.inner.abort().await
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cancellation_during_footer_write_waits_then_aborts_before_close() {
|
||||
use object_store::layers::mock::{MockLayerBuilder, MockWriterFactory};
|
||||
let started = Arc::new(tokio::sync::Notify::new());
|
||||
let release = Arc::new(tokio::sync::Notify::new());
|
||||
let closed = Arc::new(std::sync::atomic::AtomicBool::new(false));
|
||||
let factory: MockWriterFactory = Arc::new({
|
||||
let (started, release, closed) = (started.clone(), release.clone(), closed.clone());
|
||||
move |_, _, inner| {
|
||||
Box::new(PausedFooterWriter {
|
||||
inner,
|
||||
paused: false,
|
||||
started: started.clone(),
|
||||
release: release.clone(),
|
||||
closed: closed.clone(),
|
||||
})
|
||||
}
|
||||
});
|
||||
let store = ObjectStore::new(object_store::services::Memory::default())
|
||||
.unwrap()
|
||||
.layer(
|
||||
MockLayerBuilder::default()
|
||||
.writer_factory(factory)
|
||||
.build()
|
||||
.unwrap(),
|
||||
);
|
||||
let mut writer =
|
||||
ParquetFileWriter::open(batch().schema(), store.clone(), "cancel.parquet", 1, None)
|
||||
.await
|
||||
.unwrap();
|
||||
// Force footer bytes through the sink before its close operation.
|
||||
writer.sink = store.writer_with("cancel.parquet").chunk(1).await.unwrap();
|
||||
let cancellation = CancellationToken::new();
|
||||
let result = {
|
||||
let finish = writer.finish(Some(&cancellation));
|
||||
tokio::pin!(finish);
|
||||
tokio::select! {
|
||||
result = &mut finish => panic!("finished before footer write: {result:?}"),
|
||||
_ = started.notified() => {},
|
||||
}
|
||||
cancellation.cancel();
|
||||
assert!(futures::poll!(&mut finish).is_pending());
|
||||
release.notify_one();
|
||||
finish.await
|
||||
};
|
||||
assert!(matches!(
|
||||
result,
|
||||
Err(error::Error::ParquetWriteCancelled {})
|
||||
));
|
||||
assert!(!closed.load(std::sync::atomic::Ordering::SeqCst));
|
||||
writer.abort().await.unwrap();
|
||||
assert!(!store.exists("cancel.parquet").await.unwrap());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -20,7 +20,7 @@ use datafusion::datasource::file_format::file_compression_type::FileCompressionT
|
||||
use datafusion::datasource::listing::PartitionedFile;
|
||||
use datafusion::datasource::object_store::ObjectStoreUrl;
|
||||
use datafusion::datasource::physical_plan::{
|
||||
CsvSource, FileGroup, FileScanConfig, FileScanConfigBuilder, FileSource, FileStream,
|
||||
CsvSource, FileGroup, FileScanConfig, FileScanConfigBuilder, FileSource, FileStreamBuilder,
|
||||
JsonOpener, JsonSource,
|
||||
};
|
||||
use datafusion::physical_plan::metrics::ExecutionPlanMetricsSet;
|
||||
@@ -110,13 +110,12 @@ pub async fn setup_stream_to_json_test(origin_path: &str, threshold: impl Fn(usi
|
||||
let size = store.read(origin_path).await.unwrap().len();
|
||||
|
||||
let config = scan_config(None, origin_path, Arc::new(JsonSource::new(schema)));
|
||||
let stream = FileStream::new(
|
||||
&config,
|
||||
0,
|
||||
Arc::new(json_opener),
|
||||
&ExecutionPlanMetricsSet::new(),
|
||||
)
|
||||
.unwrap();
|
||||
let stream = FileStreamBuilder::new(&config)
|
||||
.with_partition(0)
|
||||
.with_file_opener(Arc::new(json_opener))
|
||||
.with_metrics(&ExecutionPlanMetricsSet::new())
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
let (tmp_store, dir) = test_tmp_store("test_stream_to_json");
|
||||
|
||||
@@ -162,7 +161,12 @@ pub async fn setup_stream_to_csv_test(
|
||||
0,
|
||||
)
|
||||
.unwrap();
|
||||
let stream = FileStream::new(&config, 0, csv_opener, &ExecutionPlanMetricsSet::new()).unwrap();
|
||||
let stream = FileStreamBuilder::new(&config)
|
||||
.with_partition(0)
|
||||
.with_file_opener(csv_opener)
|
||||
.with_metrics(&ExecutionPlanMetricsSet::new())
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
let (tmp_store, dir) = test_tmp_store("test_stream_to_csv");
|
||||
|
||||
|
||||
@@ -33,15 +33,15 @@ use datafusion::functions_aggregate::count::Count;
|
||||
use datafusion::functions_aggregate::min_max::{Max, Min};
|
||||
use datafusion::optimizer::AnalyzerRule;
|
||||
use datafusion::optimizer::analyzer::type_coercion::TypeCoercion;
|
||||
use datafusion::physical_planner::create_aggregate_expr_and_maybe_filter;
|
||||
use datafusion_common::{Column, ScalarValue};
|
||||
use datafusion_expr::expr::{AggregateFunction, AggregateFunctionParams};
|
||||
use datafusion_expr::function::StateFieldsArgs;
|
||||
use datafusion_expr::physical_planning_context::PhysicalPlanningContext;
|
||||
use datafusion_expr::{
|
||||
Accumulator, Aggregate, AggregateUDF, AggregateUDFImpl, EmitTo, Expr, ExprSchemable,
|
||||
GroupsAccumulator, LogicalPlan, Signature,
|
||||
};
|
||||
use datafusion_physical_expr::aggregate::AggregateFunctionExpr;
|
||||
use datafusion_physical_expr::aggregate::{AggregateFunctionExpr, LoweredAggregateBuilder};
|
||||
use datatypes::arrow::datatypes::{DataType, Field};
|
||||
|
||||
use crate::aggrs::aggr_wrapper::fix_order::FixStateUdafOrderingAnalyzer;
|
||||
@@ -210,12 +210,25 @@ impl StateMergeHelper {
|
||||
lower_aggr_exprs.push(expr);
|
||||
|
||||
// then create the merge function using the physical expression of the original aggregate function
|
||||
let (original_phy_expr, _filter, _ordering) = create_aggregate_expr_and_maybe_filter(
|
||||
let (name, human_display) = match aggr_expr {
|
||||
Expr::Alias(alias) => (alias.name.clone(), aggr_expr.human_display().to_string()),
|
||||
Expr::AggregateFunction(_) => (
|
||||
aggr_expr.schema_name().to_string(),
|
||||
aggr_expr.human_display().to_string(),
|
||||
),
|
||||
_ => unreachable!("aggregate expression was validated above"),
|
||||
};
|
||||
let original_phy_expr = LoweredAggregateBuilder::new(
|
||||
aggr_expr,
|
||||
aggr.input.schema(),
|
||||
aggr.input.schema().as_arrow(),
|
||||
&Default::default(),
|
||||
)?;
|
||||
&PhysicalPlanningContext::default(),
|
||||
)
|
||||
.with_name(name)
|
||||
.with_human_display(human_display)
|
||||
.build()?
|
||||
.aggregate;
|
||||
|
||||
let merge_func = MergeWrapper::new(
|
||||
(*aggr_func.func).clone(),
|
||||
@@ -371,9 +384,6 @@ impl AggregateUDFImpl for StateWrapper {
|
||||
Ok(Box::new(StateGroupsAccum::new(inner, state_type)?))
|
||||
}
|
||||
|
||||
fn as_any(&self) -> &dyn std::any::Any {
|
||||
self
|
||||
}
|
||||
fn name(&self) -> &str {
|
||||
self.name.as_str()
|
||||
}
|
||||
@@ -443,7 +453,7 @@ impl AggregateUDFImpl for StateWrapper {
|
||||
&self,
|
||||
statistics_args: &datafusion_expr::StatisticsArgs,
|
||||
) -> Option<ScalarValue> {
|
||||
let inner = self.inner().inner().as_any();
|
||||
let inner = self.inner().inner();
|
||||
// only count/min/max need special handling here, for getting result from statistics
|
||||
// the result of count/min/max is also the result of count_state so can return directly
|
||||
let can_use_stat = inner.is::<Count>() || inner.is::<Max>() || inner.is::<Min>();
|
||||
@@ -572,11 +582,10 @@ impl GroupsAccumulator for StateGroupsAccum {
|
||||
&mut self,
|
||||
values: &[ArrayRef],
|
||||
group_indices: &[usize],
|
||||
opt_filter: Option<&BooleanArray>,
|
||||
total_num_groups: usize,
|
||||
) -> datafusion_common::Result<()> {
|
||||
self.inner
|
||||
.merge_batch(values, group_indices, opt_filter, total_num_groups)
|
||||
.merge_batch(values, group_indices, total_num_groups)
|
||||
}
|
||||
|
||||
fn evaluate(&mut self, emit_to: EmitTo) -> datafusion_common::Result<ArrayRef> {
|
||||
@@ -596,10 +605,6 @@ impl GroupsAccumulator for StateGroupsAccum {
|
||||
self.inner.convert_to_state(values, opt_filter)
|
||||
}
|
||||
|
||||
fn supports_convert_to_state(&self) -> bool {
|
||||
self.inner.supports_convert_to_state()
|
||||
}
|
||||
|
||||
fn size(&self) -> usize {
|
||||
self.inner.size()
|
||||
}
|
||||
@@ -800,10 +805,6 @@ impl AggregateUDFImpl for DeltaMergeWrapper {
|
||||
}))
|
||||
}
|
||||
|
||||
fn as_any(&self) -> &dyn std::any::Any {
|
||||
self
|
||||
}
|
||||
|
||||
fn name(&self) -> &str {
|
||||
&self.name
|
||||
}
|
||||
@@ -973,9 +974,6 @@ impl AggregateUDFImpl for MergeWrapper {
|
||||
Ok(Box::new(MergeAccum::new(inner_accum, &fields)))
|
||||
}
|
||||
|
||||
fn as_any(&self) -> &dyn std::any::Any {
|
||||
self
|
||||
}
|
||||
fn name(&self) -> &str {
|
||||
self.name.as_str()
|
||||
}
|
||||
|
||||
@@ -144,7 +144,6 @@ fn rewrite_expr(
|
||||
let Some(old_state_wrapper) = aggregate_function
|
||||
.func
|
||||
.inner()
|
||||
.as_any()
|
||||
.downcast_ref::<StateWrapper>()
|
||||
else {
|
||||
return Ok(Transformed::no(Expr::AggregateFunction(aggregate_function)));
|
||||
|
||||
@@ -12,7 +12,6 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use std::any::Any;
|
||||
use std::pin::Pin;
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::task::{Context, Poll};
|
||||
@@ -25,6 +24,7 @@ use arrow::record_batch::RecordBatch;
|
||||
use arrow_schema::SchemaRef;
|
||||
use common_telemetry::init_default_ut_logging;
|
||||
use datafusion::catalog::{Session, TableProvider};
|
||||
use datafusion::common::tree_node::TreeNodeRecursion;
|
||||
use datafusion::datasource::DefaultTableSource;
|
||||
use datafusion::execution::{RecordBatchStream, SendableRecordBatchStream, TaskContext};
|
||||
use datafusion::functions_aggregate::average::avg_udaf;
|
||||
@@ -39,16 +39,16 @@ use datafusion::physical_planner::{DefaultPhysicalPlanner, PhysicalPlanner};
|
||||
use datafusion::prelude::SessionContext;
|
||||
use datafusion_common::arrow::array::AsArray;
|
||||
use datafusion_common::arrow::datatypes::{Float64Type, UInt64Type};
|
||||
use datafusion_common::{Column, TableReference};
|
||||
use datafusion_common::{Column, Result, TableReference};
|
||||
use datafusion_expr::expr::{AggregateFunction, NullTreatment};
|
||||
use datafusion_expr::function::AccumulatorArgs;
|
||||
use datafusion_expr::{
|
||||
Aggregate, AggregateUDFImpl, ColumnarValue, Expr, LogicalPlan, ScalarFunctionArgs, SortExpr,
|
||||
TableScan, TypeSignature, lit,
|
||||
TableScanBuilder, TypeSignature, lit,
|
||||
};
|
||||
use datafusion_physical_expr::aggregate::AggregateExprBuilder;
|
||||
use datafusion_physical_expr::expressions::{Column as PhysicalColumn, col, lit as physical_lit};
|
||||
use datafusion_physical_expr::{EquivalenceProperties, Partitioning};
|
||||
use datafusion_physical_expr::{EquivalenceProperties, Partitioning, PhysicalExpr};
|
||||
use futures::{Stream, StreamExt as _};
|
||||
use hyperloglogplus::HyperLogLog;
|
||||
use pretty_assertions::assert_eq;
|
||||
@@ -97,10 +97,6 @@ impl ExecutionPlan for MockInputExec {
|
||||
"MockInputExec"
|
||||
}
|
||||
|
||||
fn as_any(&self) -> &dyn Any {
|
||||
self
|
||||
}
|
||||
|
||||
fn properties(&self) -> &Arc<PlanProperties> {
|
||||
&self.properties
|
||||
}
|
||||
@@ -109,6 +105,13 @@ impl ExecutionPlan for MockInputExec {
|
||||
vec![]
|
||||
}
|
||||
|
||||
fn apply_expressions(
|
||||
&self,
|
||||
_f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,
|
||||
) -> Result<TreeNodeRecursion> {
|
||||
Ok(TreeNodeRecursion::Continue)
|
||||
}
|
||||
|
||||
fn with_new_children(
|
||||
self: Arc<Self>,
|
||||
_children: Vec<Arc<dyn ExecutionPlan>>,
|
||||
@@ -203,10 +206,6 @@ impl Default for DummyTableProvider {
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl TableProvider for DummyTableProvider {
|
||||
fn as_any(&self) -> &dyn std::any::Any {
|
||||
self
|
||||
}
|
||||
|
||||
fn schema(&self) -> Arc<arrow_schema::Schema> {
|
||||
self.schema.clone()
|
||||
}
|
||||
@@ -237,14 +236,12 @@ fn dummy_table_scan() -> LogicalPlan {
|
||||
let table_provider = Arc::new(DummyTableProvider::default());
|
||||
let table_source = DefaultTableSource::new(table_provider);
|
||||
LogicalPlan::TableScan(
|
||||
TableScan::try_new(
|
||||
TableReference::bare("Number"),
|
||||
Arc::new(table_source),
|
||||
None,
|
||||
vec![],
|
||||
None,
|
||||
)
|
||||
.unwrap(),
|
||||
TableScanBuilder::new(TableReference::bare("Number"), Arc::new(table_source))
|
||||
.with_projection(None)
|
||||
.with_filters(vec![])
|
||||
.with_fetch(None)
|
||||
.build()
|
||||
.unwrap(),
|
||||
)
|
||||
}
|
||||
|
||||
@@ -252,14 +249,12 @@ fn dummy_table_scan_with_ts() -> LogicalPlan {
|
||||
let table_provider = Arc::new(DummyTableProvider::with_ts(None));
|
||||
let table_source = DefaultTableSource::new(table_provider);
|
||||
LogicalPlan::TableScan(
|
||||
TableScan::try_new(
|
||||
TableReference::bare("Number"),
|
||||
Arc::new(table_source),
|
||||
None,
|
||||
vec![],
|
||||
None,
|
||||
)
|
||||
.unwrap(),
|
||||
TableScanBuilder::new(TableReference::bare("Number"), Arc::new(table_source))
|
||||
.with_projection(None)
|
||||
.with_filters(vec![])
|
||||
.with_fetch(None)
|
||||
.build()
|
||||
.unwrap(),
|
||||
)
|
||||
}
|
||||
|
||||
@@ -381,10 +376,7 @@ async fn test_sum_udaf() {
|
||||
.create_physical_plan(&res.lower_state, &ctx.state())
|
||||
.await
|
||||
.unwrap();
|
||||
let aggr_exec = phy_aggr_state_plan
|
||||
.as_any()
|
||||
.downcast_ref::<AggregateExec>()
|
||||
.unwrap();
|
||||
let aggr_exec = phy_aggr_state_plan.downcast_ref::<AggregateExec>().unwrap();
|
||||
let aggr_func_expr = &aggr_exec.aggr_expr()[0];
|
||||
let mut state_accum = aggr_func_expr.create_accumulator().unwrap();
|
||||
|
||||
@@ -414,10 +406,7 @@ async fn test_sum_udaf() {
|
||||
.create_physical_plan(&res.upper_merge, &ctx.state())
|
||||
.await
|
||||
.unwrap();
|
||||
let aggr_exec = phy_aggr_merge_plan
|
||||
.as_any()
|
||||
.downcast_ref::<AggregateExec>()
|
||||
.unwrap();
|
||||
let aggr_exec = phy_aggr_merge_plan.downcast_ref::<AggregateExec>().unwrap();
|
||||
let aggr_func_expr = &aggr_exec.aggr_expr()[0];
|
||||
let mut merge_accum = aggr_func_expr.create_accumulator().unwrap();
|
||||
|
||||
@@ -543,10 +532,7 @@ async fn test_avg_udaf() {
|
||||
.create_physical_plan(&coerced_aggr_state_plan, &ctx.state())
|
||||
.await
|
||||
.unwrap();
|
||||
let aggr_exec = phy_aggr_state_plan
|
||||
.as_any()
|
||||
.downcast_ref::<AggregateExec>()
|
||||
.unwrap();
|
||||
let aggr_exec = phy_aggr_state_plan.downcast_ref::<AggregateExec>().unwrap();
|
||||
let aggr_func_expr = &aggr_exec.aggr_expr()[0];
|
||||
let mut state_accum = aggr_func_expr.create_accumulator().unwrap();
|
||||
|
||||
@@ -582,10 +568,7 @@ async fn test_avg_udaf() {
|
||||
.create_physical_plan(&res.upper_merge, &ctx.state())
|
||||
.await
|
||||
.unwrap();
|
||||
let aggr_exec = phy_aggr_merge_plan
|
||||
.as_any()
|
||||
.downcast_ref::<AggregateExec>()
|
||||
.unwrap();
|
||||
let aggr_exec = phy_aggr_merge_plan.downcast_ref::<AggregateExec>().unwrap();
|
||||
let aggr_func_expr = &aggr_exec.aggr_expr()[0];
|
||||
|
||||
let mut merge_accum = aggr_func_expr.create_accumulator().unwrap();
|
||||
@@ -703,10 +686,7 @@ async fn test_last_value_order_by_udaf() {
|
||||
.create_physical_plan(&fixed_aggr_state_plan, &ctx.state())
|
||||
.await
|
||||
.unwrap();
|
||||
let aggr_exec = phy_aggr_state_plan
|
||||
.as_any()
|
||||
.downcast_ref::<AggregateExec>()
|
||||
.unwrap();
|
||||
let aggr_exec = phy_aggr_state_plan.downcast_ref::<AggregateExec>().unwrap();
|
||||
let aggr_func_expr = &aggr_exec.aggr_expr()[0];
|
||||
|
||||
let merge_input_fields = vec![Arc::new(Field::new(
|
||||
@@ -796,10 +776,7 @@ async fn test_last_value_order_by_udaf() {
|
||||
.create_physical_plan(&res.upper_merge, &ctx.state())
|
||||
.await
|
||||
.unwrap();
|
||||
let aggr_exec = phy_aggr_merge_plan
|
||||
.as_any()
|
||||
.downcast_ref::<AggregateExec>()
|
||||
.unwrap();
|
||||
let aggr_exec = phy_aggr_merge_plan.downcast_ref::<AggregateExec>().unwrap();
|
||||
let aggr_func_expr = &aggr_exec.aggr_expr()[0];
|
||||
|
||||
let mut merge_accum = aggr_func_expr.create_accumulator().unwrap();
|
||||
@@ -900,7 +877,7 @@ fn test_avg_state_groups_accumulator_state_merge_evaluate() {
|
||||
.update_batch(&merged_values, &merged_group_indices, None, 3)
|
||||
.unwrap();
|
||||
merged_accum
|
||||
.merge_batch(&source_state, &[1, 2, 0], None, 3)
|
||||
.merge_batch(&source_state, &[1, 2, 0], 3)
|
||||
.unwrap();
|
||||
|
||||
let result = merged_accum.evaluate(EmitTo::All).unwrap();
|
||||
@@ -1324,14 +1301,12 @@ async fn test_udaf_correct_eval_result() {
|
||||
);
|
||||
let table_source = DefaultTableSource::new(Arc::new(table_provider));
|
||||
let logical_plan = LogicalPlan::TableScan(
|
||||
TableScan::try_new(
|
||||
test_table_ref.clone(),
|
||||
Arc::new(table_source),
|
||||
None,
|
||||
vec![],
|
||||
None,
|
||||
)
|
||||
.unwrap(),
|
||||
TableScanBuilder::new(test_table_ref.clone(), Arc::new(table_source))
|
||||
.with_projection(None)
|
||||
.with_filters(vec![])
|
||||
.with_fetch(None)
|
||||
.build()
|
||||
.unwrap(),
|
||||
);
|
||||
|
||||
let args = case.args;
|
||||
|
||||
@@ -123,7 +123,6 @@ impl UddSketchState {
|
||||
|
||||
fn downcast_accumulator_args(args: AccumulatorArgs) -> DfResult<(u32, f64)> {
|
||||
let bucket_size = match args.exprs[0]
|
||||
.as_any()
|
||||
.downcast_ref::<Literal>()
|
||||
.map(|lit| lit.value())
|
||||
{
|
||||
@@ -140,7 +139,6 @@ fn downcast_accumulator_args(args: AccumulatorArgs) -> DfResult<(u32, f64)> {
|
||||
};
|
||||
|
||||
let error_rate = match args.exprs[1]
|
||||
.as_any()
|
||||
.downcast_ref::<Literal>()
|
||||
.map(|lit| lit.value())
|
||||
{
|
||||
|
||||
@@ -26,7 +26,7 @@ use std::sync::Arc;
|
||||
use ahash::RandomState;
|
||||
use datafusion_common::cast::as_list_array;
|
||||
use datafusion_common::error::Result;
|
||||
use datafusion_common::hash_utils::create_hashes;
|
||||
use datafusion_common::hash_utils::create_hashes_with_hasher;
|
||||
use datafusion_common::utils::SingleRowListArrayBuilder;
|
||||
use datafusion_common::{ScalarValue, internal_err, not_impl_err};
|
||||
use datafusion_expr::function::{AccumulatorArgs, StateFieldsArgs};
|
||||
@@ -74,10 +74,6 @@ pub struct CountHash {
|
||||
}
|
||||
|
||||
impl AggregateUDFImpl for CountHash {
|
||||
fn as_any(&self) -> &dyn std::any::Any {
|
||||
self
|
||||
}
|
||||
|
||||
fn name(&self) -> &str {
|
||||
"count_hash"
|
||||
}
|
||||
@@ -208,7 +204,7 @@ impl GroupsAccumulator for CountHashGroupAccumulator {
|
||||
let array = &values[0];
|
||||
self.batch_hashes.clear();
|
||||
self.batch_hashes.resize(array.len(), 0);
|
||||
let hashes = create_hashes(
|
||||
let hashes = create_hashes_with_hasher(
|
||||
&[ArrayRef::clone(array)],
|
||||
&self.random_state,
|
||||
&mut self.batch_hashes,
|
||||
@@ -279,7 +275,6 @@ impl GroupsAccumulator for CountHashGroupAccumulator {
|
||||
&mut self,
|
||||
values: &[ArrayRef],
|
||||
group_indices: &[usize],
|
||||
_opt_filter: Option<&BooleanArray>,
|
||||
total_num_groups: usize,
|
||||
) -> Result<()> {
|
||||
assert_eq!(
|
||||
@@ -364,10 +359,6 @@ impl GroupsAccumulator for CountHashGroupAccumulator {
|
||||
Ok(vec![Arc::new(list_array)])
|
||||
}
|
||||
|
||||
fn supports_convert_to_state(&self) -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
fn size(&self) -> usize {
|
||||
// Base size of the struct
|
||||
let mut size = size_of::<Self>();
|
||||
@@ -423,7 +414,7 @@ impl Accumulator for CountHashAccumulator {
|
||||
|
||||
self.batch_hashes.clear();
|
||||
self.batch_hashes.resize(arr.len(), 0);
|
||||
let hashes = create_hashes(
|
||||
let hashes = create_hashes_with_hasher(
|
||||
&[ArrayRef::clone(arr)],
|
||||
&self.random_state,
|
||||
&mut self.batch_hashes,
|
||||
@@ -522,6 +513,28 @@ mod tests {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_count_hash_accumulator_typed_null_state_merge() -> Result<()> {
|
||||
let typed_nulls = Arc::new(Int32Array::from(vec![None, None])) as ArrayRef;
|
||||
|
||||
let mut fresh = create_test_accumulator();
|
||||
fresh.update_batch(&[typed_nulls])?;
|
||||
let fresh_state = fresh.state()?;
|
||||
assert_eq!(fresh.evaluate()?, ScalarValue::Int64(Some(1)));
|
||||
|
||||
let persisted_state = Arc::new(
|
||||
SingleRowListArrayBuilder::new(Arc::new(UInt64Array::from(vec![0])) as ArrayRef)
|
||||
.build_list_array(),
|
||||
) as ArrayRef;
|
||||
let mut restored = create_test_accumulator();
|
||||
restored.merge_batch(&[persisted_state])?;
|
||||
|
||||
assert_eq!(restored.evaluate()?, fresh.evaluate()?);
|
||||
assert_eq!(restored.state()?, fresh_state);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_count_hash_accumulator_merge() -> Result<()> {
|
||||
// Accumulator 1
|
||||
@@ -622,7 +635,7 @@ mod tests {
|
||||
// We will merge acc1's group 0 into acc2's group 0
|
||||
// and acc1's group 1 into acc2's group 2
|
||||
let merge_group_indices = vec![0, 2];
|
||||
acc2.merge_batch(&state1, &merge_group_indices, None, 3)?;
|
||||
acc2.merge_batch(&state1, &merge_group_indices, 3)?;
|
||||
|
||||
let result_array = acc2.evaluate(EmitTo::All)?;
|
||||
let result = result_array.as_any().downcast_ref::<Int64Array>().unwrap();
|
||||
|
||||
@@ -183,8 +183,6 @@ mod tests {
|
||||
use std::sync::Arc;
|
||||
|
||||
use arrow::array::StringArray;
|
||||
use datatypes::scalars::ScalarVector;
|
||||
use datatypes::vectors::{ConstantVector, StringVector, Vector};
|
||||
|
||||
use super::*;
|
||||
|
||||
@@ -252,14 +250,12 @@ mod tests {
|
||||
vec_avg.evaluate().unwrap()
|
||||
);
|
||||
|
||||
// test update with constant vector
|
||||
// test update with repeated values
|
||||
let mut vec_avg = VectorAvg::default();
|
||||
let v: Vec<ArrayRef> = vec![
|
||||
Arc::new(ConstantVector::new(
|
||||
Arc::new(StringVector::from_vec(vec!["[1.0,2.0,3.0]".to_string()])),
|
||||
4,
|
||||
))
|
||||
.to_arrow_array(),
|
||||
let v = vec![
|
||||
ScalarValue::Utf8(Some("[1.0,2.0,3.0]".to_string()))
|
||||
.to_array_of_size(4)
|
||||
.unwrap(),
|
||||
];
|
||||
vec_avg.update_batch(&v).unwrap();
|
||||
assert_eq!(
|
||||
|
||||
@@ -159,9 +159,6 @@ impl Accumulator for VectorProduct {
|
||||
mod tests {
|
||||
use std::sync::Arc;
|
||||
|
||||
use datatypes::scalars::ScalarVector;
|
||||
use datatypes::vectors::{ConstantVector, StringVector, Vector};
|
||||
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
@@ -213,14 +210,12 @@ mod tests {
|
||||
vec_product.update_batch(&v).unwrap();
|
||||
assert_eq!(ScalarValue::Binary(None), vec_product.evaluate().unwrap());
|
||||
|
||||
// test update with constant vector
|
||||
// test update with repeated values
|
||||
let mut vec_product = VectorProduct::default();
|
||||
let v: Vec<ArrayRef> = vec![
|
||||
Arc::new(ConstantVector::new(
|
||||
Arc::new(StringVector::from_vec(vec!["[1.0,2.0,3.0]".to_string()])),
|
||||
4,
|
||||
))
|
||||
.to_arrow_array(),
|
||||
let v = vec![
|
||||
ScalarValue::Utf8(Some("[1.0,2.0,3.0]".to_string()))
|
||||
.to_array_of_size(4)
|
||||
.unwrap(),
|
||||
];
|
||||
|
||||
vec_product.update_batch(&v).unwrap();
|
||||
|
||||
@@ -171,8 +171,6 @@ mod tests {
|
||||
use std::sync::Arc;
|
||||
|
||||
use arrow::array::StringArray;
|
||||
use datatypes::scalars::ScalarVector;
|
||||
use datatypes::vectors::{ConstantVector, StringVector, Vector};
|
||||
|
||||
use super::*;
|
||||
|
||||
@@ -225,14 +223,12 @@ mod tests {
|
||||
vec_sum.update_batch(&v).unwrap();
|
||||
assert_eq!(ScalarValue::Binary(None), vec_sum.evaluate().unwrap());
|
||||
|
||||
// test update with constant vector
|
||||
// test update with repeated values
|
||||
let mut vec_sum = VectorSum::default();
|
||||
let v: Vec<ArrayRef> = vec![
|
||||
Arc::new(ConstantVector::new(
|
||||
Arc::new(StringVector::from_vec(vec!["[1.0,2.0,3.0]".to_string()])),
|
||||
4,
|
||||
))
|
||||
.to_arrow_array(),
|
||||
let v = vec![
|
||||
ScalarValue::Utf8(Some("[1.0,2.0,3.0]".to_string()))
|
||||
.to_array_of_size(4)
|
||||
.unwrap(),
|
||||
];
|
||||
vec_sum.update_batch(&v).unwrap();
|
||||
assert_eq!(
|
||||
|
||||
@@ -23,6 +23,33 @@ use datatypes::types::cast::cast;
|
||||
use datatypes::value::ValueRef;
|
||||
use snafu::{OptionExt, ResultExt};
|
||||
|
||||
/// Integer types accepted by geospatial function signatures.
|
||||
pub(crate) const INTEGER_TYPES: &[DataType] = &[
|
||||
DataType::Int8,
|
||||
DataType::Int16,
|
||||
DataType::Int32,
|
||||
DataType::Int64,
|
||||
DataType::UInt8,
|
||||
DataType::UInt16,
|
||||
DataType::UInt32,
|
||||
DataType::UInt64,
|
||||
];
|
||||
|
||||
/// Legacy primitive numeric signature types; Decimal values are coerced to `Float64`.
|
||||
pub(crate) const NUMERICS: &[DataType] = &[
|
||||
DataType::Int8,
|
||||
DataType::Int16,
|
||||
DataType::Int32,
|
||||
DataType::Int64,
|
||||
DataType::UInt8,
|
||||
DataType::UInt16,
|
||||
DataType::UInt32,
|
||||
DataType::UInt64,
|
||||
DataType::Float16,
|
||||
DataType::Float32,
|
||||
DataType::Float64,
|
||||
];
|
||||
|
||||
/// Create a function signature with oneof signatures of interleaving two arguments.
|
||||
pub(crate) fn one_of_sigs2(args1: Vec<DataType>, args2: Vec<DataType>) -> Signature {
|
||||
let mut sigs = Vec::with_capacity(args1.len() * args2.len());
|
||||
|
||||
@@ -24,7 +24,6 @@
|
||||
//! When IQR = 0 (constant quartiles), returns 0.0 if value is on the fence,
|
||||
//! or +inf if value is outside.
|
||||
|
||||
use std::any::Any;
|
||||
use std::fmt::Debug;
|
||||
use std::ops::Range;
|
||||
use std::sync::Arc;
|
||||
@@ -32,11 +31,11 @@ use std::sync::Arc;
|
||||
use arrow::array::{Array, ArrayRef, Float64Array};
|
||||
use arrow::datatypes::{DataType, Field, FieldRef};
|
||||
use datafusion_common::{DataFusionError, Result, ScalarValue};
|
||||
use datafusion_expr::type_coercion::aggregates::NUMERICS;
|
||||
use datafusion_expr::{PartitionEvaluator, Signature, Volatility, WindowUDFImpl};
|
||||
use datafusion_functions_window_common::field::WindowUDFFieldArgs;
|
||||
use datafusion_functions_window_common::partition::PartitionEvaluatorArgs;
|
||||
|
||||
use crate::helper::NUMERICS;
|
||||
use crate::scalars::anomaly::utils::{cast_to_f64, collect_window_values, percentile_sorted};
|
||||
|
||||
/// Minimum valid samples for IQR (linear-interpolated Q1 != Q3 is possible at n >= 3).
|
||||
@@ -56,10 +55,6 @@ impl AnomalyScoreIqr {
|
||||
}
|
||||
|
||||
impl WindowUDFImpl for AnomalyScoreIqr {
|
||||
fn as_any(&self) -> &dyn Any {
|
||||
self
|
||||
}
|
||||
|
||||
fn name(&self) -> &str {
|
||||
"anomaly_score_iqr"
|
||||
}
|
||||
|
||||
@@ -20,7 +20,6 @@
|
||||
//! When MAD = 0 (majority-constant window), returns 0.0 if value equals
|
||||
//! median, or +inf otherwise.
|
||||
|
||||
use std::any::Any;
|
||||
use std::fmt::Debug;
|
||||
use std::ops::Range;
|
||||
use std::sync::Arc;
|
||||
@@ -28,11 +27,11 @@ use std::sync::Arc;
|
||||
use arrow::array::{Array, ArrayRef, Float64Array};
|
||||
use arrow::datatypes::{DataType, Field, FieldRef};
|
||||
use datafusion_common::{DataFusionError, Result, ScalarValue};
|
||||
use datafusion_expr::type_coercion::aggregates::NUMERICS;
|
||||
use datafusion_expr::{PartitionEvaluator, Signature, Volatility, WindowUDFImpl};
|
||||
use datafusion_functions_window_common::field::WindowUDFFieldArgs;
|
||||
use datafusion_functions_window_common::partition::PartitionEvaluatorArgs;
|
||||
|
||||
use crate::helper::NUMERICS;
|
||||
use crate::scalars::anomaly::utils::{
|
||||
anomaly_ratio, cast_to_f64, collect_window_values, median_f64,
|
||||
};
|
||||
@@ -57,10 +56,6 @@ impl AnomalyScoreMad {
|
||||
}
|
||||
|
||||
impl WindowUDFImpl for AnomalyScoreMad {
|
||||
fn as_any(&self) -> &dyn Any {
|
||||
self
|
||||
}
|
||||
|
||||
fn name(&self) -> &str {
|
||||
"anomaly_score_mad"
|
||||
}
|
||||
|
||||
@@ -19,7 +19,6 @@
|
||||
//! When stddev = 0 (constant window), returns 0.0 if value equals mean,
|
||||
//! or +inf otherwise.
|
||||
|
||||
use std::any::Any;
|
||||
use std::fmt::Debug;
|
||||
use std::ops::Range;
|
||||
use std::sync::Arc;
|
||||
@@ -27,11 +26,11 @@ use std::sync::Arc;
|
||||
use arrow::array::{Array, ArrayRef, Float64Array};
|
||||
use arrow::datatypes::{DataType, Field, FieldRef};
|
||||
use datafusion_common::{DataFusionError, Result, ScalarValue};
|
||||
use datafusion_expr::type_coercion::aggregates::NUMERICS;
|
||||
use datafusion_expr::{PartitionEvaluator, Signature, Volatility, WindowUDFImpl};
|
||||
use datafusion_functions_window_common::field::WindowUDFFieldArgs;
|
||||
use datafusion_functions_window_common::partition::PartitionEvaluatorArgs;
|
||||
|
||||
use crate::helper::NUMERICS;
|
||||
use crate::scalars::anomaly::utils::{anomaly_ratio, cast_to_f64, collect_window_values};
|
||||
|
||||
/// Minimum valid samples for zscore (stddev requires n >= 2).
|
||||
@@ -51,10 +50,6 @@ impl AnomalyScoreZscore {
|
||||
}
|
||||
|
||||
impl WindowUDFImpl for AnomalyScoreZscore {
|
||||
fn as_any(&self) -> &dyn Any {
|
||||
self
|
||||
}
|
||||
|
||||
fn name(&self) -> &str {
|
||||
"anomaly_score_zscore"
|
||||
}
|
||||
|
||||
@@ -12,13 +12,11 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
mod binary;
|
||||
mod ctx;
|
||||
mod if_func;
|
||||
mod is_null;
|
||||
mod unary;
|
||||
|
||||
pub use binary::scalar_binary_op;
|
||||
pub use ctx::EvalContext;
|
||||
pub use unary::scalar_unary_op;
|
||||
|
||||
|
||||
@@ -1,90 +0,0 @@
|
||||
// Copyright 2023 Greptime Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use std::iter;
|
||||
|
||||
use common_query::error::Result;
|
||||
use datatypes::prelude::*;
|
||||
use datatypes::vectors::{ConstantVector, Helper};
|
||||
|
||||
use crate::scalars::expression::ctx::EvalContext;
|
||||
|
||||
pub fn scalar_binary_op<L: Scalar, R: Scalar, O: Scalar, F>(
|
||||
l: &VectorRef,
|
||||
r: &VectorRef,
|
||||
f: F,
|
||||
ctx: &mut EvalContext,
|
||||
) -> Result<<O as Scalar>::VectorType>
|
||||
where
|
||||
F: Fn(Option<L::RefType<'_>>, Option<R::RefType<'_>>, &mut EvalContext) -> Option<O>,
|
||||
{
|
||||
debug_assert!(
|
||||
l.len() == r.len(),
|
||||
"Size of vectors must match to apply binary expression"
|
||||
);
|
||||
|
||||
let result = match (l.is_const(), r.is_const()) {
|
||||
(false, true) => {
|
||||
let left: &<L as Scalar>::VectorType = unsafe { Helper::static_cast(l) };
|
||||
let right: &ConstantVector = unsafe { Helper::static_cast(r) };
|
||||
let right: &<R as Scalar>::VectorType = unsafe { Helper::static_cast(right.inner()) };
|
||||
let b = right.get_data(0);
|
||||
|
||||
let it = left.iter_data().map(|a| f(a, b.clone(), ctx));
|
||||
<O as Scalar>::VectorType::from_owned_iterator(it)
|
||||
}
|
||||
|
||||
(false, false) => {
|
||||
let left: &<L as Scalar>::VectorType = unsafe { Helper::static_cast(l) };
|
||||
let right: &<R as Scalar>::VectorType = unsafe { Helper::static_cast(r) };
|
||||
|
||||
let it = left
|
||||
.iter_data()
|
||||
.zip(right.iter_data())
|
||||
.map(|(a, b)| f(a, b, ctx));
|
||||
<O as Scalar>::VectorType::from_owned_iterator(it)
|
||||
}
|
||||
|
||||
(true, false) => {
|
||||
let left: &ConstantVector = unsafe { Helper::static_cast(l) };
|
||||
let left: &<L as Scalar>::VectorType = unsafe { Helper::static_cast(left.inner()) };
|
||||
let a = left.get_data(0);
|
||||
|
||||
let right: &<R as Scalar>::VectorType = unsafe { Helper::static_cast(r) };
|
||||
let it = right.iter_data().map(|b| f(a.clone(), b, ctx));
|
||||
<O as Scalar>::VectorType::from_owned_iterator(it)
|
||||
}
|
||||
|
||||
(true, true) => {
|
||||
let left: &ConstantVector = unsafe { Helper::static_cast(l) };
|
||||
let left: &<L as Scalar>::VectorType = unsafe { Helper::static_cast(left.inner()) };
|
||||
let a = left.get_data(0);
|
||||
|
||||
let right: &ConstantVector = unsafe { Helper::static_cast(r) };
|
||||
let right: &<R as Scalar>::VectorType = unsafe { Helper::static_cast(right.inner()) };
|
||||
let b = right.get_data(0);
|
||||
|
||||
let it = iter::repeat(a)
|
||||
.zip(iter::repeat(b))
|
||||
.map(|(a, b)| f(a, b, ctx))
|
||||
.take(left.len());
|
||||
<O as Scalar>::VectorType::from_owned_iterator(it)
|
||||
}
|
||||
};
|
||||
|
||||
if let Some(error) = ctx.error.take() {
|
||||
return Err(error);
|
||||
}
|
||||
Ok(result)
|
||||
}
|
||||
@@ -22,12 +22,12 @@ use datafusion::arrow::array::{Array, AsArray, ListBuilder, StringViewBuilder};
|
||||
use datafusion::arrow::datatypes::{DataType, Field, Float64Type, UInt8Type};
|
||||
use datafusion::logical_expr::ColumnarValue;
|
||||
use datafusion_common::DataFusionError;
|
||||
use datafusion_expr::type_coercion::aggregates::INTEGERS;
|
||||
use datafusion_expr::{ScalarFunctionArgs, Signature, TypeSignature, Volatility};
|
||||
use geohash::Coord;
|
||||
use snafu::ResultExt;
|
||||
|
||||
use crate::function::{Function, extract_args};
|
||||
use crate::helper::INTEGER_TYPES;
|
||||
use crate::scalars::geo::helpers;
|
||||
|
||||
fn ensure_resolution_usize(v: u8) -> datafusion_common::Result<usize> {
|
||||
@@ -49,7 +49,7 @@ impl Default for GeohashFunction {
|
||||
fn default() -> Self {
|
||||
let mut signatures = Vec::new();
|
||||
for coord_type in &[DataType::Float32, DataType::Float64] {
|
||||
for resolution_type in INTEGERS {
|
||||
for resolution_type in INTEGER_TYPES {
|
||||
signatures.push(TypeSignature::Exact(vec![
|
||||
// latitude
|
||||
coord_type.clone(),
|
||||
@@ -146,7 +146,7 @@ impl Default for GeohashNeighboursFunction {
|
||||
fn default() -> Self {
|
||||
let mut signatures = Vec::new();
|
||||
for coord_type in &[DataType::Float32, DataType::Float64] {
|
||||
for resolution_type in INTEGERS {
|
||||
for resolution_type in INTEGER_TYPES {
|
||||
signatures.push(TypeSignature::Exact(vec![
|
||||
// latitude
|
||||
coord_type.clone(),
|
||||
|
||||
@@ -26,7 +26,6 @@ use datafusion::arrow::compute;
|
||||
use datafusion::arrow::datatypes::{Float64Type, Int64Type, UInt8Type, UInt64Type};
|
||||
use datafusion::logical_expr::ColumnarValue;
|
||||
use datafusion_common::{DataFusionError, ScalarValue};
|
||||
use datafusion_expr::type_coercion::aggregates::INTEGERS;
|
||||
use datafusion_expr::{ScalarFunctionArgs, Signature, TypeSignature, Volatility};
|
||||
use datatypes::arrow::datatypes::{DataType, Field};
|
||||
use derive_more::Display;
|
||||
@@ -34,6 +33,7 @@ use h3o::{CellIndex, LatLng, Resolution};
|
||||
use snafu::prelude::*;
|
||||
|
||||
use crate::function::{Function, extract_args};
|
||||
use crate::helper::INTEGER_TYPES;
|
||||
use crate::scalars::geo::helpers;
|
||||
|
||||
static CELL_TYPES: LazyLock<Vec<DataType>> =
|
||||
@@ -42,11 +42,11 @@ static CELL_TYPES: LazyLock<Vec<DataType>> =
|
||||
static COORDINATE_TYPES: LazyLock<Vec<DataType>> =
|
||||
LazyLock::new(|| vec![DataType::Float32, DataType::Float64]);
|
||||
|
||||
static RESOLUTION_TYPES: &[DataType] = INTEGERS;
|
||||
static RESOLUTION_TYPES: &[DataType] = INTEGER_TYPES;
|
||||
|
||||
static DISTANCE_TYPES: &[DataType] = INTEGERS;
|
||||
static DISTANCE_TYPES: &[DataType] = INTEGER_TYPES;
|
||||
|
||||
static POSITION_TYPES: &[DataType] = INTEGERS;
|
||||
static POSITION_TYPES: &[DataType] = INTEGER_TYPES;
|
||||
|
||||
/// Function that returns [h3] encoding cellid for a given geospatial coordinate.
|
||||
///
|
||||
|
||||
@@ -25,6 +25,7 @@ use s2::latlng::LatLng;
|
||||
use snafu::ensure;
|
||||
|
||||
use crate::function::{Function, extract_args};
|
||||
use crate::helper::INTEGER_TYPES;
|
||||
use crate::scalars::geo::helpers;
|
||||
use crate::scalars::geo::helpers::ensure_and_coerce;
|
||||
|
||||
@@ -34,7 +35,7 @@ static CELL_TYPES: LazyLock<Vec<DataType>> =
|
||||
static COORDINATE_TYPES: LazyLock<Vec<DataType>> =
|
||||
LazyLock::new(|| vec![DataType::Float32, DataType::Float64]);
|
||||
|
||||
static LEVEL_TYPES: &[DataType] = datafusion_expr::type_coercion::aggregates::INTEGERS;
|
||||
static LEVEL_TYPES: &[DataType] = INTEGER_TYPES;
|
||||
|
||||
/// Function that returns [s2] encoding cellid for a given geospatial coordinate.
|
||||
///
|
||||
|
||||
@@ -532,7 +532,7 @@ mod tests {
|
||||
use datafusion_common::arrow::datatypes::{Float64Type, Int64Type};
|
||||
use datatypes::extension::json::Json2ExtensionType;
|
||||
use datatypes::types::parse_string_to_jsonb;
|
||||
use serde_json::json;
|
||||
use serde_json::{Value, json};
|
||||
|
||||
use super::*;
|
||||
|
||||
@@ -595,6 +595,34 @@ mod tests {
|
||||
})
|
||||
}
|
||||
|
||||
fn assert_json_or_string_eq(actual: Option<&str>, expected: Option<&str>) {
|
||||
let is_json_container = |value: &str| {
|
||||
matches!(
|
||||
serde_json::from_str::<Value>(value),
|
||||
Ok(Value::Object(_) | Value::Array(_))
|
||||
)
|
||||
};
|
||||
|
||||
match (actual, expected) {
|
||||
(Some(actual), Some(expected))
|
||||
if is_json_container(actual) || is_json_container(expected) =>
|
||||
{
|
||||
let actual_value = serde_json::from_str::<Value>(actual).unwrap_or_else(|error| {
|
||||
panic!("failed to parse actual JSON result {actual:?}: {error}")
|
||||
});
|
||||
let expected_value =
|
||||
serde_json::from_str::<Value>(expected).unwrap_or_else(|error| {
|
||||
panic!("failed to parse expected JSON result {expected:?}: {error}")
|
||||
});
|
||||
assert_eq!(
|
||||
actual_value, expected_value,
|
||||
"JSON result mismatch: actual {actual:?}, expected {expected:?}"
|
||||
);
|
||||
}
|
||||
_ => assert_eq!(actual, expected),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_json_get_int() {
|
||||
let json_get_int = JsonGetInt::default();
|
||||
@@ -895,7 +923,7 @@ mod tests {
|
||||
let result = result.as_string_view();
|
||||
assert_eq!(1, result.len());
|
||||
let actual = result.is_valid(0).then(|| result.value(0));
|
||||
assert_eq!(actual, expect);
|
||||
assert_json_or_string_eq(actual, expect);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1033,7 +1061,7 @@ mod tests {
|
||||
let result = result.as_string_view();
|
||||
assert_eq!(1, result.len());
|
||||
let actual = result.is_valid(0).then(|| result.value(0));
|
||||
assert_eq!(actual, expect);
|
||||
assert_json_or_string_eq(actual, expect);
|
||||
}
|
||||
|
||||
let json_strings = [
|
||||
|
||||
@@ -59,10 +59,8 @@ impl FunctionRewrite for JsonGetRewriter {
|
||||
// json_get(column, path, <data_type>)
|
||||
// )
|
||||
fn inject_type_from_cast_expr(cast: Cast) -> Result<Transformed<Expr>> {
|
||||
let Cast {
|
||||
expr,
|
||||
mut data_type,
|
||||
} = cast;
|
||||
let Cast { expr, field } = cast;
|
||||
let mut data_type = field.data_type().clone();
|
||||
|
||||
let mut json_get = match *expr {
|
||||
Expr::ScalarFunction(f)
|
||||
@@ -73,7 +71,7 @@ fn inject_type_from_cast_expr(cast: Cast) -> Result<Transformed<Expr>> {
|
||||
expr => {
|
||||
return Ok(Transformed::no(Expr::Cast(Cast {
|
||||
expr: Box::new(expr),
|
||||
data_type,
|
||||
field,
|
||||
})));
|
||||
}
|
||||
};
|
||||
@@ -204,10 +202,7 @@ mod tests {
|
||||
});
|
||||
|
||||
// Create a cast expression: json_get(...)::int8
|
||||
let cast_expr = Expr::Cast(Cast {
|
||||
expr: Box::new(json_expr),
|
||||
data_type: DataType::Int8,
|
||||
});
|
||||
let cast_expr = Expr::Cast(Cast::new(Box::new(json_expr), DataType::Int8));
|
||||
|
||||
// Apply the rewriter
|
||||
let result = rewriter.rewrite(cast_expr, &schema, &config).unwrap();
|
||||
@@ -279,10 +274,7 @@ mod tests {
|
||||
|
||||
// Create an arrow cast function: cast(json_get(...), 'Int64')
|
||||
// Note: ArrowCastFunc doesn't exist in this codebase, so this test uses a simple cast instead
|
||||
let arrow_cast_expr = Expr::Cast(Cast {
|
||||
expr: Box::new(json_get_expr),
|
||||
data_type: DataType::Int64,
|
||||
});
|
||||
let arrow_cast_expr = Expr::Cast(Cast::new(Box::new(json_get_expr), DataType::Int64));
|
||||
|
||||
// Apply the rewriter
|
||||
let result = rewriter.rewrite(arrow_cast_expr, &schema, &config).unwrap();
|
||||
|
||||
@@ -24,6 +24,7 @@ use datafusion::execution::SessionStateBuilder;
|
||||
use datafusion::logical_expr::{self, ColumnarValue, Expr, Volatility};
|
||||
use datafusion::physical_planner::{DefaultPhysicalPlanner, PhysicalPlanner};
|
||||
use datafusion_common::DataFusionError;
|
||||
use datafusion_expr::physical_planning_context::PhysicalPlanningContext;
|
||||
use datafusion_expr::{ScalarFunctionArgs, Signature};
|
||||
use datatypes::arrow::array::RecordBatch;
|
||||
use datatypes::arrow::datatypes::{DataType, Field};
|
||||
@@ -110,8 +111,12 @@ impl MatchesFunction {
|
||||
let input_schema = Self::input_schema();
|
||||
let session_state = SessionStateBuilder::new().with_default_features().build();
|
||||
let planner = DefaultPhysicalPlanner::default();
|
||||
let physical_expr =
|
||||
planner.create_physical_expr(&like_expr, &input_schema, &session_state)?;
|
||||
let physical_expr = planner.create_physical_expr(
|
||||
&like_expr,
|
||||
&input_schema,
|
||||
&session_state,
|
||||
&PhysicalPlanningContext::default(),
|
||||
)?;
|
||||
|
||||
let arrow_schema = Arc::new(input_schema.as_arrow().clone());
|
||||
let input_record_batch = RecordBatch::try_new(arrow_schema, vec![data_array]).unwrap();
|
||||
|
||||
@@ -19,10 +19,10 @@ use datafusion::arrow::array::{Array, ArrayRef, AsArray, PrimitiveArray};
|
||||
use datafusion::arrow::datatypes::DataType as ArrowDataType;
|
||||
use datafusion::logical_expr::{ColumnarValue, Volatility};
|
||||
use datafusion_common::{DataFusionError, ScalarValue, utils};
|
||||
use datafusion_expr::type_coercion::aggregates::NUMERICS;
|
||||
use datafusion_expr::{ScalarFunctionArgs, Signature};
|
||||
|
||||
use crate::function::Function;
|
||||
use crate::helper::NUMERICS;
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct ClampFunction {
|
||||
@@ -338,7 +338,9 @@ mod test {
|
||||
|
||||
use arrow_schema::Field;
|
||||
use datafusion_common::config::ConfigOptions;
|
||||
use datatypes::arrow::array::{ArrayRef, Float64Array, Int64Array, UInt64Array};
|
||||
use datatypes::arrow::array::{
|
||||
ArrayRef, Decimal128Array, Float64Array, Int64Array, UInt64Array,
|
||||
};
|
||||
use datatypes::arrow_array::StringArray;
|
||||
|
||||
use super::*;
|
||||
@@ -370,6 +372,79 @@ mod test {
|
||||
impl_test_eval!(ClampMinFunction);
|
||||
impl_test_eval!(ClampMaxFunction);
|
||||
|
||||
fn decimal_array(values: Vec<i128>) -> ColumnarValue {
|
||||
ColumnarValue::Array(Arc::new(
|
||||
Decimal128Array::from(values)
|
||||
.with_precision_and_scale(10, 2)
|
||||
.unwrap(),
|
||||
))
|
||||
}
|
||||
|
||||
fn decimal_scalar(value: i128) -> ColumnarValue {
|
||||
ColumnarValue::Scalar(ScalarValue::Decimal128(Some(value), 10, 2))
|
||||
}
|
||||
|
||||
#[allow(deprecated)]
|
||||
fn evaluate_decimal(
|
||||
function: &dyn Function,
|
||||
args: Vec<ColumnarValue>,
|
||||
) -> datafusion_common::Result<ArrayRef> {
|
||||
let input_types = args
|
||||
.iter()
|
||||
.map(ColumnarValue::data_type)
|
||||
.collect::<Vec<_>>();
|
||||
let planned_types = datafusion_expr::type_coercion::functions::data_types(
|
||||
function.name(),
|
||||
&input_types,
|
||||
function.signature(),
|
||||
)?;
|
||||
let args = args
|
||||
.into_iter()
|
||||
.zip(planned_types)
|
||||
.map(|(arg, planned_type)| arg.cast_to(&planned_type, None))
|
||||
.collect::<datafusion_common::Result<Vec<_>>>()?;
|
||||
function
|
||||
.invoke_with_args(ScalarFunctionArgs {
|
||||
args,
|
||||
arg_fields: vec![],
|
||||
number_rows: 3,
|
||||
return_field: Arc::new(Field::new("x", ArrowDataType::Float64, false)),
|
||||
config_options: Arc::new(ConfigOptions::new()),
|
||||
})
|
||||
.and_then(|value| value.to_array(3))
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn clamp_decimal_coercion_executes_as_float64() {
|
||||
let test_cases = [
|
||||
(
|
||||
Box::new(ClampFunction::default()) as Box<dyn Function>,
|
||||
vec![
|
||||
decimal_array(vec![100, 300, 500]),
|
||||
decimal_scalar(200),
|
||||
decimal_scalar(400),
|
||||
],
|
||||
vec![2.0, 3.0, 4.0],
|
||||
),
|
||||
(
|
||||
Box::new(ClampMinFunction::default()),
|
||||
vec![decimal_array(vec![100, 300, 500]), decimal_scalar(200)],
|
||||
vec![2.0, 3.0, 5.0],
|
||||
),
|
||||
(
|
||||
Box::new(ClampMaxFunction::default()),
|
||||
vec![decimal_array(vec![100, 300, 500]), decimal_scalar(200)],
|
||||
vec![1.0, 2.0, 2.0],
|
||||
),
|
||||
];
|
||||
|
||||
for (function, args, expected) in test_cases {
|
||||
let result = evaluate_decimal(function.as_ref(), args).unwrap();
|
||||
let expected: ArrayRef = Arc::new(Float64Array::from(expected));
|
||||
assert_eq!(expected.as_ref(), result.as_ref());
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn clamp_i64() {
|
||||
let inputs = [
|
||||
|
||||
@@ -18,10 +18,10 @@ use std::fmt::Display;
|
||||
use datafusion_common::arrow::compute;
|
||||
use datafusion_common::arrow::compute::kernels::numeric;
|
||||
use datafusion_common::arrow::datatypes::DataType;
|
||||
use datafusion_expr::type_coercion::aggregates::NUMERICS;
|
||||
use datafusion_expr::{ColumnarValue, ScalarFunctionArgs, Signature, Volatility};
|
||||
|
||||
use crate::function::{Function, extract_args};
|
||||
use crate::helper::NUMERICS;
|
||||
|
||||
const NAME: &str = "mod";
|
||||
|
||||
@@ -90,12 +90,74 @@ mod tests {
|
||||
use std::sync::Arc;
|
||||
|
||||
use arrow_schema::Field;
|
||||
use datafusion_common::ScalarValue;
|
||||
use datafusion_common::arrow::array::{
|
||||
AsArray, Float64Array, Int32Array, StringViewArray, UInt32Array,
|
||||
AsArray, Decimal128Array, Float64Array, Int32Array, StringViewArray, UInt32Array,
|
||||
};
|
||||
use datafusion_common::arrow::datatypes::{Float64Type, Int64Type, UInt64Type};
|
||||
|
||||
use super::*;
|
||||
fn decimal_array(values: Vec<i128>) -> ColumnarValue {
|
||||
ColumnarValue::Array(Arc::new(
|
||||
Decimal128Array::from(values)
|
||||
.with_precision_and_scale(10, 2)
|
||||
.unwrap(),
|
||||
))
|
||||
}
|
||||
|
||||
fn decimal_scalar(value: i128) -> ColumnarValue {
|
||||
ColumnarValue::Scalar(ScalarValue::Decimal128(Some(value), 10, 2))
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[allow(deprecated)]
|
||||
fn modulo_decimal_coercion_executes_as_float64() {
|
||||
let function = ModuloFunction::default();
|
||||
let test_cases = [
|
||||
(
|
||||
vec![decimal_array(vec![500, 600]), decimal_scalar(200)],
|
||||
vec![1.0, 0.0],
|
||||
),
|
||||
(
|
||||
vec![decimal_scalar(500), decimal_array(vec![200, 300])],
|
||||
vec![1.0, 2.0],
|
||||
),
|
||||
];
|
||||
|
||||
for (args, expected) in test_cases {
|
||||
let input_types = args
|
||||
.iter()
|
||||
.map(ColumnarValue::data_type)
|
||||
.collect::<Vec<_>>();
|
||||
let planned_types = datafusion_expr::type_coercion::functions::data_types(
|
||||
function.name(),
|
||||
&input_types,
|
||||
function.signature(),
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(vec![DataType::Float64; 2], planned_types);
|
||||
let args = args
|
||||
.into_iter()
|
||||
.zip(planned_types)
|
||||
.map(|(arg, planned_type)| arg.cast_to(&planned_type, None))
|
||||
.collect::<datafusion_common::Result<Vec<_>>>()
|
||||
.unwrap();
|
||||
let result = function
|
||||
.invoke_with_args(ScalarFunctionArgs {
|
||||
args,
|
||||
arg_fields: vec![],
|
||||
number_rows: 2,
|
||||
return_field: Arc::new(Field::new("x", DataType::Float64, false)),
|
||||
config_options: Arc::new(Default::default()),
|
||||
})
|
||||
.unwrap()
|
||||
.to_array(2)
|
||||
.unwrap();
|
||||
let result = result.as_primitive::<Float64Type>();
|
||||
assert_eq!(&Float64Array::from(expected), result);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_mod_function_signed() {
|
||||
let function = ModuloFunction::default();
|
||||
|
||||
@@ -18,11 +18,11 @@ use common_query::error;
|
||||
use datafusion::arrow::compute::kernels::numeric;
|
||||
use datafusion_common::arrow::compute::kernels::cast;
|
||||
use datafusion_common::arrow::datatypes::DataType;
|
||||
use datafusion_expr::type_coercion::aggregates::NUMERICS;
|
||||
use datafusion_expr::{ColumnarValue, ScalarFunctionArgs, Signature, Volatility};
|
||||
use snafu::ResultExt;
|
||||
|
||||
use crate::function::{Function, extract_args};
|
||||
use crate::helper::NUMERICS;
|
||||
|
||||
/// generates rates from a sequence of adjacent data points.
|
||||
#[derive(Clone, Debug)]
|
||||
@@ -96,12 +96,13 @@ mod tests {
|
||||
let rate = RateFunction::default();
|
||||
assert_eq!("rate", rate.name());
|
||||
assert_eq!(DataType::Float64, rate.return_type(&[]).unwrap());
|
||||
assert!(matches!(rate.signature(),
|
||||
Signature {
|
||||
type_signature: TypeSignature::Uniform(2, valid_types),
|
||||
volatility: Volatility::Immutable,
|
||||
..
|
||||
} if valid_types == NUMERICS
|
||||
assert!(matches!(
|
||||
rate.signature(),
|
||||
Signature {
|
||||
type_signature: TypeSignature::Uniform(2, valid_types),
|
||||
volatility: Volatility::Immutable,
|
||||
..
|
||||
} if valid_types == NUMERICS
|
||||
));
|
||||
let values = vec![1.0, 3.0, 6.0];
|
||||
let ts = vec![0, 1, 2];
|
||||
|
||||
@@ -14,13 +14,43 @@
|
||||
|
||||
use std::fmt;
|
||||
|
||||
use common_query::error::Result;
|
||||
use datafusion::logical_expr::ColumnarValue;
|
||||
use datafusion_expr::{ScalarFunctionArgs, Signature, Volatility};
|
||||
use datatypes::arrow::datatypes::DataType;
|
||||
use datatypes::prelude::{Scalar, ScalarVector, VectorRef};
|
||||
use datatypes::vectors::{Helper, Vector};
|
||||
|
||||
use crate::function::{Function, extract_args};
|
||||
use crate::scalars::expression::{EvalContext, scalar_binary_op};
|
||||
use crate::scalars::expression::EvalContext;
|
||||
|
||||
fn scalar_binary_op<L: Scalar, R: Scalar, O: Scalar, F>(
|
||||
l: &VectorRef,
|
||||
r: &VectorRef,
|
||||
f: F,
|
||||
ctx: &mut EvalContext,
|
||||
) -> Result<<O as Scalar>::VectorType>
|
||||
where
|
||||
F: Fn(Option<L::RefType<'_>>, Option<R::RefType<'_>>, &mut EvalContext) -> Option<O>,
|
||||
{
|
||||
debug_assert!(
|
||||
l.len() == r.len(),
|
||||
"Size of vectors must match to apply binary expression"
|
||||
);
|
||||
|
||||
let left: &<L as Scalar>::VectorType = unsafe { Helper::static_cast(l) };
|
||||
let right: &<R as Scalar>::VectorType = unsafe { Helper::static_cast(r) };
|
||||
let result = <O as Scalar>::VectorType::from_owned_iterator(
|
||||
left.iter_data()
|
||||
.zip(right.iter_data())
|
||||
.map(|(a, b)| f(a, b, ctx)),
|
||||
);
|
||||
|
||||
if let Some(error) = ctx.error.take() {
|
||||
return Err(error);
|
||||
}
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub(crate) struct TestAndFunction {
|
||||
|
||||
@@ -12,7 +12,6 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use std::any::Any;
|
||||
use std::fmt::{Debug, Formatter};
|
||||
use std::hash::{Hash, Hasher};
|
||||
|
||||
@@ -49,10 +48,6 @@ impl Hash for ScalarUdf {
|
||||
}
|
||||
|
||||
impl ScalarUDFImpl for ScalarUdf {
|
||||
fn as_any(&self) -> &dyn Any {
|
||||
self
|
||||
}
|
||||
|
||||
fn name(&self) -> &str {
|
||||
self.function.name()
|
||||
}
|
||||
|
||||
@@ -21,7 +21,8 @@ mod version;
|
||||
|
||||
use build::BuildFunction;
|
||||
use database::{
|
||||
ConnectionIdFunction, DatabaseFunction, PgBackendPidFunction, ReadPreferenceFunction,
|
||||
ConnectionIdFunction, CurrentUserFunction, DatabaseFunction, PgBackendPidFunction,
|
||||
ReadPreferenceFunction, SchemaFunction, SystemUserFunction, UserFunction,
|
||||
};
|
||||
use pg_catalog::PGCatalogFunction;
|
||||
use procedure_state::ProcedureStateFunction;
|
||||
@@ -37,6 +38,10 @@ impl SystemFunction {
|
||||
registry.register_scalar(BuildFunction::default());
|
||||
registry.register_scalar(VersionFunction::default());
|
||||
registry.register_scalar(DatabaseFunction::default());
|
||||
registry.register_scalar(SchemaFunction::default());
|
||||
registry.register_scalar(UserFunction::default());
|
||||
registry.register_scalar(CurrentUserFunction::default());
|
||||
registry.register_scalar(SystemUserFunction::default());
|
||||
registry.register_scalar(ReadPreferenceFunction::default());
|
||||
registry.register_scalar(PgBackendPidFunction::default());
|
||||
registry.register_scalar(ConnectionIdFunction::default());
|
||||
|
||||
@@ -20,39 +20,91 @@ use crate::function::{Function, find_function_context};
|
||||
use crate::system::define_nullary_udf;
|
||||
|
||||
define_nullary_udf!(DatabaseFunction);
|
||||
define_nullary_udf!(SchemaFunction);
|
||||
define_nullary_udf!(UserFunction);
|
||||
define_nullary_udf!(CurrentUserFunction);
|
||||
define_nullary_udf!(SystemUserFunction);
|
||||
define_nullary_udf!(ReadPreferenceFunction);
|
||||
define_nullary_udf!(PgBackendPidFunction);
|
||||
define_nullary_udf!(ConnectionIdFunction);
|
||||
|
||||
const DATABASE_FUNCTION_NAME: &str = "database";
|
||||
const SCHEMA_FUNCTION_NAME: &str = "schema";
|
||||
const USER_FUNCTION_NAME: &str = "user";
|
||||
const CURRENT_USER_FUNCTION_NAME: &str = "current_user";
|
||||
const SYSTEM_USER_FUNCTION_NAME: &str = "system_user";
|
||||
const READ_PREFERENCE_FUNCTION_NAME: &str = "read_preference";
|
||||
const PG_BACKEND_PID: &str = "pg_backend_pid";
|
||||
const CONNECTION_ID: &str = "connection_id";
|
||||
|
||||
impl Function for DatabaseFunction {
|
||||
fn name(&self) -> &str {
|
||||
DATABASE_FUNCTION_NAME
|
||||
}
|
||||
macro_rules! impl_current_schema_function {
|
||||
($name: ident, $fn_name: expr) => {
|
||||
impl Function for $name {
|
||||
fn name(&self) -> &str {
|
||||
$fn_name
|
||||
}
|
||||
|
||||
fn return_type(&self, _: &[DataType]) -> datafusion_common::Result<DataType> {
|
||||
Ok(DataType::Utf8View)
|
||||
}
|
||||
fn return_type(&self, _: &[DataType]) -> datafusion_common::Result<DataType> {
|
||||
Ok(DataType::Utf8View)
|
||||
}
|
||||
|
||||
fn signature(&self) -> &Signature {
|
||||
&self.signature
|
||||
}
|
||||
fn signature(&self) -> &Signature {
|
||||
&self.signature
|
||||
}
|
||||
|
||||
fn invoke_with_args(
|
||||
&self,
|
||||
args: ScalarFunctionArgs,
|
||||
) -> datafusion_common::Result<ColumnarValue> {
|
||||
let func_ctx = find_function_context(&args)?;
|
||||
let db = func_ctx.query_ctx.current_schema();
|
||||
fn invoke_with_args(
|
||||
&self,
|
||||
args: ScalarFunctionArgs,
|
||||
) -> datafusion_common::Result<ColumnarValue> {
|
||||
let func_ctx = find_function_context(&args)?;
|
||||
let db = func_ctx.query_ctx.current_schema();
|
||||
|
||||
Ok(ColumnarValue::Scalar(ScalarValue::Utf8View(Some(db))))
|
||||
}
|
||||
Ok(ColumnarValue::Scalar(ScalarValue::Utf8View(Some(db))))
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
impl_current_schema_function!(DatabaseFunction, DATABASE_FUNCTION_NAME);
|
||||
// MySQL's `SCHEMA()` is a synonym for `DATABASE()`.
|
||||
impl_current_schema_function!(SchemaFunction, SCHEMA_FUNCTION_NAME);
|
||||
|
||||
macro_rules! impl_current_user_function {
|
||||
($name: ident, $fn_name: expr) => {
|
||||
impl Function for $name {
|
||||
fn name(&self) -> &str {
|
||||
$fn_name
|
||||
}
|
||||
|
||||
fn return_type(&self, _: &[DataType]) -> datafusion_common::Result<DataType> {
|
||||
Ok(DataType::Utf8View)
|
||||
}
|
||||
|
||||
fn signature(&self) -> &Signature {
|
||||
&self.signature
|
||||
}
|
||||
|
||||
fn invoke_with_args(
|
||||
&self,
|
||||
args: ScalarFunctionArgs,
|
||||
) -> datafusion_common::Result<ColumnarValue> {
|
||||
let func_ctx = find_function_context(&args)?;
|
||||
let user = func_ctx.query_ctx.current_user();
|
||||
|
||||
Ok(ColumnarValue::Scalar(ScalarValue::Utf8View(Some(
|
||||
user.username().to_string(),
|
||||
))))
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
// GreptimeDB has no notion of a user switching identity mid-session, so `USER()`,
|
||||
// `CURRENT_USER()` and `SYSTEM_USER()` all report the authenticated user.
|
||||
impl_current_user_function!(UserFunction, USER_FUNCTION_NAME);
|
||||
impl_current_user_function!(CurrentUserFunction, CURRENT_USER_FUNCTION_NAME);
|
||||
impl_current_user_function!(SystemUserFunction, SYSTEM_USER_FUNCTION_NAME);
|
||||
|
||||
impl Function for ReadPreferenceFunction {
|
||||
fn name(&self) -> &str {
|
||||
READ_PREFERENCE_FUNCTION_NAME
|
||||
|
||||
@@ -389,6 +389,9 @@ impl PGCatalogFunction {
|
||||
registry.register(pg_catalog::create_pg_get_partition_ancestors_udf());
|
||||
registry.register(pg_catalog::quote_ident_udf::create_quote_ident_udf());
|
||||
registry.register(pg_catalog::quote_ident_udf::create_parse_ident_udf());
|
||||
// Register array bound UDFs used by pg_catalog views.
|
||||
registry.register(pg_catalog::array_bounds_udf::create_array_upper_udf());
|
||||
registry.register(pg_catalog::array_bounds_udf::create_array_lower_udf());
|
||||
registry.register_scalar(ObjDescriptionFunction::new());
|
||||
registry.register_scalar(ColDescriptionFunction::new());
|
||||
registry.register_scalar(ShobjDescriptionFunction::new());
|
||||
|
||||
@@ -37,6 +37,35 @@ use vec1::{Vec1, vec1};
|
||||
use crate::error;
|
||||
use crate::error::{DecodeFlightDataSnafu, InvalidFlightDataSnafu, Result};
|
||||
|
||||
/// Encodes a batch into the schema, record-batch header and payload used by bulk inserts.
|
||||
///
|
||||
/// Uses the default Flight compression and rejects dictionary batches, whose
|
||||
/// additional Flight messages cannot be represented by this three-part format.
|
||||
pub fn record_batch_to_ipc(
|
||||
record_batch: DfRecordBatch,
|
||||
) -> Result<(ProstBytes, ProstBytes, ProstBytes)> {
|
||||
let mut encoder = FlightEncoder::default();
|
||||
let schema = encoder.encode_schema(record_batch.schema().as_ref());
|
||||
let mut iter = encoder
|
||||
.encode(FlightMessage::RecordBatch(record_batch))
|
||||
.into_iter();
|
||||
let flight_data = iter.next().context(InvalidFlightDataSnafu {
|
||||
reason: "Failed to encode empty flight data",
|
||||
})?;
|
||||
if iter.next().is_some() {
|
||||
return error::NotSupportedSnafu {
|
||||
feat: "bulk insert RecordBatch with dictionary arrays",
|
||||
}
|
||||
.fail();
|
||||
}
|
||||
|
||||
Ok((
|
||||
schema.data_header,
|
||||
flight_data.data_header,
|
||||
flight_data.data_body,
|
||||
))
|
||||
}
|
||||
|
||||
/// Flight metadata key used to carry flow query extensions as JSON pairs.
|
||||
pub const FLOW_EXTENSIONS_METADATA_KEY: &str = "x-greptime-flow-extensions";
|
||||
/// Flight metadata key used to carry query snapshot read upper bounds as JSON.
|
||||
@@ -374,8 +403,52 @@ mod test {
|
||||
use datatypes::arrow::buffer::OffsetBuffer;
|
||||
use datatypes::arrow::datatypes::{DataType, Field, Schema};
|
||||
|
||||
use super::*;
|
||||
use crate::Error;
|
||||
use crate::flight::*;
|
||||
|
||||
#[test]
|
||||
fn test_record_batch_to_ipc_preserves_wire_bytes() {
|
||||
let schema = Arc::new(Schema::new(vec![Field::new("n", DataType::Int32, true)]));
|
||||
for values in [vec![], vec![Some(1), None, Some(3)]] {
|
||||
let batch =
|
||||
DfRecordBatch::try_new(schema.clone(), vec![Arc::new(Int32Array::from(values))])
|
||||
.unwrap();
|
||||
// Reproduce the previous Prom encoding sequence and compare all bytes.
|
||||
let mut encoder = FlightEncoder::default();
|
||||
let expected_schema = encoder.encode_schema(batch.schema().as_ref());
|
||||
let messages = encoder.encode(FlightMessage::RecordBatch(batch.clone()));
|
||||
assert_eq!(messages.len(), 1);
|
||||
let expected_batch = messages.first();
|
||||
assert_eq!(
|
||||
record_batch_to_ipc(batch).unwrap(),
|
||||
(
|
||||
expected_schema.data_header,
|
||||
expected_batch.data_header.clone(),
|
||||
expected_batch.data_body.clone(),
|
||||
)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_record_batch_to_ipc_rejects_dictionary() {
|
||||
let schema = Arc::new(Schema::new(vec![Field::new_dictionary(
|
||||
"tag",
|
||||
DataType::UInt32,
|
||||
DataType::Utf8,
|
||||
true,
|
||||
)]));
|
||||
let dictionary = DictionaryArray::new(
|
||||
UInt32Array::from_value(0, 3),
|
||||
Arc::new(StringArray::from_iter_values(["x"])),
|
||||
);
|
||||
let batch = DfRecordBatch::try_new(schema, vec![Arc::new(dictionary)]).unwrap();
|
||||
assert!(matches!(
|
||||
record_batch_to_ipc(batch),
|
||||
Err(Error::NotSupported { feat })
|
||||
if feat == "bulk insert RecordBatch with dictionary arrays"
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_try_decode() -> Result<()> {
|
||||
|
||||
@@ -239,10 +239,6 @@ fn build_struct(
|
||||
|
||||
// Implement DataFusion's ScalarUDFImpl trait
|
||||
impl datafusion::logical_expr::ScalarUDFImpl for #name {
|
||||
fn as_any(&self) -> &dyn std::any::Any {
|
||||
self
|
||||
}
|
||||
|
||||
fn name(&self) -> &str {
|
||||
#display_name
|
||||
}
|
||||
@@ -304,7 +300,7 @@ fn build_struct(
|
||||
.and_then(|cv| match cv {
|
||||
common_query::prelude::ColumnarValue::Vector(v) => Ok(v),
|
||||
common_query::prelude::ColumnarValue::Scalar(s) => {
|
||||
datatypes::vectors::Helper::try_from_scalar_value(s, args.number_rows)
|
||||
datatypes::vectors::Helper::try_from_scalar_value(s, args.number_rows, None)
|
||||
.context(common_query::error::FromScalarValueSnafu)
|
||||
}
|
||||
})
|
||||
|
||||
@@ -31,6 +31,7 @@ use common_procedure::{
|
||||
EventTrigger, LockKey, PoisonKey, PoisonKeys, Procedure, ProcedureId, Status, StringKey,
|
||||
};
|
||||
use common_telemetry::{error, info, warn};
|
||||
use common_wal::options::WalOptions;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use snafu::{ResultExt, ensure};
|
||||
use store_api::metadata::ColumnMetadata;
|
||||
@@ -47,12 +48,12 @@ use crate::ddl::event::table::{
|
||||
TableDdlEvent, TableDdlEventType, TableDdlLocator, alter_table_kind_name,
|
||||
};
|
||||
use crate::ddl::utils::{
|
||||
MultipleResults, extract_column_metadatas, handle_multiple_results, map_to_procedure_error,
|
||||
sync_follower_regions,
|
||||
MultipleResults, extract_column_metadatas, extract_region_wal_options, handle_multiple_results,
|
||||
map_to_procedure_error, sync_follower_regions,
|
||||
};
|
||||
use crate::error::{
|
||||
AbortProcedureSnafu, ConvertAlterTableRequestSnafu, NoLeaderSnafu, PutPoisonSnafu, Result,
|
||||
RetryLaterSnafu, UnsupportedSnafu,
|
||||
RetryLaterSnafu, UnexpectedSnafu, UnsupportedSnafu,
|
||||
};
|
||||
use crate::key::table_info::TableInfoValue;
|
||||
use crate::key::{DeserializedValueWithBytes, RegionDistribution};
|
||||
@@ -148,11 +149,11 @@ impl AlterTableProcedure {
|
||||
|
||||
// Safety: Checked in `AlterTableProcedure::new`.
|
||||
let alter_kind = self.data.task.alter_table.kind.as_ref().unwrap();
|
||||
if enables_skip_wal(alter_kind) {
|
||||
if sets_skip_wal(alter_kind) {
|
||||
ensure!(
|
||||
only_enables_skip_wal(alter_kind),
|
||||
only_sets_skip_wal(alter_kind),
|
||||
UnsupportedSnafu {
|
||||
operation: "combining skip_wal = 'true' with other table options".to_string()
|
||||
operation: "combining skip_wal with other table options".to_string()
|
||||
}
|
||||
);
|
||||
let engine = table_info_value.table_info.meta.engine.as_str();
|
||||
@@ -162,7 +163,8 @@ impl AlterTableProcedure {
|
||||
operation: format!("setting skip_wal on {engine} engine tables")
|
||||
}
|
||||
);
|
||||
// Persist the irreversible intent before any region stops writing WAL.
|
||||
// Keep skip-WAL changes serialized with region migration and use the same
|
||||
// route snapshot for validation and RPC delivery.
|
||||
let (physical_table_id, physical_table_route) = self
|
||||
.context
|
||||
.table_metadata_manager
|
||||
@@ -199,6 +201,37 @@ impl AlterTableProcedure {
|
||||
table_id: physical_table_id
|
||||
}
|
||||
);
|
||||
// validate_alter_table_expr() has already parsed this single skip-WAL option.
|
||||
let Some(skip_wal) = skip_wal_value(alter_kind) else {
|
||||
return UnexpectedSnafu {
|
||||
err_msg: "missing or invalid skip_wal value after validation".to_string(),
|
||||
}
|
||||
.fail();
|
||||
};
|
||||
if !skip_wal {
|
||||
let datanode_table_values = self
|
||||
.context
|
||||
.table_metadata_manager
|
||||
.datanode_table_manager()
|
||||
.regions(physical_table_id, &physical_table_route)
|
||||
.await?;
|
||||
let region_wal_options = extract_region_wal_options(&datanode_table_values)?;
|
||||
ensure!(
|
||||
current_region_ids.iter().all(|region_id| {
|
||||
matches!(
|
||||
region_wal_options
|
||||
.get(®ion_id.region_number())
|
||||
.cloned()
|
||||
.unwrap_or_default(),
|
||||
WalOptions::RaftEngine | WalOptions::Kafka(_)
|
||||
)
|
||||
}),
|
||||
UnsupportedSnafu {
|
||||
operation: "setting skip_wal = 'false' without an existing WAL provider"
|
||||
.to_string()
|
||||
}
|
||||
);
|
||||
}
|
||||
self.data.region_distribution =
|
||||
Some(region_distribution(&physical_table_route.region_routes));
|
||||
}
|
||||
@@ -265,8 +298,8 @@ impl AlterTableProcedure {
|
||||
MultipleResults::PartialRetryable(error) => Err(error),
|
||||
MultipleResults::PartialNonRetryable(error)
|
||||
| MultipleResults::AllNonRetryable(error) => {
|
||||
// The metadata already enables skip-WAL. Retry the idempotent request
|
||||
// so later attempts can update the remaining replicas.
|
||||
// The metadata already contains the requested skip-WAL value. Retry the
|
||||
// idempotent request so later attempts can update the remaining replicas.
|
||||
Err(BoxedError::new(error)).context(RetryLaterSnafu {
|
||||
clean_poisons: true,
|
||||
})
|
||||
@@ -450,24 +483,35 @@ impl AlterTableProcedure {
|
||||
}
|
||||
}
|
||||
|
||||
fn enables_skip_wal(alter_kind: &Kind) -> bool {
|
||||
fn sets_skip_wal(alter_kind: &Kind) -> bool {
|
||||
let Kind::SetTableOptions(SetTableOptions { table_options }) = alter_kind else {
|
||||
return false;
|
||||
};
|
||||
|
||||
table_options
|
||||
.iter()
|
||||
.any(|option| option.key == SKIP_WAL_KEY && option.value == "true")
|
||||
.any(|option| option.key == SKIP_WAL_KEY)
|
||||
}
|
||||
|
||||
pub(crate) fn only_enables_skip_wal(alter_kind: &Kind) -> bool {
|
||||
pub(crate) fn only_sets_skip_wal(alter_kind: &Kind) -> bool {
|
||||
let Kind::SetTableOptions(SetTableOptions { table_options }) = alter_kind else {
|
||||
return false;
|
||||
};
|
||||
|
||||
table_options.len() == 1
|
||||
&& table_options[0].key == SKIP_WAL_KEY
|
||||
&& table_options[0].value == "true"
|
||||
table_options.len() == 1 && table_options[0].key == SKIP_WAL_KEY
|
||||
}
|
||||
|
||||
fn skip_wal_value(alter_kind: &Kind) -> Option<bool> {
|
||||
let Kind::SetTableOptions(SetTableOptions { table_options }) = alter_kind else {
|
||||
return None;
|
||||
};
|
||||
let [option] = table_options.as_slice() else {
|
||||
return None;
|
||||
};
|
||||
if option.key != SKIP_WAL_KEY {
|
||||
return None;
|
||||
}
|
||||
option.value.parse().ok()
|
||||
}
|
||||
|
||||
fn is_metadata_only_alter(alter_kind: &Kind) -> Result<bool> {
|
||||
@@ -487,7 +531,7 @@ enum AlterTableFlow {
|
||||
|
||||
impl AlterTableFlow {
|
||||
fn from_kind(kind: &Kind) -> Result<Self> {
|
||||
Ok(if only_enables_skip_wal(kind) {
|
||||
Ok(if only_sets_skip_wal(kind) {
|
||||
Self::MetadataFirst
|
||||
} else if is_metadata_only_alter(kind)? {
|
||||
Self::MetadataOnly
|
||||
@@ -611,7 +655,7 @@ pub struct AlterTableData {
|
||||
table_info_value: Option<DeserializedValueWithBytes<TableInfoValue>>,
|
||||
/// Region distribution for table in case we need to update region options.
|
||||
region_distribution: Option<RegionDistribution>,
|
||||
/// Region locks held by irreversible region-option alters.
|
||||
/// Region locks held by skip-WAL alters.
|
||||
#[serde(default)]
|
||||
region_locks: Vec<RegionId>,
|
||||
}
|
||||
|
||||
@@ -174,7 +174,7 @@ impl AlterTableExecutor {
|
||||
.await
|
||||
}
|
||||
|
||||
/// Alters all replicas for the irreversible skip-WAL flow.
|
||||
/// Alters all replicas for the skip-WAL flow.
|
||||
pub(crate) async fn on_alter_skip_wal_regions(
|
||||
&self,
|
||||
node_manager: &NodeManagerRef,
|
||||
|
||||
@@ -568,6 +568,21 @@ pub fn validate_flow_options(flow_task: &CreateFlowTask) -> Result<()> {
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(value) = flow_task
|
||||
.flow_options
|
||||
.get(FLOW_EXPERIMENTAL_ENABLE_INCREMENTAL_READ_KEY)
|
||||
&& value != FLOW_EXPERIMENTAL_ENABLE_INCREMENTAL_READ_SEQUENCE_RANGE
|
||||
{
|
||||
value.parse::<bool>().map_err(|_| {
|
||||
UnexpectedSnafu {
|
||||
err_msg: format!(
|
||||
"Invalid flow option {FLOW_EXPERIMENTAL_ENABLE_INCREMENTAL_READ_KEY}: {value}"
|
||||
),
|
||||
}
|
||||
.build()
|
||||
})?;
|
||||
}
|
||||
|
||||
defer_on_missing_source(flow_task)?;
|
||||
get_flow_type_from_options(flow_task)?;
|
||||
Ok(())
|
||||
@@ -770,6 +785,9 @@ pub enum FlowType {
|
||||
|
||||
pub const FLOW_EXPERIMENTAL_ENABLE_INCREMENTAL_READ_KEY: &str =
|
||||
"experimental_enable_incremental_read";
|
||||
/// Reserved internal value for flows requiring exact sequence ranges.
|
||||
pub const FLOW_EXPERIMENTAL_ENABLE_INCREMENTAL_READ_SEQUENCE_RANGE: &str =
|
||||
"__greptime_internal_exact_sequence_range";
|
||||
|
||||
impl FlowType {
|
||||
pub const BATCHING: &str = "batching";
|
||||
|
||||
@@ -19,7 +19,7 @@ use std::sync::Arc;
|
||||
use api::region::RegionResponse;
|
||||
use api::v1::alter_table_expr::Kind;
|
||||
use api::v1::region::sync_request::ManifestInfo;
|
||||
use api::v1::region::{RegionRequest, region_request};
|
||||
use api::v1::region::{RegionRequest, alter_request, region_request};
|
||||
use api::v1::{
|
||||
AddColumn, AddColumns, AlterTableExpr, ColumnDataType, ColumnDef as PbColumnDef, DropColumn,
|
||||
DropColumns, SemanticType, SetTableOptions,
|
||||
@@ -30,6 +30,7 @@ use common_error::status_code::StatusCode;
|
||||
use common_procedure::store::poison_store::PoisonStore;
|
||||
use common_procedure::{Procedure, ProcedureId, Status};
|
||||
use common_procedure_test::{MockContextProvider, execute_procedure_until_done};
|
||||
use common_wal::options::{KafkaWalOptions, WalOptions};
|
||||
use datatypes::prelude::ConcreteDataType;
|
||||
use datatypes::schema::ColumnSchema;
|
||||
use store_api::metadata::ColumnMetadata;
|
||||
@@ -164,6 +165,29 @@ fn assert_alter_request(
|
||||
assert_eq!(req.region_id, expected_region_id);
|
||||
}
|
||||
|
||||
fn assert_skip_wal_alter_request(
|
||||
peer: Peer,
|
||||
request: RegionRequest,
|
||||
expected_peer_id: u64,
|
||||
expected_region_id: RegionId,
|
||||
expected_skip_wal: bool,
|
||||
) {
|
||||
assert_eq!(peer.id, expected_peer_id);
|
||||
let Some(region_request::Body::Alter(req)) = request.body else {
|
||||
unreachable!();
|
||||
};
|
||||
assert_eq!(req.region_id, expected_region_id);
|
||||
let Some(alter_request::Kind::SetTableOptions(options)) = req.kind else {
|
||||
unreachable!();
|
||||
};
|
||||
assert_eq!(1, options.table_options.len());
|
||||
assert_eq!(SKIP_WAL_KEY, options.table_options[0].key);
|
||||
assert_eq!(
|
||||
expected_skip_wal.to_string(),
|
||||
options.table_options[0].value
|
||||
);
|
||||
}
|
||||
|
||||
fn assert_sync_request(
|
||||
peer: Peer,
|
||||
request: RegionRequest,
|
||||
@@ -729,39 +753,42 @@ async fn test_skip_wal_rejects_file_engine_table() {
|
||||
fn test_skip_wal_holds_region_locks() {
|
||||
let table_id = 1024;
|
||||
let region_ids = vec![RegionId::new(table_id, 2), RegionId::new(table_id, 1)];
|
||||
let task = AlterTableTask {
|
||||
alter_table: AlterTableExpr {
|
||||
catalog_name: DEFAULT_CATALOG_NAME.to_string(),
|
||||
schema_name: DEFAULT_SCHEMA_NAME.to_string(),
|
||||
table_name: "foo".to_string(),
|
||||
kind: Some(Kind::SetTableOptions(SetTableOptions {
|
||||
table_options: vec![api::v1::Option {
|
||||
key: SKIP_WAL_KEY.to_string(),
|
||||
value: "true".to_string(),
|
||||
}],
|
||||
})),
|
||||
},
|
||||
};
|
||||
let context = new_ddl_context(Arc::new(MockDatanodeManager::new(())));
|
||||
let procedure = AlterTableProcedure::new_with_region_locks(
|
||||
table_id,
|
||||
task,
|
||||
region_ids.clone(),
|
||||
context.clone(),
|
||||
)
|
||||
.unwrap();
|
||||
for value in ["true", "false"] {
|
||||
let task = AlterTableTask {
|
||||
alter_table: AlterTableExpr {
|
||||
catalog_name: DEFAULT_CATALOG_NAME.to_string(),
|
||||
schema_name: DEFAULT_SCHEMA_NAME.to_string(),
|
||||
table_name: "foo".to_string(),
|
||||
kind: Some(Kind::SetTableOptions(SetTableOptions {
|
||||
table_options: vec![api::v1::Option {
|
||||
key: SKIP_WAL_KEY.to_string(),
|
||||
value: value.to_string(),
|
||||
}],
|
||||
})),
|
||||
},
|
||||
};
|
||||
let procedure = AlterTableProcedure::new_with_region_locks(
|
||||
table_id,
|
||||
task,
|
||||
region_ids.clone(),
|
||||
context.clone(),
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let lock_key = procedure.lock_key();
|
||||
for region_id in ®ion_ids {
|
||||
assert!(
|
||||
lock_key
|
||||
.keys_to_lock()
|
||||
.any(|key| key == &RegionLock::Write(*region_id).into())
|
||||
);
|
||||
let lock_key = procedure.lock_key();
|
||||
for region_id in ®ion_ids {
|
||||
assert!(
|
||||
lock_key
|
||||
.keys_to_lock()
|
||||
.any(|key| key == &RegionLock::Write(*region_id).into())
|
||||
);
|
||||
}
|
||||
|
||||
let recovered =
|
||||
AlterTableProcedure::from_json(&procedure.dump().unwrap(), context.clone()).unwrap();
|
||||
assert_eq!(lock_key, recovered.lock_key());
|
||||
}
|
||||
|
||||
let recovered = AlterTableProcedure::from_json(&procedure.dump().unwrap(), context).unwrap();
|
||||
assert_eq!(lock_key, recovered.lock_key());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -875,6 +902,252 @@ async fn test_skip_wal_rejects_no_leader_before_updating_metadata() {
|
||||
assert!(!table_info.meta.options.skip_wal);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_enable_wal_rejects_noop_provider_before_updating_metadata() {
|
||||
let ddl_context = new_ddl_context(Arc::new(MockDatanodeManager::new(())));
|
||||
let table_name = "foo";
|
||||
let table_id = 1024;
|
||||
let mut task = test_create_table_task(table_name, table_id);
|
||||
task.table_info.meta.options.skip_wal = true;
|
||||
task.table_info
|
||||
.meta
|
||||
.options
|
||||
.extra_options
|
||||
.insert(SKIP_WAL_KEY.to_string(), "true".to_string());
|
||||
let table_route = prepare_table_route(table_id);
|
||||
let region_locks = table_route
|
||||
.region_routes()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.map(|route| route.region.id)
|
||||
.collect();
|
||||
let region_wal_options = HashMap::from([
|
||||
(1, WalOptions::Noop),
|
||||
(2, WalOptions::Noop),
|
||||
(3, WalOptions::Noop),
|
||||
]);
|
||||
ddl_context
|
||||
.table_metadata_manager
|
||||
.create_table_metadata(task.table_info, table_route, region_wal_options)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let alter_task = AlterTableTask {
|
||||
alter_table: AlterTableExpr {
|
||||
catalog_name: DEFAULT_CATALOG_NAME.to_string(),
|
||||
schema_name: DEFAULT_SCHEMA_NAME.to_string(),
|
||||
table_name: table_name.to_string(),
|
||||
kind: Some(Kind::SetTableOptions(SetTableOptions {
|
||||
table_options: vec![api::v1::Option {
|
||||
key: SKIP_WAL_KEY.to_string(),
|
||||
value: "false".to_string(),
|
||||
}],
|
||||
})),
|
||||
},
|
||||
};
|
||||
let mut procedure = AlterTableProcedure::new_with_region_locks(
|
||||
table_id,
|
||||
alter_task,
|
||||
region_locks,
|
||||
ddl_context.clone(),
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let error = procedure.on_prepare().await.unwrap_err();
|
||||
assert_matches!(error, Error::Unsupported { .. });
|
||||
assert!(
|
||||
error
|
||||
.to_string()
|
||||
.contains("without an existing WAL provider")
|
||||
);
|
||||
let table_info = ddl_context
|
||||
.table_metadata_manager
|
||||
.table_info_manager()
|
||||
.get(table_id)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
.into_inner()
|
||||
.table_info;
|
||||
assert!(table_info.meta.options.skip_wal);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_enable_wal_accepts_legacy_raft_regions_without_wal_options() {
|
||||
let ddl_context = new_ddl_context(Arc::new(MockDatanodeManager::new(())));
|
||||
let table_name = "legacy_raft";
|
||||
let table_id = 1024;
|
||||
let mut task = test_create_table_task(table_name, table_id);
|
||||
task.table_info.meta.options.skip_wal = true;
|
||||
task.table_info
|
||||
.meta
|
||||
.options
|
||||
.extra_options
|
||||
.insert(SKIP_WAL_KEY.to_string(), "true".to_string());
|
||||
let table_route = prepare_table_route(table_id);
|
||||
let region_locks = table_route
|
||||
.region_routes()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.map(|route| route.region.id)
|
||||
.collect();
|
||||
ddl_context
|
||||
.table_metadata_manager
|
||||
.create_table_metadata(task.table_info, table_route, HashMap::new())
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let alter_task = AlterTableTask {
|
||||
alter_table: AlterTableExpr {
|
||||
catalog_name: DEFAULT_CATALOG_NAME.to_string(),
|
||||
schema_name: DEFAULT_SCHEMA_NAME.to_string(),
|
||||
table_name: table_name.to_string(),
|
||||
kind: Some(Kind::SetTableOptions(SetTableOptions {
|
||||
table_options: vec![api::v1::Option {
|
||||
key: SKIP_WAL_KEY.to_string(),
|
||||
value: "false".to_string(),
|
||||
}],
|
||||
})),
|
||||
},
|
||||
};
|
||||
let mut procedure = AlterTableProcedure::new_with_region_locks(
|
||||
table_id,
|
||||
alter_task,
|
||||
region_locks,
|
||||
ddl_context.clone(),
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
procedure.on_prepare().await.unwrap();
|
||||
procedure.on_update_metadata().await.unwrap();
|
||||
let table_info = ddl_context
|
||||
.table_metadata_manager
|
||||
.table_info_manager()
|
||||
.get(table_id)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
.into_inner()
|
||||
.table_info;
|
||||
assert!(!table_info.meta.options.skip_wal);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_enable_wal_updates_metadata_and_region_request() {
|
||||
let (tx, mut rx) = mpsc::channel(8);
|
||||
let node_manager = Arc::new(MockDatanodeManager::new(DatanodeWatcher::new(tx)));
|
||||
let ddl_context = new_ddl_context(node_manager);
|
||||
let table_name = "foo";
|
||||
let table_id = 1024;
|
||||
let mut task = test_create_table_task(table_name, table_id);
|
||||
task.table_info.meta.options.skip_wal = true;
|
||||
task.table_info
|
||||
.meta
|
||||
.options
|
||||
.extra_options
|
||||
.insert(SKIP_WAL_KEY.to_string(), "true".to_string());
|
||||
let table_route = prepare_table_route(table_id);
|
||||
let region_locks = table_route
|
||||
.region_routes()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.map(|route| route.region.id)
|
||||
.collect();
|
||||
let region_wal_options = HashMap::from([
|
||||
(1, WalOptions::RaftEngine),
|
||||
(
|
||||
2,
|
||||
WalOptions::Kafka(KafkaWalOptions::new("topic-2".to_string())),
|
||||
),
|
||||
(
|
||||
3,
|
||||
WalOptions::Kafka(KafkaWalOptions::new("topic-3".to_string())),
|
||||
),
|
||||
]);
|
||||
ddl_context
|
||||
.table_metadata_manager
|
||||
.create_table_metadata(task.table_info, table_route, region_wal_options)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let alter_task = AlterTableTask {
|
||||
alter_table: AlterTableExpr {
|
||||
catalog_name: DEFAULT_CATALOG_NAME.to_string(),
|
||||
schema_name: DEFAULT_SCHEMA_NAME.to_string(),
|
||||
table_name: table_name.to_string(),
|
||||
kind: Some(Kind::SetTableOptions(SetTableOptions {
|
||||
table_options: vec![api::v1::Option {
|
||||
key: SKIP_WAL_KEY.to_string(),
|
||||
value: "false".to_string(),
|
||||
}],
|
||||
})),
|
||||
},
|
||||
};
|
||||
let mut procedure = AlterTableProcedure::new_with_region_locks(
|
||||
table_id,
|
||||
alter_task,
|
||||
region_locks,
|
||||
ddl_context.clone(),
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
procedure.on_prepare().await.unwrap();
|
||||
let persisted: serde_json::Value = serde_json::from_str(&procedure.dump().unwrap()).unwrap();
|
||||
assert_eq!("UpdateMetadata", persisted["state"]);
|
||||
|
||||
let Some(alter_request::Kind::SetTableOptions(options)) =
|
||||
procedure.make_region_alter_kind().unwrap()
|
||||
else {
|
||||
unreachable!();
|
||||
};
|
||||
assert_eq!(1, options.table_options.len());
|
||||
assert_eq!(SKIP_WAL_KEY, options.table_options[0].key);
|
||||
assert_eq!("false", options.table_options[0].value);
|
||||
|
||||
procedure.on_update_metadata().await.unwrap();
|
||||
let table_info = ddl_context
|
||||
.table_metadata_manager
|
||||
.table_info_manager()
|
||||
.get(table_id)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
.into_inner()
|
||||
.table_info;
|
||||
assert!(!table_info.meta.options.skip_wal);
|
||||
assert_eq!(
|
||||
Some("false"),
|
||||
table_info
|
||||
.meta
|
||||
.options
|
||||
.extra_options
|
||||
.get(SKIP_WAL_KEY)
|
||||
.map(String::as_str)
|
||||
);
|
||||
|
||||
let provider = Arc::new(MockContextProvider::default());
|
||||
procedure
|
||||
.submit_alter_region_requests(ProcedureId::random(), provider.as_ref())
|
||||
.await
|
||||
.unwrap();
|
||||
let mut requests = Vec::new();
|
||||
while let Ok(request) = rx.try_recv() {
|
||||
requests.push(request);
|
||||
}
|
||||
requests.sort_unstable_by_key(|(peer, _)| peer.id);
|
||||
let expected = [
|
||||
(1, RegionId::new(table_id, 1)),
|
||||
(2, RegionId::new(table_id, 2)),
|
||||
(3, RegionId::new(table_id, 3)),
|
||||
(4, RegionId::new(table_id, 2)),
|
||||
(5, RegionId::new(table_id, 1)),
|
||||
];
|
||||
assert_eq!(expected.len(), requests.len());
|
||||
for ((peer, request), (peer_id, region_id)) in requests.into_iter().zip(expected) {
|
||||
assert_skip_wal_alter_request(peer, request, peer_id, region_id, false);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_skip_wal_updates_metadata_before_all_replicas() {
|
||||
let (tx, mut rx) = mpsc::channel(8);
|
||||
|
||||
@@ -35,7 +35,7 @@ use table::table_name::TableName;
|
||||
|
||||
use crate::ddl::alter_database::AlterDatabaseProcedure;
|
||||
use crate::ddl::alter_logical_tables::AlterLogicalTablesProcedure;
|
||||
use crate::ddl::alter_table::{AlterTableProcedure, RegionRouteChanged, only_enables_skip_wal};
|
||||
use crate::ddl::alter_table::{AlterTableProcedure, RegionRouteChanged, only_sets_skip_wal};
|
||||
use crate::ddl::comment_on::CommentOnProcedure;
|
||||
use crate::ddl::create_database::{CreateDatabaseMetadataCommitterRef, CreateDatabaseProcedure};
|
||||
use crate::ddl::create_flow::CreateFlowProcedure;
|
||||
@@ -433,7 +433,7 @@ impl DdlManager {
|
||||
.alter_table
|
||||
.kind
|
||||
.as_ref()
|
||||
.is_some_and(only_enables_skip_wal);
|
||||
.is_some_and(only_sets_skip_wal);
|
||||
|
||||
let mut route_change_retries = 0;
|
||||
loop {
|
||||
|
||||
@@ -269,6 +269,20 @@ pub enum Error {
|
||||
error: prost::DecodeError,
|
||||
},
|
||||
|
||||
#[snafu(display("Failed to decode packed file references"))]
|
||||
DecodePackedFileRefs {
|
||||
#[snafu(implicit)]
|
||||
location: Location,
|
||||
#[snafu(source)]
|
||||
error: base64::DecodeError,
|
||||
},
|
||||
|
||||
#[snafu(display("Invalid packed file references framing"))]
|
||||
InvalidPackedFileRefs {
|
||||
#[snafu(implicit)]
|
||||
location: Location,
|
||||
},
|
||||
|
||||
#[snafu(display("Failed to encode object into json"))]
|
||||
EncodeJson {
|
||||
#[snafu(implicit)]
|
||||
@@ -1212,6 +1226,8 @@ impl ErrorExt for Error {
|
||||
| PayloadNotExist { .. }
|
||||
| ConvertRawKey { .. }
|
||||
| DecodeProto { .. }
|
||||
| DecodePackedFileRefs { .. }
|
||||
| InvalidPackedFileRefs { .. }
|
||||
| BuildTableMeta { .. }
|
||||
| TableRouteNotFound { .. }
|
||||
| TableRepartNotFound { .. }
|
||||
|
||||
@@ -94,7 +94,21 @@ impl HeartbeatResponseHandlerExecutor for HandlerGroupExecutor {
|
||||
Ok(HandleControl::Done) => break,
|
||||
Ok(HandleControl::Continue) => {}
|
||||
Err(e) => {
|
||||
error!(e;"Error while handling: {:?}", ctx.response);
|
||||
let mailbox_message_id = ctx
|
||||
.response
|
||||
.mailbox_message
|
||||
.as_ref()
|
||||
.map(|message| message.id);
|
||||
let json_payload_len =
|
||||
ctx.response.mailbox_message.as_ref().and_then(|message| {
|
||||
message.payload.as_ref().map(|payload| match payload {
|
||||
api::v1::meta::mailbox_message::Payload::Json(json) => json.len(),
|
||||
})
|
||||
});
|
||||
error!(
|
||||
%e;
|
||||
"Error while handling heartbeat response: mailbox_message_id={mailbox_message_id:?}, json_payload_len={json_payload_len:?}"
|
||||
);
|
||||
break;
|
||||
}
|
||||
}
|
||||
@@ -102,3 +116,86 @@ impl HeartbeatResponseHandlerExecutor for HandlerGroupExecutor {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod error_log_tests {
|
||||
use std::fmt::Debug;
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
use api::v1::meta::mailbox_message::Payload;
|
||||
use api::v1::meta::{HeartbeatResponse, MailboxMessage};
|
||||
use common_telemetry::tracing::field::{Field, Visit};
|
||||
use common_telemetry::tracing::{Event, Subscriber};
|
||||
use common_telemetry::tracing_subscriber::layer::{Context, SubscriberExt};
|
||||
use common_telemetry::{tracing, tracing_subscriber};
|
||||
|
||||
use super::parse_mailbox_message::ParseMailboxMessageHandler;
|
||||
use super::*;
|
||||
use crate::heartbeat::mailbox::HeartbeatMailbox;
|
||||
|
||||
#[derive(Clone, Default)]
|
||||
struct LogCapture(Arc<Mutex<Vec<String>>>);
|
||||
|
||||
impl<S> tracing_subscriber::Layer<S> for LogCapture
|
||||
where
|
||||
S: Subscriber,
|
||||
{
|
||||
fn on_event(&self, event: &Event<'_>, _ctx: Context<'_, S>) {
|
||||
let mut visitor = FieldVisitor::default();
|
||||
event.record(&mut visitor);
|
||||
self.0.lock().unwrap().push(visitor.fields.join(", "));
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct FieldVisitor {
|
||||
fields: Vec<String>,
|
||||
}
|
||||
|
||||
impl Visit for FieldVisitor {
|
||||
fn record_debug(&mut self, field: &Field, value: &dyn Debug) {
|
||||
self.fields.push(format!("{}={value:?}", field.name()));
|
||||
}
|
||||
|
||||
fn record_str(&mut self, field: &Field, value: &str) {
|
||||
self.fields.push(format!("{}={value:?}", field.name()));
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
async fn test_handler_error_log_excludes_mailbox_payload() {
|
||||
let payload_sentinel = "MALFORMED_PACKED_PAYLOAD_MUST_NOT_BE_LOGGED".repeat(256);
|
||||
let payload = format!(
|
||||
r#"{{"PackedGcRegions":{{"regions":[],"packed_file_refs_manifest":"{}"}}"#,
|
||||
payload_sentinel
|
||||
);
|
||||
let payload_len = payload.len();
|
||||
let capture = LogCapture::default();
|
||||
let subscriber = tracing_subscriber::registry().with(capture.clone());
|
||||
let _guard = tracing::subscriber::set_default(subscriber);
|
||||
let (mailbox_tx, _) = tokio::sync::mpsc::channel(1);
|
||||
let ctx = HeartbeatResponseHandlerContext::new(
|
||||
Arc::new(HeartbeatMailbox::new(mailbox_tx)),
|
||||
HeartbeatResponse {
|
||||
mailbox_message: Some(MailboxMessage {
|
||||
id: 42,
|
||||
subject: "unsafe subject".to_string(),
|
||||
to: "unsafe recipient".to_string(),
|
||||
from: "unsafe sender".to_string(),
|
||||
payload: Some(Payload::Json(payload)),
|
||||
..Default::default()
|
||||
}),
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
let executor = HandlerGroupExecutor::new(vec![Arc::new(ParseMailboxMessageHandler)]);
|
||||
|
||||
executor.handle(ctx).await.unwrap();
|
||||
|
||||
let logs = capture.0.lock().unwrap().join("\n");
|
||||
assert!(logs.contains("Error while handling heartbeat response"));
|
||||
assert!(logs.contains("mailbox_message_id=Some(42)"));
|
||||
assert!(logs.contains(&format!("json_payload_len=Some({payload_len})")));
|
||||
assert!(!logs.contains(&payload_sentinel));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -16,16 +16,19 @@ use std::collections::{HashMap, HashSet};
|
||||
use std::fmt::{Display, Formatter};
|
||||
use std::time::Duration;
|
||||
|
||||
use base64::Engine;
|
||||
use common_error::ext::{ErrorExt, RetryHint};
|
||||
use common_error::status_code::StatusCode;
|
||||
use serde::{Deserialize, Deserializer, Serialize, Serializer};
|
||||
use snafu::ResultExt as _;
|
||||
use store_api::region_engine::SyncRegionFromRequest;
|
||||
use store_api::region_request::{RegionFlushReason, RegionRequirements};
|
||||
use store_api::storage::{FileRefsManifest, GcReport, RegionId, RegionNumber};
|
||||
use store_api::storage::{FileId, FileRef, FileRefsManifest, GcReport, RegionId, RegionNumber};
|
||||
use strum::Display;
|
||||
use table::metadata::TableId;
|
||||
use table::table_name::TableName;
|
||||
|
||||
use crate::error::{DecodePackedFileRefsSnafu, InvalidPackedFileRefsSnafu};
|
||||
use crate::flow_name::FlowName;
|
||||
use crate::key::schema_name::SchemaName;
|
||||
use crate::key::{FlowId, FlowPartitionId};
|
||||
@@ -579,6 +582,24 @@ impl Display for GetFileRefs {
|
||||
}
|
||||
}
|
||||
|
||||
/// Instruction to get file references using the packed manifest format.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||
pub struct GetPackedFileRefs {
|
||||
/// List of region IDs to get file references from active FileHandles (in-memory).
|
||||
pub query_regions: Vec<RegionId>,
|
||||
/// Mapping from the src region IDs (whose file references to look for) to
|
||||
/// the dst region IDs (where to read the manifests).
|
||||
/// Key: The source region IDs (where files originally came from).
|
||||
/// Value: The set of destination region IDs (whose manifests need to be read).
|
||||
pub related_regions: HashMap<RegionId, HashSet<RegionId>>,
|
||||
}
|
||||
|
||||
impl Display for GetPackedFileRefs {
|
||||
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
|
||||
write!(f, "GetPackedFileRefs(region_ids={:?})", self.query_regions)
|
||||
}
|
||||
}
|
||||
|
||||
/// Instruction to trigger garbage collection for a region.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||
pub struct GcRegions {
|
||||
@@ -590,6 +611,147 @@ pub struct GcRegions {
|
||||
pub full_file_listing: bool,
|
||||
}
|
||||
|
||||
/// Instruction to trigger garbage collection with a packed file-reference manifest.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||
pub struct PackedGcRegions {
|
||||
/// The region ID to perform GC on.
|
||||
pub regions: Vec<RegionId>,
|
||||
/// Packed manifest used by the packed-only GC protocol.
|
||||
pub packed_file_refs_manifest: PackedFileRefsManifest,
|
||||
/// Whether to perform a full file listing to find orphan files.
|
||||
pub full_file_listing: bool,
|
||||
}
|
||||
|
||||
impl Display for PackedGcRegions {
|
||||
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
|
||||
let file_refs_count = self.packed_file_refs_manifest.file_refs.len();
|
||||
write!(
|
||||
f,
|
||||
"PackedGcRegions(regions={:?}, file_refs_count={}, full_file_listing={})",
|
||||
self.regions, file_refs_count, self.full_file_listing
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
/// A packed, JSON-compatible encoding of a file reference manifest.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, Default)]
|
||||
pub struct PackedFileRefsManifest {
|
||||
/// Required packed file references for each region.
|
||||
pub file_refs: HashMap<RegionId, PackedRegionFileRefs>,
|
||||
#[serde(default)]
|
||||
pub manifest_version: HashMap<RegionId, u64>,
|
||||
#[serde(default)]
|
||||
pub cross_region_refs: HashMap<RegionId, HashSet<RegionId>>,
|
||||
}
|
||||
|
||||
/// Packed file references for one region, encoded with STANDARD Base64.
|
||||
///
|
||||
/// `files` is required and contains 16-byte UUID records for references whose
|
||||
/// `index_version` is `None`. `indexed` is required and contains 24-byte records
|
||||
/// for references whose `index_version` is `Some`: a 16-byte UUID followed by
|
||||
/// an 8-byte big-endian `u64` version. Decoding fails when either field is not
|
||||
/// valid STANDARD Base64 or its decoded bytes do not align to the corresponding
|
||||
/// record width.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, Default)]
|
||||
pub struct PackedRegionFileRefs {
|
||||
/// Required STANDARD Base64-encoded 16-byte UUID records.
|
||||
pub files: String,
|
||||
/// Required STANDARD Base64-encoded 16-byte UUID and 8-byte version records.
|
||||
pub indexed: String,
|
||||
}
|
||||
|
||||
impl PackedFileRefsManifest {
|
||||
/// Packs a file reference manifest using required [`PackedRegionFileRefs`]
|
||||
/// STANDARD Base64 fields.
|
||||
///
|
||||
/// References whose `index_version` is `None` are encoded as 16-byte UUID
|
||||
/// `files` records; references whose `index_version` is `Some` are encoded
|
||||
/// as 24-byte `indexed` records: a 16-byte UUID followed by an 8-byte
|
||||
/// big-endian `u64` version.
|
||||
pub fn from_manifest(manifest: &FileRefsManifest) -> Self {
|
||||
Self {
|
||||
file_refs: manifest
|
||||
.file_refs
|
||||
.iter()
|
||||
.map(|(region, refs)| (*region, PackedRegionFileRefs::from_refs(refs)))
|
||||
.collect(),
|
||||
manifest_version: manifest.manifest_version.clone(),
|
||||
cross_region_refs: manifest.cross_region_refs.clone(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Decodes this packed representation into a file reference manifest.
|
||||
///
|
||||
/// Returns a typed error for invalid STANDARD Base64 or malformed record
|
||||
/// framing without retaining encoded payloads.
|
||||
pub fn into_manifest(self) -> crate::error::Result<FileRefsManifest> {
|
||||
let mut file_refs = HashMap::new();
|
||||
for (region, encoded) in self.file_refs {
|
||||
file_refs.insert(region, encoded.into_refs(region)?);
|
||||
}
|
||||
Ok(FileRefsManifest {
|
||||
file_refs,
|
||||
manifest_version: self.manifest_version,
|
||||
cross_region_refs: self.cross_region_refs,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl PackedRegionFileRefs {
|
||||
/// Packs file references as the documented STANDARD Base64 record streams.
|
||||
pub fn from_refs(refs: &HashSet<FileRef>) -> Self {
|
||||
let engine = base64::engine::general_purpose::STANDARD;
|
||||
let mut files = Vec::new();
|
||||
let mut indexed = Vec::new();
|
||||
for file_ref in refs {
|
||||
match file_ref.index_version {
|
||||
None => files.extend_from_slice(file_ref.file_id.as_bytes()),
|
||||
Some(version) => {
|
||||
indexed.extend_from_slice(file_ref.file_id.as_bytes());
|
||||
indexed.extend_from_slice(&version.to_be_bytes());
|
||||
}
|
||||
}
|
||||
}
|
||||
Self {
|
||||
files: engine.encode(files),
|
||||
indexed: engine.encode(indexed),
|
||||
}
|
||||
}
|
||||
|
||||
/// Decodes the documented STANDARD Base64 record streams for `region`.
|
||||
fn into_refs(self, region: RegionId) -> crate::error::Result<HashSet<FileRef>> {
|
||||
let engine = base64::engine::general_purpose::STANDARD;
|
||||
let files = engine
|
||||
.decode(self.files)
|
||||
.context(DecodePackedFileRefsSnafu)?;
|
||||
let indexed = engine
|
||||
.decode(self.indexed)
|
||||
.context(DecodePackedFileRefsSnafu)?;
|
||||
if files.len() % 16 != 0 || indexed.len() % 24 != 0 {
|
||||
return InvalidPackedFileRefsSnafu.fail();
|
||||
}
|
||||
|
||||
let mut refs = HashSet::new();
|
||||
for bytes in files.chunks_exact(16) {
|
||||
let mut id = [0; 16];
|
||||
id.copy_from_slice(bytes);
|
||||
refs.insert(FileRef::new(region, FileId::from_bytes(id), None));
|
||||
}
|
||||
for bytes in indexed.chunks_exact(24) {
|
||||
let mut id = [0; 16];
|
||||
id.copy_from_slice(&bytes[..16]);
|
||||
let mut version = [0; 8];
|
||||
version.copy_from_slice(&bytes[16..]);
|
||||
refs.insert(FileRef::new(
|
||||
region,
|
||||
FileId::from_bytes(id),
|
||||
Some(u64::from_be_bytes(version)),
|
||||
));
|
||||
}
|
||||
Ok(refs)
|
||||
}
|
||||
}
|
||||
|
||||
impl Display for GcRegions {
|
||||
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
|
||||
write!(
|
||||
@@ -625,6 +787,26 @@ impl Display for GetFileRefsReply {
|
||||
}
|
||||
}
|
||||
|
||||
/// Reply for GetPackedFileRefs instruction.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||
pub struct GetPackedFileRefsReply {
|
||||
pub packed_file_refs_manifest: PackedFileRefsManifest,
|
||||
pub success: bool,
|
||||
pub error: Option<InstructionError>,
|
||||
}
|
||||
|
||||
impl Display for GetPackedFileRefsReply {
|
||||
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
|
||||
write!(
|
||||
f,
|
||||
"GetPackedFileRefsReply(success={}, file_refs_count={}, error={:?})",
|
||||
self.success,
|
||||
self.packed_file_refs_manifest.file_refs.len(),
|
||||
self.error
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
/// Reply for GC instruction.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||
pub struct GcRegionsReply {
|
||||
@@ -827,8 +1009,12 @@ pub enum Instruction {
|
||||
FlushRegions(FlushRegions),
|
||||
/// Gets file references for regions.
|
||||
GetFileRefs(GetFileRefs),
|
||||
/// Gets file references using the packed manifest format.
|
||||
GetPackedFileRefs(GetPackedFileRefs),
|
||||
/// Triggers garbage collection for a region.
|
||||
GcRegions(GcRegions),
|
||||
/// Triggers garbage collection with a packed file-reference manifest.
|
||||
PackedGcRegions(PackedGcRegions),
|
||||
/// Temporary suspend serving reads or writes
|
||||
Suspend,
|
||||
/// Makes regions enter staging state.
|
||||
@@ -1111,6 +1297,7 @@ pub enum InstructionReply {
|
||||
DowngradeRegions(DowngradeRegionsReply),
|
||||
FlushRegions(FlushRegionReply),
|
||||
GetFileRefs(GetFileRefsReply),
|
||||
GetPackedFileRefs(GetPackedFileRefsReply),
|
||||
GcRegions(GcRegionsReply),
|
||||
EnterStagingRegions(EnterStagingRegionsReply),
|
||||
SyncRegions(SyncRegionsReply),
|
||||
@@ -1131,6 +1318,9 @@ impl Display for InstructionReply {
|
||||
}
|
||||
Self::FlushRegions(reply) => write!(f, "InstructionReply::FlushRegions({})", reply),
|
||||
Self::GetFileRefs(reply) => write!(f, "InstructionReply::GetFileRefs({})", reply),
|
||||
Self::GetPackedFileRefs(reply) => {
|
||||
write!(f, "InstructionReply::GetPackedFileRefs({})", reply)
|
||||
}
|
||||
Self::GcRegions(reply) => write!(f, "InstructionReply::GcRegion({})", reply),
|
||||
Self::EnterStagingRegions(reply) => {
|
||||
write!(
|
||||
@@ -1221,6 +1411,7 @@ impl InstructionReply {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::collections::HashSet;
|
||||
use std::error::Error as _;
|
||||
|
||||
use common_error::mock::MockError;
|
||||
use common_wal::options::WalOptions;
|
||||
@@ -1771,6 +1962,161 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_legacy_gc_file_refs_compatibility() {
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct LegacyGetFileRefs {
|
||||
query_regions: Vec<RegionId>,
|
||||
related_regions: HashMap<RegionId, HashSet<RegionId>>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct LegacyGetFileRefsReply {
|
||||
file_refs_manifest: FileRefsManifest,
|
||||
success: bool,
|
||||
error: Option<InstructionError>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
enum LegacyInstruction {
|
||||
GetFileRefs(LegacyGetFileRefs),
|
||||
}
|
||||
|
||||
let get_file_refs = Instruction::GetFileRefs(GetFileRefs {
|
||||
query_regions: vec![RegionId::new(7, 3)],
|
||||
related_regions: HashMap::new(),
|
||||
});
|
||||
let get_file_refs_json = serde_json::to_string(&get_file_refs).unwrap();
|
||||
let legacy_get_file_refs: LegacyInstruction =
|
||||
serde_json::from_str(&get_file_refs_json).unwrap();
|
||||
let LegacyInstruction::GetFileRefs(legacy_get_file_refs) = legacy_get_file_refs;
|
||||
assert_eq!(legacy_get_file_refs.query_regions.len(), 1);
|
||||
assert!(legacy_get_file_refs.related_regions.is_empty());
|
||||
|
||||
let reply = GetFileRefsReply {
|
||||
file_refs_manifest: FileRefsManifest::default(),
|
||||
success: true,
|
||||
error: None,
|
||||
};
|
||||
let reply_json = serde_json::to_string(&reply).unwrap();
|
||||
let legacy_reply: LegacyGetFileRefsReply = serde_json::from_str(&reply_json).unwrap();
|
||||
let current_reply: GetFileRefsReply = serde_json::from_str(&reply_json).unwrap();
|
||||
assert!(legacy_reply.success);
|
||||
assert!(legacy_reply.error.is_none());
|
||||
assert!(legacy_reply.file_refs_manifest.file_refs.is_empty());
|
||||
assert_eq!(current_reply, reply);
|
||||
|
||||
let packed = Instruction::PackedGcRegions(PackedGcRegions {
|
||||
regions: vec![],
|
||||
packed_file_refs_manifest: PackedFileRefsManifest::default(),
|
||||
full_file_listing: false,
|
||||
});
|
||||
let packed_json = serde_json::to_string(&packed).unwrap();
|
||||
assert!(serde_json::from_str::<LegacyInstruction>(&packed_json).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_packed_gc_regions_round_trip_is_distinct() {
|
||||
let instruction = Instruction::PackedGcRegions(PackedGcRegions {
|
||||
regions: vec![RegionId::new(7, 3)],
|
||||
packed_file_refs_manifest: PackedFileRefsManifest::default(),
|
||||
full_file_listing: true,
|
||||
});
|
||||
let serialized = serde_json::to_string(&instruction).unwrap();
|
||||
assert!(serialized.contains("PackedGcRegions"));
|
||||
assert_eq!(
|
||||
serde_json::from_str::<Instruction>(&serialized).unwrap(),
|
||||
instruction
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_packed_file_refs_manifest_round_trip_and_validation() {
|
||||
let region = RegionId::new(7, 3);
|
||||
let file_id = FileId::parse_str("00000000-0000-0000-0000-000000000001").unwrap();
|
||||
let mut manifest = FileRefsManifest::default();
|
||||
manifest.file_refs.insert(
|
||||
region,
|
||||
HashSet::from([
|
||||
FileRef::new(region, file_id, None),
|
||||
FileRef::new(region, file_id, Some(0)),
|
||||
FileRef::new(region, file_id, Some(u64::MAX)),
|
||||
]),
|
||||
);
|
||||
manifest.manifest_version.insert(region, 42);
|
||||
manifest
|
||||
.cross_region_refs
|
||||
.insert(region, HashSet::from([RegionId::new(7, 4)]));
|
||||
let packed = PackedFileRefsManifest::from_manifest(&manifest);
|
||||
assert_eq!(packed.clone().into_manifest().unwrap(), manifest);
|
||||
let mut missing_files = serde_json::to_value(&packed).unwrap();
|
||||
missing_files
|
||||
.get_mut("file_refs")
|
||||
.and_then(serde_json::Value::as_object_mut)
|
||||
.and_then(|file_refs| file_refs.values_mut().next())
|
||||
.and_then(serde_json::Value::as_object_mut)
|
||||
.unwrap()
|
||||
.remove("files");
|
||||
assert!(serde_json::from_value::<PackedFileRefsManifest>(missing_files).is_err());
|
||||
|
||||
let mut missing_indexed = serde_json::to_value(&packed).unwrap();
|
||||
missing_indexed
|
||||
.get_mut("file_refs")
|
||||
.and_then(serde_json::Value::as_object_mut)
|
||||
.and_then(|file_refs| file_refs.values_mut().next())
|
||||
.and_then(serde_json::Value::as_object_mut)
|
||||
.unwrap()
|
||||
.remove("indexed");
|
||||
assert!(serde_json::from_value::<PackedFileRefsManifest>(missing_indexed).is_err());
|
||||
|
||||
let mut malformed = packed.clone();
|
||||
malformed.file_refs.get_mut(®ion).unwrap().indexed = "!".to_string();
|
||||
let err = malformed.into_manifest().unwrap_err();
|
||||
assert!(matches!(
|
||||
&err,
|
||||
crate::error::Error::DecodePackedFileRefs { .. }
|
||||
));
|
||||
assert!(
|
||||
err.source()
|
||||
.is_some_and(|source| source.is::<base64::DecodeError>())
|
||||
);
|
||||
assert_eq!(err.status_code(), StatusCode::Unexpected);
|
||||
assert_eq!(err.retry_hint(), RetryHint::NonRetryable);
|
||||
|
||||
let mut malformed = packed;
|
||||
malformed.file_refs.get_mut(®ion).unwrap().indexed =
|
||||
base64::engine::general_purpose::STANDARD.encode([0; 1]);
|
||||
let err = malformed.into_manifest().unwrap_err();
|
||||
assert!(matches!(
|
||||
&err,
|
||||
crate::error::Error::InvalidPackedFileRefs { .. }
|
||||
));
|
||||
assert!(err.source().is_none());
|
||||
assert_eq!(err.status_code(), StatusCode::Unexpected);
|
||||
assert_eq!(err.retry_hint(), RetryHint::NonRetryable);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_packed_file_refs_is_smaller_than_legacy() {
|
||||
let region = RegionId::new(7, 3);
|
||||
let mut manifest = FileRefsManifest::default();
|
||||
manifest.file_refs.insert(
|
||||
region,
|
||||
(0..500)
|
||||
.map(|i| {
|
||||
let id =
|
||||
FileId::parse_str(&format!("00000000-0000-0000-0000-{i:012}")).unwrap();
|
||||
FileRef::new(region, id, None)
|
||||
})
|
||||
.collect(),
|
||||
);
|
||||
let legacy = serde_json::to_vec(&manifest).unwrap().len();
|
||||
let packed = serde_json::to_vec(&PackedFileRefsManifest::from_manifest(&manifest))
|
||||
.unwrap()
|
||||
.len();
|
||||
assert!(packed * 2 < legacy, "packed={packed}, legacy={legacy}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_serialize_get_file_refs_instruction_reply() {
|
||||
let mut manifest = FileRefsManifest::default();
|
||||
|
||||
@@ -270,18 +270,158 @@ pub fn datafusion_status_code<T: ErrorExt + 'static>(
|
||||
e: &DataFusionError,
|
||||
default_status: Option<StatusCode>,
|
||||
) -> StatusCode {
|
||||
match e {
|
||||
let mut error = e;
|
||||
loop {
|
||||
error = match error {
|
||||
DataFusionError::Shared(inner) => inner,
|
||||
DataFusionError::Context(_, inner) | DataFusionError::Diagnostic(_, inner) => inner,
|
||||
_ => break,
|
||||
};
|
||||
}
|
||||
|
||||
match error {
|
||||
DataFusionError::Internal(_) => StatusCode::Internal,
|
||||
DataFusionError::NotImplemented(_) => StatusCode::Unsupported,
|
||||
DataFusionError::Plan(_) => StatusCode::PlanQuery,
|
||||
DataFusionError::External(e) => {
|
||||
if let Some(ext) = (*e).downcast_ref::<T>() {
|
||||
DataFusionError::External(error) => {
|
||||
if let Some(ext) = (*error).downcast_ref::<T>() {
|
||||
ext.status_code()
|
||||
} else if let Some(ext) = (*error).downcast_ref::<BoxedError>() {
|
||||
ext.status_code()
|
||||
} else {
|
||||
default_status.unwrap_or(StatusCode::EngineExecuteQuery)
|
||||
}
|
||||
}
|
||||
DataFusionError::Diagnostic(_, e) => datafusion_status_code::<T>(e, default_status),
|
||||
_ => default_status.unwrap_or(StatusCode::EngineExecuteQuery),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::sync::Arc;
|
||||
|
||||
use common_error::ext::PlainError;
|
||||
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_datafusion_status_code_preserves_external_errors_through_wrappers() {
|
||||
let boxed_error = |status| {
|
||||
DataFusionError::External(Box::new(BoxedError::new(PlainError::new(
|
||||
"neutral error".to_string(),
|
||||
status,
|
||||
))))
|
||||
};
|
||||
|
||||
for status in [
|
||||
StatusCode::RequestOutdated,
|
||||
StatusCode::Unknown,
|
||||
StatusCode::Unsupported,
|
||||
] {
|
||||
let errors = [
|
||||
boxed_error(status),
|
||||
DataFusionError::Shared(Arc::new(boxed_error(status))),
|
||||
DataFusionError::Context("context".to_string(), Box::new(boxed_error(status))),
|
||||
DataFusionError::Diagnostic(
|
||||
Box::new(datafusion_common::Diagnostic::new_error("diagnostic", None)),
|
||||
Box::new(boxed_error(status)),
|
||||
),
|
||||
DataFusionError::Shared(Arc::new(DataFusionError::Context(
|
||||
"context".to_string(),
|
||||
Box::new(DataFusionError::Diagnostic(
|
||||
Box::new(datafusion_common::Diagnostic::new_error("diagnostic", None)),
|
||||
Box::new(boxed_error(status)),
|
||||
)),
|
||||
))),
|
||||
];
|
||||
|
||||
for error in errors {
|
||||
assert_eq!(datafusion_status_code::<Error>(&error, None), status);
|
||||
assert_eq!(
|
||||
datafusion_status_code::<Error>(&error, Some(StatusCode::PlanQuery)),
|
||||
status
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
let direct_error = DataFusionError::External(Box::new(Error::DynFilterPayloadTooLarge {
|
||||
payload_size_bytes: 2,
|
||||
max_payload_bytes: 1,
|
||||
location: Location::default(),
|
||||
}));
|
||||
assert_eq!(
|
||||
datafusion_status_code::<Error>(&direct_error, Some(StatusCode::Internal)),
|
||||
StatusCode::PlanQuery
|
||||
);
|
||||
|
||||
let boundary_error =
|
||||
DataFusionError::Shared(Arc::new(DataFusionError::External(Box::new(
|
||||
BoxedError::new(common_recordbatch::error::Error::PhysicalExpr {
|
||||
error: DataFusionError::NotImplemented("inner error".to_string()),
|
||||
location: Location::default(),
|
||||
}),
|
||||
))));
|
||||
assert_eq!(
|
||||
datafusion_status_code::<Error>(&boundary_error, Some(StatusCode::PlanQuery)),
|
||||
StatusCode::Internal
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_datafusion_status_code_uses_default_for_untyped_errors() {
|
||||
let wrap = |error| {
|
||||
DataFusionError::Shared(Arc::new(DataFusionError::Context(
|
||||
"context".to_string(),
|
||||
Box::new(DataFusionError::Diagnostic(
|
||||
Box::new(datafusion_common::Diagnostic::new_error("diagnostic", None)),
|
||||
Box::new(error),
|
||||
)),
|
||||
)))
|
||||
};
|
||||
let errors = || {
|
||||
[
|
||||
(
|
||||
DataFusionError::External(Box::new(std::io::Error::other("neutral io error"))),
|
||||
StatusCode::EngineExecuteQuery,
|
||||
StatusCode::Unknown,
|
||||
),
|
||||
(
|
||||
DataFusionError::Internal("neutral internal error".to_string()),
|
||||
StatusCode::Internal,
|
||||
StatusCode::Internal,
|
||||
),
|
||||
(
|
||||
DataFusionError::NotImplemented("neutral not implemented error".to_string()),
|
||||
StatusCode::Unsupported,
|
||||
StatusCode::Unsupported,
|
||||
),
|
||||
(
|
||||
DataFusionError::Plan("neutral plan error".to_string()),
|
||||
StatusCode::PlanQuery,
|
||||
StatusCode::PlanQuery,
|
||||
),
|
||||
(
|
||||
DataFusionError::External(Box::new(DataFusionError::Internal(
|
||||
"inner error".to_string(),
|
||||
))),
|
||||
StatusCode::EngineExecuteQuery,
|
||||
StatusCode::Unknown,
|
||||
),
|
||||
]
|
||||
};
|
||||
|
||||
for (error, none_expected, default_expected) in
|
||||
errors()
|
||||
.into_iter()
|
||||
.chain(errors().map(|(error, none_expected, default_expected)| {
|
||||
(wrap(error), none_expected, default_expected)
|
||||
}))
|
||||
{
|
||||
assert_eq!(datafusion_status_code::<Error>(&error, None), none_expected);
|
||||
assert_eq!(
|
||||
datafusion_status_code::<Error>(&error, Some(StatusCode::Unknown)),
|
||||
default_expected
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -137,9 +137,7 @@ fn portable_remote_dyn_filter_expr(
|
||||
bounds_only: bool,
|
||||
) -> DataFusionResult<Arc<dyn PhysicalExpr>> {
|
||||
expr.transform_up(|node| {
|
||||
if node.as_any().is::<HashTableLookupExpr>()
|
||||
|| (bounds_only && node.as_any().is::<InListExpr>())
|
||||
{
|
||||
if node.is::<HashTableLookupExpr>() || (bounds_only && node.is::<InListExpr>()) {
|
||||
Ok(Transformed::yes(lit(true)))
|
||||
} else {
|
||||
Ok(Transformed::no(node))
|
||||
@@ -183,7 +181,7 @@ fn validate_payload_size(
|
||||
|
||||
fn validate_supported_payload_expr(expr: &Arc<dyn PhysicalExpr>) -> DataFusionResult<()> {
|
||||
expr.apply(|node| {
|
||||
if node.as_any().is::<HashTableLookupExpr>() {
|
||||
if node.is::<HashTableLookupExpr>() {
|
||||
return Err(DataFusionError::Plan(
|
||||
"HashTableLookupExpr cannot be encoded into DynFilterPayload::Datafusion"
|
||||
.to_string(),
|
||||
@@ -207,7 +205,7 @@ fn validate_decoded_payload_expr(
|
||||
input_schema: &datafusion::arrow::datatypes::Schema,
|
||||
) -> DataFusionResult<()> {
|
||||
expr.apply(|node| {
|
||||
if let Some(column) = node.as_any().downcast_ref::<Column>() {
|
||||
if let Some(column) = node.downcast_ref::<Column>() {
|
||||
let Some(field) = input_schema.fields().get(column.index()) else {
|
||||
return Err(DataFusionError::Plan(format!(
|
||||
"Decoded Column '{}' references out-of-bounds index {} for input schema of size {}",
|
||||
@@ -391,8 +389,8 @@ mod tests {
|
||||
.decode_datafusion_expr(&TaskContext::default(), &schema, 1024)
|
||||
.unwrap();
|
||||
|
||||
let original = expr.as_any().downcast_ref::<Column>().unwrap();
|
||||
let decoded = decoded.as_any().downcast_ref::<Column>().unwrap();
|
||||
let original = expr.downcast_ref::<Column>().unwrap();
|
||||
let decoded = decoded.downcast_ref::<Column>().unwrap();
|
||||
|
||||
assert_eq!(decoded.name(), original.name());
|
||||
assert_eq!(decoded.index(), original.index());
|
||||
@@ -457,7 +455,7 @@ mod tests {
|
||||
)) as Arc<dyn PhysicalExpr>;
|
||||
let lookup = Arc::new(HashTableLookupExpr::new(
|
||||
vec![Arc::clone(&device_id)],
|
||||
SeededRandomState::with_seeds(0, 0, 0, 0),
|
||||
SeededRandomState::with_seed(0),
|
||||
Arc::new(Map::HashMap(Box::new(JoinHashMapU32::with_capacity(0)))),
|
||||
"hash_lookup".to_string(),
|
||||
)) as Arc<dyn PhysicalExpr>;
|
||||
@@ -539,10 +537,10 @@ mod tests {
|
||||
));
|
||||
}
|
||||
|
||||
fn contains_expr<T: 'static>(expr: &Arc<dyn PhysicalExpr>) -> bool {
|
||||
fn contains_expr<T: PhysicalExpr>(expr: &Arc<dyn PhysicalExpr>) -> bool {
|
||||
let mut found = false;
|
||||
expr.apply(|node| {
|
||||
if node.as_any().is::<T>() {
|
||||
if node.is::<T>() {
|
||||
found = true;
|
||||
Ok(TreeNodeRecursion::Stop)
|
||||
} else {
|
||||
|
||||
@@ -371,7 +371,7 @@ mod tests {
|
||||
let decoded = reg
|
||||
.decode_children(&TaskContext::default(), &schema, 1024)
|
||||
.unwrap();
|
||||
let decoded = decoded[0].as_any().downcast_ref::<Column>().unwrap();
|
||||
let decoded = decoded[0].downcast_ref::<Column>().unwrap();
|
||||
|
||||
assert_eq!(reg.filter_id, "filter-1");
|
||||
assert_eq!(decoded.name(), "host");
|
||||
|
||||
@@ -12,7 +12,6 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use std::any::Any;
|
||||
use std::fmt::{Debug, Formatter};
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
@@ -22,8 +21,11 @@ use datafusion::execution::SendableRecordBatchStream as DfSendableRecordBatchStr
|
||||
use datafusion::execution::context::TaskContext;
|
||||
use datafusion::physical_expr::{EquivalenceProperties, Partitioning, PhysicalSortExpr};
|
||||
use datafusion::physical_plan::execution_plan::{Boundedness, EmissionType};
|
||||
use datafusion::physical_plan::{DisplayAs, DisplayFormatType, ExecutionPlan, PlanProperties};
|
||||
use datafusion::physical_plan::{
|
||||
DisplayAs, DisplayFormatType, ExecutionPlan, PhysicalExpr, PlanProperties,
|
||||
};
|
||||
use datafusion_common::DataFusionError;
|
||||
use datafusion_common::tree_node::TreeNodeRecursion;
|
||||
use datatypes::arrow::datatypes::SchemaRef as ArrowSchemaRef;
|
||||
use datatypes::schema::SchemaRef;
|
||||
|
||||
@@ -99,10 +101,6 @@ impl DisplayAs for StreamScanAdapter {
|
||||
}
|
||||
|
||||
impl ExecutionPlan for StreamScanAdapter {
|
||||
fn as_any(&self) -> &dyn Any {
|
||||
self
|
||||
}
|
||||
|
||||
fn schema(&self) -> ArrowSchemaRef {
|
||||
self.arrow_schema.clone()
|
||||
}
|
||||
@@ -115,6 +113,13 @@ impl ExecutionPlan for StreamScanAdapter {
|
||||
vec![]
|
||||
}
|
||||
|
||||
fn apply_expressions(
|
||||
&self,
|
||||
_f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> datafusion_common::Result<TreeNodeRecursion>,
|
||||
) -> datafusion_common::Result<TreeNodeRecursion> {
|
||||
Ok(TreeNodeRecursion::Continue)
|
||||
}
|
||||
|
||||
// DataFusion will swap children unconditionally.
|
||||
// But since this node is leaf node, it's safe to just return self.
|
||||
fn with_new_children(
|
||||
|
||||
@@ -29,6 +29,7 @@ use datafusion::arrow::datatypes::SchemaRef as DfSchemaRef;
|
||||
use datafusion::error::Result as DfResult;
|
||||
use datafusion::execution::context::ExecutionProps;
|
||||
use datafusion::logical_expr::Expr;
|
||||
use datafusion::logical_expr::physical_planning_context::PhysicalPlanningContext;
|
||||
use datafusion::logical_expr::utils::conjunction;
|
||||
use datafusion::physical_expr::create_physical_expr;
|
||||
use datafusion::physical_plan::metrics::{BaselineMetrics, MetricValue};
|
||||
@@ -98,8 +99,13 @@ where
|
||||
.to_dfschema_ref()
|
||||
.context(error::PhysicalExprSnafu)?;
|
||||
|
||||
let filters = create_physical_expr(&expr, &df_schema, &ExecutionProps::new())
|
||||
.context(error::PhysicalExprSnafu)?;
|
||||
let filters = create_physical_expr(
|
||||
&expr,
|
||||
&df_schema,
|
||||
&ExecutionProps::new(),
|
||||
&PhysicalPlanningContext::default(),
|
||||
)
|
||||
.context(error::PhysicalExprSnafu)?;
|
||||
Some(filters)
|
||||
} else {
|
||||
None
|
||||
@@ -931,7 +937,6 @@ fn convert_map_to_json_binary(
|
||||
|
||||
#[cfg(test)]
|
||||
mod test {
|
||||
use std::any::Any;
|
||||
use std::time::Duration;
|
||||
|
||||
use common_error::ext::BoxedError;
|
||||
@@ -942,6 +947,7 @@ mod test {
|
||||
use datafusion::physical_plan::execution_plan::{Boundedness, EmissionType};
|
||||
use datafusion::physical_plan::metrics::{ExecutionPlanMetricsSet, MetricBuilder, MetricsSet};
|
||||
use datafusion::physical_plan::{DisplayAs, PlanProperties};
|
||||
use datafusion_common::tree_node::TreeNodeRecursion;
|
||||
use datatypes::arrow::array::{ArrayRef, MapArray, StringArray, StructArray};
|
||||
use datatypes::arrow::buffer::OffsetBuffer;
|
||||
use datatypes::arrow::datatypes::Field;
|
||||
@@ -1021,10 +1027,6 @@ mod test {
|
||||
REGION_SCAN_EXEC_NAME
|
||||
}
|
||||
|
||||
fn as_any(&self) -> &dyn Any {
|
||||
self
|
||||
}
|
||||
|
||||
fn properties(&self) -> &Arc<PlanProperties> {
|
||||
&self.properties
|
||||
}
|
||||
@@ -1033,6 +1035,15 @@ mod test {
|
||||
vec![]
|
||||
}
|
||||
|
||||
fn apply_expressions(
|
||||
&self,
|
||||
_f: &mut dyn FnMut(
|
||||
&Arc<dyn PhysicalExpr>,
|
||||
) -> datafusion_common::Result<TreeNodeRecursion>,
|
||||
) -> datafusion_common::Result<TreeNodeRecursion> {
|
||||
Ok(TreeNodeRecursion::Continue)
|
||||
}
|
||||
|
||||
fn with_new_children(
|
||||
self: Arc<Self>,
|
||||
_children: Vec<Arc<dyn ExecutionPlan>>,
|
||||
|
||||
@@ -205,7 +205,26 @@ impl ErrorExt for Error {
|
||||
| Error::PhysicalExpr { .. }
|
||||
| Error::RecordBatchSliceIndexOverflow { .. } => StatusCode::Internal,
|
||||
|
||||
Error::PollStream { .. } => StatusCode::EngineExecuteQuery,
|
||||
Error::PollStream { error, .. } => {
|
||||
let mut error = error;
|
||||
loop {
|
||||
error = match error {
|
||||
datafusion::error::DataFusionError::Shared(inner) => inner,
|
||||
datafusion::error::DataFusionError::Context(_, inner)
|
||||
| datafusion::error::DataFusionError::Diagnostic(_, inner) => inner,
|
||||
_ => break,
|
||||
};
|
||||
}
|
||||
|
||||
match error {
|
||||
datafusion::error::DataFusionError::External(source) => source
|
||||
.downcast_ref::<BoxedError>()
|
||||
.map_or(StatusCode::EngineExecuteQuery, |source| {
|
||||
source.status_code()
|
||||
}),
|
||||
_ => StatusCode::EngineExecuteQuery,
|
||||
}
|
||||
}
|
||||
|
||||
Error::ArrowCompute { .. } => StatusCode::IllegalState,
|
||||
|
||||
@@ -242,3 +261,106 @@ impl ErrorExt for Error {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::sync::Arc;
|
||||
|
||||
use common_error::ext::PlainError;
|
||||
use datafusion::error::DataFusionError;
|
||||
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn poll_stream_status_code_preserves_boxed_error_through_wrappers() {
|
||||
let boxed_error = |status| {
|
||||
DataFusionError::External(Box::new(BoxedError::new(PlainError::new(
|
||||
"neutral error".to_string(),
|
||||
status,
|
||||
))))
|
||||
};
|
||||
|
||||
for status in [
|
||||
StatusCode::RequestOutdated,
|
||||
StatusCode::Unknown,
|
||||
StatusCode::Unsupported,
|
||||
] {
|
||||
let errors = [
|
||||
boxed_error(status),
|
||||
DataFusionError::Shared(Arc::new(boxed_error(status))),
|
||||
DataFusionError::Context("context".to_string(), Box::new(boxed_error(status))),
|
||||
DataFusionError::Diagnostic(
|
||||
Box::new(datafusion::common::Diagnostic::new_error(
|
||||
"diagnostic",
|
||||
None,
|
||||
)),
|
||||
Box::new(boxed_error(status)),
|
||||
),
|
||||
DataFusionError::Shared(Arc::new(DataFusionError::Context(
|
||||
"context".to_string(),
|
||||
Box::new(DataFusionError::Diagnostic(
|
||||
Box::new(datafusion::common::Diagnostic::new_error(
|
||||
"diagnostic",
|
||||
None,
|
||||
)),
|
||||
Box::new(boxed_error(status)),
|
||||
)),
|
||||
))),
|
||||
];
|
||||
|
||||
for error in errors {
|
||||
let error = Error::PollStream {
|
||||
error,
|
||||
location: Location::default(),
|
||||
};
|
||||
assert_eq!(error.status_code(), status);
|
||||
}
|
||||
}
|
||||
|
||||
let error = Error::PollStream {
|
||||
error: DataFusionError::Shared(Arc::new(DataFusionError::External(Box::new(
|
||||
BoxedError::new(Error::PhysicalExpr {
|
||||
error: DataFusionError::NotImplemented("inner error".to_string()),
|
||||
location: Location::default(),
|
||||
}),
|
||||
)))),
|
||||
location: Location::default(),
|
||||
};
|
||||
assert_eq!(error.status_code(), StatusCode::Internal);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn poll_stream_status_code_defaults_for_other_datafusion_errors() {
|
||||
let wrap = |error| {
|
||||
DataFusionError::Shared(Arc::new(DataFusionError::Context(
|
||||
"context".to_string(),
|
||||
Box::new(DataFusionError::Diagnostic(
|
||||
Box::new(datafusion::common::Diagnostic::new_error(
|
||||
"diagnostic",
|
||||
None,
|
||||
)),
|
||||
Box::new(error),
|
||||
)),
|
||||
)))
|
||||
};
|
||||
let errors = || {
|
||||
[
|
||||
DataFusionError::External(Box::new(std::io::Error::other("neutral io error"))),
|
||||
DataFusionError::Internal("neutral internal error".to_string()),
|
||||
DataFusionError::NotImplemented("neutral not implemented error".to_string()),
|
||||
DataFusionError::Plan("neutral plan error".to_string()),
|
||||
DataFusionError::External(Box::new(DataFusionError::Internal(
|
||||
"inner error".to_string(),
|
||||
))),
|
||||
]
|
||||
};
|
||||
|
||||
for error in errors().into_iter().chain(errors().map(wrap)) {
|
||||
let error = Error::PollStream {
|
||||
error,
|
||||
location: Location::default(),
|
||||
};
|
||||
assert_eq!(error.status_code(), StatusCode::EngineExecuteQuery);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -616,6 +616,7 @@ mod test {
|
||||
use std::sync::Arc;
|
||||
|
||||
use datafusion::execution::context::ExecutionProps;
|
||||
use datafusion::logical_expr::physical_planning_context::PhysicalPlanningContext;
|
||||
use datafusion::logical_expr::{BinaryExpr, col, lit};
|
||||
use datafusion::physical_expr::create_physical_expr;
|
||||
use datafusion_common::{Column, DFSchema};
|
||||
@@ -737,7 +738,13 @@ mod test {
|
||||
]);
|
||||
let df_schema = DFSchema::try_from(schema.clone()).unwrap();
|
||||
let props = ExecutionProps::new();
|
||||
let physical_expr = create_physical_expr(&expr, &df_schema, &props).unwrap();
|
||||
let physical_expr = create_physical_expr(
|
||||
&expr,
|
||||
&df_schema,
|
||||
&props,
|
||||
&PhysicalPlanningContext::default(),
|
||||
)
|
||||
.unwrap();
|
||||
let batch = RecordBatch::try_new(
|
||||
Arc::new(schema),
|
||||
vec![
|
||||
@@ -783,7 +790,13 @@ mod test {
|
||||
let schema = Schema::new(vec![Field::new("col", DataType::Utf8, false)]);
|
||||
let df_schema = DFSchema::try_from(schema.clone()).unwrap();
|
||||
let props = ExecutionProps::new();
|
||||
let physical_expr = create_physical_expr(&col_or_expr, &df_schema, &props).unwrap();
|
||||
let physical_expr = create_physical_expr(
|
||||
&col_or_expr,
|
||||
&df_schema,
|
||||
&props,
|
||||
&PhysicalPlanningContext::default(),
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
// Create test data
|
||||
let col_data = Arc::new(datatypes::arrow::array::StringArray::from(vec![
|
||||
|
||||
@@ -44,7 +44,9 @@ pub use datatypes::arrow::record_batch::RecordBatch as DfRecordBatch;
|
||||
use datatypes::arrow::util::display::{
|
||||
ArrayFormatter, ArrayFormatterFactory, DisplayIndex, FormatOptions, FormatResult,
|
||||
};
|
||||
use datatypes::arrow::util::pretty::pretty_format_batches_with_options;
|
||||
use datatypes::arrow::util::pretty::{
|
||||
pretty_format_batches_with_options, pretty_format_batches_with_schema,
|
||||
};
|
||||
use datatypes::extension::json::is_any_json_extension_type;
|
||||
use datatypes::prelude::{ConcreteDataType, DataType, VectorRef};
|
||||
use datatypes::schema::{ColumnSchema, Schema, SchemaRef};
|
||||
@@ -396,12 +398,19 @@ impl RecordBatches {
|
||||
.iter()
|
||||
.map(|x| x.df_record_batch().clone())
|
||||
.collect::<Vec<_>>();
|
||||
let options =
|
||||
FormatOptions::default().with_formatter_factory(Some(&BinaryFormatterFactory));
|
||||
let result =
|
||||
pretty_format_batches_with_options(df_batches, &options).context(error::FormatSnafu)?;
|
||||
let result: String = if df_batches.is_empty() {
|
||||
pretty_format_batches_with_schema(self.schema.arrow_schema().clone(), df_batches)
|
||||
.context(error::FormatSnafu)?
|
||||
.to_string()
|
||||
} else {
|
||||
let options =
|
||||
FormatOptions::default().with_formatter_factory(Some(&BinaryFormatterFactory));
|
||||
pretty_format_batches_with_options(df_batches, &options)
|
||||
.context(error::FormatSnafu)?
|
||||
.to_string()
|
||||
};
|
||||
|
||||
Ok(result.to_string())
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
pub fn try_new(schema: SchemaRef, batches: Vec<RecordBatch>) -> Result<Self> {
|
||||
@@ -1082,6 +1091,35 @@ mod tests {
|
||||
assert_eq!(r.take(), expected);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_recordbatches_pretty_print_empty_batches_preserves_schema() {
|
||||
let schema = Arc::new(Schema::new(vec![
|
||||
ColumnSchema::new("unit", ConcreteDataType::string_datatype(), false),
|
||||
ColumnSchema::new(
|
||||
"ts",
|
||||
ConcreteDataType::timestamp_millisecond_datatype(),
|
||||
false,
|
||||
),
|
||||
ColumnSchema::new(
|
||||
"lhs.degrees(val) + rhs.radians(val)",
|
||||
ConcreteDataType::float64_datatype(),
|
||||
false,
|
||||
),
|
||||
]));
|
||||
let batches =
|
||||
RecordBatches::try_collect(Box::pin(EmptyRecordBatchStream::new(schema.clone())))
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(schema, batches.schema());
|
||||
let expected = "\
|
||||
+------+----+-------------------------------------+
|
||||
| unit | ts | lhs.degrees(val) + rhs.radians(val) |
|
||||
+------+----+-------------------------------------+
|
||||
+------+----+-------------------------------------+";
|
||||
assert_eq!(expected, batches.pretty_print().unwrap());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_recordbatches_try_new() {
|
||||
let column_a = ColumnSchema::new("a", ConcreteDataType::int32_datatype(), false);
|
||||
|
||||
@@ -18,6 +18,7 @@ use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::time::Duration;
|
||||
|
||||
use api::v1::meta::heartbeat_request::NodeWorkloads;
|
||||
use api::v1::meta::mailbox_message::Payload;
|
||||
use api::v1::meta::{DatanodeWorkloads, HeartbeatRequest, NodeInfo, Peer, RegionRole, RegionStat};
|
||||
use common_base::Plugins;
|
||||
use common_meta::cache_invalidator::CacheInvalidatorRef;
|
||||
@@ -146,7 +147,15 @@ impl HeartbeatTask {
|
||||
None
|
||||
}) {
|
||||
if let Some(msg) = res.mailbox_message.as_ref() {
|
||||
info!("Received mailbox message: {msg:?}, meta_client id: {client_id:?}");
|
||||
let payload_len = msg
|
||||
.payload
|
||||
.as_ref()
|
||||
.map(|Payload::Json(payload)| payload.len())
|
||||
.unwrap_or_default();
|
||||
info!(
|
||||
message_id = msg.id,
|
||||
payload_len, client_id, "Received mailbox message"
|
||||
);
|
||||
}
|
||||
if let Some(lease) = res.region_lease.as_ref() {
|
||||
metrics::LAST_RECEIVED_HEARTBEAT_ELAPSED
|
||||
@@ -193,7 +202,8 @@ impl HeartbeatTask {
|
||||
ctx: HeartbeatResponseHandlerContext,
|
||||
handler_executor: HeartbeatResponseHandlerExecutorRef,
|
||||
) -> Result<()> {
|
||||
trace!("Heartbeat response: {:?}", ctx.response);
|
||||
let mailbox_message_id = ctx.response.mailbox_message.as_ref().map(|msg| msg.id);
|
||||
trace!(?mailbox_message_id, "Handling heartbeat response");
|
||||
handler_executor
|
||||
.handle(ctx)
|
||||
.await
|
||||
@@ -379,7 +389,8 @@ impl HeartbeatTask {
|
||||
.set(last_sent.elapsed().as_millis() as i64);
|
||||
// Resets the timer.
|
||||
last_sent = Instant::now();
|
||||
debug!("Sending heartbeat request: {:?}", req);
|
||||
let mailbox_message_id = req.mailbox_message.as_ref().map(|msg| msg.id);
|
||||
debug!(?mailbox_message_id, "Sending heartbeat request");
|
||||
if let Err(e) = tx.send(req).await {
|
||||
error!(e; "Failed to send heartbeat to metasrv");
|
||||
match Self::create_streams(
|
||||
|
||||
@@ -41,9 +41,9 @@ use crate::heartbeat::handler::apply_staging_manifest::ApplyStagingManifestsHand
|
||||
use crate::heartbeat::handler::close_region::CloseRegionsHandler;
|
||||
use crate::heartbeat::handler::downgrade_region::DowngradeRegionsHandler;
|
||||
use crate::heartbeat::handler::enter_staging::EnterStagingRegionsHandler;
|
||||
use crate::heartbeat::handler::file_ref::GetFileRefsHandler;
|
||||
use crate::heartbeat::handler::file_ref::{GetFileRefsHandler, GetPackedFileRefsHandler};
|
||||
use crate::heartbeat::handler::flush_region::FlushRegionsHandler;
|
||||
use crate::heartbeat::handler::gc_worker::GcRegionsHandler;
|
||||
use crate::heartbeat::handler::gc_worker::{GcRegionsHandler, PackedGcRegionsHandler};
|
||||
use crate::heartbeat::handler::open_region::OpenRegionsHandler;
|
||||
use crate::heartbeat::handler::remap_manifest::RemapManifestHandler;
|
||||
use crate::heartbeat::handler::sync_region::SyncRegionHandler;
|
||||
@@ -135,7 +135,11 @@ impl RegionHeartbeatResponseHandler {
|
||||
.into(),
|
||||
))),
|
||||
Instruction::GetFileRefs(_) => Ok(Some(Box::new(GetFileRefsHandler.into()))),
|
||||
Instruction::GetPackedFileRefs(_) => {
|
||||
Ok(Some(Box::new(GetPackedFileRefsHandler.into())))
|
||||
}
|
||||
Instruction::GcRegions(_) => Ok(Some(Box::new(GcRegionsHandler.into()))),
|
||||
Instruction::PackedGcRegions(_) => Ok(Some(Box::new(PackedGcRegionsHandler.into()))),
|
||||
Instruction::InvalidateCaches(_) => InvalidHeartbeatResponseSnafu.fail(),
|
||||
Instruction::Suspend => Ok(None),
|
||||
Instruction::EnterStagingRegions(_) => {
|
||||
@@ -159,7 +163,9 @@ pub enum InstructionHandlers {
|
||||
DowngradeRegions(DowngradeRegionsHandler),
|
||||
UpgradeRegions(UpgradeRegionsHandler),
|
||||
GetFileRefs(GetFileRefsHandler),
|
||||
GetPackedFileRefs(GetPackedFileRefsHandler),
|
||||
GcRegions(GcRegionsHandler),
|
||||
PackedGcRegions(PackedGcRegionsHandler),
|
||||
EnterStagingRegions(EnterStagingRegionsHandler),
|
||||
SyncRegions(SyncRegionHandler),
|
||||
RemapManifest(RemapManifestHandler),
|
||||
@@ -192,7 +198,9 @@ impl_from_handler!(
|
||||
DowngradeRegionsHandler => DowngradeRegions,
|
||||
UpgradeRegionsHandler => UpgradeRegions,
|
||||
GetFileRefsHandler => GetFileRefs,
|
||||
GetPackedFileRefsHandler => GetPackedFileRefs,
|
||||
GcRegionsHandler => GcRegions,
|
||||
PackedGcRegionsHandler => PackedGcRegions,
|
||||
EnterStagingRegionsHandler => EnterStagingRegions,
|
||||
SyncRegionHandler => SyncRegions,
|
||||
RemapManifestHandler => RemapManifest,
|
||||
@@ -240,7 +248,9 @@ dispatch_instr!(
|
||||
DowngradeRegions => DowngradeRegions,
|
||||
UpgradeRegions => UpgradeRegions,
|
||||
GetFileRefs => GetFileRefs,
|
||||
GetPackedFileRefs => GetPackedFileRefs,
|
||||
GcRegions => GcRegions,
|
||||
PackedGcRegions => PackedGcRegions,
|
||||
EnterStagingRegions => EnterStagingRegions,
|
||||
SyncRegions => SyncRegions,
|
||||
RemapManifest => RemapManifest,
|
||||
|
||||
@@ -13,7 +13,10 @@
|
||||
// limitations under the License.
|
||||
|
||||
use common_error::ext::ErrorExt;
|
||||
use common_meta::instruction::{GetFileRefs, GetFileRefsReply, InstructionError, InstructionReply};
|
||||
use common_meta::instruction::{
|
||||
GetFileRefs, GetFileRefsReply, GetPackedFileRefs, GetPackedFileRefsReply, InstructionError,
|
||||
InstructionReply, PackedFileRefsManifest,
|
||||
};
|
||||
use store_api::storage::FileRefsManifest;
|
||||
|
||||
use crate::heartbeat::handler::{HandlerContext, InstructionHandler};
|
||||
@@ -65,3 +68,54 @@ impl InstructionHandler for GetFileRefsHandler {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub struct GetPackedFileRefsHandler;
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl InstructionHandler for GetPackedFileRefsHandler {
|
||||
type Instruction = GetPackedFileRefs;
|
||||
|
||||
async fn handle(
|
||||
&self,
|
||||
ctx: &HandlerContext,
|
||||
get_file_refs: Self::Instruction,
|
||||
) -> Option<InstructionReply> {
|
||||
let region_server = &ctx.region_server;
|
||||
let Some(mito_engine) = region_server.mito_engine() else {
|
||||
return Some(InstructionReply::GetPackedFileRefs(
|
||||
GetPackedFileRefsReply {
|
||||
packed_file_refs_manifest: PackedFileRefsManifest::default(),
|
||||
success: false,
|
||||
error: Some(InstructionError::legacy_internal_retryable(
|
||||
"MitoEngine not found",
|
||||
)),
|
||||
},
|
||||
));
|
||||
};
|
||||
match mito_engine
|
||||
.get_snapshot_of_file_refs(get_file_refs.query_regions, get_file_refs.related_regions)
|
||||
.await
|
||||
{
|
||||
Ok(all_file_refs) => Some(InstructionReply::GetPackedFileRefs(
|
||||
GetPackedFileRefsReply {
|
||||
packed_file_refs_manifest: PackedFileRefsManifest::from_manifest(
|
||||
&all_file_refs,
|
||||
),
|
||||
success: true,
|
||||
error: None,
|
||||
},
|
||||
)),
|
||||
Err(e) => Some(InstructionReply::GetPackedFileRefs(
|
||||
GetPackedFileRefsReply {
|
||||
packed_file_refs_manifest: PackedFileRefsManifest::default(),
|
||||
success: false,
|
||||
error: Some(InstructionError::new(
|
||||
e.status_code(),
|
||||
format!("Failed to get file refs: {}", e.output_msg()),
|
||||
e.retry_hint(),
|
||||
)),
|
||||
},
|
||||
)),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user