mirror of
https://github.com/GreptimeTeam/greptimedb.git
synced 2026-01-05 21:02:58 +00:00
4e615e8906
* feat/bulk-wal: ### Refactor: Simplify Data Handling in LogStore Implementations - **`kafka/log_store.rs`, `raft_engine/log_store.rs`, `wal.rs`, `raw_entry_reader.rs`, `logstore.rs`:** - Refactored `entry` and `build_entry` functions to accept `Vec<u8>` directly instead of `&mut Vec<u8>`. - Removed usage of `std::mem::take` for data handling, simplifying the code and improving readability. - Updated test cases to align with the new function signatures. * feat/bulk-wal: ### Add Support for Bulk WAL Entries and Flight Data Encoding - **Add `raw_data` field to `BulkPart` and related structs**: Updated `BulkPart` and related structures in `src/mito2/src/memtable/bulk/part.rs`, `src/mito2/src/memtable/simple_bulk_memtable.rs`, `src/mito2/src/memtable/time_partition.rs`, `src/mito2/src/region_write_ctx.rs`, `src/mito2/src/worker/handle_bulk_insert.rs`, and `src/store-api/src/region_request.rs` to include a new `raw_data` field for handling Arrow IPC data. - **Implement Flight Data Encoding**: Added a new module `flight` in `src/common/test-util/src/flight.rs` to encode record batches to Flight data format. - **Update `greptime-proto` dependency**: Changed the revision of the `greptime-proto` dependency in `Cargo.lock` and `Cargo.toml`. - **Enhance WAL Writer and Tests**: Modified `src/mito2/src/wal.rs` and related test files to support bulk WAL entries and added tests for encoding and handling bulk data. * feat/bulk-wal: - **Update `greptime-proto` Dependency**: Updated the `greptime-proto` dependency to a new revision in `Cargo.lock` and `Cargo.toml`. - **Add `common-grpc` Dependency**: Added `common-grpc` as a dependency in `Cargo.lock` and `src/mito2/Cargo.toml`. - **Refactor `BulkPart` Structure**: Removed `num_rows` field and added `num_rows()` method in `src/mito2/src/memtable/bulk/part.rs`. Updated related usages in `src/mito2/src/memtable/simple_bulk_memtable.rs`, `src/mito2/src/memtable/time_partition.rs`, `src/mito2/src/memtable/time_series.rs`, `src/mito2/src/region_write_ctx.rs`, and `src/mito2/src/worker/handle_bulk_insert.rs`. - **Implement `TryFrom` and `From` for `BulkWalEntry`**: Added implementations for converting between `BulkPart` and `BulkWalEntry` in `src/mito2/src/memtable/bulk/part.rs`. - **Handle Bulk Entries in Region Opener**: Added logic to process bulk entries in `src/mito2/src/region/opener.rs`. - **Fix `BulkInsertRequest` Handling**: Corrected `region_id` handling in `src/operator/src/bulk_insert.rs` and `src/store-api/src/region_request.rs`. - **Add Error Variant for `ConvertBulkWalEntry`**: Added a new error variant in `src/mito2/src/error.rs` for handling bulk WAL entry conversion errors. * fix: ci * feat/bulk-wal: Add bulk write operation in `opener.rs` - Enhanced the region write context by adding a call to `write_bulk()` after `write_memtable()` in `opener.rs`. - This change aims to improve the efficiency of writing operations by enabling bulk writes. * feat/bulk-wal: Enhance error handling and metrics in `bulk_insert.rs` - Updated `Inserter` to improve error handling by capturing the result of `datanode.handle(request)` and incrementing the `DIST_INGEST_ROW_COUNT` metric with the number of affected rows. * feat/bulk-wal: ### Remove Encode Error Handling for WAL Entries - **`error.rs`**: Removed the `EncodeWal` error variant and its associated handling. - **`wal.rs`**: Eliminated the `entry_encode_buf` buffer and its usage for encoding WAL entries. Replaced with direct encoding to a vector using `encode_to_vec()`.
106 lines
2.7 KiB
TOML
106 lines
2.7 KiB
TOML
[package]
|
|
name = "mito2"
|
|
version.workspace = true
|
|
edition.workspace = true
|
|
license.workspace = true
|
|
|
|
[features]
|
|
default = []
|
|
test = ["common-test-util", "rstest", "rstest_reuse", "rskafka"]
|
|
|
|
[lints]
|
|
workspace = true
|
|
|
|
[dependencies]
|
|
api.workspace = true
|
|
aquamarine.workspace = true
|
|
async-channel = "1.9"
|
|
async-stream.workspace = true
|
|
async-trait.workspace = true
|
|
bytemuck.workspace = true
|
|
bytes.workspace = true
|
|
common-base.workspace = true
|
|
common-config.workspace = true
|
|
common-datasource.workspace = true
|
|
common-decimal.workspace = true
|
|
common-error.workspace = true
|
|
common-grpc.workspace = true
|
|
common-macro.workspace = true
|
|
common-meta.workspace = true
|
|
common-query.workspace = true
|
|
common-recordbatch.workspace = true
|
|
common-runtime.workspace = true
|
|
common-telemetry.workspace = true
|
|
common-test-util = { workspace = true, optional = true }
|
|
common-time.workspace = true
|
|
common-wal.workspace = true
|
|
crc32fast = "1"
|
|
crossbeam-utils.workspace = true
|
|
datafusion.workspace = true
|
|
datafusion-common.workspace = true
|
|
datafusion-expr.workspace = true
|
|
datatypes.workspace = true
|
|
dotenv.workspace = true
|
|
futures.workspace = true
|
|
humantime-serde.workspace = true
|
|
index.workspace = true
|
|
itertools.workspace = true
|
|
lazy_static = "1.4"
|
|
log-store = { workspace = true }
|
|
memcomparable = "0.2"
|
|
moka = { workspace = true, features = ["sync", "future"] }
|
|
object-store.workspace = true
|
|
parquet = { workspace = true, features = ["async"] }
|
|
paste.workspace = true
|
|
pin-project.workspace = true
|
|
prometheus.workspace = true
|
|
prost.workspace = true
|
|
puffin.workspace = true
|
|
rand.workspace = true
|
|
regex = "1.5"
|
|
rskafka = { workspace = true, optional = true }
|
|
rstest = { workspace = true, optional = true }
|
|
rstest_reuse = { workspace = true, optional = true }
|
|
scopeguard = "1.2"
|
|
serde.workspace = true
|
|
serde_json.workspace = true
|
|
serde_with.workspace = true
|
|
smallvec.workspace = true
|
|
snafu.workspace = true
|
|
store-api.workspace = true
|
|
strum.workspace = true
|
|
table.workspace = true
|
|
tokio.workspace = true
|
|
tokio-stream.workspace = true
|
|
tokio-util.workspace = true
|
|
uuid.workspace = true
|
|
|
|
[dev-dependencies]
|
|
common-function.workspace = true
|
|
common-meta = { workspace = true, features = ["testing"] }
|
|
common-test-util.workspace = true
|
|
criterion = "0.4"
|
|
dotenv.workspace = true
|
|
log-store.workspace = true
|
|
object-store = { workspace = true, features = ["services-memory"] }
|
|
rskafka.workspace = true
|
|
rstest.workspace = true
|
|
rstest_reuse.workspace = true
|
|
session.workspace = true
|
|
toml.workspace = true
|
|
|
|
[[bench]]
|
|
name = "memtable_bench"
|
|
harness = false
|
|
required-features = ["test"]
|
|
|
|
[[bench]]
|
|
name = "bench_filter_time_partition"
|
|
harness = false
|
|
required-features = ["test"]
|
|
|
|
[[bench]]
|
|
name = "bench_compaction_picker"
|
|
harness = false
|
|
required-features = ["test"]
|