feat: add FlatConvertFormat to convert record batches in old format to the flat format (#6786)

* feat: add convert format to FlatReadFormat

Signed-off-by: evenyag <realevenyag@gmail.com>

* test: test convert format

Signed-off-by: evenyag <realevenyag@gmail.com>

* fix: only convert string pks to dictionary

Signed-off-by: evenyag <realevenyag@gmail.com>

---------

Signed-off-by: evenyag <realevenyag@gmail.com>
This commit is contained in:
Yingwen
2025-08-25 14:47:06 +08:00
committed by GitHub
parent 83a65a81c0
commit d5575d3fa4
5 changed files with 658 additions and 54 deletions

View File

@@ -93,6 +93,8 @@ pub const COLUMN_ID_ENCODE_SIZE: usize = 4;
impl SparsePrimaryKeyCodec {
/// Creates a new [`SparsePrimaryKeyCodec`] instance.
///
/// The `region_metadata` should be the metadata of the logical region.
pub fn new(region_metadata: &RegionMetadataRef) -> Self {
Self {
inner: Arc::new(SparsePrimaryKeyCodecInner {
@@ -123,6 +125,7 @@ impl SparsePrimaryKeyCodec {
}
}
/// Creates a new [`SparsePrimaryKeyCodec`] instance with additional label `fields`.
pub fn with_fields(fields: Vec<(ColumnId, SortField)>) -> Self {
Self {
inner: Arc::new(SparsePrimaryKeyCodecInner {