rust/greptimedb
1
0
Fork 0
mirror of https://github.com/GreptimeTeam/greptimedb.git synced 2026-01-06 05:12:54 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(mito2): Define basic structs for MitoEngine (#1928)

Browse Source 
* chore: metadata wip

* docs(mito2): Add struct relationships

* feat(mito2): define basic structs

* feat: add version and refactor other metadata

* docs: remove generics param from MitoEngine

* chore: Update src/mito2/Cargo.toml

Co-authored-by: Lei, HUANG <6406592+v0y4g3r@users.noreply.github.com>

* chore: Apply suggestions from code review

Co-authored-by: Ruihang Xia <waynestxia@gmail.com>

---------

Co-authored-by: Lei, HUANG <6406592+v0y4g3r@users.noreply.github.com>
Co-authored-by: Ruihang Xia <waynestxia@gmail.com>
This commit is contained in:
Yingwen
2023-07-10 21:25:33 +09:00
committed by GitHub
parent ed756288b3
commit 65f5349767
10 changed files with 420 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

40
Cargo.lock generated
View File

@@ -228,6 +228,20 @@ version = "0.1.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b3f9eb837c6a783fbf002e3e5cc7925a3aa6893d6d42f9169517528983777590"
[[package]]
name = "aquamarine"
version = "0.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "df752953c49ce90719c7bf1fc587bc8227aed04732ea0c0f85e5397d7fdbd1a1"
dependencies = [
"include_dir",
"itertools",
"proc-macro-error",
"proc-macro2",
"quote",
"syn 1.0.109",
]
[[package]]
name = "arc-swap"
version = "1.6.0"
@@ -4428,6 +4442,25 @@ dependencies = [
"hashbrown 0.12.3",
]
[[package]]
name = "include_dir"
version = "0.7.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "18762faeff7122e89e0857b02f7ce6fcc0d101d5e9ad2ad7846cc01d61b7f19e"
dependencies = [
"include_dir_macros",
]
[[package]]
name = "include_dir_macros"
version = "0.7.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b139284b5cf57ecfa712bcc66950bb635b31aff41c188e8a4cfc758eca374a3f"
dependencies = [
"proc-macro2",
"quote",
]
[[package]]
name = "indexmap"
version = "1.9.3"
@@ -5397,6 +5430,13 @@ dependencies = [
[[package]]
name = "mito2"
version = "0.3.2"
dependencies = [
"aquamarine",
"common-error",
"object-store",
"snafu",
"store-api",
]
[[package]]
name = "moka"

5
src/mito2/Cargo.toml
View File

@@ -5,3 +5,8 @@ edition.workspace = true
license.workspace = true
[dependencies]
aquamarine = "0.3"
common-error = { path = "../common/error" }
object-store = { path = "../object-store" }
snafu.workspace = true
store-api = { path = "../store-api" }

19
src/mito2/src/config.rs Normal file
View File

@@ -0,0 +1,19 @@
// Copyright 2023 Greptime Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Configurations.
/// Configuration for [MitoEngine](crate::engine::MitoEngine).
#[derive(Debug)]
pub struct MitoConfig {}

35
src/mito2/src/engine.rs
View File

@@ -12,6 +12,39 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use std::sync::Arc;
use object_store::ObjectStore;
use crate::config::MitoConfig;
use crate::worker::WorkerGroup;
/// Region engine implementation for timeseries data.
#[derive(Clone)]
pub struct MitoEngine {}
pub struct MitoEngine {
inner: Arc<EngineInner>,
}
impl MitoEngine {
/// Returns a new [MitoEngine] with specific `config`, `log_store` and `object_store`.
pub fn new<S>(config: MitoConfig, log_store: S, object_store: ObjectStore) -> MitoEngine {
MitoEngine {
inner: Arc::new(EngineInner::new(config, log_store, object_store)),
}
}
}
/// Inner struct of [MitoEngine].
struct EngineInner {
/// Region workers group.
workers: WorkerGroup,
}
impl EngineInner {
/// Returns a new [EngineInner] with specific `config`, `log_store` and `object_store`.
fn new<S>(_config: MitoConfig, _log_store: S, _object_store: ObjectStore) -> EngineInner {
EngineInner {
workers: WorkerGroup::default(),
}
}
}

21
src/mito2/src/error.rs Normal file
View File

@@ -0,0 +1,21 @@
// Copyright 2023 Greptime Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Error of mito engine.
use snafu::Snafu;
#[derive(Debug, Snafu)]
#[snafu(visibility(pub))]
pub enum Error {}

174
src/mito2/src/lib.rs
View File

@@ -12,4 +12,178 @@
// See the License for the specific language governing permissions and
// limitations under the License.
//! # Mito
//!
//! Mito is the a region engine to store timeseries data.
// TODO(yingwen): Remove all `allow(dead_code)` after finish refactoring mito.
pub mod config;
#[allow(dead_code)]
pub mod engine;
pub mod error;
#[allow(dead_code)]
mod region;
#[allow(dead_code)]
mod worker;
#[cfg_attr(doc, aquamarine::aquamarine)]
/// # Mito developer document
///
/// ## Engine
///
/// Engine hierarchy:
///
/// ```mermaid
/// classDiagram
/// class MitoEngine {
/// -WorkerGroup workers
/// }
/// class MitoRegion {
/// +VersionControlRef version_control
/// -RegionId region_id
/// -String manifest_dir
/// -AtomicI64 last_flush_millis
/// +region_id() RegionId
/// +scan() ChunkReaderImpl
/// }
/// class RegionMap {
/// -HashMap&lt;RegionId, MitoRegionRef&gt; regions
/// }
/// class ChunkReaderImpl
///
/// class WorkerGroup {
/// -Vec~RegionWorker~ workers
/// }
/// class RegionWorker {
/// -RegionMap regions
/// -Sender sender
/// -JoinHandle handle
/// }
/// class RegionWorkerThread~LogStore~ {
/// -RegionMap regions
/// -Receiver receiver
/// -Wal~LogStore~ wal
/// -ObjectStore object_store
/// -MemtableBuilderRef memtable_builder
/// -FlushSchedulerRef~LogStore~ flush_scheduler
/// -FlushStrategy flush_strategy
/// -CompactionSchedulerRef~LogStore~ compaction_scheduler
/// -FilePurgerRef file_purger
/// }
/// class Wal~LogStore~ {
/// -LogStore log_store
/// }
/// class MitoConfig
///
/// MitoEngine o-- MitoConfig
/// MitoEngine o-- MitoRegion
/// MitoEngine o-- WorkerGroup
/// MitoRegion o-- VersionControl
/// MitoRegion -- ChunkReaderImpl
/// WorkerGroup o-- RegionWorker
/// RegionWorker o-- RegionMap
/// RegionWorker -- RegionWorkerThread~LogStore~
/// RegionWorkerThread~LogStore~ o-- RegionMap
/// RegionWorkerThread~LogStore~ o-- Wal~LogStore~
/// ```
///
/// ## Metadata
///
/// Metadata hierarchy:
///
/// ```mermaid
/// classDiagram
/// class VersionControl {
/// -CowCell~Version~ version
/// -AtomicU64 committed_sequence
/// }
/// class Version {
/// -RegionMetadataRef metadata
/// -MemtableVersionRef memtables
/// -LevelMetasRef ssts
/// -SequenceNumber flushed_sequence
/// -ManifestVersion manifest_version
/// }
/// class MemtableVersion {
/// -MemtableRef mutable
/// -Vec~MemtableRef~ immutables
/// +mutable_memtable() MemtableRef
/// +immutable_memtables() &[MemtableRef]
/// +freeze_mutable(MemtableRef new_mutable) MemtableVersion
/// }
/// class LevelMetas {
/// -LevelMetaVec levels
/// -AccessLayerRef sst_layer
/// -FilePurgerRef file_purger
/// -Option~i64~ compaction_time_window
/// }
/// class LevelMeta {
/// -Level level
/// -HashMap&lt;FileId, FileHandle&gt; files
/// }
/// class FileHandle {
/// -FileMeta meta
/// -bool compacting
/// -AtomicBool deleted
/// -AccessLayerRef sst_layer
/// -FilePurgerRef file_purger
/// }
/// class FileMeta {
/// +RegionId region_id
/// +FileId file_id
/// +Option&lt;Timestamp, Timestamp&gt; time_range
/// +Level level
/// +u64 file_size
/// }
///
/// VersionControl o-- Version
/// Version o-- RegionMetadata
/// Version o-- MemtableVersion
/// Version o-- LevelMetas
/// LevelMetas o-- LevelMeta
/// LevelMeta o-- FileHandle
/// FileHandle o-- FileMeta
///
/// class RegionMetadata {
/// +RegionId region_id
/// +VersionNumber version
/// +SchemaRef table_schema
/// +Vec~usize~ primary_key_indices
/// +Vec~usize~ value_indices
/// +ColumnId next_column_id
/// +TableOptions region_options
/// +DateTime~Utc~ created_on
/// +RegionSchemaRef region_schema
/// }
/// class RegionSchema {
/// -SchemaRef user_schema
/// -StoreSchemaRef store_schema
/// -ColumnsMetadataRef columns
/// }
/// class Schema
/// class StoreSchema {
/// -Vec~ColumnMetadata~ columns
/// -SchemaRef schema
/// -usize row_key_end
/// -usize user_column_end
/// }
/// class ColumnsMetadata {
/// -Vec~ColumnMetadata~ columns
/// -HashMap&lt;String, usize&gt; name_to_col_index
/// -usize row_key_end
/// -usize timestamp_key_index
/// -usize user_column_end
/// }
/// class ColumnMetadata
///
/// RegionMetadata o-- RegionSchema
/// RegionMetadata o-- Schema
/// RegionSchema o-- StoreSchema
/// RegionSchema o-- Schema
/// RegionSchema o-- ColumnsMetadata
/// StoreSchema o-- ColumnsMetadata
/// StoreSchema o-- Schema
/// StoreSchema o-- ColumnMetadata
/// ColumnsMetadata o-- ColumnMetadata
/// ```
mod docs {}

41
src/mito2/src/region.rs Normal file
View File

@@ -0,0 +1,41 @@
// Copyright 2023 Greptime Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Mito region.
mod metadata;
mod version;
use std::collections::HashMap;
use std::sync::{Arc, RwLock};
use store_api::storage::RegionId;
use crate::region::version::VersionControlRef;
/// Metadata and runtime status of a region.
#[derive(Debug)]
pub(crate) struct MitoRegion {
version_control: VersionControlRef,
}
pub(crate) type MitoRegionRef = Arc<MitoRegion>;
/// Regions indexed by ids.
#[derive(Debug, Default)]
pub(crate) struct RegionMap {
regions: RwLock<HashMap<RegionId, MitoRegionRef>>,
}
pub(crate) type RegionMapRef = Arc<RegionMap>;

23
src/mito2/src/region/metadata.rs Normal file
View File

@@ -0,0 +1,23 @@
// Copyright 2023 Greptime Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Metadata of mito regions.
use std::sync::Arc;
/// Static metadata of a region.
#[derive(Debug)]
pub(crate) struct RegionMetadata {}
pub(crate) type RegionMetadataRef = Arc<RegionMetadata>;

32
src/mito2/src/region/version.rs Normal file
View File

@@ -0,0 +1,32 @@
// Copyright 2023 Greptime Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Version control of mito engine.
//!
//! Version is an immutable snapshot of region's metadata.
//!
//! To read latest data from `VersionControl`, we should
//! 1. Acquire `Version` from `VersionControl`.
//! 2. Then acquire last sequence.
//!
//! Reason: data may be flushed/compacted and some data with old sequence may be removed
//! and became invisible between step 1 and 2, so need to acquire version at first.
use std::sync::Arc;
/// Controls version of in memory metadata for a region.
#[derive(Debug)]
pub(crate) struct VersionControl {}
pub(crate) type VersionControlRef = Arc<VersionControl>;

31
src/mito2/src/worker.rs Normal file
View File

@@ -0,0 +1,31 @@
// Copyright 2023 Greptime Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Structs and utilities for writing regions.
use crate::region::RegionMapRef;
/// A fixed size group of [RegionWorker]s.
///
/// The group binds each region to a specific [RegionWorker].
#[derive(Debug, Default)]
pub(crate) struct WorkerGroup {
workers: Vec<RegionWorker>,
}
/// Worker to write and alter regions bound to it.
#[derive(Debug, Default)]
struct RegionWorker {
regions: RegionMapRef,
}