fix: database base ttl (#4926)

* main:
 Add common-meta dependency and implement SchemaMetadataManager

 - Introduce `common-meta` as a new dependency in `mito2`.
 - Implement `SchemaMetadataManager` for managing schema-level metadata.
 - Update `DatanodeBuilder` and `MitoEngine` to pass `KvBackendRef` for schema metadata management.
 - Add `SchemaMetadataManager` to `RegionWorkerLoop` for compaction handling.
 - Include `SchemaNameKey` usage in compaction-related code.
 - Add `database_metadata_manager` module with `SchemaMetadataManager` struct and associated logic.

* fix/database-base-ttl:
 Refactor metadata management and update compaction logic

 - Remove `database_metadata_manager` and introduce `schema_metadata_manager`
 - Update compaction logic to handle TTL based on schema metadata
 - Adjust tests to use `schema_metadata_manager` for setting up schema options
 - Fix engine creation in tests to pass `kv_backend` explicitly
 - Remove unused imports and apply minor code cleanups

* fix/database-base-ttl:
 Extend CREATE TABLE LIKE to inherit schema options

 - Implement inheritance of database level options for CREATE TABLE LIKE
 - Add schema options to SHOW CREATE TABLE output
 - Refactor create_table_stmt to include schema_options in SQL generation
 - Update error handling to include TableMetadataManagerSnafu

* fix/database-base-ttl:
 Refactor error handling and remove schema dependency in table creation

 - Replace expect with the ? operator for error handling in open_compaction_region
 - Simplify create_logical_tables by removing catalog and schema name parameters
 - Remove unnecessary schema retrieval and merging of schema options in create_table_info
 - Clean up unused imports and redundant code

* fix/database-base-ttl:
 Refactor error handling and update documentation comments

 - Update comment to reflect retrieval of schema options instead of metadata
 - Introduce new error type `GetSchemaMetadataSnafu` for schema metadata retrieval failures
 - Implement error handling for schema metadata retrieval in `find_ttl` function

* fix: toml

* fix/database-base-ttl:
 Refactor SchemaMetadataManager and adjust Cargo.toml dependencies

 - Remove unused imports in schema_metadata_manager.rs
 - Add conditional compilation for SchemaMetadataManager::new
 - Update Cargo.toml to remove "testing" feature from common-meta dependency in main section and add it to dev-dependencies

* fix/database-base-ttl:
 Fix typos in comments and function names across multiple modules

 - Correct spelling of 'parallelism' in region_server, engine, and scan_region modules
 - Amend typo in TODO comment from 'persisent' to 'persistent' in server module
 - Update incorrect test query from 'versiona' to 'version' in federated module tests

* fix/database-base-ttl: Add schema existence check in StatementExecutor for CREATE TABLE operation

* fix/database-base-ttl: Add warning log for failed TTL retrieval in compaction region open function

* fix/database-base-ttl:
 Refactor to use SchemaMetadataManagerRef in Datanode and MitoEngine

 - Replace KvBackendRef with SchemaMetadataManagerRef across various components.
 - Update DatanodeBuilder and MitoEngine to pass SchemaMetadataManagerRef instead of KvBackendRef.
 - Adjust test cases to use get_schema_metadata_manager method for consistency.
This commit is contained in:
Lei, HUANG
2024-11-05 10:51:32 +08:00
committed by GitHub
parent f221ee30fd
commit 3dcd6b8e51
34 changed files with 689 additions and 81 deletions

View File

@@ -32,6 +32,7 @@ use common_datasource::lister::{Lister, Source};
use common_datasource::object_store::build_backend;
use common_datasource::util::find_dir_and_filename;
use common_meta::key::flow::flow_info::FlowInfoValue;
use common_meta::SchemaOptions;
use common_query::prelude::GREPTIME_TIMESTAMP;
use common_query::Output;
use common_recordbatch::adapter::RecordBatchStreamAdapter;
@@ -703,6 +704,7 @@ pub fn show_create_database(database_name: &str, options: OptionMap) -> Result<O
pub fn show_create_table(
table: TableRef,
schema_options: Option<SchemaOptions>,
partitions: Option<Partitions>,
query_ctx: QueryContextRef,
) -> Result<Output> {
@@ -711,7 +713,7 @@ pub fn show_create_table(
let quote_style = query_ctx.quote_style();
let mut stmt = create_table_stmt(&table_info, quote_style)?;
let mut stmt = create_table_stmt(&table_info, schema_options, quote_style)?;
stmt.partitions = partitions.map(|mut p| {
p.set_quote(quote_style);
p

View File

@@ -16,6 +16,7 @@
use std::collections::HashMap;
use common_meta::SchemaOptions;
use datatypes::schema::{ColumnDefaultConstraint, ColumnSchema, SchemaRef, COMMENT_KEY};
use humantime::format_duration;
use snafu::ResultExt;
@@ -36,7 +37,8 @@ use crate::error::{
ConvertSqlTypeSnafu, ConvertSqlValueSnafu, GetFulltextOptionsSnafu, Result, SqlSnafu,
};
fn create_sql_options(table_meta: &TableMeta) -> OptionMap {
/// Generates CREATE TABLE options from given table metadata and schema-level options.
fn create_sql_options(table_meta: &TableMeta, schema_options: Option<SchemaOptions>) -> OptionMap {
let table_opts = &table_meta.options;
let mut options = OptionMap::default();
if let Some(write_buffer_size) = table_opts.write_buffer_size {
@@ -47,7 +49,12 @@ fn create_sql_options(table_meta: &TableMeta) -> OptionMap {
}
if let Some(ttl) = table_opts.ttl {
options.insert(TTL_KEY.to_string(), format_duration(ttl).to_string());
}
} else if let Some(database_ttl) = schema_options.and_then(|o| o.ttl) {
options.insert(
TTL_KEY.to_string(),
format_duration(database_ttl).to_string(),
);
};
for (k, v) in table_opts
.extra_options
.iter()
@@ -169,7 +176,11 @@ fn create_table_constraints(
}
/// Create a CreateTable statement from table info.
pub fn create_table_stmt(table_info: &TableInfoRef, quote_style: char) -> Result<CreateTable> {
pub fn create_table_stmt(
table_info: &TableInfoRef,
schema_options: Option<SchemaOptions>,
quote_style: char,
) -> Result<CreateTable> {
let table_meta = &table_info.meta;
let table_name = &table_info.name;
let schema = &table_info.meta.schema;
@@ -195,7 +206,7 @@ pub fn create_table_stmt(table_info: &TableInfoRef, quote_style: char) -> Result
columns,
engine: table_meta.engine.clone(),
constraints,
options: create_sql_options(table_meta),
options: create_sql_options(table_meta, schema_options),
partitions: None,
})
}
@@ -271,7 +282,7 @@ mod tests {
.unwrap(),
);
let stmt = create_table_stmt(&info, '"').unwrap();
let stmt = create_table_stmt(&info, None, '"').unwrap();
let sql = format!("\n{}", stmt);
assert_eq!(
@@ -337,7 +348,7 @@ ENGINE=mito
.unwrap(),
);
let stmt = create_table_stmt(&info, '"').unwrap();
let stmt = create_table_stmt(&info, None, '"').unwrap();
let sql = format!("\n{}", stmt);
assert_eq!(