rust/greptimedb
1
0
Fork 0
mirror of https://github.com/GreptimeTeam/greptimedb.git synced 2026-05-31 04:10:38 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor!: trying to replace TableGlobalValue, part 2 (#1985)

Browse Source 
* refactor!: using the new table metadata values

* fix: resolve PR comments

* fix: resolve PR comments

* fix: resolve PR comments
This commit is contained in:
LFC
2023-07-19 20:01:43 +08:00
committed by GitHub
parent 2ef0d06cdb
commit 172febb1af
41 changed files with 936 additions and 1499 deletions
Download Patch File Download Diff File Expand all files Collapse all files

43
src/frontend/src/catalog.rs
View File

@@ -30,9 +30,7 @@ use catalog::{
use client::client_manager::DatanodeClients;
use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, INFORMATION_SCHEMA_NAME};
use common_error::ext::BoxedError;
use common_meta::helper::{
build_catalog_prefix, build_schema_prefix, CatalogKey, SchemaKey, TableGlobalKey,
};
use common_meta::helper::{build_catalog_prefix, build_schema_prefix, CatalogKey, SchemaKey};
use common_meta::key::table_info::TableInfoKey;
use common_meta::key::table_name::TableNameKey;
use common_meta::key::table_region::TableRegionKey;
@@ -120,17 +118,6 @@ impl FrontendCatalogManager {
table: &str,
table_id: TableId,
) {
let tg_key = TableGlobalKey {
catalog_name: catalog.into(),
schema_name: schema.into(),
table_name: table.into(),
}
.to_string();
let tg_key = tg_key.as_bytes();
self.backend_cache_invalidator.invalidate_key(tg_key).await;
let key = TableNameKey::new(catalog, schema, table);
self.backend_cache_invalidator
.invalidate_key(&key.as_raw_key())
@@ -160,7 +147,7 @@ impl FrontendCatalogManager {
self.partition_manager
.table_routes()
.invalidate_table_route(&TableName::new(catalog, schema, table))
.invalidate_table_route(table_id)
.await;
}
}
@@ -343,10 +330,9 @@ impl CatalogManager for FrontendCatalogManager {
let mut tables = self
.table_metadata_manager
.table_name_manager()
.tables_old(catalog, schema)
.tables(catalog, schema)
.await
.context(TableMetadataManagerSnafu)?;
if catalog == DEFAULT_CATALOG_NAME && schema == DEFAULT_SCHEMA_NAME {
tables.push("numbers".to_string());
}
@@ -359,13 +345,11 @@ impl CatalogManager for FrontendCatalogManager {
catalog_name: catalog.to_string(),
}
.to_string();
Ok(self
.backend
self.backend
.get(key.as_bytes())
.await
.context(TableMetadataManagerSnafu)?
.is_some())
.context(TableMetadataManagerSnafu)
.map(|x| x.is_some())
}
async fn schema_exist(&self, catalog: &str, schema: &str) -> CatalogResult<bool> {
@@ -374,7 +358,6 @@ impl CatalogManager for FrontendCatalogManager {
schema_name: schema.to_string(),
}
.to_string();
Ok(self
.backend()
.get(schema_key.as_bytes())
@@ -387,7 +370,7 @@ impl CatalogManager for FrontendCatalogManager {
let key = TableNameKey::new(catalog, schema, table);
self.table_metadata_manager
.table_name_manager()
.get_old(&key)
.get(key)
.await
.context(TableMetadataManagerSnafu)
.map(|x| x.is_some())
@@ -423,19 +406,19 @@ impl CatalogManager for FrontendCatalogManager {
let key = TableNameKey::new(catalog, schema, table_name);
let Some(table_name_value) = self.table_metadata_manager
.table_name_manager()
.get_old(&key)
.get(key)
.await
.context(TableMetadataManagerSnafu)? else { return Ok(None) };
let _table_id = table_name_value.table_id();
let table_id = table_name_value.table_id();
let Some(v) = self.table_metadata_manager
let Some(table_info_value) = self.table_metadata_manager
.table_info_manager()
.get_old(&key.into())
.get(table_id)
.await
.context(TableMetadataManagerSnafu)? else { return Ok(None) };
let table_info = Arc::new(
v.table_info
table_info_value
.table_info
.try_into()
.context(catalog_err::InvalidTableInfoInCatalogSnafu)?,
);

4
src/frontend/src/error.rs
View File

@@ -603,7 +603,8 @@ impl ErrorExt for Error {
Error::NotSupported { .. } => StatusCode::Unsupported,
Error::HandleHeartbeatResponse { source, .. } => source.status_code(),
Error::HandleHeartbeatResponse { source, .. }
| Error::TableMetadataManager { source, .. } => source.status_code(),
Error::RuntimeResource { source, .. } => source.status_code(),
Error::PromStoreRemoteQueryPlan { source, .. }
@@ -702,7 +703,6 @@ impl ErrorExt for Error {
Error::WriteParquet { source, .. } => source.status_code(),
Error::InvalidCopyParameter { .. } => StatusCode::InvalidArguments,
Error::TableMetadataManager { source, .. } => source.status_code(),
}
}

40
src/frontend/src/heartbeat/handler/invalidate_table_cache.rs
View File

@@ -18,13 +18,13 @@ use common_meta::error::Result as MetaResult;
use common_meta::heartbeat::handler::{
HandleControl, HeartbeatResponseHandler, HeartbeatResponseHandlerContext,
};
use common_meta::helper::TableGlobalKey;
use common_meta::ident::TableIdent;
use common_meta::instruction::{Instruction, InstructionReply, SimpleReply};
use common_meta::key::table_info::TableInfoKey;
use common_meta::key::table_name::TableNameKey;
use common_meta::key::table_region::TableRegionKey;
use common_meta::key::TableMetaKey;
use common_meta::table_name::TableName;
use common_telemetry::{error, info};
use common_telemetry::error;
use partition::manager::TableRouteCacheInvalidatorRef;
#[derive(Clone)]
@@ -83,26 +83,28 @@ impl InvalidateTableCacheHandler {
}
async fn invalidate_table_cache(&self, table_ident: TableIdent) {
let tg_key = TableGlobalKey {
catalog_name: table_ident.catalog.clone(),
schema_name: table_ident.schema.clone(),
table_name: table_ident.table.clone(),
}
.to_string();
info!("invalidate table cache: {}", tg_key);
let tg_key = tg_key.as_bytes();
let table_id = table_ident.table_id;
self.backend_cache_invalidator
.invalidate_key(&TableInfoKey::new(table_id).as_raw_key())
.await;
self.backend_cache_invalidator.invalidate_key(tg_key).await;
self.backend_cache_invalidator
.invalidate_key(&TableRegionKey::new(table_id).as_raw_key())
.await;
let key = &TableRegionKey::new(table_ident.table_id).as_raw_key();
self.backend_cache_invalidator.invalidate_key(key).await;
self.backend_cache_invalidator
.invalidate_key(
&TableNameKey::new(
&table_ident.catalog,
&table_ident.schema,
&table_ident.table,
)
.as_raw_key(),
)
.await;
self.table_route_cache_invalidator
.invalidate_table_route(&TableName::new(
table_ident.catalog,
table_ident.schema,
table_ident.table,
))
.invalidate_table_route(table_id)
.await;
}
}

39
src/frontend/src/heartbeat/handler/tests.rs
View File

@@ -22,11 +22,12 @@ use common_meta::heartbeat::handler::{
HandlerGroupExecutor, HeartbeatResponseHandlerContext, HeartbeatResponseHandlerExecutor,
};
use common_meta::heartbeat::mailbox::{HeartbeatMailbox, MessageMeta};
use common_meta::helper::TableGlobalKey;
use common_meta::ident::TableIdent;
use common_meta::instruction::{Instruction, InstructionReply, SimpleReply};
use common_meta::table_name::TableName;
use common_meta::key::table_region::TableRegionKey;
use common_meta::key::TableMetaKey;
use partition::manager::TableRouteCacheInvalidator;
use table::metadata::TableId;
use tokio::sync::mpsc;
use super::invalidate_table_cache::InvalidateTableCacheHandler;
@@ -44,30 +45,26 @@ impl KvCacheInvalidator for MockKvCacheInvalidator {
}
pub struct MockTableRouteCacheInvalidator {
inner: Mutex<HashMap<String, i32>>,
inner: Mutex<HashMap<TableId, i32>>,
}
#[async_trait::async_trait]
impl TableRouteCacheInvalidator for MockTableRouteCacheInvalidator {
async fn invalidate_table_route(&self, table: &TableName) {
let _ = self.inner.lock().unwrap().remove(&table.to_string());
async fn invalidate_table_route(&self, table_id: TableId) {
let _ = self.inner.lock().unwrap().remove(&table_id);
}
}
#[tokio::test]
async fn test_invalidate_table_cache_handler() {
let table_key = TableGlobalKey {
catalog_name: "test".to_string(),
schema_name: "greptime".to_string(),
table_name: "foo_table".to_string(),
};
let inner = HashMap::from([(table_key.to_string().as_bytes().to_vec(), 1)]);
let table_id = 1;
let table_region_key = TableRegionKey::new(table_id);
let inner = HashMap::from([(table_region_key.as_raw_key(), 1)]);
let backend = Arc::new(MockKvCacheInvalidator {
inner: Mutex::new(inner),
});
let inner = HashMap::from([(table_key.to_string(), 1)]);
let inner = HashMap::from([(table_id, 1)]);
let table_route = Arc::new(MockTableRouteCacheInvalidator {
inner: Mutex::new(inner),
});
@@ -87,7 +84,7 @@ async fn test_invalidate_table_cache_handler() {
catalog: "test".to_string(),
schema: "greptime".to_string(),
table: "foo_table".to_string(),
table_id: 0,
table_id,
engine: "mito".to_string(),
}),
)
@@ -102,19 +99,9 @@ async fn test_invalidate_table_cache_handler() {
.inner
.lock()
.unwrap()
.contains_key(table_key.to_string().as_bytes()));
.contains_key(&table_region_key.as_raw_key()));
let table_name = TableName {
catalog_name: "test".to_string(),
schema_name: "greptime".to_string(),
table_name: "foo_table".to_string(),
};
assert!(!table_route
.inner
.lock()
.unwrap()
.contains_key(&table_name.to_string()));
assert!(!table_route.inner.lock().unwrap().contains_key(&table_id));
// removes a invalid key
handle_instruction(

18
src/frontend/src/instance/distributed.rs
View File

@@ -25,7 +25,7 @@ use api::v1::{
FlushTableExpr, InsertRequests,
};
use async_trait::async_trait;
use catalog::{CatalogManager, RegisterTableRequest};
use catalog::{CatalogManager, DeregisterTableRequest, RegisterTableRequest};
use chrono::DateTime;
use client::client_manager::DatanodeClients;
use client::Database;
@@ -176,11 +176,20 @@ impl DistInstance {
.with_context(|| TableNotFoundSnafu {
table_name: table_name.to_string(),
})?;
let table_id = table.table_info().ident.table_id;
let table_id = table.table_info().table_id();
self.drop_table_procedure(&table_name, table_id).await?;
let request = DeregisterTableRequest {
catalog: table_name.catalog_name.clone(),
schema: table_name.schema_name.clone(),
table_name: table_name.table_name.clone(),
};
self.catalog_manager
.deregister_table(request)
.await
.context(CatalogSnafu)?;
// Since the table information dropped on meta does not go through KvBackend, so we
// manually invalidate the cache here.
//
@@ -279,7 +288,6 @@ impl DistInstance {
let route_response = self
.meta_client
.route(RouteRequest {
table_names: vec![table_name.clone()],
table_ids: vec![table_id],
})
.await
@@ -383,7 +391,7 @@ impl DistInstance {
let partitions = self
.catalog_manager
.partition_manager()
.find_table_partitions(&table_name)
.find_table_partitions(table.table_info().table_id())
.await
.context(error::FindTablePartitionRuleSnafu {
table_name: &table_name.table_name,

19
src/frontend/src/instance/distributed/inserter.rs
View File

@@ -103,13 +103,14 @@ impl DistInserter {
let table_info = self.find_table_info(&request.table_name).await?;
let table_meta = &table_info.meta;
let table_id = table_info.table_id();
let split = partition_manager
.split_insert_request(&table_name, request, table_meta.schema.as_ref())
.split_insert_request(table_id, request, table_meta.schema.as_ref())
.await
.context(SplitInsertSnafu)?;
let table_route = partition_manager
.find_table_route(&table_name)
.find_table_route(table_id)
.await
.with_context(|_| FindTableRouteSnafu {
table_name: table_name.to_string(),
@@ -252,23 +253,23 @@ mod tests {
.table_name_manager()
.create(&key, table_id)
.await
.unwrap()
.is_none());
.is_ok());
assert!(table_metadata_manager
.table_info_manager()
.put_old(table_info)
.compare_and_put(table_id, None, table_info)
.await
.is_ok());
assert!(table_metadata_manager
let _ = table_metadata_manager
.table_region_manager()
.put_old(
&key.into(),
.compare_and_put(
1,
None,
RegionDistribution::from([(1, vec![1]), (2, vec![2]), (3, vec![3])]),
)
.await
.is_ok());
.unwrap();
}
#[tokio::test]

28
src/frontend/src/table.rs
View File

@@ -101,8 +101,9 @@ impl Table for DistTable {
let partition_manager = self.catalog_manager.partition_manager();
let datanode_clients = self.catalog_manager.datanode_clients();
let table_id = self.table_info.table_id();
let partition_rule = partition_manager
.find_table_partition_rule(&self.table_name)
.find_table_partition_rule(table_id)
.await
.map_err(BoxedError::new)
.context(TableOperationSnafu)?;
@@ -112,7 +113,7 @@ impl Table for DistTable {
.map_err(BoxedError::new)
.context(TableOperationSnafu)?;
let datanodes = partition_manager
.find_region_datanodes(&self.table_name, regions)
.find_region_datanodes(table_id, regions)
.await
.map_err(BoxedError::new)
.context(TableOperationSnafu)?;
@@ -171,8 +172,9 @@ impl Table for DistTable {
async fn delete(&self, request: DeleteRequest) -> table::Result<usize> {
let partition_manager = self.catalog_manager.partition_manager();
let table_id = self.table_info.table_id();
let partition_rule = partition_manager
.find_table_partition_rule(&self.table_name)
.find_table_partition_rule(table_id)
.await
.map_err(BoxedError::new)
.context(TableOperationSnafu)?;
@@ -242,7 +244,7 @@ impl DistTable {
let route = self
.catalog_manager
.partition_manager()
.find_table_route(table_name)
.find_table_route(self.table_info.table_id())
.await
.with_context(|_| FindTableRouteSnafu {
table_name: table_name.to_string(),
@@ -479,7 +481,7 @@ pub(crate) mod test {
],
);
table_routes
.insert_table_route(table_name.clone(), Arc::new(table_route))
.insert_table_route(1, Arc::new(table_route))
.await;
let table_name = TableName::new(
@@ -554,7 +556,7 @@ pub(crate) mod test {
],
);
table_routes
.insert_table_route(table_name.clone(), Arc::new(table_route))
.insert_table_route(2, Arc::new(table_route))
.await;
partition_manager
@@ -564,12 +566,9 @@ pub(crate) mod test {
async fn test_find_partition_rule() {
let partition_manager = create_partition_rule_manager().await;
// "one_column_partitioning_table" has id 1
let partition_rule = partition_manager
.find_table_partition_rule(&TableName::new(
DEFAULT_CATALOG_NAME,
DEFAULT_SCHEMA_NAME,
"one_column_partitioning_table",
))
.find_table_partition_rule(1)
.await
.unwrap();
let range_rule = partition_rule
@@ -580,12 +579,9 @@ pub(crate) mod test {
assert_eq!(range_rule.all_regions(), &vec![3, 2, 1]);
assert_eq!(range_rule.bounds(), &vec![10_i32.into(), 50_i32.into()]);
// "two_column_partitioning_table" has table 2
let partition_rule = partition_manager
.find_table_partition_rule(&TableName::new(
DEFAULT_CATALOG_NAME,
DEFAULT_SCHEMA_NAME,
"two_column_partitioning_table",
))
.find_table_partition_rule(2)
.await
.unwrap();
let range_columns_rule = partition_rule