refactor: refactor frontend cache (#3912)

* feat: implement the `TableCache`

* refactor: use `TableCache`

* refactor: replace `TableFlowManager` with `TableFlownodeSetCache`

* refactor: introduce cache crate

* chore: add comments

* chore: update comments

* chore: apply suggestions from CR

* chore: rename `cache_invalidator` to `local_cache_invalidator`

* chore(fuzz): set `acquire_timeout` to 30s

tests-integration/Cargo.toml

@@ -17,6 +17,7 @@ async-stream.workspace = true
 async-trait = "0.1"
 auth.workspace = true
 axum.workspace = true
+cache.workspace = true
 catalog.workspace = true
 chrono.workspace = true
 clap.workspace = true
@@ -43,6 +44,7 @@ futures.workspace = true
 futures-util.workspace = true
 meta-client.workspace = true
 meta-srv = { workspace = true, features = ["mock"] }
+moka.workspace = true
 mysql_async = { version = "0.33", default-features = false, features = [
     "default-rustls",
 ] }

tests-integration/src/cluster.rs

@@ -19,12 +19,12 @@ use std::time::Duration;
 
 use api::v1::region::region_server::RegionServer;
 use arrow_flight::flight_service_server::FlightServiceServer;
+use cache::default_cache_registry_builder;
 use catalog::kvbackend::{CachedMetaKvBackendBuilder, KvBackendCatalogManager, MetaKvBackend};
 use client::client_manager::DatanodeClients;
 use client::Client;
 use common_base::Plugins;
 use common_grpc::channel_manager::{ChannelConfig, ChannelManager};
-use common_meta::cache_invalidator::MultiCacheInvalidator;
 use common_meta::heartbeat::handler::parse_mailbox_message::ParseMailboxMessageHandler;
 use common_meta::heartbeat::handler::HandlerGroupExecutor;
 use common_meta::kv_backend::chroot::ChrootKvBackend;
@@ -346,22 +346,25 @@ impl GreptimeDbClusterBuilder {
 
         let cached_meta_backend =
             Arc::new(CachedMetaKvBackendBuilder::new(meta_client.clone()).build());
-        let multi_cache_invalidator = Arc::new(MultiCacheInvalidator::with_invalidators(vec![
-            cached_meta_backend.clone(),
-        ]));
+        let cache_registry_builder =
+            default_cache_registry_builder(Arc::new(MetaKvBackend::new(meta_client.clone())));
+        let cache_registry = Arc::new(
+            cache_registry_builder
+                .add_cache(cached_meta_backend.clone())
+                .build(),
+        );
+        let table_cache = cache_registry.get().unwrap();
         let catalog_manager = KvBackendCatalogManager::new(
             Mode::Distributed,
             Some(meta_client.clone()),
             cached_meta_backend.clone(),
-            multi_cache_invalidator.clone(),
+            table_cache,
         )
         .await;
 
         let handlers_executor = HandlerGroupExecutor::new(vec![
             Arc::new(ParseMailboxMessageHandler),
-            Arc::new(InvalidateTableCacheHandler::new(
-                multi_cache_invalidator.clone(),
-            )),
+            Arc::new(InvalidateTableCacheHandler::new(cache_registry.clone())),
         ]);
 
         let heartbeat_task = HeartbeatTask::new(
@@ -373,11 +376,12 @@ impl GreptimeDbClusterBuilder {
 
         let instance = FrontendBuilder::new(
             cached_meta_backend.clone(),
+            cache_registry.clone(),
             catalog_manager,
             datanode_clients,
             meta_client,
         )
-        .with_cache_invalidator(multi_cache_invalidator)
+        .with_local_cache_invalidator(cache_registry)
         .with_heartbeat_task(heartbeat_task)
         .try_build()
         .await

tests-integration/src/standalone.rs

@@ -14,12 +14,12 @@
 
 use std::sync::Arc;
 
+use cache::default_cache_registry_builder;
 use catalog::kvbackend::KvBackendCatalogManager;
 use cmd::standalone::StandaloneOptions;
 use common_base::Plugins;
 use common_catalog::consts::{MIN_USER_FLOW_ID, MIN_USER_TABLE_ID};
 use common_config::KvBackendConfig;
-use common_meta::cache_invalidator::MultiCacheInvalidator;
 use common_meta::ddl::flow_meta::FlowMetadataAllocator;
 use common_meta::ddl::table_meta::TableMetadataAllocator;
 use common_meta::ddl::DdlContext;
@@ -128,12 +128,12 @@ impl GreptimeDbStandaloneBuilder {
         let table_metadata_manager = Arc::new(TableMetadataManager::new(kv_backend.clone()));
         table_metadata_manager.init().await.unwrap();
         let flow_metadata_manager = Arc::new(FlowMetadataManager::new(kv_backend.clone()));
-        let multi_cache_invalidator = Arc::new(MultiCacheInvalidator::default());
+        let cache_registry = Arc::new(default_cache_registry_builder(kv_backend.clone()).build());
         let catalog_manager = KvBackendCatalogManager::new(
             Mode::Standalone,
             None,
             kv_backend.clone(),
-            multi_cache_invalidator.clone(),
+            cache_registry.get().unwrap(),
         )
         .await;
 
@@ -167,7 +167,7 @@ impl GreptimeDbStandaloneBuilder {
             DdlManager::try_new(
                 DdlContext {
                     node_manager: node_manager.clone(),
-                    cache_invalidator: multi_cache_invalidator,
+                    cache_invalidator: cache_registry.clone(),
                     memory_region_keeper: Arc::new(MemoryRegionKeeper::default()),
                     table_metadata_manager,
                     table_metadata_allocator,
@@ -182,6 +182,7 @@ impl GreptimeDbStandaloneBuilder {
 
         let instance = FrontendBuilder::new(
             kv_backend.clone(),
+            cache_registry,
             catalog_manager,
             node_manager,
             ddl_task_executor,