feat: introduce TableRouteCache to PartitionRuleManager (#3922)

* chore: add `CompositeTableRouteCacheRef` to `PartitionRuleManager`

* chore: update comments

* fix: add metrics for `get`

* chore: apply suggestions from CR

* chore: correct cache name

* feat: implement `LayeredCacheRegistry`

* fix: invalidate logical tables by physical table id

* refactor: replace `CacheRegistry` with `LayeredCacheRegistry`

* chore: update comments

* chore: apply suggestions from CR

* chore: fix fmt

* refactor: use `TableRouteCache` instead

* chore: apply suggestions from CR

* chore: fix clippy

src/cache/Cargo.toml

@@ -6,5 +6,8 @@ license.workspace = true
 
 [dependencies]
 catalog.workspace = true
+common-error.workspace = true
+common-macro.workspace = true
 common-meta.workspace = true
 moka.workspace = true
+snafu.workspace = true

src/cache/src/error.rs

@@ -0,0 +1,44 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use common_error::ext::ErrorExt;
+use common_error::status_code::StatusCode;
+use common_macro::stack_trace_debug;
+use snafu::{Location, Snafu};
+
+#[derive(Snafu)]
+#[snafu(visibility(pub))]
+#[stack_trace_debug]
+pub enum Error {
+    #[snafu(display("Failed to get cache from cache registry: {}", name))]
+    CacheRequired {
+        #[snafu(implicit)]
+        location: Location,
+        name: String,
+    },
+}
+
+pub type Result<T> = std::result::Result<T, Error>;
+
+impl ErrorExt for Error {
+    fn status_code(&self) -> StatusCode {
+        match self {
+            Error::CacheRequired { .. } => StatusCode::Internal,
+        }
+    }
+
+    fn as_any(&self) -> &dyn std::any::Any {
+        self
+    }
+}

src/cache/src/lib.rs

@@ -12,15 +12,21 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+pub mod error;
+
 use std::sync::Arc;
 use std::time::Duration;
 
 use catalog::kvbackend::new_table_cache;
 use common_meta::cache::{
-    new_table_flownode_set_cache, new_table_info_cache, new_table_name_cache, CacheRegistryBuilder,
+    new_table_flownode_set_cache, new_table_info_cache, new_table_name_cache,
+    new_table_route_cache, CacheRegistry, CacheRegistryBuilder, LayeredCacheRegistryBuilder,
 };
 use common_meta::kv_backend::KvBackendRef;
 use moka::future::CacheBuilder;
+use snafu::OptionExt;
+
+use crate::error::Result;
 
 const DEFAULT_CACHE_MAX_CAPACITY: u64 = 65536;
 const DEFAULT_CACHE_TTL: Duration = Duration::from_secs(10 * 60);
@@ -30,9 +36,9 @@ pub const TABLE_INFO_CACHE_NAME: &str = "table_info_cache";
 pub const TABLE_NAME_CACHE_NAME: &str = "table_name_cache";
 pub const TABLE_CACHE_NAME: &str = "table_cache";
 pub const TABLE_FLOWNODE_SET_CACHE_NAME: &str = "table_flownode_set_cache";
+pub const TABLE_ROUTE_CACHE_NAME: &str = "table_route_cache";
 
-// TODO(weny): Make the cache configurable.
-pub fn default_cache_registry_builder(kv_backend: KvBackendRef) -> CacheRegistryBuilder {
+pub fn build_fundamental_cache_registry(kv_backend: KvBackendRef) -> CacheRegistry {
     // Builds table info cache
     let cache = CacheBuilder::new(DEFAULT_CACHE_MAX_CAPACITY)
         .time_to_live(DEFAULT_CACHE_TTL)
@@ -55,16 +61,15 @@ pub fn default_cache_registry_builder(kv_backend: KvBackendRef) -> CacheRegistry
         kv_backend.clone(),
     ));
 
-    // Builds table cache
+    // Builds table route cache
     let cache = CacheBuilder::new(DEFAULT_CACHE_MAX_CAPACITY)
         .time_to_live(DEFAULT_CACHE_TTL)
         .time_to_idle(DEFAULT_CACHE_TTI)
         .build();
-    let table_cache = Arc::new(new_table_cache(
-        TABLE_CACHE_NAME.to_string(),
+    let table_route_cache = Arc::new(new_table_route_cache(
+        TABLE_ROUTE_CACHE_NAME.to_string(),
         cache,
-        table_info_cache.clone(),
-        table_name_cache.clone(),
+        kv_backend.clone(),
     ));
 
     // Builds table flownode set cache
@@ -81,6 +86,37 @@ pub fn default_cache_registry_builder(kv_backend: KvBackendRef) -> CacheRegistry
     CacheRegistryBuilder::default()
         .add_cache(table_info_cache)
         .add_cache(table_name_cache)
-        .add_cache(table_cache)
+        .add_cache(table_route_cache)
         .add_cache(table_flownode_set_cache)
+        .build()
+}
+
+// TODO(weny): Make the cache configurable.
+pub fn with_default_composite_cache_registry(
+    builder: LayeredCacheRegistryBuilder,
+) -> Result<LayeredCacheRegistryBuilder> {
+    let table_info_cache = builder.get().context(error::CacheRequiredSnafu {
+        name: TABLE_INFO_CACHE_NAME,
+    })?;
+    let table_name_cache = builder.get().context(error::CacheRequiredSnafu {
+        name: TABLE_NAME_CACHE_NAME,
+    })?;
+
+    // Builds table cache
+    let cache = CacheBuilder::new(DEFAULT_CACHE_MAX_CAPACITY)
+        .time_to_live(DEFAULT_CACHE_TTL)
+        .time_to_idle(DEFAULT_CACHE_TTI)
+        .build();
+    let table_cache = Arc::new(new_table_cache(
+        TABLE_CACHE_NAME.to_string(),
+        cache,
+        table_info_cache,
+        table_name_cache,
+    ));
+
+    let registry = CacheRegistryBuilder::default()
+        .add_cache(table_cache)
+        .build();
+
+    Ok(builder.add_cache_registry(registry))
 }