From e4cd08c750cb830e21b7ecf2ee60ff331282244f Mon Sep 17 00:00:00 2001 From: Hao Date: Wed, 19 Apr 2023 10:52:02 +0800 Subject: [PATCH] feat: add table id and engine to information_schema.TABLES (#1407) * feat: add table id and engine to informatin_schema.TABLES * Update src/catalog/src/information_schema/tables.rs Co-authored-by: Yingwen * chore: change table_engine to engine * test: update sqlness for information schema * test: update information_schema test in frontend::tests::instance_test.rs * fix: github action sqlness information_schema test fail * test: ignore table_id in information_schema * test: support distribute and standalone have different output --------- Co-authored-by: Yingwen --- src/catalog/src/information_schema/tables.rs | 26 +++++++- src/frontend/src/tests/instance_test.rs | 61 ++++++++++++++----- src/frontend/src/tests/test_util.rs | 10 +++ .../common/system/information_schema.result | 14 ++--- .../common/system/information_schema.sql | 2 +- 5 files changed, 87 insertions(+), 26 deletions(-) diff --git a/src/catalog/src/information_schema/tables.rs b/src/catalog/src/information_schema/tables.rs index 311964c5aa..17650d2d9a 100644 --- a/src/catalog/src/information_schema/tables.rs +++ b/src/catalog/src/information_schema/tables.rs @@ -23,7 +23,7 @@ use datafusion::physical_plan::stream::RecordBatchStreamAdapter as DfRecordBatch use datafusion::physical_plan::SendableRecordBatchStream as DfSendableRecordBatchStream; use datatypes::prelude::{ConcreteDataType, ScalarVectorBuilder, VectorRef}; use datatypes::schema::{ColumnSchema, Schema, SchemaRef}; -use datatypes::vectors::StringVectorBuilder; +use datatypes::vectors::{StringVectorBuilder, UInt32VectorBuilder}; use snafu::ResultExt; use table::metadata::TableType; @@ -44,6 +44,8 @@ impl InformationSchemaTables { ColumnSchema::new("table_schema", ConcreteDataType::string_datatype(), false), ColumnSchema::new("table_name", ConcreteDataType::string_datatype(), false), ColumnSchema::new("table_type", ConcreteDataType::string_datatype(), false), + ColumnSchema::new("table_id", ConcreteDataType::uint32_datatype(), true), + ColumnSchema::new("engine", ConcreteDataType::string_datatype(), true), ])); Self { schema, @@ -73,6 +75,8 @@ struct InformationSchemaTablesBuilder { schema_names: StringVectorBuilder, table_names: StringVectorBuilder, table_types: StringVectorBuilder, + table_ids: UInt32VectorBuilder, + engines: StringVectorBuilder, } impl InformationSchemaTablesBuilder { @@ -85,6 +89,8 @@ impl InformationSchemaTablesBuilder { schema_names: StringVectorBuilder::with_capacity(42), table_names: StringVectorBuilder::with_capacity(42), table_types: StringVectorBuilder::with_capacity(42), + table_ids: UInt32VectorBuilder::with_capacity(42), + engines: StringVectorBuilder::with_capacity(42), } } @@ -100,7 +106,15 @@ impl InformationSchemaTablesBuilder { let Some(schema) = self.catalog_provider.schema(&schema_name)? else { continue }; for table_name in schema.table_names()? { let Some(table) = schema.table(&table_name).await? else { continue }; - self.add_table(&catalog_name, &schema_name, &table_name, table.table_type()); + let table_info = table.table_info(); + self.add_table( + &catalog_name, + &schema_name, + &table_name, + table.table_type(), + Some(table_info.ident.table_id), + Some(&table_info.meta.engine), + ); } } @@ -110,6 +124,8 @@ impl InformationSchemaTablesBuilder { INFORMATION_SCHEMA_NAME, TABLES, TableType::View, + None, + None, ); self.finish() @@ -121,6 +137,8 @@ impl InformationSchemaTablesBuilder { schema_name: &str, table_name: &str, table_type: TableType, + table_id: Option, + engine: Option<&str>, ) { self.catalog_names.push(Some(catalog_name)); self.schema_names.push(Some(schema_name)); @@ -130,6 +148,8 @@ impl InformationSchemaTablesBuilder { TableType::View => "VIEW", TableType::Temporary => "LOCAL TEMPORARY", })); + self.table_ids.push(table_id); + self.engines.push(engine); } fn finish(&mut self) -> Result { @@ -138,6 +158,8 @@ impl InformationSchemaTablesBuilder { Arc::new(self.schema_names.finish()), Arc::new(self.table_names.finish()), Arc::new(self.table_types.finish()), + Arc::new(self.table_ids.finish()), + Arc::new(self.engines.finish()), ]; RecordBatch::new(self.schema.clone(), columns).context(CreateRecordBatchSnafu) } diff --git a/src/frontend/src/tests/instance_test.rs b/src/frontend/src/tests/instance_test.rs index 49eb3a6c24..d18593c850 100644 --- a/src/frontend/src/tests/instance_test.rs +++ b/src/frontend/src/tests/instance_test.rs @@ -904,6 +904,7 @@ async fn test_execute_copy_from_s3(instance: Arc) { #[apply(both_instances_cases)] async fn test_information_schema(instance: Arc) { + let is_distributed_mode = instance.is_distributed_mode(); let instance = instance.frontend(); let sql = "create table another_table(i bigint time index)"; @@ -913,27 +914,55 @@ async fn test_information_schema(instance: Arc) { // User can only see information schema under current catalog. // A necessary requirement to GreptimeCloud. - let sql = "select table_catalog, table_schema, table_name, table_type from information_schema.tables where table_type != 'SYSTEM VIEW' order by table_name"; + let sql = "select table_catalog, table_schema, table_name, table_type, table_id, engine from information_schema.tables where table_type != 'SYSTEM VIEW' order by table_name"; let output = execute_sql(&instance, sql).await; - let expected = "\ -+---------------+--------------------+------------+------------+ -| table_catalog | table_schema | table_name | table_type | -+---------------+--------------------+------------+------------+ -| greptime | public | numbers | BASE TABLE | -| greptime | public | scripts | BASE TABLE | -| greptime | information_schema | tables | VIEW | -+---------------+--------------------+------------+------------+"; + let expected = match is_distributed_mode { + true => { + "\ ++---------------+--------------------+------------+------------+----------+-------------+ +| table_catalog | table_schema | table_name | table_type | table_id | engine | ++---------------+--------------------+------------+------------+----------+-------------+ +| greptime | public | numbers | BASE TABLE | 1 | test_engine | +| greptime | public | scripts | BASE TABLE | 1024 | mito | +| greptime | information_schema | tables | VIEW | | | ++---------------+--------------------+------------+------------+----------+-------------+" + } + false => { + "\ ++---------------+--------------------+------------+------------+----------+-------------+ +| table_catalog | table_schema | table_name | table_type | table_id | engine | ++---------------+--------------------+------------+------------+----------+-------------+ +| greptime | public | numbers | BASE TABLE | 1 | test_engine | +| greptime | public | scripts | BASE TABLE | 1 | mito | +| greptime | information_schema | tables | VIEW | | | ++---------------+--------------------+------------+------------+----------+-------------+" + } + }; + check_output_stream(output, expected).await; let output = execute_sql_with(&instance, sql, query_ctx).await; - let expected = "\ -+-----------------+--------------------+---------------+------------+ -| table_catalog | table_schema | table_name | table_type | -+-----------------+--------------------+---------------+------------+ -| another_catalog | another_schema | another_table | BASE TABLE | -| another_catalog | information_schema | tables | VIEW | -+-----------------+--------------------+---------------+------------+"; + let expected = match is_distributed_mode { + true => { + "\ ++-----------------+--------------------+---------------+------------+----------+--------+ +| table_catalog | table_schema | table_name | table_type | table_id | engine | ++-----------------+--------------------+---------------+------------+----------+--------+ +| another_catalog | another_schema | another_table | BASE TABLE | 1025 | mito | +| another_catalog | information_schema | tables | VIEW | | | ++-----------------+--------------------+---------------+------------+----------+--------+" + } + false => { + "\ ++-----------------+--------------------+---------------+------------+----------+--------+ +| table_catalog | table_schema | table_name | table_type | table_id | engine | ++-----------------+--------------------+---------------+------------+----------+--------+ +| another_catalog | another_schema | another_table | BASE TABLE | 1024 | mito | +| another_catalog | information_schema | tables | VIEW | | | ++-----------------+--------------------+---------------+------------+----------+--------+" + } + }; check_output_stream(output, expected).await; } diff --git a/src/frontend/src/tests/test_util.rs b/src/frontend/src/tests/test_util.rs index e2854211b5..c40e2752f2 100644 --- a/src/frontend/src/tests/test_util.rs +++ b/src/frontend/src/tests/test_util.rs @@ -26,18 +26,28 @@ use crate::tests::{ pub(crate) trait MockInstance { fn frontend(&self) -> Arc; + + fn is_distributed_mode(&self) -> bool; } impl MockInstance for MockStandaloneInstance { fn frontend(&self) -> Arc { self.instance.clone() } + + fn is_distributed_mode(&self) -> bool { + false + } } impl MockInstance for MockDistributedInstance { fn frontend(&self) -> Arc { self.frontend.clone() } + + fn is_distributed_mode(&self) -> bool { + true + } } pub(crate) async fn standalone() -> Arc { diff --git a/tests/cases/standalone/common/system/information_schema.result b/tests/cases/standalone/common/system/information_schema.result index e375153620..ca9b0ffa73 100644 --- a/tests/cases/standalone/common/system/information_schema.result +++ b/tests/cases/standalone/common/system/information_schema.result @@ -27,18 +27,18 @@ order by table_name; | foo | +------------+ -select table_catalog, table_schema, table_name, table_type +select table_catalog, table_schema, table_name, table_type, engine from information_schema.tables where table_catalog = 'greptime' and table_schema != 'public' order by table_schema, table_name; -+---------------+--------------------+------------+------------+ -| table_catalog | table_schema | table_name | table_type | -+---------------+--------------------+------------+------------+ -| greptime | information_schema | tables | VIEW | -| greptime | my_db | foo | BASE TABLE | -+---------------+--------------------+------------+------------+ ++---------------+--------------------+------------+------------+--------+ +| table_catalog | table_schema | table_name | table_type | engine | ++---------------+--------------------+------------+------------+--------+ +| greptime | information_schema | tables | VIEW | | +| greptime | my_db | foo | BASE TABLE | mito | ++---------------+--------------------+------------+------------+--------+ use public; diff --git a/tests/cases/standalone/common/system/information_schema.sql b/tests/cases/standalone/common/system/information_schema.sql index 871c01c49d..ac3a683d02 100644 --- a/tests/cases/standalone/common/system/information_schema.sql +++ b/tests/cases/standalone/common/system/information_schema.sql @@ -14,7 +14,7 @@ from information_schema.tables where table_schema = 'my_db' order by table_name; -select table_catalog, table_schema, table_name, table_type +select table_catalog, table_schema, table_name, table_type, engine from information_schema.tables where table_catalog = 'greptime' and table_schema != 'public'