From c679ca5fdcc67f0195a65ee33ad9c7df79bad527 Mon Sep 17 00:00:00 2001 From: Ning Sun Date: Mon, 16 Nov 2026 17:00:51 +0000 Subject: [PATCH] feat(ingestion): configurable time index unit for auto-created physical metric tables (#10189) * feat(ingestion): configurable time index unit for auto-created physical metric tables Auto-created physical metric tables hardcoded a millisecond time index. Add an `auto_create_physical_table_timestamp_unit` option (frontend and standalone configs, default "Millisecond") that selects the time index unit for physical metric tables created by auto table creation. New logical tables created under such a physical table inherit its unit, so Prometheus remote write, OTLP metrics, and other metric-engine ingestion paths write timestamps in the configured unit. The option threads through all auto-create sites: the operator inserter's `create_physical_table_on_demand`, the frontend instance's batched logical-table flush path (`create_metric_physical_table_if_missing`), the logical-table batcher's unit alignment fallback, and the OTLP metric preflight validation unit. Millisecond remains the default, so existing behavior is unchanged unless configured. Signed-off-by: Ning Sun * fix(otlp): match default physical schema to configured timestamp unit Signed-off-by: Ning Sun --------- Signed-off-by: Ning Sun --- config/config.md | 2 + config/frontend.example.toml | 7 + config/standalone.example.toml | 7 + src/cmd/tests/load_config_test.rs | 3 + src/frontend/src/frontend.rs | 7 + src/frontend/src/instance.rs | 4 + src/frontend/src/instance/builder.rs | 6 + src/frontend/src/instance/logical_batcher.rs | 1 + src/frontend/src/instance/prom_store.rs | 4 +- src/operator/src/insert.rs | 16 +- src/operator/src/insert/otlp.rs | 23 +- src/servers/src/batcher/logical_table.rs | 12 +- .../src/batcher/logical_table/tables.rs | 4 +- .../src/batcher/logical_table/test_util.rs | 2 + src/standalone/src/options.rs | 9 + tests-integration/src/otlp.rs | 88 ++++++++ tests-integration/src/prom_store.rs | 196 ++++++++++++++++++ tests-integration/src/standalone.rs | 15 ++ tests-integration/src/test_util.rs | 1 + tests-integration/tests/http.rs | 1 + 20 files changed, 389 insertions(+), 19 deletions(-) diff --git a/config/config.md b/config/config.md index 20e785565f9..27aeabab52c 100644 --- a/config/config.md +++ b/config/config.md @@ -15,6 +15,7 @@ | `default_timezone` | String | Unset | The default timezone of the server. | | `default_column_prefix` | String | Unset | The default column prefix for auto-created time index, value, native histogram, and exemplar-label columns.
Legacy OTLP summary columns keep their historical `greptime_` prefix. | | `auto_create_table` | Bool | `true` | Server-side global switch for auto table creation on write.
When `false`, a missing table is never auto-created even if the request sets the `auto_create_table` hint to `true`. Default: `true`. | +| `auto_create_physical_table_timestamp_unit` | String | Unset | Time index unit for physical metric tables created by auto table creation
(Prometheus remote write, OTLP metrics, and other metric-engine ingestion).
New logical tables created under such a physical table inherit its unit.
Options: "Second", "Millisecond" (default), "Microsecond", "Nanosecond". | | `user_provider` | String | Unset | The user provider for authentication.
Examples: "static_user_provider:file:/path/to/users", "static_user_provider:cmd:greptime_user=greptime_pwd"
Password verifier formats: "plain:", "pbkdf2_sha256:::",
"mysql_native_password:",
"pg_scram_sha256::::"
"pbkdf2_sha256" and "pg_scram_sha256" protect passwords at rest, but such users cannot log in over MySQL:
`static_user_provider` and `watch_file_user_provider` only negotiate `mysql_native_password`, which cannot verify these formats.
"mysql_native_password" is MySQL-specific and cannot authenticate over PostgreSQL at all.
PostgreSQL SCRAM only covers "plain" and "pg_scram_sha256" users; if any user is "pbkdf2_sha256" or
"mysql_native_password", PostgreSQL falls back to cleartext password auth for every user.
For "pg_scram_sha256" users, keep the default iteration count (4096) and salt length (16): both are
observable in the SCRAM server-first message, and non-default values weaken resistance to username
enumeration. | | `max_in_flight_write_bytes` | String | 0 | Shared quota for concurrent write request bodies and messages (HTTP, gRPC, Flight)
plus retained decoder memory and estimated PB-to-Arrow conversion memory for ordinary-table batching.
A batch must fit together with its request's existing reservations; only additional memory is acquired.
Set to 0 to disable the limit. Default: "0" (unlimited) | | `write_bytes_exhausted_policy` | String | `wait` | Policy when write bytes quota is exhausted.
Options: "wait" (default, 10s timeout), "wait()" (e.g., "wait(30s)"), "fail"
Table-batch conversion never waits for additional memory. Under a wait policy,
temporary contention returns a retryable error; requests that cannot fit are rejected. | @@ -293,6 +294,7 @@ | `default_timezone` | String | Unset | The default timezone of the server. | | `default_column_prefix` | String | Unset | The default column prefix for auto-created time index, value, native histogram, and exemplar-label columns.
Legacy OTLP summary columns keep their historical `greptime_` prefix. | | `auto_create_table` | Bool | `true` | Server-side global switch for auto table creation on write.
When `false`, a missing table is never auto-created even if the request sets the `auto_create_table` hint to `true`. Default: `true`. | +| `auto_create_physical_table_timestamp_unit` | String | Unset | Time index unit for physical metric tables created by auto table creation
(Prometheus remote write, OTLP metrics, and other metric-engine ingestion).
New logical tables created under such a physical table inherit its unit.
Options: "Second", "Millisecond" (default), "Microsecond", "Nanosecond". | | `user_provider` | String | Unset | The user provider for authentication.
Examples: "static_user_provider:file:/path/to/users", "static_user_provider:cmd:greptime_user=greptime_pwd"
Password verifier formats: "plain:", "pbkdf2_sha256:::",
"mysql_native_password:",
"pg_scram_sha256::::"
"pbkdf2_sha256" and "pg_scram_sha256" protect passwords at rest, but such users cannot log in over MySQL:
`static_user_provider` and `watch_file_user_provider` only negotiate `mysql_native_password`, which cannot verify these formats.
"mysql_native_password" is MySQL-specific and cannot authenticate over PostgreSQL at all.
PostgreSQL SCRAM only covers "plain" and "pg_scram_sha256" users; if any user is "pbkdf2_sha256" or
"mysql_native_password", PostgreSQL falls back to cleartext password auth for every user.
For "pg_scram_sha256" users, keep the default iteration count (4096) and salt length (16): both are
observable in the SCRAM server-first message, and non-default values weaken resistance to username
enumeration. | | `max_in_flight_write_bytes` | String | 0 | Shared quota for concurrent write request bodies and messages (HTTP, gRPC, Flight)
plus retained decoder memory and estimated PB-to-Arrow conversion memory for ordinary-table batching.
A batch must fit together with its request's existing reservations; only additional memory is acquired.
Set to 0 to disable the limit. Default: "0" (unlimited) | | `write_bytes_exhausted_policy` | String | `wait` | Policy when write bytes quota is exhausted.
Options: "wait" (default, 10s timeout), "wait()" (e.g., "wait(30s)"), "fail"
Table-batch conversion never waits for additional memory. Under a wait policy,
temporary contention returns a retryable error; requests that cannot fit are rejected. | diff --git a/config/frontend.example.toml b/config/frontend.example.toml index 9d15ca4c88e..b8a73240345 100644 --- a/config/frontend.example.toml +++ b/config/frontend.example.toml @@ -11,6 +11,13 @@ default_column_prefix = "greptime" ## When `false`, a missing table is never auto-created even if the request sets the `auto_create_table` hint to `true`. Default: `true`. #+ auto_create_table = true +## Time index unit for physical metric tables created by auto table creation +## (Prometheus remote write, OTLP metrics, and other metric-engine ingestion). +## New logical tables created under such a physical table inherit its unit. +## Options: "Second", "Millisecond" (default), "Microsecond", "Nanosecond". +## @toml2docs:none-default +#+ auto_create_physical_table_timestamp_unit = "Millisecond" + ## The user provider for authentication. ## Examples: "static_user_provider:file:/path/to/users", "static_user_provider:cmd:greptime_user=greptime_pwd" ## Password verifier formats: "plain:", "pbkdf2_sha256:::", diff --git a/config/standalone.example.toml b/config/standalone.example.toml index 2bd671640f7..616b6f4f795 100644 --- a/config/standalone.example.toml +++ b/config/standalone.example.toml @@ -11,6 +11,13 @@ default_column_prefix = "greptime" ## When `false`, a missing table is never auto-created even if the request sets the `auto_create_table` hint to `true`. Default: `true`. #+ auto_create_table = true +## Time index unit for physical metric tables created by auto table creation +## (Prometheus remote write, OTLP metrics, and other metric-engine ingestion). +## New logical tables created under such a physical table inherit its unit. +## Options: "Second", "Millisecond" (default), "Microsecond", "Nanosecond". +## @toml2docs:none-default +#+ auto_create_physical_table_timestamp_unit = "Millisecond" + ## The user provider for authentication. ## Examples: "static_user_provider:file:/path/to/users", "static_user_provider:cmd:greptime_user=greptime_pwd" ## Password verifier formats: "plain:", "pbkdf2_sha256:::", diff --git a/src/cmd/tests/load_config_test.rs b/src/cmd/tests/load_config_test.rs index 129fbc5a81c..4f2a68ba988 100644 --- a/src/cmd/tests/load_config_test.rs +++ b/src/cmd/tests/load_config_test.rs @@ -25,6 +25,7 @@ use common_options::datanode::{ClientOptions, DatanodeClientOptions}; use common_runtime::global::RuntimeOptions; use common_telemetry::logging::{DEFAULT_LOGGING_DIR, DEFAULT_OTLP_HTTP_ENDPOINT, LoggingOptions}; use common_test_util::temp_dir::create_named_temp_file; +use common_time::timestamp::TimeUnit; use common_wal::config::DatanodeWalConfig; use common_wal::config::raft_engine::RaftEngineConfig; use datanode::config::{DatanodeOptions, RegionEngineConfig, StorageConfig}; @@ -249,6 +250,7 @@ fn test_load_frontend_example_config() { default_timezone: Some("UTC".to_string()), default_column_prefix: Some("greptime".to_string()), auto_create_table: true, + auto_create_physical_table_timestamp_unit: TimeUnit::Millisecond, meta_client: Some(MetaClientOptions { metasrv_addrs: vec!["127.0.0.1:3002".to_string()], timeout: Duration::from_secs(3), @@ -430,6 +432,7 @@ fn test_load_standalone_example_config() { default_timezone: Some("UTC".to_string()), default_column_prefix: Some("greptime".to_string()), auto_create_table: true, + auto_create_physical_table_timestamp_unit: TimeUnit::Millisecond, wal: DatanodeWalConfig::RaftEngine(RaftEngineConfig { dir: Some(format!("{}/{}", DEFAULT_DATA_HOME, WAL_DIR)), sync_period: Some(Duration::from_secs(5)), diff --git a/src/frontend/src/frontend.rs b/src/frontend/src/frontend.rs index 865a64a5d7e..dc669313a55 100644 --- a/src/frontend/src/frontend.rs +++ b/src/frontend/src/frontend.rs @@ -23,6 +23,7 @@ use common_memory_manager::OnExhaustedPolicy; use common_options::datanode::DatanodeClientOptions; use common_options::memory::MemoryOptions; use common_telemetry::logging::{LoggingOptions, SlowQueryOptions, TracingOptions}; +use common_time::timestamp::TimeUnit; use meta_client::{MetaClientOptions, MetaClientRef}; use pipeline::PipelineOptions; use query::options::QueryOptions; @@ -53,6 +54,11 @@ pub struct FrontendOptions { /// even if a request sets the `auto_create_table` hint to `true`. When `true` /// (default), the per-request hint still applies. Default: `true`. pub auto_create_table: bool, + /// Time index unit for physical metric tables created by auto table creation + /// (Prometheus remote write, OTLP metrics, and other metric-engine ingestion). + /// New logical tables created under such a physical table inherit its unit. + /// Options: "Second", "Millisecond" (default), "Microsecond", "Nanosecond". + pub auto_create_physical_table_timestamp_unit: TimeUnit, /// Shared quota for write bodies/messages, retained decoder memory, and estimated /// ordinary-table batch conversion memory. A batch must fit alongside its request's /// existing reservations; only additional memory is acquired. @@ -109,6 +115,7 @@ impl Default for FrontendOptions { default_timezone: None, default_column_prefix: None, auto_create_table: true, + auto_create_physical_table_timestamp_unit: TimeUnit::Millisecond, max_in_flight_write_bytes: ReadableSize(0), write_bytes_exhausted_policy: OnExhaustedPolicy::default(), http: HttpOptions::default(), diff --git a/src/frontend/src/instance.rs b/src/frontend/src/instance.rs index ba735d1226d..6d8f98fdc0c 100644 --- a/src/frontend/src/instance.rs +++ b/src/frontend/src/instance.rs @@ -64,6 +64,7 @@ use common_recordbatch::RecordBatchStreamWrapper; use common_recordbatch::error::StreamTimeoutSnafu; use common_telemetry::logging::SlowQueryOptions; use common_telemetry::{debug, error, tracing}; +use common_time::timestamp::TimeUnit; use datafusion::dataframe::DataFrame; use datafusion::physical_plan::ExecutionPlan; use datafusion_expr::LogicalPlan; @@ -133,6 +134,9 @@ pub struct Instance { process_manager: ProcessManagerRef, slow_query_options: SlowQueryOptions, influxdb_default_merge_mode: InfluxdbMergeMode, + /// Time index unit for physical metric tables created by auto table creation + /// through the frontend instance (e.g., batched logical-table flushes). + auto_create_physical_table_timestamp_unit: TimeUnit, trace_ingest_chunk_size: usize, trace_aux_cache: otlp::TraceAuxCache, trace_aux_limiter: Option, diff --git a/src/frontend/src/instance/builder.rs b/src/frontend/src/instance/builder.rs index 4056d78a480..3703c5bedea 100644 --- a/src/frontend/src/instance/builder.rs +++ b/src/frontend/src/instance/builder.rs @@ -253,6 +253,9 @@ impl FrontendBuilder { table_flownode_cache.clone(), self.options.auto_create_table, ) + .with_auto_create_physical_table_timestamp_unit( + self.options.auto_create_physical_table_timestamp_unit, + ) .with_write_dispatcher(write_dispatcher.clone()) }; // The execution-only inserter owns no batchers, avoiding an Arc cycle. @@ -439,6 +442,9 @@ impl FrontendBuilder { process_manager, slow_query_options: self.options.slow_query.clone(), influxdb_default_merge_mode: self.options.influxdb.default_merge_mode, + auto_create_physical_table_timestamp_unit: self + .options + .auto_create_physical_table_timestamp_unit, trace_ingest_chunk_size: self.options.otlp.trace_ingest_chunk_size, trace_aux_cache: TraceAuxCache::new(self.options.otlp.trace_aux_cache_size.as_bytes()), trace_aux_limiter, diff --git a/src/frontend/src/instance/logical_batcher.rs b/src/frontend/src/instance/logical_batcher.rs index b4755ebc974..3d1d7165179 100644 --- a/src/frontend/src/instance/logical_batcher.rs +++ b/src/frontend/src/instance/logical_batcher.rs @@ -53,6 +53,7 @@ impl Instance { self.table_flownode_set_cache().clone(), true, Arc::new(LogicalTables(Arc::downgrade(self))), + options.auto_create_physical_table_timestamp_unit, options_batcher.pending_rows_flush_interval, options_batcher.max_batch_rows, options_batcher.max_concurrent_flushes, diff --git a/src/frontend/src/instance/prom_store.rs b/src/frontend/src/instance/prom_store.rs index 963a8ae22ed..4d13d391843 100644 --- a/src/frontend/src/instance/prom_store.rs +++ b/src/frontend/src/instance/prom_store.rs @@ -788,7 +788,9 @@ impl Instance { let default_schema = vec![ api::v1::ColumnSchema { column_name: common_query::prelude::greptime_timestamp().to_string(), - datatype: api::v1::ColumnDataType::TimestampMillisecond as i32, + datatype: api::helper::timestamp_datatype( + self.auto_create_physical_table_timestamp_unit, + ) as i32, semantic_type: api::v1::SemanticType::Timestamp as i32, datatype_extension: None, options: None, diff --git a/src/operator/src/insert.rs b/src/operator/src/insert.rs index 7a6907edc9c..e69063734db 100644 --- a/src/operator/src/insert.rs +++ b/src/operator/src/insert.rs @@ -120,6 +120,11 @@ pub struct Inserter { /// When `false`, missing tables are never auto-created regardless of the /// per-request `auto_create_table` hint. When `true`, the hint still applies. auto_create_table: bool, + /// Time index unit for physical metric tables created by auto table creation. + /// New logical tables created under such a physical table inherit its unit, + /// so all metric-engine ingestion paths write timestamps in this unit. + /// Defaults to milliseconds for compatibility. + auto_create_physical_table_timestamp_unit: TimeUnit, pending_rows_batcher: Option>, /// Rows handed to detached flow mirror tasks that have not finished yet. /// Bounds how much row data in-flight mirror writes may keep alive, see @@ -342,11 +347,18 @@ impl Inserter { node_manager, table_flownode_set_cache, auto_create_table, + auto_create_physical_table_timestamp_unit: TimeUnit::Millisecond, pending_rows_batcher: None, mirror_pending_rows: Arc::new(AtomicU64::new(0)), } } + /// Sets the time index unit for auto-created physical metric tables. + pub fn with_auto_create_physical_table_timestamp_unit(mut self, unit: TimeUnit) -> Self { + self.auto_create_physical_table_timestamp_unit = unit; + self + } + /// Installs the shared dispatcher used by direct writes and batch flushes. pub fn with_write_dispatcher(mut self, dispatcher: WriteDispatcherRef) -> Self { self.write_dispatcher = dispatcher; @@ -1868,7 +1880,9 @@ impl Inserter { let default_schema = vec![ ColumnSchema { column_name: greptime_timestamp().to_string(), - datatype: ColumnDataType::TimestampMillisecond as _, + datatype: api::helper::timestamp_datatype( + self.auto_create_physical_table_timestamp_unit, + ) as _, semantic_type: SemanticType::Timestamp as _, datatype_extension: None, options: None, diff --git a/src/operator/src/insert/otlp.rs b/src/operator/src/insert/otlp.rs index 9405480d3a2..aa6bf665eb6 100644 --- a/src/operator/src/insert/otlp.rs +++ b/src/operator/src/insert/otlp.rs @@ -118,12 +118,9 @@ impl Inserter { tables.insert(backing.clone(), table); } physical_columns.entry(backing.as_str()).or_insert_with(|| { - PhysicalColumns::new( - tables[backing] - .as_ref() - .map(table_columns) - .unwrap_or_else(default_physical_columns), - ) + PhysicalColumns::new(tables[backing].as_ref().map(table_columns).unwrap_or_else( + || default_physical_columns(self.auto_create_physical_table_timestamp_unit), + )) }); } // Match ordinary insertion: impure defaults are evaluated once per table @@ -176,7 +173,9 @@ impl Inserter { let unit = target .and_then(table_time_index_unit) .or_else(|| physical.and_then(table_time_index_unit)) - .unwrap_or(TimeUnit::Millisecond); + // The physical table will be auto-created in the configured + // unit, so validate its rows in that unit as well. + .unwrap_or(self.auto_create_physical_table_timestamp_unit); if let Some(rows) = &mut request.rows { convert_rows_time_unit(rows, unit)?; } @@ -378,13 +377,13 @@ fn table_columns(table: &TableRef) -> Vec<(DataColumn, SemanticType)> { .collect() } -fn default_physical_columns() -> Vec<(DataColumn, SemanticType)> { +fn default_physical_columns(unit: TimeUnit) -> Vec<(DataColumn, SemanticType)> { use datatypes::prelude::ConcreteDataType; vec![ ( DataColumn::new( greptime_timestamp(), - ConcreteDataType::timestamp_millisecond_datatype(), + ConcreteDataType::timestamp_datatype(unit), false, ) .with_time_index(true), @@ -698,7 +697,7 @@ mod tests { #[test] fn test_preflight_omissions_defaults_and_explicit_nulls() { - let mut columns = default_physical_columns(); + let mut columns = default_physical_columns(TimeUnit::Millisecond); columns.push(( DataColumn::new("host", ConcreteDataType::string_datatype(), false) .with_default_constraint(Some(ColumnDefaultConstraint::Value(DataValue::String( @@ -739,7 +738,7 @@ mod tests { #[test] fn test_preflight_existing_columns_and_physical_definitions() { - let columns = default_physical_columns(); + let columns = default_physical_columns(TimeUnit::Millisecond); let mut rows = Rows { schema: vec![ColumnSchema { column_name: greptime_timestamp().into(), @@ -805,7 +804,7 @@ mod tests { (defaulted, None), (impure, Some("impure default for physical field `impure`")), ] { - let mut columns = default_physical_columns(); + let mut columns = default_physical_columns(TimeUnit::Millisecond); columns.extend((0..1024).map(|index| { ( DataColumn::new( diff --git a/src/servers/src/batcher/logical_table.rs b/src/servers/src/batcher/logical_table.rs index 707afe2ea0f..b7c0817e03a 100644 --- a/src/servers/src/batcher/logical_table.rs +++ b/src/servers/src/batcher/logical_table.rs @@ -108,6 +108,10 @@ pub struct LogicalTablePendingRowsBatcher { worker_channel_capacity: usize, prom_store_with_metric_engine: bool, schema_alterer: PendingRowsSchemaAltererRef, + /// Time index unit the schema alterer uses when auto-creating a missing + /// physical metric table; keeps the batcher's logical-create alignment + /// consistent with the created physical table. + auto_create_physical_table_timestamp_unit: TimeUnit, pending_rows_batch_sync: bool, shutdown: broadcast::Sender<()>, } @@ -123,6 +127,7 @@ impl LogicalTablePendingRowsBatcher { table_flownode_set_cache: TableFlownodeSetCacheRef, prom_store_with_metric_engine: bool, schema_alterer: PendingRowsSchemaAltererRef, + auto_create_physical_table_timestamp_unit: TimeUnit, flush_interval: Duration, max_batch_rows: usize, max_concurrent_flushes: usize, @@ -163,6 +168,7 @@ impl LogicalTablePendingRowsBatcher { flow_notification_tx, prom_store_with_metric_engine, schema_alterer, + auto_create_physical_table_timestamp_unit, flush_limiter, request_limiter, worker_channel_capacity, @@ -184,8 +190,8 @@ impl LogicalTablePendingRowsBatcher { } /// Returns the physical metric table's time index unit resolved from - /// `ctx`, defaulting to millisecond when the table does not exist yet - /// (the schema alterer auto-creates it as millisecond). + /// `ctx`, falling back to the configured auto-create unit when the table + /// does not exist yet (the schema alterer auto-creates it with that unit). async fn physical_time_index_unit_or_default(&self, ctx: &QueryContextRef) -> TimeUnit { let key = batch_key_from_ctx(ctx); let Ok(Some(table)) = self @@ -193,7 +199,7 @@ impl LogicalTablePendingRowsBatcher { .table(&key.catalog, &key.schema, &key.physical_table, None) .await else { - return TimeUnit::Millisecond; + return self.auto_create_physical_table_timestamp_unit; }; table .table_info() diff --git a/src/servers/src/batcher/logical_table/tables.rs b/src/servers/src/batcher/logical_table/tables.rs index 140dd2d4f23..8aef0df0b39 100644 --- a/src/servers/src/batcher/logical_table/tables.rs +++ b/src/servers/src/batcher/logical_table/tables.rs @@ -94,8 +94,8 @@ impl LogicalTablePendingRowsBatcher { // New tables are created on the request's selected physical table; // their time index must use the physical table's unit (a missing - // physical table is auto-created as millisecond by the schema - // alterer, matching the default here). + // physical table is auto-created in the configured auto-create unit + // by the schema alterer, matching the fallback here). if !plan.tables_to_create.is_empty() { let physical_unit = self.physical_time_index_unit_or_default(ctx).await; for (_, request_schema) in &mut plan.tables_to_create { diff --git a/src/servers/src/batcher/logical_table/test_util.rs b/src/servers/src/batcher/logical_table/test_util.rs index 364cf0c8f5a..6cf95af1f1c 100644 --- a/src/servers/src/batcher/logical_table/test_util.rs +++ b/src/servers/src/batcher/logical_table/test_util.rs @@ -32,6 +32,7 @@ use common_meta::node_manager::{ }; use common_meta::test_util::MockDatanodeManager; use common_query::prelude::GREPTIME_PHYSICAL_TABLE; +use common_time::timestamp::TimeUnit; use datatypes::prelude::ConcreteDataType; use datatypes::schema::{ColumnSchema as DtColumnSchema, Schema as DtSchema}; use operator::test_util::{create_partition_rule_manager, prepare_mocked_backend}; @@ -110,6 +111,7 @@ pub(crate) async fn submission_batcher( mock_table_flownode_cache(1, vec![]).await, true, Arc::new(UnexpectedSchemaAlterer), + TimeUnit::Millisecond, Duration::from_secs(1), 2, 1, diff --git a/src/standalone/src/options.rs b/src/standalone/src/options.rs index e2971712e4b..3c7a9e6ce13 100644 --- a/src/standalone/src/options.rs +++ b/src/standalone/src/options.rs @@ -21,6 +21,7 @@ use common_event_recorder::EventRecorderOptions; use common_memory_manager::OnExhaustedPolicy; use common_options::memory::MemoryOptions; use common_telemetry::logging::{LoggingOptions, SlowQueryOptions, TracingOptions}; +use common_time::timestamp::TimeUnit; use common_wal::config::DatanodeWalConfig; use datanode::config::{DatanodeOptions, ProcedureConfig, RegionEngineConfig, StorageConfig}; use file_engine::config::EngineConfig as FileEngineConfig; @@ -47,6 +48,11 @@ pub struct StandaloneOptions { /// Upper bound: when `false`, missing tables are never auto-created even if a /// request sets the `auto_create_table` hint to `true`. Default: `true`. pub auto_create_table: bool, + /// Time index unit for physical metric tables created by auto table creation + /// (Prometheus remote write, OTLP metrics, and other metric-engine ingestion). + /// New logical tables created under such a physical table inherit its unit. + /// Options: "Second", "Millisecond" (default), "Microsecond", "Nanosecond". + pub auto_create_physical_table_timestamp_unit: TimeUnit, /// Shared quota for write bodies/messages, retained decoder memory, and estimated /// ordinary-table batch conversion memory. A batch must fit alongside its request's /// existing reservations; only additional memory is acquired. @@ -111,6 +117,7 @@ impl Default for StandaloneOptions { default_timezone: None, default_column_prefix: None, auto_create_table: true, + auto_create_physical_table_timestamp_unit: TimeUnit::Millisecond, max_in_flight_write_bytes: ReadableSize(0), write_bytes_exhausted_policy: OnExhaustedPolicy::default(), http: HttpOptions::default(), @@ -181,6 +188,8 @@ impl StandaloneOptions { FrontendOptions { default_timezone: cloned_opts.default_timezone, auto_create_table: cloned_opts.auto_create_table, + auto_create_physical_table_timestamp_unit: cloned_opts + .auto_create_physical_table_timestamp_unit, max_in_flight_write_bytes: cloned_opts.max_in_flight_write_bytes, write_bytes_exhausted_policy: cloned_opts.write_bytes_exhausted_policy, http: cloned_opts.http, diff --git a/tests-integration/src/otlp.rs b/tests-integration/src/otlp.rs index 70133835aa0..8e193843627 100644 --- a/tests-integration/src/otlp.rs +++ b/tests-integration/src/otlp.rs @@ -3999,6 +3999,94 @@ WITH( } } + #[tokio::test(flavor = "multi_thread")] + async fn test_otlp_initial_write_auto_create_physical_timestamp_unit() { + use common_time::timestamp::TimeUnit as ConfigTimeUnit; + + for (unit, arrow_unit, expected) in [ + (ConfigTimeUnit::Second, TimeUnit::Second, 60), + (ConfigTimeUnit::Millisecond, TimeUnit::Millisecond, 60_123), + ( + ConfigTimeUnit::Microsecond, + TimeUnit::Microsecond, + 60_123_456, + ), + ( + ConfigTimeUnit::Nanosecond, + TimeUnit::Nanosecond, + 60_123_456_789, + ), + ] { + let standalone = GreptimeDbStandaloneBuilder::new(&format!("otlp_initial_{unit:?}")) + .with_auto_create_physical_table_timestamp_unit(unit) + .build() + .await; + let instance = standalone.fe_instance(); + let mut ctx = QueryContext::with(DEFAULT_CATALOG_NAME, "public"); + ctx.set_protocol_ctx(ProtocolCtx::OtlpMetric(OtlpMetricCtx { + with_metric_engine: true, + ..Default::default() + })); + let ctx = Arc::new(ctx); + + // No pre-created tables: preflight must accept the first point so + // ingestion can create both tables with the configured time index. + let outcome = instance + .metrics( + metric_request(vec![gauge_metric( + "initial_metric", + &[(60_123_456_789, 1.0, None)], + )]), + ctx.clone(), + None, + ) + .await + .unwrap(); + assert_eq!( + (outcome.accepted_data_points, outcome.rejected_data_points), + (1, 0), + "unit={unit:?}" + ); + + for table in ["greptime_physical_table", "initial_metric"] { + let output = instance + .do_query( + &format!("SELECT greptime_timestamp FROM {table}"), + ctx.clone(), + ) + .await + .remove(0) + .unwrap(); + let OutputData::Stream(stream) = output.data else { + panic!("expected stream") + }; + let batches = RecordBatches::try_collect(stream).await.unwrap().take(); + assert_eq!( + batches.iter().map(|batch| batch.num_rows()).sum::(), + 1 + ); + let timestamp = batches[0].column(0); + assert_eq!( + timestamp.data_type(), + &DataType::Timestamp(arrow_unit, None) + ); + let actual = match arrow_unit { + TimeUnit::Second => timestamp.as_primitive::().value(0), + TimeUnit::Millisecond => timestamp + .as_primitive::() + .value(0), + TimeUnit::Microsecond => timestamp + .as_primitive::() + .value(0), + TimeUnit::Nanosecond => { + timestamp.as_primitive::().value(0) + } + }; + assert_eq!(actual, expected, "unit={unit:?}, table={table}"); + } + } + } + #[tokio::test(flavor = "multi_thread")] async fn test_otlp_preflight_shared_physical_and_fanout_conflicts() { let standalone = GreptimeDbStandaloneBuilder::new("otlp_physical_conflicts") diff --git a/tests-integration/src/prom_store.rs b/tests-integration/src/prom_store.rs index de72d6501ec..c248f5124f3 100644 --- a/tests-integration/src/prom_store.rs +++ b/tests-integration/src/prom_store.rs @@ -478,6 +478,202 @@ mod tests { .await; } + #[tokio::test(flavor = "multi_thread")] + async fn test_standalone_prom_remote_write_auto_create_microsecond_physical_table() { + common_telemetry::init_default_ut_logging(); + let standalone = GreptimeDbStandaloneBuilder::new("test_prom_rw_auto_create_us_physical") + .with_auto_create_physical_table_timestamp_unit( + common_time::timestamp::TimeUnit::Microsecond, + ) + .build() + .await; + let instance = standalone.fe_instance(); + + let db = "prometheus_auto_create_us"; + let ctx = Arc::new(QueryContext::with(DEFAULT_CATALOG_NAME, db)); + assert!( + SqlQueryHandler::do_query( + instance.as_ref(), + &format!("CREATE DATABASE IF NOT EXISTS {db}"), + ctx.clone(), + ) + .await + .first() + .unwrap() + .is_ok() + ); + + // No manual table creation: both the physical table and the logical + // table `metric1` must be auto-created by the write, with a + // microsecond time index inherited from the physical table. + let write_request = WriteRequest { + timeseries: vec![prom_store::mock_timeseries()[0].clone()], + ..Default::default() + }; + let (row_inserts, _) = to_grpc_row_insert_requests(&write_request).unwrap(); + instance + .write(row_inserts, ctx.clone(), true) + .await + .unwrap(); + + for table in ["greptime_physical_table", "metric1"] { + let mut output = instance + .do_query( + &format!("SELECT greptime_timestamp FROM {table}"), + ctx.clone(), + ) + .await; + let OutputData::Stream(stream) = output.remove(0).unwrap().data else { + unreachable!() + }; + let batches = common_recordbatch::RecordBatches::try_collect(stream) + .await + .unwrap() + .take(); + assert_eq!(batches[0].num_rows(), 2, "unexpected rows in {table}"); + let ts_column = batches[0].column(0); + assert_eq!( + ts_column.data_type(), + &DataType::Timestamp(TimeUnit::Microsecond, None), + "unexpected time index type in {table}" + ); + let timestamps = ts_column + .as_primitive::() + .values(); + assert_eq!(timestamps.to_vec(), vec![1_000_000, 2_000_000]); + } + + // Remote read narrows microsecond timestamps back to milliseconds. + let read_request = ReadRequest { + queries: vec![Query { + start_timestamp_ms: 1000, + end_timestamp_ms: 2000, + matchers: vec![LabelMatcher { + name: prom_store::METRIC_NAME_LABEL.to_string(), + value: "metric1".to_string(), + r#type: 0, + }], + ..Default::default() + }], + ..Default::default() + }; + let resp = instance.read(read_request, ctx.clone()).await.unwrap(); + let body = prom_store::snappy_decompress(&resp.body).unwrap(); + let read_response = ReadResponse::decode(&body[..]).unwrap(); + let samples = &read_response.results[0].timeseries[0].samples; + assert_eq!( + samples.clone(), + vec![ + Sample { + value: 1.0, + timestamp: 1000 + }, + Sample { + value: 2.0, + timestamp: 2000 + } + ] + ); + } + + #[tokio::test(flavor = "multi_thread")] + async fn test_standalone_prom_batcher_auto_create_microsecond_physical_table() { + use servers::batcher::logical_table::PendingRowsSchemaAlterer; + + common_telemetry::init_default_ut_logging(); + let standalone = GreptimeDbStandaloneBuilder::new("test_prom_batcher_auto_create_us") + .with_auto_create_physical_table_timestamp_unit( + common_time::timestamp::TimeUnit::Microsecond, + ) + .build() + .await; + let instance = standalone.fe_instance(); + + // Drives the batcher's schema-alterer path: a missing physical table is + // auto-created with the configured microsecond unit, and new logical + // tables aligned to it. + let db = "prometheus_batcher_auto_create_us"; + let ctx = Arc::new(QueryContext::with(DEFAULT_CATALOG_NAME, db)); + assert!( + SqlQueryHandler::do_query( + instance.as_ref(), + &format!("CREATE DATABASE IF NOT EXISTS {db}"), + ctx.clone(), + ) + .await + .first() + .unwrap() + .is_ok() + ); + // The batcher aligns request schemas to the physical table's unit + // before invoking the schema alterer, so the schema below arrives in + // the configured microsecond unit. A missing physical table must be + // auto-created with the same unit. + let request_schema = vec![ + api::v1::ColumnSchema { + column_name: "greptime_timestamp".to_string(), + datatype: ColumnDataType::TimestampMicrosecond as i32, + semantic_type: SemanticType::Timestamp as i32, + datatype_extension: None, + options: None, + }, + api::v1::ColumnSchema { + column_name: "greptime_value".to_string(), + datatype: ColumnDataType::Float64 as i32, + semantic_type: SemanticType::Field as i32, + datatype_extension: None, + options: None, + }, + api::v1::ColumnSchema { + column_name: "job".to_string(), + datatype: ColumnDataType::String as i32, + semantic_type: SemanticType::Tag as i32, + datatype_extension: None, + options: None, + }, + ]; + instance + .create_tables_if_missing_batch( + DEFAULT_CATALOG_NAME, + db, + &[("metric1", request_schema.as_slice())], + true, + ctx.clone(), + ) + .await + .unwrap(); + + for table in ["greptime_physical_table", "metric1"] { + let mut output = instance + .do_query( + &format!( + "SELECT greptime_data_type FROM information_schema.columns \ + WHERE table_schema='{db}' AND table_name='{table}' \ + AND column_name='greptime_timestamp'" + ), + ctx.clone(), + ) + .await; + let OutputData::Stream(stream) = output.remove(0).unwrap().data else { + unreachable!() + }; + let batches = common_recordbatch::RecordBatches::try_collect(stream) + .await + .unwrap() + .take(); + assert_eq!( + batches[0].num_rows(), + 1, + "missing time index column in {table}" + ); + let data_type = batches[0].column(0).as_string::().value(0); + assert_eq!( + data_type, "TimestampMicrosecond", + "unexpected time index type in {table}" + ); + } + } + /// Regression test for : /// prometheus remote write/read against a physical metric table /// pre-created with a non-millisecond time index. Millisecond samples are diff --git a/tests-integration/src/standalone.rs b/tests-integration/src/standalone.rs index 8c604858186..159b1fb6085 100644 --- a/tests-integration/src/standalone.rs +++ b/tests-integration/src/standalone.rs @@ -91,6 +91,7 @@ pub struct GreptimeDbStandaloneBuilder { slow_query_options: SlowQueryOptions, event_recorder_options: EventRecorderOptions, auto_create_table: bool, + auto_create_physical_table_timestamp_unit: Option, logical_batcher: Option, table_batcher: BatcherOptions, max_in_flight_write_bytes: ReadableSize, @@ -114,6 +115,7 @@ impl GreptimeDbStandaloneBuilder { }, event_recorder_options: EventRecorderOptions::default(), auto_create_table: true, + auto_create_physical_table_timestamp_unit: None, logical_batcher: None, table_batcher: BatcherOptions::default(), max_in_flight_write_bytes: ReadableSize(0), @@ -163,6 +165,16 @@ impl GreptimeDbStandaloneBuilder { self } + /// Sets the time index unit for auto-created physical metric tables. + #[must_use] + pub fn with_auto_create_physical_table_timestamp_unit( + mut self, + unit: common_time::timestamp::TimeUnit, + ) -> Self { + self.auto_create_physical_table_timestamp_unit = Some(unit); + self + } + /// Sets the event recorder options for the standalone instance. #[must_use] pub fn with_event_recorder_options( @@ -420,6 +432,9 @@ impl GreptimeDbStandaloneBuilder { slow_query: self.slow_query_options.clone(), event_recorder: self.event_recorder_options.clone(), auto_create_table: self.auto_create_table, + auto_create_physical_table_timestamp_unit: self + .auto_create_physical_table_timestamp_unit + .unwrap_or(common_time::timestamp::TimeUnit::Millisecond), pipeline: self.pipeline_options.clone(), max_in_flight_write_bytes: self.max_in_flight_write_bytes, pending_rows_batcher: PendingRowsBatcherOptions { diff --git a/tests-integration/src/test_util.rs b/tests-integration/src/test_util.rs index 5eee6ea1fed..9742fb5bb61 100644 --- a/tests-integration/src/test_util.rs +++ b/tests-integration/src/test_util.rs @@ -913,6 +913,7 @@ pub fn build_test_prom_server( frontend_ref.table_flownode_set_cache().clone(), true, frontend_ref.clone(), + common_time::timestamp::TimeUnit::Millisecond, Duration::from_millis(50), 1000, 4, diff --git a/tests-integration/tests/http.rs b/tests-integration/tests/http.rs index 950feb8a6ca..f044e91cc09 100644 --- a/tests-integration/tests/http.rs +++ b/tests-integration/tests/http.rs @@ -3418,6 +3418,7 @@ providers = []"#, r#" enable_telemetry = true auto_create_table = true +auto_create_physical_table_timestamp_unit = "Millisecond" max_in_flight_write_bytes = "0KiB" write_bytes_exhausted_policy = "wait" init_regions_in_background = false