diff --git a/config/config.md b/config/config.md
index 20e785565f9..27aeabab52c 100644
--- a/config/config.md
+++ b/config/config.md
@@ -15,6 +15,7 @@
| `default_timezone` | String | Unset | The default timezone of the server. |
| `default_column_prefix` | String | Unset | The default column prefix for auto-created time index, value, native histogram, and exemplar-label columns.
Legacy OTLP summary columns keep their historical `greptime_` prefix. |
| `auto_create_table` | Bool | `true` | Server-side global switch for auto table creation on write.
When `false`, a missing table is never auto-created even if the request sets the `auto_create_table` hint to `true`. Default: `true`. |
+| `auto_create_physical_table_timestamp_unit` | String | Unset | Time index unit for physical metric tables created by auto table creation
(Prometheus remote write, OTLP metrics, and other metric-engine ingestion).
New logical tables created under such a physical table inherit its unit.
Options: "Second", "Millisecond" (default), "Microsecond", "Nanosecond". |
| `user_provider` | String | Unset | The user provider for authentication.
Examples: "static_user_provider:file:/path/to/users", "static_user_provider:cmd:greptime_user=greptime_pwd"
Password verifier formats: "plain:", "pbkdf2_sha256:::",
"mysql_native_password:",
"pg_scram_sha256::::"
"pbkdf2_sha256" and "pg_scram_sha256" protect passwords at rest, but such users cannot log in over MySQL:
`static_user_provider` and `watch_file_user_provider` only negotiate `mysql_native_password`, which cannot verify these formats.
"mysql_native_password" is MySQL-specific and cannot authenticate over PostgreSQL at all.
PostgreSQL SCRAM only covers "plain" and "pg_scram_sha256" users; if any user is "pbkdf2_sha256" or
"mysql_native_password", PostgreSQL falls back to cleartext password auth for every user.
For "pg_scram_sha256" users, keep the default iteration count (4096) and salt length (16): both are
observable in the SCRAM server-first message, and non-default values weaken resistance to username
enumeration. |
| `max_in_flight_write_bytes` | String | 0 | Shared quota for concurrent write request bodies and messages (HTTP, gRPC, Flight)
plus retained decoder memory and estimated PB-to-Arrow conversion memory for ordinary-table batching.
A batch must fit together with its request's existing reservations; only additional memory is acquired.
Set to 0 to disable the limit. Default: "0" (unlimited) |
| `write_bytes_exhausted_policy` | String | `wait` | Policy when write bytes quota is exhausted.
Options: "wait" (default, 10s timeout), "wait()" (e.g., "wait(30s)"), "fail"
Table-batch conversion never waits for additional memory. Under a wait policy,
temporary contention returns a retryable error; requests that cannot fit are rejected. |
@@ -293,6 +294,7 @@
| `default_timezone` | String | Unset | The default timezone of the server. |
| `default_column_prefix` | String | Unset | The default column prefix for auto-created time index, value, native histogram, and exemplar-label columns.
Legacy OTLP summary columns keep their historical `greptime_` prefix. |
| `auto_create_table` | Bool | `true` | Server-side global switch for auto table creation on write.
When `false`, a missing table is never auto-created even if the request sets the `auto_create_table` hint to `true`. Default: `true`. |
+| `auto_create_physical_table_timestamp_unit` | String | Unset | Time index unit for physical metric tables created by auto table creation
(Prometheus remote write, OTLP metrics, and other metric-engine ingestion).
New logical tables created under such a physical table inherit its unit.
Options: "Second", "Millisecond" (default), "Microsecond", "Nanosecond". |
| `user_provider` | String | Unset | The user provider for authentication.
Examples: "static_user_provider:file:/path/to/users", "static_user_provider:cmd:greptime_user=greptime_pwd"
Password verifier formats: "plain:", "pbkdf2_sha256:::",
"mysql_native_password:",
"pg_scram_sha256::::"
"pbkdf2_sha256" and "pg_scram_sha256" protect passwords at rest, but such users cannot log in over MySQL:
`static_user_provider` and `watch_file_user_provider` only negotiate `mysql_native_password`, which cannot verify these formats.
"mysql_native_password" is MySQL-specific and cannot authenticate over PostgreSQL at all.
PostgreSQL SCRAM only covers "plain" and "pg_scram_sha256" users; if any user is "pbkdf2_sha256" or
"mysql_native_password", PostgreSQL falls back to cleartext password auth for every user.
For "pg_scram_sha256" users, keep the default iteration count (4096) and salt length (16): both are
observable in the SCRAM server-first message, and non-default values weaken resistance to username
enumeration. |
| `max_in_flight_write_bytes` | String | 0 | Shared quota for concurrent write request bodies and messages (HTTP, gRPC, Flight)
plus retained decoder memory and estimated PB-to-Arrow conversion memory for ordinary-table batching.
A batch must fit together with its request's existing reservations; only additional memory is acquired.
Set to 0 to disable the limit. Default: "0" (unlimited) |
| `write_bytes_exhausted_policy` | String | `wait` | Policy when write bytes quota is exhausted.
Options: "wait" (default, 10s timeout), "wait()" (e.g., "wait(30s)"), "fail"
Table-batch conversion never waits for additional memory. Under a wait policy,
temporary contention returns a retryable error; requests that cannot fit are rejected. |
diff --git a/config/frontend.example.toml b/config/frontend.example.toml
index 9d15ca4c88e..b8a73240345 100644
--- a/config/frontend.example.toml
+++ b/config/frontend.example.toml
@@ -11,6 +11,13 @@ default_column_prefix = "greptime"
## When `false`, a missing table is never auto-created even if the request sets the `auto_create_table` hint to `true`. Default: `true`.
#+ auto_create_table = true
+## Time index unit for physical metric tables created by auto table creation
+## (Prometheus remote write, OTLP metrics, and other metric-engine ingestion).
+## New logical tables created under such a physical table inherit its unit.
+## Options: "Second", "Millisecond" (default), "Microsecond", "Nanosecond".
+## @toml2docs:none-default
+#+ auto_create_physical_table_timestamp_unit = "Millisecond"
+
## The user provider for authentication.
## Examples: "static_user_provider:file:/path/to/users", "static_user_provider:cmd:greptime_user=greptime_pwd"
## Password verifier formats: "plain:", "pbkdf2_sha256:::",
diff --git a/config/standalone.example.toml b/config/standalone.example.toml
index 2bd671640f7..616b6f4f795 100644
--- a/config/standalone.example.toml
+++ b/config/standalone.example.toml
@@ -11,6 +11,13 @@ default_column_prefix = "greptime"
## When `false`, a missing table is never auto-created even if the request sets the `auto_create_table` hint to `true`. Default: `true`.
#+ auto_create_table = true
+## Time index unit for physical metric tables created by auto table creation
+## (Prometheus remote write, OTLP metrics, and other metric-engine ingestion).
+## New logical tables created under such a physical table inherit its unit.
+## Options: "Second", "Millisecond" (default), "Microsecond", "Nanosecond".
+## @toml2docs:none-default
+#+ auto_create_physical_table_timestamp_unit = "Millisecond"
+
## The user provider for authentication.
## Examples: "static_user_provider:file:/path/to/users", "static_user_provider:cmd:greptime_user=greptime_pwd"
## Password verifier formats: "plain:", "pbkdf2_sha256:::",
diff --git a/src/cmd/tests/load_config_test.rs b/src/cmd/tests/load_config_test.rs
index 129fbc5a81c..4f2a68ba988 100644
--- a/src/cmd/tests/load_config_test.rs
+++ b/src/cmd/tests/load_config_test.rs
@@ -25,6 +25,7 @@ use common_options::datanode::{ClientOptions, DatanodeClientOptions};
use common_runtime::global::RuntimeOptions;
use common_telemetry::logging::{DEFAULT_LOGGING_DIR, DEFAULT_OTLP_HTTP_ENDPOINT, LoggingOptions};
use common_test_util::temp_dir::create_named_temp_file;
+use common_time::timestamp::TimeUnit;
use common_wal::config::DatanodeWalConfig;
use common_wal::config::raft_engine::RaftEngineConfig;
use datanode::config::{DatanodeOptions, RegionEngineConfig, StorageConfig};
@@ -249,6 +250,7 @@ fn test_load_frontend_example_config() {
default_timezone: Some("UTC".to_string()),
default_column_prefix: Some("greptime".to_string()),
auto_create_table: true,
+ auto_create_physical_table_timestamp_unit: TimeUnit::Millisecond,
meta_client: Some(MetaClientOptions {
metasrv_addrs: vec!["127.0.0.1:3002".to_string()],
timeout: Duration::from_secs(3),
@@ -430,6 +432,7 @@ fn test_load_standalone_example_config() {
default_timezone: Some("UTC".to_string()),
default_column_prefix: Some("greptime".to_string()),
auto_create_table: true,
+ auto_create_physical_table_timestamp_unit: TimeUnit::Millisecond,
wal: DatanodeWalConfig::RaftEngine(RaftEngineConfig {
dir: Some(format!("{}/{}", DEFAULT_DATA_HOME, WAL_DIR)),
sync_period: Some(Duration::from_secs(5)),
diff --git a/src/frontend/src/frontend.rs b/src/frontend/src/frontend.rs
index 865a64a5d7e..dc669313a55 100644
--- a/src/frontend/src/frontend.rs
+++ b/src/frontend/src/frontend.rs
@@ -23,6 +23,7 @@ use common_memory_manager::OnExhaustedPolicy;
use common_options::datanode::DatanodeClientOptions;
use common_options::memory::MemoryOptions;
use common_telemetry::logging::{LoggingOptions, SlowQueryOptions, TracingOptions};
+use common_time::timestamp::TimeUnit;
use meta_client::{MetaClientOptions, MetaClientRef};
use pipeline::PipelineOptions;
use query::options::QueryOptions;
@@ -53,6 +54,11 @@ pub struct FrontendOptions {
/// even if a request sets the `auto_create_table` hint to `true`. When `true`
/// (default), the per-request hint still applies. Default: `true`.
pub auto_create_table: bool,
+ /// Time index unit for physical metric tables created by auto table creation
+ /// (Prometheus remote write, OTLP metrics, and other metric-engine ingestion).
+ /// New logical tables created under such a physical table inherit its unit.
+ /// Options: "Second", "Millisecond" (default), "Microsecond", "Nanosecond".
+ pub auto_create_physical_table_timestamp_unit: TimeUnit,
/// Shared quota for write bodies/messages, retained decoder memory, and estimated
/// ordinary-table batch conversion memory. A batch must fit alongside its request's
/// existing reservations; only additional memory is acquired.
@@ -109,6 +115,7 @@ impl Default for FrontendOptions {
default_timezone: None,
default_column_prefix: None,
auto_create_table: true,
+ auto_create_physical_table_timestamp_unit: TimeUnit::Millisecond,
max_in_flight_write_bytes: ReadableSize(0),
write_bytes_exhausted_policy: OnExhaustedPolicy::default(),
http: HttpOptions::default(),
diff --git a/src/frontend/src/instance.rs b/src/frontend/src/instance.rs
index ba735d1226d..6d8f98fdc0c 100644
--- a/src/frontend/src/instance.rs
+++ b/src/frontend/src/instance.rs
@@ -64,6 +64,7 @@ use common_recordbatch::RecordBatchStreamWrapper;
use common_recordbatch::error::StreamTimeoutSnafu;
use common_telemetry::logging::SlowQueryOptions;
use common_telemetry::{debug, error, tracing};
+use common_time::timestamp::TimeUnit;
use datafusion::dataframe::DataFrame;
use datafusion::physical_plan::ExecutionPlan;
use datafusion_expr::LogicalPlan;
@@ -133,6 +134,9 @@ pub struct Instance {
process_manager: ProcessManagerRef,
slow_query_options: SlowQueryOptions,
influxdb_default_merge_mode: InfluxdbMergeMode,
+ /// Time index unit for physical metric tables created by auto table creation
+ /// through the frontend instance (e.g., batched logical-table flushes).
+ auto_create_physical_table_timestamp_unit: TimeUnit,
trace_ingest_chunk_size: usize,
trace_aux_cache: otlp::TraceAuxCache,
trace_aux_limiter: Option,
diff --git a/src/frontend/src/instance/builder.rs b/src/frontend/src/instance/builder.rs
index 4056d78a480..3703c5bedea 100644
--- a/src/frontend/src/instance/builder.rs
+++ b/src/frontend/src/instance/builder.rs
@@ -253,6 +253,9 @@ impl FrontendBuilder {
table_flownode_cache.clone(),
self.options.auto_create_table,
)
+ .with_auto_create_physical_table_timestamp_unit(
+ self.options.auto_create_physical_table_timestamp_unit,
+ )
.with_write_dispatcher(write_dispatcher.clone())
};
// The execution-only inserter owns no batchers, avoiding an Arc cycle.
@@ -439,6 +442,9 @@ impl FrontendBuilder {
process_manager,
slow_query_options: self.options.slow_query.clone(),
influxdb_default_merge_mode: self.options.influxdb.default_merge_mode,
+ auto_create_physical_table_timestamp_unit: self
+ .options
+ .auto_create_physical_table_timestamp_unit,
trace_ingest_chunk_size: self.options.otlp.trace_ingest_chunk_size,
trace_aux_cache: TraceAuxCache::new(self.options.otlp.trace_aux_cache_size.as_bytes()),
trace_aux_limiter,
diff --git a/src/frontend/src/instance/logical_batcher.rs b/src/frontend/src/instance/logical_batcher.rs
index b4755ebc974..3d1d7165179 100644
--- a/src/frontend/src/instance/logical_batcher.rs
+++ b/src/frontend/src/instance/logical_batcher.rs
@@ -53,6 +53,7 @@ impl Instance {
self.table_flownode_set_cache().clone(),
true,
Arc::new(LogicalTables(Arc::downgrade(self))),
+ options.auto_create_physical_table_timestamp_unit,
options_batcher.pending_rows_flush_interval,
options_batcher.max_batch_rows,
options_batcher.max_concurrent_flushes,
diff --git a/src/frontend/src/instance/prom_store.rs b/src/frontend/src/instance/prom_store.rs
index 963a8ae22ed..4d13d391843 100644
--- a/src/frontend/src/instance/prom_store.rs
+++ b/src/frontend/src/instance/prom_store.rs
@@ -788,7 +788,9 @@ impl Instance {
let default_schema = vec![
api::v1::ColumnSchema {
column_name: common_query::prelude::greptime_timestamp().to_string(),
- datatype: api::v1::ColumnDataType::TimestampMillisecond as i32,
+ datatype: api::helper::timestamp_datatype(
+ self.auto_create_physical_table_timestamp_unit,
+ ) as i32,
semantic_type: api::v1::SemanticType::Timestamp as i32,
datatype_extension: None,
options: None,
diff --git a/src/operator/src/insert.rs b/src/operator/src/insert.rs
index 7a6907edc9c..e69063734db 100644
--- a/src/operator/src/insert.rs
+++ b/src/operator/src/insert.rs
@@ -120,6 +120,11 @@ pub struct Inserter {
/// When `false`, missing tables are never auto-created regardless of the
/// per-request `auto_create_table` hint. When `true`, the hint still applies.
auto_create_table: bool,
+ /// Time index unit for physical metric tables created by auto table creation.
+ /// New logical tables created under such a physical table inherit its unit,
+ /// so all metric-engine ingestion paths write timestamps in this unit.
+ /// Defaults to milliseconds for compatibility.
+ auto_create_physical_table_timestamp_unit: TimeUnit,
pending_rows_batcher: Option>,
/// Rows handed to detached flow mirror tasks that have not finished yet.
/// Bounds how much row data in-flight mirror writes may keep alive, see
@@ -342,11 +347,18 @@ impl Inserter {
node_manager,
table_flownode_set_cache,
auto_create_table,
+ auto_create_physical_table_timestamp_unit: TimeUnit::Millisecond,
pending_rows_batcher: None,
mirror_pending_rows: Arc::new(AtomicU64::new(0)),
}
}
+ /// Sets the time index unit for auto-created physical metric tables.
+ pub fn with_auto_create_physical_table_timestamp_unit(mut self, unit: TimeUnit) -> Self {
+ self.auto_create_physical_table_timestamp_unit = unit;
+ self
+ }
+
/// Installs the shared dispatcher used by direct writes and batch flushes.
pub fn with_write_dispatcher(mut self, dispatcher: WriteDispatcherRef) -> Self {
self.write_dispatcher = dispatcher;
@@ -1868,7 +1880,9 @@ impl Inserter {
let default_schema = vec![
ColumnSchema {
column_name: greptime_timestamp().to_string(),
- datatype: ColumnDataType::TimestampMillisecond as _,
+ datatype: api::helper::timestamp_datatype(
+ self.auto_create_physical_table_timestamp_unit,
+ ) as _,
semantic_type: SemanticType::Timestamp as _,
datatype_extension: None,
options: None,
diff --git a/src/operator/src/insert/otlp.rs b/src/operator/src/insert/otlp.rs
index 9405480d3a2..aa6bf665eb6 100644
--- a/src/operator/src/insert/otlp.rs
+++ b/src/operator/src/insert/otlp.rs
@@ -118,12 +118,9 @@ impl Inserter {
tables.insert(backing.clone(), table);
}
physical_columns.entry(backing.as_str()).or_insert_with(|| {
- PhysicalColumns::new(
- tables[backing]
- .as_ref()
- .map(table_columns)
- .unwrap_or_else(default_physical_columns),
- )
+ PhysicalColumns::new(tables[backing].as_ref().map(table_columns).unwrap_or_else(
+ || default_physical_columns(self.auto_create_physical_table_timestamp_unit),
+ ))
});
}
// Match ordinary insertion: impure defaults are evaluated once per table
@@ -176,7 +173,9 @@ impl Inserter {
let unit = target
.and_then(table_time_index_unit)
.or_else(|| physical.and_then(table_time_index_unit))
- .unwrap_or(TimeUnit::Millisecond);
+ // The physical table will be auto-created in the configured
+ // unit, so validate its rows in that unit as well.
+ .unwrap_or(self.auto_create_physical_table_timestamp_unit);
if let Some(rows) = &mut request.rows {
convert_rows_time_unit(rows, unit)?;
}
@@ -378,13 +377,13 @@ fn table_columns(table: &TableRef) -> Vec<(DataColumn, SemanticType)> {
.collect()
}
-fn default_physical_columns() -> Vec<(DataColumn, SemanticType)> {
+fn default_physical_columns(unit: TimeUnit) -> Vec<(DataColumn, SemanticType)> {
use datatypes::prelude::ConcreteDataType;
vec![
(
DataColumn::new(
greptime_timestamp(),
- ConcreteDataType::timestamp_millisecond_datatype(),
+ ConcreteDataType::timestamp_datatype(unit),
false,
)
.with_time_index(true),
@@ -698,7 +697,7 @@ mod tests {
#[test]
fn test_preflight_omissions_defaults_and_explicit_nulls() {
- let mut columns = default_physical_columns();
+ let mut columns = default_physical_columns(TimeUnit::Millisecond);
columns.push((
DataColumn::new("host", ConcreteDataType::string_datatype(), false)
.with_default_constraint(Some(ColumnDefaultConstraint::Value(DataValue::String(
@@ -739,7 +738,7 @@ mod tests {
#[test]
fn test_preflight_existing_columns_and_physical_definitions() {
- let columns = default_physical_columns();
+ let columns = default_physical_columns(TimeUnit::Millisecond);
let mut rows = Rows {
schema: vec![ColumnSchema {
column_name: greptime_timestamp().into(),
@@ -805,7 +804,7 @@ mod tests {
(defaulted, None),
(impure, Some("impure default for physical field `impure`")),
] {
- let mut columns = default_physical_columns();
+ let mut columns = default_physical_columns(TimeUnit::Millisecond);
columns.extend((0..1024).map(|index| {
(
DataColumn::new(
diff --git a/src/servers/src/batcher/logical_table.rs b/src/servers/src/batcher/logical_table.rs
index 707afe2ea0f..b7c0817e03a 100644
--- a/src/servers/src/batcher/logical_table.rs
+++ b/src/servers/src/batcher/logical_table.rs
@@ -108,6 +108,10 @@ pub struct LogicalTablePendingRowsBatcher {
worker_channel_capacity: usize,
prom_store_with_metric_engine: bool,
schema_alterer: PendingRowsSchemaAltererRef,
+ /// Time index unit the schema alterer uses when auto-creating a missing
+ /// physical metric table; keeps the batcher's logical-create alignment
+ /// consistent with the created physical table.
+ auto_create_physical_table_timestamp_unit: TimeUnit,
pending_rows_batch_sync: bool,
shutdown: broadcast::Sender<()>,
}
@@ -123,6 +127,7 @@ impl LogicalTablePendingRowsBatcher {
table_flownode_set_cache: TableFlownodeSetCacheRef,
prom_store_with_metric_engine: bool,
schema_alterer: PendingRowsSchemaAltererRef,
+ auto_create_physical_table_timestamp_unit: TimeUnit,
flush_interval: Duration,
max_batch_rows: usize,
max_concurrent_flushes: usize,
@@ -163,6 +168,7 @@ impl LogicalTablePendingRowsBatcher {
flow_notification_tx,
prom_store_with_metric_engine,
schema_alterer,
+ auto_create_physical_table_timestamp_unit,
flush_limiter,
request_limiter,
worker_channel_capacity,
@@ -184,8 +190,8 @@ impl LogicalTablePendingRowsBatcher {
}
/// Returns the physical metric table's time index unit resolved from
- /// `ctx`, defaulting to millisecond when the table does not exist yet
- /// (the schema alterer auto-creates it as millisecond).
+ /// `ctx`, falling back to the configured auto-create unit when the table
+ /// does not exist yet (the schema alterer auto-creates it with that unit).
async fn physical_time_index_unit_or_default(&self, ctx: &QueryContextRef) -> TimeUnit {
let key = batch_key_from_ctx(ctx);
let Ok(Some(table)) = self
@@ -193,7 +199,7 @@ impl LogicalTablePendingRowsBatcher {
.table(&key.catalog, &key.schema, &key.physical_table, None)
.await
else {
- return TimeUnit::Millisecond;
+ return self.auto_create_physical_table_timestamp_unit;
};
table
.table_info()
diff --git a/src/servers/src/batcher/logical_table/tables.rs b/src/servers/src/batcher/logical_table/tables.rs
index 140dd2d4f23..8aef0df0b39 100644
--- a/src/servers/src/batcher/logical_table/tables.rs
+++ b/src/servers/src/batcher/logical_table/tables.rs
@@ -94,8 +94,8 @@ impl LogicalTablePendingRowsBatcher {
// New tables are created on the request's selected physical table;
// their time index must use the physical table's unit (a missing
- // physical table is auto-created as millisecond by the schema
- // alterer, matching the default here).
+ // physical table is auto-created in the configured auto-create unit
+ // by the schema alterer, matching the fallback here).
if !plan.tables_to_create.is_empty() {
let physical_unit = self.physical_time_index_unit_or_default(ctx).await;
for (_, request_schema) in &mut plan.tables_to_create {
diff --git a/src/servers/src/batcher/logical_table/test_util.rs b/src/servers/src/batcher/logical_table/test_util.rs
index 364cf0c8f5a..6cf95af1f1c 100644
--- a/src/servers/src/batcher/logical_table/test_util.rs
+++ b/src/servers/src/batcher/logical_table/test_util.rs
@@ -32,6 +32,7 @@ use common_meta::node_manager::{
};
use common_meta::test_util::MockDatanodeManager;
use common_query::prelude::GREPTIME_PHYSICAL_TABLE;
+use common_time::timestamp::TimeUnit;
use datatypes::prelude::ConcreteDataType;
use datatypes::schema::{ColumnSchema as DtColumnSchema, Schema as DtSchema};
use operator::test_util::{create_partition_rule_manager, prepare_mocked_backend};
@@ -110,6 +111,7 @@ pub(crate) async fn submission_batcher(
mock_table_flownode_cache(1, vec![]).await,
true,
Arc::new(UnexpectedSchemaAlterer),
+ TimeUnit::Millisecond,
Duration::from_secs(1),
2,
1,
diff --git a/src/standalone/src/options.rs b/src/standalone/src/options.rs
index e2971712e4b..3c7a9e6ce13 100644
--- a/src/standalone/src/options.rs
+++ b/src/standalone/src/options.rs
@@ -21,6 +21,7 @@ use common_event_recorder::EventRecorderOptions;
use common_memory_manager::OnExhaustedPolicy;
use common_options::memory::MemoryOptions;
use common_telemetry::logging::{LoggingOptions, SlowQueryOptions, TracingOptions};
+use common_time::timestamp::TimeUnit;
use common_wal::config::DatanodeWalConfig;
use datanode::config::{DatanodeOptions, ProcedureConfig, RegionEngineConfig, StorageConfig};
use file_engine::config::EngineConfig as FileEngineConfig;
@@ -47,6 +48,11 @@ pub struct StandaloneOptions {
/// Upper bound: when `false`, missing tables are never auto-created even if a
/// request sets the `auto_create_table` hint to `true`. Default: `true`.
pub auto_create_table: bool,
+ /// Time index unit for physical metric tables created by auto table creation
+ /// (Prometheus remote write, OTLP metrics, and other metric-engine ingestion).
+ /// New logical tables created under such a physical table inherit its unit.
+ /// Options: "Second", "Millisecond" (default), "Microsecond", "Nanosecond".
+ pub auto_create_physical_table_timestamp_unit: TimeUnit,
/// Shared quota for write bodies/messages, retained decoder memory, and estimated
/// ordinary-table batch conversion memory. A batch must fit alongside its request's
/// existing reservations; only additional memory is acquired.
@@ -111,6 +117,7 @@ impl Default for StandaloneOptions {
default_timezone: None,
default_column_prefix: None,
auto_create_table: true,
+ auto_create_physical_table_timestamp_unit: TimeUnit::Millisecond,
max_in_flight_write_bytes: ReadableSize(0),
write_bytes_exhausted_policy: OnExhaustedPolicy::default(),
http: HttpOptions::default(),
@@ -181,6 +188,8 @@ impl StandaloneOptions {
FrontendOptions {
default_timezone: cloned_opts.default_timezone,
auto_create_table: cloned_opts.auto_create_table,
+ auto_create_physical_table_timestamp_unit: cloned_opts
+ .auto_create_physical_table_timestamp_unit,
max_in_flight_write_bytes: cloned_opts.max_in_flight_write_bytes,
write_bytes_exhausted_policy: cloned_opts.write_bytes_exhausted_policy,
http: cloned_opts.http,
diff --git a/tests-integration/src/otlp.rs b/tests-integration/src/otlp.rs
index 70133835aa0..8e193843627 100644
--- a/tests-integration/src/otlp.rs
+++ b/tests-integration/src/otlp.rs
@@ -3999,6 +3999,94 @@ WITH(
}
}
+ #[tokio::test(flavor = "multi_thread")]
+ async fn test_otlp_initial_write_auto_create_physical_timestamp_unit() {
+ use common_time::timestamp::TimeUnit as ConfigTimeUnit;
+
+ for (unit, arrow_unit, expected) in [
+ (ConfigTimeUnit::Second, TimeUnit::Second, 60),
+ (ConfigTimeUnit::Millisecond, TimeUnit::Millisecond, 60_123),
+ (
+ ConfigTimeUnit::Microsecond,
+ TimeUnit::Microsecond,
+ 60_123_456,
+ ),
+ (
+ ConfigTimeUnit::Nanosecond,
+ TimeUnit::Nanosecond,
+ 60_123_456_789,
+ ),
+ ] {
+ let standalone = GreptimeDbStandaloneBuilder::new(&format!("otlp_initial_{unit:?}"))
+ .with_auto_create_physical_table_timestamp_unit(unit)
+ .build()
+ .await;
+ let instance = standalone.fe_instance();
+ let mut ctx = QueryContext::with(DEFAULT_CATALOG_NAME, "public");
+ ctx.set_protocol_ctx(ProtocolCtx::OtlpMetric(OtlpMetricCtx {
+ with_metric_engine: true,
+ ..Default::default()
+ }));
+ let ctx = Arc::new(ctx);
+
+ // No pre-created tables: preflight must accept the first point so
+ // ingestion can create both tables with the configured time index.
+ let outcome = instance
+ .metrics(
+ metric_request(vec![gauge_metric(
+ "initial_metric",
+ &[(60_123_456_789, 1.0, None)],
+ )]),
+ ctx.clone(),
+ None,
+ )
+ .await
+ .unwrap();
+ assert_eq!(
+ (outcome.accepted_data_points, outcome.rejected_data_points),
+ (1, 0),
+ "unit={unit:?}"
+ );
+
+ for table in ["greptime_physical_table", "initial_metric"] {
+ let output = instance
+ .do_query(
+ &format!("SELECT greptime_timestamp FROM {table}"),
+ ctx.clone(),
+ )
+ .await
+ .remove(0)
+ .unwrap();
+ let OutputData::Stream(stream) = output.data else {
+ panic!("expected stream")
+ };
+ let batches = RecordBatches::try_collect(stream).await.unwrap().take();
+ assert_eq!(
+ batches.iter().map(|batch| batch.num_rows()).sum::(),
+ 1
+ );
+ let timestamp = batches[0].column(0);
+ assert_eq!(
+ timestamp.data_type(),
+ &DataType::Timestamp(arrow_unit, None)
+ );
+ let actual = match arrow_unit {
+ TimeUnit::Second => timestamp.as_primitive::().value(0),
+ TimeUnit::Millisecond => timestamp
+ .as_primitive::()
+ .value(0),
+ TimeUnit::Microsecond => timestamp
+ .as_primitive::()
+ .value(0),
+ TimeUnit::Nanosecond => {
+ timestamp.as_primitive::().value(0)
+ }
+ };
+ assert_eq!(actual, expected, "unit={unit:?}, table={table}");
+ }
+ }
+ }
+
#[tokio::test(flavor = "multi_thread")]
async fn test_otlp_preflight_shared_physical_and_fanout_conflicts() {
let standalone = GreptimeDbStandaloneBuilder::new("otlp_physical_conflicts")
diff --git a/tests-integration/src/prom_store.rs b/tests-integration/src/prom_store.rs
index de72d6501ec..c248f5124f3 100644
--- a/tests-integration/src/prom_store.rs
+++ b/tests-integration/src/prom_store.rs
@@ -478,6 +478,202 @@ mod tests {
.await;
}
+ #[tokio::test(flavor = "multi_thread")]
+ async fn test_standalone_prom_remote_write_auto_create_microsecond_physical_table() {
+ common_telemetry::init_default_ut_logging();
+ let standalone = GreptimeDbStandaloneBuilder::new("test_prom_rw_auto_create_us_physical")
+ .with_auto_create_physical_table_timestamp_unit(
+ common_time::timestamp::TimeUnit::Microsecond,
+ )
+ .build()
+ .await;
+ let instance = standalone.fe_instance();
+
+ let db = "prometheus_auto_create_us";
+ let ctx = Arc::new(QueryContext::with(DEFAULT_CATALOG_NAME, db));
+ assert!(
+ SqlQueryHandler::do_query(
+ instance.as_ref(),
+ &format!("CREATE DATABASE IF NOT EXISTS {db}"),
+ ctx.clone(),
+ )
+ .await
+ .first()
+ .unwrap()
+ .is_ok()
+ );
+
+ // No manual table creation: both the physical table and the logical
+ // table `metric1` must be auto-created by the write, with a
+ // microsecond time index inherited from the physical table.
+ let write_request = WriteRequest {
+ timeseries: vec![prom_store::mock_timeseries()[0].clone()],
+ ..Default::default()
+ };
+ let (row_inserts, _) = to_grpc_row_insert_requests(&write_request).unwrap();
+ instance
+ .write(row_inserts, ctx.clone(), true)
+ .await
+ .unwrap();
+
+ for table in ["greptime_physical_table", "metric1"] {
+ let mut output = instance
+ .do_query(
+ &format!("SELECT greptime_timestamp FROM {table}"),
+ ctx.clone(),
+ )
+ .await;
+ let OutputData::Stream(stream) = output.remove(0).unwrap().data else {
+ unreachable!()
+ };
+ let batches = common_recordbatch::RecordBatches::try_collect(stream)
+ .await
+ .unwrap()
+ .take();
+ assert_eq!(batches[0].num_rows(), 2, "unexpected rows in {table}");
+ let ts_column = batches[0].column(0);
+ assert_eq!(
+ ts_column.data_type(),
+ &DataType::Timestamp(TimeUnit::Microsecond, None),
+ "unexpected time index type in {table}"
+ );
+ let timestamps = ts_column
+ .as_primitive::()
+ .values();
+ assert_eq!(timestamps.to_vec(), vec![1_000_000, 2_000_000]);
+ }
+
+ // Remote read narrows microsecond timestamps back to milliseconds.
+ let read_request = ReadRequest {
+ queries: vec![Query {
+ start_timestamp_ms: 1000,
+ end_timestamp_ms: 2000,
+ matchers: vec![LabelMatcher {
+ name: prom_store::METRIC_NAME_LABEL.to_string(),
+ value: "metric1".to_string(),
+ r#type: 0,
+ }],
+ ..Default::default()
+ }],
+ ..Default::default()
+ };
+ let resp = instance.read(read_request, ctx.clone()).await.unwrap();
+ let body = prom_store::snappy_decompress(&resp.body).unwrap();
+ let read_response = ReadResponse::decode(&body[..]).unwrap();
+ let samples = &read_response.results[0].timeseries[0].samples;
+ assert_eq!(
+ samples.clone(),
+ vec![
+ Sample {
+ value: 1.0,
+ timestamp: 1000
+ },
+ Sample {
+ value: 2.0,
+ timestamp: 2000
+ }
+ ]
+ );
+ }
+
+ #[tokio::test(flavor = "multi_thread")]
+ async fn test_standalone_prom_batcher_auto_create_microsecond_physical_table() {
+ use servers::batcher::logical_table::PendingRowsSchemaAlterer;
+
+ common_telemetry::init_default_ut_logging();
+ let standalone = GreptimeDbStandaloneBuilder::new("test_prom_batcher_auto_create_us")
+ .with_auto_create_physical_table_timestamp_unit(
+ common_time::timestamp::TimeUnit::Microsecond,
+ )
+ .build()
+ .await;
+ let instance = standalone.fe_instance();
+
+ // Drives the batcher's schema-alterer path: a missing physical table is
+ // auto-created with the configured microsecond unit, and new logical
+ // tables aligned to it.
+ let db = "prometheus_batcher_auto_create_us";
+ let ctx = Arc::new(QueryContext::with(DEFAULT_CATALOG_NAME, db));
+ assert!(
+ SqlQueryHandler::do_query(
+ instance.as_ref(),
+ &format!("CREATE DATABASE IF NOT EXISTS {db}"),
+ ctx.clone(),
+ )
+ .await
+ .first()
+ .unwrap()
+ .is_ok()
+ );
+ // The batcher aligns request schemas to the physical table's unit
+ // before invoking the schema alterer, so the schema below arrives in
+ // the configured microsecond unit. A missing physical table must be
+ // auto-created with the same unit.
+ let request_schema = vec![
+ api::v1::ColumnSchema {
+ column_name: "greptime_timestamp".to_string(),
+ datatype: ColumnDataType::TimestampMicrosecond as i32,
+ semantic_type: SemanticType::Timestamp as i32,
+ datatype_extension: None,
+ options: None,
+ },
+ api::v1::ColumnSchema {
+ column_name: "greptime_value".to_string(),
+ datatype: ColumnDataType::Float64 as i32,
+ semantic_type: SemanticType::Field as i32,
+ datatype_extension: None,
+ options: None,
+ },
+ api::v1::ColumnSchema {
+ column_name: "job".to_string(),
+ datatype: ColumnDataType::String as i32,
+ semantic_type: SemanticType::Tag as i32,
+ datatype_extension: None,
+ options: None,
+ },
+ ];
+ instance
+ .create_tables_if_missing_batch(
+ DEFAULT_CATALOG_NAME,
+ db,
+ &[("metric1", request_schema.as_slice())],
+ true,
+ ctx.clone(),
+ )
+ .await
+ .unwrap();
+
+ for table in ["greptime_physical_table", "metric1"] {
+ let mut output = instance
+ .do_query(
+ &format!(
+ "SELECT greptime_data_type FROM information_schema.columns \
+ WHERE table_schema='{db}' AND table_name='{table}' \
+ AND column_name='greptime_timestamp'"
+ ),
+ ctx.clone(),
+ )
+ .await;
+ let OutputData::Stream(stream) = output.remove(0).unwrap().data else {
+ unreachable!()
+ };
+ let batches = common_recordbatch::RecordBatches::try_collect(stream)
+ .await
+ .unwrap()
+ .take();
+ assert_eq!(
+ batches[0].num_rows(),
+ 1,
+ "missing time index column in {table}"
+ );
+ let data_type = batches[0].column(0).as_string::().value(0);
+ assert_eq!(
+ data_type, "TimestampMicrosecond",
+ "unexpected time index type in {table}"
+ );
+ }
+ }
+
/// Regression test for :
/// prometheus remote write/read against a physical metric table
/// pre-created with a non-millisecond time index. Millisecond samples are
diff --git a/tests-integration/src/standalone.rs b/tests-integration/src/standalone.rs
index 8c604858186..159b1fb6085 100644
--- a/tests-integration/src/standalone.rs
+++ b/tests-integration/src/standalone.rs
@@ -91,6 +91,7 @@ pub struct GreptimeDbStandaloneBuilder {
slow_query_options: SlowQueryOptions,
event_recorder_options: EventRecorderOptions,
auto_create_table: bool,
+ auto_create_physical_table_timestamp_unit: Option,
logical_batcher: Option,
table_batcher: BatcherOptions,
max_in_flight_write_bytes: ReadableSize,
@@ -114,6 +115,7 @@ impl GreptimeDbStandaloneBuilder {
},
event_recorder_options: EventRecorderOptions::default(),
auto_create_table: true,
+ auto_create_physical_table_timestamp_unit: None,
logical_batcher: None,
table_batcher: BatcherOptions::default(),
max_in_flight_write_bytes: ReadableSize(0),
@@ -163,6 +165,16 @@ impl GreptimeDbStandaloneBuilder {
self
}
+ /// Sets the time index unit for auto-created physical metric tables.
+ #[must_use]
+ pub fn with_auto_create_physical_table_timestamp_unit(
+ mut self,
+ unit: common_time::timestamp::TimeUnit,
+ ) -> Self {
+ self.auto_create_physical_table_timestamp_unit = Some(unit);
+ self
+ }
+
/// Sets the event recorder options for the standalone instance.
#[must_use]
pub fn with_event_recorder_options(
@@ -420,6 +432,9 @@ impl GreptimeDbStandaloneBuilder {
slow_query: self.slow_query_options.clone(),
event_recorder: self.event_recorder_options.clone(),
auto_create_table: self.auto_create_table,
+ auto_create_physical_table_timestamp_unit: self
+ .auto_create_physical_table_timestamp_unit
+ .unwrap_or(common_time::timestamp::TimeUnit::Millisecond),
pipeline: self.pipeline_options.clone(),
max_in_flight_write_bytes: self.max_in_flight_write_bytes,
pending_rows_batcher: PendingRowsBatcherOptions {
diff --git a/tests-integration/src/test_util.rs b/tests-integration/src/test_util.rs
index 5eee6ea1fed..9742fb5bb61 100644
--- a/tests-integration/src/test_util.rs
+++ b/tests-integration/src/test_util.rs
@@ -913,6 +913,7 @@ pub fn build_test_prom_server(
frontend_ref.table_flownode_set_cache().clone(),
true,
frontend_ref.clone(),
+ common_time::timestamp::TimeUnit::Millisecond,
Duration::from_millis(50),
1000,
4,
diff --git a/tests-integration/tests/http.rs b/tests-integration/tests/http.rs
index 950feb8a6ca..f044e91cc09 100644
--- a/tests-integration/tests/http.rs
+++ b/tests-integration/tests/http.rs
@@ -3418,6 +3418,7 @@ providers = []"#,
r#"
enable_telemetry = true
auto_create_table = true
+auto_create_physical_table_timestamp_unit = "Millisecond"
max_in_flight_write_bytes = "0KiB"
write_bytes_exhausted_policy = "wait"
init_regions_in_background = false