mirror of
https://github.com/GreptimeTeam/greptimedb.git
synced 2026-11-18 00:34:11 +00:00
feat(ingestion): configurable time index unit for auto-created physical metric tables (#10189)
* feat(ingestion): configurable time index unit for auto-created physical metric tables Auto-created physical metric tables hardcoded a millisecond time index. Add an `auto_create_physical_table_timestamp_unit` option (frontend and standalone configs, default "Millisecond") that selects the time index unit for physical metric tables created by auto table creation. New logical tables created under such a physical table inherit its unit, so Prometheus remote write, OTLP metrics, and other metric-engine ingestion paths write timestamps in the configured unit. The option threads through all auto-create sites: the operator inserter's `create_physical_table_on_demand`, the frontend instance's batched logical-table flush path (`create_metric_physical_table_if_missing`), the logical-table batcher's unit alignment fallback, and the OTLP metric preflight validation unit. Millisecond remains the default, so existing behavior is unchanged unless configured. Signed-off-by: Ning Sun <sunning@greptime.com> * fix(otlp): match default physical schema to configured timestamp unit Signed-off-by: Ning Sun <sunning@greptime.com> --------- Signed-off-by: Ning Sun <sunning@greptime.com>
This commit is contained in:
@@ -15,6 +15,7 @@
|
||||
| `default_timezone` | String | Unset | The default timezone of the server. |
|
||||
| `default_column_prefix` | String | Unset | The default column prefix for auto-created time index, value, native histogram, and exemplar-label columns.<br/>Legacy OTLP summary columns keep their historical `greptime_` prefix. |
|
||||
| `auto_create_table` | Bool | `true` | Server-side global switch for auto table creation on write.<br/>When `false`, a missing table is never auto-created even if the request sets the `auto_create_table` hint to `true`. Default: `true`. |
|
||||
| `auto_create_physical_table_timestamp_unit` | String | Unset | Time index unit for physical metric tables created by auto table creation<br/>(Prometheus remote write, OTLP metrics, and other metric-engine ingestion).<br/>New logical tables created under such a physical table inherit its unit.<br/>Options: "Second", "Millisecond" (default), "Microsecond", "Nanosecond". |
|
||||
| `user_provider` | String | Unset | The user provider for authentication.<br/>Examples: "static_user_provider:file:/path/to/users", "static_user_provider:cmd:greptime_user=greptime_pwd"<br/>Password verifier formats: "plain:<password>", "pbkdf2_sha256:<iterations>:<hex_salt>:<hex_hash>",<br/>"mysql_native_password:<hex_sha1_sha1_password>",<br/>"pg_scram_sha256:<iterations>:<hex_salt>:<hex_stored_key>:<hex_server_key>"<br/>"pbkdf2_sha256" and "pg_scram_sha256" protect passwords at rest, but such users cannot log in over MySQL:<br/>`static_user_provider` and `watch_file_user_provider` only negotiate `mysql_native_password`, which cannot verify these formats.<br/>"mysql_native_password" is MySQL-specific and cannot authenticate over PostgreSQL at all.<br/>PostgreSQL SCRAM only covers "plain" and "pg_scram_sha256" users; if any user is "pbkdf2_sha256" or<br/>"mysql_native_password", PostgreSQL falls back to cleartext password auth for every user.<br/>For "pg_scram_sha256" users, keep the default iteration count (4096) and salt length (16): both are<br/>observable in the SCRAM server-first message, and non-default values weaken resistance to username<br/>enumeration. |
|
||||
| `max_in_flight_write_bytes` | String | 0 | Shared quota for concurrent write request bodies and messages (HTTP, gRPC, Flight)<br/>plus retained decoder memory and estimated PB-to-Arrow conversion memory for ordinary-table batching.<br/>A batch must fit together with its request's existing reservations; only additional memory is acquired.<br/>Set to 0 to disable the limit. Default: "0" (unlimited) |
|
||||
| `write_bytes_exhausted_policy` | String | `wait` | Policy when write bytes quota is exhausted.<br/>Options: "wait" (default, 10s timeout), "wait(<duration>)" (e.g., "wait(30s)"), "fail"<br/>Table-batch conversion never waits for additional memory. Under a wait policy,<br/>temporary contention returns a retryable error; requests that cannot fit are rejected. |
|
||||
@@ -293,6 +294,7 @@
|
||||
| `default_timezone` | String | Unset | The default timezone of the server. |
|
||||
| `default_column_prefix` | String | Unset | The default column prefix for auto-created time index, value, native histogram, and exemplar-label columns.<br/>Legacy OTLP summary columns keep their historical `greptime_` prefix. |
|
||||
| `auto_create_table` | Bool | `true` | Server-side global switch for auto table creation on write.<br/>When `false`, a missing table is never auto-created even if the request sets the `auto_create_table` hint to `true`. Default: `true`. |
|
||||
| `auto_create_physical_table_timestamp_unit` | String | Unset | Time index unit for physical metric tables created by auto table creation<br/>(Prometheus remote write, OTLP metrics, and other metric-engine ingestion).<br/>New logical tables created under such a physical table inherit its unit.<br/>Options: "Second", "Millisecond" (default), "Microsecond", "Nanosecond". |
|
||||
| `user_provider` | String | Unset | The user provider for authentication.<br/>Examples: "static_user_provider:file:/path/to/users", "static_user_provider:cmd:greptime_user=greptime_pwd"<br/>Password verifier formats: "plain:<password>", "pbkdf2_sha256:<iterations>:<hex_salt>:<hex_hash>",<br/>"mysql_native_password:<hex_sha1_sha1_password>",<br/>"pg_scram_sha256:<iterations>:<hex_salt>:<hex_stored_key>:<hex_server_key>"<br/>"pbkdf2_sha256" and "pg_scram_sha256" protect passwords at rest, but such users cannot log in over MySQL:<br/>`static_user_provider` and `watch_file_user_provider` only negotiate `mysql_native_password`, which cannot verify these formats.<br/>"mysql_native_password" is MySQL-specific and cannot authenticate over PostgreSQL at all.<br/>PostgreSQL SCRAM only covers "plain" and "pg_scram_sha256" users; if any user is "pbkdf2_sha256" or<br/>"mysql_native_password", PostgreSQL falls back to cleartext password auth for every user.<br/>For "pg_scram_sha256" users, keep the default iteration count (4096) and salt length (16): both are<br/>observable in the SCRAM server-first message, and non-default values weaken resistance to username<br/>enumeration. |
|
||||
| `max_in_flight_write_bytes` | String | 0 | Shared quota for concurrent write request bodies and messages (HTTP, gRPC, Flight)<br/>plus retained decoder memory and estimated PB-to-Arrow conversion memory for ordinary-table batching.<br/>A batch must fit together with its request's existing reservations; only additional memory is acquired.<br/>Set to 0 to disable the limit. Default: "0" (unlimited) |
|
||||
| `write_bytes_exhausted_policy` | String | `wait` | Policy when write bytes quota is exhausted.<br/>Options: "wait" (default, 10s timeout), "wait(<duration>)" (e.g., "wait(30s)"), "fail"<br/>Table-batch conversion never waits for additional memory. Under a wait policy,<br/>temporary contention returns a retryable error; requests that cannot fit are rejected. |
|
||||
|
||||
@@ -11,6 +11,13 @@ default_column_prefix = "greptime"
|
||||
## When `false`, a missing table is never auto-created even if the request sets the `auto_create_table` hint to `true`. Default: `true`.
|
||||
#+ auto_create_table = true
|
||||
|
||||
## Time index unit for physical metric tables created by auto table creation
|
||||
## (Prometheus remote write, OTLP metrics, and other metric-engine ingestion).
|
||||
## New logical tables created under such a physical table inherit its unit.
|
||||
## Options: "Second", "Millisecond" (default), "Microsecond", "Nanosecond".
|
||||
## @toml2docs:none-default
|
||||
#+ auto_create_physical_table_timestamp_unit = "Millisecond"
|
||||
|
||||
## The user provider for authentication.
|
||||
## Examples: "static_user_provider:file:/path/to/users", "static_user_provider:cmd:greptime_user=greptime_pwd"
|
||||
## Password verifier formats: "plain:<password>", "pbkdf2_sha256:<iterations>:<hex_salt>:<hex_hash>",
|
||||
|
||||
@@ -11,6 +11,13 @@ default_column_prefix = "greptime"
|
||||
## When `false`, a missing table is never auto-created even if the request sets the `auto_create_table` hint to `true`. Default: `true`.
|
||||
#+ auto_create_table = true
|
||||
|
||||
## Time index unit for physical metric tables created by auto table creation
|
||||
## (Prometheus remote write, OTLP metrics, and other metric-engine ingestion).
|
||||
## New logical tables created under such a physical table inherit its unit.
|
||||
## Options: "Second", "Millisecond" (default), "Microsecond", "Nanosecond".
|
||||
## @toml2docs:none-default
|
||||
#+ auto_create_physical_table_timestamp_unit = "Millisecond"
|
||||
|
||||
## The user provider for authentication.
|
||||
## Examples: "static_user_provider:file:/path/to/users", "static_user_provider:cmd:greptime_user=greptime_pwd"
|
||||
## Password verifier formats: "plain:<password>", "pbkdf2_sha256:<iterations>:<hex_salt>:<hex_hash>",
|
||||
|
||||
@@ -25,6 +25,7 @@ use common_options::datanode::{ClientOptions, DatanodeClientOptions};
|
||||
use common_runtime::global::RuntimeOptions;
|
||||
use common_telemetry::logging::{DEFAULT_LOGGING_DIR, DEFAULT_OTLP_HTTP_ENDPOINT, LoggingOptions};
|
||||
use common_test_util::temp_dir::create_named_temp_file;
|
||||
use common_time::timestamp::TimeUnit;
|
||||
use common_wal::config::DatanodeWalConfig;
|
||||
use common_wal::config::raft_engine::RaftEngineConfig;
|
||||
use datanode::config::{DatanodeOptions, RegionEngineConfig, StorageConfig};
|
||||
@@ -249,6 +250,7 @@ fn test_load_frontend_example_config() {
|
||||
default_timezone: Some("UTC".to_string()),
|
||||
default_column_prefix: Some("greptime".to_string()),
|
||||
auto_create_table: true,
|
||||
auto_create_physical_table_timestamp_unit: TimeUnit::Millisecond,
|
||||
meta_client: Some(MetaClientOptions {
|
||||
metasrv_addrs: vec!["127.0.0.1:3002".to_string()],
|
||||
timeout: Duration::from_secs(3),
|
||||
@@ -430,6 +432,7 @@ fn test_load_standalone_example_config() {
|
||||
default_timezone: Some("UTC".to_string()),
|
||||
default_column_prefix: Some("greptime".to_string()),
|
||||
auto_create_table: true,
|
||||
auto_create_physical_table_timestamp_unit: TimeUnit::Millisecond,
|
||||
wal: DatanodeWalConfig::RaftEngine(RaftEngineConfig {
|
||||
dir: Some(format!("{}/{}", DEFAULT_DATA_HOME, WAL_DIR)),
|
||||
sync_period: Some(Duration::from_secs(5)),
|
||||
|
||||
@@ -23,6 +23,7 @@ use common_memory_manager::OnExhaustedPolicy;
|
||||
use common_options::datanode::DatanodeClientOptions;
|
||||
use common_options::memory::MemoryOptions;
|
||||
use common_telemetry::logging::{LoggingOptions, SlowQueryOptions, TracingOptions};
|
||||
use common_time::timestamp::TimeUnit;
|
||||
use meta_client::{MetaClientOptions, MetaClientRef};
|
||||
use pipeline::PipelineOptions;
|
||||
use query::options::QueryOptions;
|
||||
@@ -53,6 +54,11 @@ pub struct FrontendOptions {
|
||||
/// even if a request sets the `auto_create_table` hint to `true`. When `true`
|
||||
/// (default), the per-request hint still applies. Default: `true`.
|
||||
pub auto_create_table: bool,
|
||||
/// Time index unit for physical metric tables created by auto table creation
|
||||
/// (Prometheus remote write, OTLP metrics, and other metric-engine ingestion).
|
||||
/// New logical tables created under such a physical table inherit its unit.
|
||||
/// Options: "Second", "Millisecond" (default), "Microsecond", "Nanosecond".
|
||||
pub auto_create_physical_table_timestamp_unit: TimeUnit,
|
||||
/// Shared quota for write bodies/messages, retained decoder memory, and estimated
|
||||
/// ordinary-table batch conversion memory. A batch must fit alongside its request's
|
||||
/// existing reservations; only additional memory is acquired.
|
||||
@@ -109,6 +115,7 @@ impl Default for FrontendOptions {
|
||||
default_timezone: None,
|
||||
default_column_prefix: None,
|
||||
auto_create_table: true,
|
||||
auto_create_physical_table_timestamp_unit: TimeUnit::Millisecond,
|
||||
max_in_flight_write_bytes: ReadableSize(0),
|
||||
write_bytes_exhausted_policy: OnExhaustedPolicy::default(),
|
||||
http: HttpOptions::default(),
|
||||
|
||||
@@ -64,6 +64,7 @@ use common_recordbatch::RecordBatchStreamWrapper;
|
||||
use common_recordbatch::error::StreamTimeoutSnafu;
|
||||
use common_telemetry::logging::SlowQueryOptions;
|
||||
use common_telemetry::{debug, error, tracing};
|
||||
use common_time::timestamp::TimeUnit;
|
||||
use datafusion::dataframe::DataFrame;
|
||||
use datafusion::physical_plan::ExecutionPlan;
|
||||
use datafusion_expr::LogicalPlan;
|
||||
@@ -133,6 +134,9 @@ pub struct Instance {
|
||||
process_manager: ProcessManagerRef,
|
||||
slow_query_options: SlowQueryOptions,
|
||||
influxdb_default_merge_mode: InfluxdbMergeMode,
|
||||
/// Time index unit for physical metric tables created by auto table creation
|
||||
/// through the frontend instance (e.g., batched logical-table flushes).
|
||||
auto_create_physical_table_timestamp_unit: TimeUnit,
|
||||
trace_ingest_chunk_size: usize,
|
||||
trace_aux_cache: otlp::TraceAuxCache,
|
||||
trace_aux_limiter: Option<RequestLimiter>,
|
||||
|
||||
@@ -253,6 +253,9 @@ impl FrontendBuilder {
|
||||
table_flownode_cache.clone(),
|
||||
self.options.auto_create_table,
|
||||
)
|
||||
.with_auto_create_physical_table_timestamp_unit(
|
||||
self.options.auto_create_physical_table_timestamp_unit,
|
||||
)
|
||||
.with_write_dispatcher(write_dispatcher.clone())
|
||||
};
|
||||
// The execution-only inserter owns no batchers, avoiding an Arc cycle.
|
||||
@@ -439,6 +442,9 @@ impl FrontendBuilder {
|
||||
process_manager,
|
||||
slow_query_options: self.options.slow_query.clone(),
|
||||
influxdb_default_merge_mode: self.options.influxdb.default_merge_mode,
|
||||
auto_create_physical_table_timestamp_unit: self
|
||||
.options
|
||||
.auto_create_physical_table_timestamp_unit,
|
||||
trace_ingest_chunk_size: self.options.otlp.trace_ingest_chunk_size,
|
||||
trace_aux_cache: TraceAuxCache::new(self.options.otlp.trace_aux_cache_size.as_bytes()),
|
||||
trace_aux_limiter,
|
||||
|
||||
@@ -53,6 +53,7 @@ impl Instance {
|
||||
self.table_flownode_set_cache().clone(),
|
||||
true,
|
||||
Arc::new(LogicalTables(Arc::downgrade(self))),
|
||||
options.auto_create_physical_table_timestamp_unit,
|
||||
options_batcher.pending_rows_flush_interval,
|
||||
options_batcher.max_batch_rows,
|
||||
options_batcher.max_concurrent_flushes,
|
||||
|
||||
@@ -788,7 +788,9 @@ impl Instance {
|
||||
let default_schema = vec![
|
||||
api::v1::ColumnSchema {
|
||||
column_name: common_query::prelude::greptime_timestamp().to_string(),
|
||||
datatype: api::v1::ColumnDataType::TimestampMillisecond as i32,
|
||||
datatype: api::helper::timestamp_datatype(
|
||||
self.auto_create_physical_table_timestamp_unit,
|
||||
) as i32,
|
||||
semantic_type: api::v1::SemanticType::Timestamp as i32,
|
||||
datatype_extension: None,
|
||||
options: None,
|
||||
|
||||
@@ -120,6 +120,11 @@ pub struct Inserter {
|
||||
/// When `false`, missing tables are never auto-created regardless of the
|
||||
/// per-request `auto_create_table` hint. When `true`, the hint still applies.
|
||||
auto_create_table: bool,
|
||||
/// Time index unit for physical metric tables created by auto table creation.
|
||||
/// New logical tables created under such a physical table inherit its unit,
|
||||
/// so all metric-engine ingestion paths write timestamps in this unit.
|
||||
/// Defaults to milliseconds for compatibility.
|
||||
auto_create_physical_table_timestamp_unit: TimeUnit,
|
||||
pending_rows_batcher: Option<Arc<dyn PendingRowsBatcher>>,
|
||||
/// Rows handed to detached flow mirror tasks that have not finished yet.
|
||||
/// Bounds how much row data in-flight mirror writes may keep alive, see
|
||||
@@ -342,11 +347,18 @@ impl Inserter {
|
||||
node_manager,
|
||||
table_flownode_set_cache,
|
||||
auto_create_table,
|
||||
auto_create_physical_table_timestamp_unit: TimeUnit::Millisecond,
|
||||
pending_rows_batcher: None,
|
||||
mirror_pending_rows: Arc::new(AtomicU64::new(0)),
|
||||
}
|
||||
}
|
||||
|
||||
/// Sets the time index unit for auto-created physical metric tables.
|
||||
pub fn with_auto_create_physical_table_timestamp_unit(mut self, unit: TimeUnit) -> Self {
|
||||
self.auto_create_physical_table_timestamp_unit = unit;
|
||||
self
|
||||
}
|
||||
|
||||
/// Installs the shared dispatcher used by direct writes and batch flushes.
|
||||
pub fn with_write_dispatcher(mut self, dispatcher: WriteDispatcherRef) -> Self {
|
||||
self.write_dispatcher = dispatcher;
|
||||
@@ -1868,7 +1880,9 @@ impl Inserter {
|
||||
let default_schema = vec![
|
||||
ColumnSchema {
|
||||
column_name: greptime_timestamp().to_string(),
|
||||
datatype: ColumnDataType::TimestampMillisecond as _,
|
||||
datatype: api::helper::timestamp_datatype(
|
||||
self.auto_create_physical_table_timestamp_unit,
|
||||
) as _,
|
||||
semantic_type: SemanticType::Timestamp as _,
|
||||
datatype_extension: None,
|
||||
options: None,
|
||||
|
||||
@@ -118,12 +118,9 @@ impl Inserter {
|
||||
tables.insert(backing.clone(), table);
|
||||
}
|
||||
physical_columns.entry(backing.as_str()).or_insert_with(|| {
|
||||
PhysicalColumns::new(
|
||||
tables[backing]
|
||||
.as_ref()
|
||||
.map(table_columns)
|
||||
.unwrap_or_else(default_physical_columns),
|
||||
)
|
||||
PhysicalColumns::new(tables[backing].as_ref().map(table_columns).unwrap_or_else(
|
||||
|| default_physical_columns(self.auto_create_physical_table_timestamp_unit),
|
||||
))
|
||||
});
|
||||
}
|
||||
// Match ordinary insertion: impure defaults are evaluated once per table
|
||||
@@ -176,7 +173,9 @@ impl Inserter {
|
||||
let unit = target
|
||||
.and_then(table_time_index_unit)
|
||||
.or_else(|| physical.and_then(table_time_index_unit))
|
||||
.unwrap_or(TimeUnit::Millisecond);
|
||||
// The physical table will be auto-created in the configured
|
||||
// unit, so validate its rows in that unit as well.
|
||||
.unwrap_or(self.auto_create_physical_table_timestamp_unit);
|
||||
if let Some(rows) = &mut request.rows {
|
||||
convert_rows_time_unit(rows, unit)?;
|
||||
}
|
||||
@@ -378,13 +377,13 @@ fn table_columns(table: &TableRef) -> Vec<(DataColumn, SemanticType)> {
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn default_physical_columns() -> Vec<(DataColumn, SemanticType)> {
|
||||
fn default_physical_columns(unit: TimeUnit) -> Vec<(DataColumn, SemanticType)> {
|
||||
use datatypes::prelude::ConcreteDataType;
|
||||
vec![
|
||||
(
|
||||
DataColumn::new(
|
||||
greptime_timestamp(),
|
||||
ConcreteDataType::timestamp_millisecond_datatype(),
|
||||
ConcreteDataType::timestamp_datatype(unit),
|
||||
false,
|
||||
)
|
||||
.with_time_index(true),
|
||||
@@ -698,7 +697,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn test_preflight_omissions_defaults_and_explicit_nulls() {
|
||||
let mut columns = default_physical_columns();
|
||||
let mut columns = default_physical_columns(TimeUnit::Millisecond);
|
||||
columns.push((
|
||||
DataColumn::new("host", ConcreteDataType::string_datatype(), false)
|
||||
.with_default_constraint(Some(ColumnDefaultConstraint::Value(DataValue::String(
|
||||
@@ -739,7 +738,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn test_preflight_existing_columns_and_physical_definitions() {
|
||||
let columns = default_physical_columns();
|
||||
let columns = default_physical_columns(TimeUnit::Millisecond);
|
||||
let mut rows = Rows {
|
||||
schema: vec![ColumnSchema {
|
||||
column_name: greptime_timestamp().into(),
|
||||
@@ -805,7 +804,7 @@ mod tests {
|
||||
(defaulted, None),
|
||||
(impure, Some("impure default for physical field `impure`")),
|
||||
] {
|
||||
let mut columns = default_physical_columns();
|
||||
let mut columns = default_physical_columns(TimeUnit::Millisecond);
|
||||
columns.extend((0..1024).map(|index| {
|
||||
(
|
||||
DataColumn::new(
|
||||
|
||||
@@ -108,6 +108,10 @@ pub struct LogicalTablePendingRowsBatcher {
|
||||
worker_channel_capacity: usize,
|
||||
prom_store_with_metric_engine: bool,
|
||||
schema_alterer: PendingRowsSchemaAltererRef,
|
||||
/// Time index unit the schema alterer uses when auto-creating a missing
|
||||
/// physical metric table; keeps the batcher's logical-create alignment
|
||||
/// consistent with the created physical table.
|
||||
auto_create_physical_table_timestamp_unit: TimeUnit,
|
||||
pending_rows_batch_sync: bool,
|
||||
shutdown: broadcast::Sender<()>,
|
||||
}
|
||||
@@ -123,6 +127,7 @@ impl LogicalTablePendingRowsBatcher {
|
||||
table_flownode_set_cache: TableFlownodeSetCacheRef,
|
||||
prom_store_with_metric_engine: bool,
|
||||
schema_alterer: PendingRowsSchemaAltererRef,
|
||||
auto_create_physical_table_timestamp_unit: TimeUnit,
|
||||
flush_interval: Duration,
|
||||
max_batch_rows: usize,
|
||||
max_concurrent_flushes: usize,
|
||||
@@ -163,6 +168,7 @@ impl LogicalTablePendingRowsBatcher {
|
||||
flow_notification_tx,
|
||||
prom_store_with_metric_engine,
|
||||
schema_alterer,
|
||||
auto_create_physical_table_timestamp_unit,
|
||||
flush_limiter,
|
||||
request_limiter,
|
||||
worker_channel_capacity,
|
||||
@@ -184,8 +190,8 @@ impl LogicalTablePendingRowsBatcher {
|
||||
}
|
||||
|
||||
/// Returns the physical metric table's time index unit resolved from
|
||||
/// `ctx`, defaulting to millisecond when the table does not exist yet
|
||||
/// (the schema alterer auto-creates it as millisecond).
|
||||
/// `ctx`, falling back to the configured auto-create unit when the table
|
||||
/// does not exist yet (the schema alterer auto-creates it with that unit).
|
||||
async fn physical_time_index_unit_or_default(&self, ctx: &QueryContextRef) -> TimeUnit {
|
||||
let key = batch_key_from_ctx(ctx);
|
||||
let Ok(Some(table)) = self
|
||||
@@ -193,7 +199,7 @@ impl LogicalTablePendingRowsBatcher {
|
||||
.table(&key.catalog, &key.schema, &key.physical_table, None)
|
||||
.await
|
||||
else {
|
||||
return TimeUnit::Millisecond;
|
||||
return self.auto_create_physical_table_timestamp_unit;
|
||||
};
|
||||
table
|
||||
.table_info()
|
||||
|
||||
@@ -94,8 +94,8 @@ impl LogicalTablePendingRowsBatcher {
|
||||
|
||||
// New tables are created on the request's selected physical table;
|
||||
// their time index must use the physical table's unit (a missing
|
||||
// physical table is auto-created as millisecond by the schema
|
||||
// alterer, matching the default here).
|
||||
// physical table is auto-created in the configured auto-create unit
|
||||
// by the schema alterer, matching the fallback here).
|
||||
if !plan.tables_to_create.is_empty() {
|
||||
let physical_unit = self.physical_time_index_unit_or_default(ctx).await;
|
||||
for (_, request_schema) in &mut plan.tables_to_create {
|
||||
|
||||
@@ -32,6 +32,7 @@ use common_meta::node_manager::{
|
||||
};
|
||||
use common_meta::test_util::MockDatanodeManager;
|
||||
use common_query::prelude::GREPTIME_PHYSICAL_TABLE;
|
||||
use common_time::timestamp::TimeUnit;
|
||||
use datatypes::prelude::ConcreteDataType;
|
||||
use datatypes::schema::{ColumnSchema as DtColumnSchema, Schema as DtSchema};
|
||||
use operator::test_util::{create_partition_rule_manager, prepare_mocked_backend};
|
||||
@@ -110,6 +111,7 @@ pub(crate) async fn submission_batcher(
|
||||
mock_table_flownode_cache(1, vec![]).await,
|
||||
true,
|
||||
Arc::new(UnexpectedSchemaAlterer),
|
||||
TimeUnit::Millisecond,
|
||||
Duration::from_secs(1),
|
||||
2,
|
||||
1,
|
||||
|
||||
@@ -21,6 +21,7 @@ use common_event_recorder::EventRecorderOptions;
|
||||
use common_memory_manager::OnExhaustedPolicy;
|
||||
use common_options::memory::MemoryOptions;
|
||||
use common_telemetry::logging::{LoggingOptions, SlowQueryOptions, TracingOptions};
|
||||
use common_time::timestamp::TimeUnit;
|
||||
use common_wal::config::DatanodeWalConfig;
|
||||
use datanode::config::{DatanodeOptions, ProcedureConfig, RegionEngineConfig, StorageConfig};
|
||||
use file_engine::config::EngineConfig as FileEngineConfig;
|
||||
@@ -47,6 +48,11 @@ pub struct StandaloneOptions {
|
||||
/// Upper bound: when `false`, missing tables are never auto-created even if a
|
||||
/// request sets the `auto_create_table` hint to `true`. Default: `true`.
|
||||
pub auto_create_table: bool,
|
||||
/// Time index unit for physical metric tables created by auto table creation
|
||||
/// (Prometheus remote write, OTLP metrics, and other metric-engine ingestion).
|
||||
/// New logical tables created under such a physical table inherit its unit.
|
||||
/// Options: "Second", "Millisecond" (default), "Microsecond", "Nanosecond".
|
||||
pub auto_create_physical_table_timestamp_unit: TimeUnit,
|
||||
/// Shared quota for write bodies/messages, retained decoder memory, and estimated
|
||||
/// ordinary-table batch conversion memory. A batch must fit alongside its request's
|
||||
/// existing reservations; only additional memory is acquired.
|
||||
@@ -111,6 +117,7 @@ impl Default for StandaloneOptions {
|
||||
default_timezone: None,
|
||||
default_column_prefix: None,
|
||||
auto_create_table: true,
|
||||
auto_create_physical_table_timestamp_unit: TimeUnit::Millisecond,
|
||||
max_in_flight_write_bytes: ReadableSize(0),
|
||||
write_bytes_exhausted_policy: OnExhaustedPolicy::default(),
|
||||
http: HttpOptions::default(),
|
||||
@@ -181,6 +188,8 @@ impl StandaloneOptions {
|
||||
FrontendOptions {
|
||||
default_timezone: cloned_opts.default_timezone,
|
||||
auto_create_table: cloned_opts.auto_create_table,
|
||||
auto_create_physical_table_timestamp_unit: cloned_opts
|
||||
.auto_create_physical_table_timestamp_unit,
|
||||
max_in_flight_write_bytes: cloned_opts.max_in_flight_write_bytes,
|
||||
write_bytes_exhausted_policy: cloned_opts.write_bytes_exhausted_policy,
|
||||
http: cloned_opts.http,
|
||||
|
||||
@@ -3999,6 +3999,94 @@ WITH(
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
async fn test_otlp_initial_write_auto_create_physical_timestamp_unit() {
|
||||
use common_time::timestamp::TimeUnit as ConfigTimeUnit;
|
||||
|
||||
for (unit, arrow_unit, expected) in [
|
||||
(ConfigTimeUnit::Second, TimeUnit::Second, 60),
|
||||
(ConfigTimeUnit::Millisecond, TimeUnit::Millisecond, 60_123),
|
||||
(
|
||||
ConfigTimeUnit::Microsecond,
|
||||
TimeUnit::Microsecond,
|
||||
60_123_456,
|
||||
),
|
||||
(
|
||||
ConfigTimeUnit::Nanosecond,
|
||||
TimeUnit::Nanosecond,
|
||||
60_123_456_789,
|
||||
),
|
||||
] {
|
||||
let standalone = GreptimeDbStandaloneBuilder::new(&format!("otlp_initial_{unit:?}"))
|
||||
.with_auto_create_physical_table_timestamp_unit(unit)
|
||||
.build()
|
||||
.await;
|
||||
let instance = standalone.fe_instance();
|
||||
let mut ctx = QueryContext::with(DEFAULT_CATALOG_NAME, "public");
|
||||
ctx.set_protocol_ctx(ProtocolCtx::OtlpMetric(OtlpMetricCtx {
|
||||
with_metric_engine: true,
|
||||
..Default::default()
|
||||
}));
|
||||
let ctx = Arc::new(ctx);
|
||||
|
||||
// No pre-created tables: preflight must accept the first point so
|
||||
// ingestion can create both tables with the configured time index.
|
||||
let outcome = instance
|
||||
.metrics(
|
||||
metric_request(vec![gauge_metric(
|
||||
"initial_metric",
|
||||
&[(60_123_456_789, 1.0, None)],
|
||||
)]),
|
||||
ctx.clone(),
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
(outcome.accepted_data_points, outcome.rejected_data_points),
|
||||
(1, 0),
|
||||
"unit={unit:?}"
|
||||
);
|
||||
|
||||
for table in ["greptime_physical_table", "initial_metric"] {
|
||||
let output = instance
|
||||
.do_query(
|
||||
&format!("SELECT greptime_timestamp FROM {table}"),
|
||||
ctx.clone(),
|
||||
)
|
||||
.await
|
||||
.remove(0)
|
||||
.unwrap();
|
||||
let OutputData::Stream(stream) = output.data else {
|
||||
panic!("expected stream")
|
||||
};
|
||||
let batches = RecordBatches::try_collect(stream).await.unwrap().take();
|
||||
assert_eq!(
|
||||
batches.iter().map(|batch| batch.num_rows()).sum::<usize>(),
|
||||
1
|
||||
);
|
||||
let timestamp = batches[0].column(0);
|
||||
assert_eq!(
|
||||
timestamp.data_type(),
|
||||
&DataType::Timestamp(arrow_unit, None)
|
||||
);
|
||||
let actual = match arrow_unit {
|
||||
TimeUnit::Second => timestamp.as_primitive::<TimestampSecondType>().value(0),
|
||||
TimeUnit::Millisecond => timestamp
|
||||
.as_primitive::<TimestampMillisecondType>()
|
||||
.value(0),
|
||||
TimeUnit::Microsecond => timestamp
|
||||
.as_primitive::<TimestampMicrosecondType>()
|
||||
.value(0),
|
||||
TimeUnit::Nanosecond => {
|
||||
timestamp.as_primitive::<TimestampNanosecondType>().value(0)
|
||||
}
|
||||
};
|
||||
assert_eq!(actual, expected, "unit={unit:?}, table={table}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
async fn test_otlp_preflight_shared_physical_and_fanout_conflicts() {
|
||||
let standalone = GreptimeDbStandaloneBuilder::new("otlp_physical_conflicts")
|
||||
|
||||
@@ -478,6 +478,202 @@ mod tests {
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
async fn test_standalone_prom_remote_write_auto_create_microsecond_physical_table() {
|
||||
common_telemetry::init_default_ut_logging();
|
||||
let standalone = GreptimeDbStandaloneBuilder::new("test_prom_rw_auto_create_us_physical")
|
||||
.with_auto_create_physical_table_timestamp_unit(
|
||||
common_time::timestamp::TimeUnit::Microsecond,
|
||||
)
|
||||
.build()
|
||||
.await;
|
||||
let instance = standalone.fe_instance();
|
||||
|
||||
let db = "prometheus_auto_create_us";
|
||||
let ctx = Arc::new(QueryContext::with(DEFAULT_CATALOG_NAME, db));
|
||||
assert!(
|
||||
SqlQueryHandler::do_query(
|
||||
instance.as_ref(),
|
||||
&format!("CREATE DATABASE IF NOT EXISTS {db}"),
|
||||
ctx.clone(),
|
||||
)
|
||||
.await
|
||||
.first()
|
||||
.unwrap()
|
||||
.is_ok()
|
||||
);
|
||||
|
||||
// No manual table creation: both the physical table and the logical
|
||||
// table `metric1` must be auto-created by the write, with a
|
||||
// microsecond time index inherited from the physical table.
|
||||
let write_request = WriteRequest {
|
||||
timeseries: vec![prom_store::mock_timeseries()[0].clone()],
|
||||
..Default::default()
|
||||
};
|
||||
let (row_inserts, _) = to_grpc_row_insert_requests(&write_request).unwrap();
|
||||
instance
|
||||
.write(row_inserts, ctx.clone(), true)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
for table in ["greptime_physical_table", "metric1"] {
|
||||
let mut output = instance
|
||||
.do_query(
|
||||
&format!("SELECT greptime_timestamp FROM {table}"),
|
||||
ctx.clone(),
|
||||
)
|
||||
.await;
|
||||
let OutputData::Stream(stream) = output.remove(0).unwrap().data else {
|
||||
unreachable!()
|
||||
};
|
||||
let batches = common_recordbatch::RecordBatches::try_collect(stream)
|
||||
.await
|
||||
.unwrap()
|
||||
.take();
|
||||
assert_eq!(batches[0].num_rows(), 2, "unexpected rows in {table}");
|
||||
let ts_column = batches[0].column(0);
|
||||
assert_eq!(
|
||||
ts_column.data_type(),
|
||||
&DataType::Timestamp(TimeUnit::Microsecond, None),
|
||||
"unexpected time index type in {table}"
|
||||
);
|
||||
let timestamps = ts_column
|
||||
.as_primitive::<TimestampMicrosecondType>()
|
||||
.values();
|
||||
assert_eq!(timestamps.to_vec(), vec![1_000_000, 2_000_000]);
|
||||
}
|
||||
|
||||
// Remote read narrows microsecond timestamps back to milliseconds.
|
||||
let read_request = ReadRequest {
|
||||
queries: vec![Query {
|
||||
start_timestamp_ms: 1000,
|
||||
end_timestamp_ms: 2000,
|
||||
matchers: vec![LabelMatcher {
|
||||
name: prom_store::METRIC_NAME_LABEL.to_string(),
|
||||
value: "metric1".to_string(),
|
||||
r#type: 0,
|
||||
}],
|
||||
..Default::default()
|
||||
}],
|
||||
..Default::default()
|
||||
};
|
||||
let resp = instance.read(read_request, ctx.clone()).await.unwrap();
|
||||
let body = prom_store::snappy_decompress(&resp.body).unwrap();
|
||||
let read_response = ReadResponse::decode(&body[..]).unwrap();
|
||||
let samples = &read_response.results[0].timeseries[0].samples;
|
||||
assert_eq!(
|
||||
samples.clone(),
|
||||
vec![
|
||||
Sample {
|
||||
value: 1.0,
|
||||
timestamp: 1000
|
||||
},
|
||||
Sample {
|
||||
value: 2.0,
|
||||
timestamp: 2000
|
||||
}
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
async fn test_standalone_prom_batcher_auto_create_microsecond_physical_table() {
|
||||
use servers::batcher::logical_table::PendingRowsSchemaAlterer;
|
||||
|
||||
common_telemetry::init_default_ut_logging();
|
||||
let standalone = GreptimeDbStandaloneBuilder::new("test_prom_batcher_auto_create_us")
|
||||
.with_auto_create_physical_table_timestamp_unit(
|
||||
common_time::timestamp::TimeUnit::Microsecond,
|
||||
)
|
||||
.build()
|
||||
.await;
|
||||
let instance = standalone.fe_instance();
|
||||
|
||||
// Drives the batcher's schema-alterer path: a missing physical table is
|
||||
// auto-created with the configured microsecond unit, and new logical
|
||||
// tables aligned to it.
|
||||
let db = "prometheus_batcher_auto_create_us";
|
||||
let ctx = Arc::new(QueryContext::with(DEFAULT_CATALOG_NAME, db));
|
||||
assert!(
|
||||
SqlQueryHandler::do_query(
|
||||
instance.as_ref(),
|
||||
&format!("CREATE DATABASE IF NOT EXISTS {db}"),
|
||||
ctx.clone(),
|
||||
)
|
||||
.await
|
||||
.first()
|
||||
.unwrap()
|
||||
.is_ok()
|
||||
);
|
||||
// The batcher aligns request schemas to the physical table's unit
|
||||
// before invoking the schema alterer, so the schema below arrives in
|
||||
// the configured microsecond unit. A missing physical table must be
|
||||
// auto-created with the same unit.
|
||||
let request_schema = vec![
|
||||
api::v1::ColumnSchema {
|
||||
column_name: "greptime_timestamp".to_string(),
|
||||
datatype: ColumnDataType::TimestampMicrosecond as i32,
|
||||
semantic_type: SemanticType::Timestamp as i32,
|
||||
datatype_extension: None,
|
||||
options: None,
|
||||
},
|
||||
api::v1::ColumnSchema {
|
||||
column_name: "greptime_value".to_string(),
|
||||
datatype: ColumnDataType::Float64 as i32,
|
||||
semantic_type: SemanticType::Field as i32,
|
||||
datatype_extension: None,
|
||||
options: None,
|
||||
},
|
||||
api::v1::ColumnSchema {
|
||||
column_name: "job".to_string(),
|
||||
datatype: ColumnDataType::String as i32,
|
||||
semantic_type: SemanticType::Tag as i32,
|
||||
datatype_extension: None,
|
||||
options: None,
|
||||
},
|
||||
];
|
||||
instance
|
||||
.create_tables_if_missing_batch(
|
||||
DEFAULT_CATALOG_NAME,
|
||||
db,
|
||||
&[("metric1", request_schema.as_slice())],
|
||||
true,
|
||||
ctx.clone(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
for table in ["greptime_physical_table", "metric1"] {
|
||||
let mut output = instance
|
||||
.do_query(
|
||||
&format!(
|
||||
"SELECT greptime_data_type FROM information_schema.columns \
|
||||
WHERE table_schema='{db}' AND table_name='{table}' \
|
||||
AND column_name='greptime_timestamp'"
|
||||
),
|
||||
ctx.clone(),
|
||||
)
|
||||
.await;
|
||||
let OutputData::Stream(stream) = output.remove(0).unwrap().data else {
|
||||
unreachable!()
|
||||
};
|
||||
let batches = common_recordbatch::RecordBatches::try_collect(stream)
|
||||
.await
|
||||
.unwrap()
|
||||
.take();
|
||||
assert_eq!(
|
||||
batches[0].num_rows(),
|
||||
1,
|
||||
"missing time index column in {table}"
|
||||
);
|
||||
let data_type = batches[0].column(0).as_string::<i32>().value(0);
|
||||
assert_eq!(
|
||||
data_type, "TimestampMicrosecond",
|
||||
"unexpected time index type in {table}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// Regression test for <https://github.com/GreptimeTeam/greptimedb/issues/9231>:
|
||||
/// prometheus remote write/read against a physical metric table
|
||||
/// pre-created with a non-millisecond time index. Millisecond samples are
|
||||
|
||||
@@ -91,6 +91,7 @@ pub struct GreptimeDbStandaloneBuilder {
|
||||
slow_query_options: SlowQueryOptions,
|
||||
event_recorder_options: EventRecorderOptions,
|
||||
auto_create_table: bool,
|
||||
auto_create_physical_table_timestamp_unit: Option<common_time::timestamp::TimeUnit>,
|
||||
logical_batcher: Option<BatcherOptions>,
|
||||
table_batcher: BatcherOptions,
|
||||
max_in_flight_write_bytes: ReadableSize,
|
||||
@@ -114,6 +115,7 @@ impl GreptimeDbStandaloneBuilder {
|
||||
},
|
||||
event_recorder_options: EventRecorderOptions::default(),
|
||||
auto_create_table: true,
|
||||
auto_create_physical_table_timestamp_unit: None,
|
||||
logical_batcher: None,
|
||||
table_batcher: BatcherOptions::default(),
|
||||
max_in_flight_write_bytes: ReadableSize(0),
|
||||
@@ -163,6 +165,16 @@ impl GreptimeDbStandaloneBuilder {
|
||||
self
|
||||
}
|
||||
|
||||
/// Sets the time index unit for auto-created physical metric tables.
|
||||
#[must_use]
|
||||
pub fn with_auto_create_physical_table_timestamp_unit(
|
||||
mut self,
|
||||
unit: common_time::timestamp::TimeUnit,
|
||||
) -> Self {
|
||||
self.auto_create_physical_table_timestamp_unit = Some(unit);
|
||||
self
|
||||
}
|
||||
|
||||
/// Sets the event recorder options for the standalone instance.
|
||||
#[must_use]
|
||||
pub fn with_event_recorder_options(
|
||||
@@ -420,6 +432,9 @@ impl GreptimeDbStandaloneBuilder {
|
||||
slow_query: self.slow_query_options.clone(),
|
||||
event_recorder: self.event_recorder_options.clone(),
|
||||
auto_create_table: self.auto_create_table,
|
||||
auto_create_physical_table_timestamp_unit: self
|
||||
.auto_create_physical_table_timestamp_unit
|
||||
.unwrap_or(common_time::timestamp::TimeUnit::Millisecond),
|
||||
pipeline: self.pipeline_options.clone(),
|
||||
max_in_flight_write_bytes: self.max_in_flight_write_bytes,
|
||||
pending_rows_batcher: PendingRowsBatcherOptions {
|
||||
|
||||
@@ -913,6 +913,7 @@ pub fn build_test_prom_server(
|
||||
frontend_ref.table_flownode_set_cache().clone(),
|
||||
true,
|
||||
frontend_ref.clone(),
|
||||
common_time::timestamp::TimeUnit::Millisecond,
|
||||
Duration::from_millis(50),
|
||||
1000,
|
||||
4,
|
||||
|
||||
@@ -3418,6 +3418,7 @@ providers = []"#,
|
||||
r#"
|
||||
enable_telemetry = true
|
||||
auto_create_table = true
|
||||
auto_create_physical_table_timestamp_unit = "Millisecond"
|
||||
max_in_flight_write_bytes = "0KiB"
|
||||
write_bytes_exhausted_policy = "wait"
|
||||
init_regions_in_background = false
|
||||
|
||||
Reference in New Issue
Block a user