mirror of
https://github.com/GreptimeTeam/greptimedb.git
synced 2026-08-19 12:38:25 +00:00
b30d17f89c
The example TOMLs and generated config.md documented the default of wal.sync_period as "10s", but since #5677 moved the WAL sync task to a background RepeatedTask, an unset sync_period falls back to 5s in RaftEngineLogStore. The two paths therefore had different fsync periods: deployments based on the example configs used 10s while bare configs used 5s. Align the documentation with the actual code behavior (5s) instead of changing the code fallback to 10s, so that no existing deployment silently gets a larger data-loss window on host power loss. - config/datanode.example.toml, config/standalone.example.toml: 10s -> 5s - config/config.md: regenerated via make config-docs - src/cmd/tests/load_config_test.rs: update assertions accordingly Signed-off-by: jeremyhi <fengjiachun@gmail.com>
937 lines
32 KiB
TOML
937 lines
32 KiB
TOML
## The default timezone of the server.
|
|
## @toml2docs:none-default
|
|
default_timezone = "UTC"
|
|
|
|
## The default column prefix for auto-created time index, value, and native histogram columns.
|
|
## Legacy OTLP summary columns keep their historical `greptime_` prefix.
|
|
## @toml2docs:none-default
|
|
default_column_prefix = "greptime"
|
|
|
|
## Server-side global switch for auto table creation on write.
|
|
## When `false`, a missing table is never auto-created even if the request sets the `auto_create_table` hint to `true`. Default: `true`.
|
|
#+ auto_create_table = true
|
|
|
|
## The user provider for authentication.
|
|
## Examples: "static_user_provider:file:/path/to/users", "static_user_provider:cmd:greptime_user=greptime_pwd"
|
|
## Password verifier formats: "plain:<password>", "pbkdf2_sha256:<iterations>:<hex_salt>:<hex_hash>",
|
|
## "mysql_native_password:<hex_sha1_sha1_password>",
|
|
## "pg_scram_sha256:<iterations>:<hex_salt>:<hex_stored_key>:<hex_server_key>"
|
|
## "pbkdf2_sha256" and "pg_scram_sha256" protect passwords at rest, but cannot authenticate over MySQL's
|
|
## native password handshake; a MySQL client must send the password in cleartext for such users.
|
|
## "mysql_native_password" is MySQL-specific and cannot authenticate over PostgreSQL at all.
|
|
## PostgreSQL SCRAM only covers "plain" and "pg_scram_sha256" users; if any user is "pbkdf2_sha256" or
|
|
## "mysql_native_password", PostgreSQL falls back to cleartext password auth for every user.
|
|
## For "pg_scram_sha256" users, keep the default iteration count (4096) and salt length (16): both are
|
|
## observable in the SCRAM server-first message, and non-default values weaken resistance to username
|
|
## enumeration.
|
|
## @toml2docs:none-default
|
|
#+ user_provider = "static_user_provider:file:/path/to/users"
|
|
|
|
## Maximum total memory for all concurrent write request bodies and messages (HTTP, gRPC, Flight).
|
|
## Set to 0 to disable the limit. Default: "0" (unlimited)
|
|
## @toml2docs:none-default
|
|
#+ max_in_flight_write_bytes = "1GB"
|
|
|
|
## Policy when write bytes quota is exhausted.
|
|
## Options: "wait" (default, 10s timeout), "wait(<duration>)" (e.g., "wait(30s)"), "fail"
|
|
## @toml2docs:none-default
|
|
#+ write_bytes_exhausted_policy = "wait"
|
|
|
|
## Initialize all regions in the background during the startup.
|
|
## By default, it provides services after all regions have been initialized.
|
|
init_regions_in_background = false
|
|
|
|
## Parallelism of initializing regions.
|
|
init_regions_parallelism = 16
|
|
|
|
## The maximum concurrent queries allowed to be executed. Zero means unlimited.
|
|
max_concurrent_queries = 0
|
|
|
|
## Enable telemetry to collect anonymous usage data. Enabled by default.
|
|
#+ enable_telemetry = true
|
|
|
|
## The runtime options.
|
|
#+ [runtime]
|
|
## The number of threads to execute the runtime for global read operations.
|
|
#+ global_rt_size = 8
|
|
## The number of threads to execute compact operations.
|
|
#+ compact_rt_size = 4
|
|
## The maximum number of blocking threads for compact operations.
|
|
## Defaults to max(num_cpus / 2, 1).
|
|
#+ compact_rt_max_blocking_threads = 4
|
|
|
|
## The HTTP server options.
|
|
[http]
|
|
## The address to bind the HTTP server.
|
|
addr = "127.0.0.1:4000"
|
|
## HTTP request timeout. Set to 0 to disable timeout.
|
|
## When Prometheus pending-row batching is enabled, a nonzero timeout less than or equal to the
|
|
## `prom_store.pending_rows_flush_interval` plus 1 second is adjusted to that value.
|
|
timeout = "0s"
|
|
## HTTP request body limit.
|
|
## The following units are supported: `B`, `KB`, `KiB`, `MB`, `MiB`, `GB`, `GiB`, `TB`, `TiB`, `PB`, `PiB`.
|
|
## Set to 0 to disable limit.
|
|
body_limit = "64MB"
|
|
## HTTP CORS support, it's turned on by default
|
|
## This allows browser to access http APIs without CORS restrictions
|
|
enable_cors = true
|
|
## Customize allowed origins for HTTP CORS.
|
|
## @toml2docs:none-default
|
|
cors_allowed_origins = ["https://example.com"]
|
|
|
|
## Experimental: enable POST /v1/sql/analyze/stream for streaming EXPLAIN ANALYZE VERBOSE metrics.
|
|
experimental_enable_explain_analyze_stream = true
|
|
|
|
## Whether to start the dedicated public HTTP **API** server. This server serves
|
|
## only the `v1` interfaces plus the dashboard, and shares every other `[http]`
|
|
## option with the main server. It is disabled by default; set to `true` to enable.
|
|
enable_api_server = false
|
|
## The address to bind the dedicated HTTP API server, in the same form as `addr`.
|
|
## Defaults to `127.0.0.1:4006`.
|
|
api_server_addr = "127.0.0.1:4006"
|
|
|
|
## The gRPC server options.
|
|
[grpc]
|
|
## The address to bind the gRPC server.
|
|
bind_addr = "127.0.0.1:4001"
|
|
## The number of server worker threads.
|
|
runtime_size = 8
|
|
## The maximum connection age for gRPC connection.
|
|
## The value can be a human-readable time string. For example: `10m` for ten minutes or `1h` for one hour.
|
|
## Refer to https://grpc.io/docs/guides/keepalive/ for more details.
|
|
## @toml2docs:none-default
|
|
#+ max_connection_age = "10m"
|
|
|
|
## gRPC server TLS options, see `mysql.tls` section.
|
|
[grpc.tls]
|
|
## TLS mode.
|
|
mode = "disable"
|
|
|
|
## Certificate file path.
|
|
## @toml2docs:none-default
|
|
cert_path = ""
|
|
|
|
## Private key file path.
|
|
## @toml2docs:none-default
|
|
key_path = ""
|
|
|
|
## Watch for Certificate and key file change and auto reload.
|
|
## For now, gRPC tls config does not support auto reload.
|
|
watch = false
|
|
|
|
## MySQL server options.
|
|
[mysql]
|
|
## Whether to enable.
|
|
enable = true
|
|
## The addr to bind the MySQL server.
|
|
addr = "127.0.0.1:4002"
|
|
## The number of server worker threads.
|
|
runtime_size = 2
|
|
## Server-side keep-alive time.
|
|
## Set to 0 (default) to disable.
|
|
keep_alive = "0s"
|
|
## Maximum entries in the MySQL prepared statement cache; default is 10,000.
|
|
prepared_stmt_cache_size= 10000
|
|
# MySQL server TLS options.
|
|
[mysql.tls]
|
|
|
|
## TLS mode, refer to https://www.postgresql.org/docs/current/libpq-ssl.html
|
|
## - `disable` (default value)
|
|
## - `prefer`
|
|
## - `require`
|
|
## - `verify-ca`
|
|
## - `verify-full`
|
|
mode = "disable"
|
|
|
|
## Certificate file path.
|
|
## @toml2docs:none-default
|
|
cert_path = ""
|
|
|
|
## Private key file path.
|
|
## @toml2docs:none-default
|
|
key_path = ""
|
|
|
|
## Watch for Certificate and key file change and auto reload
|
|
watch = false
|
|
|
|
## PostgresSQL server options.
|
|
[postgres]
|
|
## Whether to enable
|
|
enable = true
|
|
## The addr to bind the PostgresSQL server.
|
|
addr = "127.0.0.1:4003"
|
|
## The number of server worker threads.
|
|
runtime_size = 2
|
|
## Server-side keep-alive time.
|
|
## Set to 0 (default) to disable.
|
|
keep_alive = "0s"
|
|
|
|
## PostgresSQL server TLS options, see `mysql.tls` section.
|
|
[postgres.tls]
|
|
## TLS mode.
|
|
mode = "disable"
|
|
|
|
## Certificate file path.
|
|
## @toml2docs:none-default
|
|
cert_path = ""
|
|
|
|
## Private key file path.
|
|
## @toml2docs:none-default
|
|
key_path = ""
|
|
|
|
## Watch for Certificate and key file change and auto reload
|
|
watch = false
|
|
|
|
## OpenTSDB protocol options.
|
|
[opentsdb]
|
|
## Whether to enable OpenTSDB put in HTTP API.
|
|
enable = true
|
|
|
|
## InfluxDB protocol options.
|
|
[influxdb]
|
|
## Whether to enable InfluxDB protocol in HTTP API.
|
|
enable = true
|
|
## Default merge mode for tables automatically created by InfluxDB protocol.
|
|
## Available values: "last_non_null", "last_row".
|
|
default_merge_mode = "last_non_null"
|
|
|
|
## Jaeger protocol options.
|
|
[jaeger]
|
|
## Whether to enable Jaeger protocol in HTTP API.
|
|
enable = true
|
|
|
|
## OpenTelemetry protocol options.
|
|
[otlp]
|
|
## Whether to enable OpenTelemetry protocol in HTTP API.
|
|
enable = true
|
|
## Maximum spans per trace ingest chunk. Set to 0 to disable splitting.
|
|
trace_ingest_chunk_size = 512
|
|
|
|
## Prometheus remote storage options
|
|
[prom_store]
|
|
## Whether to enable Prometheus remote write and read in HTTP API.
|
|
enable = true
|
|
## Whether to store the data from Prometheus remote write in metric engine.
|
|
with_metric_engine = true
|
|
## Whether to enable validation for Prometheus remote write requests.
|
|
## Available options:
|
|
## - strict: deny invalid UTF-8 strings (default).
|
|
## - lossy: allow invalid UTF-8 strings, replace invalid characters with REPLACEMENT_CHARACTER(U+FFFD).
|
|
## - unchecked: do not valid strings.
|
|
prom_validation_mode = "strict"
|
|
## Experimental: enable Prometheus remote write v2 native histogram ingestion.
|
|
experimental_enable_prometheus_native_histogram = false
|
|
## Interval to flush pending rows batcher.
|
|
## Set to "0s" to disable batching mode in Prometheus Remote Write endpoint
|
|
#+pending_rows_flush_interval = "0s"
|
|
## Max rows per pending batch before triggering a flush.
|
|
#+max_batch_rows = 100000
|
|
## Max number of concurrent batch flushes.
|
|
#+max_concurrent_flushes = 256
|
|
## Capacity of the pending batch worker channel.
|
|
#+worker_channel_capacity = 65526
|
|
## Max inflight write requests before backpressure.
|
|
#+max_inflight_requests = 3000
|
|
## Maximum number of logical-table flow notifications waiting in the shared queue.
|
|
#+flow_notification_queue_capacity = 1024
|
|
|
|
## The WAL options.
|
|
[wal]
|
|
## The provider of the WAL.
|
|
## - `raft_engine`: the wal is stored in the local file system by raft-engine.
|
|
## - `kafka`: it's remote wal that data is stored in Kafka.
|
|
provider = "raft_engine"
|
|
|
|
## The directory to store the WAL files.
|
|
## **It's only used when the provider is `raft_engine`**.
|
|
## @toml2docs:none-default
|
|
dir = "./greptimedb_data/wal"
|
|
|
|
## The size of the WAL segment file.
|
|
## **It's only used when the provider is `raft_engine`**.
|
|
file_size = "128MB"
|
|
|
|
## The threshold of the WAL size to trigger a purge.
|
|
## **It's only used when the provider is `raft_engine`**.
|
|
purge_threshold = "1GB"
|
|
|
|
## The interval to trigger a purge.
|
|
## **It's only used when the provider is `raft_engine`**.
|
|
purge_interval = "1m"
|
|
|
|
## The read batch size.
|
|
## **It's only used when the provider is `raft_engine`**.
|
|
read_batch_size = 128
|
|
|
|
## Whether to use sync write.
|
|
## **It's only used when the provider is `raft_engine`**.
|
|
sync_write = false
|
|
|
|
## Whether to reuse logically truncated log files.
|
|
## **It's only used when the provider is `raft_engine`**.
|
|
enable_log_recycle = true
|
|
|
|
## Whether to pre-create log files on start up.
|
|
## **It's only used when the provider is `raft_engine`**.
|
|
prefill_log_files = false
|
|
|
|
## Duration for fsyncing log files.
|
|
## **It's only used when the provider is `raft_engine`**.
|
|
sync_period = "5s"
|
|
|
|
## Parallelism during WAL recovery.
|
|
recovery_parallelism = 2
|
|
|
|
## The Kafka broker endpoints.
|
|
## **It's only used when the provider is `kafka`**.
|
|
broker_endpoints = ["127.0.0.1:9092"]
|
|
|
|
## The connect timeout for kafka client.
|
|
## **It's only used when the provider is `kafka`**.
|
|
#+ connect_timeout = "3s"
|
|
|
|
## The timeout for kafka client.
|
|
## **It's only used when the provider is `kafka`**.
|
|
#+ timeout = "3s"
|
|
|
|
## Automatically create topics for WAL.
|
|
## Set to `true` to automatically create topics for WAL.
|
|
## Otherwise, use topics named `topic_name_prefix_[0..num_topics)`
|
|
auto_create_topics = true
|
|
|
|
## Number of topics.
|
|
## **It's only used when the provider is `kafka`**.
|
|
num_topics = 64
|
|
|
|
## Topic selector type.
|
|
## Available selector types:
|
|
## - `round_robin` (default)
|
|
## **It's only used when the provider is `kafka`**.
|
|
selector_type = "round_robin"
|
|
|
|
## A Kafka topic is constructed by concatenating `topic_name_prefix` and `topic_id`.
|
|
## i.g., greptimedb_wal_topic_0, greptimedb_wal_topic_1.
|
|
## **It's only used when the provider is `kafka`**.
|
|
topic_name_prefix = "greptimedb_wal_topic"
|
|
|
|
## Expected number of replicas of each partition.
|
|
## **It's only used when the provider is `kafka`**.
|
|
replication_factor = 1
|
|
|
|
## Above which a topic creation operation will be cancelled.
|
|
## **It's only used when the provider is `kafka`**.
|
|
create_topic_timeout = "30s"
|
|
|
|
## The max size of a single producer batch.
|
|
## Warning: Kafka has a default limit of 1MB per message in a topic.
|
|
## **It's only used when the provider is `kafka`**.
|
|
max_batch_bytes = "1MB"
|
|
|
|
## The consumer wait timeout.
|
|
## **It's only used when the provider is `kafka`**.
|
|
consumer_wait_timeout = "100ms"
|
|
|
|
## Ignore missing entries during read WAL.
|
|
## **It's only used when the provider is `kafka`**.
|
|
##
|
|
## This option ensures that when Kafka messages are deleted, the system
|
|
## can still successfully replay memtable data without throwing an
|
|
## out-of-range error.
|
|
## However, enabling this option might lead to unexpected data loss,
|
|
## as the system will skip over missing entries instead of treating
|
|
## them as critical errors.
|
|
overwrite_entry_start_id = false
|
|
|
|
# The Kafka SASL configuration.
|
|
# **It's only used when the provider is `kafka`**.
|
|
# Available SASL mechanisms:
|
|
# - `PLAIN`
|
|
# - `SCRAM-SHA-256`
|
|
# - `SCRAM-SHA-512`
|
|
# [wal.sasl]
|
|
# type = "SCRAM-SHA-512"
|
|
# username = "user_kafka"
|
|
# password = "secret"
|
|
|
|
# The Kafka TLS configuration.
|
|
# **It's only used when the provider is `kafka`**.
|
|
# [wal.tls]
|
|
# server_ca_cert_path = "/path/to/server_cert"
|
|
# client_cert_path = "/path/to/client_cert"
|
|
# client_key_path = "/path/to/key"
|
|
|
|
## Metadata storage options.
|
|
[metadata_store]
|
|
## The size of the metadata store log file.
|
|
file_size = "64MB"
|
|
## The threshold of the metadata store size to trigger a purge.
|
|
purge_threshold = "256MB"
|
|
## The interval of the metadata store to trigger a purge.
|
|
purge_interval = "1m"
|
|
|
|
## Procedure storage options.
|
|
[procedure]
|
|
## Procedure max retry time.
|
|
max_retry_times = 3
|
|
## Initial retry delay of procedures, increases exponentially
|
|
retry_delay = "500ms"
|
|
## Max running procedures.
|
|
## The maximum number of procedures that can be running at the same time.
|
|
## If the number of running procedures exceeds this limit, the procedure will be rejected.
|
|
max_running_procedures = 128
|
|
|
|
## flow engine options.
|
|
[flow]
|
|
## The number of flow worker in flownode.
|
|
## Not setting(or set to 0) this value will use the number of CPU cores divided by 2.
|
|
#+num_workers=0
|
|
|
|
# Example of using S3 as the storage.
|
|
# [storage]
|
|
# type = "S3"
|
|
# bucket = "greptimedb"
|
|
# root = "data"
|
|
# access_key_id = "test"
|
|
# secret_access_key = "123456"
|
|
# endpoint = "https://s3.amazonaws.com"
|
|
# region = "us-west-2"
|
|
# enable_virtual_host_style = false
|
|
# disable_ec2_metadata = false
|
|
|
|
# Example of using Oss as the storage.
|
|
# [storage]
|
|
# type = "Oss"
|
|
# bucket = "greptimedb"
|
|
# root = "data"
|
|
# access_key_id = "test"
|
|
# access_key_secret = "123456"
|
|
# endpoint = "https://oss-cn-hangzhou.aliyuncs.com"
|
|
|
|
# Example of using Azblob as the storage.
|
|
# [storage]
|
|
# type = "Azblob"
|
|
# container = "greptimedb"
|
|
# root = "data"
|
|
# account_name = "test"
|
|
# account_key = "123456"
|
|
# endpoint = "https://greptimedb.blob.core.windows.net"
|
|
# sas_token = ""
|
|
|
|
# Example of using Gcs as the storage.
|
|
# [storage]
|
|
# type = "Gcs"
|
|
# bucket = "greptimedb"
|
|
# root = "data"
|
|
# scope = "test"
|
|
# credential_path = "123456"
|
|
# credential = "base64-credential"
|
|
# endpoint = "https://storage.googleapis.com"
|
|
|
|
## The query engine options.
|
|
[query]
|
|
## Parallelism of the query engine.
|
|
## Default to 0, which means the number of CPU cores.
|
|
parallelism = 0
|
|
|
|
## Memory pool size for query execution operators (aggregation, sorting, join).
|
|
## Supports absolute size (e.g., "2GB", "4GB") or percentage of system memory (e.g., "20%").
|
|
## Setting it to 0 disables the limit (unbounded, default behavior).
|
|
## When this limit is reached, queries will fail with ResourceExhausted error.
|
|
## NOTE: This does NOT limit memory used by table scans.
|
|
memory_pool_size = "50%"
|
|
|
|
## The data storage options.
|
|
[storage]
|
|
## The working home directory.
|
|
data_home = "./greptimedb_data"
|
|
|
|
## Root directory for standalone SQL access to local files.
|
|
## Relative SQL paths are resolved below this directory. Absolute paths are accepted only when
|
|
## they are inside this directory. Defaults to `<data_home>/copy`.
|
|
## Distributed deployments always reject SQL access to local files.
|
|
## Upgrade note: COPY commands and existing external tables that reference paths outside this
|
|
## directory will fail. Move those files below the copy root, set this option to a dedicated
|
|
## directory containing them, or migrate the files to object storage before upgrading.
|
|
#+ copy_root = "./greptimedb_data/copy"
|
|
|
|
## The storage type used to store the data.
|
|
## - `File`: the data is stored in the local file system.
|
|
## - `S3`: the data is stored in the S3 object storage.
|
|
## - `Gcs`: the data is stored in the Google Cloud Storage.
|
|
## - `Azblob`: the data is stored in the Azure Blob Storage.
|
|
## - `Oss`: the data is stored in the Aliyun OSS.
|
|
type = "File"
|
|
|
|
## The S3 bucket name.
|
|
## **It's only used when the storage type is `S3`, `Oss` and `Gcs`**.
|
|
## @toml2docs:none-default
|
|
bucket = "greptimedb"
|
|
|
|
## The S3 data will be stored in the specified prefix, for example, `s3://${bucket}/${root}`.
|
|
## **It's only used when the storage type is `S3`, `Oss` and `Azblob`**.
|
|
## @toml2docs:none-default
|
|
root = "greptimedb"
|
|
|
|
## The access key id of the aws account.
|
|
## It's **highly recommended** to use AWS IAM roles instead of hardcoding the access key id and secret key.
|
|
## **It's only used when the storage type is `S3` and `Oss`**.
|
|
## @toml2docs:none-default
|
|
access_key_id = "test"
|
|
|
|
## The secret access key of the aws account.
|
|
## It's **highly recommended** to use AWS IAM roles instead of hardcoding the access key id and secret key.
|
|
## **It's only used when the storage type is `S3`**.
|
|
## @toml2docs:none-default
|
|
secret_access_key = "test"
|
|
|
|
## The secret access key of the aliyun account.
|
|
## **It's only used when the storage type is `Oss`**.
|
|
## @toml2docs:none-default
|
|
access_key_secret = "test"
|
|
|
|
## The account key of the azure account.
|
|
## **It's only used when the storage type is `Azblob`**.
|
|
## @toml2docs:none-default
|
|
account_name = "test"
|
|
|
|
## The account key of the azure account.
|
|
## **It's only used when the storage type is `Azblob`**.
|
|
## @toml2docs:none-default
|
|
account_key = "test"
|
|
|
|
## The scope of the google cloud storage.
|
|
## **It's only used when the storage type is `Gcs`**.
|
|
## @toml2docs:none-default
|
|
scope = "test"
|
|
|
|
## The credential path of the google cloud storage.
|
|
## **It's only used when the storage type is `Gcs`**.
|
|
## @toml2docs:none-default
|
|
credential_path = "test"
|
|
|
|
## The credential of the google cloud storage.
|
|
## **It's only used when the storage type is `Gcs`**.
|
|
## @toml2docs:none-default
|
|
credential = "base64-credential"
|
|
|
|
## The container of the azure account.
|
|
## **It's only used when the storage type is `Azblob`**.
|
|
## @toml2docs:none-default
|
|
container = "greptimedb"
|
|
|
|
## The sas token of the azure account.
|
|
## **It's only used when the storage type is `Azblob`**.
|
|
## @toml2docs:none-default
|
|
sas_token = ""
|
|
|
|
## The endpoint of the S3 service.
|
|
## **It's only used when the storage type is `S3`, `Oss`, `Gcs` and `Azblob`**.
|
|
## @toml2docs:none-default
|
|
endpoint = "https://s3.amazonaws.com"
|
|
|
|
## The region of the S3 service.
|
|
## **It's only used when the storage type is `S3`, `Oss`, `Gcs` and `Azblob`**.
|
|
## @toml2docs:none-default
|
|
region = "us-west-2"
|
|
|
|
## The http client options to the storage.
|
|
## **It's only used when the storage type is `S3`, `Oss`, `Gcs` and `Azblob`**.
|
|
[storage.http_client]
|
|
|
|
## The maximum idle connection per host allowed in the pool.
|
|
pool_max_idle_per_host = 1024
|
|
|
|
## The timeout for only the connect phase of a http client.
|
|
connect_timeout = "30s"
|
|
|
|
## The total request timeout, applied from when the request starts connecting until the response body has finished.
|
|
## Also considered a total deadline.
|
|
timeout = "30s"
|
|
|
|
## The timeout for idle sockets being kept-alive.
|
|
pool_idle_timeout = "90s"
|
|
|
|
## To skip the ssl verification
|
|
## **Security Notice**: Setting `skip_ssl_validation = true` disables certificate verification, making connections vulnerable to man-in-the-middle attacks. Only use this in development or trusted private networks.
|
|
skip_ssl_validation = false
|
|
|
|
# Custom storage options
|
|
# [[storage.providers]]
|
|
# name = "S3"
|
|
# type = "S3"
|
|
# bucket = "greptimedb"
|
|
# root = "data"
|
|
# access_key_id = "test"
|
|
# secret_access_key = "123456"
|
|
# endpoint = "https://s3.amazonaws.com"
|
|
# region = "us-west-2"
|
|
# [[storage.providers]]
|
|
# name = "Gcs"
|
|
# type = "Gcs"
|
|
# bucket = "greptimedb"
|
|
# root = "data"
|
|
# scope = "test"
|
|
# credential_path = "123456"
|
|
# credential = "base64-credential"
|
|
# endpoint = "https://storage.googleapis.com"
|
|
|
|
## The region engine options. You can configure multiple region engines.
|
|
[[region_engine]]
|
|
|
|
## The Mito engine options.
|
|
[region_engine.mito]
|
|
|
|
## Number of region workers.
|
|
#+ num_workers = 8
|
|
|
|
## Request channel size of each worker.
|
|
worker_channel_size = 128
|
|
|
|
## Max batch size for a worker to handle requests.
|
|
worker_request_batch_size = 64
|
|
|
|
## Number of meta action updated to trigger a new checkpoint for the manifest.
|
|
manifest_checkpoint_distance = 10
|
|
|
|
## Whether to compress manifest and checkpoint file by gzip (default false).
|
|
compress_manifest = false
|
|
|
|
## Max number of running background flush jobs (default: 1/2 of cpu cores).
|
|
## @toml2docs:none-default="Auto"
|
|
#+ max_background_flushes = 4
|
|
|
|
## Max number of running background compaction jobs (default: 1/4 of cpu cores).
|
|
## @toml2docs:none-default="Auto"
|
|
#+ max_background_compactions = 2
|
|
|
|
## Max number of running background purge jobs (default: number of cpu cores).
|
|
## @toml2docs:none-default="Auto"
|
|
#+ max_background_purges = 8
|
|
|
|
## Memory budget for compaction tasks.
|
|
## Supports absolute size (e.g., "2GiB", "512MB") or percentage of system memory (e.g., "50%").
|
|
## Setting it to 0 or "unlimited" disables the limit.
|
|
## @toml2docs:none-default="0"
|
|
#+ experimental_compaction_memory_limit = "0"
|
|
|
|
## Behavior when compaction cannot acquire memory from the budget.
|
|
## Options: "wait" (default, 10s), "wait(<duration>)", "fail"
|
|
## @toml2docs:none-default="wait"
|
|
#+ experimental_compaction_on_exhausted = "wait"
|
|
|
|
## Interval to auto flush a region if it has not flushed yet.
|
|
auto_flush_interval = "1h"
|
|
|
|
## Global write buffer size for all regions. If not set, it's default to 1/8 of OS memory with a max limitation of 1GB.
|
|
## @toml2docs:none-default="Auto"
|
|
#+ global_write_buffer_size = "1GB"
|
|
|
|
## Global write buffer size threshold to reject write requests. If not set, it's default to 2 times of `global_write_buffer_size`.
|
|
## @toml2docs:none-default="Auto"
|
|
#+ global_write_buffer_reject_size = "2GB"
|
|
|
|
## Default write buffer size for each region. Regions stall at this size and reject writes at twice this size. Setting it to 0 disables both limits unless the table specifies `write_buffer_size`.
|
|
#+ default_region_write_buffer_size = "0"
|
|
|
|
## Cache size for SST metadata. Setting it to 0 to disable the cache.
|
|
## If not set, it's default to 1/8 of OS memory with a max limitation of 512MB.
|
|
## @toml2docs:none-default="Auto"
|
|
#+ sst_meta_cache_size = "512MB"
|
|
|
|
## Cache size for vectors and arrow arrays. Setting it to 0 to disable the cache.
|
|
## If not set, it's default to 1/16 of OS memory with a max limitation of 512MB.
|
|
## @toml2docs:none-default="Auto"
|
|
#+ vector_cache_size = "512MB"
|
|
|
|
## Cache size for pages of SST row groups. Setting it to 0 to disable the cache.
|
|
## If not set, it's default to 1/8 of OS memory.
|
|
## @toml2docs:none-default="Auto"
|
|
#+ page_cache_size = "512MB"
|
|
|
|
## Cache size for time series selector (e.g. `last_value()`). Setting it to 0 to disable the cache.
|
|
## If not set, it's default to 1/16 of OS memory with a max limitation of 512MB.
|
|
## @toml2docs:none-default="Auto"
|
|
#+ selector_result_cache_size = "512MB"
|
|
|
|
## Cache size for flat range scan results. Setting it to 0 to disable the cache.
|
|
## If not set, it's default to 1/16 of OS memory with a max limitation of 512MB.
|
|
## @toml2docs:none-default="Auto"
|
|
#+ range_result_cache_size = "512MB"
|
|
|
|
## Cache size for prefilter results. Setting it to 0 to disable the cache.
|
|
## If not set, it's default to 1/32 of OS memory with a max limitation of 128MB.
|
|
## @toml2docs:none-default="Auto"
|
|
#+ prefilter_result_cache_size = "128MB"
|
|
|
|
## Whether to enable the write cache, it's enabled by default when using object storage. It is recommended to enable it when using object storage for better performance.
|
|
enable_write_cache = false
|
|
|
|
## File system path for write cache, defaults to `{data_home}`.
|
|
write_cache_path = ""
|
|
|
|
## Capacity for write cache. If your disk space is sufficient, it is recommended to set it larger.
|
|
write_cache_size = "5GiB"
|
|
|
|
## TTL for write cache.
|
|
## @toml2docs:none-default
|
|
write_cache_ttl = "8h"
|
|
|
|
## Preload index (puffin) files into cache on region open (default: true).
|
|
## When enabled, index files are loaded into the write cache during region initialization,
|
|
## which can improve query performance at the cost of longer startup times.
|
|
preload_index_cache = true
|
|
|
|
## Percentage of write cache capacity allocated for index (puffin) files (default: 20).
|
|
## The remaining capacity is used for data (parquet) files.
|
|
## Must be between 0 and 100 (exclusive). For example, with a 5GiB write cache and 20% allocation,
|
|
## 1GiB is reserved for index files and 4GiB for data files.
|
|
index_cache_percent = 20
|
|
|
|
## Enable refilling cache on read operations (default: true).
|
|
## When disabled, cache refilling on read won't happen.
|
|
enable_refill_cache_on_read = true
|
|
|
|
## Capacity for manifest cache (default: 256MB).
|
|
manifest_cache_size = "256MB"
|
|
|
|
## Buffer size for SST writing.
|
|
sst_write_buffer_size = "8MB"
|
|
|
|
## Maximum number of SST files to scan concurrently.
|
|
max_concurrent_scan_files = 384
|
|
|
|
## Whether to allow stale WAL entries read during replay.
|
|
allow_stale_entries = false
|
|
|
|
## Memory limit for table scans across all queries.
|
|
## Supports absolute size (e.g., "2GB") or percentage of system memory (e.g., "20%").
|
|
## Setting it to 0 or "unlimited" disables the limit.
|
|
scan_memory_limit = "unlimited"
|
|
## Controls what happens when a scan cannot get memory immediately.
|
|
## "fail" (default) fails fast and is the recommended option for most users.
|
|
## "wait" / "wait(<duration>)" waits for memory to become available. This is mainly
|
|
## for advanced tuning in bursty workloads where temporary contention is common and
|
|
## higher latency is acceptable.
|
|
## "wait" means "wait(10s)", not unlimited waiting.
|
|
scan_memory_on_exhausted = "fail"
|
|
|
|
## Minimum time interval between two compactions.
|
|
## To align with the old behavior, the default value is 0 (no restrictions).
|
|
min_compaction_interval = "0m"
|
|
|
|
## Whether to allow to schedule a compaction after a successful region edit.
|
|
##
|
|
## Setting this to "true" is a necessary but not sufficient condition for scheduling compaction after a region edit.
|
|
## Other constraints, such as "min_compaction_interval", may still prevent compaction from being scheduled.
|
|
## Setting this to "false", however, guarantees that compaction will not be scheduled after a region edit.
|
|
schedule_compaction_after_edit = true
|
|
|
|
## Whether to enable flat format as the default SST format.
|
|
default_flat_format = true
|
|
|
|
## The options for index in Mito engine.
|
|
[region_engine.mito.index]
|
|
|
|
## Auxiliary directory path for the index in filesystem, used to store intermediate files for
|
|
## creating the index and staging files for searching the index, defaults to `{data_home}/index_intermediate`.
|
|
## The default name for this directory is `index_intermediate` for backward compatibility.
|
|
##
|
|
## This path contains two subdirectories:
|
|
## - `__intm`: for storing intermediate files used during creating index.
|
|
## - `staging`: for storing staging files used during searching index.
|
|
aux_path = ""
|
|
|
|
## The max capacity of the staging directory.
|
|
staging_size = "2GB"
|
|
|
|
## The TTL of the staging directory.
|
|
## Defaults to 7 days.
|
|
## Setting it to "0s" to disable TTL.
|
|
staging_ttl = "7d"
|
|
|
|
## Cache size for inverted index metadata.
|
|
metadata_cache_size = "64MiB"
|
|
|
|
## Cache size for inverted index content.
|
|
content_cache_size = "128MiB"
|
|
|
|
## Page size for inverted index content cache.
|
|
content_cache_page_size = "64KiB"
|
|
|
|
## Cache size for index result.
|
|
result_cache_size = "128MiB"
|
|
|
|
## The options for inverted index in Mito engine.
|
|
[region_engine.mito.inverted_index]
|
|
|
|
## Whether to create the index on flush.
|
|
## - `auto`: automatically (default)
|
|
## - `disable`: never
|
|
create_on_flush = "auto"
|
|
|
|
## Whether to create the index on compaction.
|
|
## - `auto`: automatically (default)
|
|
## - `disable`: never
|
|
create_on_compaction = "auto"
|
|
|
|
## Whether to apply the index on query
|
|
## - `auto`: automatically (default)
|
|
## - `disable`: never
|
|
apply_on_query = "auto"
|
|
|
|
## Memory threshold for performing an external sort during index creation.
|
|
## - `auto`: automatically determine the threshold based on the system memory size (default)
|
|
## - `unlimited`: no memory limit
|
|
## - `[size]` e.g. `64MB`: fixed memory threshold
|
|
mem_threshold_on_create = "auto"
|
|
|
|
## Deprecated, use `region_engine.mito.index.aux_path` instead.
|
|
intermediate_path = ""
|
|
|
|
## The options for full-text index in Mito engine.
|
|
[region_engine.mito.fulltext_index]
|
|
|
|
## Whether to create the index on flush.
|
|
## - `auto`: automatically (default)
|
|
## - `disable`: never
|
|
create_on_flush = "auto"
|
|
|
|
## Whether to create the index on compaction.
|
|
## - `auto`: automatically (default)
|
|
## - `disable`: never
|
|
create_on_compaction = "auto"
|
|
|
|
## Whether to apply the index on query
|
|
## - `auto`: automatically (default)
|
|
## - `disable`: never
|
|
apply_on_query = "auto"
|
|
|
|
## Memory threshold for index creation.
|
|
## - `auto`: automatically determine the threshold based on the system memory size (default)
|
|
## - `unlimited`: no memory limit
|
|
## - `[size]` e.g. `64MB`: fixed memory threshold
|
|
mem_threshold_on_create = "auto"
|
|
|
|
## The options for bloom filter in Mito engine.
|
|
[region_engine.mito.bloom_filter_index]
|
|
|
|
## Whether to create the bloom filter on flush.
|
|
## - `auto`: automatically (default)
|
|
## - `disable`: never
|
|
create_on_flush = "auto"
|
|
|
|
## Whether to create the bloom filter on compaction.
|
|
## - `auto`: automatically (default)
|
|
## - `disable`: never
|
|
create_on_compaction = "auto"
|
|
|
|
## Whether to apply the bloom filter on query
|
|
## - `auto`: automatically (default)
|
|
## - `disable`: never
|
|
apply_on_query = "auto"
|
|
|
|
## Memory threshold for bloom filter creation.
|
|
## - `auto`: automatically determine the threshold based on the system memory size (default)
|
|
## - `unlimited`: no memory limit
|
|
## - `[size]` e.g. `64MB`: fixed memory threshold
|
|
mem_threshold_on_create = "auto"
|
|
|
|
[[region_engine]]
|
|
## Enable the file engine.
|
|
[region_engine.file]
|
|
|
|
[[region_engine]]
|
|
## Metric engine options.
|
|
[region_engine.metric]
|
|
|
|
## The logging options.
|
|
[logging]
|
|
## The directory to store the log files. If set to empty, logs will not be written to files.
|
|
dir = "./greptimedb_data/logs"
|
|
|
|
## The log level. Can be `info`/`debug`/`warn`/`error`.
|
|
## @toml2docs:none-default
|
|
level = "info"
|
|
|
|
## Enable OTLP tracing.
|
|
enable_otlp_tracing = false
|
|
|
|
## The OTLP tracing endpoint.
|
|
otlp_endpoint = "http://localhost:4318/v1/traces"
|
|
|
|
## Whether to append logs to stdout.
|
|
append_stdout = true
|
|
|
|
## Whether to write logs to files in `dir`.
|
|
enable_file_logging = true
|
|
|
|
## The log format. Can be `text`/`json`.
|
|
log_format = "text"
|
|
|
|
## The maximum amount of log files.
|
|
max_log_files = 720
|
|
|
|
## Whether to enable per-region metrics.
|
|
## Default to false.
|
|
enable_per_region_metrics = false
|
|
|
|
## The OTLP tracing export protocol. Can be `grpc`/`http`.
|
|
otlp_export_protocol = "http"
|
|
|
|
## Additional OTLP headers, only valid when using OTLP http
|
|
[logging.otlp_headers]
|
|
## @toml2docs:none-default
|
|
#Authorization = "Bearer my-token"
|
|
## @toml2docs:none-default
|
|
#Database = "My database"
|
|
|
|
## The percentage of tracing will be sampled and exported.
|
|
## Valid range `[0, 1]`, 1 means all traces are sampled, 0 means all traces are not sampled, the default value is 1.
|
|
## ratio > 1 are treated as 1. Fractions < 0 are treated as 0
|
|
[logging.tracing_sample_ratio]
|
|
default_ratio = 1.0
|
|
|
|
## The slow query log options.
|
|
[slow_query]
|
|
## Whether to enable slow query log.
|
|
#+ enable = false
|
|
|
|
## The record type of slow queries. It can be `system_table` or `log`.
|
|
## @toml2docs:none-default
|
|
#+ record_type = "system_table"
|
|
|
|
## The threshold of slow query.
|
|
## @toml2docs:none-default
|
|
#+ threshold = "10s"
|
|
|
|
## The sampling ratio of slow query log. The value should be in the range of (0, 1].
|
|
## @toml2docs:none-default
|
|
#+ sample_ratio = 1.0
|
|
|
|
## The tracing options. Only effect when compiled with `tokio-console` feature.
|
|
#+ [tracing]
|
|
## The tokio console address.
|
|
## @toml2docs:none-default
|
|
#+ tokio_console_addr = "127.0.0.1"
|
|
|
|
## Configuration options for the event recorder.
|
|
[event_recorder]
|
|
## TTL for the events table that will be used to store the events. Default is `90d`.
|
|
ttl = "90d"
|
|
## Event types to record. Current available event types: `create_database`,
|
|
## `alter_database`, `drop_database`, `create_flow`, `drop_flow`,
|
|
## `create_table`, `create_logical_tables`, `alter_table`, `alter_logical_tables`,
|
|
## `drop_table`, `undrop_table`, `purge_dropped_table`, `truncate_table`,
|
|
## `create_view`, `drop_view`.
|
|
## When omitted, all current and future event types are recorded.
|
|
## Set to an empty array to disable event recording.
|
|
#+ event_types = ["create_database"]
|
|
|
|
## The memory options.
|
|
[memory]
|
|
## Whether to enable heap profiling activation during startup.
|
|
## When enabled, heap profiling will be activated if the `MALLOC_CONF` environment variable
|
|
## is set to "prof:true,prof_active:false". The official image adds this env variable.
|
|
## Default is true.
|
|
enable_heap_profiling = true
|