mirror of
https://github.com/openobserve/openobserve.git
synced 2026-11-04 00:08:55 +00:00
Huaijin
37e5f55684
The query half of the TSID-major layout: #13882 wrote `tsid-major-v3-*` files ordered by `(__hash__, _timestamp)`; this PR adds the `.midx` series index next to them and makes PromQL prune through it. Supersedes the remaining scope of #13839. ## Write side - Every `tsid-major-v3-*` file written by the hour-end merge gets a `.midx` series index (Arrow IPC + zstd), stored like the Tantivy index under a derived root distinct from the data tree: `files/{org}/metrics/{stream}/{date}/{hour}/tsid-major-v3-{id}.parquet` → `files/{org}/midx/{stream}/{date}/{hour}/tsid-major-v3-{id}.midx` (Tantivy: `files/{org}/index/{stream}_{type}/…/{id}.ttv`): **one row per contiguous TSID run** — its `row_count: u32` plus the label columns of the run's first row (source types preserved). Runs tile the data file, so each run's starting row is the prefix sum of the preceding counts and is not stored; `__hash__` is excluded (matchers never reference it). The whole index is one ZSTD-compressed IPC batch. Runs crossing an input RecordBatch boundary appear as two adjacent entries; query-time range coalescing makes that representation equivalent without buffering samples. File and row-group boundaries stay independent of TSID boundaries. - `TsidSeriesIndexWriter` lives inside `search/datafusion/merge/tsid_major.rs` and collects run entries alongside the Parquet writer (one buffered row per run), encoding and compressing once at finish. `MergedFile` carries `series_index: Option<Vec<u8>>`; the compactor uploads it to the `midx/` tree (compliance mode included). - `.midx` is intentionally **not** tracked in `file_list`; its path is derived from the data file's key (`MetricsFileLayout::series_index_path`) and it is deleted together with the parent in the delayed-deletion pass (`deleted.rs`); like the `index/` tree, the `midx/` day directories are covered by retention's local-disk deletion and the gc tool. ## Query side (PromQL) - `promql_service::…::tsid_series_index::search` runs where Tantivy used to, before the metrics table is registered: for each TSID-major file it loads the `.midx` (concurrent, `2×target_partitions ≤ 64`), projects the row-range columns plus the matched labels **by name against each series index's own schema** (label sets / column orders differ between files), builds the matcher filter **per series index** with `physical_expr::create_physical_expr` — the exact same predicates as `apply_matchers` (`matcher_predicates`) — evaluates it over the run rows, coalesces adjacent/overlapping ranges and attaches them as `FileSelection::RowRanges` (converted into a Parquet access plan by the generic helper from #13861, no per-row bitmap). - **Applied per file**: files of any other layout in the same query (still-open hours, legacy data, late arrivals) are left in place and scanned in full. A matched label missing from a series index over-selects that file; a query whose matchers cannot be evaluated at all selects the whole file; any series index failure logs and falls back to a full scan. The final PromQL matchers are always applied, so results are exact in every case. - Evaluated selections are cached (LRU, 256 MiB) keyed by account + series index path + the full matcher set. - The Tantivy path in the PromQL storage scan is removed (`convert_matchers_to_index_condition`, index-field plumbing, `is_add_filter_back` bookkeeping). SQL search is untouched. ## Benchmark (from the #13839 POC, same layout + series index) 478,144,896 rows, 7 CPUs, one-hour PromQL range, step 15s, result cache off, release + mimalloc: | Layout | Filter steady median | No-filter steady median | Data files | Total storage | |---|---:|---:|---:|---:| | main, 2 GiB | 695.709 ms | 4.638 s | 88 | 4,925.81 MiB | | TSID-major + `.midx` | 269.496 ms | 2.793 s | 80 | 1,294.61 MiB | ## Validation - `cargo fmt --check`; `cargo check` / `clippy` on config, search, compaction, promql-service, jobs, search_service (`--tests`) clean - `config tsid_layout` 3 passed (incl. `series_index_path`); `search tsid_major mode:: merge::` 10 passed (rotation test now also asserts the series index, exact-run and coalescing series index tests); `compaction merge deleted` 33 passed; `promql-service` 20 passed + 1 pre-existing wall-clock flake (`cache::test_promql_cache_set_and_get`, fails intermittently on main too) - End-to-end on the #13839 branch (same code paths): hour-end merge writes N `tsid-major-v3-*` + `.midx`; PromQL takes the series index path (`selected 39 ranges across 4/4 files`), counts/sums match the synthetic data, cold-cache identical after restart; a mixed hour (3 major + 1 late `tsid-sorted`) reports `1 files without a series index left to tantivy/full scan` with unchanged results; delayed deletion removes `.parquet` **and** `.midx`
109 lines
4.6 KiB
TOML
109 lines
4.6 KiB
TOML
# Configuration documentation:
|
||
# https://embarkstudios.github.io/cargo-deny/index.html
|
||
|
||
[advisories]
|
||
yanked = "deny"
|
||
git-fetch-with-cli = true
|
||
ignore = [
|
||
"RUSTSEC-2023-0071", # from rsa, transitive dep of sqlx
|
||
"RUSTSEC-2021-0139", # from ansi_term, transitive dep of vrl
|
||
"RUSTSEC-2024-0370", # we need it even it is no longer maintained
|
||
"RUSTSEC-2024-0384", # from instant, dep of parking_lot
|
||
"RUSTSEC-2024-0388", # from derivative, dep of vrl
|
||
"RUSTSEC-2024-0421", # from idna, transitive dep of vrl
|
||
"RUSTSEC-2024-0436", # from paste, transitive dep of datafusion
|
||
"RUSTSEC-2024-0437", # from protobuf, transitive dep of prometheus
|
||
]
|
||
|
||
[licenses]
|
||
private = { ignore = true }
|
||
|
||
exceptions = [
|
||
# We should probably NOT bundle CA certs but use the OS ones.
|
||
{ name = "webpki-roots", allow = ["MPL-2.0","CDLA-Permissive-2.0"] },
|
||
{ name = "webpki-root-certs", allow = ["CDLA-Permissive-2.0"] }, # transitive dep via rustls-platform-verifier
|
||
{ name = "openobserve", allow = ["AGPL-3.0"] },
|
||
{ name = "openobserve-api-common", allow = ["AGPL-3.0"] },
|
||
{ name = "openobserve-api-grpc", allow = ["AGPL-3.0"] },
|
||
{ name = "openobserve-api-http", allow = ["AGPL-3.0"] },
|
||
{ name = "openobserve-api-ingest", allow = ["AGPL-3.0"] },
|
||
{ name = "openobserve-api-management", allow = ["AGPL-3.0"] },
|
||
{ name = "openobserve-api-pipelines", allow = ["AGPL-3.0"] },
|
||
{ name = "openobserve-api-search", allow = ["AGPL-3.0"] },
|
||
{ name = "openobserve-builtins", allow = ["AGPL-3.0"] },
|
||
{ name = "common", allow = ["AGPL-3.0"] },
|
||
{ name = "openobserve-core", allow = ["AGPL-3.0"] },
|
||
{ name = "openobserve-node", allow = ["AGPL-3.0"] },
|
||
{ name = "openobserve-jobs", allow = ["AGPL-3.0"] },
|
||
{ name = "openobserve-mcp", allow = ["AGPL-3.0"] },
|
||
{ name = "openobserve-org-storage", allow = ["AGPL-3.0"] },
|
||
{ name = "openobserve-sourcemaps", allow = ["AGPL-3.0"] },
|
||
{ name = "openobserve-tls", allow = ["AGPL-3.0"] },
|
||
{ name = "transform", allow = ["AGPL-3.0"] },
|
||
{ name = "super_cluster_queue", allow = ["AGPL-3.0"] },
|
||
{ name = "search", allow = ["AGPL-3.0"] },
|
||
{ name = "openobserve-cipher", allow = ["AGPL-3.0"] },
|
||
{ name = "db", allow = ["AGPL-3.0"] },
|
||
{ name = "promql", allow = ["AGPL-3.0"] },
|
||
{ name = "audit", allow = ["AGPL-3.0"] },
|
||
{ name = "search_service", allow = ["AGPL-3.0"] },
|
||
{ name = "usage_reporting", allow = ["AGPL-3.0"] },
|
||
{ name = "vortex", allow = ["AGPL-3.0"] },
|
||
{ name = "vortex-datafusion", allow = ["AGPL-3.0"] },
|
||
{ name = "metrics_index", allow = ["AGPL-3.0"] },
|
||
{ name = "config", allow = ["AGPL-3.0"] },
|
||
{ name = "infra", allow = ["AGPL-3.0"] },
|
||
{ name = "ingester", allow = ["AGPL-3.0"] },
|
||
{ name = "wal", allow = ["AGPL-3.0"] },
|
||
{ name = "proto", allow = ["AGPL-3.0"] },
|
||
{ name = "flight", allow = ["AGPL-3.0"] },
|
||
{ name = "tantivy_utils", allow = ["AGPL-3.0"] },
|
||
{ name = "report_server", allow = ["AGPL-3.0"] },
|
||
{ name = "web", allow = ["AGPL-3.0"] },
|
||
{ name = "arrayref", allow = ["BSD-2-Clause"] },
|
||
{ name = "mach", allow = ["BSD-2-Clause"] },
|
||
{ name = "fastdivide", allow = ["zlib-acknowledgement"] },
|
||
{ name = "foldhash", allow = ["Zlib"] },
|
||
{ name = "adler32", allow = ["Zlib"] },
|
||
{ name = "libz-rs-sys", allow = ["Zlib"] },
|
||
{ name = "libz-rs", allow = ["Zlib"] },
|
||
{ name = "zlib-rs", allow = ["Zlib"] },
|
||
# These two are transitive deps of vrl via dep rustyline.
|
||
# However, rustyline is needed only when we enable feature cli
|
||
# which we haven't. Probably in Cargo.lock due to old code, need
|
||
# to take a look and remove.
|
||
{ name = "clipboard-win", allow = ["BSL-1.0"] },
|
||
{ name = "error-code", allow = ["BSL-1.0"] },
|
||
{ name = "base16", allow = ["CC0-1.0"] },
|
||
{ name = "tiny-keccak", allow = ["CC0-1.0"] },
|
||
{ name = "ring", allow = ["BSD-4-Clause", "OpenSSL"] },
|
||
{ name = "inferno", allow = ["CDDL-1.0"] },
|
||
{ name = "libbz2-rs-sys", allow = ["bzip2-1.0.6"] },
|
||
{ name = "aws-lc-sys", allow = ["OpenSSL"] },
|
||
{ name = "borrow-or-share", allow = ["MIT-0"] },
|
||
]
|
||
|
||
allow = [
|
||
"Apache-2.0",
|
||
"Apache-2.0 WITH LLVM-exception",
|
||
"MPL-2.0",
|
||
"MIT",
|
||
"BSD-3-Clause", # OSI + fsf free
|
||
"Unicode-3.0", # OSI
|
||
"ISC", # OSI + fsf free
|
||
"BSL-1.0",
|
||
"CC0-1.0",
|
||
"0BSD", # OSI approved - BSD Zero Clause License
|
||
]
|
||
|
||
[[licenses.clarify]]
|
||
name = "ring"
|
||
expression = "BSD-4-Clause AND ISC AND MIT AND OpenSSL"
|
||
license-files = [{ path = "LICENSE", hash = 0xbd0eed23 }]
|
||
|
||
[sources.allow-org]
|
||
github = ["openobserve", "apache"]
|
||
|
||
[sources]
|
||
allow-git = ["https://github.com/mattsse/chromiumoxide"]
|