diff --git a/Cargo.lock b/Cargo.lock index cb8c44311a..ef30aa41ec 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1105,7 +1105,7 @@ dependencies = [ "log", "pin-project-lite", "tokio", - "tungstenite 0.20.1", + "tungstenite", ] [[package]] @@ -1733,6 +1733,15 @@ dependencies = [ "serde", ] +[[package]] +name = "bitpacking" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c1d3e2bfd8d06048a179f7b17afc3188effa10385e7b00dc65af6aae732ea92" +dependencies = [ + "crunchy", +] + [[package]] name = "bitvec" version = "1.0.1" @@ -1995,6 +2004,12 @@ dependencies = [ "shlex", ] +[[package]] +name = "census" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f4c707c6a209cbe82d10abd08e1ea8995e9ea937d2550646e02798948992be0" + [[package]] name = "cesu8" version = "1.1.0" @@ -2451,6 +2466,7 @@ dependencies = [ "arrow-json", "arrow-schema", "async-recursion", + "async-trait", "async-walkdir", "aws-config", "aws-sdk-sns", @@ -2496,7 +2512,9 @@ dependencies = [ "sqlx", "svix-ksuid", "sysinfo", + "tantivy", "tokio", + "tokio-util", "tracing", "tracing-log", "tracing-subscriber", @@ -3530,6 +3548,12 @@ version = "0.15.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b" +[[package]] +name = "downcast-rs" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75b325c5dbd37f80359721ad39aca5a29fb04c89279657cffdda8736d0c0b9d2" + [[package]] name = "dunce" version = "1.0.5" @@ -3764,6 +3788,12 @@ dependencies = [ "regex-syntax 0.8.5", ] +[[package]] +name = "fastdivide" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59668941c55e5c186b8b58c391629af56774ec768f73c08bbcd56f09348eb00b" + [[package]] name = "fastrand" version = "1.9.0" @@ -3873,6 +3903,12 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "foldhash" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f81ec6369c545a7d40e4589b5597581fa1c441fe1cce96dd1de43159910a36a2" + [[package]] name = "form_urlencoded" version = "1.2.1" @@ -3882,6 +3918,16 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "fs4" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7e180ac76c23b45e767bd7ae9579bc0bb458618c4bc71835926e098e61d15f8" +dependencies = [ + "rustix", + "windows-sys 0.52.0", +] + [[package]] name = "fst" version = "0.4.7" @@ -4217,6 +4263,17 @@ dependencies = [ "serde", ] +[[package]] +name = "hashbrown" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e087f84d4f86bf4b218b927129862374b72199ae7d8657835f1e89000eea4fb" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash", +] + [[package]] name = "hashlink" version = "0.9.1" @@ -4327,6 +4384,12 @@ dependencies = [ "windows", ] +[[package]] +name = "htmlescape" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9025058dae765dee5070ec375f591e2ba14638c63feff74f13805a72e523163" + [[package]] name = "http" version = "0.2.12" @@ -5115,6 +5178,12 @@ dependencies = [ "webpki-roots 0.26.6", ] +[[package]] +name = "levenshtein_automata" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c2cdeb66e45e9f36bfad5bbdb4d2384e70936afbee843c6f6543f0c551ebb25" + [[package]] name = "lexical-core" version = "1.0.2" @@ -5382,6 +5451,15 @@ dependencies = [ "vob", ] +[[package]] +name = "lru" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "234cf4f4a04dc1f57e24b96cc0cd600cf2af460d4161ac5ecdd0af8e1f3b2a38" +dependencies = [ + "hashbrown 0.15.0", +] + [[package]] name = "lz4_flex" version = "0.11.3" @@ -5460,6 +5538,16 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "490cc448043f947bae3cbee9c203358d62dbee0db12107a74be5c30ccfd09771" +[[package]] +name = "measure_time" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbefd235b0aadd181626f281e1d684e116972988c14c264e42069d5e8a5775cc" +dependencies = [ + "instant", + "log", +] + [[package]] name = "mediatype" version = "0.19.18" @@ -5591,6 +5679,12 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9252111cf132ba0929b6f8e030cac2a24b507f3a4d6db6fb2896f27b354c714b" +[[package]] +name = "murmurhash32" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2195bf6aa996a481483b29d62a7663eed3fe39600c460e323f8ff41e90bdd89b" + [[package]] name = "names" version = "0.14.0" @@ -5927,6 +6021,12 @@ version = "1.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775" +[[package]] +name = "oneshot" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e296cf87e61c9cfc1a61c3c63a0f7f286ed4554e0e22be84e8a38e1d264a2a29" + [[package]] name = "onig" version = "6.4.0" @@ -6058,12 +6158,11 @@ dependencies = [ "strum 0.25.0", "sysinfo", "syslog_loose 0.18.0", + "tantivy", "thiserror", "tikv-jemallocator", "time", "tokio", - "tokio-stream", - "tokio-tungstenite", "tonic 0.12.3", "tracing", "tracing-appender", @@ -6275,6 +6374,15 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" +[[package]] +name = "ownedbytes" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3a059efb063b8f425b948e042e6b9bd85edfe60e913630ed727b23e2dfcc558" +dependencies = [ + "stable_deref_trait", +] + [[package]] name = "owo-colors" version = "3.5.0" @@ -7118,7 +7226,7 @@ dependencies = [ "pin-project-lite", "quinn-proto", "quinn-udp", - "rustc-hash", + "rustc-hash 2.0.0", "rustls 0.23.15", "socket2", "thiserror", @@ -7135,7 +7243,7 @@ dependencies = [ "bytes", "rand", "ring", - "rustc-hash", + "rustc-hash 2.0.0", "rustls 0.23.15", "slab", "thiserror", @@ -7207,6 +7315,16 @@ dependencies = [ "getrandom", ] +[[package]] +name = "rand_distr" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32cb0b9bc82b0a0876c2dd994a7e7a2683d3e7390ca40e6886785ef0c7e3ee31" +dependencies = [ + "num-traits", + "rand", +] + [[package]] name = "raw-cpuid" version = "10.7.0" @@ -7585,6 +7703,16 @@ dependencies = [ "walkdir", ] +[[package]] +name = "rust-stemmers" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e46a2036019fdb888131db7a4c847a1063a7493f971ed94ea82c67eada63ca54" +dependencies = [ + "serde", + "serde_derive", +] + [[package]] name = "rust_decimal" version = "1.36.0" @@ -7607,6 +7735,12 @@ version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" +[[package]] +name = "rustc-hash" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" + [[package]] name = "rustc-hash" version = "2.0.0" @@ -8195,6 +8329,15 @@ version = "0.3.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "38b58827f4464d87d377d175e90bf58eb00fd8716ff0a62f80356b5e61555d0d" +[[package]] +name = "sketches-ddsketch" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85636c14b73d81f541e525f585c0a2109e6744e1565b5c1668e31c70c10ed65c" +dependencies = [ + "serde", +] + [[package]] name = "slab" version = "0.4.9" @@ -8850,6 +8993,149 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417" +[[package]] +name = "tantivy" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8d0582f186c0a6d55655d24543f15e43607299425c5ad8352c242b914b31856" +dependencies = [ + "aho-corasick", + "arc-swap", + "base64 0.22.1", + "bitpacking", + "byteorder", + "census", + "crc32fast", + "crossbeam-channel", + "downcast-rs", + "fastdivide", + "fnv", + "fs4", + "futures-util", + "htmlescape", + "itertools 0.12.1", + "levenshtein_automata", + "log", + "lru", + "lz4_flex", + "measure_time", + "memmap2", + "num_cpus", + "once_cell", + "oneshot", + "rayon", + "regex", + "rust-stemmers", + "rustc-hash 1.1.0", + "serde", + "serde_json", + "sketches-ddsketch", + "smallvec", + "tantivy-bitpacker", + "tantivy-columnar", + "tantivy-common", + "tantivy-fst", + "tantivy-query-grammar", + "tantivy-sstable", + "tantivy-stacker", + "tantivy-tokenizer-api", + "tempfile", + "thiserror", + "time", + "uuid", + "winapi 0.3.9", +] + +[[package]] +name = "tantivy-bitpacker" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "284899c2325d6832203ac6ff5891b297fc5239c3dc754c5bc1977855b23c10df" +dependencies = [ + "bitpacking", +] + +[[package]] +name = "tantivy-columnar" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12722224ffbe346c7fec3275c699e508fd0d4710e629e933d5736ec524a1f44e" +dependencies = [ + "downcast-rs", + "fastdivide", + "itertools 0.12.1", + "serde", + "tantivy-bitpacker", + "tantivy-common", + "tantivy-sstable", + "tantivy-stacker", +] + +[[package]] +name = "tantivy-common" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8019e3cabcfd20a1380b491e13ff42f57bb38bf97c3d5fa5c07e50816e0621f4" +dependencies = [ + "async-trait", + "byteorder", + "ownedbytes", + "serde", + "time", +] + +[[package]] +name = "tantivy-fst" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d60769b80ad7953d8a7b2c70cdfe722bbcdcac6bccc8ac934c40c034d866fc18" +dependencies = [ + "byteorder", + "regex-syntax 0.8.5", + "utf8-ranges", +] + +[[package]] +name = "tantivy-query-grammar" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "847434d4af57b32e309f4ab1b4f1707a6c566656264caa427ff4285c4d9d0b82" +dependencies = [ + "nom", +] + +[[package]] +name = "tantivy-sstable" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c69578242e8e9fc989119f522ba5b49a38ac20f576fc778035b96cc94f41f98e" +dependencies = [ + "tantivy-bitpacker", + "tantivy-common", + "tantivy-fst", + "zstd", +] + +[[package]] +name = "tantivy-stacker" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c56d6ff5591fc332739b3ce7035b57995a3ce29a93ffd6012660e0949c956ea8" +dependencies = [ + "murmurhash32", + "rand_distr", + "tantivy-common", +] + +[[package]] +name = "tantivy-tokenizer-api" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a0dcade25819a89cfe6f17d932c9cedff11989936bf6dd4f336d50392053b04" +dependencies = [ + "serde", +] + [[package]] name = "tap" version = "1.0.1" @@ -9104,18 +9390,6 @@ dependencies = [ "tokio", ] -[[package]] -name = "tokio-tungstenite" -version = "0.23.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6989540ced10490aaf14e6bad2e3d33728a2813310a0c71d1574304c49631cd" -dependencies = [ - "futures-util", - "log", - "tokio", - "tungstenite 0.23.0", -] - [[package]] name = "tokio-util" version = "0.7.12" @@ -9124,6 +9398,7 @@ checksum = "61e7c3654c13bcd040d4a03abee2c75b1d14a37b423cf5a813ceae1cc903ec6a" dependencies = [ "bytes", "futures-core", + "futures-io", "futures-sink", "pin-project-lite", "tokio", @@ -9446,24 +9721,6 @@ dependencies = [ "utf-8", ] -[[package]] -name = "tungstenite" -version = "0.23.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e2e2ce1e47ed2994fd43b04c8f618008d4cabdd5ee34027cf14f9d918edd9c8" -dependencies = [ - "byteorder", - "bytes", - "data-encoding", - "http 1.1.0", - "httparse", - "log", - "rand", - "sha1", - "thiserror", - "utf-8", -] - [[package]] name = "twox-hash" version = "1.6.3" diff --git a/Cargo.toml b/Cargo.toml index 8b2e4cf1bf..3b4f70b59f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,16 +3,16 @@ description = "OpenObserve is an observability platform that allows you to captu edition = "2021" homepage = "https://openobserve.ai/" keywords = [ - "search", - "log", - "metrics", - "traces", - "analytics", - "openobserve", - "observability", - "open source", - "open-source", - "opensource", + "search", + "log", + "metrics", + "traces", + "analytics", + "openobserve", + "observability", + "open source", + "open-source", + "opensource", ] license = "AGPL-3.0-only" name = "openobserve" @@ -54,12 +54,11 @@ actix-web-lab = "0.20" actix-web-opentelemetry = { version = "0.17", features = ["metrics"] } actix-web-prometheus.workspace = true actix-web-rust-embed-responder = { version = "2.2", default-features = false, features = [ - "support-rust-embed-for-web", - "base64", + "support-rust-embed-for-web", + "base64", ] } actix-ws = "0.2.5" actix = "0.13.5" -tokio-tungstenite = "0.23.1" actix-web-actors = "4.3.0" ahash.workspace = true anyhow.workspace = true @@ -74,11 +73,11 @@ blake3 = { version = "1.4", features = ["rayon"] } bytes.workspace = true chrono.workspace = true clap = { version = "4.1", default-features = false, features = [ - "std", - "help", - "usage", - "suggestions", - "cargo", + "std", + "help", + "usage", + "suggestions", + "cargo", ] } cloudevents-sdk = { version = "0.7.0", features = ["actix"] } cron = "0.12.1" @@ -112,23 +111,23 @@ mimalloc = { version = "0.1", default-features = false, optional = true } once_cell.workspace = true opentelemetry = "0.25" opentelemetry_sdk = { version = "0.25", features = [ - "rt-tokio", - "trace", - "metrics", + "rt-tokio", + "trace", + "metrics", ] } opentelemetry-otlp = { version = "0.25", features = [ - "http-proto", - "serialize", - "serde", - "trace", - "reqwest-client", + "http-proto", + "serialize", + "serde", + "trace", + "reqwest-client", ] } opentelemetry-proto = { version = "0.25", features = [ - "gen-tonic", - "with-serde", - "logs", - "metrics", - "trace", + "gen-tonic", + "with-serde", + "logs", + "metrics", + "trace", ] } parking_lot.workspace = true prometheus.workspace = true @@ -158,7 +157,6 @@ thiserror.workspace = true time.workspace = true tikv-jemallocator = { version = "0.5", optional = true } tokio.workspace = true -tokio-stream.workspace = true console-subscriber = { version = "0.2", optional = true } tonic.workspace = true tracing.workspace = true @@ -179,6 +177,7 @@ ingester.workspace = true report_server.workspace = true chromiumoxide.workspace = true lettre.workspace = true +tantivy.workspace = true [build-dependencies] chrono = { version = "0.4", default-features = false, features = ["clock"] } @@ -190,12 +189,12 @@ float-cmp = "0.9" [workspace] members = [ - "src/config", - "src/infra", - "src/ingester", - "src/wal", - "src/proto", - "src/report_server", + "src/config", + "src/infra", + "src/ingester", + "src/wal", + "src/proto", + "src/report_server", ] resolver = "2" @@ -227,8 +226,8 @@ bitvec = "1.0" bytes = "1.4" byteorder = "1.4.3" chromiumoxide = { git = "https://github.com/mattsse/chromiumoxide", features = [ - "tokio-runtime", - "_fetcher-rusttls-tokio", + "tokio-runtime", + "_fetcher-rusttls-tokio", ], default-features = false, rev = "348967500868c2b5dfcb3930fe483b2ff17b35ff" } chrono = { version = "0.4", default-features = false, features = ["clock"] } cityhasher = { version = "0.1", default-features = false } @@ -260,12 +259,12 @@ indexmap = { version = "2.0", features = ["serde"] } ipnetwork = "0.20" itertools = "0.12" lettre = { version = "0.11", default-features = false, features = [ - "builder", - "hostname", - "smtp-transport", - "pool", - "tokio1", - "tokio1-rustls-tls", + "builder", + "hostname", + "smtp-transport", + "pool", + "tokio1", + "tokio1-rustls-tls", ] } log = "0.4" md5 = "0.7.0" @@ -281,13 +280,13 @@ rayon = "1.10" regex = "1.7" regex-syntax = "0.8" reqwest = { version = "0.12", default-features = false, features = [ - "rustls-tls-native-roots", - "stream", + "rustls-tls-native-roots", + "stream", ] } sea-orm = { version = "1.1.0", features = [ - "sqlx-all", - "runtime-tokio-rustls", - "macros", + "sqlx-all", + "runtime-tokio-rustls", + "macros", ] } segment = "~0.2.4" serde = { version = "1", features = ["derive"] } @@ -296,18 +295,20 @@ sha256 = "1.4.0" snafu = "0.7.5" snap = "1" sqlx = { version = "0.8.1", features = [ - "runtime-tokio-rustls", - "postgres", - "mysql", - "sqlite", - "chrono", + "runtime-tokio-rustls", + "postgres", + "mysql", + "sqlite", + "chrono", ] } svix-ksuid = { version = "0.8", features = ["serde"] } sysinfo = "0.29" +tantivy = { version = "0.22.0", features = ["quickwit"] } tempfile = "3" thiserror = "1.0" time = "0.3" tokio = { version = "1", features = ["full"] } +tokio-util = { version = "0.7.12", features = ["compat"] } tokio-stream = "0.1" tonic = { version = "0.12.3", features = ["prost", "gzip"] } tracing = "0.1.40" @@ -315,9 +316,9 @@ tracing-appender = "0.2.3" tracing-log = "0.2" tracing-opentelemetry = "0.26" tracing-subscriber = { version = "0.3.0", features = [ - "registry", - "env-filter", - "json", + "registry", + "env-filter", + "json", ] } url = "2.2" urlencoding = "2.1" diff --git a/deny.toml b/deny.toml index 9659841859..fb0cb3cd55 100644 --- a/deny.toml +++ b/deny.toml @@ -20,6 +20,8 @@ exceptions = [ { name = "report_server", allow = ["AGPL-3.0"] }, { name = "arrayref", allow = ["BSD-2-Clause"] }, { name = "mach", allow = ["BSD-2-Clause"] }, + { name = "fastdivide", allow = ["zlib-acknowledgement"] }, + { name = "foldhash", allow = ["Zlib"] }, # These two are transitive deps of vrl via dep rustyline. # However, rustyline is needed only when we enable feature cli @@ -32,7 +34,7 @@ exceptions = [ { name = "tiny-keccak", allow = ["CC0-1.0"] }, { name = "enum-iterator-derive", allow = ["0BSD"] }, { name = "quoted_printable", allow = ["0BSD"] }, - { name = "ring", allow = ["BSD-4-Clause","OpenSSL"] }, + { name = "ring", allow = ["BSD-4-Clause", "OpenSSL"] }, { name = "unicode-ident", allow = ["Unicode-DFS-2016"] }, { name = "adler32", allow = ["Zlib"] }, ] @@ -42,9 +44,9 @@ allow = [ "Apache-2.0 WITH LLVM-exception", "MPL-2.0", "MIT", - "BSD-3-Clause", # OSI + fsf free - "Unicode-3.0", # OSI - "ISC", # OSI + fsf free + "BSD-3-Clause", # OSI + fsf free + "Unicode-3.0", # OSI + "ISC", # OSI + fsf free ] [[licenses.clarify]] diff --git a/src/cli/basic/cli.rs b/src/cli/basic/cli.rs index 674e89efd6..4f64470ac8 100644 --- a/src/cli/basic/cli.rs +++ b/src/cli/basic/cli.rs @@ -220,10 +220,6 @@ pub async fn cli() -> Result { } } "migrate-file-list" => { - let prefix = match command.get_one::("prefix") { - Some(prefix) => prefix.to_string(), - None => "".to_string(), - }; let from = match command.get_one::("from") { Some(from) => from.to_string(), None => "".to_string(), @@ -232,11 +228,8 @@ pub async fn cli() -> Result { Some(to) => to.to_string(), None => "".to_string(), }; - println!( - "Running migration file_list from {} to {}, with prefix: {}", - from, to, prefix - ); - migration::file_list::run(&prefix, &from, &to).await?; + println!("Running migration file_list from {} to {}", from, to); + migration::file_list::run(&from, &to).await?; } "migrate-meta" => { let from = match command.get_one::("from") { diff --git a/src/cli/data/export.rs b/src/cli/data/export.rs index 9e0ce99d6c..6c4eabebc8 100644 --- a/src/cli/data/export.rs +++ b/src/cli/data/export.rs @@ -76,7 +76,6 @@ impl Context for Export { timeout: 0, search_type, search_event_context, - index_type: "".to_string(), }; match SearchService::search("", &c.org, stream_type, None, &req).await { diff --git a/src/common/migration/file_list.rs b/src/common/migration/file_list.rs index 59a983f070..0fbc10acd4 100644 --- a/src/common/migration/file_list.rs +++ b/src/common/migration/file_list.rs @@ -15,31 +15,19 @@ use config::{ get_config, - meta::stream::{FileKey, PartitionTimeLevel, StreamType}, + meta::stream::{FileKey, PartitionTimeLevel, ALL_STREAM_TYPES}, utils::time::BASE_TIME, }; use infra::file_list as infra_file_list; -use crate::{ - job::file_list, - service::{compact::stats::update_stats_from_file_list, db}, -}; +use crate::service::{compact::stats::update_stats_from_file_list, db}; -pub async fn run(prefix: &str, from: &str, to: &str) -> Result<(), anyhow::Error> { +pub async fn run(from: &str, to: &str) -> Result<(), anyhow::Error> { // check wal dir std::fs::create_dir_all(&get_config().common.data_wal_dir).expect("create wal dir success"); // load stream list - let mut from_storage = false; let src: Box = match from.to_lowercase().as_str().trim() { - "local" | "s3" | "sled" | "etcd" => { - // load file list to db - db::file_list::remote::cache(prefix, false) - .await - .expect("file list migration failed"); - from_storage = true; - Box::::default() - } "sqlite" => Box::::default(), "mysql" => Box::::default(), "postgres" | "postgresql" => Box::::default(), @@ -55,24 +43,12 @@ pub async fn run(prefix: &str, from: &str, to: &str) -> Result<(), anyhow::Error dest.create_table().await?; db::schema::cache().await?; - // move file_list from wal for disk - if let Err(e) = file_list::move_file_list_to_storage(false).await { - log::error!("Error moving disk files to remote: {}", e); - } - // load stream list - let stream_types = [ - StreamType::Logs, - StreamType::Metrics, - StreamType::Traces, - StreamType::EnrichmentTables, - StreamType::Metadata, - ]; let start_time = BASE_TIME.timestamp_micros(); let end_time = chrono::Utc::now().timestamp_micros(); let orgs = db::schema::list_organizations_from_cache().await; for org_id in orgs.iter() { - for stream_type in stream_types { + for stream_type in ALL_STREAM_TYPES { let streams = db::schema::list_streams_from_cache(org_id, stream_type).await; for stream_name in streams.iter() { // load file_list from source @@ -97,64 +73,13 @@ pub async fn run(prefix: &str, from: &str, to: &str) -> Result<(), anyhow::Error } } - // load file_list_deleted from storage - if from_storage { - let files = - crate::service::compact::file_list_deleted::load_prefix_from_s3(org_id).await?; - if !files.is_empty() { - let files = files - .values() - .flatten() - .map(|file| file.0.to_owned()) - .collect::>(); - if let Err(e) = dest - .batch_add_deleted(org_id, false, end_time, &files) - .await - { - log::error!("load file_list_deleted into db err: {}", e); - } - } - } - // load file_list_deleted from source let files = src .query_deleted(org_id, end_time, 1_000_000) .await .expect("load file_list_deleted failed"); - let files_need_flatten = files - .iter() - .filter_map(|(file, flattened)| { - if *flattened { - Some(file.to_string()) - } else { - None - } - }) - .collect::>(); - let files_noneed_flatten = files - .iter() - .filter_map(|(file, flattened)| { - if !*flattened { - Some(file.to_string()) - } else { - None - } - }) - .collect::>(); - if !files_need_flatten.is_empty() { - if let Err(e) = dest - .batch_add_deleted(org_id, true, end_time, &files_need_flatten) - .await - { - log::error!("load file_list_deleted into db err: {}", e); - continue; - } - } - if !files_noneed_flatten.is_empty() { - if let Err(e) = dest - .batch_add_deleted(org_id, false, end_time, &files_noneed_flatten) - .await - { + if !files.is_empty() { + if let Err(e) = dest.batch_add_deleted(org_id, end_time, &files).await { log::error!("load file_list_deleted into db err: {}", e); continue; } diff --git a/src/common/migration/mod.rs b/src/common/migration/mod.rs index ffbba5cb5e..25fa742d5d 100644 --- a/src/common/migration/mod.rs +++ b/src/common/migration/mod.rs @@ -36,12 +36,6 @@ pub async fn check_upgrade(old_ver: &str, new_ver: &str) -> Result<(), anyhow::E } log::info!("Upgrading from {} to {}", old_ver, new_ver); - let v053 = Version::from("v0.5.3").unwrap(); - if old_ver < v053 && new_ver.to_string().starts_with("v0.6.") { - upgrade_052_053().await?; - return Ok(()); - } - let v093 = Version::from("v0.9.3").unwrap(); if old_ver < v093 { upgrade_092_093().await?; @@ -55,16 +49,6 @@ pub async fn check_upgrade(old_ver: &str, new_ver: &str) -> Result<(), anyhow::E Ok(()) } -async fn upgrade_052_053() -> Result<(), anyhow::Error> { - // migration for metadata - meta::run("sled", "sqlite").await?; - - // migration for file_list - file_list::run("", "sled", "sqlite").await?; - - Ok(()) -} - async fn upgrade_092_093() -> Result<(), anyhow::Error> { // migration schema schema::run().await?; diff --git a/src/common/utils/http.rs b/src/common/utils/http.rs index c201312673..84896c9a6d 100644 --- a/src/common/utils/http.rs +++ b/src/common/utils/http.rs @@ -21,12 +21,9 @@ use std::{ use actix_web::{http::header::HeaderName, web::Query}; use awc::http::header::HeaderMap; -use config::{ - get_config, - meta::{ - search::{SearchEventContext, SearchEventType}, - stream::StreamType, - }, +use config::meta::{ + search::{SearchEventContext, SearchEventType}, + stream::StreamType, }; use opentelemetry::{global, propagation::Extractor, trace::TraceContextExt}; use tracing_opentelemetry::OpenTelemetrySpanExt; @@ -104,41 +101,6 @@ pub(crate) fn get_search_event_context_from_request( } } -/// Index type for a search can be either `parquet` or `fst`. It's only effective when env -/// `ZO_INVERTED_INDEX_STORE_FORMAT` is set as `both`. -/// Otherwise 'index_type' is set by env `ZO_INVERTED_INDEX_SEARCH_FORMAT`, which is also -/// the same as store format if store format is not `both`. -/// -/// For performance testing phrase only. Will be deprecated after 1 month. -#[inline(always)] -pub(crate) fn get_index_type_from_request( - query: &Query>, -) -> Result { - let cfg = get_config(); - let index_type = query - .get("index_type") - .cloned() - .unwrap_or_default() - .to_lowercase(); - if index_type.is_empty() || index_type == cfg.common.inverted_index_search_format { - Ok(cfg.common.inverted_index_search_format.to_string()) - } else if cfg.common.inverted_index_store_format == "both" { - match index_type.as_str() { - "parquet" => Ok("parquet".to_string()), - "fst" => Ok("fst".to_string()), - _ => Err(Error::new( - ErrorKind::Other, - "'index_type' query param with value 'parquet' or 'fst' allowed", - )), - } - } else { - Err(Error::new( - ErrorKind::Other, - "'index_type' query param with value 'parquet' or 'fst' allowed", - )) - } -} - #[inline(always)] pub(crate) fn get_use_cache_from_request(query: &Query>) -> bool { let Some(v) = query.get("use_cache") else { diff --git a/src/common/utils/stream.rs b/src/common/utils/stream.rs index c809d01c88..40a0f07838 100644 --- a/src/common/utils/stream.rs +++ b/src/common/utils/stream.rs @@ -179,6 +179,7 @@ mod tests { original_size: 1000, compressed_size: 700, flattened: false, + index_size: 0, }; populate_file_meta(&[&batch], &mut file_meta, None, None) .await @@ -215,6 +216,7 @@ mod tests { original_size: 1000, compressed_size: 700, flattened: false, + index_size: 0, }; populate_file_meta(&[&batch], &mut file_meta, Some("time"), Some("time")) .await diff --git a/src/config/Cargo.toml b/src/config/Cargo.toml index 76478f549e..7e78e0651b 100644 --- a/src/config/Cargo.toml +++ b/src/config/Cargo.toml @@ -16,6 +16,7 @@ arc-swap.workspace = true arrow.workspace = true arrow-json.workspace = true arrow-schema.workspace = true +async-trait.workspace = true async-recursion.workspace = true async-walkdir.workspace = true aws-config.workspace = true @@ -62,7 +63,9 @@ sqlparser.workspace = true sqlx.workspace = true svix-ksuid.workspace = true sysinfo.workspace = true +tantivy.workspace = true tokio.workspace = true +tokio-util.workspace = true tracing.workspace = true tracing-log.workspace = true tracing-subscriber.workspace = true diff --git a/src/config/src/config.rs b/src/config/src/config.rs index d5a372e67e..6dd6a34b30 100644 --- a/src/config/src/config.rs +++ b/src/config/src/config.rs @@ -61,6 +61,7 @@ pub const FILE_EXT_JSON: &str = ".json"; pub const FILE_EXT_ARROW: &str = ".arrow"; pub const FILE_EXT_PARQUET: &str = ".parquet"; pub const FILE_EXT_PUFFIN: &str = ".puffin"; +pub const FILE_EXT_TANTIVY: &str = ".ttv"; pub const INDEX_FIELD_NAME_FOR_ALL: &str = "_all"; @@ -727,13 +728,13 @@ pub struct Common { #[env_config( name = "ZO_INVERTED_INDEX_STORE_FORMAT", default = "parquet", - help = "InvertedIndex store format, parquet(default), or both." + help = "InvertedIndex store format, parquet(default), tantivy, both" )] pub inverted_index_store_format: String, #[env_config( name = "ZO_INVERTED_INDEX_SEARCH_FORMAT", default = "parquet", - help = "InvertedIndex search format. Can only be configured when store format is both. Otherwise, it's set by store format" + help = "InvertedIndex search format, parquet(default), tantivy." )] pub inverted_index_search_format: String, #[env_config( @@ -1583,16 +1584,26 @@ fn check_common_config(cfg: &mut Config) -> Result<(), anyhow::Error> { // check default inverted index search format cfg.common.inverted_index_store_format = cfg.common.inverted_index_store_format.to_lowercase(); if cfg.common.inverted_index_store_format.is_empty() { - cfg.common.inverted_index_search_format = "parquet".to_string(); + cfg.common.inverted_index_store_format = "parquet".to_string(); } - if !["both", "parquet"].contains(&cfg.common.inverted_index_store_format.as_str()) { + if !["both", "parquet", "tantivy"].contains(&cfg.common.inverted_index_store_format.as_str()) { return Err(anyhow::anyhow!( - "ZO_INVERTED_INDEX_STORE_FORMAT must be one of both, parquet." + "ZO_INVERTED_INDEX_STORE_FORMAT must be one of parquet, tantivy, both." )); } - if cfg.common.inverted_index_store_format != "both" { + cfg.common.inverted_index_search_format = + cfg.common.inverted_index_search_format.to_lowercase(); + if cfg.common.inverted_index_search_format.is_empty() { cfg.common.inverted_index_search_format = cfg.common.inverted_index_store_format.clone(); } + if cfg.common.inverted_index_search_format == "both" { + cfg.common.inverted_index_search_format = "parquet".to_string(); + } + if !["parquet", "tantivy"].contains(&cfg.common.inverted_index_search_format.as_str()) { + return Err(anyhow::anyhow!( + "ZO_INVERTED_INDEX_SEARCH_FORMAT must be one of parquet, tantivy." + )); + } Ok(()) } diff --git a/src/config/src/meta/inverted_index/mod.rs b/src/config/src/meta/inverted_index/mod.rs index 450b1bf1eb..7db51a3abc 100644 --- a/src/config/src/meta/inverted_index/mod.rs +++ b/src/config/src/meta/inverted_index/mod.rs @@ -54,23 +54,23 @@ const fn default_min_len() -> usize { usize::MAX } -/// Currently supports two InvertedIndexFormat -/// Parquet -> v2 -/// FST -> v3 -/// BOTH -> use both +/// Supported inverted index formats: +/// - Parquet (v2): Index is stored in parquet format +/// - Tantivy (v3): Index is stored in custom puffin format files +/// - both: Use both Parquet and Tantivy. Note that this will generate two inverted index files. #[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize, Default)] pub enum InvertedIndexFormat { #[default] Parquet, - FST, Both, + Tantivy, } impl From<&String> for InvertedIndexFormat { fn from(s: &String) -> Self { match s.to_lowercase().as_str() { - "fst" => InvertedIndexFormat::FST, "both" => InvertedIndexFormat::Both, + "tantivy" => InvertedIndexFormat::Tantivy, _ => InvertedIndexFormat::Parquet, } } @@ -80,8 +80,8 @@ impl std::fmt::Display for InvertedIndexFormat { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { InvertedIndexFormat::Parquet => write!(f, "parquet"), - InvertedIndexFormat::FST => write!(f, "fst"), InvertedIndexFormat::Both => write!(f, "both"), + InvertedIndexFormat::Tantivy => write!(f, "tantivy"), } } } diff --git a/src/config/src/meta/mod.rs b/src/config/src/meta/mod.rs index 547ce765ef..c483fd2f8f 100644 --- a/src/config/src/meta/mod.rs +++ b/src/config/src/meta/mod.rs @@ -22,6 +22,7 @@ pub mod logger; pub mod meta_store; pub mod pipeline; pub mod puffin; +pub mod puffin_directory; pub mod search; pub mod short_url; pub mod sql; diff --git a/src/config/src/meta/puffin/mod.rs b/src/config/src/meta/puffin/mod.rs index 4bffe8d67d..f4633b3960 100644 --- a/src/config/src/meta/puffin/mod.rs +++ b/src/config/src/meta/puffin/mod.rs @@ -100,6 +100,8 @@ pub enum BlobTypes { #[default] #[serde(rename = "o2-fst-v1")] O2FstV1, + #[serde(rename = "o2-ttv-v1")] + O2TtvV1, } #[derive(Default)] @@ -135,6 +137,11 @@ impl BlobMetadataBuilder { self } + pub fn properties(mut self, properties: HashMap) -> Self { + self.properties = properties; + self + } + pub fn build(self) -> Result { Ok(BlobMetadata { blob_type: self.blob_type.ok_or("blob_type is required")?, diff --git a/src/config/src/meta/puffin/reader.rs b/src/config/src/meta/puffin/reader.rs index 68970f88c5..34cdf4e7af 100644 --- a/src/config/src/meta/puffin/reader.rs +++ b/src/config/src/meta/puffin/reader.rs @@ -24,6 +24,7 @@ use super::{ }; use crate::meta::puffin::{CompressionCodec, MIN_FILE_SIZE}; +#[derive(Debug)] pub struct PuffinBytesReader { source: R, metadata: Option, @@ -45,38 +46,34 @@ impl PuffinBytesReader { .seek(SeekFrom::Start(blob_metadata.offset as _)) .await?; - let mut raw_data = vec![0u8; blob_metadata.length as usize]; - self.source.read_exact(&mut raw_data).await?; + let mut compressed = vec![0u8; blob_metadata.length as usize]; + self.source.read_exact(&mut compressed).await?; - let data = match blob_metadata.compression_codec { - None => raw_data, + let decompressed = match blob_metadata.compression_codec { + Some(CompressionCodec::Lz4) => { + return Err(anyhow!("Lz4 compression is not supported")); + } Some(CompressionCodec::Zstd) => { let mut decompressed = Vec::new(); - let mut decoder = zstd::Decoder::new(&raw_data[..])?; + let mut decoder = zstd::Decoder::new(&compressed[..])?; decoder.read_to_end(&mut decompressed)?; decompressed } - Some(CompressionCodec::Lz4) => { - todo!("Lz4 decompression is not implemented yet") - } + None => compressed, }; - Ok(data) + Ok(decompressed) } pub async fn get_field(&mut self, field: &str) -> Result> { self.parse_footer().await?; match self.metadata.as_ref() { None => Err(anyhow!("Metadata not found")), - Some(v) => { - let Some(idx) = v.properties.get(field) else { - return Ok(None); - }; - let idx = idx - .parse::() - .map_err(|_| anyhow!("Field not found"))?; - Ok(v.blobs.get(idx).cloned()) - } + Some(v) => Ok(v + .blobs + .iter() + .find(|b| b.properties.get("blob_tag").is_some_and(|val| val == field)) + .cloned()), } } @@ -118,7 +115,6 @@ impl PuffinBytesReader { struct PuffinFooterBytesReader { source: R, file_size: u64, - chunk: FooterChunk, flags: PuffinFooterFlags, payload_size: u64, metadata: Option, @@ -129,7 +125,6 @@ impl PuffinFooterBytesReader { Self { source, file_size, - chunk: FooterChunk::FootMagic, flags: PuffinFooterFlags::empty(), payload_size: 0, metadata: None, @@ -139,71 +134,46 @@ impl PuffinFooterBytesReader { impl PuffinFooterBytesReader { async fn parse(mut self) -> Result { - let mut buf = Vec::new(); - while let Some((offset, size)) = self.next_chunk_to_read() { - self.source.seek(SeekFrom::Start(offset)).await?; - let size = size as usize; - buf.resize(size, 0); - let buf = &mut buf[..size]; - self.source.read_exact(buf).await?; - self.parse_chunk(buf)?; - } + // read the footer magic and + self.source + .seek(SeekFrom::Start(self.footer_magic_offset())) + .await?; + let mut magic = [0u8; MAGIC_SIZE as usize]; + self.source.read_exact(&mut magic).await?; + ensure!(magic == MAGIC, anyhow!("Footer MAGIC mismatch")); - ensure!( - self.chunk == FooterChunk::Finished, - anyhow!("Error parsing Puffin footer from bytes") - ); + self.source + .seek(SeekFrom::Start(self.flags_offset())) + .await?; + let mut flags = [0u8; FLAGS_SIZE as usize]; + self.source.read_exact(&mut flags).await?; + self.flags = PuffinFooterFlags::from_bits(u32::from_le_bytes(flags)) + .ok_or_else(|| anyhow!("Error parsing Puffin flags from bytes"))?; + + self.source + .seek(SeekFrom::Start(self.payload_size_offset())) + .await?; + let mut payload_size = [0u8; FOOTER_PAYLOAD_SIZE_SIZE as usize]; + self.source.read_exact(&mut payload_size).await?; + self.payload_size = i32::from_le_bytes(payload_size) as u64; + + self.source + .seek(SeekFrom::Start(self.payload_offset())) + .await?; + let mut payload: Vec = vec![0; self.payload_size as usize]; + self.source.read_exact(&mut payload).await?; + self.metadata = Some(self.parse_payload(&payload)?); + self.validate_payload()?; + + self.source + .seek(SeekFrom::Start(self.head_magic_offset())) + .await?; + self.source.read_exact(&mut magic).await?; + ensure!(magic == MAGIC, anyhow!("Footer MAGIC mismatch")); Ok(self.metadata.unwrap()) } - fn next_chunk_to_read(&mut self) -> Option<(u64, u64)> { - match self.chunk { - FooterChunk::FootMagic => Some((self.footer_magic_offset(), MAGIC_SIZE)), - FooterChunk::Flags => Some((self.flags_offset(), FLAGS_SIZE)), - FooterChunk::PayloadSize => { - Some((self.payload_size_offset(), FOOTER_PAYLOAD_SIZE_SIZE)) - } - FooterChunk::Payload => Some((self.payload_offset(), self.payload_size)), - FooterChunk::HeadMagic => Some((self.head_magic_offset(), MAGIC_SIZE)), - FooterChunk::Finished => None, - } - } - - fn parse_chunk(&mut self, bytes: &[u8]) -> Result<()> { - match self.chunk { - FooterChunk::FootMagic => { - ensure!(bytes == MAGIC, anyhow!("Head Magic mismatch")); - self.chunk = FooterChunk::Flags; - } - FooterChunk::Flags => { - let flag_bits = u32::from_le_bytes(bytes.try_into()?); - self.flags = PuffinFooterFlags::from_bits_truncate(flag_bits); - self.chunk = FooterChunk::PayloadSize; - } - FooterChunk::PayloadSize => { - let size = i32::from_le_bytes(bytes.try_into()?); - ensure!( - size >= 0, - anyhow!("Unexpected footer payload size {size}. Should be non-negative") - ); - self.payload_size = size as _; - self.chunk = FooterChunk::Payload; - } - FooterChunk::Payload => { - self.metadata = Some(self.parse_payload(bytes)?); - self.validate_payload()?; - self.chunk = FooterChunk::HeadMagic; - } - FooterChunk::HeadMagic => { - ensure!(bytes == MAGIC, anyhow!("Magic Mismatch")); - self.chunk = FooterChunk::Finished; - } - FooterChunk::Finished => {} - } - Ok(()) - } - fn parse_payload(&self, bytes: &[u8]) -> Result { if self.flags.contains(PuffinFooterFlags::COMPRESSED) { let decoder = zstd::Decoder::new(bytes)?; @@ -211,7 +181,7 @@ impl PuffinFooterBytesReader { .map_err(|e| anyhow!("Error decompress footer payload {}", e.to_string())) } else { serde_json::from_slice(bytes) - .map_err(|e| anyhow!("Error decompress footer payload {}", e.to_string())) + .map_err(|e| anyhow!("Error serializing footer {}", e.to_string())) } } @@ -259,13 +229,3 @@ impl PuffinFooterBytesReader { self.payload_offset() - MAGIC_SIZE } } - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum FooterChunk { - FootMagic, - Flags, - PayloadSize, - Payload, - HeadMagic, - Finished, -} diff --git a/src/config/src/meta/puffin/writer.rs b/src/config/src/meta/puffin/writer.rs index 3ccb4d0328..e0e9e521a6 100644 --- a/src/config/src/meta/puffin/writer.rs +++ b/src/config/src/meta/puffin/writer.rs @@ -54,49 +54,63 @@ impl PuffinBytesWriter { &self, blob_type: BlobTypes, compression_codec: Option, + properties: HashMap, + size: u64, ) -> BlobMetadata { BlobMetadataBuilder::default() .blob_type(blob_type) .compression_codec(compression_codec) + .properties(properties) .offset(self.written_bytes as _) + .length(size) .build() .expect("Missing required fields") } } impl PuffinBytesWriter { - pub fn add_blob(&mut self, field: String, raw_data: Vec) -> Result<()> { + pub fn add_blob( + &mut self, + raw_data: &Vec, + blob_type: BlobTypes, + // blob_tag will be added in the puffin footer, with its respective offset + blob_tag: String, + compress: bool, + ) -> Result<()> { self.add_header_if_needed() .context("Error writing puffin header")?; - // build blob metadata - let mut metadata = self.build_blob_metadata(BlobTypes::O2FstV1, None); - - // compress blob raw data - match metadata.compression_codec { - None => { - metadata.length = raw_data.len() as u64; - self.writer.write_all(&raw_data)?; - } - Some(CompressionCodec::Zstd) => { - let mut encoder = zstd::Encoder::new(vec![], 3)?; - encoder - .write_all(&raw_data) - .context("Error encoding blob raw data")?; - let compressed_bytes = encoder.finish()?; - self.writer.write_all(&compressed_bytes)?; - metadata.length = compressed_bytes.len() as u64; - } - Some(CompressionCodec::Lz4) => { - todo!("Lz4 compression is not implemented yet") - } + let (final_data, final_size, compression_codec) = if compress { + // compress blob raw data + let mut encoder = zstd::Encoder::new(vec![], 3)?; + encoder + .write_all(raw_data) + .context("Error encoding blob raw data")?; + let compressed_bytes = encoder.finish()?; + let compressed_bytes_len = compressed_bytes.len() as u64; + ( + compressed_bytes, + compressed_bytes_len, + Some(CompressionCodec::Zstd), + ) + } else { + let raw_data_len = raw_data.len() as u64; + // use raw data directly + (raw_data.to_owned(), raw_data_len, None) }; - self.written_bytes += metadata.length; - self.properties - .insert(field, self.blobs_metadata.len().to_string()); - self.blobs_metadata.push(metadata); + self.writer.write_all(&final_data)?; + let properties = { + let mut properties = HashMap::new(); + properties.insert("blob_tag".to_string(), blob_tag); + properties + }; + // add metadata for this blob + let blob_metadata = + self.build_blob_metadata(blob_type, compression_codec, properties, final_size); + self.blobs_metadata.push(blob_metadata); + self.written_bytes += final_size; Ok(()) } diff --git a/src/config/src/meta/puffin_directory/mod.rs b/src/config/src/meta/puffin_directory/mod.rs new file mode 100644 index 0000000000..e45fa5038a --- /dev/null +++ b/src/config/src/meta/puffin_directory/mod.rs @@ -0,0 +1,119 @@ +// Copyright 2024 OpenObserve Inc. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +use std::{io::Write, path::PathBuf, sync::LazyLock}; + +use anyhow::Result; +use tantivy::{ + directory::{Directory, OwnedBytes}, + doc, + schema::Schema, +}; +use writer::PuffinDirWriter; + +use crate::get_config; + +pub mod reader; +pub mod writer; + +// We do not need all of the tantivy files, only the .term and .idx files +// for getting doc IDs and also the meta.json file +// This might change in the future when we add more features to the index +const ALLOWED_FILE_EXT: &[&str] = &["term", "idx", "pos"]; +const META_JSON: &str = "meta.json"; + +// Lazy loaded global instance of RAM directory which will contain +// all the files of an empty tantivy index. This instance will be used to fill the missing files +// from the `.ttv` file, as tantivy needs them regardless of the configuration of a field. +static EMPTY_PUFFIN_DIRECTORY: LazyLock = LazyLock::new(|| { + let puffin_dir = PuffinDirWriter::new(); + let puffin_dir_clone = puffin_dir.clone(); + let schema = Schema::builder().build(); + let mut index_writer = tantivy::IndexBuilder::new() + .schema(schema) + .single_segment_index_writer(puffin_dir_clone, 50_000_000) + .expect("Failed to create index writer for EMPTY_PUFFIN_DIRECTORY"); + let _ = index_writer.add_document(doc!()); + index_writer + .finalize() + .expect("Failed to finalize index writer for EMPTY_PUFFIN_DIRECTORY"); + puffin_dir +}); + +// Lazy loaded global segment id of the empty puffin directory which will be used to construct the +// path of a file +static EMPTY_PUFFIN_SEG_ID: LazyLock = LazyLock::new(|| { + EMPTY_PUFFIN_DIRECTORY + .list_files() + .iter() + .find(|path| path.extension().is_some_and(|ext| ext != "json")) + .unwrap() + .file_stem() + .unwrap() + .to_str() + .unwrap() + .to_owned() +}); + +pub fn get_file_from_empty_puffin_dir_with_ext(file_ext: &str) -> Result { + let empty_puffin_dir = &EMPTY_PUFFIN_DIRECTORY; + let seg_id = &EMPTY_PUFFIN_SEG_ID; + let file_path = format!("{}.{}", seg_id.as_str(), file_ext); + let file_data = empty_puffin_dir.open_read(&PathBuf::from(file_path))?; + Ok(file_data.read_bytes()?) +} + +pub fn convert_puffin_dir_to_tantivy_dir( + mut puffin_dir_path: PathBuf, + puffin_dir: PuffinDirWriter, +) -> Result<()> { + // create directory + let cfg = get_config(); + let file_name = puffin_dir_path + .file_name() + .ok_or_else(|| anyhow::anyhow!("Failed to get file name from path"))?; + let mut file_name = file_name.to_os_string(); + file_name.push(".folder"); + puffin_dir_path.set_file_name(file_name); + let mut tantivy_folder_path = PathBuf::from(&cfg.common.data_stream_dir); + tantivy_folder_path.push(PathBuf::from(&puffin_dir_path)); + + // Check if the folder already exists + if !tantivy_folder_path.exists() { + std::fs::create_dir_all(&tantivy_folder_path)?; + log::info!( + "Created folder for index at {}", + tantivy_folder_path.to_str().unwrap_or("") + ); + } else { + log::warn!( + "Folder already exists for index at {}", + tantivy_folder_path.to_str().unwrap_or("") + ); + } + + for file in puffin_dir.list_files() { + let file_data = puffin_dir.open_read(&file.clone())?; + let mut file_handle = std::fs::OpenOptions::new() + .write(true) + .create(true) + .truncate(true) + .open(tantivy_folder_path.join(&file))?; + file_handle.write_all(&file_data.read_bytes()?)?; + file_handle.flush()?; + } + + Ok(()) +} diff --git a/src/config/src/meta/puffin_directory/reader/file_reader.rs b/src/config/src/meta/puffin_directory/reader/file_reader.rs new file mode 100644 index 0000000000..6de99a3cb0 --- /dev/null +++ b/src/config/src/meta/puffin_directory/reader/file_reader.rs @@ -0,0 +1,196 @@ +// Copyright 2024 OpenObserve Inc. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +use std::{ + io, + path::{Path, PathBuf}, + sync::Arc, +}; + +use anyhow::Result; +use hashbrown::HashMap; +use tantivy::{ + directory::{error::OpenReadError, Directory, FileHandle, OwnedBytes}, + HasLen, +}; +use tokio::{fs::File, sync::Mutex}; +use tokio_util::compat::TokioAsyncReadCompatExt; + +use crate::meta::{ + puffin::{reader::PuffinBytesReader, BlobMetadata}, + puffin_directory::{EMPTY_PUFFIN_DIRECTORY, EMPTY_PUFFIN_SEG_ID}, +}; + +#[derive(Debug)] +pub struct StdFsReader { + source: Arc>>>, + blob_metadata_map: Arc>>, +} + +impl StdFsReader { + /// Open a puffin direcotry from the given bytes data + pub async fn from_dir(path: &Path) -> Result { + let file = tokio::fs::OpenOptions::new().read(true).open(path).await?; + let mut puffin_reader = PuffinBytesReader::new(file.compat()); + let puffin_meta = puffin_reader.get_metadata().await?; + + let mut blob_meta_map = HashMap::new(); + + for blob_meta in puffin_meta.blob_metadata { + // Fetch the files names from the blob_meta itself + if let Some(file_name) = blob_meta.properties.get("file_name") { + let path = PathBuf::from(file_name); + blob_meta_map.insert(path, blob_meta); + } + } + + let puffin_dir = StdFsReader { + source: Arc::new(Mutex::new(puffin_reader)), + blob_metadata_map: Arc::new(std::sync::RwLock::new(blob_meta_map)), + }; + + Ok(puffin_dir) + } +} + +impl Clone for StdFsReader { + fn clone(&self) -> Self { + StdFsReader { + source: self.source.clone(), + blob_metadata_map: self.blob_metadata_map.clone(), + } + } +} + +// Version 1: Keep the blob withing the file handle and return it when read +#[derive(Debug)] +struct StdFsPuffinSliceHandle { + source: Arc>>>, + metadata: BlobMetadata, +} + +impl HasLen for StdFsPuffinSliceHandle { + fn len(&self) -> usize { + self.metadata.length as usize + } +} + +#[async_trait::async_trait] +impl FileHandle for StdFsPuffinSliceHandle { + fn read_bytes(&self, _byte_range: core::ops::Range) -> io::Result { + let msg = "Unsupported operation for StdFsDirectory"; + Err(io::Error::new(io::ErrorKind::Other, msg)) + } + + async fn read_bytes_async( + &self, + byte_range: core::ops::Range, + ) -> io::Result { + let bytes = self + .source + .lock() + .await + .read_slice(&self.metadata, byte_range) + .await + .map_err(|msg| io::Error::new(io::ErrorKind::Other, msg.to_string()))?; + Ok(OwnedBytes::new(bytes)) + } +} + +impl Directory for StdFsReader { + fn get_file_handle( + &self, + path: &Path, + ) -> std::result::Result, OpenReadError> { + match self.blob_metadata_map.read().unwrap().get(path) { + Some(blob_meta) => { + let file_handle = StdFsPuffinSliceHandle { + source: self.source.clone(), + metadata: blob_meta.clone(), + }; + Ok(Arc::new(file_handle)) + } + None => { + let ext = path.extension().unwrap().to_str().unwrap(); + let empty_puffin_dir = &EMPTY_PUFFIN_DIRECTORY; + let seg_id = &EMPTY_PUFFIN_SEG_ID; + let file_path = PathBuf::from(format!("{}.{}", seg_id.as_str(), ext)); + empty_puffin_dir.get_file_handle(&file_path) + } + } + } + + fn exists(&self, path: &Path) -> std::result::Result { + let exists = self.blob_metadata_map.read().unwrap().contains_key(path); + + if !exists { + let ext = path.extension().unwrap().to_str().unwrap(); + let dir_path = format!("{}.{}", &EMPTY_PUFFIN_SEG_ID.as_str(), ext); + return EMPTY_PUFFIN_DIRECTORY.exists(&PathBuf::from(dir_path)); + } + + return Ok(exists); + } + + fn atomic_read(&self, path: &Path) -> std::result::Result, OpenReadError> { + unimplemented!() + // match self.blob_metadata_map.read().unwrap().get(path) { + // Some(blob_meta) => { + // let blob = self.source.lock().atomic_read(blob_meta)?; + // Ok(blob) + // } + // None => { + // let ext = path.extension().unwrap().to_str().unwrap(); + // let empty_puffin_dir = &EMPTY_PUFFIN_DIRECTORY; + // let seg_id = &EMPTY_PUFFIN_SEG_ID; + // let file_path = PathBuf::from(format!("{}.{}", seg_id.as_str(), ext)); + // empty_puffin_dir.atomic_read(&file_path) + // } + // } + } + + fn atomic_write(&self, _path: &Path, _data: &[u8]) -> io::Result<()> { + unimplemented!("read-only") + } + + fn delete(&self, _path: &Path) -> Result<(), tantivy::directory::error::DeleteError> { + unimplemented!("read-only") + } + + fn open_write( + &self, + _path: &Path, + ) -> Result { + unimplemented!("read-only") + } + + fn sync_directory(&self) -> io::Result<()> { + unimplemented!("read-only") + } + + fn watch( + &self, + _watch_callback: tantivy::directory::WatchCallback, + ) -> tantivy::Result { + Ok(tantivy::directory::WatchHandle::empty()) + } + + fn acquire_lock( + &self, + _lock: &tantivy::directory::Lock, + ) -> Result { + Ok(tantivy::directory::DirectoryLock::from(Box::new(|| {}))) + } +} diff --git a/src/config/src/meta/puffin_directory/reader/mem_reader.rs b/src/config/src/meta/puffin_directory/reader/mem_reader.rs new file mode 100644 index 0000000000..409db9a100 --- /dev/null +++ b/src/config/src/meta/puffin_directory/reader/mem_reader.rs @@ -0,0 +1,203 @@ +// Copyright 2024 OpenObserve Inc. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +use std::{ + io, + path::{Path, PathBuf}, + sync::{Arc, RwLock}, +}; + +use anyhow::{Context, Result}; +use futures::{AsyncRead, AsyncSeek}; +use hashbrown::HashMap; +use tantivy::{ + directory::{error::OpenReadError, Directory, FileHandle, OwnedBytes}, + HasLen, +}; + +use crate::meta::{ + puffin::reader::PuffinBytesReader, + puffin_directory::{ + get_file_from_empty_puffin_dir_with_ext, EMPTY_PUFFIN_DIRECTORY, EMPTY_PUFFIN_SEG_ID, + }, +}; + +#[derive(Debug)] +pub struct RamDirectoryReader { + source: Arc>, + blob_metadata_map: Arc>>, +} + +impl RamDirectoryReader +where + R: AsyncRead + AsyncSeek + Send + Unpin + Sync + Clone, +{ + /// Open a puffin direcotry from the given bytes data + pub async fn from_bytes(data: R) -> Result { + let mut puffin_reader = PuffinBytesReader::new(data); + let puffin_meta = puffin_reader + .get_metadata() + .await + .context("Failed to get blobs meta")? + .ok_or_else(|| anyhow::anyhow!("Corrupted tantivy index file without blob tag"))?; + + let mut blob_meta_map = HashMap::new(); + + for blob in puffin_meta.blobs { + // Fetch the files names from the blob_meta itself + if let Some(file_name) = blob.properties.get("blob_tag") { + let path = PathBuf::from(file_name); + let blob_bytes = puffin_reader.read_blob_bytes(&blob).await?; + blob_meta_map.insert(path, OwnedBytes::new(blob_bytes)); + } + } + + let puffin_dir = RamDirectoryReader { + source: Arc::new(puffin_reader), + blob_metadata_map: Arc::new(RwLock::new(blob_meta_map)), + }; + + Ok(puffin_dir) + } +} + +impl Clone for RamDirectoryReader +where + R: Clone, +{ + fn clone(&self) -> Self { + RamDirectoryReader { + source: self.source.clone(), + blob_metadata_map: self.blob_metadata_map.clone(), + } + } +} + +// Version 1: Keep the blob withing the file handle and return it when read +#[derive(Debug)] +struct PuffinSliceHandle { + blob: OwnedBytes, +} + +impl HasLen for PuffinSliceHandle { + fn len(&self) -> usize { + self.blob.len() + } +} + +#[async_trait::async_trait] +impl FileHandle for PuffinSliceHandle { + fn read_bytes(&self, byte_range: core::ops::Range) -> io::Result { + Ok(self.blob.slice(byte_range)) + } + + async fn read_bytes_async( + &self, + byte_range: core::ops::Range, + ) -> io::Result { + Ok(self.blob.slice(byte_range)) + } +} + +impl Directory for RamDirectoryReader +where + R: AsyncRead + AsyncSeek + Send + Unpin + Sync + Clone + std::fmt::Debug + 'static, +{ + fn get_file_handle( + &self, + path: &Path, + ) -> std::result::Result, OpenReadError> { + match self.blob_metadata_map.read().unwrap().get(path) { + Some(blob) => { + let file_handle = PuffinSliceHandle { blob: blob.clone() }; + Ok(Arc::new(file_handle)) + } + None => { + let ext = match path.extension().and_then(|ext| ext.to_str()) { + Some(ext) => ext, + None => return Err(OpenReadError::FileDoesNotExist(path.to_path_buf())), + }; + match get_file_from_empty_puffin_dir_with_ext(ext) { + Ok(blob) => Ok(Arc::new(PuffinSliceHandle { blob })), + Err(_) => Err(OpenReadError::FileDoesNotExist(path.to_path_buf())), + } + } + } + } + + fn exists(&self, path: &Path) -> std::result::Result { + let exists = self.blob_metadata_map.read().unwrap().contains_key(path); + + if !exists { + let ext = match path.extension().and_then(|ext| ext.to_str()) { + Some(ext) => ext, + None => return Ok(false), + }; + let dir_path = format!("{}.{}", &EMPTY_PUFFIN_SEG_ID.as_str(), ext); + return EMPTY_PUFFIN_DIRECTORY.exists(&PathBuf::from(dir_path)); + } + + Ok(exists) + } + + fn atomic_read(&self, path: &Path) -> std::result::Result, OpenReadError> { + match self.blob_metadata_map.read().unwrap().get(path) { + Some(blob) => Ok(blob.to_vec()), + None => { + let ext = match path.extension().and_then(|ext| ext.to_str()) { + Some(ext) => ext, + None => return Err(OpenReadError::FileDoesNotExist(path.to_path_buf())), + }; + match get_file_from_empty_puffin_dir_with_ext(ext) { + Ok(blob) => Ok(blob.to_vec()), + Err(_) => Err(OpenReadError::FileDoesNotExist(path.to_path_buf())), + } + } + } + } + + fn atomic_write(&self, _path: &Path, _data: &[u8]) -> io::Result<()> { + unimplemented!("read-only") + } + + fn delete(&self, _path: &Path) -> Result<(), tantivy::directory::error::DeleteError> { + unimplemented!("read-only") + } + + fn open_write( + &self, + _path: &Path, + ) -> Result { + unimplemented!("read-only") + } + + fn sync_directory(&self) -> io::Result<()> { + unimplemented!("read-only") + } + + fn watch( + &self, + _watch_callback: tantivy::directory::WatchCallback, + ) -> tantivy::Result { + Ok(tantivy::directory::WatchHandle::empty()) + } + + fn acquire_lock( + &self, + _lock: &tantivy::directory::Lock, + ) -> Result { + Ok(tantivy::directory::DirectoryLock::from(Box::new(|| {}))) + } +} diff --git a/src/config/src/meta/puffin_directory/reader/mod.rs b/src/config/src/meta/puffin_directory/reader/mod.rs new file mode 100644 index 0000000000..ddfe756fed --- /dev/null +++ b/src/config/src/meta/puffin_directory/reader/mod.rs @@ -0,0 +1,19 @@ +// Copyright 2024 OpenObserve Inc. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +// pub mod file_reader; +pub mod mem_reader; + +pub use mem_reader::RamDirectoryReader; diff --git a/src/config/src/meta/puffin_directory/writer.rs b/src/config/src/meta/puffin_directory/writer.rs new file mode 100644 index 0000000000..b2db921dcb --- /dev/null +++ b/src/config/src/meta/puffin_directory/writer.rs @@ -0,0 +1,216 @@ +// Copyright 2024 OpenObserve Inc. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +use std::{ + io::{self, Write}, + path::{Path, PathBuf}, + sync::{Arc, RwLock}, +}; + +use ahash::HashSet; +use anyhow::{Context, Result}; +use tantivy::{ + directory::{error::OpenReadError, Directory, RamDirectory, WatchCallback, WatchHandle}, + HasLen, +}; + +use crate::{ + get_config, + meta::{ + puffin::{writer::PuffinBytesWriter, BlobTypes}, + puffin_directory::{ALLOWED_FILE_EXT, META_JSON}, + }, +}; +/// Puffin directory is a puffin file which contains all the tantivy files. +/// Each tantivy file is stored as a blob in the puffin file, along with their file name. +#[derive(Debug)] +pub struct PuffinDirWriter { + ram_directory: Arc, + /// record all the files paths in the puffin file + file_paths: Arc>>, +} + +impl Default for PuffinDirWriter { + fn default() -> Self { + Self::new() + } +} + +impl Clone for PuffinDirWriter { + fn clone(&self) -> Self { + PuffinDirWriter { + ram_directory: self.ram_directory.clone(), + file_paths: self.file_paths.clone(), + } + } +} + +pub fn convert_puffin_dir_to_tantivy_dir( + mut puffin_dir_path: PathBuf, + puffin_dir: PuffinDirWriter, +) { + // create directory + let cfg = get_config(); + let file_name = puffin_dir_path.file_name().unwrap(); + let mut file_name = file_name.to_os_string(); + file_name.push(".folder"); + puffin_dir_path.set_file_name(file_name); + let mut tantivy_folder_path = PathBuf::from(&cfg.common.data_stream_dir); + tantivy_folder_path.push(PathBuf::from(&puffin_dir_path)); + + // Check if the folder already exists + if !tantivy_folder_path.exists() { + std::fs::create_dir_all(&tantivy_folder_path).unwrap(); + log::info!( + "Created folder for index at {}", + tantivy_folder_path.to_str().unwrap() + ); + } else { + log::warn!( + "Folder already exists for index at {}", + tantivy_folder_path.to_str().unwrap() + ); + } + + for file in puffin_dir.list_files() { + let file_data = puffin_dir.open_read(&file.clone()).unwrap(); + let mut file_handle = std::fs::OpenOptions::new() + .write(true) + .create(true) + .truncate(true) + .open(tantivy_folder_path.join(&file)) + .unwrap(); + file_handle + .write_all(&file_data.read_bytes().unwrap()) + .unwrap(); + file_handle.flush().unwrap(); + } +} + +impl PuffinDirWriter { + pub fn new() -> Self { + PuffinDirWriter { + ram_directory: Arc::new(RamDirectory::create()), + file_paths: Arc::new(RwLock::new(HashSet::default())), + } + } + + pub fn list_files(&self) -> Vec { + self.file_paths + .read() + .expect("poisoned lock") + .iter() + .cloned() + .collect() + } + + // This function will serialize the directory into a single puffin file + pub fn to_puffin_bytes(&self) -> Result> { + let mut puffin_buf: Vec = Vec::new(); + let mut puffin_writer = PuffinBytesWriter::new(&mut puffin_buf); + let mut segment_id = String::new(); + + let file_paths = self.file_paths.read().expect("poisoned lock"); + let allowed_file_paths = file_paths.iter().filter(|path| { + let mut allowed = false; + if let Some(path_ext) = path.extension() { + if ALLOWED_FILE_EXT.contains(&path_ext.to_str().unwrap()) { + allowed = true; + } + } + + // check if its meta.json file + if !allowed && path.to_str().unwrap() == META_JSON { + allowed = true; + }; + allowed + }); + for path in allowed_file_paths.clone() { + if segment_id.is_empty() && path.extension().is_some_and(|ext| ext != "json") { + segment_id = path.file_stem().unwrap().to_str().unwrap().to_owned(); + } + + let file_data = self.ram_directory.open_read(path)?; + log::debug!( + "Serializing file to puffin: len: {}, path: {}", + file_data.len(), + path.to_str().unwrap() + ); + puffin_writer + .add_blob( + &file_data + .read_bytes() + .expect("failed to read file") + .to_vec(), + BlobTypes::O2TtvV1, + path.to_str().unwrap().to_owned(), + false, + ) + .context("Failed to add blob")?; + } + + puffin_writer.finish().context("Failed to finish writing")?; + Ok(puffin_buf) + } +} + +/// This implementation is used during index creation. +impl Directory for PuffinDirWriter { + fn get_file_handle( + &self, + path: &Path, + ) -> Result, OpenReadError> { + self.ram_directory.get_file_handle(path) + } + + fn delete(&self, path: &Path) -> Result<(), tantivy::directory::error::DeleteError> { + self.ram_directory.delete(path) + } + + fn exists(&self, path: &Path) -> Result { + self.ram_directory.exists(path) + } + + fn open_write( + &self, + path: &Path, + ) -> Result { + // capture the files being written to ram directory + self.file_paths.write().unwrap().insert(path.to_path_buf()); + self.ram_directory.open_write(path) + } + + // this should read from the puffin source + fn atomic_read(&self, path: &Path) -> Result, OpenReadError> { + self.ram_directory.atomic_read(path) + } + + fn atomic_write(&self, path: &Path, data: &[u8]) -> io::Result<()> { + // capture the files being written to ram directory + self.file_paths + .write() + .expect("poisoned lock") + .insert(path.to_path_buf()); + self.ram_directory.atomic_write(path, data) + } + + fn sync_directory(&self) -> io::Result<()> { + self.ram_directory.sync_directory() + } + + fn watch(&self, watch_callback: WatchCallback) -> tantivy::Result { + self.ram_directory.watch(watch_callback) + } +} diff --git a/src/config/src/meta/search.rs b/src/config/src/meta/search.rs index 07a9f43f89..5aaddc9e8d 100644 --- a/src/config/src/meta/search.rs +++ b/src/config/src/meta/search.rs @@ -59,8 +59,6 @@ pub struct Request { #[serde(default, flatten)] #[serde(skip_serializing_if = "Option::is_none")] pub search_event_context: Option, - #[serde(default)] - pub index_type: String, } #[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize, ToSchema)] @@ -439,7 +437,6 @@ impl SearchHistoryRequest { timeout: 0, search_type: Some(SearchEventType::Other), search_event_context: None, - index_type: "".to_string(), }; Ok(search_req) } @@ -651,7 +648,6 @@ impl From for cluster_rpc::SearchRequest { work_group: "".to_string(), user_id: None, search_event_type: req.search_type.map(|event| event.to_string()), - index_type: req.index_type.clone(), } } } @@ -990,7 +986,6 @@ impl MultiStreamRequest { timeout: self.timeout, search_type: self.search_type, search_event_context: self.search_event_context.clone(), - index_type: self.index_type.clone(), }); } res @@ -1076,7 +1071,6 @@ mod tests { timeout: 0, search_type: None, search_event_context: None, - index_type: "".to_string(), }; let rpc_req = cluster_rpc::SearchRequest::from(req.clone()); diff --git a/src/config/src/meta/stream.rs b/src/config/src/meta/stream.rs index bf30cf3675..4f39890907 100644 --- a/src/config/src/meta/stream.rs +++ b/src/config/src/meta/stream.rs @@ -15,7 +15,6 @@ use std::{cmp::max, fmt::Display}; -use byteorder::{ByteOrder, LittleEndian}; use chrono::Duration; use hashbrown::HashMap; use proto::cluster_rpc; @@ -206,6 +205,7 @@ pub struct FileMeta { pub records: i64, pub original_size: i64, pub compressed_size: i64, + pub index_size: i64, pub flattened: bool, } @@ -215,44 +215,6 @@ impl FileMeta { } } -impl From<&FileMeta> for Vec { - fn from(value: &FileMeta) -> Vec { - let mut bytes = [0; 40]; - LittleEndian::write_i64(&mut bytes[0..8], value.min_ts); - LittleEndian::write_i64(&mut bytes[8..16], value.max_ts); - LittleEndian::write_i64(&mut bytes[16..24], value.records); - LittleEndian::write_i64(&mut bytes[24..32], value.original_size); - LittleEndian::write_i64(&mut bytes[32..40], value.compressed_size); - bytes.to_vec() - } -} - -impl TryFrom<&[u8]> for FileMeta { - type Error = std::io::Error; - - fn try_from(value: &[u8]) -> Result { - if value.len() < 40 { - return Err(std::io::Error::new( - std::io::ErrorKind::Other, - "Invalid FileMeta", - )); - } - let min_ts = LittleEndian::read_i64(&value[0..8]); - let max_ts = LittleEndian::read_i64(&value[8..16]); - let records = LittleEndian::read_i64(&value[16..24]); - let original_size = LittleEndian::read_i64(&value[24..32]); - let compressed_size = LittleEndian::read_i64(&value[32..40]); - Ok(Self { - min_ts, - max_ts, - records, - original_size, - compressed_size, - flattened: false, - }) - } -} - impl From<&[parquet::file::metadata::KeyValue]> for FileMeta { fn from(values: &[parquet::file::metadata::KeyValue]) -> Self { let mut meta = FileMeta::default(); @@ -272,6 +234,13 @@ impl From<&[parquet::file::metadata::KeyValue]> for FileMeta { } } +#[derive(Clone, Debug, Default)] +pub struct FileListDeleted { + pub file: String, + pub index_file: bool, + pub flattened: bool, +} + #[derive(Debug, Clone, PartialEq)] pub enum QueryPartitionStrategy { FileNum, @@ -315,6 +284,7 @@ pub struct StreamStats { pub file_num: i64, pub storage_size: i64, pub compressed_size: i64, + pub index_size: i64, } impl StreamStats { @@ -353,6 +323,7 @@ impl StreamStats { self.doc_time_max = self.doc_time_max.max(meta.max_ts); self.storage_size += meta.original_size; self.compressed_size += meta.compressed_size; + self.index_size += meta.index_size; if self.doc_time_min == 0 { self.doc_time_min = meta.min_ts; } @@ -362,6 +333,9 @@ impl StreamStats { if self.compressed_size < 0 { self.compressed_size = 0; } + if self.index_size < 0 { + self.index_size = 0; + } } pub fn format_by(&mut self, stats: &StreamStats) { @@ -369,6 +343,7 @@ impl StreamStats { self.doc_num = stats.doc_num; self.storage_size = stats.storage_size; self.compressed_size = stats.compressed_size; + self.index_size = stats.index_size; self.doc_time_min = self.doc_time_min.min(stats.doc_time_min); self.doc_time_max = self.doc_time_max.max(stats.doc_time_max); if self.doc_time_min == 0 { @@ -405,6 +380,7 @@ impl From for StreamStats { file_num: 0, storage_size: meta.original_size as i64, compressed_size: meta.compressed_size.unwrap_or_default() as i64, + index_size: meta.index_size.unwrap_or_default() as i64, } } } @@ -421,6 +397,7 @@ impl std::ops::Sub for StreamStats { doc_time_max: self.doc_time_max.max(rhs.max_ts), storage_size: self.storage_size - rhs.original_size, compressed_size: self.compressed_size - rhs.compressed_size, + index_size: self.index_size - rhs.index_size, }; if ret.doc_time_min == 0 { ret.doc_time_min = rhs.min_ts; @@ -437,6 +414,7 @@ impl From<&FileMeta> for cluster_rpc::FileMeta { records: req.records, original_size: req.original_size, compressed_size: req.compressed_size, + index_size: req.index_size, } } } @@ -450,6 +428,7 @@ impl From<&cluster_rpc::FileMeta> for FileMeta { original_size: req.original_size, compressed_size: req.compressed_size, flattened: false, + index_size: req.index_size, } } } @@ -955,6 +934,7 @@ mod tests { original_size: 10, compressed_size: 1, flattened: false, + index_size: 0, }; let rpc_meta = cluster_rpc::FileMeta::from(&file_meta); diff --git a/src/config/src/meta/usage.rs b/src/config/src/meta/usage.rs index 5db53d3422..2c8c1ee754 100644 --- a/src/config/src/meta/usage.rs +++ b/src/config/src/meta/usage.rs @@ -351,4 +351,6 @@ pub struct Stats { pub max_ts: i64, #[serde(default)] pub compressed_size: Option, + #[serde(default)] + pub index_size: Option, } diff --git a/src/config/src/utils/inverted_index.rs b/src/config/src/utils/inverted_index.rs index e17ce2d15f..0b43467b8c 100644 --- a/src/config/src/utils/inverted_index.rs +++ b/src/config/src/utils/inverted_index.rs @@ -21,7 +21,7 @@ use itertools::Itertools; use crate::{ meta::{inverted_index::reader::IndexReader, stream::StreamType}, - FILE_EXT_PARQUET, FILE_EXT_PUFFIN, INDEX_MIN_CHAR_LEN, + FILE_EXT_PARQUET, FILE_EXT_PUFFIN, FILE_EXT_TANTIVY, INDEX_MIN_CHAR_LEN, }; /// Split a string into tokens based on a delimiter. if delimiter is empty, split by whitespace and @@ -71,6 +71,43 @@ pub async fn create_index_reader_from_puffin_bytes( Ok(IndexReader::new(Cursor::new(buf))) } +/// FST inverted index solution has a 1:1 mapping between parquet and idx files. +/// This is a helper function to convert the paruqet file name to idx file name. +/// e.g. +/// from: files/default/logs/quickstart1/2024/02/16/16/7164299619311026293.parquet +/// to: files/default/index/quickstart1_logs/2024/02/16/16/7164299619311026293.ttv +pub fn convert_parquet_idx_file_name_to_tantivy_file(from: &str) -> Option { + let mut parts: Vec> = from.split('/').map(Cow::Borrowed).collect(); + + if parts.len() < 4 { + return None; + } + + // Replace the stream_type part + let stream_type_pos = 2; + let stream_type = match parts[stream_type_pos].as_ref() { + "logs" => StreamType::Logs, + "metrics" => StreamType::Metrics, + "traces" => StreamType::Traces, + _ => return None, + }; + parts[stream_type_pos] = Cow::Borrowed("index"); + + // Replace the stream_name part + let stream_name_pos = stream_type_pos + 1; + parts[stream_name_pos] = Cow::Owned(format!("{}_{}", parts[stream_name_pos], stream_type)); + + // Replace the file extension + let file_name_pos = parts.len() - 1; + if !parts[file_name_pos].ends_with(FILE_EXT_PARQUET) { + return None; + } + parts[file_name_pos] = + Cow::Owned(parts[file_name_pos].replace(FILE_EXT_PARQUET, FILE_EXT_TANTIVY)); + + Some(parts.join("/")) +} + /// FST inverted index solution has a 1:1 mapping between parquet and idx files. /// This is a helper function to convert the paruqet file name to idx file name. /// e.g. @@ -274,7 +311,49 @@ mod tests { } #[test] - fn test_convert_parquet_idx_file_name() { + fn test_convert_parquet_idx_file_name_to_tantivy_file() { + let test_cases = vec![ + ( + "files/default/logs/quickstart1/2024/02/16/16/7164299619311026293.parquet", + Some( + "files/default/index/quickstart1_logs/2024/02/16/16/7164299619311026293.ttv" + .to_string(), + ), + ), + ( + "files/default/metrics/quickstart1/2024/02/16/16/7164299619311026293.parquet", + Some( + "files/default/index/quickstart1_metrics/2024/02/16/16/7164299619311026293.ttv" + .to_string(), + ), + ), + ( + "files/default/traces/quickstart1/2024/02/16/16/7164299619311026293.parquet", + Some( + "files/default/index/quickstart1_traces/2024/02/16/16/7164299619311026293.ttv" + .to_string(), + ), + ), + ( + "files/default/metadata/quickstart1/2024/02/16/16/7164299619311026293.parquet", + None, + ), + ( + "files/default/index/quickstart1/2024/02/16/16/7164299619311026293.parquet", + None, + ), + ]; + + for (input, expected) in test_cases { + assert_eq!( + convert_parquet_idx_file_name_to_tantivy_file(input), + expected + ); + } + } + + #[test] + fn test_convert_parquet_idx_file_name_to_fst() { let test_cases = vec![ ( "files/default/logs/quickstart1/2024/02/16/16/7164299619311026293.parquet", @@ -306,6 +385,7 @@ mod tests { None, ), ]; + for (input, expected) in test_cases { assert_eq!(convert_parquet_idx_file_name(input), expected); } diff --git a/src/handler/http/request/search/mod.rs b/src/handler/http/request/search/mod.rs index 887eb7af82..9919452204 100644 --- a/src/handler/http/request/search/mod.rs +++ b/src/handler/http/request/search/mod.rs @@ -39,9 +39,9 @@ use crate::{ utils::{ functions, http::{ - get_index_type_from_request, get_or_create_trace_id, - get_search_event_context_from_request, get_search_type_from_request, - get_stream_type_from_request, get_use_cache_from_request, get_work_group, + get_or_create_trace_id, get_search_event_context_from_request, + get_search_type_from_request, get_stream_type_from_request, + get_use_cache_from_request, get_work_group, }, }, }, @@ -160,14 +160,6 @@ pub async fn search( .and_then(|event_type| get_search_event_context_from_request(event_type, &query)); } - // set index_type - if req.index_type.is_empty() { - req.index_type = match get_index_type_from_request(&query) { - Ok(typ) => typ, - Err(e) => return Ok(MetaHttpResponse::bad_request(e)), - }; - } - // get stream name let stream_names = match resolve_stream_names(&req.query.sql) { Ok(v) => v.clone(), @@ -481,7 +473,6 @@ pub async fn around( timeout, search_type: Some(SearchEventType::UI), search_event_context: None, - index_type: "".to_string(), }; let search_res = SearchService::search(&trace_id, &org_id, stream_type, user_id.clone(), &req) .instrument(http_span.clone()) @@ -533,7 +524,6 @@ pub async fn around( timeout, search_type: Some(SearchEventType::UI), search_event_context: None, - index_type: "".to_string(), }; let search_res = SearchService::search(&trace_id, &org_id, stream_type, user_id.clone(), &req) .instrument(http_span) @@ -912,7 +902,6 @@ async fn values_v1( timeout, search_type: Some(SearchEventType::Values), search_event_context: None, - index_type: "".to_string(), }; // skip fields which aren't part of the schema @@ -1233,7 +1222,6 @@ async fn values_v2( timeout, search_type: Some(SearchEventType::Values), search_event_context: None, - index_type: "".to_string(), }; let search_res = SearchService::search( &trace_id, diff --git a/src/handler/http/request/search/multi_streams.rs b/src/handler/http/request/search/multi_streams.rs index 0f24df278d..e6b7cf0157 100644 --- a/src/handler/http/request/search/multi_streams.rs +++ b/src/handler/http/request/search/multi_streams.rs @@ -933,7 +933,6 @@ pub async fn around_multi( timeout, search_type: Some(search::SearchEventType::UI), search_event_context: None, - index_type: "".to_string(), }; let search_res = SearchService::search(&trace_id, &org_id, stream_type, user_id.clone(), &req) @@ -1007,7 +1006,6 @@ pub async fn around_multi( timeout, search_type: Some(search::SearchEventType::UI), search_event_context: None, - index_type: "".to_string(), }; let search_res = SearchService::search(&trace_id, &org_id, stream_type, user_id.clone(), &req) diff --git a/src/handler/http/request/traces/mod.rs b/src/handler/http/request/traces/mod.rs index bb1d84a7bf..9155194975 100644 --- a/src/handler/http/request/traces/mod.rs +++ b/src/handler/http/request/traces/mod.rs @@ -273,7 +273,6 @@ pub async fn get_latest_traces( timeout, search_type: None, search_event_context: None, - index_type: "".to_string(), }; let stream_type = StreamType::Traces; let user_id = in_req diff --git a/src/infra/src/cache/file_data/mod.rs b/src/infra/src/cache/file_data/mod.rs index 0f994729a9..9546edd3bf 100644 --- a/src/infra/src/cache/file_data/mod.rs +++ b/src/infra/src/cache/file_data/mod.rs @@ -133,7 +133,11 @@ pub async fn download(trace_id: &str, file: &str) -> Result<(), anyhow::Error> { } } -pub async fn get(file: &str, range: Option>) -> Result { +pub async fn get( + _trace_id: &str, + file: &str, + range: Option>, +) -> Result { let cfg = config::get_config(); // get from memory cache if cfg.memory_cache.enabled { diff --git a/src/infra/src/cache/stats.rs b/src/infra/src/cache/stats.rs index 3a54859034..9191d77ad6 100644 --- a/src/infra/src/cache/stats.rs +++ b/src/infra/src/cache/stats.rs @@ -83,6 +83,8 @@ pub fn incr_stream_stats(key: &str, val: &FileMeta) -> Result<(), anyhow::Error> stats.file_num += 1; stats.storage_size += val.original_size; stats.compressed_size += val.compressed_size; + // increase the stats here for inverted index + stats.index_size += val.index_size; Ok(()) } @@ -117,6 +119,7 @@ mod tests { file_num: 1, storage_size: 200, compressed_size: 3, + index_size: 120000, }; set_stream_stats("nexus", "default", StreamType::Logs, val.clone()); diff --git a/src/infra/src/file_list/mod.rs b/src/infra/src/file_list/mod.rs index 4df8f2ba80..7b0480bb4e 100644 --- a/src/infra/src/file_list/mod.rs +++ b/src/infra/src/file_list/mod.rs @@ -19,7 +19,7 @@ use async_trait::async_trait; use config::{ meta::{ meta_store::MetaStore, - stream::{FileKey, FileMeta, PartitionTimeLevel, StreamStats, StreamType}, + stream::{FileKey, FileListDeleted, FileMeta, PartitionTimeLevel, StreamStats, StreamType}, }, utils::time::second_micros, }; @@ -62,9 +62,8 @@ pub trait FileList: Sync + Send + 'static { async fn batch_add_deleted( &self, org_id: &str, - flattened: bool, created_at: i64, - files: &[String], + files: &[FileListDeleted], ) -> Result<()>; async fn batch_remove_deleted(&self, files: &[String]) -> Result<()>; async fn get(&self, file: &str) -> Result; @@ -100,7 +99,7 @@ pub trait FileList: Sync + Send + 'static { org_id: &str, time_max: i64, limit: i64, - ) -> Result>; + ) -> Result>; // stream stats async fn get_min_ts( &self, @@ -200,13 +199,10 @@ pub async fn batch_remove(files: &[String]) -> Result<()> { #[inline] pub async fn batch_add_deleted( org_id: &str, - flattened: bool, created_at: i64, - files: &[String], + files: &[FileListDeleted], ) -> Result<()> { - CLIENT - .batch_add_deleted(org_id, flattened, created_at, files) - .await + CLIENT.batch_add_deleted(org_id, created_at, files).await } #[inline] @@ -292,7 +288,11 @@ pub async fn query_old_data_hours( } #[inline] -pub async fn query_deleted(org_id: &str, time_max: i64, limit: i64) -> Result> { +pub async fn query_deleted( + org_id: &str, + time_max: i64, + limit: i64, +) -> Result> { CLIENT.query_deleted(org_id, time_max, limit).await } @@ -482,6 +482,8 @@ pub struct FileRecord { pub original_size: i64, pub compressed_size: i64, #[sqlx(default)] + pub index_size: i64, + #[sqlx(default)] pub flattened: bool, } @@ -493,6 +495,7 @@ impl From<&FileRecord> for FileMeta { records: record.records, original_size: record.original_size, compressed_size: record.compressed_size, + index_size: record.index_size, flattened: record.flattened, } } @@ -507,6 +510,7 @@ pub struct StatsRecord { pub records: i64, pub original_size: i64, pub compressed_size: i64, + pub index_size: i64, } impl From<&StatsRecord> for StreamStats { @@ -519,6 +523,7 @@ impl From<&StatsRecord> for StreamStats { file_num: record.file_num, storage_size: record.original_size, compressed_size: record.compressed_size, + index_size: record.index_size, } } } @@ -528,6 +533,7 @@ pub struct FileDeletedRecord { pub stream: String, pub date: String, pub file: String, + pub index_file: bool, pub flattened: bool, } diff --git a/src/infra/src/file_list/mysql.rs b/src/infra/src/file_list/mysql.rs index 6b46d64a0d..364dc32de0 100644 --- a/src/infra/src/file_list/mysql.rs +++ b/src/infra/src/file_list/mysql.rs @@ -18,7 +18,9 @@ use std::collections::HashMap as stdHashMap; use async_trait::async_trait; use config::{ get_config, - meta::stream::{FileKey, FileMeta, PartitionTimeLevel, StreamStats, StreamType}, + meta::stream::{ + FileKey, FileListDeleted, FileMeta, PartitionTimeLevel, StreamStats, StreamType, + }, metrics::{DB_QUERY_NUMS, DB_QUERY_TIME}, utils::{ hash::Sum64, @@ -159,9 +161,8 @@ impl super::FileList for MysqlFileList { async fn batch_add_deleted( &self, org_id: &str, - flattened: bool, created_at: i64, - files: &[String], + files: &[FileListDeleted], ) -> Result<()> { if files.is_empty() { return Ok(()); @@ -171,16 +172,17 @@ impl super::FileList for MysqlFileList { for files in chunks { let mut tx = pool.begin().await?; let mut query_builder: QueryBuilder = QueryBuilder::new( - "INSERT INTO file_list_deleted (org, stream, date, file, flattened, created_at)", + "INSERT INTO file_list_deleted (org, stream, date, file, index_file, flattened, created_at)", ); query_builder.push_values(files, |mut b, item| { let (stream_key, date_key, file_name) = - parse_file_key_columns(item).expect("parse file key failed"); + parse_file_key_columns(&item.file).expect("parse file key failed"); b.push_bind(org_id) .push_bind(stream_key) .push_bind(date_key) .push_bind(file_name) - .push_bind(flattened) + .push_bind(item.index_file) + .push_bind(item.flattened) .push_bind(created_at); }); DB_QUERY_NUMS @@ -263,7 +265,7 @@ impl super::FileList for MysqlFileList { let start = std::time::Instant::now(); let ret = sqlx::query_as::<_, super::FileRecord>( r#" -SELECT stream, date, file, deleted, min_ts, max_ts, records, original_size, compressed_size, flattened +SELECT stream, date, file, deleted, min_ts, max_ts, records, original_size, compressed_size, index_size, flattened FROM file_list WHERE stream = ? AND date = ? AND file = ?; "#, ) @@ -352,7 +354,7 @@ SELECT stream, date, file, deleted, min_ts, max_ts, records, original_size, comp let ret = if flattened.is_some() { sqlx::query_as::<_, super::FileRecord>( r#" -SELECT stream, date, file, deleted, min_ts, max_ts, records, original_size, compressed_size, flattened +SELECT stream, date, file, deleted, min_ts, max_ts, records, original_size, compressed_size, index_size, flattened FROM file_list WHERE stream = ? AND flattened = ? LIMIT 1000; "#, @@ -366,7 +368,7 @@ SELECT stream, date, file, deleted, min_ts, max_ts, records, original_size, comp let max_ts_upper_bound = super::calculate_max_ts_upper_bound(time_end, stream_type); sqlx::query_as::<_, super::FileRecord>( r#" -SELECT stream, date, file, deleted, min_ts, max_ts, records, original_size, compressed_size, flattened +SELECT stream, date, file, deleted, min_ts, max_ts, records, original_size, compressed_size, index_size, flattened FROM file_list WHERE stream = ? AND max_ts >= ? AND max_ts <= ? AND min_ts <= ?; "#, @@ -410,7 +412,7 @@ SELECT stream, date, file, deleted, min_ts, max_ts, records, original_size, comp .collect::>() .join(","); let query_str = format!( - "SELECT id, stream, date, file, min_ts, max_ts, records, original_size, compressed_size FROM file_list WHERE id IN ({ids})" + "SELECT id, stream, date, file, min_ts, max_ts, records, original_size, compressed_size, index_size FROM file_list WHERE id IN ({ids})" ); DB_QUERY_NUMS .with_label_values(&["select", "file_list"]) @@ -567,7 +569,7 @@ SELECT date org_id: &str, time_max: i64, limit: i64, - ) -> Result> { + ) -> Result> { if time_max == 0 { return Ok(Vec::new()); } @@ -576,7 +578,7 @@ SELECT date .with_label_values(&["select", "file_list_deleted"]) .inc(); let ret = sqlx::query_as::<_, super::FileDeletedRecord>( - r#"SELECT stream, date, file, flattened FROM file_list_deleted WHERE org = ? AND created_at < ? LIMIT ?;"#, + r#"SELECT stream, date, file, index_file, flattened FROM file_list_deleted WHERE org = ? AND created_at < ? LIMIT ?;"#, ) .bind(org_id) .bind(time_max) @@ -585,11 +587,10 @@ SELECT date .await?; Ok(ret .iter() - .map(|r| { - ( - format!("files/{}/{}/{}", r.stream, r.date, r.file), - r.flattened, - ) + .map(|r| FileListDeleted { + file: format!("files/{}/{}/{}", r.stream, r.date, r.file), + index_file: r.index_file, + flattened: r.flattened, }) .collect()) } @@ -665,7 +666,7 @@ SELECT date let sql = format!( r#" SELECT stream, MIN(min_ts) AS min_ts, MAX(max_ts) AS max_ts, CAST(COUNT(*) AS SIGNED) AS file_num, - CAST(SUM(records) AS SIGNED) AS records, CAST(SUM(original_size) AS SIGNED) AS original_size, CAST(SUM(compressed_size) AS SIGNED) AS compressed_size + CAST(SUM(records) AS SIGNED) AS records, CAST(SUM(original_size) AS SIGNED) AS original_size, CAST(SUM(compressed_size) AS SIGNED) AS compressed_size, CAST(SUM(index_size) AS SIGNED) AS index_size FROM file_list WHERE {field} = '{value}' "#, @@ -768,8 +769,8 @@ SELECT stream, MIN(min_ts) AS min_ts, MAX(max_ts) AS max_ts, CAST(COUNT(*) AS SI if let Err(e) = sqlx::query( r#" INSERT INTO stream_stats - (org, stream, file_num, min_ts, max_ts, records, original_size, compressed_size) - VALUES (?, ?, 0, 0, 0, 0, 0, 0); + (org, stream, file_num, min_ts, max_ts, records, original_size, compressed_size, index_size) + VALUES (?, ?, 0, 0, 0, 0, 0, 0, 0); "#, ) .bind(org_id) @@ -796,7 +797,7 @@ INSERT INTO stream_stats if let Err(e) = sqlx::query( r#" UPDATE stream_stats - SET file_num = file_num + ?, min_ts = ?, max_ts = ?, records = records + ?, original_size = original_size + ?, compressed_size = compressed_size + ? + SET file_num = file_num + ?, min_ts = ?, max_ts = ?, records = records + ?, original_size = original_size + ?, compressed_size = compressed_size + ?, index_size = index_size + ? WHERE stream = ?; "#, ) @@ -806,6 +807,7 @@ UPDATE stream_stats .bind(stats.doc_num) .bind(stats.storage_size) .bind(stats.compressed_size) + .bind(stats.index_size) .bind(stream_key) .execute(&mut *tx) .await @@ -829,7 +831,7 @@ UPDATE stream_stats DB_QUERY_NUMS .with_label_values(&["update", "stream_stats"]) .inc(); - sqlx::query(r#"UPDATE stream_stats SET file_num = 0, min_ts = 0, max_ts = 0, records = 0, original_size = 0, compressed_size = 0;"#) + sqlx::query(r#"UPDATE stream_stats SET file_num = 0, min_ts = 0, max_ts = 0, records = 0, original_size = 0, compressed_size = 0, index_size = 0;"#) .execute(&pool) .await?; Ok(()) @@ -1264,7 +1266,7 @@ INSERT IGNORE INTO {table} (org, stream, date, file, deleted, min_ts, max_ts, re for files in chunks { let mut tx = pool.begin().await?; let mut query_builder: QueryBuilder = QueryBuilder::new( - format!("INSERT INTO {table} (org, stream, date, file, deleted, min_ts, max_ts, records, original_size, compressed_size, flattened)").as_str(), + format!("INSERT INTO {table} (org, stream, date, file, deleted, min_ts, max_ts, records, original_size, compressed_size, index_size, flattened)").as_str(), ); query_builder.push_values(files, |mut b, item| { let (stream_key, date_key, file_name) = @@ -1280,6 +1282,7 @@ INSERT IGNORE INTO {table} (org, stream, date, file, deleted, min_ts, max_ts, re .push_bind(item.meta.records) .push_bind(item.meta.original_size) .push_bind(item.meta.compressed_size) + .push_bind(item.meta.index_size) .push_bind(item.meta.flattened); }); DB_QUERY_NUMS.with_label_values(&["insert", table]).inc(); @@ -1342,7 +1345,8 @@ CREATE TABLE IF NOT EXISTS file_list max_ts BIGINT not null, records BIGINT not null, original_size BIGINT not null, - compressed_size BIGINT not null + compressed_size BIGINT not null, + index_size BIGINT not null ); "#, ) @@ -1367,7 +1371,8 @@ CREATE TABLE IF NOT EXISTS file_list_history max_ts BIGINT not null, records BIGINT not null, original_size BIGINT not null, - compressed_size BIGINT not null + compressed_size BIGINT not null, + index_size BIGINT not null ); "#, ) @@ -1386,6 +1391,7 @@ CREATE TABLE IF NOT EXISTS file_list_deleted stream VARCHAR(256) not null, date VARCHAR(16) not null, file VARCHAR(496) not null, + index_file BOOLEAN default false not null, flattened BOOLEAN default false not null, created_at BIGINT not null ); @@ -1430,7 +1436,8 @@ CREATE TABLE IF NOT EXISTS stream_stats max_ts BIGINT not null, records BIGINT not null, original_size BIGINT not null, - compressed_size BIGINT not null + compressed_size BIGINT not null, + index_size BIGINT not null ); "#, ) @@ -1449,6 +1456,16 @@ CREATE TABLE IF NOT EXISTS stream_stats let data_type = "BIGINT default 0 not null"; add_column("file_list_jobs", column, data_type).await?; + // create column index_size for old version <= 0.13.1 + let column = "index_size"; + let data_type = "BIGINT default 0 not null"; + add_column("file_list", column, data_type).await?; + add_column("file_list_history", column, data_type).await?; + add_column("stream_stats", column, data_type).await?; + let column = "index_file"; + let data_type = "BOOLEAN default false not null"; + add_column("file_list_deleted", column, data_type).await?; + Ok(()) } diff --git a/src/infra/src/file_list/postgres.rs b/src/infra/src/file_list/postgres.rs index b011e91be9..b4d31ec734 100644 --- a/src/infra/src/file_list/postgres.rs +++ b/src/infra/src/file_list/postgres.rs @@ -18,7 +18,9 @@ use std::collections::HashMap as stdHashMap; use async_trait::async_trait; use config::{ get_config, - meta::stream::{FileKey, FileMeta, PartitionTimeLevel, StreamStats, StreamType}, + meta::stream::{ + FileKey, FileListDeleted, FileMeta, PartitionTimeLevel, StreamStats, StreamType, + }, metrics::{DB_QUERY_NUMS, DB_QUERY_TIME}, utils::{ hash::Sum64, @@ -165,9 +167,8 @@ impl super::FileList for PostgresFileList { async fn batch_add_deleted( &self, org_id: &str, - flattened: bool, created_at: i64, - files: &[String], + files: &[FileListDeleted], ) -> Result<()> { if files.is_empty() { return Ok(()); @@ -177,16 +178,17 @@ impl super::FileList for PostgresFileList { for files in chunks { let mut tx = pool.begin().await?; let mut query_builder: QueryBuilder = QueryBuilder::new( - "INSERT INTO file_list_deleted (org, stream, date, file, flattened, created_at)", + "INSERT INTO file_list_deleted (org, stream, date, file, index_file, flattened, created_at)", ); query_builder.push_values(files, |mut b, item| { let (stream_key, date_key, file_name) = - parse_file_key_columns(item).expect("parse file key failed"); + parse_file_key_columns(&item.file).expect("parse file key failed"); b.push_bind(org_id) .push_bind(stream_key) .push_bind(date_key) .push_bind(file_name) - .push_bind(flattened) + .push_bind(item.index_file) + .push_bind(item.flattened) .push_bind(created_at); }); DB_QUERY_NUMS @@ -277,7 +279,7 @@ impl super::FileList for PostgresFileList { let start = std::time::Instant::now(); let ret = sqlx::query_as::<_, super::FileRecord>( r#" -SELECT stream, date, file, deleted, min_ts, max_ts, records, original_size, compressed_size, flattened +SELECT stream, date, file, deleted, min_ts, max_ts, records, original_size, compressed_size, index_size, flattened FROM file_list WHERE stream = $1 AND date = $2 AND file = $3; "# ) @@ -366,7 +368,7 @@ SELECT stream, date, file, deleted, min_ts, max_ts, records, original_size, comp let ret = if flattened.is_some() { sqlx::query_as::<_, super::FileRecord>( r#" -SELECT stream, date, file, deleted, min_ts, max_ts, records, original_size, compressed_size, flattened +SELECT stream, date, file, deleted, min_ts, max_ts, records, original_size, compressed_size, index_size, flattened FROM file_list WHERE stream = $1 AND flattened = $2 LIMIT 1000; "# @@ -378,7 +380,7 @@ SELECT stream, date, file, deleted, min_ts, max_ts, records, original_size, comp let (time_start, time_end) = time_range.unwrap_or((0, 0)); let max_ts_upper_bound = super::calculate_max_ts_upper_bound(time_end, stream_type); let sql = r#" -SELECT stream, date, file, deleted, min_ts, max_ts, records, original_size, compressed_size, flattened +SELECT stream, date, file, deleted, min_ts, max_ts, records, original_size, compressed_size, index_size, flattened FROM file_list WHERE stream = $1 AND max_ts >= $2 AND max_ts <= $3 AND min_ts <= $4; "#; @@ -423,7 +425,7 @@ SELECT stream, date, file, deleted, min_ts, max_ts, records, original_size, comp .collect::>() .join(","); let query_str = format!( - "SELECT id, stream, date, file, min_ts, max_ts, records, original_size, compressed_size FROM file_list WHERE id IN ({ids})" + "SELECT id, stream, date, file, min_ts, max_ts, records, original_size, compressed_size, index_size FROM file_list WHERE id IN ({ids})" ); DB_QUERY_NUMS .with_label_values(&["select", "file_list"]) @@ -580,7 +582,7 @@ SELECT date org_id: &str, time_max: i64, limit: i64, - ) -> Result> { + ) -> Result> { if time_max == 0 { return Ok(Vec::new()); } @@ -590,7 +592,7 @@ SELECT date .inc(); let ret = sqlx ::query_as::<_, super::FileDeletedRecord>( - r#"SELECT stream, date, file, flattened FROM file_list_deleted WHERE org = $1 AND created_at < $2 LIMIT $3;"# + r#"SELECT stream, date, file, index_file, flattened FROM file_list_deleted WHERE org = $1 AND created_at < $2 LIMIT $3;"# ) .bind(org_id) .bind(time_max) @@ -598,11 +600,10 @@ SELECT date .fetch_all(&pool).await?; Ok(ret .iter() - .map(|r| { - ( - format!("files/{}/{}/{}", r.stream, r.date, r.file), - r.flattened, - ) + .map(|r| FileListDeleted { + file: format!("files/{}/{}/{}", r.stream, r.date, r.file), + index_file: r.index_file, + flattened: r.flattened, }) .collect()) } @@ -680,7 +681,7 @@ SELECT date let sql = format!( r#" SELECT stream, MIN(min_ts) AS min_ts, MAX(max_ts) AS max_ts, COUNT(*)::BIGINT AS file_num, - SUM(records)::BIGINT AS records, SUM(original_size)::BIGINT AS original_size, SUM(compressed_size)::BIGINT AS compressed_size + SUM(records)::BIGINT AS records, SUM(original_size)::BIGINT AS original_size, SUM(compressed_size)::BIGINT AS compressed_size, SUM(index_size)::BIGINT AS index_size FROM file_list WHERE {field} = '{value}' "# @@ -783,8 +784,8 @@ SELECT stream, MIN(min_ts) AS min_ts, MAX(max_ts) AS max_ts, COUNT(*)::BIGINT AS if let Err(e) = sqlx::query( r#" INSERT INTO stream_stats - (org, stream, file_num, min_ts, max_ts, records, original_size, compressed_size) - VALUES ($1, $2, 0, 0, 0, 0, 0, 0); + (org, stream, file_num, min_ts, max_ts, records, original_size, compressed_size, index_size) + VALUES ($1, $2, 0, 0, 0, 0, 0, 0, 0); "#, ) .bind(org_id) @@ -813,8 +814,8 @@ INSERT INTO stream_stats ::query( r#" UPDATE stream_stats - SET file_num = file_num + $1, min_ts = $2, max_ts = $3, records = records + $4, original_size = original_size + $5, compressed_size = compressed_size + $6 - WHERE stream = $7; + SET file_num = file_num + $1, min_ts = $2, max_ts = $3, records = records + $4, original_size = original_size + $5, compressed_size = compressed_size + $6, index_size = index_size + $7 + WHERE stream = $8; "# ) .bind(stats.file_num) @@ -823,6 +824,7 @@ UPDATE stream_stats .bind(stats.doc_num) .bind(stats.storage_size) .bind(stats.compressed_size) + .bind(stats.index_size) .bind(stream_key) .execute(&mut *tx).await { @@ -847,7 +849,7 @@ UPDATE stream_stats .inc(); sqlx ::query( - r#"UPDATE stream_stats SET file_num = 0, min_ts = 0, max_ts = 0, records = 0, original_size = 0, compressed_size = 0;"# + r#"UPDATE stream_stats SET file_num = 0, min_ts = 0, max_ts = 0, records = 0, original_size = 0, compressed_size = 0, index_size = 0;"# ) .execute(&pool).await?; Ok(()) @@ -1189,8 +1191,8 @@ impl PostgresFileList { ::query( format!( r#" -INSERT INTO {table} (org, stream, date, file, deleted, min_ts, max_ts, records, original_size, compressed_size, flattened) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11) +INSERT INTO {table} (org, stream, date, file, deleted, min_ts, max_ts, records, original_size, compressed_size, index_size, flattened) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12) ON CONFLICT DO NOTHING; "# ).as_str() @@ -1205,6 +1207,7 @@ INSERT INTO {table} (org, stream, date, file, deleted, min_ts, max_ts, records, .bind(meta.records) .bind(meta.original_size) .bind(meta.compressed_size) + .bind(meta.index_size) .bind(meta.flattened) .execute(&pool).await { @@ -1228,7 +1231,7 @@ INSERT INTO {table} (org, stream, date, file, deleted, min_ts, max_ts, records, let mut tx = pool.begin().await?; let mut query_builder: QueryBuilder = QueryBuilder::new( format!( - "INSERT INTO {table} (org, stream, date, file, deleted, min_ts, max_ts, records, original_size, compressed_size, flattened)" + "INSERT INTO {table} (org, stream, date, file, deleted, min_ts, max_ts, records, original_size, compressed_size, index_size, flattened)" ).as_str() ); query_builder.push_values(files, |mut b, item| { @@ -1245,6 +1248,7 @@ INSERT INTO {table} (org, stream, date, file, deleted, min_ts, max_ts, records, .push_bind(item.meta.records) .push_bind(item.meta.original_size) .push_bind(item.meta.compressed_size) + .push_bind(item.meta.index_size) .push_bind(item.meta.flattened); }); DB_QUERY_NUMS.with_label_values(&["insert", table]).inc(); @@ -1307,7 +1311,8 @@ CREATE TABLE IF NOT EXISTS file_list max_ts BIGINT not null, records BIGINT not null, original_size BIGINT not null, - compressed_size BIGINT not null + compressed_size BIGINT not null, + index_size BIGINT not null ); "#, ) @@ -1332,7 +1337,8 @@ CREATE TABLE IF NOT EXISTS file_list_history max_ts BIGINT not null, records BIGINT not null, original_size BIGINT not null, - compressed_size BIGINT not null + compressed_size BIGINT not null, + index_size BIGINT not null ); "#, ) @@ -1351,6 +1357,7 @@ CREATE TABLE IF NOT EXISTS file_list_deleted stream VARCHAR(256) not null, date VARCHAR(16) not null, file VARCHAR(1024) not null, + index_file BOOLEAN default false not null, flattened BOOLEAN default false not null, created_at BIGINT not null ); @@ -1395,7 +1402,8 @@ CREATE TABLE IF NOT EXISTS stream_stats max_ts BIGINT not null, records BIGINT not null, original_size BIGINT not null, - compressed_size BIGINT not null + compressed_size BIGINT not null, + index_size BIGINT not null ); "#, ) @@ -1414,6 +1422,16 @@ CREATE TABLE IF NOT EXISTS stream_stats let data_type = "BIGINT default 0 not null"; add_column("file_list_jobs", column, data_type).await?; + // create column index_size for old version <= 0.13.1 + let column = "index_size"; + let data_type = "BIGINT default 0 not null"; + add_column("file_list", column, data_type).await?; + add_column("file_list_history", column, data_type).await?; + add_column("stream_stats", column, data_type).await?; + let column = "index_file"; + let data_type = "BOOLEAN default false not null"; + add_column("file_list_deleted", column, data_type).await?; + Ok(()) } diff --git a/src/infra/src/file_list/sqlite.rs b/src/infra/src/file_list/sqlite.rs index 5f95f5adf3..b2194345ac 100644 --- a/src/infra/src/file_list/sqlite.rs +++ b/src/infra/src/file_list/sqlite.rs @@ -18,7 +18,9 @@ use std::collections::HashMap as stdHashMap; use async_trait::async_trait; use config::{ get_config, - meta::stream::{FileKey, FileMeta, PartitionTimeLevel, StreamStats, StreamType}, + meta::stream::{ + FileKey, FileListDeleted, FileMeta, PartitionTimeLevel, StreamStats, StreamType, + }, utils::{ parquet::parse_file_key_columns, time::{end_of_the_day, DAY_MICRO_SECS}, @@ -135,9 +137,8 @@ impl super::FileList for SqliteFileList { async fn batch_add_deleted( &self, org_id: &str, - flattened: bool, created_at: i64, - files: &[String], + files: &[FileListDeleted], ) -> Result<()> { if files.is_empty() { return Ok(()); @@ -148,16 +149,17 @@ impl super::FileList for SqliteFileList { let client = client.lock().await; let mut tx = client.begin().await?; let mut query_builder: QueryBuilder = QueryBuilder::new( - "INSERT INTO file_list_deleted (org, stream, date, file, flattened, created_at)", + "INSERT INTO file_list_deleted (org, stream, date, file, index_file, flattened, created_at)", ); - query_builder.push_values(files, |mut b, item: &String| { + query_builder.push_values(files, |mut b, item| { let (stream_key, date_key, file_name) = - parse_file_key_columns(item).expect("parse file key failed"); + parse_file_key_columns(&item.file).expect("parse file key failed"); b.push_bind(org_id) .push_bind(stream_key) .push_bind(date_key) .push_bind(file_name) - .push_bind(flattened) + .push_bind(item.index_file) + .push_bind(item.flattened) .push_bind(created_at); }); if let Err(e) = query_builder.build().execute(&mut *tx).await { @@ -229,7 +231,7 @@ impl super::FileList for SqliteFileList { parse_file_key_columns(file).map_err(|e| Error::Message(e.to_string()))?; let ret = sqlx::query_as::<_, super::FileRecord>( r#" -SELECT stream, date, file, deleted, min_ts, max_ts, records, original_size, compressed_size, flattened +SELECT stream, date, file, deleted, min_ts, max_ts, records, original_size, compressed_size, index_size, flattened FROM file_list WHERE stream = $1 AND date = $2 AND file = $3; "#, ) @@ -279,7 +281,7 @@ SELECT stream, date, file, deleted, min_ts, max_ts, records, original_size, comp async fn list(&self) -> Result> { let pool = CLIENT_RO.clone(); let ret = sqlx::query_as::<_, super::FileRecord>( - r#"SELECT stream, date, file, deleted, min_ts, max_ts, records, original_size, compressed_size, flattened FROM file_list;"#, + r#"SELECT stream, date, file, deleted, min_ts, max_ts, records, original_size, compressed_size, index_size, flattened FROM file_list;"#, ) .fetch_all(&pool) .await?; @@ -315,8 +317,8 @@ SELECT stream, date, file, deleted, min_ts, max_ts, records, original_size, comp let ret = if flattened.is_some() { sqlx::query_as::<_, super::FileRecord>( r#" -SELECT stream, date, file, deleted, min_ts, max_ts, records, original_size, compressed_size, flattened - FROM file_list +SELECT stream, date, file, deleted, min_ts, max_ts, records, original_size, compressed_size, index_size, flattened + FROM file_list WHERE stream = $1 AND flattened = $2 LIMIT 1000; "#, ) @@ -329,8 +331,8 @@ SELECT stream, date, file, deleted, min_ts, max_ts, records, original_size, comp let max_ts_upper_bound = super::calculate_max_ts_upper_bound(time_end, stream_type); sqlx::query_as::<_, super::FileRecord>( r#" -SELECT stream, date, file, deleted, min_ts, max_ts, records, original_size, compressed_size, flattened - FROM file_list +SELECT stream, date, file, deleted, min_ts, max_ts, records, original_size, compressed_size, index_size, flattened + FROM file_list WHERE stream = $1 AND max_ts >= $2 AND max_ts <= $3 AND min_ts <= $4; "#, ) @@ -370,7 +372,7 @@ SELECT stream, date, file, deleted, min_ts, max_ts, records, original_size, comp .collect::>() .join(","); let query_str = format!( - "SELECT id, stream, date, file, min_ts, max_ts, records, original_size, compressed_size FROM file_list WHERE id IN ({ids})" + "SELECT id, stream, date, file, min_ts, max_ts, records, original_size, compressed_size, index_size FROM file_list WHERE id IN ({ids})" ); let res = sqlx::query_as::<_, super::FileRecord>(&query_str) .fetch_all(&pool) @@ -477,7 +479,7 @@ SELECT stream, date, file, deleted, min_ts, max_ts, records, original_size, comp let max_ts_upper_bound = super::calculate_max_ts_upper_bound(time_end, stream_type); let sql = r#" SELECT date - FROM file_list + FROM file_list WHERE stream = $1 AND max_ts >= $2 AND max_ts <= $3 AND min_ts <= $4 AND records < $5 GROUP BY date HAVING count(*) >= $6; "#; @@ -502,13 +504,13 @@ SELECT date org_id: &str, time_max: i64, limit: i64, - ) -> Result> { + ) -> Result> { if time_max == 0 { return Ok(Vec::new()); } let pool = CLIENT_RO.clone(); let ret = sqlx::query_as::<_, super::FileDeletedRecord>( - r#"SELECT stream, date, file, flattened FROM file_list_deleted WHERE org = $1 AND created_at < $2 LIMIT $3;"#, + r#"SELECT stream, date, file, index_file, flattened FROM file_list_deleted WHERE org = $1 AND created_at < $2 LIMIT $3;"#, ) .bind(org_id) .bind(time_max) @@ -517,11 +519,10 @@ SELECT date .await?; Ok(ret .iter() - .map(|r| { - ( - format!("files/{}/{}/{}", r.stream, r.date, r.file), - r.flattened, - ) + .map(|r| FileListDeleted { + file: format!("files/{}/{}/{}", r.stream, r.date, r.file), + index_file: r.index_file, + flattened: r.flattened, }) .collect()) } @@ -593,8 +594,8 @@ SELECT date }; let sql = format!( r#" -SELECT stream, MIN(min_ts) as min_ts, MAX(max_ts) as max_ts, COUNT(*) as file_num, SUM(records) as records, SUM(original_size) as original_size, SUM(compressed_size) as compressed_size - FROM file_list +SELECT stream, MIN(min_ts) as min_ts, MAX(max_ts) as max_ts, COUNT(*) as file_num, SUM(records) as records, SUM(original_size) as original_size, SUM(compressed_size) as compressed_size, SUM(index_size) as index_size + FROM file_list WHERE {field} = '{value}' "#, ); @@ -685,9 +686,9 @@ SELECT stream, MIN(min_ts) as min_ts, MAX(max_ts) as max_ts, COUNT(*) as file_nu let org_id = stream_key[..stream_key.find('/').unwrap()].to_string(); if let Err(e) = sqlx::query( r#" - INSERT INTO stream_stats - (org, stream, file_num, min_ts, max_ts, records, original_size, compressed_size) - VALUES ($1, $2, 0, 0, 0, 0, 0, 0); + INSERT INTO stream_stats + (org, stream, file_num, min_ts, max_ts, records, original_size, compressed_size, index_size) + VALUES ($1, $2, 0, 0, 0, 0, 0, 0, 0); "#, ) .bind(org_id) @@ -710,9 +711,9 @@ SELECT stream, MIN(min_ts) as min_ts, MAX(max_ts) as max_ts, COUNT(*) as file_nu for (stream_key, stats) in update_streams { if let Err(e) = sqlx::query( r#" -UPDATE stream_stats - SET file_num = file_num + $1, min_ts = $2, max_ts = $3, records = records + $4, original_size = original_size + $5, compressed_size = compressed_size + $6 - WHERE stream = $7; +UPDATE stream_stats + SET file_num = file_num + $1, min_ts = $2, max_ts = $3, records = records + $4, original_size = original_size + $5, compressed_size = compressed_size + $6, index_size = index_size + $7 + WHERE stream = $8; "#, ) .bind(stats.file_num) @@ -721,6 +722,7 @@ UPDATE stream_stats .bind(stats.doc_num) .bind(stats.storage_size) .bind(stats.compressed_size) + .bind(stats.index_size) .bind(stream_key) .execute(&mut *tx) .await @@ -764,7 +766,7 @@ UPDATE stream_stats async fn reset_stream_stats(&self) -> Result<()> { let client = CLIENT_RW.clone(); let client = client.lock().await; - sqlx::query(r#"UPDATE stream_stats SET file_num = 0, min_ts = 0, max_ts = 0, records = 0, original_size = 0, compressed_size = 0;"#) + sqlx::query(r#"UPDATE stream_stats SET file_num = 0, min_ts = 0, max_ts = 0, records = 0, original_size = 0, compressed_size = 0, index_size = 0;"#) .execute(&*client) .await?; Ok(()) @@ -845,10 +847,10 @@ UPDATE stream_stats let ret = match sqlx::query_as::<_, super::MergeJobPendingRecord>( r#" SELECT stream, max(id) as id, COUNT(*) AS num - FROM file_list_jobs - WHERE status = $1 - GROUP BY stream - ORDER BY num DESC + FROM file_list_jobs + WHERE status = $1 + GROUP BY stream + ORDER BY num DESC LIMIT $2;"#, ) .bind(super::FileListJobStatus::Pending) @@ -1041,8 +1043,8 @@ impl SqliteFileList { let client = client.lock().await; match sqlx::query( format!(r#" -INSERT INTO {table} (id, org, stream, date, file, deleted, min_ts, max_ts, records, original_size, compressed_size, flattened) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12); +INSERT INTO {table} (id, org, stream, date, file, deleted, min_ts, max_ts, records, original_size, compressed_size, index_size, flattened) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13); "#).as_str(), ) .bind(id) @@ -1056,6 +1058,7 @@ INSERT INTO {table} (id, org, stream, date, file, deleted, min_ts, max_ts, recor .bind(meta.records) .bind(meta.original_size) .bind(meta.compressed_size) + .bind(meta.index_size) .bind(meta.flattened) .execute(&*client) .await { @@ -1088,7 +1091,7 @@ INSERT INTO {table} (id, org, stream, date, file, deleted, min_ts, max_ts, recor let client = client.lock().await; let mut tx = client.begin().await?; let mut query_builder: QueryBuilder = QueryBuilder::new( - format!("INSERT INTO {table} (id, org, stream, date, file, deleted, min_ts, max_ts, records, original_size, compressed_size, flattened)").as_str(), + format!("INSERT INTO {table} (id, org, stream, date, file, deleted, min_ts, max_ts, records, original_size, compressed_size, index_size, flattened)").as_str(), ); query_builder.push_values(files, |mut b, (id, item)| { let (stream_key, date_key, file_name) = @@ -1105,6 +1108,7 @@ INSERT INTO {table} (id, org, stream, date, file, deleted, min_ts, max_ts, recor .push_bind(item.meta.records) .push_bind(item.meta.original_size) .push_bind(item.meta.compressed_size) + .push_bind(item.meta.index_size) .push_bind(item.meta.flattened); }); let need_single_insert = match query_builder.build().execute(&mut *tx).await { @@ -1170,7 +1174,8 @@ CREATE TABLE IF NOT EXISTS file_list max_ts BIGINT not null, records BIGINT not null, original_size BIGINT not null, - compressed_size BIGINT not null + compressed_size BIGINT not null, + index_size BIGINT not null ); "#, ) @@ -1192,7 +1197,8 @@ CREATE TABLE IF NOT EXISTS file_list_history max_ts BIGINT not null, records BIGINT not null, original_size BIGINT not null, - compressed_size BIGINT not null + compressed_size BIGINT not null, + index_size BIGINT not null ); "#, ) @@ -1208,6 +1214,7 @@ CREATE TABLE IF NOT EXISTS file_list_deleted stream VARCHAR not null, date VARCHAR not null, file VARCHAR not null, + index_file BOOLEAN default false not null, flattened BOOLEAN default false not null, created_at BIGINT not null ); @@ -1246,7 +1253,8 @@ CREATE TABLE IF NOT EXISTS stream_stats max_ts BIGINT not null, records BIGINT not null, original_size BIGINT not null, - compressed_size BIGINT not null + compressed_size BIGINT not null, + index_size BIGINT not null ); "#, ) @@ -1265,6 +1273,16 @@ CREATE TABLE IF NOT EXISTS stream_stats let data_type = "BIGINT default 0 not null"; add_column(&client, "file_list_jobs", column, data_type).await?; + // create column index_size for old version <= 0.13.1 + let column = "index_size"; + let data_type = "BIGINT default 0 not null"; + add_column(&client, "file_list", column, data_type).await?; + add_column(&client, "file_list_history", column, data_type).await?; + add_column(&client, "stream_stats", column, data_type).await?; + let column = "index_file"; + let data_type = "BOOLEAN default false not null"; + add_column(&client, "file_list_deleted", column, data_type).await?; + Ok(()) } diff --git a/src/job/file_list.rs b/src/job/file_list.rs deleted file mode 100644 index 6935e6ede2..0000000000 --- a/src/job/file_list.rs +++ /dev/null @@ -1,159 +0,0 @@ -// Copyright 2024 OpenObserve Inc. -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. -// -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . - -use std::{fs, path::Path}; - -use config::{ - cluster::LOCAL_NODE, - get_config, - meta::stream::{StreamParams, StreamType}, - utils::file::scan_files, -}; -use infra::storage; -use tokio::time; - -use crate::{common::infra::wal, service::db}; - -pub async fn run() -> Result<(), anyhow::Error> { - let cfg = get_config(); - if cfg.common.local_mode || cfg.common.meta_store_external { - return Ok(()); - } - - tokio::task::spawn(async move { run_move_file_to_s3().await }); - // tokio::task::spawn(async move { run_sync_s3_to_cache().await }); - - Ok(()) -} - -pub async fn run_move_file_to_s3() -> Result<(), anyhow::Error> { - if !LOCAL_NODE.is_ingester() { - return Ok(()); // not an ingester, no need to init job - } - - let mut interval = time::interval(time::Duration::from_secs( - get_config().limit.file_push_interval, - )); - interval.tick().await; // trigger the first run - loop { - interval.tick().await; - if let Err(e) = move_file_list_to_storage(true).await { - log::error!("Error moving file_list to remote: {}", e); - } - } -} - -// upload compressed file_list to storage & delete moved files from local -pub async fn move_file_list_to_storage(check_in_use: bool) -> Result<(), anyhow::Error> { - let cfg = get_config(); - let data_dir = Path::new(&cfg.common.data_wal_dir).canonicalize().unwrap(); - - let pattern = format!("{}file_list/", &cfg.common.data_wal_dir); - let files = scan_files(&pattern, "json", None).unwrap_or_default(); - - for file in files { - let local_file = file.to_owned(); - let local_path = Path::new(&file).canonicalize().unwrap(); - let file_path = local_path - .strip_prefix(&data_dir) - .unwrap() - .to_str() - .unwrap() - .replace('\\', "/"); - let columns = file_path.splitn(2, '/').collect::>(); - - // eg: file_list/0/2023/08/21/08/7099303408192061440f3XQ2p.json - let mut file_name = columns[1].to_string(); - - // Hack: compatible for <= 0.5.1 - if !file_name.contains('/') && file_name.contains('_') { - file_name = file_name.replace('_', "/"); - } - - // check the file is using for write - if check_in_use - && wal::check_in_use(StreamParams::new("", "", StreamType::Filelist), &file_name).await - { - continue; - } - log::info!("[JOB] convert file_list: {}", file); - match upload_file(&local_file, &file_name).await { - Ok(_) => match fs::remove_file(&local_file) { - Ok(_) => {} - Err(e) => { - log::error!( - "[JOB] Failed to remove file_list {}: {}", - local_file, - e.to_string() - ) - } - }, - Err(e) => log::error!("[JOB] Error while uploading file_list to storage {}", e), - } - } - Ok(()) -} - -async fn upload_file(path_str: &str, file_key: &str) -> Result<(), anyhow::Error> { - let mut file = fs::File::open(path_str).unwrap(); - let file_meta = file.metadata().unwrap(); - let file_size = file_meta.len(); - log::info!("[JOB] File_list upload begin: local: {}", path_str); - if file_size == 0 { - if let Err(e) = fs::remove_file(path_str) { - log::error!("[JOB] File_list failed to remove: {}, {}", path_str, e); - } - return Err(anyhow::anyhow!("File_list is empty: {}", path_str)); - } - - let mut encoder = zstd::Encoder::new(Vec::new(), 3)?; - std::io::copy(&mut file, &mut encoder)?; - let compressed_bytes = encoder.finish().unwrap(); - - let file_columns = file_key.split('/').collect::>(); - let new_file_key = format!( - "file_list/{}/{}/{}/{}/{}.zst", - file_columns[1], file_columns[2], file_columns[3], file_columns[4], file_columns[5] - ); - - let result = storage::put(&new_file_key, bytes::Bytes::from(compressed_bytes)).await; - match result { - Ok(_output) => { - log::info!("[JOB] File_list upload successfully: {}", new_file_key); - Ok(()) - } - Err(err) => { - log::error!("[JOB] File_list upload error: {:?}", err); - Err(anyhow::anyhow!(err)) - } - } -} - -async fn _run_sync_s3_to_cache() -> Result<(), anyhow::Error> { - if !LOCAL_NODE.is_querier() && !LOCAL_NODE.is_compactor() { - return Ok(()); // only querier or compactor need to sync - } - - let mut interval = time::interval(time::Duration::from_secs( - get_config().s3.sync_to_cache_interval, - )); - interval.tick().await; // trigger the first run - loop { - interval.tick().await; - if let Err(e) = db::file_list::remote::cache_latest_hour().await { - log::error!("[JOB] File_list sync s3 to cache error: {}", e); - } - } -} diff --git a/src/job/files/parquet.rs b/src/job/files/parquet.rs index 2a1d7a1c59..c86d2278c6 100644 --- a/src/job/files/parquet.rs +++ b/src/job/files/parquet.rs @@ -23,7 +23,7 @@ use std::{ use anyhow::Context; use arrow::{ array::{ - new_null_array, ArrayRef, BinaryBuilder, BooleanArray, BooleanBuilder, Int64Array, + new_null_array, Array, ArrayRef, BinaryBuilder, BooleanArray, BooleanBuilder, Int64Array, Int64Builder, StringArray, StringBuilder, }, datatypes::Field, @@ -37,7 +37,8 @@ use config::{ meta::{ bitvec::BitVec, inverted_index::{writer::ColumnIndexer, InvertedIndexFormat}, - puffin::writer::PuffinBytesWriter, + puffin::{writer::PuffinBytesWriter, BlobTypes}, + puffin_directory::writer::PuffinDirWriter, search::StorageType, stream::{FileKey, FileMeta, PartitionTimeLevel, StreamSettings, StreamType}, }, @@ -46,7 +47,10 @@ use config::{ arrow::record_batches_to_json_rows, asynchronism::file::get_file_meta, file::scan_files_with_channel, - inverted_index::{convert_parquet_idx_file_name, split_token}, + inverted_index::{ + convert_parquet_idx_file_name, convert_parquet_idx_file_name_to_tantivy_file, + split_token, + }, json, parquet::{ get_recordbatch_reader_from_bytes, read_metadata_from_file, read_schema_from_file, @@ -632,6 +636,7 @@ async fn merge_files( original_size: new_file_size, compressed_size: 0, flattened: false, + index_size: 0, }; if new_file_meta.records == 0 { return Err(anyhow::anyhow!("merge_files error: records is 0")); @@ -767,21 +772,23 @@ async fn merge_files( .map_err(|e| anyhow::anyhow!("generate_parquet_index_on_ingester error: {}", e))?; } - // generate fst format inverted index + // generate tantivy inverted index and write to storage if matches!( index_format, - InvertedIndexFormat::FST | InvertedIndexFormat::Both + InvertedIndexFormat::Tantivy | InvertedIndexFormat::Both ) { let (schema, mut reader) = get_recordbatch_reader_from_bytes(&buf).await?; - generate_fst_inverted_index( + let index_size = create_tantivy_index( + "INGESTER", &new_file_key, &full_text_search_fields, &index_fields, - None, schema, &mut reader, ) - .await?; + .await + .map_err(|e| anyhow::anyhow!("generate_tantivy_index_on_ingester error: {}", e))?; + new_file_meta.index_size = index_size as i64; } Ok((new_file_key, new_file_meta, retain_file_list)) @@ -1323,17 +1330,21 @@ async fn prepare_index_record_batches( /// Creates fst inverted index bytes and writes to storage. /// Called by both ingester and compactor. Compactor needs to provide `file_list_to_invalidate` /// to delete previously created small index files +#[allow(dead_code)] pub(crate) async fn generate_fst_inverted_index( parquet_file_name: &str, + file_list_to_invalidate: Option<&[FileKey]>, full_text_search_fields: &[String], index_fields: &[String], - file_list_to_invalidate: Option<&[FileKey]>, /* for compactor to delete corresponding small - * .idx files */ - schema: Arc, reader: &mut ParquetRecordBatchStream>, ) -> Result<(), anyhow::Error> { let start = std::time::Instant::now(); + let caller = if file_list_to_invalidate.is_some() { + "[COMPACTOR:JOB]" + } else { + "[INGESTER:JOB]" + }; if full_text_search_fields.is_empty() && index_fields.is_empty() { return Ok(()); @@ -1366,11 +1377,6 @@ pub(crate) async fn generate_fst_inverted_index( let Some(idx_file_name) = convert_parquet_idx_file_name(parquet_file_name) else { return Ok(()); }; - let caller = if file_list_to_invalidate.is_some() { - "[COMPACTOR:JOB]" - } else { - "[INGESTER:JOB]" - }; match storage::put(&idx_file_name, Bytes::from(compressed_bytes)).await { Ok(_) => { log::info!( @@ -1395,6 +1401,191 @@ pub(crate) async fn generate_fst_inverted_index( } } +pub(crate) async fn create_tantivy_index( + caller: &str, + parquet_file_name: &str, + full_text_search_fields: &[String], + index_fields: &[String], + schema: Arc, + reader: &mut ParquetRecordBatchStream>, +) -> Result { + let start = std::time::Instant::now(); + let caller = format!("[{caller}:JOB]"); + + let dir = PuffinDirWriter::new(); + let _index = generate_tantivy_index( + dir.clone(), + reader, + full_text_search_fields, + index_fields, + schema, + ) + .await?; + let puffin_bytes = dir.to_puffin_bytes()?; + let index_size = puffin_bytes.len(); + + // write fst bytes into disk + let Some(idx_file_name) = convert_parquet_idx_file_name_to_tantivy_file(parquet_file_name) + else { + return Ok(0); + }; + + match storage::put(&idx_file_name, Bytes::from(puffin_bytes)).await { + Ok(_) => { + log::info!( + "{} Written tantivy index file successfully: {}, index size {}, took: {} ms", + caller, + idx_file_name, + index_size, + start.elapsed().as_millis() + ); + } + Err(e) => { + log::error!( + "{} Written tantivy index file error: {}", + caller, + e.to_string() + ); + return Err(e); + } + } + + Ok(index_size) +} + +/// Create a tantivy index in the given directory for the record batch +pub(crate) async fn generate_tantivy_index( + tantivy_dir: D, + reader: &mut ParquetRecordBatchStream>, + full_text_search_fields: &[String], + index_fields: &[String], + schema: Arc, +) -> Result, anyhow::Error> { + let mut tantivy_schema_builder = tantivy::schema::SchemaBuilder::new(); + let schema_fields = schema + .fields() + .iter() + .map(|f| (f.name(), f)) + .collect::>(); + + // filter out fields that are not in schema & not of type Utf8 + let fts_fields = full_text_search_fields + .iter() + .filter(|f| { + schema_fields + .get(f) + .map(|v| v.data_type() == &DataType::Utf8) + .is_some() + }) + .map(|f| f.to_string()) + .collect::>(); + let index_fields = index_fields + .iter() + .filter(|f| { + schema_fields + .get(f) + .map(|v| v.data_type() == &DataType::Utf8) + .is_some() + }) + .map(|f| f.to_string()) + .collect::>(); + let tantivy_fields = fts_fields.union(&index_fields).collect::>(); + if tantivy_fields.is_empty() { + return Ok(None); + } + + // add fields to tantivy schema + if !fts_fields.is_empty() { + let fts_opts = tantivy::schema::TextOptions::default().set_indexing_options( + tantivy::schema::TextFieldIndexing::default() + .set_index_option(tantivy::schema::IndexRecordOption::Basic) + .set_fieldnorms(false), + ); + tantivy_schema_builder.add_text_field(INDEX_FIELD_NAME_FOR_ALL, fts_opts); + } + for field in index_fields.iter() { + let index_opts = tantivy::schema::TextOptions::default().set_indexing_options( + tantivy::schema::TextFieldIndexing::default() + .set_index_option(tantivy::schema::IndexRecordOption::Basic) + .set_tokenizer("raw") + .set_fieldnorms(false), + ); + tantivy_schema_builder.add_text_field(field, index_opts); + } + let tantivy_schema = tantivy_schema_builder.build(); + let fts_field = tantivy_schema.get_field(INDEX_FIELD_NAME_FOR_ALL).ok(); + + let mut total_num_rows = 0; + // docs per row to be added in the tantivy index + let mut docs = BTreeMap::new(); + + loop { + let batch = reader.try_next().await?; + let Some(inverted_idx_batch) = batch else { + break; + }; + let num_rows = inverted_idx_batch.num_rows(); + if num_rows == 0 { + continue; + } + + // update total_num_rows + let prev_total_num_rows = total_num_rows; + total_num_rows += num_rows; + + // process full text search fields + for column_name in tantivy_fields.iter() { + // get column + let Some(column_data) = inverted_idx_batch + .column_by_name(column_name) + .unwrap() + .as_any() + .downcast_ref::() + else { + continue; + }; + + // get field + let field = match tantivy_schema.get_field(column_name) { + Ok(f) => f, + Err(_) => fts_field.unwrap(), + }; + for i in 0..num_rows { + let doc = docs + .entry(i + prev_total_num_rows) + .or_insert(tantivy::doc!()); + doc.add_text(field, column_data.value(i)); + } + } + } + + // TODO: Check if 50MB is enough for the tantivy index for >1GB parquet file + let mut index_writer = tantivy::IndexBuilder::new() + .schema(tantivy_schema) + .single_segment_index_writer(tantivy_dir, 50_000_000) + .context("failed to create index builder")?; + + for (_rowid, doc) in docs.into_iter() { + if let Err(e) = index_writer.add_document(doc) { + log::error!( + "generate_tantivy_index: Failed to add document to index: {}", + e + ); + return Err(anyhow::anyhow!("Failed to add document to index: {}", e)); + } + } + + let index = index_writer.finalize().map_err(|e| { + log::error!( + "generate_tantivy_index: Failed to finalize the index writer: {}", + e + ); + anyhow::anyhow!("Failed to finalize the index writer: {}", e) + })?; + + Ok(Some(index)) +} + /// Create and compressed inverted index bytes using FST solution for the given RecordBatch pub(crate) async fn prepare_fst_index_bytes( schema: Arc, @@ -1526,7 +1717,6 @@ pub(crate) async fn prepare_fst_index_bytes( // create puffin file let mut puffin_buf: Vec = Vec::new(); let mut puffin_writer = PuffinBytesWriter::new(&mut puffin_buf); - let mut original_size = 0; for (column_name, indexer) in indexers { if indexer.is_empty() { @@ -1538,7 +1728,7 @@ pub(crate) async fn prepare_fst_index_bytes( column_name ))?; original_size += buf.len(); - puffin_writer.add_blob(column_name, buf)?; + puffin_writer.add_blob(&buf, BlobTypes::O2FstV1, column_name, true)?; } puffin_writer.finish()?; diff --git a/src/job/mod.rs b/src/job/mod.rs index 1802ef213d..00e2cbf21f 100644 --- a/src/job/mod.rs +++ b/src/job/mod.rs @@ -24,12 +24,11 @@ use crate::{ infra::config::SYSLOG_ENABLED, meta::{organization::DEFAULT_ORG, user::UserRequest}, }, - service::{compact::stats::update_stats_from_file_list, db, usage, users}, + service::{db, usage, users}, }; mod alert_manager; mod compactor; -pub(crate) mod file_list; pub(crate) mod files; mod flatten_compactor; pub mod metrics; @@ -169,31 +168,6 @@ pub async fn init() -> Result<(), anyhow::Error> { // cache pipeline db::pipeline::cache().await.expect("Pipeline cache failed"); - // cache file list - if !cfg.common.meta_store_external { - if LOCAL_NODE.is_ingester() { - // load the wal file_list into memory - db::file_list::local::load_wal_in_cache() - .await - .expect("load wal file list failed"); - } - if LOCAL_NODE.is_querier() || LOCAL_NODE.is_compactor() { - db::file_list::remote::cache("", false) - .await - .expect("file list remote cache failed"); - infra_file_list::create_table_index() - .await - .expect("file list create table index failed"); - infra_file_list::LOCAL_CACHE - .create_table_index() - .await - .expect("file list create table index failed"); - update_stats_from_file_list() - .await - .expect("file list remote calculate stats failed"); - } - } - infra_file_list::create_table_index().await?; infra_file_list::LOCAL_CACHE.create_table_index().await?; tokio::task::spawn(async move { db::file_list::remote::cache_stats().await }); @@ -217,7 +191,6 @@ pub async fn init() -> Result<(), anyhow::Error> { } tokio::task::spawn(async move { files::run().await }); - tokio::task::spawn(async move { file_list::run().await }); tokio::task::spawn(async move { stats::run().await }); tokio::task::spawn(async move { compactor::run().await }); tokio::task::spawn(async move { flatten_compactor::run().await }); diff --git a/src/proto/proto/cluster/common.proto b/src/proto/proto/cluster/common.proto index 91da27e9c2..e65c59fda1 100644 --- a/src/proto/proto/cluster/common.proto +++ b/src/proto/proto/cluster/common.proto @@ -20,6 +20,7 @@ message FileMeta { int64 records = 3; int64 original_size = 4; int64 compressed_size = 5; + int64 index_size = 6; } enum StreamType { diff --git a/src/proto/proto/cluster/plan.proto b/src/proto/proto/cluster/plan.proto index dc08ce538f..7dccb165be 100644 --- a/src/proto/proto/cluster/plan.proto +++ b/src/proto/proto/cluster/plan.proto @@ -5,7 +5,7 @@ option java_package = "org.openobserve.cluster"; option java_outer_classname = "planProto"; package cluster; - + import "datafusion_common/datafusion_common.proto"; import "datafusion/datafusion.proto"; import "cluster/common.proto"; @@ -26,7 +26,7 @@ message FlightSearchRequest { uint32 partition = 2; string org_id = 3; string stream_type = 4; - // used for search + // used for search bytes plan = 6; repeated int64 file_id_list = 7; repeated IdxFileName idx_file_list = 8; @@ -39,7 +39,6 @@ message FlightSearchRequest { bool is_super_cluster = 14; bool use_inverted_index = 15; optional string work_group = 16; - optional string index_type = 17; optional string user_id = 18; optional string search_event_type = 19; } diff --git a/src/proto/proto/cluster/search.proto b/src/proto/proto/cluster/search.proto index dd8f6ae696..371e04a820 100644 --- a/src/proto/proto/cluster/search.proto +++ b/src/proto/proto/cluster/search.proto @@ -41,7 +41,6 @@ message SearchRequest { repeated IdxFileName idx_files = 8; int64 timeout = 9; string work_group = 10; - string index_type = 11; optional string user_id = 12; optional string search_event_type = 13; } @@ -101,4 +100,4 @@ message CancelQueryRequest { message CancelQueryResponse { bool is_success = 1; -} \ No newline at end of file +} diff --git a/src/proto/src/generated/cluster.rs b/src/proto/src/generated/cluster.rs index edb52af095..708770b63b 100644 --- a/src/proto/src/generated/cluster.rs +++ b/src/proto/src/generated/cluster.rs @@ -32,6 +32,8 @@ pub struct FileMeta { pub original_size: i64, #[prost(int64, tag = "5")] pub compressed_size: i64, + #[prost(int64, tag = "6")] + pub index_size: i64, } /// Job information for a request #[derive(Eq)] @@ -968,8 +970,6 @@ pub struct SearchRequest { pub timeout: i64, #[prost(string, tag = "10")] pub work_group: ::prost::alloc::string::String, - #[prost(string, tag = "11")] - pub index_type: ::prost::alloc::string::String, #[prost(string, optional, tag = "12")] pub user_id: ::core::option::Option<::prost::alloc::string::String>, #[prost(string, optional, tag = "13")] @@ -2804,8 +2804,6 @@ pub struct FlightSearchRequest { pub use_inverted_index: bool, #[prost(string, optional, tag = "16")] pub work_group: ::core::option::Option<::prost::alloc::string::String>, - #[prost(string, optional, tag = "17")] - pub index_type: ::core::option::Option<::prost::alloc::string::String>, #[prost(string, optional, tag = "18")] pub user_id: ::core::option::Option<::prost::alloc::string::String>, #[prost(string, optional, tag = "19")] diff --git a/src/service/alerts/mod.rs b/src/service/alerts/mod.rs index c432eea0f6..aafb4cfab1 100644 --- a/src/service/alerts/mod.rs +++ b/src/service/alerts/mod.rs @@ -336,7 +336,6 @@ impl QueryConditionExt for QueryCondition { timeout: 0, search_type, search_event_context, - index_type: "".to_string(), }; SearchService::search(&trace_id, org_id, stream_type, None, &req).await }; diff --git a/src/service/compact/deleted.rs b/src/service/compact/deleted.rs new file mode 100644 index 0000000000..dcb10621d2 --- /dev/null +++ b/src/service/compact/deleted.rs @@ -0,0 +1,153 @@ +// Copyright 2024 OpenObserve Inc. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +use config::{ + meta::stream::FileListDeleted, + utils::inverted_index::convert_parquet_idx_file_name_to_tantivy_file, +}; +use hashbrown::HashMap; +use infra::{file_list as infra_file_list, storage}; + +pub async fn delete( + org_id: &str, + _time_min: i64, + time_max: i64, + batch_size: i64, +) -> Result { + let files = query_deleted(org_id, time_max, batch_size).await?; + if files.is_empty() { + return Ok(0); + } + let files_num = files.values().flatten().count() as i64; + + // delete files from storage + if let Err(e) = storage::del( + &files + .values() + .flatten() + .map(|file| file.file.as_str()) + .collect::>(), + ) + .await + { + // maybe the file already deleted, so we just skip the `not found` error + if !e.to_string().to_lowercase().contains("not found") { + log::error!("[COMPACT] delete files from storage failed: {}", e); + return Err(e); + } + } + + // delete related inverted index puffin files + let inverted_index_files = files + .values() + .flatten() + .filter_map(|file| { + if file.index_file { + convert_parquet_idx_file_name_to_tantivy_file(&file.file) + } else { + None + } + }) + .collect::>(); + if !inverted_index_files.is_empty() { + if let Err(e) = storage::del( + &inverted_index_files + .iter() + .map(|file| file.as_str()) + .collect::>(), + ) + .await + { + // maybe the file already deleted or there's not related index files, + // so we just skip the `not found` error + if !e.to_string().to_lowercase().contains("not found") { + log::error!("[COMPACT] delete files from storage failed: {}", e); + return Err(e); + } + } + } + + // delete flattened files from storage + let flattened_files = files + .values() + .flatten() + .filter_map(|file| { + if file.flattened { + Some(format!( + "files{}/{}", + config::get_config().common.column_all, + file.file.strip_prefix("files/").unwrap() + )) + } else { + None + } + }) + .collect::>(); + if !flattened_files.is_empty() { + if let Err(e) = storage::del( + &flattened_files + .iter() + .map(|file| file.as_str()) + .collect::>(), + ) + .await + { + // maybe the file already deleted, so we just skip the `not found` error + if !e.to_string().to_lowercase().contains("not found") { + log::error!("[COMPACT] delete files from storage failed: {}", e); + return Err(e); + } + } + } + + // delete files from file_list_deleted s3 + if files.keys().len() > 1 || !files.contains_key("") { + if let Err(e) = + storage::del(&files.keys().map(|file| file.as_str()).collect::>()).await + { + log::error!("[COMPACT] delete files from storage failed: {}", e); + return Err(e); + } + } + + // delete files from file_list_deleted table + if let Err(e) = infra_file_list::batch_remove_deleted( + &files + .values() + .flatten() + .map(|file| file.file.to_owned()) + .collect::>(), + ) + .await + { + log::error!("[COMPACT] delete files from table failed: {}", e); + return Err(e.into()); + } + + Ok(files_num) +} + +async fn query_deleted( + org_id: &str, + time_max: i64, + limit: i64, +) -> Result>, anyhow::Error> { + let files = infra_file_list::query_deleted(org_id, time_max, limit).await?; + let mut hash_files = HashMap::default(); + if !files.is_empty() { + hash_files.insert("".to_string(), files); + } + Ok(hash_files) +} diff --git a/src/service/compact/file_list.rs b/src/service/compact/file_list.rs deleted file mode 100644 index 815b1005e1..0000000000 --- a/src/service/compact/file_list.rs +++ /dev/null @@ -1,356 +0,0 @@ -// Copyright 2024 OpenObserve Inc. -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. -// -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . - -use std::{ - io::{BufRead, BufReader, Write}, - sync::Arc, -}; - -use bytes::Buf; -use chrono::{DateTime, Datelike, Duration, TimeZone, Timelike, Utc}; -use config::{ - cluster::LOCAL_NODE, - ider, - meta::stream::FileKey, - utils::{json, time::hour_micros}, -}; -use hashbrown::HashMap; -use infra::{dist_lock, schema::STREAM_SCHEMAS_LATEST, storage}; -use tokio::sync::{RwLock, Semaphore}; - -use crate::{common::infra::cluster::get_node_by_uuid, service::db}; - -pub async fn run(offset: i64) -> Result<(), anyhow::Error> { - run_merge(offset).await?; - run_delete().await?; - Ok(()) -} - -/// check all streams done compact in this hour -/// merge all small file list keys in this hour to a single file and upload to -/// storage delete all small file list keys in this hour from storage -/// node should load new file list from storage -pub async fn run_merge(offset: i64) -> Result<(), anyhow::Error> { - let time_now: DateTime = Utc::now(); - let time_now_hour = Utc - .with_ymd_and_hms( - time_now.year(), - time_now.month(), - time_now.day(), - time_now.hour(), - 0, - 0, - ) - .unwrap() - .timestamp_micros(); - - let mut offset = offset; - if offset == 0 { - // get earliest date from schema - offset = time_now.timestamp_micros(); - let r = STREAM_SCHEMAS_LATEST.read().await; - for (key, val) in r.iter() { - if let Some(val) = val.schema().metadata.get("created_at") { - let time_min = val.parse().unwrap(); - if time_min == 0 { - log::info!( - "[COMPACT] file_list stream [{}] created_at is 0, just skip", - key - ); - continue; - } - if time_min < offset { - offset = time_min; - } - } - } - drop(r); - } - - // still not found, just return - if offset == 0 { - log::info!("[COMPACT] file_list no stream, no need to compact"); - return Ok(()); // no stream - } - // only compact for the past hour - if offset >= time_now_hour { - return Ok(()); - } - - let offset_time: DateTime = Utc.timestamp_nanos(offset * 1000); - let offset_time_hour = Utc - .with_ymd_and_hms( - offset_time.year(), - offset_time.month(), - offset_time.day(), - offset_time.hour(), - 0, - 0, - ) - .unwrap() - .timestamp_micros(); - - // check compact is done - let offsets = db::compact::files::list_offset().await?; - if offsets.is_empty() { - return Ok(()); // no stream - } - // compact offset already is next hour, we need fix it, get the latest compact - // offset - let mut is_waiting_streams = false; - for (key, val) in offsets { - if (val - hour_micros(1)) < offset { - log::info!("[COMPACT] file_list is waiting for stream: {key}, offset: {val}"); - is_waiting_streams = true; - break; - } - } - - if is_waiting_streams { - // compact zero hour for daily partition - let time_zero_hour = Utc - .with_ymd_and_hms(time_now.year(), time_now.month(), time_now.day(), 0, 0, 0) - .unwrap() - .timestamp_micros(); - merge_file_list(time_zero_hour).await?; - tokio::time::sleep(tokio::time::Duration::from_secs(1)).await; - // compact last hour, because it just done compaction that generated a lot of - // small file_list files - let time_last_hour = time_now - Duration::try_hours(1).unwrap(); - let time_last_hour = Utc - .with_ymd_and_hms( - time_last_hour.year(), - time_last_hour.month(), - time_last_hour.day(), - time_last_hour.hour(), - 0, - 0, - ) - .unwrap() - .timestamp_micros(); - merge_file_list(time_last_hour).await?; - tokio::time::sleep(tokio::time::Duration::from_secs(1)).await; - // compact current hour, because it continue to generate small file_list files - merge_file_list(time_now_hour).await?; - // it waiting, no need update offset - return Ok(()); - } - - // compact zero hour for daily partition - let offset_zero_hour = Utc - .with_ymd_and_hms( - offset_time.year(), - offset_time.month(), - offset_time.day(), - 0, - 0, - 0, - ) - .unwrap() - .timestamp_micros(); - merge_file_list(offset_zero_hour).await?; - tokio::time::sleep(tokio::time::Duration::from_secs(1)).await; - - // compact offset file list - merge_file_list(offset).await?; - - // write new sync offset - offset = offset_time_hour + hour_micros(1); - db::compact::file_list::set_offset(offset).await -} - -pub async fn run_delete() -> Result<(), anyhow::Error> { - let days = db::compact::file_list::list_delete().await?; - if days.is_empty() { - return Ok(()); // no delete - } - - for day in days { - let mut t = DateTime::parse_from_rfc3339(&format!("{day}T00:00:00Z"))?.with_timezone(&Utc); - for _hour in 0..24 { - let offset = t.timestamp_micros(); - merge_file_list(offset).await?; - t += Duration::try_hours(1).unwrap(); - } - - // delete day - db::compact::file_list::del_delete(&day).await?; - } - - Ok(()) -} - -/// merge and delete the small file list keys in this hour from etcd -/// upload new file list into storage -async fn merge_file_list(offset: i64) -> Result<(), anyhow::Error> { - let lock_key = format!("/compact/file_list/{offset}"); - let locker = dist_lock::lock(&lock_key, 0, None).await?; - let node = db::compact::file_list::get_process(offset).await; - if !node.is_empty() && LOCAL_NODE.uuid.ne(&node) && get_node_by_uuid(&node).await.is_some() { - dist_lock::unlock(&locker).await?; - return Ok(()); // not this node, just skip - } - - // before start merging, set current node to lock the offset - let ret = db::compact::file_list::set_process(offset, &LOCAL_NODE.uuid.clone()).await; - // already bind to this node, we can unlock now - dist_lock::unlock(&locker).await?; - drop(locker); - ret?; - - // get all small file list keys in this hour - let offset_time = Utc.timestamp_nanos(offset * 1000); - let offset_prefix = offset_time.format("/%Y/%m/%d/%H/").to_string(); - let key = format!("file_list{offset_prefix}"); - log::info!("[COMPACT] file_list is merging, prefix: {key}"); - let file_list = storage::list(&key).await?; - if file_list.len() <= 1 { - db::compact::file_list::del_process(offset).await?; - return Ok(()); // only one file list, no need merge - } - log::info!( - "[COMPACT] file_list is merging, prefix: {key}, got files: {}", - file_list.len() - ); - - // filter deleted file keys - let filter_file_keys: Arc>> = - Arc::new(RwLock::new(HashMap::new())); - let semaphore = std::sync::Arc::new(Semaphore::new( - config::get_config().limit.file_merge_thread_num, - )); - let mut tasks = Vec::new(); - for file in file_list.clone() { - let filter_file_keys = filter_file_keys.clone(); - let permit = semaphore.clone().acquire_owned().await.unwrap(); - let task: tokio::task::JoinHandle> = - tokio::task::spawn(async move { - log::info!("[COMPACT] file_list merge small files: {}", file); - let data = match storage::get(&file).await { - Ok(val) => val, - Err(err) => { - drop(permit); - return Err(err); - } - }; - // uncompress file - let uncompress = match zstd::decode_all(data.reader()) { - Ok(val) => val, - Err(err) => { - drop(permit); - return Err(err.into()); - } - }; - let uncompress_reader = BufReader::new(uncompress.reader()); - // parse file list - for line in uncompress_reader.lines() { - let line = match line { - Ok(val) => val, - Err(err) => { - drop(permit); - return Err(err.into()); - } - }; - if line.is_empty() { - continue; - } - let item: FileKey = match json::from_slice(line.as_bytes()) { - Ok(val) => val, - Err(err) => { - drop(permit); - return Err(err.into()); - } - }; - let mut filter_file_keys = filter_file_keys.write().await; - match filter_file_keys.get(&item.key) { - Some(_) => { - if item.deleted { - filter_file_keys.insert(item.key.clone(), item); - } - } - None => { - filter_file_keys.insert(item.key.clone(), item); - } - } - } - drop(permit); - Ok(()) - }); - tasks.push(task); - } - - // wait all tasks done - for task in tasks { - match task.await { - Ok(ret) => match ret { - Ok(_) => {} - Err(err) => { - log::error!("[COMPACT] file_list merge small files failed: {}", err); - } - }, - Err(err) => { - log::error!("[COMPACT] file_list merge small files failed: {}", err); - } - } - } - - // write new file list - let id = ider::generate(); - let file_name = format!("file_list{offset_prefix}{id}.json.zst"); - let mut buf = zstd::Encoder::new(Vec::new(), 3)?; - let mut has_content = false; - let filter_file_keys = filter_file_keys.read().await; - for (_, item) in filter_file_keys.iter() { - if item.deleted { - continue; - } - let val = json::to_vec(&item)?; - buf.write_all(val.as_slice())?; - buf.write_all(b"\n")?; - has_content = true; - } - let compressed_bytes = buf.finish().unwrap(); - - let new_file_ok = if has_content { - match storage::put(&file_name, compressed_bytes.into()).await { - Ok(_) => { - log::info!( - "[COMPACT] file_list merge successfully, {} files into a new file: {}", - file_list.len(), - file_name - ); - true - } - Err(err) => { - log::error!("[COMPACT] file_list upload failed: {}", err); - false - } - } - } else { - true - }; - if new_file_ok { - // delete all small file list keys in this hour from storage - if let Err(e) = - storage::del(&file_list.iter().map(|v| v.as_str()).collect::>()).await - { - log::error!("[COMPACT] file_list delete small file failed: {}", e); - } - } - - // clean progress mark - db::compact::file_list::del_process(offset).await?; - Ok(()) -} diff --git a/src/service/compact/file_list_deleted.rs b/src/service/compact/file_list_deleted.rs deleted file mode 100644 index 48e324feec..0000000000 --- a/src/service/compact/file_list_deleted.rs +++ /dev/null @@ -1,284 +0,0 @@ -// Copyright 2024 OpenObserve Inc. -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. -// -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . - -use std::io::{BufRead, BufReader}; - -use bytes::Buf; -use chrono::{DateTime, TimeZone, Utc}; -use config::utils::time::hour_micros; -use futures::future::try_join_all; -use hashbrown::HashMap; -use infra::{file_list as infra_file_list, storage}; - -pub async fn delete( - org_id: &str, - time_min: i64, - time_max: i64, - batch_size: i64, -) -> Result { - let files = query_deleted(org_id, time_min, time_max, batch_size).await?; - if files.is_empty() { - return Ok(0); - } - let files_num = files.values().flatten().count() as i64; - - // delete files from storage - if let Err(e) = storage::del( - &files - .values() - .flatten() - .map(|file| file.0.as_str()) - .collect::>(), - ) - .await - { - // maybe the file already deleted, so we just skip the `not found` error - if !e.to_string().to_lowercase().contains("not found") { - log::error!("[COMPACT] delete files from storage failed: {}", e); - return Err(e); - } - } - - // TODO: disable it for now, we haven't enable inverted index puffin files - // delete related inverted index puffin files - // let inverted_index_files = files - // .values() - // .flatten() - // .filter_map(|file| convert_parquet_idx_file_name(&file.0)) - // .collect::>(); - // if !inverted_index_files.is_empty() { - // if let Err(e) = storage::del( - // &inverted_index_files - // .iter() - // .map(|file| file.as_str()) - // .collect::>(), - // ) - // .await - // { - // // maybe the file already deleted or there's not related index files, - // // so we just skip the `not found` error - // if !e.to_string().to_lowercase().contains("not found") { - // log::error!("[COMPACT] delete files from storage failed: {}", e); - // return Err(e); - // } - // } - // } - - // delete flattened files from storage - let flattened_files = files - .values() - .flatten() - .filter_map(|file| { - if file.1 { - Some(format!( - "files{}/{}", - config::get_config().common.column_all, - file.0.strip_prefix("files/").unwrap() - )) - } else { - None - } - }) - .collect::>(); - if !flattened_files.is_empty() { - if let Err(e) = storage::del( - &flattened_files - .iter() - .map(|file| file.as_str()) - .collect::>(), - ) - .await - { - // maybe the file already deleted, so we just skip the `not found` error - if !e.to_string().to_lowercase().contains("not found") { - log::error!("[COMPACT] delete files from storage failed: {}", e); - return Err(e); - } - } - } - - // delete files from file_list_deleted s3 - if files.keys().len() > 1 || !files.contains_key("") { - if let Err(e) = - storage::del(&files.keys().map(|file| file.as_str()).collect::>()).await - { - log::error!("[COMPACT] delete files from storage failed: {}", e); - return Err(e); - } - } - - // delete files from file_list_deleted table - if let Err(e) = infra_file_list::batch_remove_deleted( - &files - .values() - .flatten() - .map(|file| file.0.to_owned()) - .collect::>(), - ) - .await - { - log::error!("[COMPACT] delete files from table failed: {}", e); - return Err(e.into()); - } - - Ok(files_num) -} - -async fn query_deleted( - org_id: &str, - time_min: i64, - time_max: i64, - limit: i64, -) -> Result>, anyhow::Error> { - if config::get_config().common.meta_store_external { - query_deleted_from_table(org_id, time_min, time_max, limit).await - } else { - query_deleted_from_s3(org_id, time_min, time_max, limit).await - } -} - -async fn query_deleted_from_table( - org_id: &str, - _time_min: i64, - time_max: i64, - limit: i64, -) -> Result>, anyhow::Error> { - let files = infra_file_list::query_deleted(org_id, time_max, limit).await?; - let mut hash_files = HashMap::default(); - if !files.is_empty() { - hash_files.insert("".to_string(), files); - } - Ok(hash_files) -} - -async fn query_deleted_from_s3( - org_id: &str, - time_min: i64, - time_max: i64, - _limit: i64, -) -> Result>, anyhow::Error> { - let mut cur_time = if time_min != 0 { - time_min - } else { - let prefix = format!("file_list_deleted/{org_id}/"); - let mut files = storage::list(&prefix).await?; - if files.is_empty() { - return Ok(HashMap::default()); - } - files.sort(); - // file_list_deleted/{org_id}/{year}/{month}/{day}/{hour}/{file}.json.zst - let file = files.first().unwrap(); - let columns = file.split('/').collect::>(); - let time = Utc - .with_ymd_and_hms( - columns[2].parse::().unwrap(), - columns[3].parse::().unwrap(), - columns[4].parse::().unwrap(), - columns[5].parse::().unwrap(), - 0, - 0, - ) - .unwrap(); - time.timestamp_micros() - }; - let mut files = HashMap::with_capacity(16); - while cur_time <= time_max { - let offset_time: DateTime = Utc.timestamp_nanos(cur_time * 1000); - let file_list_prefix = format!("{org_id}/{}", offset_time.format("%Y/%m/%d/%H/")); - let prefix_files = load_prefix_from_s3(&file_list_prefix).await?; - for (file, records) in prefix_files { - let entry = files.entry(file).or_insert_with(Vec::new); - entry.extend(records); - } - cur_time += hour_micros(1); - } - Ok(files) -} - -pub async fn load_prefix_from_s3( - prefix: &str, -) -> Result>, anyhow::Error> { - let prefix = format!("file_list_deleted/{prefix}/"); - let files = storage::list(&prefix).await?; - let files_num = files.len(); - if files.is_empty() { - return Ok(HashMap::default()); - } - log::info!("Load file_list_deleted gets {} files", files_num); - - let cfg = config::get_config(); - let mut tasks = Vec::with_capacity(cfg.limit.query_thread_num + 1); - let chunk_size = std::cmp::max(1, files_num / cfg.limit.query_thread_num); - for chunk in files.chunks(chunk_size) { - let chunk = chunk.to_vec(); - let task: tokio::task::JoinHandle>, anyhow::Error>> = - tokio::task::spawn(async move { - let mut files = HashMap::with_capacity(chunk.len()); - for file in chunk { - match process_file(&file).await { - Err(err) => { - log::error!("Error processing file: {:?} {:?}", file, err); - continue; - } - Ok(records) => { - if records.is_empty() { - continue; - } - let entry = files.entry(file).or_insert_with(Vec::new); - entry.extend(records); - } - } - tokio::task::yield_now().await; - } - Ok(files) - }); - tasks.push(task); - } - - let mut files = HashMap::with_capacity(files_num); - let task_results = try_join_all(tasks).await?; - for task_result in task_results { - for (file, records) in task_result? { - let entry = files.entry(file).or_insert_with(Vec::new); - entry.extend(records); - } - } - Ok(files) -} - -async fn process_file(file: &str) -> Result, anyhow::Error> { - // download file list from storage - let data = match storage::get(file).await { - Ok(data) => data, - Err(_) => { - return Ok(Vec::new()); - } - }; - - // uncompress file - let uncompress = zstd::decode_all(data.reader())?; - let uncompress_reader = BufReader::new(uncompress.reader()); - // parse file list - let mut records = Vec::with_capacity(16); - for line in uncompress_reader.lines() { - let line = line?; - if line.is_empty() { - continue; - } - records.push((line, false)); - } - - Ok(records) -} diff --git a/src/service/compact/merge.rs b/src/service/compact/merge.rs index beb552bcdb..d8e1a78ac2 100644 --- a/src/service/compact/merge.rs +++ b/src/service/compact/merge.rs @@ -13,7 +13,7 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . -use std::{io::Write, sync::Arc}; +use std::sync::Arc; use ::datafusion::{arrow::datatypes::Schema, error::DataFusionError}; use arrow::array::RecordBatch; @@ -26,14 +26,14 @@ use config::{ meta::{ inverted_index::InvertedIndexFormat, search::StorageType, - stream::{FileKey, FileMeta, MergeStrategy, PartitionTimeLevel, StreamStats, StreamType}, + stream::{ + FileKey, FileListDeleted, FileMeta, MergeStrategy, PartitionTimeLevel, StreamStats, + StreamType, + }, }, metrics, utils::{ - json, - parquet::{ - get_recordbatch_reader_from_bytes, parse_file_key_columns, read_schema_from_bytes, - }, + parquet::{get_recordbatch_reader_from_bytes, read_schema_from_bytes}, record_batch_ext::concat_batches, schema_ext::SchemaExt, time::hour_micros, @@ -58,7 +58,7 @@ use tokio::{ use crate::{ common::infra::cluster::get_node_by_uuid, - job::files::parquet::{generate_fst_inverted_index, generate_index_on_compactor}, + job::files::parquet::{create_tantivy_index, generate_index_on_compactor}, service::{ db, file_list, schema::generate_schema_for_defined_schema_fields, search::datafusion::exec, stream, @@ -663,6 +663,7 @@ pub async fn merge_files( original_size: new_file_size, compressed_size: 0, flattened: false, + index_size: 0, }; if new_file_meta.records == 0 { return Err(anyhow::anyhow!("merge_files error: records is 0")); @@ -700,7 +701,7 @@ pub async fn merge_files( for file in new_file_list.iter() { fi += 1; log::info!("[COMPACT:{thread_id}:{fi}] merge small file: {}", &file.key); - let buf = file_data::get(&file.key, None).await?; + let buf = file_data::get("", &file.key, None).await?; let schema = read_schema_from_bytes(&buf).await?; let schema = schema.as_ref().clone().with_metadata(Default::default()); let schema_key = schema.hash_key(); @@ -820,7 +821,8 @@ pub async fn merge_files( .await .map_err(|e| { anyhow::anyhow!( - "generate_index_on_compactor error: {}, need delete files: {:?}", + "generate_index_on_compactor for file: {}, error: {}, need delete files: {:?}", + new_file_key, e, retain_file_list ) @@ -855,22 +857,28 @@ pub async fn merge_files( } } - // generate fst format inverted index if matches!( index_format, - InvertedIndexFormat::FST | InvertedIndexFormat::Both + InvertedIndexFormat::Tantivy | InvertedIndexFormat::Both ) { let (schema, mut reader) = get_recordbatch_reader_from_bytes(&buf).await?; - // generate fst inverted index and write to storage - generate_fst_inverted_index( + let index_size = create_tantivy_index( + "COMPACTOR", &new_file_key, &full_text_search_fields, &index_fields, - Some(&retain_file_list), schema, &mut reader, ) - .await?; + .await.map_err(|e| { + anyhow::anyhow!( + "create_tantivy_index_on_compactor for file: {}, error: {}, need delete files: {:?}", + new_file_key, + e, + retain_file_list + ) + })?; + new_file_meta.index_size = index_size as i64; } Ok((new_file_key, new_file_meta, retain_file_list)) @@ -880,59 +888,28 @@ async fn write_file_list(org_id: &str, events: &[FileKey]) -> Result<(), anyhow: if events.is_empty() { return Ok(()); } - if get_config().common.meta_store_external { - write_file_list_db_only(org_id, events).await - } else { - write_file_list_s3(org_id, events).await - } -} -async fn write_file_list_db_only(org_id: &str, events: &[FileKey]) -> Result<(), anyhow::Error> { let put_items = events .iter() .filter(|v| !v.deleted) .map(|v| v.to_owned()) .collect::>(); - let del_items_need_flatten = events + let del_items = events .iter() - .filter(|v| v.deleted && v.meta.flattened) - .map(|v| v.key.clone()) - .collect::>(); - let del_items_noneed_flatten = events - .iter() - .filter(|v| v.deleted && !v.meta.flattened) - .map(|v| v.key.clone()) + .filter(|v| v.deleted) + .map(|v| FileListDeleted { + file: v.key.clone(), + index_file: v.meta.index_size > 0, + flattened: v.meta.flattened, + }) .collect::>(); // set to external db // retry 5 times let mut success = false; let created_at = config::utils::time::now_micros(); for _ in 0..5 { - if !del_items_need_flatten.is_empty() { - if let Err(e) = infra_file_list::batch_add_deleted( - org_id, - true, - created_at, - &del_items_need_flatten, - ) - .await - { - log::error!( - "[COMPACT] batch_add_deleted to external db failed, retrying: {}", - e - ); - tokio::time::sleep(tokio::time::Duration::from_secs(1)).await; - continue; - } - } - if !del_items_noneed_flatten.is_empty() { - if let Err(e) = infra_file_list::batch_add_deleted( - org_id, - false, - created_at, - &del_items_noneed_flatten, - ) - .await + if !del_items.is_empty() { + if let Err(e) = infra_file_list::batch_add_deleted(org_id, created_at, &del_items).await { log::error!( "[COMPACT] batch_add_deleted to external db failed, retrying: {}", @@ -947,18 +924,9 @@ async fn write_file_list_db_only(org_id: &str, events: &[FileKey]) -> Result<(), tokio::time::sleep(tokio::time::Duration::from_secs(1)).await; continue; } - if !del_items_need_flatten.is_empty() { - if let Err(e) = infra_file_list::batch_remove(&del_items_need_flatten).await { - log::error!( - "[COMPACT] batch_delete to external db failed, retrying: {}", - e - ); - tokio::time::sleep(tokio::time::Duration::from_secs(1)).await; - continue; - } - } - if !del_items_noneed_flatten.is_empty() { - if let Err(e) = infra_file_list::batch_remove(&del_items_noneed_flatten).await { + if !del_items.is_empty() { + let del_files = del_items.iter().map(|v| v.file.clone()).collect::>(); + if let Err(e) = infra_file_list::batch_remove(&del_files).await { log::error!( "[COMPACT] batch_delete to external db failed, retrying: {}", e @@ -984,76 +952,6 @@ async fn write_file_list_db_only(org_id: &str, events: &[FileKey]) -> Result<(), } } -async fn write_file_list_s3(org_id: &str, events: &[FileKey]) -> Result<(), anyhow::Error> { - // write new data into file_list - let (_stream_key, date_key, _file_name) = parse_file_key_columns(&events.first().unwrap().key)?; - // upload the new file_list to storage - let new_file_list_key = format!("file_list/{}/{}.json.zst", date_key, ider::generate()); - let mut buf = zstd::Encoder::new(Vec::new(), 3)?; - for file in events.iter() { - let mut write_buf = json::to_vec(&file)?; - write_buf.push(b'\n'); - buf.write_all(&write_buf)?; - } - let compressed_bytes = buf.finish().unwrap(); - storage::put(&new_file_list_key, compressed_bytes.into()).await?; - - // write deleted files into file_list_deleted - let del_items = events - .iter() - .filter(|v| v.deleted) - .map(|v| v.key.clone()) - .collect::>(); - if !del_items.is_empty() { - let deleted_file_list_key = format!( - "file_list_deleted/{org_id}/{}/{}.json.zst", - Utc::now().format("%Y/%m/%d/%H"), - ider::generate() - ); - let mut buf = zstd::Encoder::new(Vec::new(), 3)?; - for file in del_items.iter() { - buf.write_all((file.to_owned() + "\n").as_bytes())?; - } - let compressed_bytes = buf.finish().unwrap(); - storage::put(&deleted_file_list_key, compressed_bytes.into()).await?; - } - - // set to local cache & send broadcast - // retry 5 times - for _ in 0..5 { - // set to local cache - let mut cache_success = true; - for event in events.iter() { - if let Err(e) = - db::file_list::progress(&event.key, Some(&event.meta), event.deleted).await - { - cache_success = false; - log::error!( - "[COMPACT] set local cache for file_list failed, retrying: {}", - e - ); - tokio::time::sleep(tokio::time::Duration::from_secs(1)).await; - break; - } - } - if !cache_success { - continue; - } - // send broadcast to other nodes - if let Err(e) = db::file_list::broadcast::send(events, None).await { - log::error!( - "[COMPACT] send broadcast for file_list failed, retrying: {}", - e - ); - tokio::time::sleep(tokio::time::Duration::from_secs(1)).await; - continue; - } - // broadcast success - break; - } - Ok(()) -} - pub fn generate_inverted_idx_recordbatch( schema: Arc, batches: &[RecordBatch], diff --git a/src/service/compact/mod.rs b/src/service/compact/mod.rs index 0d7b2acb45..7a5c525a50 100644 --- a/src/service/compact/mod.rs +++ b/src/service/compact/mod.rs @@ -30,8 +30,7 @@ use tokio::sync::{mpsc, Semaphore}; use crate::{common::infra::cluster::get_node_from_consistent_hash, service::db}; -mod file_list; -pub mod file_list_deleted; +pub mod deleted; pub mod flatten; pub mod merge; pub mod retention; @@ -364,13 +363,6 @@ pub async fn run_merge( task.await?; } - // after compact, compact file list from storage - if !cfg.common.meta_store_external { - if let Err(e) = file_list::run(min_offset).await { - log::error!("[COMPACTOR] merge file list error: {}", e); - } - } - Ok(()) } @@ -406,7 +398,7 @@ pub async fn run_delay_deletion() -> Result<(), anyhow::Error> { let (offset, _) = db::compact::organization::get_offset(&org_id, "file_list_deleted").await; let batch_size = 10000; loop { - match file_list_deleted::delete(&org_id, offset, time_max, batch_size).await { + match deleted::delete(&org_id, offset, time_max, batch_size).await { Ok(affected) => { if affected == 0 { break; diff --git a/src/service/compact/retention.rs b/src/service/compact/retention.rs index ae96b66657..f524d43732 100644 --- a/src/service/compact/retention.rs +++ b/src/service/compact/retention.rs @@ -13,19 +13,18 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . -use std::{ - collections::{HashMap, HashSet}, - io::Write, -}; +use std::collections::HashMap; use chrono::{DateTime, Duration, TimeZone, Utc}; use config::{ cluster::LOCAL_NODE, - get_config, ider, is_local_disk_storage, - meta::stream::{FileKey, FileMeta, PartitionTimeLevel, StreamStats, StreamType}, - utils::{json, time::BASE_TIME}, + get_config, is_local_disk_storage, + meta::stream::{ + FileKey, FileListDeleted, FileMeta, PartitionTimeLevel, StreamStats, StreamType, + }, + utils::time::BASE_TIME, }; -use infra::{cache, dist_lock, file_list as infra_file_list, storage}; +use infra::{cache, dist_lock, file_list as infra_file_list}; use crate::{ common::infra::cluster::get_node_by_uuid, @@ -338,14 +337,11 @@ async fn delete_from_file_list( // collect stream stats let mut stream_stats = StreamStats::default(); - let mut file_list_days: HashSet = HashSet::new(); let mut hours_files: HashMap> = HashMap::with_capacity(24); for file in files { stream_stats = stream_stats - file.meta; let file_name = file.key.clone(); let columns: Vec<_> = file_name.split('/').collect(); - let day_key = format!("{}-{}-{}", columns[4], columns[5], columns[6]); - file_list_days.insert(day_key); let hour_key = format!( "{}/{}/{}/{}", columns[4], columns[5], columns[6], columns[7] @@ -360,7 +356,7 @@ async fn delete_from_file_list( } // write file list to storage - write_file_list(org_id, file_list_days, hours_files).await?; + write_file_list(org_id, hours_files).await?; // update stream stats if stream_stats.doc_num != 0 { @@ -378,18 +374,6 @@ async fn delete_from_file_list( } async fn write_file_list( - org_id: &str, - file_list_days: HashSet, - hours_files: HashMap>, -) -> Result<(), anyhow::Error> { - if get_config().common.meta_store_external { - write_file_list_db_only(org_id, hours_files).await - } else { - write_file_list_s3(org_id, file_list_days, hours_files).await - } -} - -async fn write_file_list_db_only( org_id: &str, hours_files: HashMap>, ) -> Result<(), anyhow::Error> { @@ -402,15 +386,18 @@ async fn write_file_list_db_only( let del_items = events .iter() .filter(|v| v.deleted) - .map(|v| v.key.clone()) + .map(|v| FileListDeleted { + file: v.key.clone(), + index_file: v.meta.index_size > 0, + flattened: v.meta.flattened, + }) .collect::>(); // set to external db // retry 5 times let mut success = false; let created_at = Utc::now().timestamp_micros(); for _ in 0..5 { - if let Err(e) = - infra_file_list::batch_add_deleted(org_id, false, created_at, &del_items).await + if let Err(e) = infra_file_list::batch_add_deleted(org_id, created_at, &del_items).await { log::error!( "[COMPACT] batch_add_deleted to external db failed, retrying: {}", @@ -424,13 +411,16 @@ async fn write_file_list_db_only( tokio::time::sleep(tokio::time::Duration::from_secs(1)).await; continue; } - if let Err(e) = infra_file_list::batch_remove(&del_items).await { - log::error!( - "[COMPACT] batch_delete to external db failed, retrying: {}", - e - ); - tokio::time::sleep(tokio::time::Duration::from_secs(1)).await; - continue; + if !del_items.is_empty() { + let del_files = del_items.iter().map(|v| v.file.clone()).collect::>(); + if let Err(e) = infra_file_list::batch_remove(&del_files).await { + log::error!( + "[COMPACT] batch_delete to external db failed, retrying: {}", + e + ); + tokio::time::sleep(tokio::time::Duration::from_secs(1)).await; + continue; + } } success = true; break; @@ -444,84 +434,6 @@ async fn write_file_list_db_only( Ok(()) } -async fn write_file_list_s3( - org_id: &str, - file_list_days: HashSet, - hours_files: HashMap>, -) -> Result<(), anyhow::Error> { - for (key, events) in hours_files { - // upload the new file_list to storage - let new_file_list_key = format!("file_list/{key}/{}.json.zst", ider::generate()); - let mut buf = zstd::Encoder::new(Vec::new(), 3)?; - for file in events.iter() { - let mut write_buf = json::to_vec(&file)?; - write_buf.push(b'\n'); - buf.write_all(&write_buf)?; - } - let compressed_bytes = buf.finish().unwrap(); - storage::put(&new_file_list_key, compressed_bytes.into()).await?; - - // write deleted files into file_list_deleted - let del_items = events - .iter() - .filter(|v| v.deleted) - .map(|v| v.key.clone()) - .collect::>(); - if !del_items.is_empty() { - let deleted_file_list_key = format!( - "file_list_deleted/{org_id}/{}/{}.json.zst", - Utc::now().format("%Y/%m/%d/%H"), - ider::generate() - ); - let mut buf = zstd::Encoder::new(Vec::new(), 3)?; - for file in del_items.iter() { - buf.write_all((file.to_owned() + "\n").as_bytes())?; - } - let compressed_bytes = buf.finish().unwrap(); - storage::put(&deleted_file_list_key, compressed_bytes.into()).await?; - } - - // set to local cache & send broadcast - // retry 5 times - for _ in 0..5 { - // set to local cache - let mut cache_success = true; - for event in &events { - if let Err(e) = - db::file_list::progress(&event.key, Some(&event.meta), event.deleted).await - { - cache_success = false; - log::error!( - "[COMPACT] delete_from_file_list set local cache failed, retrying: {}", - e - ); - tokio::time::sleep(tokio::time::Duration::from_secs(1)).await; - break; - } - } - if !cache_success { - continue; - } - // send broadcast to other nodes - if let Err(e) = db::file_list::broadcast::send(&events, None).await { - log::error!( - "[COMPACT] delete_from_file_list send broadcast failed, retrying: {}", - e - ); - tokio::time::sleep(tokio::time::Duration::from_secs(1)).await; - continue; - } - break; - } - } - - // mark file list need to do merge again - for key in file_list_days { - db::compact::file_list::set_delete(&key).await?; - } - Ok(()) -} - #[cfg(test)] mod tests { use super::*; diff --git a/src/service/db/enrichment_table.rs b/src/service/db/enrichment_table.rs index 3b282bf2c7..7bd564e42e 100644 --- a/src/service/db/enrichment_table.rs +++ b/src/service/db/enrichment_table.rs @@ -52,7 +52,6 @@ pub async fn get(org_id: &str, name: &str) -> Result, any timeout: 0, search_type: None, search_event_context: None, - index_type: "".to_string(), }; // do search match SearchService::search("", org_id, StreamType::EnrichmentTables, None, &req).await { diff --git a/src/service/metrics/prom.rs b/src/service/metrics/prom.rs index cf71b7fc1d..908a3ae3c4 100644 --- a/src/service/metrics/prom.rs +++ b/src/service/metrics/prom.rs @@ -738,7 +738,6 @@ pub(crate) async fn get_series( timeout: 0, search_type: None, search_event_context: None, - index_type: "".to_string(), }; let series = match search_service::search("", org_id, StreamType::Metrics, None, &req).await { Err(err) => { @@ -884,7 +883,6 @@ pub(crate) async fn get_label_values( timeout: 0, search_type: None, search_event_context: None, - index_type: "".to_string(), }; let mut label_values = match search_service::search("", org_id, stream_type, None, &req).await { Ok(resp) => resp diff --git a/src/service/promql/value.rs b/src/service/promql/value.rs index b7987404d7..9348b1a0e0 100644 --- a/src/service/promql/value.rs +++ b/src/service/promql/value.rs @@ -677,7 +677,7 @@ mod tests { assert!(approx_eq!(f64, increase, rate * 60.0)); let delta = extrapolate(&samples, ExtrapolationKind::Delta); - assert!(approx_eq!(f64, dbg!(delta), 4.0)); + assert!(approx_eq!(f64, delta, 4.0)); } #[test] diff --git a/src/service/search/cluster/flight.rs b/src/service/search/cluster/flight.rs index 43253c97e6..5f9b29d5f1 100644 --- a/src/service/search/cluster/flight.rs +++ b/src/service/search/cluster/flight.rs @@ -116,10 +116,10 @@ pub async fn search( }; // 2. get inverted index file list - let (use_fst_inverted_index, idx_file_list, idx_scan_size, idx_took) = + let (use_ttv_inverted_index, idx_file_list, idx_scan_size, idx_took) = get_inverted_index_file_lists(trace_id, &req, &sql, &query).await?; scan_stats.idx_scan_size = idx_scan_size as i64; - req.set_use_inverted_index(use_fst_inverted_index); + req.set_use_inverted_index(use_ttv_inverted_index); // 3. get nodes let node_group = req @@ -354,7 +354,7 @@ pub async fn run_datafusion( // check inverted index prefix search if sql.stream_type == StreamType::Index - && cfg.common.inverted_index_search_format.to_lowercase() != "contains" + && cfg.common.full_text_search_type.to_lowercase() != "contains" { for (stream, items) in sql.prefix_items.iter() { equal_keys @@ -773,25 +773,17 @@ async fn get_inverted_index_file_lists( query: &SearchQuery, ) -> Result<(bool, Vec, usize, usize)> { let cfg = get_config(); - let inverted_index_type = if req.inverted_index_type.is_none() - || req.inverted_index_type.as_ref().unwrap().is_empty() - { - cfg.common.inverted_index_search_format.clone() - } else { - req.inverted_index_type.as_ref().unwrap().to_string() - }; + let inverted_index_type = cfg.common.inverted_index_search_format.clone(); let (use_inverted_index, index_terms) = super::super::is_use_inverted_index(sql); - let use_parquet_inverted_index = - use_inverted_index && (inverted_index_type == "parquet" || inverted_index_type == "both"); - let use_fst_inverted_index = - use_inverted_index && (inverted_index_type == "fst" || inverted_index_type == "both"); + let use_parquet_inverted_index = use_inverted_index && inverted_index_type == "parquet"; + let use_ttv_inverted_index = use_inverted_index && inverted_index_type == "tantivy"; log::info!( "[trace_id {trace_id}] flight->search: use_inverted_index with parquet format {}", use_parquet_inverted_index ); if !use_parquet_inverted_index { - return Ok((use_fst_inverted_index, vec![], 0, 0)); + return Ok((use_ttv_inverted_index, vec![], 0, 0)); } let stream_name = sql.stream_names.first().unwrap(); @@ -814,7 +806,7 @@ async fn get_inverted_index_file_lists( ); Ok(( - use_fst_inverted_index, + use_ttv_inverted_index, idx_file_list, idx_scan_size, idx_took, diff --git a/src/service/search/datafusion/distributed_plan/remote_scan.rs b/src/service/search/datafusion/distributed_plan/remote_scan.rs index fae8c5550c..98367ff48a 100644 --- a/src/service/search/datafusion/distributed_plan/remote_scan.rs +++ b/src/service/search/datafusion/distributed_plan/remote_scan.rs @@ -255,7 +255,6 @@ async fn get_remote_batch( user_id: req.user_id.clone(), search_event_type: req.search_event_type, use_inverted_index: req.use_inverted_index, - index_type: req.inverted_index_type.clone(), }; log::info!( diff --git a/src/service/search/datafusion/table_provider/helpers.rs b/src/service/search/datafusion/table_provider/helpers.rs index f7009ffd1e..3d71e628b6 100644 --- a/src/service/search/datafusion/table_provider/helpers.rs +++ b/src/service/search/datafusion/table_provider/helpers.rs @@ -164,13 +164,22 @@ pub fn split_files( } pub fn generate_access_plan(file: &PartitionedFile) -> Option> { + let index_segment_length = if config::get_config() + .common + .inverted_index_search_format + .eq("tantivy") + { + 1 + } else { + INDEX_SEGMENT_LENGTH + }; let segment_ids = storage::file_list::get_segment_ids(file.path().as_ref())?; let stats = file.statistics.as_ref()?; let Precision::Exact(num_rows) = stats.num_rows else { return None; }; let row_group_count = (num_rows + PARQUET_MAX_ROW_GROUP_SIZE - 1) / PARQUET_MAX_ROW_GROUP_SIZE; - let segment_count = (num_rows + INDEX_SEGMENT_LENGTH - 1) / INDEX_SEGMENT_LENGTH; + let segment_count = (num_rows + index_segment_length - 1) / index_segment_length; let mut access_plan = ParquetAccessPlan::new_none(row_group_count); let mut selection = Vec::with_capacity(segment_ids.len()); let mut last_group_id = 0; @@ -178,7 +187,7 @@ pub fn generate_access_plan(file: &PartitionedFile) -> Option= segment_count { break; } - let row_group_id = (segment_id * INDEX_SEGMENT_LENGTH) / PARQUET_MAX_ROW_GROUP_SIZE; + let row_group_id = (segment_id * index_segment_length) / PARQUET_MAX_ROW_GROUP_SIZE; if row_group_id != last_group_id && !selection.is_empty() { if selection.iter().any(|s: &RowSelector| !s.skip) { access_plan.scan(last_group_id); @@ -187,10 +196,10 @@ pub fn generate_access_plan(file: &PartitionedFile) -> Option num_rows { - num_rows % INDEX_SEGMENT_LENGTH + let length = if (segment_id + 1) * index_segment_length > num_rows { + num_rows % index_segment_length } else { - INDEX_SEGMENT_LENGTH + index_segment_length }; if *val { selection.push(RowSelector::select(length)); diff --git a/src/service/search/grpc/flight.rs b/src/service/search/grpc/flight.rs index faf666b873..08ba7ea19f 100644 --- a/src/service/search/grpc/flight.rs +++ b/src/service/search/grpc/flight.rs @@ -138,7 +138,6 @@ pub async fn search( time_range: Some((req.start_time, req.end_time)), work_group: work_group.clone(), use_inverted_index: req.use_inverted_index, - inverted_index_type: req.index_type.clone(), }); // get all tables diff --git a/src/service/search/grpc/mod.rs b/src/service/search/grpc/mod.rs index cda7568fd4..16ba921a0c 100644 --- a/src/service/search/grpc/mod.rs +++ b/src/service/search/grpc/mod.rs @@ -36,7 +36,6 @@ pub struct QueryParams { pub time_range: Option<(i64, i64)>, pub work_group: Option, pub use_inverted_index: bool, - pub inverted_index_type: Option, } fn check_memory_circuit_breaker(trace_id: &str, scan_stats: &ScanStats) -> Result<()> { diff --git a/src/service/search/grpc/storage.rs b/src/service/search/grpc/storage.rs index bd5a5d8610..1151b1fb52 100644 --- a/src/service/search/grpc/storage.rs +++ b/src/service/search/grpc/storage.rs @@ -13,33 +13,34 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . -use std::sync::Arc; +use std::{cmp, sync::Arc}; +use anyhow::Context; use arrow_schema::Schema; use config::{ get_config, is_local_disk_storage, meta::{ bitvec::BitVec, - inverted_index::search::{ExactSearch, PrefixSearch, SubstringSearch}, + puffin_directory::reader::RamDirectoryReader, search::{ScanStats, StorageType}, stream::FileKey, }, - utils::inverted_index::{ - convert_parquet_idx_file_name, create_index_reader_from_puffin_bytes, split_token, - }, - FILE_EXT_PARQUET, INDEX_FIELD_NAME_FOR_ALL, + utils::inverted_index::convert_parquet_idx_file_name_to_tantivy_file, }; use datafusion::execution::cache::cache_manager::FileStatisticsCache; -use futures::future::try_join_all; -use hashbrown::HashMap; +use futures::{future::try_join_all, io::Cursor}; +use hashbrown::{HashMap, HashSet}; use infra::{ cache::file_data, errors::{Error, ErrorCodes}, schema::get_stream_setting_index_fields, - storage, }; use itertools::Itertools; use proto::cluster_rpc::KvItem; +use tantivy::{ + query::{PhrasePrefixQuery, Query, QueryParser, RegexQuery}, + Directory, Term, +}; use tokio::sync::Semaphore; use tracing::Instrument; @@ -110,24 +111,19 @@ pub async fn search( // check inverted index let cfg = get_config(); - let inverted_index_type = if query.inverted_index_type.is_none() - || query.inverted_index_type.as_ref().unwrap().is_empty() - { - cfg.common.inverted_index_search_format.clone() - } else { - query.inverted_index_type.as_ref().unwrap().to_string() - }; - let use_inverted_index = - query.use_inverted_index && (inverted_index_type == "fst" || inverted_index_type == "both"); - log::info!( - "[trace_id {}] flight->search: use_inverted_index with fst format {}", - query.trace_id, - use_inverted_index - ); + let inverted_index_type = cfg.common.inverted_index_search_format.clone(); + let use_inverted_index = query.use_inverted_index && inverted_index_type == "tantivy"; + if use_inverted_index { + log::info!( + "[trace_id {}] flight->search: use_inverted_index with tantivy format {}", + query.trace_id, + use_inverted_index + ); + } let mut idx_took = 0; if use_inverted_index { - idx_took = filter_file_list_by_inverted_index( + idx_took = filter_file_list_by_tantivy_index( query.clone(), &mut files, req_equal_terms, @@ -135,11 +131,12 @@ pub async fn search( ) .await?; log::info!( - "[trace_id {}] search->storage: stream {}/{}/{}, FST inverted index reduced file_list num to {} in {} ms", + "[trace_id {}] search->storage: stream {}/{}/{}, {} inverted index reduced file_list num to {} in {} ms", query.trace_id, query.org_id, query.stream_type, query.stream_name, + inverted_index_type, files.len(), idx_took ); @@ -212,6 +209,7 @@ pub async fn search( } // load files to local cache + let cache_start = std::time::Instant::now(); let (cache_type, deleted_files, (mem_cached_files, disk_cached_files)) = cache_files( &query.trace_id, &files.iter().map(|f| f.key.as_ref()).collect_vec(), @@ -231,7 +229,7 @@ pub async fn search( scan_stats.querier_memory_cached_files = mem_cached_files as i64; scan_stats.querier_disk_cached_files = disk_cached_files as i64; log::info!( - "[trace_id {}] search->storage: stream {}/{}/{}, load files {}, memory cached {}, disk cached {}, download others into {:?} cache done", + "[trace_id {}] search->storage: stream {}/{}/{}, load files {}, memory cached {}, disk cached {}, download others into {:?} cache done, took: {} ms", query.trace_id, query.org_id, query.stream_type, @@ -240,6 +238,7 @@ pub async fn search( scan_stats.querier_memory_cached_files, scan_stats.querier_disk_cached_files, cache_type, + cache_start.elapsed().as_millis() ); // set target partitions based on cache type @@ -327,8 +326,8 @@ async fn cache_files( let trace_id = trace_id.to_string(); let file_name = file.to_string(); let permit = semaphore.clone().acquire_owned().await.unwrap(); - let task: tokio::task::JoinHandle<(Option, bool, bool)> = tokio::task::spawn( - async move { + let task: tokio::task::JoinHandle<(Option, bool, bool)> = + tokio::task::spawn(async move { let cfg = get_config(); let ret = match cache_type { file_data::CacheType::Memory => { @@ -363,43 +362,19 @@ async fn cache_files( } _ => (None, false, false), }; - // In case where the parquet file is not found or has no data, we assume that it - // must have been deleted by some external entity, and hence we - // should remove the entry from file_list table. - // - // Caching Index files is a little different from caching parquet filesd as index - // files are not present in the file_list table as of now - // (TODO: part of phase 2). + // return file_name if download failed let file_name = if let Some(e) = ret.0 { - if (e.to_string().to_lowercase().contains("not found") - || e.to_string().to_lowercase().contains("data size is zero")) - // only proceed if the file_name has parquet extension - // FIXME: Revisit, after phase 2 of FST Index - && file_name.ends_with(FILE_EXT_PARQUET) - { - // delete file from file list - log::warn!("found invalid file: {}", file_name); - if let Err(e) = file_list::delete_parquet_file(&file_name, true).await { - log::error!( - "[trace_id {trace_id}] search->storage: delete from file_list err: {}", - e - ); - } - Some(file_name) - } else { - log::warn!( - "[trace_id {trace_id}] search->storage: download file to cache err: {}", - e - ); - None - } + log::warn!( + "[trace_id {trace_id}] search->storage: download file to cache err: {}", + e + ); + Some(file_name) } else { None }; drop(permit); (file_name, ret.1, ret.2) - }, - ); + }); tasks.push(task); } @@ -438,13 +413,14 @@ async fn cache_files( /// If the query does not match any FST in the index file, the file will be filtered out. /// If the query does match then the segment IDs for the file will be updated. /// If the query not find corresponding index file, the file will *not* be filtered out. -async fn filter_file_list_by_inverted_index( +async fn filter_file_list_by_tantivy_index( query: Arc, file_list: &mut Vec, equal_terms: &[KvItem], match_terms: &[String], ) -> Result { let start = std::time::Instant::now(); + let cfg = get_config(); // construct partition filters let equal_terms: Vec<(String, String)> = equal_terms @@ -459,20 +435,27 @@ async fn filter_file_list_by_inverted_index( let index_terms = generate_filter_from_equal_items(&index_terms_orig); // Cache the corresponding Index files - let cfg = get_config(); let mut scan_stats = ScanStats::new(); - let mut file_list_map = file_list.drain(..).into_group_map_by(|f| f.key.clone()); + let mut file_list_map = file_list + .drain(..) + .map(|f| (f.key.clone(), f)) + .collect::>(); let index_file_names = file_list_map - .keys() - .filter_map(|f| convert_parquet_idx_file_name(f)) + .iter() + .filter_map(|(_, f)| { + if f.meta.index_size > 0 { + convert_parquet_idx_file_name_to_tantivy_file(&f.key).map(|v| (f.key.clone(), v)) + } else { + None + } + }) .collect_vec(); let (cache_type, _, (mem_cached_files, disk_cached_files)) = cache_files( &query.trace_id, - index_file_names + &index_file_names .iter() - .map(|f| f.as_str()) - .collect_vec() - .as_ref(), + .map(|(_parquet_file, ttv_file)| ttv_file.as_str()) + .collect_vec(), &scan_stats, ) .await?; @@ -480,7 +463,7 @@ async fn filter_file_list_by_inverted_index( scan_stats.querier_memory_cached_files = mem_cached_files as i64; scan_stats.querier_disk_cached_files = disk_cached_files as i64; log::info!( - "[trace_id {}] search->storage: stream {}/{}/{}, load puffin index files {}, memory cached {}, disk cached {}, download others into {:?} cache done", + "[trace_id {}] search->storage: stream {}/{}/{}, load puffin index files {}, memory cached {}, disk cached {}, download others into {:?} cache done, took: {} ms", query.trace_id, query.org_id, query.stream_type, @@ -489,56 +472,39 @@ async fn filter_file_list_by_inverted_index( scan_stats.querier_memory_cached_files, scan_stats.querier_disk_cached_files, cache_type, + start.elapsed().as_millis() ); - let full_text_terms = Arc::new( - match_terms - .iter() - .filter_map(|t| { - let tokens = split_token(t, &cfg.common.inverted_index_split_chars); - // If tokens empty return None so that full_text_terms will not have empty strings - if tokens.is_empty() { - return None; - } - Some( - tokens - .into_iter() - .max_by_key(|key| key.len()) - .unwrap_or_default(), - ) - }) - .collect_vec(), - ); + let full_text_terms = Arc::new(match_terms.to_vec()); let index_terms = Arc::new(index_terms); - let semaphore = std::sync::Arc::new(Semaphore::new(cfg.limit.query_thread_num)); + + let semaphore = std::sync::Arc::new(Semaphore::new(cfg.limit.cpu_num)); let mut tasks = Vec::new(); - for file in file_list_map.keys() { + + for (parquet_file, _ttv_file) in index_file_names { + let file = parquet_file; let full_text_term_clone = full_text_terms.clone(); let index_terms_clone = index_terms.clone(); - let file_name = file.clone(); let trace_id = query.trace_id.to_string(); let permit = semaphore.clone().acquire_owned().await.unwrap(); // Spawn a task for each file, wherein full text search and // secondary index search queries are executed let task = tokio::task::spawn(async move { - let res = inverted_index_search_in_file( - &trace_id, - &file_name, - full_text_term_clone, - index_terms_clone, - ) - .await; + let res = + search_tantivy_index(&trace_id, &file, full_text_term_clone, index_terms_clone) + .await; drop(permit); res }); - tasks.push(task) } - let results = try_join_all(tasks) + let mut total_hits = 0; + for result in try_join_all(tasks) .await - .map_err(|e| Error::ErrorCode(ErrorCodes::ServerInternalError(e.to_string())))?; - for result in results { + .map_err(|e| Error::ErrorCode(ErrorCodes::ServerInternalError(e.to_string())))? + { + let result: anyhow::Result<(String, Option)> = result; // Each result corresponds to a file in the file list match result { Ok((file_name, bitvec)) => { @@ -546,17 +512,18 @@ async fn filter_file_list_by_inverted_index( // Replace the segment IDs in the existing `FileKey` with the new found segments let file = file_list_map .get_mut(&file_name) - .unwrap() - .first_mut() // we expect each file name has atleast 1 file .unwrap(); - file.segment_ids = Some(res.clone().into_vec()); + let hits_per_file = res.count_ones(); + total_hits += hits_per_file; + file.segment_ids = Some(res.into_vec()); log::debug!( - "[trace_id {}] search->storage: Final bitmap for fts_terms {:?} and index_terms: {:?} total: {}", + "[trace_id {}] search->storage: hits for fts_terms {:?} and index_terms: {:?} found {} in {}", query.trace_id, *full_text_terms, index_terms, - res.len(), + hits_per_file, + file_name ); } else { // if the bitmap is empty then we remove the file from the list @@ -578,121 +545,151 @@ async fn filter_file_list_by_inverted_index( } } } - file_list.extend(file_list_map.into_values().flatten()); + log::info!( + "[trace_id {}] search->storage: Total hits for fts_terms {:?} and index_terms: {:?} found {}", + query.trace_id, + *full_text_terms, + index_terms, + total_hits + ); + file_list.extend(file_list_map.into_values()); Ok(start.elapsed().as_millis() as usize) } -/// Fetches the file from cache if it exists, otherwise fetch from storage -async fn fetch_file(file_name: &str) -> anyhow::Result> { - // first get from memory cache - if file_data::memory::exist(file_name).await { - return file_data::memory::get(file_name, None) - .await - .map(|bytes| bytes.to_vec()) - .ok_or(anyhow::anyhow!("memory cache get failed")); +fn search_tantivy_index_with_field( + query_parser: QueryParser, + field: tantivy::schema::Field, + tantivy_searcher: tantivy::Searcher, + fts_terms: &[String], +) -> anyhow::Result<(HashSet, u32)> { + let cfg = get_config(); + let mut max_doc_id = 0; + let mut matched_docs = HashSet::new(); + + // TODO: improve this logic + // The issue is joining the full text search terms + // Ideally there should be multiple queries for each term + // and their result should be intersected + let query: Box = match cfg.common.full_text_search_type.as_str() { + "eq" => query_parser.parse_query(&fts_terms.join(" "))?, + "contains" => Box::new(RegexQuery::from_pattern( + &format!("*{}*", &fts_terms.join(" ")), + field, + )?), + // Default to prefix search + _ => Box::new(PhrasePrefixQuery::new(vec![Term::from_field_text( + field, + // we join all the fields that are full text searchable + &fts_terms.join(" "), + )])), + }; + + let docs = tantivy_searcher.search(&query, &tantivy::collector::DocSetCollector)?; + for doc in docs { + max_doc_id = cmp::max(doc.doc_id, max_doc_id); + matched_docs.insert(doc); } - if file_data::disk::exist(file_name).await { - // check disk next - return file_data::disk::get(file_name, None) - .await - .map(|bytes| bytes.to_vec()) - .ok_or(anyhow::anyhow!("disk cache get failed")); - } - // finally get from storage - storage::get(file_name).await.map(|bytes| bytes.to_vec()) + + Ok((matched_docs, max_doc_id)) } -async fn inverted_index_search_in_file( +pub async fn get_tantivy_directory( + trace_id: &str, + file_name: &str, +) -> anyhow::Result> { + let index_file_bytes = infra::cache::file_data::get(trace_id, file_name, None).await?; + let puffin_dir = RamDirectoryReader::from_bytes(Cursor::new(index_file_bytes)).await?; + Ok(Box::new(puffin_dir)) +} + +async fn search_tantivy_index( trace_id: &str, parquet_file_name: &str, fts_terms: Arc>, index_terms: Arc)>>, ) -> anyhow::Result<(String, Option)> { - let cfg = config::get_config(); - let Some(index_file_name) = convert_parquet_idx_file_name(parquet_file_name) else { + let Some(ttv_file_name) = convert_parquet_idx_file_name_to_tantivy_file(parquet_file_name) + else { return Err(anyhow::anyhow!( - "[trace_id {trace_id}] search->storage: Unable to convert parquet file name {} to index file name", + "[trace_id {trace_id}] search->storage: Unable to find tantivy index files for parquet file {}", parquet_file_name )); }; - let compressed_index_blob = match fetch_file(&index_file_name).await { - Err(e) => { - log::warn!( - "[trace_id {trace_id}] search->storage: Unable to load corresponding FST index - file for parquet file {}, err: {}", - parquet_file_name, - e - ); - return Err(e); - } - Ok(bytes) => bytes, - }; - let mut res = BitVec::new(); - let mut index_reader = create_index_reader_from_puffin_bytes(compressed_index_blob).await?; - if let Some(mut field_reader) = index_reader.field(INDEX_FIELD_NAME_FOR_ALL).await? { - let column_index_meta = field_reader.metadata().await?; - let matched_bv = match cfg.common.full_text_search_type.as_str() { - "eq" => { - let mut searcher = ExactSearch::new(fts_terms.as_ref(), &column_index_meta); - searcher.search(&mut field_reader).await - } - "contains" => { - let mut searcher = SubstringSearch::new(fts_terms.as_ref(), &column_index_meta); - searcher.search(&mut field_reader).await - } - // Default to prefix search - _ => { - let mut searcher = PrefixSearch::new(fts_terms.as_ref(), &column_index_meta); - searcher.search(&mut field_reader).await - } + let puffin_dir = get_tantivy_directory(trace_id, &ttv_file_name).await?; + let tantivy_index = tantivy::Index::open(puffin_dir)?; + let tantivy_schema = tantivy_index.schema(); + let tantivy_reader = tantivy_index.reader()?; + let tantivy_searcher = tantivy_reader.searcher(); + + let mut matched_docs = HashSet::new(); + let mut max_doc_id = 0u32; + if !fts_terms.is_empty() { + let fts_field = tantivy_schema.get_field("_all").unwrap(); + let query_parser = QueryParser::for_index(&tantivy_index, vec![fts_field]); + let tantivy_searcher_clone = tantivy_searcher.clone(); + if let Ok((docs, max_row)) = tokio::task::spawn_blocking(move || { + search_tantivy_index_with_field( + query_parser, + fts_field, + tantivy_searcher_clone, + &fts_terms, + ) + }) + .await? + { + matched_docs.extend(docs); + max_doc_id = max_row; + } else { + return Err(anyhow::anyhow!( + "[trace_id {}] search->storage: search_full_text_index failed", + trace_id + )); }; + } + for (field_name, terms) in index_terms.iter() { + let field = tantivy_schema.get_field(field_name).unwrap(); + let query_parser = QueryParser::for_index(&tantivy_index, vec![field]); + let tantivy_searcher_clone = tantivy_searcher.clone(); + let terms = terms.clone(); - match matched_bv { - Ok(bitmap) => { - if res.len() < bitmap.len() { - res.resize(bitmap.len(), false); - } - res |= bitmap; - } - Err(e) => { - log::warn!( - "[trace_id {trace_id}] search->storage: Error loading FST map from index file {} for full text search with error {}. Keep the file", - index_file_name, - e.to_string() - ); - } - } + if let Ok((docs, max_row)) = tokio::task::spawn_blocking(move || { + search_tantivy_index_with_field(query_parser, field, tantivy_searcher_clone, &terms) + }) + .await? + { + matched_docs.extend(docs); + max_doc_id = max_row; + } else { + return Err(anyhow::anyhow!( + "[trace_id {}] search->storage: search_full_text_index failed", + trace_id + )); + }; } - if !index_terms.is_empty() { - for (col, index_terms) in index_terms.iter() { - if let Some(mut field_reader) = index_reader.field(col).await? { - let column_index_meta = field_reader.metadata().await?; - let mut secondary_index_match = ExactSearch::new(index_terms, &column_index_meta); - match secondary_index_match.search(&mut field_reader).await { - Ok(bitmap) => { - if res.len() < bitmap.len() { - res.resize(bitmap.len(), false); - } - res &= bitmap; - } - Err(e) => { - log::warn!( - "[trace_id {trace_id}] search->storage: Error loading FST map from index file {} for column {} with error {}. Keep the file", - index_file_name, - col, - e.to_string() - ); - } - } - } - } + // return early if no matches in tantivy + if matched_docs.is_empty() { + return Ok((parquet_file_name.to_string(), None)); } - Ok(if res.is_empty() { - (parquet_file_name.into(), None) // no match -> skip the file in search + // Prepare a vec of segment offsets + // this is useful when there are more than one segments + let seg_metas = tantivy_index + .searchable_segment_metas() + .context("Count segments")?; + let mut res = BitVec::with_capacity(max_doc_id as usize + 1); + // rewrite the empty bitvec + res.resize((max_doc_id + 1) as usize, false); + // Return early if there is only one segment + if seg_metas.len() == 1 { + for doc in matched_docs { + res.set(doc.doc_id as usize, true); + } + Ok((parquet_file_name.to_string(), Some(res))) } else { - (parquet_file_name.into(), Some(res)) // match -> take the file in search - }) + Err(anyhow::anyhow!( + "[trace_id {trace_id}] search->storage: Multiple segments in tantivy index not supported" + )) + } } diff --git a/src/service/search/mod.rs b/src/service/search/mod.rs index 3ed9a3d532..cc93e4a1c1 100644 --- a/src/service/search/mod.rs +++ b/src/service/search/mod.rs @@ -28,10 +28,7 @@ use config::{ usage::{RequestStats, UsageType}, }, metrics, - utils::{ - base64, json, schema::filter_source_by_partition_key, sql::is_aggregate_query, - str::StringExt, - }, + utils::{base64, json, schema::filter_source_by_partition_key, sql::is_aggregate_query}, }; use hashbrown::HashMap; use infra::{ @@ -423,7 +420,6 @@ pub async fn search( user_id.clone(), Some((query.start_time, query.end_time)), in_req.search_type.map(|v| v.to_string()), - in_req.index_type.optional(), ); let span = tracing::span::Span::current(); diff --git a/src/service/search/request.rs b/src/service/search/request.rs index ca9f65558b..87b0f6e7d8 100644 --- a/src/service/search/request.rs +++ b/src/service/search/request.rs @@ -26,7 +26,6 @@ pub struct Request { pub work_group: Option, pub time_range: Option<(i64, i64)>, pub search_event_type: Option, // node rule - pub inverted_index_type: Option, pub use_inverted_index: bool, } @@ -41,7 +40,6 @@ impl Default for Request { work_group: None, time_range: None, search_event_type: None, - inverted_index_type: None, use_inverted_index: false, } } @@ -57,7 +55,6 @@ impl Request { user_id: Option, time_range: Option<(i64, i64)>, search_event_type: Option, - inverted_index_type: Option, ) -> Self { Self { trace_id, @@ -68,7 +65,6 @@ impl Request { work_group: None, time_range, search_event_type, - inverted_index_type, use_inverted_index: false, } } @@ -92,10 +88,6 @@ impl Request { pub fn set_use_inverted_index(&mut self, use_inverted_index: bool) { self.use_inverted_index = use_inverted_index; } - - pub fn set_inverted_index_type(&mut self, index_type: Option) { - self.inverted_index_type = index_type; - } } impl From for Request { @@ -109,7 +101,6 @@ impl From for Request { work_group: request.work_group, time_range: Some((request.start_time, request.end_time)), search_event_type: request.search_event_type, - inverted_index_type: request.index_type, use_inverted_index: request.use_inverted_index, } } diff --git a/src/service/search/super_cluster/follower.rs b/src/service/search/super_cluster/follower.rs index e56c7edd1d..01ce9493c9 100644 --- a/src/service/search/super_cluster/follower.rs +++ b/src/service/search/super_cluster/follower.rs @@ -251,15 +251,8 @@ async fn get_inverted_index_file_lists( match_terms: &[String], ) -> Result<(bool, Vec, usize, usize)> { let cfg = config::get_config(); - let inverted_index_type = if req.inverted_index_type.is_none() - || req.inverted_index_type.as_ref().unwrap().is_empty() - { - cfg.common.inverted_index_search_format.clone() - } else { - req.inverted_index_type.as_ref().unwrap().to_string() - }; - let use_inverted_index = req.use_inverted_index - && (inverted_index_type == "parquet" || inverted_index_type == "both"); + let use_inverted_index = + req.use_inverted_index && cfg.common.inverted_index_search_format == "parquet"; log::info!( "[trace_id {trace_id}] flight->follower_leader: use_inverted_index with parquet format {}", use_inverted_index diff --git a/src/service/stream.rs b/src/service/stream.rs index 60a8304426..c4633ead91 100644 --- a/src/service/stream.rs +++ b/src/service/stream.rs @@ -496,6 +496,7 @@ pub async fn delete_stream( fn transform_stats(stats: &mut StreamStats) { stats.storage_size /= SIZE_IN_MB; stats.compressed_size /= SIZE_IN_MB; + stats.index_size /= SIZE_IN_MB; } pub fn stream_created(schema: &Schema) -> Option {