rust/tantivy
1
0
Fork 0
mirror of https://github.com/quickwit-oss/tantivy.git synced 2026-08-18 03:58:21 +00:00
Code Issues Packages Projects Releases Wiki Activity

set docstore cache size at construction

Browse Source
This commit is contained in:
Pascal Seitz
2022-07-04 13:54:22 +08:00
parent 9db2f0e82b
commit 5750224d4c
9 changed files with 26 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/core/searcher.rs
+1 -5
View File
@@ -81,13 +81,9 @@ impl Searcher {
) -> io::Result<Searcher> {
let store_readers: Vec<StoreReader> = segment_readers
.iter()
.map(SegmentReader::get_store_reader)
.map(|segment_reader| segment_reader.get_store_reader(doc_store_cache_size))
.collect::<io::Result<Vec<_>>>()?;
for store_reader in &store_readers {
store_reader.set_cache_size(doc_store_cache_size);
}
Ok(Searcher {
schema,
index,
src/core/segment_reader.rs
+3 -3
View File
@@ -133,8 +133,8 @@ impl SegmentReader {
}
/// Accessor to the segment's `StoreReader`.
pub fn get_store_reader(&self) -> io::Result<StoreReader> {
StoreReader::open(self.store_file.clone())
pub fn get_store_reader(&self, cache_size: usize) -> io::Result<StoreReader> {
StoreReader::open(self.store_file.clone(), cache_size)
}
/// Open a new segment for reading.
@@ -326,7 +326,7 @@ impl SegmentReader {
self.positions_composite.space_usage(),
self.fast_fields_readers.space_usage(),
self.fieldnorm_readers.space_usage(),
self.get_store_reader()?.space_usage(),
self.get_store_reader(0)?.space_usage(),
self.alive_bitset_opt
.as_ref()
.map(AliveBitSet::space_usage)
src/functional_test.rs
+1 -1
View File
@@ -9,7 +9,7 @@ fn check_index_content(searcher: &Searcher, vals: &[u64]) -> crate::Result<()> {
assert!(searcher.segment_readers().len() < 20);
assert_eq!(searcher.num_docs() as usize, vals.len());
for segment_reader in searcher.segment_readers() {
let store_reader = segment_reader.get_store_reader()?;
let store_reader = segment_reader.get_store_reader(1)?;
for doc_id in 0..segment_reader.max_doc() {
let _doc = store_reader.get(doc_id)?;
}
src/indexer/index_writer.rs
+4 -1
View File
@@ -792,6 +792,7 @@ mod tests {
self, Cardinality, Facet, FacetOptions, IndexRecordOption, NumericOptions,
TextFieldIndexing, TextOptions, FAST, INDEXED, STORED, STRING, TEXT,
};
use crate::store::DOCSTORE_CACHE_CAPACITY;
use crate::{DocAddress, Index, IndexSettings, IndexSortByField, Order, ReloadPolicy, Term};
const LOREM: &str = "Doc Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do \
@@ -1550,7 +1551,9 @@ mod tests {
// doc store tests
for segment_reader in searcher.segment_readers().iter() {
let store_reader = segment_reader.get_store_reader().unwrap();
let store_reader = segment_reader
.get_store_reader(DOCSTORE_CACHE_CAPACITY)
.unwrap();
// test store iterator
for doc in store_reader.iter(segment_reader.alive_bitset()) {
let id = doc.unwrap().get_first(id_field).unwrap().as_u64().unwrap();
src/indexer/merger.rs
+2 -7
View File
@@ -1036,13 +1036,9 @@ impl IndexMerger {
let store_readers: Vec<_> = self
.readers
.iter()
.map(|reader| reader.get_store_reader())
.map(|reader| reader.get_store_reader(50))
.collect::<Result<_, _>>()?;
for store_reader in &store_readers {
store_reader.set_cache_size(50);
}
let mut document_iterators: Vec<_> = store_readers
.iter()
.enumerate()
@@ -1065,8 +1061,7 @@ impl IndexMerger {
} else {
debug!("trivial-doc-id-mapping");
for reader in &self.readers {
let store_reader = reader.get_store_reader()?;
store_reader.set_cache_size(1);
let store_reader = reader.get_store_reader(1)?;
if reader.has_deletes()
// If there is not enough data in the store, we avoid stacking in order to
// avoid creating many small blocks in the doc store. Once we have 5 full blocks,
src/indexer/segment_writer.rs
+1
View File
@@ -389,6 +389,7 @@ fn remap_and_write(
serializer
.segment()
.open_read(SegmentComponent::TempStore)?,
50,
)?;
for old_doc_id in doc_id_map.iter_old_doc_ids() {
let doc_bytes = store_read.get_document_bytes(old_doc_id)?;
src/reader/mod.rs
+2 -2
View File
@@ -13,7 +13,7 @@ use self::pool::Pool;
use self::warming::WarmingState;
use crate::core::searcher::SearcherGeneration;
use crate::directory::{Directory, WatchCallback, WatchHandle, META_LOCK};
use crate::store::LRU_CACHE_CAPACITY;
use crate::store::DOCSTORE_CACHE_CAPACITY;
use crate::{Index, Inventory, Searcher, SegmentReader, TrackedObject};
/// Defines when a new version of the index should be reloaded.
@@ -60,7 +60,7 @@ impl IndexReaderBuilder {
index,
warmers: Vec::new(),
num_warming_threads: 1,
doc_store_cache_size: LRU_CACHE_CAPACITY,
doc_store_cache_size: DOCSTORE_CACHE_CAPACITY,
}
}
src/store/mod.rs
+8 -8
View File
@@ -40,7 +40,7 @@ mod reader;
mod writer;
pub use self::compressors::{Compressor, ZstdCompressor};
pub use self::decompressors::Decompressor;
pub(crate) use self::reader::LRU_CACHE_CAPACITY;
pub(crate) use self::reader::DOCSTORE_CACHE_CAPACITY;
pub use self::reader::{CacheStats, StoreReader};
pub use self::writer::StoreWriter;
@@ -115,7 +115,7 @@ pub mod tests {
let schema = write_lorem_ipsum_store(store_wrt, NUM_DOCS, Compressor::Lz4, BLOCK_SIZE);
let field_title = schema.get_field("title").unwrap();
let store_file = directory.open_read(path)?;
let store = StoreReader::open(store_file)?;
let store = StoreReader::open(store_file, 10)?;
for i in 0..NUM_DOCS as u32 {
assert_eq!(
*store
@@ -155,7 +155,7 @@ pub mod tests {
let schema = write_lorem_ipsum_store(store_wrt, NUM_DOCS, compressor, blocksize);
let field_title = schema.get_field("title").unwrap();
let store_file = directory.open_read(path)?;
let store = StoreReader::open(store_file)?;
let store = StoreReader::open(store_file, 10)?;
for i in 0..NUM_DOCS as u32 {
assert_eq!(
*store
@@ -232,7 +232,7 @@ pub mod tests {
let searcher = index.reader()?.searcher();
let reader = searcher.segment_reader(0);
let store = reader.get_store_reader()?;
let store = reader.get_store_reader(10)?;
for doc in store.iter(reader.alive_bitset()) {
assert_eq!(
*doc?.get_first(text_field).unwrap().as_text().unwrap(),
@@ -268,7 +268,7 @@ pub mod tests {
}
assert_eq!(
index.reader().unwrap().searcher().segment_readers()[0]
.get_store_reader()
.get_store_reader(10)
.unwrap()
.decompressor(),
Decompressor::Lz4
@@ -289,7 +289,7 @@ pub mod tests {
let searcher = index.reader().unwrap().searcher();
assert_eq!(searcher.segment_readers().len(), 1);
let reader = searcher.segment_readers().iter().last().unwrap();
let store = reader.get_store_reader().unwrap();
let store = reader.get_store_reader(10).unwrap();
for doc in store.iter(reader.alive_bitset()).take(50) {
assert_eq!(
@@ -336,7 +336,7 @@ pub mod tests {
let searcher = index.reader()?.searcher();
assert_eq!(searcher.segment_readers().len(), 1);
let reader = searcher.segment_readers().iter().last().unwrap();
let store = reader.get_store_reader()?;
let store = reader.get_store_reader(10)?;
assert_eq!(store.block_checkpoints().count(), 1);
Ok(())
}
@@ -380,7 +380,7 @@ mod bench {
16_384,
);
let store_file = directory.open_read(path).unwrap();
let store = StoreReader::open(store_file).unwrap();
let store = StoreReader::open(store_file, 10).unwrap();
b.iter(|| store.iter(None).collect::<Vec<_>>());
}
}
src/store/reader.rs
+4 -13
View File
@@ -19,7 +19,7 @@ use crate::space_usage::StoreSpaceUsage;
use crate::store::index::Checkpoint;
use crate::DocId;
pub(crate) const LRU_CACHE_CAPACITY: usize = 100;
pub(crate) const DOCSTORE_CACHE_CAPACITY: usize = 100;
type Block = OwnedBytes;
@@ -64,10 +64,6 @@ impl BlockCache {
self.cache.lock().unwrap().len()
}
fn set_size(&self, size: usize) {
self.cache.lock().unwrap().resize(size);
}
#[cfg(test)]
fn peek_lru(&self) -> Option<usize> {
self.cache
@@ -111,7 +107,7 @@ impl Sum for CacheStats {
impl StoreReader {
/// Opens a store reader
pub fn open(store_file: FileSlice) -> io::Result<StoreReader> {
pub fn open(store_file: FileSlice, cache_size: usize) -> io::Result<StoreReader> {
let (footer, data_and_offset) = DocStoreFooter::extract_footer(store_file)?;
let (data_file, offset_index_file) = data_and_offset.split(footer.offset as usize);
@@ -122,7 +118,7 @@ impl StoreReader {
decompressor: footer.decompressor,
data: data_file,
cache: BlockCache {
cache: Mutex::new(LruCache::new(LRU_CACHE_CAPACITY)),
cache: Mutex::new(LruCache::new(cache_size)),
cache_hits: Default::default(),
cache_misses: Default::default(),
},
@@ -144,11 +140,6 @@ impl StoreReader {
self.cache.stats()
}
/// Set lru cache size for decompressed blocks. Defaults to 100 (LRU_CACHE_CAPACITY).
pub(crate) fn set_cache_size(&self, size: usize) {
self.cache.set_size(size)
}
/// Get checkpoint for DocId. The checkpoint can be used to load a block containing the
/// document.
///
@@ -405,7 +396,7 @@ mod tests {
let schema = write_lorem_ipsum_store(writer, 500, Compressor::default(), BLOCK_SIZE);
let title = schema.get_field("title").unwrap();
let store_file = directory.open_read(path)?;
let store = StoreReader::open(store_file)?;
let store = StoreReader::open(store_file, DOCSTORE_CACHE_CAPACITY)?;
assert_eq!(store.cache.len(), 0);
assert_eq!(store.cache_stats().cache_hits, 0);