mirror of
https://github.com/quickwit-oss/tantivy.git
synced 2025-12-23 02:29:57 +00:00
rename merge_holes_under => merge_holes_under_bytes
This commit is contained in:
@@ -230,7 +230,7 @@ impl InvertedIndexReader {
|
||||
terms: impl std::ops::RangeBounds<Term>,
|
||||
automaton: A,
|
||||
limit: Option<u64>,
|
||||
merge_holes_under: usize,
|
||||
merge_holes_under_bytes: usize,
|
||||
) -> io::Result<impl Iterator<Item = TermInfo> + 'a>
|
||||
where
|
||||
A::State: Clone,
|
||||
@@ -254,7 +254,7 @@ impl InvertedIndexReader {
|
||||
};
|
||||
|
||||
let mut stream = range_builder
|
||||
.into_stream_async_merging_holes(merge_holes_under)
|
||||
.into_stream_async_merging_holes(merge_holes_under_bytes)
|
||||
.await?;
|
||||
|
||||
let iter = std::iter::from_fn(move || stream.next().map(|(_k, v)| v.clone()));
|
||||
@@ -345,11 +345,11 @@ impl InvertedIndexReader {
|
||||
{
|
||||
// merge holes under 4MiB, that's how many bytes we can hope to receive during a TTFB from
|
||||
// S3 (~80MiB/s, and 50ms latency)
|
||||
let merge_holes_under = (80 * 1024 * 1024 * 50) / 1000;
|
||||
let merge_holes_under_bytes = (80 * 1024 * 1024 * 50) / 1000;
|
||||
// we build a first iterator to download everything. Simply calling the function already
|
||||
// loads everything, but doesn't start iterating over the sstable.
|
||||
let mut _term_info = self
|
||||
.get_term_range_async(.., automaton.clone(), None, merge_holes_under)
|
||||
.get_term_range_async(.., automaton.clone(), None, merge_holes_under_bytes)
|
||||
.await?;
|
||||
|
||||
// we build a 2nd iterator, this one with no holes, so we don't go through blocks we can't
|
||||
@@ -364,7 +364,7 @@ impl InvertedIndexReader {
|
||||
let range_to_load = term_info
|
||||
.map(|term_info| term_info.postings_range)
|
||||
.coalesce(|range1, range2| {
|
||||
if range1.end + merge_holes_under >= range2.start {
|
||||
if range1.end + merge_holes_under_bytes >= range2.start {
|
||||
Ok(range1.start..range2.end)
|
||||
} else {
|
||||
Err((range1, range2))
|
||||
|
||||
@@ -101,7 +101,7 @@ impl<TSSTable: SSTable> Dictionary<TSSTable> {
|
||||
key_range: impl RangeBounds<[u8]>,
|
||||
limit: Option<u64>,
|
||||
automaton: &impl Automaton,
|
||||
merge_holes_under: usize,
|
||||
merge_holes_under_bytes: usize,
|
||||
) -> io::Result<DeltaReader<TSSTable::ValueReader>> {
|
||||
let match_all = automaton.will_always_match(&automaton.start());
|
||||
if match_all {
|
||||
@@ -112,7 +112,7 @@ impl<TSSTable: SSTable> Dictionary<TSSTable> {
|
||||
let blocks = stream::iter(self.get_block_iterator_for_range_and_automaton(
|
||||
key_range,
|
||||
automaton,
|
||||
merge_holes_under,
|
||||
merge_holes_under_bytes,
|
||||
));
|
||||
let data = blocks
|
||||
.map(|block_addr| {
|
||||
@@ -242,7 +242,7 @@ impl<TSSTable: SSTable> Dictionary<TSSTable> {
|
||||
&'a self,
|
||||
key_range: impl RangeBounds<[u8]>,
|
||||
automaton: &'a impl Automaton,
|
||||
merge_holes_under: usize,
|
||||
merge_holes_under_bytes: usize,
|
||||
) -> impl Iterator<Item = BlockAddr> + 'a {
|
||||
let lower_bound = match key_range.start_bound() {
|
||||
Bound::Included(key) | Bound::Excluded(key) => {
|
||||
@@ -263,7 +263,7 @@ impl<TSSTable: SSTable> Dictionary<TSSTable> {
|
||||
.filter(move |(block_id, _)| block_range.contains(block_id))
|
||||
.map(|(_, block_addr)| block_addr)
|
||||
.coalesce(move |first, second| {
|
||||
if first.byte_range.end + merge_holes_under >= second.byte_range.start {
|
||||
if first.byte_range.end + merge_holes_under_bytes >= second.byte_range.start {
|
||||
Ok(BlockAddr {
|
||||
first_ordinal: first.first_ordinal,
|
||||
byte_range: first.byte_range.start..second.byte_range.end,
|
||||
|
||||
@@ -91,7 +91,7 @@ where
|
||||
|
||||
async fn delta_reader_async(
|
||||
&self,
|
||||
merge_holes_under: usize,
|
||||
merge_holes_under_bytes: usize,
|
||||
) -> io::Result<DeltaReader<TSSTable::ValueReader>> {
|
||||
let key_range = (
|
||||
bound_as_byte_slice(&self.lower),
|
||||
@@ -102,7 +102,7 @@ where
|
||||
key_range,
|
||||
self.limit,
|
||||
&self.automaton,
|
||||
merge_holes_under,
|
||||
merge_holes_under_bytes,
|
||||
)
|
||||
.await
|
||||
}
|
||||
@@ -142,12 +142,12 @@ where
|
||||
}
|
||||
|
||||
/// Same as `into_stream_async`, but tries to issue a single io operation when requesting
|
||||
/// blocks that are not consecutive, but also less than `merge_holes_under` bytes appart.
|
||||
/// blocks that are not consecutive, but also less than `merge_holes_under_bytes` bytes appart.
|
||||
pub async fn into_stream_async_merging_holes(
|
||||
self,
|
||||
merge_holes_under: usize,
|
||||
merge_holes_under_bytes: usize,
|
||||
) -> io::Result<Streamer<'a, TSSTable, A>> {
|
||||
let delta_reader = self.delta_reader_async(merge_holes_under).await?;
|
||||
let delta_reader = self.delta_reader_async(merge_holes_under_bytes).await?;
|
||||
self.into_stream_given_delta_reader(delta_reader)
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user