mirror of
https://github.com/quickwit-oss/tantivy.git
synced 2026-10-30 07:17:07 +00:00
Minor refactoring (#1266)
This commit is contained in:
+13
-15
@@ -1,23 +1,19 @@
|
||||
mod pool;
|
||||
mod warming;
|
||||
|
||||
use std::convert::TryInto;
|
||||
use std::io;
|
||||
use std::sync::atomic::AtomicU64;
|
||||
use std::sync::{atomic, Arc, Weak};
|
||||
|
||||
pub use warming::Warmer;
|
||||
|
||||
pub use self::pool::LeasedItem;
|
||||
use self::pool::Pool;
|
||||
use self::warming::WarmingState;
|
||||
use crate::core::searcher::SearcherGeneration;
|
||||
use crate::directory::WatchHandle;
|
||||
use crate::directory::META_LOCK;
|
||||
use crate::directory::{Directory, WatchCallback};
|
||||
use crate::Index;
|
||||
use crate::Searcher;
|
||||
use crate::SegmentReader;
|
||||
use crate::{Inventory, TrackedObject};
|
||||
use std::sync::atomic;
|
||||
use std::sync::atomic::AtomicU64;
|
||||
use std::sync::Arc;
|
||||
use std::sync::Weak;
|
||||
use std::{convert::TryInto, io};
|
||||
pub use warming::Warmer;
|
||||
use crate::directory::{Directory, WatchCallback, WatchHandle, META_LOCK};
|
||||
use crate::{Index, Inventory, Searcher, SegmentReader, TrackedObject};
|
||||
|
||||
/// Defines when a new version of the index should be reloaded.
|
||||
///
|
||||
@@ -29,7 +25,8 @@ pub enum ReloadPolicy {
|
||||
/// The index is entirely reloaded manually.
|
||||
/// All updates of the index should be manual.
|
||||
///
|
||||
/// No change is reflected automatically. You are required to call `IndexReader::reload()` manually.
|
||||
/// No change is reflected automatically. You are required to call `IndexReader::reload()`
|
||||
/// manually.
|
||||
Manual,
|
||||
/// The index is reloaded within milliseconds after a new commit is available.
|
||||
/// This is made possible by watching changes in the `meta.json` file.
|
||||
@@ -139,7 +136,8 @@ impl IndexReaderBuilder {
|
||||
|
||||
/// Sets the number of warming threads.
|
||||
///
|
||||
/// This allows parallelizing warming work when there are multiple [Warmer] registered with the [IndexReader].
|
||||
/// This allows parallelizing warming work when there are multiple [Warmer] registered with the
|
||||
/// [IndexReader].
|
||||
pub fn num_warming_threads(mut self, num_warming_threads: usize) -> IndexReaderBuilder {
|
||||
self.num_warming_threads = num_warming_threads;
|
||||
self
|
||||
|
||||
+7
-7
@@ -1,10 +1,9 @@
|
||||
use crossbeam::channel::unbounded;
|
||||
use crossbeam::channel::{Receiver, RecvError, Sender};
|
||||
use std::ops::{Deref, DerefMut};
|
||||
use std::sync::atomic::AtomicUsize;
|
||||
use std::sync::atomic::Ordering;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
use std::sync::Arc;
|
||||
|
||||
use crossbeam::channel::{unbounded, Receiver, RecvError, Sender};
|
||||
|
||||
pub struct GenerationItem<T> {
|
||||
generation: usize,
|
||||
item: T,
|
||||
@@ -196,11 +195,12 @@ impl<T> Drop for LeasedItem<T> {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
|
||||
use super::Pool;
|
||||
use super::Queue;
|
||||
use crossbeam::channel;
|
||||
use std::{iter, mem};
|
||||
|
||||
use crossbeam::channel;
|
||||
|
||||
use super::{Pool, Queue};
|
||||
|
||||
#[test]
|
||||
fn test_pool() {
|
||||
let items10: Vec<usize> = iter::repeat(10).take(10).collect();
|
||||
|
||||
+24
-29
@@ -1,13 +1,10 @@
|
||||
use std::{
|
||||
collections::HashSet,
|
||||
ops::Deref,
|
||||
sync::{Arc, Mutex, Weak},
|
||||
thread::JoinHandle,
|
||||
time::Duration,
|
||||
};
|
||||
use std::collections::HashSet;
|
||||
use std::ops::Deref;
|
||||
use std::sync::{Arc, Mutex, Weak};
|
||||
use std::thread::JoinHandle;
|
||||
use std::time::Duration;
|
||||
|
||||
use crate::Inventory;
|
||||
use crate::{Executor, Searcher, SearcherGeneration, TantivyError};
|
||||
use crate::{Executor, Inventory, Searcher, SearcherGeneration, TantivyError};
|
||||
|
||||
pub const GC_INTERVAL: Duration = Duration::from_secs(1);
|
||||
|
||||
@@ -41,9 +38,11 @@ impl WarmingState {
|
||||
}))))
|
||||
}
|
||||
|
||||
/// Start tracking a new generation of [Searcher], and [Warmer::warm] it if there are active warmers.
|
||||
/// Start tracking a new generation of [Searcher], and [Warmer::warm] it if there are active
|
||||
/// warmers.
|
||||
///
|
||||
/// A background GC thread for [Warmer::garbage_collect] calls is uniquely created if there are active warmers.
|
||||
/// A background GC thread for [Warmer::garbage_collect] calls is uniquely created if there are
|
||||
/// active warmers.
|
||||
pub fn warm_new_searcher_generation(&self, searcher: &Searcher) -> crate::Result<()> {
|
||||
self.0
|
||||
.lock()
|
||||
@@ -70,8 +69,9 @@ struct WarmingStateInner {
|
||||
|
||||
impl WarmingStateInner {
|
||||
/// Start tracking provided searcher as an exemplar of a new generation.
|
||||
/// If there are active warmers, warm them with the provided searcher, and kick background GC thread if it has not yet been kicked.
|
||||
/// Otherwise, prune state for dropped searcher generations inline.
|
||||
/// If there are active warmers, warm them with the provided searcher, and kick background GC
|
||||
/// thread if it has not yet been kicked. Otherwise, prune state for dropped searcher
|
||||
/// generations inline.
|
||||
fn warm_new_searcher_generation(
|
||||
&mut self,
|
||||
searcher: &Searcher,
|
||||
@@ -102,7 +102,8 @@ impl WarmingStateInner {
|
||||
strong_warmers
|
||||
}
|
||||
|
||||
/// [Warmer::garbage_collect] active warmers if some searcher generation is observed to have been dropped.
|
||||
/// [Warmer::garbage_collect] active warmers if some searcher generation is observed to have
|
||||
/// been dropped.
|
||||
fn gc_maybe(&mut self) -> bool {
|
||||
let live_generations = self.searcher_generation_inventory.list();
|
||||
let live_generation_ids: HashSet<u64> = live_generations
|
||||
@@ -143,7 +144,8 @@ impl WarmingStateInner {
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
/// Every [GC_INTERVAL] attempt to GC, with panics caught and logged using [std::panic::catch_unwind].
|
||||
/// Every [GC_INTERVAL] attempt to GC, with panics caught and logged using
|
||||
/// [std::panic::catch_unwind].
|
||||
fn gc_loop(inner: Weak<Mutex<WarmingStateInner>>) {
|
||||
for _ in crossbeam::channel::tick(GC_INTERVAL) {
|
||||
if let Some(inner) = inner.upgrade() {
|
||||
@@ -170,22 +172,15 @@ fn warming_executor(num_threads: usize) -> crate::Result<Executor> {
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::{
|
||||
collections::HashSet,
|
||||
sync::{
|
||||
atomic::{self, AtomicUsize},
|
||||
Arc, RwLock, Weak,
|
||||
},
|
||||
};
|
||||
|
||||
use crate::{
|
||||
core::searcher::SearcherGeneration,
|
||||
directory::RamDirectory,
|
||||
schema::{Schema, INDEXED},
|
||||
Index, IndexSettings, ReloadPolicy, Searcher, SegmentId,
|
||||
};
|
||||
use std::collections::HashSet;
|
||||
use std::sync::atomic::{self, AtomicUsize};
|
||||
use std::sync::{Arc, RwLock, Weak};
|
||||
|
||||
use super::Warmer;
|
||||
use crate::core::searcher::SearcherGeneration;
|
||||
use crate::directory::RamDirectory;
|
||||
use crate::schema::{Schema, INDEXED};
|
||||
use crate::{Index, IndexSettings, ReloadPolicy, Searcher, SegmentId};
|
||||
|
||||
#[derive(Default)]
|
||||
struct TestWarmer {
|
||||
|
||||
Reference in New Issue
Block a user