Minor refactoring (#1266)

This commit is contained in:
Paul Masurel
2022-01-28 15:55:55 +09:00
committed by GitHub
parent 9679c5f306
commit eca6628b3c
208 changed files with 2034 additions and 2350 deletions
+13 -15
View File
@@ -1,23 +1,19 @@
mod pool;
mod warming;
use std::convert::TryInto;
use std::io;
use std::sync::atomic::AtomicU64;
use std::sync::{atomic, Arc, Weak};
pub use warming::Warmer;
pub use self::pool::LeasedItem;
use self::pool::Pool;
use self::warming::WarmingState;
use crate::core::searcher::SearcherGeneration;
use crate::directory::WatchHandle;
use crate::directory::META_LOCK;
use crate::directory::{Directory, WatchCallback};
use crate::Index;
use crate::Searcher;
use crate::SegmentReader;
use crate::{Inventory, TrackedObject};
use std::sync::atomic;
use std::sync::atomic::AtomicU64;
use std::sync::Arc;
use std::sync::Weak;
use std::{convert::TryInto, io};
pub use warming::Warmer;
use crate::directory::{Directory, WatchCallback, WatchHandle, META_LOCK};
use crate::{Index, Inventory, Searcher, SegmentReader, TrackedObject};
/// Defines when a new version of the index should be reloaded.
///
@@ -29,7 +25,8 @@ pub enum ReloadPolicy {
/// The index is entirely reloaded manually.
/// All updates of the index should be manual.
///
/// No change is reflected automatically. You are required to call `IndexReader::reload()` manually.
/// No change is reflected automatically. You are required to call `IndexReader::reload()`
/// manually.
Manual,
/// The index is reloaded within milliseconds after a new commit is available.
/// This is made possible by watching changes in the `meta.json` file.
@@ -139,7 +136,8 @@ impl IndexReaderBuilder {
/// Sets the number of warming threads.
///
/// This allows parallelizing warming work when there are multiple [Warmer] registered with the [IndexReader].
/// This allows parallelizing warming work when there are multiple [Warmer] registered with the
/// [IndexReader].
pub fn num_warming_threads(mut self, num_warming_threads: usize) -> IndexReaderBuilder {
self.num_warming_threads = num_warming_threads;
self
+7 -7
View File
@@ -1,10 +1,9 @@
use crossbeam::channel::unbounded;
use crossbeam::channel::{Receiver, RecvError, Sender};
use std::ops::{Deref, DerefMut};
use std::sync::atomic::AtomicUsize;
use std::sync::atomic::Ordering;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::Arc;
use crossbeam::channel::{unbounded, Receiver, RecvError, Sender};
pub struct GenerationItem<T> {
generation: usize,
item: T,
@@ -196,11 +195,12 @@ impl<T> Drop for LeasedItem<T> {
#[cfg(test)]
mod tests {
use super::Pool;
use super::Queue;
use crossbeam::channel;
use std::{iter, mem};
use crossbeam::channel;
use super::{Pool, Queue};
#[test]
fn test_pool() {
let items10: Vec<usize> = iter::repeat(10).take(10).collect();
+24 -29
View File
@@ -1,13 +1,10 @@
use std::{
collections::HashSet,
ops::Deref,
sync::{Arc, Mutex, Weak},
thread::JoinHandle,
time::Duration,
};
use std::collections::HashSet;
use std::ops::Deref;
use std::sync::{Arc, Mutex, Weak};
use std::thread::JoinHandle;
use std::time::Duration;
use crate::Inventory;
use crate::{Executor, Searcher, SearcherGeneration, TantivyError};
use crate::{Executor, Inventory, Searcher, SearcherGeneration, TantivyError};
pub const GC_INTERVAL: Duration = Duration::from_secs(1);
@@ -41,9 +38,11 @@ impl WarmingState {
}))))
}
/// Start tracking a new generation of [Searcher], and [Warmer::warm] it if there are active warmers.
/// Start tracking a new generation of [Searcher], and [Warmer::warm] it if there are active
/// warmers.
///
/// A background GC thread for [Warmer::garbage_collect] calls is uniquely created if there are active warmers.
/// A background GC thread for [Warmer::garbage_collect] calls is uniquely created if there are
/// active warmers.
pub fn warm_new_searcher_generation(&self, searcher: &Searcher) -> crate::Result<()> {
self.0
.lock()
@@ -70,8 +69,9 @@ struct WarmingStateInner {
impl WarmingStateInner {
/// Start tracking provided searcher as an exemplar of a new generation.
/// If there are active warmers, warm them with the provided searcher, and kick background GC thread if it has not yet been kicked.
/// Otherwise, prune state for dropped searcher generations inline.
/// If there are active warmers, warm them with the provided searcher, and kick background GC
/// thread if it has not yet been kicked. Otherwise, prune state for dropped searcher
/// generations inline.
fn warm_new_searcher_generation(
&mut self,
searcher: &Searcher,
@@ -102,7 +102,8 @@ impl WarmingStateInner {
strong_warmers
}
/// [Warmer::garbage_collect] active warmers if some searcher generation is observed to have been dropped.
/// [Warmer::garbage_collect] active warmers if some searcher generation is observed to have
/// been dropped.
fn gc_maybe(&mut self) -> bool {
let live_generations = self.searcher_generation_inventory.list();
let live_generation_ids: HashSet<u64> = live_generations
@@ -143,7 +144,8 @@ impl WarmingStateInner {
Ok(true)
}
/// Every [GC_INTERVAL] attempt to GC, with panics caught and logged using [std::panic::catch_unwind].
/// Every [GC_INTERVAL] attempt to GC, with panics caught and logged using
/// [std::panic::catch_unwind].
fn gc_loop(inner: Weak<Mutex<WarmingStateInner>>) {
for _ in crossbeam::channel::tick(GC_INTERVAL) {
if let Some(inner) = inner.upgrade() {
@@ -170,22 +172,15 @@ fn warming_executor(num_threads: usize) -> crate::Result<Executor> {
#[cfg(test)]
mod tests {
use std::{
collections::HashSet,
sync::{
atomic::{self, AtomicUsize},
Arc, RwLock, Weak,
},
};
use crate::{
core::searcher::SearcherGeneration,
directory::RamDirectory,
schema::{Schema, INDEXED},
Index, IndexSettings, ReloadPolicy, Searcher, SegmentId,
};
use std::collections::HashSet;
use std::sync::atomic::{self, AtomicUsize};
use std::sync::{Arc, RwLock, Weak};
use super::Warmer;
use crate::core::searcher::SearcherGeneration;
use crate::directory::RamDirectory;
use crate::schema::{Schema, INDEXED};
use crate::{Index, IndexSettings, ReloadPolicy, Searcher, SegmentId};
#[derive(Default)]
struct TestWarmer {