rust/tantivy
1
0
Fork 0
mirror of https://github.com/quickwit-oss/tantivy.git synced 2026-06-03 09:00:42 +00:00
Code Issues Packages Projects Releases Wiki Activity

issue/77 Added managed directory

Browse Source
This commit is contained in:
Paul Masurel
2017-03-03 22:41:30 +09:00
parent 590a8582c9
commit 4b7afa2ae7
13 changed files with 168 additions and 124 deletions
Download Patch File Download Diff File Expand all files Collapse all files

76
src/core/index.rs
View File

@@ -19,6 +19,7 @@ use super::pool::LeasedItem;
use std::path::Path;
use core::IndexMeta;
use IndexWriter;
use directory::ManagedDirectory;
use core::META_FILEPATH;
use super::segment::create_segment;
use indexer::segment_updater::save_new_metas;
@@ -26,7 +27,6 @@ use directory::error::FileError;
const NUM_SEARCHERS: usize = 12;
fn load_metas(directory: &Directory) -> Result<IndexMeta> {
let meta_data = directory.atomic_read(&META_FILEPATH)?;
let meta_string = String::from_utf8_lossy(&meta_data);
@@ -36,58 +36,19 @@ fn load_metas(directory: &Directory) -> Result<IndexMeta> {
/// Tantivy's Search Index
pub struct Index {
directory: Box<Directory>,
directory: ManagedDirectory,
schema: Schema,
searcher_pool: Arc<Pool<Searcher>>,
}
/// Deletes all of the files of the segment.
/// This is called when there is a merge or a rollback.
///
/// # Disclaimer
/// If deletion of a file fails (e.g. a file
/// was read-only.), the method does not
/// fail and just logs an error when it fails.
#[doc(hidden)]
pub fn delete_segment(directory: &Directory, segment_id: SegmentId) {
info!("Deleting segment {:?}", segment_id);
let segment_filepaths_res = directory.ls_starting_with(
&*segment_id.uuid_string()
);
match segment_filepaths_res {
Ok(segment_filepaths) => {
for segment_filepath in &segment_filepaths {
if let Err(err) = directory.delete(&segment_filepath) {
match err {
FileError::FileDoesNotExist(_) => {
// this is normal behavior.
// the position file for instance may not exists.
}
FileError::IOError(err) => {
error!("Failed to remove {:?} : {:?}", segment_id, err);
}
}
}
}
}
Err(_) => {
error!("Failed to list files of segment {:?} for deletion.", segment_id.uuid_string());
}
}
}
impl Index {
/// Creates a new index using the `RAMDirectory`.
///
/// The index will be allocated in anonymous memory.
/// This should only be used for unit tests.
pub fn create_in_ram(schema: Schema) -> Index {
let directory = Box::new(RAMDirectory::create());
let directory = ManagedDirectory::new(RAMDirectory::create());
Index::from_directory(directory, schema).expect("Creating a RAMDirectory should never fail") // unwrap is ok here
}
@@ -96,8 +57,8 @@ impl Index {
///
/// If a previous index was in this directory, then its meta file will be destroyed.
pub fn create(directory_path: &Path, schema: Schema) -> Result<Index> {
let directory = MmapDirectory::open(directory_path)?;
Index::from_directory(box directory, schema)
let directory = ManagedDirectory::new(MmapDirectory::open(directory_path)?);
Index::from_directory(directory, schema)
}
/// Creates a new index in a temp directory.
@@ -109,12 +70,12 @@ impl Index {
/// The temp directory is only used for testing the `MmapDirectory`.
/// For other unit tests, prefer the `RAMDirectory`, see: `create_in_ram`.
pub fn create_from_tempdir(schema: Schema) -> Result<Index> {
let directory = Box::new(try!(MmapDirectory::create_from_tempdir()));
let directory = ManagedDirectory::new(MmapDirectory::create_from_tempdir()?);
Index::from_directory(directory, schema)
}
/// Creates a new index given a directory and an `IndexMeta`.
fn create_from_metas(directory: Box<Directory>, metas: IndexMeta) -> Result<Index> {
fn create_from_metas(directory: ManagedDirectory, metas: IndexMeta) -> Result<Index> {
let schema = metas.schema.clone();
let index = Index {
directory: directory,
@@ -126,16 +87,16 @@ impl Index {
}
/// Create a new index from a directory.
pub fn from_directory(mut directory: Box<Directory>, schema: Schema) -> Result<Index> {
pub fn from_directory(mut directory: ManagedDirectory, schema: Schema) -> Result<Index> {
save_new_metas(schema.clone(), 0, directory.borrow_mut())?;
Index::create_from_metas(directory, IndexMeta::with_schema(schema))
}
/// Opens a new directory from an index path.
pub fn open(directory_path: &Path) -> Result<Index> {
let directory = try!(MmapDirectory::open(directory_path));
let directory = ManagedDirectory::new(MmapDirectory::open(directory_path)?);
let metas = try!(load_metas(&directory));
Index::create_from_metas(directory.box_clone(), metas)
Index::create_from_metas(directory, metas)
}
/// Returns the index opstamp.
@@ -196,16 +157,7 @@ impl Index {
.map(|segment_meta| self.segment(segment_meta))
.collect())
}
/// Remove all of the file associated with the segment.
///
/// This method cannot fail. If a problem occurs,
/// some files may end up never being removed.
/// The error will only be logged.
pub fn delete_segment(&self, segment_id: SegmentId) {
delete_segment(self.directory(), segment_id);
}
#[doc(hidden)]
pub fn segment(&self, segment_meta: SegmentMeta) -> Segment {
create_segment(self.clone(), segment_meta)
@@ -219,12 +171,12 @@ impl Index {
/// Return a reference to the index directory.
pub fn directory(&self) -> &Directory {
&*self.directory
&self.directory
}
/// Return a mutable reference to the index directory.
pub fn directory_mut(&mut self) -> &mut Directory {
&mut *self.directory
&mut self.directory
}
/// Reads the meta.json and returns the list of
@@ -288,7 +240,7 @@ impl fmt::Debug for Index {
impl Clone for Index {
fn clone(&self) -> Index {
Index {
directory: self.directory.box_clone(),
directory: self.directory.clone(),
schema: self.schema.clone(),
searcher_pool: self.searcher_pool.clone(),
}

1
src/core/mod.rs
View File

@@ -26,4 +26,5 @@ use std::path::PathBuf;
lazy_static! {
pub static ref META_FILEPATH: PathBuf = PathBuf::from("meta.json");
pub static ref MANAGED_FILEPATH: PathBuf = PathBuf::from(".managed.json");
}

14
src/core/segment.rs
View File

@@ -62,19 +62,7 @@ impl Segment {
/// It just joins the segment id with the extension
/// associated to a segment component.
pub fn relative_path(&self, component: SegmentComponent) -> PathBuf {
use self::SegmentComponent::*;
let mut path = self.id().uuid_string();
path.push_str(&*match component {
POSITIONS => ".pos".to_string(),
INFO => ".info".to_string(),
POSTINGS => ".idx".to_string(),
TERMS => ".term".to_string(),
STORE => ".store".to_string(),
FASTFIELDS => ".fast".to_string(),
FIELDNORMS => ".fieldnorm".to_string(),
DELETE => {format!(".{}.del", self.meta.delete_opstamp().unwrap_or(0))},
});
PathBuf::from(path)
self.meta.relative_path(component)
}
/// Open one of the component file for read.

19
src/core/segment_component.rs
View File

@@ -10,5 +10,20 @@ pub enum SegmentComponent {
DELETE
}
impl SegmentComponent {
pub fn iterator() -> impl Iterator<Item=&'static SegmentComponent> {
static SEGMENT_COMPONENTS: [SegmentComponent; 8] = [
SegmentComponent::INFO,
SegmentComponent::POSTINGS,
SegmentComponent::POSITIONS,
SegmentComponent::FASTFIELDS,
SegmentComponent::FIELDNORMS,
SegmentComponent::TERMS,
SegmentComponent::STORE,
SegmentComponent::DELETE
];
SEGMENT_COMPONENTS.into_iter()
}
}

31
src/core/segment_meta.rs
View File

@@ -1,4 +1,6 @@
use core::SegmentId;
use super::SegmentComponent;
use std::path::PathBuf;
#[derive(Clone, Debug, RustcDecodable,RustcEncodable)]
@@ -43,6 +45,35 @@ impl SegmentMeta {
.unwrap_or(0u32)
}
pub fn alive_files(&self) -> Vec<PathBuf> {
SegmentComponent::iterator()
.map(|component| {
self.relative_path(*component)
})
.collect::<Vec<PathBuf>>()
}
/// Returns the relative path of a component of our segment.
///
/// It just joins the segment id with the extension
/// associated to a segment component.
pub fn relative_path(&self, component: SegmentComponent) -> PathBuf {
use self::SegmentComponent::*;
let mut path = self.id().uuid_string();
path.push_str(&*match component {
POSITIONS => ".pos".to_string(),
INFO => ".info".to_string(),
POSTINGS => ".idx".to_string(),
TERMS => ".term".to_string(),
STORE => ".store".to_string(),
FASTFIELDS => ".fast".to_string(),
FIELDNORMS => ".fieldnorm".to_string(),
DELETE => {format!(".{}.del", self.delete_opstamp().unwrap_or(0))},
});
PathBuf::from(path)
}
/// Return the highest doc id + 1
///
/// If there are no deletes, then num_docs = max_docs