rust/tantivy
1
0
Fork 0
mirror of https://github.com/quickwit-oss/tantivy.git synced 2026-05-30 07:00:41 +00:00
Code Issues Packages Projects Releases Wiki Activity

Minor refactoring (#1266)

Browse Source
This commit is contained in:
Paul Masurel
2022-01-28 15:55:55 +09:00
committed by GitHub
parent 9679c5f306
commit eca6628b3c
208 changed files with 2034 additions and 2350 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
src/tokenizer/tokenized_string.rs
View File

@@ -1,7 +1,9 @@
use crate::tokenizer::{BoxTokenStream, Token, TokenStream, TokenStreamChain};
use serde::{Deserialize, Serialize};
use std::cmp::Ordering;
use serde::{Deserialize, Serialize};
use crate::tokenizer::{BoxTokenStream, Token, TokenStream, TokenStreamChain};
/// Struct representing pre-tokenized text
#[derive(Debug, Clone, Serialize, Deserialize, Eq, PartialEq)]
pub struct PreTokenizedString {
@@ -91,7 +93,6 @@ impl TokenStream for PreTokenizedStream {
mod tests {
use super::*;
use crate::tokenizer::Token;
#[test]