rust/tantivy
1
0
Fork 0
mirror of https://github.com/quickwit-oss/tantivy.git synced 2026-01-06 01:02:55 +00:00
Code Issues Packages Projects Releases Wiki Activity

Compare commits

base: rust:0.10.0
Branches Tags
rust:main rust:bucket_id_agg rust:paul.masurel/skip-first-doc-on-intersection rust:stuhood.lazy-scorers-blocks-upstream rust:paul.masurel/seek_into_the_danger_zone_result rust:paul.masurel/search_into_danger_zone_fastfield rust:stuhood.lazy-scorers-blocks rust:paradedb/fix-agg-validation rust:flatheadmill-geo rust:paradedb-paradedb/fix-overflow-issue rust:low_card_optimization rust:paul.masurel/clippy rust:qw-airmail-20250916 rust:unit-test-aggregation-sum-other rust:clippy-and-cleanups rust:trinity.pointard/opti-sstable-sorted-ords-to-term rust:release_0.24 rust:dependabot/cargo/rand-0.9.1 rust:dependabot/cargo/lru-0.14.0 rust:dependabot/cargo/fs4-0.13.1 rust:tagged-user-input-ast rust:1686a/parse-agg-res rust:dependabot/cargo/rand_distr-0.5.1 rust:issue/2411-adding-panic-handler-to-rayon-merge-thread-pool rust:dependabot/cargo/nom-8 rust:upgrade_deps rust:test_parser rust:dependabot/github_actions/codecov/codecov-action-5 rust:binggan-0.12.0 rust:agg_format rust:path_collector rust:test_order_bug rust:test_columnar_multi rust:columnar_optionalmask_in_multivalues rust:prefix-phrase-query-optim rust:tantivy-with-oneshot rust:update_example rust:store-reader-enumerate rust:document-deserialize-seed rust:check_doc_mem rust:experimental-do-not-merge rust:expdonotmerge rust:cherry-pick-vec-instead-of-btreemap rust:validate_length rust:raphael_bench_avx_filter rust:raphael_optim_memset2 rust:query-field-name-avoid-alloc rust:check_bug rust:missing_test rust:list_fields rust:block-cache-control rust:trinity--multilayer-sstable rust:tracing-tantivy2 rust:tracing-tantivy rust:index_writer_err rust:default_fast_field_tokenizer rust:fmassot/add-preserve-original-ascii-folding rust:numeric_with_str_col rust:straightforward-simplify rust:fmassot/fake-test-branch rust:fmassot/dynamic-tokenizer-poc-follow-up rust:fmassot/add-box-token-filter-and-refactor rust:dynamic-tokenizer-proof-of-concept rust:fmassot/add-box-tokenizer-and-token-filter rust:tokenizer-api-0.1.1 rust:addconversion rust:removed-extra-enum-alternative-for-datetime-precision rust:issue/1981-reproduce-bm25-bug rust:tokenizer_docs rust:merge-proptests rust:columnar-todo rust:quickwit-0.5 rust:remove-byteorder rust:remove_dyn rust:pr/alexcole/1855-inject-bm25-statistics rust:evance-expose-build-query-from-user-input-ast rust:missing-specialized-method-arcdyncolumn rust:fast-u64-range-query rust:0.19.2 rust:columnar-cli2 rust:trinity--pub-term-as-slice rust:typed-column3 rust:columnar-merge3 rust:columnar-merge-exploring-fn rust:use_columnar rust:typed-column2 rust:term_agg_bucket_limit rust:range rust:typed-column rust:columnar-main rust:forced-types rust:u128-columnar rust:use_stats rust:truncation_comment rust:0.19.1 rust:trinity/yoke-documents rust:column-handle rust:columnar-cli rust:columnar rust:revert-1711-sparse_dense_index rust:commit-changes rust:sparse_dense_index_headroom rust:optional_column rust:no-column-for-multivalued rust:fmassot--bench-hdfs-with-array rust:bugfix-position-broken rust:float rust:simplify rust:debug_time rust:sparse_codecs rust:fix_estimate rust:column-reader rust:column-reader-b rust:columnreader-simple rust:col-trait-refact rust:ip_fastfield rust:column-trait-2 rust:refact-dyn-column rust:column-trait rust:refact-codec-trait rust:pr/PSeitz/1485-1 rust:fastfieldcodec rust:termmap_per_field rust:fastfield-codecs-tests rust:rayon-indexing-writers rust:issue/1251 rust:index-bench rust:fix_open_bytes_index rust:warming rust:revert-1222-issue/1195 rust:add-random-bench-for-fast-field rust:wasm-friendly-exploratory rust:revert-1159-patch-1 rust:debug-position-panic-0.15.3 rust:fix-issue-1111 rust:remove_rand rust:unused-deps rust:hfb/phrase_match_slop rust:0.15.3 rust:expose-min-max-on-multi-fast-field-reader rust:revert-1094-expose-min-max-on-multi-fast-field-reader rust:hotfix-1088 rust:owned-bytes-instead-of-rawdoc rust:rihardsk-date-ranges rust:issue/997 rust:0.13.3 rust:fail-fast-store-checkpoint rust:debugging-store rust:issue/reproduce-897 rust:issue/969-reproduce rust:troublescooter-static rust:intermediate-mergeable-troublescooter rust:barrotsteindev-filter-collector-tpredicatevalue rust:issue/938b rust:issue/938 rust:barrotsteindev-filter-collector rust:issue/922 rust:issue/896 rust:quickfix-explain rust:slog rust:limit-random-seeks rust:0.13.1 rust:issue/866b rust:bugfix-unittest-macos rust:githubactions rust:fieldnorm_readers rust:blockwand rust:nodeffeatfails rust:removedaliveiterator rust:robyoung-introduce-offset-to-top-docs rust:specialization-of-foreach-for-union-scorer rust:elshize-bmw rust:cutt rust:nrt rust:storewriter-out rust:segment_writer_in_ram2 rust:polling rust:audunhalland-query-boost rust:poll-meta rust:segment_writer_in_ram rust:bundle rust:petr-tik-n662_footer_is_incompatible_impl rust:race-condition-in-tests rust:hotfix-0.10.3-issue681 rust:remove-tru rust:kkoziara-pre-tokenized-text rust:issue/681 rust:issue/680 rust:refactoring/field rust:updating/uuid rust:fix-bench rust:hotfix-656 rust:readd-macos-ci rust:incremental-search rust:crate-division rust:streamer-with-state rust:criterion rust:python-binding-changes rust:bump-version rust:gh-pages rust:issue/554 rust:perf/experimental-union-for_each rust:issue/526b rust:softcommits rust:0.9.1-hotfix rust:revert-521-fix-non-english-stemmers rust:issue/500-hotfix rust:hotfix/0.8.2 rust:issue/483 rust:bug/merge-deted rust:issue/407 rust:issue/457 rust:segment_fruits rust:doc rust:dds/lenient rust:issue/webasm rust:issue/396 rust:broken_collector_with_result rust:dss/automaton-weight-suggestion rust:wasm rust:common-crawl rust:tantivy-imhotep
rust:0.25.0 rust:0.22.1 rust:0.24.2 rust:0.24.1 rust:tantivy-query-grammar-v0.24.0 rust:tantivy-columnar-v0.5.0 rust:tantivy-bitpacker-v0.8.0 rust:ownedbytes-v0.9.0 rust:tantivy-stacker-v0.5.0 rust:tantivy-sstable-v0.5.0 rust:tantivy-common-v0.9.0 rust:tantivy-tokenizer-api-v0.5.0 rust:0.24 rust:qw-airmail-20250224-hotfix-merge-panic rust:qw-airmail-20250219-hotfix-merge-panic rust:qw-airmail-202406 rust:qw-airmail-20240611 rust:qw-airmail-20240606 rust:quickwit-rev rust:0.22.0 rust:tantivy-stacker-v0.3.0 rust:ownedbytes-v0.7.0 rust:tantivy-tokenizer-api-v0.3.0 rust:tantivy-columnar-v0.3.0 rust:tantivy-common-v0.7.0 rust:tantivy-query-grammar-v0.22.0 rust:tantivy-sstable-v0.3.0 rust:tantivy-bitpacker-v0.6.0 rust:0.21.1 rust:0.21 rust:0.20.2 rust:0.20.1 rust:0.20 rust:quickwit-0.5-rev rust:0.19.2 rust:0.19.1 rust:0.19 rust:0.18.1 rust:0.18 rust:0.17 rust:0.16.1 rust:0.15.3 rust:0.15.2 rust:0.15.1 rust:0.15 rust:0.14 rust:0.13.3 rust:0.13.2 rust:0.13.1 rust:0.13 rust:0.12 rust:0.11.3 rust:0.11.1 rust:0.11 rust:0.10.3 rust:0.1 rust:0.10.2 rust:0.10.1 rust:0.10.0 rust:0.9.1 rust:0.9 rust:0.8.2 rust:0.8.2b rust:0.8.1 rust:0.8.0 rust:0.7.2 rust:0.7.1 rust:0.7.0 rust:0.6.1 rust:0.6.0 rust:0.5.2 rust:0.5.1 rust:0.5.0 rust:0.4.3 rust:0.4.2 rust:0.4.0 rust:0.3.1 rust:0.3.0 rust:0.2.0
..
compare: rust:issue/554
Branches Tags
rust:bucket_id_agg rust:paul.masurel/skip-first-doc-on-intersection rust:main rust:stuhood.lazy-scorers-blocks-upstream rust:paul.masurel/seek_into_the_danger_zone_result rust:paul.masurel/search_into_danger_zone_fastfield rust:stuhood.lazy-scorers-blocks rust:paradedb/fix-agg-validation rust:flatheadmill-geo rust:paradedb-paradedb/fix-overflow-issue rust:low_card_optimization rust:paul.masurel/clippy rust:qw-airmail-20250916 rust:unit-test-aggregation-sum-other rust:clippy-and-cleanups rust:trinity.pointard/opti-sstable-sorted-ords-to-term rust:release_0.24 rust:dependabot/cargo/rand-0.9.1 rust:dependabot/cargo/lru-0.14.0 rust:dependabot/cargo/fs4-0.13.1 rust:tagged-user-input-ast rust:1686a/parse-agg-res rust:dependabot/cargo/rand_distr-0.5.1 rust:issue/2411-adding-panic-handler-to-rayon-merge-thread-pool rust:dependabot/cargo/nom-8 rust:upgrade_deps rust:test_parser rust:dependabot/github_actions/codecov/codecov-action-5 rust:binggan-0.12.0 rust:agg_format rust:path_collector rust:test_order_bug rust:test_columnar_multi rust:columnar_optionalmask_in_multivalues rust:prefix-phrase-query-optim rust:tantivy-with-oneshot rust:update_example rust:store-reader-enumerate rust:document-deserialize-seed rust:check_doc_mem rust:experimental-do-not-merge rust:expdonotmerge rust:cherry-pick-vec-instead-of-btreemap rust:validate_length rust:raphael_bench_avx_filter rust:raphael_optim_memset2 rust:query-field-name-avoid-alloc rust:check_bug rust:missing_test rust:list_fields rust:block-cache-control rust:trinity--multilayer-sstable rust:tracing-tantivy2 rust:tracing-tantivy rust:index_writer_err rust:default_fast_field_tokenizer rust:fmassot/add-preserve-original-ascii-folding rust:numeric_with_str_col rust:straightforward-simplify rust:fmassot/fake-test-branch rust:fmassot/dynamic-tokenizer-poc-follow-up rust:fmassot/add-box-token-filter-and-refactor rust:dynamic-tokenizer-proof-of-concept rust:fmassot/add-box-tokenizer-and-token-filter rust:tokenizer-api-0.1.1 rust:addconversion rust:removed-extra-enum-alternative-for-datetime-precision rust:issue/1981-reproduce-bm25-bug rust:tokenizer_docs rust:merge-proptests rust:columnar-todo rust:quickwit-0.5 rust:remove-byteorder rust:remove_dyn rust:pr/alexcole/1855-inject-bm25-statistics rust:evance-expose-build-query-from-user-input-ast rust:missing-specialized-method-arcdyncolumn rust:fast-u64-range-query rust:0.19.2 rust:columnar-cli2 rust:trinity--pub-term-as-slice rust:typed-column3 rust:columnar-merge3 rust:columnar-merge-exploring-fn rust:use_columnar rust:typed-column2 rust:term_agg_bucket_limit rust:range rust:typed-column rust:columnar-main rust:forced-types rust:u128-columnar rust:use_stats rust:truncation_comment rust:0.19.1 rust:trinity/yoke-documents rust:column-handle rust:columnar-cli rust:columnar rust:revert-1711-sparse_dense_index rust:commit-changes rust:sparse_dense_index_headroom rust:optional_column rust:no-column-for-multivalued rust:fmassot--bench-hdfs-with-array rust:bugfix-position-broken rust:float rust:simplify rust:debug_time rust:sparse_codecs rust:fix_estimate rust:column-reader rust:column-reader-b rust:columnreader-simple rust:col-trait-refact rust:ip_fastfield rust:column-trait-2 rust:refact-dyn-column rust:column-trait rust:refact-codec-trait rust:pr/PSeitz/1485-1 rust:fastfieldcodec rust:termmap_per_field rust:fastfield-codecs-tests rust:rayon-indexing-writers rust:issue/1251 rust:index-bench rust:fix_open_bytes_index rust:warming rust:revert-1222-issue/1195 rust:add-random-bench-for-fast-field rust:wasm-friendly-exploratory rust:revert-1159-patch-1 rust:debug-position-panic-0.15.3 rust:fix-issue-1111 rust:remove_rand rust:unused-deps rust:hfb/phrase_match_slop rust:0.15.3 rust:expose-min-max-on-multi-fast-field-reader rust:revert-1094-expose-min-max-on-multi-fast-field-reader rust:hotfix-1088 rust:owned-bytes-instead-of-rawdoc rust:rihardsk-date-ranges rust:issue/997 rust:0.13.3 rust:fail-fast-store-checkpoint rust:debugging-store rust:issue/reproduce-897 rust:issue/969-reproduce rust:troublescooter-static rust:intermediate-mergeable-troublescooter rust:barrotsteindev-filter-collector-tpredicatevalue rust:issue/938b rust:issue/938 rust:barrotsteindev-filter-collector rust:issue/922 rust:issue/896 rust:quickfix-explain rust:slog rust:limit-random-seeks rust:0.13.1 rust:issue/866b rust:bugfix-unittest-macos rust:githubactions rust:fieldnorm_readers rust:blockwand rust:nodeffeatfails rust:removedaliveiterator rust:robyoung-introduce-offset-to-top-docs rust:specialization-of-foreach-for-union-scorer rust:elshize-bmw rust:cutt rust:nrt rust:storewriter-out rust:segment_writer_in_ram2 rust:polling rust:audunhalland-query-boost rust:poll-meta rust:segment_writer_in_ram rust:bundle rust:petr-tik-n662_footer_is_incompatible_impl rust:race-condition-in-tests rust:hotfix-0.10.3-issue681 rust:remove-tru rust:kkoziara-pre-tokenized-text rust:issue/681 rust:issue/680 rust:refactoring/field rust:updating/uuid rust:fix-bench rust:hotfix-656 rust:readd-macos-ci rust:incremental-search rust:crate-division rust:streamer-with-state rust:criterion rust:python-binding-changes rust:bump-version rust:gh-pages rust:issue/554 rust:perf/experimental-union-for_each rust:issue/526b rust:softcommits rust:0.9.1-hotfix rust:revert-521-fix-non-english-stemmers rust:issue/500-hotfix rust:hotfix/0.8.2 rust:issue/483 rust:bug/merge-deted rust:issue/407 rust:issue/457 rust:segment_fruits rust:doc rust:dds/lenient rust:issue/webasm rust:issue/396 rust:broken_collector_with_result rust:dss/automaton-weight-suggestion rust:wasm rust:common-crawl rust:tantivy-imhotep
rust:0.25.0 rust:0.22.1 rust:0.24.2 rust:0.24.1 rust:tantivy-query-grammar-v0.24.0 rust:tantivy-columnar-v0.5.0 rust:tantivy-bitpacker-v0.8.0 rust:ownedbytes-v0.9.0 rust:tantivy-stacker-v0.5.0 rust:tantivy-sstable-v0.5.0 rust:tantivy-common-v0.9.0 rust:tantivy-tokenizer-api-v0.5.0 rust:0.24 rust:qw-airmail-20250224-hotfix-merge-panic rust:qw-airmail-20250219-hotfix-merge-panic rust:qw-airmail-202406 rust:qw-airmail-20240611 rust:qw-airmail-20240606 rust:quickwit-rev rust:0.22.0 rust:tantivy-stacker-v0.3.0 rust:ownedbytes-v0.7.0 rust:tantivy-tokenizer-api-v0.3.0 rust:tantivy-columnar-v0.3.0 rust:tantivy-common-v0.7.0 rust:tantivy-query-grammar-v0.22.0 rust:tantivy-sstable-v0.3.0 rust:tantivy-bitpacker-v0.6.0 rust:0.21.1 rust:0.21 rust:0.20.2 rust:0.20.1 rust:0.20 rust:quickwit-0.5-rev rust:0.19.2 rust:0.19.1 rust:0.19 rust:0.18.1 rust:0.18 rust:0.17 rust:0.16.1 rust:0.15.3 rust:0.15.2 rust:0.15.1 rust:0.15 rust:0.14 rust:0.13.3 rust:0.13.2 rust:0.13.1 rust:0.13 rust:0.12 rust:0.11.3 rust:0.11.1 rust:0.11 rust:0.10.3 rust:0.1 rust:0.10.2 rust:0.10.1 rust:0.10.0 rust:0.9.1 rust:0.9 rust:0.8.2 rust:0.8.2b rust:0.8.1 rust:0.8.0 rust:0.7.2 rust:0.7.1 rust:0.7.0 rust:0.6.1 rust:0.6.0 rust:0.5.2 rust:0.5.1 rust:0.5.0 rust:0.4.3 rust:0.4.2 rust:0.4.0 rust:0.3.1 rust:0.3.0 rust:0.2.0

1 Commits
0.10.0 ... issue/554

Author SHA1 Message Date
Paul Masurel
94c73363e4 added unit test 2019-06-02 14:53:03 +09:00
168 changed files with 2168 additions and 2898 deletions
Expand all files Collapse all files

17
.travis.yml
View File

@@ -10,7 +10,7 @@ env:
global: global:
- CRATE_NAME=tantivy - CRATE_NAME=tantivy
- TRAVIS_CARGO_NIGHTLY_FEATURE="" - TRAVIS_CARGO_NIGHTLY_FEATURE=""
# - secure: eC8HjTi1wgRVCsMAeXEXt8Ckr0YBSGOEnQkkW4/Nde/OZ9jJjz2nmP1ELQlDE7+czHub2QvYtDMG0parcHZDx/Kus0yvyn08y3g2rhGIiE7y8OCvQm1Mybu2D/p7enm6shXquQ6Z5KRfRq+18mHy80wy9ABMA/ukEZdvnfQ76/Een8/Lb0eHaDoXDXn3PqLVtByvSfQQ7OhS60dEScu8PWZ6/l1057P5NpdWbMExBE7Ro4zYXNhkJeGZx0nP/Bd4Jjdt1XfPzMEybV6NZ5xsTILUBFTmOOt603IsqKGov089NExqxYu5bD3K+S4MzF1Nd6VhomNPJqLDCfhlymJCUj5n5Ku4yidlhQbM4Ej9nGrBalJnhcjBjPua5tmMF2WCxP9muKn/2tIOu1/+wc0vMf9Yd3wKIkf5+FtUxCgs2O+NslWvmOMAMI/yD25m7hb4t1IwE/4Bk+GVcWJRWXbo0/m6ZUHzRzdjUY2a1qvw7C9udzdhg7gcnXwsKrSWi2NjMiIVw86l+Zim0nLpKIN41sxZHLaFRG63Ki8zQ/481LGn32awJ6i3sizKS0WD+N1DfR2qYMrwYHaMN0uR0OFXYTJkFvTFttAeUY3EKmRKAuMhmO2YRdSr4/j/G5E9HMc1gSGJj6PxgpQU7EpvxRsmoVAEJr0mszmOj9icGHep/FM= - secure: eC8HjTi1wgRVCsMAeXEXt8Ckr0YBSGOEnQkkW4/Nde/OZ9jJjz2nmP1ELQlDE7+czHub2QvYtDMG0parcHZDx/Kus0yvyn08y3g2rhGIiE7y8OCvQm1Mybu2D/p7enm6shXquQ6Z5KRfRq+18mHy80wy9ABMA/ukEZdvnfQ76/Een8/Lb0eHaDoXDXn3PqLVtByvSfQQ7OhS60dEScu8PWZ6/l1057P5NpdWbMExBE7Ro4zYXNhkJeGZx0nP/Bd4Jjdt1XfPzMEybV6NZ5xsTILUBFTmOOt603IsqKGov089NExqxYu5bD3K+S4MzF1Nd6VhomNPJqLDCfhlymJCUj5n5Ku4yidlhQbM4Ej9nGrBalJnhcjBjPua5tmMF2WCxP9muKn/2tIOu1/+wc0vMf9Yd3wKIkf5+FtUxCgs2O+NslWvmOMAMI/yD25m7hb4t1IwE/4Bk+GVcWJRWXbo0/m6ZUHzRzdjUY2a1qvw7C9udzdhg7gcnXwsKrSWi2NjMiIVw86l+Zim0nLpKIN41sxZHLaFRG63Ki8zQ/481LGn32awJ6i3sizKS0WD+N1DfR2qYMrwYHaMN0uR0OFXYTJkFvTFttAeUY3EKmRKAuMhmO2YRdSr4/j/G5E9HMc1gSGJj6PxgpQU7EpvxRsmoVAEJr0mszmOj9icGHep/FM=
addons: addons:
apt: apt:
@@ -38,12 +38,12 @@ matrix:
# Linux # Linux
#- env: TARGET=aarch64-unknown-linux-gnu #- env: TARGET=aarch64-unknown-linux-gnu
#- env: TARGET=i686-unknown-linux-gnu #- env: TARGET=i686-unknown-linux-gnu
- env: TARGET=x86_64-unknown-linux-gnu CODECOV=1 UPLOAD_DOCS=1 - env: TARGET=x86_64-unknown-linux-gnu CODECOV=1
# - env: TARGET=x86_64-unknown-linux-musl CODECOV=1 # - env: TARGET=x86_64-unknown-linux-musl CODECOV=1
# OSX # OSX
#- env: TARGET=x86_64-apple-darwin - env: TARGET=x86_64-apple-darwin
# os: osx os: osx
before_install: before_install:
- set -e - set -e
@@ -52,7 +52,6 @@ before_install:
install: install:
- sh ci/install.sh - sh ci/install.sh
- source ~/.cargo/env || true - source ~/.cargo/env || true
- env | grep "TRAVIS"
before_script: before_script:
- export PATH=$HOME/.cargo/bin:$PATH - export PATH=$HOME/.cargo/bin:$PATH
@@ -62,14 +61,12 @@ before_script:
script: script:
- bash ci/script.sh - bash ci/script.sh
after_success:
- cargo doc-upload
before_deploy: before_deploy:
- sh ci/before_deploy.sh - sh ci/before_deploy.sh
after_success:
# Needs GH_TOKEN env var to be set in travis settings
- if [[ -v GH_TOKEN ]]; then echo "GH TOKEN IS SET"; else echo "GH TOKEN NOT SET"; fi
- if [[ -v UPLOAD_DOCS ]]; then cargo doc; cargo doc-upload; else echo "doc upload disabled."; fi
cache: cargo cache: cargo
before_cache: before_cache:
# Travis can't cache files that are not readable by "others" # Travis can't cache files that are not readable by "others"

11
CHANGELOG.md
View File

@@ -3,17 +3,11 @@ Tantivy 0.10.0
*Tantivy 0.10.0 index format is compatible with the index format in 0.9.0.* *Tantivy 0.10.0 index format is compatible with the index format in 0.9.0.*
- Added an API to easily tweak or entirely replace the
default score. See `TopDocs::tweak_score`and `TopScore::custom_score` (@pmasurel)
- Added an ASCII folding filter (@drusellers) - Added an ASCII folding filter (@drusellers)
- Bugfix in `query.count` in presence of deletes (@pmasurel) - Bugfix in `query.count` in presence of deletes (@pmasurel)
- Added `.explain(...)` in `Query` and `Weight` to (@pmasurel)
- Added an efficient way to `delete_all_documents` in `IndexWriter` (@petr-tik).
All segments are simply removed.
Minor Minor
--------- ---------
- Switched to Rust 2018 (@uvd)
- Small simplification of the code. - Small simplification of the code.
Calling .freq() or .doc() when .advance() has never been called Calling .freq() or .doc() when .advance() has never been called
on segment postings should panic from now on. on segment postings should panic from now on.
@@ -25,12 +19,11 @@ Arc<RwLock<IndexWriter>>`. `add_document` and `delete_term` can
only require a read lock. (@pmasurel) only require a read lock. (@pmasurel)
- Introducing `Opstamp` as an expressive type alias for `u64`. (@petr-tik) - Introducing `Opstamp` as an expressive type alias for `u64`. (@petr-tik)
- Stamper now relies on `AtomicU64` on all platforms (@petr-tik) - Stamper now relies on `AtomicU64` on all platforms (@petr-tik)
- Bugfix - Files get deleted slightly earlier
- Compilation resources improved (@fdb-hiroshima)
## How to update? ## How to update?
Your program should be usable as is. Your existing indexes are usable as is, but you may need some
trivial updates.
### Fast fields ### Fast fields

11
Cargo.toml
View File

@@ -1,6 +1,6 @@
[package] [package]
name = "tantivy" name = "tantivy"
version = "0.10.0" version = "0.10.0-dev"
authors = ["Paul Masurel <paul.masurel@gmail.com>"] authors = ["Paul Masurel <paul.masurel@gmail.com>"]
license = "MIT" license = "MIT"
categories = ["database-implementations", "data-structures"] categories = ["database-implementations", "data-structures"]
@@ -10,12 +10,11 @@ homepage = "https://github.com/tantivy-search/tantivy"
repository = "https://github.com/tantivy-search/tantivy" repository = "https://github.com/tantivy-search/tantivy"
readme = "README.md" readme = "README.md"
keywords = ["search", "information", "retrieval"] keywords = ["search", "information", "retrieval"]
edition = "2018"
[dependencies] [dependencies]
base64 = "0.10.0" base64 = "0.10.0"
byteorder = "1.0" byteorder = "1.0"
once_cell = "0.2" lazy_static = "1"
regex = "1.0" regex = "1.0"
tantivy-fst = "0.1" tantivy-fst = "0.1"
memmap = {version = "0.7", optional=true} memmap = {version = "0.7", optional=true}
@@ -43,7 +42,7 @@ owning_ref = "0.4"
stable_deref_trait = "1.0.0" stable_deref_trait = "1.0.0"
rust-stemmers = "1.1" rust-stemmers = "1.1"
downcast-rs = { version="1.0" } downcast-rs = { version="1.0" }
bitpacking = {version="0.8", default-features = false, features=["bitpacker4x"]} bitpacking = "0.6"
census = "0.2" census = "0.2"
fnv = "1.0.6" fnv = "1.0.6"
owned-read = "0.4" owned-read = "0.4"
@@ -55,10 +54,10 @@ murmurhash32 = "0.2"
chrono = "0.4" chrono = "0.4"
[target.'cfg(windows)'.dependencies] [target.'cfg(windows)'.dependencies]
winapi = "0.3" winapi = "0.2"
[dev-dependencies] [dev-dependencies]
rand = "0.7" rand = "0.6"
maplit = "1" maplit = "1"
matches = "0.1.8" matches = "0.1.8"
time = "0.1.42" time = "0.1.42"

59
README.md
View File

@@ -4,7 +4,6 @@
[![Join the chat at https://gitter.im/tantivy-search/tantivy](https://badges.gitter.im/tantivy-search/tantivy.svg)](https://gitter.im/tantivy-search/tantivy?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) [![Join the chat at https://gitter.im/tantivy-search/tantivy](https://badges.gitter.im/tantivy-search/tantivy.svg)](https://gitter.im/tantivy-search/tantivy?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT) [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
[![Build status](https://ci.appveyor.com/api/projects/status/r7nb13kj23u8m9pj/branch/master?svg=true)](https://ci.appveyor.com/project/fulmicoton/tantivy/branch/master) [![Build status](https://ci.appveyor.com/api/projects/status/r7nb13kj23u8m9pj/branch/master?svg=true)](https://ci.appveyor.com/project/fulmicoton/tantivy/branch/master)
[![Crates.io](https://img.shields.io/crates/v/tantivy.svg)](https://crates.io/crates/tantivy)
[![Say Thanks!](https://img.shields.io/badge/Say%20Thanks-!-1EAEDB.svg)](https://saythanks.io/to/fulmicoton) [![Say Thanks!](https://img.shields.io/badge/Say%20Thanks-!-1EAEDB.svg)](https://saythanks.io/to/fulmicoton)
![Tantivy](https://tantivy-search.github.io/logo/tantivy-logo.png) ![Tantivy](https://tantivy-search.github.io/logo/tantivy-logo.png)
@@ -65,6 +64,7 @@ performance for different type of queries / collection.
library upon which one could build a distributed search. Serializable/mergeable collector state for instance, library upon which one could build a distributed search. Serializable/mergeable collector state for instance,
are within the scope of tantivy. are within the scope of tantivy.
# Supported OS and compiler # Supported OS and compiler
Tantivy works on stable rust (>= 1.27) and supports Linux, MacOS and Windows. Tantivy works on stable rust (>= 1.27) and supports Linux, MacOS and Windows.
@@ -80,61 +80,30 @@ It will walk you through getting a wikipedia search engine up and running in a f
- [For the last released version](https://docs.rs/tantivy/) - [For the last released version](https://docs.rs/tantivy/)
- [For the last master branch](https://tantivy-search.github.io/tantivy/tantivy/index.html) - [For the last master branch](https://tantivy-search.github.io/tantivy/tantivy/index.html)
# How can I support this project? # Compiling
There are many ways to support this project. ## Development
- Use tantivy and tell us about your experience on [gitter](https://gitter.im/tantivy-search/tantivy) or by email (paul.masurel@gmail.com)
- Report bugs
- Write a blog post
- Help with documentation by asking questions or submitting PRs
- Contribute code (you can join [our gitter](https://gitter.im/tantivy-search/tantivy) )
- Talk about tantivy around you
- Drop a word on on [![Say Thanks!](https://img.shields.io/badge/Say%20Thanks-!-1EAEDB.svg)](https://saythanks.io/to/fulmicoton) or even [![Become a patron](https://c5.patreon.com/external/logo/become_a_patron_button.png)](https://www.patreon.com/fulmicoton)
# Contributing code
We use the GitHub Pull Request workflow - reference a GitHub ticket and/or include a comprehensive commit message when opening a PR.
## Clone and build locally
Tantivy compiles on stable rust but requires `Rust >= 1.27`. Tantivy compiles on stable rust but requires `Rust >= 1.27`.
To check out and run tests, you can simply run : To check out and run tests, you can simply run :
```bash
git clone https://github.com/tantivy-search/tantivy.git git clone https://github.com/tantivy-search/tantivy.git
cd tantivy cd tantivy
cargo build cargo build
```
## Run tests ## Running tests
Some tests will not run with just `cargo test` because of `fail-rs`. Some tests will not run with just `cargo test` because of `fail-rs`.
To run the tests exhaustively, run `./run-tests.sh` To run the tests exhaustively, run `./run-tests.sh`.
## Debug # How can I support this project ?
You might find it useful to step through the programme with a debugger. There are many ways to support this project.
### A failing test - If you use tantivy, tell us about your experience on [gitter](https://gitter.im/tantivy-search/tantivy) or by email (paul.masurel@gmail.com)
- Report bugs
Make sure you haven't run `cargo clean` after the most recent `cargo test` or `cargo build` to guarantee that `target/` dir exists. Use this bash script to find the most name of the most recent debug build of tantivy and run it under rust-gdb. - Write a blog post
- Complete documentation
```bash - Contribute code (you can join [our gitter](https://gitter.im/tantivy-search/tantivy) )
find target/debug/ -maxdepth 1 -executable -type f -name "tantivy*" -printf '%TY-%Tm-%Td %TT %p\n' | sort -r | cut -d " " -f 3 | xargs -I RECENT_DBG_TANTIVY rust-gdb RECENT_DBG_TANTIVY - Talk about tantivy around you
``` - Drop a word on on [![Say Thanks!](https://img.shields.io/badge/Say%20Thanks-!-1EAEDB.svg)](https://saythanks.io/to/fulmicoton) or even [![Become a patron](https://c5.patreon.com/external/logo/become_a_patron_button.png)](https://www.patreon.com/fulmicoton)
Now that you are in rust-gdb, you can set breakpoints on lines and methods that match your source-code and run the debug executable with flags that you normally pass to `cargo test` to like this
```bash
$gdb run --test-threads 1 --test $NAME_OF_TEST
```
### An example
By default, rustc compiles everything in the `examples/` dir in debug mode. This makes it easy for you to make examples to reproduce bugs.
```bash
rust-gdb target/debug/examples/$EXAMPLE_NAME
$ gdb run
```

2
examples/basic_search.rs
View File

@@ -10,6 +10,8 @@
// - search for the best document matchings "sea whale" // - search for the best document matchings "sea whale"
// - retrieve the best document original content. // - retrieve the best document original content.
extern crate tempdir;
// --- // ---
// Importing tantivy... // Importing tantivy...
#[macro_use] #[macro_use]

2
examples/custom_collector.rs
View File

@@ -7,6 +7,8 @@
// Of course, you can have a look at the tantivy's built-in collectors // Of course, you can have a look at the tantivy's built-in collectors
// such as the `CountCollector` for more examples. // such as the `CountCollector` for more examples.
extern crate tempdir;
// --- // ---
// Importing tantivy... // Importing tantivy...
#[macro_use] #[macro_use]

2
examples/faceted_search.rs
View File

@@ -10,6 +10,8 @@
// - search for the best document matchings "sea whale" // - search for the best document matchings "sea whale"
// - retrieve the best document original content. // - retrieve the best document original content.
extern crate tempdir;
// --- // ---
// Importing tantivy... // Importing tantivy...
#[macro_use] #[macro_use]

2
examples/multiple_producer.rs
View File

@@ -23,6 +23,8 @@
// index a single document?), but aims at demonstrating the mechanism that makes indexing // index a single document?), but aims at demonstrating the mechanism that makes indexing
// from several threads possible. // from several threads possible.
extern crate tempdir;
// --- // ---
// Importing tantivy... // Importing tantivy...
#[macro_use] #[macro_use]

1
examples/snippet.rs
View File

@@ -4,6 +4,7 @@
// your hit result. // your hit result.
// Snippet are an extracted of a target document, and returned in HTML format. // Snippet are an extracted of a target document, and returned in HTML format.
// The keyword searched by the user are highlighted with a `<b>` tag. // The keyword searched by the user are highlighted with a `<b>` tag.
extern crate tempdir;
// --- // ---
// Importing tantivy... // Importing tantivy...

2
examples/stop_words.rs
View File

@@ -9,6 +9,8 @@
// - add a few stop words // - add a few stop words
// - index few documents in our index // - index few documents in our index
extern crate tempdir;
// --- // ---
// Importing tantivy... // Importing tantivy...
#[macro_use] #[macro_use]

2
examples/working_with_json.rs
View File

@@ -1,4 +1,4 @@
use tantivy; extern crate tantivy;
use tantivy::schema::*; use tantivy::schema::*;
// # Document from json // # Document from json

16
src/collector/count_collector.rs
View File

@@ -1,10 +1,10 @@
use super::Collector; use super::Collector;
use crate::collector::SegmentCollector; use collector::SegmentCollector;
use crate::DocId; use DocId;
use crate::Result; use Result;
use crate::Score; use Score;
use crate::SegmentLocalId; use SegmentLocalId;
use crate::SegmentReader; use SegmentReader;
/// `CountCollector` collector only counts how many /// `CountCollector` collector only counts how many
/// documents match the query. /// documents match the query.
@@ -94,8 +94,8 @@ impl SegmentCollector for SegmentCountCollector {
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use super::{Count, SegmentCountCollector}; use super::{Count, SegmentCountCollector};
use crate::collector::Collector; use collector::Collector;
use crate::collector::SegmentCollector; use collector::SegmentCollector;
#[test] #[test]
fn test_count_collect_does_not_requires_scoring() { fn test_count_collect_does_not_requires_scoring() {

126
src/collector/custom_score_top_collector.rs
View File

@@ -1,126 +0,0 @@
use crate::collector::top_collector::{TopCollector, TopSegmentCollector};
use crate::collector::{Collector, SegmentCollector};
use crate::Result;
use crate::{DocAddress, DocId, Score, SegmentReader};
pub(crate) struct CustomScoreTopCollector<TCustomScorer, TScore = Score> {
custom_scorer: TCustomScorer,
collector: TopCollector<TScore>,
}
impl<TCustomScorer, TScore> CustomScoreTopCollector<TCustomScorer, TScore>
where
TScore: Clone + PartialOrd,
{
pub fn new(
custom_scorer: TCustomScorer,
limit: usize,
) -> CustomScoreTopCollector<TCustomScorer, TScore> {
CustomScoreTopCollector {
custom_scorer,
collector: TopCollector::with_limit(limit),
}
}
}
/// A custom segment scorer makes it possible to define any kind of score
/// for a given document belonging to a specific segment.
///
/// It is the segment local version of the [`CustomScorer`](./trait.CustomScorer.html).
pub trait CustomSegmentScorer<TScore>: 'static {
/// Computes the score of a specific `doc`.
fn score(&self, doc: DocId) -> TScore;
}
/// `CustomScorer` makes it possible to define any kind of score.
///
/// The `CustomerScorer` itself does not make much of the computation itself.
/// Instead, it helps constructing `Self::Child` instances that will compute
/// the score at a segment scale.
pub trait CustomScorer<TScore>: Sync {
/// Type of the associated [`CustomSegmentScorer`](./trait.CustomSegmentScorer.html).
type Child: CustomSegmentScorer<TScore>;
/// Builds a child scorer for a specific segment. The child scorer is associated to
/// a specific segment.
fn segment_scorer(&self, segment_reader: &SegmentReader) -> Result<Self::Child>;
}
impl<TCustomScorer, TScore> Collector for CustomScoreTopCollector<TCustomScorer, TScore>
where
TCustomScorer: CustomScorer<TScore>,
TScore: 'static + PartialOrd + Clone + Send + Sync,
{
type Fruit = Vec<(TScore, DocAddress)>;
type Child = CustomScoreTopSegmentCollector<TCustomScorer::Child, TScore>;
fn for_segment(
&self,
segment_local_id: u32,
segment_reader: &SegmentReader,
) -> Result<Self::Child> {
let segment_scorer = self.custom_scorer.segment_scorer(segment_reader)?;
let segment_collector = self
.collector
.for_segment(segment_local_id, segment_reader)?;
Ok(CustomScoreTopSegmentCollector {
segment_collector,
segment_scorer,
})
}
fn requires_scoring(&self) -> bool {
false
}
fn merge_fruits(&self, segment_fruits: Vec<Self::Fruit>) -> Result<Self::Fruit> {
self.collector.merge_fruits(segment_fruits)
}
}
pub struct CustomScoreTopSegmentCollector<T, TScore>
where
TScore: 'static + PartialOrd + Clone + Send + Sync + Sized,
T: CustomSegmentScorer<TScore>,
{
segment_collector: TopSegmentCollector<TScore>,
segment_scorer: T,
}
impl<T, TScore> SegmentCollector for CustomScoreTopSegmentCollector<T, TScore>
where
TScore: 'static + PartialOrd + Clone + Send + Sync,
T: 'static + CustomSegmentScorer<TScore>,
{
type Fruit = Vec<(TScore, DocAddress)>;
fn collect(&mut self, doc: DocId, _score: Score) {
let score = self.segment_scorer.score(doc);
self.segment_collector.collect(doc, score);
}
fn harvest(self) -> Vec<(TScore, DocAddress)> {
self.segment_collector.harvest()
}
}
impl<F, TScore, T> CustomScorer<TScore> for F
where
F: 'static + Send + Sync + Fn(&SegmentReader) -> T,
T: CustomSegmentScorer<TScore>,
{
type Child = T;
fn segment_scorer(&self, segment_reader: &SegmentReader) -> Result<Self::Child> {
Ok((self)(segment_reader))
}
}
impl<F, TScore> CustomSegmentScorer<TScore> for F
where
F: 'static + Sync + Send + Fn(DocId) -> TScore,
{
fn score(&self, doc: DocId) -> TScore {
(self)(doc)
}
}

39
src/collector/facet_collector.rs
View File

@@ -1,15 +1,9 @@
use crate::collector::Collector; use collector::Collector;
use crate::collector::SegmentCollector; use collector::SegmentCollector;
use crate::docset::SkipResult; use docset::SkipResult;
use crate::fastfield::FacetReader; use fastfield::FacetReader;
use crate::schema::Facet; use schema::Facet;
use crate::schema::Field; use schema::Field;
use crate::DocId;
use crate::Result;
use crate::Score;
use crate::SegmentLocalId;
use crate::SegmentReader;
use crate::TantivyError;
use std::cmp::Ordering; use std::cmp::Ordering;
use std::collections::btree_map; use std::collections::btree_map;
use std::collections::BTreeMap; use std::collections::BTreeMap;
@@ -18,6 +12,12 @@ use std::collections::BinaryHeap;
use std::collections::Bound; use std::collections::Bound;
use std::iter::Peekable; use std::iter::Peekable;
use std::{u64, usize}; use std::{u64, usize};
use DocId;
use Result;
use Score;
use SegmentLocalId;
use SegmentReader;
use TantivyError;
struct Hit<'a> { struct Hit<'a> {
count: u64, count: u64,
@@ -27,13 +27,13 @@ struct Hit<'a> {
impl<'a> Eq for Hit<'a> {} impl<'a> Eq for Hit<'a> {}
impl<'a> PartialEq<Hit<'a>> for Hit<'a> { impl<'a> PartialEq<Hit<'a>> for Hit<'a> {
fn eq(&self, other: &Hit<'_>) -> bool { fn eq(&self, other: &Hit) -> bool {
self.count == other.count self.count == other.count
} }
} }
impl<'a> PartialOrd<Hit<'a>> for Hit<'a> { impl<'a> PartialOrd<Hit<'a>> for Hit<'a> {
fn partial_cmp(&self, other: &Hit<'_>) -> Option<Ordering> { fn partial_cmp(&self, other: &Hit) -> Option<Ordering> {
Some(self.cmp(other)) Some(self.cmp(other))
} }
} }
@@ -398,7 +398,7 @@ impl<'a> Iterator for FacetChildIterator<'a> {
} }
impl FacetCounts { impl FacetCounts {
pub fn get<T>(&self, facet_from: T) -> FacetChildIterator<'_> pub fn get<T>(&self, facet_from: T) -> FacetChildIterator
where where
Facet: From<T>, Facet: From<T>,
{ {
@@ -412,8 +412,7 @@ impl FacetCounts {
let facet_after = Facet::from_encoded_string(facet_after_bytes); let facet_after = Facet::from_encoded_string(facet_after_bytes);
Bound::Excluded(facet_after) Bound::Excluded(facet_after)
}; };
let underlying: btree_map::Range<'_, _, _> = let underlying: btree_map::Range<_, _> = self.facet_counts.range((left_bound, right_bound));
self.facet_counts.range((left_bound, right_bound));
FacetChildIterator { underlying } FacetChildIterator { underlying }
} }
@@ -454,12 +453,12 @@ impl FacetCounts {
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use super::{FacetCollector, FacetCounts}; use super::{FacetCollector, FacetCounts};
use crate::core::Index; use core::Index;
use crate::query::AllQuery; use query::AllQuery;
use crate::schema::{Document, Facet, Field, Schema};
use rand::distributions::Uniform; use rand::distributions::Uniform;
use rand::prelude::SliceRandom; use rand::prelude::SliceRandom;
use rand::{thread_rng, Rng}; use rand::{thread_rng, Rng};
use schema::{Document, Facet, Field, Schema};
use std::iter; use std::iter;
#[test] #[test]

21
src/collector/mod.rs
View File

@@ -66,7 +66,7 @@ let (doc_count, top_docs): (usize, Vec<(Score, DocAddress)>) =
The `Collector` trait is implemented for up to 4 collectors. The `Collector` trait is implemented for up to 4 collectors.
If you have more than 4 collectors, you can either group them into If you have more than 4 collectors, you can either group them into
tuples of tuples `(a,(b,(c,d)))`, or rely on [`MultiCollector`](./struct.MultiCollector.html). tuples of tuples `(a,(b,(c,d)))`, or rely on `MultiCollector`'s.
# Combining several collectors dynamically # Combining several collectors dynamically
@@ -85,12 +85,12 @@ See the `custom_collector` example.
*/ */
use crate::DocId; use downcast_rs;
use crate::Result; use DocId;
use crate::Score; use Result;
use crate::SegmentLocalId; use Score;
use crate::SegmentReader; use SegmentLocalId;
use downcast_rs::impl_downcast; use SegmentReader;
mod count_collector; mod count_collector;
pub use self::count_collector::Count; pub use self::count_collector::Count;
@@ -103,11 +103,8 @@ mod top_collector;
mod top_score_collector; mod top_score_collector;
pub use self::top_score_collector::TopDocs; pub use self::top_score_collector::TopDocs;
mod custom_score_top_collector; mod top_field_collector;
pub use self::custom_score_top_collector::{CustomScorer, CustomSegmentScorer}; pub use self::top_field_collector::TopDocsByField;
mod tweak_score_top_collector;
pub use self::tweak_score_top_collector::{ScoreSegmentTweaker, ScoreTweaker};
mod facet_collector; mod facet_collector;
pub use self::facet_collector::FacetCollector; pub use self::facet_collector::FacetCollector;

55
src/collector/multi_collector.rs
View File

@@ -1,30 +1,30 @@
use super::Collector; use super::Collector;
use super::SegmentCollector; use super::SegmentCollector;
use crate::collector::Fruit; use collector::Fruit;
use crate::DocId;
use crate::Result;
use crate::Score;
use crate::SegmentLocalId;
use crate::SegmentReader;
use crate::TantivyError;
use std::marker::PhantomData; use std::marker::PhantomData;
use std::ops::Deref; use std::ops::Deref;
use DocId;
use Result;
use Score;
use SegmentLocalId;
use SegmentReader;
use TantivyError;
pub struct MultiFruit { pub struct MultiFruit {
sub_fruits: Vec<Option<Box<dyn Fruit>>>, sub_fruits: Vec<Option<Box<Fruit>>>,
} }
pub struct CollectorWrapper<TCollector: Collector>(TCollector); pub struct CollectorWrapper<TCollector: Collector>(TCollector);
impl<TCollector: Collector> Collector for CollectorWrapper<TCollector> { impl<TCollector: Collector> Collector for CollectorWrapper<TCollector> {
type Fruit = Box<dyn Fruit>; type Fruit = Box<Fruit>;
type Child = Box<dyn BoxableSegmentCollector>; type Child = Box<BoxableSegmentCollector>;
fn for_segment( fn for_segment(
&self, &self,
segment_local_id: u32, segment_local_id: u32,
reader: &SegmentReader, reader: &SegmentReader,
) -> Result<Box<dyn BoxableSegmentCollector>> { ) -> Result<Box<BoxableSegmentCollector>> {
let child = self.0.for_segment(segment_local_id, reader)?; let child = self.0.for_segment(segment_local_id, reader)?;
Ok(Box::new(SegmentCollectorWrapper(child))) Ok(Box::new(SegmentCollectorWrapper(child)))
} }
@@ -33,7 +33,7 @@ impl<TCollector: Collector> Collector for CollectorWrapper<TCollector> {
self.0.requires_scoring() self.0.requires_scoring()
} }
fn merge_fruits(&self, children: Vec<<Self as Collector>::Fruit>) -> Result<Box<dyn Fruit>> { fn merge_fruits(&self, children: Vec<<Self as Collector>::Fruit>) -> Result<Box<Fruit>> {
let typed_fruit: Vec<TCollector::Fruit> = children let typed_fruit: Vec<TCollector::Fruit> = children
.into_iter() .into_iter()
.map(|untyped_fruit| { .map(|untyped_fruit| {
@@ -50,21 +50,21 @@ impl<TCollector: Collector> Collector for CollectorWrapper<TCollector> {
} }
} }
impl SegmentCollector for Box<dyn BoxableSegmentCollector> { impl SegmentCollector for Box<BoxableSegmentCollector> {
type Fruit = Box<dyn Fruit>; type Fruit = Box<Fruit>;
fn collect(&mut self, doc: u32, score: f32) { fn collect(&mut self, doc: u32, score: f32) {
self.as_mut().collect(doc, score); self.as_mut().collect(doc, score);
} }
fn harvest(self) -> Box<dyn Fruit> { fn harvest(self) -> Box<Fruit> {
BoxableSegmentCollector::harvest_from_box(self) BoxableSegmentCollector::harvest_from_box(self)
} }
} }
pub trait BoxableSegmentCollector { pub trait BoxableSegmentCollector {
fn collect(&mut self, doc: u32, score: f32); fn collect(&mut self, doc: u32, score: f32);
fn harvest_from_box(self: Box<Self>) -> Box<dyn Fruit>; fn harvest_from_box(self: Box<Self>) -> Box<Fruit>;
} }
pub struct SegmentCollectorWrapper<TSegmentCollector: SegmentCollector>(TSegmentCollector); pub struct SegmentCollectorWrapper<TSegmentCollector: SegmentCollector>(TSegmentCollector);
@@ -76,7 +76,7 @@ impl<TSegmentCollector: SegmentCollector> BoxableSegmentCollector
self.0.collect(doc, score); self.0.collect(doc, score);
} }
fn harvest_from_box(self: Box<Self>) -> Box<dyn Fruit> { fn harvest_from_box(self: Box<Self>) -> Box<Fruit> {
Box::new(self.0.harvest()) Box::new(self.0.harvest())
} }
} }
@@ -157,9 +157,8 @@ impl<TFruit: Fruit> FruitHandle<TFruit> {
#[allow(clippy::type_complexity)] #[allow(clippy::type_complexity)]
#[derive(Default)] #[derive(Default)]
pub struct MultiCollector<'a> { pub struct MultiCollector<'a> {
collector_wrappers: Vec< collector_wrappers:
Box<dyn Collector<Child = Box<dyn BoxableSegmentCollector>, Fruit = Box<dyn Fruit>> + 'a>, Vec<Box<Collector<Child = Box<BoxableSegmentCollector>, Fruit = Box<Fruit>> + 'a>>,
>,
} }
impl<'a> MultiCollector<'a> { impl<'a> MultiCollector<'a> {
@@ -208,7 +207,7 @@ impl<'a> Collector for MultiCollector<'a> {
} }
fn merge_fruits(&self, segments_multifruits: Vec<MultiFruit>) -> Result<MultiFruit> { fn merge_fruits(&self, segments_multifruits: Vec<MultiFruit>) -> Result<MultiFruit> {
let mut segment_fruits_list: Vec<Vec<Box<dyn Fruit>>> = (0..self.collector_wrappers.len()) let mut segment_fruits_list: Vec<Vec<Box<Fruit>>> = (0..self.collector_wrappers.len())
.map(|_| Vec::with_capacity(segments_multifruits.len())) .map(|_| Vec::with_capacity(segments_multifruits.len()))
.collect::<Vec<_>>(); .collect::<Vec<_>>();
for segment_multifruit in segments_multifruits { for segment_multifruit in segments_multifruits {
@@ -231,7 +230,7 @@ impl<'a> Collector for MultiCollector<'a> {
} }
pub struct MultiCollectorChild { pub struct MultiCollectorChild {
children: Vec<Box<dyn BoxableSegmentCollector>>, children: Vec<Box<BoxableSegmentCollector>>,
} }
impl SegmentCollector for MultiCollectorChild { impl SegmentCollector for MultiCollectorChild {
@@ -258,12 +257,12 @@ impl SegmentCollector for MultiCollectorChild {
mod tests { mod tests {
use super::*; use super::*;
use crate::collector::{Count, TopDocs}; use collector::{Count, TopDocs};
use crate::query::TermQuery; use query::TermQuery;
use crate::schema::IndexRecordOption; use schema::IndexRecordOption;
use crate::schema::{Schema, TEXT}; use schema::{Schema, TEXT};
use crate::Index; use Index;
use crate::Term; use Term;
#[test] #[test]
fn test_multi_collector() { fn test_multi_collector() {

16
src/collector/tests.rs
View File

@@ -1,12 +1,12 @@
use super::*; use super::*;
use crate::core::SegmentReader; use core::SegmentReader;
use crate::fastfield::BytesFastFieldReader; use fastfield::BytesFastFieldReader;
use crate::fastfield::FastFieldReader; use fastfield::FastFieldReader;
use crate::schema::Field; use schema::Field;
use crate::DocAddress; use DocAddress;
use crate::DocId; use DocId;
use crate::Score; use Score;
use crate::SegmentLocalId; use SegmentLocalId;
/// Stores all of the doc ids. /// Stores all of the doc ids.
/// This collector is only used for tests. /// This collector is only used for tests.

21
src/collector/top_collector.rs
View File

@@ -1,11 +1,11 @@
use crate::DocAddress;
use crate::DocId;
use crate::Result;
use crate::SegmentLocalId;
use crate::SegmentReader;
use serde::export::PhantomData; use serde::export::PhantomData;
use std::cmp::Ordering; use std::cmp::Ordering;
use std::collections::BinaryHeap; use std::collections::BinaryHeap;
use DocAddress;
use DocId;
use Result;
use SegmentLocalId;
use SegmentReader;
/// Contains a feature (field, score, etc.) of a document along with the document address. /// Contains a feature (field, score, etc.) of a document along with the document address.
/// ///
@@ -177,8 +177,9 @@ impl<T: PartialOrd + Clone> TopSegmentCollector<T> {
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use super::TopSegmentCollector; use super::{TopCollector, TopSegmentCollector};
use crate::DocAddress; use DocAddress;
use Score;
#[test] #[test]
fn test_top_collector_not_at_capacity() { fn test_top_collector_not_at_capacity() {
@@ -214,4 +215,10 @@ mod tests {
] ]
); );
} }
#[test]
#[should_panic]
fn test_top_0() {
let _collector: TopCollector<Score> = TopCollector::with_limit(0);
}
} }

271
src/collector/top_field_collector.rs Normal file
View File

@@ -0,0 +1,271 @@
use super::Collector;
use collector::top_collector::TopCollector;
use collector::top_collector::TopSegmentCollector;
use collector::SegmentCollector;
use fastfield::FastFieldReader;
use fastfield::FastValue;
use schema::Field;
use std::marker::PhantomData;
use DocAddress;
use Result;
use SegmentLocalId;
use SegmentReader;
use TantivyError;
/// The Top Field Collector keeps track of the K documents
/// sorted by a fast field in the index
///
/// The implementation is based on a `BinaryHeap`.
/// The theorical complexity for collecting the top `K` out of `n` documents
/// is `O(n log K)`.
///
/// ```rust
/// #[macro_use]
/// extern crate tantivy;
/// # use tantivy::schema::{Schema, Field, FAST, TEXT};
/// # use tantivy::{Index, Result, DocAddress};
/// # use tantivy::query::{Query, QueryParser};
/// use tantivy::Searcher;
/// use tantivy::collector::TopDocs;
///
/// # fn main() -> tantivy::Result<()> {
/// # let mut schema_builder = Schema::builder();
/// # let title = schema_builder.add_text_field("title", TEXT);
/// # let rating = schema_builder.add_u64_field("rating", FAST);
/// # let schema = schema_builder.build();
/// # let index = Index::create_in_ram(schema);
/// # let mut index_writer = index.writer_with_num_threads(1, 3_000_000)?;
/// # index_writer.add_document(doc!(
/// # title => "The Name of the Wind",
/// # rating => 92u64,
/// # ));
/// # index_writer.add_document(doc!(title => "The Diary of Muadib", rating => 97u64));
/// # index_writer.add_document(doc!(title => "A Dairy Cow", rating => 63u64));
/// # index_writer.add_document(doc!(title => "The Diary of a Young Girl", rating => 80u64));
/// # index_writer.commit()?;
/// # let reader = index.reader()?;
/// # let query = QueryParser::for_index(&index, vec![title]).parse_query("diary")?;
/// # let top_docs = docs_sorted_by_rating(&reader.searcher(), &query, rating)?;
/// # assert_eq!(top_docs,
/// # vec![(97u64, DocAddress(0u32, 1)),
/// # (80u64, DocAddress(0u32, 3))]);
/// # Ok(())
/// # }
/// #
/// /// Searches the document matching the given query, and
/// /// collects the top 10 documents, order by the `field`
/// /// given in argument.
/// ///
/// /// `field` is required to be a FAST field.
/// fn docs_sorted_by_rating(searcher: &Searcher,
/// query: &Query,
/// sort_by_field: Field)
/// -> Result<Vec<(u64, DocAddress)>> {
///
/// // This is where we build our collector!
/// let top_docs_by_rating = TopDocs::with_limit(2).order_by_field(sort_by_field);
///
/// // ... and here is our documents. Not this is a simple vec.
/// // The `u64` in the pair is the value of our fast field for each documents.
/// searcher.search(query, &top_docs_by_rating)
/// }
/// ```
pub struct TopDocsByField<T> {
collector: TopCollector<T>,
field: Field,
}
impl<T: FastValue + PartialOrd + Clone> TopDocsByField<T> {
/// Creates a top field collector, with a number of documents equal to "limit".
///
/// The given field name must be a fast field, otherwise the collector have an error while
/// collecting results.
///
/// This constructor is crate-private. Client are supposed to call
/// build `TopDocsByField` object using the `TopDocs` API.
///
/// e.g.:
/// `TopDocs::with_limit(2).order_by_field(sort_by_field)`
///
/// # Panics
/// The method panics if limit is 0
pub(crate) fn new(field: Field, limit: usize) -> TopDocsByField<T> {
TopDocsByField {
collector: TopCollector::with_limit(limit),
field,
}
}
}
impl<T: FastValue + PartialOrd + Send + Sync + 'static> Collector for TopDocsByField<T> {
type Fruit = Vec<(T, DocAddress)>;
type Child = TopFieldSegmentCollector<T>;
fn for_segment(
&self,
segment_local_id: SegmentLocalId,
reader: &SegmentReader,
) -> Result<TopFieldSegmentCollector<T>> {
let collector = self.collector.for_segment(segment_local_id, reader)?;
let reader = reader.fast_fields().u64(self.field).ok_or_else(|| {
let field_name = reader.schema().get_field_name(self.field);
TantivyError::SchemaError(format!("Failed to find fast field reader {:?}", field_name))
})?;
Ok(TopFieldSegmentCollector {
collector,
reader,
_type: PhantomData,
})
}
fn requires_scoring(&self) -> bool {
false
}
fn merge_fruits(
&self,
segment_fruits: Vec<Vec<(T, DocAddress)>>,
) -> Result<Vec<(T, DocAddress)>> {
self.collector.merge_fruits(segment_fruits)
}
}
pub struct TopFieldSegmentCollector<T> {
collector: TopSegmentCollector<u64>,
reader: FastFieldReader<u64>,
_type: PhantomData<T>,
}
impl<T: FastValue + PartialOrd + Send + Sync + 'static> SegmentCollector
for TopFieldSegmentCollector<T>
{
type Fruit = Vec<(T, DocAddress)>;
fn collect(&mut self, doc: u32, _score: f32) {
let field_value = self.reader.get(doc);
self.collector.collect(doc, field_value);
}
fn harvest(self) -> Vec<(T, DocAddress)> {
self.collector
.harvest()
.into_iter()
.map(|(val, doc_address)| (T::from_u64(val), doc_address))
.collect()
}
}
#[cfg(test)]
mod tests {
use super::TopDocsByField;
use collector::Collector;
use collector::TopDocs;
use query::Query;
use query::QueryParser;
use schema::Field;
use schema::IntOptions;
use schema::{Schema, FAST, TEXT};
use DocAddress;
use Index;
use IndexWriter;
use TantivyError;
const TITLE: &str = "title";
const SIZE: &str = "size";
#[test]
fn test_top_collector_not_at_capacity() {
let mut schema_builder = Schema::builder();
let title = schema_builder.add_text_field(TITLE, TEXT);
let size = schema_builder.add_u64_field(SIZE, FAST);
let schema = schema_builder.build();
let (index, query) = index("beer", title, schema, |index_writer| {
index_writer.add_document(doc!(
title => "bottle of beer",
size => 12u64,
));
index_writer.add_document(doc!(
title => "growler of beer",
size => 64u64,
));
index_writer.add_document(doc!(
title => "pint of beer",
size => 16u64,
));
});
let searcher = index.reader().unwrap().searcher();
let top_collector = TopDocs::with_limit(4).order_by_field(size);
let top_docs: Vec<(u64, DocAddress)> = searcher.search(&query, &top_collector).unwrap();
assert_eq!(
top_docs,
vec![
(64, DocAddress(0, 1)),
(16, DocAddress(0, 2)),
(12, DocAddress(0, 0))
]
);
}
#[test]
#[should_panic]
fn test_field_does_not_exist() {
let mut schema_builder = Schema::builder();
let title = schema_builder.add_text_field(TITLE, TEXT);
let size = schema_builder.add_u64_field(SIZE, FAST);
let schema = schema_builder.build();
let (index, _) = index("beer", title, schema, |index_writer| {
index_writer.add_document(doc!(
title => "bottle of beer",
size => 12u64,
));
});
let searcher = index.reader().unwrap().searcher();
let top_collector: TopDocsByField<u64> = TopDocs::with_limit(4).order_by_field(Field(2));
let segment_reader = searcher.segment_reader(0u32);
top_collector
.for_segment(0, segment_reader)
.expect("should panic");
}
#[test]
fn test_field_not_fast_field() {
let mut schema_builder = Schema::builder();
let title = schema_builder.add_text_field(TITLE, TEXT);
let size = schema_builder.add_u64_field(SIZE, IntOptions::default());
let schema = schema_builder.build();
let (index, _) = index("beer", title, schema, |index_writer| {
index_writer.add_document(doc!(
title => "bottle of beer",
size => 12u64,
));
});
let searcher = index.reader().unwrap().searcher();
let segment = searcher.segment_reader(0);
let top_collector: TopDocsByField<u64> = TopDocs::with_limit(4).order_by_field(size);
assert_matches!(
top_collector
.for_segment(0, segment)
.map(|_| ())
.unwrap_err(),
TantivyError::SchemaError(_)
);
}
fn index(
query: &str,
query_field: Field,
schema: Schema,
mut doc_adder: impl FnMut(&mut IndexWriter) -> (),
) -> (Index, Box<Query>) {
let index = Index::create_in_ram(schema);
let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
doc_adder(&mut index_writer);
index_writer.commit().unwrap();
let query_parser = QueryParser::for_index(&index, vec![query_field]);
let query = query_parser.parse_query(query).unwrap();
(index, query)
}
}

440
src/collector/top_score_collector.rs
View File

@@ -1,18 +1,16 @@
use super::Collector; use super::Collector;
use crate::collector::custom_score_top_collector::CustomScoreTopCollector; use collector::top_collector::TopCollector;
use crate::collector::top_collector::TopCollector; use collector::top_collector::TopSegmentCollector;
use crate::collector::top_collector::TopSegmentCollector; use collector::SegmentCollector;
use crate::collector::tweak_score_top_collector::TweakedScoreTopCollector; use collector::TopDocsByField;
use crate::collector::{ use fastfield::FastValue;
CustomScorer, CustomSegmentScorer, ScoreSegmentTweaker, ScoreTweaker, SegmentCollector, use schema::Field;
}; use DocAddress;
use crate::schema::Field; use DocId;
use crate::DocAddress; use Result;
use crate::DocId; use Score;
use crate::Result; use SegmentLocalId;
use crate::Score; use SegmentReader;
use crate::SegmentLocalId;
use crate::SegmentReader;
/// The Top Score Collector keeps track of the K documents /// The Top Score Collector keeps track of the K documents
/// sorted by their score. /// sorted by their score.
@@ -79,311 +77,13 @@ impl TopDocs {
/// Set top-K to rank documents by a given fast field. /// Set top-K to rank documents by a given fast field.
/// ///
/// ```rust /// (By default, `TopDocs` collects the top-K documents sorted by
/// #[macro_use] /// the similarity score.)
/// extern crate tantivy; pub fn order_by_field<T: PartialOrd + FastValue + Clone>(
/// # use tantivy::schema::{Schema, FAST, TEXT};
/// # use tantivy::{Index, Result, DocAddress};
/// # use tantivy::query::{Query, QueryParser};
/// use tantivy::Searcher;
/// use tantivy::collector::TopDocs;
/// use tantivy::schema::Field;
///
/// # fn main() -> tantivy::Result<()> {
/// # let mut schema_builder = Schema::builder();
/// # let title = schema_builder.add_text_field("title", TEXT);
/// # let rating = schema_builder.add_u64_field("rating", FAST);
/// # let schema = schema_builder.build();
/// #
/// # let index = Index::create_in_ram(schema);
/// # let mut index_writer = index.writer_with_num_threads(1, 3_000_000)?;
/// # index_writer.add_document(doc!(
/// # title => "The Name of the Wind",
/// # rating => 92u64,
/// # ));
/// # index_writer.add_document(doc!(title => "The Diary of Muadib", rating => 97u64));
/// # index_writer.add_document(doc!(title => "A Dairy Cow", rating => 63u64));
/// # index_writer.add_document(doc!(title => "The Diary of a Young Girl", rating => 80u64));
/// # index_writer.commit()?;
/// # let reader = index.reader()?;
/// # let query = QueryParser::for_index(&index, vec![title]).parse_query("diary")?;
/// # let top_docs = docs_sorted_by_rating(&reader.searcher(), &query, rating)?;
/// # assert_eq!(top_docs,
/// # vec![(97u64, DocAddress(0u32, 1)),
/// # (80u64, DocAddress(0u32, 3))]);
/// # Ok(())
/// # }
///
///
/// /// Searches the document matching the given query, and
/// /// collects the top 10 documents, order by the u64-`field`
/// /// given in argument.
/// ///
/// /// `field` is required to be a FAST field.
/// fn docs_sorted_by_rating(searcher: &Searcher,
/// query: &Query,
/// sort_by_field: Field)
/// -> Result<Vec<(u64, DocAddress)>> {
///
/// // This is where we build our topdocs collector
/// //
/// // Note the generics parameter that needs to match the
/// // type `sort_by_field`.
/// let top_docs_by_rating = TopDocs
/// ::with_limit(10)
/// .order_by_u64_field(sort_by_field);
///
/// // ... and here are our documents. Note this is a simple vec.
/// // The `u64` in the pair is the value of our fast field for
/// // each documents.
/// //
/// // The vec is sorted decreasingly by `sort_by_field`, and has a
/// // length of 10, or less if not enough documents matched the
/// // query.
/// let resulting_docs: Vec<(u64, DocAddress)> =
/// searcher.search(query, &top_docs_by_rating)?;
///
/// Ok(resulting_docs)
/// }
/// ```
///
/// # Panics
///
/// May panic if the field requested is not a fast field.
///
pub fn order_by_u64_field(
self, self,
field: Field, field: Field,
) -> impl Collector<Fruit = Vec<(u64, DocAddress)>> { ) -> TopDocsByField<T> {
self.custom_score(move |segment_reader: &SegmentReader| { TopDocsByField::new(field, self.0.limit())
let ff_reader = segment_reader
.fast_fields()
.u64(field)
.expect("Field requested is not a i64/u64 fast field.");
move |doc: DocId| ff_reader.get(doc)
})
}
/// Ranks the documents using a custom score.
///
/// This method offers a convenient way to tweak or replace
/// the documents score. As suggested by the prototype you can
/// manually define your own [`ScoreTweaker`](./trait.ScoreTweaker.html)
/// and pass it as an argument, but there is a much simpler way to
/// tweak your score: you can use a closure as in the following
/// example.
///
/// # Example
///
/// Typically, you will want to rely on one or more fast fields,
/// to alter the original relevance `Score`.
///
/// For instance, in the following, we assume that we are implementing
/// an e-commerce website that has a fast field called `popularity`
/// that rates whether a product is typically often bought by users.
///
/// In the following example will will tweak our ranking a bit by
/// boosting popular products a notch.
///
/// In more serious application, this tweaking could involved running a
/// learning-to-rank model over various features
///
/// ```rust
/// #[macro_use]
/// extern crate tantivy;
/// # use tantivy::schema::{Schema, FAST, TEXT};
/// # use tantivy::{Index, DocAddress, DocId, Score};
/// # use tantivy::query::QueryParser;
/// use tantivy::SegmentReader;
/// use tantivy::collector::TopDocs;
/// use tantivy::schema::Field;
///
/// # fn create_schema() -> Schema {
/// # let mut schema_builder = Schema::builder();
/// # schema_builder.add_text_field("product_name", TEXT);
/// # schema_builder.add_u64_field("popularity", FAST);
/// # schema_builder.build()
/// # }
/// #
/// # fn main() -> tantivy::Result<()> {
/// # let schema = create_schema();
/// # let index = Index::create_in_ram(schema);
/// # let mut index_writer = index.writer_with_num_threads(1, 3_000_000)?;
/// # let product_name = index.schema().get_field("product_name").unwrap();
/// #
/// let popularity: Field = index.schema().get_field("popularity").unwrap();
/// # index_writer.add_document(doc!(product_name => "The Diary of Muadib", popularity => 1u64));
/// # index_writer.add_document(doc!(product_name => "A Dairy Cow", popularity => 10u64));
/// # index_writer.add_document(doc!(product_name => "The Diary of a Young Girl", popularity => 15u64));
/// # index_writer.commit()?;
/// // ...
/// # let user_query = "diary";
/// # let query = QueryParser::for_index(&index, vec![product_name]).parse_query(user_query)?;
///
/// // This is where we build our collector with our custom score.
/// let top_docs_by_custom_score = TopDocs
/// ::with_limit(10)
/// .tweak_score(move |segment_reader: &SegmentReader| {
/// // The argument is a function that returns our scoring
/// // function.
/// //
/// // The point of this "mother" function is to gather all
/// // of the segment level information we need for scoring.
/// // Typically, fast_fields.
/// //
/// // In our case, we will get a reader for the popularity
/// // fast field.
/// let popularity_reader =
/// segment_reader.fast_fields().u64(popularity).unwrap();
///
/// // We can now define our actual scoring function
/// move |doc: DocId, original_score: Score| {
/// let popularity: u64 = popularity_reader.get(doc);
/// // Well.. For the sake of the example we use a simple logarithm
/// // function.
/// let popularity_boost_score = ((2u64 + popularity) as f32).log2();
/// popularity_boost_score * original_score
/// }
/// });
/// # let reader = index.reader()?;
/// # let searcher = reader.searcher();
/// // ... and here are our documents. Note this is a simple vec.
/// // The `Score` in the pair is our tweaked score.
/// let resulting_docs: Vec<(Score, DocAddress)> =
/// searcher.search(&*query, &top_docs_by_custom_score)?;
///
/// # Ok(())
/// # }
/// ```
///
/// # See also
/// [custom_score(...)](#method.custom_score).
pub fn tweak_score<TScore, TScoreSegmentTweaker, TScoreTweaker>(
self,
score_tweaker: TScoreTweaker,
) -> impl Collector<Fruit = Vec<(TScore, DocAddress)>>
where
TScore: 'static + Send + Sync + Clone + PartialOrd,
TScoreSegmentTweaker: ScoreSegmentTweaker<TScore> + 'static,
TScoreTweaker: ScoreTweaker<TScore, Child = TScoreSegmentTweaker>,
{
TweakedScoreTopCollector::new(score_tweaker, self.0.limit())
}
/// Ranks the documents using a custom score.
///
/// This method offers a convenient way to use a different score.
///
/// As suggested by the prototype you can manually define your
/// own [`CustomScorer`](./trait.CustomScorer.html)
/// and pass it as an argument, but there is a much simpler way to
/// tweak your score: you can use a closure as in the following
/// example.
///
/// # Limitation
///
/// This method only makes it possible to compute the score from a given
/// `DocId`, fastfield values for the doc and any information you could
/// have precomputed beforehands. It does not make it possible for instance
/// to compute something like TfIdf as it does not have access to the list of query
/// terms present in the document, nor the term frequencies for the different terms.
///
/// It can be used if your search engine relies on a learning-to-rank model for instance,
/// which does not rely on the term frequencies or positions as features.
///
/// # Example
///
/// ```rust
/// # #[macro_use]
/// # extern crate tantivy;
/// # use tantivy::schema::{Schema, FAST, TEXT};
/// # use tantivy::{Index, DocAddress, DocId};
/// # use tantivy::query::QueryParser;
/// use tantivy::SegmentReader;
/// use tantivy::collector::TopDocs;
/// use tantivy::schema::Field;
///
/// # fn create_schema() -> Schema {
/// # let mut schema_builder = Schema::builder();
/// # schema_builder.add_text_field("product_name", TEXT);
/// # schema_builder.add_u64_field("popularity", FAST);
/// # schema_builder.add_u64_field("boosted", FAST);
/// # schema_builder.build()
/// # }
/// #
/// # fn main() -> tantivy::Result<()> {
/// # let schema = create_schema();
/// # let index = Index::create_in_ram(schema);
/// # let mut index_writer = index.writer_with_num_threads(1, 3_000_000)?;
/// # let product_name = index.schema().get_field("product_name").unwrap();
/// #
/// let popularity: Field = index.schema().get_field("popularity").unwrap();
/// let boosted: Field = index.schema().get_field("boosted").unwrap();
/// # index_writer.add_document(doc!(boosted=>1u64, product_name => "The Diary of Muadib", popularity => 1u64));
/// # index_writer.add_document(doc!(boosted=>0u64, product_name => "A Dairy Cow", popularity => 10u64));
/// # index_writer.add_document(doc!(boosted=>0u64, product_name => "The Diary of a Young Girl", popularity => 15u64));
/// # index_writer.commit()?;
/// // ...
/// # let user_query = "diary";
/// # let query = QueryParser::for_index(&index, vec![product_name]).parse_query(user_query)?;
///
/// // This is where we build our collector with our custom score.
/// let top_docs_by_custom_score = TopDocs
/// ::with_limit(10)
/// .custom_score(move |segment_reader: &SegmentReader| {
/// // The argument is a function that returns our scoring
/// // function.
/// //
/// // The point of this "mother" function is to gather all
/// // of the segment level information we need for scoring.
/// // Typically, fast_fields.
/// //
/// // In our case, we will get a reader for the popularity
/// // fast field and a boosted field.
/// //
/// // We want to get boosted items score, and when we get
/// // a tie, return the item with the highest popularity.
/// //
/// // Note that this is implemented by using a `(u64, u64)`
/// // as a score.
/// let popularity_reader =
/// segment_reader.fast_fields().u64(popularity).unwrap();
/// let boosted_reader =
/// segment_reader.fast_fields().u64(boosted).unwrap();
///
/// // We can now define our actual scoring function
/// move |doc: DocId| {
/// let popularity: u64 = popularity_reader.get(doc);
/// let boosted: u64 = boosted_reader.get(doc);
/// // Score do not have to be `f64` in tantivy.
/// // Here we return a couple to get lexicographical order
/// // for free.
/// (boosted, popularity)
/// }
/// });
/// # let reader = index.reader()?;
/// # let searcher = reader.searcher();
/// // ... and here are our documents. Note this is a simple vec.
/// // The `Score` in the pair is our tweaked score.
/// let resulting_docs: Vec<((u64, u64), DocAddress)> =
/// searcher.search(&*query, &top_docs_by_custom_score)?;
///
/// # Ok(())
/// # }
/// ```
///
/// # See also
/// [tweak_score(...)](#method.tweak_score).
pub fn custom_score<TScore, TCustomSegmentScorer, TCustomScorer>(
self,
custom_score: TCustomScorer,
) -> impl Collector<Fruit = Vec<(TScore, DocAddress)>>
where
TScore: 'static + Send + Sync + Clone + PartialOrd,
TCustomSegmentScorer: CustomSegmentScorer<TScore> + 'static,
TCustomScorer: CustomScorer<TScore, Child = TCustomSegmentScorer>,
{
CustomScoreTopCollector::new(custom_score, self.0.limit())
} }
} }
@@ -428,13 +128,12 @@ impl SegmentCollector for TopScoreSegmentCollector {
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use super::TopDocs; use super::TopDocs;
use crate::collector::Collector; use query::QueryParser;
use crate::query::{Query, QueryParser}; use schema::Schema;
use crate::schema::{Field, Schema, FAST, STORED, TEXT}; use schema::TEXT;
use crate::DocAddress; use DocAddress;
use crate::Index; use Index;
use crate::IndexWriter; use Score;
use crate::Score;
fn make_index() -> Index { fn make_index() -> Index {
let mut schema_builder = Schema::builder(); let mut schema_builder = Schema::builder();
@@ -501,97 +200,4 @@ mod tests {
TopDocs::with_limit(0); TopDocs::with_limit(0);
} }
const TITLE: &str = "title";
const SIZE: &str = "size";
#[test]
fn test_top_field_collector_not_at_capacity() {
let mut schema_builder = Schema::builder();
let title = schema_builder.add_text_field(TITLE, TEXT);
let size = schema_builder.add_u64_field(SIZE, FAST);
let schema = schema_builder.build();
let (index, query) = index("beer", title, schema, |index_writer| {
index_writer.add_document(doc!(
title => "bottle of beer",
size => 12u64,
));
index_writer.add_document(doc!(
title => "growler of beer",
size => 64u64,
));
index_writer.add_document(doc!(
title => "pint of beer",
size => 16u64,
));
});
let searcher = index.reader().unwrap().searcher();
let top_collector = TopDocs::with_limit(4).order_by_u64_field(size);
let top_docs: Vec<(u64, DocAddress)> = searcher.search(&query, &top_collector).unwrap();
assert_eq!(
top_docs,
vec![
(64, DocAddress(0, 1)),
(16, DocAddress(0, 2)),
(12, DocAddress(0, 0))
]
);
}
#[test]
#[should_panic]
fn test_field_does_not_exist() {
let mut schema_builder = Schema::builder();
let title = schema_builder.add_text_field(TITLE, TEXT);
let size = schema_builder.add_u64_field(SIZE, FAST);
let schema = schema_builder.build();
let (index, _) = index("beer", title, schema, |index_writer| {
index_writer.add_document(doc!(
title => "bottle of beer",
size => 12u64,
));
});
let searcher = index.reader().unwrap().searcher();
let top_collector = TopDocs::with_limit(4).order_by_u64_field(Field(2));
let segment_reader = searcher.segment_reader(0u32);
top_collector
.for_segment(0, segment_reader)
.expect("should panic");
}
#[test]
#[should_panic(expected = "Field requested is not a i64/u64 fast field")]
fn test_field_not_fast_field() {
let mut schema_builder = Schema::builder();
let title = schema_builder.add_text_field(TITLE, TEXT);
let size = schema_builder.add_u64_field(SIZE, STORED);
let schema = schema_builder.build();
let (index, _) = index("beer", title, schema, |index_writer| {
index_writer.add_document(doc!(
title => "bottle of beer",
size => 12u64,
));
});
let searcher = index.reader().unwrap().searcher();
let segment = searcher.segment_reader(0);
let top_collector = TopDocs::with_limit(4).order_by_u64_field(size);
assert!(top_collector.for_segment(0, segment).is_ok());
}
fn index(
query: &str,
query_field: Field,
schema: Schema,
mut doc_adder: impl FnMut(&mut IndexWriter) -> (),
) -> (Index, Box<Query>) {
let index = Index::create_in_ram(schema);
let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
doc_adder(&mut index_writer);
index_writer.commit().unwrap();
let query_parser = QueryParser::for_index(&index, vec![query_field]);
let query = query_parser.parse_query(query).unwrap();
(index, query)
}
} }

129
src/collector/tweak_score_top_collector.rs
View File

@@ -1,129 +0,0 @@
use crate::collector::top_collector::{TopCollector, TopSegmentCollector};
use crate::collector::{Collector, SegmentCollector};
use crate::DocAddress;
use crate::{DocId, Result, Score, SegmentReader};
pub(crate) struct TweakedScoreTopCollector<TScoreTweaker, TScore = Score> {
score_tweaker: TScoreTweaker,
collector: TopCollector<TScore>,
}
impl<TScoreTweaker, TScore> TweakedScoreTopCollector<TScoreTweaker, TScore>
where
TScore: Clone + PartialOrd,
{
pub fn new(
score_tweaker: TScoreTweaker,
limit: usize,
) -> TweakedScoreTopCollector<TScoreTweaker, TScore> {
TweakedScoreTopCollector {
score_tweaker,
collector: TopCollector::with_limit(limit),
}
}
}
/// A `ScoreSegmentTweaker` makes it possible to modify the default score
/// for a given document belonging to a specific segment.
///
/// It is the segment local version of the [`ScoreTweaker`](./trait.ScoreTweaker.html).
pub trait ScoreSegmentTweaker<TScore>: 'static {
/// Tweak the given `score` for the document `doc`.
fn score(&self, doc: DocId, score: Score) -> TScore;
}
/// `ScoreTweaker` makes it possible to tweak the score
/// emitted by the scorer into another one.
///
/// The `ScoreTweaker` itself does not make much of the computation itself.
/// Instead, it helps constructing `Self::Child` instances that will compute
/// the score at a segment scale.
pub trait ScoreTweaker<TScore>: Sync {
/// Type of the associated [`ScoreSegmentTweaker`](./trait.ScoreSegmentTweaker.html).
type Child: ScoreSegmentTweaker<TScore>;
/// Builds a child tweaker for a specific segment. The child scorer is associated to
/// a specific segment.
fn segment_tweaker(&self, segment_reader: &SegmentReader) -> Result<Self::Child>;
}
impl<TScoreTweaker, TScore> Collector for TweakedScoreTopCollector<TScoreTweaker, TScore>
where
TScoreTweaker: ScoreTweaker<TScore>,
TScore: 'static + PartialOrd + Clone + Send + Sync,
{
type Fruit = Vec<(TScore, DocAddress)>;
type Child = TopTweakedScoreSegmentCollector<TScoreTweaker::Child, TScore>;
fn for_segment(
&self,
segment_local_id: u32,
segment_reader: &SegmentReader,
) -> Result<Self::Child> {
let segment_scorer = self.score_tweaker.segment_tweaker(segment_reader)?;
let segment_collector = self
.collector
.for_segment(segment_local_id, segment_reader)?;
Ok(TopTweakedScoreSegmentCollector {
segment_collector,
segment_scorer,
})
}
fn requires_scoring(&self) -> bool {
true
}
fn merge_fruits(&self, segment_fruits: Vec<Self::Fruit>) -> Result<Self::Fruit> {
self.collector.merge_fruits(segment_fruits)
}
}
pub struct TopTweakedScoreSegmentCollector<TSegmentScoreTweaker, TScore>
where
TScore: 'static + PartialOrd + Clone + Send + Sync + Sized,
TSegmentScoreTweaker: ScoreSegmentTweaker<TScore>,
{
segment_collector: TopSegmentCollector<TScore>,
segment_scorer: TSegmentScoreTweaker,
}
impl<TSegmentScoreTweaker, TScore> SegmentCollector
for TopTweakedScoreSegmentCollector<TSegmentScoreTweaker, TScore>
where
TScore: 'static + PartialOrd + Clone + Send + Sync,
TSegmentScoreTweaker: 'static + ScoreSegmentTweaker<TScore>,
{
type Fruit = Vec<(TScore, DocAddress)>;
fn collect(&mut self, doc: DocId, score: Score) {
let score = self.segment_scorer.score(doc, score);
self.segment_collector.collect(doc, score);
}
fn harvest(self) -> Vec<(TScore, DocAddress)> {
self.segment_collector.harvest()
}
}
impl<F, TScore, TSegmentScoreTweaker> ScoreTweaker<TScore> for F
where
F: 'static + Send + Sync + Fn(&SegmentReader) -> TSegmentScoreTweaker,
TSegmentScoreTweaker: ScoreSegmentTweaker<TScore>,
{
type Child = TSegmentScoreTweaker;
fn segment_tweaker(&self, segment_reader: &SegmentReader) -> Result<Self::Child> {
Ok((self)(segment_reader))
}
}
impl<F, TScore> ScoreSegmentTweaker<TScore> for F
where
F: 'static + Sync + Send + Fn(DocId, Score) -> TScore,
{
fn score(&self, doc: DocId, score: Score) -> TScore {
(self)(doc, score)
}
}

10
src/common/bitset.rs
View File

@@ -5,7 +5,7 @@ use std::u64;
pub(crate) struct TinySet(u64); pub(crate) struct TinySet(u64);
impl fmt::Debug for TinySet { impl fmt::Debug for TinySet {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
self.into_iter().collect::<Vec<u32>>().fmt(f) self.into_iter().collect::<Vec<u32>>().fmt(f)
} }
} }
@@ -204,12 +204,12 @@ mod tests {
use super::BitSet; use super::BitSet;
use super::TinySet; use super::TinySet;
use crate::docset::DocSet; use docset::DocSet;
use crate::query::BitSetDocSet; use query::BitSetDocSet;
use crate::tests;
use crate::tests::generate_nonunique_unsorted;
use std::collections::BTreeSet; use std::collections::BTreeSet;
use std::collections::HashSet; use std::collections::HashSet;
use tests;
use tests::generate_nonunique_unsorted;
#[test] #[test]
fn test_tiny_set() { fn test_tiny_set() {

24
src/common/composite_file.rs
View File

@@ -1,11 +1,11 @@
use crate::common::BinarySerializable; use common::BinarySerializable;
use crate::common::CountingWriter; use common::CountingWriter;
use crate::common::VInt; use common::VInt;
use crate::directory::ReadOnlySource; use directory::ReadOnlySource;
use crate::directory::WritePtr; use directory::WritePtr;
use crate::schema::Field; use schema::Field;
use crate::space_usage::FieldUsage; use space_usage::FieldUsage;
use crate::space_usage::PerFieldSpaceUsage; use space_usage::PerFieldSpaceUsage;
use std::collections::HashMap; use std::collections::HashMap;
use std::io::Write; use std::io::Write;
use std::io::{self, Read}; use std::io::{self, Read};
@@ -185,10 +185,10 @@ impl CompositeFile {
mod test { mod test {
use super::{CompositeFile, CompositeWrite}; use super::{CompositeFile, CompositeWrite};
use crate::common::BinarySerializable; use common::BinarySerializable;
use crate::common::VInt; use common::VInt;
use crate::directory::{Directory, RAMDirectory}; use directory::{Directory, RAMDirectory};
use crate::schema::Field; use schema::Field;
use std::io::Write; use std::io::Write;
use std::path::Path; use std::path::Path;

6
src/common/serialize.rs
View File

@@ -1,6 +1,6 @@
use crate::common::Endianness;
use crate::common::VInt;
use byteorder::{ReadBytesExt, WriteBytesExt}; use byteorder::{ReadBytesExt, WriteBytesExt};
use common::Endianness;
use common::VInt;
use std::fmt; use std::fmt;
use std::io; use std::io;
use std::io::Read; use std::io::Read;
@@ -136,7 +136,7 @@ impl BinarySerializable for String {
pub mod test { pub mod test {
use super::*; use super::*;
use crate::common::VInt; use common::VInt;
pub fn fixed_size_test<O: BinarySerializable + FixedSize + Default>() { pub fn fixed_size_test<O: BinarySerializable + FixedSize + Default>() {
let mut buffer = Vec::new(); let mut buffer = Vec::new();

10
src/common/vint.rs
View File

@@ -30,16 +30,16 @@ pub fn serialize_vint_u32(val: u32) -> (u64, usize) {
let val = u64::from(val); let val = u64::from(val);
const STOP_BIT: u64 = 128u64; const STOP_BIT: u64 = 128u64;
match val { match val {
0..=STOP_1 => (val | STOP_BIT, 1), 0...STOP_1 => (val | STOP_BIT, 1),
START_2..=STOP_2 => ( START_2...STOP_2 => (
(val & MASK_1) | ((val & MASK_2) << 1) | (STOP_BIT << (8)), (val & MASK_1) | ((val & MASK_2) << 1) | (STOP_BIT << (8)),
2, 2,
), ),
START_3..=STOP_3 => ( START_3...STOP_3 => (
(val & MASK_1) | ((val & MASK_2) << 1) | ((val & MASK_3) << 2) | (STOP_BIT << (8 * 2)), (val & MASK_1) | ((val & MASK_2) << 1) | ((val & MASK_3) << 2) | (STOP_BIT << (8 * 2)),
3, 3,
), ),
START_4..=STOP_4 => ( START_4...STOP_4 => (
(val & MASK_1) (val & MASK_1)
| ((val & MASK_2) << 1) | ((val & MASK_2) << 1)
| ((val & MASK_3) << 2) | ((val & MASK_3) << 2)
@@ -171,8 +171,8 @@ mod tests {
use super::serialize_vint_u32; use super::serialize_vint_u32;
use super::VInt; use super::VInt;
use crate::common::BinarySerializable;
use byteorder::{ByteOrder, LittleEndian}; use byteorder::{ByteOrder, LittleEndian};
use common::BinarySerializable;
fn aux_test_vint(val: u64) { fn aux_test_vint(val: u64) {
let mut v = [14u8; 10]; let mut v = [14u8; 10];

2
src/core/executor.rs
View File

@@ -1,6 +1,6 @@
use crate::Result;
use crossbeam::channel; use crossbeam::channel;
use scoped_pool::{Pool, ThreadConfig}; use scoped_pool::{Pool, ThreadConfig};
use Result;
/// Search executor whether search request are single thread or multithread. /// Search executor whether search request are single thread or multithread.
/// ///

104
src/core/index.rs
View File

@@ -1,44 +1,44 @@
use super::segment::create_segment; use super::segment::create_segment;
use super::segment::Segment; use super::segment::Segment;
use crate::core::Executor; use core::Executor;
use crate::core::IndexMeta; use core::IndexMeta;
use crate::core::SegmentId; use core::SegmentId;
use crate::core::SegmentMeta; use core::SegmentMeta;
use crate::core::META_FILEPATH; use core::META_FILEPATH;
use crate::directory::ManagedDirectory; use directory::ManagedDirectory;
#[cfg(feature = "mmap")] #[cfg(feature = "mmap")]
use crate::directory::MmapDirectory; use directory::MmapDirectory;
use crate::directory::INDEX_WRITER_LOCK; use directory::INDEX_WRITER_LOCK;
use crate::directory::{Directory, RAMDirectory}; use directory::{Directory, RAMDirectory};
use crate::error::DataCorruption; use error::DataCorruption;
use crate::error::TantivyError; use error::TantivyError;
use crate::indexer::index_writer::open_index_writer; use indexer::index_writer::open_index_writer;
use crate::indexer::index_writer::HEAP_SIZE_MIN; use indexer::index_writer::HEAP_SIZE_MIN;
use crate::indexer::segment_updater::save_new_metas; use indexer::segment_updater::save_new_metas;
use crate::reader::IndexReader;
use crate::reader::IndexReaderBuilder;
use crate::schema::Field;
use crate::schema::FieldType;
use crate::schema::Schema;
use crate::tokenizer::BoxedTokenizer;
use crate::tokenizer::TokenizerManager;
use crate::IndexWriter;
use crate::Result;
use num_cpus; use num_cpus;
use reader::IndexReader;
use reader::IndexReaderBuilder;
use schema::Field;
use schema::FieldType;
use schema::Schema;
use serde_json; use serde_json;
use std::borrow::BorrowMut; use std::borrow::BorrowMut;
use std::fmt; use std::fmt;
#[cfg(feature = "mmap")] #[cfg(feature = "mmap")]
use std::path::Path; use std::path::Path;
use std::sync::Arc; use std::sync::Arc;
use tokenizer::BoxedTokenizer;
use tokenizer::TokenizerManager;
use IndexWriter;
use Result;
fn load_metas(directory: &dyn Directory) -> Result<IndexMeta> { fn load_metas(directory: &Directory) -> Result<IndexMeta> {
let meta_data = directory.atomic_read(&META_FILEPATH)?; let meta_data = directory.atomic_read(&META_FILEPATH)?;
let meta_string = String::from_utf8_lossy(&meta_data); let meta_string = String::from_utf8_lossy(&meta_data);
serde_json::from_str(&meta_string) serde_json::from_str(&meta_string)
.map_err(|e| { .map_err(|e| {
DataCorruption::new( DataCorruption::new(
META_FILEPATH.to_path_buf(), META_FILEPATH.clone(),
format!("Meta file cannot be deserialized. {:?}.", e), format!("Meta file cannot be deserialized. {:?}.", e),
) )
}) })
@@ -169,11 +169,11 @@ impl Index {
} }
/// Helper to access the tokenizer associated to a specific field. /// Helper to access the tokenizer associated to a specific field.
pub fn tokenizer_for_field(&self, field: Field) -> Result<Box<dyn BoxedTokenizer>> { pub fn tokenizer_for_field(&self, field: Field) -> Result<Box<BoxedTokenizer>> {
let field_entry = self.schema.get_field_entry(field); let field_entry = self.schema.get_field_entry(field);
let field_type = field_entry.field_type(); let field_type = field_entry.field_type();
let tokenizer_manager: &TokenizerManager = self.tokenizers(); let tokenizer_manager: &TokenizerManager = self.tokenizers();
let tokenizer_name_opt: Option<Box<dyn BoxedTokenizer>> = match field_type { let tokenizer_name_opt: Option<Box<BoxedTokenizer>> = match field_type {
FieldType::Str(text_options) => text_options FieldType::Str(text_options) => text_options
.get_indexing_options() .get_indexing_options()
.map(|text_indexing_options| text_indexing_options.tokenizer().to_string()) .map(|text_indexing_options| text_indexing_options.tokenizer().to_string())
@@ -346,22 +346,22 @@ impl Index {
} }
impl fmt::Debug for Index { impl fmt::Debug for Index {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
write!(f, "Index({:?})", self.directory) write!(f, "Index({:?})", self.directory)
} }
} }
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use crate::directory::RAMDirectory; use directory::RAMDirectory;
use crate::schema::Field; use schema::Field;
use crate::schema::{Schema, INDEXED, TEXT}; use schema::{Schema, INDEXED, TEXT};
use crate::Index;
use crate::IndexReader;
use crate::IndexWriter;
use crate::ReloadPolicy;
use std::thread; use std::thread;
use std::time::Duration; use std::time::Duration;
use Index;
use IndexReader;
use IndexWriter;
use ReloadPolicy;
#[test] #[test]
fn test_indexer_for_field() { fn test_indexer_for_field() {
@@ -537,38 +537,4 @@ mod tests {
} }
assert_eq!(count, 2); assert_eq!(count, 2);
} }
// This test will not pass on windows, because windows
// prevent deleting files that are MMapped.
#[cfg(not(target_os = "windows"))]
#[test]
fn garbage_collect_works_as_intended() {
let directory = RAMDirectory::create();
let schema = throw_away_schema();
let field = schema.get_field("num_likes").unwrap();
let index = Index::create(directory.clone(), schema).unwrap();
let mut writer = index.writer_with_num_threads(8, 24_000_000).unwrap();
for i in 0u64..8_000u64 {
writer.add_document(doc!(field => i));
}
writer.commit().unwrap();
let mem_right_after_commit = directory.total_mem_usage();
thread::sleep(Duration::from_millis(1_000));
let reader = index
.reader_builder()
.reload_policy(ReloadPolicy::Manual)
.try_into()
.unwrap();
assert_eq!(reader.searcher().num_docs(), 8_000);
writer.wait_merging_threads().unwrap();
let mem_right_after_merge_finished = directory.total_mem_usage();
reader.reload().unwrap();
let searcher = reader.searcher();
assert_eq!(searcher.num_docs(), 8_000);
assert!(mem_right_after_merge_finished < mem_right_after_commit);
}
} }

23
src/core/index_meta.rs
View File

@@ -1,8 +1,8 @@
use crate::core::SegmentMeta; use core::SegmentMeta;
use crate::schema::Schema; use schema::Schema;
use crate::Opstamp;
use serde_json; use serde_json;
use std::fmt; use std::fmt;
use Opstamp;
/// Meta information about the `Index`. /// Meta information about the `Index`.
/// ///
@@ -14,27 +14,14 @@ use std::fmt;
/// ///
#[derive(Clone, Serialize, Deserialize)] #[derive(Clone, Serialize, Deserialize)]
pub struct IndexMeta { pub struct IndexMeta {
/// List of `SegmentMeta` informations associated to each finalized segment of the index.
pub segments: Vec<SegmentMeta>, pub segments: Vec<SegmentMeta>,
/// Index `Schema`
pub schema: Schema, pub schema: Schema,
/// Opstamp associated to the last `commit` operation.
pub opstamp: Opstamp, pub opstamp: Opstamp,
#[serde(skip_serializing_if = "Option::is_none")] #[serde(skip_serializing_if = "Option::is_none")]
/// Payload associated to the last commit.
///
/// Upon commit, clients can optionally add a small `Striing` payload to their commit
/// to help identify this commit.
/// This payload is entirely unused by tantivy.
pub payload: Option<String>, pub payload: Option<String>,
} }
impl IndexMeta { impl IndexMeta {
/// Create an `IndexMeta` object representing a brand new `Index`
/// with the given index.
///
/// This new index does not contains any segments.
/// Opstamp will the value `0u64`.
pub fn with_schema(schema: Schema) -> IndexMeta { pub fn with_schema(schema: Schema) -> IndexMeta {
IndexMeta { IndexMeta {
segments: vec![], segments: vec![],
@@ -46,7 +33,7 @@ impl IndexMeta {
} }
impl fmt::Debug for IndexMeta { impl fmt::Debug for IndexMeta {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
write!( write!(
f, f,
"{}", "{}",
@@ -60,7 +47,7 @@ impl fmt::Debug for IndexMeta {
mod tests { mod tests {
use super::IndexMeta; use super::IndexMeta;
use crate::schema::{Schema, TEXT}; use schema::{Schema, TEXT};
use serde_json; use serde_json;
#[test] #[test]

18
src/core/inverted_index_reader.rs
View File

@@ -1,13 +1,13 @@
use crate::common::BinarySerializable; use common::BinarySerializable;
use crate::directory::ReadOnlySource; use directory::ReadOnlySource;
use crate::positions::PositionReader;
use crate::postings::TermInfo;
use crate::postings::{BlockSegmentPostings, SegmentPostings};
use crate::schema::FieldType;
use crate::schema::IndexRecordOption;
use crate::schema::Term;
use crate::termdict::TermDictionary;
use owned_read::OwnedRead; use owned_read::OwnedRead;
use positions::PositionReader;
use postings::TermInfo;
use postings::{BlockSegmentPostings, SegmentPostings};
use schema::FieldType;
use schema::IndexRecordOption;
use schema::Term;
use termdict::TermDictionary;
/// The inverted index reader is in charge of accessing /// The inverted index reader is in charge of accessing
/// the inverted index associated to a specific field. /// the inverted index associated to a specific field.

24
src/core/mod.rs
View File

@@ -21,16 +21,18 @@ pub use self::segment_id::SegmentId;
pub use self::segment_meta::SegmentMeta; pub use self::segment_meta::SegmentMeta;
pub use self::segment_reader::SegmentReader; pub use self::segment_reader::SegmentReader;
use once_cell::sync::Lazy; use std::path::PathBuf;
use std::path::Path;
/// The meta file contains all the information about the list of segments and the schema lazy_static! {
/// of the index.
pub static META_FILEPATH: Lazy<&'static Path> = Lazy::new(|| Path::new("meta.json"));
/// The managed file contains a list of files that were created by the tantivy /// The meta file contains all the information about the list of segments and the schema
/// and will therefore be garbage collected when they are deemed useless by tantivy. /// of the index.
/// pub static ref META_FILEPATH: PathBuf = PathBuf::from("meta.json");
/// Removing this file is safe, but will prevent the garbage collection of all of the file that
/// are currently in the directory /// The managed file contains a list of files that were created by the tantivy
pub static MANAGED_FILEPATH: Lazy<&'static Path> = Lazy::new(|| Path::new(".managed.json")); /// and will therefore be garbage collected when they are deemed useless by tantivy.
///
/// Removing this file is safe, but will prevent the garbage collection of all of the file that
/// are currently in the directory
pub static ref MANAGED_FILEPATH: PathBuf = PathBuf::from(".managed.json");
}

46
src/core/searcher.rs
View File

@@ -1,26 +1,26 @@
use crate::collector::Collector; use collector::Collector;
use crate::collector::SegmentCollector; use collector::SegmentCollector;
use crate::core::Executor; use core::Executor;
use crate::core::InvertedIndexReader; use core::InvertedIndexReader;
use crate::core::SegmentReader; use core::SegmentReader;
use crate::query::Query; use query::Query;
use crate::query::Scorer; use query::Scorer;
use crate::query::Weight; use query::Weight;
use crate::schema::Document; use schema::Document;
use crate::schema::Schema; use schema::Schema;
use crate::schema::{Field, Term}; use schema::{Field, Term};
use crate::space_usage::SearcherSpaceUsage; use space_usage::SearcherSpaceUsage;
use crate::store::StoreReader;
use crate::termdict::TermMerger;
use crate::DocAddress;
use crate::Index;
use crate::Result;
use std::fmt; use std::fmt;
use std::sync::Arc; use std::sync::Arc;
use store::StoreReader;
use termdict::TermMerger;
use DocAddress;
use Index;
use Result;
fn collect_segment<C: Collector>( fn collect_segment<C: Collector>(
collector: &C, collector: &C,
weight: &dyn Weight, weight: &Weight,
segment_ord: u32, segment_ord: u32,
segment_reader: &SegmentReader, segment_reader: &SegmentReader,
) -> Result<C::Fruit> { ) -> Result<C::Fruit> {
@@ -28,7 +28,7 @@ fn collect_segment<C: Collector>(
let mut segment_collector = collector.for_segment(segment_ord as u32, segment_reader)?; let mut segment_collector = collector.for_segment(segment_ord as u32, segment_reader)?;
if let Some(delete_bitset) = segment_reader.delete_bitset() { if let Some(delete_bitset) = segment_reader.delete_bitset() {
scorer.for_each(&mut |doc, score| { scorer.for_each(&mut |doc, score| {
if delete_bitset.is_alive(doc) { if !delete_bitset.is_deleted(doc) {
segment_collector.collect(doc, score); segment_collector.collect(doc, score);
} }
}); });
@@ -132,7 +132,7 @@ impl Searcher {
/// ///
/// Finally, the Collector merges each of the child collectors into itself for result usability /// Finally, the Collector merges each of the child collectors into itself for result usability
/// by the caller. /// by the caller.
pub fn search<C: Collector>(&self, query: &dyn Query, collector: &C) -> Result<C::Fruit> { pub fn search<C: Collector>(&self, query: &Query, collector: &C) -> Result<C::Fruit> {
let executor = self.index.search_executor(); let executor = self.index.search_executor();
self.search_with_executor(query, collector, executor) self.search_with_executor(query, collector, executor)
} }
@@ -151,7 +151,7 @@ impl Searcher {
/// hurt it. It will however, decrease the average response time. /// hurt it. It will however, decrease the average response time.
pub fn search_with_executor<C: Collector>( pub fn search_with_executor<C: Collector>(
&self, &self,
query: &dyn Query, query: &Query,
collector: &C, collector: &C,
executor: &Executor, executor: &Executor,
) -> Result<C::Fruit> { ) -> Result<C::Fruit> {
@@ -203,7 +203,7 @@ impl FieldSearcher {
/// Returns a Stream over all of the sorted unique terms of /// Returns a Stream over all of the sorted unique terms of
/// for the given field. /// for the given field.
pub fn terms(&self) -> TermMerger<'_> { pub fn terms(&self) -> TermMerger {
let term_streamers: Vec<_> = self let term_streamers: Vec<_> = self
.inv_index_readers .inv_index_readers
.iter() .iter()
@@ -214,7 +214,7 @@ impl FieldSearcher {
} }
impl fmt::Debug for Searcher { impl fmt::Debug for Searcher {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
let segment_ids = self let segment_ids = self
.segment_readers .segment_readers
.iter() .iter()

22
src/core/segment.rs
View File

@@ -1,17 +1,17 @@
use super::SegmentComponent; use super::SegmentComponent;
use crate::core::Index; use core::Index;
use crate::core::SegmentId; use core::SegmentId;
use crate::core::SegmentMeta; use core::SegmentMeta;
use crate::directory::error::{OpenReadError, OpenWriteError}; use directory::error::{OpenReadError, OpenWriteError};
use crate::directory::Directory; use directory::Directory;
use crate::directory::{ReadOnlySource, WritePtr}; use directory::{ReadOnlySource, WritePtr};
use crate::indexer::segment_serializer::SegmentSerializer; use indexer::segment_serializer::SegmentSerializer;
use crate::schema::Schema; use schema::Schema;
use crate::Opstamp;
use crate::Result;
use std::fmt; use std::fmt;
use std::path::PathBuf; use std::path::PathBuf;
use std::result; use std::result;
use Opstamp;
use Result;
/// A segment is a piece of the index. /// A segment is a piece of the index.
#[derive(Clone)] #[derive(Clone)]
@@ -21,7 +21,7 @@ pub struct Segment {
} }
impl fmt::Debug for Segment { impl fmt::Debug for Segment {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
write!(f, "Segment({:?})", self.id().uuid_string()) write!(f, "Segment({:?})", self.id().uuid_string())
} }
} }

14
src/core/segment_id.rs
View File

@@ -2,8 +2,6 @@ use std::cmp::{Ord, Ordering};
use std::fmt; use std::fmt;
use uuid::Uuid; use uuid::Uuid;
#[cfg(test)]
use once_cell::sync::Lazy;
#[cfg(test)] #[cfg(test)]
use std::sync::atomic; use std::sync::atomic;
@@ -19,10 +17,10 @@ use std::sync::atomic;
pub struct SegmentId(Uuid); pub struct SegmentId(Uuid);
#[cfg(test)] #[cfg(test)]
static AUTO_INC_COUNTER: Lazy<atomic::AtomicUsize> = Lazy::new(|| atomic::AtomicUsize::default()); lazy_static! {
static ref AUTO_INC_COUNTER: atomic::AtomicUsize = atomic::AtomicUsize::default();
#[cfg(test)] static ref ZERO_ARRAY: [u8; 8] = [0u8; 8];
const ZERO_ARRAY: [u8; 8] = [0u8; 8]; }
// During tests, we generate the segment id in a autoincrement manner // During tests, we generate the segment id in a autoincrement manner
// for consistency of segment id between run. // for consistency of segment id between run.
@@ -32,7 +30,7 @@ const ZERO_ARRAY: [u8; 8] = [0u8; 8];
#[cfg(test)] #[cfg(test)]
fn create_uuid() -> Uuid { fn create_uuid() -> Uuid {
let new_auto_inc_id = (*AUTO_INC_COUNTER).fetch_add(1, atomic::Ordering::SeqCst); let new_auto_inc_id = (*AUTO_INC_COUNTER).fetch_add(1, atomic::Ordering::SeqCst);
Uuid::from_fields(new_auto_inc_id as u32, 0, 0, &ZERO_ARRAY).unwrap() Uuid::from_fields(new_auto_inc_id as u32, 0, 0, &*ZERO_ARRAY).unwrap()
} }
#[cfg(not(test))] #[cfg(not(test))]
@@ -64,7 +62,7 @@ impl SegmentId {
} }
impl fmt::Debug for SegmentId { impl fmt::Debug for SegmentId {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
write!(f, "Seg({:?})", self.short_uuid_string()) write!(f, "Seg({:?})", self.short_uuid_string())
} }
} }

11
src/core/segment_meta.rs
View File

@@ -1,14 +1,15 @@
use super::SegmentComponent; use super::SegmentComponent;
use crate::core::SegmentId;
use crate::Opstamp;
use census::{Inventory, TrackedObject}; use census::{Inventory, TrackedObject};
use once_cell::sync::Lazy; use core::SegmentId;
use serde; use serde;
use std::collections::HashSet; use std::collections::HashSet;
use std::fmt; use std::fmt;
use std::path::PathBuf; use std::path::PathBuf;
use Opstamp;
static INVENTORY: Lazy<Inventory<InnerSegmentMeta>> = Lazy::new(Inventory::new); lazy_static! {
static ref INVENTORY: Inventory<InnerSegmentMeta> = { Inventory::new() };
}
#[derive(Clone, Debug, Serialize, Deserialize)] #[derive(Clone, Debug, Serialize, Deserialize)]
struct DeleteMeta { struct DeleteMeta {
@@ -26,7 +27,7 @@ pub struct SegmentMeta {
} }
impl fmt::Debug for SegmentMeta { impl fmt::Debug for SegmentMeta {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> Result<(), fmt::Error> { fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> {
self.tracked.fmt(f) self.tracked.fmt(f)
} }
} }

56
src/core/segment_reader.rs
View File

@@ -1,27 +1,26 @@
use crate::common::CompositeFile; use common::CompositeFile;
use crate::common::HasLen; use common::HasLen;
use crate::core::InvertedIndexReader; use core::InvertedIndexReader;
use crate::core::Segment; use core::Segment;
use crate::core::SegmentComponent; use core::SegmentComponent;
use crate::core::SegmentId; use core::SegmentId;
use crate::directory::ReadOnlySource; use directory::ReadOnlySource;
use crate::fastfield::DeleteBitSet; use fastfield::DeleteBitSet;
use crate::fastfield::FacetReader; use fastfield::FacetReader;
use crate::fastfield::FastFieldReaders; use fastfield::FastFieldReaders;
use crate::fieldnorm::FieldNormReader; use fieldnorm::FieldNormReader;
use crate::schema::Field; use schema::Field;
use crate::schema::FieldType; use schema::FieldType;
use crate::schema::Schema; use schema::Schema;
use crate::space_usage::SegmentSpaceUsage; use space_usage::SegmentSpaceUsage;
use crate::store::StoreReader;
use crate::termdict::TermDictionary;
use crate::DocId;
use crate::Result;
use fail::fail_point;
use std::collections::HashMap; use std::collections::HashMap;
use std::fmt; use std::fmt;
use std::sync::Arc; use std::sync::Arc;
use std::sync::RwLock; use std::sync::RwLock;
use store::StoreReader;
use termdict::TermDictionary;
use DocId;
use Result;
/// Entry point to access all of the datastructures of the `Segment` /// Entry point to access all of the datastructures of the `Segment`
/// ///
@@ -244,9 +243,10 @@ impl SegmentReader {
let postings_source = postings_source_opt.unwrap(); let postings_source = postings_source_opt.unwrap();
let termdict_source = self.termdict_composite.open_read(field).expect( let termdict_source = self
"Failed to open field term dictionary in composite file. Is the field indexed?", .termdict_composite
); .open_read(field)
.expect("Failed to open field term dictionary in composite file. Is the field indexed");
let positions_source = self let positions_source = self
.positions_composite .positions_composite
@@ -296,7 +296,7 @@ impl SegmentReader {
} }
/// Returns an iterator that will iterate over the alive document ids /// Returns an iterator that will iterate over the alive document ids
pub fn doc_ids_alive(&self) -> SegmentReaderAliveDocsIterator<'_> { pub fn doc_ids_alive(&self) -> SegmentReaderAliveDocsIterator {
SegmentReaderAliveDocsIterator::new(&self) SegmentReaderAliveDocsIterator::new(&self)
} }
@@ -320,7 +320,7 @@ impl SegmentReader {
} }
impl fmt::Debug for SegmentReader { impl fmt::Debug for SegmentReader {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
write!(f, "SegmentReader({:?})", self.segment_id) write!(f, "SegmentReader({:?})", self.segment_id)
} }
} }
@@ -373,9 +373,9 @@ impl<'a> Iterator for SegmentReaderAliveDocsIterator<'a> {
#[cfg(test)] #[cfg(test)]
mod test { mod test {
use crate::core::Index; use core::Index;
use crate::schema::{Schema, Term, STORED, TEXT}; use schema::{Schema, Term, STORED, TEXT};
use crate::DocId; use DocId;
#[test] #[test]
fn test_alive_docs_iterator() { fn test_alive_docs_iterator() {

22
src/directory/directory.rs
View File

@@ -1,9 +1,9 @@
use crate::directory::directory_lock::Lock; use directory::directory_lock::Lock;
use crate::directory::error::LockError; use directory::error::LockError;
use crate::directory::error::{DeleteError, OpenReadError, OpenWriteError}; use directory::error::{DeleteError, OpenReadError, OpenWriteError};
use crate::directory::WatchCallback; use directory::WatchCallback;
use crate::directory::WatchHandle; use directory::WatchHandle;
use crate::directory::{ReadOnlySource, WritePtr}; use directory::{ReadOnlySource, WritePtr};
use std::fmt; use std::fmt;
use std::io; use std::io;
use std::io::Write; use std::io::Write;
@@ -48,10 +48,10 @@ impl RetryPolicy {
/// ///
/// It is transparently associated to a lock file, that gets deleted /// It is transparently associated to a lock file, that gets deleted
/// on `Drop.` The lock is released automatically on `Drop`. /// on `Drop.` The lock is released automatically on `Drop`.
pub struct DirectoryLock(Box<dyn Drop + Send + Sync + 'static>); pub struct DirectoryLock(Box<Drop + Send + Sync + 'static>);
struct DirectoryLockGuard { struct DirectoryLockGuard {
directory: Box<dyn Directory>, directory: Box<Directory>,
path: PathBuf, path: PathBuf,
} }
@@ -76,7 +76,7 @@ enum TryAcquireLockError {
fn try_acquire_lock( fn try_acquire_lock(
filepath: &Path, filepath: &Path,
directory: &mut dyn Directory, directory: &mut Directory,
) -> Result<DirectoryLock, TryAcquireLockError> { ) -> Result<DirectoryLock, TryAcquireLockError> {
let mut write = directory.open_write(filepath).map_err(|e| match e { let mut write = directory.open_write(filepath).map_err(|e| match e {
OpenWriteError::FileAlreadyExists(_) => TryAcquireLockError::FileExists, OpenWriteError::FileAlreadyExists(_) => TryAcquireLockError::FileExists,
@@ -210,14 +210,14 @@ pub trait Directory: DirectoryClone + fmt::Debug + Send + Sync + 'static {
/// DirectoryClone /// DirectoryClone
pub trait DirectoryClone { pub trait DirectoryClone {
/// Clones the directory and boxes the clone /// Clones the directory and boxes the clone
fn box_clone(&self) -> Box<dyn Directory>; fn box_clone(&self) -> Box<Directory>;
} }
impl<T> DirectoryClone for T impl<T> DirectoryClone for T
where where
T: 'static + Directory + Clone, T: 'static + Directory + Clone,
{ {
fn box_clone(&self) -> Box<dyn Directory> { fn box_clone(&self) -> Box<Directory> {
Box::new(self.clone()) Box::new(self.clone())
} }
} }

51
src/directory/directory_lock.rs
View File

@@ -1,4 +1,3 @@
use once_cell::sync::Lazy;
use std::path::PathBuf; use std::path::PathBuf;
/// A directory lock. /// A directory lock.
@@ -29,27 +28,29 @@ pub struct Lock {
pub is_blocking: bool, pub is_blocking: bool,
} }
/// Only one process should be able to write tantivy's index at a time. lazy_static! {
/// This lock file, when present, is in charge of preventing other processes to open an IndexWriter. /// Only one process should be able to write tantivy's index at a time.
/// /// This lock file, when present, is in charge of preventing other processes to open an IndexWriter.
/// If the process is killed and this file remains, it is safe to remove it manually. ///
/// /// If the process is killed and this file remains, it is safe to remove it manually.
/// Failing to acquire this lock usually means a misuse of tantivy's API, ///
/// (creating more than one instance of the `IndexWriter`), are a spurious /// Failing to acquire this lock usually means a misuse of tantivy's API,
/// lock file remaining after a crash. In the latter case, removing the file after /// (creating more than one instance of the `IndexWriter`), are a spurious
/// checking no process running tantivy is running is safe. /// lock file remaining after a crash. In the latter case, removing the file after
pub static INDEX_WRITER_LOCK: Lazy<Lock> = Lazy::new(|| Lock { /// checking no process running tantivy is running is safe.
filepath: PathBuf::from(".tantivy-writer.lock"), pub static ref INDEX_WRITER_LOCK: Lock = Lock {
is_blocking: false, filepath: PathBuf::from(".tantivy-writer.lock"),
}); is_blocking: false
/// The meta lock file is here to protect the segment files being opened by };
/// `IndexReader::reload()` from being garbage collected. /// The meta lock file is here to protect the segment files being opened by
/// It makes it possible for another process to safely consume /// `IndexReader::reload()` from being garbage collected.
/// our index in-writing. Ideally, we may have prefered `RWLock` semantics /// It makes it possible for another process to safely consume
/// here, but it is difficult to achieve on Windows. /// our index in-writing. Ideally, we may have prefered `RWLock` semantics
/// /// here, but it is difficult to achieve on Windows.
/// Opening segment readers is a very fast process. ///
pub static META_LOCK: Lazy<Lock> = Lazy::new(|| Lock { /// Opening segment readers is a very fast process.
filepath: PathBuf::from(".tantivy-meta.lock"), pub static ref META_LOCK: Lock = Lock {
is_blocking: true, filepath: PathBuf::from(".tantivy-meta.lock"),
}); is_blocking: true
};
}

22
src/directory/error.rs
View File

@@ -6,7 +6,7 @@ use std::path::PathBuf;
/// Error while trying to acquire a directory lock. /// Error while trying to acquire a directory lock.
#[derive(Debug, Fail)] #[derive(Debug, Fail)]
pub enum LockError { pub enum LockError {
/// Failed to acquired a lock as it is already held by another /// Failed to acquired a lock as it is already hold by another
/// client. /// client.
/// - In the context of a blocking lock, this means the lock was not released within some `timeout` period. /// - In the context of a blocking lock, this means the lock was not released within some `timeout` period.
/// - In the context of a non-blocking lock, this means the lock was busy at the moment of the call. /// - In the context of a non-blocking lock, this means the lock was busy at the moment of the call.
@@ -33,7 +33,7 @@ impl Into<io::Error> for IOError {
} }
impl fmt::Display for IOError { impl fmt::Display for IOError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
match self.path { match self.path {
Some(ref path) => write!(f, "io error occurred on path '{:?}': '{}'", path, self.err), Some(ref path) => write!(f, "io error occurred on path '{:?}': '{}'", path, self.err),
None => write!(f, "io error occurred: '{}'", self.err), None => write!(f, "io error occurred: '{}'", self.err),
@@ -46,7 +46,7 @@ impl StdError for IOError {
"io error occurred" "io error occurred"
} }
fn cause(&self) -> Option<&dyn StdError> { fn cause(&self) -> Option<&StdError> {
Some(&self.err) Some(&self.err)
} }
} }
@@ -84,7 +84,7 @@ impl From<io::Error> for OpenDirectoryError {
} }
impl fmt::Display for OpenDirectoryError { impl fmt::Display for OpenDirectoryError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
match *self { match *self {
OpenDirectoryError::DoesNotExist(ref path) => { OpenDirectoryError::DoesNotExist(ref path) => {
write!(f, "the underlying directory '{:?}' does not exist", path) write!(f, "the underlying directory '{:?}' does not exist", path)
@@ -106,7 +106,7 @@ impl StdError for OpenDirectoryError {
"error occurred while opening a directory" "error occurred while opening a directory"
} }
fn cause(&self) -> Option<&dyn StdError> { fn cause(&self) -> Option<&StdError> {
None None
} }
} }
@@ -129,7 +129,7 @@ impl From<IOError> for OpenWriteError {
} }
impl fmt::Display for OpenWriteError { impl fmt::Display for OpenWriteError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
match *self { match *self {
OpenWriteError::FileAlreadyExists(ref path) => { OpenWriteError::FileAlreadyExists(ref path) => {
write!(f, "the file '{:?}' already exists", path) write!(f, "the file '{:?}' already exists", path)
@@ -148,7 +148,7 @@ impl StdError for OpenWriteError {
"error occurred while opening a file for writing" "error occurred while opening a file for writing"
} }
fn cause(&self) -> Option<&dyn StdError> { fn cause(&self) -> Option<&StdError> {
match *self { match *self {
OpenWriteError::FileAlreadyExists(_) => None, OpenWriteError::FileAlreadyExists(_) => None,
OpenWriteError::IOError(ref err) => Some(err), OpenWriteError::IOError(ref err) => Some(err),
@@ -173,7 +173,7 @@ impl From<IOError> for OpenReadError {
} }
impl fmt::Display for OpenReadError { impl fmt::Display for OpenReadError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
match *self { match *self {
OpenReadError::FileDoesNotExist(ref path) => { OpenReadError::FileDoesNotExist(ref path) => {
write!(f, "the file '{:?}' does not exist", path) write!(f, "the file '{:?}' does not exist", path)
@@ -192,7 +192,7 @@ impl StdError for OpenReadError {
"error occurred while opening a file for reading" "error occurred while opening a file for reading"
} }
fn cause(&self) -> Option<&dyn StdError> { fn cause(&self) -> Option<&StdError> {
match *self { match *self {
OpenReadError::FileDoesNotExist(_) => None, OpenReadError::FileDoesNotExist(_) => None,
OpenReadError::IOError(ref err) => Some(err), OpenReadError::IOError(ref err) => Some(err),
@@ -217,7 +217,7 @@ impl From<IOError> for DeleteError {
} }
impl fmt::Display for DeleteError { impl fmt::Display for DeleteError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
match *self { match *self {
DeleteError::FileDoesNotExist(ref path) => { DeleteError::FileDoesNotExist(ref path) => {
write!(f, "the file '{:?}' does not exist", path) write!(f, "the file '{:?}' does not exist", path)
@@ -234,7 +234,7 @@ impl StdError for DeleteError {
"error occurred while deleting a file" "error occurred while deleting a file"
} }
fn cause(&self) -> Option<&dyn StdError> { fn cause(&self) -> Option<&StdError> {
match *self { match *self {
DeleteError::FileDoesNotExist(_) => None, DeleteError::FileDoesNotExist(_) => None,
DeleteError::IOError(ref err) => Some(err), DeleteError::IOError(ref err) => Some(err),

37
src/directory/managed_directory.rs
View File

@@ -1,13 +1,11 @@
use crate::core::MANAGED_FILEPATH; use core::MANAGED_FILEPATH;
use crate::directory::error::{DeleteError, IOError, LockError, OpenReadError, OpenWriteError}; use directory::error::{DeleteError, IOError, LockError, OpenReadError, OpenWriteError};
use crate::directory::DirectoryLock; use directory::DirectoryLock;
use crate::directory::Lock; use directory::Lock;
use crate::directory::META_LOCK; use directory::META_LOCK;
use crate::directory::{ReadOnlySource, WritePtr}; use directory::{ReadOnlySource, WritePtr};
use crate::directory::{WatchCallback, WatchHandle}; use directory::{WatchCallback, WatchHandle};
use crate::error::DataCorruption; use error::DataCorruption;
use crate::Directory;
use crate::Result;
use serde_json; use serde_json;
use std::collections::HashSet; use std::collections::HashSet;
use std::io; use std::io;
@@ -16,6 +14,8 @@ use std::path::{Path, PathBuf};
use std::result; use std::result;
use std::sync::RwLockWriteGuard; use std::sync::RwLockWriteGuard;
use std::sync::{Arc, RwLock}; use std::sync::{Arc, RwLock};
use Directory;
use Result;
/// Returns true iff the file is "managed". /// Returns true iff the file is "managed".
/// Non-managed file are not subject to garbage collection. /// Non-managed file are not subject to garbage collection.
@@ -39,7 +39,7 @@ fn is_managed(path: &Path) -> bool {
/// useful anymore. /// useful anymore.
#[derive(Debug)] #[derive(Debug)]
pub struct ManagedDirectory { pub struct ManagedDirectory {
directory: Box<dyn Directory>, directory: Box<Directory>,
meta_informations: Arc<RwLock<MetaInformation>>, meta_informations: Arc<RwLock<MetaInformation>>,
} }
@@ -51,8 +51,8 @@ struct MetaInformation {
/// Saves the file containing the list of existing files /// Saves the file containing the list of existing files
/// that were created by tantivy. /// that were created by tantivy.
fn save_managed_paths( fn save_managed_paths(
directory: &mut dyn Directory, directory: &mut Directory,
wlock: &RwLockWriteGuard<'_, MetaInformation>, wlock: &RwLockWriteGuard<MetaInformation>,
) -> io::Result<()> { ) -> io::Result<()> {
let mut w = serde_json::to_vec(&wlock.managed_paths)?; let mut w = serde_json::to_vec(&wlock.managed_paths)?;
writeln!(&mut w)?; writeln!(&mut w)?;
@@ -69,7 +69,7 @@ impl ManagedDirectory {
let managed_files: HashSet<PathBuf> = serde_json::from_str(&managed_files_json) let managed_files: HashSet<PathBuf> = serde_json::from_str(&managed_files_json)
.map_err(|e| { .map_err(|e| {
DataCorruption::new( DataCorruption::new(
MANAGED_FILEPATH.to_path_buf(), MANAGED_FILEPATH.clone(),
format!("Managed file cannot be deserialized: {:?}. ", e), format!("Managed file cannot be deserialized: {:?}. ", e),
) )
})?; })?;
@@ -264,14 +264,15 @@ mod tests {
mod mmap_specific { mod mmap_specific {
use super::super::*; use super::super::*;
use once_cell::sync::Lazy;
use std::path::Path; use std::path::Path;
use tempdir::TempDir; use tempdir::TempDir;
static TEST_PATH1: Lazy<&'static Path> = Lazy::new(|| Path::new("some_path_for_test")); lazy_static! {
static TEST_PATH2: Lazy<&'static Path> = Lazy::new(|| Path::new("some_path_for_test2")); static ref TEST_PATH1: &'static Path = Path::new("some_path_for_test");
static ref TEST_PATH2: &'static Path = Path::new("some_path_for_test2");
}
use crate::directory::MmapDirectory; use directory::MmapDirectory;
use std::io::Write; use std::io::Write;
#[test] #[test]

40
src/directory/mmap_directory.rs
View File

@@ -1,25 +1,23 @@
use fs2; extern crate fs2;
use notify; extern crate notify;
use self::fs2::FileExt; use self::fs2::FileExt;
use self::notify::RawEvent; use self::notify::RawEvent;
use self::notify::RecursiveMode; use self::notify::RecursiveMode;
use self::notify::Watcher; use self::notify::Watcher;
use crate::core::META_FILEPATH;
use crate::directory::error::LockError;
use crate::directory::error::{
DeleteError, IOError, OpenDirectoryError, OpenReadError, OpenWriteError,
};
use crate::directory::read_only_source::BoxedData;
use crate::directory::Directory;
use crate::directory::DirectoryLock;
use crate::directory::Lock;
use crate::directory::ReadOnlySource;
use crate::directory::WatchCallback;
use crate::directory::WatchCallbackList;
use crate::directory::WatchHandle;
use crate::directory::WritePtr;
use atomicwrites; use atomicwrites;
use core::META_FILEPATH;
use directory::error::LockError;
use directory::error::{DeleteError, IOError, OpenDirectoryError, OpenReadError, OpenWriteError};
use directory::read_only_source::BoxedData;
use directory::Directory;
use directory::DirectoryLock;
use directory::Lock;
use directory::ReadOnlySource;
use directory::WatchCallback;
use directory::WatchCallbackList;
use directory::WatchHandle;
use directory::WritePtr;
use memmap::Mmap; use memmap::Mmap;
use std::collections::HashMap; use std::collections::HashMap;
use std::convert::From; use std::convert::From;
@@ -256,7 +254,7 @@ impl MmapDirectoryInner {
} }
impl fmt::Debug for MmapDirectory { impl fmt::Debug for MmapDirectory {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
write!(f, "MmapDirectory({:?})", self.inner.root_path) write!(f, "MmapDirectory({:?})", self.inner.root_path)
} }
} }
@@ -322,7 +320,7 @@ impl MmapDirectory {
#[cfg(windows)] #[cfg(windows)]
{ {
use std::os::windows::fs::OpenOptionsExt; use std::os::windows::fs::OpenOptionsExt;
use winapi::um::winbase; use winapi::winbase;
open_opts open_opts
.write(true) .write(true)
@@ -527,13 +525,13 @@ mod tests {
// The following tests are specific to the MmapDirectory // The following tests are specific to the MmapDirectory
use super::*; use super::*;
use crate::schema::{Schema, SchemaBuilder, TEXT}; use schema::{Schema, SchemaBuilder, TEXT};
use crate::Index;
use crate::ReloadPolicy;
use std::fs; use std::fs;
use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::atomic::{AtomicUsize, Ordering};
use std::thread; use std::thread;
use std::time::Duration; use std::time::Duration;
use Index;
use ReloadPolicy;
#[test] #[test]
fn test_open_non_existant_path() { fn test_open_non_existant_path() {

8
src/directory/mod.rs
View File

@@ -24,18 +24,22 @@ pub use self::ram_directory::RAMDirectory;
pub use self::read_only_source::ReadOnlySource; pub use self::read_only_source::ReadOnlySource;
pub(crate) use self::watch_event_router::WatchCallbackList; pub(crate) use self::watch_event_router::WatchCallbackList;
pub use self::watch_event_router::{WatchCallback, WatchHandle}; pub use self::watch_event_router::{WatchCallback, WatchHandle};
use std::io::{BufWriter, Write}; use std::io::{BufWriter, Seek, Write};
#[cfg(feature = "mmap")] #[cfg(feature = "mmap")]
pub use self::mmap_directory::MmapDirectory; pub use self::mmap_directory::MmapDirectory;
pub(crate) use self::managed_directory::ManagedDirectory; pub(crate) use self::managed_directory::ManagedDirectory;
/// Synonym of Seek + Write
pub trait SeekableWrite: Seek + Write {}
impl<T: Seek + Write> SeekableWrite for T {}
/// Write object for Directory. /// Write object for Directory.
/// ///
/// `WritePtr` are required to implement both Write /// `WritePtr` are required to implement both Write
/// and Seek. /// and Seek.
pub type WritePtr = BufWriter<Box<dyn Write>>; pub type WritePtr = BufWriter<Box<SeekableWrite>>;
#[cfg(test)] #[cfg(test)]
mod tests; mod tests;

23
src/directory/ram_directory.rs
View File

@@ -1,9 +1,8 @@
use crate::core::META_FILEPATH; use core::META_FILEPATH;
use crate::directory::error::{DeleteError, OpenReadError, OpenWriteError}; use directory::error::{DeleteError, OpenReadError, OpenWriteError};
use crate::directory::WatchCallbackList; use directory::WatchCallbackList;
use crate::directory::WritePtr; use directory::WritePtr;
use crate::directory::{Directory, ReadOnlySource, WatchCallback, WatchHandle}; use directory::{Directory, ReadOnlySource, WatchCallback, WatchHandle};
use fail::fail_point;
use std::collections::HashMap; use std::collections::HashMap;
use std::fmt; use std::fmt;
use std::io::{self, BufWriter, Cursor, Seek, SeekFrom, Write}; use std::io::{self, BufWriter, Cursor, Seek, SeekFrom, Write};
@@ -104,14 +103,10 @@ impl InnerDirectory {
fn watch(&mut self, watch_handle: WatchCallback) -> WatchHandle { fn watch(&mut self, watch_handle: WatchCallback) -> WatchHandle {
self.watch_router.subscribe(watch_handle) self.watch_router.subscribe(watch_handle)
} }
fn total_mem_usage(&self) -> usize {
self.fs.values().map(|f| f.len()).sum()
}
} }
impl fmt::Debug for RAMDirectory { impl fmt::Debug for RAMDirectory {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
write!(f, "RAMDirectory") write!(f, "RAMDirectory")
} }
} }
@@ -131,12 +126,6 @@ impl RAMDirectory {
pub fn create() -> RAMDirectory { pub fn create() -> RAMDirectory {
Self::default() Self::default()
} }
/// Returns the sum of the size of the different files
/// in the RAMDirectory.
pub fn total_mem_usage(&self) -> usize {
self.fs.read().unwrap().total_mem_usage()
}
} }
impl Directory for RAMDirectory { impl Directory for RAMDirectory {

4
src/directory/read_only_source.rs
View File

@@ -1,9 +1,9 @@
use crate::common::HasLen; use common::HasLen;
use stable_deref_trait::{CloneStableDeref, StableDeref}; use stable_deref_trait::{CloneStableDeref, StableDeref};
use std::ops::Deref; use std::ops::Deref;
use std::sync::Arc; use std::sync::Arc;
pub type BoxedData = Box<dyn Deref<Target = [u8]> + Send + Sync + 'static>; pub type BoxedData = Box<Deref<Target = [u8]> + Send + Sync + 'static>;
/// Read object that represents files in tantivy. /// Read object that represents files in tantivy.
/// ///

41
src/directory/tests.rs
View File

@@ -1,6 +1,5 @@
use super::*; use super::*;
use once_cell::sync::Lazy; use std::io::{Seek, SeekFrom, Write};
use std::io::Write;
use std::mem; use std::mem;
use std::path::{Path, PathBuf}; use std::path::{Path, PathBuf};
use std::sync::atomic::AtomicUsize; use std::sync::atomic::AtomicUsize;
@@ -10,7 +9,9 @@ use std::thread;
use std::time; use std::time;
use std::time::Duration; use std::time::Duration;
static TEST_PATH: Lazy<&'static Path> = Lazy::new(|| Path::new("some_path_for_test")); lazy_static! {
static ref TEST_PATH: &'static Path = Path::new("some_path_for_test");
}
#[test] #[test]
fn test_ram_directory() { fn test_ram_directory() {
@@ -33,7 +34,7 @@ fn ram_directory_panics_if_flush_forgotten() {
assert!(write_file.write_all(&[4]).is_ok()); assert!(write_file.write_all(&[4]).is_ok());
} }
fn test_simple(directory: &mut dyn Directory) { fn test_simple(directory: &mut Directory) {
{ {
let mut write_file = directory.open_write(*TEST_PATH).unwrap(); let mut write_file = directory.open_write(*TEST_PATH).unwrap();
assert!(directory.exists(*TEST_PATH)); assert!(directory.exists(*TEST_PATH));
@@ -51,7 +52,24 @@ fn test_simple(directory: &mut dyn Directory) {
assert!(!directory.exists(*TEST_PATH)); assert!(!directory.exists(*TEST_PATH));
} }
fn test_rewrite_forbidden(directory: &mut dyn Directory) { fn test_seek(directory: &mut Directory) {
{
{
let mut write_file = directory.open_write(*TEST_PATH).unwrap();
write_file.write_all(&[4, 3, 7, 3, 5]).unwrap();
write_file.seek(SeekFrom::Start(0)).unwrap();
write_file.write_all(&[3, 1]).unwrap();
write_file.flush().unwrap();
}
let read_file = directory.open_read(*TEST_PATH).unwrap();
let data: &[u8] = &*read_file;
assert_eq!(data, &[3u8, 1u8, 7u8, 3u8, 5u8]);
}
assert!(directory.delete(*TEST_PATH).is_ok());
}
fn test_rewrite_forbidden(directory: &mut Directory) {
{ {
directory.open_write(*TEST_PATH).unwrap(); directory.open_write(*TEST_PATH).unwrap();
assert!(directory.exists(*TEST_PATH)); assert!(directory.exists(*TEST_PATH));
@@ -62,7 +80,7 @@ fn test_rewrite_forbidden(directory: &mut dyn Directory) {
assert!(directory.delete(*TEST_PATH).is_ok()); assert!(directory.delete(*TEST_PATH).is_ok());
} }
fn test_write_create_the_file(directory: &mut dyn Directory) { fn test_write_create_the_file(directory: &mut Directory) {
{ {
assert!(directory.open_read(*TEST_PATH).is_err()); assert!(directory.open_read(*TEST_PATH).is_err());
let _w = directory.open_write(*TEST_PATH).unwrap(); let _w = directory.open_write(*TEST_PATH).unwrap();
@@ -72,7 +90,7 @@ fn test_write_create_the_file(directory: &mut dyn Directory) {
} }
} }
fn test_directory_delete(directory: &mut dyn Directory) { fn test_directory_delete(directory: &mut Directory) {
assert!(directory.open_read(*TEST_PATH).is_err()); assert!(directory.open_read(*TEST_PATH).is_err());
let mut write_file = directory.open_write(*TEST_PATH).unwrap(); let mut write_file = directory.open_write(*TEST_PATH).unwrap();
write_file.write_all(&[1, 2, 3, 4]).unwrap(); write_file.write_all(&[1, 2, 3, 4]).unwrap();
@@ -100,8 +118,9 @@ fn test_directory_delete(directory: &mut dyn Directory) {
assert!(directory.delete(*TEST_PATH).is_err()); assert!(directory.delete(*TEST_PATH).is_err());
} }
fn test_directory(directory: &mut dyn Directory) { fn test_directory(directory: &mut Directory) {
test_simple(directory); test_simple(directory);
test_seek(directory);
test_rewrite_forbidden(directory); test_rewrite_forbidden(directory);
test_write_create_the_file(directory); test_write_create_the_file(directory);
test_directory_delete(directory); test_directory_delete(directory);
@@ -110,7 +129,7 @@ fn test_directory(directory: &mut dyn Directory) {
test_watch(directory); test_watch(directory);
} }
fn test_watch(directory: &mut dyn Directory) { fn test_watch(directory: &mut Directory) {
let counter: Arc<AtomicUsize> = Default::default(); let counter: Arc<AtomicUsize> = Default::default();
let counter_clone = counter.clone(); let counter_clone = counter.clone();
let watch_callback = Box::new(move || { let watch_callback = Box::new(move || {
@@ -144,7 +163,7 @@ fn test_watch(directory: &mut dyn Directory) {
assert_eq!(10, counter.load(Ordering::SeqCst)); assert_eq!(10, counter.load(Ordering::SeqCst));
} }
fn test_lock_non_blocking(directory: &mut dyn Directory) { fn test_lock_non_blocking(directory: &mut Directory) {
{ {
let lock_a_res = directory.acquire_lock(&Lock { let lock_a_res = directory.acquire_lock(&Lock {
filepath: PathBuf::from("a.lock"), filepath: PathBuf::from("a.lock"),
@@ -169,7 +188,7 @@ fn test_lock_non_blocking(directory: &mut dyn Directory) {
assert!(lock_a_res.is_ok()); assert!(lock_a_res.is_ok());
} }
fn test_lock_blocking(directory: &mut dyn Directory) { fn test_lock_blocking(directory: &mut Directory) {
let lock_a_res = directory.acquire_lock(&Lock { let lock_a_res = directory.acquire_lock(&Lock {
filepath: PathBuf::from("a.lock"), filepath: PathBuf::from("a.lock"),
is_blocking: true, is_blocking: true,

4
src/directory/watch_event_router.rs
View File

@@ -3,7 +3,7 @@ use std::sync::RwLock;
use std::sync::Weak; use std::sync::Weak;
/// Type alias for callbacks registered when watching files of a `Directory`. /// Type alias for callbacks registered when watching files of a `Directory`.
pub type WatchCallback = Box<dyn Fn() -> () + Sync + Send>; pub type WatchCallback = Box<Fn() -> () + Sync + Send>;
/// Helper struct to implement the watch method in `Directory` implementations. /// Helper struct to implement the watch method in `Directory` implementations.
/// ///
@@ -67,7 +67,7 @@ impl WatchCallbackList {
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use crate::directory::WatchCallbackList; use directory::WatchCallbackList;
use std::mem; use std::mem;
use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::Arc; use std::sync::Arc;

6
src/docset.rs
View File

@@ -1,9 +1,9 @@
use crate::common::BitSet; use common::BitSet;
use crate::fastfield::DeleteBitSet; use fastfield::DeleteBitSet;
use crate::DocId;
use std::borrow::Borrow; use std::borrow::Borrow;
use std::borrow::BorrowMut; use std::borrow::BorrowMut;
use std::cmp::Ordering; use std::cmp::Ordering;
use DocId;
/// Expresses the outcome of a call to `DocSet`'s `.skip_next(...)`. /// Expresses the outcome of a call to `DocSet`'s `.skip_next(...)`.
#[derive(PartialEq, Eq, Debug)] #[derive(PartialEq, Eq, Debug)]

17
src/error.rs
View File

@@ -2,11 +2,11 @@
use std::io; use std::io;
use crate::directory::error::LockError; use directory::error::LockError;
use crate::directory::error::{IOError, OpenDirectoryError, OpenReadError, OpenWriteError}; use directory::error::{IOError, OpenDirectoryError, OpenReadError, OpenWriteError};
use crate::fastfield::FastFieldNotAvailableError; use fastfield::FastFieldNotAvailableError;
use crate::query; use query;
use crate::schema; use schema;
use serde_json; use serde_json;
use std::fmt; use std::fmt;
use std::path::PathBuf; use std::path::PathBuf;
@@ -34,7 +34,7 @@ impl DataCorruption {
} }
impl fmt::Debug for DataCorruption { impl fmt::Debug for DataCorruption {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> Result<(), fmt::Error> { fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> {
write!(f, "Data corruption: ")?; write!(f, "Data corruption: ")?;
if let Some(ref filepath) = &self.filepath { if let Some(ref filepath) = &self.filepath {
write!(f, "(in file `{:?}`)", filepath)?; write!(f, "(in file `{:?}`)", filepath)?;
@@ -77,6 +77,9 @@ pub enum TantivyError {
/// An Error appeared related to the schema. /// An Error appeared related to the schema.
#[fail(display = "Schema error: '{}'", _0)] #[fail(display = "Schema error: '{}'", _0)]
SchemaError(String), SchemaError(String),
/// Tried to access a fastfield reader for a field not configured accordingly.
#[fail(display = "Fast field not available: '{:?}'", _0)]
FastFieldError(#[cause] FastFieldNotAvailableError),
/// System error. (e.g.: We failed spawning a new thread) /// System error. (e.g.: We failed spawning a new thread)
#[fail(display = "System error.'{}'", _0)] #[fail(display = "System error.'{}'", _0)]
SystemError(String), SystemError(String),
@@ -90,7 +93,7 @@ impl From<DataCorruption> for TantivyError {
impl From<FastFieldNotAvailableError> for TantivyError { impl From<FastFieldNotAvailableError> for TantivyError {
fn from(fastfield_error: FastFieldNotAvailableError) -> TantivyError { fn from(fastfield_error: FastFieldNotAvailableError) -> TantivyError {
TantivyError::SchemaError(format!("{}", fastfield_error)) TantivyError::FastFieldError(fastfield_error)
} }
} }

4
src/fastfield/bytes/mod.rs
View File

@@ -6,8 +6,8 @@ pub use self::writer::BytesFastFieldWriter;
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use crate::schema::Schema; use schema::Schema;
use crate::Index; use Index;
#[test] #[test]
fn test_bytes() { fn test_bytes() {

6
src/fastfield/bytes/reader.rs
View File

@@ -1,8 +1,8 @@
use owning_ref::OwningRef; use owning_ref::OwningRef;
use crate::directory::ReadOnlySource; use directory::ReadOnlySource;
use crate::fastfield::FastFieldReader; use fastfield::FastFieldReader;
use crate::DocId; use DocId;
/// Reader for byte array fast fields /// Reader for byte array fast fields
/// ///

6
src/fastfield/bytes/writer.rs
View File

@@ -1,8 +1,8 @@
use std::io; use std::io;
use crate::fastfield::serializer::FastFieldSerializer; use fastfield::serializer::FastFieldSerializer;
use crate::schema::{Document, Field, Value}; use schema::{Document, Field, Value};
use crate::DocId; use DocId;
/// Writer for byte array (as in, any number of bytes per document) fast fields /// Writer for byte array (as in, any number of bytes per document) fast fields
/// ///

12
src/fastfield/delete.rs
View File

@@ -1,11 +1,11 @@
use crate::common::HasLen;
use crate::directory::ReadOnlySource;
use crate::directory::WritePtr;
use crate::space_usage::ByteCount;
use crate::DocId;
use bit_set::BitSet; use bit_set::BitSet;
use common::HasLen;
use directory::ReadOnlySource;
use directory::WritePtr;
use space_usage::ByteCount;
use std::io; use std::io;
use std::io::Write; use std::io::Write;
use DocId;
/// Write a delete `BitSet` /// Write a delete `BitSet`
/// ///
@@ -82,8 +82,8 @@ impl HasLen for DeleteBitSet {
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use super::*; use super::*;
use crate::directory::*;
use bit_set::BitSet; use bit_set::BitSet;
use directory::*;
use std::path::PathBuf; use std::path::PathBuf;
fn test_delete_bitset_helper(bitset: &BitSet) { fn test_delete_bitset_helper(bitset: &BitSet) {

4
src/fastfield/error.rs
View File

@@ -1,11 +1,11 @@
use crate::schema::FieldEntry; use schema::FieldEntry;
use std::result; use std::result;
/// `FastFieldNotAvailableError` is returned when the /// `FastFieldNotAvailableError` is returned when the
/// user requested for a fast field reader, and the field was not /// user requested for a fast field reader, and the field was not
/// defined in the schema as a fast field. /// defined in the schema as a fast field.
#[derive(Debug, Fail)] #[derive(Debug, Fail)]
#[fail(display = "Fast field not available: '{:?}'", field_name)] #[fail(display = "field not available: '{:?}'", field_name)]
pub struct FastFieldNotAvailableError { pub struct FastFieldNotAvailableError {
field_name: String, field_name: String,
} }

8
src/fastfield/facet_reader.rs
View File

@@ -1,9 +1,9 @@
use super::MultiValueIntFastFieldReader; use super::MultiValueIntFastFieldReader;
use crate::schema::Facet; use schema::Facet;
use crate::termdict::TermDictionary;
use crate::termdict::TermOrdinal;
use crate::DocId;
use std::str; use std::str;
use termdict::TermDictionary;
use termdict::TermOrdinal;
use DocId;
/// The facet reader makes it possible to access the list of /// The facet reader makes it possible to access the list of
/// facets associated to a given document in a specific /// facets associated to a given document in a specific

40
src/fastfield/mod.rs
View File

@@ -33,10 +33,10 @@ pub use self::reader::FastFieldReader;
pub use self::readers::FastFieldReaders; pub use self::readers::FastFieldReaders;
pub use self::serializer::FastFieldSerializer; pub use self::serializer::FastFieldSerializer;
pub use self::writer::{FastFieldsWriter, IntFastFieldWriter}; pub use self::writer::{FastFieldsWriter, IntFastFieldWriter};
use crate::common; use common;
use crate::schema::Cardinality; use schema::Cardinality;
use crate::schema::FieldType; use schema::FieldType;
use crate::schema::Value; use schema::Value;
mod bytes; mod bytes;
mod delete; mod delete;
@@ -49,7 +49,7 @@ mod serializer;
mod writer; mod writer;
/// Trait for types that are allowed for fast fields: (u64 or i64). /// Trait for types that are allowed for fast fields: (u64 or i64).
pub trait FastValue: Default + Clone + Copy + Send + Sync + PartialOrd { pub trait FastValue: Default + Clone + Copy {
/// Converts a value from u64 /// Converts a value from u64
/// ///
/// Internally all fast field values are encoded as u64. /// Internally all fast field values are encoded as u64.
@@ -126,27 +126,27 @@ fn value_to_u64(value: &Value) -> u64 {
mod tests { mod tests {
use super::*; use super::*;
use crate::common::CompositeFile; use common::CompositeFile;
use crate::directory::{Directory, RAMDirectory, WritePtr}; use directory::{Directory, RAMDirectory, WritePtr};
use crate::fastfield::FastFieldReader; use fastfield::FastFieldReader;
use crate::schema::Document;
use crate::schema::Field;
use crate::schema::Schema;
use crate::schema::FAST;
use once_cell::sync::Lazy;
use rand::prelude::SliceRandom; use rand::prelude::SliceRandom;
use rand::rngs::StdRng; use rand::rngs::StdRng;
use rand::SeedableRng; use rand::SeedableRng;
use schema::Document;
use schema::Field;
use schema::Schema;
use schema::FAST;
use std::collections::HashMap; use std::collections::HashMap;
use std::path::Path; use std::path::Path;
pub static SCHEMA: Lazy<Schema> = Lazy::new(|| { lazy_static! {
let mut schema_builder = Schema::builder(); pub static ref SCHEMA: Schema = {
schema_builder.add_u64_field("field", FAST); let mut schema_builder = Schema::builder();
schema_builder.build() schema_builder.add_u64_field("field", FAST);
}); schema_builder.build()
};
pub static FIELD: Lazy<Field> = Lazy::new(|| SCHEMA.get_field("field").unwrap()); pub static ref FIELD: Field = { SCHEMA.get_field("field").unwrap() };
}
#[test] #[test]
pub fn test_fastfield() { pub fn test_fastfield() {

16
src/fastfield/multivalued/mod.rs
View File

@@ -7,16 +7,16 @@ pub use self::writer::MultiValueIntFastFieldWriter;
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use time; extern crate time;
use self::time::Duration; use self::time::Duration;
use crate::collector::TopDocs; use collector::TopDocs;
use crate::query::QueryParser; use query::QueryParser;
use crate::schema::Cardinality; use schema::Cardinality;
use crate::schema::Facet; use schema::Facet;
use crate::schema::IntOptions; use schema::IntOptions;
use crate::schema::Schema; use schema::Schema;
use crate::Index; use Index;
#[test] #[test]
fn test_multivalued_u64() { fn test_multivalued_u64() {

8
src/fastfield/multivalued/reader.rs
View File

@@ -1,5 +1,5 @@
use crate::fastfield::{FastFieldReader, FastValue}; use fastfield::{FastFieldReader, FastValue};
use crate::DocId; use DocId;
/// Reader for a multivalued `u64` fast field. /// Reader for a multivalued `u64` fast field.
/// ///
@@ -64,8 +64,8 @@ impl<Item: FastValue> MultiValueIntFastFieldReader<Item> {
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use crate::core::Index; use core::Index;
use crate::schema::{Facet, Schema}; use schema::{Facet, Schema};
#[test] #[test]
fn test_multifastfield_reader() { fn test_multifastfield_reader() {

16
src/fastfield/multivalued/writer.rs
View File

@@ -1,13 +1,13 @@
use crate::fastfield::serializer::FastSingleFieldSerializer; use fastfield::serializer::FastSingleFieldSerializer;
use crate::fastfield::value_to_u64; use fastfield::value_to_u64;
use crate::fastfield::FastFieldSerializer; use fastfield::FastFieldSerializer;
use crate::postings::UnorderedTermId;
use crate::schema::{Document, Field};
use crate::termdict::TermOrdinal;
use crate::DocId;
use itertools::Itertools; use itertools::Itertools;
use postings::UnorderedTermId;
use schema::{Document, Field};
use std::collections::HashMap; use std::collections::HashMap;
use std::io; use std::io;
use termdict::TermOrdinal;
use DocId;
/// Writer for multi-valued (as in, more than one value per document) /// Writer for multi-valued (as in, more than one value per document)
/// int fast field. /// int fast field.
@@ -116,7 +116,7 @@ impl MultiValueIntFastFieldWriter {
} }
{ {
// writing the values themselves. // writing the values themselves.
let mut value_serializer: FastSingleFieldSerializer<'_, _>; let mut value_serializer: FastSingleFieldSerializer<_>;
match mapping_opt { match mapping_opt {
Some(mapping) => { Some(mapping) => {
value_serializer = serializer.new_u64_fast_field_with_idx( value_serializer = serializer.new_u64_fast_field_with_idx(

20
src/fastfield/reader.rs
View File

@@ -1,18 +1,18 @@
use super::FastValue; use super::FastValue;
use crate::common::bitpacker::BitUnpacker; use common::bitpacker::BitUnpacker;
use crate::common::compute_num_bits; use common::compute_num_bits;
use crate::common::BinarySerializable; use common::BinarySerializable;
use crate::common::CompositeFile; use common::CompositeFile;
use crate::directory::ReadOnlySource; use directory::ReadOnlySource;
use crate::directory::{Directory, RAMDirectory, WritePtr}; use directory::{Directory, RAMDirectory, WritePtr};
use crate::fastfield::{FastFieldSerializer, FastFieldsWriter}; use fastfield::{FastFieldSerializer, FastFieldsWriter};
use crate::schema::Schema;
use crate::schema::FAST;
use crate::DocId;
use owning_ref::OwningRef; use owning_ref::OwningRef;
use schema::Schema;
use schema::FAST;
use std::collections::HashMap; use std::collections::HashMap;
use std::marker::PhantomData; use std::marker::PhantomData;
use std::path::Path; use std::path::Path;
use DocId;
/// Trait for accessing a fastfield. /// Trait for accessing a fastfield.
/// ///

14
src/fastfield/readers.rs
View File

@@ -1,11 +1,11 @@
use crate::common::CompositeFile; use common::CompositeFile;
use crate::fastfield::BytesFastFieldReader; use fastfield::BytesFastFieldReader;
use crate::fastfield::MultiValueIntFastFieldReader; use fastfield::MultiValueIntFastFieldReader;
use crate::fastfield::{FastFieldNotAvailableError, FastFieldReader}; use fastfield::{FastFieldNotAvailableError, FastFieldReader};
use crate::schema::{Cardinality, Field, FieldType, Schema}; use schema::{Cardinality, Field, FieldType, Schema};
use crate::space_usage::PerFieldSpaceUsage; use space_usage::PerFieldSpaceUsage;
use crate::Result;
use std::collections::HashMap; use std::collections::HashMap;
use Result;
/// Provides access to all of the FastFieldReader. /// Provides access to all of the FastFieldReader.
/// ///

24
src/fastfield/serializer.rs
View File

@@ -1,10 +1,10 @@
use crate::common::bitpacker::BitPacker; use common::bitpacker::BitPacker;
use crate::common::compute_num_bits; use common::compute_num_bits;
use crate::common::BinarySerializable; use common::BinarySerializable;
use crate::common::CompositeWrite; use common::CompositeWrite;
use crate::common::CountingWriter; use common::CountingWriter;
use crate::directory::WritePtr; use directory::WritePtr;
use crate::schema::Field; use schema::Field;
use std::io::{self, Write}; use std::io::{self, Write};
/// `FastFieldSerializer` is in charge of serializing /// `FastFieldSerializer` is in charge of serializing
@@ -45,7 +45,7 @@ impl FastFieldSerializer {
field: Field, field: Field,
min_value: u64, min_value: u64,
max_value: u64, max_value: u64,
) -> io::Result<FastSingleFieldSerializer<'_, CountingWriter<WritePtr>>> { ) -> io::Result<FastSingleFieldSerializer<CountingWriter<WritePtr>>> {
self.new_u64_fast_field_with_idx(field, min_value, max_value, 0) self.new_u64_fast_field_with_idx(field, min_value, max_value, 0)
} }
@@ -56,7 +56,7 @@ impl FastFieldSerializer {
min_value: u64, min_value: u64,
max_value: u64, max_value: u64,
idx: usize, idx: usize,
) -> io::Result<FastSingleFieldSerializer<'_, CountingWriter<WritePtr>>> { ) -> io::Result<FastSingleFieldSerializer<CountingWriter<WritePtr>>> {
let field_write = self.composite_write.for_field_with_idx(field, idx); let field_write = self.composite_write.for_field_with_idx(field, idx);
FastSingleFieldSerializer::open(field_write, min_value, max_value) FastSingleFieldSerializer::open(field_write, min_value, max_value)
} }
@@ -66,7 +66,7 @@ impl FastFieldSerializer {
&mut self, &mut self,
field: Field, field: Field,
idx: usize, idx: usize,
) -> io::Result<FastBytesFieldSerializer<'_, CountingWriter<WritePtr>>> { ) -> io::Result<FastBytesFieldSerializer<CountingWriter<WritePtr>>> {
let field_write = self.composite_write.for_field_with_idx(field, idx); let field_write = self.composite_write.for_field_with_idx(field, idx);
FastBytesFieldSerializer::open(field_write) FastBytesFieldSerializer::open(field_write)
} }
@@ -79,7 +79,7 @@ impl FastFieldSerializer {
} }
} }
pub struct FastSingleFieldSerializer<'a, W: Write> { pub struct FastSingleFieldSerializer<'a, W: Write + 'a> {
bit_packer: BitPacker, bit_packer: BitPacker,
write: &'a mut W, write: &'a mut W,
min_value: u64, min_value: u64,
@@ -127,7 +127,7 @@ impl<'a, W: Write> FastSingleFieldSerializer<'a, W> {
} }
} }
pub struct FastBytesFieldSerializer<'a, W: Write> { pub struct FastBytesFieldSerializer<'a, W: Write + 'a> {
write: &'a mut W, write: &'a mut W,
} }

14
src/fastfield/writer.rs
View File

@@ -1,13 +1,13 @@
use super::multivalued::MultiValueIntFastFieldWriter; use super::multivalued::MultiValueIntFastFieldWriter;
use crate::common; use common;
use crate::common::BinarySerializable; use common::BinarySerializable;
use crate::common::VInt; use common::VInt;
use crate::fastfield::{BytesFastFieldWriter, FastFieldSerializer}; use fastfield::{BytesFastFieldWriter, FastFieldSerializer};
use crate::postings::UnorderedTermId; use postings::UnorderedTermId;
use crate::schema::{Cardinality, Document, Field, FieldType, Schema}; use schema::{Cardinality, Document, Field, FieldType, Schema};
use crate::termdict::TermOrdinal;
use std::collections::HashMap; use std::collections::HashMap;
use std::io; use std::io;
use termdict::TermOrdinal;
/// The fastfieldswriter regroup all of the fast field writers. /// The fastfieldswriter regroup all of the fast field writers.
pub struct FastFieldsWriter { pub struct FastFieldsWriter {

4
src/fieldnorm/reader.rs
View File

@@ -1,6 +1,6 @@
use super::{fieldnorm_to_id, id_to_fieldnorm}; use super::{fieldnorm_to_id, id_to_fieldnorm};
use crate::directory::ReadOnlySource; use directory::ReadOnlySource;
use crate::DocId; use DocId;
/// Reads the fieldnorm associated to a document. /// Reads the fieldnorm associated to a document.
/// The fieldnorm represents the length associated to /// The fieldnorm represents the length associated to

6
src/fieldnorm/serializer.rs
View File

@@ -1,6 +1,6 @@
use crate::common::CompositeWrite; use common::CompositeWrite;
use crate::directory::WritePtr; use directory::WritePtr;
use crate::schema::Field; use schema::Field;
use std::io; use std::io;
use std::io::Write; use std::io::Write;

6
src/fieldnorm/writer.rs
View File

@@ -1,9 +1,9 @@
use crate::DocId; use DocId;
use super::fieldnorm_to_id; use super::fieldnorm_to_id;
use super::FieldNormsSerializer; use super::FieldNormsSerializer;
use crate::schema::Field; use schema::Field;
use crate::schema::Schema; use schema::Schema;
use std::io; use std::io;
/// The `FieldNormsWriter` is in charge of tracking the fieldnorm byte /// The `FieldNormsWriter` is in charge of tracking the fieldnorm byte

6
src/functional_test.rs
View File

@@ -1,10 +1,10 @@
use rand::thread_rng; use rand::thread_rng;
use std::collections::HashSet; use std::collections::HashSet;
use crate::schema::*;
use crate::Index;
use crate::Searcher;
use rand::Rng; use rand::Rng;
use schema::*;
use Index;
use Searcher;
fn check_index_content(searcher: &Searcher, vals: &HashSet<u64>) { fn check_index_content(searcher: &Searcher, vals: &HashSet<u64>) {
assert!(searcher.segment_readers().len() < 20); assert!(searcher.segment_readers().len() < 20);

4
src/indexer/delete_queue.rs
View File

@@ -1,8 +1,8 @@
use super::operation::DeleteOperation; use super::operation::DeleteOperation;
use crate::Opstamp;
use std::mem; use std::mem;
use std::ops::DerefMut; use std::ops::DerefMut;
use std::sync::{Arc, RwLock}; use std::sync::{Arc, RwLock};
use Opstamp;
// The DeleteQueue is similar in conceptually to a multiple // The DeleteQueue is similar in conceptually to a multiple
// consumer single producer broadcast channel. // consumer single producer broadcast channel.
@@ -250,7 +250,7 @@ impl DeleteCursor {
mod tests { mod tests {
use super::{DeleteOperation, DeleteQueue}; use super::{DeleteOperation, DeleteQueue};
use crate::schema::{Field, Term}; use schema::{Field, Term};
#[test] #[test]
fn test_deletequeue() { fn test_deletequeue() {

4
src/indexer/doc_opstamp_mapping.rs
View File

@@ -1,6 +1,6 @@
use crate::DocId;
use crate::Opstamp;
use std::sync::Arc; use std::sync::Arc;
use DocId;
use Opstamp;
// Doc to opstamp is used to identify which // Doc to opstamp is used to identify which
// document should be deleted. // document should be deleted.

272
src/indexer/index_writer.rs
View File

@@ -1,37 +1,37 @@
use super::operation::{AddOperation, UserOperation}; use super::operation::{AddOperation, UserOperation};
use super::segment_updater::SegmentUpdater; use super::segment_updater::SegmentUpdater;
use super::PreparedCommit; use super::PreparedCommit;
use crate::core::Index;
use crate::core::Segment;
use crate::core::SegmentComponent;
use crate::core::SegmentId;
use crate::core::SegmentMeta;
use crate::core::SegmentReader;
use crate::directory::DirectoryLock;
use crate::docset::DocSet;
use crate::error::TantivyError;
use crate::fastfield::write_delete_bitset;
use crate::indexer::delete_queue::{DeleteCursor, DeleteQueue};
use crate::indexer::doc_opstamp_mapping::DocToOpstampMapping;
use crate::indexer::operation::DeleteOperation;
use crate::indexer::stamper::Stamper;
use crate::indexer::MergePolicy;
use crate::indexer::SegmentEntry;
use crate::indexer::SegmentWriter;
use crate::postings::compute_table_size;
use crate::schema::Document;
use crate::schema::IndexRecordOption;
use crate::schema::Term;
use crate::Opstamp;
use crate::Result;
use bit_set::BitSet; use bit_set::BitSet;
use core::Index;
use core::Segment;
use core::SegmentComponent;
use core::SegmentId;
use core::SegmentMeta;
use core::SegmentReader;
use crossbeam::channel; use crossbeam::channel;
use directory::DirectoryLock;
use docset::DocSet;
use error::TantivyError;
use fastfield::write_delete_bitset;
use futures::{Canceled, Future}; use futures::{Canceled, Future};
use indexer::delete_queue::{DeleteCursor, DeleteQueue};
use indexer::doc_opstamp_mapping::DocToOpstampMapping;
use indexer::operation::DeleteOperation;
use indexer::stamper::Stamper;
use indexer::MergePolicy;
use indexer::SegmentEntry;
use indexer::SegmentWriter;
use postings::compute_table_size;
use schema::Document;
use schema::IndexRecordOption;
use schema::Term;
use std::mem; use std::mem;
use std::ops::Range; use std::ops::Range;
use std::sync::Arc; use std::sync::Arc;
use std::thread; use std::thread;
use std::thread::JoinHandle; use std::thread::JoinHandle;
use Opstamp;
use Result;
// Size of the margin for the heap. A segment is closed when the remaining memory // Size of the margin for the heap. A segment is closed when the remaining memory
// in the heap goes below MARGIN_IN_BYTES. // in the heap goes below MARGIN_IN_BYTES.
@@ -268,7 +268,7 @@ fn index_documents(
memory_budget: usize, memory_budget: usize,
segment: &Segment, segment: &Segment,
generation: usize, generation: usize,
document_iterator: &mut dyn Iterator<Item = Vec<AddOperation>>, document_iterator: &mut Iterator<Item = Vec<AddOperation>>,
segment_updater: &mut SegmentUpdater, segment_updater: &mut SegmentUpdater,
mut delete_cursor: DeleteCursor, mut delete_cursor: DeleteCursor,
) -> Result<bool> { ) -> Result<bool> {
@@ -332,8 +332,7 @@ fn index_documents(
} }
impl IndexWriter { impl IndexWriter {
/// If there are some merging threads, blocks until they all finish their work and /// The index writer
/// then drop the `IndexWriter`.
pub fn wait_merging_threads(mut self) -> Result<()> { pub fn wait_merging_threads(mut self) -> Result<()> {
// this will stop the indexing thread, // this will stop the indexing thread,
// dropping the last reference to the segment_updater. // dropping the last reference to the segment_updater.
@@ -384,6 +383,7 @@ impl IndexWriter {
/// Spawns a new worker thread for indexing. /// Spawns a new worker thread for indexing.
/// The thread consumes documents from the pipeline. /// The thread consumes documents from the pipeline.
///
fn add_indexing_worker(&mut self) -> Result<()> { fn add_indexing_worker(&mut self) -> Result<()> {
let document_receiver_clone = self.operation_receiver.clone(); let document_receiver_clone = self.operation_receiver.clone();
let mut segment_updater = self.segment_updater.clone(); let mut segment_updater = self.segment_updater.clone();
@@ -440,12 +440,12 @@ impl IndexWriter {
} }
/// Accessor to the merge policy. /// Accessor to the merge policy.
pub fn get_merge_policy(&self) -> Arc<Box<dyn MergePolicy>> { pub fn get_merge_policy(&self) -> Arc<Box<MergePolicy>> {
self.segment_updater.get_merge_policy() self.segment_updater.get_merge_policy()
} }
/// Set the merge policy. /// Set the merge policy.
pub fn set_merge_policy(&self, merge_policy: Box<dyn MergePolicy>) { pub fn set_merge_policy(&self, merge_policy: Box<MergePolicy>) {
self.segment_updater.set_merge_policy(merge_policy); self.segment_updater.set_merge_policy(merge_policy);
} }
@@ -462,52 +462,6 @@ impl IndexWriter {
self.segment_updater.garbage_collect_files() self.segment_updater.garbage_collect_files()
} }
/// Deletes all documents from the index
///
/// Requires `commit`ing
/// Enables users to rebuild the index,
/// by clearing and resubmitting necessary documents
///
/// ```rust
/// #[macro_use]
/// extern crate tantivy;
/// use tantivy::query::QueryParser;
/// use tantivy::collector::TopDocs;
/// use tantivy::schema::*;
/// use tantivy::Index;
///
/// fn main() -> tantivy::Result<()> {
/// let mut schema_builder = Schema::builder();
/// let title = schema_builder.add_text_field("title", TEXT | STORED);
/// let schema = schema_builder.build();
///
/// let index = Index::create_in_ram(schema.clone());
///
/// let mut index_writer = index.writer_with_num_threads(1, 50_000_000)?;
/// index_writer.add_document(doc!(title => "The modern Promotheus"));
/// index_writer.commit()?;
///
/// let clear_res = index_writer.delete_all_documents().unwrap();
/// // have to commit, otherwise deleted terms remain available
/// index_writer.commit()?;
///
/// let searcher = index.reader()?.searcher();
/// let query_parser = QueryParser::for_index(&index, vec![title]);
/// let query_promo = query_parser.parse_query("Promotheus")?;
/// let top_docs_promo = searcher.search(&query_promo, &TopDocs::with_limit(1))?;
///
/// assert!(top_docs_promo.is_empty());
/// Ok(())
/// }
/// ```
pub fn delete_all_documents(&mut self) -> Result<Opstamp> {
// Delete segments
self.segment_updater.remove_all_segments();
// Return new stamp - reverted stamp
self.stamper.revert(self.committed_opstamp);
Ok(self.committed_opstamp)
}
/// Merges a given list of segments /// Merges a given list of segments
/// ///
/// `segment_ids` is required to be non-empty. /// `segment_ids` is required to be non-empty.
@@ -535,22 +489,19 @@ impl IndexWriter {
/// Rollback to the last commit /// Rollback to the last commit
/// ///
/// This cancels all of the updates that /// This cancels all of the update that
/// happened after the last commit. /// happened before after the last commit.
/// After calling rollback, the index is in the same /// After calling rollback, the index is in the same
/// state as it was after the last commit. /// state as it was after the last commit.
/// ///
/// The opstamp at the last commit is returned. /// The opstamp at the last commit is returned.
pub fn rollback(&mut self) -> Result<Opstamp> { pub fn rollback(&mut self) -> Result<Opstamp> {
info!("Rolling back to opstamp {}", self.committed_opstamp); info!("Rolling back to opstamp {}", self.committed_opstamp);
self.rollback_impl()
}
/// Private, implementation of rollback
fn rollback_impl(&mut self) -> Result<Opstamp> {
// marks the segment updater as killed. From now on, all // marks the segment updater as killed. From now on, all
// segment updates will be ignored. // segment updates will be ignored.
self.segment_updater.kill(); self.segment_updater.kill();
let document_receiver = self.operation_receiver.clone(); let document_receiver = self.operation_receiver.clone();
// take the directory lock to create a new index_writer. // take the directory lock to create a new index_writer.
@@ -603,7 +554,7 @@ impl IndexWriter {
/// It is also possible to add a payload to the `commit` /// It is also possible to add a payload to the `commit`
/// using this API. /// using this API.
/// See [`PreparedCommit::set_payload()`](PreparedCommit.html) /// See [`PreparedCommit::set_payload()`](PreparedCommit.html)
pub fn prepare_commit(&mut self) -> Result<PreparedCommit<'_>> { pub fn prepare_commit(&mut self) -> Result<PreparedCommit> {
// Here, because we join all of the worker threads, // Here, because we join all of the worker threads,
// all of the segment update for this commit have been // all of the segment update for this commit have been
// sent. // sent.
@@ -773,15 +724,15 @@ mod tests {
use super::super::operation::UserOperation; use super::super::operation::UserOperation;
use super::initial_table_size; use super::initial_table_size;
use crate::collector::TopDocs; use collector::TopDocs;
use crate::directory::error::LockError; use directory::error::LockError;
use crate::error::*; use error::*;
use crate::indexer::NoMergePolicy; use indexer::NoMergePolicy;
use crate::query::TermQuery; use query::TermQuery;
use crate::schema::{self, IndexRecordOption}; use schema::{self, IndexRecordOption};
use crate::Index; use Index;
use crate::ReloadPolicy; use ReloadPolicy;
use crate::Term; use Term;
#[test] #[test]
fn test_operations_group() { fn test_operations_group() {
@@ -1098,145 +1049,4 @@ mod tests {
assert_eq!(num_docs_containing("b"), 0); assert_eq!(num_docs_containing("b"), 0);
fail::cfg("RAMDirectory::atomic_write", "off").unwrap(); fail::cfg("RAMDirectory::atomic_write", "off").unwrap();
} }
#[test]
fn test_add_then_delete_all_documents() {
let mut schema_builder = schema::Schema::builder();
let text_field = schema_builder.add_text_field("text", schema::TEXT);
let index = Index::create_in_ram(schema_builder.build());
let reader = index
.reader_builder()
.reload_policy(ReloadPolicy::Manual)
.try_into()
.unwrap();
let num_docs_containing = |s: &str| {
reader.reload().unwrap();
let searcher = reader.searcher();
let term = Term::from_field_text(text_field, s);
searcher.doc_freq(&term)
};
let mut index_writer = index.writer_with_num_threads(4, 12_000_000).unwrap();
let add_tstamp = index_writer.add_document(doc!(text_field => "a"));
let commit_tstamp = index_writer.commit().unwrap();
assert!(commit_tstamp > add_tstamp);
index_writer.delete_all_documents().unwrap();
index_writer.commit().unwrap();
// Search for documents with the same term that we added
assert_eq!(num_docs_containing("a"), 0);
}
#[test]
fn test_delete_all_documents_rollback_correct_stamp() {
let mut schema_builder = schema::Schema::builder();
let text_field = schema_builder.add_text_field("text", schema::TEXT);
let index = Index::create_in_ram(schema_builder.build());
let mut index_writer = index.writer_with_num_threads(4, 12_000_000).unwrap();
let add_tstamp = index_writer.add_document(doc!(text_field => "a"));
// commit documents - they are now available
let first_commit = index_writer.commit();
assert!(first_commit.is_ok());
let first_commit_tstamp = first_commit.unwrap();
assert!(first_commit_tstamp > add_tstamp);
// delete_all_documents the index
let clear_tstamp = index_writer.delete_all_documents().unwrap();
assert_eq!(clear_tstamp, add_tstamp);
// commit the clear command - now documents aren't available
let second_commit = index_writer.commit();
assert!(second_commit.is_ok());
let second_commit_tstamp = second_commit.unwrap();
// add new documents again
for _ in 0..100 {
index_writer.add_document(doc!(text_field => "b"));
}
// rollback to last commit, when index was empty
let rollback = index_writer.rollback();
assert!(rollback.is_ok());
let rollback_tstamp = rollback.unwrap();
assert_eq!(rollback_tstamp, second_commit_tstamp);
// working with an empty index == no documents
let term_b = Term::from_field_text(text_field, "b");
assert_eq!(index.reader().unwrap().searcher().doc_freq(&term_b), 0);
}
#[test]
fn test_delete_all_documents_then_add() {
let mut schema_builder = schema::Schema::builder();
let text_field = schema_builder.add_text_field("text", schema::TEXT);
let index = Index::create_in_ram(schema_builder.build());
// writing the segment
let mut index_writer = index.writer_with_num_threads(4, 12_000_000).unwrap();
let res = index_writer.delete_all_documents();
assert!(res.is_ok());
assert!(index_writer.commit().is_ok());
// add one simple doc
index_writer.add_document(doc!(text_field => "a"));
assert!(index_writer.commit().is_ok());
let term_a = Term::from_field_text(text_field, "a");
// expect the document with that term to be in the index
assert_eq!(index.reader().unwrap().searcher().doc_freq(&term_a), 1);
}
#[test]
fn test_delete_all_documents_and_rollback() {
let mut schema_builder = schema::Schema::builder();
let text_field = schema_builder.add_text_field("text", schema::TEXT);
let index = Index::create_in_ram(schema_builder.build());
let mut index_writer = index.writer_with_num_threads(4, 12_000_000).unwrap();
// add one simple doc
index_writer.add_document(doc!(text_field => "a"));
let comm = index_writer.commit();
assert!(comm.is_ok());
let commit_tstamp = comm.unwrap();
// clear but don't commit!
let clear_tstamp = index_writer.delete_all_documents().unwrap();
// clear_tstamp should reset to before the last commit
assert!(clear_tstamp < commit_tstamp);
// rollback
let _rollback_tstamp = index_writer.rollback().unwrap();
// Find original docs in the index
let term_a = Term::from_field_text(text_field, "a");
// expect the document with that term to be in the index
assert_eq!(index.reader().unwrap().searcher().doc_freq(&term_a), 1);
}
#[test]
fn test_delete_all_documents_empty_index() {
let schema_builder = schema::Schema::builder();
let index = Index::create_in_ram(schema_builder.build());
let mut index_writer = index.writer_with_num_threads(4, 12_000_000).unwrap();
let clear = index_writer.delete_all_documents();
let commit = index_writer.commit();
assert!(clear.is_ok());
assert!(commit.is_ok());
}
#[test]
fn test_delete_all_documents_index_twice() {
let schema_builder = schema::Schema::builder();
let index = Index::create_in_ram(schema_builder.build());
let mut index_writer = index.writer_with_num_threads(4, 12_000_000).unwrap();
let clear = index_writer.delete_all_documents();
let commit = index_writer.commit();
assert!(clear.is_ok());
assert!(commit.is_ok());
let clear_again = index_writer.delete_all_documents();
let commit_again = index_writer.commit();
assert!(clear_again.is_ok());
assert!(commit_again.is_ok());
}
} }

6
src/indexer/log_merge_policy.rs
View File

@@ -1,5 +1,5 @@
use super::merge_policy::{MergeCandidate, MergePolicy}; use super::merge_policy::{MergeCandidate, MergePolicy};
use crate::core::SegmentMeta; use core::SegmentMeta;
use std::cmp; use std::cmp;
use std::f64; use std::f64;
@@ -95,8 +95,8 @@ impl Default for LogMergePolicy {
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use super::*; use super::*;
use crate::core::{SegmentId, SegmentMeta}; use core::{SegmentId, SegmentMeta};
use crate::indexer::merge_policy::MergePolicy; use indexer::merge_policy::MergePolicy;
fn test_merge_policy() -> LogMergePolicy { fn test_merge_policy() -> LogMergePolicy {
let mut log_merge_policy = LogMergePolicy::default(); let mut log_merge_policy = LogMergePolicy::default();

4
src/indexer/merge_operation.rs
View File

@@ -1,7 +1,7 @@
use crate::Opstamp;
use crate::SegmentId;
use census::{Inventory, TrackedObject}; use census::{Inventory, TrackedObject};
use std::collections::HashSet; use std::collections::HashSet;
use Opstamp;
use SegmentId;
#[derive(Default)] #[derive(Default)]
pub struct MergeOperationInventory(Inventory<InnerMergeOperation>); pub struct MergeOperationInventory(Inventory<InnerMergeOperation>);

8
src/indexer/merge_policy.rs
View File

@@ -1,5 +1,5 @@
use crate::core::SegmentId; use core::SegmentId;
use crate::core::SegmentMeta; use core::SegmentMeta;
use std::fmt::Debug; use std::fmt::Debug;
use std::marker; use std::marker;
@@ -39,8 +39,8 @@ impl MergePolicy for NoMergePolicy {
pub mod tests { pub mod tests {
use super::*; use super::*;
use crate::core::SegmentId; use core::SegmentId;
use crate::core::SegmentMeta; use core::SegmentMeta;
/// `MergePolicy` useful for test purposes. /// `MergePolicy` useful for test purposes.
/// ///

88
src/indexer/merger.rs
View File

@@ -1,31 +1,31 @@
use crate::common::MAX_DOC_LIMIT; use common::MAX_DOC_LIMIT;
use crate::core::Segment; use core::Segment;
use crate::core::SegmentReader; use core::SegmentReader;
use crate::core::SerializableSegment; use core::SerializableSegment;
use crate::docset::DocSet; use docset::DocSet;
use crate::fastfield::BytesFastFieldReader; use fastfield::BytesFastFieldReader;
use crate::fastfield::DeleteBitSet; use fastfield::DeleteBitSet;
use crate::fastfield::FastFieldReader; use fastfield::FastFieldReader;
use crate::fastfield::FastFieldSerializer; use fastfield::FastFieldSerializer;
use crate::fastfield::MultiValueIntFastFieldReader; use fastfield::MultiValueIntFastFieldReader;
use crate::fieldnorm::FieldNormReader; use fieldnorm::FieldNormReader;
use crate::fieldnorm::FieldNormsSerializer; use fieldnorm::FieldNormsSerializer;
use crate::fieldnorm::FieldNormsWriter; use fieldnorm::FieldNormsWriter;
use crate::indexer::SegmentSerializer; use indexer::SegmentSerializer;
use crate::postings::InvertedIndexSerializer;
use crate::postings::Postings;
use crate::schema::Cardinality;
use crate::schema::FieldType;
use crate::schema::{Field, Schema};
use crate::store::StoreWriter;
use crate::termdict::TermMerger;
use crate::termdict::TermOrdinal;
use crate::DocId;
use crate::Result;
use crate::TantivyError;
use itertools::Itertools; use itertools::Itertools;
use postings::InvertedIndexSerializer;
use postings::Postings;
use schema::Cardinality;
use schema::FieldType;
use schema::{Field, Schema};
use std::cmp; use std::cmp;
use std::collections::HashMap; use std::collections::HashMap;
use store::StoreWriter;
use termdict::TermMerger;
use termdict::TermOrdinal;
use DocId;
use Result;
use TantivyError;
fn compute_total_num_tokens(readers: &[SegmentReader], field: Field) -> u64 { fn compute_total_num_tokens(readers: &[SegmentReader], field: Field) -> u64 {
let mut total_tokens = 0u64; let mut total_tokens = 0u64;
@@ -692,28 +692,28 @@ impl SerializableSegment for IndexMerger {
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use crate::collector::tests::TestCollector;
use crate::collector::tests::{BytesFastFieldTestCollector, FastFieldTestCollector};
use crate::collector::{Count, FacetCollector};
use crate::core::Index;
use crate::query::AllQuery;
use crate::query::BooleanQuery;
use crate::query::TermQuery;
use crate::schema;
use crate::schema::Cardinality;
use crate::schema::Document;
use crate::schema::Facet;
use crate::schema::IndexRecordOption;
use crate::schema::IntOptions;
use crate::schema::Term;
use crate::schema::TextFieldIndexing;
use crate::schema::INDEXED;
use crate::DocAddress;
use crate::IndexWriter;
use crate::Searcher;
use byteorder::{BigEndian, ReadBytesExt, WriteBytesExt}; use byteorder::{BigEndian, ReadBytesExt, WriteBytesExt};
use collector::tests::TestCollector;
use collector::tests::{BytesFastFieldTestCollector, FastFieldTestCollector};
use collector::{Count, FacetCollector};
use core::Index;
use futures::Future; use futures::Future;
use query::AllQuery;
use query::BooleanQuery;
use query::TermQuery;
use schema;
use schema::Cardinality;
use schema::Document;
use schema::Facet;
use schema::IndexRecordOption;
use schema::IntOptions;
use schema::Term;
use schema::TextFieldIndexing;
use schema::INDEXED;
use std::io::Cursor; use std::io::Cursor;
use DocAddress;
use IndexWriter;
use Searcher;
#[test] #[test]
fn test_index_merger_no_deletes() { fn test_index_merger_no_deletes() {

6
src/indexer/operation.rs
View File

@@ -1,6 +1,6 @@
use crate::schema::Document; use schema::Document;
use crate::schema::Term; use schema::Term;
use crate::Opstamp; use Opstamp;
/// Timestamped Delete operation. /// Timestamped Delete operation.
#[derive(Clone, Eq, PartialEq, Debug)] #[derive(Clone, Eq, PartialEq, Debug)]

6
src/indexer/prepared_commit.rs
View File

@@ -1,6 +1,6 @@
use super::IndexWriter; use super::IndexWriter;
use crate::Opstamp; use Opstamp;
use crate::Result; use Result;
/// A prepared commit /// A prepared commit
pub struct PreparedCommit<'a> { pub struct PreparedCommit<'a> {
@@ -10,7 +10,7 @@ pub struct PreparedCommit<'a> {
} }
impl<'a> PreparedCommit<'a> { impl<'a> PreparedCommit<'a> {
pub(crate) fn new(index_writer: &'a mut IndexWriter, opstamp: Opstamp) -> PreparedCommit<'_> { pub(crate) fn new(index_writer: &'a mut IndexWriter, opstamp: Opstamp) -> PreparedCommit {
PreparedCommit { PreparedCommit {
index_writer, index_writer,
payload: None, payload: None,

8
src/indexer/segment_entry.rs
View File

@@ -1,7 +1,7 @@
use crate::core::SegmentId;
use crate::core::SegmentMeta;
use crate::indexer::delete_queue::DeleteCursor;
use bit_set::BitSet; use bit_set::BitSet;
use core::SegmentId;
use core::SegmentMeta;
use indexer::delete_queue::DeleteCursor;
use std::fmt; use std::fmt;
/// A segment entry describes the state of /// A segment entry describes the state of
@@ -67,7 +67,7 @@ impl SegmentEntry {
} }
impl fmt::Debug for SegmentEntry { impl fmt::Debug for SegmentEntry {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
write!(formatter, "SegmentEntry({:?})", self.meta) write!(formatter, "SegmentEntry({:?})", self.meta)
} }
} }

28
src/indexer/segment_manager.rs
View File

@@ -1,16 +1,16 @@
use super::segment_register::SegmentRegister; use super::segment_register::SegmentRegister;
use crate::core::SegmentId; use core::SegmentId;
use crate::core::SegmentMeta; use core::SegmentMeta;
use crate::core::META_FILEPATH; use core::META_FILEPATH;
use crate::error::TantivyError; use error::TantivyError;
use crate::indexer::delete_queue::DeleteCursor; use indexer::delete_queue::DeleteCursor;
use crate::indexer::SegmentEntry; use indexer::SegmentEntry;
use crate::Result as TantivyResult;
use std::collections::hash_set::HashSet; use std::collections::hash_set::HashSet;
use std::fmt::{self, Debug, Formatter}; use std::fmt::{self, Debug, Formatter};
use std::path::PathBuf; use std::path::PathBuf;
use std::sync::RwLock; use std::sync::RwLock;
use std::sync::{RwLockReadGuard, RwLockWriteGuard}; use std::sync::{RwLockReadGuard, RwLockWriteGuard};
use Result as TantivyResult;
#[derive(Default)] #[derive(Default)]
struct SegmentRegisters { struct SegmentRegisters {
@@ -29,7 +29,7 @@ pub struct SegmentManager {
} }
impl Debug for SegmentManager { impl Debug for SegmentManager {
fn fmt(&self, f: &mut Formatter<'_>) -> Result<(), fmt::Error> { fn fmt(&self, f: &mut Formatter) -> Result<(), fmt::Error> {
let lock = self.read(); let lock = self.read();
write!( write!(
f, f,
@@ -81,7 +81,7 @@ impl SegmentManager {
/// but have not yet been deleted by the garbage collector. /// but have not yet been deleted by the garbage collector.
pub fn list_files(&self) -> HashSet<PathBuf> { pub fn list_files(&self) -> HashSet<PathBuf> {
let mut files = HashSet::new(); let mut files = HashSet::new();
files.insert(META_FILEPATH.to_path_buf()); files.insert(META_FILEPATH.clone());
for segment_meta in SegmentMeta::all() { for segment_meta in SegmentMeta::all() {
files.extend(segment_meta.list_files()); files.extend(segment_meta.list_files());
} }
@@ -91,13 +91,13 @@ impl SegmentManager {
// Lock poisoning should never happen : // Lock poisoning should never happen :
// The lock is acquired and released within this class, // The lock is acquired and released within this class,
// and the operations cannot panic. // and the operations cannot panic.
fn read(&self) -> RwLockReadGuard<'_, SegmentRegisters> { fn read(&self) -> RwLockReadGuard<SegmentRegisters> {
self.registers self.registers
.read() .read()
.expect("Failed to acquire read lock on SegmentManager.") .expect("Failed to acquire read lock on SegmentManager.")
} }
fn write(&self) -> RwLockWriteGuard<'_, SegmentRegisters> { fn write(&self) -> RwLockWriteGuard<SegmentRegisters> {
self.registers self.registers
.write() .write()
.expect("Failed to acquire write lock on SegmentManager.") .expect("Failed to acquire write lock on SegmentManager.")
@@ -118,12 +118,6 @@ impl SegmentManager {
}); });
} }
pub(crate) fn remove_all_segments(&self) {
let mut registers_lock = self.write();
registers_lock.committed.clear();
registers_lock.uncommitted.clear();
}
pub fn commit(&self, segment_entries: Vec<SegmentEntry>) { pub fn commit(&self, segment_entries: Vec<SegmentEntry>) {
let mut registers_lock = self.write(); let mut registers_lock = self.write();
registers_lock.committed.clear(); registers_lock.committed.clear();

16
src/indexer/segment_register.rs
View File

@@ -1,7 +1,7 @@
use crate::core::SegmentId; use core::SegmentId;
use crate::core::SegmentMeta; use core::SegmentMeta;
use crate::indexer::delete_queue::DeleteCursor; use indexer::delete_queue::DeleteCursor;
use crate::indexer::segment_entry::SegmentEntry; use indexer::segment_entry::SegmentEntry;
use std::collections::HashMap; use std::collections::HashMap;
use std::collections::HashSet; use std::collections::HashSet;
use std::fmt::{self, Debug, Formatter}; use std::fmt::{self, Debug, Formatter};
@@ -20,7 +20,7 @@ pub struct SegmentRegister {
} }
impl Debug for SegmentRegister { impl Debug for SegmentRegister {
fn fmt(&self, f: &mut Formatter<'_>) -> Result<(), fmt::Error> { fn fmt(&self, f: &mut Formatter) -> Result<(), fmt::Error> {
write!(f, "SegmentRegister(")?; write!(f, "SegmentRegister(")?;
for k in self.segment_states.keys() { for k in self.segment_states.keys() {
write!(f, "{}, ", k.short_uuid_string())?; write!(f, "{}, ", k.short_uuid_string())?;
@@ -93,9 +93,9 @@ impl SegmentRegister {
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use super::*; use super::*;
use crate::core::SegmentId; use core::SegmentId;
use crate::core::SegmentMeta; use core::SegmentMeta;
use crate::indexer::delete_queue::*; use indexer::delete_queue::*;
fn segment_ids(segment_register: &SegmentRegister) -> Vec<SegmentId> { fn segment_ids(segment_register: &SegmentRegister) -> Vec<SegmentId> {
segment_register segment_register

14
src/indexer/segment_serializer.rs
View File

@@ -1,11 +1,11 @@
use crate::Result; use Result;
use crate::core::Segment; use core::Segment;
use crate::core::SegmentComponent; use core::SegmentComponent;
use crate::fastfield::FastFieldSerializer; use fastfield::FastFieldSerializer;
use crate::fieldnorm::FieldNormsSerializer; use fieldnorm::FieldNormsSerializer;
use crate::postings::InvertedIndexSerializer; use postings::InvertedIndexSerializer;
use crate::store::StoreWriter; use store::StoreWriter;
/// Segment serializer is in charge of laying out on disk /// Segment serializer is in charge of laying out on disk
/// the data accumulated and sorted by the `SegmentWriter`. /// the data accumulated and sorted by the `SegmentWriter`.

158
src/indexer/segment_updater.rs
View File

@@ -1,31 +1,29 @@
use super::segment_manager::{get_mergeable_segments, SegmentManager}; use super::segment_manager::{get_mergeable_segments, SegmentManager};
use crate::core::Index; use core::Index;
use crate::core::IndexMeta; use core::IndexMeta;
use crate::core::Segment; use core::Segment;
use crate::core::SegmentId; use core::SegmentId;
use crate::core::SegmentMeta; use core::SegmentMeta;
use crate::core::SerializableSegment; use core::SerializableSegment;
use crate::core::META_FILEPATH; use core::META_FILEPATH;
use crate::directory::{Directory, DirectoryClone}; use directory::{Directory, DirectoryClone};
use crate::error::TantivyError; use error::TantivyError;
use crate::indexer::delete_queue::DeleteCursor;
use crate::indexer::index_writer::advance_deletes;
use crate::indexer::merge_operation::MergeOperationInventory;
use crate::indexer::merger::IndexMerger;
use crate::indexer::stamper::Stamper;
use crate::indexer::MergeOperation;
use crate::indexer::SegmentEntry;
use crate::indexer::SegmentSerializer;
use crate::indexer::{DefaultMergePolicy, MergePolicy};
use crate::schema::Schema;
use crate::Opstamp;
use crate::Result;
use futures::oneshot; use futures::oneshot;
use futures::sync::oneshot::Receiver; use futures::sync::oneshot::Receiver;
use futures::Future; use futures::Future;
use futures_cpupool::Builder as CpuPoolBuilder; use futures_cpupool::Builder as CpuPoolBuilder;
use futures_cpupool::CpuFuture; use futures_cpupool::CpuFuture;
use futures_cpupool::CpuPool; use futures_cpupool::CpuPool;
use indexer::delete_queue::DeleteCursor;
use indexer::index_writer::advance_deletes;
use indexer::merge_operation::MergeOperationInventory;
use indexer::merger::IndexMerger;
use indexer::stamper::Stamper;
use indexer::MergeOperation;
use indexer::SegmentEntry;
use indexer::SegmentSerializer;
use indexer::{DefaultMergePolicy, MergePolicy};
use schema::Schema;
use serde_json; use serde_json;
use std::borrow::BorrowMut; use std::borrow::BorrowMut;
use std::collections::HashMap; use std::collections::HashMap;
@@ -38,17 +36,19 @@ use std::sync::Arc;
use std::sync::RwLock; use std::sync::RwLock;
use std::thread; use std::thread;
use std::thread::JoinHandle; use std::thread::JoinHandle;
use Opstamp;
use Result;
/// Save the index meta file. /// Save the index meta file.
/// This operation is atomic : /// This operation is atomic :
/// Either /// Either
/// - it fails, in which case an error is returned, // - it fails, in which case an error is returned,
/// and the `meta.json` remains untouched, /// and the `meta.json` remains untouched,
/// - it succeeds, and `meta.json` is written /// - it success, and `meta.json` is written
/// and flushed. /// and flushed.
/// ///
/// This method is not part of tantivy's public API /// This method is not part of tantivy's public API
pub fn save_new_metas(schema: Schema, directory: &mut dyn Directory) -> Result<()> { pub fn save_new_metas(schema: Schema, directory: &mut Directory) -> Result<()> {
save_metas( save_metas(
&IndexMeta { &IndexMeta {
segments: Vec::new(), segments: Vec::new(),
@@ -69,8 +69,7 @@ pub fn save_new_metas(schema: Schema, directory: &mut dyn Directory) -> Result<(
/// and flushed. /// and flushed.
/// ///
/// This method is not part of tantivy's public API /// This method is not part of tantivy's public API
fn save_metas(metas: &IndexMeta, directory: &mut dyn Directory) -> Result<()> { fn save_metas(metas: &IndexMeta, directory: &mut Directory) -> Result<()> {
info!("save metas");
let mut buffer = serde_json::to_vec_pretty(metas)?; let mut buffer = serde_json::to_vec_pretty(metas)?;
// Just adding a new line at the end of the buffer. // Just adding a new line at the end of the buffer.
writeln!(&mut buffer)?; writeln!(&mut buffer)?;
@@ -142,7 +141,7 @@ struct InnerSegmentUpdater {
pool: CpuPool, pool: CpuPool,
index: Index, index: Index,
segment_manager: SegmentManager, segment_manager: SegmentManager,
merge_policy: RwLock<Arc<Box<dyn MergePolicy>>>, merge_policy: RwLock<Arc<Box<MergePolicy>>>,
merging_thread_id: AtomicUsize, merging_thread_id: AtomicUsize,
merging_threads: RwLock<HashMap<usize, JoinHandle<Result<()>>>>, merging_threads: RwLock<HashMap<usize, JoinHandle<Result<()>>>>,
generation: AtomicUsize, generation: AtomicUsize,
@@ -179,11 +178,11 @@ impl SegmentUpdater {
}))) })))
} }
pub fn get_merge_policy(&self) -> Arc<Box<dyn MergePolicy>> { pub fn get_merge_policy(&self) -> Arc<Box<MergePolicy>> {
self.0.merge_policy.read().unwrap().clone() self.0.merge_policy.read().unwrap().clone()
} }
pub fn set_merge_policy(&self, merge_policy: Box<dyn MergePolicy>) { pub fn set_merge_policy(&self, merge_policy: Box<MergePolicy>) {
let arc_merge_policy = Arc::new(merge_policy); let arc_merge_policy = Arc::new(merge_policy);
*self.0.merge_policy.write().unwrap() = arc_merge_policy; *self.0.merge_policy.write().unwrap() = arc_merge_policy;
} }
@@ -214,11 +213,6 @@ impl SegmentUpdater {
} }
} }
/// Orders `SegmentManager` to remove all segments
pub(crate) fn remove_all_segments(&self) {
self.0.segment_manager.remove_all_segments();
}
pub fn kill(&mut self) { pub fn kill(&mut self) {
self.0.killed.store(true, Ordering::Release); self.0.killed.store(true, Ordering::Release);
} }
@@ -229,7 +223,7 @@ impl SegmentUpdater {
/// Apply deletes up to the target opstamp to all segments. /// Apply deletes up to the target opstamp to all segments.
/// ///
/// The method returns copies of the segment entries, /// Tne method returns copies of the segment entries,
/// updated with the delete information. /// updated with the delete information.
fn purge_deletes(&self, target_opstamp: Opstamp) -> Result<Vec<SegmentEntry>> { fn purge_deletes(&self, target_opstamp: Opstamp) -> Result<Vec<SegmentEntry>> {
let mut segment_entries = self.0.segment_manager.segment_entries(); let mut segment_entries = self.0.segment_manager.segment_entries();
@@ -452,41 +446,38 @@ impl SegmentUpdater {
) -> Result<()> { ) -> Result<()> {
self.run_async(move |segment_updater| { self.run_async(move |segment_updater| {
info!("End merge {:?}", after_merge_segment_entry.meta()); info!("End merge {:?}", after_merge_segment_entry.meta());
{ let mut delete_cursor = after_merge_segment_entry.delete_cursor().clone();
let mut delete_cursor = after_merge_segment_entry.delete_cursor().clone(); if let Some(delete_operation) = delete_cursor.get() {
if let Some(delete_operation) = delete_cursor.get() { let committed_opstamp = segment_updater.load_metas().opstamp;
let committed_opstamp = segment_updater.load_metas().opstamp; if delete_operation.opstamp < committed_opstamp {
if delete_operation.opstamp < committed_opstamp { let index = &segment_updater.0.index;
let index = &segment_updater.0.index; let segment = index.segment(after_merge_segment_entry.meta().clone());
let segment = index.segment(after_merge_segment_entry.meta().clone()); if let Err(e) =
if let Err(e) = advance_deletes( advance_deletes(segment, &mut after_merge_segment_entry, committed_opstamp)
segment, {
&mut after_merge_segment_entry, error!(
committed_opstamp, "Merge of {:?} was cancelled (advancing deletes failed): {:?}",
) { merge_operation.segment_ids(),
error!( e
"Merge of {:?} was cancelled (advancing deletes failed): {:?}", );
merge_operation.segment_ids(), if cfg!(test) {
e panic!("Merge failed.");
);
if cfg!(test) {
panic!("Merge failed.");
}
// ... cancel merge
// `merge_operations` are tracked. As it is dropped, the
// the segment_ids will be available again for merge.
return;
} }
// ... cancel merge
// `merge_operations` are tracked. As it is dropped, the
// the segment_ids will be available again for merge.
return;
} }
} }
let previous_metas = segment_updater.load_metas(); }
segment_updater segment_updater
.0 .0
.segment_manager .segment_manager
.end_merge(merge_operation.segment_ids(), after_merge_segment_entry); .end_merge(merge_operation.segment_ids(), after_merge_segment_entry);
segment_updater.consider_merge_options(); segment_updater.consider_merge_options();
segment_updater.save_metas(previous_metas.opstamp, previous_metas.payload.clone()); info!("save metas");
} // we drop all possible handle to a now useless `SegmentMeta`. let previous_metas = segment_updater.load_metas();
segment_updater.save_metas(previous_metas.opstamp, previous_metas.payload.clone());
segment_updater.garbage_collect_files_exec(); segment_updater.garbage_collect_files_exec();
}) })
.wait() .wait()
@@ -533,9 +524,9 @@ impl SegmentUpdater {
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use crate::indexer::merge_policy::tests::MergeWheneverPossible; use indexer::merge_policy::tests::MergeWheneverPossible;
use crate::schema::*; use schema::*;
use crate::Index; use Index;
#[test] #[test]
fn test_delete_during_merge() { fn test_delete_during_merge() {
@@ -660,31 +651,4 @@ mod tests {
assert!(index.searchable_segment_metas().unwrap().is_empty()); assert!(index.searchable_segment_metas().unwrap().is_empty());
assert!(reader.searcher().segment_readers().is_empty()); assert!(reader.searcher().segment_readers().is_empty());
} }
#[test]
fn test_remove_all_segments() {
let mut schema_builder = Schema::builder();
let text_field = schema_builder.add_text_field("text", TEXT);
let schema = schema_builder.build();
let index = Index::create_in_ram(schema);
// writing the segment
let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
{
for _ in 0..100 {
index_writer.add_document(doc!(text_field=>"a"));
index_writer.add_document(doc!(text_field=>"b"));
}
assert!(index_writer.commit().is_ok());
}
index_writer.segment_updater().remove_all_segments();
let seg_vec = index_writer
.segment_updater()
.0
.segment_manager
.segment_entries();
assert!(seg_vec.is_empty());
}
} }

38
src/indexer/segment_writer.rs
View File

@@ -1,26 +1,26 @@
use super::operation::AddOperation; use super::operation::AddOperation;
use crate::core::Segment; use core::Segment;
use crate::core::SerializableSegment; use core::SerializableSegment;
use crate::fastfield::FastFieldsWriter; use fastfield::FastFieldsWriter;
use crate::fieldnorm::FieldNormsWriter; use fieldnorm::FieldNormsWriter;
use crate::indexer::segment_serializer::SegmentSerializer; use indexer::segment_serializer::SegmentSerializer;
use crate::postings::MultiFieldPostingsWriter; use postings::MultiFieldPostingsWriter;
use crate::schema::FieldEntry; use schema::FieldEntry;
use crate::schema::FieldType; use schema::FieldType;
use crate::schema::Schema; use schema::Schema;
use crate::schema::Term; use schema::Term;
use crate::schema::Value; use schema::Value;
use crate::tokenizer::BoxedTokenizer;
use crate::tokenizer::FacetTokenizer;
use crate::tokenizer::{TokenStream, Tokenizer};
use crate::DocId;
use crate::Opstamp;
use crate::Result;
use std::io; use std::io;
use std::str; use std::str;
use tokenizer::BoxedTokenizer;
use tokenizer::FacetTokenizer;
use tokenizer::{TokenStream, Tokenizer};
use DocId;
use Opstamp;
use Result;
/// A `SegmentWriter` is in charge of creating segment index from a /// A `SegmentWriter` is in charge of creating segment index from a
/// set of documents. /// documents.
/// ///
/// They creates the postings list in anonymous memory. /// They creates the postings list in anonymous memory.
/// The segment is layed on disk when the segment gets `finalized`. /// The segment is layed on disk when the segment gets `finalized`.
@@ -31,7 +31,7 @@ pub struct SegmentWriter {
fast_field_writers: FastFieldsWriter, fast_field_writers: FastFieldsWriter,
fieldnorms_writer: FieldNormsWriter, fieldnorms_writer: FieldNormsWriter,
doc_opstamps: Vec<Opstamp>, doc_opstamps: Vec<Opstamp>,
tokenizers: Vec<Option<Box<dyn BoxedTokenizer>>>, tokenizers: Vec<Option<Box<BoxedTokenizer>>>,
} }
impl SegmentWriter { impl SegmentWriter {

21
src/indexer/stamper.rs
View File

@@ -1,7 +1,7 @@
use crate::Opstamp;
use std::ops::Range; use std::ops::Range;
use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::Arc; use std::sync::Arc;
use Opstamp;
/// Stamper provides Opstamps, which is just an auto-increment id to label /// Stamper provides Opstamps, which is just an auto-increment id to label
/// an operation. /// an operation.
@@ -28,12 +28,6 @@ impl Stamper {
end: start + n, end: start + n,
} }
} }
/// Reverts the stamper to a given `Opstamp` value and returns it
pub fn revert(&self, to_opstamp: Opstamp) -> Opstamp {
self.0.store(to_opstamp, Ordering::SeqCst);
to_opstamp
}
} }
#[cfg(test)] #[cfg(test)]
@@ -56,17 +50,4 @@ mod test {
assert_eq!(stamper.stamp(), 15u64); assert_eq!(stamper.stamp(), 15u64);
} }
#[test]
fn test_stamper_revert() {
let stamper = Stamper::new(7u64);
assert_eq!(stamper.stamp(), 7u64);
assert_eq!(stamper.stamp(), 8u64);
let stamper_clone = stamper.clone();
assert_eq!(stamper_clone.stamp(), 9u64);
stamper.revert(6);
assert_eq!(stamper.stamp(), 6);
assert_eq!(stamper_clone.stamp(), 7);
}
} }

108
src/lib.rs
View File

@@ -108,6 +108,9 @@
//! [literate programming](http://fulmicoton.com/tantivy-examples/simple_search.html) / //! [literate programming](http://fulmicoton.com/tantivy-examples/simple_search.html) /
//! [source code](https://github.com/fulmicoton/tantivy/blob/master/examples/simple_search.rs)) //! [source code](https://github.com/fulmicoton/tantivy/blob/master/examples/simple_search.rs))
#[macro_use]
extern crate lazy_static;
#[macro_use] #[macro_use]
extern crate serde_derive; extern crate serde_derive;
@@ -120,9 +123,57 @@ extern crate log;
#[macro_use] #[macro_use]
extern crate failure; extern crate failure;
#[cfg(feature = "mmap")]
extern crate atomicwrites;
extern crate base64;
extern crate bit_set;
extern crate bitpacking;
extern crate byteorder;
extern crate combine;
extern crate crossbeam;
extern crate fnv;
extern crate futures;
extern crate futures_cpupool;
extern crate htmlescape;
extern crate itertools;
extern crate levenshtein_automata;
#[cfg(feature = "mmap")]
extern crate memmap;
extern crate num_cpus;
extern crate owning_ref;
extern crate regex;
extern crate rust_stemmers;
extern crate scoped_pool;
extern crate serde;
extern crate stable_deref_trait;
extern crate tantivy_fst;
extern crate tempdir;
extern crate tempfile;
extern crate uuid;
#[cfg(test)]
#[macro_use]
extern crate matches;
#[cfg(windows)]
extern crate winapi;
#[cfg(test)]
extern crate rand;
#[cfg(test)]
#[macro_use]
extern crate maplit;
#[cfg(all(test, feature = "unstable"))] #[cfg(all(test, feature = "unstable"))]
extern crate test; extern crate test;
#[macro_use]
extern crate downcast_rs;
#[macro_use]
extern crate fail;
#[cfg(feature = "mmap")] #[cfg(feature = "mmap")]
#[cfg(test)] #[cfg(test)]
mod functional_test; mod functional_test;
@@ -130,11 +181,14 @@ mod functional_test;
#[macro_use] #[macro_use]
mod macros; mod macros;
pub use crate::error::TantivyError; pub use error::TantivyError;
#[deprecated(since = "0.7.0", note = "please use `tantivy::TantivyError` instead")] #[deprecated(since = "0.7.0", note = "please use `tantivy::TantivyError` instead")]
pub use crate::error::TantivyError as Error; pub use error::TantivyError as Error;
pub use chrono;
extern crate census;
pub extern crate chrono;
extern crate owned_read;
/// Tantivy result. /// Tantivy result.
pub type Result<T> = std::result::Result<T, error::TantivyError>; pub type Result<T> = std::result::Result<T, error::TantivyError>;
@@ -171,15 +225,15 @@ pub use self::snippet::{Snippet, SnippetGenerator};
mod docset; mod docset;
pub use self::docset::{DocSet, SkipResult}; pub use self::docset::{DocSet, SkipResult};
pub use crate::core::SegmentComponent; pub use core::SegmentComponent;
pub use crate::core::{Index, IndexMeta, Searcher, Segment, SegmentId, SegmentMeta}; pub use core::{Index, Searcher, Segment, SegmentId, SegmentMeta, IndexMeta};
pub use crate::core::{InvertedIndexReader, SegmentReader}; pub use core::{InvertedIndexReader, SegmentReader};
pub use crate::directory::Directory; pub use directory::Directory;
pub use crate::indexer::IndexWriter; pub use indexer::IndexWriter;
pub use crate::postings::Postings; pub use postings::Postings;
pub use crate::schema::{Document, Term}; pub use schema::{Document, Term};
pub use crate::common::{i64_to_u64, u64_to_i64}; pub use common::{i64_to_u64, u64_to_i64};
/// Expose the current version of tantivy, as well /// Expose the current version of tantivy, as well
/// whether it was compiled with the simd compression. /// whether it was compiled with the simd compression.
@@ -189,10 +243,10 @@ pub fn version() -> &'static str {
/// Defines tantivy's merging strategy /// Defines tantivy's merging strategy
pub mod merge_policy { pub mod merge_policy {
pub use crate::indexer::DefaultMergePolicy; pub use indexer::DefaultMergePolicy;
pub use crate::indexer::LogMergePolicy; pub use indexer::LogMergePolicy;
pub use crate::indexer::MergePolicy; pub use indexer::MergePolicy;
pub use crate::indexer::NoMergePolicy; pub use indexer::NoMergePolicy;
} }
/// A `u32` identifying a document within a segment. /// A `u32` identifying a document within a segment.
@@ -250,20 +304,20 @@ pub struct DocAddress(pub SegmentLocalId, pub DocId);
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use crate::collector::tests::TestCollector; use collector::tests::TestCollector;
use crate::core::SegmentReader; use core::SegmentReader;
use crate::docset::DocSet; use docset::DocSet;
use crate::query::BooleanQuery; use query::BooleanQuery;
use crate::schema::*;
use crate::DocAddress;
use crate::Index;
use crate::IndexWriter;
use crate::Postings;
use crate::ReloadPolicy;
use rand::distributions::Bernoulli; use rand::distributions::Bernoulli;
use rand::distributions::Uniform; use rand::distributions::Uniform;
use rand::rngs::StdRng; use rand::rngs::StdRng;
use rand::{Rng, SeedableRng}; use rand::{Rng, SeedableRng};
use schema::*;
use DocAddress;
use Index;
use IndexWriter;
use Postings;
use ReloadPolicy;
pub fn assert_nearly_equals(expected: f32, val: f32) { pub fn assert_nearly_equals(expected: f32, val: f32) {
assert!( assert!(
@@ -288,7 +342,7 @@ mod tests {
pub fn sample_with_seed(n: u32, ratio: f64, seed_val: u8) -> Vec<u32> { pub fn sample_with_seed(n: u32, ratio: f64, seed_val: u8) -> Vec<u32> {
StdRng::from_seed([seed_val; 32]) StdRng::from_seed([seed_val; 32])
.sample_iter(&Bernoulli::new(ratio).unwrap()) .sample_iter(&Bernoulli::new(ratio))
.take(n as usize) .take(n as usize)
.enumerate() .enumerate()
.filter_map(|(val, keep)| if keep { Some(val as u32) } else { None }) .filter_map(|(val, keep)| if keep { Some(val as u32) } else { None })
@@ -426,7 +480,7 @@ mod tests {
} }
} }
fn advance_undeleted(docset: &mut dyn DocSet, reader: &SegmentReader) -> bool { fn advance_undeleted(docset: &mut DocSet, reader: &SegmentReader) -> bool {
while docset.advance() { while docset.advance() {
if !reader.is_deleted(docset.doc()) { if !reader.is_deleted(docset.doc()) {
return true; return true;

2
src/macros.rs
View File

@@ -67,7 +67,7 @@ macro_rules! doc(
#[cfg(test)] #[cfg(test)]
mod test { mod test {
use crate::schema::{Schema, FAST, TEXT}; use schema::{Schema, FAST, TEXT};
#[test] #[test]
fn test_doc_basic() { fn test_doc_basic() {

4
src/positions/mod.rs
View File

@@ -38,8 +38,8 @@ const LONG_SKIP_INTERVAL: u64 = (LONG_SKIP_IN_BLOCKS * COMPRESSION_BLOCK_SIZE) a
pub mod tests { pub mod tests {
use super::{PositionReader, PositionSerializer}; use super::{PositionReader, PositionSerializer};
use crate::directory::ReadOnlySource; use directory::ReadOnlySource;
use crate::positions::COMPRESSION_BLOCK_SIZE; use positions::COMPRESSION_BLOCK_SIZE;
use std::iter; use std::iter;
fn create_stream_buffer(vals: &[u32]) -> (ReadOnlySource, ReadOnlySource) { fn create_stream_buffer(vals: &[u32]) -> (ReadOnlySource, ReadOnlySource) {

12
src/positions/reader.rs
View File

@@ -1,9 +1,3 @@
use crate::common::{BinarySerializable, FixedSize};
use crate::directory::ReadOnlySource;
use crate::positions::COMPRESSION_BLOCK_SIZE;
use crate::positions::LONG_SKIP_INTERVAL;
use crate::positions::LONG_SKIP_IN_BLOCKS;
use crate::postings::compression::compressed_block_size;
/// Positions works as a long sequence of compressed block. /// Positions works as a long sequence of compressed block.
/// All terms are chained one after the other. /// All terms are chained one after the other.
/// ///
@@ -25,7 +19,13 @@ use crate::postings::compression::compressed_block_size;
/// so skipping a block without decompressing it is just a matter of advancing that many /// so skipping a block without decompressing it is just a matter of advancing that many
/// bytes. /// bytes.
use bitpacking::{BitPacker, BitPacker4x}; use bitpacking::{BitPacker, BitPacker4x};
use common::{BinarySerializable, FixedSize};
use directory::ReadOnlySource;
use owned_read::OwnedRead; use owned_read::OwnedRead;
use positions::COMPRESSION_BLOCK_SIZE;
use positions::LONG_SKIP_INTERVAL;
use positions::LONG_SKIP_IN_BLOCKS;
use postings::compression::compressed_block_size;
struct Positions { struct Positions {
bit_packer: BitPacker4x, bit_packer: BitPacker4x,

6
src/positions/serializer.rs
View File

@@ -1,8 +1,8 @@
use crate::common::BinarySerializable;
use crate::common::CountingWriter;
use crate::positions::{COMPRESSION_BLOCK_SIZE, LONG_SKIP_INTERVAL};
use bitpacking::BitPacker; use bitpacking::BitPacker;
use bitpacking::BitPacker4x; use bitpacking::BitPacker4x;
use common::BinarySerializable;
use common::CountingWriter;
use positions::{COMPRESSION_BLOCK_SIZE, LONG_SKIP_INTERVAL};
use std::io::{self, Write}; use std::io::{self, Write};
pub struct PositionSerializer<W: io::Write> { pub struct PositionSerializer<W: io::Write> {

10
src/postings/block_search.rs
View File

@@ -1,4 +1,4 @@
use crate::postings::compression::AlignedBuffer; use postings::compression::AlignedBuffer;
/// This modules define the logic used to search for a doc in a given /// This modules define the logic used to search for a doc in a given
/// block. (at most 128 docs) /// block. (at most 128 docs)
@@ -8,7 +8,7 @@ use crate::postings::compression::AlignedBuffer;
#[cfg(target_arch = "x86_64")] #[cfg(target_arch = "x86_64")]
mod sse2 { mod sse2 {
use crate::postings::compression::{AlignedBuffer, COMPRESSION_BLOCK_SIZE}; use postings::compression::{AlignedBuffer, COMPRESSION_BLOCK_SIZE};
use std::arch::x86_64::__m128i as DataType; use std::arch::x86_64::__m128i as DataType;
use std::arch::x86_64::_mm_add_epi32 as op_add; use std::arch::x86_64::_mm_add_epi32 as op_add;
use std::arch::x86_64::_mm_cmplt_epi32 as op_lt; use std::arch::x86_64::_mm_cmplt_epi32 as op_lt;
@@ -49,7 +49,7 @@ mod sse2 {
#[cfg(test)] #[cfg(test)]
mod test { mod test {
use super::linear_search_sse2_128; use super::linear_search_sse2_128;
use crate::postings::compression::{AlignedBuffer, COMPRESSION_BLOCK_SIZE}; use postings::compression::{AlignedBuffer, COMPRESSION_BLOCK_SIZE};
#[test] #[test]
fn test_linear_search_sse2_128_u32() { fn test_linear_search_sse2_128_u32() {
@@ -140,7 +140,7 @@ impl BlockSearcher {
) -> usize { ) -> usize {
#[cfg(target_arch = "x86_64")] #[cfg(target_arch = "x86_64")]
{ {
use crate::postings::compression::COMPRESSION_BLOCK_SIZE; use postings::compression::COMPRESSION_BLOCK_SIZE;
if self == BlockSearcher::SSE2 && len == COMPRESSION_BLOCK_SIZE { if self == BlockSearcher::SSE2 && len == COMPRESSION_BLOCK_SIZE {
return sse2::linear_search_sse2_128(block_docs, target); return sse2::linear_search_sse2_128(block_docs, target);
} }
@@ -166,7 +166,7 @@ mod tests {
use super::exponential_search; use super::exponential_search;
use super::linear_search; use super::linear_search;
use super::BlockSearcher; use super::BlockSearcher;
use crate::postings::compression::{AlignedBuffer, COMPRESSION_BLOCK_SIZE}; use postings::compression::{AlignedBuffer, COMPRESSION_BLOCK_SIZE};
#[test] #[test]
fn test_linear_search() { fn test_linear_search() {

2
src/postings/compression/mod.rs
View File

@@ -1,5 +1,5 @@
use crate::common::FixedSize;
use bitpacking::{BitPacker, BitPacker4x}; use bitpacking::{BitPacker, BitPacker4x};
use common::FixedSize;
pub const COMPRESSION_BLOCK_SIZE: usize = BitPacker4x::BLOCK_LEN; pub const COMPRESSION_BLOCK_SIZE: usize = BitPacker4x::BLOCK_LEN;
const COMPRESSED_BLOCK_MAX_SIZE: usize = COMPRESSION_BLOCK_SIZE * u32::SIZE_IN_BYTES; const COMPRESSED_BLOCK_MAX_SIZE: usize = COMPRESSION_BLOCK_SIZE * u32::SIZE_IN_BYTES;

120
src/postings/mod.rs
View File

@@ -31,7 +31,7 @@ pub use self::segment_postings::{BlockSegmentPostings, SegmentPostings};
pub(crate) use self::stacker::compute_table_size; pub(crate) use self::stacker::compute_table_size;
pub use crate::common::HasLen; pub use common::HasLen;
pub(crate) const USE_SKIP_INFO_LIMIT: u32 = COMPRESSION_BLOCK_SIZE as u32; pub(crate) const USE_SKIP_INFO_LIMIT: u32 = COMPRESSION_BLOCK_SIZE as u32;
pub(crate) type UnorderedTermId = u64; pub(crate) type UnorderedTermId = u64;
@@ -48,25 +48,24 @@ pub(crate) enum FreqReadingOption {
pub mod tests { pub mod tests {
use super::*; use super::*;
use crate::core::Index; use core::Index;
use crate::core::SegmentComponent; use core::SegmentComponent;
use crate::core::SegmentReader; use core::SegmentReader;
use crate::docset::{DocSet, SkipResult}; use docset::{DocSet, SkipResult};
use crate::fieldnorm::FieldNormReader; use fieldnorm::FieldNormReader;
use crate::indexer::operation::AddOperation; use indexer::operation::AddOperation;
use crate::indexer::SegmentWriter; use indexer::SegmentWriter;
use crate::merge_policy::NoMergePolicy; use merge_policy::NoMergePolicy;
use crate::query::Scorer; use query::Scorer;
use crate::schema::{Document, Schema, Term, INDEXED, STRING, TEXT};
use crate::schema::{Field, TextOptions};
use crate::schema::{IndexRecordOption, TextFieldIndexing};
use crate::tokenizer::{SimpleTokenizer, MAX_TOKEN_LEN};
use crate::DocId;
use crate::Score;
use once_cell::sync::Lazy;
use rand::rngs::StdRng; use rand::rngs::StdRng;
use rand::{Rng, SeedableRng}; use rand::{Rng, SeedableRng};
use schema::{Document, Schema, Term, INDEXED, STRING, TEXT};
use schema::{Field, TextOptions};
use schema::{IndexRecordOption, TextFieldIndexing};
use std::iter; use std::iter;
use tokenizer::{SimpleTokenizer, MAX_TOKEN_LEN};
use DocId;
use Score;
#[test] #[test]
pub fn test_position_write() { pub fn test_position_write() {
@@ -510,52 +509,53 @@ pub mod tests {
} }
} }
pub static TERM_A: Lazy<Term> = Lazy::new(|| { lazy_static! {
let field = Field(0); pub static ref TERM_A: Term = {
Term::from_field_text(field, "a") let field = Field(0);
}); Term::from_field_text(field, "a")
pub static TERM_B: Lazy<Term> = Lazy::new(|| { };
let field = Field(0); pub static ref TERM_B: Term = {
Term::from_field_text(field, "b") let field = Field(0);
}); Term::from_field_text(field, "b")
pub static TERM_C: Lazy<Term> = Lazy::new(|| { };
let field = Field(0); pub static ref TERM_C: Term = {
Term::from_field_text(field, "c") let field = Field(0);
}); Term::from_field_text(field, "c")
pub static TERM_D: Lazy<Term> = Lazy::new(|| { };
let field = Field(0); pub static ref TERM_D: Term = {
Term::from_field_text(field, "d") let field = Field(0);
}); Term::from_field_text(field, "d")
};
pub static ref INDEX: Index = {
let mut schema_builder = Schema::builder();
let text_field = schema_builder.add_text_field("text", STRING);
let schema = schema_builder.build();
pub static INDEX: Lazy<Index> = Lazy::new(|| { let mut rng: StdRng = StdRng::from_seed([1u8; 32]);
let mut schema_builder = Schema::builder();
let text_field = schema_builder.add_text_field("text", STRING);
let schema = schema_builder.build();
let mut rng: StdRng = StdRng::from_seed([1u8; 32]); let index = Index::create_in_ram(schema);
let posting_list_size = 1_000_000;
let index = Index::create_in_ram(schema); {
let posting_list_size = 1_000_000; let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap();
{ for _ in 0..posting_list_size {
let mut index_writer = index.writer_with_num_threads(1, 3_000_000).unwrap(); let mut doc = Document::default();
for _ in 0..posting_list_size { if rng.gen_bool(1f64 / 15f64) {
let mut doc = Document::default(); doc.add_text(text_field, "a");
if rng.gen_bool(1f64 / 15f64) { }
doc.add_text(text_field, "a"); if rng.gen_bool(1f64 / 10f64) {
doc.add_text(text_field, "b");
}
if rng.gen_bool(1f64 / 5f64) {
doc.add_text(text_field, "c");
}
doc.add_text(text_field, "d");
index_writer.add_document(doc);
} }
if rng.gen_bool(1f64 / 10f64) { assert!(index_writer.commit().is_ok());
doc.add_text(text_field, "b");
}
if rng.gen_bool(1f64 / 5f64) {
doc.add_text(text_field, "c");
}
doc.add_text(text_field, "d");
index_writer.add_document(doc);
} }
assert!(index_writer.commit().is_ok()); index
} };
index }
});
/// Wraps a given docset, and forward alls call but the /// Wraps a given docset, and forward alls call but the
/// `.skip_next(...)`. This is useful to test that a specialized /// `.skip_next(...)`. This is useful to test that a specialized
@@ -589,7 +589,7 @@ pub mod tests {
} }
} }
pub fn test_skip_against_unoptimized<F: Fn() -> Box<dyn DocSet>>( pub fn test_skip_against_unoptimized<F: Fn() -> Box<DocSet>>(
postings_factory: F, postings_factory: F,
targets: Vec<u32>, targets: Vec<u32>,
) { ) {

2
src/postings/postings.rs
View File

@@ -1,4 +1,4 @@
use crate::docset::DocSet; use docset::DocSet;
/// Postings (also called inverted list) /// Postings (also called inverted list)
/// ///

43
src/postings/postings_writer.rs
View File

@@ -1,23 +1,23 @@
use super::stacker::{Addr, MemoryArena, TermHashMap}; use super::stacker::{Addr, MemoryArena, TermHashMap};
use crate::postings::recorder::{ use postings::recorder::{
BufferLender, NothingRecorder, Recorder, TFAndPositionRecorder, TermFrequencyRecorder, BufferLender, NothingRecorder, Recorder, TFAndPositionRecorder, TermFrequencyRecorder,
}; };
use crate::postings::UnorderedTermId; use postings::UnorderedTermId;
use crate::postings::{FieldSerializer, InvertedIndexSerializer}; use postings::{FieldSerializer, InvertedIndexSerializer};
use crate::schema::IndexRecordOption; use schema::IndexRecordOption;
use crate::schema::{Field, FieldEntry, FieldType, Schema, Term}; use schema::{Field, FieldEntry, FieldType, Schema, Term};
use crate::termdict::TermOrdinal;
use crate::tokenizer::TokenStream;
use crate::tokenizer::{Token, MAX_TOKEN_LEN};
use crate::DocId;
use crate::Result;
use std::collections::HashMap; use std::collections::HashMap;
use std::io; use std::io;
use std::marker::PhantomData; use std::marker::PhantomData;
use std::ops::DerefMut; use std::ops::DerefMut;
use termdict::TermOrdinal;
use tokenizer::TokenStream;
use tokenizer::{Token, MAX_TOKEN_LEN};
use DocId;
use Result;
fn posting_from_field_entry(field_entry: &FieldEntry) -> Box<dyn PostingsWriter> { fn posting_from_field_entry(field_entry: &FieldEntry) -> Box<PostingsWriter> {
match *field_entry.field_type() { match *field_entry.field_type() {
FieldType::Str(ref text_options) => text_options FieldType::Str(ref text_options) => text_options
.get_indexing_options() .get_indexing_options()
@@ -49,7 +49,7 @@ pub struct MultiFieldPostingsWriter {
heap: MemoryArena, heap: MemoryArena,
schema: Schema, schema: Schema,
term_index: TermHashMap, term_index: TermHashMap,
per_field_postings_writers: Vec<Box<dyn PostingsWriter>>, per_field_postings_writers: Vec<Box<PostingsWriter>>,
} }
fn make_field_partition( fn make_field_partition(
@@ -99,12 +99,7 @@ impl MultiFieldPostingsWriter {
self.term_index.mem_usage() + self.heap.mem_usage() self.term_index.mem_usage() + self.heap.mem_usage()
} }
pub fn index_text( pub fn index_text(&mut self, doc: DocId, field: Field, token_stream: &mut TokenStream) -> u32 {
&mut self,
doc: DocId,
field: Field,
token_stream: &mut dyn TokenStream,
) -> u32 {
let postings_writer = self.per_field_postings_writers[field.0 as usize].deref_mut(); let postings_writer = self.per_field_postings_writers[field.0 as usize].deref_mut();
postings_writer.index_text( postings_writer.index_text(
&mut self.term_index, &mut self.term_index,
@@ -143,10 +138,10 @@ impl MultiFieldPostingsWriter {
FieldType::Str(_) | FieldType::HierarchicalFacet => { FieldType::Str(_) | FieldType::HierarchicalFacet => {
// populating the (unordered term ord) -> (ordered term ord) mapping // populating the (unordered term ord) -> (ordered term ord) mapping
// for the field. // for the field.
let unordered_term_ids = term_offsets[start..stop] let mut unordered_term_ids = term_offsets[start..stop]
.iter() .iter()
.map(|&(_, _, bucket)| bucket); .map(|&(_, _, bucket)| bucket);
let mapping: HashMap<UnorderedTermId, TermOrdinal> = unordered_term_ids let mut mapping: HashMap<UnorderedTermId, TermOrdinal> = unordered_term_ids
.enumerate() .enumerate()
.map(|(term_ord, unord_term_id)| { .map(|(term_ord, unord_term_id)| {
(unord_term_id as UnorderedTermId, term_ord as TermOrdinal) (unord_term_id as UnorderedTermId, term_ord as TermOrdinal)
@@ -199,7 +194,7 @@ pub trait PostingsWriter {
fn serialize( fn serialize(
&self, &self,
term_addrs: &[(&[u8], Addr, UnorderedTermId)], term_addrs: &[(&[u8], Addr, UnorderedTermId)],
serializer: &mut FieldSerializer<'_>, serializer: &mut FieldSerializer,
term_heap: &MemoryArena, term_heap: &MemoryArena,
heap: &MemoryArena, heap: &MemoryArena,
) -> io::Result<()>; ) -> io::Result<()>;
@@ -210,7 +205,7 @@ pub trait PostingsWriter {
term_index: &mut TermHashMap, term_index: &mut TermHashMap,
doc_id: DocId, doc_id: DocId,
field: Field, field: Field,
token_stream: &mut dyn TokenStream, token_stream: &mut TokenStream,
heap: &mut MemoryArena, heap: &mut MemoryArena,
) -> u32 { ) -> u32 {
let mut term = Term::for_field(field); let mut term = Term::for_field(field);
@@ -251,7 +246,7 @@ impl<Rec: Recorder + 'static> SpecializedPostingsWriter<Rec> {
} }
/// Builds a `SpecializedPostingsWriter` storing its data in a heap. /// Builds a `SpecializedPostingsWriter` storing its data in a heap.
pub fn new_boxed() -> Box<dyn PostingsWriter> { pub fn new_boxed() -> Box<PostingsWriter> {
Box::new(SpecializedPostingsWriter::<Rec>::new()) Box::new(SpecializedPostingsWriter::<Rec>::new())
} }
} }
@@ -288,7 +283,7 @@ impl<Rec: Recorder + 'static> PostingsWriter for SpecializedPostingsWriter<Rec>
fn serialize( fn serialize(
&self, &self,
term_addrs: &[(&[u8], Addr, UnorderedTermId)], term_addrs: &[(&[u8], Addr, UnorderedTermId)],
serializer: &mut FieldSerializer<'_>, serializer: &mut FieldSerializer,
termdict_heap: &MemoryArena, termdict_heap: &MemoryArena,
heap: &MemoryArena, heap: &MemoryArena,
) -> io::Result<()> { ) -> io::Result<()> {

14
src/postings/recorder.rs
View File

@@ -1,8 +1,8 @@
use super::stacker::{ExpUnrolledLinkedList, MemoryArena}; use super::stacker::{ExpUnrolledLinkedList, MemoryArena};
use crate::common::{read_u32_vint, write_u32_vint}; use common::{read_u32_vint, write_u32_vint};
use crate::postings::FieldSerializer; use postings::FieldSerializer;
use crate::DocId;
use std::io; use std::io;
use DocId;
const POSITION_END: u32 = 0; const POSITION_END: u32 = 0;
@@ -72,7 +72,7 @@ pub(crate) trait Recorder: Copy + 'static {
fn serialize( fn serialize(
&self, &self,
buffer_lender: &mut BufferLender, buffer_lender: &mut BufferLender,
serializer: &mut FieldSerializer<'_>, serializer: &mut FieldSerializer,
heap: &MemoryArena, heap: &MemoryArena,
) -> io::Result<()>; ) -> io::Result<()>;
} }
@@ -108,7 +108,7 @@ impl Recorder for NothingRecorder {
fn serialize( fn serialize(
&self, &self,
buffer_lender: &mut BufferLender, buffer_lender: &mut BufferLender,
serializer: &mut FieldSerializer<'_>, serializer: &mut FieldSerializer,
heap: &MemoryArena, heap: &MemoryArena,
) -> io::Result<()> { ) -> io::Result<()> {
let buffer = buffer_lender.lend_u8(); let buffer = buffer_lender.lend_u8();
@@ -159,7 +159,7 @@ impl Recorder for TermFrequencyRecorder {
fn serialize( fn serialize(
&self, &self,
buffer_lender: &mut BufferLender, buffer_lender: &mut BufferLender,
serializer: &mut FieldSerializer<'_>, serializer: &mut FieldSerializer,
heap: &MemoryArena, heap: &MemoryArena,
) -> io::Result<()> { ) -> io::Result<()> {
let buffer = buffer_lender.lend_u8(); let buffer = buffer_lender.lend_u8();
@@ -208,7 +208,7 @@ impl Recorder for TFAndPositionRecorder {
fn serialize( fn serialize(
&self, &self,
buffer_lender: &mut BufferLender, buffer_lender: &mut BufferLender,
serializer: &mut FieldSerializer<'_>, serializer: &mut FieldSerializer,
heap: &MemoryArena, heap: &MemoryArena,
) -> io::Result<()> { ) -> io::Result<()> {
let (buffer_u8, buffer_positions) = buffer_lender.lend_all(); let (buffer_u8, buffer_positions) = buffer_lender.lend_all();

50
src/postings/segment_postings.rs
View File

@@ -1,21 +1,21 @@
use crate::common::BitSet; use common::BitSet;
use crate::common::HasLen; use common::HasLen;
use crate::common::{BinarySerializable, VInt}; use common::{BinarySerializable, VInt};
use crate::docset::{DocSet, SkipResult}; use docset::{DocSet, SkipResult};
use crate::positions::PositionReader;
use crate::postings::compression::{compressed_block_size, AlignedBuffer};
use crate::postings::compression::{BlockDecoder, VIntDecoder, COMPRESSION_BLOCK_SIZE};
use crate::postings::serializer::PostingsSerializer;
use crate::postings::BlockSearcher;
use crate::postings::FreqReadingOption;
use crate::postings::Postings;
use crate::postings::SkipReader;
use crate::postings::USE_SKIP_INFO_LIMIT;
use crate::schema::IndexRecordOption;
use crate::DocId;
use owned_read::OwnedRead; use owned_read::OwnedRead;
use positions::PositionReader;
use postings::compression::{compressed_block_size, AlignedBuffer};
use postings::compression::{BlockDecoder, VIntDecoder, COMPRESSION_BLOCK_SIZE};
use postings::serializer::PostingsSerializer;
use postings::BlockSearcher;
use postings::FreqReadingOption;
use postings::Postings;
use postings::SkipReader;
use postings::USE_SKIP_INFO_LIMIT;
use schema::IndexRecordOption;
use std::cmp::Ordering; use std::cmp::Ordering;
use tantivy_fst::Streamer; use tantivy_fst::Streamer;
use DocId;
struct PositionComputer { struct PositionComputer {
// store the amount of position int // store the amount of position int
@@ -611,17 +611,17 @@ mod tests {
use super::BlockSegmentPostings; use super::BlockSegmentPostings;
use super::BlockSegmentPostingsSkipResult; use super::BlockSegmentPostingsSkipResult;
use super::SegmentPostings; use super::SegmentPostings;
use crate::common::HasLen; use common::HasLen;
use crate::core::Index; use core::Index;
use crate::docset::DocSet; use docset::DocSet;
use crate::postings::postings::Postings; use postings::postings::Postings;
use crate::schema::IndexRecordOption; use schema::IndexRecordOption;
use crate::schema::Schema; use schema::Schema;
use crate::schema::Term; use schema::Term;
use crate::schema::INDEXED; use schema::INDEXED;
use crate::DocId;
use crate::SkipResult;
use tantivy_fst::Streamer; use tantivy_fst::Streamer;
use DocId;
use SkipResult;
#[test] #[test]
fn test_empty_segment_postings() { fn test_empty_segment_postings() {

30
src/postings/serializer.rs
View File

@@ -1,18 +1,18 @@
use super::TermInfo; use super::TermInfo;
use crate::common::{BinarySerializable, VInt}; use common::{BinarySerializable, VInt};
use crate::common::{CompositeWrite, CountingWriter}; use common::{CompositeWrite, CountingWriter};
use crate::core::Segment; use core::Segment;
use crate::directory::WritePtr; use directory::WritePtr;
use crate::positions::PositionSerializer; use positions::PositionSerializer;
use crate::postings::compression::{BlockEncoder, VIntEncoder, COMPRESSION_BLOCK_SIZE}; use postings::compression::{BlockEncoder, VIntEncoder, COMPRESSION_BLOCK_SIZE};
use crate::postings::skip::SkipSerializer; use postings::skip::SkipSerializer;
use crate::postings::USE_SKIP_INFO_LIMIT; use postings::USE_SKIP_INFO_LIMIT;
use crate::schema::Schema; use schema::Schema;
use crate::schema::{Field, FieldEntry, FieldType}; use schema::{Field, FieldEntry, FieldType};
use crate::termdict::{TermDictionaryBuilder, TermOrdinal};
use crate::DocId;
use crate::Result;
use std::io::{self, Write}; use std::io::{self, Write};
use termdict::{TermDictionaryBuilder, TermOrdinal};
use DocId;
use Result;
/// `InvertedIndexSerializer` is in charge of serializing /// `InvertedIndexSerializer` is in charge of serializing
/// postings on disk, in the /// postings on disk, in the
@@ -73,7 +73,7 @@ impl InvertedIndexSerializer {
/// Open a new `PostingsSerializer` for the given segment /// Open a new `PostingsSerializer` for the given segment
pub fn open(segment: &mut Segment) -> Result<InvertedIndexSerializer> { pub fn open(segment: &mut Segment) -> Result<InvertedIndexSerializer> {
use crate::SegmentComponent::{POSITIONS, POSITIONSSKIP, POSTINGS, TERMS}; use SegmentComponent::{POSITIONS, POSITIONSSKIP, POSTINGS, TERMS};
InvertedIndexSerializer::create( InvertedIndexSerializer::create(
CompositeWrite::wrap(segment.open_write(TERMS)?), CompositeWrite::wrap(segment.open_write(TERMS)?),
CompositeWrite::wrap(segment.open_write(POSTINGS)?), CompositeWrite::wrap(segment.open_write(POSTINGS)?),
@@ -91,7 +91,7 @@ impl InvertedIndexSerializer {
&mut self, &mut self,
field: Field, field: Field,
total_num_tokens: u64, total_num_tokens: u64,
) -> io::Result<FieldSerializer<'_>> { ) -> io::Result<FieldSerializer> {
let field_entry: &FieldEntry = self.schema.get_field_entry(field); let field_entry: &FieldEntry = self.schema.get_field_entry(field);
let term_dictionary_write = self.terms_write.for_field(field); let term_dictionary_write = self.terms_write.for_field(field);
let postings_write = self.postings_write.for_field(field); let postings_write = self.postings_write.for_field(field);

8
src/postings/skip.rs
View File

@@ -1,8 +1,8 @@
use crate::common::BinarySerializable; use common::BinarySerializable;
use crate::postings::compression::COMPRESSION_BLOCK_SIZE;
use crate::schema::IndexRecordOption;
use crate::DocId;
use owned_read::OwnedRead; use owned_read::OwnedRead;
use postings::compression::COMPRESSION_BLOCK_SIZE;
use schema::IndexRecordOption;
use DocId;
pub struct SkipSerializer { pub struct SkipSerializer {
buffer: Vec<u8>, buffer: Vec<u8>,

8
src/postings/stacker/expull.rs
View File

@@ -1,7 +1,7 @@
use super::{Addr, MemoryArena}; use super::{Addr, MemoryArena};
use crate::postings::stacker::memory_arena::load; use postings::stacker::memory_arena::load;
use crate::postings::stacker::memory_arena::store; use postings::stacker::memory_arena::store;
use std::io; use std::io;
use std::mem; use std::mem;
@@ -16,8 +16,8 @@ enum CapacityResult {
fn len_to_capacity(len: u32) -> CapacityResult { fn len_to_capacity(len: u32) -> CapacityResult {
match len { match len {
0..=15 => CapacityResult::Available(FIRST_BLOCK as u32 - len), 0...15 => CapacityResult::Available(FIRST_BLOCK as u32 - len),
16..=MAX_BLOCK_LEN => { 16...MAX_BLOCK_LEN => {
let cap = 1 << (32u32 - (len - 1u32).leading_zeros()); let cap = 1 << (32u32 - (len - 1u32).leading_zeros());
let available = cap - len; let available = cap - len;
if available == 0 { if available == 0 {

8
src/postings/stacker/term_hashmap.rs
View File

@@ -1,11 +1,11 @@
use murmurhash32; extern crate murmurhash32;
use self::murmurhash32::murmurhash2; use self::murmurhash32::murmurhash2;
use super::{Addr, MemoryArena}; use super::{Addr, MemoryArena};
use crate::postings::stacker::memory_arena::store;
use crate::postings::UnorderedTermId;
use byteorder::{ByteOrder, NativeEndian}; use byteorder::{ByteOrder, NativeEndian};
use postings::stacker::memory_arena::store;
use postings::UnorderedTermId;
use std::iter; use std::iter;
use std::mem; use std::mem;
use std::slice; use std::slice;
@@ -154,7 +154,7 @@ impl TermHashMap {
unordered_term_id unordered_term_id
} }
pub fn iter(&self) -> Iter<'_> { pub fn iter(&self) -> Iter {
Iter { Iter {
inner: self.occupied.iter(), inner: self.occupied.iter(),
hashmap: &self, hashmap: &self,

4
src/postings/term_info.rs
View File

@@ -1,4 +1,4 @@
use crate::common::{BinarySerializable, FixedSize}; use common::{BinarySerializable, FixedSize};
use std::io; use std::io;
/// `TermInfo` wraps the metadata associated to a Term. /// `TermInfo` wraps the metadata associated to a Term.
@@ -45,7 +45,7 @@ impl BinarySerializable for TermInfo {
mod tests { mod tests {
use super::TermInfo; use super::TermInfo;
use crate::common::test::fixed_size_test; use common::test::fixed_size_test;
#[test] #[test]
fn test_fixed_size() { fn test_fixed_size() {

32
src/query/all_query.rs
View File

@@ -1,11 +1,10 @@
use crate::core::Searcher; use core::Searcher;
use crate::core::SegmentReader; use core::SegmentReader;
use crate::docset::DocSet; use docset::DocSet;
use crate::query::explanation::does_not_match; use query::{Query, Scorer, Weight};
use crate::query::{Explanation, Query, Scorer, Weight}; use DocId;
use crate::DocId; use Result;
use crate::Result; use Score;
use crate::Score;
/// Query that matches all of the documents. /// Query that matches all of the documents.
/// ///
@@ -14,7 +13,7 @@ use crate::Score;
pub struct AllQuery; pub struct AllQuery;
impl Query for AllQuery { impl Query for AllQuery {
fn weight(&self, _: &Searcher, _: bool) -> Result<Box<dyn Weight>> { fn weight(&self, _: &Searcher, _: bool) -> Result<Box<Weight>> {
Ok(Box::new(AllWeight)) Ok(Box::new(AllWeight))
} }
} }
@@ -23,20 +22,13 @@ impl Query for AllQuery {
pub struct AllWeight; pub struct AllWeight;
impl Weight for AllWeight { impl Weight for AllWeight {
fn scorer(&self, reader: &SegmentReader) -> Result<Box<dyn Scorer>> { fn scorer(&self, reader: &SegmentReader) -> Result<Box<Scorer>> {
Ok(Box::new(AllScorer { Ok(Box::new(AllScorer {
state: State::NotStarted, state: State::NotStarted,
doc: 0u32, doc: 0u32,
max_doc: reader.max_doc(), max_doc: reader.max_doc(),
})) }))
} }
fn explain(&self, reader: &SegmentReader, doc: DocId) -> Result<Explanation> {
if doc >= reader.max_doc() {
return Err(does_not_match(doc));
}
Ok(Explanation::new("AllQuery", 1f32))
}
} }
enum State { enum State {
@@ -93,9 +85,9 @@ impl Scorer for AllScorer {
mod tests { mod tests {
use super::AllQuery; use super::AllQuery;
use crate::query::Query; use query::Query;
use crate::schema::{Schema, TEXT}; use schema::{Schema, TEXT};
use crate::Index; use Index;
#[test] #[test]
fn test_all_query() { fn test_all_query() {

31
src/query/automaton_weight.rs
View File

@@ -1,14 +1,12 @@
use crate::common::BitSet; use common::BitSet;
use crate::core::SegmentReader; use core::SegmentReader;
use crate::query::ConstScorer; use query::BitSetDocSet;
use crate::query::{BitSetDocSet, Explanation}; use query::ConstScorer;
use crate::query::{Scorer, Weight}; use query::{Scorer, Weight};
use crate::schema::{Field, IndexRecordOption}; use schema::{Field, IndexRecordOption};
use crate::termdict::{TermDictionary, TermStreamer};
use crate::DocId;
use crate::TantivyError;
use crate::{Result, SkipResult};
use tantivy_fst::Automaton; use tantivy_fst::Automaton;
use termdict::{TermDictionary, TermStreamer};
use Result;
/// A weight struct for Fuzzy Term and Regex Queries /// A weight struct for Fuzzy Term and Regex Queries
pub struct AutomatonWeight<A> pub struct AutomatonWeight<A>
@@ -38,7 +36,7 @@ impl<A> Weight for AutomatonWeight<A>
where where
A: Automaton + Send + Sync + 'static, A: Automaton + Send + Sync + 'static,
{ {
fn scorer(&self, reader: &SegmentReader) -> Result<Box<dyn Scorer>> { fn scorer(&self, reader: &SegmentReader) -> Result<Box<Scorer>> {
let max_doc = reader.max_doc(); let max_doc = reader.max_doc();
let mut doc_bitset = BitSet::with_max_value(max_doc); let mut doc_bitset = BitSet::with_max_value(max_doc);
@@ -58,15 +56,4 @@ where
let doc_bitset = BitSetDocSet::from(doc_bitset); let doc_bitset = BitSetDocSet::from(doc_bitset);
Ok(Box::new(ConstScorer::new(doc_bitset))) Ok(Box::new(ConstScorer::new(doc_bitset)))
} }
fn explain(&self, reader: &SegmentReader, doc: DocId) -> Result<Explanation> {
let mut scorer = self.scorer(reader)?;
if scorer.skip_next(doc) == SkipResult::Reached {
Ok(Explanation::new("AutomatonScorer", 1.0f32))
} else {
Err(TantivyError::InvalidArgument(
"Document does not exist".to_string(),
))
}
}
} }

Some files were not shown because too many files have changed in this diff Show More