mirror of
https://github.com/GreptimeTeam/greptimedb.git
synced 2026-05-20 06:50:37 +00:00
5 lines
6.0 KiB
HTML
5 lines
6.0 KiB
HTML
<!DOCTYPE html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1.0"><meta name="generator" content="rustdoc"><meta name="description" content="`Tokenizer` tokenizes a text into a list of tokens."><title>Tokenizer in index::fulltext_index::tokenizer - Rust</title><script>if(window.location.protocol!=="file:")document.head.insertAdjacentHTML("beforeend","SourceSerif4-Regular-6b053e98.ttf.woff2,FiraSans-Italic-81dc35de.woff2,FiraSans-Regular-0fe48ade.woff2,FiraSans-MediumItalic-ccf7e434.woff2,FiraSans-Medium-e1aa3f0a.woff2,SourceCodePro-Regular-8badfe75.ttf.woff2,SourceCodePro-Semibold-aa29a496.ttf.woff2".split(",").map(f=>`<link rel="preload" as="font" type="font/woff2"href="../../../static.files/${f}">`).join(""))</script><link rel="stylesheet" href="../../../static.files/normalize-9960930a.css"><link rel="stylesheet" href="../../../static.files/rustdoc-e56847b5.css"><meta name="rustdoc-vars" data-root-path="../../../" data-static-root-path="../../../static.files/" data-current-crate="index" data-themes="" data-resource-suffix="" data-rustdoc-version="1.92.0-nightly (fa3155a64 2025-09-30)" data-channel="nightly" data-search-js="search-e256b49e.js" data-stringdex-js="stringdex-828709d0.js" data-settings-js="settings-c38705f0.js" ><script src="../../../static.files/storage-e2aeef58.js"></script><script defer src="sidebar-items.js"></script><script defer src="../../../static.files/main-ce535bd0.js"></script><noscript><link rel="stylesheet" href="../../../static.files/noscript-263c88ec.css"></noscript><link rel="alternate icon" type="image/png" href="../../../static.files/favicon-32x32-eab170b8.png"><link rel="icon" type="image/svg+xml" href="../../../static.files/favicon-044be391.svg"></head><body class="rustdoc trait"><!--[if lte IE 11]><div class="warning">This old browser is unsupported and will most likely display funky things.</div><![endif]--><rustdoc-topbar><h2><a href="#">Tokenizer</a></h2></rustdoc-topbar><nav class="sidebar"><div class="sidebar-crate"><h2><a href="../../../index/index.html">index</a><span class="version">1.0.0-rc.2</span></h2></div><div class="sidebar-elems"><section id="rustdoc-toc"><h2 class="location"><a href="#">Tokenizer</a></h2><h3><a href="#required-methods">Required Methods</a></h3><ul class="block"><li><a href="#tymethod.tokenize" title="tokenize">tokenize</a></li></ul><h3><a href="#implementors">Implementors</a></h3></section><div id="rustdoc-modnav"><h2><a href="index.html">In index::<wbr>fulltext_<wbr>index::<wbr>tokenizer</a></h2></div></div></nav><div class="sidebar-resizer" title="Drag to resize sidebar"></div><main><div class="width-limiter"><section id="main-content" class="content"><div class="main-heading"><div class="rustdoc-breadcrumbs"><a href="../../index.html">index</a>::<wbr><a href="../index.html">fulltext_index</a>::<wbr><a href="index.html">tokenizer</a></div><h1>Trait <span class="trait">Tokenizer</span> <button id="copy-path" title="Copy item path to clipboard">Copy item path</button></h1><rustdoc-toolbar></rustdoc-toolbar><span class="sub-heading"><a class="src" href="../../../src/index/fulltext_index/tokenizer.rs.html#46-48">Source</a> </span></div><pre class="rust item-decl"><code>pub trait Tokenizer: <a class="trait" href="https://doc.rust-lang.org/nightly/core/marker/trait.Send.html" title="trait core::marker::Send">Send</a> {
|
|
// Required method
|
|
fn <a href="#tymethod.tokenize" class="fn">tokenize</a><'a>(&self, text: &'a <a class="primitive" href="https://doc.rust-lang.org/nightly/std/primitive.str.html">str</a>) -> <a class="struct" href="https://doc.rust-lang.org/nightly/alloc/vec/struct.Vec.html" title="struct alloc::vec::Vec">Vec</a><&'a <a class="primitive" href="https://doc.rust-lang.org/nightly/std/primitive.str.html">str</a>>;
|
|
}</code></pre><details class="toggle top-doc" open><summary class="hideme"><span>Expand description</span></summary><div class="docblock"><p><code>Tokenizer</code> tokenizes a text into a list of tokens.</p>
|
|
</div></details><h2 id="required-methods" class="section-header">Required Methods<a href="#required-methods" class="anchor">§</a></h2><div class="methods"><section id="tymethod.tokenize" class="method"><a class="src rightside" href="../../../src/index/fulltext_index/tokenizer.rs.html#47">Source</a><h4 class="code-header">fn <a href="#tymethod.tokenize" class="fn">tokenize</a><'a>(&self, text: &'a <a class="primitive" href="https://doc.rust-lang.org/nightly/std/primitive.str.html">str</a>) -> <a class="struct" href="https://doc.rust-lang.org/nightly/alloc/vec/struct.Vec.html" title="struct alloc::vec::Vec">Vec</a><&'a <a class="primitive" href="https://doc.rust-lang.org/nightly/std/primitive.str.html">str</a>></h4></section></div><h2 id="implementors" class="section-header">Implementors<a href="#implementors" class="anchor">§</a></h2><div id="implementors-list"><section id="impl-Tokenizer-for-ChineseTokenizer" class="impl"><a class="src rightside" href="../../../src/index/fulltext_index/tokenizer.rs.html#89-97">Source</a><a href="#impl-Tokenizer-for-ChineseTokenizer" class="anchor">§</a><h3 class="code-header">impl <a class="trait" href="trait.Tokenizer.html" title="trait index::fulltext_index::tokenizer::Tokenizer">Tokenizer</a> for <a class="struct" href="struct.ChineseTokenizer.html" title="struct index::fulltext_index::tokenizer::ChineseTokenizer">ChineseTokenizer</a></h3></section><section id="impl-Tokenizer-for-EnglishTokenizer" class="impl"><a class="src rightside" href="../../../src/index/fulltext_index/tokenizer.rs.html#56-81">Source</a><a href="#impl-Tokenizer-for-EnglishTokenizer" class="anchor">§</a><h3 class="code-header">impl <a class="trait" href="trait.Tokenizer.html" title="trait index::fulltext_index::tokenizer::Tokenizer">Tokenizer</a> for <a class="struct" href="struct.EnglishTokenizer.html" title="struct index::fulltext_index::tokenizer::EnglishTokenizer">EnglishTokenizer</a></h3></section></div><script src="../../../trait.impl/index/fulltext_index/tokenizer/trait.Tokenizer.js" async></script></section></div></main></body></html> |