diff --git a/Cargo.lock b/Cargo.lock index bf919d7..ce96f4c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -17,6 +17,17 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627" +[[package]] +name = "ahash" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "891477e0c6a8957309ee5c45a6368af3ae14bb510732d2684ffa19af310920f9" +dependencies = [ + "getrandom", + "once_cell", + "version_check", +] + [[package]] name = "aho-corasick" version = "1.1.3" @@ -81,6 +92,44 @@ version = "1.0.89" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "86fdf8605db99b54d3cd748a44c6d04df638eb5dafb219b135d0149bd0db01f6" +[[package]] +name = "argmin" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5698c8cd3510117a4e6b96749a8061ba7dce1a19578ce4ecdb12dd36d94a7f8d" +dependencies = [ + "anyhow", + "argmin-math", + "bincode 1.3.3", + "instant", + "num-traits", + "paste", + "rand", + "rand_xoshiro", + "serde", + "serde_json", + "slog", + "slog-async", + "slog-json", + "slog-term", + "thiserror", +] + +[[package]] +name = "argmin-math" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75f2b0dada81340718682df780c9a696b090b6ef7e83c3dcc770af6de9302995" +dependencies = [ + "anyhow", + "cfg-if", + "num-complex", + "num-integer", + "num-traits", + "rand", + "thiserror", +] + [[package]] name = "async-trait" version = "0.1.82" @@ -189,6 +238,25 @@ dependencies = [ "serde", ] +[[package]] +name = "bincode" +version = "2.0.0-rc.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f11ea1a0346b94ef188834a65c068a03aec181c94896d481d7a0a40d85b0ce95" +dependencies = [ + "bincode_derive", + "serde", +] + +[[package]] +name = "bincode_derive" +version = "2.0.0-rc.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e30759b3b99a1b802a7a3aa21c85c3ded5c28e1c83170d82d70f08bbf7f3e4c" +dependencies = [ + "virtue", +] + [[package]] name = "bit-set" version = "0.5.3" @@ -288,6 +356,12 @@ dependencies = [ "libc", ] +[[package]] +name = "crawdad" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87fbd1ecd2ed790e11c8fbe034f9b3e7687404818d1bdfd8218d26ec645ec7c5" + [[package]] name = "crc32fast" version = "1.4.2" @@ -297,6 +371,15 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "crossbeam-channel" +version = "0.5.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33480d6946193aa8033910124896ca395333cae7e2d1113d1fef6c3272217df2" +dependencies = [ + "crossbeam-utils", +] + [[package]] name = "crossbeam-deque" version = "0.8.5" @@ -394,6 +477,15 @@ dependencies = [ "syn 2.0.77", ] +[[package]] +name = "deranged" +version = "0.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b42b6fa04a440b495c8b04d0e71b707c585f83cb9cb28cf8cd0d976c315e31b4" +dependencies = [ + "powerfmt", +] + [[package]] name = "derive_builder" version = "0.20.1" @@ -435,6 +527,27 @@ dependencies = [ "crypto-common", ] +[[package]] +name = "dirs-next" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b98cf8ebf19c3d1b223e151f99a4f9f0690dca41414773390fc824184ac833e1" +dependencies = [ + "cfg-if", + "dirs-sys-next", +] + +[[package]] +name = "dirs-sys-next" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ebda144c4fe02d1f7ea1a7d9641b6fc6b580adcfa024ae48797ecdeb6825b4d" +dependencies = [ + "libc", + "redox_users", + "winapi", +] + [[package]] name = "dotenvy" version = "0.15.7" @@ -709,6 +822,15 @@ dependencies = [ "crunchy", ] +[[package]] +name = "hashbrown" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" +dependencies = [ + "ahash", +] + [[package]] name = "hashbrown" version = "0.14.5" @@ -727,6 +849,12 @@ version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" +[[package]] +name = "hermit-abi" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fbf6a919d6cf397374f7dfeeea91d974c7c0a7221d0d0f4f20d859d329e53fcc" + [[package]] name = "hound" version = "3.5.1" @@ -844,7 +972,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68b900aa2f7301e21c36462b170ee99994de34dff39a4a6a528e80e7376d07e5" dependencies = [ "equivalent", - "hashbrown", + "hashbrown 0.14.5", "serde", ] @@ -876,6 +1004,17 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "is-terminal" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "261f68e344040fbd0edea105bef17c66edf46f984ddb1115b775ce31be948f4b" +dependencies = [ + "hermit-abi 0.4.0", + "libc", + "windows-sys 0.52.0", +] + [[package]] name = "is_terminal_polyfill" version = "1.70.1" @@ -950,7 +1089,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fe30c65ff4c092320f1bba3418ac111443a4827a9155442f6a7d8d0e3707cb51" dependencies = [ "aho-corasick", - "bincode", + "bincode 1.3.3", "lindera-core", "lindera-tokenizer", "once_cell", @@ -966,7 +1105,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c156a875d35ef6fedf31cb9d6bb3c562d16faad4506a5be27e2ed44357d755d4" dependencies = [ "anyhow", - "bincode", + "bincode 1.3.3", "byteorder", "jpreprocess-core", "lindera-core", @@ -983,7 +1122,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0cd89b119949a5071e6f49d805829f3dc17169eb7c6aab809e4f373c70098709" dependencies = [ "anyhow", - "bincode", + "bincode 1.3.3", "byteorder", "csv", "glob", @@ -1103,7 +1242,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c28191456debc98af6aa5f7db77872471983e9fa2a737b1c232b6ef543aed62" dependencies = [ "anyhow", - "bincode", + "bincode 1.3.3", "byteorder", "encoding_rs", "log", @@ -1131,7 +1270,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bdf5f91725e32b9a21b1656baa7030766c9bafc4de4b4ddeb8ffdde7224dd2f6" dependencies = [ "anyhow", - "bincode", + "bincode 1.3.3", "byteorder", "lindera-cc-cedict-builder", "lindera-core", @@ -1151,7 +1290,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e41f00ba7ac541b0ffd8c30e7a73f2dd197546cc5780462ec4f2e4782945a780" dependencies = [ "anyhow", - "bincode", + "bincode 1.3.3", "byteorder", "csv", "derive_builder", @@ -1207,7 +1346,7 @@ version = "0.32.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "903e558981bcb6f59870aa7d6b4bcb09e8f7db778886a6a70f67fd74c9fa2ca3" dependencies = [ - "bincode", + "bincode 1.3.3", "lindera-core", "lindera-dictionary", "once_cell", @@ -1323,7 +1462,7 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "80e04d1dcff3aae0704555fe5fee3bcfaf3d1fdf8a7e521d5b9d2b42acb52cec" dependencies = [ - "hermit-abi", + "hermit-abi 0.3.9", "libc", "wasi", "windows-sys 0.52.0", @@ -1384,6 +1523,12 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-conv" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" + [[package]] name = "num-integer" version = "0.1.46" @@ -1408,7 +1553,7 @@ version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" dependencies = [ - "hermit-abi", + "hermit-abi 0.3.9", "libc", ] @@ -1609,6 +1754,12 @@ dependencies = [ "portable-atomic", ] +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + [[package]] name = "ppv-lite86" version = "0.2.20" @@ -1733,6 +1884,7 @@ dependencies = [ "libc", "rand_chacha", "rand_core", + "serde", ] [[package]] @@ -1752,6 +1904,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" dependencies = [ "getrandom", + "serde", +] + +[[package]] +name = "rand_xoshiro" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f97cdb2a36ed4183de61b2f824cc45c9f1037f28afe0a322e9fff4c108b5aaa" +dependencies = [ + "rand_core", + "serde", ] [[package]] @@ -1800,6 +1963,17 @@ dependencies = [ "bitflags 2.6.0", ] +[[package]] +name = "redox_users" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba009ff324d1fc1b900bd1fdb31564febe58a8ccc8a6fdbb93b543d33b13ca43" +dependencies = [ + "getrandom", + "libredox", + "thiserror", +] + [[package]] name = "regex" version = "1.11.0" @@ -1844,6 +2018,19 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "rucrf" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "640271497e95c3a3a9502740187b5fc3a19485cad172e7be9a2fd8d86ffc1c28" +dependencies = [ + "argmin", + "argmin-math", + "bincode 2.0.0-rc.3", + "crossbeam-channel", + "hashbrown 0.12.3", +] + [[package]] name = "rustc-demangle" version = "0.1.24" @@ -1952,6 +2139,7 @@ dependencies = [ "tar", "thiserror", "tokenizers", + "vibrato", "zstd", ] @@ -2059,6 +2247,49 @@ version = "0.3.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "38b58827f4464d87d377d175e90bf58eb00fd8716ff0a62f80356b5e61555d0d" +[[package]] +name = "slog" +version = "2.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8347046d4ebd943127157b94d63abb990fcf729dc4e9978927fdf4ac3c998d06" + +[[package]] +name = "slog-async" +version = "2.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72c8038f898a2c79507940990f05386455b3a317d8f18d4caea7cbc3d5096b84" +dependencies = [ + "crossbeam-channel", + "slog", + "take_mut", + "thread_local", +] + +[[package]] +name = "slog-json" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e1e53f61af1e3c8b852eef0a9dee29008f55d6dd63794f3f12cef786cf0f219" +dependencies = [ + "serde", + "serde_json", + "slog", + "time", +] + +[[package]] +name = "slog-term" +version = "2.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6e022d0b998abfe5c3782c1f03551a596269450ccd677ea51c56f8b214610e8" +dependencies = [ + "is-terminal", + "slog", + "term", + "thread_local", + "time", +] + [[package]] name = "smallvec" version = "1.13.2" @@ -2171,6 +2402,12 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7065abeca94b6a8a577f9bd45aa0867a2238b74e8eb67cf10d492bc39351394" +[[package]] +name = "take_mut" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f764005d11ee5f36500a149ace24e00e3da98b0158b3e2d53a7495660d3f4d60" + [[package]] name = "tar" version = "0.4.42" @@ -2188,6 +2425,17 @@ version = "0.12.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "61c41af27dd6d1e27b1b16b489db798443478cef1f06a660c96db617ba5de3b1" +[[package]] +name = "term" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c59df8ac95d96ff9bede18eb7300b0fda5e5d8d90960e76f8e14ae765eedbf1f" +dependencies = [ + "dirs-next", + "rustversion", + "winapi", +] + [[package]] name = "thiserror" version = "1.0.64" @@ -2208,6 +2456,47 @@ dependencies = [ "syn 2.0.77", ] +[[package]] +name = "thread_local" +version = "1.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b9ef9bad013ada3808854ceac7b46812a6465ba368859a37e2100283d2d719c" +dependencies = [ + "cfg-if", + "once_cell", +] + +[[package]] +name = "time" +version = "0.3.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5dfd88e563464686c916c7e46e623e520ddc6d79fa6641390f2e3fa86e83e885" +dependencies = [ + "deranged", + "itoa", + "num-conv", + "powerfmt", + "serde", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" + +[[package]] +name = "time-macros" +version = "0.2.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f252a68540fde3a3877aeea552b832b40ab9a69e318efd078774a01ddee1ccf" +dependencies = [ + "num-conv", + "time-core", +] + [[package]] name = "tinyvec" version = "1.8.0" @@ -2490,6 +2779,26 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" +[[package]] +name = "vibrato" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df95d99b268877738d8f644c38604c4c9c09950219b1d1a725ada94d62e98722" +dependencies = [ + "bincode 2.0.0-rc.3", + "crawdad", + "csv-core", + "hashbrown 0.12.3", + "regex", + "rucrf", +] + +[[package]] +name = "virtue" +version = "0.0.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dcc60c0624df774c82a0ef104151231d37da4962957d691c011c852b2473314" + [[package]] name = "wasi" version = "0.11.0+wasi-snapshot-preview1" diff --git a/sbv2_core/Cargo.toml b/sbv2_core/Cargo.toml index 209470d..312e885 100644 --- a/sbv2_core/Cargo.toml +++ b/sbv2_core/Cargo.toml @@ -24,6 +24,7 @@ serde_json = "1.0.128" tar = "0.4.41" thiserror = "1.0.63" tokenizers = { version = "0.20.0", default-features = false } +vibrato = { version = "0.5.1", optional = true } zstd = "0.13.2" [features] @@ -35,4 +36,5 @@ directml = ["ort/directml", "std"] tensorrt = ["ort/tensorrt", "std"] coreml = ["ort/coreml", "std"] default = ["std"] -no_std = ["tokenizers/unstable_wasm"] \ No newline at end of file +no_std = ["tokenizers/unstable_wasm"] +mecab = ["vibrato"]