build(deps): bump tokenizers from 0.21.2 to 0.21.4

Bumps [tokenizers](https://github.com/huggingface/tokenizers) from 0.21.2 to 0.21.4.
- [Release notes](https://github.com/huggingface/tokenizers/releases)
- [Changelog](https://github.com/huggingface/tokenizers/blob/main/RELEASE.md)
- [Commits](https://github.com/huggingface/tokenizers/compare/v0.21.2...v0.21.4)

---
updated-dependencies:
- dependency-name: tokenizers
  dependency-version: 0.21.4
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
This commit is contained in:
dependabot[bot]
2025-08-04 10:18:59 +00:00
committed by GitHub
parent ee56e9591d
commit 0b822f704a
2 changed files with 3 additions and 3 deletions

4
Cargo.lock generated
View File

@@ -2766,9 +2766,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"
[[package]]
name = "tokenizers"
version = "0.21.2"
version = "0.21.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4c3846d8588abed0daba25a0e47edd58ea15e450a6088b2575f5116fdb0b27ca"
checksum = "a620b996116a59e184c2fa2dfd8251ea34a36d0a514758c6f966386bd2e03476"
dependencies = [
"ahash",
"aho-corasick",

View File

@@ -25,7 +25,7 @@ serde = { version = "1.0.210", features = ["derive"] }
serde_json = "1.0.141"
tar = "0.4.41"
thiserror = "2.0.11"
tokenizers = { version = "0.21.2", default-features = false }
tokenizers = { version = "0.21.4", default-features = false }
zstd = "0.13.2"
[features]