Merge pull request #144 from tuna2134/renovate/tokenizers-0.x

fix(deps): update rust crate tokenizers to 0.21.0

Cargo.lock

@@ -2251,9 +2251,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"
 
 [[package]]
 name = "tokenizers"
-version = "0.20.3"
+version = "0.21.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "67b67c92f6d705e2a1d106fb0b28c696f9074901a9c656ee5d9f5de204c39bf7"
+checksum = "9ecededfed68a69bc657e486510089e255e53c3d38cc7d4d59c8742668ca2cae"
 dependencies = [
  "aho-corasick",
  "derive_builder",

sbv2_core/Cargo.toml

@@ -25,7 +25,7 @@ serde = { version = "1.0.210", features = ["derive"] }
 serde_json = "1.0.128"
 tar = "0.4.41"
 thiserror = "1.0.63"
-tokenizers = { version = "0.20.0", default-features = false }
+tokenizers = { version = "0.21.0", default-features = false }
 zstd = "0.13.2"
 
 [features]