fix: enable default tokenizer truncation (#2537)

2026-08-21 08:02:26 +00:00 · 2023-11-01 14:45:24 +01:00
parent 6c47358c78
commit 480a9465a5
1 changed files with 0 additions and 2 deletions
@@ -162,8 +162,6 @@ impl ModelInstance {
            Tokenizer::from_file(tokenizer_filename)
                .map_err(Error::msg)?
                .with_padding(None)
-                .with_truncation(None)
-                .map_err(Error::msg)?
                .to_owned(),
        );