diff options
| author | Nicolas Patry <patry.nicolas@protonmail.com> | 2024-02-11 08:52:36 +0100 |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2024-02-11 08:52:36 +0100 |
| commit | 74497e6bf738366d4c599b42826e204fbcb96f37 (patch) | |
| tree | 60e03936f0f213b4eb2fed9ee3d46014d3007360 | |
| parent | 8ab384e63ddc20537dbf9c0e6983b6f45509f8a5 (diff) | |
| download | candle-74497e6bf738366d4c599b42826e204fbcb96f37.tar.gz candle-74497e6bf738366d4c599b42826e204fbcb96f37.tar.bz2 candle-74497e6bf738366d4c599b42826e204fbcb96f37.zip | |
Fixing the qwen tokenizer location. (#1693)
Using the chatglm one causes a bug where the "<|endoftext|>" is not
found.
| -rw-r--r-- | candle-examples/examples/qwen/main.rs | 4 |
1 files changed, 1 insertions, 3 deletions
diff --git a/candle-examples/examples/qwen/main.rs b/candle-examples/examples/qwen/main.rs index add0d519..d040d4b0 100644 --- a/candle-examples/examples/qwen/main.rs +++ b/candle-examples/examples/qwen/main.rs @@ -235,9 +235,7 @@ fn main() -> Result<()> { )); let tokenizer_filename = match args.tokenizer_file { Some(file) => std::path::PathBuf::from(file), - None => api - .model("lmz/candle-chatglm".to_string()) - .get("chatglm-tokenizer.json")?, + None => repo.get("tokenizer.json")?, }; let filenames = match args.weight_files { Some(files) => files |