Remove the end of text tokens. (#289)

1 files changed, 2 insertions, 1 deletions

diff --git a/candle-examples/examples/llama2-c/main.rs b/candle-examples/examples/llama2-c/main.rs
index 65641b3c..d710652f 100644
--- a/candle-examples/examples/llama2-c/main.rs
+++ b/candle-examples/examples/llama2-c/main.rs
@@ -266,7 +266,8 @@ fn run_eval(tokenizer: Tokenizer, config_path: &std::path::PathBuf, args: Args)
     let file = std::io::BufReader::new(file);
     let mut tokens = vec![];
     for line in file.lines() {
-        let line = tokenizer.encode(line?, false).map_err(E::msg)?;
+        let line = line?.replace("<|endoftext|>", "");
+        let line = tokenizer.encode(line, false).map_err(E::msg)?;
         tokens.push(line.get_ids().to_vec())
     }
     let tokens = tokens.concat();