Add the StarCoder2 model. (#1779)

* Add the StarCoder2 model.

* Add the example code and get things to work.

* And also tweak the readme.

1 files changed, 1 insertions, 1 deletions

diff --git a/candle-examples/src/token_output_stream.rs b/candle-examples/src/token_output_stream.rs
index 07f33620..1f507c5e 100644
--- a/candle-examples/src/token_output_stream.rs
+++ b/candle-examples/src/token_output_stream.rs
@@ -40,7 +40,7 @@ impl TokenOutputStream {
         };
         self.tokens.push(token);
         let text = self.decode(&self.tokens[self.prev_index..])?;
-        if text.len() > prev_text.len() && text.chars().last().unwrap().is_alphabetic() {
+        if text.len() > prev_text.len() && text.chars().last().unwrap().is_alphanumeric() {
             let text = text.split_at(prev_text.len());
             self.prev_index = self.current_index;
             self.current_index = self.tokens.len();