diff options
| author | Laurent Mazare <laurent.mazare@gmail.com> | 2023-09-29 08:06:54 +0200 |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2023-09-29 07:06:54 +0100 |
| commit | 53510ce427160674268199349b22bbb62cd2b9ba (patch) | |
| tree | 6c1ebc56c07ffac61180c7936983809fb508c018 /candle-nn/src | |
| parent | 23b3576c478ee46633da2b703c7961a6341f9d0f (diff) | |
| download | candle-53510ce427160674268199349b22bbb62cd2b9ba.tar.gz candle-53510ce427160674268199349b22bbb62cd2b9ba.tar.bz2 candle-53510ce427160674268199349b22bbb62cd2b9ba.zip | |
Use a silu activation in mistral. (#991)
Diffstat (limited to 'candle-nn/src')
| -rw-r--r-- | candle-nn/src/activation.rs | 4 |
1 files changed, 4 insertions, 0 deletions
diff --git a/candle-nn/src/activation.rs b/candle-nn/src/activation.rs index 1e67ed53..ddc211a7 100644 --- a/candle-nn/src/activation.rs +++ b/candle-nn/src/activation.rs @@ -9,6 +9,8 @@ pub enum Activation { #[serde(rename = "gated-gelu")] NewGelu, Relu, + Silu, + Sigmoid, Elu(f64), LeakyRelu(f64), } @@ -20,6 +22,8 @@ impl super::Module for Activation { // https://github.com/huggingface/transformers/blob/12f043eaeaabfef6f6efea411d98e6f6d3c094b7/src/transformers/activations.py#L49-L78 Self::NewGelu => xs.gelu(), Self::Relu => xs.relu(), + Self::Silu => crate::ops::silu(xs), + Self::Sigmoid => crate::ops::sigmoid(xs), &Self::Elu(alpha) => xs.elu(alpha), &Self::LeakyRelu(negative_slope) => crate::ops::leaky_relu(xs, negative_slope), } |