diff options
| author | OlivierDehaene <Olivier.dehaene@gmail.com> | 2024-02-14 10:27:22 +0100 |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2024-02-14 10:27:22 +0100 |
| commit | b60064780d09ab6733f5287b322ea5cb057d3136 (patch) | |
| tree | 38bd95dc351a046dd5b7c67fdf9203a32b06e38b /candle-nn | |
| parent | 14010a8498af3383b004be1f55a2fa39bce5389d (diff) | |
| download | candle-b60064780d09ab6733f5287b322ea5cb057d3136.tar.gz candle-b60064780d09ab6733f5287b322ea5cb057d3136.tar.bz2 candle-b60064780d09ab6733f5287b322ea5cb057d3136.zip | |
feat: add silu activation function (#1706)
* feat: add silu activation function
* use silu/arg in grad
* update candle-nn
* use node
Diffstat (limited to 'candle-nn')
| -rw-r--r-- | candle-nn/src/activation.rs | 2 | ||||
| -rw-r--r-- | candle-nn/src/ops.rs | 5 |
2 files changed, 3 insertions, 4 deletions
diff --git a/candle-nn/src/activation.rs b/candle-nn/src/activation.rs index e00463f0..60a7a6d1 100644 --- a/candle-nn/src/activation.rs +++ b/candle-nn/src/activation.rs @@ -30,7 +30,7 @@ impl super::Module for Activation { Self::Relu => xs.relu(), Self::Relu2 => xs.relu()?.sqr(), Self::Relu6 => xs.clamp(0f32, 6f32), - Self::Silu => crate::ops::silu(xs), + Self::Silu => xs.silu(), Self::Sigmoid => crate::ops::sigmoid(xs), Self::HardSigmoid => crate::ops::hard_sigmoid(xs), Self::Swiglu => crate::ops::swiglu(xs), diff --git a/candle-nn/src/ops.rs b/candle-nn/src/ops.rs index abe33350..aaec8b56 100644 --- a/candle-nn/src/ops.rs +++ b/candle-nn/src/ops.rs @@ -35,13 +35,12 @@ pub fn log_softmax<D: candle::shape::Dim>(xs: &Tensor, d: D) -> Result<Tensor> { } pub fn silu(xs: &Tensor) -> Result<Tensor> { - // TODO: Should we have a specialized op for this? - xs / (xs.neg()?.exp()? + 1.0)? + xs.silu() } pub fn swiglu(xs: &Tensor) -> Result<Tensor> { let xs = xs.chunk(2, candle::D::Minus1)?; - crate::ops::silu(&xs[0])? * &xs[1] + &xs[0].silu()? * &xs[1] } pub fn sigmoid(xs: &Tensor) -> Result<Tensor> { |