diff options
Diffstat (limited to 'candle-nn/src/ops.rs')
| -rw-r--r-- | candle-nn/src/ops.rs | 5 |
1 files changed, 5 insertions, 0 deletions
diff --git a/candle-nn/src/ops.rs b/candle-nn/src/ops.rs index a51ef2e3..a0269e59 100644 --- a/candle-nn/src/ops.rs +++ b/candle-nn/src/ops.rs @@ -39,6 +39,11 @@ pub fn silu(xs: &Tensor) -> Result<Tensor> { xs / (xs.neg()?.exp()? + 1.0)? } +pub fn swiglu(xs: &Tensor) -> Result<Tensor> { + let xs = xs.chunk(2, candle::D::Minus1)?; + crate::ops::silu(&xs[0])? * &xs[1] +} + pub fn sigmoid(xs: &Tensor) -> Result<Tensor> { // TODO: Should we have a specialized op for this? (xs.neg()?.exp()? + 1.0)?.recip() |