From 3aac1047fec43a4d756ae4e60a8ae82f7c3e636e Mon Sep 17 00:00:00 2001 From: laurent Date: Tue, 4 Jul 2023 10:52:34 +0100 Subject: Sketch the conv1d op. --- candle-core/src/tensor.rs | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) (limited to 'candle-core/src/tensor.rs') diff --git a/candle-core/src/tensor.rs b/candle-core/src/tensor.rs index a468d879..26d44718 100644 --- a/candle-core/src/tensor.rs +++ b/candle-core/src/tensor.rs @@ -432,6 +432,28 @@ impl Tensor { Ok(from_storage(storage, dims, op, false)) } + pub fn conv1d(&self, kernel: &Self, padding: usize, stride: usize) -> Result { + let storage = self.storage.conv1d( + self.layout(), + &kernel.storage, + kernel.layout(), + padding, + stride, + )?; + let op = if self.track_op() || kernel.track_op() { + Some(Op::Conv1D { + arg: self.clone(), + kernel: kernel.clone(), + padding, + stride, + }) + } else { + None + }; + let dims = self.dims(); + Ok(from_storage(storage, dims, op, false)) + } + pub fn matmul(&self, rhs: &Self) -> Result { let a_dims = self.shape().dims(); let b_dims = rhs.shape().dims(); -- cgit v1.2.3 From a424d95473ea9268ffb1dde4d73ce0cff9904845 Mon Sep 17 00:00:00 2001 From: laurent Date: Tue, 4 Jul 2023 11:15:45 +0100 Subject: Add more of the conv1d op. --- candle-core/src/conv.rs | 24 ++++++++++++++++++++++++ candle-core/src/cpu_backend.rs | 3 +-- candle-core/src/cuda_backend.rs | 3 +-- candle-core/src/dummy_cuda_backend.rs | 3 +-- candle-core/src/lib.rs | 1 + candle-core/src/storage.rs | 7 +++---- candle-core/src/tensor.rs | 27 ++++++++++++++++++++------- candle-examples/examples/whisper/main.rs | 3 +-- 8 files changed, 52 insertions(+), 19 deletions(-) create mode 100644 candle-core/src/conv.rs (limited to 'candle-core/src/tensor.rs') diff --git a/candle-core/src/conv.rs b/candle-core/src/conv.rs new file mode 100644 index 00000000..90bb5229 --- /dev/null +++ b/candle-core/src/conv.rs @@ -0,0 +1,24 @@ +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct ParamsConv1D { + pub(crate) b_size: Option, + pub(crate) c_out: usize, + pub(crate) c_in: usize, + pub(crate) k_size: usize, + pub(crate) padding: usize, + pub(crate) stride: usize, +} + +impl ParamsConv1D { + pub(crate) fn l_out(&self, l_in: usize) -> usize { + let dilation = 1; + (l_in + 2 * self.padding - dilation * (self.k_size - 1) - 1) / self.stride + 1 + } + + pub(crate) fn out_dims(&self, l_in: usize) -> Vec { + let l_out = self.l_out(l_in); + match self.b_size { + None => vec![self.c_out, l_out], + Some(n) => vec![n, self.c_out, l_out], + } + } +} diff --git a/candle-core/src/cpu_backend.rs b/candle-core/src/cpu_backend.rs index ed3a5998..54002184 100644 --- a/candle-core/src/cpu_backend.rs +++ b/candle-core/src/cpu_backend.rs @@ -632,8 +632,7 @@ impl CpuStorage { _l: &Layout, _kernel: &Self, _kernel_l: &Layout, - _padding: usize, - _stride: usize, + _params: &crate::conv::ParamsConv1D, ) -> Result { todo!() } diff --git a/candle-core/src/cuda_backend.rs b/candle-core/src/cuda_backend.rs index ec69688c..917655fc 100644 --- a/candle-core/src/cuda_backend.rs +++ b/candle-core/src/cuda_backend.rs @@ -806,8 +806,7 @@ impl CudaStorage { _l: &Layout, _kernel: &Self, _kernel_l: &Layout, - _padding: usize, - _stride: usize, + _params: &crate::conv::ParamsConv1D, ) -> Result { todo!() } diff --git a/candle-core/src/dummy_cuda_backend.rs b/candle-core/src/dummy_cuda_backend.rs index eca5961b..0dbd8d54 100644 --- a/candle-core/src/dummy_cuda_backend.rs +++ b/candle-core/src/dummy_cuda_backend.rs @@ -105,8 +105,7 @@ impl CudaStorage { _l: &Layout, _kernel: &Self, _kernel_l: &Layout, - _padding: usize, - _stride: usize, + _params: &crate::conv::ParamsConv1D, ) -> Result { Err(Error::NotCompiledWithCudaSupport) } diff --git a/candle-core/src/lib.rs b/candle-core/src/lib.rs index 0d4c2a8d..2365a34d 100644 --- a/candle-core/src/lib.rs +++ b/candle-core/src/lib.rs @@ -1,4 +1,5 @@ mod backprop; +mod conv; mod cpu_backend; #[cfg(feature = "cuda")] mod cuda_backend; diff --git a/candle-core/src/storage.rs b/candle-core/src/storage.rs index 235080c0..53ea1544 100644 --- a/candle-core/src/storage.rs +++ b/candle-core/src/storage.rs @@ -149,18 +149,17 @@ impl Storage { l: &Layout, kernel: &Self, kernel_l: &Layout, - padding: usize, - stride: usize, + params: &crate::conv::ParamsConv1D, ) -> Result { self.same_device(kernel, "conv1d")?; self.same_dtype(kernel, "conv1d")?; match (self, &kernel) { (Storage::Cpu(inp), Storage::Cpu(kernel)) => { - let s = inp.conv1d(l, kernel, kernel_l, padding, stride)?; + let s = inp.conv1d(l, kernel, kernel_l, params)?; Ok(Self::Cpu(s)) } (Storage::Cuda(inp), Storage::Cuda(kernel)) => { - let s = inp.conv1d(l, kernel, kernel_l, padding, stride)?; + let s = inp.conv1d(l, kernel, kernel_l, params)?; Ok(Self::Cuda(s)) } (lhs, rhs) => Err(Error::DeviceMismatchBinaryOp { diff --git a/candle-core/src/tensor.rs b/candle-core/src/tensor.rs index 26d44718..590b81c4 100644 --- a/candle-core/src/tensor.rs +++ b/candle-core/src/tensor.rs @@ -433,13 +433,26 @@ impl Tensor { } pub fn conv1d(&self, kernel: &Self, padding: usize, stride: usize) -> Result { - let storage = self.storage.conv1d( - self.layout(), - &kernel.storage, - kernel.layout(), + let (c_out, c_in_k, k_size) = kernel.shape().r3()?; + let (b_size, c_in, l_in) = match *self.dims() { + [b_size, c_in, l_in] => (Some(b_size), c_in, l_in), + [c_in, l_in] => (None, c_in, l_in), + _ => todo!("proper error message"), + }; + if c_in != c_in_k { + todo!("proper error message") + } + let params = crate::conv::ParamsConv1D { + b_size, + c_out, + c_in, + k_size, padding, stride, - )?; + }; + let storage = + self.storage + .conv1d(self.layout(), &kernel.storage, kernel.layout(), ¶ms)?; let op = if self.track_op() || kernel.track_op() { Some(Op::Conv1D { arg: self.clone(), @@ -450,8 +463,8 @@ impl Tensor { } else { None }; - let dims = self.dims(); - Ok(from_storage(storage, dims, op, false)) + let out_dims = params.out_dims(l_in); + Ok(from_storage(storage, out_dims, op, false)) } pub fn matmul(&self, rhs: &Self) -> Result { diff --git a/candle-examples/examples/whisper/main.rs b/candle-examples/examples/whisper/main.rs index 75ab2189..a380d30e 100644 --- a/candle-examples/examples/whisper/main.rs +++ b/candle-examples/examples/whisper/main.rs @@ -236,8 +236,7 @@ impl Conv1D { fn forward(&self, x: &Tensor) -> candle::Result { let (bsize, _, _) = x.shape().r3()?; let w = self.weight.broadcast_left(bsize)?.t()?; - // TODO: Add the conv1d operation - let x = x.matmul(&w)?; + let x = x.conv1d(&w, self.config.padding, self.config.stride)?; match &self.bias { None => Ok(x), Some(bias) => x.broadcast_add(bias), -- cgit v1.2.3 From 950b4af49e56b640b87eb273e839b2fd466e1424 Mon Sep 17 00:00:00 2001 From: laurent Date: Tue, 4 Jul 2023 11:29:28 +0100 Subject: Proper conv1d dispatch. --- candle-core/src/conv.rs | 11 +++++++---- candle-core/src/cpu_backend.rs | 30 +++++++++++++++++++++++++----- candle-core/src/tensor.rs | 3 ++- 3 files changed, 34 insertions(+), 10 deletions(-) (limited to 'candle-core/src/tensor.rs') diff --git a/candle-core/src/conv.rs b/candle-core/src/conv.rs index 90bb5229..041bb6fb 100644 --- a/candle-core/src/conv.rs +++ b/candle-core/src/conv.rs @@ -1,6 +1,9 @@ #[derive(Debug, Clone, PartialEq, Eq)] pub(crate) struct ParamsConv1D { pub(crate) b_size: Option, + // Maybe we should have a version without l_in as this bit depends on the input and not only on + // the weights. + pub(crate) l_in: usize, pub(crate) c_out: usize, pub(crate) c_in: usize, pub(crate) k_size: usize, @@ -9,13 +12,13 @@ pub(crate) struct ParamsConv1D { } impl ParamsConv1D { - pub(crate) fn l_out(&self, l_in: usize) -> usize { + pub(crate) fn l_out(&self) -> usize { let dilation = 1; - (l_in + 2 * self.padding - dilation * (self.k_size - 1) - 1) / self.stride + 1 + (self.l_in + 2 * self.padding - dilation * (self.k_size - 1) - 1) / self.stride + 1 } - pub(crate) fn out_dims(&self, l_in: usize) -> Vec { - let l_out = self.l_out(l_in); + pub(crate) fn out_dims(&self) -> Vec { + let l_out = self.l_out(); match self.b_size { None => vec![self.c_out, l_out], Some(n) => vec![n, self.c_out, l_out], diff --git a/candle-core/src/cpu_backend.rs b/candle-core/src/cpu_backend.rs index 54002184..718b071c 100644 --- a/candle-core/src/cpu_backend.rs +++ b/candle-core/src/cpu_backend.rs @@ -202,6 +202,26 @@ fn copy_strided_src_( } } +struct Conv1D<'a>(&'a crate::conv::ParamsConv1D); + +impl<'a> Map2 for Conv1D<'a> { + const OP: &'static str = "conv1d"; + fn f( + &self, + _inp: &[T], + _inp_l: &Layout, + _k: &[T], + _k_l: &Layout, + ) -> Result> { + let p = self.0; + let l_out = p.l_out(); + let out_elems = p.c_out * l_out * p.b_size.unwrap_or(1); + let dst = vec![T::zero(); out_elems]; + // TODO: actually implement the ops. + Ok(dst) + } +} + struct MatMul((usize, usize, usize, usize)); impl Map2 for MatMul { @@ -629,12 +649,12 @@ impl CpuStorage { pub(crate) fn conv1d( &self, - _l: &Layout, - _kernel: &Self, - _kernel_l: &Layout, - _params: &crate::conv::ParamsConv1D, + l: &Layout, + kernel: &Self, + kernel_l: &Layout, + params: &crate::conv::ParamsConv1D, ) -> Result { - todo!() + Conv1D(params).map(self, l, kernel, kernel_l) } pub(crate) fn embedding(&self, ids_l: &Layout, rhs: &Self, rhs_l: &Layout) -> Result { diff --git a/candle-core/src/tensor.rs b/candle-core/src/tensor.rs index 590b81c4..25ab0a9b 100644 --- a/candle-core/src/tensor.rs +++ b/candle-core/src/tensor.rs @@ -444,6 +444,7 @@ impl Tensor { } let params = crate::conv::ParamsConv1D { b_size, + l_in, c_out, c_in, k_size, @@ -463,7 +464,7 @@ impl Tensor { } else { None }; - let out_dims = params.out_dims(l_in); + let out_dims = params.out_dims(); Ok(from_storage(storage, out_dims, op, false)) } -- cgit v1.2.3 From 6d1e79d3782c93c7f6a097ce71a91a9a277e52ed Mon Sep 17 00:00:00 2001 From: laurent Date: Wed, 5 Jul 2023 06:42:29 +0100 Subject: Bugfix for to_scalar (use the proper start offset). --- candle-core/src/tensor.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'candle-core/src/tensor.rs') diff --git a/candle-core/src/tensor.rs b/candle-core/src/tensor.rs index 25ab0a9b..95f663f0 100644 --- a/candle-core/src/tensor.rs +++ b/candle-core/src/tensor.rs @@ -326,7 +326,7 @@ impl Tensor { } let from_cpu_storage = |cpu_storage: &crate::CpuStorage| { let data = S::cpu_storage_as_slice(cpu_storage)?; - Ok::<_, Error>(data[0]) + Ok::<_, Error>(data[self.layout().start_offset()]) }; match self.storage.as_ref() { Storage::Cpu(cpu_storage) => from_cpu_storage(cpu_storage), -- cgit v1.2.3