*.dylib

*.so

*.swp

trace-*.json

candle-wasm-examples/*/build

# Changelog

This documents the main changes to the `candle` crate.

### Added

### Modified

- Dilations are now supported in conv-transpose2d.

[671](https://github.com/huggingface/candle/pull/671).

## v0.2.0 - 2023-08-30

"candle-pyo3",

"candle-transformers",

"candle-wasm-examples/llama2-c",

"candle-wasm-examples/whisper",

"candle-wasm-examples/yolo",

]

resolver = "2"

[workspace.package]

edition = "2021"

description = "Minimalist ML framework."

repository = "https://github.com/huggingface/candle"

clap = { version = "4.2.4", features = ["derive"] }

cudarc = { version = "0.9.14", features = ["f16"] }

# TODO: Switch back to the official gemm implementation once it has caught up.

hf-hub = "0.3.0"

half = { version = "2.3.1", features = ["num-traits", "use-intrinsics", "rand_distr"] }

image = { version = "0.24.7", default-features = false, features = ["jpeg", "png"] }

and ease of use. Try our online demos:

[whisper](https://huggingface.co/spaces/lmz/candle-whisper),

[LLaMA2](https://huggingface.co/spaces/lmz/candle-llama2),

## Get started

## Check out our examples

- [LLaMA and LLaMA-v2](./candle-examples/examples/llama/): general LLM.

- [Falcon](./candle-examples/examples/falcon/): general LLM.

- [StarCoder](./candle-examples/examples/bigcode/): LLM specialized to code

generation.

- [Quantized LLaMA](./candle-examples/examples/quantized/): quantized version of

the LLaMA model using the same quantization techniques as

[llama.cpp](https://github.com/ggerganov/llama.cpp).

- [yolo-v3](./candle-examples/examples/yolo-v3/) and

[yolo-v8](./candle-examples/examples/yolo-v8/): object detection and pose

estimation models.

cargo run --example quantized --release

```

In order to use **CUDA** add `--features cuda` to the example command line. If

[llama2.c](https://github.com/karpathy/llama2.c). You can either build them with

`trunk` or try them online:

[whisper](https://huggingface.co/spaces/lmz/candle-whisper),

For LLaMA2, run the following command to retrieve the weight files and start a

test server:

And then head over to

[http://localhost:8081/](http://localhost:8081/).

<!--- ANCHOR: features --->

## Features

- CUDA backend for efficiently running on GPUs, multiple GPU distribution via NCCL.

- WASM support, run your models in a browser.

- Included models.

- Whisper (multi-lingual support).

- File formats: load models from safetensors, npz, ggml, or PyTorch files.

- Serverless (on CPU), small and fast deployments.

- Quantization support using the llama.cpp quantized types.

git submodule update --init

```

#### Tracking down errors

You can set `RUST_BACKTRACE=1` to be provided with backtraces when a candle

[dependencies]

accelerate-src = { workspace = true, optional = true }

safetensors = { workspace = true }

serde = { workspace = true }

serde_json = { workspace = true }

# Reference Guide

- [Using the hub](inference/hub.md)

- [Error management](error_manage.md)

- [Simplified](training/simplified.md)

- [MNIST](training/mnist.md)

- [Fine-tuning]()

Error: WithBacktrace { inner: ShapeMismatchBinaryOp { lhs: [1, 784], rhs: [1, 784], op: "matmul" }, backtrace: Backtrace [{ fn: "candle::error::Error::bt", file: "/home/nicolas/.cargo/git/checkouts/candle-5bb8ef7e0626d693/f291065/candle-core/src/error.rs", line: 200 }, { fn: "candle::tensor::Tensor::matmul", file: "/home/nicolas/.cargo/git/checkouts/candle-5bb8ef7e0626d693/f291065/candle-core/src/tensor.rs", line: 816 }, { fn: "myapp::main", file: "./src/main.rs", line: 29 }, { fn: "core::ops::function::FnOnce::call_once", file: "/rustc/8ede3aae28fe6e4d52b38157d7bfe0d3bceef225/library/core/src/ops/function.rs", line: 250 }, { fn: "std::sys_common::backtrace::__rust_begin_short_backtrace", file: "/rustc/8ede3aae28fe6e4d52b38157d7bfe0d3bceef225/library/std/src/sys_common/backtrace.rs", line: 135 }, { fn: "std::rt::lang_start::{{closure}}", file: "/rustc/8ede3aae28fe6e4d52b38157d7bfe0d3bceef225/library/std/src/rt.rs", line: 166 }, { fn: "core::ops::function::impls::<impl core::ops::function::FnOnce<A> for &F>::call_once", file: "/rustc/8ede3aae28fe6e4d52b38157d7bfe0d3bceef225/library/core/src/ops/function.rs", line: 284 }, { fn: "std::panicking::try::do_call", file: "/rustc/8ede3aae28fe6e4d52b38157d7bfe0d3bceef225/library/std/src/panicking.rs", line: 500 }, { fn: "std::panicking::try", file: "/rustc/8ede3aae28fe6e4d52b38157d7bfe0d3bceef225/library/std/src/panicking.rs", line: 464 }, { fn: "std::panic::catch_unwind", file: "/rustc/8ede3aae28fe6e4d52b38157d7bfe0d3bceef225/library/std/src/panic.rs", line: 142 }, { fn: "std::rt::lang_start_internal::{{closure}}", file: "/rustc/8ede3aae28fe6e4d52b38157d7bfe0d3bceef225/library/std/src/rt.rs", line: 148 }, { fn: "std::panicking::try::do_call", file: "/rustc/8ede3aae28fe6e4d52b38157d7bfe0d3bceef225/library/std/src/panicking.rs", line: 500 }, { fn: "std::panicking::try", file: "/rustc/8ede3aae28fe6e4d52b38157d7bfe0d3bceef225/library/std/src/panicking.rs", line: 464 }, { fn: "std::panic::catch_unwind", file: "/rustc/8ede3aae28fe6e4d52b38157d7bfe0d3bceef225/library/std/src/panic.rs", line: 142 }, { fn: "std::rt::lang_start_internal", file: "/rustc/8ede3aae28fe6e4d52b38157d7bfe0d3bceef225/library/std/src/rt.rs", line: 148 }, { fn: "std::rt::lang_start", file: "/rustc/8ede3aae28fe6e4d52b38157d7bfe0d3bceef225/library/std/src/rt.rs", line: 165 }, { fn: "main" }, { fn: "__libc_start_main" }, { fn: "_start" }] }

```

Another thing to note, is that since Rust is compiled it is not necessarily as easy to recover proper stacktraces

```rust

# extern crate candle_core;

struct Model {

first: Tensor,

// Use Device::new_cuda(0)?; to use the GPU.

let device = Device::Cpu;

let model = Model { first, second };

let digit = model.forward(&dummy_image)?;

println!("Digit {digit:?} digit");

```rust

# extern crate candle_core;

struct Linear{

weight: Tensor,

bias: Tensor,

```rust

# extern crate candle_core;

# struct Linear{

# weight: Tensor,

# bias: Tensor,

let device = Device::cuda_if_available(0)?;

// Creating a dummy model

let first = Linear{weight, bias};

let second = Linear{weight, bias};

let model = Model { first, second };

// Inference on the model

let digit = model.forward(&dummy_image)?;

```rust

# extern crate candle_core;

# extern crate candle_nn;

use candle_nn::{Linear, Module};

struct Model {

let device = Device::Cpu;

// This has changed (784, 100) -> (100, 784) !

let first = Linear::new(weight, Some(bias));

let second = Linear::new(weight, Some(bias));

let model = Model { first, second };

let digit = model.forward(&dummy_image)?;

println!("Digit {digit:?} digit");

Now that we have the running dummy code we can get to more advanced topics:

[dependencies]

accelerate-src = { workspace = true, optional = true }

byteorder = { workspace = true }

cudarc = { workspace = true, optional = true }

gemm = { workspace = true }

half = { workspace = true }

Ok(())

}

fn run_quantize(

in_file: std::path::PathBuf,

out_file: std::path::PathBuf,

q: Quantization,

qmode: QuantizationMode,

) -> Result<()> {

// Open the out file early so as to fail directly on missing directories etc.

let mut out_file = std::fs::File::create(out_file)?;

let mut in_ = std::fs::File::open(&in_file)?;

y.iter_mut().zip(a.iter()).for_each(|(y, a)| *y = *a * *a)

}

macro_rules! binary_op {

($fn_name:ident, $ty:ty, $accelerate_name:ident) => {

#[inline]

fn avg_pool2d(&self, _: &Layout, _: (usize, usize), _: (usize, usize)) -> Result<Self>;

fn max_pool2d(&self, _: &Layout, _: (usize, usize), _: (usize, usize)) -> Result<Self>;

fn upsample_nearest2d(&self, _: &Layout, _: usize, _: usize) -> Result<Self>;

fn gather(&self, _: &Layout, _: &Self, _: &Layout, _: usize) -> Result<Self>;

}

}

Op::Reshape(node)

| Op::UpsampleNearest2D(node)

| Op::AvgPool2D { arg: node, .. }

| Op::MaxPool2D { arg: node, .. }

| Op::Copy(node)

| Op::Broadcast(node)

| Op::Cmp(node, _)

| Op::ToDType(node)

| Op::ToDevice(node)

| Op::Transpose(node, _, _)

track_grad |= tg;

nodes

}

}

} else {

nodes

let sum_grad = grads.or_insert(arg)?;

*sum_grad = sum_grad.add(&grad_arg)?;

}

Op::UpsampleNearest2D { .. } => Err(Error::BackwardNotSupported {

op: "upsample-nearest2d",

})?,

*sum_grad = sum_grad.add(&arg_grad)?

}

Op::Unary(_, UnaryOp::Gelu) => Err(Error::BackwardNotSupported { op: "gelu" })?,

Op::Unary(arg, UnaryOp::Relu) => {

let sum_grad = grads.or_insert(arg)?;

let relu_grad = arg.ge(&arg.zeros_like()?)?.to_dtype(arg.dtype())?;

}

}

pub struct GradStore(HashMap<TensorId, Tensor>);

impl GradStore {

/// Dot-product of two vectors.

///

/// # Safety

unsafe fn vec_reduce_max(xs: *const Self, res: *mut Self, len: usize) {

*res = *xs;

for i in 1..len {

}

}

unsafe fn vec_reduce_min(xs: *const Self, res: *mut Self, len: usize) {

*res = *xs;

for i in 1..len {

}

}

}

impl VecOps for f32 {

#[inline(always)]

unsafe fn vec_dot(lhs: *const Self, rhs: *const Self, res: *mut Self, len: usize) {

super::vec_dot_f32(lhs, rhs, res, len)

}

impl VecOps for half::f16 {

#[inline(always)]

unsafe fn vec_dot(lhs: *const Self, rhs: *const Self, res: *mut Self, len: usize) {

let mut res_f32 = 0f32;

super::vec_dot_f16(lhs, rhs, &mut res_f32, len);

}

}

#[inline(always)]

pub fn par_for_each(n_threads: usize, func: impl Fn(usize) + Send + Sync) {

pub mod kernels;

trait Cpu<const ARR: usize> {

use crate::op::{BinaryOpT, CmpOp, ReduceOp, UnaryOpT};

use crate::{DType, Error, IntDType, Layout, Result, Shape, WithDType};

use half::{bf16, f16};

// TODO: Maybe we should not implement [Clone] here and instead have an explicit allocator +

// intercept the oom errors to avoid panicking and provide a proper error.

}

// This function maps over two strided index sequences.

lhs_l: &Layout,

rhs_l: &Layout,

lhs: &[T],

}

// Similar to binary_map but with vectorized variants.

lhs_l: &Layout,

rhs_l: &Layout,

lhs: &[T],

}

}

struct UpsampleNearest2D(usize, usize);

impl Map1 for UpsampleNearest2D {

}

}

for offset in 0..p.k_size {

let dst_idx = dst_c_idx * l_out;

let k_cont = (0..p.c_in)

.map(|c_in_idx| k[dst_c_idx * k_s0 + c_in_idx * k_s1 + offset * k_s2])

}

}

struct Conv2D<'a>(&'a crate::conv::ParamsConv2D);

impl<'a> Map2 for Conv2D<'a> {

}

}

for offset_h in 0..p.k_h {

for offset_w in 0..p.k_w {

let dst_idx = dst_c_idx * out_w * out_h;

let k_cont = (0..p.c_in)

.map(|c_in_idx| {

}

}

}

for k_y in 0..p.k_h {

for k_x in 0..p.k_w {

let k_cont = (0..p.c_in)

.map(|c_in_idx| {

k[c_in_idx * k_s0 + dst_c_idx * k_s1 + k_y * k_s2 + k_x * k_s3]

) -> Result<Vec<T>> {

use gemm::{gemm, Parallelism};

}

let (b, m, n, k) = self.0;

MaxPool2D(kernel_size, stride).map(self, layout)

}

fn upsample_nearest2d(&self, layout: &Layout, h: usize, w: usize) -> Result<Self> {

UpsampleNearest2D(h, w).map(self, layout)

}

kernel_l: &Layout,

params: &crate::conv::ParamsConv1D,

) -> Result<Self> {

}

fn conv2d(

kernel_l: &Layout,

params: &crate::conv::ParamsConv2D,

) -> Result<Self> {

}

fn conv_transpose2d(

use crate::backend::{BackendDevice, BackendStorage};

use crate::op::{BinaryOpT, CmpOp, ReduceOp, UnaryOpT};

use crate::{CpuStorage, DType, Layout, Result, Shape, WithDType};

pub use cudarc;

use cudarc::cublas::{Gemm, GemmConfig, StridedBatchedConfig};

use cudarc::driver::{

// cudarc changes.

let elem_count = shape.elem_count();

let curand = self.curand.lock().unwrap();

let slice = match dtype {

DType::U8 | DType::U32 | DType::I64 | DType::F16 | DType::BF16 => {

Err(CudaError::UnsupportedDtype {

.w()?

}

DType::F32 => {

curand

.0

.fill_with_normal(&mut data, mean as f32, std as f32)

CudaStorageSlice::F32(data)

}

DType::F64 => {

curand.0.fill_with_normal(&mut data, mean, std).w()?;

CudaStorageSlice::F64(data)

}

}

#[derive(Debug)]

U8(CudaSlice<u8>),

U32(CudaSlice<u32>),

I64(CudaSlice<i64>),

}

type S = CudaStorageSlice;

fn f<T: DeviceRepr + WithDType + ValidAsZeroBits>(

&self,

src: &CudaSlice<T>,

}

}

fn f<T: DeviceRepr + WithDType + ValidAsZeroBits>(

&self,

src1: &CudaSlice<T>,

}

}

fn f<T: DeviceRepr + WithDType + ValidAsZeroBits>(

&self,

dst: &mut CudaSlice<T>,

}

}

fn f<T: DeviceRepr + WithDType + ValidAsZeroBits, W: Fn(CudaSlice<T>) -> S>(

&self,

src: &CudaSlice<T>,

}

}

fn f<T: DeviceRepr + WithDType + ValidAsZeroBits>(

&self,

src1: &CudaSlice<T>,

}

}

let dtype = T::DTYPE.as_str();

format!("{root}_{dtype}")

}

}

}

struct Powf(f64);

impl Map1 for Powf {

fn f<T: DeviceRepr + WithDType>(

#[derive(Debug)]

pub struct CudaStorage {

}

pub trait CudaDType: Sized {

kernel_l: &Layout,

params: &crate::conv::ParamsConv1D,

) -> Result<Self> {

let device = self.device().clone();

}

#[cfg(not(feature = "cudnn"))]

kernel_l: &Layout,

params: &crate::conv::ParamsConv2D,

) -> Result<Self> {

let device = self.device().clone();

}

#[cfg(feature = "cudnn")]

Ok(Self { slice, device })

}

fn upsample_nearest2d(&self, l: &Layout, out_w: usize, out_h: usize) -> Result<Self> {

let device = self.device().clone();

let slice = UpsampleNearest2D(out_w, out_h).map(&self.slice, &device, l)?;

let src_shape = src_l.shape();

let dims = src_shape.dims();

let el_count = src_shape.elem_count();

let cfg = LaunchConfig::for_num_elems(el_count as u32);

let dev = &self.device;

let ds = dev.htod_copy([dims, src_l.stride()].concat()).w()?;

let x_shape = [

params.b_size as i32,

params.c_in as i32,

params.i_h as i32,

];

// Note that `src` already starts at the proper offset.

let x = if src_l.is_contiguous() {

[

params.c_out as i32,

params.c_in as i32,

params.k_h as i32,

],

)?;

let (w_out, h_out) = (params.out_w() as i32, params.out_h() as i32);

let y = cudnn.create_4d_tensor(

cudarc::cudnn::sys::cudnnTensorFormat_t::CUDNN_TENSOR_NCHW,

)?;

let conv2d = Conv2dForward {

conv: &conv,

#![allow(clippy::redundant_closure_call)]

use crate::backend::BackendStorage;

use crate::{CpuStorage, Error, Result};

#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]

pub enum DType {

U8,

U32,

I64,

BF16,

F16,

F32,

F64,

}

}

impl DType {

pub fn as_str(&self) -> &'static str {

match self {

Self::U8 => "u8",

}

}

pub fn size_in_bytes(&self) -> usize {

match self {

Self::U8 => 1,

Err(Error::NotCompiledWithCudaSupport)

}

fn upsample_nearest2d(&self, _: &Layout, _: usize, _: usize) -> Result<Self> {

Err(Error::NotCompiledWithCudaSupport)

}

UnsupportedDTypeForOp(DType, &'static str),

// === Dimension Index Errors ===

DimOutOfRange {

shape: Shape,

dim: i32,

impl Error {

pub fn wrap(err: impl std::error::Error + Send + Sync + 'static) -> Self {

}

pub fn msg(err: impl std::error::Error + Send + Sync + 'static) -> Self {

}

pub fn bt(self) -> Self {

current_dim += 1;

out

}

};

}

Ok(x)

}

}

/// Generic structure used to index a slice of the tensor

pub enum TensorIndexer {

/// This selects the elemnts for which an index has some specific value.

Select(usize),

/// This is a regular slice, purely indexing a chunk of the tensor

Narrow(Bound<usize>, Bound<usize>),

}

impl From<usize> for TensorIndexer {

}

}

macro_rules! impl_from_range {

($range_type:ty) => {

impl From<$range_type> for TensorIndexer {

pub mod pickle;

pub mod quantized;

pub mod safetensors;

pub mod shape;

mod storage;

mod strided_index;

}

// A simple trait defining a module with forward method using a single argument.

fn forward(&self, xs: &Tensor) -> Result<Tensor>;

}

impl Module for quantized::QMatMul {

self.forward(xs)

}

}

Sqr,

Sqrt,

Gelu,

Relu,

Tanh,

}

stride: (usize, usize),

},

UpsampleNearest2D(Tensor),

Cat(Vec<Tensor>, usize),

pub(crate) struct Sqr;

pub(crate) struct Sqrt;

pub(crate) struct Gelu;

pub(crate) struct Relu;

pub(crate) struct Tanh;

fn f64_vec(xs: &[f64], ys: &mut [f64]) {

crate::mkl::vd_gelu(xs, ys)

}

}

impl UnaryOpT for Relu {

pub struct BlockQ8_1 {

pub(crate) d: f16,

pub(crate) s: f16,

}

const _: () = assert!(std::mem::size_of::<BlockQ8_1>() == 36);

}

sumf += sumi as f32 * f16::to_f32(xs.d) * f16::to_f32(ys.d)

}

Ok(sumf)

}

}

sumf += sumi as f32 * f16::to_f32(xs.d) * f16::to_f32(ys.d)

}

Ok(sumf)

}

for j in 0..Self::BLCK_SIZE / 2 {

let v0 = xs[j] * id;

let v1 = xs[j + Self::BLCK_SIZE / 2] * id;

sum += ys.qs[j] as i32 + ys.qs[j + Self::BLCK_SIZE / 2] as i32;

}

ys.s = f16::from_f32(sum as f32) * ys.d;

}

}

pub struct QMatMul(std::sync::Arc<QTensor>);

impl QMatMul {

}

impl Tensor {

let data = [(name, self.clone())];

Ok(st::serialize_to_file(data, &None, filename.as_ref())?)

}

/// # Safety

///

/// The unsafe is inherited from [`memmap2::MmapOptions`].

let p = p.as_ref();

let file = std::fs::File::open(p).map_err(|e| Error::from(e).with_path(p))?;

let inner = memmap2::MmapOptions::new()

#![allow(clippy::redundant_closure_call)]

use crate::{Error, Result};

}

}

impl From<Vec<usize>> for Shape {

fn from(dims: Vec<usize>) -> Self {

Self(dims)

Self(dims.to_vec())

}

pub fn rank(&self) -> usize {

self.0.len()

}

self.0

}

pub fn dims(&self) -> &[usize] {

&self.0

}

pub fn elem_count(&self) -> usize {

self.0.iter().product()

}

true

}

pub fn extend(mut self, additional_dims: &[usize]) -> Self {

self.0.extend(additional_dims);

self

}

}

extract_dims!(dims0, 0, |_: &[usize]| (), ());

extract_dims!(dims1, 1, |d: &[usize]| d[0], usize);

extract_dims!(dims2, 2, |d: &[usize]| (d[0], d[1]), (usize, usize));

assert_eq!(shape.stride_contiguous(), [458 * 792, 458, 1]);

}

}

}

}

pub(crate) fn upsample_nearest2d(&self, layout: &Layout, h: usize, w: usize) -> Result<Self> {

match self {

Storage::Cpu(storage) => {

#![allow(clippy::redundant_closure_call)]

use crate::backend::{BackendDevice, BackendStorage};

use crate::op::{

BackpropOp, BinaryOp, CmpOp, CustomOp1, CustomOp2, CustomOp3, Op, ReduceOp, UnaryOp,

};

use crate::shape::{Dim, Dims};

use crate::{storage::Storage, DType, Device, Error, Layout, Result, Shape};

use std::sync::{Arc, RwLock};

};

}

macro_rules! broadcast_binary_op {

($fn_name:ident, $inner_fn_name:ident) => {

pub fn $fn_name(&self, rhs: &Self) -> Result<Self> {

binary_op!(mul, Mul);

binary_op!(sub, Sub);

binary_op!(div, Div);

broadcast_binary_op!(broadcast_add, add);

broadcast_binary_op!(broadcast_mul, mul);

broadcast_binary_op!(broadcast_sub, sub);

unary_op!(sqr, Sqr);

unary_op!(sqrt, Sqrt);

unary_op!(gelu, Gelu);

unary_op!(relu, Relu);

/// Retrieves the single scalar value hold in the tensor. If the tensor contains multiple

let storage = self.storage().reduce_op(op, self.layout(), &[dim])?;

let mut dims = self.dims().to_vec();

dims[dim] = 1;

let res = from_storage(storage, dims, op, false);

if keepdim {

Ok(res)

/// comparison operation is specified by the `op` argument.

///

/// The returned tensor has the same shape as the original tensors and uses `u8` elements.

let storage = self

.storage()

.cmp(op, &rhs.storage(), self.layout(), rhs.layout())?;

}

/// Element-wise equality.

self.cmp(rhs, CmpOp::Eq)

}

/// Element-wise non-equality.

self.cmp(rhs, CmpOp::Ne)

}

/// Element-wise comparison with lower-than, the returned tensor uses value 1 where `self <

/// rhs` and 0 otherwise.

self.cmp(rhs, CmpOp::Lt)

}

/// Element-wise comparison with greater-than, the returned tensor uses value 1 where `self >

/// rhs` and 0 otherwise.

self.cmp(rhs, CmpOp::Gt)

}

/// Element-wise comparison with greater-equal, the returned tensor uses value 1 where `self >=

/// rhs` and 0 otherwise.

self.cmp(rhs, CmpOp::Ge)

}

/// Element-wise comparison with lower-equal, the returned tensor uses value 1 where `self <=

/// rhs` and 0 otherwise.

self.cmp(rhs, CmpOp::Le)

}

/// nearest element.

///

/// The input tensor should have four dimensions, `(batch, channels, h, w)`, the returned

/// tensor also has four dimensions, `(batch, channels, target_h, target_w)`.

let (n, c, _h, _w) = self.dims4()?;

let op = BackpropOp::new1(self, Op::UpsampleNearest2D);

let storage = self

Ok(from_storage(storage, (n, c, target_h, target_w), op, false))

}

/// 2D average pooling over an input tensor with multiple channels.

///

/// The input tensor should have four dimensions, `(batch, channels, h, w)`, the returned

Ok(from_storage(storage, shape, BackpropOp::none(), true))

}

/// Reshape returns a tensor with the target shape provided that the number of elements of the

/// original tensor is the same.

/// If the input tensor is contiguous, this is a view on the original data. Otherwise this uses

/// a new storage and copies the data over, the returned tensor is always contiguous.

///

/// ```rust

/// # use candle_core::{Tensor, DType, Device, D};

/// let a = Tensor::zeros((2, 3), DType::F32, &Device::Cpu)?;

///

/// let c = a.reshape((3, 2))?;

/// assert_eq!(c.shape().dims(), &[3, 2]);

/// # Ok::<(), candle_core::Error>(())

/// ```

if shape.elem_count() != self.elem_count() {

return Err(Error::ShapeMismatchBinaryOp {

lhs: self.shape().clone(),

for arg in args {

arg.as_ref().check_dim(dim, "cat")?;

}

if dim == 0 {

Self::cat0(args)

} else {

fn zeros(device: &Device) -> Result<()> {

let tensor = Tensor::zeros((5, 2), DType::F32, device)?;

Ok(())

}

fn binary_op(device: &Device) -> Result<()> {

let data = &[[3f32, 1., 4., 1., 5.], [2., 1., 7., 8., 2.]];

let tensor1 = Tensor::new(data, device)?;

Ok(())

}

test_device!(zeros, zeros_cpu, zeros_gpu);

test_device!(add_mul, add_mul_cpu, add_mul_gpu);

test_device!(tensor_2d, tensor_2d_cpu, tensor_2d_gpu);

test_device!(argmax, argmax_cpu, argmax_gpu);

test_device!(argmin, argmin_cpu, argmin_gpu);

test_device!(transpose, transpose_cpu, transpose_gpu);

test_device!(binary_op, binary_op_cpu, binary_op_gpu);

test_device!(embeddings, embeddings_cpu, embeddings_gpu);

test_device!(cmp, cmp_cpu, cmp_gpu);

test_device!(index_add, index_add_cpu, index_add_gpu);

test_device!(gather, gather_cpu, gather_gpu);

test_device!(scatter_add, scatter_add_cpu, scatter_add_gpu);

// There was originally a bug on the CPU implementation for randn

// https://github.com/huggingface/candle/issues/381

[dependencies]

byteorder = { workspace = true }

hf-hub = { workspace = true}

intel-mkl-src = { workspace = true, optional = true }

memmap2 = { workspace = true }

use std::fs::File;

use std::io::{self, BufReader, Read};

}

fn check_magic_number<T: Read>(reader: &mut T, expected: u32) -> Result<()> {

[dependencies]

accelerate-src = { workspace = true, optional = true }

cudarc = { workspace = true, optional = true }

half = { workspace = true, optional = true }

image = { workspace = true }

[dev-dependencies]

anyhow = { workspace = true }

accelerate = ["dep:accelerate-src", "candle/accelerate", "candle-nn/accelerate", "candle-transformers/accelerate"]

cuda = ["candle/cuda", "candle-nn/cuda", "candle-transformers/cuda"]

cudnn = ["candle/cudnn"]

mkl = ["dep:intel-mkl-src", "candle/mkl", "candle-nn/mkl", "candle-transformers/mkl"]

nccl = ["cuda", "cudarc/nccl", "dep:half"]

#[cfg(feature = "accelerate")]

extern crate accelerate_src;

use anyhow::{anyhow, Error as E, Result};

use candle::Tensor;

use candle_nn::VarBuilder;

use clap::Parser;

use hf_hub::{api::sync::Api, Cache, Repo, RepoType};

use tokenizers::{PaddingParams, Tokenizer};

#[derive(Parser, Debug)]

use anyhow::{Error as E, Result};

use clap::Parser;

use candle::{DType, Device, Tensor};

use candle_nn::VarBuilder;

tokenizer: Tokenizer,

seed: u64,

temp: Option<f64>,

device: &Device,

) -> Self {

Self {

model,

tokenizer,

#[arg(long)]

temperature: Option<f64>,

/// The seed to use when generating random samples.

#[arg(long, default_value_t = 299792458)]

seed: u64,

let model = GPTBigCode::load(vb, config)?;

println!("loaded the model in {:?}", start.elapsed());

pipeline.run(&args.prompt, args.sample_len)?;

Ok(())

}

use clap::Parser;

#[derive(Parser)]

struct Args {

#[arg(long)]

let weights = unsafe { candle::safetensors::MmapedFile::new(model_file)? };

let weights = weights.deserialize()?;

let vb = VarBuilder::from_safetensors(vec![weights], DType::F32, &device);

println!("model built");

let logits = model.forward(&image.unsqueeze(0)?)?;

let prs = candle_nn::ops::softmax(&logits, D::Minus1)?

#[cfg(feature = "accelerate")]

extern crate accelerate_src;

use clap::{Parser, ValueEnum};

#[derive(Clone, Copy, Debug, ValueEnum)]

enum Which {

B0,

use hf_hub::{api::sync::Api, Repo, RepoType};

use tokenizers::Tokenizer;

struct TextGeneration {

model: Falcon,

repeat_last_n: usize,

}

impl TextGeneration {

fn new(

model: Falcon,

tokenizer: Tokenizer,

seed: u64,

device: &Device,

) -> Self {

Self {

model,

tokenizer,

#[arg(long)]

temperature: Option<f64>,

/// The seed to use when generating random samples.

#[arg(long, default_value_t = 299792458)]

seed: u64,

let model = Falcon::load(vb, config)?;

println!("loaded the model in {:?}", start.elapsed());

pipeline.run(&args.prompt, args.sample_len)?;

Ok(())

}

use hf_hub::{api::sync::Api, Repo, RepoType};

use std::io::Write;

use model::{Config, Llama, LlamaConfig};

const EOS_TOKEN: &str = "</s>";

const DEFAULT_PROMPT: &str = "My favorite theorem is ";

#[derive(Parser, Debug)]

#[arg(long)]

temperature: Option<f64>,

/// The seed to use when generating random samples.

#[arg(long, default_value_t = 299792458)]

seed: u64,

println!("starting the inference loop");

print!("{prompt}");

let start_gen = std::time::Instant::now();

let mut index_pos = 0;

let mut token_generated = 0;

#[arg(long)]

temperature: Option<f64>,

#[arg(long, default_value = "")]

prompt: String,

None => {

let cmd = InferenceCmd {

temperature: None,

prompt: "".to_string(),

config: None,

model_id: "karpathy/tinyllamas".to_string(),

let model = Llama::load(vb, &cache, config)?;

println!("starting the inference loop");

let mut index_pos = 0;

print!("{}", args.prompt);

#[arg(long)]

temperature: Option<f64>,

/// The seed to use when generating random samples.

#[arg(long, default_value_t = 299792458)]

seed: u64,

.to_vec();

println!("starting the inference loop");

let mut new_tokens = vec![];

let start_gen = std::time::Instant::now();

let mut index_pos = 0;

mod encodec_model;

mod musicgen_model;

mod nn;

use musicgen_model::{GenConfig, MusicgenForConditionalGeneration};

let model = model.deserialize()?;

let vb = VarBuilder::from_safetensors(vec![model], DTYPE, &device);

let config = GenConfig::small();

let tokens = tokenizer

.encode(args.prompt.as_str(), true)

use candle::{DType, Device, Result, Tensor, D};

use candle_nn::{

embedding, layer_norm, linear_no_bias, Activation, Embedding, LayerNorm, Linear, Module,

VarBuilder,

};

// https://github.com/huggingface/transformers/blob/cd4584e3c809bb9e1392ccd3fe38b40daba5519a/src/transformers/models/musicgen/configuration_musicgen.py#L83

#[derive(Debug, Clone, PartialEq)]

#[derive(Debug)]

pub struct MusicgenForConditionalGeneration {

pub audio_encoder: crate::encodec_model::EncodecModel,

pub decoder: MusicgenForCausalLM,

cfg: GenConfig,

#[derive(Debug, Clone, PartialEq)]

pub struct GenConfig {

musicgen: Config,

encodec: crate::encodec_model::Config,

}

pub fn small() -> Self {

Self {

musicgen: Config::musicgen_small(),

encodec: encodec_model::Config::musicgen_small(),

}

}

}

pub fn load(vb: VarBuilder, cfg: GenConfig) -> Result<Self> {

let audio_encoder =

encodec_model::EncodecModel::load(vb.pp("audio_encoder"), &cfg.encodec)?;

let decoder = MusicgenForCausalLM::load(vb.pp("decoder"), &cfg.musicgen)?;

use candle::{Device, Tensor};

use candle_transformers::generation::LogitsProcessor;

use model::ModelWeights;

const DEFAULT_PROMPT: &str = "My favorite theorem is ";

#[arg(long, default_value_t = 0.8)]

temperature: f64,

/// The seed to use when generating random samples.

#[arg(long, default_value_t = 299792458)]

seed: u64,

prompt_tokens

};

let mut all_tokens = vec![];

let start_prompt_processing = std::time::Instant::now();

let mut next_token = {