#[cfg(feature = "accelerate")] extern crate accelerate_src; #[cfg(feature = "mkl")] extern crate intel_mkl_src; mod attention; mod clip; mod ddim; mod embeddings; mod resnet; mod schedulers; mod stable_diffusion; mod unet_2d; mod unet_2d_blocks; mod utils; mod vae; use anyhow::{Error as E, Result}; use candle::{DType, Device, IndexOp, Tensor}; use clap::Parser; use tokenizers::Tokenizer; const GUIDANCE_SCALE: f64 = 7.5; #[derive(Parser)] #[command(author, version, about, long_about = None)] struct Args { /// The prompt to be used for image generation. #[arg( long, default_value = "A very realistic photo of a rusty robot walking on a sandy beach" )] prompt: String, #[arg(long, default_value = "")] uncond_prompt: String, /// Run on CPU rather than on GPU. #[arg(long)] cpu: bool, /// Enable tracing (generates a trace-timestamp.json file). #[arg(long)] tracing: bool, /// The height in pixels of the generated image. #[arg(long)] height: Option, /// The width in pixels of the generated image. #[arg(long)] width: Option, /// The UNet weight file, in .safetensors format. #[arg(long, value_name = "FILE")] unet_weights: Option, /// The CLIP weight file, in .safetensors format. #[arg(long, value_name = "FILE")] clip_weights: Option, /// The VAE weight file, in .safetensors format. #[arg(long, value_name = "FILE")] vae_weights: Option, #[arg(long, value_name = "FILE")] /// The file specifying the tokenizer to used for tokenization. tokenizer: Option, /// The size of the sliced attention or 0 for automatic slicing (disabled by default) #[arg(long)] sliced_attention_size: Option, /// The number of steps to run the diffusion for. #[arg(long, default_value_t = 30)] n_steps: usize, /// The number of samples to generate. #[arg(long, default_value_t = 1)] num_samples: i64, /// The name of the final image to generate. #[arg(long, value_name = "FILE", default_value = "sd_final.png")] final_image: String, #[arg(long, value_enum, default_value = "v2-1")] sd_version: StableDiffusionVersion, /// Generate intermediary images at each step. #[arg(long, action)] intermediary_images: bool, #[arg(long)] use_flash_attn: bool, #[arg(long)] use_f16: bool, } #[derive(Debug, Clone, Copy, clap::ValueEnum)] enum StableDiffusionVersion { V1_5, V2_1, } #[derive(Debug, Clone, Copy, PartialEq, Eq)] enum ModelFile { Tokenizer, Clip, Unet, Vae, } impl StableDiffusionVersion { fn repo(&self) -> &'static str { match self { Self::V2_1 => "stabilityai/stable-diffusion-2-1", Self::V1_5 => "runwayml/stable-diffusion-v1-5", } } fn unet_file(&self, use_f16: bool) -> &'static str { match self { Self::V1_5 | Self::V2_1 => { if use_f16 { "unet/diffusion_pytorch_model.fp16.safetensors" } else { "unet/diffusion_pytorch_model.safetensors" } } } } fn vae_file(&self, use_f16: bool) -> &'static str { match self { Self::V1_5 | Self::V2_1 => { if use_f16 { "vae/diffusion_pytorch_model.fp16.safetensors" } else { "vae/diffusion_pytorch_model.safetensors" } } } } fn clip_file(&self, use_f16: bool) -> &'static str { match self { Self::V1_5 | Self::V2_1 => { if use_f16 { "text_encoder/model.fp16.safetensors" } else { "text_encoder/model.safetensors" } } } } } impl ModelFile { const TOKENIZER_REPO: &str = "openai/clip-vit-base-patch32"; const TOKENIZER_PATH: &str = "tokenizer.json"; fn get( &self, filename: Option, version: StableDiffusionVersion, use_f16: bool, ) -> Result { use hf_hub::api::sync::Api; match filename { Some(filename) => Ok(std::path::PathBuf::from(filename)), None => { let (repo, path) = match self { Self::Tokenizer => (Self::TOKENIZER_REPO, Self::TOKENIZER_PATH), Self::Clip => (version.repo(), version.clip_file(use_f16)), Self::Unet => (version.repo(), version.unet_file(use_f16)), Self::Vae => (version.repo(), version.vae_file(use_f16)), }; let filename = Api::new()?.model(repo.to_string()).get(path)?; Ok(filename) } } } } fn output_filename( basename: &str, sample_idx: i64, num_samples: i64, timestep_idx: Option, ) -> String { let filename = if num_samples > 1 { match basename.rsplit_once('.') { None => format!("{basename}.{sample_idx}.png"), Some((filename_no_extension, extension)) => { format!("{filename_no_extension}.{sample_idx}.{extension}") } } } else { basename.to_string() }; match timestep_idx { None => filename, Some(timestep_idx) => match filename.rsplit_once('.') { None => format!("{filename}-{timestep_idx}.png"), Some((filename_no_extension, extension)) => { format!("{filename_no_extension}-{timestep_idx}.{extension}") } }, } } fn run(args: Args) -> Result<()> { use tracing_chrome::ChromeLayerBuilder; use tracing_subscriber::prelude::*; let Args { prompt, uncond_prompt, cpu, height, width, n_steps, tokenizer, final_image, sliced_attention_size, num_samples, sd_version, clip_weights, vae_weights, unet_weights, tracing, use_f16, use_flash_attn, .. } = args; let _guard = if tracing { let (chrome_layer, guard) = ChromeLayerBuilder::new().build(); tracing_subscriber::registry().with(chrome_layer).init(); Some(guard) } else { None }; let dtype = if use_f16 { DType::F16 } else { DType::F32 }; let sd_config = match sd_version { StableDiffusionVersion::V1_5 => { stable_diffusion::StableDiffusionConfig::v1_5(sliced_attention_size, height, width) } StableDiffusionVersion::V2_1 => { stable_diffusion::StableDiffusionConfig::v2_1(sliced_attention_size, height, width) } }; let scheduler = sd_config.build_scheduler(n_steps)?; let device = candle_examples::device(cpu)?; let tokenizer = ModelFile::Tokenizer.get(tokenizer, sd_version, use_f16)?; let tokenizer = Tokenizer::from_file(tokenizer).map_err(E::msg)?; let pad_id = match &sd_config.clip.pad_with { Some(padding) => *tokenizer.get_vocab(true).get(padding.as_str()).unwrap(), None => *tokenizer.get_vocab(true).get("<|endoftext|>").unwrap(), }; println!("Running with prompt \"{prompt}\"."); let mut tokens = tokenizer .encode(prompt, true) .map_err(E::msg)? .get_ids() .to_vec(); while tokens.len() < sd_config.clip.max_position_embeddings { tokens.push(pad_id) } let tokens = Tensor::new(tokens.as_slice(), &device)?.unsqueeze(0)?; let mut uncond_tokens = tokenizer .encode(uncond_prompt, true) .map_err(E::msg)? .get_ids() .to_vec(); while uncond_tokens.len() < sd_config.clip.max_position_embeddings { uncond_tokens.push(pad_id) } let uncond_tokens = Tensor::new(uncond_tokens.as_slice(), &device)?.unsqueeze(0)?; println!("Building the Clip transformer."); let text_embeddings = { let clip_weights = ModelFile::Clip.get(clip_weights, sd_version, false)?; let text_model = sd_config.build_clip_transformer(&clip_weights, &device, DType::F32)?; let text_embeddings = text_model.forward(&tokens)?; let uncond_embeddings = text_model.forward(&uncond_tokens)?; Tensor::cat(&[uncond_embeddings, text_embeddings], 0)?.to_dtype(dtype)? }; println!("Building the autoencoder."); let vae_weights = ModelFile::Vae.get(vae_weights, sd_version, use_f16)?; let vae = sd_config.build_vae(&vae_weights, &device, dtype)?; println!("Building the unet."); let unet_weights = ModelFile::Unet.get(unet_weights, sd_version, use_f16)?; let unet = sd_config.build_unet(&unet_weights, &device, 4, use_flash_attn, dtype)?; let bsize = 1; for idx in 0..num_samples { let mut latents = Tensor::randn( 0f32, 1f32, (bsize, 4, sd_config.height / 8, sd_config.width / 8), &device, )? .to_dtype(dtype)?; // scale the initial noise by the standard deviation required by the scheduler latents = (latents * scheduler.init_noise_sigma())?; println!("starting sampling"); for (timestep_index, ×tep) in scheduler.timesteps().iter().enumerate() { let start_time = std::time::Instant::now(); let latent_model_input = Tensor::cat(&[&latents, &latents], 0)?; let latent_model_input = scheduler.scale_model_input(latent_model_input, timestep)?; let noise_pred = unet.forward(&latent_model_input, timestep as f64, &text_embeddings)?; let noise_pred = noise_pred.chunk(2, 0)?; let (noise_pred_uncond, noise_pred_text) = (&noise_pred[0], &noise_pred[1]); let noise_pred = (noise_pred_uncond + ((noise_pred_text - noise_pred_uncond)? * GUIDANCE_SCALE)?)?; latents = scheduler.step(&noise_pred, timestep, &latents)?; let dt = start_time.elapsed().as_secs_f32(); println!("step {}/{n_steps} done, {:.2}s", timestep_index + 1, dt); if args.intermediary_images { let image = vae.decode(&(&latents / 0.18215)?)?; let image = ((image / 2.)? + 0.5)?.to_device(&Device::Cpu)?; let image = (image * 255.)?.to_dtype(DType::U8)?.i(0)?; let image_filename = output_filename(&final_image, idx + 1, num_samples, Some(timestep_index + 1)); candle_examples::save_image(&image, image_filename)? } } println!( "Generating the final image for sample {}/{}.", idx + 1, num_samples ); let image = vae.decode(&(&latents / 0.18215)?)?; // TODO: Add the clamping between 0 and 1. let image = ((image / 2.)? + 0.5)?.to_device(&Device::Cpu)?; let image = (image * 255.)?.to_dtype(DType::U8)?.i(0)?; let image_filename = output_filename(&final_image, idx + 1, num_samples, None); candle_examples::save_image(&image, image_filename)? } Ok(()) } fn main() -> Result<()> { let args = Args::parse(); run(args) }