From d0f9e14039dadd5cdc8928db61d9a54230d08ef5 Mon Sep 17 00:00:00 2001 From: ailab Date: Fri, 7 Jun 2024 15:50:28 +0800 Subject: [PATCH] first commit --- .gitattributes | 41 +++++++++ README.md | 106 +++++++++++++++++++++++ config.json | 57 ++++++++++++ diffusion_pytorch_model.bin | 3 + diffusion_pytorch_model.fp16.bin | 3 + diffusion_pytorch_model.fp16.safetensors | 3 + diffusion_pytorch_model.safetensors | 3 + out_bird.png | 3 + out_couple.png | 3 + out_hug_lab_7.png | 3 + out_room.png | 3 + out_tornado.png | 3 + out_women.png | 3 + 13 files changed, 234 insertions(+) create mode 100644 .gitattributes create mode 100644 README.md create mode 100644 config.json create mode 100644 diffusion_pytorch_model.bin create mode 100644 diffusion_pytorch_model.fp16.bin create mode 100644 diffusion_pytorch_model.fp16.safetensors create mode 100644 diffusion_pytorch_model.safetensors create mode 100644 out_bird.png create mode 100644 out_couple.png create mode 100644 out_hug_lab_7.png create mode 100644 out_room.png create mode 100644 out_tornado.png create mode 100644 out_women.png diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..9b9898b --- /dev/null +++ b/.gitattributes @@ -0,0 +1,41 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +out_bird.png filter=lfs diff=lfs merge=lfs -text +out_couple.png filter=lfs diff=lfs merge=lfs -text +out_room.png filter=lfs diff=lfs merge=lfs -text +out_tornado.png filter=lfs diff=lfs merge=lfs -text +out_women.png filter=lfs diff=lfs merge=lfs -text +out_hug_lab_7.png filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md new file mode 100644 index 0000000..e0d3061 --- /dev/null +++ b/README.md @@ -0,0 +1,106 @@ +--- +license: openrail++ +base_model: runwayml/stable-diffusion-v1-5 +tags: +- stable-diffusion-xl +- stable-diffusion-xl-diffusers +- text-to-image +- diffusers +inference: false +--- + +# SDXL-controlnet: Canny + +These are controlnet weights trained on [stabilityai/stable-diffusion-xl-base-1.0](https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0) with canny conditioning. You can find some example images in the following. + +prompt: a couple watching a romantic sunset, 4k photo +![images_0)](./out_couple.png) + +prompt: ultrarealistic shot of a furry blue bird +![images_1)](./out_bird.png) + +prompt: a woman, close up, detailed, beautiful, street photography, photorealistic, detailed, Kodak ektar 100, natural, candid shot +![images_2)](./out_women.png) + +prompt: Cinematic, neoclassical table in the living room, cinematic, contour, lighting, highly detailed, winter, golden hour +![images_3)](./out_room.png) + +prompt: a tornado hitting grass field, 1980's film grain. overcast, muted colors. +![images_0)](./out_tornado.png) + +## Usage + +Make sure to first install the libraries: + +```bash +pip install accelerate transformers safetensors opencv-python diffusers +``` + +And then we're ready to go: + +```python +from diffusers import ControlNetModel, StableDiffusionXLControlNetPipeline, AutoencoderKL +from diffusers.utils import load_image +from PIL import Image +import torch +import numpy as np +import cv2 + +prompt = "aerial view, a futuristic research complex in a bright foggy jungle, hard lighting" +negative_prompt = 'low quality, bad quality, sketches' + +image = load_image("https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/sd_controlnet/hf-logo.png") + +controlnet_conditioning_scale = 0.5 # recommended for good generalization + +controlnet = ControlNetModel.from_pretrained( + "diffusers/controlnet-canny-sdxl-1.0", + torch_dtype=torch.float16 +) +vae = AutoencoderKL.from_pretrained("madebyollin/sdxl-vae-fp16-fix", torch_dtype=torch.float16) +pipe = StableDiffusionXLControlNetPipeline.from_pretrained( + "stabilityai/stable-diffusion-xl-base-1.0", + controlnet=controlnet, + vae=vae, + torch_dtype=torch.float16, +) +pipe.enable_model_cpu_offload() + +image = np.array(image) +image = cv2.Canny(image, 100, 200) +image = image[:, :, None] +image = np.concatenate([image, image, image], axis=2) +image = Image.fromarray(image) + +images = pipe( + prompt, negative_prompt=negative_prompt, image=image, controlnet_conditioning_scale=controlnet_conditioning_scale, + ).images + +images[0].save(f"hug_lab.png") +``` + +![images_10)](./out_hug_lab_7.png) + +To more details, check out the official documentation of [`StableDiffusionXLControlNetPipeline`](https://huggingface.co/docs/diffusers/main/en/api/pipelines/controlnet_sdxl). + +### Training + +Our training script was built on top of the official training script that we provide [here](https://github.com/huggingface/diffusers/blob/main/examples/controlnet/README_sdxl.md). + +#### Training data +This checkpoint was first trained for 20,000 steps on laion 6a resized to a max minimum dimension of 384. +It was then further trained for 20,000 steps on laion 6a resized to a max minimum dimension of 1024 and +then filtered to contain only minimum 1024 images. We found the further high resolution finetuning was +necessary for image quality. + +#### Compute +one 8xA100 machine + +#### Batch size +Data parallel with a single gpu batch size of 8 for a total batch size of 64. + +#### Hyper Parameters +Constant learning rate of 1e-4 scaled by batch size for total learning rate of 64e-4 + +#### Mixed precision +fp16 \ No newline at end of file diff --git a/config.json b/config.json new file mode 100644 index 0000000..c38f38a --- /dev/null +++ b/config.json @@ -0,0 +1,57 @@ +{ + "_class_name": "ControlNetModel", + "_diffusers_version": "0.20.0.dev0", + "_name_or_path": "../controlnet-1-0-canny/checkpoint-20000/controlnet", + "act_fn": "silu", + "addition_embed_type": "text_time", + "addition_embed_type_num_heads": 64, + "addition_time_embed_dim": 256, + "attention_head_dim": [ + 5, + 10, + 20 + ], + "block_out_channels": [ + 320, + 640, + 1280 + ], + "class_embed_type": null, + "conditioning_channels": 3, + "conditioning_embedding_out_channels": [ + 16, + 32, + 96, + 256 + ], + "controlnet_conditioning_channel_order": "rgb", + "cross_attention_dim": 2048, + "down_block_types": [ + "DownBlock2D", + "CrossAttnDownBlock2D", + "CrossAttnDownBlock2D" + ], + "downsample_padding": 1, + "encoder_hid_dim": null, + "encoder_hid_dim_type": null, + "flip_sin_to_cos": true, + "freq_shift": 0, + "global_pool_conditions": false, + "in_channels": 4, + "layers_per_block": 2, + "mid_block_scale_factor": 1, + "norm_eps": 1e-05, + "norm_num_groups": 32, + "num_attention_heads": null, + "num_class_embeds": null, + "only_cross_attention": false, + "projection_class_embeddings_input_dim": 2816, + "resnet_time_scale_shift": "default", + "transformer_layers_per_block": [ + 1, + 2, + 10 + ], + "upcast_attention": null, + "use_linear_projection": true +} diff --git a/diffusion_pytorch_model.bin b/diffusion_pytorch_model.bin new file mode 100644 index 0000000..541b064 --- /dev/null +++ b/diffusion_pytorch_model.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:982e12f72cb41031c060de08a5d78ff6912e9b02c6ad91fc480f05a72cad10cb +size 5004438321 diff --git a/diffusion_pytorch_model.fp16.bin b/diffusion_pytorch_model.fp16.bin new file mode 100644 index 0000000..8518d03 --- /dev/null +++ b/diffusion_pytorch_model.fp16.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a42da57d6e2fd6ec786ccfea1cf1a06d2c1d91b2d8a14c7de3a67553b10b2948 +size 2502401039 diff --git a/diffusion_pytorch_model.fp16.safetensors b/diffusion_pytorch_model.fp16.safetensors new file mode 100644 index 0000000..264c8b7 --- /dev/null +++ b/diffusion_pytorch_model.fp16.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b2e7d3921058a442cc80430d1ec8847f42599c705e2451c95e77cf4dcf8d6c25 +size 2502139136 diff --git a/diffusion_pytorch_model.safetensors b/diffusion_pytorch_model.safetensors new file mode 100644 index 0000000..e8ac2fb --- /dev/null +++ b/diffusion_pytorch_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ea99040544a999f814fd854575a3aee069a005d026864c8d321b82576706a221 +size 5004167864 diff --git a/out_bird.png b/out_bird.png new file mode 100644 index 0000000..947f423 --- /dev/null +++ b/out_bird.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:596aecc5291798f1f25b665a92110627f7661d13d4eef3ae038b175de9db93c2 +size 6935069 diff --git a/out_couple.png b/out_couple.png new file mode 100644 index 0000000..01979e8 --- /dev/null +++ b/out_couple.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3613a6c3119ccc39d687ed917c8b364f05c17e16c1d9cf0c36816ef07da80868 +size 7286999 diff --git a/out_hug_lab_7.png b/out_hug_lab_7.png new file mode 100644 index 0000000..a8643c0 --- /dev/null +++ b/out_hug_lab_7.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:33d7a1f77d34f565df9910bf8a3276817cb21fa7f4025174f93f1f3517b2a4f1 +size 1967716 diff --git a/out_room.png b/out_room.png new file mode 100644 index 0000000..4014b8d --- /dev/null +++ b/out_room.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fce39ad0ca4c081aecd90a2c16118bc38d89ac27c7ca25d59a80d470b1c4ed31 +size 6420362 diff --git a/out_tornado.png b/out_tornado.png new file mode 100644 index 0000000..8fca3e2 --- /dev/null +++ b/out_tornado.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b2ecd942a61e5ee4f8b5d1c8d108232a0e3a012036eaa4b8865ebfd0b7e15346 +size 8511428 diff --git a/out_women.png b/out_women.png new file mode 100644 index 0000000..7823f48 --- /dev/null +++ b/out_women.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ada3827272bdd1599372d5d8842a173e054bc1164097c4e347c4bcd7cfcd3c7c +size 7586978