ContextQMD
Back to Peft

So that we can populate it later.

examples/multi_adapter_examples/multi_adapter_weighted_inference_diffusers.ipynb

0.19.14.7 KB
Original Source

This notebook shows how to use the adapter merging methods from peft and apply them image generation models using diffusers.

Turn diffusers LoRA checkpoints into PeftModel

python
!pip install diffusers accelerate transformers -U -q
!pip install git+https://github.com/huggingface/peft -q
python
from google.colab import userdata
TOKEN = userdata.get("HF_TOKEN")
python
from diffusers import UNet2DConditionModel
import torch

device = torch.accelerator.current_accelerator().type if hasattr(torch, "accelerator") else "cuda"

model_id = "stabilityai/stable-diffusion-xl-base-1.0"
unet = UNet2DConditionModel.from_pretrained(
    model_id, subfolder="unet", torch_dtype=torch.float16, use_safetensors=True, variant="fp16"
).to(device)
python
# So that we can populate it later.
import copy

sdxl_unet = copy.deepcopy(unet)
python
# Load the pipeline too.
from diffusers import DiffusionPipeline

pipe = DiffusionPipeline.from_pretrained(
    model_id, variant="fp16", torch_dtype=torch.float16, unet=unet
).to(device)
python
# Only UNet
pipe.load_lora_weights("CiroN2022/toy-face", weight_name="toy_face_sdxl.safetensors", adapter_name="toy")
python
from peft import get_peft_model, LoraConfig

toy_peft_model = get_peft_model(
    sdxl_unet,
    pipe.unet.peft_config["toy"],
    adapter_name="toy"
)
python
original_state_dict = {f"base_model.model.{k}": v for k, v in pipe.unet.state_dict().items()}

toy_peft_model.load_state_dict(original_state_dict, strict=True)
toy_peft_model.push_to_hub("toy_peft_model-new", token=TOKEN)
python
pipe.delete_adapters("toy")
sdxl_unet.delete_adapters("toy")
python
pipe.load_lora_weights("nerijs/pixel-art-xl", weight_name="pixel-art-xl.safetensors", adapter_name="pixel")
pipe.set_adapters(adapter_names="pixel")
python
pixel_peft_model = get_peft_model(
    sdxl_unet,
    pipe.unet.peft_config["pixel"],
    adapter_name="pixel"
)

original_state_dict = {f"base_model.model.{k}": v for k, v in pipe.unet.state_dict().items()}
pixel_peft_model.load_state_dict(original_state_dict, strict=True)
pixel_peft_model.push_to_hub("pixel_peft_model-new", token=TOKEN)
python
del pipe, sdxl_unet, toy_peft_model, pixel_peft_model

Weighted adapter inference

python
from peft import PeftModel

base_unet = UNet2DConditionModel.from_pretrained(
    model_id, subfolder="unet", torch_dtype=torch.float16, use_safetensors=True, variant="fp16"
).to(device)

toy_id = "sayakpaul/toy_peft_model-new"
model = PeftModel.from_pretrained(base_unet, toy_id, use_safetensors=True, subfolder="toy", adapter_name="toy")
model.load_adapter("sayakpaul/pixel_peft_model-new", use_safetensors=True, subfolder="pixel", adapter_name="pixel")

# https://huggingface.co/docs/peft/main/en/package_reference/lora#peft.LoraModel.add_weighted_adapter
model.add_weighted_adapter(
    adapters=["toy", "pixel"],
    weights=[0.7, 0.3],
    combination_type="linear",
    adapter_name="toy-pixel"
)
model.set_adapters("toy-pixel")
python
type(model.base_model.model)
python
model = model.to(torch_dtype=torch.float16, device=device)

pipe = DiffusionPipeline.from_pretrained(
    model_id, unet=model, variant="fp16", torch_dtype=torch.float16,
).to(device)

prompt = "toy_face of a hacker with a hoodie, pixel art"
image = pipe(prompt, num_inference_steps=30, generator=torch.manual_seed(0)).images[0]
image
python
del pipe
python
base_unet = UNet2DConditionModel.from_pretrained(
    model_id, subfolder="unet", torch_dtype=torch.float16, use_safetensors=True, variant="fp16"
).to(device)

toy_id = "sayakpaul/toy_peft_model-new"
model = PeftModel.from_pretrained(base_unet, toy_id, use_safetensors=True, subfolder="toy", adapter_name="toy")
model.load_adapter("sayakpaul/pixel_peft_model-new", use_safetensors=True, subfolder="pixel", adapter_name="pixel")

# https://huggingface.co/docs/peft/main/en/package_reference/lora#peft.LoraModel.add_weighted_adapter
model.add_weighted_adapter(
    adapters=["toy", "pixel"],
    weights=[0.5, 0.5],
    combination_type="cat",
    adapter_name="toy-pixel"
)
model.set_adapters("toy-pixel")
python
model = model.to(torch_dtype=torch.float16, device=device)

pipe = DiffusionPipeline.from_pretrained(
   model_id, unet=model, variant="fp16", torch_dtype=torch.float16,
).to(device)

prompt = "toy_face of a hacker with a hoodie, pixel art"
image = pipe(prompt, num_inference_steps=30, generator=torch.manual_seed(0)).images[0]
image
python
del pipe

pipe = DiffusionPipeline.from_pretrained(
    model_id, variant="fp16", dtype=torch.float16,
).to(device)

prompt = "toy_face of a hacker with a hoodie, pixel art"
image = pipe(prompt, num_inference_steps=30, generator=torch.manual_seed(0)).images[0]
image