hyper

README.md

@@ -1,5 +1,5 @@
 ---
-title: InstantStyle + Hyper SDXL
+title: InstantStyle + Hyper SD
 emoji: 🖼🎨⚡️
 colorFrom: blue
 colorTo: purple

app.py

@@ -5,15 +5,17 @@ import tempfile
 import numpy as np
 from pathlib import Path
 from PIL import Image
-from diffusers import ControlNetModel, StableDiffusionXLControlNetPipeline, TCDScheduler
+from diffusers import (
+    ControlNetModel,
+    StableDiffusionControlNetPipeline,
+    TCDScheduler,
+)
 import spaces
 import gradio as gr
 from huggingface_hub import hf_hub_download, snapshot_download
-from ip_adapter import IPAdapterXL
+from ip_adapter import IPAdapter
 
-snapshot_download(
-    repo_id="h94/IP-Adapter", allow_patterns="sdxl_models/*", local_dir="."
-)
+snapshot_download(repo_id="h94/IP-Adapter", allow_patterns="models/*", local_dir=".")
 
 # global variable
 MAX_SEED = np.iinfo(np.int32).max
@@ -21,48 +23,44 @@ device = "cuda" if torch.cuda.is_available() else "cpu"
 dtype = torch.float16 if str(device).__contains__("cuda") else torch.float32
 
 # initialization
-base_model_path = "stabilityai/stable-diffusion-xl-base-1.0"
-image_encoder_path = "sdxl_models/image_encoder"
-ip_ckpt = "sdxl_models/ip-adapter_sdxl.bin"
+base_model_path = "runwayml/stable-diffusion-v1-5"
+image_encoder_path = "models/image_encoder"
+ip_ckpt = "models/ip-adapter_sd15.bin"
 
-controlnet_path = "diffusers/controlnet-canny-sdxl-1.0"
+controlnet_path = "lllyasviel/control_v11p_sd15_canny"
 controlnet = ControlNetModel.from_pretrained(
     controlnet_path, use_safetensors=False, torch_dtype=torch.float16
 ).to(device)
 
 # load Hyper SD
 
-pipe = StableDiffusionXLControlNetPipeline.from_pretrained(
+pipe = StableDiffusionControlNetPipeline.from_pretrained(
     base_model_path,
     controlnet=controlnet,
     torch_dtype=torch.float16,
     variant="fp16",
-    add_watermarker=False,
 ).to(device)
 pipe.set_progress_bar_config(disable=True)
 pipe.scheduler = TCDScheduler.from_config(pipe.scheduler.config)
 pipe.load_lora_weights(
-    hf_hub_download("ByteDance/Hyper-SD", "Hyper-SDXL-1step-lora.safetensors")
+    hf_hub_download("ByteDance/Hyper-SD", "Hyper-SD15-1step-lora.safetensors")
 )
+pipe.enable_vae_tiling()
+
 eta = 1.0
 
-# load ip-adapter
-# target_blocks=["block"] for original IP-Adapter
-# target_blocks=["up_blocks.0.attentions.1"] for style blocks only
-# target_blocks = ["up_blocks.0.attentions.1", "down_blocks.2.attentions.1"] # for style+layout blocks
-ip_model = IPAdapterXL(
+ip_model = IPAdapter(
     pipe,
     image_encoder_path,
     ip_ckpt,
     device,
-    target_blocks=["up_blocks.0.attentions.1"],
 )
 
 
 def resize_img(
     input_image,
     max_side=1280,
-    min_side=1024,
+    min_side=512,
     size=None,
     pad_to_max_side=False,
     mode=Image.BILINEAR,
@@ -165,42 +163,44 @@ def create_image(
     seed = random.randint(0, MAX_SEED) if seed == -1 else seed
     if target == "Load original IP-Adapter":
         # target_blocks=["blocks"] for original IP-Adapter
-        ip_model = IPAdapterXL(
+        ip_model = IPAdapter(
             pipe, image_encoder_path, ip_ckpt, device, target_blocks=["blocks"]
         )
     elif target == "Load only style blocks":
         # target_blocks=["up_blocks.0.attentions.1"] for style blocks only
-        ip_model = IPAdapterXL(
+        ip_model = IPAdapter(
             pipe,
             image_encoder_path,
             ip_ckpt,
             device,
-            target_blocks=["up_blocks.0.attentions.1"],
+            target_blocks=["up_blocks.1"],
         )
     elif target == "Load style+layout block":
         # target_blocks = ["up_blocks.0.attentions.1", "down_blocks.2.attentions.1"] # for style+layout blocks
-        ip_model = IPAdapterXL(
+        ip_model = IPAdapter(
             pipe,
             image_encoder_path,
             ip_ckpt,
             device,
-            target_blocks=["up_blocks.0.attentions.1", "down_blocks.2.attentions.1"],
+            target_blocks=["down_blocks.2", "mid_block", "up_blocks.1"],
         )
 
     if input_image is not None:
-        input_image = resize_img(input_image, max_side=1024)
+        input_image = resize_img(input_image, max_side=512)
         cv_input_image = pil_to_cv2(input_image)
         detected_map = cv2.Canny(cv_input_image, 50, 200)
         canny_map = Image.fromarray(cv2.cvtColor(detected_map, cv2.COLOR_BGR2RGB))
     else:
-        canny_map = Image.new("RGB", (1024, 1024), color=(255, 255, 255))
+        canny_map = Image.new("RGB", (512, 512), color=(255, 255, 255))
         control_scale = 0
 
     if float(control_scale) == 0:
-        canny_map = canny_map.resize((1024, 1024))
+        canny_map = canny_map.resize((512, 512))
 
     if len(neg_content_prompt) > 0 and neg_content_scale != 0:
         images = ip_model.generate(
+            width=512,
+            height=512,
             pil_image=image_pil,
             prompt=prompt,
             negative_prompt=n_prompt,
@@ -217,6 +217,8 @@ def create_image(
         )
     else:
         images = ip_model.generate(
+            width=512,
+            height=512,
             pil_image=image_pil,
             prompt=prompt,
             negative_prompt=n_prompt,
@@ -340,7 +342,7 @@ with block:
                         minimum=1,
                         maximum=10.0,
                         step=1.0,
-                        value=1,
+                        value=3,
                         label="num inference steps",
                     )
                     seed = gr.Slider(