Update README.md

README.md

@@ -11,23 +11,18 @@ base_model: black-forest-labs/FLUX.1-dev
 ---
 
 
-# FLUX.1-dev Controlnet
-
-
+# FLUX.1-dev-Controlnet-Union
 
 <img src="./images/image_union.png" width = "1000" />
 
 
 ## Release
 
-- [2024/08/20] 🔥 Release the first beta version.
-Until the next Diffusers pypi release,
-please install Diffusers from source and use [this PR](https://github.com/huggingface/diffusers/pull/9175) to be able to use. 
-Before merging into the official main branch of diffusers, you can use this [diffusers_flux](https://github.com/instantX-research/diffusers_flux).
-
-- [2024/08/14] Release the alpha version.
+- [2024/08/26] 🔥 Release [FLUX.1-dev-ControlNet-Union-Pro](https://huggingface.co/Shakker-Labs/FLUX.1-dev-ControlNet-Union-Pro). Please install from [the source](https://github.com/huggingface/diffusers) before the next release. We have supported CN-Union and Multi-ControlNets via [this PR](https://github.com/huggingface/diffusers/pull/9175).
 
+- [2024/08/20] Release the beta version.
 
+- [2024/08/14] Release the alpha version.
 
 
 
@@ -56,63 +51,27 @@ However, as training progresses, the performance of the Union model will continu
 |6|lq|🟢high|
 
 
-
-
-# Demo
+# Inference
 ```python
 import torch
 from diffusers.utils import load_image
-from diffusers.pipelines.flux.pipeline_flux_controlnet import FluxControlNetPipeline
-from diffusers.models.controlnet_flux import FluxControlNetModel
+from diffusers import FluxControlNetPipeline, FluxControlNetModel
 
-# load
 base_model = 'black-forest-labs/FLUX.1-dev'
 controlnet_model = 'InstantX/FLUX.1-dev-Controlnet-Union'
+
 controlnet = FluxControlNetModel.from_pretrained(controlnet_model, torch_dtype=torch.bfloat16)
 pipe = FluxControlNetPipeline.from_pretrained(base_model, controlnet=controlnet, torch_dtype=torch.bfloat16)
 pipe.to("cuda")
 
-# image cfg
-width, height = 1024, 1024
+control_image_canny = load_image("https://huggingface.co/InstantX/FLUX.1-dev-Controlnet-Union-alpha/resolve/main/images/canny.jpg")
 controlnet_conditioning_scale = 0.5
-seed = 6666
-
-# canny
-control_image = load_image("https://huggingface.co/InstantX/FLUX.1-dev-Controlnet-Union-alpha/resolve/main/images/canny.jpg")
-prompt = "A girl in city, 25 years old, cool, futuristic."
 control_mode = 0
 
-# tile
-control_image = load_image("https://huggingface.co/InstantX/FLUX.1-dev-Controlnet-Union-alpha/resolve/main/images/tile.jpg")
-prompt = "A girl, 25 years old."
-control_mode = 1
-
-# depth
-control_image = load_image("https://huggingface.co/InstantX/FLUX.1-dev-Controlnet-Union-alpha/resolve/main/images/depth.jpg")
-prompt = "A girl in city, 25 years old, cool, futuristic."
-control_mode = 2
-
-# blur
-control_image = load_image("https://huggingface.co/InstantX/FLUX.1-dev-Controlnet-Union-alpha/resolve/main/images/blur.jpg")
-prompt = "A girl, 25 years old."
-control_mode = 3
-
-# pose
-control_image = load_image("https://huggingface.co/InstantX/FLUX.1-dev-Controlnet-Union-alpha/resolve/main/images/pose.jpg")
-prompt = "A girl in city, 25 years old, cool, futuristic."
-control_mode = 4
-
-# gray
-control_image = load_image("https://huggingface.co/InstantX/FLUX.1-dev-Controlnet-Union-alpha/resolve/main/images/gray.jpg")
-prompt = "A girl, 25 years old."
-control_mode = 5
-
-# low quality
-control_image = load_image("https://huggingface.co/InstantX/FLUX.1-dev-Controlnet-Union-alpha/resolve/main/images/lq.jpg")
-prompt = "A girl in city"
-control_mode = 6
-
-# go go go
+width, height = control_image.size
+
+prompt = 'A bohemian-style female travel blogger with sun-kissed skin and messy beach waves.'
+
 image = pipe(
     prompt, 
     control_image=control_image,
@@ -120,15 +79,55 @@ image = pipe(
     width=width,
     height=height,
     controlnet_conditioning_scale=controlnet_conditioning_scale,
-    num_inference_steps=28, 
+    num_inference_steps=24, 
     guidance_scale=3.5,
-    generator=torch.manual_seed(seed),
 ).images[0]
 image.save("image.jpg")
 ```
 
+# Multi-Controls Inference
+```python
+import torch
+from diffusers.utils import load_image
+from diffusers import FluxControlNetPipeline, FluxControlNetModel, FluxMultiControlNetModel
 
+base_model = 'black-forest-labs/FLUX.1-dev'
+controlnet_model_union = './InstantX/FLUX.1-dev-Controlnet-Union'
 
-# Acknowledgements
+controlnet_union = FluxControlNetModel.from_pretrained(controlnet_model_union, torch_dtype=torch.bfloat16)
+controlnet = FluxMultiControlNetModel([controlnet_union]) # we always recommend loading via FluxMultiControlNetModel
 
-Thank you, [zzzzzero](https://github.com/zzzzzero), for pointing out the bug in the model.
+pipe = FluxControlNetPipeline.from_pretrained(base_model, controlnet=controlnet, torch_dtype=torch.bfloat16)
+pipe.to("cuda")
+
+prompt = 'A bohemian-style female travel blogger with sun-kissed skin and messy beach waves.'
+control_image_depth = load_image("https://huggingface.co/InstantX/FLUX.1-dev-Controlnet-Union/resolve/main/images/depth.jpg")
+control_mode_depth = 2
+
+control_image_canny = load_image("https://huggingface.co/InstantX/FLUX.1-dev-Controlnet-Union/resolve/main/images/canny.jpg")
+control_mode_canny = 0
+
+width, height = control_image.size
+
+image = pipe(
+    prompt, 
+    control_image=[control_image_depth, control_image_canny],
+    control_mode=[control_mode_depth, control_mode_canny],
+    width=width,
+    height=height,
+    controlnet_conditioning_scale=[0.2, 0.4],
+    num_inference_steps=24, 
+    guidance_scale=3.5,
+    generator=torch.manual_seed(42),
+).images[0]
+
+```
+
+# Resources
+- [InstantX/FLUX.1-dev-Controlnet-Canny](https://huggingface.co/InstantX/FLUX.1-dev-Controlnet-Canny)
+- [InstantX/FLUX.1-dev-Controlnet-Union](https://huggingface.co/InstantX/FLUX.1-dev-Controlnet-Union)
+- [Shakker-Labs/FLUX.1-dev-ControlNet-Depth](https://huggingface.co/Shakker-Labs/FLUX.1-dev-ControlNet-Depth)
+- [Shakker-Labs/FLUX.1-dev-ControlNet-Union-Pro](https://huggingface.co/Shakker-Labs/FLUX.1-dev-ControlNet-Union-Pro)
+
+# Acknowledgements
+Thanks [zzzzzero](https://github.com/zzzzzero) for help us pointing out some bugs in the training.