File size: 10,358 Bytes
0f8ec45
 
 
 
 
2d30d63
0f8ec45
1fd4564
5683e67
e4cb9b4
0f8ec45
c9e95c1
d50653b
5683e67
5079df0
169bac6
c0b809f
 
 
a6abdc9
 
0f8ec45
ed2d2b6
0f8ec45
 
 
 
 
089c3bd
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2d30d63
ed2d2b6
089c3bd
 
ac08ca7
089c3bd
 
0f8ec45
1a36e0e
928f3b9
cad0ecd
 
1a36e0e
4ded2e3
1a36e0e
ed2d2b6
519d843
3142fb1
 
 
bc20327
32b570e
57237e8
0f8ec45
a6abdc9
0f8ec45
 
909646e
1a36e0e
928f3b9
0f8ec45
afeabee
e8a7086
dd6d711
a6abdc9
afeabee
 
909646e
25be712
d035873
7921b80
d035873
7921b80
a26a344
 
e00c12d
d035873
 
 
afeabee
dd6d711
a6abdc9
dd6d711
a1bd179
 
 
 
 
a6abdc9
a1bd179
1a36e0e
a1bd179
1a36e0e
cad0ecd
 
a1bd179
c0b809f
909646e
c0b809f
5079df0
a6abdc9
 
1a36e0e
5079df0
cad0ecd
c0b809f
 
 
 
cad0ecd
1f46b21
0f8ec45
6027158
ed2d2b6
2656341
0f8ec45
 
 
50098a7
 
 
 
 
 
 
 
 
 
0a795c5
bf4a496
 
 
 
0f8ec45
 
 
a26a344
0f8ec45
 
 
 
 
8f1a540
cf80990
8f1a540
 
cf80990
c1d1b49
1332b31
8f1a540
 
 
cf80990
1332b31
 
8f1a540
 
 
cf80990
1332b31
 
8f1a540
 
 
2d30d63
bf4a496
8f1a540
 
 
1332b31
8f1a540
 
 
 
928f3b9
0f8ec45
 
909646e
 
a6abdc9
 
9061f6a
314fe06
1a36e0e
0f8ec45
e8a7086
0f8ec45
 
a6abdc9
e8e4ed0
57237e8
a6abdc9
 
909646e
a6abdc9
 
519d843
909646e
a6abdc9
 
 
 
 
 
 
 
 
 
 
 
0f8ec45
57237e8
e8e4ed0
8f1a540
cf80990
8f1a540
 
 
cd04efe
8f1a540
 
57237e8
5079df0
0f8ec45
5079df0
6aef7b0
 
2d30d63
 
 
 
0f8ec45
 
 
 
909646e
1a36e0e
0f8ec45
 
 
e91ae6b
 
a26a344
a6abdc9
0f8ec45
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
from __future__ import annotations
import math
import random
import spaces
import gradio as gr
import numpy as np
import torch
from PIL import Image
from diffusers import StableDiffusionXLImg2ImgPipeline, StableDiffusionXLPipeline, EDMEulerScheduler, StableDiffusionXLInstructPix2PixPipeline, AutoencoderKL, EulerAncestralDiscreteScheduler
from huggingface_hub import hf_hub_download, InferenceClient

vae = AutoencoderKL.from_pretrained("madebyollin/sdxl-vae-fp16-fix", torch_dtype=torch.float16)

refiner = StableDiffusionXLImg2ImgPipeline.from_pretrained("stabilityai/stable-diffusion-xl-refiner-1.0", vae=vae, torch_dtype=torch.float16, use_safetensors=True, variant="fp16")
refiner.to("cuda")

pipe_fast = StableDiffusionXLPipeline.from_pretrained("SG161222/RealVisXL_V4.0_Lightning", torch_dtype=torch.float16, vae=vae)
pipe_fast.load_lora_weights("KingNish/Better-Image-XL-Lora", weight_name="example-03.safetensors", adapter_name="lora")
pipe_fast.set_adapters("lora")
pipe_fast.to("cuda")

help_text = """
To optimize image results:
- Adjust the **Image CFG weight** if the image isn't changing enough or is changing too much. Lower it to allow bigger changes, or raise it to preserve original details.
- Modify the **Text CFG weight** to influence how closely the edit follows text instructions. Increase it to adhere more to the text, or decrease it for subtler changes.
- Experiment with different **random seeds** and **CFG values** for varied outcomes.
- **Rephrase your instructions** for potentially better results.
- **Increase the number of steps** for enhanced edits.
"""

def set_timesteps_patched(self, num_inference_steps: int, device = None):
    self.num_inference_steps = num_inference_steps
    
    ramp = np.linspace(0, 1, self.num_inference_steps)
    sigmas = torch.linspace(math.log(self.config.sigma_min), math.log(self.config.sigma_max), len(ramp)).exp().flip(0)
    
    sigmas = (sigmas).to(dtype=torch.float32, device=device)
    self.timesteps = self.precondition_noise(sigmas)
    
    self.sigmas = torch.cat([sigmas, torch.zeros(1, device=sigmas.device)])
    self._step_index = None
    self._begin_index = None
    self.sigmas = self.sigmas.to("cpu") 

# Image Editor
edit_file = hf_hub_download(repo_id="stabilityai/cosxl", filename="cosxl_edit.safetensors")
EDMEulerScheduler.set_timesteps = set_timesteps_patched
pipe_edit = StableDiffusionXLInstructPix2PixPipeline.from_single_file( edit_file, num_in_channels=8, is_cosxl_edit=True, vae=vae, torch_dtype=torch.float16 )
pipe_edit.scheduler = EDMEulerScheduler(sigma_min=0.002, sigma_max=120.0, sigma_data=1.0, prediction_type="v_prediction")
pipe_edit.to("cuda")

def promptifier(prompt):
    client1 = InferenceClient("mistralai/Mistral-7B-Instruct-v0.3")
    system_instructions1 = "<s>[SYSTEM] Your task is to modify prompt by USER to more better prompt for Image Generation in Stable Diffusion XL, you have to optiomize prompt and also add some keywords like, 4k, realistic, featuristic according to prompt and also break prompt into sub-lines using comma, Your task is to reply with final optimized prompt only. Just reply with prompt only.[USER]"
    formatted_prompt = f"{system_instructions1} {prompt} [PROMPT]"
    stream = client1.text_generation(formatted_prompt, max_new_tokens=80, stream=True, details=True, return_full_text=False)
    return "".join([response.token.text for response in stream if response.token.text != "</s>"])

# Generator
@spaces.GPU(duration=60, queue=False)
def king(type ,
        input_image ,
        instruction: str ,
        negative_prompt: str ="",
        enhance_prompt: bool = True,
        steps: int = 25,
        randomize_seed: bool = False,
        seed: int = 2404,
        width: int = 1024,
        height: int = 1024,
        guidance_scale: float = 6,
        fast=False,
        progress=gr.Progress(track_tqdm=True)
    ):
    if type=="Image Editing" :
        raw_image = Image.open(input_image).convert('RGB')
        if randomize_seed:
            seed = random.randint(0, 999999)
        generator = torch.manual_seed(seed)
        output_image = pipe_edit(
            instruction, negative_prompt=negative_prompt, image=raw_image,
            guidance_scale=guidance_scale, image_guidance_scale=1.5,
            num_inference_steps=steps, generator=generator, output_type="latent",
        ).images
        refine = refiner(
            prompt=instruction,
            guidance_scale=guidance_scale,
            num_inference_steps=steps,
            image=output_image,
            generator=generator,
        ).images[0]  
        return seed, refine
    else :
        if randomize_seed:
            seed = random.randint(0, 999999)
        generator = torch.Generator().manual_seed(seed)
        if enhance_prompt:
            print(f"BEFORE: {instruction} ")
            instruction = promptifier(instruction)
            print(f"AFTER: {instruction} ")
        guidance_scale2=(guidance_scale/2)
        if fast:
            image = pipe_fast(prompt = instruction,
            guidance_scale = guidance_scale2, 
            num_inference_steps = int(steps/2.5),
            width = width, height = height,
            generator = generator, output_type="latent",
            ).images
        else:            
            image = pipe_fast( prompt = instruction,
            negative_prompt=negative_prompt,
            guidance_scale = guidance_scale2, 
            num_inference_steps = steps, 
            width = width, height = height,
            generator = generator, output_type="latent",
            ).images 

        refine = refiner( prompt=instruction,
                negative_prompt = negative_prompt,
                guidance_scale = guidance_scale,
                num_inference_steps=  steps,
                image=image, generator=generator,
        ).images[0]        
        return seed, refine

client = InferenceClient()
# Prompt classifier
def response(instruction, input_image=None ):
    if input_image is None:
        output="Image Generation"
    else:
        try:
            text = instruction
            labels = ["Image Editing", "Image Generation"]
            classification = client.zero_shot_classification(text, labels, multi_label=True)
            output = classification[0]
            output = str(output)
            if "Editing" in output:
                output = "Image Editing"
            else:
                output = "Image Generation"
        except:
            if input_image is None:
                output="Image Generation"
            else:
                output="Image Editing"
    return output

css = '''
.gradio-container{max-width: 700px !important}
h1{text-align:center}
footer {
    visibility: hidden
}
'''

examples=[
        [
            "Image Generation",
            None,
            "A luxurious supercar with a unique design. The car should have a pearl white finish, and gold accents. 4k, realistic.",

        ],
        [
            "Image Editing",
            "./supercar.png",
            "make it red",

        ],
        [
            "Image Editing",
            "./red_car.png",
            "add some snow",

        ],
        [
            "Image Generation",
            None,
            "An alien grasping a sign board contain word 'ALIEN' with Neon Glow, neon, futuristic, neonpunk, neon lights",
        ],
        [
            "Image Generation",
            None,
            "Beautiful Eiffel Tower at Night",
        ],
    ]

with gr.Blocks(css=css) as demo:
    gr.Markdown("# Image Generator Pro")
    with gr.Row():
        instruction = gr.Textbox(lines=1, label="Instruction", interactive=True)
        generate_button = gr.Button("Run", scale=0)
    with gr.Row():
        type = gr.Dropdown(["Image Generation","Image Editing"], label="Task", value="Image Generation",interactive=True)
        enhance_prompt = gr.Checkbox(label="Enhance prompt", value=False, scale=0)
        fast = gr.Checkbox(label="FAST Generation", value=True, scale=0)
        
    with gr.Row():
        input_image = gr.Image(label="Image", type='filepath', interactive=True)

    with gr.Row():
        guidance_scale = gr.Number(value=6.0, step=0.1, label="Guidance Scale", interactive=True)
        steps = gr.Number(value=25, step=1, label="Steps", interactive=True)

    with gr.Accordion("Advanced options", open=False):
        with gr.Row():
            negative_prompt = gr.Text(
                    label="Negative prompt",
                    max_lines=1,
                    value="(deformed, distorted, disfigured:1.3), poorly drawn, bad anatomy, wrong anatomy, extra limb, missing limb, floating limbs, (mutated hands and fingers:1.4), disconnected limbs, mutation, mutated, ugly, disgusting, blurry, amputation",
                    visible=True)
        with gr.Row():
            width =  gr.Slider( label="Width", minimum=256, maximum=2048, step=64, value=1024)
            height =  gr.Slider( label="Height", minimum=256, maximum=2048, step=64, value=1024)
        with gr.Row():
            randomize_seed = gr.Radio(
                    ["Fix Seed", "Randomize Seed"],
                    value="Randomize Seed",
                    type="index",
                    show_label=False,
                    interactive=True,
                )
            seed = gr.Number(value=1371, step=1, label="Seed", interactive=True)



    gr.Examples(
        examples=examples,
        inputs=[type,input_image, instruction],
        fn=king,
        outputs=[input_image],
        cache_examples=False,
    )

    # gr.Markdown(help_text)

    instruction.change(fn=response, inputs=[instruction,input_image], outputs=type, queue=False)

    input_image.upload(fn=response, inputs=[instruction,input_image], outputs=type, queue=False)
    
    gr.on(triggers=[
            generate_button.click,
            instruction.submit
        ],
            fn=king,
            inputs=[type,
                input_image,
                instruction,
                negative_prompt,
                enhance_prompt,
                steps,
                randomize_seed,
                seed,
                width,
                height,
                guidance_scale,
                fast,
            ],
            outputs=[seed, input_image],
        )

demo.queue(max_size=99999).launch()