zR commited on
Commit
0198f90
1 Parent(s): ed6f29c
Files changed (2) hide show
  1. app.py +54 -39
  2. rife_model.py +6 -0
app.py CHANGED
@@ -6,7 +6,7 @@ import time
6
 
7
  import gradio as gr
8
  import torch
9
- from diffusers import CogVideoXPipeline, CogVideoXDDIMScheduler
10
  from datetime import datetime, timedelta
11
 
12
  from diffusers.image_processor import VaeImageProcessor
@@ -22,7 +22,7 @@ hf_hub_download(repo_id="ai-forever/Real-ESRGAN", filename="RealESRGAN_x4.pth",
22
  snapshot_download(repo_id="AlexWortega/RIFE", local_dir="model_rife")
23
 
24
  pipe = CogVideoXPipeline.from_pretrained("THUDM/CogVideoX-5b", torch_dtype=torch.bfloat16).to(device)
25
- pipe.scheduler = CogVideoXDDIMScheduler.from_config(pipe.scheduler.config, timestep_spacing="trailing")
26
 
27
  pipe.transformer.to(memory_format=torch.channels_last)
28
  pipe.transformer = torch.compile(pipe.transformer, mode="max-autotune", fullgraph=True)
@@ -111,6 +111,7 @@ def infer(
111
  num_videos_per_prompt=1,
112
  num_inference_steps=num_inference_steps,
113
  num_frames=49,
 
114
  output_type="pt",
115
  guidance_scale=guidance_scale,
116
  generator=torch.Generator(device="cpu").manual_seed(seed),
@@ -198,44 +199,58 @@ with gr.Blocks() as demo:
198
  seed_text = gr.Number(label="Seed Used for Video Generation", visible=False)
199
 
200
  gr.Markdown("""
201
- <table border="0" style="width: 100%; text-align: left; margin-top: 20px;">
202
- <div style="text-align: center; font-size: 32px; font-weight: bold; margin-bottom: 20px;">
203
- 🎥 Video Gallery
 
 
 
 
204
  </div>
205
- <div style="text-align: center; font-size: 18px; font-weight: bold; margin-bottom: 20px;">
206
- The following videos were generated purely by CogVideoX-5B without using super-resolution or frame interpolation techniques. 🌟
207
- <br>
208
- <span style="color: pink;">seed=42, num_inference_steps=50 and guidance_scale=6.0.</span>
 
209
  </div>
210
- <tr>
211
- <td style="width: 25%; vertical-align: top; font-size: 0.8em;">
212
- <p>A detailed wooden toy ship with intricately carved masts and sails is seen gliding smoothly over a plush, blue carpet that mimics the waves of the sea. The ship's hull is painted a rich brown, with tiny windows. The carpet, soft and textured, provides a perfect backdrop, resembling an oceanic expanse. Surrounding the ship are various other toys and children's items, hinting at a playful environment. The scene captures the innocence and imagination of childhood, with the toy ship's journey symbolizing endless adventures in a whimsical, indoor setting.</p>
213
- </td>
214
- <td style="width: 25%; vertical-align: top;">
215
- <video src="https://github.com/user-attachments/assets/ea3af39a-3160-4999-90ec-2f7863c5b0e9" width="100%" controls autoplay loop></video>
216
- </td>
217
- <td style="width: 25%; vertical-align: top; font-size: 0.8em;">
218
- <p>The camera follows behind a white vintage SUV with a black roof rack as it speeds up a steep dirt road surrounded by pine trees on a steep mountain slope, dust kicks up from its tires, the sunlight shines on the SUV as it speeds along the dirt road, casting a warm glow over the scene. The dirt road curves gently into the distance, with no other cars or vehicles in sight. The trees on either side of the road are redwoods, with patches of greenery scattered throughout. The car is seen from the rear following the curve with ease, making it seem as if it is on a rugged drive through the rugged terrain. The dirt road itself is surrounded by steep hills and mountains, with a clear blue sky above with wispy clouds.</p>
219
- </td>
220
- <td style="width: 25%; vertical-align: top;">
221
- <video src="https://github.com/user-attachments/assets/9de41efd-d4d1-4095-aeda-246dd834e91d" width="100%" controls autoplay loop></video>
222
- </td>
223
- </tr>
224
- <tr>
225
- <td style="width: 25%; vertical-align: top; font-size: 0.8em;">
226
- <p>A street artist, clad in a worn-out denim jacket and a colorful bandana, stands before a vast concrete wall in the heart, holding a can of spray paint, spray-painting a colorful bird on a mottled wall.</p>
227
- </td>
228
- <td style="width: 25%; vertical-align: top;">
229
- <video src="https://github.com/user-attachments/assets/941d6661-6a8d-4a1b-b912-59606f0b2841" width="100%" controls autoplay loop></video>
230
- </td>
231
- <td style="width: 25%; vertical-align: top; font-size: 0.8em;">
232
- <p>In the haunting backdrop of a war-torn city, where ruins and crumbled walls tell a story of devastation, a poignant close-up frames a young girl. Her face is smudged with ash, a silent testament to the chaos around her. Her eyes glistening with a mix of sorrow and resilience, capturing the raw emotion of a world that has lost its innocence to the ravages of conflict.</p>
233
- </td>
234
- <td style="width: 25%; vertical-align: top;">
235
- <video src="https://github.com/user-attachments/assets/938529c4-91ae-4f60-b96b-3c3947fa63cb" width="100%" controls autoplay loop></video>
236
- </td>
237
- </tr>
238
- </table>
 
 
 
 
 
 
 
 
 
239
  """)
240
 
241
 
@@ -244,7 +259,7 @@ with gr.Blocks() as demo:
244
  latents, seed = infer(
245
  prompt,
246
  num_inference_steps=50, # NOT Changed
247
- guidance_scale=6, # NOT Changed
248
  seed=seed_value,
249
  progress=progress,
250
  )
 
6
 
7
  import gradio as gr
8
  import torch
9
+ from diffusers import CogVideoXPipeline, CogVideoXDDIMScheduler,CogVideoXDPMScheduler
10
  from datetime import datetime, timedelta
11
 
12
  from diffusers.image_processor import VaeImageProcessor
 
22
  snapshot_download(repo_id="AlexWortega/RIFE", local_dir="model_rife")
23
 
24
  pipe = CogVideoXPipeline.from_pretrained("THUDM/CogVideoX-5b", torch_dtype=torch.bfloat16).to(device)
25
+ pipe.scheduler = CogVideoXDPMScheduler.from_config(pipe.scheduler.config, timestep_spacing="trailing")
26
 
27
  pipe.transformer.to(memory_format=torch.channels_last)
28
  pipe.transformer = torch.compile(pipe.transformer, mode="max-autotune", fullgraph=True)
 
111
  num_videos_per_prompt=1,
112
  num_inference_steps=num_inference_steps,
113
  num_frames=49,
114
+ use_dynamic_cfg=True,
115
  output_type="pt",
116
  guidance_scale=guidance_scale,
117
  generator=torch.Generator(device="cpu").manual_seed(seed),
 
199
  seed_text = gr.Number(label="Seed Used for Video Generation", visible=False)
200
 
201
  gr.Markdown("""
202
+ <body>
203
+ <div class="video-container">
204
+ <div class="video-item">
205
+ <video width="100%" controls>
206
+ <source src="https://github.com/user-attachments/assets/cf5953ea-96d3-48fd-9907-c4708752c714" type="video/mp4">
207
+ </video>
208
+ <div class="caption">A garden comes to life as a kaleidoscope of butterflies flutters amidst the blossoms, their delicate wings casting shadows on the petals below. In the background, a grand fountain cascades water with a gentle splendor, its rhythmic sound providing a soothing backdrop. Beneath the cool shade of a mature tree, a solitary wooden chair invites solitude and reflection, its smooth surface worn by the touch of countless visitors seeking a moment of tranquility in nature's embrace.</div>
209
  </div>
210
+ <div class="video-item">
211
+ <video width="100%" controls>
212
+ <source src="https://github.com/user-attachments/assets/fe0a78e6-b669-4800-8cf0-b5f9b5145b52" type="video/mp4">
213
+ </video>
214
+ <div class="caption">A small boy, head bowed and determination etched on his face, sprints through the torrential downpour as lightning crackles and thunder rumbles in the distance. The relentless rain pounds the ground, creating a chaotic dance of water droplets that mirror the dramatic sky's anger. In the far background, the silhouette of a cozy home beckons, a faint beacon of safety and warmth amidst the fierce weather. The scene is one of perseverance and the unyielding spirit of a child braving the elements.</div>
215
  </div>
216
+ <div class="video-item">
217
+ <video width="100%" controls>
218
+ <source src="https://github.com/user-attachments/assets/c182f606-8f8c-421d-b414-8487070fcfcb" type="video/mp4">
219
+ </video>
220
+ <div class="caption">A suited astronaut, with the red dust of Mars clinging to their boots, reaches out to shake hands with an alien being, their skin a shimmering blue, under the pink-tinged sky of the fourth planet. In the background, a sleek silver rocket, a beacon of human ingenuity, stands tall, its engines powered down, as the two representatives of different worlds exchange a historic greeting amidst the desolate beauty of the Martian landscape.</div>
221
+ </div>
222
+ <div class="video-item">
223
+ <video width="100%" controls>
224
+ <source src="https://github.com/user-attachments/assets/7db2bbce-194d-434d-a605-350254b6c298" type="video/mp4">
225
+ </video>
226
+ <div class="caption">An elderly gentleman, with a serene expression, sits at the water's edge, a steaming cup of tea by his side. He is engrossed in his artwork, brush in hand, as he renders an oil painting on a canvas that's propped up against a small, weathered table. The sea breeze whispers through his silver hair, gently billowing his loose-fitting white shirt, while the salty air adds an intangible element to his masterpiece in progress. The scene is one of tranquility and inspiration, with the artist's canvas capturing the vibrant hues of the setting sun reflecting off the tranquil sea.</div>
227
+ </div>
228
+ <div class="video-item">
229
+ <video width="100%" controls>
230
+ <source src="https://github.com/user-attachments/assets/62b01046-8cab-44cc-bd45-4d965bb615ec" type="video/mp4">
231
+ </video>
232
+ <div class="caption">In a dimly lit bar, purplish light bathes the face of a mature man, his eyes blinking thoughtfully as he ponders in close-up, the background artfully blurred to focus on his introspective expression, the ambiance of the bar a mere suggestion of shadows and soft lighting.</div>
233
+ </div>
234
+ <div class="video-item">
235
+ <video width="100%" controls>
236
+ <source src="https://github.com/user-attachments/assets/d78e552a-4b3f-4b81-ac3f-3898079554f6" type="video/mp4">
237
+ </video>
238
+ <div class="caption">A golden retriever, sporting sleek black sunglasses, with its lengthy fur flowing in the breeze, sprints playfully across a rooftop terrace, recently refreshed by a light rain. The scene unfolds from a distance, the dog's energetic bounds growing larger as it approaches the camera, its tail wagging with unrestrained joy, while droplets of water glisten on the concrete behind it. The overcast sky provides a dramatic backdrop, emphasizing the vibrant golden coat of the canine as it dashes towards the viewer.</div>
239
+ </div>
240
+ <div class="video-item">
241
+ <video width="100%" controls>
242
+ <source src="https://github.com/user-attachments/assets/30894f12-c741-44a2-9e6e-ddcacc231e5b" type="video/mp4">
243
+ </video>
244
+ <div class="caption">On a brilliant sunny day, the lakeshore is lined with an array of willow trees, their slender branches swaying gently in the soft breeze. The tranquil surface of the lake reflects the clear blue sky, while several elegant swans glide gracefully through the still water, leaving behind delicate ripples that disturb the mirror-like quality of the lake. The scene is one of serene beauty, with the willows' greenery providing a picturesque frame for the peaceful avian visitors.</div>
245
+ </div>
246
+ <div class="video-item">
247
+ <video width="100%" controls>
248
+ <source src="https://github.com/user-attachments/assets/926575ca-7150-435b-a0ff-4900a963297b" type="video/mp4">
249
+ </video>
250
+ <div class="caption">A Chinese mother, draped in a soft, pastel-colored robe, gently rocks back and forth in a cozy rocking chair positioned in the tranquil setting of a nursery. The dimly lit bedroom is adorned with whimsical mobiles dangling from the ceiling, casting shadows that dance on the walls. Her baby, swaddled in a delicate, patterned blanket, rests against her chest, the child's earlier cries now replaced by contented coos as the mother's soothing voice lulls the little one to sleep. The scent of lavender fills the air, adding to the serene atmosphere, while a warm, orange glow from a nearby nightlight illuminates the scene with a gentle hue, capturing a moment of tender love and comfort.</div>
251
+ </div>
252
+ </div>
253
+ </body>
254
  """)
255
 
256
 
 
259
  latents, seed = infer(
260
  prompt,
261
  num_inference_steps=50, # NOT Changed
262
+ guidance_scale=7, # NOT Changed
263
  seed=seed_value,
264
  progress=progress,
265
  )
rife_model.py CHANGED
@@ -79,6 +79,12 @@ def ssim_interpolation_rife(model, samples, exp=1, upscale_amount=1, output_devi
79
 
80
 
81
  def load_rife_model(model_path):
 
 
 
 
 
 
82
  model = Model()
83
  model.load_model(model_path, -1)
84
  model.eval()
 
79
 
80
 
81
  def load_rife_model(model_path):
82
+ torch.set_grad_enabled(False)
83
+ if torch.cuda.is_available():
84
+ torch.backends.cudnn.enabled = True
85
+ torch.backends.cudnn.benchmark = True
86
+ torch.set_default_tensor_type(torch.cuda.FloatTensor)
87
+
88
  model = Model()
89
  model.load_model(model_path, -1)
90
  model.eval()