Update app.py

app.py

@@ -1,5 +1,5 @@
 import torch
-from transformers import AutoTokenizer, AutoModelForCausalLM
+from transformers import AutoTokenizer, AutoModelForCausalLM, BitsAndBytesConfig
 import gradio as gr
 
 # Constants for the Model
@@ -10,11 +10,12 @@ MODEL_DESC = "A demo for the SeaLLMs-v3-7B-Chat language model."
 # Load the tokenizer
 tokenizer = AutoTokenizer.from_pretrained(MODEL_PATH)
 
-# Load the model with efficient settings
+# Load the model with efficient settings, using 8-bit quantization to reduce memory usage
+quantization_config = BitsAndBytesConfig(load_in_8bit=True)
 model = AutoModelForCausalLM.from_pretrained(
     MODEL_PATH,
     device_map="auto",
-    torch_dtype=torch.float32,  # Switching to float32 to avoid numerical instability
+    quantization_config=quantization_config,
     low_cpu_mem_usage=True
 )
 
@@ -31,7 +32,7 @@ def generate_response(prompt):
     try:
         outputs = model.generate(
             **inputs,
-            max_length=512,
+            max_length=256,  # Reduced max_length to lower memory usage
             num_return_sequences=1,
             no_repeat_ngram_size=2,
             early_stopping=True,