Update app.py

app.py

@@ -1,36 +1,26 @@
 import gradio as gr
-from transformers import AutoModelForCausalLM, AutoTokenizer
-import torch
+from ctransformers import AutoModelForCausalLM
 
-# Model name
-MODEL_NAME = "Qwen/Qwen2.5-0.5B-Instruct-GGUF"
+# Model path or name
+MODEL_PATH = "Qwen/Qwen2.5-0.5B-Instruct-GGUF"
 
-# Load tokenizer and model
-tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME)
-model = AutoModelForCausalLM.from_pretrained(MODEL_NAME, torch_dtype=torch.float16, device_map="auto")
+# Load the model
+model = AutoModelForCausalLM.from_pretrained(MODEL_PATH, model_type="qwen", device="cpu")
 
-def respond(message, history: list[tuple[str, str]], system_message, max_tokens, temperature, top_p):
-    messages = [{"role": "system", "content": system_message}]
-    
-    # Add chat history to messages
+def respond(message, history, system_message, max_tokens, temperature, top_p):
+    # Prepare the prompt with system message and history
+    prompt = system_message + "\n"
     for user_msg, assistant_msg in history:
-        if user_msg:
-            messages.append({"role": "user", "content": user_msg})
-        if assistant_msg:
-            messages.append({"role": "assistant", "content": assistant_msg})
-    
-    messages.append({"role": "user", "content": message})
-    
-    # Tokenize input
-    inputs = tokenizer(message, return_tensors="pt").to("cpu")
-    
+        prompt += f"User: {user_msg}\nAssistant: {assistant_msg}\n"
+    prompt += f"User: {message}\nAssistant:"
+
     # Generate response
-    with torch.no_grad():
-        outputs = model.generate(
-            **inputs, max_length=max_tokens, temperature=temperature, top_p=top_p
-        )
-    
-    response = tokenizer.decode(outputs[0], skip_special_tokens=True)
+    response = model(
+        prompt,
+        max_new_tokens=max_tokens,
+        temperature=temperature,
+        top_p=top_p,
+    )
     return response
 
 # Define Gradio interface