req.txt

app.py

@@ -6,8 +6,7 @@ import json
 """
 For more information on `huggingface_hub` Inference API support, please check the docs: https://huggingface.co/docs/huggingface_hub/v0.22.2/en/guides/inference
 """
-model = AutoModelForCausalLM.from_pretrained("KishoreK/ActionGemma-9B", load_in_4bit=True, device_map="auto")
-tokenizer = AutoTokenizer.from_pretrained("KishoreK/ActionGemma-9B")
+
 
 def respond(
     message,
@@ -87,7 +86,11 @@ If the given question lacks the parameters required by the function, also point
         "content": json.dumps(convert_to_xlam_tool(tools))
     }]
 
-    return tokenizer.decode(tokenizer.apply_chat_template(messages, add_generation_prompt=True))
+    model = AutoModelForCausalLM.from_pretrained("KishoreK/ActionGemma-9B", load_in_4bit=True, device_map="auto", use_cache=True,low_cpu_mem_usage=True )
+    tokenizer = AutoTokenizer.from_pretrained("KishoreK/ActionGemma-9B")
+    inputs = tokenizer.apply_chat_template(messages, add_generation_prompt=True)
+    outputs = model.generate(inputs, max_new_tokens=512, do_sample=False, num_return_sequences=1, eos_token_id=tokenizer.eos_token_id)
+    return tokenizer.decode(outputs[0][len(inputs[0]):], skip_special_tokens=True)
 
 """
 For information on how to customize the ChatInterface, peruse the gradio docs: https://www.gradio.app/docs/chatinterface

requirements.txt

@@ -1,3 +1,4 @@
 huggingface_hub==0.23.2
 transformers==4.43.3
-torch
+torch
+accelerate