Initial commit

app.py

@@ -0,0 +1,112 @@
+import os
+import gradio as gr
+import requests
+from typing import List, Dict, Tuple
+
+# Define the API URL (adjust according to your server address)
+API_URL = "http://127.0.0.1:5000/chat"
+
+History = List[Tuple[str, str]]
+Messages = List[Dict[str, str]]
+
+def clear_session() -> History:
+    return []
+
+def history_to_messages(history: History) -> Messages:
+    messages = []
+    for h in history:
+        messages.append({'role': 'user', 'content': h[0].strip()})
+        messages.append({'role': 'assistant', 'content': h[1].strip()})
+    return messages
+
+def messages_to_history(messages: Messages) -> History:
+    history = []
+    for q, r in zip(messages[0::2], messages[1::2]):
+        history.append((q['content'], r['content']))
+    return history
+
+def model_chat(query: str, history: History) -> Tuple[str, History]:
+    if not query.strip():
+        return '', history
+    
+    messages = history_to_messages(history)
+    messages.append({'role': 'user', 'content': query.strip()})
+
+    response = requests.post(API_URL, json={"messages": messages})
+    if response.status_code != 200:
+        return "Error: Failed to get response from the API", history
+    
+    response_json = response.json()
+    response_text = response_json["response"]
+
+    history.append((query.strip(), response_text.strip()))
+    return response_text.strip(), history
+
+with gr.Blocks(css='''
+    .gr-group {direction: rtl;}
+    .chatbot{text-align:right;}
+    .dicta-header {
+        background-color: var(--input-background-fill);
+        border-radius: 10px;
+        padding: 20px;
+        text-align: center;
+        display: flex;
+        flex-direction: row;
+        align-items: center;
+        box-shadow: var(--block-shadow);
+        border-color: var(--block-border-color);
+        border-width: 1px;
+    }
+    @media (max-width: 768px) {
+        .dicta-header {
+            flex-direction: column;
+        }
+    }
+    .chatbot.prose {
+        font-size: 1.2em;
+    }
+    .dicta-logo {
+        width: 150px;
+        height: auto;
+        margin-bottom: 20px;
+    }
+    .dicta-intro-text {
+        margin-bottom: 20px;
+        text-align: center;
+        display: flex;
+        flex-direction: column;
+        align-items: center;
+        width: 100%;
+        font-size: 1.1em;
+    }
+    textarea {
+        font-size: 1.2em;
+    }
+''', js=None) as demo:
+    gr.Markdown("""
+<div class="dicta-header">
+  <a href="">
+    <img src="\\logo111.png" alt="Logo" class="dicta-logo">
+  </a>  
+  <div class="dicta-intro-text">
+    <h1>צ'אט מערכי - הדגמה ראשונית</h1>
+    <span dir='rtl'>ברוכים הבאים לדמו האינטראקטיבי הראשון. חקרו את יכולות המודל וראו כיצד הוא יכול לסייע לכן במשימותיכם</span><br/>
+    <span dir='rtl'>הדמו נכתב על ידי סרן רועי רתם תוך שימוש במודל שפה דיקטה שפותח על ידי מפא"ת</span><br/>
+  </div>
+</div>
+""")
+    
+    chatbot = gr.Chatbot(height=600)
+    query = gr.Textbox(placeholder="הכנס שאלה בעברית (או באנגלית!)", rtl=True)
+    clear_btn = gr.Button("נקה שיחה")
+
+    def respond(query, history):
+        response, history = model_chat(query, history)
+        return history, gr.update(value="", interactive=True)
+
+    demo_state = gr.State([])
+
+    query.submit(respond, [query, demo_state], [chatbot, query, demo_state])
+    clear_btn.click(clear_session, [], demo_state, chatbot)
+
+demo.launch(share=True)

app_api.py

@@ -0,0 +1,60 @@
+from flask import Flask, request, jsonify
+from transformers import AutoModelForCausalLM, AutoTokenizer
+import torch
+
+app = Flask(__name__)
+
+# Load the model and tokenizer
+model_name = "dicta-il/dictalm2.0-instruct"
+model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype=torch.bfloat16)
+tokenizer = AutoTokenizer.from_pretrained(model_name)
+
+# Ensure the tokenizer has a pad token, if not, add one.
+if tokenizer.pad_token is None:
+    tokenizer.add_special_tokens({'pad_token': '[PAD]'})
+    model.resize_token_embeddings(len(tokenizer))
+
+# Set the device to load the model onto
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+model.to(device)
+
+@app.route('/chat', methods=['POST'])
+def chat():
+    data = request.json
+    messages = data.get("messages", [])
+
+    if not messages:
+        return jsonify({"error": "No messages provided"}), 400
+
+    # Combine messages into a single input string with the correct template
+    conversation = "<s>"
+    for i, message in enumerate(messages):
+        role = message["role"]
+        content = message["content"]
+        if role == "user":
+            if i == 0:
+                conversation += f"[INST] {content} [/INST]"
+            else:
+                conversation += f" [INST] {content} [/INST]"
+        elif role == "assistant":
+            conversation += f" {content}"
+    conversation += "</s>"
+
+    # Tokenize the combined conversation
+    encoded = tokenizer(conversation, return_tensors="pt").to(device)
+
+    # Generate response using the model
+    generated_ids = model.generate(
+        input_ids=encoded['input_ids'],
+        attention_mask=encoded['attention_mask'],
+        max_new_tokens=50,
+        pad_token_id=tokenizer.pad_token_id,
+        do_sample=True
+    )
+
+    decoded = tokenizer.decode(generated_ids[0], skip_special_tokens=True)
+
+    return jsonify({"response": decoded})
+
+if __name__ == '__main__':
+    app.run(host='0.0.0.0', port=5000)