app.py

import gradio as gr
from gradio_leaderboard import Leaderboard, ColumnFilter, SelectColumns
import pandas as pd
from apscheduler.schedulers.background import BackgroundScheduler
from huggingface_hub import snapshot_download
from functools import lru_cache
import logging
import os

from src_maia.about import CITATION_BUTTON_LABEL, CITATION_BUTTON_TEXT, EVALUATION_QUEUE_TEXT,  \
    LLM_BENCHMARKS_TEXT
#from src.tasks import TASK_DESCRIPTIONS, MEASURE_DESCRIPTION
from src_maia.tasks import TASK_DESCRIPTIONS, MEASURE_DESCRIPTION, INTRODUCTION_TEXT, TITLE
from src_maia.display.css_html_js import custom_css
from src_maia.display.utils import BENCHMARK_COLS, COLS, EVAL_COLS, EVAL_TYPES, AutoEvalColumn, ModelType, fields, \
    WeightType, Precision
from src_maia.envs import API, EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH, QUEUE_REPO, REPO_ID, RESULTS_REPO, TOKEN
from src_maia.populate import get_evaluation_queue_df, get_leaderboard_df
from src_maia.submission.submit import add_new_eval
import matplotlib.pyplot as plt
import re
import plotly.express as px
import plotly.graph_objects as go
import numpy as np

import requests

# Configure logging
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)

# EVALITA results
BASELINES = {
    "TE": 71.00, "SA": 66.38, "HS": 80.88, "AT": 82.40, "WIC": 85.00,
    "LS": 38.82, "SU": 38.91, "NER": 88.00, "REL": 62.99
}

# GPT-4o results
REFERENCES = {
    "NER": 79.11, "REL": 63.32, "LS": 59.25, "SU": 33.04
}

TASK_METADATA_MULTIPLECHOICE = {
    "TE": {"icon": "📊", "name": "Textual Entailment", "tooltip": ""},
    "SA": {"icon": "😃", "name": "Sentiment Analysis", "tooltip": ""},
    "HS": {"icon": "⚠️", "name": "Hate Speech", "tooltip": ""},
    "AT": {"icon": "🏥", "name": "Admission Test", "tooltip": ""},
    "WIC": {"icon": "🔤", "name": "Word in Context", "tooltip": ""},
    "FAQ": {"icon": "❓", "name": "Frequently Asked Questions", "tooltip": ""},
    "MAIA-MC": {"icon": "🖼️", "name": "MAIA Multiple-Choice", "tooltip": ""}
}

TASK_METADATA_GENERATIVE = {
    "LS": {"icon": "🔄", "name": "Lexical Substitution", "tooltip": "1"},
    "SU": {"icon": "📝", "name": "Summarization", "tooltip": "2"},
    "NER": {"icon": "🏷️", "name": "Named Entity Recognition", "tooltip": "3"},
    "REL": {"icon": "🔗", "name": "Relation Extraction", "tooltip": "4"},
    "MAIA-GEN": {"icon": "🧩", "name": "MAIA Generative", "tooltip": "5"}
}

# Function to send a Slack notification for a new model submission for evaluation
def send_slack_notification(model_name, user_name, user_affiliation):
    # Insert your Slack webhook URL here
    webhook_url = os.getenv("WEBHOOK_URL")

    # Create the messag to be sent to Slack
    message = {
        "text": f"New model submission for EVALITA-LLM leaderboard:\n\n"
                f"**Model Name**: {model_name}\n"
                f"**User**: {user_name}\n"
                f"**Affiliation**: {user_affiliation}\n"
                f"Check out the model on HuggingFace: https://huggingface.co/{model_name}"
    }

    # Send the message to Slack
    response = requests.post(webhook_url, json=message)

    # Check if the request was successful and return the appropriate message
    if response.status_code == 200:
        return "✅ **Notification sent successfully!**"
    else:
        return f"❌ **Failed to send notification**: {response.text}"


# Funcion to validate the model submission and send the request for processing
def validate_and_submit_request(model_name, user_email, user_affiliation):
    # Check if model name is provided and not empt
    if not model_name or not model_name.strip():
        return "❌ **Error:** Model name is required."

    # Check if user email is provided and not empty
    if not user_email or not user_email.strip():
        return "❌ **Error:** Email address is required."

    # Validate email format using regex
    email_regex = r'^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$'
    if not re.match(email_regex, user_email.strip()):
        return "❌ **Error:** Invalid email format. Please enter a valid email address."

    # Check if user affiliation is provided and not empty
    if not user_affiliation or not user_affiliation.strip():
        return "❌ **Error:** Affiliation is required."

    # Check if model name follows the correct format (organization/model-name)
    if "/" not in model_name:
        return "❌ **Error:** Model name must be in format 'organization/model-name' (e.g., 'microsoft/DialoGPT-medium')."

    # Check if the model name contains only valid characters
    if not re.match(r'^[a-zA-Z0-9._/-]+$', model_name):
        return "❌ **Error:** Model name contains invalid characters."

    slack_response = send_slack_notification(model_name.strip(), user_email.strip(), user_affiliation.strip())

    # Return the Slack response (success or failure message)
    return slack_response


# Funzione per calcolare la sensibilità del prompt (PSI)
def calculate_prompt_sensitivity(dataframe, tasks, prompt_ids):
    # Elenco dei task generativi
    generative_tasks = ["LS", "SU", "NER", "REL", "MAIA-GEN"]

    cv_per_task = []  # Lista per memorizzare il CV per ogni task

    for task in tasks:
        prompt_col = f"{task} Best Prompt Id"
        task_accuracies = []  # Lista per memorizzare le accuratezze dei prompt per un task

        for pid in prompt_ids:
            pid_int = int(pid)

            # Applicazione dei filtri sui prompt per ogni task
            if pid_int <= 6 and task in generative_tasks:  # Prompt 1-6 solo per task non generativi
                continue  # Ignoriamo i prompt 1-6 per i task generativi

            elif pid_int in [7, 8] and task not in ["SU", "MAIA-GEN"]:  # Prompt 7-8 solo per il task SU
                continue  # Ignoriamo i prompt 7-8 per task diversi da SU

            elif pid_int in [9, 10] and task not in ["LS", "NER", "REL"]:  # Prompt 9-10 solo per LS, NER, REL, MAIA-GEN
                continue  # Ignoriamo i prompt 9-10 per task che non sono LS, NER, o REL

            # Calcolo della percentuale di modelli che hanno ottenuto il miglior prompt per il task
            total = len(dataframe[prompt_col].dropna())
            count = (dataframe[prompt_col] == pid).sum()
            accuracy = (count / total * 100) if total > 0 else 0
            task_accuracies.append(accuracy)

        # Calcoliamo la media e la deviazione standard delle accuratezze per il task
        if task_accuracies:
            mean_acc = np.mean(task_accuracies)
            std_acc = np.std(task_accuracies)
            # Calcoliamo il Coefficiente di Variazione (CV) solo se la media è maggiore di 0
            if mean_acc > 0:
                cv = std_acc / mean_acc
                cv_per_task.append(cv)
            else:
                cv_per_task.append(0)
        else:
            cv_per_task.append(0)  # Se non ci sono dati per il task, CV è 0

    # Calcola la media dei CV
    mean_cv = np.mean(cv_per_task) if cv_per_task else 0

    # Normalizza il CV per ottenere il PSI
    if mean_cv >= 0.5:
        psi = 1.0
    else:
        psi = mean_cv / 0.5

    return psi, mean_cv, cv_per_task

def map_prompt_ids_for_generation(dataframe):
    """
    Map original prompt IDs (1 or 2) to their corresponding generative prompt IDs.

    - For task 'SU': 1 -> 7, 2 -> 8
    - For tasks 'NER', 'REL', 'LS': 1 -> 9, 2 -> 10
    """

    # Mapping for SU task
    task = "SU"
    best_prompt_col = f"{task} Best Prompt Id"
    if best_prompt_col in dataframe.columns:
        dataframe[best_prompt_col] = dataframe[best_prompt_col].apply(
            lambda x: 7 if x == 1 else (8 if x == 2 else x)
        )

    # Mapping for SU task
    task = "MAIA-GEN"
    best_prompt_col = f"{task} Best Prompt Id"
    if best_prompt_col in dataframe.columns:
        dataframe[best_prompt_col] = dataframe[best_prompt_col].apply(
            lambda x: 7 if x == 1 else (8 if x == 2 else x)
        )

    # Mapping for other tasks
    for task in ["NER", "REL", "LS"]:
        best_prompt_col = f"{task} Best Prompt Id"
        if best_prompt_col in dataframe.columns:
            dataframe[best_prompt_col] = dataframe[best_prompt_col].apply(
                lambda x: 9 if x == 1 else (10 if x == 2 else x)
            )

    return dataframe


def create_best_model_comparison_table(dataframe):
    """
    Tabella interattiva con dettagli dei modelli migliori per ogni task.
    """
    tasks = ["TE", "SA", "HS", "AT", "WIC", "FAQ", "LS", "SU", "NER", "REL", "MAIA-MC", "MAIA-GEN"]

    table_data = {
        'Task': [],
        'Best Overall Model': [],
        'CPS': [],
        'Best Prompt Model': [],
        'Acc.': []
    }

    for task in tasks:
        if task in dataframe.columns:
            # Miglior modello per CPS
            max_idx = dataframe[task].idxmax()
            model_raw = dataframe.loc[max_idx, 'Model']
            if isinstance(model_raw, str) and '<' in model_raw:
                match = re.search(r'>([^<]+)<', model_raw)
                model_name = match.group(1) if match else model_raw
            else:
                model_name = str(model_raw)

            comb_perf_value = dataframe.loc[max_idx, task]

            # Miglior modello per Best Prompt
            best_prompt_column = f"{task} Best Prompt"
            best_prompt_idx = dataframe[best_prompt_column].idxmax()
            best_prompt_model_raw = dataframe.loc[best_prompt_idx, 'Model']
            if isinstance(best_prompt_model_raw, str) and '<' in best_prompt_model_raw:
                match = re.search(r'>([^<]+)<', best_prompt_model_raw)
                best_prompt_model = match.group(1) if match else best_prompt_model_raw
            else:
                best_prompt_model = str(best_prompt_model_raw)

            best_prompt_accuracy = dataframe.loc[best_prompt_idx, best_prompt_column]

            # Aggiungi alla tabella
            table_data['Task'].append(task)
            table_data['Best Overall Model'].append(model_name)
            table_data['CPS'].append(f"{comb_perf_value:.2f}")
            table_data['Best Prompt Model'].append(best_prompt_model)
            table_data['Acc.'].append(f"{best_prompt_accuracy:.2f}")

    # Calcola altezza dinamica
    row_height = 30
    header_height = 30
    caption_height = 100
    fig_height = header_height + row_height * len(table_data['Task']) + caption_height
    #fig_height = header_height + row_height * len(table_data['Task']) + caption_height + 50

    fig = go.Figure(data=[go.Table(
        columnwidth=[50, 195, 40, 195, 40],
        header=dict(
            values=[f'<b>{col}</b>' for col in table_data.keys()],
            fill_color=['#2171b5', '#2171b5', '#2171b5', '#4292c6', '#4292c6'],
            font=dict(color='white', size=12, family='Arial'),
            align='center',
            height=header_height
        ),
        cells=dict(
            values=list(table_data.values()),
            fill_color=[['#f0f0f0' if i % 2 == 0 else 'white' for i in range(len(table_data['Task']))]],
            font=dict(color='#2c3e50', size=11, family='Arial'),
            align=['center', 'left', 'center', 'left', 'center'],
            height=row_height
        )
    )])

    fig.update_layout(
        title={'text': "Top Model per Task: CPS & Best Prompt",
               'font': {'family': 'Arial', 'size': 14, 'color': '#2c3e50'}},
        font=dict(family="Arial", size=11),
        height=fig_height,
        margin=dict(l=20, r=20, t=60, b=100)
    )

    # Caption
    fig.add_annotation(
        text="Best Overall Models: Scored using the primary metric, CPS, across all prompts. <br>"
             "Best Prompt Model: Scored with the highest accuracy (unofficial) based on its best-performing prompt. <br>"
             "No single model achieves the highest performance across all tasks.",
        xref="paper", yref="paper",
        x=0.5, y=-0.20,
        showarrow=False,
        font=dict(size=11, color="gray", family="Arial"),
        align="center",
        xanchor="center"
    )

    return fig



def create_prompt_heatmap(dataframe):
    """
    Heatmap con percentuale di modelli che hanno ottenuto le best performance con ciascun prompt per ogni task,
    mostrando solo i valori pertinenti:
    - Prompt 1-6: solo per task multiple-choice
    - Prompt 7-8: solo per SU
    - Prompt 9-10: solo per LS, NER, REL
    """

    tasks = ["TE", "SA", "HS", "AT", "WIC", "FAQ", "MAIA-MC", "LS", "SU", "NER", "REL", "MAIA-GEN"]
    generative_tasks = ["LS", "SU", "NER", "REL", "MAIA-GEN"]
    mc_tasks = [t for t in tasks if t not in generative_tasks]

    all_prompt_ids = set()
    for task in tasks:
        prompt_col = f"{task} Best Prompt Id"
        if prompt_col in dataframe.columns:
            all_prompt_ids.update(dataframe[prompt_col].dropna().unique())

    prompt_ids = sorted(all_prompt_ids, key=int)

    matrix = []
    hover_texts = []

    # Calcola la sensibilità al prompt (PSI, mean_cv, cv_per_task)
    psi, mean_cv, cv_per_task = calculate_prompt_sensitivity(dataframe, tasks, prompt_ids)
    #print(f"Prompt Sensitivity Index (PSI): {psi:.3f}")
    #print(f"Mean CV: {mean_cv:.3f}")
    #print(f"CV per task: {cv_per_task}")

    for pid in prompt_ids:
        row = []
        hover_row = []
        for task in tasks:
            prompt_col = f"{task} Best Prompt Id"

            pid_int = int(pid)
            # Filtri personalizzati
            if pid_int <= 6 and task in generative_tasks:
                row.append(None)
                hover_row.append("")
            elif pid_int in [7, 8] and task not in ["SU", "MAIA-GEN"]:
                row.append(None)
                hover_row.append("")
            elif pid_int in [9, 10] and task not in ["LS", "NER", "REL"]:
                row.append(None)
                hover_row.append("")
            elif prompt_col in dataframe.columns:
                total = len(dataframe[prompt_col].dropna())
                count = (dataframe[prompt_col] == pid).sum()
                percentage = (count / total * 100) if total > 0 else 0
                row.append(percentage)
                hover_row.append(
                    f"<b>Prompt {pid} - {task}</b><br>"
                    f"Models: {count}/{total}<br>"
                    f"Percentage: {percentage:.1f}%"
                )
            else:
                row.append(0)
                hover_row.append(f"<b>Prompt {pid} - {task}</b><br>No data")
        matrix.append(row)
        hover_texts.append(hover_row)

    # Ticktext colorati: blu per 1-6, arancio per 7-10
    ticktext = []
    for pid in prompt_ids:
        pid_int = int(pid)
        #if pid_int <= 6:
        ticktext.append(f'<span style="color:#1f77b4;">P{pid} </span>')  # blu
        #else:
        #ticktext.append(f'<span style="color:#ff7f0e;">P{pid}</span>')  # arancio

    fig = go.Figure(data=go.Heatmap(
        z=matrix,
        x=tasks,
        y=prompt_ids,
        colorscale=[
            [0, '#f7fbff'],
            [0.2, '#deebf7'],
            [0.4, '#9ecae1'],
            [0.6, '#4292c6'],
            [0.8, '#2171b5'],
            [1, '#08519c']
        ],
        text=[[f"{val:.0f}%" if val is not None else "" for val in row] for row in matrix],
        texttemplate="%{text}",
        textfont={"size": 11, "family": "Arial"},
        hovertemplate='%{customdata}<extra></extra>',
        customdata=hover_texts,
        colorbar=dict(title="% Models", ticksuffix="%"),
        zmin=0,
        zmax=100
    ))

    fig.update_yaxes(
        tickmode='array',
        tickvals=prompt_ids,
        ticktext=ticktext,
        tickfont={"size": 11, "family": "Arial"}
    )

    fig.update_layout(
        title={'text': "Most Effective Prompts per Task Across Models",
               'font': {'family': 'Arial', 'size': 14, 'color': '#2c3e50'}},
        #xaxis_title="Task",
        yaxis_title="Prompt Variant",
        font=dict(family="Arial", size=11),  # allinea font con line_chart
        margin=dict(b=150),
        template="plotly_white",
        dragmode=False,
        height=500
    )

    fig.add_annotation(
        text=f"<b style='font-size:14px; color:#2c3e50;'>Mean CV: {mean_cv:.2f}</b>",
        # Testo in grassetto e con colore personalizzato
        xref="paper", yref="paper",
        x=0.3, y=0.85,  # Posizione sotto il grafico
        showarrow=False,
        font=dict(size=14, color="#2c3e50", family="Verdana"),  # Cambiato font a 'Verdana' per un aspetto più elegante
        align="center",
        xanchor="center",
        bgcolor="#f7f7f7",  # Aggiunta di uno sfondo chiaro per migliorare la leggibilità
        borderpad=5,  # Padding per distanziare il testo dal bordo
        bordercolor="#ccc",  # Colore del bordo
        borderwidth=1  # Larghezza del bordo
    )

    fig.add_annotation(
        text=(
            "Prompts 1–6 are for multiple-choice tasks, 7–10 for generative tasks. Darker cells represent the number of times, across <br>"
            "all model configurations tested, that a prompt achieved the top performance. With a Mean CV (Coefficient of Variation averaged across tasks) <br>"
            "above 0.3 there is high variability between prompts, suggesting the use of multiple prompts for more stable evaluation."
        ),
        xref="paper", yref="paper",
        x=0.5, y=-0.40,
        showarrow=False,
        font=dict(size=11, color="gray", family="Arial"),
        align="center",
        xanchor="center"
    )

    fig.update_xaxes(fixedrange=True)
    fig.update_yaxes(fixedrange=True)

    return fig


def highlight_best_per_task(df):
    """Add 🟡 symbol next to the maximum value in each task column"""

    task_columns = ["TE", "SA", "HS", "AT", "WIC", "FAQ", "LS", "SU", "NER", "REL", "MAIA-MC", "MAIA-GEN"]

    df = df.copy()
    for col in task_columns:
        if col in df.columns:
            max_val = df[col].max()
            df[col] = df[col].apply(
                lambda x: f"{x:.1f}🔺" if x == max_val else f"{x:.1f}"
            )
    return df

def theoretical_performance(df_hash):
    """
    Theoretical performance of a model that scores the highest on every individual task
    """
    # This is a placeholder - you'd need to pass the actual dataframe
    # In practice, you'd compute this once and store it
    #fields = ["TE", "SA", "HS", "AT", "WIC", "FAQ", "LS", "SU", "NER", "REL"]
    return 75.0  # Placeholder value


def scale_sizes(values, min_size=8, max_size=30):
    """Normalize sizes for scatter plot markers """
    if not values:
        return []
    vmin, vmax = min(values), max(values)
    if vmax == vmin:
        return [(min_size + max_size) / 2] * len(values)
    return [
        min_size + (val - vmin) / (vmax - vmin) * (max_size - min_size)
        for val in values
    ]


def extract_model_name(model_string):
    """Extract model name from HTML string."""
    match = re.search(r'>([^<]+)<', model_string)
    return match.group(1) if match else model_string


def create_line_chart(dataframe):
    """Create left chart."""

    def scale_sizes(values, min_size=8, max_size=30):
        vmin, vmax = min(values), max(values)
        return [
            min_size + (val - vmin) / (vmax - vmin) * (max_size - min_size) if vmax > vmin
            else (min_size + max_size) / 2
            for val in values
        ]

    fig = go.Figure()

    # Loop su 5-Shot e 0-Shot
    for shot, color in [(True, "blue"), (False, "red")]:
        df = dataframe[dataframe["IS_FS"] == shot]

        x = df["#Params (B)"].tolist()
        y = df["Avg. Comb. Perf. ⬆️"].tolist()
        labels = [
            re.search(r'>([^<]+)<', m).group(1) if isinstance(m, str) and re.search(r'>([^<]+)<', m) else str(m)
            for m in df["Model"].tolist()
        ]

        fig.add_trace(go.Scatter(
            x=x,
            y=y,
            mode="markers",
            name="5-Shot" if shot else "0-Shot",
            marker=dict(color=color, size=scale_sizes(x)),
            hovertemplate="<b>%{customdata}</b><br>#Params: %{x}<br>Performance: %{y}<extra></extra>",
            customdata=labels,
        ))

    '''
    # Show the best model
    all_y = dataframe["Avg. Comb. Perf. ⬆️"].tolist()
    if all_y:
        max_idx = all_y.index(max(all_y))
        max_x = dataframe["#Params (B)"].iloc[max_idx]
        max_y = all_y[max_idx]
        max_label = re.search(r'>([^<]+)<', dataframe["Model"].iloc[max_idx]).group(1)

        fig.add_annotation(
            x=max_x,
            y=max_y,
            text=max_label,
            showarrow=True,
            arrowhead=2,
            arrowsize=1,
            arrowwidth=2,
            arrowcolor="black",
            font=dict(size=11, color="black"),
            xshift=10, yshift=10,
            ax=-30, ay=-20,
            xanchor="right"
        )
    '''

    # Layout
    fig.update_layout(
        title="Model Accuracy vs #Params",
        xaxis_title="#Params (B)", yaxis_title="Avgerage CPS",
        template="plotly_white", hovermode="closest",
        font=dict(family="Arial", size=10), dragmode=False,
        xaxis=dict(tickvals=[0, 25, 50, 75, 100, 125], ticktext=["0", "25", "50", "75", "100"]),
        yaxis=dict(tickvals=[0, 20, 40, 60, 80, 100], range=[0, 100])
    )

    # Caption
    fig.add_annotation(
        text="Accuracy generally rises with #Params, but smaller models <br>"
             "with 5-shot can outperform larger zero-shot models.",
        xref="paper", yref="paper", x=0.5, y=-0.3,
        showarrow=False, font=dict(size=11, color="gray"),
        align="center", xanchor="center"
    )

    fig.update_xaxes(fixedrange=True, rangeslider_visible=False)
    fig.update_yaxes(fixedrange=True)

    return fig


def create_boxplot_task(dataframe=None, baselines=None, references=None):
    """Create right chart"""

    tasks = ["TE", "SA", "HS", "AT", "WIC", "FAQ", "LS", "SU", "NER", "REL", "MAIA-MC", "MAIA-GEN"]

    # Dati di default se non forniti
    if dataframe is None:
        np.random.seed(42)
        dataframe = pd.DataFrame({task: np.random.uniform(0.4, 0.9, 20) * 100 for task in tasks})

    if baselines is None:
        baselines = {task: np.random.randint(50, 70) for task in tasks}

    if references is None:
        references = {}

    colors = ["#1f77b4", "#ff7f0e", "#2ca02c", "#d62728", "#9467bd",
              "#8c564b", "#e377c2", "#7f7f7f", "#bcbd22", "#17becf",
              "#ff69b4",  # colore per MAIA-MC
              "#00ced1"]  # colore per MAIA-GEN]

    fig = go.Figure()

    for i, task in enumerate(tasks):
        if task not in dataframe.columns:
            continue

        y_data = dataframe[task].dropna().tolist()
        #print(task, y_data )

        # Boxplot
        fig.add_trace(go.Box(
            y=y_data,
            name=task,
            marker=dict(color=colors[i]),
            line=dict(color="black", width=2),
            fillcolor=colors[i],
            opacity=0.7,
            hovertemplate="<b>"+task+"</b><br>Accuracy: %{y:.2f}%<extra></extra>",
            hoverlabel=dict(bgcolor=colors[i], font_color="white"),
            width=0.6,
            whiskerwidth=0.2,
            quartilemethod="linear"
        ))

        # Linea baseline
        baseline_value = baselines.get(task)
        if baseline_value is not None:
            fig.add_shape(
                type="line",
                x0=i - 0.3, x1=i + 0.3,
                y0=baseline_value, y1=baseline_value,
                line=dict(color="black", width=2, dash="dot"),
                xref="x", yref="y"
            )

        # Linea reference GPT-4o
        reference_value = references.get(task)
        if reference_value is not None:
            fig.add_shape(
                type="line",
                x0=i - 0.3, x1=i + 0.3,
                y0=reference_value, y1=reference_value,
                line=dict(color="red", width=2, dash="dashdot"),
                xref="x", yref="y"
            )

    # Layout
    fig.update_layout(
        title="Distribution of Model Accuracy by Task",
        #xaxis_title="Task",
        yaxis_title="Average CPS",
        template="plotly_white",
        boxmode="group",
        dragmode=False,
        font=dict(family="Arial", size=10),
        margin=dict(b=80),
    )

    # Caption
    fig.add_annotation(
        text=(
            "In tasks like TE and SA, models approach the accuracy of supervised models at EVALITA (dashed black line).<br>"
            "In NER and REL they remain lower. Dashed red lines show GPT-4o reference results for generative tasks."
        ),
        xref="paper", yref="paper",
        x=0.5, y=-0.30,
        showarrow=False,
        font=dict(size=11, color="gray"),
        align="center"
    )

    fig.update_yaxes(range=[0, 100], fixedrange=True)
    fig.update_xaxes(fixedrange=True)

    return fig


def create_medal_assignments2(sorted_df):
    """Function for medal assignment logic"""
    medals = {
        'large_fs': False, 'medium_fs': False, 'small_fs': False,
        'large_0shot': False, 'medium_0shot': False, 'small_0shot': False
    }

    new_model_column = []

    for _, row in sorted_df.iterrows():
        model_name = row['Model']
        size = row["Size"]
        is_fs = row['IS_FS']

        if is_fs:  # 5-Few-Shot
            if size == "🔵🔵🔵" and not medals['large_fs']:
                model_name = f"{model_name} 🔵🔵🔵🏆"
                medals['large_fs'] = True
            elif size == "🔵🔵" and not medals['medium_fs']:
                model_name = f"{model_name} 🔵🔵🏆"
                medals['medium_fs'] = True
            elif size == "🔵" and not medals['small_fs']:
                model_name = f"{model_name} 🔵🏆"
                medals['small_fs'] = True
        else:  # 0-Shot
            if size == "🔵🔵🔵" and not medals['large_0shot']:
                model_name = f"{model_name} 🔵🔵🔵🎖️"
                medals['large_0shot'] = True
            elif size == "🔵🔵" and not medals['medium_0shot']:
                model_name = f"{model_name} 🔵🔵🎖️"
                medals['medium_0shot'] = True
            elif size == "🔵" and not medals['small_0shot']:
                model_name = f"{model_name} 🔵🎖️"
                medals['small_0shot'] = True

        new_model_column.append(model_name)

    return new_model_column



def create_medal_assignments(sorted_df):
    """
    Assign cups 🏆 to the top model in each category:
    Dimension (Size) × Learning mode (FS / 0-Shot) × Model type (Textual / Multimodal)
    """

    # Initialize a dictionary to keep track of which category already has a cup
    cups = {
        # Few-shot
        ('🔵🔵🔵', True, '🔤'): False,
        ('🔵🔵', True, '🔤'): False,
        ('🔵', True, '🔤'): False,
        ('🔵🔵🔵', True, '🔤🖼️'): False,
        ('🔵🔵', True, '🔤🖼️'): False,
        ('🔵', True, '🔤🖼️'): False,
        # Zero-shot
        ('🔵🔵🔵', False, '🔤'): False,
        ('🔵🔵', False, '🔤'): False,
        ('🔵', False, '🔤'): False,
        ('🔵🔵🔵', False, '🔤🖼️'): False,
        ('🔵🔵', False, '🔤🖼️'): False,
        ('🔵', False, '🔤🖼️'): False,
    }

    new_model_column = []

    for _, row in sorted_df.iterrows():
        model_name = row['Model']
        size = row["Size"]
        is_fs = row['IS_FS']          # True = Few-shot, False = Zero-shot
        mode = row['Mode']             # '🔤' or '🔤🖼️'

        category_key = (size, is_fs, mode)
        if category_key in cups and not cups[category_key]:
            model_name = f"{model_name} 🏆"
            cups[category_key] = True

        new_model_column.append(model_name)

    return new_model_column



def create_leaderboard_base(sorted_dataframe, field_list, hidden_columns):
    """Base leaderboard creation with common parameters. """

    return Leaderboard(
        value=sorted_dataframe,
        datatype=[c.type for c in field_list],
        search_columns=[AutoEvalColumn.model.name],
        hide_columns=hidden_columns,
        filter_columns=[
            ColumnFilter(AutoEvalColumn.fewshot_symbol.name, type="checkboxgroup", label="Learning mode (🅾️0-shot vs 5️⃣5-shot)"),
            ColumnFilter(AutoEvalColumn.mode_symbol.name, type="checkboxgroup", label="Model type (🔤Textual vs 🔤🖼️Multimodal)"),
            ColumnFilter(AutoEvalColumn.params.name, type="slider", min=0, max=100, default=[0, 100],
                         label="Select the number of parameters (B)"),
        ],
        bool_checkboxgroup_label="Evaluation Mode",
        interactive=False,
    )


def init_leaderboard(dataframe, default_selection=None, hidden_columns=None):
    """Leaderboard initialization """
    if dataframe is None or dataframe.empty:
        raise ValueError("Leaderboard DataFrame is empty or None.")

    # Sort and reset index
    sorted_dataframe = dataframe.sort_values(by="Avg. Comb. Perf. ⬆️", ascending=False).reset_index(drop=True)
    sorted_dataframe["Rank"] = sorted_dataframe.index + 1

    # Apply medal assignments
    sorted_dataframe["Model"] = create_medal_assignments(sorted_dataframe)

    # Show the best values for tasks
    #sorted_dataframe = highlight_best_per_task(sorted_dataframe)

    field_list = fields(AutoEvalColumn)

    return create_leaderboard_base(sorted_dataframe, field_list, hidden_columns)


def update_task_leaderboard(dataframe, default_selection=None, hidden_columns=None):

    """ Task-specific leaderboard update."""
    if dataframe is None or dataframe.empty:
        raise ValueError("Leaderboard DataFrame is empty or None.")

    # Sort and reset index
    sorted_dataframe = dataframe.sort_values(by="Comb. Perf. ⬆️", ascending=False).reset_index(drop=True)
    sorted_dataframe["Rank"] = sorted_dataframe.index + 1

    # Apply medal assignments
    sorted_dataframe["Model"] = create_medal_assignments(sorted_dataframe)

    field_list = fields(AutoEvalColumn)

    return Leaderboard(
        value=sorted_dataframe,
        datatype=[c.type for c in field_list] + [int],
        search_columns=[AutoEvalColumn.model.name],
        hide_columns=hidden_columns,
        filter_columns=[
            ColumnFilter(AutoEvalColumn.fewshot_symbol.name, type="checkboxgroup", label="N-Shot Learning (FS)"),
            ColumnFilter(AutoEvalColumn.params.name, type="slider", min=0, max=100, default=[0, 100],
                         label="Select the number of parameters (B)"),
        ],
        bool_checkboxgroup_label="Evaluation Mode",
        interactive=False
    )


def download_snapshot(repo, local_dir, max_retries=3):
    """Snapshot download with retry logic."""
    for attempt in range(max_retries):
        try:
            logger.info(f"Downloading from {repo} to {local_dir} (attempt {attempt + 1}/{max_retries})")
            snapshot_download(
                repo_id=repo,
                local_dir=local_dir,
                repo_type="dataset",
                tqdm_class=None,
                etag_timeout=30,
                token=TOKEN
            )
            return True
        except Exception as e:
            logger.error(f"Error downloading {repo} (attempt {attempt + 1}): {e}")
            if attempt == max_retries - 1:
                logger.error(f"Failed to download {repo} after {max_retries} attempts")
                return False
    return False


def restart_space():
    """Restart the Hugging Face space."""
    try:
        logger.info("Restarting space... ")
        API.restart_space(repo_id=REPO_ID)
    except Exception as e:
        logger.error(f"Error restarting space: {e}")


def create_title_html():
    """Function for title HTML."""
    return """
    <div class="title-header">
        <h1 class="title-text">
            EVALITA-LLM Leaderboard
        </h1>
        <a href="https://huggingface.co/spaces/mii-llm/open_ita_llm_leaderboard" target="_blank" class="title-link">
            <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24">
                <path d="M3.9 12a5 5 0 0 1 7.07-7.07l1.41 1.41-1.41 1.41-1.42-1.42a3 3 0 1 0 4.24 4.24l3.54-3.54a5 5 0 0 1-7.07 7.07l-1.41-1.41 1.41-1.41 1.42 1.42z"/>
                <path d="M20.1 12a5 5 0 0 1-7.07 7.07l-1.41-1.41 1.41-1.41 1.42 1.42a3 3 0 1 0-4.24-4.24l-3.54 3.54a5 5 0 0 1 7.07-7.07l1.41 1.41-1.41 1.41-1.42-1.42z"/>
            </svg>
            Open Italian LLM Leaderboard
        </a>
    </div>
    """


def create_credits_markdown():
    """Credits section."""
    return """
**This project has benefited from the following support:**

- 🧠 **Codebase**: Based on and extended from the Open Italian LLM Leaderboard, developed by **Alessandro Ercolani** and **Samuele Colombo**. We warmly thank them for their invaluable support and guidance in implementing this leaderboard.

- 💶 **Funding**: Partially supported by the PNRR project **FAIR - Future AI Research (PE00000013)**, under the NRRP MUR program funded by **NextGenerationEU**.

- 🖥️ **Computation**: We gratefully acknowledge **CINECA** for granting access to the **LEONARDO** supercomputer.
"""


# Main initialization
def initialize_app():
    """Initialize the application ."""
    try:
        # Download snapshots
        queue_success = download_snapshot(QUEUE_REPO, EVAL_REQUESTS_PATH)
        results_success = download_snapshot(RESULTS_REPO, EVAL_RESULTS_PATH)

        if not (queue_success and results_success):
            logger.error("Failed to download required data")
            return None, None, None, None, None

        # Load leaderboard data
        leaderboard_df = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, COLS, BENCHMARK_COLS)
        finished_eval_queue_df, running_eval_queue_df, pending_eval_queue_df = get_evaluation_queue_df(
            EVAL_REQUESTS_PATH, EVAL_COLS)

        # Calculate theoretical max performance
        theoretical_max = theoretical_performance(hash(str(leaderboard_df.values.tobytes())))

        return leaderboard_df, finished_eval_queue_df, running_eval_queue_df, pending_eval_queue_df, theoretical_max

    except Exception as e:
        logger.error(f"Error initializing app: {e}")
        return None, None, None, None, None
    

def initialize_app2():
    """Initialize the application using local cache if available."""
    try:
        # Controlla se la cache locale contiene già i dati
        if os.path.exists(EVAL_REQUESTS_PATH) and os.listdir(EVAL_REQUESTS_PATH):
            logger.info(f"Usando cache locale per le richieste di valutazione: {EVAL_REQUESTS_PATH}")
            queue_success = True
        else:
            queue_success = download_snapshot(QUEUE_REPO, EVAL_REQUESTS_PATH)

        if os.path.exists(EVAL_RESULTS_PATH) and os.listdir(EVAL_RESULTS_PATH):
            logger.info(f"Usando cache locale per i risultati di valutazione: {EVAL_RESULTS_PATH}")
            results_success = True
        else:
            results_success = download_snapshot(RESULTS_REPO, EVAL_RESULTS_PATH)

        if not (queue_success and results_success):
            logger.error("Impossibile scaricare i dati richiesti")
            return None, None, None, None, None

        # Carica i DataFrame dalla cache locale
        leaderboard_df = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, COLS, BENCHMARK_COLS)
        finished_eval_queue_df, running_eval_queue_df, pending_eval_queue_df = get_evaluation_queue_df(EVAL_REQUESTS_PATH, EVAL_COLS)

         # Ottieni tutti i nomi delle colonne del DataFrame
        all_columns = leaderboard_df.columns.tolist()
        
        #print("Colonne totali del DataFrame:", all_columns)

        # Calcola la performance teorica massima
        theoretical_max = theoretical_performance(hash(str(leaderboard_df.values.tobytes())))

        return leaderboard_df, finished_eval_queue_df, running_eval_queue_df, pending_eval_queue_df, theoretical_max

    except Exception as e:
        logger.error(f"Errore durante l'inizializzazione dell'app: {e}")
        return None, None, None, None, None


# Initialize data
LEADERBOARD_DF, finished_eval_queue_df, running_eval_queue_df, pending_eval_queue_df, theoretical_max_combined_perf = initialize_app()
LEADERBOARD_DF = map_prompt_ids_for_generation(LEADERBOARD_DF)


if LEADERBOARD_DF is None:
    # Fallback behavior
    logger.error("Failed to initialize app data")
    theoretical_max_combined_perf = 0.0


# Main Gradio interface
def create_gradio_interface():
    """The main Gradio interface."""
    demo = gr.Blocks(css=custom_css)

    with demo:
        # Titolo
        gr.HTML(create_title_html())
        gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")

        # Tabs principali
        with gr.Tabs(elem_classes="tab-buttons") as tabs:
            # 🏅 Benchmark
            with gr.TabItem("🏅 Benchmark"):
                if LEADERBOARD_DF is not None:
                    # Labels dei campi affiancate
                    '''
                    with gr.Row():
                       
                        gr.HTML(f"""
                        <div class="performance-metrics">
                            <div class="metric-label" title="Total number of configurations (zero-shot and 5-few-shot) of the models evaluated in the leaderboard." style="color: #333333;">
                                Models tested: {len(LEADERBOARD_DF)}
                            </div>
                            <div class="metric-label" title="Average accuracy of the evaluated models." style="color: #333333;">
                                Avg combined perf.: {LEADERBOARD_DF['Avg. Comb. Perf. ⬆️'].mean():.2f}
                            </div>
                            <div class="metric-label" title="Standard deviation of the evaluated models' performance." style="color: #333333;">
                                Std. Dev. {LEADERBOARD_DF['Avg. Comb. Perf. ⬆️'].std():.2f}
                            </div>
                            <div class="metric-label" title="Best evaluated model." style="color: #333333;">
                                Best model: {LEADERBOARD_DF.loc[LEADERBOARD_DF['Avg. Comb. Perf. ⬆️'].idxmax(), 'Model']}
                            </div>
                            <div class="metric-label" title="Accuracy of the best evaluated model." style="color: #333333;">
                                Best model accuracy: {LEADERBOARD_DF.loc[LEADERBOARD_DF['Avg. Comb. Perf. ⬆️'].idxmax(), 'Avg. Comb. Perf. ⬆️']:.2f}
                            </div>
                            <div class="metric-label" title="Maximum achievable accuracy based on the highest performance for each task by any model in the leaderboard." style="color: #333333;">
                                Ideal model: {theoretical_max_combined_perf:.2f}
                            </div>
                        </div>
                        """)
                    '''

                    #adv_stats = calculate_advanced_statistics(LEADERBOARD_DF)
                    # Separa i modelli per tipo
                    textual_df = LEADERBOARD_DF[LEADERBOARD_DF['Mode'] == '🔤']
                    multimodal_df = LEADERBOARD_DF[LEADERBOARD_DF['Mode'].str.contains('🔤🖼️', na=False)]
                    # Separa per modalità di apprendimento (SOLO TESTUALI)
                    #textual_fs = textual_df[textual_df['IS_FS'] == True]
                    #textual_0shot = textual_df[textual_df['IS_FS'] == False]
                    
                    # Statistiche per taglia
                    #small_models = LEADERBOARD_DF[LEADERBOARD_DF['Size'] == '🔵']
                    #medium_models = LEADERBOARD_DF[LEADERBOARD_DF['Size'] == '🔵🔵']
                    #large_models = LEADERBOARD_DF[LEADERBOARD_DF['Size'] == '🔵🔵🔵']
                    # Conteggi
                    num_total_models = len(LEADERBOARD_DF)
                    num_textual_models = len(textual_df)
                    num_multimodal_models = len(multimodal_df)
                    
                    # Miglior modello sui task testuali
                    best_textual_model = textual_df.loc[textual_df['Avg. Comb. Perf. ⬆️'].idxmax(), 'Model']
                    best_textual_score = round(textual_df['Avg. Comb. Perf. ⬆️'].max(), 1)

                    # Miglior modello sui task multimodali
                    best_multimodal_model = multimodal_df.loc[multimodal_df['Avg. Comb. Perf. ⬆️'].idxmax(), 'Model']
                    best_multimodal_score = round(multimodal_df['Avg. Comb. Perf. ⬆️'].max(), 1)

                    # Assicurati che i punteggi siano numerici prima di formattarli
                    best_textual_score = float(best_textual_score)
                    best_multimodal_score = float(best_multimodal_score)

                    # Poi puoi formattarli come stringhe con un solo decimale
                    best_textual_score = f"{best_textual_score:.1f}"
                    best_multimodal_score = f"{best_multimodal_score:.1f}"

                    # Dati per l'ultimo aggiornamento
                    last_update_date = "2025-12-07"

                    # Restante codice per visualizzare i dati su Gradio
                    with gr.Row():
                        gr.HTML(f"""
                        <div class="performance-metrics" style="display: flex; flex-wrap: wrap; gap: 8px; font-size: 10px;">

                            <!-- MODEL COUNTS -->
                            <div class="metric-label" style="padding: 4px; font-weight: bold; display: flex; align-items: center; background-color: #ffffff;">
                                <span style="font-size: 12px; color: #4a90e2;">Model Configurations:</span>
                                <span style="display: inline-flex; justify-content: center; align-items: center; width: 20px; height: 20px; border-radius: 50%; background-color: #4a90e2; color: white; text-align: center; font-size: 10px; margin-left: 5px;">{num_total_models}</span>
                            </div>

                            <div class="metric-label" style="padding: 4px; font-weight: bold; display: flex; align-items: center; background-color: #ffffff;">
                                <span style="font-size: 12px; color: #4a90e2;">Textual Configurations:</span>
                                <span style="display: inline-flex; justify-content: center; align-items: center; width: 20px; height: 20px; border-radius: 50%; background-color: #4a90e2; color: white; text-align: center; font-size: 10px; margin-left: 5px;">{num_textual_models}</span>
                            </div>

                            <div class="metric-label" style="padding: 4px; font-weight: bold; display: flex; align-items: center; background-color: #ffffff;">
                                <span style="font-size: 12px; color: #4a90e2;">Multimodal Configurations:</span>
                                <span style="display: inline-flex; justify-content: center; align-items: center; width: 20px; height: 20px; border-radius: 50%; background-color: #4a90e2; color: white; text-align: center; font-size: 10px; margin-left: 5px;">{num_multimodal_models}</span>
                            </div>

                            <!-- BEST MODELS -->
                            <div class="metric-label" title="Best model evaluated on textual data (regardless of modality)" style="padding: 4px; display: flex; align-items: center; background-color: #ffffff;">
                                <span style="font-size: 12px; color: #4a90e2;">Best Textual Model:</span>
                                <span style="font-size: 12px; color: #333333; margin-left: 5px;">{best_textual_model}</span>
                                <span style="display: inline-flex; justify-content: center; align-items: center; width: 20px; height: 20px; border-radius: 50%; background-color: #e94e77; color: white; text-align: center; font-size: 10px; margin-left: 5px;">{best_textual_score}</span>
                            </div>

                            <div class="metric-label" title="Best model evaluated on textual + visual/video data" style="padding: 4px; display: flex; align-items: center; background-color: #ffffff;">
                                <span style="font-size: 12px; color: #4a90e2;">Best Multimodal Model:</span>
                                <span style="font-size: 12px; color: #333333; margin-left: 5px;">{best_multimodal_model}</span>
                                <span style="display: inline-flex; justify-content: center; align-items: center; width: 20px; height: 20px; border-radius: 50%; background-color: #e94e77; color: white; text-align: center; font-size: 10px; margin-left: 5px;">{best_multimodal_score}</span>
                            </div>

                            <!-- LAST UPDATE -->
                            <div class="metric-label" style="padding: 4px; font-weight: bold; display: flex; align-items: center; background-color: #ffffff; border-radius: 5px; height: 40px;">
                                <span style="font-size: 12px; color: #4a90e2;">Last Update:</span>
                                <span style="font-size: 12px; color: #333333; margin-left: 5px;">{last_update_date}</span>
                            </div>

                        </div>
                        """)

                    # Grafici affiancati
                    with gr.Row():
                        gr.Plot(value=create_line_chart(LEADERBOARD_DF), elem_id="line-chart")
                        gr.Plot(value=create_boxplot_task(LEADERBOARD_DF, BASELINES, REFERENCES), elem_id="line-chart")

                    with gr.Row():
                        gr.Plot(value=create_prompt_heatmap(LEADERBOARD_DF), elem_id="line-chart")
                        gr.Plot(value=create_best_model_comparison_table(LEADERBOARD_DF), elem_id="line-chart")

                    # Leaderboard
                    leaderboard = init_leaderboard(
                        LEADERBOARD_DF,
                        default_selection=['Rank', 'Size', 'FS', 'Mode', 'Model', "Avg. Comb. Perf. ⬆️",
                                           "TE", "SA", "HS", "AT", "WIC", "FAQ", "LS", "SU", "NER", "REL", "MAIA-MC",  "MAIA-GEN" ],
                        hidden_columns=[col for col in LEADERBOARD_DF.columns if
                                        col not in ['Rank', 'Size', 'FS', 'Mode', 'Model', "Avg. Comb. Perf. ⬆️",
                                                    "TE", "SA", "HS", "AT", "WIC", "FAQ", "LS", "SU", "NER", "REL", "MAIA-MC",  "MAIA-GEN" ]]
                    )

            
            
            
            
            
            
            
            
            
            
            with gr.TabItem("🧪 Tasks"):

                with gr.Tabs(elem_classes="sub-tabs"):

                    # ----------------------------------------------------
                    # 1) MULTIPLE CHOICE TASKS
                    # ----------------------------------------------------
                    with gr.TabItem("🔢 Multiple-Choice"):
                        #gr.Markdown("### Multiple Choice Tasks")

                        with gr.Tabs():
                            for task, metadata in TASK_METADATA_MULTIPLECHOICE.items():
                                #with gr.TabItem(f"{metadata['icon']} {metadata['name']} ({task})"):
                                with gr.TabItem(f"{metadata['icon']}{task}"):
                                    task_description = TASK_DESCRIPTIONS.get(task, "Description not available.")
                                    gr.Markdown(task_description, elem_classes="markdown-text")

                                    leaderboard_task = update_task_leaderboard(
                                        LEADERBOARD_DF.rename(columns={
                                            f"{task} Prompt Average": "Prompt Average",
                                            f"{task} Prompt Std": "Prompt Std",
                                            f"{task} Best Prompt": "Best Prompt",
                                            f"{task} Best Prompt Id": "Best Prompt Id",
                                            task: "Comb. Perf. ⬆️"
                                        }),
                                        default_selection=['Rank', 'Size', 'FS', 'Mode', 'Model', 'Comb. Perf. ⬆️',
                                                        'Prompt Average', 'Prompt Std', 'Best Prompt', 'Best Prompt Id'],
                                        hidden_columns=[col for col in LEADERBOARD_DF.columns if
                                                        col not in ['Rank', 'Size', 'FS', 'Mode', 'Model', 'Comb. Perf. ⬆️',
                                                                    'Prompt Average', 'Prompt Std', 'Best Prompt',
                                                                    'Best Prompt Id']]
                                    )

                    # ----------------------------------------------------
                    # 2) GENERATIVE TASKS
                    # ----------------------------------------------------
                    with gr.TabItem("📝 Generative"):
                        #gr.Markdown("### Generative Tasks")

                        with gr.Tabs():
                            for task, metadata in TASK_METADATA_GENERATIVE.items():
                                #with gr.TabItem(f"{metadata['icon']} {metadata['name']} ({task})"):
                                with gr.TabItem(f"{metadata['icon']}{task}"):
                                    task_description = TASK_DESCRIPTIONS.get(task, "Description not available.")
                                    gr.Markdown(task_description, elem_classes="markdown-text")

                                    leaderboard_task = update_task_leaderboard(
                                        LEADERBOARD_DF.rename(columns={
                                            f"{task} Prompt Average": "Prompt Average",
                                            f"{task} Prompt Std": "Prompt Std",
                                            f"{task} Best Prompt": "Best Prompt",
                                            f"{task} Best Prompt Id": "Best Prompt Id",
                                            task: "Comb. Perf. ⬆️"
                                        }),
                                        default_selection=['Rank', 'Size', 'FS', 'Mode', 'Model', 'Comb. Perf. ⬆️',
                                                        'Prompt Average', 'Prompt Std', 'Best Prompt', 'Best Prompt Id'],
                                        hidden_columns=[col for col in LEADERBOARD_DF.columns if
                                                        col not in ['Rank', 'Size', 'FS', 'Mode', 'Model', 'Comb. Perf. ⬆️',
                                                                    'Prompt Average', 'Prompt Std', 'Best Prompt',
                                                                    'Best Prompt Id']]
                                    )

                    '''
                    # ----------------------------------------------------
                    # 3) MULTI-MODAL TASKS (NEW)
                    # ----------------------------------------------------
                    with gr.TabItem("🎥 Multi-Modal"):
                        gr.Markdown("### Multi-Modal Tasks")

                        with gr.Tabs():
                            # Only MAIA for now, but scalable for more
                            with gr.TabItem("🧩 MAIA"):
                                task_description = TASK_DESCRIPTIONS.get("MAIA", "Description not available.")
                                gr.Markdown(task_description, elem_classes="markdown-text")

                                leaderboard_task = update_task_leaderboard(
                                    LEADERBOARD_DF.rename(columns={
                                        "MAIA Prompt Average": "Prompt Average",
                                        "MAIA Prompt Std": "Prompt Std",
                                        "MAIA Best Prompt": "Best Prompt",
                                        "MAIA Best Prompt Id": "Best Prompt Id",
                                        "MAIA": "Comb. Perf. ⬆️"
                                    }),
                                    default_selection=['Rank', 'Size', 'FS', 'Model', 'Comb. Perf. ⬆️',
                                                    'Prompt Average', 'Prompt Std', 'Best Prompt', 'Best Prompt Id'],
                                    hidden_columns=[col for col in LEADERBOARD_DF.columns if
                                                    col not in ['Rank', 'Size', 'FS', 'Model', 'Comb. Perf. ⬆️',
                                                                'Prompt Average', 'Prompt Std', 'Best Prompt',
                                                                'Best Prompt Id']]
                                )
                    '''
            
            
            # 📝 About
            with gr.TabItem("📝 About"):
                gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text")
               
            
            # 🚀 Submit a new model to evaluate
            with gr.TabItem("🚀 Submit"):
                gr.Markdown("# 📝 Model Evaluation Request", elem_classes="markdown-text")
                gr.Markdown("""
                            **Fill out the form below to request evaluation of your model on EVALITA-LLM.**

                            Once submitted, our team will automatically receive a notification. We will evaluate the 
                            submission’s relevance for both research and commercial purposes, as well as assess its feasibility.
                            """, elem_classes="markdown-text")

                with gr.Row():
                    with gr.Column():
                        # HuggingFace model name field
                        model_name_input = gr.Textbox(
                            label="HuggingFace Model Name",
                            placeholder="e.g., microsoft/DialoGPT-medium",
                            info="Enter the complete model name as it appears on HuggingFace Hub (organization/model-name)",
                            elem_id="model-name-input"
                        )

                        # User email field
                        user_name_input = gr.Textbox(
                            label="Your email address",
                            placeholder="e.g., mario.rossi@example.com",
                            info="Enter your email address for communication",
                            elem_id="user-email-input"
                        )

                        # Affiliation field
                        user_affiliation_input = gr.Textbox(
                            label="Affiliation",
                            placeholder="e.g., University of Milan, Google Research, Freelancer",
                            info="Enter your affiliation (university, company, organization)",
                            elem_id="user-affiliation-input"
                        )

                # Submit button
                submit_request_button = gr.Button(
                    "📤 Submit Request",
                    variant="primary",
                    elem_id="submit-request-button"
                )

                # Result  area
                submission_status = gr.Markdown(elem_id="submission-status")

                # Connect button to function
                submit_request_button.click(
                    validate_and_submit_request,
                    inputs=[model_name_input, user_name_input, user_affiliation_input],
                    outputs=submission_status
                )

                # Additional information
                with gr.Accordion("ℹ️ Additional Information", open=False):
                    gr.Markdown("""
                                **What happens after submission:**
                                1. Your request is automatically sent to the EVALITA-LLM team
                                2. We verify that the model is accessible on HuggingFace
                                3. We contact you to confirm inclusion in the evaluation
                                4. The model is added to the evaluation queue

                                **Model requirements:**
                                - Model must be publicly accessible on HuggingFace Hub
                                - Must be compatible with the EleutherAI/lm-evaluation-harness framework
                                - Must have a license that allows evaluation

                                **Evaluation tasks:**
                                Your model will be evaluated on all tasks: TE, SA, HS, AT, WIC, FAQ, LS, SU, NER, REL.
                                """, elem_classes="markdown-text")


            '''
            # Separators
            with gr.TabItem("║", interactive=False):
                gr.Markdown("", elem_classes="markdown-text")

            # Task-specific tabs (Multiple Choice)
            if LEADERBOARD_DF is not None:
                for task, metadata in TASK_METADATA_MULTIPLECHOICE.items():
                    with gr.TabItem(f"{metadata['icon']}{task}"):
                        task_description = TASK_DESCRIPTIONS.get(task, "Description not available.")
                        gr.Markdown(task_description, elem_classes="markdown-text")

                        leaderboard_task = update_task_leaderboard(
                            LEADERBOARD_DF.rename(columns={
                                f"{task} Prompt Average": "Prompt Average",
                                f"{task} Prompt Std": "Prompt Std",
                                f"{task} Best Prompt": "Best Prompt",
                                f"{task} Best Prompt Id": "Best Prompt Id",
                                task: "Comb. Perf. ⬆️"
                            }),
                            default_selection=['Rank', 'Size', 'FS', 'Model', 'Comb. Perf. ⬆️',
                                               'Prompt Average', 'Prompt Std', 'Best Prompt', 'Best Prompt Id'],
                            hidden_columns=[col for col in LEADERBOARD_DF.columns if
                                            col not in ['Rank', 'Size', 'FS', 'Model', 'Comb. Perf. ⬆️',
                                                        'Prompt Average', 'Prompt Std', 'Best Prompt',
                                                        'Best Prompt Id']]
                        )

            # Separators
            with gr.TabItem("│", interactive=False):
                gr.Markdown("", elem_classes="markdown-text")

            # Task-specific tabs (Generative)
            if LEADERBOARD_DF is not None:
                for task, metadata in TASK_METADATA_GENERATIVE.items():
                    with gr.TabItem(f"{metadata['icon']}{task}"):
                        task_description = TASK_DESCRIPTIONS.get(task, "Description not available.")
                        gr.Markdown(task_description, elem_classes="markdown-text")

                        leaderboard_task = update_task_leaderboard(
                            LEADERBOARD_DF.rename(columns={
                                f"{task} Prompt Average": "Prompt Average",
                                f"{task} Prompt Std": "Prompt Std",
                                f"{task} Best Prompt": "Best Prompt",
                                f"{task} Best Prompt Id": "Best Prompt Id",
                                task: "Comb. Perf. ⬆️"
                            }),
                            default_selection=['Rank', 'Size', 'FS', 'Model', 'Comb. Perf. ⬆️',
                                               'Prompt Average', 'Prompt Std', 'Best Prompt', 'Best Prompt Id'],
                            hidden_columns=[col for col in LEADERBOARD_DF.columns if
                                            col not in ['Rank', 'Size', 'FS', 'Model', 'Comb. Perf. ⬆️',
                                                        'Prompt Average', 'Prompt Std', 'Best Prompt',
                                                        'Best Prompt Id']]
                        )
            '''   

        # Citation e Credits
        with gr.Accordion("📙 Citation", open=False):
            gr.Textbox(
                value=CITATION_BUTTON_TEXT,
                label=CITATION_BUTTON_LABEL,
                lines=20,
                elem_id="citation-button",
                show_copy_button=True
            )

        with gr.Accordion("📙 Credits", open=False):
            gr.Markdown(create_credits_markdown())

    return demo


# Create and configure the demo
demo = create_gradio_interface()

# Background scheduler for space restart
scheduler = BackgroundScheduler()
scheduler.add_job(restart_space, "interval", seconds=1800)
scheduler.start()

# Launch configuration
if __name__ == "__main__":
    demo.queue(default_concurrency_limit=40).launch(
        debug=True,
        show_error=True
    )