import csv import gradio as gr import os import time import os import pandas as pd app = None os.environ['GRADIO_ANALYTICS_ENABLED'] = 'False' CURR_BASE_DIR = os.getcwd() BASE_AUDIOS_DIR = os.path.join(os.getcwd(), "results") BASE_VIDEO_DIR = BASE_AUDIOS_DIR should_stop = False current_process = None if not os.path.exists(BASE_AUDIOS_DIR): os.makedirs(BASE_AUDIOS_DIR) custom_css = """ .gradio-container{ background-color: unset; } input, textarea { font-family: 'Noto Naskh Arabic', 'Arial', sans-serif !important; } .large_text input{ font-size:20px !important } .tab-container button{ font-weight: bold !important; } .tabs{ gap:0px !important; } .tabitem{ padding:0px !important; } button.secondary:hover{ background-color:steelblue !important; color:white; } .tab-container button { font-size:20px !important; } .tabitem{ padding-top:0px !important; } #tool-header{ padding:20px 0px 20px 0px !important; } #tool-header h1{ display: flex; align-items: center; } #tool-header img{ width: 80px; display: inline-block; margin-right: 10px; border-radius: 5px; } """ def natural_sort_key(file_name): if ".DS_Store" in file_name: return 0 return int(file_name.split("_")[1]) def get_data(folders_list, language="msa"): results_csv_df = pd.read_csv(f"{CURR_BASE_DIR}/results/{language}/Ar_{language}_TTS_benchmark.csv") data_map = {} try: for folder_name in folders_list: full_audio_folder_path = os.path.join(BASE_AUDIOS_DIR,language ,folder_name) if not os.path.exists(full_audio_folder_path): print(f"Folder not found: {full_audio_folder_path}") continue if folder_name not in data_map: data_map[folder_name] = [] ##get all wavs and mp3s from full_audio_folder_path for file in sorted(os.listdir(full_audio_folder_path), key=natural_sort_key): if ".DS_Store" in file: continue if file.endswith(".wav") or file.endswith(".mp3"): abs_path = os.path.join(full_audio_folder_path, file) data_map[folder_name].append(abs_path) final_df = pd.DataFrame(columns=["Text"].extend(folders_list)) final_df["Text"] = results_csv_df["Text"] cache_random_hash = str(time.time()) for folder_name in data_map: folder_files_list = data_map[folder_name] formatted_column = pd.Series(folder_files_list).apply( lambda file_path: f'' ) ## append column to dataframe final_df[folder_name] = formatted_column return final_df except Exception as e: print(f"An error occurred: {e}") gr.Warning(f"An error occurred: {e}") raise e with gr.Blocks(css=custom_css) as app: gr.HTML( f"""

Arabic TTS Benchmark


This benchmark represents a foundational step by SILMA.AI towards establishing a high-quality benchmark for Arabic TTS models. We have found that standard quantitative metrics (such as WER, CER, SIM, and UTMOS) are often insufficient for accurately capturing the nuances of Arabic speech. Consequently, for this release we have prioritized direct auditory assessment, allowing listeners to evaluate the naturalness and quality of the models themselves. More advanced releases will follow as we develop the necessary technology. You can also check our Open-source Arabic TTS Benchmark

""", elem_id="tool-header" ) with gr.Tabs(): with gr.TabItem("Modern Standard Arabic (MSA)"): folders_list = ["silmatts_v2_results","hamsa_msa_results","geminiflash3.1_results","elevenlab_results_v3","elevenlab_results", "gemini_results","chirp_arabic_results", "amazontts_results","openai_mini_results"] df_header = ["Text","SILMA TTS v2","Hamsa","Gemini 3.1 Flash","Eleven Labs V3","Eleven Labs V2","Gemini 2.5 Pro","Google Chirp","Amazon Polly","OpenAI TTS Mini"] data_df = get_data(folders_list, language="msa") data_df.columns = df_header data_viewer = gr.Dataframe( datatype=["html"]+(["html"] * len(folders_list)), value=data_df, column_widths=[200]+([200]*len(folders_list)), wrap=True, show_label=True, show_row_numbers=True, interactive=True, elem_classes="data_viewer_dataframe", show_search="search", visible=True, ) with gr.TabItem("KSA"): folders_list = ["silmatts_v2_ksa_results","hamsa_results","geminiflash3.1_results","elevenlabs_ksa_results","google_chirp_results","gemini_ksa_results"] df_header = ["Text","SILMA TTS v2 KSA","Hamsa","Google Gemini 3.1","Eleven Labs V3","Google TTS (Chirp)","Google Gemini 2.5 Pro"] data_df = get_data(folders_list, language="ksa") data_df.columns = df_header data_viewer = gr.Dataframe( datatype=["html"]+(["html"] * len(folders_list)), value=data_df, column_widths=[200,200,200,200,200], wrap=True, show_label=True, show_row_numbers=True, interactive=True, elem_classes="data_viewer_dataframe", show_search="search", visible=True ) with gr.Row(): ## add link for each model with gr.Column(): gr.Markdown("Benchmark model links:") gr.Markdown(""" - SILMA TTS - Hamsa - ElevenLabs - Google Gemini TTS - Google Chirp 3 - Amazon Polly - OpenAI TTS Mini """) def main(): global app print("Starting app...") app.queue().launch(favicon_path="images/silma-favicon.png", allowed_paths=[CURR_BASE_DIR+"/images/",CURR_BASE_DIR+"/results/"]) if __name__ == "__main__": main()