Delete nemo_asr

nemo_asr/run_eval.py

@@ -1,241 +0,0 @@
-import argparse
-
-import io
-import os
-import torch
-import evaluate
-import soundfile
-
-from tqdm import tqdm
-from normalizer import data_utils
-import numpy as np
-
-from nemo.collections.asr.models import ASRModel
-import time
-
-
-wer_metric = evaluate.load("wer")
-
-
-def main(args):
-
-    data_cache_root = args.data_cache_root if args.data_cache_root is not None else os.getcwd()
-    DATA_CACHE_DIR = os.path.join(data_cache_root, "audio_cache")
-    DATASET_NAME = args.dataset
-    SPLIT_NAME = args.split
-
-    CACHE_DIR = os.path.join(DATA_CACHE_DIR, DATASET_NAME, SPLIT_NAME)
-    if not os.path.exists(CACHE_DIR):
-        os.makedirs(CACHE_DIR)
-
-    if args.device >= 0:
-        device = torch.device(f"cuda:{args.device}")
-        compute_dtype=torch.bfloat16
-    else:
-        device = torch.device("cpu")
-        compute_dtype=torch.float32
-
-    if args.model_id.endswith(".nemo"):
-        asr_model = ASRModel.restore_from(args.model_id, map_location=device)
-    else:
-        asr_model = ASRModel.from_pretrained(args.model_id, map_location=device)  # type: ASRModel
-    
-    asr_model.to(compute_dtype)
-    asr_model.eval()
-    print(f"Model size: {sum(p.numel() for p in asr_model.parameters()) / 1e9:.2f}B parameters")
-
-    dataset = data_utils.load_data(args)
-
-    if args.max_eval_samples is not None and args.max_eval_samples > 0:
-        print(f"Subsampling dataset to first {args.max_eval_samples} samples !")
-        dataset = dataset.take(args.max_eval_samples)
-
-    # Prepare data FIRST - this casts audio to proper format with "array" and "sampling_rate" keys
-    dataset = data_utils.prepare_data(dataset)
-
-    def download_audio_files(batch):
-
-        # download audio files and write the paths, transcriptions and durations to a manifest file
-        audio_paths = []
-        original_audio_paths = []
-        durations = []
-        file_names = batch.get("file_name", [None] * len(batch["audio"]))
-
-        # Use 'id' column if available, otherwise generate sequential IDs
-        if "id" in batch:
-            ids = batch["id"]
-        else:
-            # Generate IDs based on index
-            start_idx = len([f for f in os.listdir(CACHE_DIR) if f.endswith('.wav')]) if os.path.exists(CACHE_DIR) else 0
-            ids = [f"sample_{start_idx + i}" for i in range(len(batch["audio"]))]
-
-        for id, file_name, audio_sample in zip(ids, file_names, batch["audio"]):
-
-            # first step added here to make ID and wav filenames unique
-            # several datasets like earnings22 have a hierarchical structure
-            # for eg. earnings22/test/4432298/281.wav, earnings22/test/4450488/281.wav
-            # lhotse uses the filename (281.wav) here as unique ID to create and name cuts
-            # ref: https://github.com/lhotse-speech/lhotse/blob/master/lhotse/dataset/collation.py#L186
-            original_id = id  # preserve before sanitization for use as audio_filepath
-            id = id.replace('/', '_').removesuffix('.wav')
-
-            audio_path = os.path.join(CACHE_DIR, f"{id}.wav")
-            audio_array = np.float32(audio_sample["array"])
-            sample_rate = audio_sample["sampling_rate"]
-
-            if not os.path.exists(audio_path):
-                os.makedirs(os.path.dirname(audio_path), exist_ok=True)
-                soundfile.write(audio_path, audio_array, sample_rate)
-
-            audio_paths.append(audio_path)
-            # Prefer the original file_name from the dataset; fall back to the
-            # sample id (before path-sanitization) so audio_filepath in the
-            # JSONL is always a meaningful identifier rather than "sample_N".
-            if file_name is not None:
-                original_audio_paths.append(os.path.basename(str(file_name)))
-            else:
-                original_audio_paths.append(original_id)
-            durations.append(len(audio_array) / sample_rate)
-
-        
-        batch["references"] = batch["norm_text"]
-        batch["audio_filepaths"] = audio_paths
-        batch["original_audio_filepaths"] = original_audio_paths
-        batch["durations"] = durations
-
-        return batch
-
-    if asr_model.cfg.decoding.strategy != "beam":
-        asr_model.cfg.decoding.strategy = "greedy_batch"
-        asr_model.change_decoding_strategy(asr_model.cfg.decoding)
-    
-    # prepraing the offline dataset
-    dataset = dataset.map(download_audio_files, batch_size=args.batch_size, batched=True, remove_columns=["audio"])
-
-    # Write manifest from daraset batch using json and keys audio_filepath, duration, text
-
-    all_data = {
-        "audio_filepaths": [],
-        "original_audio_filepaths": [],
-        "durations": [],
-        "references": [],
-    }
-
-    data_itr = iter(dataset)
-    for data in tqdm(data_itr, desc="Downloading Samples"):
-        for key in all_data:
-            all_data[key].append(data[key])
-    
-    # Sort audio_filepaths and references based on durations values
-    sorted_indices = sorted(range(len(all_data["durations"])), key=lambda k: all_data["durations"][k], reverse=True)
-    all_data["audio_filepaths"] = [all_data["audio_filepaths"][i] for i in sorted_indices]
-    all_data["original_audio_filepaths"] = [all_data["original_audio_filepaths"][i] for i in sorted_indices]
-    all_data["references"] = [all_data["references"][i] for i in sorted_indices]
-    all_data["durations"] = [all_data["durations"][i] for i in sorted_indices]
-    
-    
-    total_time = 0
-    for _ in range(2): # warmup once and calculate rtf
-        if _ == 0:
-            audio_files = all_data["audio_filepaths"][:args.batch_size * 4] # warmup with 4 batches
-        else:
-            audio_files = all_data["audio_filepaths"]
-        start_time = time.time()
-        with torch.inference_mode(), torch.no_grad(): 
-
-            if 'canary' in args.model_id and 'v2' not in args.model_id:
-                pnc = 'nopnc'
-            else:
-                pnc = 'pnc'
-                
-            if 'canary' in args.model_id:
-                transcriptions = asr_model.transcribe(audio_files, batch_size=args.batch_size, verbose=False, pnc=pnc, num_workers=1)
-            else:
-                transcriptions = asr_model.transcribe(audio_files, batch_size=args.batch_size, verbose=False, num_workers=1)
-        end_time = time.time()
-        if _ == 1:
-            total_time += end_time - start_time
-    total_time = total_time
-
-    # normalize transcriptions with English normalizer
-    if isinstance(transcriptions, tuple) and len(transcriptions) == 2:
-        transcriptions = transcriptions[0]
-    predictions = [data_utils.normalizer(pred.text) for pred in transcriptions]
-
-    avg_time = total_time / len(all_data["audio_filepaths"])
-
-    # Write manifest results (WER and RTFX)
-    manifest_path = data_utils.write_manifest(
-        all_data["references"],
-        predictions,
-        args.model_id,
-        args.dataset_path,
-        args.dataset,
-        args.split,
-        audio_length=all_data["durations"],
-        transcription_time=[avg_time] * len(all_data["audio_filepaths"]),
-        audio_filepaths=all_data["original_audio_filepaths"],
-    )
-
-    print("Results saved at path:", os.path.abspath(manifest_path))
-
-    wer = wer_metric.compute(references=all_data['references'], predictions=predictions)
-    wer = round(100 * wer, 2)
-
-    # transcription_time = sum(all_results["transcription_time"])
-    audio_length = sum(all_data["durations"])
-    rtfx = audio_length / total_time
-    rtfx = round(rtfx, 2)
-
-    print("RTFX:", rtfx)
-    print("WER:", wer, "%")
-
-
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser()
-
-    parser.add_argument(
-        "--model_id", type=str, required=True, help="Model identifier. Should be loadable with NVIDIA NeMo.",
-    )
-    parser.add_argument(
-        '--dataset_path', type=str, default='hf-audio/open-asr-leaderboard', help='Dataset path. By default, it is `hf-audio/open-asr-leaderboard`'
-    )
-    parser.add_argument(
-        '--data_cache_root', type=str, default=None, help='Root directory for audio cache. By default, it is the current working directory.'
-    )
-    parser.add_argument(
-        "--dataset",
-        type=str,
-        required=True,
-        help="Dataset name. *E.g.* `'librispeech_asr` for the LibriSpeech ASR dataset, or `'common_voice'` for Common Voice. The full list of dataset names "
-        "can be found at `https://huggingface.co/datasets/hf-audio/open-asr-leaderboard`",
-    )
-    parser.add_argument(
-        "--split",
-        type=str,
-        default="test",
-        help="Split of the dataset. *E.g.* `'validation`' for the dev split, or `'test'` for the test split.",
-    )
-    parser.add_argument(
-        "--device",
-        type=int,
-        default=-1,
-        help="The device to run the pipeline on. -1 for CPU (default), 0 for the first GPU and so on.",
-    )
-    parser.add_argument(
-        "--batch_size", type=int, default=32, help="Number of samples to go through each streamed batch.",
-    )
-    parser.add_argument(
-        "--max_eval_samples",
-        type=int,
-        default=None,
-        help="Number of samples to be evaluated. Put a lower number e.g. 64 for testing this script.",
-    )
-    parser.add_argument(
-        "--streaming",
-        action="store_true",
-        help="Stream the dataset lazily over the network instead of downloading it in full before the evaluation. Off by default for reproducible benchmark timings.",
-    )
-    args = parser.parse_args()
-
-    main(args)

nemo_asr/run_eval_long.py

@@ -1,224 +0,0 @@
-import argparse
-
-import io
-import os
-import torch
-import evaluate
-import soundfile
-
-from tqdm import tqdm
-from normalizer import data_utils
-import numpy as np
-
-from nemo.collections.asr.models import ASRModel
-import time
-
-
-wer_metric = evaluate.load("wer")
-
-
-def main(args):
-
-    DATA_CACHE_DIR = os.path.join(os.getcwd(), "audio_cache")
-    DATASET_NAME = args.dataset
-    SPLIT_NAME = args.split
-
-    CACHE_DIR = os.path.join(DATA_CACHE_DIR, DATASET_NAME, SPLIT_NAME)
-    if not os.path.exists(CACHE_DIR):
-        os.makedirs(CACHE_DIR)
-
-    if args.device >= 0:
-        device = torch.device(f"cuda:{args.device}")
-        compute_dtype=torch.bfloat16
-    else:
-        device = torch.device("cpu")
-        compute_dtype=torch.float32
-        
-
-    if args.model_id.endswith(".nemo"):
-        asr_model = ASRModel.restore_from(args.model_id, map_location=device)
-    else:
-        asr_model = ASRModel.from_pretrained(args.model_id, map_location=device)  # type: ASRModel
-       
-    if args.longform:
-        asr_model.change_attention_model("rel_pos_local_attn", [128, 128])  # local attn
-    asr_model.to(compute_dtype)
-    asr_model.eval()
-    print(f"Model size: {sum(p.numel() for p in asr_model.parameters()) / 1e9:.2f}B parameters")
-
-    dataset = data_utils.load_data(args)
-
-    def download_audio_files(batch, indices):
-
-        # download audio files and write the paths, transcriptions and durations to a manifest file
-        audio_paths = []
-        durations = []
-
-        # Use global indices for unique filenames across all batches
-        for global_idx, sample in zip(indices, batch["audio"]):
-            # Use a unique filename based on global index
-            audio_path = os.path.join(CACHE_DIR, f"sample_{global_idx}.wav")
-
-            if "array" in sample:
-                audio_array = np.float32(sample["array"])
-                sample_rate = 16000
-
-            elif "bytes" in sample: # added to be compatible with latest datasets library (3.x.x) that produces byte stream
-                with io.BytesIO(sample["bytes"]) as audio_file:
-                    audio_array, sample_rate = soundfile.read(audio_file, dtype="float32")
-
-            else:
-                raise ValueError("Sample must have either 'array' or 'bytes' key")
-
-            if not os.path.exists(audio_path):
-                os.makedirs(os.path.dirname(audio_path), exist_ok=True)
-                soundfile.write(audio_path, audio_array, sample_rate)
-
-            audio_paths.append(audio_path)
-            durations.append(len(audio_array) / sample_rate)
-
-        
-        batch["references"] = batch["norm_text"]
-        batch["audio_filepaths"] = audio_paths
-        batch["durations"] = durations
-
-        return batch
-
-
-    if args.max_eval_samples is not None and args.max_eval_samples > 0:
-        print(f"Subsampling dataset to first {args.max_eval_samples} samples !")
-        dataset = dataset.take(args.max_eval_samples)
-
-    dataset = data_utils.prepare_data(dataset)
-    if asr_model.cfg.decoding.strategy != "beam":
-        asr_model.cfg.decoding.strategy = "greedy_batch"
-        asr_model.change_decoding_strategy(asr_model.cfg.decoding)
-    
-    # prepraing the offline dataset
-    dataset = dataset.map(download_audio_files, batch_size=args.batch_size, batched=True, with_indices=True, remove_columns=["audio"])
-
-    # Write manifest from daraset batch using json and keys audio_filepath, duration, text
-
-    all_data = {
-        "audio_filepaths": [],
-        "durations": [],
-        "references": [],
-    }
-
-    data_itr = iter(dataset)
-    for data in tqdm(data_itr, desc="Downloading Samples"):
-        for key in all_data:
-            all_data[key].append(data[key])
-    
-    # Sort audio_filepaths and references based on durations values
-    sorted_indices = sorted(range(len(all_data["durations"])), key=lambda k: all_data["durations"][k], reverse=True)
-    all_data["audio_filepaths"] = [all_data["audio_filepaths"][i] for i in sorted_indices]
-    all_data["references"] = [all_data["references"][i] for i in sorted_indices]
-    all_data["durations"] = [all_data["durations"][i] for i in sorted_indices]
-
-    total_time = 0
-    for _ in range(2): # warmup once and calculate rtf
-        if _ == 0:
-            audio_files = all_data["audio_filepaths"][:args.batch_size * 4] # warmup with 4 batches
-        else:
-            audio_files = all_data["audio_filepaths"]
-        start_time = time.time()
-        with torch.inference_mode(), torch.no_grad(): 
-
-            if 'canary' in args.model_id and 'v2' not in args.model_id:
-                pnc = 'nopnc'
-            else:
-                pnc = 'pnc'
-                
-            if 'canary' in args.model_id:
-                transcriptions = asr_model.transcribe(audio_files, batch_size=args.batch_size, verbose=False, pnc=pnc, num_workers=1)
-            else:
-                transcriptions = asr_model.transcribe(audio_files, batch_size=args.batch_size, verbose=False, num_workers=1)
-        end_time = time.time()
-        if _ == 1:
-            total_time += end_time - start_time
-    total_time = total_time
-
-    # normalize transcriptions with English normalizer
-    if isinstance(transcriptions, tuple) and len(transcriptions) == 2:
-        transcriptions = transcriptions[0]
-    predictions = [data_utils.normalizer(pred.text) for pred in transcriptions]
-
-    avg_time = total_time / len(all_data["audio_filepaths"])
-
-    # Write manifest results (WER and RTFX)
-    manifest_path = data_utils.write_manifest(
-        all_data["references"],
-        predictions,
-        args.model_id,
-        args.dataset_path,
-        args.dataset,
-        args.split,
-        audio_length=all_data["durations"],
-        transcription_time=[avg_time] * len(all_data["audio_filepaths"]),
-    )
-
-    print("Results saved at path:", os.path.abspath(manifest_path))
-
-    wer = wer_metric.compute(references=all_data['references'], predictions=predictions)
-    wer = round(100 * wer, 2)
-
-    # transcription_time = sum(all_results["transcription_time"])
-    audio_length = sum(all_data["durations"])
-    rtfx = audio_length / total_time
-    rtfx = round(rtfx, 2)
-
-    print("RTFX:", rtfx)
-    print("WER:", wer, "%")
-
-
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser()
-
-    parser.add_argument(
-        "--model_id", type=str, required=True, help="Model identifier. Should be loadable with NVIDIA NeMo.",
-    )
-    parser.add_argument(
-        '--dataset_path', type=str, default='hf-audio/open-asr-leaderboard', help='Dataset path. By default, it is `hf-audio/open-asr-leaderboard`'
-    )
-    parser.add_argument(
-        "--dataset",
-        type=str,
-        required=True,
-        help="Dataset name. *E.g.* `'librispeech_asr` for the LibriSpeech ASR dataset, or `'common_voice'` for Common Voice. The full list of dataset names "
-        "can be found at `https://huggingface.co/datasets/hf-audio/open-asr-leaderboard`",
-    )
-    parser.add_argument(
-        "--split",
-        type=str,
-        default="test",
-        help="Split of the dataset. *E.g.* `'validation`' for the dev split, or `'test'` for the test split.",
-    )
-    parser.add_argument(
-        "--device",
-        type=int,
-        default=-1,
-        help="The device to run the pipeline on. -1 for CPU (default), 0 for the first GPU and so on.",
-    )
-    parser.add_argument(
-        "--batch_size", type=int, default=32, help="Number of samples to go through each streamed batch.",
-    )
-    parser.add_argument(
-        "--max_eval_samples",
-        type=int,
-        default=None,
-        help="Number of samples to be evaluated. Put a lower number e.g. 64 for testing this script.",
-    )
-    parser.add_argument(
-        "--streaming",
-        action="store_true",
-        help="Stream the dataset lazily over the network instead of downloading it in full before the evaluation. Off by default for reproducible benchmark timings.",
-    )
-    parser.add_argument(
-        "--longform",
-        action="store_true",
-        help="Whether to use longform mode.",
-    )
-    args = parser.parse_args()
-
-    main(args)

nemo_asr/run_eval_ml.py

@@ -1,280 +0,0 @@
-# This script is used to evaluate NeMo ASR models on the Multi-Lingual datasets
-
-import argparse
-import io
-import os
-os.environ["DATASETS_USE_TORCHCODEC"] = "0"
-import torch
-import evaluate
-import soundfile
-import numpy as np
-from tqdm import tqdm
-from datasets import load_dataset
-from normalizer import data_utils
-from normalizer.eval_utils import normalize_compound_pairs
-from nemo.collections.asr.models import ASRModel
-from omegaconf import OmegaConf
-import time
-
-
-wer_metric = evaluate.load("wer")
-
-
-def main(args):
-    DATA_CACHE_DIR = os.path.join(os.getcwd(), "audio_cache")
-    CONFIG_NAME = args.config_name
-    SPLIT_NAME = args.split
-    
-    # Extract language from config_name if not provided
-    if args.language:
-        LANGUAGE = args.language
-    else:
-        # Extract language from config_name (e.g., "fleurs_en" -> "en")
-        try:
-            LANGUAGE = CONFIG_NAME.split('_', 1)[1]
-        except IndexError:
-            LANGUAGE = "en"  # Default fallback
-    
-    print(f"Detected language: {LANGUAGE}")
-
-    CACHE_DIR = os.path.join(DATA_CACHE_DIR, CONFIG_NAME, SPLIT_NAME)
-    if not os.path.exists(CACHE_DIR):
-        os.makedirs(CACHE_DIR)
-
-    if args.device >= 0:
-        device = torch.device(f"cuda:{args.device}")
-        compute_dtype = torch.bfloat16
-    else:
-        device = torch.device("cpu")
-        compute_dtype = torch.float32
-
-    # Load ASR model
-    if args.model_id.endswith(".nemo"):
-        asr_model = ASRModel.restore_from(args.model_id, map_location=device)
-    else:
-        asr_model = ASRModel.from_pretrained(args.model_id, map_location=device)
-    
-    asr_model.to(compute_dtype)
-    asr_model.eval()
-    print(f"Model size: {sum(p.numel() for p in asr_model.parameters()) / 1e9:.2f}B parameters")
-
-    # Load dataset using the HuggingFace dataset repository
-    print(f"Loading dataset: {args.dataset} with config: {CONFIG_NAME}")
-
-    dataset = load_dataset(args.dataset, CONFIG_NAME, split=SPLIT_NAME, streaming=args.streaming)
-    
-    if args.max_eval_samples is not None and args.max_eval_samples > 0:
-        print(f"Subsampling dataset to first {args.max_eval_samples} samples!")
-        dataset = dataset.select(range(min(args.max_eval_samples, len(dataset))))
-
-    # Configure decoding strategy
-    if asr_model.cfg.decoding.strategy != "beam":
-        asr_model.cfg.decoding.strategy = "greedy_batch"
-        if hasattr(asr_model.cfg.decoding, "greedy"):
-            OmegaConf.update(asr_model.cfg.decoding, "greedy.use_cuda_graph_decoder", False, force_add=True)
-        asr_model.change_decoding_strategy(asr_model.cfg.decoding)
-
-    def download_audio_files(batch):
-        """Process audio files and prepare them for evaluation."""
-        audio_paths = []
-        durations = []
-
-        for i, (file_name, sample, duration, text) in enumerate(zip(
-            batch["file_name"], batch["audio"], batch["duration"], batch["text"]
-        )):
-            # Create unique filename using index to avoid conflicts
-            unique_id = f"{CONFIG_NAME}_{i}_{os.path.basename(file_name).replace('.wav', '')}"
-            audio_path = os.path.join(CACHE_DIR, f"{unique_id}.wav")
-
-            if "array" in sample:
-                audio_array = np.float32(sample["array"])
-                sample_rate = sample.get("sampling_rate", 16000)
-            elif "bytes" in sample:
-                with io.BytesIO(sample["bytes"]) as audio_file:
-                    audio_array, sample_rate = soundfile.read(audio_file, dtype="float32")
-            else:
-                raise ValueError("Sample must have either 'array' or 'bytes' key")
-
-            if not os.path.exists(audio_path):
-                os.makedirs(os.path.dirname(audio_path), exist_ok=True)
-                soundfile.write(audio_path, audio_array, sample_rate)
-
-            audio_paths.append(audio_path)
-            # Use duration from dataset if available, otherwise calculate
-            if duration is not None:
-                durations.append(duration)
-            else:
-                durations.append(len(audio_array) / sample_rate)
-
-        batch["references"] = [text for text in batch["text"]]
-        batch["audio_filepaths"] = audio_paths
-        batch["durations"] = durations
-
-        return batch
-
-    # Process the dataset
-    print("Processing audio files...")
-    dataset = dataset.map(
-        download_audio_files, 
-        batch_size=args.batch_size, 
-        batched=True, 
-        remove_columns=["audio"]
-    )
-
-    # Collect all data
-    all_data = {
-        "audio_filepaths": [],
-        "durations": [],
-        "references": [],
-    }
-
-    print("Collecting data...")
-    for data in tqdm(dataset, desc="Collecting samples"):
-        all_data["audio_filepaths"].append(data["audio_filepaths"])
-        all_data["durations"].append(data["durations"])
-        all_data["references"].append(data["references"])
-
-    # Sort by duration for efficient batch processing
-    print("Sorting by duration...")
-    sorted_indices = sorted(range(len(all_data["durations"])), key=lambda k: all_data["durations"][k], reverse=True)
-    all_data["audio_filepaths"] = [all_data["audio_filepaths"][i] for i in sorted_indices]
-    all_data["references"] = [all_data["references"][i] for i in sorted_indices]
-    all_data["durations"] = [all_data["durations"][i] for i in sorted_indices]
-
-    # Run evaluation with warmup
-    total_time = 0
-    for warmup_round in range(2):  # warmup once and calculate rtf
-        if warmup_round == 0:
-            audio_files = all_data["audio_filepaths"][:args.batch_size * 4]  # warmup with 4 batches
-            print("Running warmup...")
-        else:
-            audio_files = all_data["audio_filepaths"]
-            print("Running full evaluation...")
-            
-        start_time = time.time()
-        with torch.inference_mode(), torch.no_grad():
-            # for canary-1b and canary-1b-flash, we need to set pnc='no' for English and for other languages, we need to set pnc='pnc' but for canary-1b-v2 pnc='yes' for all languages
-            if 'canary' in args.model_id and 'v2' not in args.model_id:
-                pnc = 'nopnc' if LANGUAGE == "en" else 'pnc'
-            else:
-                pnc = 'pnc'
-
-            if 'canary' in args.model_id:
-                transcriptions = asr_model.transcribe(audio_files, batch_size=args.batch_size, verbose=False, pnc=pnc, num_workers=1, source_lang=LANGUAGE, target_lang=LANGUAGE)
-            else:
-                transcriptions = asr_model.transcribe(audio_files, batch_size=args.batch_size, verbose=False, num_workers=1)
-        end_time = time.time()
-        
-        if warmup_round == 1:
-            total_time = end_time - start_time
-
-    # Process transcriptions
-    if isinstance(transcriptions, tuple) and len(transcriptions) == 2:
-        transcriptions = transcriptions[0]
-    
-    references = all_data["references"] 
-    if LANGUAGE == "en": # English is handled by the English normalizer
-        references = [data_utils.normalizer(ref) for ref in references]
-        predictions = [data_utils.normalizer(pred.text) for pred in transcriptions]
-    else:
-        references = [data_utils.ml_normalizer(ref, lang=LANGUAGE) for ref in references]
-        predictions = [data_utils.ml_normalizer(pred.text, lang=LANGUAGE) for pred in transcriptions]
-
-    # Filter empty references (consistent with English pipeline)
-    filtered = [
-        (ref, pred, dur)
-        for ref, pred, dur in zip(references, predictions, all_data["durations"])
-        if data_utils.is_target_text_in_range(ref)
-    ]
-    if filtered:
-        references, predictions, all_data["durations"] = zip(*filtered)
-        references, predictions = list(references), list(predictions)
-        all_data["durations"] = list(all_data["durations"])
-
-    avg_time = total_time / len(all_data["audio_filepaths"])
-
-    # Write results using eval_utils.write_manifest
-    manifest_path = data_utils.write_manifest(
-        references,
-        predictions,
-        args.model_id,
-        args.dataset,  # dataset_path for filename
-        CONFIG_NAME,  # dataset_name
-        SPLIT_NAME,
-        audio_length=all_data["durations"],
-        transcription_time=[avg_time] * len(all_data["audio_filepaths"]),
-    )
-
-    print("Results saved at path:", os.path.abspath(manifest_path))
-
-    # Calculate metrics
-    wer_refs, wer_preds = normalize_compound_pairs(references, predictions)
-    wer = wer_metric.compute(references=wer_refs, predictions=wer_preds)
-    wer = round(100 * wer, 2)
-
-    audio_length = sum(all_data["durations"])
-    rtfx = audio_length / total_time
-    rtfx = round(rtfx, 2)
-
-    print(f"Dataset: {args.dataset}")
-    print(f"Language: {LANGUAGE}")
-    print(f"Config: {CONFIG_NAME}")
-    print(f"Model: {args.model_id}")
-    print(f"RTFX: {rtfx}")
-    print(f"WER: {wer}%")
-
-
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser()
-
-    parser.add_argument(
-        "--model_id", type=str, required=True, help="Model identifier. Should be loadable with NVIDIA NeMo.",
-    )
-    parser.add_argument(
-        "--dataset",
-        type=str,
-        default="nithinraok/asr-leaderboard-datasets",
-        help="Dataset name. Default is 'nithinraok/asr-leaderboard-datasets'"
-    )
-    parser.add_argument(
-        "--config_name",
-        type=str,
-        required=True,
-        help="Config name in format <dataset>_<lang> (e.g., fleurs_en, mcv_de, mls_es)"
-    )
-    parser.add_argument(
-        "--language",
-        type=str,
-        default=None,
-        help="Language code (e.g., en, de, es). If not provided, will be extracted from config_name."
-    )
-    parser.add_argument(
-        "--split",
-        type=str,
-        default="test",
-        help="Split of the dataset. Default is 'test'.",
-    )
-    parser.add_argument(
-        "--device",
-        type=int,
-        default=-1,
-        help="The device to run the pipeline on. -1 for CPU (default), 0 for the first GPU and so on.",
-    )
-    parser.add_argument(
-        "--batch_size", type=int, default=32, help="Number of samples to go through each streamed batch.",
-    )
-    parser.add_argument(
-        "--max_eval_samples",
-        type=int,
-        default=None,
-        help="Number of samples to be evaluated. Put a lower number e.g. 64 for testing this script.",
-    )
-
-    parser.add_argument(
-        "--streaming",
-        action="store_true",
-        help="Stream the dataset lazily over the network instead of downloading it in full before the evaluation. Off by default for reproducible benchmark timings.",
-    )
-    args = parser.parse_args()
-
-    main(args) 

nemo_asr/run_eval_salm.py

@@ -1,276 +0,0 @@
-import argparse
-
-import io
-import os
-import torch
-import evaluate
-import soundfile
-import lhotse
-
-from tqdm import tqdm
-from normalizer import data_utils
-import numpy as np
-
-from nemo.collections.asr.models import ASRModel
-import time
-
-
-from nemo.collections.speechlm2.models.salm import SALM
-from omegaconf import OmegaConf
-from pathlib import Path
-from transformers import GenerationConfig
-
-
-
-wer_metric = evaluate.load("wer")
-
-
-class ToAudio(torch.utils.data.Dataset):
-    def __getitem__(self, cuts):
-        cuts = lhotse.CutSet([c.to_mono(mono_downmix=True) if isinstance(c, lhotse.MultiCut) else c for c in cuts])
-        audios, audio_lens = cuts.load_audio(collate=True)
-        return {"cuts": cuts, "audios": audios, "audio_lens": audio_lens}
-
-
-def setup_dloader(audio_files, batch_size, num_workers):
-    cuts = lhotse.CutSet([lhotse.Recording.from_file(p).to_cut() for p in audio_files])
-    cuts = cuts.resample(16000)
-    return torch.utils.data.DataLoader(
-        dataset=ToAudio(),
-        sampler=lhotse.dataset.DynamicCutSampler(cuts, max_cuts=batch_size),
-        num_workers=num_workers,
-        batch_size=None,
-    )
-
-
-def transcribe(model, dloader) -> list[str]:
-    hyps = []
-    eos_tokens = torch.tensor([model.text_eos_id])
-    for batch_idx, batch in enumerate(dloader):
-        answer_ids = model.generate(
-            prompts=[
-                [
-                    {"role": "user", "slots": {"message": f"Transcribe the following: {model.audio_locator_tag}"}}
-                ]
-            ] * len(batch["cuts"]),
-            audios=batch["audios"].to(model.device, non_blocking=True),
-            audio_lens=batch["audio_lens"].to(model.device, non_blocking=True),
-            generation_config=GenerationConfig(
-                max_new_tokens=128,
-                bos_token_id=model.text_bos_id,
-                eos_token_id=eos_tokens,
-                pad_token_id=model.text_pad_id,
-            ),
-        )
-        answer_ids = [parse_hyp(ans, eos_tokens) for ans in answer_ids.cpu()]
-        hyps.extend(model.tokenizer.ids_to_text(ans).strip() for ans in answer_ids)
-    return hyps
-
-
-def parse_hyp(answer: torch.Tensor, eos_tokens):
-    end = (answer == torch.isin(answer, eos_tokens)).nonzero(as_tuple=True)[0]
-    if end.numel() == 0:
-        return answer
-    end = end[0]
-    return answer[:end]
-
-
-def main(args):
-
-    data_cache_root = args.data_cache_root if args.data_cache_root is not None else os.getcwd()
-    DATA_CACHE_DIR = os.path.join(data_cache_root, "audio_cache")
-    DATASET_NAME = args.dataset
-    SPLIT_NAME = args.split
-
-    CACHE_DIR = os.path.join(DATA_CACHE_DIR, DATASET_NAME, SPLIT_NAME)
-    if not os.path.exists(CACHE_DIR):
-        os.makedirs(CACHE_DIR)
-
-    torch.set_float32_matmul_precision("medium")
-
-    device = torch.device(f"cuda:{args.device}")
-    model = SALM.from_pretrained(args.model_id).eval().to(torch.bfloat16).to(device)
-    print(f"Model size: {sum(p.numel() for p in model.parameters()) / 1e9:.2f}B parameters")
-
-    dataset = data_utils.load_data(args)
-
-    def download_audio_files(batch):
-
-        # download audio files and write the paths, transcriptions and durations to a manifest file
-        audio_paths = []
-        original_audio_paths = []
-        durations = []
-        file_names = batch.get("file_name", [None] * len(batch["audio"]))
-
-        # Use 'id' column if available, otherwise generate sequential IDs
-        if "id" in batch:
-            ids = batch["id"]
-        else:
-            # Generate IDs based on index
-            start_idx = len([f for f in os.listdir(CACHE_DIR) if f.endswith('.wav')]) if os.path.exists(CACHE_DIR) else 0
-            ids = [f"sample_{start_idx + i}" for i in range(len(batch["audio"]))]
-
-        for id, file_name, sample in zip(ids, file_names, batch["audio"]):
-
-            # first step added here to make ID and wav filenames unique
-            # several datasets like earnings22 have a hierarchical structure
-            # for eg. earnings22/test/4432298/281.wav, earnings22/test/4450488/281.wav
-            # lhotse uses the filename (281.wav) here as unique ID to create and name cuts
-            # ref: https://github.com/lhotse-speech/lhotse/blob/master/lhotse/dataset/collation.py#L186
-            original_id = id  # preserve before sanitization for use as audio_filepath
-            id = id.replace('/', '_').removesuffix('.wav')
-
-            audio_path = os.path.join(CACHE_DIR, f"{id}.wav")
-            audio_array = np.float32(sample["array"])
-            sample_rate = sample["sampling_rate"]
-
-            if not os.path.exists(audio_path):
-                os.makedirs(os.path.dirname(audio_path), exist_ok=True)
-                soundfile.write(audio_path, audio_array, sample_rate)
-
-            audio_paths.append(audio_path)
-            if file_name is not None:
-                original_audio_paths.append(os.path.basename(str(file_name)))
-            else:
-                original_audio_paths.append(original_id)
-            durations.append(len(audio_array) / sample_rate)
-
-
-        batch["references"] = batch["norm_text"]
-        batch["audio_filepaths"] = audio_paths
-        batch["original_audio_filepaths"] = original_audio_paths
-        batch["durations"] = durations
-
-        return batch
-
-
-    if args.max_eval_samples is not None and args.max_eval_samples > 0:
-        print(f"Subsampling dataset to first {args.max_eval_samples} samples !")
-        dataset = dataset.take(args.max_eval_samples)
-
-    dataset = data_utils.prepare_data(dataset)
-
-    # prepraing the offline dataset
-    dataset = dataset.map(download_audio_files, batch_size=args.batch_size, batched=True, remove_columns=["audio"])
-
-    # Write manifest from daraset batch using json and keys audio_filepath, duration, text
-
-    all_data = {
-        "audio_filepaths": [],
-        "original_audio_filepaths": [],
-        "durations": [],
-        "references": [],
-    }
-
-    data_itr = iter(dataset)
-    for data in tqdm(data_itr, desc="Downloading Samples"):
-        for key in all_data:
-            all_data[key].append(data[key])
-
-    # Sort audio_filepaths and references based on durations values
-    sorted_indices = sorted(range(len(all_data["durations"])), key=lambda k: all_data["durations"][k], reverse=True)
-    all_data["audio_filepaths"] = [all_data["audio_filepaths"][i] for i in sorted_indices]
-    all_data["original_audio_filepaths"] = [all_data["original_audio_filepaths"][i] for i in sorted_indices]
-    all_data["references"] = [all_data["references"][i] for i in sorted_indices]
-    all_data["durations"] = [all_data["durations"][i] for i in sorted_indices]
-
-
-    total_time = 0
-    for _ in range(2): # warmup once and calculate rtf
-        if _ == 0:
-            audio_files = all_data["audio_filepaths"][:args.batch_size * 4] # warmup with 4 batches
-        else:
-            audio_files = all_data["audio_filepaths"]
-        dloader = setup_dloader(audio_files=audio_files, batch_size=args.batch_size, num_workers=1)
-        with torch.inference_mode():
-            start_time = time.time()
-            transcriptions = transcribe(model, dloader)
-            end_time = time.time()
-        if _ == 1:
-            total_time += end_time - start_time
-    total_time = total_time
-
-    # normalize transcriptions with English normalizer
-    if isinstance(transcriptions, tuple) and len(transcriptions) == 2:
-        transcriptions = transcriptions[0]
-    predictions = [data_utils.normalizer(pred) for pred in transcriptions]
-
-    avg_time = total_time / len(all_data["audio_filepaths"])
-
-    # Write manifest results (WER and RTFX)
-    manifest_path = data_utils.write_manifest(
-        all_data["references"],
-        predictions,
-        args.model_id,
-        args.dataset_path,
-        args.dataset,
-        args.split,
-        audio_length=all_data["durations"],
-        transcription_time=[avg_time] * len(all_data["audio_filepaths"]),
-        audio_filepaths=all_data["original_audio_filepaths"],
-    )
-
-    print("Results saved at path:", os.path.abspath(manifest_path))
-
-    wer = wer_metric.compute(references=all_data['references'], predictions=predictions)
-    wer = round(100 * wer, 2)
-
-    audio_length = sum(all_data["durations"])
-    rtfx = audio_length / total_time
-    rtfx = round(rtfx, 2)
-
-    print("RTFX:", rtfx)
-    print("WER:", wer, "%")
-
-
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser()
-
-    parser.add_argument(
-        "--model_id", type=str, required=True, help="Model identifier. Should be loadable with NVIDIA NeMo.",
-    )
-    parser.add_argument(
-        '--dataset_path', type=str, default='hf-audio/open-asr-leaderboard', help='Dataset path. By default, it is `hf-audio/open-asr-leaderboard`'
-    )
-    parser.add_argument(
-        "--dataset",
-        type=str,
-        required=True,
-        help="Dataset name. *E.g.* `'librispeech_asr` for the LibriSpeech ASR dataset, or `'common_voice'` for Common Voice. The full list of dataset names "
-        "can be found at `https://huggingface.co/datasets/hf-audio/open-asr-leaderboard`",
-    )
-    parser.add_argument(
-        "--split",
-        type=str,
-        default="test",
-        help="Split of the dataset. *E.g.* `'validation`' for the dev split, or `'test'` for the test split.",
-    )
-    parser.add_argument(
-        "--device",
-        type=int,
-        default=-1,
-        help="The device to run the pipeline on. -1 for CPU (default), 0 for the first GPU and so on.",
-    )
-    parser.add_argument(
-        "--batch_size", type=int, default=32, help="Number of samples to go through each streamed batch.",
-    )
-    parser.add_argument(
-        "--max_eval_samples",
-        type=int,
-        default=None,
-        help="Number of samples to be evaluated. Put a lower number e.g. 64 for testing this script.",
-    )
-    parser.add_argument(
-        "--streaming",
-        action="store_true",
-        help="Stream the dataset lazily over the network instead of downloading it in full before the evaluation. Off by default for reproducible benchmark timings.",
-    )
-    parser.add_argument(
-        "--data_cache_root",
-        type=str,
-        default=None,
-        help="Root directory for caching audio files. Defaults to 'audio_cache' in current directory.",
-    )
-    args = parser.parse_args()
-
-    main(args)