Upload folder using huggingface_hub

.gitattributes

@@ -1,35 +1,4 @@
-*.7z filter=lfs diff=lfs merge=lfs -text
-*.arrow filter=lfs diff=lfs merge=lfs -text
-*.bin filter=lfs diff=lfs merge=lfs -text
-*.bz2 filter=lfs diff=lfs merge=lfs -text
-*.ckpt filter=lfs diff=lfs merge=lfs -text
-*.ftz filter=lfs diff=lfs merge=lfs -text
-*.gz filter=lfs diff=lfs merge=lfs -text
-*.h5 filter=lfs diff=lfs merge=lfs -text
-*.joblib filter=lfs diff=lfs merge=lfs -text
-*.lfs.* filter=lfs diff=lfs merge=lfs -text
-*.mlmodel filter=lfs diff=lfs merge=lfs -text
-*.model filter=lfs diff=lfs merge=lfs -text
-*.msgpack filter=lfs diff=lfs merge=lfs -text
-*.npy filter=lfs diff=lfs merge=lfs -text
-*.npz filter=lfs diff=lfs merge=lfs -text
-*.onnx filter=lfs diff=lfs merge=lfs -text
-*.ot filter=lfs diff=lfs merge=lfs -text
-*.parquet filter=lfs diff=lfs merge=lfs -text
-*.pb filter=lfs diff=lfs merge=lfs -text
-*.pickle filter=lfs diff=lfs merge=lfs -text
-*.pkl filter=lfs diff=lfs merge=lfs -text
-*.pt filter=lfs diff=lfs merge=lfs -text
-*.pth filter=lfs diff=lfs merge=lfs -text
-*.rar filter=lfs diff=lfs merge=lfs -text
-*.safetensors filter=lfs diff=lfs merge=lfs -text
-saved_model/**/* filter=lfs diff=lfs merge=lfs -text
-*.tar.* filter=lfs diff=lfs merge=lfs -text
-*.tar filter=lfs diff=lfs merge=lfs -text
-*.tflite filter=lfs diff=lfs merge=lfs -text
-*.tgz filter=lfs diff=lfs merge=lfs -text
-*.wasm filter=lfs diff=lfs merge=lfs -text
-*.xz filter=lfs diff=lfs merge=lfs -text
-*.zip filter=lfs diff=lfs merge=lfs -text
-*.zst filter=lfs diff=lfs merge=lfs -text
-*tfevents* filter=lfs diff=lfs merge=lfs -text
+mix-clean/backbone/models/checkpoint_150.pt filter=lfs diff=lfs merge=lfs -text
+mix-clean/kce/epoch_149.pt filter=lfs diff=lfs merge=lfs -text
+backbone/models/checkpoint_150.pt filter=lfs diff=lfs merge=lfs -text
+kce/epoch_149.pt filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,18 @@
+# DAE-TSE Pretrained Models
+
+## KCE (Keyword-guided Cue Encoder)
+
+- **File**: `kce/epoch_149.pt`
+- **Architecture**: `AEDKWSASRPhone`
+- **Training data**: LibriSpeech train-clean-360 + train-other-500
+- **Validation data**: LibriSpeech train-clean-100
+- **Config**: `kce/model.yaml`, `kce/data.yaml`
+
+## Backbone (DAE-BSRNN)
+
+- **File**: `backbone/models/checkpoint_150.pt`
+- **Architecture**: `DAE-BSRNN` (text-aware BSRNN with TC-ASR encoder)
+- **Training data**: Libri2Mix-100 mix-clean
+- **Config**: `backbone/config.yaml`
+
+> After downloading, update `model_args.tse_model.asr_encoder.checkpoint_dir` in `backbone/config.yaml` to the absolute path of `kce/epoch_149.pt`.

backbone/config.yaml

@@ -0,0 +1,106 @@
+clip_grad: 5.0
+config: confs/whole_utt/dae-bsrnn.yaml
+data_type: shard
+dataloader_args:
+  batch_size: 1
+  drop_last: true
+  num_workers: 1
+  pin_memory: true
+  prefetch_factor: 6
+dataset_args:
+  SSA_enroll_prob: 0
+  additional:
+    train:
+      data_list: data/text_cue/dump/ready/train-100.jsonl
+      data_type: TC-ASR
+    val:
+      data_list: data/text_cue/dump/ready/dev.jsonl
+      data_type: TC-ASR
+  chunk_len: 48000
+  fbank_args:
+    dither: 1.0
+    frame_length: 25
+    frame_shift: 10
+    num_mel_bins: 80
+  noise_enroll_prob: 0
+  noise_lmdb_file: ./data/musan/lmdb
+  noise_prob: 0
+  resample_rate: 16000
+  reverb_enroll_prob: 0
+  sample_num_per_epoch: 0
+  shuffle: false
+  shuffle_args:
+    shuffle_size: 2500
+  speaker_feat: true
+  specaug_enroll_prob: 0
+enable_amp: false
+exp_dir: exp/whole_utt/dae-bsrnn
+gpus:
+- 0
+- 1
+- 2
+- 3
+- 4
+- 5
+- 6
+- 7
+log_batch_interval: 100
+loss: SISDR
+loss_args: {}
+model:
+  tse_model: DAE-BSRNN
+model_args:
+  tse_model:
+    cue_emb_dim: 128
+    asr_encoder:
+      model_name: 'TCASREncoder'
+      checkpoint_path: exp/kce/epoch_149.pt #'checkpoint directory to the kce encoder', e.g. Pretrained/mix-clean/kce/epoch_149.pt
+      fnn:
+        input_dim: 256
+        num_layer: 3
+        output_dim: 128
+      multi_task: false
+      spksInTrain: 251
+    cue_fuse:
+      fuse_type: FiLM
+      multi_fuse: false
+    feature_dim: 128
+    multi_task: false
+    num_repeat: 6
+    spk_optim_head: asr
+    sr: 16000
+    stride: 128
+    win: 512
+model_init:
+  discriminator: null
+  spk_model: null
+  tse_model: null
+num_avg: 10
+num_epochs: 150
+optimizer:
+  tse_model: Adam
+optimizer_args:
+  tse_model:
+    lr: 0.001
+    weight_decay: 0.0001
+save_epoch_interval: 1
+scheduler:
+  tse_model: ExponentialDecrease
+scheduler_args:
+  scale_ratio: 1.0
+  tse_model:
+    epoch_iter: 1737
+    final_lr: 2.5e-05
+    initial_lr: 0.001
+    num_epochs: 150
+    warm_from_zero: false
+    warm_up_epoch: 0
+seed: 42
+train_data: data/clean/train-100/shard.list
+train_spk2utt: data/clean/train-100/spk2enroll.json
+train_utt2spk: data/clean/train-100/single.utt2spk
+val_data: data/clean/dev/shard.list
+val_spk1_enroll: data/clean/dev/spk1.enroll
+val_spk2_enroll: data/clean/dev/spk2.enroll
+val_spk2utt: data/clean/dev/single.wav.scp
+whole_utt: true

backbone/models/checkpoint_150.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:404a63efc4c6df122c2902a0b72339ce86725e685c978357d8ce3bb81b4a911e
+size 220761736

kce/data.yaml

@@ -0,0 +1,71 @@
+batch_size: 48
+epoch: 150
+fetch_key: mixspeech,mixspeech_len,phn_label,phn_label_len,keyword,keyword_len,bpe_label,bpe_label_len,target,spk_label
+keyword_selection_conf:
+  config:
+    positive_prob: 1
+    sample_config:
+      continuous: true
+      keyword_length_range:
+      - 2
+      - 6
+    special_token:
+      eok: 5003
+      peok: 72
+      psok: 71
+      punk: 73
+      sok: 5002
+      unk: 1
+      with_trans: true
+    use_filler_label_for_neg: false
+  selection_strategy: random_sample
+length_key: mixspeech,speech,keyword
+meta_save_config:
+  save_dir: xxx
+  use: false
+num_workers: 9
+shuffle: true
+speech_config:
+  data_type: raw
+  feats_config:
+    frame_length: 25
+    frame_shift: 10
+    num_mel_bins: 80
+  feats_type: fbank
+  mix_config:
+    max_or_min: max
+    wav_shift_config:
+      sampler: uniform
+      sampler_config:
+        high: 1.0
+        low: 0.0
+      shift_seconds: 3
+      wav_shift_prob: 0.3
+    wav_shift_type: fixed_shift_duration
+  noise_interference:
+    corrupt_list: aishell_ke_data/noise.cutted.scp
+    num_corrupt: 1
+    prob: 0.5
+    sampler: uniform
+    sampler_config:
+      low: 0.2
+    use: false
+  post_feats_config:
+    splice_config:
+      left_context: 5
+      right_context: 5
+    subsample_rate: 3
+    use: false
+  return_raw: true
+  speech_interference:
+    num_corrupt: 1
+    sampler: uniform
+    sampler_config:
+      high: 0.9
+      low: 0.1
+    use: true
+start_epoch: 0
+sv_config:
+  spk2cls: data/dump/speaker_label/spk2cls.json
+train_list: data/dump/datalist/train-860.jsonl
+valid_list: data/dump/datalist/valid.jsonl

kce/epoch_149.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bab0fa440b264185ed5b90b805b783a9d75890f350cd87cfb0910ecedb54d9e3
+size 134200203

kce/model.yaml

@@ -0,0 +1,83 @@
+audio_net_config:
+  input_trans:
+    act: ReLU
+    dim:
+    - 256
+    - 256
+    norm: LayerNorm
+    num_block: 1
+  transformer_config:
+    conv_config:
+      channels: 256
+    corss_att_config:
+      n_feats: 256
+      n_head: 1
+      norm: LayerNorm
+    cross_att: MultiHeadCrossAtt
+    feed_forward_config: &id001
+      act: ReLU
+      dim:
+      - 256
+      - 2048
+      - 256
+      norm: LayerNorm
+      num_block: 1
+    self_att: MultiHeadAtt
+    self_att_config: &id002
+      n_feats: 256
+      n_head: 4
+    size: 256
+decoder_net_config:
+  input_trans:
+    act: ReLU
+    dim: 256
+    norm: LayerNorm
+    num_block: 1
+  transformer_config:
+    corss_att_config:
+      n_feats: 256
+      n_head: 1
+      norm: LayerNorm
+    cross_att: MultiHeadCrossAtt
+    feed_forward_config: *id001
+    self_att: MultiHeadAtt
+    self_att_config: *id002
+    size: 256
+eos: 5004
+kw_net_config:
+  input_trans:
+    act: ReLU
+    dim: 256
+    norm: LayerNorm
+    num_block: 1
+  num_bpe_token: 5005
+  num_phn_token: 74
+  transformer_config:
+    feed_forward_config: *id001
+    self_att: MultiHeadAtt
+    self_att_config: *id002
+    size: 256
+loss_weight:
+- 0.3
+- 0.1
+- 0.3
+num_audio_block: 9
+num_doecoder_block: 4
+num_kw_block: 4
+sos: 2
+sv_net_config:
+  front_output_size: 256
+  num_classes: 2338
+  pooling_conf:
+    use_softmax: False
+    layer_indices:
+    - 0
+    - 1
+    - 2
+    - 3
+    - 4
+    - 5
+    - 6
+    - 7
+    - 8
+    type: learnable_weights