File size: 9,882 Bytes
a4badd2 264e471 a4badd2 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 | from transformers.configuration_utils import PretrainedConfig
from transformers.models.auto import CONFIG_MAPPING
class HiggsAudioEncoderConfig(PretrainedConfig):
"""Configuration of the Audio encoder in Higgs-Audio."""
model_type = "higgs_audio_encoder"
def __init__(
self,
num_mel_bins=128,
encoder_layers=32,
encoder_attention_heads=20,
encoder_ffn_dim=5120,
encoder_layerdrop=0.0,
d_model=1280,
dropout=0.0,
attention_dropout=0.0,
activation_function="gelu",
activation_dropout=0.0,
scale_embedding=False,
init_std=0.02,
max_source_positions=1500, # not change this because we use the pre-trained whisper model
pad_token_id=128001,
**kwargs,
):
super().__init__(**kwargs)
self.num_mel_bins = num_mel_bins
self.d_model = d_model
self.encoder_layers = encoder_layers
self.encoder_attention_heads = encoder_attention_heads
self.encoder_ffn_dim = encoder_ffn_dim
self.dropout = dropout
self.attention_dropout = attention_dropout
self.activation_function = activation_function
self.activation_dropout = activation_dropout
self.encoder_layerdrop = encoder_layerdrop
self.num_hidden_layers = encoder_layers
self.init_std = init_std
self.scale_embedding = scale_embedding # scale factor will be sqrt(d_model) if True
self.max_source_positions = max_source_positions
self.pad_token_id = pad_token_id
self.frame_rate = 25 # constant for whisper encoder
class HiggsAudio3Config(PretrainedConfig):
r"""
This is the configuration class for the HiggsAudioModel.
Args:
text_config (`Union[AutoConfig, dict]`):
The config object or dictionary of the text backbone.
audio_encoder_config (`Union[AutoConfig, dict]`):
The config object or dictionary of the whisper encoder.
The audio encoder will be bidirectional and will be only available for audio understanding.
audio_tokenizer_config
The config object or dictionary of the audio tokenizer.
audio_adapter_type
The type of audio adapter to use. We support two types of adapter:
- stack:
We stack additional Transformer layers after the main LLM backbone for audio generation.
audio_embed_avg (`bool`, *optional*, defaults to False):
Whether to average the audio embeddings before sending them to the text attention layer.
audio_decoder_proj_num_attention (`int`, *optional*, defaults to 0):
The number of attention heads in the audio decoder projection layer.
use_delay_pattern (`bool`, *optional*, defaults to False):
Whether to use delay pattern in the audio decoder.
skip_audio_tower (`bool`, *optional*, defaults to False):
Whether to skip the audio tower in the audio encoder.
use_audio_out_embed_projector (`bool`, *optional*, defaults to False):
Whether to use an embedding projector to map audio out embeddings.
use_audio_out_self_attention (`bool`, *optional*, defaults to False):
Whether to use self-attention to aggregate information from audio-tokens before sending to the text attention layer.
audio_num_codebooks (`int`, *optional*, defaults to 12):
The number of codebooks in RVQGAN.
audio_codebook_size (`int`, *optional*, defaults to 1024):
The size of each codebook in RVQGAN.
audio_stream_bos_id
The id of the bos in the audio stream
audio_stream_eos_id
The id of the eos in the audio stream
audio_bos_token (`str`, *optional*, defaults to "<|audio_bos|>"):
The special `<|audio_bos|>` token. In Higgs-Audio, it is mapped to 128011,
which is the index of `<|reserved_special_token_3|>` in Llama-3.1-8B-Instruct's tokenizer.
audio_eos_token (`str`, *optional*, defaults to "<|audio_eos|>"):
The special `<|audio_eos|>` token. We use 128012 as the default value,
which is the index of `<|reserved_special_token_4|>` in Llama-3.1-8B-Instruct's tokenizer.
audio_out_bos_token (`str`, *optional*, defaults to "<|audio_out_bos|>"):
The special `<|audio_out_bos|>` token. We use 128013 as the default value,
which is the index of `<|reserved_special_token_5|>` in Llama-3.1-8B-Instruct's tokenizer.
audio_token (`str`, *optional*, defaults to "<|AUDIO|>"):
The special `<|AUDIO|>` token. We use 128015 as the default value,
which is the index of `<|reserved_special_token_7|>` in Llama-3.1-8B-Instruct's tokenizer.
This token indicates that the location should be filled in with whisper features.
audio_out_token (`str`, *optional*, defaults to "<|AUDIO_OUT|>"):
The special `<|AUDIO_OUT|>` token. We use 128016 as the default value,
which is the index of `<|reserved_special_token_8|>` in Llama-3.1-8B-Instruct's tokenizer.
This token indicates that the location should be filled in with audio tokens extracted via audio tokenizer.
"""
model_type = "higgs_audio_3"
is_composition = True
def __init__(
self,
text_config=None,
audio_encoder_config=None,
audio_tokenizer_config=None,
audio_adapter_type="stack",
audio_embed_avg=False,
audio_decoder_proj_num_layers=0,
encode_whisper_embed=True,
encode_audio_in_tokens=False,
use_delay_pattern=False,
skip_audio_tower=False,
use_audio_out_embed_projector=False,
use_audio_out_self_attention=False,
audio_num_codebooks=12,
audio_codebook_size=1024,
audio_stream_bos_id=1024,
audio_stream_eos_id=1025,
audio_bos_token="<|audio_bos|>",
audio_eos_token="<|audio_eos|>",
audio_out_bos_token="<|audio_out_bos|>",
audio_in_token="<|AUDIO|>",
audio_out_token="<|AUDIO_OUT|>",
audio_in_token_idx=128015,
audio_out_token_idx=128016,
pad_token_id=128001,
audio_out_bos_token_id=128013,
audio_eos_token_id=128012,
projector_temporal_downsample=1,
projector_type="mlp",
projector_kernel_size=None,
projector_padding=None,
chunk_size_seconds=None,
encoder_padding_method=None,
duplicate_audio_triplet=None,
**kwargs
):
if isinstance(audio_encoder_config, dict):
audio_encoder_config["model_type"] = (
audio_encoder_config["model_type"] if "model_type" in audio_encoder_config else "higgs_audio_encoder"
)
if audio_encoder_config["model_type"] == "higgs_audio_encoder":
audio_encoder_config = HiggsAudioEncoderConfig(**audio_encoder_config)
else:
audio_encoder_config = CONFIG_MAPPING[audio_encoder_config["model_type"]](**audio_encoder_config)
elif audio_encoder_config is None:
audio_encoder_config = HiggsAudioEncoderConfig()
if isinstance(text_config, dict):
text_config["model_type"] = text_config["model_type"] if "model_type" in text_config else "qwen3"
text_config = CONFIG_MAPPING[text_config["model_type"]](**text_config)
elif text_config is None:
text_config = CONFIG_MAPPING["llama"]()
assert audio_adapter_type in ["stack"], f"Invalid audio adapter type: {audio_adapter_type}"
self.text_config = text_config
self.audio_encoder_config = audio_encoder_config
self.audio_tokenizer_config = audio_tokenizer_config
self.audio_adapter_type = audio_adapter_type
self.audio_embed_avg = audio_embed_avg
self.audio_decoder_proj_num_layers = audio_decoder_proj_num_layers
self.encode_whisper_embed = encode_whisper_embed
self.encode_audio_in_tokens = encode_audio_in_tokens
self.use_delay_pattern = use_delay_pattern
self.skip_audio_tower = skip_audio_tower
self.use_audio_out_embed_projector = use_audio_out_embed_projector
self.use_audio_out_self_attention = use_audio_out_self_attention
self.audio_num_codebooks = audio_num_codebooks
self.audio_codebook_size = audio_codebook_size
self.audio_bos_token = audio_bos_token
self.audio_eos_token = audio_eos_token
self.audio_out_bos_token = audio_out_bos_token
self.audio_in_token = audio_in_token
self.audio_out_token = audio_out_token
self.audio_in_token_idx = audio_in_token_idx
self.audio_out_token_idx = audio_out_token_idx
self.audio_stream_bos_id = audio_stream_bos_id
self.audio_stream_eos_id = audio_stream_eos_id
self.audio_out_bos_token_id = audio_out_bos_token_id
self.audio_eos_token_id = audio_eos_token_id
self.projector_temporal_downsample = projector_temporal_downsample
self.projector_type = projector_type
self.projector_kernel_size = projector_kernel_size
self.projector_padding = projector_padding
self.tps = self.audio_encoder_config.frame_rate / self.projector_temporal_downsample
super().__init__(**kwargs)
self.pad_token_id = pad_token_id
self.chunk_size_seconds = kwargs.pop("chunk_size_seconds", chunk_size_seconds if chunk_size_seconds is not None else 30.0)
self.encoder_padding_method = kwargs.pop("encoder_padding_method", encoder_padding_method if encoder_padding_method is not None else "max_length")
self.duplicate_audio_triplet = kwargs.pop("duplicate_audio_triplet", duplicate_audio_triplet if duplicate_audio_triplet is not None else True) |