| """ |
| Configuration class for Canopy-R3 Recurrent Mixture-of-Experts (MoE) model. |
| """ |
|
|
| from transformers import PretrainedConfig |
|
|
|
|
| class CanopyConfig(PretrainedConfig): |
| model_type = "canopy" |
| keys_to_ignore_at_inference = ["past_key_values"] |
|
|
| def __init__( |
| self, |
| vocab_size: int = 49152, |
| max_seq_len: int = 2048, |
| d_model: int = 768, |
| num_heads: int = 12, |
| num_kv_heads: int = 4, |
| head_dim: int = 64, |
| num_layers: int = 12, |
| prelude_layers: int = 3, |
| recurrent_layers: int = 6, |
| coda_layers: int = 3, |
| recurrent_visits: int = 2, |
| loop_residual_scale: float = 0.5, |
| dense_intermediate_size: int = 2048, |
| moe_num_experts: int = 8, |
| moe_top_k: int = 2, |
| moe_intermediate_size: int = 1536, |
| use_thought_bus: bool = True, |
| thought_bus_width: int = 192, |
| use_visit_adapter: bool = True, |
| visit_adapter_rank: int = 8, |
| router_aux_loss_coef: float = 0.01, |
| router_z_loss_coef: float = 0.001, |
| norm_eps: float = 1e-6, |
| rope_theta: float = 10000.0, |
| tie_word_embeddings: bool = True, |
| initializer_range: float = 0.02, |
| quant_mode: str = "none", |
| **kwargs, |
| ): |
| self.vocab_size = vocab_size |
| self.max_seq_len = max_seq_len |
| self.d_model = d_model |
| self.num_heads = num_heads |
| self.num_kv_heads = num_kv_heads |
| self.head_dim = head_dim |
| self.num_layers = num_layers |
| self.prelude_layers = prelude_layers |
| self.recurrent_layers = recurrent_layers |
| self.coda_layers = coda_layers |
| self.recurrent_visits = recurrent_visits |
| self.loop_residual_scale = loop_residual_scale |
| self.dense_intermediate_size = dense_intermediate_size |
| self.moe_num_experts = moe_num_experts |
| self.moe_top_k = moe_top_k |
| self.moe_intermediate_size = moe_intermediate_size |
| self.use_thought_bus = use_thought_bus |
| self.thought_bus_width = thought_bus_width |
| self.use_visit_adapter = use_visit_adapter |
| self.visit_adapter_rank = visit_adapter_rank |
| self.router_aux_loss_coef = router_aux_loss_coef |
| self.router_z_loss_coef = router_z_loss_coef |
| self.norm_eps = norm_eps |
| self.rope_theta = rope_theta |
| self.tie_word_embeddings = tie_word_embeddings |
| self.initializer_range = initializer_range |
| self.quant_mode = quant_mode |
| super().__init__(tie_word_embeddings=tie_word_embeddings, **kwargs) |
|
|
