Upload configuration_nanbeige.py

configuration_nanbeige.py

@@ -21,18 +21,14 @@ logger = logging.get_logger(__name__)
 
 class NanbeigeConfig(PretrainedConfig):
     r"""
-    This is the configuration class to store the configuration of a [`NanbeigeModel`]. It is used to instantiate an LLaMA
-    model according to the specified arguments, defining the model architecture. Instantiating a configuration with the
-    defaults will yield a similar configuration to that of the LLaMA-7B.
-
+    This is the configuration class to store the configuration of a [`NanbeigeModel`]. It is used to instantiate a Nanbeige model
+    according to the specified arguments, defining the model architecture. 
     Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
     documentation from [`PretrainedConfig`] for more information.
-
-
     Args:
         vocab_size (`int`, *optional*, defaults to 32000):
-            Vocabulary size of the LLaMA model. Defines the number of different tokens that can be represented by the
-            `inputs_ids` passed when calling [`NanbeigeModel`]
+            Vocabulary size of the Nanbeige model. Defines the number of different tokens that can be represented by the
+            `input_ids` passed when calling [`NanbeigeModel`]
         hidden_size (`int`, *optional*, defaults to 4096):
             Dimension of the hidden representations.
         intermediate_size (`int`, *optional*, defaults to 11008):
@@ -54,8 +50,7 @@ class NanbeigeConfig(PretrainedConfig):
         hidden_act (`str` or `function`, *optional*, defaults to `"silu"`):
             The non-linear activation function (function or string) in the decoder.
         max_position_embeddings (`int`, *optional*, defaults to 2048):
-            The maximum sequence length that this model might ever be used with. Nanbeige 1 supports up to 2048 tokens,
-            Nanbeige 2 up to 4096, CodeNanbeige up to 16384.
+            The maximum sequence length that this model might ever be used with.
         initializer_range (`float`, *optional*, defaults to 0.02):
             The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
         rms_norm_eps (`float`, *optional*, defaults to 1e-06):
@@ -128,16 +123,75 @@ class NanbeigeConfig(PretrainedConfig):
             Whether to fuse averaged N-gram embeddings before attention in every decoder layer.
         ngram_layer_downproject_size (`int`, *optional*):
             Optional hidden size for N-gram layer fusion projections. If None, fusion uses `hidden_size`.
-
+        num_loops (`int`, *optional*, defaults to 1):
+            Number of times the complete decoder-layer stack is executed with shared parameters. Increasing this value
+            increases the model's effective depth and FLOPs without adding a separate set of decoder-layer weights.
+            This value is ignored when `loop_loss_weights` is non-empty or `enable_double_loop_split=True`.
+        loop_loss_weights (`List[float]`, *optional*):
+            Weights associated with intermediate loop outputs during multi-loop training. When this list is non-empty,
+            the model executes `len(loop_loss_weights) + 1` loops instead of using `num_loops`. For the standard loop
+            layout, the weights must sum to at most 1.0. Defaults to an empty list.
+        skip_loop_final_norm (`bool`, *optional*, defaults to `False`):
+            Whether to skip the final RMS normalization between loops. If `True`, normalization is applied only after
+            the last loop; if `False`, every loop output is normalized before it is passed to the next loop.
+        enable_double_loop_split (`bool`, *optional*, defaults to `False`):
+            Whether to enable LoopSplit. LoopSplit keeps the outer decoder layers unlooped and repeatedly executes a
+            contiguous middle block, providing different effective depths for different parts of the network. When
+            enabled, the execution order is controlled by `loop_middle_layers` rather than `num_loops`.
+        loop_middle_layers (`int`, *optional*):
+            Number of contiguous middle decoder layers repeatedly executed by LoopSplit. It must be a positive factor
+            of `num_hidden_layers`. If omitted while LoopSplit is enabled, it defaults to half of
+            `num_hidden_layers`, which therefore must be even.
+        loop_share_kv (`bool`, *optional*, defaults to `False`):
+            Whether repeated executions of a LoopSplit middle layer reuse the key and value states produced by that
+            layer's first execution. Requires `enable_double_loop_split=True`.
+        mhc_diff_for_loop (`bool`, *optional*, defaults to `False`):
+            Whether each repeated execution of a LoopSplit middle layer uses separate mHC connection modules instead
+            of sharing one set across repetitions. Requires both `enable_double_loop_split=True` and `enable_mhc=True`.
+        mhc_double_stream_position_for_loop (`str`, *optional*):
+            Selects where LoopSplit doubles the configured number of residual streams. Accepted values are `"mid"`,
+            which doubles streams in the looped middle block, and `"edge"`, which doubles streams in the unlooped outer
+            blocks. Requires `enable_double_loop_split=True`.
+        enable_hyper_connection (`bool`, *optional*, defaults to `False`):
+            Whether to replace the standard single residual path with multiple residual streams connected around each
+            attention and MLP sublayer by learned hyper-connection modules.
+        enable_mhc (`bool`, *optional*, defaults to `False`):
+            Whether to use manifold-constrained hyper-connections (mHC), which constrain the learned residual-stream
+            mixing matrices with Sinkhorn normalization. Requires `enable_hyper_connection=True`.
+        enable_h_res_identity (`bool`, *optional*, defaults to `False`):
+            Whether the residual-stream mixing matrix includes an explicit identity component. Requires
+            `enable_hyper_connection=True`.
+        mhc_identity_nohresparam (`bool`, *optional*, defaults to `False`):
+            Whether the identity component is used without a separately learned residual mixing parameter. Requires
+            both `enable_mhc=True` and `enable_h_res_identity=True`.
+        num_residual_streams (`int`, *optional*, defaults to 4):
+            Base number of residual streams used by hyper-connections. It must be at least 2 when
+            `enable_hyper_connection=True`; LoopSplit may double it in the region selected by
+            `mhc_double_stream_position_for_loop`.
+        mhc_sinkhorn_iterations (`int`, *optional*, defaults to 20):
+            Number of Sinkhorn normalization iterations used to constrain mHC residual-stream mixing matrices. It must
+            be at least 1 when `enable_mhc=True`.
+        mhc_init_gating_factor (`float`, *optional*, defaults to 0.01):
+            Initial scale of the learned mHC gating terms that perturb the identity-like initialization of the
+            hyper-connection matrices.
+        enable_depth_attention (`bool`, *optional*, defaults to `False`):
+            Whether to enable depth attention. At each decoder layer, the current query selects and mixes value states
+            from cached anchor depths before normal token-level self-attention is applied.
+        depth_attention_stride (`int`, *optional*):
+            Layer interval at which key/value states are added as depth-attention anchors. It must be positive and
+            defaults to `num_hidden_layers // 2` when depth attention is enabled.
+        depth_attention_recent_window (`int`, *optional*, defaults to 0):
+            Reserved size of a recent-depth window in addition to anchor depths. The current anchor-only implementation
+            requires this value to be 0.
+        depth_attention_static_anchor_once (`bool`, *optional*, defaults to `True`):
+            Whether depth-attention anchors are collected once and reused across repeated LoopSplit executions. This
+            must be `True` when depth attention and LoopSplit are enabled together.
     ```python
     >>> from transformers import NanbeigeModel, NanbeigeConfig
-
-    >>> # Initializing a LLaMA llama-7b style configuration
+    >>> # Initializing a Nanbeige style configuration
     >>> configuration = NanbeigeConfig()
-
-    >>> # Initializing a model from the llama-7b style configuration
+    >>> # Initializing a model from the Nanbeige style configuration
     >>> model = NanbeigeModel(configuration)
-
     >>> # Accessing the model configuration
     >>> configuration = model.config
     ```"""