j0no12 commited on
Commit
a011b19
·
verified ·
1 Parent(s): 90101de

Use NeroXSAForCausalLM architecture name

Browse files
Files changed (3) hide show
  1. README.md +2 -2
  2. config.json +2 -2
  3. modeling_nero_xs_2.py +7 -6
README.md CHANGED
@@ -106,7 +106,7 @@ text = generate(
106
  print(text)
107
  ```
108
 
109
- Nero XS 2 uses the custom PyTorch class `NeroXSA11ForCausalLM`. It is not a drop-in Transformers `AutoModelForCausalLM` checkpoint. The complete standalone implementation is included in `modeling_nero_xs_2.py`.
110
 
111
  ## Results
112
 
@@ -129,7 +129,7 @@ HellaSwag, ARC, and PIQA were evaluated with `lm-eval` 0.4.12 using exact contin
129
 
130
  | Component | Configuration |
131
  |---|---|
132
- | Architecture class | `NeroXSA11ForCausalLM` |
133
  | Stored parameters | **2,993,152** |
134
  | Physical / effective blocks | 10 / 14 |
135
  | Recurrent layout | 1 prelude + 4 middle blocks × 2 passes + 5 coda |
 
106
  print(text)
107
  ```
108
 
109
+ Nero XS 2 uses the custom PyTorch class `NeroXSAForCausalLM`. It is not a drop-in Transformers `AutoModelForCausalLM` checkpoint. The complete standalone implementation is included in `modeling_nero_xs_2.py`.
110
 
111
  ## Results
112
 
 
129
 
130
  | Component | Configuration |
131
  |---|---|
132
+ | Architecture class | `NeroXSAForCausalLM` |
133
  | Stored parameters | **2,993,152** |
134
  | Physical / effective blocks | 10 / 14 |
135
  | Recurrent layout | 1 prelude + 4 middle blocks × 2 passes + 5 coda |
config.json CHANGED
@@ -1,6 +1,6 @@
1
  {
2
- "architectures": ["NeroXSA11ForCausalLM"],
3
- "model_type": "nero_xsa11",
4
  "vocab_size": 2048,
5
  "width": 128,
6
  "heads": 4,
 
1
  {
2
+ "architectures": ["NeroXSAForCausalLM"],
3
+ "model_type": "nero_xsa",
4
  "vocab_size": 2048,
5
  "width": 128,
6
  "heads": 4,
modeling_nero_xs_2.py CHANGED
@@ -199,7 +199,7 @@ class ReleasedNeroXSControl(nn.Module):
199
  return F.linear(self.norm(x), self.embedding.weight)
200
 
201
 
202
- class NeroXSA11ForCausalLM(nn.Module):
203
  """XSA + selective recurrence + loop conditioning + EngramLite."""
204
 
205
  def __init__(self, cfg: NeroConfig = NeroConfig()):
@@ -257,7 +257,7 @@ def variant_config(name: str) -> NeroConfig:
257
  def build_variant(name: str) -> nn.Module:
258
  if name == "released_control":
259
  return ReleasedNeroXSControl()
260
- return NeroXSA11ForCausalLM(variant_config(name))
261
 
262
 
263
  def architecture_audit() -> dict[str, int | float | bool]:
@@ -290,7 +290,7 @@ def load_model(model_dir: str | Path, device: str | torch.device = "cpu"):
290
  raw = json.loads((model_dir / "config.json").read_text())
291
  fields = NeroConfig.__dataclass_fields__
292
  cfg = NeroConfig(**{key: value for key, value in raw.items() if key in fields})
293
- model = NeroXSA11ForCausalLM(cfg)
294
  model.load_state_dict(load_file(model_dir / "model.safetensors"), strict=True)
295
  model.to(device).eval()
296
  tokenizer = AutoTokenizer.from_pretrained(model_dir)
@@ -299,7 +299,7 @@ def load_model(model_dir: str | Path, device: str | torch.device = "cpu"):
299
 
300
  @torch.inference_mode()
301
  def generate(
302
- model: NeroXSA11ForCausalLM,
303
  tokenizer,
304
  prompt: str,
305
  max_new_tokens: int = 64,
@@ -338,8 +338,9 @@ def generate(
338
  return tokenizer.decode(ids, skip_special_tokens=True)
339
 
340
 
341
- # Backward-compatible import alias for the initial release name.
342
- NeroXSA2ForCausalLM = NeroXSA11ForCausalLM
 
343
 
344
 
345
  if __name__ == "__main__":
 
199
  return F.linear(self.norm(x), self.embedding.weight)
200
 
201
 
202
+ class NeroXSAForCausalLM(nn.Module):
203
  """XSA + selective recurrence + loop conditioning + EngramLite."""
204
 
205
  def __init__(self, cfg: NeroConfig = NeroConfig()):
 
257
  def build_variant(name: str) -> nn.Module:
258
  if name == "released_control":
259
  return ReleasedNeroXSControl()
260
+ return NeroXSAForCausalLM(variant_config(name))
261
 
262
 
263
  def architecture_audit() -> dict[str, int | float | bool]:
 
290
  raw = json.loads((model_dir / "config.json").read_text())
291
  fields = NeroConfig.__dataclass_fields__
292
  cfg = NeroConfig(**{key: value for key, value in raw.items() if key in fields})
293
+ model = NeroXSAForCausalLM(cfg)
294
  model.load_state_dict(load_file(model_dir / "model.safetensors"), strict=True)
295
  model.to(device).eval()
296
  tokenizer = AutoTokenizer.from_pretrained(model_dir)
 
299
 
300
  @torch.inference_mode()
301
  def generate(
302
+ model: NeroXSAForCausalLM,
303
  tokenizer,
304
  prompt: str,
305
  max_new_tokens: int = 64,
 
338
  return tokenizer.decode(ids, skip_special_tokens=True)
339
 
340
 
341
+ # Backward-compatible import aliases for the earlier release names.
342
+ NeroXSA11ForCausalLM = NeroXSAForCausalLM
343
+ NeroXSA2ForCausalLM = NeroXSAForCausalLM
344
 
345
 
346
  if __name__ == "__main__":