Use NeroXSAForCausalLM architecture name
Browse files- README.md +2 -2
- config.json +2 -2
- modeling_nero_xs_2.py +7 -6
README.md
CHANGED
|
@@ -106,7 +106,7 @@ text = generate(
|
|
| 106 |
print(text)
|
| 107 |
```
|
| 108 |
|
| 109 |
-
Nero XS 2 uses the custom PyTorch class `
|
| 110 |
|
| 111 |
## Results
|
| 112 |
|
|
@@ -129,7 +129,7 @@ HellaSwag, ARC, and PIQA were evaluated with `lm-eval` 0.4.12 using exact contin
|
|
| 129 |
|
| 130 |
| Component | Configuration |
|
| 131 |
|---|---|
|
| 132 |
-
| Architecture class | `
|
| 133 |
| Stored parameters | **2,993,152** |
|
| 134 |
| Physical / effective blocks | 10 / 14 |
|
| 135 |
| Recurrent layout | 1 prelude + 4 middle blocks × 2 passes + 5 coda |
|
|
|
|
| 106 |
print(text)
|
| 107 |
```
|
| 108 |
|
| 109 |
+
Nero XS 2 uses the custom PyTorch class `NeroXSAForCausalLM`. It is not a drop-in Transformers `AutoModelForCausalLM` checkpoint. The complete standalone implementation is included in `modeling_nero_xs_2.py`.
|
| 110 |
|
| 111 |
## Results
|
| 112 |
|
|
|
|
| 129 |
|
| 130 |
| Component | Configuration |
|
| 131 |
|---|---|
|
| 132 |
+
| Architecture class | `NeroXSAForCausalLM` |
|
| 133 |
| Stored parameters | **2,993,152** |
|
| 134 |
| Physical / effective blocks | 10 / 14 |
|
| 135 |
| Recurrent layout | 1 prelude + 4 middle blocks × 2 passes + 5 coda |
|
config.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
| 1 |
{
|
| 2 |
-
"architectures": ["
|
| 3 |
-
"model_type": "
|
| 4 |
"vocab_size": 2048,
|
| 5 |
"width": 128,
|
| 6 |
"heads": 4,
|
|
|
|
| 1 |
{
|
| 2 |
+
"architectures": ["NeroXSAForCausalLM"],
|
| 3 |
+
"model_type": "nero_xsa",
|
| 4 |
"vocab_size": 2048,
|
| 5 |
"width": 128,
|
| 6 |
"heads": 4,
|
modeling_nero_xs_2.py
CHANGED
|
@@ -199,7 +199,7 @@ class ReleasedNeroXSControl(nn.Module):
|
|
| 199 |
return F.linear(self.norm(x), self.embedding.weight)
|
| 200 |
|
| 201 |
|
| 202 |
-
class
|
| 203 |
"""XSA + selective recurrence + loop conditioning + EngramLite."""
|
| 204 |
|
| 205 |
def __init__(self, cfg: NeroConfig = NeroConfig()):
|
|
@@ -257,7 +257,7 @@ def variant_config(name: str) -> NeroConfig:
|
|
| 257 |
def build_variant(name: str) -> nn.Module:
|
| 258 |
if name == "released_control":
|
| 259 |
return ReleasedNeroXSControl()
|
| 260 |
-
return
|
| 261 |
|
| 262 |
|
| 263 |
def architecture_audit() -> dict[str, int | float | bool]:
|
|
@@ -290,7 +290,7 @@ def load_model(model_dir: str | Path, device: str | torch.device = "cpu"):
|
|
| 290 |
raw = json.loads((model_dir / "config.json").read_text())
|
| 291 |
fields = NeroConfig.__dataclass_fields__
|
| 292 |
cfg = NeroConfig(**{key: value for key, value in raw.items() if key in fields})
|
| 293 |
-
model =
|
| 294 |
model.load_state_dict(load_file(model_dir / "model.safetensors"), strict=True)
|
| 295 |
model.to(device).eval()
|
| 296 |
tokenizer = AutoTokenizer.from_pretrained(model_dir)
|
|
@@ -299,7 +299,7 @@ def load_model(model_dir: str | Path, device: str | torch.device = "cpu"):
|
|
| 299 |
|
| 300 |
@torch.inference_mode()
|
| 301 |
def generate(
|
| 302 |
-
model:
|
| 303 |
tokenizer,
|
| 304 |
prompt: str,
|
| 305 |
max_new_tokens: int = 64,
|
|
@@ -338,8 +338,9 @@ def generate(
|
|
| 338 |
return tokenizer.decode(ids, skip_special_tokens=True)
|
| 339 |
|
| 340 |
|
| 341 |
-
# Backward-compatible import
|
| 342 |
-
|
|
|
|
| 343 |
|
| 344 |
|
| 345 |
if __name__ == "__main__":
|
|
|
|
| 199 |
return F.linear(self.norm(x), self.embedding.weight)
|
| 200 |
|
| 201 |
|
| 202 |
+
class NeroXSAForCausalLM(nn.Module):
|
| 203 |
"""XSA + selective recurrence + loop conditioning + EngramLite."""
|
| 204 |
|
| 205 |
def __init__(self, cfg: NeroConfig = NeroConfig()):
|
|
|
|
| 257 |
def build_variant(name: str) -> nn.Module:
|
| 258 |
if name == "released_control":
|
| 259 |
return ReleasedNeroXSControl()
|
| 260 |
+
return NeroXSAForCausalLM(variant_config(name))
|
| 261 |
|
| 262 |
|
| 263 |
def architecture_audit() -> dict[str, int | float | bool]:
|
|
|
|
| 290 |
raw = json.loads((model_dir / "config.json").read_text())
|
| 291 |
fields = NeroConfig.__dataclass_fields__
|
| 292 |
cfg = NeroConfig(**{key: value for key, value in raw.items() if key in fields})
|
| 293 |
+
model = NeroXSAForCausalLM(cfg)
|
| 294 |
model.load_state_dict(load_file(model_dir / "model.safetensors"), strict=True)
|
| 295 |
model.to(device).eval()
|
| 296 |
tokenizer = AutoTokenizer.from_pretrained(model_dir)
|
|
|
|
| 299 |
|
| 300 |
@torch.inference_mode()
|
| 301 |
def generate(
|
| 302 |
+
model: NeroXSAForCausalLM,
|
| 303 |
tokenizer,
|
| 304 |
prompt: str,
|
| 305 |
max_new_tokens: int = 64,
|
|
|
|
| 338 |
return tokenizer.decode(ids, skip_special_tokens=True)
|
| 339 |
|
| 340 |
|
| 341 |
+
# Backward-compatible import aliases for the earlier release names.
|
| 342 |
+
NeroXSA11ForCausalLM = NeroXSAForCausalLM
|
| 343 |
+
NeroXSA2ForCausalLM = NeroXSAForCausalLM
|
| 344 |
|
| 345 |
|
| 346 |
if __name__ == "__main__":
|