Publish accepted Memory Fusion prefix [0]

README.md

@@ -0,0 +1,43 @@
+---
+library_name: transformers
+pipeline_tag: text-generation
+tags:
+- tinycenn
+- cenn
+- language-modeling
+- text-generation
+- research
+---
+
+# SmolLM2-135M-MemoryFusion-Sequential-R64
+
+Research artifact from **TinyCeNN-LM**. Architecture: `TinyCeNN-LM experiment`.
+
+## Architecture
+
+- Architecture/run type: `TinyCeNN-LM experiment`
+- Base model: `not recorded`
+- Dataset: `Not recorded`
+- Source code: https://github.com/vtavakkoli/TinyCeNN-LM
+
+## Latest saved results
+
+No structured training report was found in this upload.
+
+The Hugging Face repository keeps timestamped run artifacts under `runs/`. This preserves training reports, configs and run metadata independently of the temporary Colab filesystem.
+
+## Saved experiment files
+
+- `tokenizer_config.json`
+
+## Reproducibility
+
+Run the matching notebook from the TinyCeNN-LM repository. Colab notebooks use a Hugging Face write token from the `HF_TOKEN` Colab Secret; tokens should never be pasted into notebook source.
+
+## Limitations
+
+This is a research checkpoint. Metrics saved here are the metrics produced by the corresponding training notebook/script; unless explicitly marked as held-out evaluation, they should not be treated as publication-grade benchmark results. Generation quality can differ substantially from the base model.
+
+## Citation
+
+If you use this experimental checkpoint, cite the TinyCeNN-LM repository and the upstream base model.

load_model.py

@@ -0,0 +1,27 @@
+from __future__ import annotations
+import json
+import torch
+from huggingface_hub import hf_hub_download
+from safetensors.torch import load_file
+from transformers import AutoModelForCausalLM, AutoTokenizer
+from tinycenn_lm.smollm2_memory_fusion import SmolMemoryFusionConfig, replace_attention_layers
+
+def load_model(repo_id: str, token=None, device=None):
+    device=torch.device(device or ('cuda' if torch.cuda.is_available() else 'cpu'))
+    dtype=torch.bfloat16 if device.type=='cuda' and torch.cuda.is_bf16_supported() else (torch.float16 if device.type=='cuda' else torch.float32)
+    meta_path=hf_hub_download(repo_id,'tinycenn_model.json',token=token)
+    with open(meta_path,encoding='utf-8') as f:
+        meta=json.load(f)
+    tokenizer=AutoTokenizer.from_pretrained(repo_id,token=token,use_fast=True)
+    if tokenizer.pad_token_id is None:
+        tokenizer.pad_token=tokenizer.eos_token
+    model=AutoModelForCausalLM.from_pretrained(meta['base_model'],dtype=dtype)
+    cfg=SmolMemoryFusionConfig.from_dict(meta['memory_fusion'])
+    replace_attention_layers(model,cfg,meta['accepted_layers'])
+    state_path=hf_hub_download(repo_id,'model.safetensors',token=token)
+    state=load_file(state_path,device='cpu')
+    model.load_state_dict(state,strict=True)
+    model.to(device).eval(); model.requires_grad_(False); model.config.use_cache=False
+    if hasattr(model,'generation_config'):
+        model.generation_config.use_cache=False
+    return model,tokenizer,meta

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:34d7f2d590d89bdc2f20fb85bf2dd962f500b85a9f1b2a2cf39b30f8dd5bf251
+size 326994892

prompt_smoke_test.json

@@ -0,0 +1,61 @@
+{
+  "status": "prompt_generation_completed",
+  "base_model": "HuggingFaceTB/SmolLM2-135M",
+  "accepted_layers": [
+    0
+  ],
+  "memory_fusion_config": {
+    "feature_dim": 32,
+    "memory_rank": 64,
+    "dilations": [
+      1,
+      2,
+      4,
+      8,
+      16,
+      32,
+      64,
+      128
+    ],
+    "shifted_window": 8,
+    "train_output_projection": true
+  },
+  "last_accepted_report": {
+    "layer": 0,
+    "accepted": true,
+    "steps": 125,
+    "nmse": 0.018665021285414696,
+    "cosine": 0.9919254779815674,
+    "probe_nll": 2.9270507097244263,
+    "incremental_delta_nll": 0.01395869255065918,
+    "cumulative_delta_nll": 0.01395869255065918,
+    "round": 4
+  },
+  "tests": [
+    {
+      "prompt": "Austria is a country in Central Europe. The capital of Austria is",
+      "baseline": "Austria is a country in Central Europe. The capital of Austria is Vienna.\n\nAustria is a country in Central Europe. The capital of Austria is Vienna.\n\nAustria",
+      "memory_fusion": "Austria is a country in Central Europe. The capital of Austria is Vienna. The country is bordered by Germany to the north, Italy to the east, Slovenia to the south, and Hungary"
+    },
+    {
+      "prompt": "Question: What is the capital of Austria?\nAnswer:",
+      "baseline": "Question: What is the capital of Austria?\nAnswer: Vienna.\n\nQuestion: What is the capital of Austria?\nAnswer: Vienna.\n\nQuestion: What is",
+      "memory_fusion": "Question: What is the capital of Austria?\nAnswer: Vienna.\n\nQuestion: What is the capital of Austria?\nAnswer: Vienna.\n\nQuestion: What is"
+    },
+    {
+      "prompt": "Paris is the capital of France. Vienna is the capital of",
+      "baseline": "Paris is the capital of France. Vienna is the capital of Austria. Paris is the capital of France.\n\nThe capital of France is Paris. The capital of Austria is Vienna",
+      "memory_fusion": "Paris is the capital of France. Vienna is the capital of Austria.\n\nThe capital of the Czech Republic is Prague.\n\nThe capital of the United States is Washington,"
+    },
+    {
+      "prompt": "The largest planet in the Solar System is",
+      "baseline": "The largest planet in the Solar System is Jupiter. It is the largest planet in the Solar System and the largest planet in the Solar System. It is the largest",
+      "memory_fusion": "The largest planet in the Solar System is Jupiter, which is 11 times the mass of Earth. The largest planet in the Solar System is Saturn, which"
+    },
+    {
+      "prompt": "2 + 2 =",
+      "baseline": "2 + 2 = 10\n\nThe sum of the first 10 natural numbers is 1 + 2 + 3",
+      "memory_fusion": "2 + 2 = 3\n\nThe sum of the first 3 terms of the sequence is 1 + 2 + 3"
+    }
+  ]
+}

requirements.txt

@@ -0,0 +1,5 @@
+torch
+transformers==4.57.6
+huggingface_hub>=0.34,<2
+safetensors
+git+https://github.com/vtavakkoli/TinyCeNN-LM.git

run_manifest.json

@@ -0,0 +1,14 @@
+{
+  "run_id": "hf_memory_fusion_export-20260915T220116Z",
+  "created_utc": "2026-09-15T22:01:16.053571+00:00",
+  "repo_id": "vtava/SmolLM2-135M-MemoryFusion-Sequential-R64",
+  "notebook": null,
+  "python": "3.13.15",
+  "platform": "Linux-6.6.122+-x86_64-with-glibc2.39",
+  "reports": [
+    "tokenizer_config.json"
+  ],
+  "torch": "2.11.0+cu128",
+  "cuda_available": true,
+  "gpu": "NVIDIA L4"
+}

sequential_progress.json

@@ -0,0 +1,69 @@
+{
+  "format_version": 1,
+  "stage": "accepted_layer_0",
+  "accepted_layers": [
+    0
+  ],
+  "config": {
+    "feature_dim": 32,
+    "memory_rank": 64,
+    "dilations": [
+      1,
+      2,
+      4,
+      8,
+      16,
+      32,
+      64,
+      128
+    ],
+    "shifted_window": 8,
+    "train_output_projection": true
+  },
+  "layer_reports": [
+    {
+      "layer": 0,
+      "accepted": false,
+      "steps": 275,
+      "nmse": 0.048099592328071594,
+      "cosine": 0.9809460043907166,
+      "probe_nll": 2.957142174243927,
+      "incremental_delta_nll": 0.04405015707015991,
+      "cumulative_delta_nll": 0.04405015707015991,
+      "round": 1
+    },
+    {
+      "layer": 0,
+      "accepted": false,
+      "steps": 300,
+      "nmse": 0.02023264765739441,
+      "cosine": 0.991240918636322,
+      "probe_nll": 2.9379348754882812,
+      "incremental_delta_nll": 0.02484285831451416,
+      "cumulative_delta_nll": 0.02484285831451416,
+      "round": 2
+    },
+    {
+      "layer": 0,
+      "accepted": false,
+      "steps": 275,
+      "nmse": 0.01509437058120966,
+      "cosine": 0.9934894442558289,
+      "probe_nll": 2.933886468410492,
+      "incremental_delta_nll": 0.020794451236724854,
+      "cumulative_delta_nll": 0.020794451236724854,
+      "round": 3
+    },
+    {
+      "layer": 0,
+      "accepted": true,
+      "steps": 125,
+      "nmse": 0.018665021285414696,
+      "cosine": 0.9919254779815674,
+      "probe_nll": 2.9270507097244263,
+      "incremental_delta_nll": 0.01395869255065918,
+      "cumulative_delta_nll": 0.01395869255065918,
+      "round": 4
+    }
+  ]
+}

sequential_run_status.json

@@ -0,0 +1,19 @@
+{
+  "status": "current_layer_needs_more_training",
+  "accepted_layers": [
+    0
+  ],
+  "current_layer": 1,
+  "rounds_completed": 12,
+  "last_report": {
+    "layer": 1,
+    "accepted": false,
+    "steps": 300,
+    "nmse": 0.04072427377104759,
+    "cosine": 0.9802741408348083,
+    "probe_nll": 2.9506284594535828,
+    "incremental_delta_nll": 0.023577749729156494,
+    "cumulative_delta_nll": 0.03668522834777832
+  },
+  "message": "No next Transformer layer was replaced. Rerun to continue this same layer."
+}

special_tokens_map.json

@@ -0,0 +1,43 @@
+{
+  "additional_special_tokens": [
+    "<|endoftext|>",
+    "<|im_start|>",
+    "<|im_end|>",
+    "<repo_name>",
+    "<reponame>",
+    "<file_sep>",
+    "<filename>",
+    "<gh_stars>",
+    "<issue_start>",
+    "<issue_comment>",
+    "<issue_closed>",
+    "<jupyter_start>",
+    "<jupyter_text>",
+    "<jupyter_code>",
+    "<jupyter_output>",
+    "<jupyter_script>",
+    "<empty_output>"
+  ],
+  "bos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "<|endoftext|>",
+  "unk_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tinycenn_model.json

@@ -0,0 +1,57 @@
+{
+  "format_version": 1,
+  "architecture": "smollm2-memory-fusion-sequential-accepted-prefix",
+  "base_model": "HuggingFaceTB/SmolLM2-135M",
+  "source_repository": "https://github.com/vtavakkoli/TinyCeNN-LM",
+  "accepted_layers": [
+    0
+  ],
+  "num_hidden_layers": 30,
+  "memory_fusion": {
+    "feature_dim": 32,
+    "memory_rank": 64,
+    "dilations": [
+      1,
+      2,
+      4,
+      8,
+      16,
+      32,
+      64,
+      128
+    ],
+    "shifted_window": 8,
+    "train_output_projection": true
+  },
+  "last_accepted_report": {
+    "layer": 0,
+    "accepted": true,
+    "steps": 125,
+    "nmse": 0.018665021285414696,
+    "cosine": 0.9919254779815674,
+    "probe_nll": 2.9270507097244263,
+    "incremental_delta_nll": 0.01395869255065918,
+    "cumulative_delta_nll": 0.01395869255065918,
+    "round": 4
+  },
+  "trainer_status_at_export": {
+    "status": "current_layer_needs_more_training",
+    "accepted_layers": [
+      0
+    ],
+    "current_layer": 1,
+    "rounds_completed": 12,
+    "last_report": {
+      "layer": 1,
+      "accepted": false,
+      "steps": 300,
+      "nmse": 0.04072427377104759,
+      "cosine": 0.9802741408348083,
+      "probe_nll": 2.9506284594535828,
+      "incremental_delta_nll": 0.023577749729156494,
+      "cumulative_delta_nll": 0.03668522834777832
+    },
+    "message": "No next Transformer layer was replaced. Rerun to continue this same layer."
+  },
+  "important": "Only formally accepted layers are included. Any sequential_in_progress layer is excluded."
+}

tokenizer_config.json

@@ -0,0 +1,169 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "<repo_name>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "4": {
+      "content": "<reponame>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "5": {
+      "content": "<file_sep>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "6": {
+      "content": "<filename>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "7": {
+      "content": "<gh_stars>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "8": {
+      "content": "<issue_start>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "9": {
+      "content": "<issue_comment>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "10": {
+      "content": "<issue_closed>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "11": {
+      "content": "<jupyter_start>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "12": {
+      "content": "<jupyter_text>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "13": {
+      "content": "<jupyter_code>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "14": {
+      "content": "<jupyter_output>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "15": {
+      "content": "<jupyter_script>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "16": {
+      "content": "<empty_output>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "<|endoftext|>",
+    "<|im_start|>",
+    "<|im_end|>",
+    "<repo_name>",
+    "<reponame>",
+    "<file_sep>",
+    "<filename>",
+    "<gh_stars>",
+    "<issue_start>",
+    "<issue_comment>",
+    "<issue_closed>",
+    "<jupyter_start>",
+    "<jupyter_text>",
+    "<jupyter_code>",
+    "<jupyter_output>",
+    "<jupyter_script>",
+    "<empty_output>"
+  ],
+  "bos_token": "<|endoftext|>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|endoftext|>",
+  "extra_special_tokens": {},
+  "model_max_length": 8192,
+  "pad_token": "<|endoftext|>",
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>",
+  "vocab_size": 49152
+}