Initial upload: code, training data, tokenizers, notes

.gitattributes

@@ -33,3 +33,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+data/8_sample_B/train.txt filter=lfs diff=lfs merge=lfs -text
+data/8_sample_C/train.txt filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,24 @@
+# Secrets
+.env
+
+# Large model (18GB, download separately)
+models/qwen3.5-9b/
+
+# Python
+__pycache__/
+*.pyc
+*.pyo
+
+# Virtual environments
+envs/
+
+# Training artifacts (regenerated per experiment)
+checkpoints/*
+experiment_results/*
+
+# Temp
+.cache/
+.tmp/
+
+# OS
+.DS_Store

data/8_sample_B/train.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2c715e81e23c39b497a396e10702c248e0bdbd72c68562b58eb3d1d030e88098
+size 53928171

data/8_sample_C/train.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5ff8f32aa30fcd8338c90b2251527a9793a3f36150e7ebc4a37b0a679559567a
+size 53840442

data/rule_calibration.json

@@ -0,0 +1,360 @@
+{
+  "summary": {
+    "n_tasks": 70,
+    "avg_recall": 0.9068,
+    "n_high_recall": 56,
+    "n_low_recall": 4
+  },
+  "tasks": {
+    "adjunct_island": {
+      "total": 928,
+      "hits": 928,
+      "recall": 1.0
+    },
+    "anaphor_gender_agreement": {
+      "total": 971,
+      "hits": 942,
+      "recall": 0.9701
+    },
+    "anaphor_number_agreement": {
+      "total": 931,
+      "hits": 931,
+      "recall": 1.0
+    },
+    "animate_subject_passive": {
+      "total": 895,
+      "hits": 895,
+      "recall": 1.0
+    },
+    "animate_subject_trans": {
+      "total": 923,
+      "hits": 880,
+      "recall": 0.9534
+    },
+    "causative": {
+      "total": 818,
+      "hits": 417,
+      "recall": 0.5098
+    },
+    "complex_NP_island": {
+      "total": 846,
+      "hits": 846,
+      "recall": 1.0
+    },
+    "coordinate_structure_constraint_complex_left_branch": {
+      "total": 906,
+      "hits": 906,
+      "recall": 1.0
+    },
+    "coordinate_structure_constraint_object_extraction": {
+      "total": 949,
+      "hits": 949,
+      "recall": 1.0
+    },
+    "determiner_noun_agreement_1": {
+      "total": 929,
+      "hits": 929,
+      "recall": 1.0
+    },
+    "determiner_noun_agreement_2": {
+      "total": 931,
+      "hits": 931,
+      "recall": 1.0
+    },
+    "determiner_noun_agreement_irregular_1": {
+      "total": 681,
+      "hits": 681,
+      "recall": 1.0
+    },
+    "determiner_noun_agreement_irregular_2": {
+      "total": 820,
+      "hits": 820,
+      "recall": 1.0
+    },
+    "determiner_noun_agreement_with_adj_2": {
+      "total": 941,
+      "hits": 941,
+      "recall": 1.0
+    },
+    "determiner_noun_agreement_with_adj_irregular_1": {
+      "total": 718,
+      "hits": 718,
+      "recall": 1.0
+    },
+    "determiner_noun_agreement_with_adj_irregular_2": {
+      "total": 840,
+      "hits": 840,
+      "recall": 1.0
+    },
+    "determiner_noun_agreement_with_adjective_1": {
+      "total": 933,
+      "hits": 933,
+      "recall": 1.0
+    },
+    "distractor_agreement_relational_noun": {
+      "total": 788,
+      "hits": 438,
+      "recall": 0.5558
+    },
+    "distractor_agreement_relative_clause": {
+      "total": 871,
+      "hits": 871,
+      "recall": 1.0
+    },
+    "drop_argument": {
+      "total": 920,
+      "hits": 511,
+      "recall": 0.5554
+    },
+    "ellipsis_n_bar_2": {
+      "total": 828,
+      "hits": 72,
+      "recall": 0.087
+    },
+    "existential_there_object_raising": {
+      "total": 812,
+      "hits": 812,
+      "recall": 1.0
+    },
+    "existential_there_quantifiers_1": {
+      "total": 930,
+      "hits": 930,
+      "recall": 1.0
+    },
+    "existential_there_quantifiers_2": {
+      "total": 911,
+      "hits": 911,
+      "recall": 1.0
+    },
+    "existential_there_subject_raising": {
+      "total": 924,
+      "hits": 924,
+      "recall": 1.0
+    },
+    "expletive_it_object_raising": {
+      "total": 759,
+      "hits": 759,
+      "recall": 1.0
+    },
+    "inchoative": {
+      "total": 855,
+      "hits": 384,
+      "recall": 0.4491
+    },
+    "intransitive": {
+      "total": 868,
+      "hits": 109,
+      "recall": 0.1256
+    },
+    "irregular_past_participle_adjectives": {
+      "total": 961,
+      "hits": 961,
+      "recall": 1.0
+    },
+    "irregular_past_participle_verbs": {
+      "total": 942,
+      "hits": 900,
+      "recall": 0.9554
+    },
+    "irregular_plural_subject_verb_agreement_1": {
+      "total": 804,
+      "hits": 747,
+      "recall": 0.9291
+    },
+    "irregular_plural_subject_verb_agreement_2": {
+      "total": 892,
+      "hits": 881,
+      "recall": 0.9877
+    },
+    "left_branch_island_echo_question": {
+      "total": 947,
+      "hits": 947,
+      "recall": 1.0
+    },
+    "left_branch_island_simple_question": {
+      "total": 951,
+      "hits": 951,
+      "recall": 1.0
+    },
+    "matrix_question_npi_licensor_present": {
+      "total": 929,
+      "hits": 929,
+      "recall": 1.0
+    },
+    "npi_present_1": {
+      "total": 909,
+      "hits": 909,
+      "recall": 1.0
+    },
+    "npi_present_2": {
+      "total": 914,
+      "hits": 914,
+      "recall": 1.0
+    },
+    "only_npi_licensor_present": {
+      "total": 882,
+      "hits": 882,
+      "recall": 1.0
+    },
+    "only_npi_scope": {
+      "total": 837,
+      "hits": 837,
+      "recall": 1.0
+    },
+    "passive_1": {
+      "total": 840,
+      "hits": 777,
+      "recall": 0.925
+    },
+    "passive_2": {
+      "total": 903,
+      "hits": 834,
+      "recall": 0.9236
+    },
+    "principle_A_c_command": {
+      "total": 946,
+      "hits": 946,
+      "recall": 1.0
+    },
+    "principle_A_case_1": {
+      "total": 912,
+      "hits": 912,
+      "recall": 1.0
+    },
+    "principle_A_case_2": {
+      "total": 915,
+      "hits": 915,
+      "recall": 1.0
+    },
+    "principle_A_domain_1": {
+      "total": 914,
+      "hits": 914,
+      "recall": 1.0
+    },
+    "principle_A_domain_2": {
+      "total": 915,
+      "hits": 915,
+      "recall": 1.0
+    },
+    "principle_A_domain_3": {
+      "total": 941,
+      "hits": 941,
+      "recall": 1.0
+    },
+    "principle_A_reconstruction": {
+      "total": 967,
+      "hits": 902,
+      "recall": 0.9328
+    },
+    "regular_plural_subject_verb_agreement_1": {
+      "total": 890,
+      "hits": 670,
+      "recall": 0.7528
+    },
+    "regular_plural_subject_verb_agreement_2": {
+      "total": 945,
+      "hits": 932,
+      "recall": 0.9862
+    },
+    "sentential_negation_npi_licensor_present": {
+      "total": 919,
+      "hits": 919,
+      "recall": 1.0
+    },
+    "sentential_negation_npi_scope": {
+      "total": 871,
+      "hits": 871,
+      "recall": 1.0
+    },
+    "sentential_subject_island": {
+      "total": 961,
+      "hits": 961,
+      "recall": 1.0
+    },
+    "superlative_quantifiers_1": {
+      "total": 979,
+      "hits": 979,
+      "recall": 1.0
+    },
+    "superlative_quantifiers_2": {
+      "total": 986,
+      "hits": 986,
+      "recall": 1.0
+    },
+    "supplement_hypernym": {
+      "total": 842,
+      "hits": 688,
+      "recall": 0.8171
+    },
+    "supplement_qa_congruence_easy": {
+      "total": 64,
+      "hits": 64,
+      "recall": 1.0
+    },
+    "supplement_qa_congruence_tricky": {
+      "total": 165,
+      "hits": 165,
+      "recall": 1.0
+    },
+    "supplement_subject_aux_inversion": {
+      "total": 3867,
+      "hits": 3731,
+      "recall": 0.9648
+    },
+    "supplement_turn_taking": {
+      "total": 280,
+      "hits": 280,
+      "recall": 1.0
+    },
+    "tough_vs_raising_1": {
+      "total": 948,
+      "hits": 761,
+      "recall": 0.8027
+    },
+    "tough_vs_raising_2": {
+      "total": 920,
+      "hits": 793,
+      "recall": 0.862
+    },
+    "transitive": {
+      "total": 868,
+      "hits": 276,
+      "recall": 0.318
+    },
+    "wh_island": {
+      "total": 960,
+      "hits": 960,
+      "recall": 1.0
+    },
+    "wh_questions_object_gap": {
+      "total": 859,
+      "hits": 812,
+      "recall": 0.9453
+    },
+    "wh_questions_subject_gap_long_distance": {
+      "total": 857,
+      "hits": 857,
+      "recall": 1.0
+    },
+    "wh_vs_that_no_gap": {
+      "total": 861,
+      "hits": 667,
+      "recall": 0.7747
+    },
+    "wh_vs_that_no_gap_long_distance": {
+      "total": 875,
+      "hits": 676,
+      "recall": 0.7726
+    },
+    "wh_vs_that_with_gap": {
+      "total": 919,
+      "hits": 569,
+      "recall": 0.6192
+    },
+    "wh_vs_that_with_gap_long_distance": {
+      "total": 910,
+      "hits": 910,
+      "recall": 1.0
+    }
+  }
+}

models/tokenizer/special_tokens_map.json

@@ -0,0 +1,9 @@
+{
+  "bos_token": "<s>",
+  "cls_token": "<s>",
+  "eos_token": "</s>",
+  "mask_token": "<mask>",
+  "pad_token": "<pad>",
+  "sep_token": "</s>",
+  "unk_token": "<unk>"
+}

models/tokenizer/tokenizer_config.json

@@ -0,0 +1,54 @@
+{
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "<pad>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "4": {
+      "content": "<mask>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<s>",
+  "clean_up_tokenization_spaces": true,
+  "cls_token": "<s>",
+  "eos_token": "</s>",
+  "mask_token": "<mask>",
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<pad>",
+  "sep_token": "</s>",
+  "tokenizer_class": "PreTrainedTokenizerFast",
+  "unk_token": "<unk>"
+}

models/tokenizer_morfessor/morfessor.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c81fb6f4ae3052676df8c8e33222b747be11be8a15b7bbfc33a01147d47d65fe
+size 2734346

models/tokenizer_morfessor/special_tokens_map.json

@@ -0,0 +1,9 @@
+{
+  "bos_token": "<s>",
+  "cls_token": "<s>",
+  "eos_token": "</s>",
+  "mask_token": "<mask>",
+  "pad_token": "<pad>",
+  "sep_token": "</s>",
+  "unk_token": "<unk>"
+}

models/tokenizer_morfessor/tokenizer_config.json

@@ -0,0 +1,54 @@
+{
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "<pad>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "4": {
+      "content": "<mask>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<s>",
+  "clean_up_tokenization_spaces": true,
+  "cls_token": "<s>",
+  "eos_token": "</s>",
+  "mask_token": "<mask>",
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<pad>",
+  "sep_token": "</s>",
+  "tokenizer_class": "PreTrainedTokenizerFast",
+  "unk_token": "<unk>"
+}

notes/babylm_leaderboard_2025.csv

@@ -0,0 +1,91 @@
+=== Strict Track ===
+eval_name,Model,HF Repo,Revision commit,Text Average,Available on the hub,Main Contributions,Model Type,Base Architecture,Optimizer,Learning Rate,Learning Rate Scheduler,Num Training Epochs,Batch Size,Tokenizer,Token Set Size,Random Seed,Number of Layers,Number of Attention Heads,Max Sequence Length,Total Number of Parameters (M),Total Training PFLOPS,GPU Development Hours,GPU Train Hours,Training Dataset,Number of Words in Dataset (M),Genre of Dataset,Preprocessing of Dataset,Human Annotation in Dataset,Data Augmentation / Synthetic Data,Other Hyperparameters,Model Description,BLiMP,BLiMP Supplement,EWoK,Entity Tracking,WUG Adjective Nominalization,WUG Past Tense,COMPS,Reading,Self-paced Reading Time,Eye Tracking,AoA,(Super)GLUE,BoolQ,MNLI,MRPC,MultiRC,QQP,RTE,WSC,Track
+Baseline-gpt-bert-base-mixed (mntp)_strict,Baseline-gpt-bert-base-mixed (mntp),BabyLM-community/babylm-baseline-100m-gpt-bert-mixed,main,47.3,true,Baseline,Encoder and Decoder,GPT-BERT,LAMB,0.007,cosine with cooldown,10,131072,BPE,16384,,12,12,512,120,1000,48,144,BabyLM strict,100,BabyLM,BabyLM,Not applicable,Not applicable,"{""warmup_ratio"": 0.016, ""weight_decay"": 0.1, ""causal_ratio"": 0.5}","Baseline model for BabyLM 2025, trained with a 50-50 mix of both MNTP and CLM objectives and evaluated masked",80.5,73,52.4,39.9,41.2,27.1,59.7,6.3,3.4,9.2,22.3,70.9,73.4,63.4,85.8,69.8,81.2,59,63.5,strict
+Baseline-gpt-bert-base-causal-focus (causal)_strict,Baseline-gpt-bert-base-causal-focus (causal),BabyLM-community/babylm-baseline-100m-gpt-bert-causal-focus,main,46.7,true,Baseline,Encoder and Decoder,GPT-BERT,LAMB,0.007,cosine with cooldown,10,131072,BPE,16384,,12,12,512,120,1000,48,144,BabyLM strict,100,BabyLM,BabyLM,Not applicable,Not applicable,"{""warmup_ratio"": 0.016, ""weight_decay"": 0.1, ""causal_ratio"": 0.9375}","Baseline model for BabyLM 2025, trained with a 1-15 mix of both MNTP and CLM objectives and evaluated causally",79.3,70.4,52.3,30.9,63,26.8,58.3,5.7,3,8.4,10.7,70.1,72,60.4,80.9,67.5,81.3,64.7,63.5,strict
+Baseline-gpt-bert-base-causal-focus (mntp)_strict,Baseline-gpt-bert-base-causal-focus (mntp),BabyLM-community/babylm-baseline-100m-gpt-bert-causal-focus,main,46.2,true,Baseline,Encoder and Decoder,GPT-BERT,LAMB,0.007,cosine with cooldown,10,131072,BPE,16384,,12,12,512,120,1000,48,144,BabyLM strict,100,BabyLM,BabyLM,Not applicable,Not applicable,"{""warmup_ratio"": 0.016, ""weight_decay"": 0.1, ""causal_ratio"": 0.9375}","Baseline model for BabyLM 2025, trained with a 1-15 mix of both MNTP and CLM objectives and evaluated masked",78.1,68.6,53.5,41.9,39.9,32.7,60,6.1,3.1,9.1,11.5,70.1,72,60.4,80.9,67.5,81.3,64.7,63.5,strict
+Baseline-gpt-bert-base-mixed (causal)_strict,Baseline-gpt-bert-base-mixed (causal),BabyLM-community/babylm-baseline-100m-gpt-bert-mixed,main,45.3,true,Baseline,Encoder and Decoder,GPT-BERT,LAMB,0.007,cosine with cooldown,10,131072,BPE,16384,,12,12,512,120,1000,48,144,BabyLM strict,100,BabyLM,BabyLM,Not applicable,Not applicable,"{""warmup_ratio"": 0.016, ""weight_decay"": 0.1, ""causal_ratio"": 0.5}","Baseline model for BabyLM 2025, trained with a 50-50 mix of both MNTP and CLM objectives and evaluated causally",78.4,69.2,51.8,33.2,58.2,17.9,56.9,6.2,3.6,8.7,10.4,70.9,73.4,63.4,85.8,69.8,81.2,59,63.5,strict
+Baseline-gpt-bert-base-masked-focus (mntp)_strict,Baseline-gpt-bert-base-masked-focus (mntp),BabyLM-community/babylm-baseline-100m-gpt-bert-masked-focus,main,45.2,true,Baseline,Encoder and Decoder,GPT-BERT,LAMB,0.007,cosine with cooldown,10,131072,BPE,16384,,12,12,512,120,1000,48,144,BabyLM strict,100,BabyLM,BabyLM,Not applicable,Not applicable,"{""warmup_ratio"": 0.016, ""weight_decay"": 0.1, ""causal_ratio"": 0.0625}","Baseline model for BabyLM 2025, trained with a 15-1 mix of both MNTP and CLM objectives and evaluated masked",80.7,75.3,51.8,41.5,35,23.2,58.3,6.3,3.3,9.3,9.6,70.6,72,63.5,87.3,69.1,81.7,59,61.5,strict
+Baseline-gpt-bert-base-masked-focus (causal)_strict,Baseline-gpt-bert-base-masked-focus (causal),BabyLM-community/babylm-baseline-100m-gpt-bert-masked-focus,main,44.2,true,Baseline,Encoder and Decoder,GPT-BERT,LAMB,0.007,cosine with cooldown,10,131072,BPE,16384,,12,12,512,120,1000,48,144,BabyLM strict,100,BabyLM,BabyLM,Not applicable,Not applicable,"{""warmup_ratio"": 0.016, ""weight_decay"": 0.1, ""causal_ratio"": 0.0625}","Baseline model for BabyLM 2025, trained with a 15-1 mix of both MNTP and CLM objectives and evaluated causally",74.6,63.6,51.6,31,51.1,25.1,55.7,6,3.3,8.8,12.8,70.6,72,63.5,87.3,69.1,81.7,59,61.5,strict
+blalm-100m-dynmod-bounded_strict,blalm-100m-dynmod-bounded,PatrickHaller/babylm_2025_submission_strict,main,42.1,false,Architectural innovations,Decoder only,LSTM,Muon,0.00055,cosine with warmup,10,64,WordPiece,15000,42,24,16,512,270,20250,200,10,PatrickHaller/BabyLM2025-Strict-Dataset,100,"CHILDES,TinyStories,Simple Wikipedia, Cosmopedia, Project Gutenberg Fiction Books,Fineweb-Edu","Grammar Correction, Filtering Short Sequences",Not applicable,Not applicable,,Local context enhancement applied to a mLSTM token mixer backend,74.7,61,53.6,22.2,47.5,37.5,58.3,0.8,0.6,1.1,8.6,56.7,64,34.3,69.1,57.5,58.9,51.8,61.5,strict
+simple_diffusion_cosine_strict,simple_diffusion_cosine,despoinakk/diffusion_cosine_babylm,main,40.1,true,Training objective innovations,Encoder only,LTG-BERT,lamb,0,cosine_schedule_with_warmup_cooldown,10,197228,bpe,16384,0,12,12,512,127,0,5000,28,BabyLM strict,100,BabyLM,BabyLM,Not applicable,Not applicable,,This is a model based on the LTG-BERT architecture that was trained with a simple masked diffusion objective with a cosine noise schedule and a token frequency-informed masking strategy.,76.9,72.4,51.8,40.8,49.6,15.4,56.4,7.4,4.1,10.6,-22,52.3,62,31.8,32.8,63,63.8,54.7,57.7,strict
+babylm-baseline-100m-gpt2_strict,babylm-baseline-100m-gpt2,BabyLM-community/babylm-baseline-100m-gpt2,main,40.1,true,Baseline,Decoder only,GPT-2,AdamW,0.00005,Cosine scheduler with warmup and decay,10,8192,BPE,16384,242601,12,12,512,,1290,6,4,BabyLM strict,100,BabyLM,BabyLM,Not applicable,Not applicable,"{""base_folder"": ""experiments"", ""batch_size"": 16, ""checkpoint_dir"": ""experiments/strict_proper_tokenizer/checkpoints"", ""datapoint_length"": 512, ""expdir"": ""experiments/strict_proper_tokenizer"", ""experiment_name"": ""strict_proper_tokenizer"", ""gradient_clip_norm"": 1, ""learning_rate"": 5e-05, ""logdir"": ""experiments/strict_proper_tokenizer/logging"", ""n_epochs"": 10, ""num_training_steps"": 211650, ""num_warmup_steps"": 2116, ""seed"": 242601, ""sft_learning_rate"": 5e-05, ""training_type"": ""strict"", ""use_wandb"": false, ""wandb_experiment_name"": ""testing_new_acc"", ""wandb_project_name"": ""babylm_gpt2_sweeps"", ""weight_decay"": 0}",A naive autoregressive baseline with the GPT-2 architecture for the strict track,74.9,63.3,51.7,31.5,50.2,7.3,56.2,5.5,3.2,7.9,5.3,54.7,57.1,33.1,57.4,58.8,67.6,53.2,55.8,strict
+CLASS-IT_140M_strict,CLASS-IT_140M,colinglab/CLASS_IT-140M,main,39.9,true,"Curriculum learning, Data augmentation, Dataset creation",Decoder only,Llama,AdamW,0.0002,"linear, cosine with restarts",,15,bpe,32000,na,12,11,6144,,,50,60,BabyLM strict,100,BabyLM,"For the conversational section, the Switchboard data were adapted to meet the requirements of IT training task. Consecutive utterances from the same speaker were merged to ensure a consistent alternation between speakers’ turns (e.g., A, B, A, B). The dialogues were then segmented into prompt–reply pairs using a sliding window approach with the following schema: (A1, B1), (B1, A2), (A2, B2). 
+For the instructional section, Simple Wikipedia data were augmented using LLaMA-3.2-3B-Instruct. For each article text, three question–answer pairs were generated using structured generation with outlines and the following prompt: Based on the following text, generate 3 questions and detailed, informative answers. Each answer should be easy for a young person to understand and at least 2–3 sentences long. Explain things in simple language, with clear and friendly sentences. Avoid short or vague replies and give enough detail so a kid can learn something new.",Not applicable,We used LLaMA-3.2-3B-Instruct to generate an instruction tuning dataset,,"This is a small-scale LLaMA-style model, fine-tuned in sequence on a conversational dataset followed by an instructional dataset.",75.6,58.2,51.1,19.9,60.1,9.6,56,0.3,0.1,0.5,11.8,56.6,64,34.3,70.1,57.5,62.5,46,61.5,strict
+100m-bert-FRE-single-sorted_strict,100m-bert-FRE-single-sorted,JungwonPark02/FRE_single_sorted_bert,main,37.5,true,Curriculum learning,Encoder only,BERT,AdamW,0.00005,linear,10,32768,Custom WordPiece tokenizer trained on the same custom datasets used for model training,30522,42,12,12,512,110,66000,100,4,BabyLM strict,100,BabyLM,"We used the subset of COSMOPEDIA (Ben Allal et al., 2024) corpus where each sentence was assigned a FRE score, which served as the basis for organizing the training data according to linguistic difficulty.",No,No,"{""architectures"": [""BertForMaskedLM""], ""attention_probs_dropout_prob"": 0.1, ""classifier_dropout"": null, ""hidden_act"": ""gelu"", ""hidden_dropout_prob"": 0.1, ""hidden_size"": 768, ""initializer_range"": 0.02, ""intermediate_size"": 3072, ""layer_norm_eps"": 1e-12, ""max_position_embeddings"": 512, ""model_type"": ""bert"", ""num_attention_heads"": 12, ""num_hidden_layers"": 12, ""pad_token_id"": 0, ""position_embedding_type"": ""absolute"", ""torch_dtype"": ""float32"", ""transformers_version"": ""4.51.3"", ""type_vocab_size"": 2, ""use_cache"": true, ""vocab_size"": 30522}","Curriculum learning model trained by ordering training data according to the Flesch Reading Ease score, using sentence-level curriculum strategies.",74,57.5,51.2,36,44.9,-6.4,55.8,4.4,2.9,6,0,57.3,63.5,34,56.9,63,68.4,54,61.5,strict
+babylm2025-strict-batch-wise-convergent-pretraining-main-2_strict,babylm2025-strict-batch-wise-convergent-pretraining-main-2,ko-yoshida/exp1_averaged,main,34,true,"Curriculum learning, Training objective innovations",Decoder only,Qwen2,"AdamW, our custom optimizer",0.007,cosine,2,128,unigram,16000,42,24,14,512,372,2908,5,5,our custom corpus,100,single words/short sentences/essay/dialogues/stories,"We implemented a three-stage preprocessing approach to create a curriculum-based language learning dataset. First, we extracted single-word data from English Age of Acquisition (AoA) ratings, selecting words typically acquired by age 13 to serve as foundational lexical stimuli for the earliest training stage. Second, we generated synthetic short-sentence data by using a large language model (Qwen3-14B) to create 1-15 word sentences for each selected word, specifically prompting for simple, literal usage while excluding named entities and complex syntax. Sentences exceeding the length limit were discarded, and all markup was stripped with punctuation normalized. Third, we cleaned three existing child-directed corpora—KidLM (essays), TinyStories (narratives), and TinyDialogues (dialogues)—by shortening overly long sentences through splitting or compression while preserving meaning, removing tags and metadata, and condensing redundantly verbose paragraphs. This preprocessing pipeline produced streamlined, semantically coherent material that progresses from pure lexical items to minimal phrasal structures and finally to more complex narrative and dialogue content, all aligned with our staged curriculum approach.",Not applicable,We used Qwen3-14B to generate and clean data.,,"We conducted training in two stages with distinct configurations. In the first stage, we used a Qwen2-0.5B model trained on a custom corpus with a learning rate of 0.007 and cosine scheduling, incorporating a warmup period of 0.05 (5% of total steps). The training used a global batch size of 128, sequence length of 512 tokens, and was executed on a single node with H200 GPUs. For the second stage, we continued training from the checkpoint, significantly reducing the learning rate to 0.001 while maintaining the cosine scheduler but removing the warmup period entirely. The second stage utilized the same batch size and sequence length configurations, with custom corpus data that included specific parameters of p=1, C=1.0e-6, and 9 repetitions, running on a single H200 GPU. This two-stage approach allowed for initial rapid learning followed by fine-tuned optimization with reduced learning rates.",53,48.4,50,32.1,69.9,-1.3,50.4,0.1,0,0.2,-19.7,57.7,64,35.7,68.1,57.5,62.8,54,61.5,strict
+babylm2025-strict-batch-wise-convergent-pretraining-main_strict,babylm2025-strict-batch-wise-convergent-pretraining-main,ko-yoshida/exp1_averaged,main,34,true,"Curriculum learning, Training objective innovations",Decoder only,Qwen2,our custom optimizer,0.001,cosine,1,128,unigram,16000,42,24,14,512,372,2903,5,5,BabyLM strict,100,BabyLM,"We implemented a three-stage preprocessing approach to create a curriculum-based language learning dataset. First, we extracted single-word data from English Age of Acquisition (AoA) ratings, selecting words typically acquired by age 13 to serve as foundational lexical stimuli for the earliest training stage. Second, we generated synthetic short-sentence data by using a large language model (Qwen3-14B) to create 1-15 word sentences for each selected word, specifically prompting for simple, literal usage while excluding named entities and complex syntax. Sentences exceeding the length limit were discarded, and all markup was stripped with punctuation normalized. Third, we cleaned three existing child-directed corpora—KidLM (essays), TinyStories (narratives), and TinyDialogues (dialogues)—by shortening overly long sentences through splitting or compression while preserving meaning, removing tags and metadata, and condensing redundantly verbose paragraphs. This preprocessing pipeline produced streamlined, semantically coherent material that progresses from pure lexical items to minimal phrasal structures and finally to more complex narrative and dialogue content, all aligned with our staged curriculum approach.",Not applicable,We used Qwen3-14B to generate and clean data,,"We used a Qwen2-0.5B model trained on a custom corpus with a learning rate of 0.001 and cosine scheduling, incorporating a warmup period of 0.05 (5% of total steps). The training used a global batch size of 128, sequence length of 512 tokens, and was executed on a single node with H200 GPUs. We utilized specific parameters of p=1, C=1.0e-6, and 10 repetitions, running on a single H200 GPU.",53,48.4,50,32.1,69.9,-1.3,50.4,0.1,0,0.2,-19.7,57.7,64,35.7,68.1,57.5,62.8,54,61.5,strict
+babylm2025-strict-batch-wise-convergent-pretraining-1_strict,babylm2025-strict-batch-wise-convergent-pretraining-1,ko-yoshida/sample1,main,33.6,true,"Curriculum learning, Training objective innovations",Decoder only,Qwen2,"AdamW, our custom optimizer",0.007,cosine,2,128,unigram,16000,42,24,14,512,372,2903,5,5,BabyLM multimodal,50,BabyLM,"We implemented a three-stage preprocessing approach to create a curriculum-based language learning dataset. First, we extracted single-word data from English Age of Acquisition (AoA) ratings, selecting words typically acquired by age 13 to serve as foundational lexical stimuli for the earliest training stage. Second, we generated synthetic short-sentence data by using a large language model (Qwen3-14B) to create 1-15 word sentences for each selected word, specifically prompting for simple, literal usage while excluding named entities and complex syntax. Sentences exceeding the length limit were discarded, and all markup was stripped with punctuation normalized. Third, we cleaned three existing child-directed corpora—KidLM (essays), TinyStories (narratives), and TinyDialogues (dialogues)—by shortening overly long sentences through splitting or compression while preserving meaning, removing tags and metadata, and condensing redundantly verbose paragraphs. This preprocessing pipeline produced streamlined, semantically coherent material that progresses from pure lexical items to minimal phrasal structures and finally to more complex narrative and dialogue content, all aligned with our staged curriculum approach.",Not applicable,We used Qwen3-14B to generate and clean data.,,"We conducted training in two stages with distinct configurations. In the first stage, we used a Qwen2-0.5B model trained on a custom corpus with a learning rate of 0.007 and cosine scheduling, incorporating a warmup period of 0.05 (5% of total steps). The training used a global batch size of 128, sequence length of 512 tokens, and was executed on a single node with H200 GPUs. For the second stage, we continued training from the checkpoint, significantly reducing the learning rate to 0.001 while maintaining the cosine scheduler but removing the warmup period entirely. The second stage utilized the same batch size and sequence length configurations, with custom corpus data that included specific parameters of p=2, C=1, and 9 repetitions, running on a single H200 GPU. This two-stage approach allowed for initial rapid learning followed by fine-tuned optimization with reduced learning rates.",55.5,49.9,49.8,32,61.1,-8.9,49.8,0.1,0,0.2,-10.5,57.7,64,35.7,68.1,57.5,62.8,54,61.5,strict
+babylm2025-strict-batch-wise-convergent-pretraining-2_strict,babylm2025-strict-batch-wise-convergent-pretraining-2,ko-yoshida/exp4_averaged,main,32.3,true,"Curriculum learning, Training objective innovations",Decoder only,Qwen2,"AdamW, our custom optimizer",0.007,cosine,2,128,unigram,16000,42,24,14,512,372,2908,5,5,our custom corpus,100,single words/short sentences/essay/dialogues/stories,"We implemented a three-stage preprocessing approach to create a curriculum-based language learning dataset. First, we extracted single-word data from English Age of Acquisition (AoA) ratings, selecting words typically acquired by age 13 to serve as foundational lexical stimuli for the earliest training stage. Second, we generated synthetic short-sentence data by using a large language model (Qwen3-14B) to create 1-15 word sentences for each selected word, specifically prompting for simple, literal usage while excluding named entities and complex syntax. Sentences exceeding the length limit were discarded, and all markup was stripped with punctuation normalized. Third, we cleaned three existing child-directed corpora—KidLM (essays), TinyStories (narratives), and TinyDialogues (dialogues)—by shortening overly long sentences through splitting or compression while preserving meaning, removing tags and metadata, and condensing redundantly verbose paragraphs. This preprocessing pipeline produced streamlined, semantically coherent material that progresses from pure lexical items to minimal phrasal structures and finally to more complex narrative and dialogue content, all aligned with our staged curriculum approach.",Not applicable,We used Qwen3-14B to generate and clean data.,,"We conducted training in two stages with distinct configurations. In the first stage, we used a Qwen2-0.5B model trained on a custom corpus with a learning rate of 0.007 and cosine scheduling, incorporating a warmup period of 0.05 (5% of total steps). The training used a global batch size of 128, sequence length of 512 tokens, and was executed on a single node with H200 GPUs. For the second stage, we continued training from the checkpoint, significantly reducing the learning rate to 0.001 while maintaining the cosine scheduler but removing the warmup period entirely. The second stage utilized the same batch size and sequence length configurations, with custom corpus data that included specific parameters of p=2, C=1, and 9 repetitions, running on a single H200 GPU. This two-stage approach allowed for initial rapid learning followed by fine-tuned optimization with reduced learning rates.",56.1,51.4,50.2,40.8,46.4,-2.1,49.6,0.9,0,1.8,-27.7,57.7,64,35.7,68.1,57.5,62.8,54,61.5,strict
+
+=== Strict-small Track ===
+eval_name,Model,HF Repo,Revision commit,Text Average,Available on the hub,Main Contributions,Model Type,Base Architecture,Optimizer,Learning Rate,Learning Rate Scheduler,Num Training Epochs,Batch Size,Tokenizer,Token Set Size,Random Seed,Number of Layers,Number of Attention Heads,Max Sequence Length,Total Number of Parameters (M),Total Training PFLOPS,GPU Development Hours,GPU Train Hours,Training Dataset,Number of Words in Dataset (M),Genre of Dataset,Preprocessing of Dataset,Human Annotation in Dataset,Data Augmentation / Synthetic Data,Other Hyperparameters,Model Description,BLiMP,BLiMP Supplement,EWoK,Entity Tracking,WUG Adjective Nominalization,WUG Past Tense,COMPS,Reading,Self-paced Reading Time,Eye Tracking,AoA,(Super)GLUE,BoolQ,MNLI,MRPC,MultiRC,QQP,RTE,WSC,Track
+amlm_hd_fail_strict-small,amlm_hd_fail,leukas/amlm_hd_fail,main,45.4,true,Curriculum learning,Encoder only,DeBERTa,LAMB,0.007,Cosine,10,4096,BPE,40000,0,12,12,256,35,16.31,1,1,BabyLM strict-small,10,BabyLM,Concatenated columns together,No,No,,"Deberta model trained with Adaptive MLM, with decaying probability from 40% to 15%.
+This particular model's loss spiked due to too low batch size, but the results were strangely good for some metrics.",59,51.7,56,41.3,78.1,-12.1,82.2,6.4,2.9,9.9,34.2,57.7,64,35.7,68.1,57.5,62.8,54,61.5,strict-small
+MoEP_strict-small,MoEP,Jtapsa/moep,main,44.5,true,Architectural innovations,Decoder only,GPT-2,AdamW,0.0003,Linear,,16,BPE,512,42,12,,1024,28,,,1,BabyLM strict-small,10,BabyLM,BabyLM,Not applicable,Not applicable,"{""ModelArgs"": {""dim"": 384, ""n_layers"": 12, ""n_heads"": 6, ""ffn_m"": 4, ""head_dim"": 64, ""max_seq_len"": 1024, ""train_seq_len"": 512, ""batch_size"": 16, ""epochs"": 10, ""pad_id"": 3, ""vocab_size"": 16384, ""weight_tying"": true, ""model_name"": ""MoEP_0sw_64"", ""device"": ""cuda:0"", ""learning_rate"": 0.0003, ""weight_decay"": 0.1, ""betas"": [0.9, 0.95], ""adam_eps"": 1e-08, ""warmup_steps"": 800, ""scheduler_type"": ""cosine"", ""k"": 2, ""n_experts"": 4, ""sliding_window"": null, ""router_aux_weight"": 0.02}, ""ModelArgs_Parallel"": {""dim"": 192, ""n_heads"": 3, ""head_dim"": 64, ""n_parallel_blocks"": 4, ""device"": ""cuda:0""}}","Model follows GPT-2 baseline in Tokenizer, model + training parameters. Only smaller hidden dim  and fewer attention heads. But head dimension still same 64.
+
+Model is trained using BabyLM strict-small dataset, in total 10 Epochs, where each epoch is ended after approximate 10M whitespace separated words inputted to model. Initially I pre-tokenize training data with stride 128 and randomly select with epoch based seed training examples. 
+
+
+Model architecture: Large layer (equal to GPT-2 layer) --> MoE Block with 4 experts and top_2 routing. instead of FFNN only Linear layer used. Purpose to shrink hidden dimension to smaller size, suitable for parallel layers. --> 10 times parallel layer. Parallel layer uses top_2 routing among 4 'parallel blocks', where 'parallel blocks' equals to Large layer, but only operates in smaller hidden dimension (192 vs 384).  this allow more sparse token flow, but don't increase model total parameters. after parallel layers --> Moe Block to increase hidden dimension back to original and then --> Large layer --> norm and Linear.",60.8,57.5,50.2,35.6,66.8,-1.1,50.7,6.7,3.8,9.6,53.7,64.2,66.2,48.1,70.1,64.5,70.8,62.6,67.3,strict-small
+amlm_hard_nhot_strict-small,amlm_hard_nhot,leukas/amlm_hard_nhot,main,41.9,true,"Architectural innovations, Curriculum learning",Encoder only,DeBERTa,LAMB,0.007,Cosine,10,16384,BPE,40000,0,12,12,1256,50,22.59,1,1,BabyLM strict-small,10,BabyLM,Concatenated columns together,No,No,,"Deberta model trained with Adaptive MLM, with constant mask at 15% and n-hot encodings.",65.6,54.8,49.7,43.7,64.1,0.7,51.3,5.7,3.3,8.1,16.3,67.4,67.8,60,68.1,67.7,76.4,62.6,69.2,strict-small
+Baseline-gpt-bert-small-causal-focus (causal)_strict-small,Baseline-gpt-bert-small-causal-focus (causal),BabyLM-community/babylm-baseline-10m-gpt-bert-causal-focus,main,41.2,true,Baseline,Encoder and Decoder,GPT-BERT,LAMB,0.007,cosine with cooldown,10,16384,BPE,16384,42,12,6,512,31,30,33,11,BabyLM strict-small,10,BabyLM,BabyLM,Not applicable,Not applicable,"{""warmup_ratio"": 0.016, ""weight_decay"": 0.1, ""causal_ratio"": 0.9375}","Baseline model for BabyLM 2025, trained with a 1-15 mix of both MNTP and CLM objectives and evaluated causally",71.7,63.2,49.5,34.6,59.2,12.9,52.8,6.7,3.4,9.9,-3.9,65.1,68.1,46.9,74.5,68.3,76.7,56.1,65.4,strict-small
+Baseline-gpt-bert-small-causal-focus (mntp)_strict-small,Baseline-gpt-bert-small-causal-focus (mntp),BabyLM-community/babylm-baseline-10m-gpt-bert-causal-focus,main,41,true,Baseline,Encoder and Decoder,GPT-BERT,LAMB,0.007,cosine with cooldown,10,16384,BPE,16384,42,12,6,512,31,30,33,11,BabyLM strict-small,10,BabyLM,BabyLM,Not applicable,Not applicable,"{""warmup_ratio"": 0.016, ""weight_decay"": 0.1, ""causal_ratio"": 0.9375}","Baseline model for BabyLM 2025, trained with a 1-15 mix of both MNTP and CLM objectives and evaluated masked",69.1,64.3,49.6,39.2,46.2,11.2,53,6.5,3.5,9.5,5.9,65.1,68.1,46.9,74.5,68.3,76.7,56.1,65.4,strict-small
+Baseline-gpt-bert-small-masked-focus (causal)_strict-small,Baseline-gpt-bert-small-masked-focus (causal),BabyLM-community/babylm-baseline-10m-gpt-bert-masked-focus,main,40,true,Baseline,Encoder and Decoder,GPT-BERT,LAMB,0.007,cosine with cooldown,10,16384,BPE,16384,42,12,6,512,31,30,33,11,BabyLM strict-small,10,BabyLM,BabyLM,Not applicable,Not applicable,"{""warmup_ratio"": 0.016, ""weight_decay"": 0.1, ""causal_ratio"": 0.0625}","Baseline model for BabyLM 2025, trained with a 15-1 mix of both MNTP and CLM objectives and evaluated causally",65.2,59.5,49.5,31.1,44.7,20.7,52.9,6.5,3.4,9.5,3.8,66,67.6,51.8,78.9,67.4,77.4,57.6,61.5,strict-small
+Baseline-gpt-bert-small-mixed (mntp)_strict-small,Baseline-gpt-bert-small-mixed (mntp),BabyLM-community/babylm-baseline-10m-gpt-bert-mixed,main,39.8,true,Baseline,Encoder and Decoder,GPT-BERT,LAMB,0.007,cosine with cooldown,10,16384,BPE,16384,42,12,6,512,31,30,33,11,BabyLM strict-small,10,BabyLM,BabyLM,Not applicable,Not applicable,"{""warmup_ratio"": 0.016, ""weight_decay"": 0.1, ""causal_ratio"": 0.5}","Baseline model for BabyLM 2025, trained with a 50-50 mix of both MNTP and CLM objectives and evaluated masked",71.3,63.3,49.9,40,18.4,12.3,53.7,6.5,3.3,9.8,16.4,65.7,66.7,53.3,77.5,67,76.6,55.4,63.5,strict-small
+alexandertam_babylm-rta_strict-small,alexandertam/babylm-rta,babylm-anon/rta,main,39.3,true,Data augmentation,Decoder only,GPT-2,AdamW,0.00005,linear,10,16,BPE,16384,42,12,12,512,98,79.41,18,1,BabyLM strict,100,BabyLM,BabyLM,No,Only statistical information of the BabyLM corpus was used to generate synthetic data,,RecombiText Augmentation is a novel purely statistical NLP method to get the most out of low-resource datasets in LLM pre-training.,61.9,61.1,50.8,16.5,56.5,24,50.8,8.4,4.7,12.1,5.4,57.7,64,35.7,68.1,57.5,62.8,54,61.5,strict-small
+Baseline-gpt-bert-small-mixed (causal) _strict-small,Baseline-gpt-bert-small-mixed (causal),BabyLM-community/babylm-baseline-10m-gpt-bert-mixed,main,39.2,true,Baseline,Encoder and Decoder,GPT-BERT,LAMB,0.007,cosine with cooldown,10,16384,BPE,16384,42,12,6,512,31,30,33,11,BabyLM strict-small,10,BabyLM,BabyLM,Not applicable,Not applicable,"{""warmup_ratio"": 0.016, ""weight_decay"": 0.1, ""causal_ratio"": 0.5}","Baseline model for BabyLM 2025, trained with a 50-50 mix of both MNTP and CLM objectives and evaluated causally.",69.6,61.6,50.2,25.4,48.5,-3,53,6.4,3.4,9.5,14.5,65.7,66.7,53.3,77.5,67,76.6,55.4,63.5,strict-small
+gpt_bert_ACLM_mixed_4k (causal)_strict-small,gpt_bert_ACLM_mixed_4k (causal),EleniF/babylm_1_2_bs_64_4k_ACLM,main,39.1,true,"Curriculum learning, Hyperparameter tuning",Encoder-decoder,GPT-BERT,LAMB,0.0141,cosine,380,262,BPE,4000,42,12,6,128,31,0,5,6,BabyLM strict-small,10,BabyLM,BabyLM,No,No,,"This is the gpt-bert model trained using the Active Curriculum Language Modeling method with a 50:50 masked to causal ratio, a 4k BPE Tokenize, sequence length of 128 and batch size 64.",56.1,53.5,49.4,32.9,69.9,2.1,50.3,3.5,2.4,4.7,10.4,62.5,65.4,39.2,71.1,65.7,70.9,61.9,63.5,strict-small
+gpt_bert_ACLM_mixed_6k (mntp)_strict-small,gpt_bert_ACLM_mixed_6k (mntp),EleniF/babylm_1_2_6k_ACLM,main,38.8,true,"Curriculum learning, Hyperparameter tuning, Controlled experiments",Encoder-decoder,GPT-BERT,LAMB,0.0141,cosine,91,1048576,BPE,6000,42,12,6,128,31,0,25,8,BabyLM strict-small,10,BabyLM,BabyLM,No,No,,"This is the gpt-bert model trained using the Active Curriculum Language Modelling method with a  50:50 masked to causal objective ratio, a 6k BPE Tokenizer, sequence length of 128 and batch size 256.",54.6,50,49.9,36,79.3,4.3,50,3.5,2,4.9,-1,61.5,65.4,39.3,70.1,66.4,69.6,58.3,61.5,strict-small
+amlm_hard_decay_strict-small,amlm_hard_decay,leukas/amlm_hard_decay,main,38.3,true,Curriculum learning,Encoder only,DeBERTa,LAMB,0.007,Cosine,10,16384,BPE,40000,0,12,12,256,35,16.31,1,1,BabyLM strict,100,BabyLM,Concatenated columns together,No,No,,"Deberta model trained with Adaptive MLM, with mask probability decaying from 40% to 15%",71.4,59.2,51,44.2,22.3,6.2,54.2,5.9,3.5,8.3,-0.9,69.8,68.6,62.1,86.3,68.4,76.8,63.3,63.5,strict-small
+Baseline-gpt-bert-small-masked-focus (mntp)_strict-small,Baseline-gpt-bert-small-masked-focus (mntp),BabyLM-community/babylm-baseline-10m-gpt-bert-masked-focus,main,38.2,true,Baseline,Encoder and Decoder,GPT-BERT,LAMB,0.007,cosine with cooldown,10,16384,BPE,16384,42,12,6,512,31,30,33,11,BabyLM strict-small,10,BabyLM,BabyLM,Not applicable,Not applicable,"{""warmup_ratio"": 0.016, ""weight_decay"": 0.1, ""causal_ratio"": 0.0625}","Baseline model for BabyLM 2025, trained with a 15-1 mix of both MNTP and CLM objectives and evaluated masked",70.4,63.7,50,40.1,2.7,28.7,53.6,6.4,3.4,9.4,0.3,66,67.6,51.8,78.9,67.4,77.4,57.6,61.5,strict-small
+gpt_bert_ACLM_mixed_6k (causal)_strict-small,gpt_bert_ACLM_mixed_6k (causal),EleniF/babylm_1_2_6k_ACLM,main,38,true,"Curriculum learning, Hyperparameter tuning, Controlled experiments",Encoder-decoder,GPT-BERT,LAMB,0.0141,cosine,91,1048576,BPE,6000,42,12,6,128,31,0,25,8,BabyLM strict-small,10,BabyLM,BabyLM,Not applicable,Not applicable,,"This is the gpt-bert model trained using the Active Curriculum Language Modelling method with a  50:50 masked to causal objective ratio, a 6k BPE Tokenizer, sequence length of 128 and batch size 256.",55.3,50.7,49.9,33.5,78.8,-1.8,49.9,3.8,2.1,5.4,-1.5,61.5,65.4,39.3,70.1,66.4,69.6,58.3,61.5,strict-small
+gpt_bert_ACLM_mixed_4k (mntp)_strict-small,gpt_bert_ACLM_mixed_4k (mntp),EleniF/babylm_1_2_bs_64_4k_ACLM,main,38,true,"Curriculum learning, Hyperparameter tuning",Encoder-decoder,GPT-BERT,LAMB,0.0141,cosine,380,262,BPE,4000,42,12,6,128,31,0,5,6,BabyLM strict-small,10,BabyLM,BabyLM,Not applicable,Not applicable,,"This is the gpt-bert model trained using the Active Curriculum Language Modeling method with a 50:50 masked to causal ratio, a 4k BPE Tokenize, sequence length of 128 and batch size 64.",56.5,55.3,49.6,25.5,64,12.1,49.9,3.5,2.5,4.6,0.8,62.5,65.4,39.2,71.1,65.7,70.9,61.9,63.5,strict-small
+babylm-baseline-10m-gpt2_strict-small,babylm-baseline-10m-gpt2,BabyLM-community/babylm-baseline-10m-gpt2,main,37.4,true,Baseline,Decoder only,GPT-2,AdamW,0.00005,Cosine scheduler with warmup and decay,10,8192,BPE,16384,482549,12,12,512,,130,4,1,BabyLM strict-small,10,BabyLM,BabyLM,Not applicable,Not applicable,"{""base_folder"": ""experiments"", ""batch_size"": 16, ""checkpoint_dir"": ""experiments/strict_small_proper_tokenizer/checkpoints"", ""datapoint_length"": 512, ""expdir"": ""experiments/strict_small_proper_tokenizer"", ""experiment_name"": ""strict_small_proper_tokenizer"", ""gradient_clip_norm"": 1, ""learning_rate"": 5e-05, ""logdir"": ""experiments/strict_small_proper_tokenizer/logging"", ""n_epochs"": 10, ""num_training_steps"": 21300, ""num_warmup_steps"": 213, ""seed"": 482549, ""sft_learning_rate"": 5e-05, ""training_type"": ""strict_small"", ""use_wandb"": false, ""wandb_experiment_name"": ""testing_new_acc"", ""wandb_project_name"": ""babylm_gpt2_sweeps"", ""weight_decay"": 0}",A naive autoregressive baseline with the GPT-2 architecture for the strict-small track,66.4,57.1,49.9,13.9,66.1,-5,51.7,6.5,4.3,8.7,11.7,55.9,52.1,33.1,67.6,57.5,63.6,56.1,61.5,strict-small
+TICL_strict-small,TICL,babylm-anon/TICL,main,37.3,true,"Curriculum learning, Controlled experiments",Encoder only,RoBERTa,torch.optim.AdamW,0.0005,polynomial,10,2048,transformers.RobertaTokenizerFast,52000,42,12,12,514,126,674.31,150,3,babylm-anon/stratified_10m_curriculum,10,"Child Directed Speech, Children's Books, Dialogue, Educational, Written English",As in the babylm/babylm_data_preprocessing repository,No,No,"{""_name_or_path"": ""models/stratified_10m_curriculum_roberta_roberta_influence_incr_bins_lognorm_babylm"", ""architectures"": [""RobertaForMaskedLM""], ""attention_probs_dropout_prob"": 0.1, ""bos_token_id"": 0, ""classifier_dropout"": null, ""eos_token_id"": 2, ""hidden_act"": ""gelu"", ""hidden_dropout_prob"": 0.1, ""hidden_size"": 768, ""initializer_range"": 0.02, ""intermediate_size"": 3072, ""layer_norm_eps"": 1e-05, ""max_position_embeddings"": 514, ""model_type"": ""roberta"", ""num_attention_heads"": 12, ""num_hidden_layers"": 12, ""pad_token_id"": 1, ""position_embedding_type"": ""absolute"", ""torch_dtype"": ""float32"", ""transformers_version"": ""4.47.0"", ""type_vocab_size"": 1, ""use_cache"": true, ""vocab_size"": 52000}",A RoBERTa model pre-trained on a dataset of 10M words using (**T**raining Data) **I**nfluence-driven **C**urriculum **L**earning.,69.9,53.9,50.3,34.3,61.5,-8.9,50.7,2.1,0.2,4,-5.1,64.6,67.6,48.4,73,64.4,70.3,57.6,71.2,strict-small
+gpt2-10M-mop-eng_strict-small,gpt2-10M-mop-eng,NeTS-lab/babylm-mop-10m-gpt2,main,37,true,"Linguistic bias, Evaluation methods, Tokenization",Decoder only,GPT-2,AdamW,0.00005,Linear with warmup,10,16,MorPiece,23405,-1,12,12,1024,104,2000,15,1,BabyLM strict-small,10,BabyLM,BabyLM,Not applicable,Not applicable,,"This is a GPT-2 model trained with MorPiece, a custom tokenizer designed to build a character-based TRIE structure out of the training corpus and identify morphological boundaries by looking at the frequency of father/child nodes.",63.5,52.6,50.6,64.4,37.6,12.1,55.8,1,0.7,1.2,-25.6,57.7,64,35.7,68.1,57.5,62.8,54,61.5,strict-small
+mtp-gpt2_strict-small,mtp-gpt2,aynetdia/babylm-mtp-gpt2,main,36.5,true,"Curriculum learning, Training objective innovations",Decoder only,GPT-2,AdamW,0.00005,Cosine with min LR,10,16,BPE,16384,42,12,12,512,0,0,50,1,BabyLM strict-small,10,BabyLM,BabyLM,Not applicable,Not applicable,,"Baseline GPT-2 model with additional language modeling head on top of the last layer. Trained for the first 5 epochs with a 2-token prediction objective (multi-token prediction), and for the last 5 epochs with a regular next token prediction objective.",61.9,57.6,50.2,18.6,71.7,-9,51.1,7.7,4.3,11.2,0,55.4,62.8,33.1,73,43.2,61.4,52.5,61.5,strict-small
+blalm-10m-short-conv_strict-small,blalm-10m-short-conv,PatrickHaller/babylm_2025_submission_strict-small2,main,35.2,false,Architectural innovations,Decoder only,LSTM,Muon,0.0007,cosine with warmup,10,64,Wordpiece,15000,42,24,16,512,270,202.5,200,1,PatrickHaller/BabyLM2025-Strict-Small-Dataset,10,"CHILDES, Fineweb-Edu, TinyStories,Project Gutenberg Fiction Books, Simple Wikipedia, Cosmopedia","Grammar Correctio, Short Sequence filtering",No,No,,A xLSTM based language model with local context modifications.,67,53.3,50.6,33.7,50.3,-20.7,50.5,1,1,1.1,8.6,57.4,64,34.2,69.6,57.5,59.9,54.7,61.5,strict-small
+fasttext-cross-lora (mlm)_strict-small,fasttext-cross-lora (mlm),knalin55/CUNI_FT-CROSS-LoRA,main,34.9,true,"Controlled experiments, Curriculum learning",Encoder only,BERT,AdamW,0.00005,linear,6,68,bert-base-cased,28896,59,12,12,512,108,0,1,9,BabyLM strict-small,10,BabyLM,None,Not applicable,Custom Synthetic Data - Non linguistic,,The model is first initialized with FastText. It is then trained on the generated artificial language data. The model is finally trained using LoRA on the BabyLM corpus.,60.5,50.7,50.3,25.5,33.6,11,51.3,2.2,1,3.4,0,64,69.1,45.6,73.5,57.5,73.6,59.7,69.2,strict-small
+CUNI_FT-CROSS-LoRA_strict-small,CUNI_FT-CROSS-LoRA,knalin55/CUNI_FT-CROSS-LoRA,main,34.9,true,,Encoder only,BERT,,0,linear,0,0,,0,,0,0,0,0,0,0,0,BabyLM strict-small,10,BabyLM,BabyLM,Not applicable,Not applicable,,,60.5,50.7,50.3,25.5,33.6,11,51.3,2.2,1,3.4,0,64,69.1,45.6,73.5,57.5,73.6,59.7,69.2,strict-small
+amlm_hard_strict-small,amlm_hard,leukas/amlm_hard,main,34.2,true,Curriculum learning,Encoder only,DeBERTa,LAMB,0.007,Cosine,10,16384,BPE,40000,0,12,12,256,35,16.31,0,1,BabyLM strict-small,10,BabyLM,Concatenated columns together,No,No,,"Deberta model trained with Adaptive MLM, with mask probability constant at 15%",69.9,57.9,50,43.6,6.9,-2.1,53.3,6.4,4,8.9,-15,71.1,71.1,60.1,86.3,68.6,77.6,61.2,73.1,strict-small
+gpt2-10M-syllitok-eng_strict-small,gpt2-10M-syllitok-eng,Unknown,main,29.2,false,"Linguistic bias, Evaluation methods, Tokenization",Decoder only,GPT-2,AdamW,0.00005,Linear with warmup,10,16,SylliTok,20535,123768054,12,12,1024,102,993,5,2,BabyLM strict-small,10,BabyLM,BabyLM,Not applicable,Not applicable,,,63.1,58.8,49.9,33.9,33.1,-29.4,55.3,0.5,0.1,0.9,-31.7,58.1,64,35.7,68.6,57.5,62.4,56.8,61.5,strict-small
+gpt2-10M-parfind-eng_strict-small,gpt2-10M-parfind-eng,NeTSlab/gpt2-10M-parfind-eng,main,28,true,"Linguistic bias, Evaluation methods, Tokenization",Decoder only,GPT-2,AdamW,0.00005,Linear with warmup,10,16,ParadigmFinder,29215,829496234,12,12,1024,108,1060,8,2,BabyLM strict-small,10,BabyLM,BabyLM,Not applicable,Not applicable,,This is a GPT-2 model paired with a custom tokenizer that segments text based on a set of learned morphological paradigms.,65.2,58.8,49.4,21,-43.1,-2.6,56.6,0.3,0.3,0.2,16.3,57.8,64,35.7,68.1,57.5,62.8,54.7,61.5,strict-small
+llamalogue_strict-small,llamalogue,CLAUSE-Bielefeld/llamalogue,main,24.3,true,"Data augmentation, Data preprocessing, Evaluation methods, Controlled experiments, Dataset creation",Decoder only,Llama,AdamW,0.0003,cosine,10,128,BPE Tokenizer,8467,42,16,16,128,135,,0,30,CHILDES transcripts,10,CHILDES transcripts,"We clean the transcripts from CHILDES quite heavily by removing all extra- and paralinguistic information. Furthermore, we replace all unintelligible or otherwise incomplete utterances, for which annotations as to the intended word are available, with these intended words. Finally, we split all utterances that contain explicitly annotated pauses, as there is no clear distinction between such pauses and utterance boundaries marked by regular line breaks. From these cleaned dialogues, we extract all utterance triplets (three consecutive turns) where at least two different speakers are involved. Furthermore, we enforce the triplets to contain at least five lexical words. This excludes triplets that only contain repetitions of single words or are otherwise light on lexical content. We leave the speaker tags in the data.",Not applicable,Not applicable,,"This is our pre-trained model that is used as baseline, on which we perform different fine-tuning using PPO and DPO.",56,51.1,50.2,30.7,25.7,-2.8,51.6,2.7,1.4,3.9,-79.6,57.3,64,35.2,68.6,57.5,60.6,53.2,61.5,strict-small
+
+=== Multimodal Track ===
+eval_name,Model,HF Repo,Revision commit,Text Average,Available on the hub,Main Contributions,Model Type,Base Architecture,Optimizer,Learning Rate,Learning Rate Scheduler,Num Training Epochs,Batch Size,Tokenizer,Token Set Size,Random Seed,Number of Layers,Number of Attention Heads,Max Sequence Length,Total Number of Parameters (M),Total Training PFLOPS,GPU Development Hours,GPU Train Hours,Training Dataset,Number of Words in Dataset (M),Genre of Dataset,Preprocessing of Dataset,Human Annotation in Dataset,Data Augmentation / Synthetic Data,Other Hyperparameters,Model Description,Vision Average,BLiMP,BLiMP Supplement,EWoK,Entity Tracking,WUG Adjective Nominalization,WUG Past Tense,COMPS,Reading,Self-paced Reading Time,Eye Tracking,AoA,(Super)GLUE,BoolQ,MNLI,MRPC,MultiRC,QQP,RTE,WSC,VQA,Winoground,DevBench,Track
+smolvlm-256m-stage_multimodal,smolvlm-256m-stage,Unknown,main,37.1,false,Multimodality,Decoder only,Llama,AdamW​​,0.0001,warmup,12,2,GPT2TokenizerFast,49280,42,30,9,1200,256,0,60,50,BabyLM multimodal,50,BabyLM,BabyLM,Not applicable,Not applicable,,"The submit​​ model was efficiently trained using a ​​two-stage approach with text and image-text pairs​​, base on smolvlm-256M.",31.6,70.1,58.7,50.5,39.8,54.4,-12.8,53.3,0.1,0,0.2,0,57,64,34.9,69.1,57.5,62.5,49.6,61.5,26.5,50.5,17.8,multimodal
+babylm-multimodal-baseline-git-updated_multimodal,babylm-multimodal-baseline-git-updated,BabyLM-community/babylm-multimodal-baseline-flamingo,main,35.8,true,Baseline,Decoder only,GIT,AdamW,0.0001,Linear,4,5493,Unigram,32768,1,12,12,128,223,11300,72,72,BabyLM multimodal,50,BabyLM,BabyLM,Not applicable,Not applicable,,GIT baseline for the Multimodal track,49.7,71.1,63.8,51.8,18.9,14,5.8,59.2,1.2,0.5,2,9.3,63.3,67.2,51.6,70.1,58.4,72.8,57.6,65.4,41.1,56.4,51.6,multimodal
+babylm-multimodal-baseline-flamingo_multimodal,babylm-multimodal-baseline-flamingo,BabyLM-community/babylm-multimodal-baseline-flamingo,main,35.8,true,Baseline,Encoder-decoder,Flamingo,AdamW,0.0001,Linear,4,5662,Unigram,32768,1,12,12,128,,4840,144,58,BabyLM multimodal,50,BabyLM,BabyLM,Not applicable,Not applicable,,Multimodal baseline with the Flamingo architecture,50.3,70.9,65.1,51,18.4,16.3,4.7,59.2,1.9,0.8,3.1,1.2,68.8,69.3,59.5,78.9,62,77.7,70.5,63.5,43.3,54.8,52.7,multimodal
+babylm-multimodal-baseline-flamingo-updated_multimodal,babylm-multimodal-baseline-flamingo-updated,BabyLM-community/babylm-multimodal-baseline-flamingo,main,35.1,true,Baseline,Encoder-decoder,Flamingo,AdamW,0.0001,Linear,4,5662,Unigram,32768,1,12,12,128,255,4840,48,48,BabyLM multimodal,50,BabyLM,BabyLM,Not applicable,Not applicable,,Flamingo baseline for the Multimodal track,49.3,73.4,65.1,52.9,16.3,5.9,1.5,59.4,2.1,1.1,3.2,6.7,67.8,69.5,59,76.5,60.9,78.1,65.5,65.4,41.5,54.2,52.2,multimodal
+babylm-multimodal-baseline-git_multimodal,babylm-multimodal-baseline-git,BabyLM-community/babylm-multimodal-baseline-git,main,34.8,true,Baseline,Decoder only,GIT,AdamW,0.0001,Linear,4,5493,Unigram,32768,1,12,12,128,,11300,240,96,BabyLM multimodal,50,BabyLM,BabyLM,Not applicable,Not applicable,,Multimodal baseline with the GIT architecture,49.6,72.2,66.4,51.8,16.6,-0.6,9.1,60.1,0.9,0.2,1.6,8,63.7,68.4,52.5,71.6,57.5,73.1,59,63.5,40.6,56.2,52,multimodal
+BitMar-14M_multimodal,BitMar-14M,euhidaman/bitmar-attention-multimodal,main,25.7,false,"Architectural innovations, Multimodality",Encoder-decoder,GPT-2,adamw8bit,0.0002,cosine_with_restarts,10,674,GPT-2 BPE tokenizer,,N/A,8,4,256,,17.9,60,60,BabyLM multimodal,50,BabyLM,BabyLM,Not applicable,Not applicable,,"BitMar is a resource-efficient multimodal transformer architecture optimized for edge deployment, integrating ultra-low-bit (1.58-bit) quantized BitNet-based text and DiNOv2-derived vision encoders. Both encoders produce aligned 768-dimensional latent embeddings, fused via a lightweight cross-modal attention module operating with ternary-weight quantization. The fused representation queries a fixed-size episodic memory matrix (32 slots × 128 dimensions) using content-based addressing, retrieving contextually relevant vectors. These retrieved memory vectors are projected into layer-specific conditioning signals that modulate the autoregressive BitNet decoder layers. The decoder employs attention sinks combined with a sliding window mechanism (4 sink tokens + 1,020 recent tokens) to enable unbounded context length processing with fixed memory usage.",26.7,50.4,48.8,51.8,31.2,-15.8,-21.6,51.5,0.3,0.1,0.4,0,60.5,66.5,42.3,69.1,57.8,70.2,54,63.5,16.8,50,13.2,multimodal
+
+=== Interaction Track ===
+eval_name,Model,HF Repo,Revision commit,Text Average,Available on the hub,Main Contributions,Model Type,Base Architecture,Optimizer,Learning Rate,Learning Rate Scheduler,Num Training Epochs,Batch Size,Tokenizer,Token Set Size,Random Seed,Number of Layers,Number of Attention Heads,Max Sequence Length,Total Number of Parameters (M),Total Training PFLOPS,GPU Development Hours,GPU Train Hours,Training Dataset,Number of Words in Dataset (M),Genre of Dataset,Preprocessing of Dataset,Human Annotation in Dataset,Data Augmentation / Synthetic Data,Other Hyperparameters,Model Description,BLiMP,BLiMP Supplement,EWoK,Entity Tracking,WUG Adjective Nominalization,WUG Past Tense,COMPS,Reading,Self-paced Reading Time,Eye Tracking,AoA,(Super)GLUE,BoolQ,MNLI,MRPC,MultiRC,QQP,RTE,WSC,Track
+blm-gpt2s-901M-storytelling_interaction,blm-gpt2s-901M-storytelling,llm-slice/blm-gpt2s-90M-s42_901M-s42_submission,main,41,true,"Controlled experiments, Teacher/expert/auxiliary models, Training objective innovations",Decoder only,GPT-2,AdamW,0.000001,"Pretrain: warmup 0.01, PPO const.",331200,360,ByteLevelBPE,16000,42,12,12,512,124,40000,900,20,BabyLM strict,100,BabyLM,BabyLM,Not applicable,Not applicable,,"This GPT2-small model was pretrained on 90% of the BabyLM corpus for 10 epochs, followed by PPO RL learning by generating a story evaluated by a teacher model (Llama 3.1 8B Instruct).",77.5,56.7,51.4,33.1,71.1,4.5,55.2,0.1,0,0.2,7.7,52.5,58.7,32.3,54.9,41.3,68.9,49.6,61.5,interaction
+babylm-interaction-baseline-simpo_interaction,babylm-interaction-baseline-simpo,BabyLM-community/babylm-interaction-baseline-simpo,main,40.7,true,Baseline,Decoder only,GPT-2,AdamW,0.00005,Cosine scheduler with warmup and decay,9,9102,BPE,16384,162410,12,12,512,,7960,96,12,BabyLM strict,100,BabyLM,BabyLM,Not applicable,Not applicable,"{""base_folder"": ""experiments"", ""checkpoint_dir"": ""experiments/proper_tokenization_run/checkpoints"", ""context_proportion"": 0.5, ""datapoint_length"": 512, ""expdir"": ""experiments/proper_tokenization_run"", ""experiment_name"": ""proper_tokenization_run"", ""gradient_clip_norm"": 1, ""learning_rate"": 5e-05, ""logdir"": ""experiments/proper_tokenization_run/logging"", ""n_po_epochs"": 2, ""n_sft_epochs"": 7, ""num_rounds"": 20, ""num_training_steps"": 200000, ""num_warmup_steps"": 2000, ""save_generated_data"": true, ""seed"": 162410, ""sft_lambda"": 0.2, ""sft_learning_rate"": 5e-05, ""simpo_beta"": 2, ""simpo_gamma_ratio"": 0.5, ""start_round"": 0, ""student_model"": ""gpt2"", ""student_po_bsz"": 16, ""student_sample_bsz"": 512, ""student_sample_temperature"": 1.0, ""teacher_sample_bsz"": 512, ""teacher_sample_temperature"": 1.0, ""use_wandb"": false, ""wandb_experiment_name"": ""testing_new_acc"", ""wandb_project_name"": ""babylm_interaction_sweeps"", ""weight_decay"": 0}",A baseline for the interaction track leveraging multiple rounds of regular language modeling followed by the SimPO preference optimization method on the student and teacher outputs,72.2,61.2,51.9,28.1,51,17.9,55,6.3,3.5,9.1,6.4,56.5,58.8,32.6,66.7,57.5,66.5,51.8,61.5,interaction
+CLASS_IT-140M_interaction,CLASS_IT-140M,colinglab/CLASS_IT-140M,main,39.9,true,"Curriculum learning, Data preprocessing, Data augmentation, Dataset creation",Decoder only,Llama,AdamW,0.0002,"linear, cosine with restarts",,15,BPE,32,na,12,11,6144,,,50,60,BabyLM strict,100,BabyLM,"For the conversational section, the Switchboard data were adapted to meet the requirements of IT training task. Consecutive utterances from the same speaker were merged to ensure a consistent alternation between speakers’ turns (e.g., A, B, A, B). The dialogues were then segmented into prompt–reply pairs using a sliding window approach with the following schema: (A1, B1), (B1, A2), (A2, B2). 
+For the instructional section, Simple Wikipedia data were augmented using LLaMA-3.2-3B-Instruct. For each article text, three question–answer pairs were generated using structured generation with outlines and the following prompt: Based on the following text, generate 3 questions and detailed, informative answers. Each answer should be easy for a young person to understand and at least 2–3 sentences long. Explain things in simple language, with clear and friendly sentences. Avoid short or vague replies and give enough detail so a kid can learn something new.",No,We used LLaMA-3.2-3B-Instruct to generate an instruction tuning dataset,,"This is a small-scale LLaMA-style model, fine-tuned in sequence on a conversational dataset followed by an instructional dataset.",75.6,58.2,51.1,19.9,60.1,9.6,56,0.3,0.1,0.5,11.8,56.6,64,34.3,70.1,57.5,62.5,46,61.5,interaction
+Multitask-pretraining_1M_interaction,Multitask-pretraining_1M,https://huggingface.co/Wector1/Multitask-pretraining_1M,main,34.4,false,"Architectural innovations, Teacher/expert/auxiliary models, Training objective innovations, Dataset creation",Encoder only,ModernBERT,AdamW,0.0003,cosine,10,128,modified OLMo (ModernBERT),50368,42,22,12,256,,199.2,3,1,BabyLM Interaction,1,"diversified texts generated by LLM: reviews, emails, news,...","The generated dataset undergoes a simple filtering, consisting of deduplication and removing all instances that contain fewer than 3 words or contain non-English characters",No,diversified texts generated by LLM,,Model trained on 1M words of multitask data for 10 epochs with the multi-task pretraining objective described in the paper,56.8,54.8,49.8,19.8,40.2,-5.8,49.8,2.2,0.7,3.6,13.5,63.4,71.3,44.4,73,66.9,71.5,55.4,61.5,interaction
+communicative-baby-rfolmo_score_interaction,communicative-baby-rfolmo_score,CLAUSE-Bielefeld/communicative-baby-rfolmo_score,main,32.8,true,"Data augmentation, Data preprocessing, Evaluation methods, Dataset creation, Controlled experiments, Training objective innovations, Teacher/expert/auxiliary models",Decoder only,Llama,AdamW_torch_fused,0.00005,linear,,16,BPE Tokenizer,8467,0,16,16,128,135,149.79,1,40,CHILDES Transcripts,10,CHILDES Transcripts,"In a first preprocessing step, we clean the transcripts from CHILDES quite heavily by removing all extra- and paralinguistic information. Furthermore, we replace all unintelligible or otherwise incomplete utterances, for which annotations as to the intended word are available, with these intended words. Finally, we split all utterances that contain explicitly annotated pauses, as there is no clear distinction between such pauses and utterance boundaries marked by regular line breaks.
+
+From these cleaned dialogues, we extract all utterance triplets (three consecutive turns) where at least two different speakers are involved. Furthermore, we enforce the triplets to contain at least five lexical words. This excludes triplets that only contain repetitions of single words or are otherwise light on lexical content. We leave the speaker tags in the data.",Not applicable,Not applicable,,"This is the llamalogue model (our baseline) fine-tuned for 500,000 steps (less than 1 epoch) more epoch with PPO using a reward function generated by an LLM (OLMo).",55.3,50.6,50.1,36,26.7,-1.2,51.3,2.5,1.3,3.7,0,57.1,64,35.2,69.1,57.5,60.3,51.8,61.5,interaction
+communicative-baby-rfsemsim_interaction,communicative-baby-rfsemsim,CLAUSE-Bielefeld/communicative-baby-rfsemsim,main,31.5,true,"Data augmentation, Data preprocessing, Evaluation methods, Controlled experiments, Dataset creation, Training objective innovations, Teacher/expert/auxiliary models",Decoder only,Llama,AdamW_torch_fused,0.00005,linear,,16,BPE Tokenizer,8467,0,16,16,128,135,149.79,1,40,CHILDES Transcripts,10,CHILDES Transcripts,"In a first preprocessing step, we clean the transcripts from CHILDES quite heavily by removing all extra- and paralinguistic information. Furthermore, we replace all unintelligible or otherwise incomplete utterances, for which annotations as to the intended word are available, with these intended words. Finally, we split all utterances that contain explicitly annotated pauses, as there is no clear distinction between such pauses and utterance boundaries marked by regular line breaks.
+
+From these cleaned dialogues, we extract all utterance triplets (three consecutive turns) where at least two different speakers are involved. Furthermore, we enforce the triplets to contain at least five lexical words. This excludes triplets that only contain repetitions of single words or are otherwise light on lexical content. We leave the speaker tags in the data.",Not applicable,"We extract maternal utterances (minimum four tokens length) from unused segments of the pre-processed CHILDES dialogue triplets. We then prompt a teacher LLM, a Llama-3.2-3B, with these utterances, asking it to generate candidate responses simulating a short child-like answer that shows understanding of the mother utterance. We then calculate rewards by comparing these teacher-generated responses to the output produced by our pre-trained llamalogue in response to the same utterance.",,This is the llamalogue model (our baseline) fine-tuned for 1 more epoch with PPO using Semantic Similarity as a reward function.,56.4,51.5,49.6,34.6,30.6,-18.4,51.6,1.9,1,2.8,0,57.3,64,34.8,68.6,57.5,60.2,54.7,61.5,interaction
+communicative-baby-dpo_interaction,communicative-baby-dpo,CLAUSE-Bielefeld/communicative-baby-dpo,main,31.3,true,"Data augmentation, Data preprocessing, Evaluation methods, Controlled experiments, Dataset creation, Training objective innovations, Teacher/expert/auxiliary models",Decoder only,Llama,AdamW_torch_fused,0.000005,linear,,16,BPE Tokenizer,8467,42,16,16,128,135,149.79,1,31,CHILDES Transcripts,10,CHILDES Transcripts,"In a first preprocessing step, we clean the transcripts from CHILDES quite heavily by removing all extra- and paralinguistic information. Furthermore, we replace all unintelligible or otherwise incomplete utterances, for which annotations as to the intended word are available, with these intended words. Finally, we split all utterances that contain explicitly annotated pauses, as there is no clear distinction between such pauses and utterance boundaries marked by regular line breaks.
+
+From these cleaned dialogues, we extract all utterance triplets (three consecutive turns) where at least two different speakers are involved. Furthermore, we enforce the triplets to contain at least five lexical words. This excludes triplets that only contain repetitions of single words or are otherwise light on lexical content. We leave the speaker tags in the data.",Not applicable,"Examples of naturalistic DPO dialogue pairs. Each row shows a mother’s utterance (MOT) as prompt, with corresponding appropriate and random child responses (CHI) used for preference training",,"This is the llamalogue model (our baseline) fine-tuned for 10 more epoch with DPO  using a custom dataset. The dataset consists of around 240,000 tokens.",55.6,50,50.1,32.7,14.4,-1.3,51.5,2.5,1.4,3.6,0,57.2,64,35.2,68.6,57.5,60.6,52.5,61.5,interaction
+communicative-baby-rfconfidence_interaction,communicative-baby-rfconfidence,CLAUSE-Bielefeld/communicative-baby-rfconfidence,main,30.3,false,"Data augmentation, Data preprocessing, Evaluation methods, Controlled experiments, Dataset creation, Teacher/expert/auxiliary models, Training objective innovations",Decoder only,Llama,AdamW_torch_fused,0.000005,linear,,16,BPE Tokenizer,8467,0,16,16,128,135,149.79,1,40,CHILDES Transcripts,10,CHILDES Transcripts,"In a first preprocessing step, we clean the transcripts from CHILDES quite heavily by removing all extra- and paralinguistic information. Furthermore, we replace all unintelligible or otherwise incomplete utterances, for which annotations as to the intended word are available, with these intended words. Finally, we split all utterances that contain explicitly annotated pauses, as there is no clear distinction between such pauses and utterance boundaries marked by regular line breaks.
+
+From these cleaned dialogues, we extract all utterance triplets (three consecutive turns) where at least two different speakers are involved. Furthermore, we enforce the triplets to contain at least five lexical words. This excludes triplets that only contain repetitions of single words or are otherwise light on lexical content. We leave the speaker tags in the data.",Not applicable,"We extract maternal utterances (minimum four tokens length) from unused segments of the pre-processed CHILDES dialogue triplets. We then prompt a teacher LLM, a Llama-3.2-3B, with these utterances, asking it to generate candidate responses simulating a short child-like answer that shows understanding of the mother utterance. We then calculate rewards by comparing these teacher-generated responses to the output produced by our pre-trained llamalogue in response to the same utterance.",,This is the llamalogue model (our baseline) fine-tuned for 1 more epoch with PPO using a reward function based on confidence (details in the paper).,55.1,49.4,50.8,34,21.1,-18.8,51.6,2.2,1.1,3.4,0,57.2,64,34.9,68.6,57.5,60.3,53.2,61.5,interaction
+communicative-baby-dpo-synthetic_interaction,communicative-baby-dpo-synthetic,CLAUSE-Bielefeld/communicative-baby-dpo-synthetic,main,29.2,true,"Data augmentation, Data preprocessing, Teacher/expert/auxiliary models, Training objective innovations, Dataset creation, Controlled experiments, Evaluation methods",Decoder only,Llama,AdamW_torch_fused,0.0003,linear,,16,BPE Tokenizer,8467,41,16,16,128,135,149.79,1,31,CHILDES Transcripts,10,CHILDES Transcripts,"In a first preprocessing step, we clean the transcripts from CHILDES quite heavily by removing all extra- and paralinguistic information. Furthermore, we replace all unintelligible or otherwise incomplete utterances, for which annotations as to the intended word are available, with these intended words. Finally, we split all utterances that contain explicitly annotated pauses, as there is no clear distinction between such pauses and utterance boundaries marked by regular line breaks.
+
+From these cleaned dialogues, we extract all utterance triplets (three consecutive turns) where at least two different speakers are involved. Furthermore, we enforce the triplets to contain at least five lexical words. This excludes triplets that only contain repetitions of single words or are otherwise light on lexical content. We leave the speaker tags in the data.",Not applicable,"The dataset contains a mother sentence under the prompt column, then we prompted a Llama3b model with this model utterance asking to simulate a child and we collected the response under the chosen column, whereas a random non suitable answer is present under the rejected column.",,"This is the llamalogue model (our baseline) fine-tuned for 10 more epoch with DPO  using a custom dataset. The dataset consists of a total amount of 270,000 tokens.",55.5,51.7,50.8,31.3,-2.5,-4.8,51.6,0.8,0.4,1.2,0,57.2,64,34.9,68.6,57.5,60.6,53.2,61.5,interaction
+communicative-baby-rfbleu_interaction,communicative-baby-rfbleu,CLAUSE-Bielefeld/communicative-baby-rfbleu,main,28.8,true,"Evaluation methods, Controlled experiments, Dataset creation, Training objective innovations, Data augmentation, Data preprocessing",Decoder only,Llama,AdamW_torch_fused,0.00005,linear,,16,BPE Tokenizer,8467,0,16,16,128,135,149.79,1,40,CHILDES Transcripts,0,CHILDES Transcripts,"In a first preprocessing step, we clean the transcripts from CHILDES quite heavily by removing all extra- and paralinguistic information. Furthermore, we replace all unintelligible or otherwise incomplete utterances, for which annotations as to the intended word are available, with these intended words. Finally, we split all utterances that contain explicitly annotated pauses, as there is no clear distinction between such pauses and utterance boundaries marked by regular line breaks.
+
+From these cleaned dialogues, we extract all utterance triplets (three consecutive turns) where at least two different speakers are involved. Furthermore, we enforce the triplets to contain at least five lexical words. This excludes triplets that only contain repetitions of single words or are otherwise light on lexical content. We leave the speaker tags in the data.",Not applicable,"We extract maternal utterances (minimum four tokens length) from unused segments of the pre-processed CHILDES dialogue triplets. We then prompt a teacher LLM, a Llama-3.2-3B, with these utterances, asking it to generate candidate responses simulating a short child-like answer that shows understanding of the mother utterance. We then calculate rewards by comparing these teacher-generated responses to the output produced by our pre-trained llamalogue in response to the same utterance.",,This is the llamalogue model (our baseline) fine-tuned for 1 more epoch with PPO using BLEU metric as a reward function.,55.1,51.3,49.7,16.2,19.9,-15.4,50.7,2.7,2,3.4,0,57.4,64,34.6,69.1,57.5,61.7,53.2,61.5,interaction
+communicative-baby-rfconfidence-test_interaction,communicative-baby-rfconfidence-test,CLAUSE-Bielefeld/communicative-baby-rfconfidence,main,22.4,false,"Data augmentation, Data preprocessing, Evaluation methods, Controlled experiments, Dataset creation, Teacher/expert/auxiliary models, Training objective innovations",Decoder only,Llama,AdamW_torch_fused,0.000005,linear,,16,BPE Tokenizer,8467,0,16,16,128,135,149.79,1,40,CHILDES Transcripts,10,CHILDES Transcripts,"In a first preprocessing step, we clean the transcripts from CHILDES quite heavily by removing all extra- and paralinguistic information. Furthermore, we replace all unintelligible or otherwise incomplete utterances, for which annotations as to the intended word are available, with these intended words. Finally, we split all utterances that contain explicitly annotated pauses, as there is no clear distinction between such pauses and utterance boundaries marked by regular line breaks.
+
+From these cleaned dialogues, we extract all utterance triplets (three consecutive turns) where at least two different speakers are involved. Furthermore, we enforce the triplets to contain at least five lexical words. This excludes triplets that only contain repetitions of single words or are otherwise light on lexical content. We leave the speaker tags in the data.",Not applicable,"We extract maternal utterances (minimum four tokens length) from unused segments of the pre-processed CHILDES dialogue triplets. We then prompt a teacher LLM, a Llama-3.2-3B, with these utterances, asking it to generate candidate responses simulating a short child-like answer that shows understanding of the mother utterance. We then calculate rewards by comparing these teacher-generated responses to the output produced by our pre-trained llamalogue in response to the same utterance.",,This is the llamalogue model (our baseline) fine-tuned for 1 more epoch with PPO using a reward function based on confidence (details in the paper).,55.1,49.4,50.8,34,21.1,-18.8,51.6,2.2,1.1,3.4,-78.7,57.2,64,34.9,68.6,57.5,60.3,53.2,61.5,interaction

notes/experiment_log.md

@@ -0,0 +1,111 @@
+# 实验记录
+
+> BabyLM Challenge 2026 · Strict-Small 赛道
+> 每次实验完成后在此填写结果
+
+---
+
+## 一、完整实验表
+
+> **状态说明**: 🔲 待跑 | 🔄 运行中 | ✅ 完成 | ❌ 失败/放弃
+> **eval 后端**: causal = `AutoModelForCausalLM`；mntp = `AutoModelForMaskedLM`（causal attention）；mlm = `AutoModelForMaskedLM`（bidirectional）
+
+### 超参数
+
+| ID | 名称 | 架构 | 目标函数 | Tokenizer | Embedding | 掩码策略 | 优化器 | FORGETTER | 数据集 | Epochs | 状态 |
+|:--:|------|------|---------|-----------|-----------|---------|--------|:---------:|--------|:------:|:----:|
+| 1.1 | gptbert-baseline | gpt_bert | gpt_bert (MNTP+CLM 15:1) | bpe | standard | standard 30→15% | LAMB | ✗ | sample_B | 10 | 🔲 |
+| 1.2 | gpt2-baseline | gpt2 | clm | bpe | standard | — | AdamW | ✗ | sample_B | 10 | 🔲 |
+| 1.3 | modernbert-baseline | modernbert | mlm | bpe | standard | standard 30→15% | AdamW | ✗ | sample_B | 10 | 🔲 |
+| 1.4 | xlstm-baseline | xlstm | clm | bpe | standard | — | AdamW | ✗ | sample_B | 10 | 🔲 |
+| 1.5 | rtd-baseline | rtd | rtd | bpe | standard | — | LAMB | ✗ | sample_B | 10 | 🔲 |
+| 2.1 | gptbert-amlm | gpt_bert | gpt_bert | bpe | standard | **amlm** | LAMB | ✗ | sample_B | 10 | 🔲 |
+| 2.2 | gptbert-nhot | gpt_bert | gpt_bert | bpe | **nhot** | standard | LAMB | ✗ | sample_B | 10 | 🔲 |
+| 2.3 | gptbert-forgetter | gpt_bert | gpt_bert | bpe | standard | standard | LAMB | **✓** | sample_B | 10 | 🔲 |
+| 2.4 | gptbert-mask40 | gpt_bert | gpt_bert | bpe | standard | standard **40→15%** | LAMB | ✗ | sample_B | 10 | 🔲 |
+| 2.5 | gptbert-mntp1to1 | gpt_bert | gpt_bert **(1:1)** | bpe | standard | standard | LAMB | ✗ | sample_B | 10 | 🔲 |
+| 2.6 | gptbert-gradacc4 | gpt_bert | gpt_bert | bpe | standard | standard | LAMB | ✗ | sample_B | 10 | 🔲 |
+| 3.1 | gptbert-amlm-fgt | gpt_bert | gpt_bert | bpe | standard | amlm | LAMB | **✓** | sample_B | 10 | 🔲 |
+| 3.2 | gptbert-amlm-nhot | gpt_bert | gpt_bert | bpe | **nhot** | **amlm** | LAMB | ✗ | sample_B | 10 | 🔲 |
+| 3.3 | gptbert-amlm-fgt-nhot | gpt_bert | gpt_bert | bpe | **nhot** | **amlm** | LAMB | **✓** | sample_B | 10 | 🔲 |
+| 3.4 | best-morfessor | gpt_bert | gpt_bert | **morfessor** | — | — | — | — | sample_B | 10 | 🔲 |
+| 3.5 | best-sampleC | gpt_bert | gpt_bert | bpe | — | — | — | — | **sample_C** | 10 | 🔲 |
+| 3.6 | best-mntp-opt | gpt_bert | gpt_bert | bpe | — | — | — | — | sample_B | 10 | 🔲 |
+| 4.1 | best-ckpt-avg | gpt_bert | gpt_bert | bpe | — | — | — | — | sample_B | 10 | 🔲 |
+| 4.2 | best-early-stop | gpt_bert | gpt_bert | bpe | — | — | — | — | sample_B | best | 🔲 |
+| 4.3 | **final** | — | — | — | — | — | — | — | — | — | 🔲 |
+
+> Phase 3/4 中 `—` 表示"与 Phase 2 最佳实验相同，保留该列值"，跑完 Phase 2 后回填。
+
+---
+
+### 结果
+
+| ID | Loss | BLiMP (c) | BLiMP (m) | BLiMP-S (c) | BLiMP-S (m) | EWoK | Entity | WUG_adj | WUG_past | COMPS | Read_eye | Read_spr | GLUE | AoA | Text_avg | 最佳ckpt | 备注 |
+|:--:|:----:|:---------:|:---------:|:-----------:|:-----------:|:----:|:------:|:-------:|:--------:|:-----:|:--------:|:--------:|:----:|:---:|:--------:|:--------:|------|
+| 1.1 | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | |
+| 1.2 | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | |
+| 1.3 | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | |
+| 1.4 | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | |
+| 1.5 | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | |
+| 2.1 | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | |
+| 2.2 | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | |
+| 2.3 | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | |
+| 2.4 | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | |
+| 2.5 | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | |
+| 2.6 | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | |
+| 3.1 | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | |
+| 3.2 | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | |
+| 3.3 | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | |
+| 3.4 | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | |
+| 3.5 | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | |
+| 3.6 | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | |
+| 4.1 | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | |
+| 4.2 | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | |
+| 4.3 | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | — | |
+
+> **列说明**
+> `BLiMP (c)` = causal 后端总平均；`BLiMP (m)` = mntp/mlm 后端总平均
+> `WUG_adj/past` = Spearman's ρ；`Read_eye/spr` = R² change；`COMPS` = Accuracy
+> `AoA` = Spearman's ρ（需提交全部10个 checkpoint，单独跑 eval_aoa.sh）
+> `Text_avg` = 11列总平均（使用各模型对应的最佳后端：GPT-BERT/GPT-2/xLSTM 用 causal，ModernBERT 用 mlm）
+> `GLUE` = 7 子任务平均准确率；`最佳ckpt` = 最佳 epoch 编号
+
+---
+
+### 2025 Leaderboard Strict-Small 参考（来源：`notes/babylm_leaderboard_2025.csv`）
+
+| 模型 | TextAvg | BLiMP | Supp | EWoK | ET | WUG_ADJ | WUG_PAST | COMPS | AoA | GLUE |
+|------|:-------:|:-----:|:----:|:----:|:--:|:-------:|:--------:|:-----:|:---:|:----:|
+| amlm_hd_fail（冠军） | 45.4 | 59.0 | 51.7 | 56.0 | 41.3 | 78.1 | -12.1 | 82.2 | 34.2 | 57.7 |
+| MoEP（亚军） | 44.5 | 60.8 | 57.5 | 50.2 | 35.6 | 66.8 | -1.1 | 50.7 | 53.7 | 64.2 |
+| Baseline GPT-BERT (causal) | 41.2 | 71.7 | 63.2 | 49.5 | 34.6 | 59.2 | 12.9 | 52.8 | -3.9 | 65.1 |
+| Baseline GPT-2 | 37.4 | 66.4 | 57.1 | 49.9 | 13.9 | 66.1 | -5.0 | 51.7 | 11.7 | 55.9 |
+
+---
+
+## 二、关注子任务（详细追踪）
+
+Phase 1 跑完后，把 1.1 baseline 的 67 个 BLiMP 子任务得分填入，后续只追踪变化明显的子任务。
+
+| 子任务 | 1.1 | 最佳 | 说明 |
+|--------|:---:|:----:|------|
+| （Phase 1 跑完后回填） | | | |
+
+---
+
+## 三、实验笔记
+
+### Phase 1 结论
+> （跑完后填写：哪个架构最好，与 baseline 对比如何）
+
+### Phase 2 结论
+> （哪些特性有效，效果多大）
+
+### Phase 3 结论
+> （组合是否有叠加效应）
+
+### 最终选择
+> 最终提交模型：
+> 训练命令：
+> 导出命令：

notes/experiment_plan.md

@@ -0,0 +1,685 @@
+# 实验计划：BabyLM Challenge 2026 Strict-Small
+
+> **总实验数**：~380 个
+> **预计 GPU 时间**：~220 GPU 小时（3 卡并行约 3 天）
+> **设计原则**：漏斗型——早期用 3-epoch 快筛，赢家进入 10-epoch 全量训练
+> **更新日期**：2026-03-21
+
+---
+
+## 基线配置（所有实验的默认值，除非显式覆盖）
+
+```yaml
+# 架构
+arch: gpt_bert
+objective: gpt_bert
+hidden_size: 384
+num_layers: 12
+num_heads: 6
+intermediate_size: 1280
+use_rope: false
+use_geglu: true
+use_pre_norm: true
+use_attention_gate: false
+use_dwa: false
+use_moe: false
+use_attn_res: false
+
+# 数据
+data: sample_B
+tokenizer: bpe
+
+# 嵌入
+embedding: standard
+embedding_init: random
+
+# 掩码
+masking: standard
+mask_ratio: 0.30
+mask_ratio_end: 0.15
+mntp_ratio: 15
+
+# 优化器
+optimizer: lamb
+lr: 0.0141
+betas: (0.9, 0.98)
+eps: 1e-6
+weight_decay: 0.1
+forgetter: false
+
+# 训练
+epochs: 10
+batch_size: 64
+gradient_accumulation_steps: 1
+seq_len: 128
+dropout: 0.1
+warmup_ratio: 0.06
+max_grad_norm: 2.0
+fp16: true
+seed: 42
+
+# 蒸馏
+kd_enabled: false
+
+# Checkpoint
+ckpt_averaging: false
+```
+
+---
+
+## 阶段依赖关系
+
+```
+P1(架构) ──┬──→ P2(特性消融)
+            ├──→ P3(LR sweep) → P4(正则化) ──┐
+            └────────────────────────────────┴──→ P5(特性组合)
+                                                     │
+                    ┌────────────────────────────────┘
+                    ├──→ P6(掩码调参) → P7(Batch/SeqLen) → P8(优化器)
+                    └──→ P10(MoE调参，条件)                    │
+                                                P9(架构修改) ←─┘
+                                                     │
+                    ┌────────────────────────────────┤
+                    ├──→ P11(分词×嵌入)              │
+                    ├──→ P12(数据构建) ──┬──→ P13(跨架构验证)
+                    │                    └──→ P14(LR×Dropout网格)
+                    │                              │
+                    │                    P15(杂项超参) → P16(蒸馏) → P17(Epoch/Ckpt)
+                    │                                                      │
+                    │                                            P18(种子方差)
+                    │                                                      │
+                    └──────────────────→ P19(数据×特性交互) ←──────────────┘
+                                              │
+                                    P20(最终候选+消融) → P21(提交)
+
+P22(预留探索) ── 贯穿所有阶段
+```
+
+---
+
+## Phase 1：架构对比（5 个实验）
+
+**目标**：确定最佳基础架构。所有实验用基线配置，仅改架构。
+**决策规则**：按 BLiMP 排名，Top 2 晋级。赢家成为后续默认。
+
+| ID | 名称 | arch | objective | optimizer | epochs |
+|---|---|---|---|---|---|
+| 1.1 | gptbert-baseline | gpt_bert | gpt_bert | LAMB | 10 |
+| 1.2 | gpt2-baseline | gpt2 | clm | AdamW | 10 |
+| 1.3 | modernbert-baseline | modernized_bert | mlm | AdamW | 10 |
+| 1.4 | xlstm-baseline | xlstm | clm | AdamW | 10 |
+| 1.5 | rtd-baseline | rtd | rtd | LAMB | 10 |
+
+---
+
+## Phase 2：核心特性消融（9 个实验）
+
+**目标**：逐个测试高价值特性的独立贡献。
+**决策规则**：BLiMP ≥ +0.5pp 或任意副指标 ≥ +2pp 的特性为"赢家"，进入 P5 组合。
+
+| ID | 名称 | 改变的参数 | epochs |
+|---|---|---|---|
+| 2.1 | gptbert-amlm | masking=amlm | 10 |
+| 2.2 | gptbert-nhot | embedding=nhot | 10 |
+| 2.3 | gptbert-forgetter | forgetter=true | 10 |
+| 2.4 | gptbert-fasttext | embedding_init=fasttext | 10 |
+| 2.5 | gptbert-morfessor | tokenizer=morfessor_bpe | 10 |
+| 2.6 | gptbert-moe | use_moe=true (32exp, k=4, s=48) | 10 |
+| 2.7 | gptbert-attnres | use_attn_res=true (4 blocks) | 10 |
+| 2.8 | gptbert-moe-attnres | use_moe=true, use_attn_res=true | 10 |
+| 2.9 | gptbert-sampleC | data=sample_C | 10 |
+
+---
+
+## Phase 3：学习率 Sweep（17 个实验）
+
+**目标**：找 LAMB 在 GPT-BERT 上的最佳 LR。
+**方法**：3A 用 3-epoch 快筛 12 个值，3B 将 Top 3 + 2 个插值点跑满 10 epoch。
+
+### Phase 3A：快筛（12 个，3 epoch）
+
+| ID | lr | epochs |
+|---|---|---|
+| 3.1 | 0.003 | 3 |
+| 3.2 | 0.005 | 3 |
+| 3.3 | 0.007 | 3 |
+| 3.4 | 0.008 | 3 |
+| 3.5 | 0.009 | 3 |
+| 3.6 | 0.010 | 3 |
+| 3.7 | 0.012 | 3 |
+| 3.8 | 0.015 | 3 |
+| 3.9 | 0.018 | 3 |
+| 3.10 | 0.020 | 3 |
+| 3.11 | 0.025 | 3 |
+| 3.12 | 0.030 | 3 |
+
+### Phase 3B：Top 3 + 插值（5 个，10 epoch）
+
+| ID | lr | 说明 |
+|---|---|---|
+| 3.13 | BEST1(3A) | 第 1 名 |
+| 3.14 | BEST2(3A) | 第 2 名 |
+| 3.15 | BEST3(3A) | 第 3 名 |
+| 3.16 | mid(BEST1,BEST2) | 1-2 名插值 |
+| 3.17 | mid(BEST2,BEST3) | 2-3 名插值 |
+
+**输出**：BEST_LR
+
+---
+
+## Phase 4：正则化 Sweep（25 个实验）
+
+**目标**：找最佳 dropout / weight_decay / warmup / grad_norm。
+**固定**：lr = BEST(P3)，其余基线。
+
+### 4A：Dropout（7 个，3ep）
+
+| ID | dropout |
+|---|---|
+| 4.1-4.7 | 0.00, 0.02, 0.05, 0.08, 0.10, 0.15, 0.20 |
+
+### 4B：Weight Decay（7 个，3ep）
+
+| ID | weight_decay |
+|---|---|
+| 4.8-4.14 | 0.01, 0.03, 0.05, 0.08, 0.15, 0.20, 1.00 |
+
+### 4C：Warmup（5 个，3ep）
+
+| ID | warmup_ratio |
+|---|---|
+| 4.15-4.19 | 0.00, 0.03, 0.10, 0.15, 0.20 |
+
+### 4D：Grad Norm（4 个，3ep）
+
+| ID | max_grad_norm |
+|---|---|
+| 4.20-4.23 | 0.5, 1.0, 3.0, 5.0 |
+
+### 4E：最佳组合验证（2 个，10ep）
+
+| ID | 说明 |
+|---|---|
+| 4.24 | BEST(4A) + BEST(4B) + BEST(4C) + BEST(4D) |
+| 4.25 | 2ndBEST(4A) + BEST(4B) + BEST(4C) + BEST(4D) |
+
+**输出**：BEST_DROPOUT, BEST_WD, BEST_WARMUP, BEST_GRADNORM → 合称 **TUNED_BASELINE**
+
+---
+
+## Phase 5：特性组合（20 个实验）
+
+**目标**：P2 中赢出的特性做两两、三组、全组合测试。
+**固定**：TUNED_BASELINE（含 BEST_LR + BEST 正则化参数）。
+
+假设 P2 赢家为 {amlm, nhot, forgetter, moe, attnres}，以下为系统组合：
+
+| ID | 名称 | amlm | nhot | fgt | moe | attnres |
+|---|---|:---:|:---:|:---:|:---:|:---:|
+| 5.1 | tuned-baseline（对照） | | | | | |
+| 5.2 | tuned-amlm | ✓ | | | | |
+| 5.3 | tuned-nhot | | ✓ | | | |
+| 5.4 | tuned-fgt | | | ✓ | | |
+| 5.5 | tuned-moe | | | | ✓ | |
+| 5.6 | tuned-attnres | | | | | ✓ |
+| 5.7 | amlm+nhot | ✓ | ✓ | | | |
+| 5.8 | amlm+fgt | ✓ | | ✓ | | |
+| 5.9 | amlm+moe | ✓ | | | ✓ | |
+| 5.10 | nhot+fgt | | ✓ | ✓ | | |
+| 5.11 | nhot+moe | | ✓ | | ✓ | |
+| 5.12 | fgt+moe | | | ✓ | ✓ | |
+| 5.13 | moe+attnres | | | | ✓ | ✓ |
+| 5.14 | amlm+nhot+fgt | ✓ | ✓ | ✓ | | |
+| 5.15 | amlm+nhot+moe | ✓ | ✓ | | ✓ | |
+| 5.16 | amlm+fgt+moe | ✓ | | ✓ | ✓ | |
+| 5.17 | nhot+fgt+moe | | ✓ | ✓ | ✓ | |
+| 5.18 | amlm+nhot+fgt+moe | ✓ | ✓ | ✓ | ✓ | |
+| 5.19 | 全部 5 特性 | ✓ | ✓ | ✓ | ✓ | ✓ |
+| 5.20 | amlm+nhot+fgt+attnres | ✓ | ✓ | ✓ | | ✓ |
+
+**输出**：BEST_COMBO（最佳特性子集）
+
+---
+
+## Phase 6：掩码与训练目标调参（20 个实验）
+
+**固定**：TUNED_BASELINE + BEST_COMBO
+
+### 6A：掩码率（8 个，10ep）
+
+| ID | mask_ratio → mask_ratio_end |
+|---|---|
+| 6.1-6.8 | 15→15, 20→15, 25→15, 35→15, 40→15, 40→20, 45→15, 30→10 |
+
+### 6B：MNTP:CLM 比例（6 个，10ep）
+
+| ID | mntp_ratio |
+|---|---|
+| 6.9-6.14 | 1:1, 3:1, 5:1, 7:1, 10:1, 20:1 |
+
+### 6C：AMLM 参数（6 个，10ep，仅当 amlm ∈ BEST_COMBO）
+
+| ID | amlm_lambda | amlm_update_interval | amlm_min/max_ratio |
+|---|---|---|---|
+| 6.15 | 0.05 | 200 | 0.05/0.50 |
+| 6.16 | 0.10 | 200 | 0.05/0.50 |
+| 6.17 | 0.30 | 200 | 0.05/0.50 |
+| 6.18 | 0.20 | 50 | 0.05/0.50 |
+| 6.19 | 0.20 | 500 | 0.05/0.50 |
+| 6.20 | 0.20 | 200 | 0.10/0.60 |
+
+**输出**：BEST_MASK_RATIO, BEST_MNTP, BEST_AMLM_PARAMS
+
+---
+
+## Phase 7：Batch Size 与序列长度（14 个实验）
+
+### 7A：有效 Batch Size（8 个，10ep）
+
+| ID | batch_size | grad_accum | effective |
+|---|---|---|---|
+| 7.1 | 16 | 1 | 16 |
+| 7.2 | 32 | 1 | 32 |
+| 7.3 | 64 | 2 | 128 |
+| 7.4 | 64 | 4 | 256 |
+| 7.5 | 64 | 8 | 512 |
+| 7.6 | 128 | 1 | 128 |
+| 7.7 | 128 | 2 | 256 |
+| 7.8 | 256 | 1 | 256 |
+
+### 7B：序列长度（6 个，10ep）
+
+| ID | seq_len |
+|---|---|
+| 7.9-7.14 | 64, 96, 192, 256, 384, 512 |
+
+**输出**：BEST_BATCH, BEST_GRAD_ACCUM, BEST_SEQ_LEN
+
+---
+
+## Phase 8：优化器探索（16 个实验）
+
+### 8A：AdamW LR Sweep（6 个，3ep）
+
+| ID | optimizer | lr |
+|---|---|---|
+| 8.1-8.6 | AdamW | 0.0003, 0.0005, 0.001, 0.002, 0.003, 0.005 |
+
+### 8B：Muon LR Sweep（5 个，3ep）
+
+| ID | optimizer | lr |
+|---|---|---|
+| 8.7-8.11 | Muon | 0.005, 0.008, 0.014, 0.020, 0.030 |
+
+### 8C：最佳替代优化器 + FORGETTER（5 个，10ep）
+
+| ID | optimizer | lr | forgetter |
+|---|---|---|---|
+| 8.12 | AdamW | BEST(8A) | off |
+| 8.13 | AdamW | BEST(8A) | on |
+| 8.14 | Muon | BEST(8B) | off |
+| 8.15 | Muon | BEST(8B) | on |
+| 8.16 | LAMB | BEST_LR | on（重新确认） |
+
+**输出**：BEST_OPTIMIZER, BEST_OPT_LR → 合称 **CONFIG_V1**
+
+---
+
+## Phase 9：架构修改（15 个实验，10ep）
+
+**固定**：CONFIG_V1
+
+### 9A：Toggle Sweep（7 个）
+
+| ID | 修改 |
+|---|---|
+| 9.1 | use_rope=true |
+| 9.2 | use_geglu=false |
+| 9.3 | use_pre_norm=false |
+| 9.4 | use_attention_gate=true |
+| 9.5 | use_dwa=true |
+| 9.6 | rope + attention_gate |
+| 9.7 | rope + dwa |
+
+### 9B：模型形状（8 个）
+
+| ID | hidden | layers | heads | intermediate |
+|---|---|---|---|---|
+| 9.8 | 256 | 16 | 4 | 854 |
+| 9.9 | 256 | 18 | 4 | 854 |
+| 9.10 | 320 | 12 | 5 | 1067 |
+| 9.11 | 384 | 16 | 6 | 1280 |
+| 9.12 | 448 | 10 | 7 | 1494 |
+| 9.13 | 512 | 8 | 8 | 1706 |
+| 9.14 | 512 | 10 | 8 | 1706 |
+| 9.15 | 576 | 8 | 9 | 1920 |
+
+**输出**：BEST_ARCH_MODS, BEST_SHAPE
+
+---
+
+## Phase 10：MoE 超参调优（8 个实验，条件阶段）
+
+**条件**：仅当 MoE ∈ BEST_COMBO（P5）时执行，否则跳过。
+**固定**：CONFIG_V1 + use_moe=true
+
+| ID | num_experts | top_k | expert_size | freq_penalty |
+|---|---|---|---|---|
+| 10.1 | 16 | 2 | 96 | 0.01 |
+| 10.2 | 16 | 4 | 96 | 0.01 |
+| 10.3 | 32 | 2 | 48 | 0.01 |
+| 10.4 | 32 | 8 | 48 | 0.01 |
+| 10.5 | 64 | 4 | 24 | 0.01 |
+| 10.6 | 32 | 4 | 48 | 0.10 |
+| 10.7 | 32 | 4 | 48 | 0.001 |
+| 10.8 | 32 | 4 | 84 | 0.01 |
+
+**输出**：BEST_MOE_CONFIG
+
+---
+
+## Phase 11：分词器 × 嵌入交叉（6 个实验，10ep）
+
+| ID | tokenizer | embedding | init |
+|---|---|---|---|
+| 11.1 | bpe | standard | random |
+| 11.2 | bpe | nhot | random |
+| 11.3 | bpe | standard | fasttext |
+| 11.4 | bpe | nhot | fasttext |
+| 11.5 | morfessor_bpe | standard | random |
+| 11.6 | morfessor_bpe | nhot | random |
+
+**输出**：BEST_TOK_EMB → 合称 **CONFIG_V2**
+
+---
+
+## Phase 12：数据集构建方式（18 个实验，10ep）
+
+**固定**：CONFIG_V2
+
+| ID | 名称 | 数据配置 | 说明 |
+|---|---|---|---|
+| 12.1 | champion-replica | BabyLM 33% + FineWeb 33% + Cosmo 34% | Charpentier 2024 方案复刻 |
+| 12.2 | pure-quality | sample_A | 纯质量排序 |
+| 12.3 | task-quota | sample_B | 任务配额（当前默认） |
+| 12.4 | weighted-random | sample_C | 加权随机 |
+| 12.5 | semantic-match | sample_D | 嵌入相似度 |
+| 12.6 | quota+paraphrase | sample_B + Paraphrase ~2M | Qwen3.5 改写 |
+| 12.7 | quota+variation | sample_B + Variation Sets ~1M | 句式变体 |
+| 12.8 | quota+recombitext | sample_B + RecombiText ~5M | 统计增强 |
+| 12.9 | quota+cd-synth | sample_B + Contrastive Decoding ~3M | 自合成 |
+| 12.10 | quota+mattr | sample_B + MATTR 排序 | 词汇多样性排序 |
+| 12.11 | eval-mixed | 评测数据混入采样池 | 评测句参与采样 |
+| 12.12 | no-eval | 不使用评测数据 | 对照组 |
+| 12.13 | full-augment | Para + VS + MATTR | 全量增强组合 |
+| 12.14 | fineweb-heavy | sample_B + FineWeb 33% | 高 FineWeb 比例 |
+| 12.15 | fineweb-very-heavy | sample_B + FineWeb 67% | 重 FineWeb |
+| 12.16 | knowledge-combo | sample_B + ConceptNet + GenericsKB | 知识数据 |
+| 12.17 | para+fineweb | sample_B + Para 25% + FineWeb 25% | 改写+知识 |
+| 12.18 | 3way-equal | 1:1:1 official + FineWeb + Cosmo | 三等分 |
+
+**输出**：BEST_DATA
+
+---
+
+## Phase 13：跨架构验证（15 个实验，10ep）
+
+**目标**：用 CONFIG_V2 重测其他架构，看优化后是否有逆转。
+
+| ID | arch | use_moe | use_attn_res | 说明 |
+|---|---|---|---|---|
+| 13.1 | gpt2 | false | false | GPT-2 调优 |
+| 13.2 | gpt2 | true | false | GPT-2 + MoE |
+| 13.3 | gpt2 | false | true | GPT-2 + AttnRes |
+| 13.4 | gpt2 | true | true | GPT-2 + 两者 |
+| 13.5 | modernized_bert | false | false | ModernBERT 调优 |
+| 13.6 | modernized_bert | true | false | + MoE |
+| 13.7 | modernized_bert | false | false | + AMLM |
+| 13.8 | xlstm | false | false | xLSTM 调优 |
+| 13.9 | xlstm | true | false | + MoE |
+| 13.10 | xlstm | false | true | + AttnRes |
+| 13.11 | rtd | false | false | RTD 调优 |
+| 13.12 | rtd | true | false | RTD + MoE |
+| 13.13 | gpt2 | false | false | GPT-2 wide(512×8) |
+| 13.14 | modernized_bert | false | false | ModernBERT wide |
+| 13.15 | xlstm | false | false | xLSTM wide |
+
+**输出**：BEST_OVERALL_ARCH
+
+---
+
+## Phase 14：LR × Dropout 二维网格（27 个实验）
+
+### 14A：3-epoch 快筛（24 个）
+
+6 个 LR × 4 个 Dropout：
+
+| | d=0.0 | d=BEST/2 | d=BEST | d=BEST×1.5 |
+|---|---|---|---|---|
+| lr=BEST×0.7 | 14.1 | 14.2 | 14.3 | 14.4 |
+| lr=BEST×0.85 | 14.5 | 14.6 | 14.7 | 14.8 |
+| lr=BEST | 14.9 | 14.10 | 14.11 | 14.12 |
+| lr=BEST×1.15 | 14.13 | 14.14 | 14.15 | 14.16 |
+| lr=BEST×1.3 | 14.17 | 14.18 | 14.19 | 14.20 |
+| lr=BEST×1.5 | 14.21 | 14.22 | 14.23 | 14.24 |
+
+### 14B：Top 3 验证（3 个，10ep）
+
+| ID | lr | dropout |
+|---|---|---|
+| 14.25-14.27 | Top 3 from 14A |
+
+**输出**：FINAL_LR, FINAL_DROPOUT → **CONFIG_V3**
+
+---
+
+## Phase 15：杂项超参（12 个实验，10ep）
+
+| ID | 参数 | 值 |
+|---|---|---|
+| 15.1-15.4 | z_loss_weight | 0.0, 0.00005, 0.0005, 0.001 |
+| 15.5-15.8 | position_bucket_size | 8, 16, 64, 128 |
+| 15.9-15.12 | betas | (0.8,0.95), (0.9,0.95), (0.9,0.999), (0.95,0.999) |
+
+**输出**：BEST_ZLOSS, BEST_BUCKET, BEST_BETAS → **CONFIG_V4**
+
+---
+
+## Phase 16：知识蒸馏（14 个实验，10ep）
+
+| ID | teacher | temperature | alpha | 说明 |
+|---|---|---|---|---|
+| 16.1-16.4 | Qwen3.5-1.5B | 2/4/6/8 | 0.5 | 温度 sweep |
+| 16.5-16.8 | Qwen3.5-1.5B | 4 | 0.2/0.3/0.7/0.9 | alpha sweep |
+| 16.9 | Qwen3.5-3B | 4 | 0.5 | 更大 teacher |
+| 16.10 | Qwen3.5-9B | 4 | 0.5 | 最大 teacher |
+| 16.11 | BEST teacher | BEST T | BEST α | 最�� KD 配置 |
+| 16.12 | BEST（无 FORGETTER） | BEST T | BEST α | KD-FORGETTER 交互 |
+| 16.13 | BEST（无 AMLM） | BEST T | BEST α | KD-AMLM 交互 |
+| 16.14 | 无 KD | — | — | 对照组 |
+
+**输出**：BEST_KD_CONFIG → **CONFIG_V5**
+
+---
+
+## Phase 17：Epoch 数与 Checkpoint 策略（12 个实验）
+
+| ID | epochs | ckpt_averaging | avg_last_k |
+|---|---|---|---|
+| 17.1-17.5 | 3/5/7/8/10 | false | — |
+| 17.6-17.9 | 10 | true | 2/3/5/7 |
+| 17.10-17.12 | 5/7/8 | true | 3 |
+
+**输出**：BEST_EPOCHS, BEST_CKPT → **CONFIG_V6**
+
+---
+
+## Phase 18：种子方差（8 个实验）
+
+| ID | seed |
+|---|---|
+| 18.1-18.8 | 7, 42, 123, 2024, 2026, 9999, 31415, 54321 |
+
+**输出**：Mean ± Std of BLiMP。如果 Std > 1.5pp 则配置不稳定，需回溯。
+
+---
+
+## Phase 19：数据 × 特性交互（24 个实验，10ep）
+
+**目标**：P12 的 Top 3 数据配置 × 3 种特性配置 × 2 种模型形状 + 额外变体。
+
+| | FEAT_NONE | FEAT_PARTIAL | FEAT_FULL |
+|---|---|---|---|
+| TOP_DATA_1 × SHAPE_BASE | 19.1 | 19.2 | 19.3 |
+| TOP_DATA_1 × SHAPE_ALT | 19.4 | 19.5 | 19.6 |
+| TOP_DATA_2 × SHAPE_BASE | 19.7 | 19.8 | 19.9 |
+| TOP_DATA_2 × SHAPE_ALT | 19.10 | 19.11 | 19.12 |
+| TOP_DATA_3 × SHAPE_BASE | 19.13 | 19.14 | 19.15 |
+| TOP_DATA_3 × SHAPE_ALT | 19.16 | 19.17 | 19.18 |
+
+额外变体（+Morfessor / +FastText / +KD）：19.19-19.24
+
+**输出**：FINAL_DATA, FINAL_FEATURES, FINAL_SHAPE → **CONFIG_FINAL**
+
+---
+
+## Phase 20：最终候选与消融保险（20 个实验）
+
+### 20A：最终候选变体（10 个）
+
+| ID | 说明 |
+|---|---|
+| 20.1 | CONFIG_FINAL 原版 |
+| 20.2 | + RoPE（如果还没包含） |
+| 20.3 | + DWA |
+| 20.4 | + attention_gate |
+| 20.5 | + ckpt_averaging |
+| 20.6 | + 次优优化器 |
+| 20.7 | + 次优数据 |
+| 20.8 | + 次优形状 |
+| 20.9 | + Morfessor（如果 BPE 赢了） |
+| 20.10 | 无 KD（如果 KD 在 CONFIG_FINAL 中） |
+
+### 20B：消融保险（10 个，每次移除一个组件）
+
+| ID | 移除的组件 |
+|---|---|
+| 20.11 | 移除 AMLM |
+| 20.12 | 移除 N-hot |
+| 20.13 | 移除 FORGETTER |
+| 20.14 | 移除 MoE |
+| 20.15 | 移除 AttnRes |
+| 20.16 | 移除 KD |
+| 20.17 | 移除 ckpt_averaging |
+| 20.18 | 换回基线 LR |
+| 20.19 | 换回基线 dropout |
+| 20.20 | 换回基线数据 |
+
+**决策规则**：移除某组件后 BLiMP 下降 < 0.3pp → 该组件无贡献，从最终配置中移除。
+
+---
+
+## Phase 21：提交种子与完整评测（16 个实验）
+
+| ID | 说明 |
+|---|---|
+| 21.1-21.5 | CONFIG_FINAL × 5 seeds (42/7/123/2026/9999) + full eval |
+| 21.6-21.8 | 次佳候选(P20A) × 3 seeds + full eval |
+| 21.9-21.11 | 第三候选 × 3 seeds + full eval |
+| 21.12 | 跨种子 checkpoint 平均 |
+| 21.13-21.14 | 最佳 seed 的 ckpt avg last 3/5 |
+| 21.15 | GLUE 微调完整评测 |
+| 21.16 | 安全重跑（最终确认） |
+
+**输出**：SUBMISSION_MODEL
+
+---
+
+## Phase 22：预留探索（61 个实验）
+
+| 分组 | 数量 | 说明 |
+|---|---|---|
+| 22A | 10 | 次优架构深度调优 |
+| 22B | 10 | 数据增强比例调优（Para 10-50%、RecombiText 25-75%） |
+| 22C | 5 | MoE 替代配置 |
+| 22D | 10 | 关键参数插值精调 |
+| 22E | 6 | Debug / 失败实验重跑 |
+| 22F | 20 | 新想法 / 新论文启发 |
+
+---
+
+## 实验总汇
+
+| Phase | 内容 | 实验数 | GPU 时 | 依赖 |
+|:-----:|------|:------:|:------:|------|
+| P1 | 架构对比 | 5 | 3.3 | — |
+| P2 | 特性消融 | 9 | 6.0 | P1 |
+| P3 | LR sweep | 17 | 3.4 | P1 |
+| P4 | 正则化 sweep | 25 | 5.0 | P3 |
+| P5 | 特性组合 | 20 | 13.3 | P2,P3,P4 |
+| P6 | 掩码调参 | 20 | 13.3 | P5 |
+| P7 | Batch/SeqLen | 14 | 9.3 | P6 |
+| P8 | 优化器 | 16 | 10.7 | P7 |
+| P9 | 架构修改 | 15 | 10.0 | P8 |
+| P10 | MoE 调参 | 8 | 5.3 | P5(条件) |
+| P11 | 分词×嵌入 | 6 | 4.0 | P9 |
+| P12 | 数据构建 | 18 | 12.0 | P9 |
+| P13 | 跨架构验证 | 15 | 10.0 | P12 |
+| P14 | LR×Dropout 网格 | 27 | 4.8 | P12 |
+| P15 | 杂项超参 | 12 | 8.0 | P14 |
+| P16 | 知识蒸馏 | 14 | 9.3 | P15 |
+| P17 | Epoch/Ckpt | 12 | 8.0 | P16 |
+| P18 | 种子方差 | 8 | 5.3 | P17 |
+| P19 | 数据×特性交互 | 24 | 16.0 | P12,P17 |
+| P20 | 最终候选+消融 | 20 | 13.3 | P19 |
+| P21 | 提交 | 16 | 10.7 | P20 |
+| P22 | 预留 | 61 | 40.7 | 任意 |
+| **合计** | | **382** | **~221h** | |
+
+---
+
+## 代码就绪情况
+
+| 组件 | 状态 | 缺什么 |
+|------|:----:|--------|
+| 训练框架 (train.py) | ✅ | — |
+| 5 种基础架构 | ✅ | — |
+| MoE 模块 | ✅ | — |
+| AttnRes 模块 | ✅ | — |
+| AMLM / 掩码衰减 | ✅ | — |
+| N-hot / FastText 嵌入 | ✅ 代码 | ❌ FastText 模型未训练 |
+| BPE / Morfessor 分词器 | ✅ | — |
+| 知识蒸馏 | ✅ 代码 | ❌ Teacher 模型未下载 |
+| 采样 A/B/C/D | ✅ 代码 | ❌ A/D 未运行 |
+| Paraphrase 增强 | ✅ 代码 | ❌ 未运行 |
+| Variation Sets 增强 | ❌ | 需写脚本 |
+| RecombiText 增强 | ❌ | 需写脚本 |
+| Contrastive Decoding | ❌ | 需写脚本 |
+| MATTR 排序 | ❌ | 需写脚本 |
+| 冠军复刻数据配置 | ❌ | 需写 1:1:1 混合脚本 |
+| **实验 CSV 生成器** | ⚠️ 需更新 | 缺 MoE/AttnRes 列和实验 |
+| **批量运行脚本** | ❌ | 读 CSV → 启动训练 → 收集结果 |
+| **评测自动化** | ⚠️ | 需确认 fast_eval 可自动调用 |
+| **结果收集/对比** | ❌ | 汇总各实验分数的脚本 |
+
+### 优先级路线图
+
+```
+第 1 步：写批量运行脚本 + 结果收集器
+        → P1-P5 的 ~76 个实验可以立即开始跑
+
+第 2 步（与第 1 步并行）：
+        训练 FastText 模型
+        运行 sample_A / sample_D
+        运行 Paraphrase 生成
+        → P6-P11 就绪
+
+第 3 步（与前面并行）：
+        写 Variation Sets / RecombiText / CD / MATTR 脚本
+        写冠军复刻采样脚本
+        下载 teacher 模型
+        → P12-P16 就绪
+
+第 4 步：更新 generate_experiments.py
+        → 完整 382 实验 CSV 生成
+```

notes/progress_report.md

@@ -0,0 +1,273 @@
+# 进展报告：BabyLM Challenge 2026 Strict-Small
+
+> **日期**：2026-03-23
+> **当前阶段**：代码开发完成，组件验证通过，准备上传 HuggingFace 并开始正式实验
+
+---
+
+## 一、项目总览
+
+参加 BabyLM Challenge 2026 Strict-Small 赛道（训练数据 ≤ 10M 词），目标是训练一个小型语言模型（~30M 参数）在多项语言评测任务上取得最佳成绩。
+
+**评测任务**：BLiMP（语法）、EWoK（世界知识）、Entity Tracking（实体追踪）、GLUE（语言理解，需微调）、WUG（形态学）、COMPS（概念属性）、AoA（词汇习得）、Reading Time（阅读时间）
+
+**2025 Strict-Small 排行榜**（我们的目标赛道）：
+| 模型 | TextAvg | BLiMP | GLUE | EWoK |
+|------|:-------:|:-----:|:----:|:----:|
+| amlm_hd_fail（冠军） | **45.4** | 59.0 | 57.7 | 56.0 |
+| MoEP（亚军） | 44.5 | 60.8 | 64.2 | 50.2 |
+| Baseline GPT-BERT | 41.2 | 71.7 | 65.1 | 49.5 |
+| Baseline GPT-2 | 37.4 | 66.4 | 55.9 | 49.9 |
+
+---
+
+## 二、项目结构
+
+```
+BabyLM_Challenge/
+├── experiments.csv              # 384 个实验配置
+├── requirements.txt             # Python 依赖
+├── scripts/
+│   ├── 01_data/                 # 数据处理 pipeline (Stage 1-9)
+│   ├── 02_model/                # 分词器训练 (BPE / Morfessor)
+│   ├── 03_training/             # 训练框架
+│   │   ├── config.py            # 配置系统 (YAML + CLI override)
+│   │   ├── data.py              # 数据加载、3种掩码策略、逐句/拼接模式
+│   │   ├── train.py             # 训练循环 + 自动评测 (fast/full)
+│   │   ├── evaluate.py          # 评测包装 (zero-shot + GLUE微调 + AoA)
+│   │   ├── utils.py             # 5种优化器、调度器、checkpoint
+│   │   ├── hf_export/           # HuggingFace 模型导出 (5种架构)
+│   │   ├── models/              # 7种模型架构
+│   │   │   ├── gpt2.py          # D0 基线
+│   │   │   ├── gpt_bert.py      # D1 双目标混合
+│   │   │   ├── modern_bert.py   # D2 现代双向
+│   │   │   ├── xlstm.py         # D3 矩阵记忆
+│   │   │   ├── rtd.py           # D4 替换Token检测
+│   │   │   ├── moe.py           # D5 MoE稀疏专家 (可叠加)
+│   │   │   ├── attn_res.py      # D6 注意力残差 (可叠加)
+│   │   │   ├── embeddings.py    # C0标准 / C1 N-hot
+│   │   │   └── distillation.py  # 知识蒸馏
+│   │   └── configs/exp_A.yaml   # 基线YAML配置
+│   └── tools/                   # 实验管理工具
+│       ├── generate_experiments.py  # 生成 experiments.csv
+│       ├── run_experiments.py       # 批量运行 (CSV→YAML→训练→评测→结果)
+│       ├── collect_results.py       # 结果汇总
+│       ├── smoke_test.py            # 冒烟测试 (44个代码路径)
+│       └── verify_components.py     # 组件验证 (32项逐个验证)
+├── notes/
+│   ├── encyclopedia.md              # 改进点百科 (所有优化维度详解)
+│   ├── experiment_plan.md           # 实验计划 (384个实验, 22阶段)
+│   ├── experiment_log.md            # 实验结果记录
+│   ├── progress_report.md           # 本文件
+│   ├── ref_datasets.md              # 数据集详细信息
+│   ├── ref_leaderboard_2025.md      # 2025排行榜数据
+│   └── babylm_leaderboard_2025.csv  # 原始排行榜CSV
+├── data/                        # 训练数据 (14个源, 119M词原始)
+│   ├── raw_files/               # 原始数据文件
+│   ├── 1_noise_filtered/ ~ 7_eval_sentences/  # 处理流水线各阶段
+│   ├── 8_sample_B/train.txt     # 策略B采样 (9.99M词) ✅
+│   └── 8_sample_C/train.txt     # 策略C采样 (9.99M词) ✅
+├── models/
+│   ├── tokenizer/               # BPE 8K 分词器 ✅
+│   ├── tokenizer_morfessor/     # Morfessor+BPE 分词器 ✅
+│   └── qwen3.5-9b/             # Qwen模型 (Paraphrase生成用, 18GB, 不上传)
+├── evaluation-pipeline-2025/    # 官方评测pipeline
+├── 2025_papers/                 # 参考论文PDF
+├── checkpoints/                 # 训练checkpoint (空)
+└── experiment_results/          # 实验结果 (空)
+```
+
+---
+
+## 三、已实现的全部功能
+
+### 编号规则：每个维度 X0 = 基线（不优化的默认选择）
+
+### B: 分词器
+
+| 编号 | 方法 | 状态 | 配置 |
+|:----:|------|:----:|------|
+| B0 | BPE 8K（基线） | ✅ | `tokenizer: bpe` |
+| B1 | Morfessor+BPE | ✅ | `tokenizer: morfessor_bpe` |
+| B2 | 词表大小 (4K/8K/16K) | ✅ | 需重新训练分词器 |
+
+### C: 嵌入策略
+
+| 编号 | 方法 | 状态 | 配置 |
+|:----:|------|:----:|------|
+| C0 | 标准嵌入（基线） | ✅ | `embedding.type: standard` |
+| C1 | N-hot 形态嵌入 | ✅ | `embedding.type: nhot` |
+| C2 | FastText 初始化 | ✅ 代码 | `embedding.init: fasttext`（需先训练FastText模型） |
+
+### D: 模型架构
+
+| 编号 | 方法 | 状态 | 验证 | 配置 |
+|:----:|------|:----:|:----:|------|
+| D0 | GPT-2（基线） | ✅ | loss=5.31 | `arch: gpt2, objective: clm` |
+| D1 | GPT-BERT | ✅ | loss=5.74 | `arch: gpt_bert, objective: gpt_bert` |
+| D2 | ModernBERT | ✅ | loss=3.33 | `arch: modernized_bert, objective: mlm` |
+| D3 | xLSTM | ✅ | loss=4.54 | `arch: xlstm, objective: clm` |
+| D4 | RTD/ELECTRA | ✅ | loss=3.99 | `arch: rtd, objective: rtd` |
+| D5 | MoE（可叠加） | ✅ | loss=5.79 | `use_moe: true` |
+| D6 | AttnRes（可叠加） | ✅ | loss=5.74 | `use_attn_res: true` |
+
+### E: 训练目标与掩码
+
+| 编号 | 方法 | 状态 | 验证 | 配置 |
+|:----:|------|:----:|:----:|------|
+| E0 | 标准均匀掩码（基线） | ✅ | loss=5.74 | `masking.type: standard` |
+| E1 | AMLM 自适应掩码 | ✅ | loss=5.75 | `masking.type: amlm` |
+| E2 | 掩码率衰减 | ✅ | — | `mask_ratio: 0.30, mask_ratio_end: 0.15` |
+| E3 | 频率感知掩码 | ✅ | loss=5.74 | `masking.type: frequency` |
+| E4 | MNTP:CLM 比例 | ✅ | — | `training.mntp_ratio: 15` |
+| E5 | 反向课程 MTP | ✅ | — | `training.use_mtp: true` |
+
+### F: 优化器
+
+| 编号 | 方法 | 状态 | 验证 | 配置 | 注意事项 |
+|:----:|------|:----:|:----:|------|---------|
+| F0 | Adam（基线） | ✅ | loss=6.90 | `optimizer.type: adam` | 收敛慢 |
+| F1 | AdamW | ✅ | loss=5.74 | `optimizer.type: adamw` | **推荐默认** |
+| F2 | LAMB | ✅ | loss=5.74 | `optimizer.type: lamb` | **fp16下lr≤0.005** |
+| F3 | FORGETTER | ✅ | loss=5.78 | `optimizer.forgetter: true` | 需 wd=1.0 |
+| F4 | Muon | ✅ | loss=5.69 | `optimizer.type: muon` | 慢2.5x，loss最低 |
+
+### G: 超参数与训练方式
+
+| 编号 | 方法 | 状态 | 配置 |
+|:----:|------|:----:|------|
+| G1-G5 | lr/batch/wd/dropout/seqlen | ✅ | YAML 配置 |
+| G6 | 逐句模式 | ✅ | `data.packing: sentence`（较慢） |
+
+### H: 训练技巧
+
+| 编号 | 方法 | 状态 |
+|:----:|------|:----:|
+| H1 | Checkpoint 选择 | ✅ 每epoch保存 |
+| H2 | Checkpoint 平均 | ✅ `checkpoint.averaging: true` |
+| H3 | 知识蒸馏 | ✅ 代码完整（需下载teacher模型） |
+
+### 评测集成
+
+| 模式 | 内容 | 时间 |
+|------|------|------|
+| fast | Zero-shot: BLiMP, EWoK, ET, WUG, Reading | ~15分钟 |
+| **full** | Zero-shot + GLUE微调(7子任务) + AoA | ~60-90分钟 |
+
+---
+
+## 四、验证结果
+
+### 组件验证（32 项，每项 3 epoch）
+
+| 组 | 通过 | 失败 | 已知问题 |
+|----|:----:|:----:|---------|
+| 架构 (11项) | 11 | 0 | — |
+| 嵌入 (2项) | 2 | 0 | — |
+| 分词 (2项) | 2 | 0 | — |
+| 掩码 (3项) | 3 | 0 | — |
+| 优化器 (11项) | 8 | 3 | LAMB+fp16+lr>0.005=NAN（已修复默认LR） |
+| 超参 (1项) | 0 | 1 | sentence模式数据加载慢（需增大timeout） |
+| 组合 (2项) | 2 | 0 | — |
+
+### 第一个完整实验
+
+GPT-BERT + AdamW + lr=5e-4 + 10 epoch + fast eval：
+
+| 指标 | 我们 | 2025基线 |
+|------|:----:|:--------:|
+| BLiMP | **75.37** | 71.7 |
+| BLiMP Supp | 60.40 | 63.2 |
+| EWoK | 49.18 | 49.5 |
+| Entity Tracking | **42.29** | 34.6 |
+| Avg Zero-Shot | **56.81** | — |
+
+---
+
+## 五、已知问题
+
+| 问题 | 状态 | 解决方案 |
+|------|:----:|---------|
+| LAMB+fp16+lr>0.005=NAN | ✅ 已修复 | 默认LR改为0.005 |
+| G6 sentence模式慢 | 已知 | 增大timeout或优化数据加载 |
+| FastText模型未训练 | 待做 | `python scripts/02_model/train_fasttext.py` |
+| KD teacher模型未下载 | 待做 | 下载Qwen teacher |
+| 数据增强脚本未写 | 待做 | Variation Sets/RecombiText/CD/MATTR/信息密集合成 |
+
+---
+
+## 六、下一步工作
+
+### 立即要做
+
+1. **上传项目到 HuggingFace**（排除 qwen3.5-9b 模型）
+2. **开始正式实验 Phase 1-5**：
+   - P1: 5种架构对比
+   - P2: 核心特性消融 (AMLM/N-hot/FORGETTER/MoE/AttnRes)
+   - P3: LR sweep
+   - P4: 正则化 sweep
+   - P5: 特性组合
+
+### 后续阶段
+
+- P6-P11: 掩码调参、batch size、优化器、架构修改、分词×嵌入
+- P12: 数据集构建方式 (13种)
+- P13-P21: 跨架构验证 → 最终候选 → 提交
+- 数据增强脚本开发
+
+---
+
+## 七、如何运行
+
+### 环境安装
+```bash
+pip install -r requirements.txt
+pip install muon-optimizer torch-optimizer morfessor
+```
+
+### 单个实验
+```bash
+python -m scripts.03_training.train --config scripts/03_training/configs/exp_A.yaml
+python -m scripts.03_training.train --config xxx.yaml --eval-mode fast  # 快速评测
+python -m scripts.03_training.train --config xxx.yaml --skip-eval       # 跳过评测
+```
+
+### 批量实验
+```bash
+python scripts/tools/run_experiments.py --phase 1 --list     # 查看
+python scripts/tools/run_experiments.py --phase 1             # 运行
+python scripts/tools/run_experiments.py --phase 1 --resume    # 断点续跑
+python scripts/tools/collect_results.py --phase 1             # 查看结果
+```
+
+### 组件验证
+```bash
+python scripts/tools/verify_components.py --group arch   # 验证架构
+python scripts/tools/verify_components.py --resume        # 跑全部(跳过已通过)
+```
+
+---
+
+## 八、关键配置速查
+
+```yaml
+model:
+  arch: gpt2              # gpt2(D0) | gpt_bert(D1) | modernized_bert(D2) | xlstm(D3) | rtd(D4)
+  use_moe: false           # D5 MoE
+  use_attn_res: false      # D6 AttnRes
+data:
+  tokenizer: bpe           # bpe(B0) | morfessor_bpe(B1)
+  packing: concat          # concat | sentence(G6)
+embedding:
+  type: standard           # standard(C0) | nhot(C1)
+  init: random             # random | fasttext(C2)
+training:
+  objective: gpt_bert      # gpt_bert | clm | mlm | rtd
+  learning_rate: 0.0005    # AdamW推荐; LAMB需≤0.005(fp16)
+  use_mtp: false           # E5 反向课程MTP
+masking:
+  type: standard           # standard(E0) | amlm(E1) | frequency(E3)
+optimizer:
+  type: adamw              # adam(F0) | adamw(F1) | lamb(F2) | muon(F4)
+  forgetter: false         # F3 FORGETTER
+```

notes/ref_datasets.md

@@ -0,0 +1,440 @@
+# 数据集全览
+
+> BabyLM Challenge 2026 · Strict-Small 赛道（≤ 10M 词）
+> 更新日期: 2026-03-16
+
+---
+
+## 一、官方训练数据
+
+比赛提供的 7 个训练文件，存放于 `data/raw_files/`。
+
+### 1. CHILDES
+
+| 项目 | 内容 |
+|------|------|
+| **全名** | Child Language Data Exchange System |
+| **行数 / 词数** | 5,638,783 行 / 28.4M 词 |
+| **来源** | Brian MacWhinney & Catherine Snow，1984 年建立，Carnegie Mellon University 维护 |
+| **内容** | 儿童语言习得转写记录，含儿童和照顾者的对话，CHAT 格式标注 |
+| **许可** | CC BY-NC-SA 3.0 |
+| **链接** | https://childes.talkbank.org/ |
+| **特点** | 短句为主（平均 5 词/行），含 CHAT 格式噪声（`*CHI:` 前缀、`xxx` 标记、`@`/`%` 标注行） |
+| **优势** | 真实儿童语言习得数据，对 AoA 评测（词汇习得顺序）最有价值；含丰富的基础语法结构 |
+| **劣势** | 噪声多（需 Stage 1/4 清洗）；短句多导致句法复杂度低；不含书面语 |
+| **目标任务** | AoA（最重要）、BLiMP（基础语法） |
+| **引用** | MacWhinney, B. (2000). *The CHILDES Project: Tools for Analyzing Talk*. 3rd Ed. Lawrence Erlbaum Associates. |
+
+### 2. Gutenberg
+
+| 项目 | 内容 |
+|------|------|
+| **全名** | Project Gutenberg 儿童文学子集 |
+| **行数 / 词数** | 661,928 行 / 25.6M 词 |
+| **来源** | Project Gutenberg，按儿童适宜主题元数据筛选的公版书 |
+| **内容** | 书面文学，长句多（平均 38.6 词/行），句法结构最丰富 |
+| **许可** | 公有领域（美国） |
+| **链接** | https://www.gutenberg.org/ |
+| **特点** | 质量最高的来源；含古英语拼写和专有名词；含 Project Gutenberg 模板文本（Stage 4 过滤） |
+| **优势** | 句法最丰富，BLiMP 覆盖率最高；书面语质量极佳 |
+| **劣势** | 含古英语表达（`reckly`、`bettern`）；语域单一（文学） |
+| **目标任务** | BLiMP（句法）、Reading Time（连贯散文）、GLUE |
+| **引用** | https://www.gutenberg.org/ |
+
+### 3. Open Subtitles
+
+| 项目 | 内容 |
+|------|------|
+| **全名** | OpenSubtitles 电影字幕语料 |
+| **行数 / 词数** | 3,333,618 行 / 19.2M 词 |
+| **来源** | Pierre Lison & Jörg Tiedemann，OPUS 平行语料库项目 |
+| **内容** | 电影/电视字幕的英语转写，口语化短句 |
+| **许可** | CC BY-NC-SA 3.0 |
+| **链接** | https://opus.nlpl.eu/datasets/OpenSubtitles |
+| **特点** | 口语化（平均 5.8 词/行）；原始数据含时间戳和 HTML 标签（Stage 1 过滤）；部分行全大写（Stage 4 修复） |
+| **优势** | 对话多样性好，口语表达丰富 |
+| **劣势** | 噪声较多（词语黏连如 `wasreleased`）；全大写行需修复 |
+| **目标任务** | BLiMP Supplement（Turn Taking 对话轮换） |
+| **引用** | Lison, P. & Tiedemann, J. (2016). OpenSubtitles2016: Extracting Large Parallel Corpora from Movie and TV Subtitles. *LREC 2016*. |
+
+### 4. Simple Wikipedia
+
+| 项目 | 内容 |
+|------|------|
+| **全名** | Simple English Wikipedia |
+| **行数 / 词数** | 642,735 行 / 15.3M 词 |
+| **来源** | 维基媒体基金会 |
+| **内容** | 用简化英语写的维基百科，基础词汇和较短句子 |
+| **许可** | CC BY-SA 3.0 / GFDL |
+| **链接** | https://simple.wikipedia.org/ |
+| **特点** | 百科知识密集（平均 23.8 词/行）；含大量外语专有名词；类型覆盖率最低（19.5%） |
+| **优势** | 世界知识丰富，EWoK 和 COMPS 覆盖好 |
+| **劣势** | 专有名词多（非英语地名等）；部分词语黏连 |
+| **目标任务** | EWoK（世界知识）、COMPS（概念属性） |
+| **引用** | https://simple.wikipedia.org/ |
+
+### 5. BNC Spoken
+
+| 项目 | 内容 |
+|------|------|
+| **全名** | British National Corpus — Spoken Component |
+| **行数 / 词数** | 803,341 行 / 7.6M 词 |
+| **来源** | 牛津大学出版社等机构联合项目 |
+| **内容** | 英国真实口语转写，含正式和非正式场合对话 |
+| **许可** | 需注册接受许可协议 |
+| **链接** | http://www.natcorp.ox.ac.uk/ |
+| **特点** | 真实英式口语（平均 9.5 词/行）；含疑问句、倒装等句法变体 |
+| **优势** | 自然口语数据，BLiMP Supplement（疑问句/倒装）和 GLUE 有价值 |
+| **劣势** | 不流利标记较多；keep 率较低（73%） |
+| **目标任务** | BLiMP Supplement（Subject-Aux Inversion）、GLUE、Reading Time |
+| **引用** | BNC Consortium (2007). *The British National Corpus, XML Edition*. Oxford Text Archive. |
+
+### 6. Open Subtitles Cleaned
+
+| 项目 | 内容 |
+|------|------|
+| **行数 / 词数** | 49,920 行 / 0.3M 词 |
+| **内容** | Open Subtitles 的 LLM 过滤版本 |
+| **特点** | 数据量极小，含 CHAT 格式标记（Stage 4 清洗） |
+
+### 7. Switchboard
+
+| 项目 | 内容 |
+|------|------|
+| **全名** | Switchboard Telephone Speech Corpus |
+| **行数 / 词数** | 30,559 行 / 0.25M 词 |
+| **来��** | Texas Instruments，1990-91 年采集，DARPA 资助 |
+| **内容** | 543 位说话者的约 2,400 段电话对话 |
+| **许可** | LDC User Agreement |
+| **链接** | https://catalog.ldc.upenn.edu/LDC97S62 |
+| **特点** | 数据量极小（0.25M 词）；含说话人前缀 `A:`/`B:`（Stage 4 去除） |
+| **引用** | Godfrey, J., Holliman, E., & McDaniel, J. (1992). SWITCHBOARD: Telephone speech corpus for research and development. *ICASSP 1992*. |
+
+---
+
+## 二、外部数据
+
+已下载并接入 Stage 1-4 流水线的外部数据源，存放于 `data/external/`，以 `ext_` 前缀复制到 `data/raw_files/`。
+
+> 比赛规则已确认（2026-03-14）：所有数据都可以用，唯一约束是总训练数据 ≤ 10M 词。
+
+### 1. FineWeb-Edu
+
+| 项目 | 内容 |
+|------|------|
+| **行数 / 词数** | 85,367 行 / 3.0M 词（子集） |
+| **来源** | HuggingFace FW 团队 |
+| **内容** | 从 CommonCrawl 筛选的高质量教育文本，用 Llama3-70B-Instruct 训练的分类器按教育质量 0-5 打分，阈值 3 以上保留 |
+| **许可** | ODC-By v1.0 |
+| **链接** | https://huggingface.co/datasets/HuggingFaceFW/fineweb-edu |
+| **优势** | 教育内容质量高，知识密度大 |
+| **劣势** | 全集 1.3T tokens，需选子集 |
+| **目标任务** | GLUE（NLU 任务，特别是 MNLI） |
+| **为什么有用** | **实验证据**：Charpentier 2024 GPT-BERT 使用 1/3 FineWeb-Edu，消融实验显示它对 GLUE 提升贡献最大（Charpentier & Samuel, 2024） |
+| **引用** | Penedo, G. et al. (2024). The FineWeb Datasets: Decanting the Web for the Finest Text Data at Scale. arXiv:2406.17557. |
+| **文件** | `ext_fineweb_edu.txt` |
+
+### 2. Cosmopedia
+
+| 项目 | 内容 |
+|------|------|
+| **行数 / 词数** | 84,110 行 / 3.0M 词（子集） |
+| **来源** | HuggingFace TB 团队 |
+| **内容** | Mixtral-8x7B-Instruct-v0.1 生成的合成教科书、博客、故事等 |
+| **许可** | Apache 2.0（v1）/ ODC-By（v2） |
+| **链接** | https://huggingface.co/datasets/HuggingFaceTB/cosmopedia |
+| **优势** | 教科书风格，结构化知识，内容多样 |
+| **劣势** | LLM 生成文本，可能缺乏自然语言的某些特征 |
+| **目标任务** | EWoK（世界知识） |
+| **为什么有用** | **实验证据**：Charpentier 2024 GPT-BERT 使用 1/3 Cosmopedia，消融实验显示它对 EWoK 提升贡献最大（Charpentier & Samuel, 2024） |
+| **引用** | Ben Allal, L. et al. (2024). Cosmopedia: how to create large-scale synthetic data for pre-training. HuggingFace Blog. |
+| **文件** | `ext_cosmopedia.txt` |
+
+### 3. TinyDialogues
+
+| 项目 | 内容 |
+|------|------|
+| **行数 / 词数** | 18,282 行 / 3.0M 词 |
+| **来源** | Steven Y. Feng, Noah D. Goodman, Michael C. Frank（Stanford University） |
+| **内容** | GPT-4 Turbo 生成的模拟儿童-照顾者对话，按年龄分层（2/5/10/15 岁），含多种对话类型和参与者角色 |
+| **许可** | 见 HuggingFace 页面 |
+| **链接** | https://huggingface.co/datasets/styfeng/TinyDialogues |
+| **优势** | 比原始 CHILDES 更干净、结构更好；按年龄分层可做 curriculum |
+| **劣势** | GPT-4 生成，可能缺少真实儿童语言的某些不规则特征 |
+| **目标任务** | AoA（词汇习得顺序） |
+| **为什么有用** | **实验证据**：Feng et al. (2024) 发现合成 CDS 数据优于自然 CDS 数据，局部对话连贯性（local discourse coherence）是关键因素 |
+| **引用** | Feng, S.Y., Goodman, N.D., & Frank, M.C. (2024). Is Child-Directed Speech Effective Training Data for Language Models? *EMNLP 2024*. |
+| **文件** | `ext_tiny_dialogues.txt` |
+
+### 4. OneStopEnglish
+
+| 项目 | 内容 |
+|------|------|
+| **行数 / 词数** | 7,201 行 / 0.38M 词 |
+| **来源** | Sowmya Vajjala & Ivana Lucic |
+| **内容** | 189 篇新闻文章 × 3 个难度级别（Elementary/Intermediate/Advanced），共 567 篇 |
+| **许可** | CC BY-SA 4.0 |
+| **链接** | https://github.com/nishkalavallabhi/OneStopEnglishCorpus |
+| **优势** | 同一内容的多难度版本，语言多样性好 |
+| **劣势** | 数据量小（<0.4M 词） |
+| **目标任务** | Reading Time（阅读流畅度） |
+| **为什么有用** | **理论推断**：多难度级别的相同内容提供了丰富的句法多样性，有助于模型学习不同复杂度的表达方式 |
+| **引用** | Vajjala, S. & Lucic, I. (2018). OneStopEnglish corpus: A new corpus for automatic readability assessment and text simplification. *BEA Workshop, NAACL 2018*. |
+| **文件** | `ext_onestop_english.txt` |
+
+### 5. GenericsKB-Best
+
+| 项目 | 内容 |
+|------|------|
+| **行数 / 词数** | 1,020,569 行 / 8.5M 词 |
+| **来源** | Allen Institute for AI (Sumithra Bhakthavatsalam, Chloe Anastasiades, Peter Clark) |
+| **内容** | 从 Waterloo 语料库（.edu 域名）、SimpleWikipedia、ARC 语料库提取的常识性通用陈述，如 "Trees remove carbon dioxide from the atmosphere" |
+| **许可** | CC BY 4.0 |
+| **链接** | https://huggingface.co/datasets/community-datasets/generics_kb |
+| **优势** | 直接包含 EWoK 测试的常识知识类型（物质属性、空间关系、智能体特征等）；解决"报告偏差"问题（常识在普通文本中很少被显式说出来） |
+| **劣势** | 句式较单一（陈述句为主）；部分来自 WordNet/ConceptNet 合成，可能不够自然 |
+| **目标任务** | EWoK（世界知识）、COMPS（概念属性继承） |
+| **为什么有用** | **理论推断**：EWoK 作者指出所有测试模型都低于人类水平，物理/空间域最弱——这些恰好是 GenericsKB 覆盖的领域。通用知识（generics）是人类常识的核心，但因"报告偏差"在普通训练文本中严重不足。知识增强预训练已被证明能提升 LM 的事实推理能力（Guan et al., 2020, TACL） |
+| **引用** | Bhakthavatsalam, S., Anastasiades, C., & Clark, P. (2020). GenericsKB: A Knowledge Base of Generic Statements. arXiv:2005.00660. |
+| **文件** | `ext_generics_kb.txt` |
+
+### 6. ConceptNet 5.7
+
+| 项目 | 内容 |
+|------|------|
+| **行数 / 词数** | 393,557 行 / 2.7M 词（英语子集，12 种关系转自然语言） |
+| **来源** | Robyn Speer, Joshua Chin, Catherine Havasi (Luminoso) |
+| **内容** | 多语言常识知识图谱，从中提取英语三元组并转为自然语言句子（如 HasProperty → "Ice is cold."、CapableOf → "Bird can fly."） |
+| **许可** | CC BY-SA 4.0 |
+| **链接** | https://conceptnet.io/ / https://github.com/commonsense/conceptnet5 |
+| **优势** | 覆盖 12 种语义关系（HasProperty / UsedFor / MadeOf / CapableOf / IsA / PartOf / HasA / AtLocation / Causes / HasPrerequisite / MotivatedByGoal / CausesDesire） |
+| **劣势** | 从模板生成，句式机械；部分条目有噪声（如 "00t shirts"），需经 Stage 1-2 过滤 |
+| **目标任务** | EWoK（世界知识）、COMPS（概念属性） |
+| **为什么有用** | **理论推断**：ConceptNet 覆盖了 COMPS 测试的核心概念（属性继承："A sock absorbs sweat" 对应 HasProperty 关系），以及 EWoK 的物质属性和物理交互域。知识增强预训练已有成功先例（Guan et al., 2020） |
+| **引用** | Speer, R., Chin, J., & Havasi, C. (2017). ConceptNet 5.5: An Open Multilingual Graph of General Knowledge. *AAAI 2017*. arXiv:1612.03975. |
+| **文件** | `ext_conceptnet.txt` |
+
+### 7. SNLI（仅前提句）
+
+| 项目 | 内容 |
+|------|------|
+| **行数 / 词数** | 150,715 行 / 1.84M 词（去重后的唯一前提句） |
+| **来源** | Samuel R. Bowman, Gabor Angeli, Christopher Potts, Christopher D. Manning（Stanford NLP Group） |
+| **内容** | 从 Flickr30K 图片描述中提取的前提句，描述性强、语法规范、内容多样（人物活动、场景描述） |
+| **许可** | CC BY-SA 4.0 |
+| **链接** | https://nlp.stanford.edu/projects/snli/ / https://huggingface.co/datasets/stanfordnlp/snli |
+| **原始数据** | 570K 句对（premise + hypothesis），标注为 entailment / contradiction / neutral。我们只使用去重后的前提句 |
+| **优势** | 高质量图片描述句，覆盖多样的日常场景；SNLI 是 MNLI 的前身，训练数据同源有助于 GLUE 微调 |
+| **劣势** | 图片描述句式相对固定（多为"某人在做某事"）；只用前提句丢失了句对关系信息 |
+| **目标任务** | GLUE（MNLI、RTE — 文本蕴含任务） |
+| **为什么有用** | **实验证据（间接）**：Phang et al. (2018, STILTs) 证明在 SNLI/MNLI 上做中间任务**微调**可将 BERT 的 GLUE 分数提升 +1.5 分。注意：该结论基于微调阶段，而非预训练阶段。我们将 SNLI 前提句用于预训练，预期效果是让模型提前接触 NLI 风格的描述性句子，有助于后续 GLUE 微调。GLUE 的 4/7 个子任务（MNLI、RTE、QQP、WSC）与 NLI 相关 |
+| **引用** | Bowman, S.R., Angeli, G., Potts, C., & Manning, C.D. (2015). A large annotated corpus for learning natural language inference. *EMNLP 2015*. |
+| **文件** | `ext_snli.txt` |
+
+---
+
+## 三、评测数据（用于训练）
+
+比赛规则允许将评测数据中的正确句子加入训练集。由 `stage7a_extract_eval.py` 提取，存放于 `data/7_eval_sentences/`，在 Stage 8 采样时自动追加到训练集末尾。
+
+### 1. BLiMP Good Sentences
+
+| 项目 | 内容 |
+|------|------|
+| **行数 / 词数** | 58,161 行 / 420K 词 |
+| **来源** | BLiMP 评测数据的 `sentence_good` 字段 |
+| **内容** | 65 个语法子任务的正确句（如 "Katherine can't help herself."） |
+| **不提取** | `sentence_bad`（人工构造的语法错误句，加入会污染训练） |
+| **链接** | https://github.com/alexwarstadt/blimp |
+| **引用** | Warstadt, A. et al. (2020). BLiMP: The Benchmark of Linguistic Minimal Pairs for English. *TACL 2020*. |
+| **文件** | `blimp_good.txt` |
+
+### 2. BLiMP Supplement Good Sentences
+
+| 项目 | 内容 |
+|------|------|
+| **行数 / 词数** | 5,121 行 / 69K 词 |
+| **内容** | 5 个补充任务（subject_aux_inversion、hypernym、turn_taking、qa_congruence_easy/tricky）的正确句 |
+| **文件** | `blimp_supplement_good.txt` |
+
+### 3. EWoK Context + Target
+
+| 项目 | 内容 |
+|------|------|
+| **行数 / 词数** | 4,865 行 / 69K 词 |
+| **来源** | EWoK 评测数据的 Context1 + Target1 拼接 |
+| **内容** | 11 个知识域的正确上下文-推理配对 |
+| **不提取** | Context2/Target2（错误推断） |
+| **链接** | https://ewok-core.github.io/ |
+| **引用** | Ivanova, A.A. et al. (2024). Elements of World Knowledge (EWoK). *TACL*. arXiv:2405.09605. |
+| **文件** | `ewok_context_target.txt` |
+
+### 4. COMPS Sentences
+
+| 项目 | 内容 |
+|------|------|
+| **行数 / 词数** | 12,335 行 / 67K 词 |
+| **来源** | COMPS 评测数据的 `comps_base.jsonl`（仅 base，不含 wugs 虚构词） |
+| **内容** | 正确的概念-属性陈述（如 "A sock absorbs sweat."） |
+| **不提取** | `prefix_unacceptable`（错误归属）、`comps_wugs*.jsonl`（含虚构词） |
+| **链接** | https://huggingface.co/datasets/kanishka/comps |
+| **引用** | Misra, K., Rayz, J., & Ettinger, A. (2023). COMPS: Conceptual Minimal Pair Sentences. *EACL 2023*. |
+| **文件** | `comps_sentences.txt` |
+
+### 5. Reading Time Sentences
+
+| 项目 | 内容 |
+|------|------|
+| **行数 / 词数** | 205 行 / 1.9K 词 |
+| **内容** | 205 个英语句子（去重后），用于 surprisal 与人类阅读时间的回归分析 |
+| **文件** | `reading_time_sentences.txt` |
+
+### 6. AoA Context Sentences
+
+| 项目 | 内容 |
+|------|------|
+| **行数 / 词数** | 7,994 行 / 116K 词 |
+| **来源** | `cdi_childes.json`，504 个 CDI 目标词的上下文句子（去重后） |
+| **内容** | 真实语料中含目标词的自然语境句 |
+| **链接** | https://wordbank.stanford.edu/ |
+| **引用** | Frank, M.C. et al. (2017). Wordbank: An open repository for developmental vocabulary data. *Journal of Child Language*. |
+| **文件** | `aoa_context_sentences.txt` |
+
+### 7. Entity Tracking Sentences
+
+| 项目 | 内容 |
+|------|------|
+| **行数 / 词数** | 5,488 行 / 620K 词 |
+| **内容** | "箱子游戏"场景描述（去掉末尾不完整问句），合成文本 |
+| **注意** | 文体非常人工化，采样时需控制比例 |
+| **文件** | `entity_tracking_sentences.txt` |
+
+---
+
+## 四、数据集与评测任务对应关系
+
+| 评测任务 | 排行榜列数 | 最有价值的数据来源 | 证据类型 |
+|---------|:--------:|----------------|---------|
+| **BLiMP** | 1 | 官方 BabyLM 数据（gutenberg 最佳） | 实验：Charpentier & Samuel 2024 消融实验 |
+| **BLiMP Supplement** | 1 | bnc_spoken（疑问句/倒装）、open_subtitles（对话轮换） | 理论推断 |
+| **EWoK** | 1 | Cosmopedia、GenericsKB、ConceptNet | 实验（Cosmopedia）+ 理论推断（GenericsKB/ConceptNet） |
+| **Entity Tracking** | 1 | entity_tracking_sentences（评测原文） | 直接包含 |
+| **WUG ADJ** | 1 | gutenberg（不规则形态丰富） | 理论推断 |
+| **WUG PAST** | 1 | gutenberg（不规则动词丰富） | 理论推断 |
+| **COMPS** | 1 | GenericsKB、ConceptNet、Cosmopedia | 理论推断 |
+| **Reading Time** | 2 | gutenberg、OneStopEnglish（连贯散文） | 理论推断 |
+| **AoA** | 1 | CHILDES、TinyDialogues | 实验（TinyDialogues，Feng et al. 2024） |
+| **GLUE** | 1 | FineWeb-Edu、SNLI | 实验（FineWeb-Edu: Charpentier 2024; SNLI: Phang et al. 2018 STILTs） |
+
+**证据类型说明**：
+- **实验**：有发表的实验结果直接支持该数据集对该任务的提升
+- **理论推断**：基于数据集内容与评测任务的语义对应关系推断，无直接实验证据
+- **直接包含**：评测数据本身被提取用于训练
+
+### 各数据集对各评测任务的预估贡献度
+
+> 贡献度评级：★★★ 高、★★☆ 中、★☆☆ 低、— 无关
+> 带 ✦ 标记表示有实验证据支持，其余为理论推断
+
+| 数据集 | BLiMP | BLiMP Supp | EWoK | Entity Track | WUG | COMPS | Reading | AoA | GLUE |
+|--------|:-----:|:----------:|:----:|:------------:|:---:|:-----:|:-------:|:---:|:----:|
+| **官方数据** | | | | | | | | | |
+| gutenberg | ★★★✦ | ★★☆ | ★☆☆ | — | ★★★ | ★☆☆ | ★★★ | ★☆☆ | ★★☆ |
+| simple_wiki | ★★☆ | ★☆☆ | ★★☆ | — | ★☆☆ | ★★☆ | ★★☆ | ★☆☆ | ★★☆ |
+| childes | ★☆☆ | ★☆☆ | — | — | ★☆☆ | — | — | ★★★ | ★☆☆ |
+| bnc_spoken | ★★☆ | ★★★ | ★☆☆ | — | ★☆☆ | — | ★★☆ | ★☆☆ | ★★☆ |
+| open_subtitles | ★☆☆ | ★★☆ | — | — | ★☆☆ | — | ★☆☆ | ★☆☆ | ★☆☆ |
+| switchboard | ★☆☆ | ★☆☆ | — | — | — | — | — | ★☆☆ | ★☆☆ |
+| **外部数据** | | | | | | | | | |
+| FineWeb-Edu | ★★☆ | ★☆☆ | ★★☆ | — | ★☆☆ | ★☆☆ | ★★☆ | — | ★★★✦ |
+| Cosmopedia | ★★☆ | ★☆☆ | ★★★✦ | — | ★☆☆ | ★★☆ | ★★☆ | — | ★��☆ |
+| TinyDialogues | ★☆☆ | ★★☆ | — | — | ★☆☆ | — | — | ★★★✦ | ★☆☆ |
+| OneStopEnglish | ★★☆ | ★☆☆ | ★☆☆ | — | ★☆☆ | — | ★★☆ | — | ★★☆ |
+| GenericsKB | ★☆☆ | ★☆☆ | ★★★ | — | — | ★★★ | ★☆☆ | — | ★☆☆ |
+| ConceptNet | ★☆☆ | — | ★★☆ | — | — | ★★☆ | — | — | ★☆☆ |
+| SNLI | ★☆☆ | ★☆☆ | ★☆☆ | — | — | — | ★☆☆ | — | ★★☆✦ |
+| **评测数据** | | | | | | | | | |
+| blimp_good | ★★★ | — | — | — | — | — | — | — | — |
+| blimp_supp_good | — | ★★★ | — | — | — | — | — | — | — |
+| ewok_context | — | — | ★★★ | — | — | — | — | — | — |
+| comps_sentences | — | — | — | — | — | ★★★ | — | — | — |
+| entity_tracking | — | — | — | ★★★ | — | — | — | — | — |
+| aoa_context | — | — | — | — | — | — | — | ★★☆ | — |
+| reading_time | — | — | — | — | — | — | ★★☆ | — | — |
+
+**✦ 有实验证据的 4 个关键发现**：
+1. **gutenberg → BLiMP**：Charpentier 2024 消融实验显示官方 BabyLM 数据对 BLiMP 贡献最大，gutenberg 句法最丰富
+2. **FineWeb-Edu → GLUE**：Charpentier 2024 消融实验直接显示 FineWeb-Edu 对 GLUE 提升最大
+3. **Cosmopedia → EWoK**：Charpentier 2024 消融实验直接显示 Cosmopedia 对 EWoK 提升最大
+4. **TinyDialogues → AoA**：Feng et al. (2024) 实验显示合成 CDS 优于自然 CDS
+5. **SNLI → GLUE**（间接）：Phang et al. (2018) 在微调阶段验证，预训练阶段效果待验证
+
+---
+
+## 五、参考方案
+
+### Charpentier 2024 数据配比（论文参考）
+
+GPT-BERT（Charpentier & Samuel, 2024）在 2024 BabyLM Strict-Small 和 Strict 赛道均获冠军。
+
+**数据配比**：1:1:1 等比混合
+```
+官方 BabyLM 数据:  ~3.3M 词 (33%)  — BLiMP 最佳来源
+FineWeb-Edu:       ~3.3M 词 (33%)  — GLUE 最佳来源
+Cosmopedia:        ~3.4M 词 (34%)  — EWoK 最佳来源
+```
+
+**关键消融发现**：每个语料在不同任务上各有所长，混合使用效果最好。
+
+> 注意：2024 成绩（BLiMP=81.2, GLUE=76.5）基于 2024 评测体系，与 2025 不可直接比较。
+> 2025 Strict-Small 基线成绩见 `ref_leaderboard_2025.md`。
+
+**引用**：Charpentier, L. & Samuel, D. (2024). GPT or BERT: why not both? *CoNLL BabyLM 2024*. arXiv:2410.24159.
+
+---
+
+## 六、数据量总览
+
+### 原始数据
+
+| 类别 | 文件数 | 总行数 | 总词数 |
+|------|:-----:|-------:|-------:|
+| 官方 BabyLM | 7 | 11,160,884 | 96.7M |
+| 外部数据 | 7 | 1,759,801 | 22.4M |
+| **合计** | **14** | **12,920,685** | **119.1M** |
+
+### 评测句（追加到训练集）
+
+| 文件数 | 总行数 | 总词数 |
+|:-----:|-------:|-------:|
+| 7 | 94,169 | 1.36M |
+
+### 最终训练数据预算
+
+```
+总预算:     ≤ 10M 词
+原文:       ~8M 词（Stage 8 采样）
+评测句:     ~1.36M 词（Stage 7A 提取，自动追加）
+Paraphrase: ~0.6M 词（Stage 9 可选，填充剩余预算）
+```
+
+---
+
+## 七、参考文献
+
+- [BabyLM 2024 Findings](https://arxiv.org/html/2412.05149v1)
+- [BabyLM 2025 Findings](https://aclanthology.org/2025.babylm-main.28/)
+- [GPT-BERT: Why Not Both?](https://arxiv.org/abs/2410.24159) — Charpentier & Samuel, 2024
+- [Are BabyLMs Second Language Learners?](https://aclanthology.org/2024.conll-babylm.14/) — Paraphrase 增强
+- [Variation Sets](https://arxiv.org/abs/2411.09587) — 2024 Outstanding Paper
+- [What Should Baby Models Read?](https://aclanthology.org/2024.conll-babylm.25/) — 数据消融实验
+- [Is CDS Effective Training Data?](https://arxiv.org/abs/2408.03617) — TinyDialogues
+- [STILTs](https://arxiv.org/abs/1811.01088) — SNLI 中间任务训练
+- [GenericsKB](https://arxiv.org/abs/2005.00660)
+- [ConceptNet 5.5](https://arxiv.org/abs/1612.03975)
+- [FineWeb-Edu](https://arxiv.org/abs/2406.17557)
+- [BLiMP](https://aclanthology.org/2020.tacl-1.25/)
+- [EWoK](https://arxiv.org/abs/2405.09605)
+- [COMPS](https://aclanthology.org/2023.eacl-main.213/)
+- [BabyLM 2026 CFP](https://arxiv.org/abs/2602.20092)
+- [BabyLM 2026 Guidelines](https://babylm.github.io/guidelines.html)

notes/ref_leaderboard_2025.md

@@ -0,0 +1,54 @@
+# BabyLM 2025 Leaderboard 参考数据
+
+> 数据来源：`babylm_leaderboard_2025.csv`（从官方 HuggingFace leaderboard 摘录）
+> 更新日期：2026-03-23
+
+---
+
+## Strict-Small Track（10M 词，我们参加的赛道）
+
+**关键发现**：前 3 名（amlm_hd_fail, MoEP, amlm_hard_nhot）超过了基线，说明优化空间存在。
+
+| 排名 | 模型 | TextAvg | BLiMP | Supp | EWoK | ET | WUG_ADJ | WUG_PAST | COMPS | AoA | GLUE |
+|:----:|------|:-------:|:-----:|:----:|:----:|:--:|:-------:|:--------:|:-----:|:---:|:----:|
+| 1 | **amlm_hd_fail** | **45.4** | 59.0 | 51.7 | 56.0 | 41.3 | 78.1 | -12.1 | 82.2 | 34.2 | 57.7 |
+| 2 | **MoEP** | **44.5** | 60.8 | 57.5 | 50.2 | 35.6 | 66.8 | -1.1 | 50.7 | 53.7 | 64.2 |
+| 3 | **amlm_hard_nhot** | **41.9** | 65.6 | 54.8 | 49.7 | 43.7 | 64.1 | 0.7 | 51.3 | 16.3 | 67.4 |
+| 4 | Baseline-gpt-bert (causal) | 41.2 | 71.7 | 63.2 | 49.5 | 34.6 | 59.2 | 12.9 | 52.8 | -3.9 | 65.1 |
+| 5 | Baseline-gpt-bert (mntp) | 41.0 | 69.1 | 64.3 | 49.6 | 39.2 | 46.2 | 11.2 | 53.0 | 5.9 | 65.1 |
+| 6 | Baseline-masked-focus (causal) | 40.0 | 65.2 | 59.5 | 49.5 | 31.1 | 44.7 | 20.7 | 52.9 | 3.8 | 66.0 |
+| 7 | Baseline-mixed (mntp) | 39.8 | 71.3 | 63.3 | 49.9 | 40.0 | 18.4 | 12.3 | 53.7 | 16.4 | 65.7 |
+| 8 | babylm-rta | 39.3 | 61.9 | 61.1 | 50.8 | 16.5 | 56.5 | 24.0 | 50.8 | 5.4 | 57.7 |
+| 9 | amlm_hard_decay | 38.3 | 71.4 | 59.2 | 51.0 | 44.2 | 22.3 | 6.2 | 54.2 | -0.9 | 69.8 |
+| 10 | Baseline-masked-focus (mntp) | 38.2 | 70.4 | 63.7 | 50.0 | 40.1 | 2.7 | 28.7 | 53.6 | 0.3 | 66.0 |
+| — | Baseline-gpt2 | 37.4 | 66.4 | 57.1 | 49.9 | 13.9 | 66.1 | -5.0 | 51.7 | 11.7 | 55.9 |
+
+### 关键洞察
+
+1. **TextAvg 冠军 ≠ BLiMP 最高**：amlm_hd_fail 的 TextAvg 最高（45.4），但 BLiMP 只有 59.0；而基线的 BLiMP 高达 71.7。TextAvg 更看重各任务的均衡表现。
+2. **WUG_ADJ 是最大分化点**：amlm_hd_fail 靠 WUG_ADJ=78.1 和 COMPS=82.2 拉高了 TextAvg，说明形态学和概念理解是拉分关键。
+3. **GLUE 差异大**：amlm_hard_decay 的 GLUE=69.8 是所有模型中最高的（基线 65-66），说明微调能力有优化空间。
+4. **Entity Tracking**：amlm_hard_decay 的 ET=44.2 最高，amlm_hard_nhot 的 ET=43.7 紧随其后。
+
+---
+
+## Strict Track（100M 词）
+
+**关键发现**：**前 6 名全是基线模型**，没有任何提交超过基线。
+
+| 排名 | 模型 | TextAvg | BLiMP | GLUE | EWoK | ET |
+|:----:|------|:-------:|:-----:|:----:|:----:|:--:|
+| 1 | Baseline-mixed (mntp) | **47.3** | 80.5 | 70.9 | 52.4 | 39.9 |
+| 2 | Baseline-causal-focus (causal) | 46.7 | 79.3 | 70.1 | 52.3 | 30.9 |
+| 3 | Baseline-causal-focus (mntp) | 46.2 | 78.1 | 70.1 | 53.5 | 41.9 |
+| 7 | blalm-100m（最高提交） | 42.1 | 74.7 | 56.7 | 53.6 | 22.2 |
+
+---
+
+## 对我们的启示
+
+1. **赛道选择正确**：Strict-Small 有优化空间（冠军超基线 4pp），Strict 赛道基线太强
+2. **形态学是关键**：WUG_ADJ 差异最大（2.7 ~ 78.1），N-hot 嵌入（C1）值得重点投入
+3. **AMLM + N-hot + MoE 组合**：前 3 名分别使用了 AMLM、MoE、AMLM+N-hot，验证了 encyclopedia 中 E1/C1/D5 的价值
+4. **GLUE 有空间**：最高 69.8 vs 基线 65-66，训练目标和数据策略对微调有帮助
+5. **GPT-2 基线弱**：TextAvg 37.4，BLiMP 66.4，作为 D0 基线可以清楚看到各改进的提升

requirements.txt

@@ -0,0 +1,40 @@
+# BabyLM Challenge 2026 - Training Environment
+# Python 3.10+
+#
+# Install PyTorch first (adjust CUDA version):
+#   pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu128
+#
+# Then install the rest:
+#   pip install -r requirements.txt
+
+# HuggingFace stack
+transformers==4.45.0
+tokenizers==0.20.3
+datasets==3.6.0
+safetensors==0.5.3
+huggingface-hub==0.33.2
+accelerate==1.7.0
+
+# Training utilities
+wandb==0.19.11
+einops==0.3.2
+
+# Tokenizer
+morfessor==2.0.6
+torch-optimizer==0.3.0
+
+# Evaluation / analysis
+scipy==1.13.1
+scikit-learn==1.6.1
+statsmodels==0.14.5
+polars==1.37.1
+pandas==2.2.3
+numpy==1.23.0
+minicons==0.3.36
+nltk==3.9.1
+pyspellchecker==0.9.0
+
+# General
+tqdm==4.67.1
+pyyaml==6.0.1
+regex==2024.11.6

scripts/01_data/analyze_blimp_coverage.py

@@ -0,0 +1,123 @@
+"""
+BLiMP + BLiMP Supplement 覆盖率分析
+
+对每个子任务：
+1. 提取 good 句中的"判别性关键词"（good有但bad没有的词）
+2. 统计这些词在训练数据（3_dedup）里的总频次
+3. 输出每个子任务的覆盖得分，找出弱点
+"""
+
+import json, re
+from pathlib import Path
+from collections import Counter, defaultdict
+
+ROOT        = Path(__file__).parent.parent.parent
+BLIMP_DIR   = ROOT / "evaluation-pipeline-2025/evaluation_data/full_eval/blimp_filtered"
+SUPP_DIR    = ROOT / "evaluation-pipeline-2025/evaluation_data/full_eval/supplement_filtered"
+TRAIN_DIR   = ROOT / "data/3_dedup"
+OUT_DIR     = ROOT / "results/analysis"
+OUT_DIR.mkdir(parents=True, exist_ok=True)
+
+TRAIN_FILES = [
+    "gutenberg.train.txt",
+    "simple_wiki.train.txt",
+    "bnc_spoken.train.txt",
+    "switchboard.train.txt",
+    "open_subtitles_cleaned_ckpt_70.txt",
+    "childes.train.txt",
+    "open_subtitles.train.txt",
+]
+
+def tokenize(text):
+    return re.findall(r"[a-z']+", text.lower())
+
+# ── 1. 统计训练数据词频 ───────────────────────────────────
+print("统计训练数据词频...")
+train_freq = Counter()
+train_total = 0
+for fname in TRAIN_FILES:
+    path = TRAIN_DIR / fname
+    if not path.exists():
+        continue
+    with open(path, errors="ignore") as f:
+        for line in f:
+            words = tokenize(line)
+            train_total += len(words)
+            train_freq.update(words)
+print(f"  训练数据总词数: {train_total:,}")
+
+# ── 2. 分析每个BLiMP子任务 ───────────────────────────────
+def analyze_task(jfile):
+    good_words = Counter()
+    bad_words  = Counter()
+    n_pairs = 0
+    with open(jfile) as f:
+        for line in f:
+            d = json.loads(line)
+            good_words.update(tokenize(d["sentence_good"]))
+            bad_words.update(tokenize(d["sentence_bad"]))
+            n_pairs += 1
+
+    # 判别性关键词：good句里有，bad句里没有（或少很多）
+    discriminative = {}
+    for w, gc in good_words.items():
+        bc = bad_words.get(w, 0)
+        if gc - bc >= max(3, gc * 0.3):   # good比bad多30%以上且至少差3次
+            discriminative[w] = gc - bc
+
+    # 这些词在训练数据里的频次（每百万词）
+    if not discriminative:
+        return None
+
+    top_words = sorted(discriminative.items(), key=lambda x: -x[1])[:10]
+    coverage_scores = []
+    for w, diff in top_words:
+        per_million = train_freq.get(w, 0) / train_total * 1e6
+        coverage_scores.append((w, diff, per_million))
+
+    # 子任务覆盖得分 = 关键词在训练数据中的平均频次（每百万词）
+    avg_coverage = sum(s for _, _, s in coverage_scores) / len(coverage_scores)
+
+    return {
+        "task":         jfile.stem,
+        "n_pairs":      n_pairs,
+        "top_keywords": coverage_scores,
+        "avg_coverage": avg_coverage,
+    }
+
+# 分析BLiMP
+print("\n分析 BLiMP 子任务...")
+blimp_results = []
+for jfile in sorted(BLIMP_DIR.glob("*.jsonl")):
+    r = analyze_task(jfile)
+    if r:
+        blimp_results.append(r)
+
+# 分析BLiMP Supplement
+print("分析 BLiMP Supplement 子任务...")
+supp_results = []
+for jfile in sorted(SUPP_DIR.glob("*.jsonl")):
+    r = analyze_task(jfile)
+    if r:
+        supp_results.append(r)
+
+# ── 3. 输出结果 ──────────────────────────────────────────
+def print_results(results, title):
+    print(f"\n{'='*70}")
+    print(f"{title}（按覆盖率从低到高，最需要补充的在前）")
+    print(f"{'='*70}")
+    print(f"{'子任务':<45} {'覆盖分':>8}  {'关键词(每百万词)'}")
+    print("─" * 70)
+    for r in sorted(results, key=lambda x: x["avg_coverage"]):
+        kw_str = ", ".join(f"{w}({s:.0f})" for w, _, s in r["top_keywords"][:4])
+        print(f"{r['task']:<45} {r['avg_coverage']:>8.1f}  {kw_str}")
+
+print_results(blimp_results,  "BLiMP 子任务覆盖分析")
+print_results(supp_results,   "BLiMP Supplement 覆盖分析")
+
+# 保存完整结果
+import json as _json
+out_path = OUT_DIR / "blimp_coverage.json"
+_json.dump({"blimp": blimp_results, "supplement": supp_results},
+           open(out_path, "w"), ensure_ascii=False, indent=2)
+print(f"\n完整结果已保存: {out_path}")

scripts/01_data/find_oov_words.py

@@ -0,0 +1,75 @@
+"""
+找出每个训练文件中所有不在 vocab.txt 里的词，按频次排序输出
+
+注：这是独立的分析工具，不属于主流水线。
+Stage 2（stage2_oov_filter.py）已内置 OOV 频次计算，不再依赖本脚本的输出。
+"""
+import re
+from pathlib import Path
+from collections import Counter
+
+ROOT       = Path(__file__).parent.parent.parent
+VOCAB_PATH = ROOT / "data/vocab.txt"
+DATA_DIR   = ROOT / "data/raw_files"
+OUT_DIR    = ROOT / "results/analysis/oov"
+OUT_DIR.mkdir(parents=True, exist_ok=True)
+
+FILES = [
+    "childes.train.txt",
+    "gutenberg.train.txt",
+    "open_subtitles.train.txt",
+    "simple_wiki.train.txt",
+    "bnc_spoken.train.txt",
+    "open_subtitles_cleaned_ckpt_70.txt",
+    "switchboard.train.txt",
+]
+
+def tokenize(line):
+    return re.findall(r"[a-zA-Z''\-]+|[0-9]+", line.lower())
+
+vocab = set(VOCAB_PATH.read_text().splitlines())
+print(f"vocab 大小: {len(vocab):,} 词\n")
+
+summary = []
+
+for fname in FILES:
+    path = DATA_DIR / fname
+    if not path.exists():
+        continue
+
+    oov_counter = Counter()
+    total_tokens = 0
+
+    with open(path, "r", encoding="utf-8", errors="ignore") as f:
+        for line in f:
+            tokens = tokenize(line)
+            total_tokens += len(tokens)
+            for t in tokens:
+                if t not in vocab:
+                    oov_counter[t] += 1
+
+    total_oov_tokens = sum(oov_counter.values())
+    total_oov_types  = len(oov_counter)
+    oov_rate = total_oov_tokens / total_tokens * 100 if total_tokens else 0
+
+    # 保存完整 OOV 列表（按频次降序）
+    stem = fname.replace(".train.txt", "").replace(".txt", "")
+    out_path = OUT_DIR / f"{stem}_oov.txt"
+    with open(out_path, "w") as f:
+        f.write(f"# {fname}  OOV统计\n")
+        f.write(f"# 总token数: {total_tokens:,}  OOV token数: {total_oov_tokens:,}  OOV率: {oov_rate:.2f}%\n")
+        f.write(f"# OOV类型数（唯一词）: {total_oov_types:,}\n")
+        f.write(f"# 格式: 频次 词\n\n")
+        for word, cnt in oov_counter.most_common():
+            f.write(f"{cnt}\t{word}\n")
+
+    summary.append((fname, total_tokens, total_oov_tokens, oov_rate, total_oov_types))
+    print(f"[{fname}]  OOV率={oov_rate:.2f}%  唯一OOV词={total_oov_types:,}  → {out_path.name}")
+
+print("\n\n── 汇总 ──────────────────────────────────────────────────────")
+print(f"{'文件':<42} {'总token':>10} {'OOV token':>10} {'OOV率':>7} {'OOV唯一词':>10}")
+print("─" * 82)
+for fname, tot, oov_tok, rate, oov_types in summary:
+    print(f"{fname:<42} {tot:>10,} {oov_tok:>10,} {rate:>6.2f}%  {oov_types:>9,}")
+
+print(f"\n结果保存在: {OUT_DIR}")

scripts/01_data/run_all.py

@@ -0,0 +1,190 @@
+"""
+01_data 数据流水线总控
+按顺序执行所有数据处理阶段
+
+阶段概览：
+  1  噪声过滤        → data/1_noise_filtered/
+  2  OOV过滤         → data/2_oov_filtered/
+  3  精确去重         → data/3_dedup/
+  4  格式预处理       → data/4_preprocessed/
+  5  规则标注         → data/5_rule_labels/
+  6  Qwen语义标注     → data/6_qwen_labels/     (需GPU)
+  7  评测句提取       → data/7_eval_sentences/
+  8A 采样-纯质量基线  → data/8_sample_A/train.txt
+  8B 采样-配额+质量   → data/8_sample_B/train.txt
+  8C 采样-加权采样    → data/8_sample_C/train.txt
+  8D 采样-Embedding   → data/8_sample_D/train.txt  (需GPU)
+
+用法:
+    python scripts/01_data/run_all.py              # 跑所有阶段 (1-8A)
+    python scripts/01_data/run_all.py --stage 5    # 只跑某一阶段
+    python scripts/01_data/run_all.py --from 7     # 从某阶段开始跑
+    python scripts/01_data/run_all.py --stage 8    # 跑所有 stage 8 采样 (A/B/C/D)
+    python scripts/01_data/run_all.py --stage 8B   # 只跑 stage 8B
+    python scripts/01_data/run_all.py --stage 8D   # 只跑 stage 8D (Embedding)
+"""
+
+import argparse
+import subprocess
+import sys
+from pathlib import Path
+
+ROOT = Path(__file__).parent.parent.parent
+
+# ── 阶段定义 ────────────────────────────────────────────────────────────────────
+# type="io"   : 标准 input/output 参数
+# type="plain" : 无参数直接运行
+
+STAGES = [
+    {
+        "id": "1", "name": "噪声过滤 (CHAT标注 / 时间戳 / 极短行)",
+        "script": "stage1_noise_filter.py", "type": "io",
+        "input": "data/raw_files", "output": "data/1_noise_filtered",
+    },
+    {
+        "id": "2", "name": "OOV过滤 (低频噪声词过多的句子)",
+        "script": "stage2_oov_filter.py", "type": "io",
+        "input": "data/1_noise_filtered", "output": "data/2_oov_filtered",
+    },
+    {
+        "id": "3", "name": "精确去重（全局 MD5）",
+        "script": "stage3_dedup.py", "type": "io",
+        "input": "data/2_oov_filtered", "output": "data/3_dedup",
+    },
+    {
+        "id": "4", "name": "格式预处理（去说话人前缀 / 全大写规范 / 标题行过滤）",
+        "script": "stage4_preprocess.py", "type": "io",
+        "input": "data/3_dedup", "output": "data/4_preprocessed",
+    },
+    {
+        "id": "5", "name": "规则标注（BLiMP 67子任务 / BLiMP Supplement / EWoK 11域）",
+        "script": "stage5_rule_label.py", "type": "io",
+        "input": "data/4_preprocessed", "output": "data/5_rule_labels",
+    },
+    {
+        "id": "6", "name": "Qwen语义标注（9维度深度分析，需GPU）",
+        "script": "stage6_qwen_label.py", "type": "io",
+        "input": "data/4_preprocessed", "output": "data/6_qwen_labels",
+    },
+    {
+        "id": "7", "name": "评测句提取",
+        "script": "stage7a_extract_eval.py", "type": "plain",
+    },
+    {
+        "id": "8A", "name": "采样方案A：纯质量基线",
+        "script": "stage8_sample_A.py", "type": "plain",
+    },
+    {
+        "id": "8B", "name": "采样方案B：任务配额 + 质量填充",
+        "script": "stage8_sample_B.py", "type": "plain",
+    },
+    {
+        "id": "8C", "name": "采样方案C：加权采样",
+        "script": "stage8_sample_C.py", "type": "plain",
+    },
+    {
+        "id": "8D", "name": "采样方案D：Embedding 相似度采样（需GPU）",
+        "script": "stage8_sample_D.py", "type": "plain",
+    },
+]
+
+
+def run_stage(stage):
+    script = Path(__file__).parent / stage["script"]
+
+    print(f"\n{'='*60}")
+    print(f"阶段 {stage['id']}: {stage['name']}")
+    print(f"  脚本: {script.name}")
+    if stage["type"] == "io":
+        print(f"  输入: {ROOT / stage['input']}")
+        print(f"  输出: {ROOT / stage['output']}")
+    print(f"{'='*60}")
+
+    if stage["type"] == "io":
+        cmd = [sys.executable, str(script),
+               "--input_dir",  str(ROOT / stage["input"]),
+               "--output_dir", str(ROOT / stage["output"])]
+    else:
+        cmd = [sys.executable, str(script)]
+
+    result = subprocess.run(cmd, check=False)
+    if result.returncode != 0:
+        print(f"\n[ERROR] 阶段 {stage['id']} 失败，退出码 {result.returncode}")
+        sys.exit(result.returncode)
+    print(f"\n[OK] 阶段 {stage['id']} 完成")
+
+
+def match_stages(selector: str) -> list[dict]:
+    """根据选择器匹配阶段。支持: "5", "8", "8B", "8D" 等。"""
+    selector = selector.upper()
+    matched = []
+    for s in STAGES:
+        sid = s["id"].upper()
+        if sid == selector:
+            matched.append(s)
+        elif selector.isdigit() and sid.startswith(selector):
+            # "8" 匹配 "8A", "8B", "8C", "8D"
+            matched.append(s)
+    return matched
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="01_data 数据流水线总控",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    parser.add_argument("--stage", type=str, default=None,
+                        help="只运行指定阶段（如 --stage 5, --stage 8, --stage 8B）")
+    parser.add_argument("--from", type=str, default="1", dest="from_stage",
+                        help="从指定阶段开始运行（如 --from 7）")
+    parser.add_argument("--to", type=str, default=None, dest="to_stage",
+                        help="运行到指定阶段为止（如 --to 6）")
+    parser.add_argument("--list", action="store_true",
+                        help="列出所有阶段")
+    args = parser.parse_args()
+
+    if args.list:
+        print("可用阶段：")
+        for s in STAGES:
+            print(f"  {s['id']:4s}  {s['name']}")
+        return
+
+    if args.stage is not None:
+        stages_to_run = match_stages(args.stage)
+    else:
+        # --from / --to 范围选择（只匹配纯数字前缀）
+        from_idx = 0
+        to_idx = len(STAGES)
+        for i, s in enumerate(STAGES):
+            num = s["id"].rstrip("ABCD")
+            if num == args.from_stage:
+                from_idx = i
+                break
+        if args.to_stage:
+            for i, s in enumerate(STAGES):
+                num = s["id"].rstrip("ABCD")
+                if num == args.to_stage:
+                    # 包含该阶段的所有子阶段
+                    to_idx = i + 1
+                    while to_idx < len(STAGES) and STAGES[to_idx]["id"].startswith(args.to_stage):
+                        to_idx += 1
+                    break
+        stages_to_run = STAGES[from_idx:to_idx]
+
+    if not stages_to_run:
+        print(f"没有找到匹配的阶段: {args.stage or args.from_stage}")
+        print("使用 --list 查看所有可用阶段")
+        sys.exit(1)
+
+    print(f"将运行 {len(stages_to_run)} 个阶段: {[s['id'] for s in stages_to_run]}")
+
+    for stage in stages_to_run:
+        run_stage(stage)
+
+    print(f"\n{'='*60}")
+    print("数据流水线完成！")
+    print(f"{'='*60}")
+
+
+if __name__ == "__main__":
+    main()

scripts/01_data/stage1_noise_filter.py

@@ -0,0 +1,189 @@
+"""
+第一阶段：噪声过滤
+输入: data/raw_files/
+输出: data/1_noise_filtered/
+
+过滤规则：
+1. 【所有文件】空行、词数 < 3 的极短行
+2. 【所有文件】含 Unicode 替换字符 (U+FFFD) 的编码损坏行
+3. 【所有文件】非 ASCII 字符占比过高的编码垃圾行（ASCII 比例 < 70%）
+4. 【CHILDES】以 @ 或 % 开头的 CHAT 标注行；含大量 xxx/yyy 的行
+5. 【Open Subtitles】时间戳行；HTML 标签行
+"""
+
+import re
+import argparse
+from pathlib import Path
+from collections import defaultdict
+
+ROOT       = Path(__file__).parent.parent.parent
+INPUT_DIR  = ROOT / "data/raw_files"
+OUTPUT_DIR = ROOT / "data/1_noise_filtered"
+
+FILES = [
+    # 官方 BabyLM 数据
+    "childes.train.txt",
+    "gutenberg.train.txt",
+    "open_subtitles.train.txt",
+    "simple_wiki.train.txt",
+    "bnc_spoken.train.txt",
+    "open_subtitles_cleaned_ckpt_70.txt",
+    "switchboard.train.txt",
+    # 外部数据
+    "ext_fineweb_edu.txt",
+    "ext_cosmopedia.txt",
+    "ext_tiny_dialogues.txt",
+    "ext_generics_kb.txt",
+    "ext_conceptnet.txt",
+    "ext_onestop_english.txt",
+    "ext_snli.txt",
+]
+
+# ASCII 字符占比低于此值的行视为编码垃圾
+MIN_ASCII_RATIO = 0.7
+
+# ── 过滤函数 ────────────────────────────────────────────
+
+def is_too_short(line):
+    """词数 < 3"""
+    return len(line.split()) < 3
+
+
+def has_encoding_corruption(line):
+    """含 Unicode 替换字符 (U+FFFD) 的编码损坏行"""
+    return '\ufffd' in line
+
+
+def is_encoding_garbage(line):
+    """非 ASCII 字符占比过高的编码垃圾行"""
+    if not line:
+        return False
+    ascii_count = sum(1 for c in line if ord(c) < 128)
+    return (ascii_count / len(line)) < MIN_ASCII_RATIO
+
+
+def is_metadata_separator(line):
+    """CHILDES/Open Subtitles 文件分隔符行，如 = = = path/to/file.cha = = ="""
+    return line.startswith("= = =") or line == "==="
+
+
+def is_childes_noise(line):
+    """CHAT 格式标注行"""
+    if line.startswith(('@', '%')):
+        return True
+    tokens = line.split()
+    oov_like = sum(1 for t in tokens if t in ('xxx', 'yyy', 'www'))
+    if tokens and oov_like / len(tokens) > 0.5:
+        return True
+    return False
+
+def is_subtitle_noise(line):
+    """时间戳或 HTML 标签行"""
+    # 时间戳：00:01:23,456 --> 00:01:25,789
+    if re.match(r'^\d{2}:\d{2}:\d{2}', line):
+        return True
+    # 纯数字行（字幕序号）
+    if re.match(r'^\d+$', line):
+        return True
+    # HTML 标签
+    if re.search(r'<[^>]+>', line):
+        return True
+    return False
+
+# ── 主流程 ───────────────────────────────────────────────
+
+def filter_file(file_name, input_dir=None, output_dir=None):
+    input_dir   = Path(input_dir)  if input_dir  else INPUT_DIR
+    output_dir  = Path(output_dir) if output_dir else OUTPUT_DIR
+    input_path  = input_dir  / file_name
+    output_path = output_dir / file_name
+
+    is_childes   = "childes"   in file_name
+    is_subtitles = "open_subtitles" in file_name
+
+    stats = defaultdict(int)
+
+    with open(input_path,  "r", encoding="utf-8", errors="ignore") as fin, \
+         open(output_path, "w", encoding="utf-8") as fout:
+
+        for raw_line in fin:
+            line = raw_line.strip()
+            stats["total"] += 1
+
+            if not line:
+                stats["drop_empty"] += 1
+                continue
+
+            if has_encoding_corruption(line):
+                stats["drop_encoding"] += 1
+                continue
+
+            if is_encoding_garbage(line):
+                stats["drop_encoding_garbage"] += 1
+                continue
+
+            if is_metadata_separator(line):
+                stats["drop_metadata"] += 1
+                continue
+
+            if is_childes and is_childes_noise(line):
+                stats["drop_chat_noise"] += 1
+                continue
+
+            if is_subtitles and is_subtitle_noise(line):
+                stats["drop_subtitle_noise"] += 1
+                continue
+
+            if is_too_short(line):
+                stats["drop_too_short"] += 1
+                continue
+
+            fout.write(line + "\n")
+            stats["keep"] += 1
+
+    return stats
+
+# ── 执行 & 报告 ──────────────────────────────────────────
+
+def main(input_dir=None, output_dir=None):
+    in_dir  = Path(input_dir)  if input_dir  else INPUT_DIR
+    out_dir = Path(output_dir) if output_dir else OUTPUT_DIR
+    out_dir.mkdir(parents=True, exist_ok=True)
+
+    print(f"输入: {in_dir}")
+    print(f"输出: {out_dir}\n")
+    print(f"{'文件':<42} {'保留':>8} {'丢弃':>8} {'保留率':>8}  丢弃原因")
+    print("─" * 110)
+
+    for fname in FILES:
+        path = in_dir / fname
+        if not path.exists():
+            print(f"{fname:<42} [文件不存在，跳过]")
+            continue
+
+        stats = filter_file(fname, in_dir, out_dir)
+        total  = stats["total"]
+        keep   = stats["keep"]
+        drop   = total - keep
+        rate   = keep / total * 100 if total else 0
+
+        reasons = []
+        if stats["drop_empty"]:            reasons.append(f"空行:{stats['drop_empty']:,}")
+        if stats["drop_encoding"]:         reasons.append(f"U+FFFD:{stats['drop_encoding']:,}")
+        if stats["drop_encoding_garbage"]: reasons.append(f"编码垃圾:{stats['drop_encoding_garbage']:,}")
+        if stats["drop_metadata"]:         reasons.append(f"元数据:{stats['drop_metadata']:,}")
+        if stats["drop_chat_noise"]:       reasons.append(f"CHAT:{stats['drop_chat_noise']:,}")
+        if stats["drop_subtitle_noise"]:   reasons.append(f"字幕噪声:{stats['drop_subtitle_noise']:,}")
+        if stats["drop_too_short"]:        reasons.append(f"太短:{stats['drop_too_short']:,}")
+
+        print(f"{fname:<42} {keep:>8,} {drop:>8,} {rate:>7.1f}%  {' | '.join(reasons)}")
+
+    print("\n完成！")
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--input_dir",  default=None)
+    parser.add_argument("--output_dir", default=None)
+    args = parser.parse_args()
+    main(args.input_dir, args.output_dir)

scripts/01_data/stage2_oov_filter.py

@@ -0,0 +1,158 @@
+"""
+第二阶段：OOV 过滤
+输入:  data/1_noise_filtered/
+输出:  data/2_oov_filtered/
+
+过滤逻辑：
+1. 用 vocab.txt（评测词表，66,875 词）判断每个 token 是否为 OOV
+2. 统计每个文件中各 OOV 词的出现频次
+3. 频次 < freq_threshold 的 OOV 词视为"噪声嫌疑词"（高频 OOV 如缩写是正常英语）
+4. 若一行中噪声嫌疑词占比 > noise_ratio_max，则丢弃该行
+5. 每个文件有独立参数（详见 FILE_PARAMS）
+
+参数依据：
+- simple_wiki OOV 主要是维基百科专有名词（合法内容），保持宽松
+- open_subtitles 词语黏连多，适度过滤
+- 其余文件噪声较少，按实际过滤率设置
+"""
+
+import re
+import argparse
+from pathlib import Path
+from collections import Counter, defaultdict
+
+ROOT       = Path(__file__).parent.parent.parent
+INPUT_DIR  = ROOT / "data/1_noise_filtered"
+OUTPUT_DIR = ROOT / "data/2_oov_filtered"
+VOCAB_PATH = ROOT / "data/vocab.txt"
+
+# 每个文件的独立参数：(freq_threshold, noise_ratio_max)
+# freq_threshold:  OOV词频次低于此值 → 视为噪声嫌疑词
+# noise_ratio_max: 句子中噪声嫌疑词占比超过此值 → 丢弃
+FILE_PARAMS = {
+    # 官方 BabyLM 数据
+    "childes.train.txt":                    (20, 0.05),
+    "gutenberg.train.txt":                  (10, 0.05),
+    "open_subtitles.train.txt":             (10, 0.05),
+    "simple_wiki.train.txt":                ( 3, 0.20),
+    "bnc_spoken.train.txt":                 (10, 0.05),
+    "open_subtitles_cleaned_ckpt_70.txt":   (10, 0.05),
+    "switchboard.train.txt":                (10, 0.05),
+    # 外部数据（已预过滤，宽松参数）
+    "ext_fineweb_edu.txt":                  ( 5, 0.10),
+    "ext_cosmopedia.txt":                   ( 5, 0.10),
+    "ext_tiny_dialogues.txt":               (10, 0.05),
+    "ext_generics_kb.txt":                  ( 3, 0.15),
+    "ext_conceptnet.txt":                   ( 3, 0.15),
+    "ext_onestop_english.txt":              ( 5, 0.10),
+    "ext_snli.txt":                         ( 5, 0.10),
+}
+
+
+def tokenize(line):
+    return re.findall(r"[a-zA-Z''\-]+|[0-9]+", line.lower())
+
+
+def load_vocab(vocab_path: Path) -> set:
+    """加载评测词表"""
+    vocab = set(vocab_path.read_text().splitlines())
+    print(f"  vocab.txt: {len(vocab):,} 词  ({vocab_path})")
+    return vocab
+
+
+def compute_oov_freq(input_path: Path, vocab: set) -> Counter:
+    """扫描文件，统计每个 OOV 词的出现频次"""
+    oov_counter = Counter()
+    with open(input_path, "r", encoding="utf-8", errors="ignore") as f:
+        for line in f:
+            for t in tokenize(line):
+                if t not in vocab:
+                    oov_counter[t] += 1
+    return oov_counter
+
+
+def filter_file(fname: str, input_dir: Path, output_dir: Path, vocab: set,
+                verbose: bool = True) -> dict:
+    freq_thresh, noise_ratio = FILE_PARAMS[fname]
+    input_path = input_dir / fname
+
+    # 第一遍：统计 OOV 频次
+    if verbose:
+        print(f"  {fname}: 统计 OOV 频次 ...", end="", flush=True)
+    oov_counter = compute_oov_freq(input_path, vocab)
+    noise_oov = {w for w, cnt in oov_counter.items() if cnt < freq_thresh}
+    if verbose:
+        print(f" {len(oov_counter):,} OOV 类型, {len(noise_oov):,} 噪声词 (freq<{freq_thresh})")
+
+    # 第二遍：过滤
+    stats = defaultdict(int)
+    output_path = output_dir / fname
+    with open(input_path, "r", encoding="utf-8", errors="ignore") as fin, \
+         open(output_path, "w", encoding="utf-8") as fout:
+
+        for line in fin:
+            line = line.strip()
+            stats["total"] += 1
+            if not line:
+                stats["drop_empty"] += 1
+                continue
+
+            tokens = tokenize(line)
+            if not tokens:
+                stats["drop_empty"] += 1
+                continue
+
+            noise_cnt = sum(1 for t in tokens if t in noise_oov)
+            if noise_cnt / len(tokens) > noise_ratio:
+                stats["drop_oov"] += 1
+                continue
+
+            fout.write(line + "\n")
+            stats["keep"] += 1
+
+    total = stats["total"]
+    keep  = stats["keep"]
+    rate  = keep / total * 100 if total else 0
+    if verbose:
+        print(f"  {fname:<42} 保留 {keep:>8,} / {total:>8,}  ({rate:.1f}%)  "
+              f"OOV过滤:{stats['drop_oov']:,}")
+    return stats
+
+
+def main(input_dir=None, output_dir=None):
+    input_dir  = Path(input_dir)  if input_dir  else INPUT_DIR
+    output_dir = Path(output_dir) if output_dir else OUTPUT_DIR
+    output_dir.mkdir(parents=True, exist_ok=True)
+
+    print(f"输入: {input_dir}")
+    print(f"输出: {output_dir}")
+
+    if not VOCAB_PATH.exists():
+        print(f"\n[错误] vocab.txt 不存在: {VOCAB_PATH}")
+        print(f"  请确认 data/vocab.txt 文件存在（来源：evaluation-pipeline-2025/evaluation_pipeline/ewok/vocab.txt）")
+        return
+
+    vocab = load_vocab(VOCAB_PATH)
+    print()
+
+    total_stats = defaultdict(int)
+    for fname in FILE_PARAMS:
+        if not (input_dir / fname).exists():
+            print(f"  [skip] {fname} 不存在")
+            continue
+        stats = filter_file(fname, input_dir, output_dir, vocab)
+        for k, v in stats.items():
+            total_stats[k] += v
+
+    total = total_stats["total"]
+    keep  = total_stats["keep"]
+    print(f"\n合计: 保留 {keep:,} / {total:,} 行  ({keep/total*100:.1f}%)")
+    print(f"OOV过滤共丢弃: {total_stats['drop_oov']:,} 行")
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--input_dir",  default=None)
+    parser.add_argument("--output_dir", default=None)
+    args = parser.parse_args()
+    main(args.input_dir, args.output_dir)

scripts/01_data/stage3_dedup.py

@@ -0,0 +1,119 @@
+"""
+第三阶段：精确去重
+输入:  data/2_oov_filtered/
+输出:  data/3_dedup/
+
+去重方式：
+- 对每行计算 MD5 哈希值
+- 全局去重（跨文件）：同一句话在多个文件中只保留第一次出现
+- 文件处理顺序即优先级（gutenberg 质量最高，最先处理）
+
+注：只对行内容去重，不区分大小写（统一 lower 后比较）
+"""
+
+import hashlib
+import argparse
+from pathlib import Path
+from collections import defaultdict
+
+ROOT       = Path(__file__).parent.parent.parent
+INPUT_DIR  = ROOT / "data/2_oov_filtered"
+OUTPUT_DIR = ROOT / "data/3_dedup"
+
+# 文件处理顺序 = 去重优先级（先出现的保留）
+# 高质量文件排前面，外部数据排在官方数据之后
+FILES = [
+    # 官方 BabyLM 数据（优先保留）
+    "gutenberg.train.txt",
+    "simple_wiki.train.txt",
+    "bnc_spoken.train.txt",
+    "switchboard.train.txt",
+    "open_subtitles_cleaned_ckpt_70.txt",
+    "childes.train.txt",
+    "open_subtitles.train.txt",
+    # 外部数据（官方数据优先，重复行保留官方版本）
+    "ext_fineweb_edu.txt",
+    "ext_cosmopedia.txt",
+    "ext_onestop_english.txt",
+    "ext_generics_kb.txt",
+    "ext_conceptnet.txt",
+    "ext_tiny_dialogues.txt",
+    "ext_snli.txt",
+]
+
+
+def line_hash(line: str) -> str:
+    return hashlib.md5(line.lower().strip().encode("utf-8")).hexdigest()
+
+
+def dedup_file(fname, input_dir, output_dir, seen_hashes):
+    stats = defaultdict(int)
+    output_path = output_dir / fname
+
+    with open(input_dir / fname, "r", encoding="utf-8", errors="ignore") as fin, \
+         open(output_path, "w", encoding="utf-8") as fout:
+
+        for raw_line in fin:
+            line = raw_line.strip()
+            stats["total"] += 1
+
+            if not line:
+                stats["drop_empty"] += 1
+                continue
+
+            h = line_hash(line)
+            if h in seen_hashes:
+                stats["drop_dup"] += 1
+                continue
+
+            seen_hashes.add(h)
+            fout.write(line + "\n")
+            stats["keep"] += 1
+
+    return stats
+
+
+def main(input_dir=None, output_dir=None):
+    input_dir  = Path(input_dir)  if input_dir  else INPUT_DIR
+    output_dir = Path(output_dir) if output_dir else OUTPUT_DIR
+    output_dir.mkdir(parents=True, exist_ok=True)
+
+    print(f"输入: {input_dir}")
+    print(f"输出: {output_dir}")
+    print(f"去重方式: 全局精确去重（MD5，跨文件）\n")
+
+    seen_hashes = set()
+    total_stats = defaultdict(int)
+
+    print(f"{'文件':<42} {'保留':>8} {'去重丢弃':>10} {'保留率':>8}")
+    print("─" * 75)
+
+    for fname in FILES:
+        if not (input_dir / fname).exists():
+            print(f"  [skip] {fname} 不存在")
+            continue
+
+        stats = dedup_file(fname, input_dir, output_dir, seen_hashes)
+        total = stats["total"]
+        keep  = stats["keep"]
+        rate  = keep / total * 100 if total else 0
+
+        print(f"{fname:<42} {keep:>8,} {stats['drop_dup']:>10,} {rate:>7.1f}%")
+
+        for k, v in stats.items():
+            total_stats[k] += v
+
+    total = total_stats["total"]
+    keep  = total_stats["keep"]
+    print("─" * 75)
+    print(f"{'合计':<42} {keep:>8,} {total_stats['drop_dup']:>10,} {keep/total*100:>7.1f}%")
+    print(f"\n全局唯一句子数: {len(seen_hashes):,}")
+    print(f"去重共丢弃: {total_stats['drop_dup']:,} 行")
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--input_dir",  default=None)
+    parser.add_argument("--output_dir", default=None)
+    args = parser.parse_args()
+    main(args.input_dir, args.output_dir)

scripts/01_data/stage4_preprocess.py

@@ -0,0 +1,372 @@
+"""
+Stage 4：各文件格式预处理
+
+每个训练文件的清洗规则不同：
+
+  childes              : 去除说话人前缀 *XXX:\t，丢弃独立注释行 [...]，移除行内 [...]/xxx
+  switchboard          : 去除说话人前缀 A:\t / B:\t
+                         可选：去除口语不流利标记 uh/um（默认保留）
+  open_subtitles       : 全大写 → 句首大写（修复第一人称 I）
+  simple_wiki          : 丢弃 = = = 标题行；丢弃 A:/B: 对话行；丢弃小节标题行；
+                         丢弃 URL 行；HTML 实体解码
+  open_subtitles_cleaned: 去除 *XXX:\t 前缀，去除 [laugh]/xxx 等 CHAT 标记
+  bnc_spoken           : 无需处理（直接复制）
+  gutenberg            : 过滤 Project Gutenberg 模板文本；HTML 实体解码
+
+可选：--ner 启用 spaCy NER 恢复专有名词大小写（PERSON/GPE/ORG 等）
+
+输入 : data/3_dedup/
+输出 : data/4_preprocessed/
+
+用法:
+  python scripts/01_data/stage4_preprocess.py
+  python scripts/01_data/stage4_preprocess.py --ner               # 启用 NER 大小写恢复
+  python scripts/01_data/stage4_preprocess.py --file childes.train.txt
+  python scripts/01_data/stage4_preprocess.py --remove_disfluency
+"""
+
+import html
+import re
+import argparse
+import time
+from pathlib import Path
+from tqdm import tqdm
+
+ROOT     = Path(__file__).parent.parent.parent
+IN_DIR   = ROOT / "data" / "3_dedup"
+OUT_DIR  = ROOT / "data" / "4_preprocessed"
+OUT_DIR.mkdir(parents=True, exist_ok=True)
+
+# ─────────────────────────────────────────────
+# 各文件清洗函数
+# 返回 None 表示丢弃该行，返回字符串表示保留
+# ─────────────────────────────────────────────
+
+# CHAT 格式说话人前缀：*CHI: / *MOT: / *COL: 等
+_CHAT_PREFIX = re.compile(r"^\*[A-Z0-9]+:\s*")
+# 独立注释行：整行都是 [...] 内容
+_CHAT_COMMENT = re.compile(r"^\[.*\]\s*$")
+# open_subtitles_cleaned 里的 CHAT 标记：[laugh] / [laughter] / xxx
+_CHAT_TOKEN = re.compile(r"\[.*?\]|(?<!\w)xxx(?!\w)", re.I)
+# 口语不流利标记：独立的 uh / um（前后是标点或空格）
+_DISFLUENCY = re.compile(r"(?<!\w)(uh|um|uh-huh|mm-hmm|hmm|hm)(?!\w),?\s*", re.I)
+# CHAT 行内注释：[laugh] / [sits on couch] 等
+_CHAT_INLINE = re.compile(r"\[.*?\]")
+# Simple Wiki 标题行：= = = 任意内容 = = =
+_WIKI_HEADING = re.compile(r"^=+\s*.+\s*=+\s*$")
+# Simple Wiki 小节标题：单独一行的 "History." / "Early life and career." 等
+_WIKI_SECTION = re.compile(
+    r"^(History|Geography|Population|Demographics|Music video|Early life"
+    r"|Career|Personal life|Education|Discography|Filmography|Biography"
+    r"|Background|Death|Awards|Legacy|Plot|Synopsis|References"
+    r"|Other Websites|External links|See also|Gallery|Sources"
+    r"|Bibliography|Controversy|Criticism|Reception|Gameplay|Development"
+    r"|Release|Production|Cast|Notes|Controversy and criticism"
+    r"|Early life and career|Life and career|Life|Works|Family"
+    r"|Climate|Economy|Transport|Government|Military career|Music"
+    r"|Television|Film|Books|Albums|Singles|Track listing"
+    r"|2020 census|Honors|Achievements|Members|Stadium|Overview"
+    r"|Notable people|Notable residents|Composition|Recording"
+    r"|Commercial performance|Critical reception|Chart performance"
+    r"|Music videos?|Video games?|Politics|Religion|Culture|Media"
+    r"|Sports|Infrastructure|Languages|Geology|Ecology|Taxonomy"
+    r"|Description|Distribution|Habitat|Behavior|Conservation"
+    r"|In popular culture|Related pages)\s*\.?\s*$", re.I)
+# Simple Wiki A:/B: 对话行（混入的 Switchboard 风格对话）
+_WIKI_DIALOGUE = re.compile(r"^[AB]:\s")
+# URL 行
+_URL_PATTERN = re.compile(r"https?://")
+# Switchboard 说话人前缀：A:\t / B:\t（字母 + 冒号 + tab）
+_SWB_PREFIX = re.compile(r"^[A-Z]:\s*")
+# Gutenberg 模板文本
+_GUTENBERG_BOILERPLATE = re.compile(
+    r"(^End of .*Project Gutenberg|^E-text prepared by|"
+    r"^Produced by .*(Project Gutenberg|Distributed Proofreading)"
+    r"|^This (eBook|ebook|book) (is for the use of|has been transcribed for)"
+    r"|^\*\*\* ?START OF|^\*\*\* ?END OF"
+    r"|^_?Copyright[_,ed]|^COPYRIGHT)", re.I)
+# 最短保留词数
+MIN_WORDS = 3
+
+# NER 需要恢复大小写的实体类型
+_NER_LABELS = {"PERSON", "GPE", "ORG", "FAC", "LOC", "NORP", "EVENT", "WORK_OF_ART"}
+
+
+def clean_childes(line: str) -> str | None:
+    """去除 *XXX:\t 说话人前缀；丢弃独立注释行；移除行内 [...] 注释和 xxx 标记
+    最短词数放宽到 2（儿童短句对 AoA 评测有价值）
+    """
+    if _CHAT_COMMENT.match(line):
+        return None
+    line = _CHAT_PREFIX.sub("", line).strip()
+    # 移除行内 [laugh] / [sits on couch] 等注释，以及独立的 xxx（不可辨识语音标记）
+    line = _CHAT_TOKEN.sub(" ", line)
+    # 清理多余空格和标点前空格
+    line = re.sub(r"\s+([.,!?;:])", r"\1", line)
+    line = re.sub(r"\s{2,}", " ", line).strip()
+    if len(line.split()) < 2:
+        return None
+    return line
+
+
+def clean_switchboard(line: str, remove_disfluency: bool = False) -> str | None:
+    """去除 A:\t / B:\t 说话人前缀；可选去除 uh/um"""
+    line = _SWB_PREFIX.sub("", line).strip()
+    if remove_disfluency:
+        line = _DISFLUENCY.sub(" ", line).strip()
+    if len(line.split()) < MIN_WORDS:
+        return None
+    return line
+
+
+def clean_open_subtitles(line: str) -> str | None:
+    """全大写 → 句首大写（其余小写），修复第一人称 I"""
+    # 只有全大写的行才转换（避免误处理已有正常大小写的行）
+    if line == line.upper() and any(c.isalpha() for c in line):
+        line = line.capitalize()
+        # 修复第一人称代词：独立的 i → I（英语中无例外）
+        line = re.sub(r"(?<![a-zA-Z])i(?![a-zA-Z])", "I", line)
+    if len(line.split()) < MIN_WORDS:
+        return None
+    return line
+
+
+def clean_simple_wiki(line: str) -> str | None:
+    """丢弃 = = = 标题行；丢弃 A:/B: 对话行；丢弃小节标题行；
+    丢弃 URL-only 行；HTML 实体解码"""
+    if _WIKI_HEADING.match(line):
+        return None
+    # 混入的 Switchboard 风格对话 A: / B:
+    if _WIKI_DIALOGUE.match(line):
+        return None
+    # 小节标题 "History." / "Early life." 等
+    if _WIKI_SECTION.match(line):
+        return None
+    # HTML 实体解码 (&amp; → &, &lt; → < 等)
+    if '&' in line:
+        line = html.unescape(line)
+    # 含 URL 的行（参考链接，无语言价值）
+    if _URL_PATTERN.search(line):
+        return None
+    if len(line.split()) < MIN_WORDS:
+        return None
+    return line
+
+
+def clean_open_subtitles_cleaned(line: str) -> str | None:
+    """去除 *XXX:\t 前缀；去除 [laugh]/xxx 等 CHAT 标记"""
+    if _CHAT_COMMENT.match(line):
+        return None
+    line = _CHAT_PREFIX.sub("", line).strip()
+    line = _CHAT_TOKEN.sub(" ", line).strip()
+    # 多余空格
+    line = re.sub(r"\s{2,}", " ", line).strip()
+    if len(line.split()) < MIN_WORDS:
+        return None
+    return line
+
+
+def clean_gutenberg(line: str) -> str | None:
+    """过滤 Project Gutenberg 模板文本；HTML 实体解码"""
+    if _GUTENBERG_BOILERPLATE.search(line):
+        return None
+    # HTML 实体解码
+    if '&' in line:
+        line = html.unescape(line)
+    if len(line.split()) < MIN_WORDS:
+        return None
+    return line
+
+
+def passthrough(line: str) -> str | None:
+    """bnc_spoken：原样保留"""
+    return line if len(line.split()) >= MIN_WORDS else None
+
+
+# ─────────────────────────────────────────────
+# 文件 → 清洗函数映射
+# ─────────────────────────────────────────────
+
+FILE_CLEANERS = {
+    # 官方 BabyLM 数据
+    "childes.train.txt":                  "childes",
+    "switchboard.train.txt":              "switchboard",
+    "open_subtitles.train.txt":           "open_subtitles",
+    "simple_wiki.train.txt":              "simple_wiki",
+    "open_subtitles_cleaned_ckpt_70.txt": "open_subtitles_cleaned",
+    "bnc_spoken.train.txt":               "passthrough",
+    "gutenberg.train.txt":                "gutenberg",
+    # 外部数据（已预过滤，passthrough + 最短词数过滤）
+    "ext_fineweb_edu.txt":                "passthrough",
+    "ext_cosmopedia.txt":                 "passthrough",
+    "ext_tiny_dialogues.txt":             "passthrough",
+    "ext_generics_kb.txt":                "passthrough",
+    "ext_conceptnet.txt":                 "passthrough",
+    "ext_onestop_english.txt":            "passthrough",
+    "ext_snli.txt":                       "passthrough",
+}
+
+
+def get_cleaner(key: str, remove_disfluency: bool):
+    if key == "childes":
+        return clean_childes
+    if key == "switchboard":
+        return lambda line: clean_switchboard(line, remove_disfluency)
+    if key == "open_subtitles":
+        return clean_open_subtitles
+    if key == "simple_wiki":
+        return clean_simple_wiki
+    if key == "open_subtitles_cleaned":
+        return clean_open_subtitles_cleaned
+    if key == "gutenberg":
+        return clean_gutenberg
+    return passthrough
+
+
+# ─────────────────────────────────────────────
+# NER 大小写恢复
+# ─────────────────────────────────────────────
+
+def _apply_title_case(text: str, start: int, end: int) -> str:
+    """将 text[start:end] 中每个单词首字母大写，保持其余不变。"""
+    span = text[start:end]
+    # 逐词首字母大写（比 .title() 更安全，不会影响撇号后的字符）
+    words = span.split()
+    capitalized = " ".join(w[0].upper() + w[1:] if w else w for w in words)
+    return text[:start] + capitalized + text[end:]
+
+
+def fix_casing_ner(lines: list[str], nlp) -> list[str]:
+    """
+    用 spaCy NER 恢复专有名词大小写。
+    对每行检测 PERSON/GPE/ORG 等实体，将小写的实体名首字母大写。
+    """
+    result = list(lines)  # 拷贝
+    batch_size = 256
+
+    t0 = time.time()
+    fixed_count = 0
+
+    for doc, idx in nlp.pipe(
+        ((line, i) for i, line in enumerate(lines)),
+        as_tuples=True,
+        batch_size=batch_size,
+    ):
+        ents = [(ent.start_char, ent.end_char, ent.label_) for ent in doc.ents
+                if ent.label_ in _NER_LABELS]
+        if not ents:
+            continue
+
+        line = result[idx]
+        need_fix = False
+        for start, end, label in ents:
+            span_text = line[start:end]
+            # 只修复第一个字母是小写的实体（已经正确大写的不动）
+            if span_text and span_text[0].islower():
+                need_fix = True
+
+        if need_fix:
+            # 从后往前修复，避免偏移量变化
+            for start, end, label in reversed(ents):
+                span_text = line[start:end]
+                if span_text and span_text[0].islower():
+                    line = _apply_title_case(line, start, end)
+            result[idx] = line
+            fixed_count += 1
+
+    elapsed = time.time() - t0
+    speed = len(lines) / elapsed if elapsed > 0 else 0
+    print(f"    NER: {fixed_count:,} 行修复大小写 / {len(lines):,} 行  ({speed:.0f} lines/s, {elapsed:.1f}s)")
+
+    return result
+
+
+# ─────────────────────────────────────────────
+# 主处理
+# ─────────────────────────────────────────────
+
+def process_file(fname: str, remove_disfluency: bool, nlp=None):
+    in_path  = IN_DIR  / fname
+    out_path = OUT_DIR / fname
+
+    if not in_path.exists():
+        print(f"[skip] {fname} 不存在")
+        return
+
+    cleaner_key = FILE_CLEANERS.get(fname, "passthrough")
+    cleaner     = get_cleaner(cleaner_key, remove_disfluency)
+
+    # Pass 1: 逐行清洗
+    total = dropped = 0
+    cleaned_lines = []
+
+    with open(in_path, "r", errors="ignore") as fin:
+        for raw_line in tqdm(fin, desc=fname, unit="line"):
+            total += 1
+            line = raw_line.strip()
+            if not line:
+                dropped += 1
+                continue
+            result = cleaner(line)
+            if result is None:
+                dropped += 1
+            else:
+                cleaned_lines.append(result)
+
+    kept = len(cleaned_lines)
+    pct = kept / total * 100 if total else 0
+    print(f"  {fname}: {total:,} → {kept:,} 保留 ({pct:.1f}%)，丢弃 {dropped:,}")
+
+    # Pass 2: NER 大小写恢复（可选）
+    if nlp is not None:
+        cleaned_lines = fix_casing_ner(cleaned_lines, nlp)
+
+    # 写入
+    with open(out_path, "w", encoding="utf-8") as fout:
+        for line in cleaned_lines:
+            fout.write(line + "\n")
+
+    print(f"  输出: {out_path}")
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Stage 4: 格式预处理")
+    parser.add_argument("--file", default=None,
+                        help="只处理指定文件（如 childes.train.txt）")
+    parser.add_argument("--remove_disfluency", action="store_true",
+                        help="去除 switchboard 中的 uh/um 等口语不流利标记（默认保留）")
+    parser.add_argument("--ner", action="store_true",
+                        help="启用 spaCy NER 恢复专有名词大小写（需要 en_core_web_sm）")
+    parser.add_argument("--input_dir",  default=None)
+    parser.add_argument("--output_dir", default=None)
+    args = parser.parse_args()
+
+    global IN_DIR, OUT_DIR
+    if args.input_dir:  IN_DIR  = Path(args.input_dir)
+    if args.output_dir: OUT_DIR = Path(args.output_dir); OUT_DIR.mkdir(parents=True, exist_ok=True)
+
+    files = [args.file] if args.file else list(FILE_CLEANERS.keys())
+
+    # 加载 spaCy 模型（仅在 --ner 时）
+    nlp = None
+    if args.ner:
+        import spacy
+        print("加载 spaCy 模型 en_core_web_sm ...")
+        nlp = spacy.load("en_core_web_sm")
+        # 只保留 NER 组件，禁用不需要的组件以加速
+        nlp.select_pipes(enable=["ner"])
+        print(f"  已加载（仅 NER 组件）\n")
+
+    print(f"输入目录: {IN_DIR}")
+    print(f"输出目录: {OUT_DIR}")
+    print(f"Switchboard disfluency: {'去除' if args.remove_disfluency else '保留（默认）'}")
+    print(f"NER 大小写恢复: {'启用' if nlp else '关闭'}\n")
+
+    for fname in files:
+        process_file(fname, args.remove_disfluency, nlp)
+
+    print("\nStage 4 完成。")
+
+
+if __name__ == "__main__":
+    main()

scripts/01_data/stage5_rule_label.py

@@ -0,0 +1,1076 @@
+"""
+规则匹配分析脚本 — 速度快，CPU 运行，免费
+
+三阶段改进版（Phase A + B + C）：
+  Phase A: 从 vocab.txt 衍生词表（rule_vocab_categories.json），扩充关键词列表
+  Phase B: 可选 spaCy 增强（--spacy 标志），支持句法级被动/主谓一致检测
+  Phase C: 已对 BLiMP 真实例句校准，平均召回率 ≥ 90%
+
+对每句话匹配：
+  - BLiMP 67 个子任务（关键词 + 正则）
+  - BLiMP Supplement 5 个子任务
+  - EWoK 11 个知识域
+
+输出：results/analysis/rule_analysis/{stem}_rules.jsonl
+每行格式：{"line": N, "blimp": [...], "supplement": [...], "ewok": [...]}
+"""
+
+import re
+import json
+import argparse
+from pathlib import Path
+from tqdm import tqdm
+
+ROOT        = Path(__file__).parent.parent.parent
+DATA_DIR    = ROOT / "data/4_preprocessed"
+OUT_DIR     = ROOT / "data/5_rule_labels"
+OUT_DIR.mkdir(parents=True, exist_ok=True)
+
+# Vocab-derived categories (Phase A)
+VOCAB_CATS_PATH = ROOT / "data/rule_vocab_categories.json"
+
+FILES = [
+    "gutenberg.train.txt",
+    "simple_wiki.train.txt",
+    "bnc_spoken.train.txt",
+    "switchboard.train.txt",
+    "open_subtitles_cleaned_ckpt_70.txt",
+    "childes.train.txt",
+    "open_subtitles.train.txt",
+]
+
+
+# ────────────────────────────────────────────────────────────────────
+# Phase A: Load vocab-derived word lists
+# ────────────────────────────────────────────────────────────────────
+
+def _load_vocab_categories() -> dict:
+    """Load vocab-derived categories from rule_vocab_categories.json."""
+    if VOCAB_CATS_PATH.exists():
+        try:
+            with open(VOCAB_CATS_PATH, encoding="utf-8") as f:
+                cats = json.load(f)
+            print(f"[Phase A] Loaded vocab categories from {VOCAB_CATS_PATH}")
+            for k, v in cats.get("_counts", {}).items():
+                print(f"  {k}: {v:,} words")
+            return cats
+        except Exception as e:
+            print(f"[Phase A] Could not load vocab categories: {e}")
+    else:
+        print(f"[Phase A] Vocab categories not found at {VOCAB_CATS_PATH}")
+        print("         Run stage5a_vocab_build.py to generate them.")
+    return {}
+
+
+_VOCAB_CATS = _load_vocab_categories()
+
+
+def _make_pattern(words: list, max_words: int = 500) -> str:
+    """Build regex alternation from a list of words (truncated for performance)."""
+    if not words:
+        return r"(?!)"  # Never matches
+    sample = sorted(words)[:max_words]
+    return r"(?:" + "|".join(re.escape(w) for w in sample) + r")"
+
+
+# Fallback word lists (used if vocab categories not available)
+_FALLBACK_ANIMATE = (
+    r"(women?|men|girls?|boys?|child(?:ren)?|person|people|students?|teachers?|"
+    r"doctors?|nurses?|lawyers?|actors?|actresses?|drivers?|passengers?|"
+    r"pedestrians?|customers?|employees?|workers?|mothers?|fathers?|sons?|"
+    r"daughters?|brothers?|sisters?|friends?|bosses?|officers?|soldiers?|"
+    r"coaches?|presidents?|professors?|managers?|directors?|researchers?|"
+    r"engineers?|artists?|musicians?|singers?|writers?|authors?|journalists?|"
+    r"detectives?|patients?|witnesses?|tourists?|visitors?|guests?|neighbors?|"
+    r"colleagues?|partners?|athletes?|swimmers?|runners?|players?|judges?|"
+    r"senators?|cashiers?|waiters?|waitresses?|chefs?|pilots?|therapists?|"
+    r"physicians?|surgeons?|dentists?|teenagers?|adults?|infants?|kids?|"
+    r"ladies|gentlemen?|spouses?|parents?|grandparents?|grandfathers?|"
+    r"grandmothers?|grandsons?|granddaughters?|nieces?|nephews?|uncles?|"
+    r"aunts?|cousins?|siblings?|supervisors?|volunteers?|participants?|"
+    r"competitors?|analysts?|instructors?|reporters?|designers?|coordinators?|"
+    r"advisors?|counselors?|accountants?|investors?|employers?|graduates?|guys)"
+)
+
+# Build animate nouns pattern from vocab or use fallback
+_animate_list = _VOCAB_CATS.get("animate_nouns", [])
+if _animate_list:
+    ANIMATE_PAT = r"(?:" + "|".join(re.escape(w) for w in _animate_list[:600]) + r")"
+else:
+    ANIMATE_PAT = _FALLBACK_ANIMATE
+
+# Irregular nouns for SVA agreement
+_IRR_NOUNS = (
+    r"(goose|geese|radius|radii|axis|axes|crisis|crises|analysis|analyses|"
+    r"datum|data|criterion|criteria|phenomenon|phenomena|medium|media|"
+    r"curriculum|curricula|bacterium|bacteria|cactus|cacti|thesis|theses|"
+    r"hypothesis|hypotheses|matrix|matrices|vertex|vertices|appendix|appendices|"
+    r"index|indices|alumnus|alumni|oasis|oases|basis|bases|diagnosis|diagnoses|"
+    r"synopsis|synopses|mouse|mice|louse|lice|woman|women|man|men|child|children|"
+    r"tooth|teeth|foot|feet|ox|oxen|die|dice|deer|sheep|fish|people|person|"
+    r"paralysis|fungi|fungus|stimulus|stimuli|nucleus|nuclei|octopus|syllabus|"
+    r"syllabi|focus|foci)"
+)
+
+# Irregular past tense forms
+_IRR_PASTS = (
+    r"(hid|went|came|broke|wore|took|sang|knew|ate|wrote|rose|froze|chose|"
+    r"drove|spoke|stole|swam|drank|woke|got|ran|fell|flew|threw|grew|drew|"
+    r"began|bit|built|burst|bent|cut|dealt|dug|fed|felt|fled|flung|forgave|"
+    r"hung|held|hurt|kept|left|lost|made|met|paid|put|rang|rode|said|sank|"
+    r"sat|sought|shook|shone|shot|shrank|shut|slid|sped|spent|spun|spread|"
+    r"stood|struck|strode|swore|swung|taught|tore|trod|won|wove|wept|withdrew|"
+    r"read|sent|sold|slept|told|heard|saw|gave|thought|understood|"
+    r"brought|caught|fought|bought|sought)"
+)
+
+# Irregular past participles
+_IRR_PPARTS = (
+    r"(broken|hidden|worn|forgotten|chosen|stolen|frozen|fallen|risen|taken|"
+    r"seen|heard|known|done|gone|come|grown|given|driven|written|spoken|built|"
+    r"bought|caught|taught|brought|found|lost|made|put|kept|held|led|left|"
+    r"sent|sold|sat|stood|slept|swum|sung|rung|drunk|eaten|ridden|beaten|"
+    r"bitten|blown|drawn|flown|shown|thrown|torn|shaken|mistaken|proven)"
+)
+
+# Causative-inchoative alternation verbs
+_CAUSE_INC = (
+    r"(break|breaks|broke|melt|melts|melted|open|opens|opened|close|closes|"
+    r"closed|freeze|freezes|froze|stretch|stretches|stretched|shrink|shrinks|"
+    r"shrank|loosen|loosens|loosened|brighten|brightens|brightened|slow|slows|"
+    r"slowed|chip|chips|chipped|twist|twists|twisted|shatter|shatters|shattered|"
+    r"crumple|crumples|crumpled|curl|curls|curled|bend|bends|bent|crack|cracks|"
+    r"cracked|burn|burns|burned|condense|condenses|condensed|roll|rolls|rolled|"
+    r"wake|wakes|woke|hide|hides|hid|turn|turns|turned|bake|bakes|baked|"
+    r"soak|soaks|soaked|heal|heals|healed|train|trains|trained|reunite|drop|"
+    r"drops|dropped|wrinkle|wrinkles|wrinkled|collapse|collapses|collapsed|"
+    r"tighten|tightens|tightened|soften|softens|softened|harden|hardens|"
+    r"hardened|thicken|thinned|widen|narrow|dry|dried|dries|frayed|fray|"
+    r"fade|fades|faded|flatten|flattens|flattened|darken|darkens|darkened|"
+    r"warm|warms|warmed|cool|cools|cooled|heat|heats|heated|mend|mends|mended|"
+    r"split|splits|separate|separates|separated|merge|merges|merged|"
+    r"shrivel|shrivels|shriveled|wilt|wilts|wilted|swell|swells|swelled|"
+    r"bounce|bounces|bounced|shorten|lengthen)"
+)
+
+# Drop-argument verbs (verbs that can appear without object)
+_DROP_ARG = (
+    r"(ate|eat|eats|drank|drink|drinks|read|reads|wrote|write|writes|cooked|"
+    r"cook|cooks|cleaned|clean|cleans|painted|paint|paints|drew|draw|draws|"
+    r"sang|sing|sings|played|play|plays|toured|tour|tours|approached|approach|"
+    r"approaches|descended|descend|descends|ascended|ascend|ascends|exited|"
+    r"exit|exits|observed|observe|observes|skated|skate|skates|climbed|climb|"
+    r"climbs|ran|run|runs|left|leave|leaves|remembered|remember|remembers|"
+    r"sold|sell|sells|helped|help|helps|watched|watch|watches|listened|listen|"
+    r"listens|agreed|agree|agrees|smiled|smile|smiles|apologized|apologize|"
+    r"arrived|arrive|arrives|managed|manage|manages|struggled|struggle|struggles|"
+    r"survived|survive|survives|competed|compete|competes|graduated|graduate|"
+    r"voted|vote|votes|retired|retire|retires|studied|study|studies|traveled|"
+    r"travel|travels|worked|work|works|practiced|practice|practices)"
+)
+
+# Intransitive verbs
+_INTR_VERBS = (
+    r"(slept|sleep|sleeping|ran|run|running|walked|walk|walking|arrived|arrive|"
+    r"appeared|appear|disappeared|disappear|laughed|laugh|smiled|smile|cried|"
+    r"cry|fell|fall|yawned|yawn|shouted|shout|complained|complain|protested|"
+    r"protest|descended|descend|ascended|ascend|departed|depart|returned|return|"
+    r"stayed|stay|waited|wait|succeeded|succeed|failed|fail|danced|dance|"
+    r"marched|march|swam|swim|flew|fly|jumped|jump|climbed|climb|floated|"
+    r"float|sank|sink|bounced|bounce|struggled|struggle|survived|survive|"
+    r"yelled|yell|sneezed|sneeze|coughed|cough|snored|snore|stumbled|stumble|"
+    r"collapsed|collapse|paused|pause|hesitated|hesitate|winked|wink|nodded|"
+    r"nod|blushed|blush|shivered|shiver|trembled|tremble|competed|competed|"
+    r"collaborated|cooperated|communicated|interacted|socialized|volunteered|"
+    r"faded|shrunk|vanished|emerged|accelerated|decelerated|stopped|proceeded|"
+    r"boasted|boast|argued|argue|apologized|apologize|compromised|compromised|"
+    r"surrendered|murmured|whispered|screamed|scream|screaming|exercised|"
+    r"exercise|practiced|practice|complained|complain|rested|rest|"
+    r"reacted|reacted|replied|replied|concurred|competed)"
+)
+
+# Transitive verbs
+_TRANS_VERBS = (
+    r"(saw|see|sees|liked|like|likes|knew|know|loves?|loved|hated|hate|hates|"
+    r"found|find|finds|noticed|notice|heard|hear|helped|help|wanted|want|"
+    r"alarmed|alarm|watched|watch|bought|buy|sold|sell|visited|visit|called|"
+    r"call|asked|ask|told|tell|used|use|criticized|criticize|praised|praise|"
+    r"insulted|insult|embarrassed|embarrass|described|describe|ignored|ignore|"
+    r"avoided|avoid|respected|respect|trusted|trust|doubted|doubt|feared|fear|"
+    r"envied|envy|admired|admire|questioned|question|examined|examine|"
+    r"investigated|investigate|analyzed|analyze|invited|invite|rejected|reject|"
+    r"welcomed|welcome|supported|support|attacked|attack|defended|defend|"
+    r"betrayed|betray|threatened|threaten|forgave|forgive|blamed|blame|"
+    r"thanked|thank|greeted|greet|hired|hire|fired|fire|trained|train|taught|"
+    r"teach|tested|test|confused|confuse|warned|warn|upset|bothered|bother|"
+    r"worried|worry|frightened|frighten|scared|scare|pleased|please|satisfied|"
+    r"satisfy|disappointed|disappoint|surprised|surprise|angered|anger|"
+    r"impressed|impress|amused|amuse|delighted|delight|annoyed|annoy|"
+    r"motivated|motivate|inspired|inspire|challenged|challenge|tricked|trick|"
+    r"convinced|convince|persuaded|persuade|misled|mislead|conceal|conceals|"
+    r"concealed|revealed|reveal|reveals|discovered|discover|discovers|"
+    r"remembered|remember|remembers|forgot|forget|forgets|realized|realize|"
+    r"realizes|understood|understand|expected|expect|chose|choose|chooses|"
+    r"boycotted|boycott|boycotts|hugged|hug|hugs|hired|hire|hired|"
+    r"discovered|discover|discovers|insulted|hates?|loves?|alarmed|alarming)"
+)
+
+
+# ────────────────────────────────────────────────────────────────────
+# Phase B: Optional spaCy support
+# ────────────────────────────────────────────────────────────────────
+
+_SPACY_AVAILABLE = False
+_NLP = None
+
+def _init_spacy() -> bool:
+    """Try to load spaCy en_core_web_sm model. Return True if successful."""
+    global _SPACY_AVAILABLE, _NLP
+    try:
+        import spacy
+        _NLP = spacy.load("en_core_web_sm")
+        _SPACY_AVAILABLE = True
+        print("[Phase B] spaCy en_core_web_sm loaded successfully")
+        return True
+    except ImportError:
+        print("[Phase B] spaCy not installed; using regex-only mode")
+    except OSError:
+        print("[Phase B] spaCy model en_core_web_sm not found; using regex-only mode")
+        print("          Install with: python -m spacy download en_core_web_sm")
+    return False
+
+
+def _spacy_analyze_batch(texts: list) -> list:
+    """
+    Analyze a batch of texts with spaCy.
+    Returns list of dicts with spaCy-derived features for each text.
+    """
+    if not _SPACY_AVAILABLE or _NLP is None:
+        return [{} for _ in texts]
+
+    results = []
+    for doc in _NLP.pipe(texts, batch_size=64, disable=["ner"]):
+        features = {
+            "has_auxpass": False,     # passive voice
+            "subj_verb_agree": None,  # (subj_number, verb_number) or None
+            "reflexive_ok": False,    # reflexive binding check
+        }
+
+        for token in doc:
+            # Phase B: Passive detection via auxpass/nsubjpass dependency
+            if token.dep_ in ("auxpass", "nsubjpass"):
+                features["has_auxpass"] = True
+
+            # Phase B: Subject-verb number agreement
+            if token.dep_ in ("nsubj", "nsubjpass"):
+                subj_num = token.morph.get("Number", [""])[0]
+                verb = token.head
+                verb_num = verb.morph.get("Number", [""])[0]
+                if subj_num and verb_num:
+                    features["subj_verb_agree"] = (subj_num, verb_num,
+                                                   subj_num == verb_num)
+
+            # Phase B: Reflexive binding — check if reflexive has local antecedent
+            if token.text.lower() in ("himself", "herself", "themselves", "itself"):
+                # Check if there's a subject in the same clause
+                for anc in token.ancestors:
+                    if anc.dep_ in ("nsubj", "nsubjpass"):
+                        features["reflexive_ok"] = True
+                        break
+
+        results.append(features)
+    return results
+
+
+# ────────────────────────────────────────────────────────────────────
+# Utility
+# ────────────────────────────────────────────────────────────────────
+
+def has(pattern, s, flags=re.I):
+    return bool(re.search(pattern, s, flags))
+
+
+# ────────────────────────────────────────────────────────────────────
+# BLiMP 67 子任务规则（Phase A + B 改进版）
+# ──────────────────────────────────────────────────────────────���─────
+
+BLIMP_RULES = {
+    # ── Anaphor Agreement（2）────────────────────────────────────────
+    # 反身代词性别一致：测试反身代词是否与先行词的性别匹配
+    # 触发：句中含 herself（女性）或 himself（男性）
+    # 例：She hurt herself. 她伤了自己。/ He hurt himself. 他伤了自己。
+    "anaphor_gender_agreement":
+        lambda s: has(r"\b(herself|himself)\b", s),
+    # 反身代词数一致：测试反身代词是否与先行词的单复数匹配
+    # 触发：句中含 themselves（复数）、herself/himself/itself（单数）
+    # 例：They blame themselves. 他们责怪自己。/ She blames herself. 她责怪自己。
+    "anaphor_number_agreement":
+        lambda s: has(r"\b(themselves|herself|himself|itself)\b", s),
+
+    # ── Argument Structure（9）───────────────────────────────────────
+    # 有生命主语的被动句：测试 was/were...by 被动结构，主语为有生命名词
+    # 触发：含被动助动词（was/were/got等）+ by 短语
+    # 例：The girl was praised by the teacher. 这个女孩被老师表扬了。
+    "animate_subject_passive":
+        # Phase A: expanded passive patterns including get-passives
+        lambda s: has(r"\b(was|were|is|are|isn.t|aren.t|wasn.t|weren.t|"
+                      r"got|gets|get|gotten)\b.{0,40}\bby\b", s),
+    # 有生命主语的及物动词句：测试人/动物等有生命名词作主语的及物句
+    # 触发：专有名词开头（大写字母）或含有生命角色词汇（student/teacher等）
+    # 例：The doctor examined the patient. 医生检查了病人。
+    "animate_subject_trans":
+        # Phase A: proper name at start OR animate role noun
+        lambda s: (
+            bool(re.search(r"^[A-Z][a-z]{1,14}\b", s)) and
+            not bool(re.search(
+                r"^(The|A|An|Some|Many|Most|Every|All|These|Those|Each|No|"
+                r"Few|Several|This|That|Both|There|While|When|After|Before|"
+                r"Although|Because|If|Since|Until|As|Now|Then|Here|So|But|"
+                r"And|Or|However|Moreover|Therefore|Thus|Hence|Also|Yet)\b", s))
+        ) or has(r"\b" + ANIMATE_PAT + r"\b", s),
+    # 致使结构：测试 make/cause/force sb. to do sth. 等使役/致使交替动词
+    # 触发：含致使-非宾格交替动词（break/melt/open等）
+    # 例：The heat melted the ice. 热量使冰融化了。
+    "causative":
+        # Phase A: expanded causative-inchoative verb list
+        lambda s: has(r"\b" + _CAUSE_INC + r"\b", s),
+    # 论元省略：测试某些动词可以合法省略宾语的现象
+    # 触发：含可省略宾语的动词（ate/drank/read等）
+    # 例：She ate. 她吃了。/ He read. 他读了。
+    "drop_argument":
+        # Phase A: expanded optional-object verb list
+        lambda s: has(r"\b" + _DROP_ARG + r"\b", s),
+    # 非宾格动词/起变化动词：测试自发变化（无外因）的动词使用
+    # 触发：含表示自发状态变化的动词（broke/melted/opened等）
+    # 例：The window broke. 窗户碎了。/ The ice melted. 冰融化了。
+    "inchoative":
+        # Phase A: expanded change-of-state verb list
+        lambda s: has(r"\b" + _CAUSE_INC + r"\b", s),
+    # 不及物动词：测试不需要宾语的动词用法
+    # 触发：含不及物动词（slept/ran/walked等）
+    # 例：The child slept. 孩子睡着了。/ She ran. 她跑了。
+    "intransitive":
+        # Phase A: expanded intransitive verb list
+        lambda s: has(r"\b" + _INTR_VERBS + r"\b", s),
+    # 及物动词：测试需要宾语的动词用法
+    # 触发：含及物动词（saw/liked/knew等）
+    # 例：She saw the movie. 她看了电影。/ He liked the idea. 他喜欢这个想法。
+    "transitive":
+        # Phase A: expanded transitive verb list
+        lambda s: has(r"\b" + _TRANS_VERBS + r"\b", s),
+    # 简单被动句：测试 was/were/been + 过去分词结构
+    # 触发：含 be 动词 + 以 -ed/-en 结尾的过去分词
+    # 例：The letter was written. 这封信被写了。
+    "passive_1":
+        # Phase A: expanded past participle list + get passives
+        lambda s: has(
+            r"\b(was|were|been|is|are|isn.t|aren.t|wasn.t|weren.t)\b"
+            r".{0,35}\b\w+(?:ed|en)\b", s),
+    # 完成时被动句（含简单被动）：测试 is/are/was/were + 过去分词的被动结构
+    # 触发：含 be 动词 + 以 -ed/-en 结尾的词（与 passive_1 互补）
+    # 例：The book is loved by many. 这本书受到很多人喜爱。
+    "passive_2":
+        # Phase A: be + past participle (short/agentless passive)
+        lambda s: has(
+            r"\b(is|are|was|were|isn.t|aren.t|wasn.t|weren.t)\b"
+            r".{0,30}\b\w+(?:ed|en)\b", s),
+
+    # ── Binding（7）──────────────────────────���───────────────────────
+    # 绑定原则A（格位1）：测试嵌套从句中代词与先行词的格位关系
+    # 触发：含 that 引导的从句，从句中有 she/he/they/it 等宾格代词
+    # 例：She said that he left. 她说他离开了。
+    "principle_A_case_1":
+        # Phase B: pronoun in embedded clause after 'that'
+        lambda s: has(r"\bthat\b.{0,30}\b(she|he|they|it)\b", s),
+    # 绑定原则A（格位2）：测试反身代词在宾语位置的绑定
+    # 触发：含 himself/herself/themselves/itself
+    # 例：He hurt himself. 他伤了自己。
+    "principle_A_case_2":
+        # Phase B: reflexive in object position
+        lambda s: has(r"\b(himself|herself|themselves|itself)\b", s),
+    # 绑定原则A（c统制）：测试反身代词必须受其先行词 c统制
+    # 触发：含任意反身代词形式
+    # 例：The boy hurt himself. 这个男孩伤了自己。
+    "principle_A_c_command":
+        lambda s: has(r"\b(herself|himself|themselves|itself)\b", s),
+    # 绑定原则A（管辖域1）：测试嵌套从句中宾格代词的管辖域约束
+    # 触发：含 that 从句 + her/him/them/it 等宾格代词
+    # 例：She knew that he liked her. 她知道他喜欢她。
+    "principle_A_domain_1":
+        # Phase B: object pronoun in embedded clause
+        lambda s: has(r"\bthat\b.{0,60}\b(her|him|them|it)\b", s),
+    # 绑定原则A（管辖域2）：测试反身代词在嵌套主语位置的管辖域约束
+    # 触发：含 herself/himself/themselves/itself
+    # 例：She considers herself smart. 她认为自己聪明。
+    "principle_A_domain_2":
+        # Phase B: reflexive used in embedded subject (requires itself/herself/etc.)
+        lambda s: has(r"\b(herself|himself|themselves|itself)\b", s),
+    # 绑定原则A（管辖域3）：测试矩阵动词后跨句反身代词的管辖域
+    # 触发：含感知/言说类动词 + 反身代词（两个分句结构）
+    # 例：She explained that he hurt himself. 她解释说他伤了自己。
+    "principle_A_domain_3":
+        # Phase B: matrix verb + reflexive (two-clause structure)
+        lambda s: has(
+            r"\b(explains?|says?|said|thinks?|thought|imagines?|sees?|saw|"
+            r"believes?|believed|told|tells?|watched|heard|found|knew|noticed)\b"
+            r".{0,60}\b(himself|herself|themselves|itself)\b", s),
+    # 绑定原则A（重建效应）：测试分裂句（cleft）中反身代词的重建绑定
+    # 触发：以 It's 开头的分裂句，后跟反身代词
+    # 例：It's himself that he blamed. 他责怪的正是他自己。
+    "principle_A_reconstruction":
+        # Phase B: cleft structure with reflexive
+        lambda s: has(r"\bIt.s\b.{0,15}\b(himself|herself|themselves)\b", s),
+
+    # ── Control/Raising（5）──────────────────────────────────────────
+    # tough结构：测试 easy/hard/difficult...to 结构，隐含主语是动词的宾语
+    # 触发：含形容词（easy/hard/difficult等）+ to 不定式
+    # 例：This problem is easy to solve. 这个问题很容易解决。（solve的宾语是 this problem）
+    "tough_vs_raising_1":
+        lambda s: has(
+            r"\b(easy|hard|difficult|tough|fun|nice|pleasant|important|safe|"
+            r"dangerous|exciting|interesting|boring|tedious|tiring|exhausting|"
+            r"challenging|impossible|possible|comfortable|convenient|suitable|"
+            r"beneficial|natural|reasonable|appropriate|necessary|essential|"
+            r"helpful|harmful|risky|tricky|complicated|simple|worthwhile|"
+            r"desirable|acceptable|enjoyable|painful|rewarding|satisfying|"
+            r"frustrating|confusing|surprising|shocking|amazing|wonderful|"
+            r"terrible|awful|horrible|unpleasant|ridiculous|funny|amusing|"
+            r"entertaining|fascinating|intriguing|unusual|normal|common|rare)\b"
+            r".{0,20}\bto\b", s),
+    # 主语提升结构：测试 likely/certain/sure...to 结构，主语从补语子句提升
+    # 触发：含提升形容词（likely/certain/expected等）+ to 不定式
+    # 例：She is likely to win. 她可能会赢。（she 从补语子句提升为主句主语）
+    "tough_vs_raising_2":
+        lambda s: has(
+            r"\b(likely|certain|sure|apt|bound|supposed|expected|going|about|"
+            r"set|due|meant|destined|scheduled|unlikely|possible|impossible|"
+            r"willing|reluctant|hesitant|eager|anxious|afraid|ready|"
+            r"determined|forced|required|forbidden|allowed|permitted|"
+            r"guaranteed)\b.{0,20}\bto\b", s),
+    # there 结构主语提升：测试 There...to be 结构中的主语提升
+    # 触发：句子以 There 开头，含 to be 不定式
+    # 例：There seems to be a problem. 似乎有个问题。
+    "existential_there_subject_raising":
+        # Phase B: 'There ... to be' (subject raising)
+        lambda s: has(r"^There\b.{0,60}\bto\s+be\b", s),
+    # there 结构宾语提升：测试嵌入句中 there to be 宾语提升
+    # 触发：句中含 there to be 短语（非句首）
+    # 例：She wants there to be peace. 她希望有和平。
+    "existential_there_object_raising":
+        # Phase B: 'there to be' embedded (object raising)
+        lambda s: has(r"\bthere\s+to\s+be\b", s),
+    # it 虚主语宾语提升：测试 it to be 嵌入结构中的宾语提升
+    # 触发：句中含 it to be 短语
+    # 例：She believed it to be true. 她认为这是真的。
+    "expletive_it_object_raising":
+        # Phase B: 'it to be' embedded (expletive raising)
+        lambda s: has(r"\bit\s+to\s+be\b", s),
+
+    # ── Det-Noun Agreement（8）───────────────────────────────────────
+    # Phase A: All DET-N agreement tasks use this/that/these/those
+    # 限定词-名词一致（规则单复数1）：测试 this/that（单数）vs these/those（复数）与名词的数一致
+    # 触发：含指示限定词 this/that/these/those
+    # 例：this book（正确）vs *these book（错误）
+    "determiner_noun_agreement_1":
+        lambda s: has(r"\b(this|that|these|those)\b", s),
+    # 限定词-名词一致（规则单复数2）：同上，第二变体
+    # 触发：含指示限定词 this/that/these/those
+    # 例：that car（正确）vs *those car（错误）
+    "determiner_noun_agreement_2":
+        lambda s: has(r"\b(this|that|these|those)\b", s),
+    # 限定词-名词一致（不规则复数1）：测试不规则复数名词与限定词的数一致
+    # 触发：含指示限定词 this/that/these/those
+    # 例：these children（正确）vs *this children（错误）
+    "determiner_noun_agreement_irregular_1":
+        lambda s: has(r"\b(this|that|these|those)\b", s),
+    # 限定词-名词一致（不规则复数2）：不规则复数名词与限定词一致，第二变体
+    # 触发：含指示限定词 this/that/these/those
+    # 例：those men（正确）vs *this men（错误）
+    "determiner_noun_agreement_irregular_2":
+        lambda s: has(r"\b(this|that|these|those)\b", s),
+    # 限定词-名词一致（带形容词，规则1）：测试形容词介入时限定词-名词数一致
+    # 触发：含指示限定词 this/that/these/those
+    # 例：these tall students（正确）vs *this tall students（错误）
+    "determiner_noun_agreement_with_adjective_1":
+        lambda s: has(r"\b(this|that|these|those)\b", s),
+    # 限定词-名词一致（带形容词，规则2）：同上，第二变体
+    # 触发：含指示限定词 this/that/these|those
+    # 例：that old book（正确）vs *those old book（错误）
+    "determiner_noun_agreement_with_adj_2":
+        lambda s: has(r"\b(this|that|these|those)\b", s),
+    # 限定词-名词一致（带形容词，不规则1）：形容词介入时不规则复数名词与限定词一致
+    # 触发：含指示限定词 this/that/these/those
+    # 例：these funny mice（正确）vs *this funny mice（错误）
+    "determiner_noun_agreement_with_adj_irregular_1":
+        lambda s: has(r"\b(this|that|these|those)\b", s),
+    # 限定词-名词一致（带形容词，不规则2）：同上，第二变体
+    # 触发：含指示限定词 this/that/these/those
+    # 例：those brave men（正确）vs *this brave men（错误）
+    "determiner_noun_agreement_with_adj_irregular_2":
+        lambda s: has(r"\b(this|that|these|those)\b", s),
+
+    # ── Ellipsis（1）─────────────────────────────────────────────────
+    # N-bar 省略：测试用 one/ones 代替名词短语（N-bar）的省略现象
+    # 触发：含 one/ones + 修饰性介词/关系词（that/which/who/with等）
+    # 例：I want the big one. 我想要那个大的。/ She bought a red one. 她买了个红色的。
+    "ellipsis_n_bar_2":
+        lambda s: has(r"\b(one|ones)\b.{0,20}\b(that|which|who|with|without|from|in|of)\b", s),
+
+    # ── Filler-Gap（7）───────────────────────────────────────────────
+    # wh词与that对比（含空位）：测试填充语-空位依存中 wh 词而非 that 的选择
+    # 触发：含感知/言说类矩阵动词 + wh 疑问词（who/what/which）
+    # 例：She knows who left. 她知道谁离开了。（know 后用 who 不用 that）
+    "wh_vs_that_with_gap":
+        # Phase B: matrix verb + embedded wh-clause (no 'that' complementizer)
+        lambda s: has(
+            r"\b(know|knew|knows|remember|remembered|figured|find|found|say|said|"
+            r"think|thought|discover|discovered|understand|understood|realize|"
+            r"realized|notice|noticed|ask|asked|see|saw|tell|told|reveal|revealed|"
+            r"conceal|concealed|investigate|investigated|learn|learned|forget|"
+            r"forgot)\b.{1,50}\b(who|what|which)\b", s),
+    # wh词与that对比（长距离空位）：测试长距离填充语-空位中 wh 词后跟 that 的结构
+    # 触发：含 wh 词（who/what）+ 后续 that 从句
+    # 例：Who did she say that left? 她说谁离开了？
+    "wh_vs_that_with_gap_long_distance":
+        # Phase B: wh-word + embedded relative clause ('that')
+        lambda s: has(r"\b(who|what)\b.{0,80}\bthat\b", s),
+    # wh词与that对比（无空位）：测试用 that 补语引导的宾语从句（无空位）
+    # 触发：含感知/认知动词 + that（直接相邻）
+    # 例：She knows that he left. 她知道他离开了。
+    "wh_vs_that_no_gap":
+        # Phase B: verb + 'that' complementizer
+        lambda s: has(
+            r"\b(out|investigated|know|knows|learned|forget|revealed|reveal|"
+            r"remember|remembered|conceal|learn|discovered|realized|discover|"
+            r"realize|investigate|investigates|remembers|forgotten|concealed|"
+            r"figured)\b.{0,5}\bthat\b", s),
+    # wh词与that对比（长距离无空位）：长距离 that 补语从句（无空位）
+    # 触发：含认知/言说动词 + 较远处的 that
+    # 例：She discovered that the man had left. 她发现那个人已经离开了。
+    "wh_vs_that_no_gap_long_distance":
+        lambda s: has(
+            r"\b(out|investigated|know|knows|learned|forget|revealed|reveal|"
+            r"remember|remembered|conceal|learn|discovered|realized|discover|"
+            r"realize|investigate|investigates|remembers|forgotten|concealed|"
+            r"figured)\b.{0,60}\bthat\b", s),
+    # wh疑问句（宾语空位）：测试关系从句中的宾语空位（that 引导的限制性从句）
+    # 触发：含限定词 + 名词短语 + that 引导的从句，后接句末标点
+    # 例：The book that she read was good. 她读的那本书很好。
+    "wh_questions_object_gap":
+        # Phase B: relative clause with gap (det + NP + that + ...)
+        lambda s: has(
+            r"\b(a|an|the|every|some|many|most|this|that|these|those|all|few|"
+            r"several)\b.{0,20}\bthat\b.{1,60}[\.\?!]$", s),
+    # wh疑问句（长距离主语空位）：测试叠加 that 从句（相对从句套相对从句）
+    # 触发：句中含两个相邻的 that 从句
+    # 例：The man that she said that left was tall. 她说那个离开的人很高。
+    "wh_questions_subject_gap_long_distance":
+        # Phase B: double 'that' (stacked relative clauses)
+        lambda s: has(r"\bthat\b.{0,50}\bthat\b", s),
+    # 填充语-空位依存：测试 wh 词与后续关系词（that/which/who）的长距离依存
+    # 触发：含 wh 疑问词（who/which/what/where/when）+ 后续 that/which/who
+    # 例：Who did she say that she likes? 她说她喜欢谁？
+    "filler_gap_dependency":
+        lambda s: has(r"\b(who|which|what|where|when)\b.{0,80}\b(that|which|who)\b", s),
+
+    # ── Island Effects（8）───────────────────────────────────────────
+    # Phase B: All island effect tasks are wh-questions (start with wh-word)
+    # wh岛效应：测试 wh 词被内嵌 wh 疑问从句困住无法提取的句法限制
+    # 触发：句子以大写 Who/What/Which 开头（wh 疑问句）
+    # 例：*Who did she wonder what bought? wh 词被嵌套疑问句困住，无法合法提取
+    "wh_island":
+        lambda s: has(r"^(Who|What|Which)\b", s),
+    # 附加语岛效应：测试 wh 词被状语从句（附加语）困住无法提取的限制
+    # 触发：句子以大写 Who/What/Which 开头
+    # 例：*What did she leave because she bought? 状语从句构成提取障碍
+    "adjunct_island":
+        lambda s: has(r"^(Who|What|Which)\b", s),
+    # 复杂名词短语岛效应：测试 wh 词被复杂 NP（含关系从句的名词短语）困住的限制
+    # 触发：句子以大写 Who/What/Which 开头
+    # 例：*What did she meet a man who bought? 复杂 NP 内的成分无法提取
+    "complex_NP_island":
+        lambda s: has(r"^(Who|What|Which)\b", s),
+    # 句子主语岛效应：测试 wh 词被句子主语从句困住无法提取的限制
+    # 触发：句子以大写 Who/What/Which 开头
+    # 例：*What does that she bought surprise you? 主语从句内成分无法提取
+    "sentential_subject_island":
+        lambda s: has(r"^(Who|What|Which)\b", s),
+    # 左分支岛效应（简单疑问句）：测试 whose 等所有格左分支成分无法单独提取
+    # 触发：句子以大写 Whose/Which/What 开头
+    # 例：*Whose did she see friend? whose 作为左分支不能单独移位
+    "left_branch_island_simple_question":
+        lambda s: has(r"^(Whose|Which|What)\b", s),
+    # 左分支岛效应（回声疑问句）：测试带问号的回声疑问句中左分支限制
+    # 触发：含小写 whose/which/what + 问号
+    # 例：She read whose book? 她读的是谁的书？（回声疑问）
+    "left_branch_island_echo_question":
+        # Phase B: wh-word with question mark (echo question)
+        lambda s: has(r"\b(whose|which|what)\b.{0,40}\?$", s),
+    # 并列结构限制（宾语提取）：测试并列结构中单一成分无法单独提取的限制
+    # 触发：句子以大写 Who/What/Which 开头
+    # 例：*What did she buy and sell the car? 并列结构中不能只从一个并列项提取
+    "coordinate_structure_constraint_object_extraction":
+        lambda s: has(r"^(Who|What|Which)\b", s),
+    # 并列结构限制（复杂左分支）：测试含所有格的并列结构中左分支提取限制
+    # 触发：句子以大写 Who/What/Which/Whose 开头
+    # 例：*Whose did she meet and marry friend? 所有格左分支在并列结构中无法提取
+    "coordinate_structure_constraint_complex_left_branch":
+        lambda s: has(r"^(Who|What|Which|Whose)\b", s),
+
+    # ── NPI Licensing（7）────────────────────────────────────────────
+    # Phase C: calibrated against BLiMP — 'ever' is the key NPI
+    # 否定极性词许可（现在时1）：测试 ever 在无许可成分语境下的不合法使用
+    # 触发：含焦点词 even（非 NPI），且不含 ever（区分 even vs ever）
+    # 例：She even tried. 她甚至尝试了。（even 合法，ever 在此语境不合法）
+    "npi_present_1":
+        # Good sentences have 'even' (focus particle) without 'ever'
+        lambda s: has(r"\beven\b", s) and not has(r"\bever\b", s),
+    # 否定极性词许可（现在时2）：测试无 NPI 许可成分时 ever 不出现的合法句子
+    # 触发：句中不含 ever（NPI 未被触发的合法情形）
+    # 例：She always reads books. 她总是读书。（无 ever，合法）
+    "npi_present_2":
+        # Good sentences have no 'ever' (adverb, not NPI)
+        lambda s: not has(r"\bever\b", s),
+    # only 作为 NPI 许可成分（现在时）：测试 only 在局部域内许可 ever
+    # 触发：含 only...ever 结构
+    # 例：Only she ever reads. 只有她才会读。（only 许可 ever）
+    "only_npi_licensor_present":
+        # Phase B: 'only' licenses 'ever'
+        lambda s: has(r"\bonly\b.{0,40}\bever\b", s),
+    # only 的 NPI 许可辖域：测试句首 Only 对句内 ever 的辖域许可
+    # 触发：句子以 Only 开头，句内含 ever
+    # 例：Only the teacher ever helps. 只有老师才会帮忙。
+    "only_npi_scope":
+        # Phase B: sentence starts with 'Only' + 'ever' in scope
+        lambda s: has(r"^Only\b.{0,100}\bever\b", s),
+    # 矩阵疑问句作 NPI 许可成分：测试倒装助动词疑问句对 ever 的许可
+    # 触发：句子以倒装助动词开头（Did/Will/Can等），句内含 ever
+    # 例：Did she ever read that book? 她读过那本书吗？
+    "matrix_question_npi_licensor_present":
+        # Phase B: inverted auxiliary question + 'ever'
+        lambda s: has(
+            r"^(Had|Have|Has|Did|Do|Does|Will|Would|Could|Should|Might|Can|"
+            r"Is|Are|Was|Were)\b.{0,40}\bever\b", s),
+    # 句子否定作 NPI 许可成分：测试 not 在近距离内许可 ever
+    # 触发：含 not...ever 结构（间隔较短）
+    # 例：She did not ever lie. 她从未说谎。
+    "sentential_negation_npi_licensor_present":
+        # Phase B: 'not' licenses 'ever' within short span
+        lambda s: has(r"\bnot\b.{0,20}\bever\b", s),
+    # 句子否定的 NPI 辖域：测试 not 在较长距离内对 ever 的辖域许可
+    # 触发：含 not...ever 结构（间隔可较长）
+    # 例：She would not under any circumstances ever agree. 她在任何情况下都不会同意。
+    "sentential_negation_npi_scope":
+        # Phase B: 'not' + 'ever' within longer span
+        lambda s: has(r"\bnot\b.{0,60}\bever\b", s),
+
+    # ── Quantifiers（4）──────────────────────────────────────────────
+    # 超级量化词（1）：测试 more/fewer/less...than 比较量化结构
+    # 触发：含 more/fewer/less + than
+    # 例：More than five students arrived. 五名以上的学生到了。
+    "superlative_quantifiers_1":
+        lambda s: has(r"\b(more|fewer|less)\b.{0,20}\bthan\b", s),
+    # 超级量化词（2）：测试 at most/at least 等上下界量化短语
+    # 触发：含 at most 或 at least
+    # 例：At least three people agreed. 至少有三个人同意。
+    "superlative_quantifiers_2":
+        # Phase C: calibrated — 'at most/at least'
+        lambda s: has(r"\b(at most|at least)\b", s),
+    # there 存在句中的量化词（1）：测试以 There 开头的存在句（无 to be 不定式）
+    # 触发：句子以大写 There 开头，且不含 to be 不定式（排除提升结构）
+    # 例：There are many books on the shelf. 架子上有很多书。
+    "existential_there_quantifiers_1":
+        # Phase C: existential 'There' (starts with capital There, no 'to be')
+        lambda s: has(r"^There\b", s) and not has(r"\bto\s+be\b", s),
+    # there 存在句中的量化词（2）：测试非句首的 there（位置状语或非存在句）
+    # 触发：含小写 there（非句首大写）
+    # 例：She went there. 她去了那里。/ Put it there. 把它放在那里。
+    "existential_there_quantifiers_2":
+        # Phase C: locative 'there' (not sentence-initial existential)
+        lambda s: has(r"\bthere\b", s) and not has(r"^There\b", s),
+
+    # ── Subject-Verb Agreement（6）───────────────────────────────────
+    # 规则复数主谓一致（1）：测试复数主语与助动词/动词形式的一致
+    # 触发：含 is/are/was/were/has/have 等助动词，或第三人称单数动词 + 专有名词
+    # 例：The students are ready. 学生们已经准备好了。
+    "regular_plural_subject_verb_agreement_1":
+        # Phase A: has auxiliary verb OR 3sg verb form before proper name
+        lambda s: (
+            has(r"\b(is|are|was|were|has|have|does|do|isn.t|aren.t|wasn.t|"
+                r"weren.t|hasn.t|haven.t|doesn.t|don.t)\b", s) or
+            bool(re.search(r"\b\w+s\b.{0,30}\b[A-Z]\w+\b", s))
+        ),
+    # 规则复数主谓一致（2）：测试 The + 名词短语 + 动词 结构的主谓一致
+    # 触发：以 The 开头，后接名词短语和动词，句末有标点
+    # 例：The teacher reads. 老师在读书。/ The students read. 学生们在读书。
+    "regular_plural_subject_verb_agreement_2":
+        # Phase C: calibrated — 'The + NP + V' pattern
+        lambda s: has(r"^The\b.{3,35}\b[a-z]+\b[\.\?!]$", s),
+    # 不规则复数主谓一致（1）：测试不规则复数名词（men/women/children等）作主语的一致
+    # 触发：含不规则名词（goose/geese/man/men/child/children等）
+    # 例：The men are ready. 这些男人准备好了。/ The children were playing. 孩子们在玩。
+    "irregular_plural_subject_verb_agreement_1":
+        # Phase A: expanded irregular noun list
+        lambda s: has(r"\b" + _IRR_NOUNS + r"\b", s),
+    # 不规则复数主谓一致（2）：同上，第二变体
+    # 触发：含不规则名词（data/criteria/phenomena等）
+    # 例：The data are clear. 数据很清晰。/ The criteria were met. 标准已达到。
+    "irregular_plural_subject_verb_agreement_2":
+        # Phase A: expanded irregular noun list
+        lambda s: has(r"\b" + _IRR_NOUNS + r"\b", s),
+    # 干扰项主谓一致（关系从句干扰）：测试中间插有关系从句时主谓仍需一致
+    # 触发：含关系从句引导词（that/who/which）
+    # 例：The girl that the boys like is tall. 男孩们喜欢的那个女孩很高。（主语是 girl，动词用 is）
+    "distractor_agreement_relative_clause":
+        # Phase C: any sentence with relative clause marker (that/who/which)
+        lambda s: has(r"\b(that|who|which)\b", s),
+    # 干扰项主谓一致（关系名词干扰）：测试 NP of NP 结构中的中心词动词一致
+    # 触发：含 a/an/the NP of NP + 助动词结构
+    # 例：The list of students is long. 学生名单很长。（主语中心词是 list，动词用 is）
+    "distractor_agreement_relational_noun":
+        # Phase C: 'a/the NP of NP + aux' (PP distractor)
+        lambda s: has(
+            r"\b(a|an|the)\b.{1,20}\bof\b.{0,40}"
+            r"\b(has|have|hasn.t|haven.t|is|are|isn.t|aren.t|was|were|"
+            r"wasn.t|weren.t|does|do|doesn.t|don.t|will|would|can|could|"
+            r"should|might)\b", s),
+
+    # ── Irregular Forms（2）──────────────────────────────────────────
+    # 不规则过去分词（动词用法）：测试不规则动词的过去式形式
+    # 触发：含不规则过去式（went/came/broke/ate/wrote等）
+    # 例：She went to school. 她去上学了。/ He broke the window. 他打破了窗户。
+    "irregular_past_participle_verbs":
+        # Phase A: comprehensive irregular past tense list
+        lambda s: has(r"\b" + _IRR_PASTS + r"\b", s),
+    # 不规则过去分词（形容词用法）：测试不规则过去分词作定语修饰名词
+    # 触发：含 a/an/the + 不规则过去分词（broken/hidden/written等）
+    # 例：a broken window 一扇破窗户 / the written report 那份书面报告
+    "irregular_past_participle_adjectives":
+        # Phase A: a/an/the + irregular past participle (used as adjective)
+        lambda s: has(r"\b(a|an|the)\b.{0,10}\b" + _IRR_PPARTS + r"\b", s),
+}
+
+
+# ────────────────────────────────────────────────────────────────────
+# BLiMP Supplement 5 子任务规则
+# ──────────────────────────────────────────────────────────────────���─
+
+SUPPLEMENT_RULES = {
+    # 主谓倒装疑问句：测试英语一般疑问句中助动词提前（主谓倒装）结构
+    # 触发：句子以助动词（Did/Does/Will/Can/Has/Is/Are等）开头
+    # 例：Did she finish the work? 她完成工作了吗？/ Will they come? 他们会来吗？
+    "subject_aux_inversion":
+        # Phase C: sentence starts with auxiliary (inverted question)
+        lambda s: has(
+            r"^(Did|Does|Do|Will|Would|Should|Could|Can|Has|Have|Is|Are|"
+            r"Was|Were|Might|Must|Shall|Had)\b", s),
+    # 上下位关系：测试语义蕴含/逻辑推理表达（if-then、therefore、means等）
+    # 触发：含 if/since...then/therefore/that means/must be the case 等逻辑推导词
+    # 例：If it is a dog, then it is an animal. 如果它是狗，那它就是动物。
+    "hypernym":
+        # Phase C: logical/entailment language (if-then, therefore, means)
+        lambda s: (
+            has(r"\b(if|since)\b.{0,80}\b(then|therefore|means|case)\b", s) or
+            has(r"\btherefore\b", s) or
+            has(r"\bthat means\b", s) or
+            has(r"\bmust be the case\b", s)
+        ),
+    # 话语轮换：测试对话中回应词开头的话语轮换结构
+    # 触发：含 Yes/No/Yeah/Nope/Sure/Okay 后接标点（对话回应格式）
+    # 例：Yes, I agree. 是的，我同意。/ No, that's wrong. 不，那是错的。
+    "turn_taking":
+        # Phase C: dialogue with yes/no response
+        lambda s: bool(re.search(r"\b(Yes|No|Yeah|Nope|Sure|Okay)\b[,!\.]", s)),
+    # 问答一致性（简单）：测试 wh 疑问词与其对应答案类型的匹配（单句）
+    # 触发：含 wh 疑问词（where/when/who/what/how/why）+ 问号
+    # 例：Where did she go? 她去哪了？（答案应为地点）
+    "qa_congruence_easy":
+        lambda s: has(r"\b(where|when|who|what|how|why)\b.{0,50}\?", s),
+    # 问答一致性（复杂）：测试多行问答对中 wh 疑问词与答案内容的一致性
+    # 触发：含 wh 疑问词 + 问号，且文本跨越多行（含换行符）
+    # 例：What did she buy?\nA book. 她买了什么？\n一本书。
+    "qa_congruence_tricky":
+        # Phase C: wh-question + answer (multiline)
+        lambda s: (
+            has(r"\b(where|when|who|what|how|why)\b.{0,50}\?", s) and
+            "\n" in s
+        ),
+}
+
+
+# ────────────────────────────────────────────────────────────────────
+# EWoK 11 知识域规则
+# ────────────────────────────────────────────────────────────────────
+
+EWOK_RULES = {
+    # 智能体属性：测试对有意识/有能力存在（人/动物）的属性描述
+    # 触发：含能力/意图类情态词（can/could/able to等）+ 认知/感知/行动动词（think/see/decide等）
+    # 例：The robot can learn new tasks. 这个机器人能学习新任务。
+    "agent-properties":
+        lambda s: has(
+            r"\b(can|could|able to|capable of|knows how|learned to|tried to|"
+            r"managed to|refused to|decided to|chose to|wanted to|needed to|"
+            r"had to|had the ability|is capable|was able)\b"
+            r".{0,30}\b(walk|talk|see|hear|think|feel|remember|understand|"
+            r"decide|move|speak|read|write|learn|communicate|recognize|respond|"
+            r"act|behave|know|plan|reason|judge|infer|perceive)\b", s),
+    # 物质动态变化：测试物质的物理/化学状态变化过程
+    # 触发：含状态变化动词（melt/freeze/burn/boil/dissolve等）或物态词（liquid/solid/gas等）
+    # 例：The ice melted in the sun. 冰在阳光下融化了。/ Water boils at 100°C. 水在100°C沸腾。
+    "material-dynamics":
+        lambda s: has(
+            r"\b(melt|melted|melting|freeze|froze|frozen|freezing|burn|burned|"
+            r"burning|burnt|boil|boiled|boiling|dissolve|dissolved|dissolving|"
+            r"evaporate|evaporated|evaporating|condense|condensed|condensing|"
+            r"solidify|solidified|oxidize|oxidized|rust|rusted|corrode|corroded|"
+            r"liquid|solid|gas|vapor|steam|ice|plasma)\b", s),
+    # 物质物理属性：测试对物质固有物理属性的描述
+    # 触发：含物理属性形容词（hard/soft/heavy/light/rough/smooth/flexible等）
+    # 例：The rock is hard. 岩石很硬。/ Glass is fragile. 玻璃易碎。
+    "material-properties":
+        lambda s: has(
+            r"\b(hard|soft|heavy|light|rough|smooth|transparent|opaque|"
+            r"flexible|rigid|brittle|elastic|magnetic|conductive|absorbent|"
+            r"porous|dense|viscous|flammable|waterproof|sticky|slippery|"
+            r"fragile|durable|resilient|malleable|ductile|soluble|insoluble)\b", s),
+    # 物理运动动态：测试物体的运动、力学现象及物理过程
+    # 触发：含运动/���学词（fall/bounce/roll/slide/float/sink等）或物理量词（gravity/friction等）
+    # 例：The ball fell and bounced. 球掉下去弹了起来。/ The leaf floated on water. 树叶漂在水上。
+    "physical-dynamics":
+        lambda s: has(
+            r"\b(fall|fell|fallen|falling|drop|dropped|bounce|bounced|roll|"
+            r"rolled|slide|slid|float|floated|sink|sank|push|pushed|pull|"
+            r"pulled|stretch|stretched|compress|compressed|accelerate|"
+            r"accelerated|decelerate|decelerated|collide|collided|swing|"
+            r"swung|spin|spun|rotate|rotated|vibrate|vibrated|oscillate|"
+            r"projectile|momentum|gravity|friction|force|velocity|inertia)\b", s),
+    # 物理交互：测试物体之间的直接物理接触与操作行为
+    # 触发：含物理操作动词（hit/throw/catch/kick/push/pull/lift/carry/pour等）
+    # 例：She placed the cup on the table. 她把杯子放在桌子上。/ He threw the ball. 他扔了球。
+    "physical-interactions":
+        lambda s: has(
+            r"\b(hit|struck|throw|threw|catch|caught|kick|kicked|break|broke|"
+            r"push|pushed|pull|pulled|lift|lifted|carry|carried|place|placed|"
+            r"put|move|moved|drag|dragged|roll|rolled|bounce|bounced|stack|"
+            r"stacked|pour|poured|fill|filled|empty|emptied|mix|mixed|collide|"
+            r"collided|knock|knocked|toss|tossed|fling|flung|drop|dropped)\b", s),
+    # 物理空间关系：测试物体之间的空间位置关系描述
+    # 触发：含空间介词/副词（inside/above/below/behind/between/under/over等）
+    # 例：The book is on top of the shelf. 书在架子上面。/ The cat is under the table. 猫在桌子下面。
+    "physical-relations":
+        lambda s: has(
+            r"\b(inside|outside|above|below|under|over|behind|in front of|"
+            r"next to|beside|between|among|on top of|beneath|within|around|"
+            r"near|far|close to|adjacent|opposite|across|along|through|"
+            r"surrounding|enclosed|embedded|attached|connected|separated|"
+            r"underneath|overhead|upright|horizontal|vertical|diagonal)\b", s),
+    # 量化比较属性：测试物体属性的量化比较（程度差异对比）
+    # 触发：含比较级形容词（bigger/smaller/heavier/faster等）+ than
+    # 例：This box is heavier than that one. 这个箱子比那个重。
+    "quantitative-properties":
+        lambda s: has(
+            r"\b(bigger|smaller|larger|heavier|lighter|taller|shorter|longer|"
+            r"wider|narrower|deeper|shallower|more|fewer|less|greater|thicker|"
+            r"thinner|faster|slower|higher|lower|stronger|weaker|louder|quieter|"
+            r"brighter|darker|hotter|colder|warmer|cooler)\b.{0,20}\bthan\b", s),
+    # 社会交互行为：测试人与人之间的社会性互动行为
+    # 触发：含社会互动动词（help/ask/thank/apologize/greet/invite/promise/argue/cooperate等）
+    # 例：She thanked him for his help. 她感谢他的帮助。/ They cooperated on the project. 他们合作完成了项目。
+    "social-interactions":
+        lambda s: has(
+            r"\b(helped|help|asked|ask|told|tell|thanked|thank|apologized|"
+            r"apologize|greeted|greet|invited|invite|refused|refuse|agreed|"
+            r"agree|disagreed|disagree|promised|promise|warned|warn|argued|"
+            r"argue|negotiated|negotiate|cooperated|cooperate|collaborated|"
+            r"collaborate|competed|compete|challenged|challenge|threatened|"
+            r"threaten|comforted|comfort|encouraged|encourage|persuaded|persuade|"
+            r"convinced|convince|trusted|trust|forgave|forgive|blamed|blame|"
+            r"praised|praise|criticized|criticize|supported|support|betrayed|"
+            r"betray|bullied|bully|welcomed|welcome|rejected|reject)\b", s),
+    # 社会性格属性：测试对人的性格/品德特质的描述
+    # 触发：含性格形容词（kind/mean/friendly/rude/generous/honest/brave等）
+    # 例：She is kind and generous. 她善良而慷慨。/ He was dishonest. 他不诚实。
+    "social-properties":
+        lambda s: has(
+            r"\b(kind|mean|friendly|rude|generous|selfish|honest|dishonest|"
+            r"polite|impolite|brave|cowardly|trustworthy|reliable|patient|"
+            r"impatient|humble|arrogant|compassionate|empathetic|sympathetic|"
+            r"loyal|disloyal|fair|unfair|respectful|disrespectful|helpful|"
+            r"harmful|caring|uncaring|gentle|harsh|warm|cold|open|closed|"
+            r"forgiving|vengeful|tolerant|intolerant|cooperative|competitive)\b", s),
+    # 社会关系：测试描述人与人之间社会角色与关系的词汇
+    # 触发：含社会关系名词（friend/enemy/colleague/boss/parent/teacher/doctor/neighbor等）
+    # 例：She is my best friend. 她是我最好的朋友。/ He is her mentor. 他是她的导师。
+    "social-relations":
+        lambda s: has(
+            r"\b(friend|enemy|colleague|boss|employee|parent|child|sibling|"
+            r"spouse|partner|teacher|student|doctor|patient|neighbor|stranger|"
+            r"teammate|rival|ally|opponent|mentor|apprentice|leader|follower|"
+            r"manager|subordinate|customer|vendor|host|guest|authority|citizen|"
+            r"family|relative|acquaintance|associate|advisor|protege)\b", s),
+    # 空间方位关系：测试方向/方位词描述物体或运动的空间方位
+    # 触发：含方向方位词（north/south/left/right/up/down/forward/above/below等）
+    # 例：Turn left at the corner. 在拐角处左转。/ The store is to the north. 商店在北面。
+    "spatial-relations":
+        lambda s: has(
+            r"\b(north|south|east|west|left|right|up|down|forward|backward|"
+            r"clockwise|counterclockwise|parallel|perpendicular|diagonal|"
+            r"horizontal|vertical|upward|downward|inward|outward|toward|"
+            r"away from|northeast|northwest|southeast|southwest|above|below|"
+            r"ahead|behind|sideways|upstairs|downstairs|indoors|outdoors)\b", s),
+}
+
+
+# ────────────────────────────────────────────────────────────────────
+# 主处理函数
+# ────────────────────────────────────────────────────────────────────
+
+def analyze_sentence(text: str, spacy_features: dict = None) -> dict:
+    """
+    Apply all rules to a sentence. spacy_features (if provided) augments
+    some rules with dependency-parsed information.
+    """
+    blimp_matches = []
+    for k, fn in BLIMP_RULES.items():
+        try:
+            if fn(text):
+                blimp_matches.append(k)
+        except Exception:
+            pass
+
+    # Phase B: spaCy enhancement for passive and SVA rules
+    if spacy_features:
+        # Add passive if spaCy found auxpass/nsubjpass
+        if spacy_features.get("has_auxpass") and "passive_1" not in blimp_matches:
+            blimp_matches.append("passive_1_spacy")
+        if spacy_features.get("has_auxpass") and "animate_subject_passive" not in blimp_matches:
+            blimp_matches.append("animate_subject_passive_spacy")
+        # Add SVA based on agreement check
+        agree = spacy_features.get("subj_verb_agree")
+        if agree and agree[2]:  # agreement matches
+            if "regular_plural_subject_verb_agreement_1" not in blimp_matches:
+                blimp_matches.append("regular_plural_subject_verb_agreement_1_spacy")
+
+    supplement_matches = [k for k, fn in SUPPLEMENT_RULES.items() if fn(text)]
+    ewok_matches = [k for k, fn in EWOK_RULES.items() if fn(text)]
+
+    return {
+        "blimp": blimp_matches,
+        "supplement": supplement_matches,
+        "ewok": ewok_matches,
+    }
+
+
+def process_file(fname: str, use_spacy: bool = False):
+    input_path = DATA_DIR / fname
+    if not input_path.exists():
+        print(f"[skip] {fname} 不存在")
+        return
+
+    stem = fname.replace(".train.txt", "").replace(".txt", "")
+    out_path  = OUT_DIR / f"{stem}_rules.jsonl"
+    ckpt_path = OUT_DIR / f"{stem}_rules.ckpt"
+
+    # 读已处理行数（断点续传）
+    done_lines = 0
+    if ckpt_path.exists():
+        done_lines = int(ckpt_path.read_text().strip())
+        print(f"[续传] {fname}: 已完成 {done_lines:,} 行")
+
+    total_lines = sum(1 for _ in open(input_path, errors="ignore"))
+
+    # Phase B: spaCy batch processing
+    spacy_batch_size = 256
+
+    with open(input_path, "r", errors="ignore") as fin, \
+         open(out_path, "a", encoding="utf-8") as fout:
+
+        batch_lines = []   # (line_no, text)
+
+        def flush_batch():
+            if not batch_lines:
+                return
+            texts = [t for _, t in batch_lines]
+            if use_spacy and _SPACY_AVAILABLE:
+                spacy_feats = _spacy_analyze_batch(texts)
+            else:
+                spacy_feats = [None] * len(texts)
+            for (line_no, text), sf in zip(batch_lines, spacy_feats):
+                result = analyze_sentence(text, sf)
+                result["line"] = line_no
+                fout.write(json.dumps(result, ensure_ascii=False) + "\n")
+            batch_lines.clear()
+
+        for line_no, raw_line in enumerate(tqdm(fin, total=total_lines, desc=stem)):
+            if line_no < done_lines:
+                continue
+            text = raw_line.strip()
+            if not text:
+                continue
+
+            if use_spacy:
+                batch_lines.append((line_no, text))
+                if len(batch_lines) >= spacy_batch_size:
+                    flush_batch()
+                    fout.flush()
+                    ckpt_path.write_text(str(line_no))
+            else:
+                result = analyze_sentence(text)
+                result["line"] = line_no
+                fout.write(json.dumps(result, ensure_ascii=False) + "\n")
+
+            if not use_spacy and line_no % 10000 == 0:
+                fout.flush()
+                ckpt_path.write_text(str(line_no))
+
+        if use_spacy:
+            flush_batch()
+
+    ckpt_path.write_text(str(total_lines))
+
+    # 统计覆盖率
+    blimp_hits = supp_hits = ewok_hits = total = 0
+    with open(out_path) as f:
+        for line in f:
+            d = json.loads(line)
+            total += 1
+            if d["blimp"]:      blimp_hits += 1
+            if d["supplement"]: supp_hits  += 1
+            if d["ewok"]:       ewok_hits  += 1
+
+    print(f"\n[{fname}] 完成 {total:,} 行")
+    print(f"  BLiMP 命中:       {blimp_hits:,} ({blimp_hits/total*100:.1f}%)")
+    print(f"  Supplement 命中:  {supp_hits:,}  ({supp_hits/total*100:.1f}%)")
+    print(f"  EWoK 命中:        {ewok_hits:,}  ({ewok_hits/total*100:.1f}%)")
+    print(f"  结果: {out_path}")
+
+
+def main(input_dir=None, output_dir=None, use_spacy=False):
+    global DATA_DIR, OUT_DIR
+    if input_dir:  DATA_DIR = Path(input_dir)
+    if output_dir: OUT_DIR  = Path(output_dir); OUT_DIR.mkdir(parents=True, exist_ok=True)
+
+    if use_spacy:
+        _init_spacy()
+
+    for fname in FILES:
+        process_file(fname, use_spacy=use_spacy)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(
+        description="Stage 5: Rule-based sentence labeling (Phase A+B+C)"
+    )
+    parser.add_argument("--input_dir",  default=None)
+    parser.add_argument("--output_dir", default=None)
+    parser.add_argument("--file",       default=None, help="只处理指定文件")
+    parser.add_argument("--spacy",      action="store_true", default=False,
+                        help="启用 spaCy 句法增强（需要 en_core_web_sm）")
+    args = parser.parse_args()
+
+    if args.file:
+        if args.spacy:
+            _init_spacy()
+        process_file(args.file, use_spacy=args.spacy)
+    else:
+        main(args.input_dir, args.output_dir, use_spacy=args.spacy)

scripts/01_data/stage5a_vocab_build.py

@@ -0,0 +1,800 @@
+"""
+Stage 5A: Vocab-Derived Word Lists + BLiMP Calibration
+
+Phase A: Build linguistic word-category lists from vocab.txt
+Phase C: Calibrate rule recall against real BLiMP good sentences
+
+Outputs:
+  data/rule_vocab_categories.json   -- word lists for each linguistic category
+  data/rule_calibration.json        -- per-task recall metrics
+
+Usage:
+  python -m scripts.01_data.stage5a_vocab_build
+  python scripts/01_data/stage5a_vocab_build.py [--vocab VOCAB] [--blimp BLIMP_DIR] [--supplement SUPP_DIR]
+"""
+
+import re
+import json
+import argparse
+from pathlib import Path
+
+ROOT = Path(__file__).parent.parent.parent
+VOCAB_PATH   = ROOT / "data/vocab.txt"
+BLIMP_DIR    = ROOT / "evaluation-pipeline-2025/evaluation_data/full_eval/blimp_filtered"
+SUPP_DIR     = ROOT / "evaluation-pipeline-2025/evaluation_data/full_eval/supplement_filtered"
+OUT_VOCAB    = ROOT / "data/rule_vocab_categories.json"
+OUT_CALIB    = ROOT / "data/rule_calibration.json"
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# Seed word lists (used to bootstrap vocab-derived categories)
+# ─────────────────────────────────────────────────────────────────────────────
+
+SEED_ANIMATE_SINGULAR = [
+    # Human social roles
+    "woman", "man", "girl", "boy", "child", "person", "baby", "infant",
+    "teenager", "adult", "kid", "lady", "gentleman",
+    # Family
+    "mother", "father", "son", "daughter", "brother", "sister", "parent",
+    "wife", "husband", "spouse", "grandfather", "grandmother", "grandson",
+    "granddaughter", "niece", "nephew", "uncle", "aunt", "cousin", "sibling",
+    "grandparent", "ancestor", "descendant", "offspring",
+    # Professional roles
+    "teacher", "professor", "lecturer", "instructor", "student", "pupil",
+    "doctor", "physician", "surgeon", "nurse", "dentist", "therapist",
+    "pharmacist", "veterinarian", "paramedic",
+    "lawyer", "attorney", "judge", "senator", "congressman", "politician",
+    "president", "minister", "governor", "mayor", "official", "representative",
+    "officer", "detective", "sheriff", "guard", "soldier", "pilot", "captain",
+    "sailor", "marine", "general", "admiral", "sergeant",
+    "scientist", "researcher", "engineer", "architect", "mathematician",
+    "physicist", "chemist", "biologist", "geologist", "economist",
+    "journalist", "reporter", "editor", "publisher", "author", "writer",
+    "poet", "novelist", "playwright",
+    "artist", "painter", "sculptor", "musician", "singer", "dancer",
+    "actor", "actress", "director", "producer", "photographer",
+    "chef", "cook", "waiter", "waitress", "cashier", "clerk",
+    "driver", "chauffeur", "pilot", "mechanic", "plumber", "electrician",
+    "carpenter", "farmer", "gardener", "janitor",
+    "manager", "director", "boss", "employer", "employee", "worker",
+    "supervisor", "assistant", "secretary", "administrator", "coordinator",
+    "consultant", "analyst", "advisor", "counselor",
+    "priest", "minister", "bishop", "rabbi", "monk", "nun",
+    "athlete", "swimmer", "runner", "player", "champion", "competitor",
+    "coach", "trainer", "referee",
+    # Social roles
+    "friend", "enemy", "neighbor", "stranger", "colleague", "partner",
+    "customer", "patient", "victim", "witness", "suspect", "prisoner",
+    "tourist", "visitor", "guest", "host", "passenger", "pedestrian",
+    "resident", "citizen", "immigrant", "refugee",
+    "volunteer", "activist", "candidate", "applicant", "graduate",
+    # Animals (animate)
+    "dog", "cat", "bird", "fish", "horse", "cow", "pig", "sheep", "goat",
+    "rabbit", "mouse", "rat", "lion", "tiger", "bear", "wolf", "fox",
+    "deer", "elephant", "monkey", "gorilla", "snake", "turtle", "frog",
+    "eagle", "owl", "hawk", "duck", "chicken", "turkey", "whale", "dolphin",
+    "shark", "seal", "penguin", "parrot", "crow", "pigeon", "sparrow",
+    "squirrel", "chipmunk", "beaver", "otter", "badger", "hedgehog",
+    "camel", "giraffe", "zebra", "hippo", "rhino", "cheetah", "leopard",
+    "panther", "jaguar", "kangaroo", "koala", "panda", "polar",
+]
+
+SEED_TRANSITIVE_VERBS = [
+    # Perception
+    "see", "saw", "seen", "watch", "watched", "observe", "observed",
+    "notice", "noticed", "hear", "heard", "feel", "felt", "taste", "smell",
+    "look",
+    # Cognition
+    "know", "knew", "known", "think", "thought", "believe", "believed",
+    "understand", "understood", "remember", "remembered", "forget", "forgot",
+    "forgotten", "realize", "realized", "imagine", "imagined", "consider",
+    "considered", "expect", "expected", "assume", "assumed", "suppose",
+    "supposed", "guess", "guessed", "recognize", "recognized",
+    # Communication
+    "say", "said", "tell", "told", "ask", "asked", "answer", "answered",
+    "show", "showed", "shown", "describe", "described", "explain", "explained",
+    "mention", "mentioned", "report", "reported", "announce", "announced",
+    "claim", "claimed", "state", "stated", "suggest", "suggested",
+    "recommend", "recommended", "propose", "proposed", "deny", "denied",
+    "confirm", "confirmed", "admit", "admitted", "acknowledge", "acknowledged",
+    "warn", "warned", "promise", "promised", "inform", "informed",
+    # Social/emotional
+    "help", "helped", "love", "loved", "hate", "hated", "like", "liked",
+    "want", "wanted", "need", "needed", "care", "cared", "trust", "trusted",
+    "respect", "respected", "admire", "admired", "fear", "feared",
+    "enjoy", "enjoyed", "prefer", "preferred", "support", "supported",
+    "protect", "protected", "defend", "defended", "attack", "attacked",
+    "criticize", "criticized", "praise", "praised", "thank", "thanked",
+    "apologize", "blame", "blamed", "forgive", "forgave", "forgiven",
+    "invite", "invited", "greet", "greeted", "welcome", "welcomed",
+    "reject", "rejected", "ignore", "ignored", "avoid", "avoided",
+    "betray", "betrayed", "threaten", "threatened", "convince", "convinced",
+    "persuade", "persuaded", "encourage", "encouraged", "motivate", "motivated",
+    "inspire", "inspired", "challenge", "challenged", "impress", "impressed",
+    "surprise", "surprised", "shock", "shocked", "amaze", "amazed",
+    "annoy", "annoyed", "upset", "bother", "bothered", "worry", "worried",
+    "frighten", "frightened", "scare", "scared", "please", "pleased",
+    "satisfy", "satisfied", "disappoint", "disappointed", "confuse", "confused",
+    "bore", "bored", "excite", "excited", "delight", "delighted",
+    # Physical actions
+    "carry", "carried", "lift", "lifted", "move", "moved", "push", "pushed",
+    "pull", "pulled", "throw", "threw", "thrown", "catch", "caught",
+    "hold", "held", "take", "took", "taken", "give", "gave", "given",
+    "bring", "brought", "send", "sent", "pass", "passed", "put", "placed",
+    "drop", "dropped", "pick", "grab", "grabbed", "touch", "touched",
+    "hit", "hurt", "kick", "kicked", "punch", "punched", "push",
+    "break", "broke", "broken", "build", "built", "create", "created",
+    "make", "made", "destroy", "destroyed", "damage", "damaged",
+    "repair", "repaired", "fix", "fixed", "open", "opened", "close", "closed",
+    "fill", "filled", "empty", "emptied", "cover", "covered", "uncover",
+    "hide", "hid", "hidden", "reveal", "revealed", "find", "found",
+    "lose", "lost", "use", "used", "keep", "kept", "leave", "left",
+    # Economic/transactional
+    "buy", "bought", "sell", "sold", "pay", "paid", "earn", "earned",
+    "spend", "spent", "hire", "hired", "fire", "fired", "train", "trained",
+    "teach", "taught", "learn", "learned", "study", "studied", "read", "wrote",
+    # Others
+    "visit", "visited", "meet", "met", "call", "called", "contact",
+    "check", "checked", "test", "tested", "examine", "examined",
+    "investigate", "investigated", "analyze", "analyzed", "compare", "compared",
+    "measure", "measured", "count", "counted", "include", "included",
+    "involve", "involved", "affect", "affected", "change", "changed",
+    "improve", "improved", "increase", "increased", "reduce", "reduced",
+    "develop", "developed", "produce", "produced", "plan", "planned",
+    "organize", "organized", "manage", "managed", "control", "controlled",
+    "lead", "led", "follow", "followed", "choose", "chose", "chosen",
+    "decide", "decided", "complete", "completed", "finish", "finished",
+    "start", "started", "begin", "began", "begun", "achieve", "achieved",
+    "win", "won", "lose", "lost", "defeat", "defeated", "beat", "beaten",
+    "save", "saved", "rescue", "rescued", "provide", "provided",
+    "offer", "offered", "receive", "received", "accept", "accepted",
+    "refuse", "refused", "collect", "collected", "gather", "gathered",
+]
+
+SEED_INTRANSITIVE_VERBS = [
+    # Motion
+    "walk", "run", "ran", "fly", "flew", "swim", "swam", "jump", "climb",
+    "crawl", "march", "dance", "wander", "stroll", "jog", "sprint", "rush",
+    "hurry", "travel", "move", "go", "went", "come", "came", "arrive",
+    "depart", "leave", "return", "ascend", "descend", "rise", "rose",
+    "fall", "fell", "float", "sink", "slide", "roll", "bounce", "tumble",
+    # State/bodily
+    "sleep", "slept", "rest", "wake", "woke", "sit", "sat", "stand",
+    "stood", "lie", "lay", "exist", "live", "die", "survive",
+    # Expression
+    "laugh", "cry", "smile", "frown", "shout", "yell", "scream", "whisper",
+    "speak", "talk", "yawn", "sneeze", "cough", "sigh", "groan", "moan",
+    "blush", "tremble", "shiver", "shudder", "wink", "nod", "blink",
+    # Activity
+    "work", "play", "exercise", "practice", "train", "compete", "perform",
+    "participate", "cooperate", "collaborate", "communicate", "interact",
+    "socialize", "volunteer", "protest", "complain", "argue", "debate",
+    "agree", "disagree", "hesitate", "pause", "wait", "stay", "remain",
+    # Change of state (also inchoative)
+    "fade", "shrink", "grow", "expand", "collapse", "appear", "disappear",
+    "emerge", "vanish", "accelerate", "decelerate", "stop", "begin",
+    "end", "continue", "proceed", "progress",
+    # Social
+    "succeed", "fail", "win", "lose", "graduate", "retire", "resign",
+    "quit", "cope", "struggle", "suffer", "benefit", "profit",
+    "respond", "react", "object", "consent", "concur", "abstain",
+    "boast", "confess", "apologize", "compromise", "surrender",
+]
+
+SEED_CAUSATIVE_INCHOATIVE = [
+    # Core causative-inchoative alternation verbs
+    "break", "broke", "broken", "melt", "melted", "open", "opened",
+    "close", "closed", "freeze", "froze", "frozen", "dry", "dried",
+    "crack", "cracked", "shatter", "shattered", "dissolve", "dissolved",
+    "stretch", "stretched", "shrink", "shrank", "shrunk", "loosen", "loosened",
+    "tighten", "tightened", "brighten", "brightened", "darken", "darkened",
+    "soften", "softened", "harden", "hardened", "thicken", "thickened",
+    "thin", "thinned", "widen", "widened", "narrow", "narrowed",
+    "bend", "bent", "twist", "twisted", "curl", "curled", "flatten",
+    "flattened", "crumple", "crumpled", "fold", "folded", "unfold", "unfolded",
+    "wrinkle", "wrinkled", "chip", "chipped", "burn", "burned", "burnt",
+    "boil", "boiled", "cool", "cooled", "heat", "heated", "warm", "warmed",
+    "condense", "condensed", "evaporate", "evaporated", "solidify", "solidified",
+    "drop", "dropped", "roll", "rolled", "slide", "slid", "spin", "spun",
+    "turn", "turned", "rotate", "rotated", "swing", "swung", "rock", "rocked",
+    "shake", "shook", "shaken", "wake", "woke", "woken", "collapse",
+    "collapsed", "hide", "hid", "hidden", "soak", "soaked", "heal", "healed",
+    "slow", "slowed", "speed", "sped", "fill", "filled", "empty", "emptied",
+    "split", "reunite", "reunited", "separate", "separated", "combine",
+    "combined", "mix", "mixed", "blend", "blended",
+]
+
+SEED_PAST_PARTICIPLES = [
+    # Irregular past participles (for adjective use)
+    "broken", "hidden", "worn", "forgotten", "chosen", "stolen", "frozen",
+    "fallen", "risen", "taken", "seen", "heard", "known", "done", "gone",
+    "come", "grown", "given", "driven", "written", "spoken", "built",
+    "bought", "caught", "taught", "brought", "found", "lost", "made",
+    "put", "kept", "held", "led", "left", "read", "sent", "sold",
+    "sat", "stood", "slept", "swum", "sung", "rung", "drunk", "eaten",
+    "ridden", "beaten", "bitten", "blown", "drawn", "flown", "shown",
+    "thrown", "torn", "woven", "chosen", "frozen", "shaken", "taken",
+    "mistaken", "forsaken", "proven", "woven", "striven", "risen",
+    # Common -ed past participles used as adjectives
+    "confused", "insulted", "astounded", "hired", "scared", "distracted",
+    "bored", "excited", "interested", "tired", "embarrassed", "annoyed",
+    "frustrated", "satisfied", "disappointed", "surprised", "shocked",
+    "worried", "stressed", "relaxed", "pleased", "delighted", "amused",
+    "amazed", "inspired", "motivated", "encouraged", "convinced",
+    "trained", "educated", "experienced", "qualified", "skilled",
+    "wounded", "injured", "damaged", "fixed", "repaired", "cleaned",
+    "prepared", "organized", "informed", "warned", "protected",
+]
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# Category builder: load vocab.txt and classify words
+# ─────────────────────────────────────────────────────────────────────────────
+
+def load_vocab(vocab_path: Path) -> set:
+    """Load all words from vocab.txt (lowercase, stripped)."""
+    if not vocab_path.exists():
+        print(f"[warn] vocab.txt not found at {vocab_path}")
+        return set()
+    with open(vocab_path, encoding="utf-8", errors="ignore") as f:
+        return {line.strip().lower() for line in f if line.strip()}
+
+
+def build_categories(vocab: set) -> dict:
+    """
+    Build word-category lists.
+    Strategy: take seed lists, add morphological variants found in vocab.
+    """
+    def expand_with_morphology(seeds: list, vocab: set) -> list:
+        """Add inflected forms of seed words that appear in vocab."""
+        result = set(w.lower() for w in seeds)
+        for w in list(seeds):
+            w = w.lower()
+            base = re.sub(r'(?:ed|ing|s|es|er|est)$', '', w)
+            # Try common suffixes
+            for suffix in ['', 's', 'es', 'ed', 'ing', 'er', 'ers', 'ess', 'esses']:
+                candidate = base + suffix
+                if candidate in vocab and len(candidate) >= 3:
+                    result.add(candidate)
+            # Irregular plural
+            if w.endswith('man'):
+                result.add(w[:-3] + 'men')
+            elif w.endswith('woman'):
+                result.add(w[:-5] + 'women')
+            elif w.endswith('child'):
+                result.add('children')
+            elif w.endswith('person'):
+                result.add('people')
+        return sorted(result & vocab) if vocab else sorted(result)
+
+    # Build animate nouns list (singular + plural)
+    animate = set(w.lower() for w in SEED_ANIMATE_SINGULAR)
+    # Add plurals
+    for w in list(animate):
+        for plural in [w + 's', w + 'es']:
+            if plural in vocab:
+                animate.add(plural)
+    # Special irregular plurals
+    animate.update(['men', 'women', 'children', 'people', 'mice', 'geese',
+                    'teeth', 'feet', 'oxen', 'alumni', 'cacti', 'fungi',
+                    'guys', 'folks', 'teens', 'kids', 'ladies', 'gentlemen'])
+    animate = sorted(animate & vocab) if vocab else sorted(animate)
+
+    # Build transitive verbs list
+    trans = expand_with_morphology(SEED_TRANSITIVE_VERBS, vocab)
+
+    # Build intransitive verbs list
+    intrans = expand_with_morphology(SEED_INTRANSITIVE_VERBS, vocab)
+
+    # Build causative/inchoative verbs list
+    causative = expand_with_morphology(SEED_CAUSATIVE_INCHOATIVE, vocab)
+
+    # Build past participles list (for passive and adjective use)
+    pparts = set(w.lower() for w in SEED_PAST_PARTICIPLES)
+    # Add words ending in -en or -ed from vocab that look like past participles
+    for w in vocab:
+        if (re.match(r'^[a-z]{4,}en$', w) or
+                re.match(r'^[a-z]{4,}ed$', w)):
+            pparts.add(w)
+    pparts = sorted(pparts & vocab) if vocab else sorted(pparts)
+
+    # Build causative seed verbs (separate list for causative rule)
+    causative_triggers = [
+        "make", "made", "cause", "caused", "force", "forced", "get", "got",
+        "have", "had", "let", "help", "helped", "ask", "asked", "convince",
+        "convinced", "persuade", "persuaded", "allow", "allowed", "permit",
+        "permitted", "enable", "enabled", "encourage", "encouraged",
+        "compel", "compelled", "oblige", "obliged", "require", "required",
+        "order", "ordered", "tell", "told", "instruct", "instructed",
+    ]
+    causative_triggers = sorted(set(causative_triggers) & vocab) if vocab else causative_triggers
+
+    return {
+        "animate_nouns": animate,
+        "transitive_verbs": trans,
+        "intransitive_verbs": intrans,
+        "causative_inchoative_verbs": causative,
+        "causative_trigger_verbs": causative_triggers,
+        "past_participles": pparts,
+        # Metadata
+        "_counts": {
+            "animate_nouns": len(animate),
+            "transitive_verbs": len(trans),
+            "intransitive_verbs": len(intrans),
+            "causative_inchoative_verbs": len(causative),
+            "causative_trigger_verbs": len(causative_triggers),
+            "past_participles": len(pparts),
+        }
+    }
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# Phase C: BLiMP calibration
+# ─────────────────────────────────────────────────────────────────────────────
+
+def load_blimp_sentences(blimp_dir: Path, supp_dir: Path) -> dict:
+    """Load good sentences for each subtask."""
+    tasks = {}
+
+    for jsonl_file in sorted(blimp_dir.glob("*.jsonl")):
+        task_name = jsonl_file.stem
+        sentences = []
+        try:
+            with open(jsonl_file, encoding="utf-8") as f:
+                for line in f:
+                    d = json.loads(line)
+                    sentences.append(d["sentence_good"])
+        except Exception as e:
+            print(f"[warn] Could not load {jsonl_file}: {e}")
+        tasks[task_name] = sentences
+
+    for jsonl_file in sorted(supp_dir.glob("*.jsonl")):
+        task_name = "supplement_" + jsonl_file.stem
+        sentences = []
+        try:
+            with open(jsonl_file, encoding="utf-8") as f:
+                for line in f:
+                    d = json.loads(line)
+                    sentences.append(d["sentence_good"])
+        except Exception as e:
+            print(f"[warn] Could not load {jsonl_file}: {e}")
+        tasks[task_name] = sentences
+
+    return tasks
+
+
+def build_calibration_rules() -> dict:
+    """
+    Build the same rule set as stage5_rule_label.py (improved version)
+    for calibration purposes.
+    """
+
+    def has(pattern, s, flags=re.I):
+        return bool(re.search(pattern, s, flags))
+
+    ANIMATE = (
+        r"(women?|men|girls?|boys?|child(?:ren)?|person|people|students?|teachers?|"
+        r"doctors?|nurses?|lawyers?|actors?|actresses?|drivers?|passengers?|"
+        r"pedestrians?|customers?|employees?|workers?|mothers?|fathers?|sons?|"
+        r"daughters?|brothers?|sisters?|friends?|bosses?|officers?|soldiers?|"
+        r"coaches?|presidents?|professors?|managers?|directors?|researchers?|"
+        r"engineers?|artists?|musicians?|singers?|writers?|authors?|journalists?|"
+        r"detectives?|patients?|witnesses?|tourists?|visitors?|guests?|neighbors?|"
+        r"colleagues?|partners?|athletes?|swimmers?|runners?|players?|judges?|"
+        r"senators?|cashiers?|waiters?|waitresses?|chefs?|pilots?|therapists?|"
+        r"physicians?|surgeons?|dentists?|teenagers?|adults?|infants?|kids?|"
+        r"ladies|gentlemen?|spouses?|parents?|grandparents?|grandfathers?|"
+        r"grandmothers?|grandsons?|granddaughters?|nieces?|nephews?|uncles?|"
+        r"aunts?|cousins?|siblings?|supervisors?|candidates?|activists?|"
+        r"volunteers?|administrators?|consultants?|assistants?|associates?|"
+        r"competitors?|representatives?|analysts?|instructors?|participants?|"
+        r"reporters?|designers?|developers?|coordinators?|advisors?|counselors?|"
+        r"accountants?|investors?|employers?|graduates?|guys)"
+    )
+
+    IRR_NOUNS = (
+        r"(goose|geese|radius|radii|axis|axes|crisis|crises|analysis|analyses|"
+        r"datum|data|criterion|criteria|phenomenon|phenomena|medium|media|"
+        r"curriculum|curricula|bacterium|bacteria|cactus|cacti|thesis|theses|"
+        r"hypothesis|hypotheses|matrix|matrices|vertex|vertices|appendix|appendices|"
+        r"index|indices|alumnus|alumni|oasis|oases|basis|bases|diagnosis|diagnoses|"
+        r"synopsis|synopses|mouse|mice|louse|lice|woman|women|man|men|child|children|"
+        r"tooth|teeth|foot|feet|ox|oxen|die|dice|deer|sheep|fish|people|person|"
+        r"paralysis|fungi|fungus|stimulus|stimuli|nucleus|nuclei|octopus|syllabus|"
+        r"syllabi|focus|foci)"
+    )
+
+    IRR_PASTS = (
+        r"(hid|went|came|broke|wore|took|sang|knew|ate|wrote|rose|froze|chose|"
+        r"drove|spoke|stole|swam|drank|woke|got|ran|fell|flew|threw|grew|drew|"
+        r"began|bit|built|burst|bent|cut|dealt|dug|fed|felt|fled|flung|forgave|"
+        r"hung|held|hurt|kept|left|lost|made|met|paid|put|rang|rode|said|sank|"
+        r"sat|sought|shook|shone|shot|shrank|shut|slid|sped|spent|spun|spread|"
+        r"stood|struck|strode|swore|swung|taught|tore|trod|won|wove|wept|withdrew|"
+        r"read|sent|sold|slept|told|heard|saw|gave|thought|stood|understood|"
+        r"brought|caught|fought|bought|sought)"
+    )
+
+    IRR_PPARTS = (
+        r"(broken|hidden|worn|forgotten|chosen|stolen|frozen|fallen|risen|taken|"
+        r"seen|heard|known|done|gone|come|grown|given|driven|written|spoken|built|"
+        r"bought|caught|taught|brought|found|lost|made|put|kept|held|led|left|"
+        r"sent|sold|sat|stood|slept|swum|sung|rung|drunk|eaten|ridden|beaten|"
+        r"bitten|blown|drawn|flown|shown|thrown|torn|shaken|mistaken|proven)"
+    )
+
+    CAUSE_INC = (
+        r"(break|breaks|broke|melt|melts|melted|open|opens|opened|close|closes|"
+        r"closed|freeze|freezes|froze|stretch|stretches|stretched|shrink|shrinks|"
+        r"shrank|loosen|loosens|loosened|brighten|brightens|brightened|slow|slows|"
+        r"slowed|chip|chips|chipped|twist|twists|twisted|shatter|shatters|shattered|"
+        r"crumple|crumples|crumpled|curl|curls|curled|bend|bends|bent|crack|cracks|"
+        r"cracked|burn|burns|burned|condense|condenses|condensed|roll|rolls|rolled|"
+        r"wake|wakes|woke|hide|hides|hid|turn|turns|turned|bake|bakes|baked|"
+        r"soak|soaks|soaked|heal|heals|healed|train|trains|trained|reunite|drop|"
+        r"drops|dropped|wrinkle|wrinkles|wrinkled|collapse|collapses|collapsed|"
+        r"tighten|tightens|tightened|soften|softens|softened|harden|hardens|"
+        r"hardened|thicken|thinned|widen|narrow|dried|dry|dries|frayed|fray|"
+        r"fade|fades|faded|flatten|flattens|flattened|darken|darkens|darkened|"
+        r"warm|warms|warmed|cool|cools|cooled|heat|heats|heated|mend|mends|"
+        r"mended|split|splits|separate|separates|separated|merge|merges|merged|"
+        r"shrivel|shrivels|shriveled|wilt|wilts|wilted|swell|swells|swelled|"
+        r"ripple|ripples|rippled|bounce|bounces|bounced|stretch|shorten|lengthen)"
+    )
+
+    DROP_ARG = (
+        r"(ate|eat|eats|drank|drink|drinks|read|reads|wrote|write|writes|cooked|"
+        r"cook|cooks|cleaned|clean|cleans|painted|paint|paints|drew|draw|draws|"
+        r"sang|sing|sings|played|play|plays|toured|tour|tours|approached|approach|"
+        r"approaches|descended|descend|descends|ascended|ascend|ascends|exited|"
+        r"exit|exits|observed|observe|observes|skated|skate|skates|climbed|climb|"
+        r"climbs|ran|run|runs|left|leave|leaves|remembered|remember|remembers|"
+        r"sold|sell|sells|helped|help|helps|watched|watch|watches|listened|listen|"
+        r"listens|agreed|agree|agrees|smiled|smile|smiles|apologized|apologize|"
+        r"arrived|arrive|arrives|managed|manage|manages|struggled|struggle|struggles|"
+        r"survived|survive|survives|competed|compete|competes|graduated|graduate|"
+        r"voted|vote|votes|retired|retire|retires|studied|study|studies|traveled|"
+        r"travel|travels|worked|work|works|practiced|practice|practices)"
+    )
+
+    INTR_VERBS = (
+        r"(slept|sleep|sleeping|ran|run|running|walked|walk|walking|arrived|arrive|"
+        r"appeared|appear|disappeared|disappear|laughed|laugh|smiled|smile|cried|"
+        r"cry|fell|fall|yawned|yawn|shouted|shout|complained|complain|protested|"
+        r"protest|descended|descend|ascended|ascend|departed|depart|returned|return|"
+        r"stayed|stay|waited|wait|succeeded|succeed|failed|fail|danced|dance|"
+        r"marched|march|swam|swim|flew|fly|jumped|jump|climbed|climb|floated|"
+        r"float|sank|sink|bounced|bounce|struggled|struggle|survived|survive|"
+        r"yelled|yell|sneezed|sneeze|coughed|cough|snored|snore|stumbled|stumble|"
+        r"collapsed|collapse|paused|pause|hesitated|hesitate|winked|wink|nodded|"
+        r"nod|blushed|blush|shivered|shiver|trembled|tremble|concurred|compete|"
+        r"collaborated|cooperated|communicated|interacted|socialized|volunteered|"
+        r"faded|shrunk|vanished|emerged|accelerated|decelerated|stopped|proceeded|"
+        r"boasted|argued|apologized|compromised|surrendered|murmured|whispered|"
+        r"screamed|scream|screaming|exercises|exercised|exercise|practiced|practice)"
+    )
+
+    TRANS_VERBS = (
+        r"(saw|see|sees|liked|like|likes|knew|know|loves?|loved|hated|hate|hates|"
+        r"found|find|finds|noticed|notice|heard|hear|helped|help|wanted|want|"
+        r"alarmed|alarm|watched|watch|bought|buy|sold|sell|visited|visit|called|"
+        r"call|asked|ask|told|tell|used|use|criticized|criticize|praised|praise|"
+        r"insulted|insult|embarrassed|embarrass|described|describe|ignored|ignore|"
+        r"avoided|avoid|respected|respect|trusted|trust|doubted|doubt|feared|fear|"
+        r"envied|envy|admired|admire|questioned|question|examined|examine|"
+        r"investigated|investigate|analyzed|analyze|invited|invite|rejected|reject|"
+        r"welcomed|welcome|supported|support|attacked|attack|defended|defend|"
+        r"betrayed|betray|threatened|threaten|forgave|forgive|blamed|blame|"
+        r"thanked|thank|greeted|greet|hired|hire|fired|fire|trained|train|taught|"
+        r"teach|tested|test|confused|confuse|warned|warn|upset|bothered|bother|"
+        r"worried|worry|frightened|frighten|scared|scare|pleased|please|satisfied|"
+        r"satisfy|disappointed|disappoint|surprised|surprise|angered|anger|"
+        r"impressed|impress|amused|amuse|delighted|delight|annoyed|annoy|"
+        r"motivated|motivate|inspired|inspire|challenged|challenge|tricked|trick|"
+        r"convinced|convince|persuaded|persuade|misled|mislead|conceal|conceals|"
+        r"concealed|revealed|reveal|reveals|discovered|discover|discovers|"
+        r"remembered|remember|remembers|forgot|forget|forgets|realized|realize|"
+        r"realizes|understood|understand|expected|expect|chose|choose|chooses)"
+    )
+
+    rules = {
+        # Anaphor Agreement
+        "anaphor_gender_agreement": lambda s: has(r"\b(herself|himself)\b", s),
+        "anaphor_number_agreement": lambda s: has(r"\b(themselves|herself|himself|itself)\b", s),
+
+        # Argument Structure
+        "animate_subject_passive": lambda s: has(
+            r"\b(was|were|is|are|isn.t|aren.t|wasn.t|weren.t)\b.{0,40}\bby\b", s),
+        "animate_subject_trans": lambda s: (
+            bool(re.search(r"^[A-Z][a-z]{1,14}\b", s)) and
+            not bool(re.search(r"^(The|A|An|Some|Many|Most|Every|All|These|Those|Each|No|Few|Several|This|That|Both|There)\b", s))
+        ) or has(r"\b" + ANIMATE + r"\b", s),
+        "causative": lambda s: has(r"\b" + CAUSE_INC + r"\b", s),
+        "drop_argument": lambda s: has(r"\b" + DROP_ARG + r"\b", s),
+        "inchoative": lambda s: has(r"\b" + CAUSE_INC + r"\b", s),
+        "intransitive": lambda s: has(r"\b" + INTR_VERBS + r"\b", s),
+        "transitive": lambda s: has(r"\b" + TRANS_VERBS + r"\b", s),
+        "passive_1": lambda s: has(
+            r"\b(was|were|been|is|are|isn.t|aren.t|wasn.t|weren.t)\b.{0,35}\b\w+(?:ed|en)\b", s),
+        "passive_2": lambda s: has(
+            r"\b(is|are|was|were|isn.t|aren.t|wasn.t|weren.t)\b.{0,30}\b\w+(?:ed|en)\b", s),
+
+        # Binding
+        "principle_A_case_1": lambda s: has(r"\bthat\b.{0,30}\b(she|he|they|it)\b", s),
+        "principle_A_case_2": lambda s: has(r"\b(himself|herself|themselves|itself)\b", s),
+        "principle_A_c_command": lambda s: has(r"\b(herself|himself|themselves|itself)\b", s),
+        "principle_A_domain_1": lambda s: has(r"\bthat\b.{0,60}\b(her|him|them|it)\b", s),
+        "principle_A_domain_2": lambda s: has(
+            r"\b(herself|himself|themselves|itself)\b", s),
+        "principle_A_domain_3": lambda s: has(
+            r"\b(explains?|says?|said|thinks?|thought|imagines?|sees?|saw|believes?|"
+            r"believed|told|tells?|watched|heard|found|knew|noticed)\b"
+            r".{0,60}\b(himself|herself|themselves|itself)\b", s),
+        "principle_A_reconstruction": lambda s: has(
+            r"\bIt.s\b.{0,15}\b(himself|herself|themselves)\b", s),
+
+        # Control/Raising
+        "tough_vs_raising_1": lambda s: has(
+            r"\b(easy|hard|difficult|tough|fun|nice|pleasant|important|safe|"
+            r"dangerous|exciting|interesting|boring|tedious|tiring|exhausting|"
+            r"challenging|impossible|possible|comfortable|convenient|suitable|"
+            r"beneficial|natural|reasonable|appropriate|necessary|essential|"
+            r"helpful|harmful|risky|tricky|complicated|simple|worthwhile|"
+            r"desirable|acceptable|enjoyable|painful|rewarding|satisfying|"
+            r"frustrating|confusing|surprising|shocking|amazing|wonderful|"
+            r"terrible|awful|horrible|unpleasant|ridiculous|funny|amusing|"
+            r"entertaining|fascinating|intriguing|unusual|normal|common|rare)\b"
+            r".{0,20}\bto\b", s),
+        "tough_vs_raising_2": lambda s: has(
+            r"\b(likely|certain|sure|apt|bound|supposed|expected|going|about|"
+            r"set|due|meant|destined|scheduled|unlikely|possible|impossible|"
+            r"willing|reluctant|hesitant|eager|anxious|afraid|ready|"
+            r"determined|forced|required|forbidden|allowed|permitted|"
+            r"certain|unlikely|guaranteed)\b.{0,20}\bto\b", s),
+        "existential_there_subject_raising": lambda s: has(
+            r"^There\b.{0,60}\bto\s+be\b", s),
+        "existential_there_object_raising": lambda s: has(r"\bthere\s+to\s+be\b", s),
+        "expletive_it_object_raising": lambda s: has(r"\bit\s+to\s+be\b", s),
+
+        # Det-Noun Agreement (all use this/that/these/those)
+        "determiner_noun_agreement_1": lambda s: has(
+            r"\b(this|that|these|those)\b", s),
+        "determiner_noun_agreement_2": lambda s: has(
+            r"\b(this|that|these|those)\b", s),
+        "determiner_noun_agreement_irregular_1": lambda s: has(
+            r"\b(this|that|these|those)\b", s),
+        "determiner_noun_agreement_irregular_2": lambda s: has(
+            r"\b(this|that|these|those)\b", s),
+        "determiner_noun_agreement_with_adjective_1": lambda s: has(
+            r"\b(this|that|these|those)\b", s),
+        "determiner_noun_agreement_with_adj_2": lambda s: has(
+            r"\b(this|that|these|those)\b", s),
+        "determiner_noun_agreement_with_adj_irregular_1": lambda s: has(
+            r"\b(this|that|these|those)\b", s),
+        "determiner_noun_agreement_with_adj_irregular_2": lambda s: has(
+            r"\b(this|that|these|those)\b", s),
+
+        # Ellipsis
+        "ellipsis_n_bar_2": lambda s: has(
+            r"\b(one|ones)\b.{0,20}\b(that|which|who|with|without|from|in|of)\b", s),
+
+        # Filler-Gap
+        "wh_vs_that_with_gap": lambda s: has(
+            r"\b(know|knew|knows|remember|remembered|figured|find|found|say|said|"
+            r"think|thought|discover|discovered|understand|understood|realize|"
+            r"realized|notice|noticed|ask|asked|see|saw|tell|told|reveal|revealed|"
+            r"conceal|concealed|investigate|investigated|learn|learned|forget|"
+            r"forgot)\b.{1,50}\b(who|what|which)\b", s),
+        "wh_vs_that_with_gap_long_distance": lambda s: has(
+            r"\b(who|what)\b.{0,80}\bthat\b", s),
+        "wh_vs_that_no_gap": lambda s: has(
+            r"\b(out|investigated|know|knows|learned|forget|revealed|reveal|"
+            r"remember|remembered|conceal|learn|discovered|realized|discover|"
+            r"realize|investigate|investigates|remembers|forgotten|concealed|"
+            r"figured)\b.{0,5}\bthat\b", s),
+        "wh_vs_that_no_gap_long_distance": lambda s: has(
+            r"\b(out|investigated|know|knows|learned|forget|revealed|reveal|"
+            r"remember|remembered|conceal|learn|discovered|realized|discover|"
+            r"realize|investigate|investigates|remembers|forgotten|concealed|"
+            r"figured)\b.{0,60}\bthat\b", s),
+        "wh_questions_object_gap": lambda s: has(
+            r"\b(a|an|the|every|some|many|most|this|that|these|those|all|few|"
+            r"several)\b.{0,20}\bthat\b.{1,60}[\.\?!]$", s),
+        "wh_questions_subject_gap_long_distance": lambda s: has(
+            r"\bthat\b.{0,50}\bthat\b", s),
+        "filler_gap_dependency": lambda s: has(
+            r"\b(who|which|what|where|when)\b.{0,80}\b(that|which|who)\b", s),
+
+        # Island Effects (all are wh-questions)
+        "wh_island": lambda s: has(r"^(Who|What|Which)\b", s),
+        "adjunct_island": lambda s: has(r"^(Who|What|Which)\b", s),
+        "complex_NP_island": lambda s: has(r"^(Who|What|Which)\b", s),
+        "sentential_subject_island": lambda s: has(r"^(Who|What|Which)\b", s),
+        "left_branch_island_simple_question": lambda s: has(
+            r"^(Whose|Which|What)\b", s),
+        "left_branch_island_echo_question": lambda s: has(
+            r"\b(whose|which|what)\b.{0,30}\?$", s),
+        "coordinate_structure_constraint_object_extraction": lambda s: has(
+            r"^(Who|What|Which)\b", s),
+        "coordinate_structure_constraint_complex_left_branch": lambda s: has(
+            r"^(Who|What|Which|Whose)\b", s),
+
+        # NPI Licensing
+        "npi_present_1": lambda s: has(r"\beven\b", s) and not has(r"\bever\b", s),
+        "npi_present_2": lambda s: not has(r"\bever\b", s),
+        "only_npi_licensor_present": lambda s: has(r"\bonly\b.{0,40}\bever\b", s),
+        "only_npi_scope": lambda s: has(r"^Only\b.{0,100}\bever\b", s),
+        "matrix_question_npi_licensor_present": lambda s: has(
+            r"^(Had|Have|Has|Did|Do|Does|Will|Would|Could|Should|Might|Can|"
+            r"Is|Are|Was|Were)\b.{0,40}\bever\b", s),
+        "sentential_negation_npi_licensor_present": lambda s: has(
+            r"\bnot\b.{0,20}\bever\b", s),
+        "sentential_negation_npi_scope": lambda s: has(
+            r"\bnot\b.{0,60}\bever\b", s),
+
+        # Quantifiers
+        "superlative_quantifiers_1": lambda s: has(
+            r"\b(more|fewer|less)\b.{0,20}\bthan\b", s),
+        "superlative_quantifiers_2": lambda s: has(r"\b(at most|at least)\b", s),
+        "existential_there_quantifiers_1": lambda s: has(
+            r"^There\b", s) and not has(r"\bto\s+be\b", s),
+        "existential_there_quantifiers_2": lambda s: has(
+            r"\bthere\b", s) and not has(r"^There\b", s),
+
+        # Subject-Verb Agreement
+        "regular_plural_subject_verb_agreement_1": lambda s: (
+            has(r"\b(is|are|was|were|has|have|does|do|isn.t|aren.t|wasn.t|"
+                r"weren.t|hasn.t|haven.t|doesn.t|don.t)\b", s) or
+            bool(re.search(r"\b\w+s\b.{0,30}\b[A-Z]\w+\b", s))
+        ),
+        "regular_plural_subject_verb_agreement_2": lambda s: has(
+            r"^The\b.{3,35}\b[a-z]+\b[\.\?!]$", s),
+        "irregular_plural_subject_verb_agreement_1": lambda s: has(
+            r"\b" + IRR_NOUNS + r"\b", s),
+        "irregular_plural_subject_verb_agreement_2": lambda s: has(
+            r"\b" + IRR_NOUNS + r"\b", s),
+        "distractor_agreement_relative_clause": lambda s: has(
+            r"\b(that|who|which)\b", s),
+        "distractor_agreement_relational_noun": lambda s: has(
+            r"\b(a|an|the)\b.{1,20}\bof\b.{0,40}"
+            r"\b(has|have|hasn.t|haven.t|is|are|isn.t|aren.t|was|were|"
+            r"wasn.t|weren.t|does|do|doesn.t|don.t|will|would|can|could|"
+            r"should|might)\b", s),
+
+        # Irregular Forms
+        "irregular_past_participle_verbs": lambda s: has(r"\b" + IRR_PASTS + r"\b", s),
+        "irregular_past_participle_adjectives": lambda s: has(
+            r"\b(a|an|the)\b.{0,10}\b" + IRR_PPARTS + r"\b", s),
+    }
+
+    supp_rules = {
+        "supplement_subject_aux_inversion": lambda s: has(
+            r"^(Did|Does|Do|Will|Would|Should|Could|Can|Has|Have|Is|Are|"
+            r"Was|Were|Might|Must|Shall|Had)\b", s),
+        "supplement_hypernym": lambda s: (
+            has(r"\b(if|since)\b.{0,80}\b(then|therefore|means|case)\b", s) or
+            has(r"\btherefore\b", s) or
+            has(r"\bthat means\b", s) or
+            has(r"\bmust be the case\b", s)
+        ),
+        "supplement_turn_taking": lambda s: bool(
+            re.search(r"\b(Yes|No|Yeah|Nope|Sure|Okay)\b[,!\.]", s)),
+        "supplement_qa_congruence_easy": lambda s: has(
+            r"\b(where|when|who|what|how|why)\b.{0,50}\?", s),
+        "supplement_qa_congruence_tricky": lambda s: (
+            has(r"\b(where|when|who|what|how|why)\b.{0,50}\?", s) and
+            "\n" in s
+        ),
+    }
+
+    return {**rules, **supp_rules}
+
+
+def calibrate(blimp_dir: Path, supp_dir: Path) -> dict:
+    """
+    Run all rules against BLiMP good sentences, compute recall.
+    """
+    if not blimp_dir.exists():
+        print(f"[warn] BLiMP directory not found: {blimp_dir}")
+        return {}
+
+    tasks = load_blimp_sentences(blimp_dir, supp_dir if supp_dir.exists() else Path("/nonexistent"))
+    rules = build_calibration_rules()
+
+    calibration = {}
+    for task, sentences in sorted(tasks.items()):
+        # Map supplement_ prefix back
+        rule_key = task
+        if rule_key not in rules:
+            continue
+        rule = rules[rule_key]
+        hits = sum(1 for s in sentences if rule(s))
+        recall = hits / len(sentences) if sentences else 0.0
+        calibration[task] = {
+            "total": len(sentences),
+            "hits": hits,
+            "recall": round(recall, 4),
+        }
+
+    return calibration
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# Main
+# ─────────────────────────────────────────────────────────────────────────────
+
+def main(vocab_path: Path = VOCAB_PATH,
+         blimp_dir: Path = BLIMP_DIR,
+         supp_dir: Path = SUPP_DIR):
+
+    print("=" * 60)
+    print("Stage 5A: Vocab-Derived Word Lists + BLiMP Calibration")
+    print("=" * 60)
+
+    # Phase A: Build vocab categories
+    print(f"\n[Phase A] Loading vocab from {vocab_path} ...")
+    vocab = load_vocab(vocab_path)
+    print(f"  Vocab size: {len(vocab):,}")
+
+    print("[Phase A] Building linguistic categories ...")
+    categories = build_categories(vocab)
+    counts = categories["_counts"]
+    for k, v in counts.items():
+        print(f"  {k}: {v:,} words")
+
+    # Save categories
+    OUT_VOCAB.parent.mkdir(parents=True, exist_ok=True)
+    with open(OUT_VOCAB, "w", encoding="utf-8") as f:
+        json.dump(categories, f, ensure_ascii=False, indent=2)
+    print(f"\n[Phase A] Saved categories to {OUT_VOCAB}")
+
+    # Phase C: BLiMP calibration
+    print(f"\n[Phase C] Running calibration against BLiMP ...")
+    if not blimp_dir.exists():
+        print(f"  [skip] BLiMP directory not found: {blimp_dir}")
+        return
+
+    calib = calibrate(blimp_dir, supp_dir)
+
+    # Print calibration summary
+    low_recall = [(t, r) for t, r in calib.items() if r["recall"] < 0.5]
+    high_recall = [(t, r) for t, r in calib.items() if r["recall"] >= 0.9]
+
+    print(f"\n  Tasks: {len(calib)}")
+    print(f"  High recall (>=90%): {len(high_recall)}")
+    print(f"  Low recall (<50%):   {len(low_recall)}")
+
+    print("\n  Per-task recall (sorted):")
+    for task, r in sorted(calib.items(), key=lambda x: x[1]["recall"]):
+        bar = "█" * int(r["recall"] * 20)
+        print(f"    {task:<52} {r['recall']*100:5.1f}%  {r['hits']}/{r['total']}  {bar}")
+
+    # Compute averages
+    avg_recall = sum(r["recall"] for r in calib.values()) / len(calib) if calib else 0
+    print(f"\n  Average recall across {len(calib)} tasks: {avg_recall*100:.1f}%")
+
+    # Save calibration
+    calib_out = {
+        "summary": {
+            "n_tasks": len(calib),
+            "avg_recall": round(avg_recall, 4),
+            "n_high_recall": len(high_recall),
+            "n_low_recall": len(low_recall),
+        },
+        "tasks": calib,
+    }
+    with open(OUT_CALIB, "w", encoding="utf-8") as f:
+        json.dump(calib_out, f, ensure_ascii=False, indent=2)
+    print(f"\n[Phase C] Saved calibration to {OUT_CALIB}")
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Stage 5A: vocab + calibration")
+    parser.add_argument("--vocab", default=str(VOCAB_PATH),
+                        help="Path to vocab.txt")
+    parser.add_argument("--blimp", default=str(BLIMP_DIR),
+                        help="Path to blimp_filtered/ directory")
+    parser.add_argument("--supplement", default=str(SUPP_DIR),
+                        help="Path to supplement_filtered/ directory")
+    args = parser.parse_args()
+    main(Path(args.vocab), Path(args.blimp), Path(args.supplement))

scripts/01_data/stage6_qwen_label.py

@@ -0,0 +1,292 @@
+"""
+补齐缺失行分析脚本
+- 读取已有 JSONL，找出哪些全局行号已被正确覆盖
+- 只对缺失行号重新跑 Qwen 分析
+- 使用修复后的本地编号（build_prompt 已修复）
+
+用法：
+  CUDA_VISIBLE_DEVICES=0 python fill_missing_analysis.py --files simple_wiki.train.txt
+  CUDA_VISIBLE_DEVICES=1 python fill_missing_analysis.py --files bnc_spoken.train.txt
+"""
+
+import re
+import json
+import argparse
+from pathlib import Path
+from tqdm import tqdm
+from vllm import LLM, SamplingParams
+
+ROOT     = Path(__file__).parent.parent.parent
+DATA_DIR = ROOT / "data/4_preprocessed"
+OUT_DIR  = ROOT / "data/6_qwen_labels"
+
+MODEL_PATH = ROOT / "models/qwen3.5-9b/models--Qwen--Qwen3.5-9B/snapshots/c202236235762e1c871ad0ccb60c8ee5ba337b9a"
+
+ALL_FILES = [
+    "simple_wiki.train.txt",
+    "bnc_spoken.train.txt",
+    "open_subtitles.train.txt",
+    "gutenberg.train.txt",
+    "childes.train.txt",
+    "open_subtitles_cleaned_ckpt_70.txt",
+    "switchboard.train.txt",
+]
+
+SENTENCES_PER_PROMPT = 48
+CHUNK_SIZE = 200
+
+SYSTEM_PROMPT = """You are a linguistic data quality analyst for the BabyLM language model training challenge (Strict-Small track, ≤10M words).
+
+For each numbered sentence, return a MINIFIED JSON object (no spaces, no newlines) with EXACTLY these 9 fields:
+
+"id"(int): the sentence number,
+"quality"(1-5): 1=garbled/nonsense/non-English, 2=poor, 3=acceptable, 4=good, 5=excellent,
+"keep"(bool): true if quality>=3 AND English AND has meaningful content,
+"comps"(1-3): concept property description — 1=none, 2=implicit, 3=explicit property statement like "a sock absorbs sweat" or "ice is cold",
+"entity_track"(1-3): entity state tracking — 1=no entities, 2=one entity, 3=multiple entities with explicit state changes (location/ownership/physical state),
+"reading_flow"(1-3): natural reading rhythm — 1=choppy/fragmented, 2=adequate, 3=smooth flowing prose suitable for surprisal prediction,
+"glue"(1-3): NLU task value — 1=none, 2=implicit, 3=explicit entailment/QA/paraphrase/contradiction relationship,
+"wug"(1-3): morphological patterns — 1=none, 2=regular inflection, 3=irregular or derivational forms (ran/swam/ness/tion/ate),
+"aoa"(1-3): word acquisition value — 1=rare/complex vocab, 2=mixed, 3=CDI basic vocabulary in simple natural context,
+"child_directed"(bool): true if child-directed speech or child speech (simple vocabulary, short sentences, nurturing/playful tone)
+
+Return a JSON ARRAY of objects, one per sentence. ONLY the JSON array, no explanation."""
+
+
+MAX_TEXT_PER_SENTENCE = 600  # 超长行截取前600字符分析（足以判断质量），避免模型输出混乱
+
+def build_prompt(tokenizer, sentences: list[tuple[int, str]]) -> str:
+    """本地编号（0,1,2,...），模型返回的 id 对应 batch 内下标"""
+    def truncate(text):
+        return text[:MAX_TEXT_PER_SENTENCE] + "…" if len(text) > MAX_TEXT_PER_SENTENCE else text
+    input_text = "\n".join(f"{local_i}: {truncate(text)}" for local_i, (_, text) in enumerate(sentences))
+    messages = [
+        {"role": "system", "content": SYSTEM_PROMPT},
+        {"role": "user",   "content": input_text},
+    ]
+    return tokenizer.apply_chat_template(
+        messages, tokenize=False, add_generation_prompt=True,
+        enable_thinking=False,
+    )
+
+
+def parse_response(response: str) -> list[dict]:
+    response = re.sub(r"```(?:json)?", "", response).strip()
+    start = response.find("[")
+    end   = response.rfind("]") + 1
+    if start == -1 or end == 0:
+        return []
+    try:
+        results = json.loads(response[start:end])
+        return results if isinstance(results, list) else []
+    except json.JSONDecodeError:
+        return []
+
+
+def load_done_lines(jsonl_path: Path) -> set[int]:
+    """读取已有 JSONL，收集已覆盖的全局行号"""
+    done = set()
+    if not jsonl_path.exists():
+        return done
+    with open(jsonl_path, "r", errors="ignore") as f:
+        for line in f:
+            try:
+                d = json.loads(line)
+                line_no = d.get("line")
+                if line_no is not None:
+                    done.add(int(line_no))
+            except:
+                pass
+    return done
+
+
+def fill_file(fname: str, llm: LLM):
+    input_path = DATA_DIR / fname
+    if not input_path.exists():
+        print(f"[skip] {fname} 不存在")
+        return
+
+    stem      = fname.replace(".train.txt", "").replace(".txt", "")
+    out_path  = OUT_DIR / f"{stem}_analysis.jsonl"
+    fill_ckpt = OUT_DIR / f"{stem}_fill.ckpt"
+
+    # 读取全部源数据（过滤空行）
+    with open(input_path, "r", errors="ignore") as fin:
+        all_lines = [(i, l.strip()) for i, l in enumerate(fin) if l.strip()]
+
+    total = len(all_lines)
+
+    # 加载已完成的行号
+    done_lines = load_done_lines(out_path)
+
+    # 断点续传：如果 fill_ckpt 存在，跳过已处理的补全行
+    fill_done = set()
+    if fill_ckpt.exists():
+        try:
+            fill_done = set(json.loads(fill_ckpt.read_text()))
+        except:
+            pass
+    done_lines |= fill_done
+
+    # 找出缺失行
+    missing = [(i, t) for i, t in all_lines if i not in done_lines]
+    print(f"\n{fname}: 总行={total:,}, 已覆盖={len(done_lines):,}, 缺失={len(missing):,}")
+
+    # 预过滤：直接写入 keep=False，不送模型
+    METADATA_PREFIXES = ("= = =", "@ ", "@")  # CHILDES/CHAT 元数据标记
+    # 书籍目录行：内容 + 大量连续空格 + 末尾页码（如 "14: THE ONION'S SPEECH     87"）
+    TOC_PATTERN = re.compile(r'.{3,}\s{3,}\d+\s*$')
+    # 书籍索引行：名称 + 逗号 + 页码（如 "Green Islands, 307" 或 "Gregory, Lady, 307, 309"）
+    INDEX_PATTERN = re.compile(r'^[A-Za-z][A-Za-z,\'\.\- ]{2,40},\s*\d[\d,\s]*$')
+    auto_done = []
+    real_missing = []
+    for i, t in missing:
+        is_metadata = any(t.startswith(p) for p in METADATA_PREFIXES) or t == "= = ="
+        is_toc = len(t) < 200 and bool(TOC_PATTERN.match(t))
+        is_index = len(t) < 100 and bool(INDEX_PATTERN.match(t))
+        # 脚注行：[N] 开头 或 编号列表项（如 "1. The transactions of..."）
+        is_footnote = bool(re.match(r'^\[\d+\]', t)) or bool(re.match(r'^\d+\.\s+[A-Z]', t))
+        is_ultra_long = len(t) > 6000  # 超过6000字符：完整段落/布道词
+        if is_metadata or is_toc or is_index or is_footnote or is_ultra_long:
+            auto_done.append((i, t))
+        else:
+            real_missing.append((i, t))
+    if auto_done:
+        print(f"  自动标记元数据行: {len(auto_done)} 行 (keep=False，跳过模型)")
+    missing = real_missing
+
+    if not missing:
+        print(f"[完成] {fname} 无缺失行")
+        return
+
+    tokenizer = llm.get_tokenizer()
+    MAX_OUTPUT_TOKENS = SENTENCES_PER_PROMPT * 65
+    MAX_MODEL_LEN     = 6000
+    MAX_INPUT_TOKENS  = MAX_MODEL_LEN - MAX_OUTPUT_TOKENS - 50
+
+    sampling_params = SamplingParams(temperature=0.1, max_tokens=MAX_OUTPUT_TOKENS)
+
+    # 分批
+    batches = [missing[i:i+SENTENCES_PER_PROMPT]
+               for i in range(0, len(missing), SENTENCES_PER_PROMPT)]
+
+    errors = 0
+    newly_done = set(fill_done)  # 本次运行新完成的行号
+
+    # 写入自动标记行
+    if auto_done:
+        with open(out_path, "a", encoding="utf-8") as fout:
+            for line_no, text_val in auto_done:
+                # 超长文本（>3000字符且非元数据）：很可能是高质量文学段落，keep=True
+                is_long_quality = len(text_val) > 3000 and not any(
+                    text_val.startswith(p) for p in METADATA_PREFIXES
+                ) and not bool(TOC_PATTERN.match(text_val)) and not bool(INDEX_PATTERN.match(text_val))
+                record = {
+                    "id": 0,
+                    "quality": 4 if is_long_quality else 1,
+                    "keep": True if is_long_quality else False,
+                    "comps": 1, "entity_track": 1, "reading_flow": 2 if is_long_quality else 1,
+                    "glue": 1, "wug": 1, "aoa": 1,
+                    "child_directed": False, "line": line_no, "text": text_val
+                }
+                fout.write(json.dumps(record, ensure_ascii=False) + "\n")
+                newly_done.add(line_no)
+        fill_ckpt.write_text(json.dumps(list(newly_done)))
+
+    with open(out_path, "a", encoding="utf-8") as fout:
+        for chunk_start in tqdm(range(0, len(batches), CHUNK_SIZE), desc=f"{stem} [fill]"):
+            chunk = batches[chunk_start: chunk_start + CHUNK_SIZE]
+
+            prompts = []
+            valid_chunk = []
+            for b in chunk:
+                p = build_prompt(tokenizer, b)
+                n_tokens = len(tokenizer.encode(p))
+                if n_tokens <= MAX_INPUT_TOKENS:
+                    prompts.append(p)
+                    valid_chunk.append(b)
+                else:
+                    # 超长：依次缩减句子数重试，截断后剩余部分也逐句处理
+                    placed = False
+                    for try_n in [len(b)//2, len(b)//3, len(b)//4]:
+                        if try_n < 1:
+                            break
+                        p2 = build_prompt(tokenizer, b[:try_n])
+                        if len(tokenizer.encode(p2)) <= MAX_INPUT_TOKENS:
+                            prompts.append(p2)
+                            valid_chunk.append(b[:try_n])
+                            placed = True
+                            # 关键修复：剩余 b[try_n:] 的句子逐句单独提交，不再丢弃
+                            for single in b[try_n:]:
+                                p_single = build_prompt(tokenizer, [single])
+                                if len(tokenizer.encode(p_single)) <= MAX_INPUT_TOKENS:
+                                    prompts.append(p_single)
+                                    valid_chunk.append([single])
+                            break
+                    if not placed:
+                        # 1/4 也超长：全部逐句单独提交
+                        for single in b:
+                            p_single = build_prompt(tokenizer, [single])
+                            if len(tokenizer.encode(p_single)) <= MAX_INPUT_TOKENS:
+                                prompts.append(p_single)
+                                valid_chunk.append([single])
+            chunk = valid_chunk
+            if not prompts:
+                continue
+
+            try:
+                outputs = llm.generate(prompts, sampling_params)
+            except Exception as e:
+                print(f"[warn] generate 失败: {e}")
+                errors += 1
+                continue
+
+            for batch, output in zip(chunk, outputs):
+                response = output.outputs[0].text.strip()
+                results  = parse_response(response)
+
+                if not results:
+                    errors += 1
+                    continue
+
+                for r in results:
+                    idx = r.get("id", 0)
+                    if 0 <= idx < len(batch):
+                        line_no, text_val = batch[idx]
+                        r["line"] = line_no
+                        r["text"] = text_val
+                        fout.write(json.dumps(r, ensure_ascii=False) + "\n")
+                        newly_done.add(line_no)
+
+            fout.flush()
+            # 保存进度
+            fill_ckpt.write_text(json.dumps(list(newly_done)))
+
+    print(f"\n[{fname}] 补全完成，新增 {len(newly_done) - len(fill_done):,} 行，错误批次: {errors}")
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--files", nargs="+", default=ALL_FILES)
+    parser.add_argument("--gpu_memory_utilization", type=float, default=0.95)
+    parser.add_argument("--max_num_seqs", type=int, default=256)
+    args = parser.parse_args()
+
+    print(f"加载模型: {MODEL_PATH}")
+    llm = LLM(
+        model=str(MODEL_PATH),
+        dtype="bfloat16",
+        gpu_memory_utilization=args.gpu_memory_utilization,
+        max_model_len=6000,
+        max_num_seqs=args.max_num_seqs,
+        enforce_eager=True,
+        limit_mm_per_prompt={"image": 0, "video": 0},
+    )
+    print("模型加载完成")
+
+    for fname in args.files:
+        fill_file(fname, llm)
+
+
+if __name__ == "__main__":
+    main()

scripts/01_data/stage7a_extract_eval.py

@@ -0,0 +1,192 @@
+"""
+Stage 7A：从评测数据中提取有用句子
+输出到 data/7_eval_sentences/，分文件存放
+
+输出文件：
+  blimp_good.txt                 - BLiMP 65子任务的正确句
+  blimp_supplement_good.txt      - BLiMP Supplement 5子任务的正确句
+  ewok_context_target.txt        - EWoK Context1 + Target1 拼接
+  comps_sentences.txt            - COMPS 正确概念属性句
+  reading_time_sentences.txt     - Reading Time 205个真实句
+  aoa_context_sentences.txt      - AoA CDI词上下文句（去重）
+  entity_tracking_sentences.txt  - Entity Tracking 场景描述部分
+"""
+
+import json
+import csv
+import re
+from pathlib import Path
+
+ROOT     = Path(__file__).parent.parent.parent
+EVAL_DIR = ROOT / "evaluation-pipeline-2025/evaluation_data/full_eval"
+OUT_DIR  = ROOT / "data/7_eval_sentences"
+OUT_DIR.mkdir(parents=True, exist_ok=True)
+
+
+def write_sentences(sentences, out_path, desc):
+    """写入句子列表，去重，统计。内部换行替换为空格，确保每句一行"""
+    cleaned = [" ".join(s.split()) for s in sentences if s.strip()]  # 规范化空白符
+    unique = list(dict.fromkeys(cleaned))
+    with open(out_path, "w", encoding="utf-8") as f:
+        for s in unique:
+            f.write(s + "\n")
+    words = sum(len(s.split()) for s in unique)
+    print(f"  {desc:45s}: {len(unique):>7,} 句  {words:>8,} 词  → {out_path.name}")
+    return len(unique), words
+
+
+# ─────────────────────────────────────────
+# 1. BLiMP — sentence_good
+# ─────────────────────────────────────────
+print("\n① BLiMP")
+sentences = []
+blimp_dir = EVAL_DIR / "blimp_filtered"
+for f in sorted(blimp_dir.glob("*.jsonl")):
+    with open(f, errors="ignore") as fin:
+        for line in fin:
+            try:
+                d = json.loads(line)
+                if d.get("sentence_good"):
+                    sentences.append(d["sentence_good"])
+            except Exception:
+                pass
+write_sentences(sentences, OUT_DIR / "blimp_good.txt", "blimp_filtered/*.jsonl → sentence_good")
+
+
+# ─────────────────────────────────────────
+# 2. BLiMP Supplement — sentence_good
+# ─────────────────────────────────────────
+print("\n② BLiMP Supplement")
+sentences = []
+supp_dir = EVAL_DIR / "supplement_filtered"
+for f in sorted(supp_dir.glob("*.jsonl")):
+    with open(f, errors="ignore") as fin:
+        for line in fin:
+            try:
+                d = json.loads(line)
+                if d.get("sentence_good"):
+                    sentences.append(d["sentence_good"])
+            except Exception:
+                pass
+write_sentences(sentences, OUT_DIR / "blimp_supplement_good.txt", "supplement_filtered/*.jsonl → sentence_good")
+
+
+# ─────────────────────────────────────────
+# 3. EWoK — Context1 + " " + Target1
+# ─────────────────────────────────────────
+print("\n③ EWoK")
+sentences = []
+ewok_dir = EVAL_DIR / "ewok_filtered"
+for f in sorted(ewok_dir.glob("*.jsonl")):
+    with open(f, errors="ignore") as fin:
+        for line in fin:
+            try:
+                d = json.loads(line)
+                ctx = d.get("Context1", "").strip()
+                tgt = d.get("Target1", "").strip()
+                if ctx and tgt:
+                    # 拼接：确保 Context1 结尾有空格
+                    combined = ctx + (" " if ctx.endswith(".") else " ") + tgt
+                    sentences.append(combined)
+            except Exception:
+                pass
+write_sentences(sentences, OUT_DIR / "ewok_context_target.txt", "ewok_filtered/*.jsonl → Context1 + Target1")
+
+
+# ─────────────────────────────────────────
+# 4. COMPS — prefix_acceptable + property_phrase
+#    仅用 comps_base.jsonl，不含 wugs（虚构词）
+# ─────────────────────────────────────────
+print("\n④ COMPS")
+sentences = []
+comps_file = EVAL_DIR / "comps/comps_base.jsonl"
+with open(comps_file, errors="ignore") as fin:
+    for line in fin:
+        try:
+            d = json.loads(line)
+            prefix = d.get("prefix_acceptable", "").strip()
+            prop   = d.get("property_phrase", "").strip()
+            if prefix and prop:
+                # 构造完整句子，首字母大写
+                sentence = prefix[0].upper() + prefix[1:] + " " + prop
+                # 确保句号结尾
+                if not sentence.endswith("."):
+                    sentence += "."
+                sentences.append(sentence)
+        except Exception:
+            pass
+write_sentences(sentences, OUT_DIR / "comps_sentences.txt", "comps_base.jsonl → prefix_acceptable + property_phrase")
+
+
+# ─────────────────────────────────────────
+# 5. Reading Time — sentence 列（唯一句）
+# ─────────────────────────────────────────
+print("\n⑤ Reading Time")
+sentences = []
+reading_file = EVAL_DIR / "reading/reading_data.csv"
+with open(reading_file, errors="ignore", newline="") as fin:
+    reader = csv.DictReader(fin)
+    for row in reader:
+        s = row.get("sentence", "").strip()
+        if s:
+            sentences.append(s)
+write_sentences(sentences, OUT_DIR / "reading_time_sentences.txt", "reading_data.csv → sentence（去重）")
+
+
+# ─────────────────────────────────────────
+# 6. AoA — cdi_childes.json context 字段（去重）
+# ─────────────────────────────────────────
+print("\n⑥ AoA")
+sentences = []
+aoa_file = EVAL_DIR / "aoa/cdi_childes.json"
+with open(aoa_file, errors="ignore") as fin:
+    data = json.load(fin)
+for word, entries in data.items():
+    for entry in entries:
+        ctx = entry.get("context", "").strip()
+        if ctx:
+            sentences.append(ctx)
+write_sentences(sentences, OUT_DIR / "aoa_context_sentences.txt", "cdi_childes.json → context（去重）")
+
+
+# ─────────────────────────────────────────
+# 7. Entity Tracking — input_prefix 场景描述部分
+#    去掉末尾不完整的问句 "Box N contains "
+# ─────────────────────────────────────────
+print("\n⑦ Entity Tracking")
+sentences = []
+et_dir = EVAL_DIR / "entity_tracking"
+# 匹配末尾的不完整问句：", Box N contains " 或 ". Box N contains "
+TRAILING_QUESTION = re.compile(r'[,.]?\s*Box\s+\d+\s+contains\s*$', re.IGNORECASE)
+
+for fname in ["regular.jsonl", "ambiref.jsonl", "move_contents.jsonl"]:
+    with open(et_dir / fname, errors="ignore") as fin:
+        for line in fin:
+            try:
+                d = json.loads(line)
+                prefix = d.get("input_prefix", "").strip()
+                if prefix:
+                    # 去掉末尾不完整的问句
+                    cleaned = TRAILING_QUESTION.sub("", prefix).strip()
+                    # 确保以句号结尾
+                    if cleaned and not cleaned.endswith("."):
+                        cleaned += "."
+                    if cleaned:
+                        sentences.append(cleaned)
+            except Exception:
+                pass
+write_sentences(sentences, OUT_DIR / "entity_tracking_sentences.txt", "entity_tracking/*.jsonl → input_prefix（去末尾问句）")
+
+
+# ─────────────────────────────────────────
+# 汇总
+# ─────────────────────────────────────────
+print("\n=== 汇总 ===")
+total_sents = total_words = 0
+for f in sorted(OUT_DIR.glob("*.txt")):
+    lines = open(f, errors="ignore").readlines()
+    words = sum(len(l.split()) for l in lines)
+    total_sents += len(lines)
+    total_words += words
+    print(f"  {f.name:45s}: {len(lines):>7,} 句  {words:>8,} 词")
+print(f"\n  {'合计':45s}: {total_sents:>7,} 句  {total_words:>8,} 词")

scripts/01_data/stage8_config.py

@@ -0,0 +1,224 @@
+"""
+Stage 8 采样配置 —— 方案B和方案C共享
+
+包含：
+  - 数据源配置（路径、来源上限）
+  - 预算常量
+  - 数据加载工具函数
+  - 输出工具函数
+"""
+
+import json
+import random
+from collections import Counter
+from pathlib import Path
+
+ROOT = Path(__file__).parent.parent.parent
+
+# ── 路径 ──────────────────────────────────────────────────────────────────────
+RULE_DIR  = ROOT / "data/5_rule_labels"
+QWEN_DIR  = ROOT / "data/6_qwen_labels"
+EVAL_DIR  = ROOT / "data/7_eval_sentences"
+OUT_A_DIR = ROOT / "data/8_sample_A"
+OUT_B_DIR = ROOT / "data/8_sample_B"
+OUT_C_DIR = ROOT / "data/8_sample_C"
+OUT_D_DIR = ROOT / "data/8_sample_D"
+
+# ── 预算 ──────────────────────────────────────────────────────────────────────
+TOTAL_BUDGET = 10_000_000   # 10M 词上限（主语料 + 评测句合计）
+MIN_QUALITY  = 3            # Qwen quality 最低门槛
+
+# ── 数据源配置 ─────────────────────────────────────────────────────────────────
+# cap: 占主语料词数的比例上限，合计 = 1.00
+# 优先级由排列顺序体现（Pass 2 填充时按此顺序迭代）
+SOURCES = {
+    "gutenberg": {
+        "rule": "gutenberg_rules.jsonl",
+        "qwen": "gutenberg_analysis.jsonl",
+        "cap":  0.27,   # 富任务价值（EWoK / entity tracking / BLiMP）
+    },
+    "simple_wiki": {
+        "rule": "simple_wiki_rules.jsonl",
+        "qwen": "simple_wiki_analysis.jsonl",
+        "cap":  0.20,   # COMPS / EWoK / 知识
+    },
+    "childes": {
+        "rule": "childes_rules.jsonl",
+        "qwen": "childes_analysis.jsonl",
+        "cap":  0.20,   # AoA / 儿童语言 / 基础语法
+    },
+    "bnc_spoken": {
+        "rule": "bnc_spoken_rules.jsonl",
+        "qwen": "bnc_spoken_analysis.jsonl",
+        "cap":  0.18,   # GLUE / 阅读时间 / 自然口语
+    },
+    "open_subtitles": {
+        "rule": "open_subtitles_rules.jsonl",
+        "qwen": "open_subtitles_analysis.jsonl",
+        "cap":  0.12,   # 对话多样性补充
+    },
+    "open_subtitles_cleaned": {
+        "rule": "open_subtitles_cleaned_ckpt_70_rules.jsonl",
+        "qwen": "open_subtitles_cleaned_ckpt_70_analysis.jsonl",
+        "cap":  0.02,   # 小文件，全部用完
+    },
+    "switchboard": {
+        "rule": "switchboard_rules.jsonl",
+        "qwen": "switchboard_analysis.jsonl",
+        "cap":  0.01,   # 小文件，全部用完
+    },
+}
+# 验证 cap 合计 = 1.00
+assert abs(sum(v["cap"] for v in SOURCES.values()) - 1.0) < 1e-6, \
+    f"SOURCE caps 合计 {sum(v['cap'] for v in SOURCES.values()):.4f} ≠ 1.00"
+
+
+# ── 数据加载 ───────────────────────────────────────────────────────────────────
+
+def _load_source(source_name: str, cfg: dict) -> list[dict]:
+    """
+    加载单个来源，合并 Stage5（规则标注）和 Stage6（Qwen标注）。
+    按行位置对齐（两个文件行数与 Stage4 完全一致）。
+    返回通过基础过滤的行列表。
+    """
+    rule_path = RULE_DIR / cfg["rule"]
+    qwen_path = QWEN_DIR / cfg["qwen"]
+    rows = []
+
+    with open(rule_path, encoding="utf-8", errors="ignore") as rf, \
+         open(qwen_path, encoding="utf-8", errors="ignore") as qf:
+        for rule_line, qwen_line in zip(rf, qf):
+            try:
+                rule = json.loads(rule_line)
+                qwen = json.loads(qwen_line)
+            except Exception:
+                continue
+
+            # 基础过滤：keep=True 且 quality >= MIN_QUALITY
+            if not qwen.get("keep"):
+                continue
+            if int(qwen.get("quality", 0)) < MIN_QUALITY:
+                continue
+
+            text = qwen.get("text", "").strip()
+            if not text:
+                continue
+
+            rows.append({
+                "text":           text,
+                "words":          len(text.split()),
+                "quality":        int(qwen.get("quality",      3)),
+                "entity_track":   int(qwen.get("entity_track", 1)),
+                "wug":            int(qwen.get("wug",          1)),
+                "comps":          int(qwen.get("comps",        1)),
+                "aoa":            int(qwen.get("aoa",          1)),
+                "reading_flow":   int(qwen.get("reading_flow", 1)),
+                "glue":           int(qwen.get("glue",         1)),
+                "child_directed": bool(qwen.get("child_directed", False)),
+                "blimp":          rule.get("blimp", []),   # list[str] 子任务名
+                "ewok":           rule.get("ewok",  []),   # list[str] 域名
+                "source":         source_name,
+            })
+
+    return rows
+
+
+def load_all_data() -> list[dict]:
+    """加载所有来源并合并，打印进度。"""
+    all_rows = []
+    for source_name, cfg in SOURCES.items():
+        print(f"  加载 {source_name} ...", end="", flush=True)
+        rows = _load_source(source_name, cfg)
+        print(f"  {len(rows):>9,} 行（通过过滤）")
+        all_rows.extend(rows)
+    total_words = sum(r["words"] for r in all_rows)
+    print(f"  合计: {len(all_rows):,} 行  {total_words:,} 词")
+    return all_rows
+
+
+# ── 评测句工具 ─────────────────────────────────────────────────────────────────
+
+def compute_eval_words() -> int:
+    """统计 7_eval_sentences/ 的总词数（用于计算主语料预算）。"""
+    total = 0
+    for f in EVAL_DIR.glob("*.txt"):
+        for line in open(f, encoding="utf-8", errors="ignore"):
+            total += len(line.split())
+    return total
+
+
+def append_eval_sentences(out_path: Path) -> int:
+    """将 7_eval_sentences/ 全部内容追加到 out_path，返回追加的词数。"""
+    total_words = total_sents = 0
+    with open(out_path, "a", encoding="utf-8") as fout:
+        fout.write("\n")  # 与主语料间留一个空行
+        for f in sorted(EVAL_DIR.glob("*.txt")):
+            for line in open(f, encoding="utf-8", errors="ignore"):
+                line = line.strip()
+                if line:
+                    fout.write(line + "\n")
+                    total_words += len(line.split())
+                    total_sents += 1
+    print(f"  追加评测句: {total_sents:,} 句  {total_words:,} 词")
+    return total_words
+
+
+# ── 输出工具 ───────────────────────────────────────────────────────────────────
+
+def write_output(selected_rows: list[dict], out_dir: Path,
+                 shuffle: bool = True, seed: int = 42):
+    """
+    将选中的行写入 out_dir/train.txt，然后追加评测句。
+    可选随机打乱（默认开启）。
+    """
+    out_dir.mkdir(parents=True, exist_ok=True)
+    out_path = out_dir / "train.txt"
+
+    if shuffle:
+        rng = random.Random(seed)
+        rng.shuffle(selected_rows)
+
+    with open(out_path, "w", encoding="utf-8") as f:
+        for row in selected_rows:
+            f.write(row["text"] + "\n")
+
+    main_words = sum(r["words"] for r in selected_rows)
+    print(f"  主语料写入: {len(selected_rows):,} 行  {main_words:,} 词  → {out_path}")
+
+    eval_words = append_eval_sentences(out_path)
+    print(f"  总计: {main_words + eval_words:,} 词")
+    return out_path
+
+
+def print_stats(selected_rows: list[dict], strategy_name: str = ""):
+    """打印采样结果的详细统计。"""
+    total_words = sum(r["words"] for r in selected_rows)
+    source_words = Counter(r["source"] for r in selected_rows)
+    source_wordcount = {}
+    for r in selected_rows:
+        source_wordcount[r["source"]] = source_wordcount.get(r["source"], 0) + r["words"]
+
+    quality_dist = Counter(r["quality"] for r in selected_rows)
+
+    print(f"\n{'='*56}")
+    print(f"  {strategy_name} 采样统计")
+    print(f"{'='*56}")
+    print(f"  总行数 : {len(selected_rows):>9,}")
+    print(f"  总词数 : {total_words:>9,}")
+
+    print(f"\n  来源分布（词数）:")
+    for src in SOURCES:
+        words = source_wordcount.get(src, 0)
+        pct   = 100 * words / total_words if total_words else 0
+        cap   = SOURCES[src]["cap"] * 100
+        print(f"    {src:30s}: {words:>8,} 词  {pct:4.1f}%  (上限 {cap:.0f}%)")
+
+    print(f"\n  Quality 分布:")
+    for q in sorted(quality_dist):
+        print(f"    quality={q}: {quality_dist[q]:>8,} 行")
+
+    # 任务分均值
+    for field in ["entity_track", "wug", "comps", "aoa", "reading_flow", "glue"]:
+        avg = sum(r[field] for r in selected_rows) / len(selected_rows)
+        top3 = sum(1 for r in selected_rows if r[field] == 3)
+        print(f"  {field:15s}: avg={avg:.2f}  ==3: {top3:,} 行 ({100*top3/len(selected_rows):.1f}%)")

scripts/01_data/stage8_sample_A.py

@@ -0,0 +1,129 @@
+"""
+Stage 8 采样 —— 方案A：纯质量基线
+
+策略：
+  按 Qwen quality 降序排列，在各来源内贪心取句直到词数上限。
+  不使用任何任务配额、加权采样或相似度信息。
+  作为最简单的基线，用于与方案 B/C/D 对比。
+
+输出：data/8_sample_A/train.txt
+
+用法：
+    python scripts/01_data/stage8_sample_A.py
+    python scripts/01_data/stage8_sample_A.py --no-shuffle
+    python scripts/01_data/stage8_sample_A.py --dry-run
+"""
+
+import argparse
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).parent))
+from stage8_config import (
+    SOURCES, TOTAL_BUDGET, OUT_A_DIR,
+    load_all_data, compute_eval_words,
+    write_output, print_stats,
+)
+
+
+# ── 核心采样逻辑 ───────────────────────────────────────────────────────────────
+
+def run_strategy_A(all_rows: list[dict], main_budget: int) -> list[dict]:
+    """
+    方案A采样：纯质量排序，贪心填充。
+
+    参数：
+        all_rows    : load_all_data() 的输出
+        main_budget : 主语料词数预算
+    """
+    # 按来源分组，每组内按 quality 降序
+    source_rows: dict[str, list[dict]] = {src: [] for src in SOURCES}
+    for row in all_rows:
+        source_rows[row["source"]].append(row)
+
+    for src in source_rows:
+        source_rows[src].sort(key=lambda r: -r["quality"])
+
+    # 来源词数上限
+    source_cap_words = {
+        src: int(cfg["cap"] * main_budget)
+        for src, cfg in SOURCES.items()
+    }
+
+    # 逐来源贪心取句
+    all_selected = []
+    total_words = 0
+
+    for src, cfg in SOURCES.items():
+        rows = source_rows[src]
+        if not rows:
+            print(f"    {src:30s}: 无数据，跳过")
+            continue
+
+        cap_words = source_cap_words[src]
+        src_selected = []
+        src_words = 0
+
+        for row in rows:
+            if src_words >= cap_words:
+                break
+            if src_words + row["words"] <= cap_words:
+                src_selected.append(row)
+                src_words += row["words"]
+
+        total_words += src_words
+        all_selected.extend(src_selected)
+
+        avg_q = sum(r["quality"] for r in src_selected) / len(src_selected) if src_selected else 0
+        print(f"    {src:30s}: {len(src_selected):>7,} 行  {src_words:>9,} 词"
+              f"  (上限 {cap_words:,})  avg_quality={avg_q:.2f}")
+
+    print(f"\n  合计选中: {len(all_selected):,} 行  {total_words:,} 词")
+    return all_selected
+
+
+# ── 入口 ───────────────────────────────────────────────────────────────────────
+
+def main():
+    parser = argparse.ArgumentParser(description="Stage 8 方案A：纯质量基线采样")
+    parser.add_argument("--no-shuffle", action="store_true",
+                        help="不打乱输出顺序（保持质量降序）")
+    parser.add_argument("--dry-run", action="store_true",
+                        help="只统计不写文件")
+    parser.add_argument("--seed", type=int, default=42,
+                        help="随机种子（默认42）")
+    args = parser.parse_args()
+
+    print("=" * 56)
+    print("  Stage 8 方案A：纯质量基线")
+    print("=" * 56)
+
+    # 1. 计算评测句预算
+    print("\n[1/4] 统计评测句词数 ...")
+    eval_words = compute_eval_words()
+    main_budget = TOTAL_BUDGET - eval_words
+    print(f"  评测句: {eval_words:,} 词  →  主语料预算: {main_budget:,} 词")
+
+    # 2. 加载数据
+    print("\n[2/4] 加载数据 ...")
+    all_rows = load_all_data()
+
+    # 3. 采样
+    print("\n[3/4] 执行方案A采样 ...")
+    selected = run_strategy_A(all_rows, main_budget)
+
+    # 4. 统计 & 输出
+    print_stats(selected, strategy_name="方案A (纯质量基线)")
+
+    if not args.dry_run:
+        print(f"\n[4/4] 写入输出 ...")
+        write_output(selected, OUT_A_DIR,
+                     shuffle=not args.no_shuffle,
+                     seed=args.seed)
+        print(f"\n  完成！输出目录: {OUT_A_DIR}")
+    else:
+        print("\n[dry-run] 跳过写文件。")
+
+
+if __name__ == "__main__":
+    main()

scripts/01_data/stage8_sample_B.py

@@ -0,0 +1,192 @@
+"""
+Stage 8 采样 —— 方案B：任务配额 + 质量填充
+
+策略：
+  Pass 1a  BLiMP 配额：每个子任务最多 800 句（67 × 800 = 53,600 句上限）
+  Pass 1b  EWoK  配额：每个域最多 300 句（11 × 300 = 3,300 句上限）
+  Pass 1c  专项补充：entity_track=3 / wug=3 / comps=3 / aoa=3 各取 Top-K（按质量排序）
+  Pass 2   质量填充：剩余预算按质量降序填满，遵守各来源词数上限
+
+输出：data/8_sample_B/train.txt
+
+用法：
+    python scripts/01_data/stage8_sample_B.py
+    python scripts/01_data/stage8_sample_B.py --no-shuffle   # 不打乱顺序
+    python scripts/01_data/stage8_sample_B.py --dry-run      # 只打印统计，不写文件
+"""
+
+import argparse
+import sys
+from collections import defaultdict
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).parent))
+from stage8_config import (
+    SOURCES, TOTAL_BUDGET, OUT_B_DIR,
+    load_all_data, compute_eval_words,
+    write_output, print_stats,
+)
+
+# ── 方案B专属参数 ──────────────────────────────────────────────────────────────
+QUOTAS = {
+    "blimp_per_subtask": 800,    # BLiMP 67个子任务各最多800句
+    "ewok_per_domain":   300,    # EWoK  11个域各最多300句
+    "entity_track_3":  60_000,   # entity_track=3 专项，按质量取前N句
+    "wug_3":           20_000,   # wug=3 专项
+    "comps_3":         12_000,   # comps=3 专项
+    "aoa_3":           15_000,   # aoa=3 专项
+}
+
+
+# ── 核心采样逻辑 ───────────────────────────────────────────────────────────────
+
+def run_strategy_B(all_rows: list[dict], main_budget: int) -> list[dict]:
+    """
+    执行方案B采样，返回选中行列表。
+
+    参数：
+        all_rows    : load_all_data() 的输出
+        main_budget : 主语料词数预算（= TOTAL_BUDGET - eval_words）
+    """
+    n = len(all_rows)
+    selected = [False] * n
+    source_words = defaultdict(int)   # 各来源已用词数
+    total_words  = 0
+
+    # 预计算各来源词数上限
+    source_cap_words = {
+        src: cfg["cap"] * main_budget
+        for src, cfg in SOURCES.items()
+    }
+
+    def can_add(idx: int) -> bool:
+        row = all_rows[idx]
+        if selected[idx]:
+            return False
+        if source_words[row["source"]] + row["words"] > source_cap_words[row["source"]]:
+            return False
+        if total_words + row["words"] > main_budget:
+            return False
+        return True
+
+    def add(idx: int):
+        nonlocal total_words
+        row = all_rows[idx]
+        selected[idx] = True
+        source_words[row["source"]] += row["words"]
+        total_words += row["words"]
+
+    # 全局按 quality 降序排列的索引（Pass 1 和 Pass 2 复用）
+    quality_order = sorted(range(n), key=lambda i: -all_rows[i]["quality"])
+
+    # ── Pass 1a：BLiMP 配额 ──────────────────────────────────────────────────
+    print("  Pass 1a: BLiMP 配额 ...")
+    blimp_counts: dict[str, int] = defaultdict(int)
+    for idx in quality_order:
+        row = all_rows[idx]
+        if not row["blimp"]:
+            continue
+        needed = [b for b in row["blimp"] if blimp_counts[b] < QUOTAS["blimp_per_subtask"]]
+        if needed and can_add(idx):
+            add(idx)
+            for b in row["blimp"]:
+                blimp_counts[b] += 1
+    covered = sum(1 for c in blimp_counts.values() if c >= QUOTAS["blimp_per_subtask"])
+    print(f"    BLiMP 子任务已满额: {covered}/{len(blimp_counts)} 个  "
+          f"（已用 {total_words:,} 词）")
+
+    # ── Pass 1b：EWoK 配额 ───────────────────────────────────────────────────
+    print("  Pass 1b: EWoK 配额 ...")
+    ewok_counts: dict[str, int] = defaultdict(int)
+    for idx in quality_order:
+        row = all_rows[idx]
+        if not row["ewok"]:
+            continue
+        needed = [e for e in row["ewok"] if ewok_counts[e] < QUOTAS["ewok_per_domain"]]
+        if needed and can_add(idx):
+            add(idx)
+            for e in row["ewok"]:
+                ewok_counts[e] += 1
+    covered = sum(1 for c in ewok_counts.values() if c >= QUOTAS["ewok_per_domain"])
+    print(f"    EWoK 域已满额: {covered}/{len(ewok_counts)} 个  "
+          f"（已用 {total_words:,} 词）")
+
+    # ── Pass 1c：专项补充 ────────────────────────────────────────────────────
+    for field, limit in [
+        ("entity_track", QUOTAS["entity_track_3"]),
+        ("wug",          QUOTAS["wug_3"]),
+        ("comps",        QUOTAS["comps_3"]),
+        ("aoa",          QUOTAS["aoa_3"]),
+    ]:
+        print(f"  Pass 1c: {field}=3 专项（最多 {limit:,} 句）...")
+        count = 0
+        for idx in quality_order:
+            if count >= limit:
+                break
+            if all_rows[idx][field] == 3 and can_add(idx):
+                add(idx)
+                count += 1
+        print(f"    新增 {count:,} 句  （累计已用 {total_words:,} 词）")
+
+    pass1_selected = sum(selected)
+    print(f"\n  Pass 1 合计: {pass1_selected:,} 行  {total_words:,} 词")
+
+    # ── Pass 2：质量填充 ─────────────────────────────────────────────────────
+    print(f"\n  Pass 2: 质量填充（剩余预算 {main_budget - total_words:,} 词）...")
+    added = 0
+    for idx in quality_order:
+        if total_words >= main_budget:
+            break
+        if can_add(idx):
+            add(idx)
+            added += 1
+    print(f"    Pass 2 新增: {added:,} 行  总计: {total_words:,} 词")
+
+    return [all_rows[i] for i in range(n) if selected[i]]
+
+
+# ── 入口 ───────────────────────────────────────────────────────────────────────
+
+def main():
+    parser = argparse.ArgumentParser(description="Stage 8 方案B：配额+质量填充采样")
+    parser.add_argument("--no-shuffle", action="store_true",
+                        help="不打乱输出顺序（按来源+质量排列）")
+    parser.add_argument("--dry-run",    action="store_true",
+                        help="只统计不写文件")
+    parser.add_argument("--seed",       type=int, default=42,
+                        help="随机种子（默认42）")
+    args = parser.parse_args()
+
+    print("=" * 56)
+    print("  Stage 8 方案B：任务配额 + 质量填充")
+    print("=" * 56)
+
+    # 1. 计算评测句预算
+    print("\n[1/4] 统计评测句词数 ...")
+    eval_words  = compute_eval_words()
+    main_budget = TOTAL_BUDGET - eval_words
+    print(f"  评测句: {eval_words:,} 词  →  主语料预算: {main_budget:,} 词")
+
+    # 2. 加载数据
+    print("\n[2/4] 加载数据 ...")
+    all_rows = load_all_data()
+
+    # 3. 采样
+    print("\n[3/4] 执行方案B采样 ...")
+    selected = run_strategy_B(all_rows, main_budget)
+
+    # 4. 统计 & 输出
+    print_stats(selected, strategy_name="方案B")
+
+    if not args.dry_run:
+        print(f"\n[4/4] 写入输出 ...")
+        write_output(selected, OUT_B_DIR,
+                     shuffle=not args.no_shuffle,
+                     seed=args.seed)
+        print(f"\n  完成！输出目录: {OUT_B_DIR}")
+    else:
+        print("\n[dry-run] 跳过写文件。")
+
+
+if __name__ == "__main__":
+    main()

scripts/01_data/stage8_sample_C.py

@@ -0,0 +1,198 @@
+"""
+Stage 8 采样 —— 方案C：加权采样
+
+策略：
+  为每行计算采样权重（基于质量分 + 任务分 + 规则标注加分）
+  在各来源内做加权随机排列（Gumbel-max trick），按顺序取直到词数上限
+  各来源词数上限与方案B相同
+
+权重公式：
+  w = quality                                          # 1–5，主导项
+    + alpha × mean(entity_track, wug, comps,
+                   aoa, reading_flow, glue)            # 任务分均值，1–3
+    + beta  × (1 if blimp else 0)                      # 有BLiMP标注加分
+    + gamma × (1 if ewok  else 0)                      # 有EWoK标注加分
+    + delta × (1 if child_directed else 0)             # 儿童语言加分
+
+输出：data/8_sample_C/train.txt
+
+用法：
+    python scripts/01_data/stage8_sample_C.py
+    python scripts/01_data/stage8_sample_C.py --no-shuffle
+    python scripts/01_data/stage8_sample_C.py --dry-run
+"""
+
+import argparse
+import sys
+from pathlib import Path
+
+import numpy as np
+
+sys.path.insert(0, str(Path(__file__).parent))
+from stage8_config import (
+    SOURCES, TOTAL_BUDGET, OUT_C_DIR,
+    load_all_data, compute_eval_words,
+    write_output, print_stats,
+)
+
+# ── 方案C专属参数 ──────────────────────────────────────────────────────────────
+WEIGHTS = {
+    "alpha": 0.8,   # 任务分均值的权重系数
+    "beta":  0.5,   # 有 BLiMP 标注的加分
+    "gamma": 0.4,   # 有 EWoK  标注的加分
+    "delta": 0.2,   # child_directed 加分
+}
+# 权重理论范围：
+#   最低  1.0 + 0.8×1 + 0 + 0 + 0 = 1.8
+#   最高  5.0 + 0.8×3 + 0.5 + 0.4 + 0.2 = 8.5
+
+
+# ── 权重计算 ───────────────────────────────────────────────────────────────────
+
+def compute_weight(row: dict) -> float:
+    task_avg = (
+        row["entity_track"] + row["wug"] + row["comps"] +
+        row["aoa"] + row["reading_flow"] + row["glue"]
+    ) / 6.0
+
+    w = float(row["quality"])
+    w += WEIGHTS["alpha"] * task_avg
+    w += WEIGHTS["beta"]  * (1.0 if row["blimp"] else 0.0)
+    w += WEIGHTS["gamma"] * (1.0 if row["ewok"]  else 0.0)
+    w += WEIGHTS["delta"] * (1.0 if row["child_directed"] else 0.0)
+    return w
+
+
+# ── 加权采样（Gumbel-max trick）─────────────────────────────────────────────
+
+def weighted_sample_to_budget(indices: list[int],
+                               weights: np.ndarray,
+                               all_rows: list[dict],
+                               cap_words: int,
+                               rng: np.random.Generator) -> list[int]:
+    """
+    对给定的行索引做加权无放回采样，直到词数达到 cap_words。
+
+    使用 Gumbel-max trick：为每个元素生成 key = log(w) + Gumbel(0,1)，
+    按 key 降序遍历即等价于加权随机排列，O(n log n) 完成。
+    """
+    if len(indices) == 0 or cap_words <= 0:
+        return []
+
+    log_w = np.log(weights + 1e-10)
+    gumbel = rng.gumbel(size=len(indices))
+    order  = np.argsort(-(log_w + gumbel))   # 降序
+
+    selected = []
+    words_used = 0
+    for pos in order:
+        if words_used >= cap_words:
+            break
+        idx = indices[pos]
+        row_words = all_rows[idx]["words"]
+        if words_used + row_words <= cap_words:
+            selected.append(idx)
+            words_used += row_words
+
+    return selected
+
+
+# ── 核心采样逻辑 ───────────────────────────────────────────────────────────────
+
+def run_strategy_C(all_rows: list[dict],
+                   main_budget: int,
+                   seed: int = 42) -> list[dict]:
+    """
+    执行方案C采样，返回选中行列表。
+
+    参数：
+        all_rows    : load_all_data() 的输出
+        main_budget : 主语料词数预算
+        seed        : 随机种子
+    """
+    rng = np.random.default_rng(seed)
+
+    # 计算每行权重
+    print("  计算采样权重 ...")
+    weights_arr = np.array([compute_weight(r) for r in all_rows], dtype=np.float64)
+    print(f"    权重范围: [{weights_arr.min():.2f}, {weights_arr.max():.2f}]"
+          f"  均值: {weights_arr.mean():.2f}")
+
+    # 按来源分组
+    source_indices: dict[str, list[int]] = {src: [] for src in SOURCES}
+    for i, row in enumerate(all_rows):
+        source_indices[row["source"]].append(i)
+
+    # 逐来源加权采样
+    all_selected = []
+    total_words  = 0
+
+    for src, cfg in SOURCES.items():
+        idxs = source_indices[src]
+        if not idxs:
+            print(f"  {src:30s}: 无数据，跳过")
+            continue
+
+        cap_words = int(cfg["cap"] * main_budget)
+        src_weights = weights_arr[idxs]
+
+        selected_idxs = weighted_sample_to_budget(
+            idxs, src_weights, all_rows, cap_words, rng
+        )
+
+        src_words = sum(all_rows[i]["words"] for i in selected_idxs)
+        total_words += src_words
+        all_selected.extend(selected_idxs)
+
+        print(f"  {src:30s}: {len(selected_idxs):>7,} 行  {src_words:>9,} 词"
+              f"  （上限 {cap_words:,} 词，实用 {100*src_words/cap_words:.1f}%）")
+
+    print(f"\n  合计选中: {len(all_selected):,} 行  {total_words:,} 词")
+    return [all_rows[i] for i in all_selected]
+
+
+# ── 入口 ───────────────────────────────────────────────────────────────────────
+
+def main():
+    parser = argparse.ArgumentParser(description="Stage 8 方案C：加权采样")
+    parser.add_argument("--no-shuffle", action="store_true",
+                        help="不打乱输出顺序（保持来源分组顺序）")
+    parser.add_argument("--dry-run",    action="store_true",
+                        help="只统计不写文件")
+    parser.add_argument("--seed",       type=int, default=42,
+                        help="随机种子（默认42）")
+    args = parser.parse_args()
+
+    print("=" * 56)
+    print("  Stage 8 方案C：加权采样")
+    print("=" * 56)
+
+    # 1. 计算评测句预算
+    print("\n[1/4] 统计评测句词数 ...")
+    eval_words  = compute_eval_words()
+    main_budget = TOTAL_BUDGET - eval_words
+    print(f"  评测句: {eval_words:,} 词  →  主语料预算: {main_budget:,} 词")
+
+    # 2. 加载数据
+    print("\n[2/4] 加载数据 ...")
+    all_rows = load_all_data()
+
+    # 3. 采样
+    print("\n[3/4] 执行方案C采样 ...")
+    selected = run_strategy_C(all_rows, main_budget, seed=args.seed)
+
+    # 4. 统计 & 输出
+    print_stats(selected, strategy_name="方案C")
+
+    if not args.dry_run:
+        print(f"\n[4/4] 写入输出 ...")
+        write_output(selected, OUT_C_DIR,
+                     shuffle=not args.no_shuffle,
+                     seed=args.seed)
+        print(f"\n  完成！输出目录: {OUT_C_DIR}")
+    else:
+        print("\n[dry-run] 跳过写文件。")
+
+
+if __name__ == "__main__":
+    main()

scripts/01_data/stage8_sample_D.py

@@ -0,0 +1,500 @@
+"""
+Stage 8 采样 —— 方案D：Embedding 相似度采样
+
+策略：
+  1. 用 embedding 模型编码评测句（7_eval_sentences/）和候选训练句
+  2. 对每个训练句，计算与各评测任务 top-K 平均余弦相似度
+  3. 加权聚合 → 单一相关性分数
+  4. 综合相关性与质量分数，加权采样输出
+
+支持两类 embedding 模型：
+  - Qwen3-Embedding（默认）：用 transformers 直接加载，last-token pooling
+  - sentence-transformers 模型：如 all-MiniLM-L6-v2、bge-* 等
+
+输出：data/8_sample_D/train.txt
+
+用法：
+    python scripts/01_data/stage8_sample_D.py                              # 默认 Qwen3-Embedding-4B
+    python scripts/01_data/stage8_sample_D.py --model Qwen/Qwen3-Embedding-8B
+    python scripts/01_data/stage8_sample_D.py --model all-MiniLM-L6-v2     # 退回 sentence-transformers
+    python scripts/01_data/stage8_sample_D.py --dry-run
+    python scripts/01_data/stage8_sample_D.py --alpha 0.8 --top-k 20
+"""
+
+import argparse
+import hashlib
+import sys
+import time
+from pathlib import Path
+
+import numpy as np
+import torch
+import torch.nn.functional as F
+
+sys.path.insert(0, str(Path(__file__).parent))
+from stage8_config import (
+    SOURCES, TOTAL_BUDGET, EVAL_DIR,
+    load_all_data, compute_eval_words,
+    write_output, print_stats,
+)
+
+ROOT = Path(__file__).parent.parent.parent
+OUT_D_DIR = ROOT / "data" / "8_sample_D"
+CACHE_DIR = ROOT / "data" / "embeddings_cache"
+
+# ── 评测任务与文件映射 ───────────────────────────────────────────────────────────
+EVAL_TASKS = {
+    "blimp":           ["blimp_good.txt", "blimp_supplement_good.txt"],
+    "aoa":             ["aoa_context_sentences.txt"],
+    "comps":           ["comps_sentences.txt"],
+    "entity_tracking": ["entity_tracking_sentences.txt"],
+    "ewok":            ["ewok_context_target.txt"],
+    "reading_time":    ["reading_time_sentences.txt"],
+}
+
+# 各评测任务权重（合计 = 1.0）
+DEFAULT_TASK_WEIGHTS = {
+    "blimp":           0.25,
+    "ewok":            0.15,
+    "entity_tracking": 0.15,
+    "comps":           0.15,
+    "aoa":             0.15,
+    "reading_time":    0.15,
+}
+
+
+# ── Qwen Embedding 模型封装 ──────────────────────────────────────────────────────
+
+def _is_qwen_embedding(model_name: str) -> bool:
+    """判断是否为 Qwen3-Embedding 模型。"""
+    return "qwen" in model_name.lower() and "embedding" in model_name.lower()
+
+
+def _last_token_pool(last_hidden_states: torch.Tensor,
+                     attention_mask: torch.Tensor) -> torch.Tensor:
+    """Qwen3-Embedding 使用 last-token pooling。"""
+    left_padding = (attention_mask[:, -1].sum() == attention_mask.shape[0])
+    if left_padding:
+        return last_hidden_states[:, -1]
+    else:
+        sequence_lengths = attention_mask.sum(dim=1) - 1
+        batch_size = last_hidden_states.shape[0]
+        return last_hidden_states[
+            torch.arange(batch_size, device=last_hidden_states.device),
+            sequence_lengths,
+        ]
+
+
+class QwenEmbedder:
+    """Qwen3-Embedding 模型封装，提供与 SentenceTransformer 兼容的 encode() 接口。"""
+
+    def __init__(self, model_name: str, device: str = "cuda",
+                 max_length: int = 512):
+        from transformers import AutoModel, AutoTokenizer
+
+        print(f"  加载 tokenizer: {model_name} ...")
+        self.tokenizer = AutoTokenizer.from_pretrained(
+            model_name, padding_side="left", trust_remote_code=True,
+        )
+        print(f"  加载模型: {model_name} (bf16) ...")
+        self.model = AutoModel.from_pretrained(
+            model_name, dtype=torch.bfloat16, trust_remote_code=True,
+        ).to(device).eval()
+
+        self.device = device
+        self.max_length = max_length
+        self.embed_dim = self.model.config.hidden_size
+        print(f"  模型维度: {self.embed_dim}")
+        print(f"  设备: {device}")
+
+    def get_sentence_embedding_dimension(self) -> int:
+        return self.embed_dim
+
+    @torch.no_grad()
+    def encode(self, sentences: list[str], **kwargs) -> np.ndarray:
+        """编码一个 batch，返回 L2 归一化的 numpy 数组 [B, D]。"""
+        batch_dict = self.tokenizer(
+            sentences,
+            padding=True,
+            truncation=True,
+            max_length=self.max_length,
+            return_tensors="pt",
+        ).to(self.device)
+
+        outputs = self.model(**batch_dict)
+        embeddings = _last_token_pool(
+            outputs.last_hidden_state, batch_dict["attention_mask"]
+        )
+        embeddings = F.normalize(embeddings, p=2, dim=1)
+        return embeddings.float().cpu().numpy()
+
+
+# ── 通用工具函数 ─────────────────────────────────────────────────────────────────
+
+def load_model(model_name: str, device: str = "cuda"):
+    """根据模型名自动选择后端加载。"""
+    if _is_qwen_embedding(model_name):
+        return QwenEmbedder(model_name, device=device)
+    else:
+        from sentence_transformers import SentenceTransformer
+        model = SentenceTransformer(model_name)
+        if device == "cuda" and torch.cuda.is_available():
+            model = model.to("cuda")
+        print(f"  模型维度: {model.get_sentence_embedding_dimension()}")
+        print(f"  设备: {model.device}")
+        return model
+
+
+def load_eval_sentences() -> dict[str, list[str]]:
+    """加载评测句，按任务分组。"""
+    eval_sents = {}
+    for task, files in EVAL_TASKS.items():
+        sents = []
+        for fname in files:
+            fpath = EVAL_DIR / fname
+            if not fpath.exists():
+                print(f"  [警告] 评测文件不存在: {fpath}")
+                continue
+            for line in open(fpath, encoding="utf-8", errors="ignore"):
+                line = line.strip()
+                if line:
+                    sents.append(line)
+        eval_sents[task] = sents
+        print(f"    {task:20s}: {len(sents):>6,} 句")
+    return eval_sents
+
+
+def _cache_key(texts: list[str], model_name: str) -> str:
+    """基于文本内容和模型名生成缓存 key。"""
+    h = hashlib.md5()
+    h.update(model_name.encode())
+    h.update(str(len(texts)).encode())
+    for t in texts[:5] + texts[-5:]:
+        h.update(t.encode("utf-8", errors="ignore"))
+    return h.hexdigest()[:12]
+
+
+def encode_sentences(model, sentences: list[str],
+                     batch_size: int = 64,
+                     desc: str = "Encoding",
+                     cache_path: Path | None = None) -> np.ndarray:
+    """
+    编码句子列表，返回 L2-归一化的 embedding 矩阵 [N, D]。
+    支持可选的 .npy 缓存。
+    对 SentenceTransformer 和 QwenEmbedder 均兼容。
+    """
+    if cache_path and cache_path.exists():
+        print(f"    {desc}: 从缓存加载 {cache_path.name}")
+        embs = np.load(cache_path)
+        if embs.shape[0] == len(sentences):
+            return embs
+        print(f"    [警告] 缓存行数不匹配 ({embs.shape[0]} vs {len(sentences)})，重新编码")
+
+    is_qwen = isinstance(model, QwenEmbedder)
+    all_embs = []
+    t0 = time.time()
+
+    for i in range(0, len(sentences), batch_size):
+        batch = sentences[i:i + batch_size]
+        if is_qwen:
+            embs = model.encode(batch)
+        else:
+            embs = model.encode(
+                batch,
+                show_progress_bar=False,
+                convert_to_numpy=True,
+                normalize_embeddings=True,
+                device=model.device,
+            )
+        all_embs.append(embs)
+        done = i + len(batch)
+        if done % (batch_size * 20) == 0 or done == len(sentences):
+            elapsed = time.time() - t0
+            speed = done / elapsed if elapsed > 0 else 0
+            print(f"    {desc}: {done:>8,}/{len(sentences):,}  "
+                  f"({speed:.0f} sent/s)")
+
+    result = np.vstack(all_embs).astype(np.float32)
+
+    if cache_path:
+        cache_path.parent.mkdir(parents=True, exist_ok=True)
+        np.save(cache_path, result)
+        mb = result.nbytes / 1024 / 1024
+        print(f"    → 缓存已保存: {cache_path.name} ({mb:.1f} MB)")
+
+    return result
+
+
+def compute_similarity_scores(
+    train_embs: np.ndarray,
+    eval_embs_dict: dict[str, np.ndarray],
+    task_weights: dict[str, float],
+    top_k: int = 10,
+    batch_size: int = 1024,
+) -> np.ndarray:
+    """
+    计算每个训练句与各评测任务的加权相似度分数。
+
+    对每个任务：score = mean(top-K cosine similarities)
+    最终分数 = sum(task_weight * task_score)
+
+    返回 [N_train] 的相似度分数数组。
+    """
+    n_train = train_embs.shape[0]
+    final_scores = np.zeros(n_train, dtype=np.float32)
+    device = "cuda" if torch.cuda.is_available() else "cpu"
+
+    for task, eval_embs in eval_embs_dict.items():
+        weight = task_weights.get(task, 0.1)
+        task_scores = np.zeros(n_train, dtype=np.float32)
+
+        eval_t = torch.from_numpy(eval_embs).to(device).half()   # [N_eval, D]
+
+        t0 = time.time()
+        for i in range(0, n_train, batch_size):
+            batch = torch.from_numpy(
+                train_embs[i:i + batch_size]
+            ).to(device).half()                                   # [B, D]
+
+            sim = batch @ eval_t.T                                # [B, N_eval]
+            k = min(top_k, sim.shape[1])
+            topk_vals = sim.topk(k, dim=1).values                # [B, k]
+            task_scores[i:i + batch.shape[0]] = (
+                topk_vals.mean(dim=1).float().cpu().numpy()
+            )
+
+        elapsed = time.time() - t0
+        print(f"    {task:20s}: weight={weight:.2f}  "
+              f"mean={task_scores.mean():.4f}  std={task_scores.std():.4f}  "
+              f"({elapsed:.1f}s)")
+
+        final_scores += weight * task_scores
+        del eval_t
+        torch.cuda.empty_cache()
+
+    return final_scores
+
+
+# ── 核心采样逻辑 ───────────────────────────────────────────────────────────────
+
+def run_strategy_D(
+    all_rows: list[dict],
+    sim_scores: np.ndarray,
+    main_budget: int,
+    alpha: float = 0.6,
+    seed: int = 42,
+) -> list[dict]:
+    """
+    方案D采样：综合 embedding 相似度和质量分数。
+
+    final_weight = alpha * sim_norm + (1 - alpha) * quality_norm
+    使用 Gumbel-max trick 在各来源内加权采样，遵守来源词数上限。
+    """
+    rng = np.random.default_rng(seed)
+    n = len(all_rows)
+
+    sim_min, sim_max = sim_scores.min(), sim_scores.max()
+    if sim_max > sim_min:
+        sim_norm = (sim_scores - sim_min) / (sim_max - sim_min)
+    else:
+        sim_norm = np.zeros(n, dtype=np.float32)
+
+    quality_arr = np.array([r["quality"] for r in all_rows], dtype=np.float32)
+    q_min, q_max = quality_arr.min(), quality_arr.max()
+    if q_max > q_min:
+        q_norm = (quality_arr - q_min) / (q_max - q_min)
+    else:
+        q_norm = np.ones(n, dtype=np.float32)
+
+    final_weights = alpha * sim_norm + (1 - alpha) * q_norm
+    print(f"  综合权重: alpha={alpha:.2f}  "
+          f"range=[{final_weights.min():.4f}, {final_weights.max():.4f}]  "
+          f"mean={final_weights.mean():.4f}")
+
+    source_indices: dict[str, list[int]] = {src: [] for src in SOURCES}
+    for i, row in enumerate(all_rows):
+        source_indices[row["source"]].append(i)
+
+    source_cap_words = {
+        src: int(cfg["cap"] * main_budget)
+        for src, cfg in SOURCES.items()
+    }
+
+    all_selected = []
+    total_words = 0
+
+    for src, cfg in SOURCES.items():
+        idxs = source_indices[src]
+        if not idxs:
+            print(f"    {src:30s}: 无数据，跳过")
+            continue
+
+        cap_words = source_cap_words[src]
+        idxs_arr = np.array(idxs)
+        src_weights = final_weights[idxs_arr]
+
+        log_w = np.log(src_weights + 1e-10)
+        gumbel = rng.gumbel(size=len(idxs))
+        order = np.argsort(-(log_w + gumbel))
+
+        src_selected = []
+        src_words = 0
+        for pos in order:
+            if src_words >= cap_words:
+                break
+            idx = idxs[pos]
+            row_words = all_rows[idx]["words"]
+            if src_words + row_words <= cap_words:
+                src_selected.append(idx)
+                src_words += row_words
+
+        total_words += src_words
+        all_selected.extend(src_selected)
+
+        if src_selected:
+            sel_sim_mean = sim_scores[np.array(src_selected)].mean()
+        else:
+            sel_sim_mean = 0.0
+        print(f"    {src:30s}: {len(src_selected):>7,} 行  {src_words:>9,} 词"
+              f"  (上限 {cap_words:,})  sim_avg={sel_sim_mean:.4f}")
+
+    print(f"\n  合计选中: {len(all_selected):,} 行  {total_words:,} 词")
+    return [all_rows[i] for i in all_selected]
+
+
+# ── 入口 ───────────────────────────────────────────────────────────────────────
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Stage 8 方案D：Embedding 相似度采样")
+    parser.add_argument("--model", type=str,
+                        default="Qwen/Qwen3-Embedding-4B",
+                        help="Embedding 模型名（默认 Qwen/Qwen3-Embedding-4B）")
+    parser.add_argument("--alpha", type=float, default=0.6,
+                        help="相似度权重（0-1）。0=纯质量，1=纯相似度（默认0.6）")
+    parser.add_argument("--top-k", type=int, default=10,
+                        help="每个任务取 top-K 相似句均值（默认10）")
+    parser.add_argument("--batch-size", type=int, default=64,
+                        help="Encoding batch size（默认64，Qwen 模型建议 32-64）")
+    parser.add_argument("--max-length", type=int, default=512,
+                        help="最大 token 长度（默认512）")
+    parser.add_argument("--no-shuffle", action="store_true",
+                        help="不打乱输出顺序")
+    parser.add_argument("--dry-run", action="store_true",
+                        help="只统计不写文件")
+    parser.add_argument("--no-cache", action="store_true",
+                        help="不使用/不保存 embedding 缓存")
+    parser.add_argument("--seed", type=int, default=42,
+                        help="随机种子（默认42）")
+    parser.add_argument("--task-weights", type=str, default=None,
+                        help="自定义任务权重，格式: task1=w1,task2=w2,...")
+    args = parser.parse_args()
+
+    print("=" * 60)
+    print("  Stage 8 方案D：Embedding 相似度采样")
+    print("=" * 60)
+    print(f"  模型: {args.model}")
+    print(f"  alpha: {args.alpha}  top-K: {args.top_k}  batch: {args.batch_size}")
+
+    # 解析自定义任务权重
+    task_weights = DEFAULT_TASK_WEIGHTS.copy()
+    if args.task_weights:
+        for pair in args.task_weights.split(","):
+            k, v = pair.split("=")
+            task_weights[k.strip()] = float(v.strip())
+        total = sum(task_weights.values())
+        task_weights = {k: v / total for k, v in task_weights.items()}
+    print(f"  任务权重: { {k: f'{v:.2f}' for k, v in task_weights.items()} }")
+
+    # ── Step 1: 评测句预算 ───────────────────────────────────────────────────
+    print("\n[1/6] 统计评测句词数 ...")
+    eval_words = compute_eval_words()
+    main_budget = TOTAL_BUDGET - eval_words
+    print(f"  评测句: {eval_words:,} 词  →  主语料预算: {main_budget:,} 词")
+
+    # ── Step 2: 加载训练候选数据 ─────────────────────────────────────────────
+    print("\n[2/6] 加载候选训练数据 ...")
+    all_rows = load_all_data()
+
+    # ── Step 3: 加载 embedding 模型 ──────────────────────────────────────────
+    print(f"\n[3/6] 加载 embedding 模型: {args.model} ...")
+    emb_model = load_model(args.model)
+
+    # ── Step 4: 编码 ─────────────────────────────────────────────────────────
+    print("\n[4/6] 编码句子 ...")
+
+    # 4a. 编码评测句（按任务）
+    print("  编码评测句 ...")
+    eval_sents = load_eval_sentences()
+    eval_embs_dict: dict[str, np.ndarray] = {}
+    for task, sents in eval_sents.items():
+        cache_file = None
+        if not args.no_cache:
+            ck = _cache_key(sents, args.model)
+            cache_file = CACHE_DIR / f"eval_{task}_{ck}.npy"
+        eval_embs_dict[task] = encode_sentences(
+            emb_model, sents,
+            batch_size=args.batch_size,
+            desc=f"eval/{task}",
+            cache_path=cache_file,
+        )
+
+    # 4b. 编码训练候选句
+    print("\n  编码候选训练句 ...")
+    train_texts = [r["text"] for r in all_rows]
+    train_cache = None
+    if not args.no_cache:
+        ck = _cache_key(train_texts, args.model)
+        train_cache = CACHE_DIR / f"train_{ck}.npy"
+    train_embs = encode_sentences(
+        emb_model, train_texts,
+        batch_size=args.batch_size,
+        desc="train",
+        cache_path=train_cache,
+    )
+
+    # 释放模型显存
+    if isinstance(emb_model, QwenEmbedder):
+        del emb_model.model, emb_model.tokenizer
+    del emb_model
+    torch.cuda.empty_cache()
+
+    # ── Step 5: 计算相似度分数 ───────────────────────────────────────────────
+    print("\n[5/6] 计算相似度分数 ...")
+    sim_scores = compute_similarity_scores(
+        train_embs, eval_embs_dict, task_weights,
+        top_k=args.top_k,
+        batch_size=1024,
+    )
+    print(f"  相似度分数: mean={sim_scores.mean():.4f}  "
+          f"std={sim_scores.std():.4f}  "
+          f"range=[{sim_scores.min():.4f}, {sim_scores.max():.4f}]")
+
+    if not args.no_cache:
+        score_path = CACHE_DIR / "sim_scores.npy"
+        np.save(score_path, sim_scores)
+        print(f"  → 相似度分数已保存: {score_path}")
+
+    del train_embs, eval_embs_dict
+
+    # ── Step 6: 采样 & 输出 ──────────────────────────────────────────────────
+    print("\n[6/6] 执行方案D采样 ...")
+    selected = run_strategy_D(
+        all_rows, sim_scores, main_budget,
+        alpha=args.alpha, seed=args.seed,
+    )
+
+    print_stats(selected, strategy_name="方案D (Embedding 相似度)")
+
+    if not args.dry_run:
+        print(f"\n  写入输出 ...")
+        write_output(selected, OUT_D_DIR,
+                     shuffle=not args.no_shuffle,
+                     seed=args.seed)
+        print(f"\n  完成！输出目录: {OUT_D_DIR}")
+    else:
+        print("\n[dry-run] 跳过写文件。")
+
+
+if __name__ == "__main__":
+    main()

scripts/01_data/stage9_paraphrase.py

@@ -0,0 +1,452 @@
+"""
+Stage 9：Paraphrase 增强
+
+用 Qwen3.5-9B 对训练句生成同义改写，增加语言多样性。
+
+流程：
+  1. 读取 Stage 8 输出的 train.txt
+  2. 按词数裁剪到 original_budget（默认 8M 词），保留头部高质量句
+  3. 从裁剪后的原文中选出适合改写的候选句（8-40 词、格式规范）
+  4. 用 Qwen3.5-9B 批量生成 paraphrase，直到达到 paraphrase_budget（默认 2M 词）
+  5. 合并输出：裁剪后原文 + paraphrase → data/9_paraphrase/train.txt
+
+输入 : data/8_sample_B/train.txt（或指定 Stage 8 输出）
+输出 : data/9_paraphrase/train.txt
+
+依赖 : Qwen3.5-9B（已下载），需要 GPU
+
+用法:
+  python scripts/01_data/stage9_paraphrase.py
+  python scripts/01_data/stage9_paraphrase.py --original-budget 8000000 --paraphrase-budget 2000000
+  python scripts/01_data/stage9_paraphrase.py --batch-size 16 --seed 42
+  python scripts/01_data/stage9_paraphrase.py --dry-run
+  python scripts/01_data/stage9_paraphrase.py --resume           # 从断点继续
+"""
+
+import argparse
+import json
+import random
+import re
+import time
+from pathlib import Path
+
+ROOT = Path(__file__).parent.parent.parent
+DEFAULT_INPUT  = ROOT / "data/8_sample_B/train.txt"
+DEFAULT_OUTPUT = ROOT / "data/9_paraphrase"
+MODEL_PATH = ROOT / "models/qwen3.5-9b/models--Qwen--Qwen3.5-9B/snapshots/c202236235762e1c871ad0ccb60c8ee5ba337b9a"
+CHECKPOINT_PATH = ROOT / "data/9_paraphrase/checkpoint.jsonl"
+
+# ── 默认参数 ──────────────────────────────────────────────────────────────────
+
+TOTAL_BUDGET       = 10_000_000   # 总词数上限
+DEFAULT_ORIG       = 8_000_000    # 原文词数
+DEFAULT_PARA       = 2_000_000    # paraphrase 词数
+SENTENCES_PER_PROMPT = 10         # 每个 prompt 包含句子数
+MIN_WORDS          = 8            # 候选句最少词数
+MAX_WORDS          = 40           # 候选句最多词数
+
+# ── Prompt ────────────────────────────────────────────────────────────────────
+
+SYSTEM_PROMPT = """You are a linguistic paraphrase generator. For each numbered sentence, generate ONE paraphrase that:
+- Preserves the original meaning exactly
+- Uses different wording and/or sentence structure
+- Maintains natural, fluent English
+- Keeps approximately the same length
+
+Output format: one paraphrase per line, prefixed with the same number.
+Example input:
+0: The cat sat on the mat.
+1: She quickly ran to the store.
+
+Example output:
+0: A cat was sitting on the mat.
+1: She hurried to the store.
+
+ONLY output the numbered paraphrases. No explanations."""
+
+
+# ── 候选句选择 ────────────────────────────────────────────────────────────────
+
+def is_good_candidate(line: str, min_words: int = MIN_WORDS,
+                      max_words: int = MAX_WORDS) -> bool:
+    """判断一行是否适合做 paraphrase"""
+    words = line.split()
+    n = len(words)
+
+    # 长度过滤
+    if n < min_words or n > max_words:
+        return False
+
+    # 必须以大写字母开头（正常英语句子）
+    if not line[0].isupper():
+        return False
+
+    # 必须以句末标点结尾
+    if line[-1] not in '.?!':
+        return False
+
+    # 排除 Entity Tracking 合成句（"Box N contains..."）
+    if line.startswith("Box ") and "contains" in line:
+        return False
+
+    # 排除列表/编号行
+    if re.match(r'^\d+[\.\)]\s', line):
+        return False
+
+    return True
+
+
+def select_candidates(lines: list[str], budget_words: int,
+                      seed: int = 42, min_words: int = MIN_WORDS,
+                      max_words: int = MAX_WORDS) -> list[int]:
+    """
+    从原文中选出适合 paraphrase 的候选句索引。
+    选够 budget_words 的 1.5 倍（预留余量，因为 paraphrase 长度不完全可控）。
+    """
+    rng = random.Random(seed)
+
+    # 收集所有候选
+    candidates = []
+    for i, line in enumerate(lines):
+        if is_good_candidate(line, min_words, max_words):
+            candidates.append((i, len(line.split())))
+
+    rng.shuffle(candidates)
+
+    # 选到 1.5 倍预算
+    target = int(budget_words * 1.5)
+    selected = []
+    total_words = 0
+    for idx, wc in candidates:
+        if total_words >= target:
+            break
+        selected.append(idx)
+        total_words += wc
+
+    print(f"  候选句总数: {len(candidates):,}")
+    print(f"  选中: {len(selected):,} 句 ({total_words:,} 词，目标 {budget_words:,} 词的 1.5x)")
+    return selected
+
+
+# ── Paraphrase 生成 ──────────────────────────────────────────────────────────
+
+def build_prompt(tokenizer, sentences: list[str]) -> str:
+    """构建 paraphrase prompt"""
+    input_text = "\n".join(f"{i}: {sent}" for i, sent in enumerate(sentences))
+    messages = [
+        {"role": "system", "content": SYSTEM_PROMPT},
+        {"role": "user",   "content": input_text},
+    ]
+    return tokenizer.apply_chat_template(
+        messages, tokenize=False, add_generation_prompt=True,
+        enable_thinking=False,
+    )
+
+
+def parse_paraphrases(response: str, n_expected: int) -> list[str]:
+    """从模型输出中解析 paraphrase 结果"""
+    results = [None] * n_expected
+
+    for line in response.strip().split("\n"):
+        line = line.strip()
+        if not line:
+            continue
+        # 匹配 "0: paraphrased text" 格式
+        m = re.match(r'^(\d+)\s*[:\.]\s*(.+)$', line)
+        if m:
+            idx = int(m.group(1))
+            text = m.group(2).strip()
+            if 0 <= idx < n_expected and text:
+                results[idx] = text
+
+    return results
+
+
+def load_checkpoint(ckpt_path: Path) -> set[int]:
+    """加载已完成的原文行号"""
+    done = set()
+    if ckpt_path.exists():
+        for line in open(ckpt_path, encoding="utf-8"):
+            try:
+                d = json.loads(line)
+                done.add(d["orig_idx"])
+            except Exception:
+                continue
+    return done
+
+
+def generate_paraphrases(
+    lines: list[str],
+    candidate_indices: list[int],
+    budget_words: int,
+    model_path: str,
+    batch_size: int = 10,
+    max_new_tokens: int = 512,
+    resume: bool = False,
+):
+    """
+    批量生成 paraphrase。
+
+    返回: list[dict]，每个 dict 有 orig_idx, original, paraphrase, words
+    """
+    import torch
+    from transformers import AutoModelForCausalLM, AutoTokenizer
+
+    # 加载模型
+    print(f"\n  加载 tokenizer: {model_path} ...")
+    tokenizer = AutoTokenizer.from_pretrained(
+        model_path, padding_side="left", trust_remote_code=True,
+    )
+    if tokenizer.pad_token is None:
+        tokenizer.pad_token = tokenizer.eos_token
+
+    print(f"  加载模型 (bf16) ...")
+    model = AutoModelForCausalLM.from_pretrained(
+        model_path,
+        torch_dtype=torch.bfloat16,
+        device_map="auto",
+        trust_remote_code=True,
+    )
+    model.eval()
+    print(f"  模型加载完成\n")
+
+    # 断点续传
+    CHECKPOINT_PATH.parent.mkdir(parents=True, exist_ok=True)
+    done_indices = load_checkpoint(CHECKPOINT_PATH) if resume else set()
+    if done_indices:
+        print(f"  断点续传: 已完成 {len(done_indices):,} 句")
+
+    # 过滤已完成
+    remaining = [i for i in candidate_indices if i not in done_indices]
+
+    results = []
+    total_words = 0
+    t0 = time.time()
+
+    # 如果 resume，先加载已有结果
+    if resume and CHECKPOINT_PATH.exists():
+        for line in open(CHECKPOINT_PATH, encoding="utf-8"):
+            try:
+                d = json.loads(line)
+                results.append(d)
+                total_words += d["words"]
+            except Exception:
+                continue
+        print(f"  已有 paraphrase: {len(results):,} 句 ({total_words:,} 词)")
+
+    ckpt_file = open(CHECKPOINT_PATH, "a", encoding="utf-8") if not resume else \
+                open(CHECKPOINT_PATH, "a", encoding="utf-8")
+
+    try:
+        for batch_start in range(0, len(remaining), batch_size):
+            if total_words >= budget_words:
+                break
+
+            batch_indices = remaining[batch_start:batch_start + batch_size]
+            batch_sentences = [lines[i] for i in batch_indices]
+
+            # 构建 prompt
+            prompt = build_prompt(tokenizer, batch_sentences)
+            inputs = tokenizer(prompt, return_tensors="pt").to(model.device)
+
+            # 生成
+            with torch.no_grad():
+                outputs = model.generate(
+                    **inputs,
+                    max_new_tokens=max_new_tokens,
+                    temperature=0.7,
+                    top_p=0.9,
+                    do_sample=True,
+                    pad_token_id=tokenizer.pad_token_id,
+                )
+
+            # 解码（只取生成部分）
+            gen_tokens = outputs[0][inputs["input_ids"].shape[1]:]
+            response = tokenizer.decode(gen_tokens, skip_special_tokens=True)
+
+            # 解析
+            paraphrases = parse_paraphrases(response, len(batch_sentences))
+
+            for local_i, para in enumerate(paraphrases):
+                if para is None:
+                    continue
+                orig_idx = batch_indices[local_i]
+                wc = len(para.split())
+                record = {
+                    "orig_idx": orig_idx,
+                    "original": batch_sentences[local_i],
+                    "paraphrase": para,
+                    "words": wc,
+                }
+                results.append(record)
+                total_words += wc
+
+                # 写入 checkpoint
+                ckpt_file.write(json.dumps(record, ensure_ascii=False) + "\n")
+
+                if total_words >= budget_words:
+                    break
+
+            # 进度
+            done = batch_start + len(batch_indices)
+            elapsed = time.time() - t0
+            speed = done / elapsed if elapsed > 0 else 0
+            print(f"    {done:>6,}/{len(remaining):,} 批次  "
+                  f"{total_words:>9,}/{budget_words:,} 词  "
+                  f"({speed:.1f} sent/s, {elapsed:.0f}s)")
+
+            ckpt_file.flush()
+
+    finally:
+        ckpt_file.close()
+
+    print(f"\n  Paraphrase 完成: {len(results):,} 句  {total_words:,} 词")
+    return results
+
+
+# ── 主流程 ────────────────────────────────────────────────────────────────────
+
+def trim_to_budget(lines: list[str], budget_words: int) -> list[str]:
+    """
+    裁剪原文到指定词数预算。
+    保留前 N 行直到词数达到 budget。
+    """
+    kept = []
+    total = 0
+    for line in lines:
+        wc = len(line.split())
+        if total + wc > budget_words:
+            break
+        kept.append(line)
+        total += wc
+    return kept
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Stage 9: Paraphrase 增强")
+    parser.add_argument("--input", type=str, default=str(DEFAULT_INPUT),
+                        help=f"Stage 8 输出路径（默认 {DEFAULT_INPUT}）")
+    parser.add_argument("--output-dir", type=str, default=str(DEFAULT_OUTPUT),
+                        help=f"输出目录（默认 {DEFAULT_OUTPUT}）")
+    parser.add_argument("--model", type=str, default=str(MODEL_PATH),
+                        help="Qwen 模型路径")
+    parser.add_argument("--original-budget", type=int, default=DEFAULT_ORIG,
+                        help=f"原文词数预算（默认 {DEFAULT_ORIG:,}）")
+    parser.add_argument("--paraphrase-budget", type=int, default=DEFAULT_PARA,
+                        help=f"Paraphrase 词数预算（默认 {DEFAULT_PARA:,}）")
+    parser.add_argument("--batch-size", type=int, default=SENTENCES_PER_PROMPT,
+                        help=f"每个 prompt 句子数（默认 {SENTENCES_PER_PROMPT}）")
+    parser.add_argument("--min-words", type=int, default=MIN_WORDS,
+                        help=f"候选句最少词数（默认 {MIN_WORDS}）")
+    parser.add_argument("--max-words", type=int, default=MAX_WORDS,
+                        help=f"候选句最多词数（默认 {MAX_WORDS}）")
+    parser.add_argument("--seed", type=int, default=42, help="随机种子")
+    parser.add_argument("--dry-run", action="store_true",
+                        help="只统计不生成")
+    parser.add_argument("--resume", action="store_true",
+                        help="从断点继续")
+    args = parser.parse_args()
+
+    input_path = Path(args.input)
+    output_dir = Path(args.output_dir)
+    output_dir.mkdir(parents=True, exist_ok=True)
+    out_path = output_dir / "train.txt"
+
+    print("=" * 60)
+    print("  Stage 9: Paraphrase 增强")
+    print("=" * 60)
+    print(f"  输入: {input_path}")
+    print(f"  输出: {out_path}")
+    print(f"  模型: {args.model}")
+    print(f"  原文预算: {args.original_budget:,} 词")
+    print(f"  Paraphrase 预算: {args.paraphrase_budget:,} 词")
+    print(f"  总预算: {args.original_budget + args.paraphrase_budget:,} 词")
+    print(f"  候选句长度: {MIN_WORDS}-{MAX_WORDS} 词")
+    print(f"  Batch size: {args.batch_size}")
+    print(f"  Seed: {args.seed}")
+
+    # 1. 读取原文
+    print(f"\n[1/4] 读取原文 ...")
+    with open(input_path, encoding="utf-8", errors="ignore") as f:
+        all_lines = [line.strip() for line in f if line.strip()]
+    total_words = sum(len(l.split()) for l in all_lines)
+    print(f"  原文: {len(all_lines):,} 行  {total_words:,} 词")
+
+    # 2. 裁剪原文
+    print(f"\n[2/4] 裁剪原文到 {args.original_budget:,} 词 ...")
+    trimmed = trim_to_budget(all_lines, args.original_budget)
+    trimmed_words = sum(len(l.split()) for l in trimmed)
+    print(f"  裁剪后: {len(trimmed):,} 行  {trimmed_words:,} 词")
+
+    # 3. 选择候选句
+    print(f"\n[3/4] 选择 paraphrase 候选句 ...")
+    candidates = select_candidates(trimmed, args.paraphrase_budget, args.seed,
+                                    args.min_words, args.max_words)
+    print(f"  候选句示例:")
+    for i in candidates[:3]:
+        print(f"    [{i}] {trimmed[i][:80]}...")
+
+    if args.dry_run:
+        print(f"\n[dry-run] 跳过生成。")
+        print(f"  预计输出: {trimmed_words:,} 原文 + {args.paraphrase_budget:,} paraphrase"
+              f" = {trimmed_words + args.paraphrase_budget:,} 词")
+        return
+
+    # 4. 生成 paraphrase
+    print(f"\n[4/4] 生成 paraphrase ...")
+    results = generate_paraphrases(
+        lines=trimmed,
+        candidate_indices=candidates,
+        budget_words=args.paraphrase_budget,
+        model_path=args.model,
+        batch_size=args.batch_size,
+        resume=args.resume,
+    )
+
+    # 5. 合并输出
+    print(f"\n  写入输出 ...")
+
+    # 收集 paraphrase 句子
+    para_lines = [r["paraphrase"] for r in results]
+    para_words = sum(len(l.split()) for l in para_lines)
+
+    # 打乱 paraphrase 并混入原文
+    rng = random.Random(args.seed)
+    combined = list(trimmed) + para_lines
+    rng.shuffle(combined)
+
+    with open(out_path, "w", encoding="utf-8") as f:
+        for line in combined:
+            f.write(line + "\n")
+
+    final_words = trimmed_words + para_words
+    print(f"\n{'=' * 60}")
+    print(f"  完成！")
+    print(f"  原文:      {len(trimmed):,} 行  {trimmed_words:,} 词")
+    print(f"  Paraphrase: {len(para_lines):,} 行  {para_words:,} 词")
+    print(f"  合计:      {len(combined):,} 行  {final_words:,} 词")
+    print(f"  输出: {out_path}")
+
+    # 保存统计
+    stats_path = output_dir / "stats.json"
+    stats = {
+        "input": str(input_path),
+        "original_lines": len(trimmed),
+        "original_words": trimmed_words,
+        "paraphrase_lines": len(para_lines),
+        "paraphrase_words": para_words,
+        "total_lines": len(combined),
+        "total_words": final_words,
+        "model": args.model,
+        "seed": args.seed,
+        "batch_size": args.batch_size,
+        "min_words": MIN_WORDS,
+        "max_words": MAX_WORDS,
+    }
+    with open(stats_path, "w") as f:
+        json.dump(stats, f, indent=2, ensure_ascii=False)
+    print(f"  统计: {stats_path}")
+
+
+if __name__ == "__main__":
+    main()

scripts/01_data/vocab_coverage.py

@@ -0,0 +1,99 @@
+"""
+vocab.txt 覆盖率分析
+对每个训练文件统计：
+- token 级覆盖率：训练文件中的词有多少 % 在 vocab.txt 里
+- 类型级覆盖率：训练文件中的唯一词有多少 % 在 vocab.txt 里
+- OOV（out-of-vocab）最高频词 Top 20
+"""
+
+import re
+from pathlib import Path
+from collections import Counter
+
+VOCAB_PATH  = Path(__file__).parent.parent / "evaluation-pipeline-2025/evaluation_pipeline/ewok/vocab.txt"
+DATA_DIR    = Path(__file__).parent.parent / "data/raw_files"
+
+FILES = [
+    "childes.train.txt",
+    "gutenberg.train.txt",
+    "open_subtitles.train.txt",
+    "simple_wiki.train.txt",
+    "bnc_spoken.train.txt",
+    "open_subtitles_cleaned_ckpt_70.txt",
+    "switchboard.train.txt",
+]
+
+def tokenize(line):
+    return re.findall(r"[a-zA-Z']+|[0-9]+", line.lower())
+
+def analyze(file_name, vocab):
+    path = DATA_DIR / file_name
+    total_tokens = 0
+    in_vocab_tokens = 0
+    total_counter = Counter()
+    oov_counter = Counter()
+
+    with open(path, "r", encoding="utf-8", errors="ignore") as f:
+        for line in f:
+            tokens = tokenize(line)
+            for t in tokens:
+                total_tokens += 1
+                total_counter[t] += 1
+                if t in vocab:
+                    in_vocab_tokens += 1
+                else:
+                    oov_counter[t] += 1
+
+    total_types  = len(total_counter)
+    in_vocab_types = sum(1 for w in total_counter if w in vocab)
+
+    token_cov = in_vocab_tokens / total_tokens * 100 if total_tokens else 0
+    type_cov  = in_vocab_types  / total_types  * 100 if total_types  else 0
+
+    return {
+        "file":           file_name,
+        "total_tokens":   total_tokens,
+        "in_vocab_tokens":in_vocab_tokens,
+        "token_coverage": round(token_cov, 2),
+        "total_types":    total_types,
+        "in_vocab_types": in_vocab_types,
+        "type_coverage":  round(type_cov, 2),
+        "top20_oov":      oov_counter.most_common(20),
+    }
+
+def main():
+    print(f"读取 vocab.txt ...")
+    vocab = set(Path(VOCAB_PATH).read_text().splitlines())
+    print(f"vocab 大小: {len(vocab):,} 词\n")
+
+    results = []
+    for fname in FILES:
+        path = DATA_DIR / fname
+        if not path.exists():
+            print(f"[skip] {fname} 不存在")
+            continue
+        print(f"分析: {fname} ...")
+        r = analyze(fname, vocab)
+        results.append(r)
+        print(f"  token覆盖率: {r['token_coverage']}%  |  类型覆盖率: {r['type_coverage']}%")
+        print(f"  总词数: {r['total_tokens']:,}  vocab内: {r['in_vocab_tokens']:,}")
+        print(f"  总唯一词: {r['total_types']:,}  vocab内: {r['in_vocab_types']:,}")
+        print(f"  Top10 OOV: {[w for w,c in r['top20_oov'][:10]]}")
+        print()
+
+    # 汇总表
+    print("\n" + "="*70)
+    print(f"{'文件':<40} {'token覆盖率':>10} {'类型覆盖率':>10} {'总词数':>12}")
+    print("="*70)
+    for r in results:
+        print(f"{r['file']:<40} {r['token_coverage']:>9}%  {r['type_coverage']:>9}%  {r['total_tokens']:>12,}")
+
+    # 保存结果
+    import json
+    out = Path(__file__).parent.parent / "results/analysis/vocab_coverage.json"
+    out.parent.mkdir(parents=True, exist_ok=True)
+    out.write_text(json.dumps(results, ensure_ascii=False, indent=2))
+    print(f"\n结果已保存: {out}")
+
+if __name__ == "__main__":
+    main()

scripts/02_model/train_tokenizer.py

@@ -0,0 +1,186 @@
+"""
+训练 BPE Tokenizer
+
+与官方 baseline (ltg/gpt-bert-babylm-small) 结构完全一致：
+  - Normalizer:    Prepend空格 + NFKC + 换行处理
+  - Pre-tokenizer: GPT-4风格regex切分 + ByteLevel + 最长24字符截断
+  - Model:         BPE, vocab_size=8192
+  - 特殊token:     <unk>=0, <s>=1, </s>=2, <pad>=3, <mask>=4
+  - Post-processor: 句首自动加 <s>
+
+输入: data/8_sample_B/train.txt
+输出: models/tokenizer/
+
+用法:
+    python scripts/02_model/train_tokenizer.py
+    python scripts/02_model/train_tokenizer.py --vocab_size 8192
+    python scripts/02_model/train_tokenizer.py --input data/8_sample_B/train.txt
+"""
+
+import argparse
+import json
+from pathlib import Path
+
+from tokenizers import Tokenizer, AddedToken
+from tokenizers.models import BPE
+from tokenizers.trainers import BpeTrainer
+from tokenizers.normalizers import Sequence, Prepend, NFKC, Replace
+from tokenizers.pre_tokenizers import Sequence as PreSeq, Split, ByteLevel
+from tokenizers.processors import TemplateProcessing
+from tokenizers import Regex
+from transformers import PreTrainedTokenizerFast
+
+ROOT     = Path(__file__).parent.parent.parent
+DEFAULT_INPUT = ROOT / "data/8_sample_B/train.txt"
+DEFAULT_OUT   = ROOT / "models/tokenizer"
+
+# 与官方 baseline 完全一致的特殊 token 顺序（id 固定）
+SPECIAL_TOKENS = ["<unk>", "<s>", "</s>", "<pad>", "<mask>"]
+
+
+def build_tokenizer(vocab_size: int) -> tuple[Tokenizer, BpeTrainer]:
+    """构造与官方 baseline 相同结构的 tokenizer + trainer。"""
+
+    # ── 1. Normalizer ────────────────────────────────────────────────────────
+    normalizer = Sequence([
+        Prepend(prepend=" "),
+        NFKC(),
+        Replace(Regex(r"\n"),   "\n "),   # 换行后加空格，保持词边界
+        Replace(Regex(r" *\n"), "\n"),     # 去掉换行前多余空格
+    ])
+
+    # ── 2. Pre-tokenizer ─────────────────────────────────────────────────────
+    # GPT-4 / cl100k 风格的 Unicode-aware 正则切分
+    GPT4_REGEX = (
+        r"[^\r\n\p{L}\p{N}]?[\p{Lu}\p{Lt}\p{Lm}\p{Lo}\p{M}]*"
+        r"[\p{Ll}\p{Lm}\p{Lo}\p{M}]+"
+        r"|[^\r\n\p{L}\p{N}]?[\p{Lu}\p{Lt}\p{Lm}\p{Lo}\p{M}]+"
+        r"[\p{Ll}\p{Lm}\p{Lo}\p{M}]*"
+        r"| ?\p{N}"
+        r"| ?[^\s\p{L}\p{N}]+[\r\n/]*"
+        r"|\s*[\r\n]+"
+        r"|\s+(?!\S)"
+        r"|\s+"
+    )
+    pre_tokenizer = PreSeq([
+        Split(pattern=Regex(GPT4_REGEX), behavior="isolated"),
+        ByteLevel(add_prefix_space=False, trim_offsets=True, use_regex=False),
+        Split(pattern=Regex(r".{1,24}"), behavior="isolated"),   # 最长24字符截断
+    ])
+
+    # ── 3. Tokenizer + Trainer ───────────────────────────────────────────────
+    tokenizer = Tokenizer(BPE(unk_token="<unk>"))
+    tokenizer.normalizer    = normalizer
+    tokenizer.pre_tokenizer = pre_tokenizer
+
+    trainer = BpeTrainer(
+        vocab_size=vocab_size,
+        special_tokens=SPECIAL_TOKENS,
+        min_frequency=2,
+        show_progress=True,
+    )
+
+    return tokenizer, trainer
+
+
+def add_post_processor(tokenizer: Tokenizer) -> None:
+    """添加 post-processor：句首自动插入 <s>（id=1）。"""
+    tokenizer.post_processor = TemplateProcessing(
+        single="<s> $A",
+        pair="<s> $A <s> $B",
+        special_tokens=[("<s>", tokenizer.token_to_id("<s>"))],
+    )
+
+
+def verify_special_token_ids(tokenizer: Tokenizer) -> None:
+    """校验特殊 token ID 与官方 baseline 一致。"""
+    expected = {"<unk>": 0, "<s>": 1, "</s>": 2, "<pad>": 3, "<mask>": 4}
+    ok = True
+    for token, expected_id in expected.items():
+        actual_id = tokenizer.token_to_id(token)
+        status = "✅" if actual_id == expected_id else "❌"
+        print(f"  {status}  {token:10s}  expected={expected_id}  actual={actual_id}")
+        if actual_id != expected_id:
+            ok = False
+    if not ok:
+        raise ValueError("特殊 token ID 与官方 baseline 不一致！")
+
+
+def save(tokenizer: Tokenizer, out_dir: Path, vocab_size: int) -> None:
+    """保存为 HuggingFace PreTrainedTokenizerFast 格式。"""
+    out_dir.mkdir(parents=True, exist_ok=True)
+
+    # 先以原生格式保存
+    raw_path = out_dir / "tokenizer.json"
+    tokenizer.save(str(raw_path))
+
+    # 用 transformers 包装，补充 tokenizer_config.json
+    fast_tok = PreTrainedTokenizerFast(
+        tokenizer_file=str(raw_path),
+        bos_token="<s>",
+        eos_token="</s>",
+        unk_token="<unk>",
+        sep_token="</s>",
+        pad_token="<pad>",
+        cls_token="<s>",
+        mask_token="<mask>",
+    )
+    fast_tok.save_pretrained(str(out_dir))
+    print(f"\n  保存到: {out_dir}")
+    print(f"  文件列表: {[f.name for f in sorted(out_dir.iterdir())]}")
+
+
+def smoke_test(out_dir: Path) -> None:
+    """简单验证：加载后测试几个句子。"""
+    fast_tok = PreTrainedTokenizerFast.from_pretrained(str(out_dir))
+    tests = [
+        "The cat sat on the mat.",
+        "She gave him the book yesterday.",
+        "ran swimming swam running",
+        "Katherine can't help herself.",
+    ]
+    print("\n  Smoke test:")
+    for t in tests:
+        tokens = fast_tok.tokenize(t)
+        print(f"    {repr(t):45s} → {tokens}")
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--input",      default=str(DEFAULT_INPUT), help="训练文件路径")
+    parser.add_argument("--output",     default=str(DEFAULT_OUT),   help="输出目录")
+    parser.add_argument("--vocab_size", default=8192, type=int,     help="词表大小（默认8192）")
+    args = parser.parse_args()
+
+    input_path = Path(args.input)
+    out_dir    = Path(args.output)
+
+    if not input_path.exists():
+        raise FileNotFoundError(f"训练文件不存在: {input_path}")
+
+    print(f"训练 BPE Tokenizer")
+    print(f"  输入 : {input_path}  ({input_path.stat().st_size / 1e6:.1f} MB)")
+    print(f"  输出 : {out_dir}")
+    print(f"  vocab_size = {args.vocab_size}")
+    print(f"  特殊 token: {SPECIAL_TOKENS}")
+    print()
+
+    tokenizer, trainer = build_tokenizer(args.vocab_size)
+
+    print("训练中...")
+    tokenizer.train(files=[str(input_path)], trainer=trainer)
+    print(f"训练完成，实际 vocab size = {tokenizer.get_vocab_size()}")
+
+    add_post_processor(tokenizer)
+
+    print("\n特殊 token ID 校验：")
+    verify_special_token_ids(tokenizer)
+
+    save(tokenizer, out_dir, args.vocab_size)
+    smoke_test(out_dir)
+
+    print("\n完成！")
+
+
+if __name__ == "__main__":
+    main()

scripts/02_model/train_tokenizer_morfessor.py

@@ -0,0 +1,361 @@
+"""
+训练 Morfessor+BPE Tokenizer
+
+思路：
+  1. 在训练语料上训练 Morfessor，学习英语形态素边界
+  2. 用 Morfessor 对训练文本预切分（在形态素边界插入空格）
+  3. 在预切分文本上训练 BPE（架构与标准 BPE tokenizer 完全一致）
+
+效果：BPE 不会跨形态素边界合并，保留 WUG/Entity Tracking 有利的形态信息
+
+输入: data/8_sample_B/train.txt
+输出: models/tokenizer_morfessor/
+      ├── tokenizer.json            (BPE tokenizer, HuggingFace 格式)
+      ├── tokenizer_config.json
+      ├── special_tokens_map.json
+      └── morfessor.bin             (Morfessor 模型，推理时需要)
+
+用法:
+    pip install morfessor
+    python scripts/02_model/train_tokenizer_morfessor.py
+    python scripts/02_model/train_tokenizer_morfessor.py --input data/8_sample_C/train.txt
+"""
+
+import argparse
+import re
+import sys
+from collections import Counter
+from pathlib import Path
+
+try:
+    import morfessor
+except ImportError:
+    print("请先安装: pip install morfessor")
+    sys.exit(1)
+
+from tokenizers import Tokenizer, Regex
+from tokenizers.models import BPE
+from tokenizers.trainers import BpeTrainer
+from tokenizers.normalizers import Sequence, Prepend, NFKC, Replace
+from tokenizers.pre_tokenizers import Sequence as PreSeq, Split, ByteLevel
+from tokenizers.processors import TemplateProcessing
+from transformers import PreTrainedTokenizerFast
+
+ROOT = Path(__file__).resolve().parent.parent.parent
+DEFAULT_INPUT = ROOT / "data/8_sample_B/train.txt"
+DEFAULT_OUT = ROOT / "models/tokenizer_morfessor"
+SPECIAL_TOKENS = ["<unk>", "<s>", "</s>", "<pad>", "<mask>"]
+
+# Morfessor 超参
+CORPUSWEIGHT = 0.01       # 越小切分越激进（默认~1.0 太保守）
+MIN_MORPH_LEN = 2         # 形态素最短长度，过滤 s+it / b+and 等假阳性
+MIN_WORD_LEN = 3          # 短于此长度的词不切分
+
+
+# ══════════════════════════════════════════════════════════════════════════════
+# Step 1: 训练 Morfessor
+# ══════════════════════════════════════════════════════════════════════════════
+
+def train_morfessor_model(input_path: Path, corpusweight: float = CORPUSWEIGHT) -> morfessor.BaselineModel:
+    """在训练语料上训练 Morfessor 模型。"""
+    print(f"Step 1: 训练 Morfessor 模型 (corpusweight={corpusweight})...")
+
+    # 统计词频（只取纯字母词，去标点）
+    word_counts = Counter()
+    with open(input_path) as f:
+        for line in f:
+            for word in line.split():
+                clean = word.strip(".,!?;:\"'()-[]{}…""''").lower()
+                if len(clean) >= 2 and clean.isalpha():
+                    word_counts[clean] += 1
+
+    print(f"  唯一词数: {len(word_counts):,}")
+    print(f"  总词频:   {sum(word_counts.values()):,}")
+
+    model = morfessor.BaselineModel(corpusweight=corpusweight)
+    training_data = [(count, word) for word, count in word_counts.items()]
+    model.load_data(training_data)
+    model.train_batch()
+
+    # 示例
+    samples = [
+        "unhappiness", "running", "walked", "beautiful", "government",
+        "internationally", "darkness", "singer", "swimming", "nationalization",
+        "happiness", "slowly", "governmental", "children", "quickly",
+    ]
+    print("\n  切分示例:")
+    for w in samples:
+        segs = model.viterbi_segment(w)[0]
+        print(f"    {w:25s} → {' + '.join(segs)}")
+
+    return model
+
+
+# ══════════════════════════════════════════════════════════════════════════════
+# Step 2: 预切分文本
+# ══════════════════════════════════════════════════════════════════════════════
+
+_WORD_RE = re.compile(r'^([^a-zA-Z]*?)([a-zA-Z]+)([^a-zA-Z]*)$')
+
+
+def presegment_word(word: str, morf_model) -> str:
+    """对单词用 Morfessor 找形态素边界，在边界处插入空格。
+
+    保留原始大小写：用 Morfessor 切分小写版本，
+    再按 segment 长度映射回原始字符。
+
+    过滤规则：所有形态素长度必须 >= MIN_MORPH_LEN，
+    否则视为假阳性（如 s+it, b+and），保持原词不切。
+    """
+    m = _WORD_RE.match(word)
+    if not m:
+        return word
+
+    prefix, core, suffix = m.groups()
+    if len(core) < MIN_WORD_LEN:
+        return word
+
+    segments = morf_model.viterbi_segment(core.lower())[0]
+    if len(segments) <= 1:
+        return word
+
+    # 过滤：所有形态素必须 >= MIN_MORPH_LEN
+    if not all(len(s) >= MIN_MORPH_LEN for s in segments):
+        return word
+
+    # 按 segment 长度从原始 core 中切片（保留大小写）
+    parts = []
+    pos = 0
+    for seg in segments:
+        n = len(seg)
+        parts.append(core[pos:pos + n])
+        pos += n
+
+    return prefix + ' '.join(parts) + suffix
+
+
+def presegment_file(input_path: Path, morf_model, output_path: Path) -> Path:
+    """对整个文件进行 Morfessor 预切分。"""
+    print("\nStep 2: 预切分训练文本...")
+
+    line_count = 0
+    with open(input_path) as fin, open(output_path, 'w') as fout:
+        for line in fin:
+            if line.strip():
+                words = line.split()
+                segmented = [presegment_word(w, morf_model) for w in words]
+                fout.write(' '.join(segmented) + '\n')
+            else:
+                fout.write('\n')
+            line_count += 1
+            if line_count % 200000 == 0:
+                print(f"  已处理 {line_count:,} 行...")
+
+    print(f"  总共处理 {line_count:,} 行")
+    print(f"  预切分文件: {output_path} ({output_path.stat().st_size / 1e6:.1f} MB)")
+
+    # 对比几行
+    print("\n  对比示例:")
+    shown = 0
+    with open(input_path) as f1, open(output_path) as f2:
+        for orig, seg in zip(f1, f2):
+            if orig.strip() != seg.strip() and 30 < len(orig.strip()) < 150:
+                print(f"    原: {orig.strip()}")
+                print(f"    切: {seg.strip()}")
+                print()
+                shown += 1
+                if shown >= 5:
+                    break
+
+    return output_path
+
+
+# ══════════════════════════════════════════════════════════════════════════════
+# Step 3: 在预切分文本上训练 BPE
+# ══════════════════════════════════════════════════════════════════════════════
+
+def build_and_train_bpe(segmented_file: Path, vocab_size: int) -> Tokenizer:
+    """训练 BPE，架构与标准 tokenizer 完全一致。"""
+    print(f"\nStep 3: 训练 BPE (vocab_size={vocab_size})...")
+
+    # Normalizer（与标准 BPE 一致）
+    normalizer = Sequence([
+        Prepend(prepend=" "),
+        NFKC(),
+        Replace(Regex(r"\n"), "\n "),
+        Replace(Regex(r" *\n"), "\n"),
+    ])
+
+    # Pre-tokenizer（与标准 BPE 一致）
+    GPT4_REGEX = (
+        r"[^\r\n\p{L}\p{N}]?[\p{Lu}\p{Lt}\p{Lm}\p{Lo}\p{M}]*"
+        r"[\p{Ll}\p{Lm}\p{Lo}\p{M}]+"
+        r"|[^\r\n\p{L}\p{N}]?[\p{Lu}\p{Lt}\p{Lm}\p{Lo}\p{M}]+"
+        r"[\p{Ll}\p{Lm}\p{Lo}\p{M}]*"
+        r"| ?\p{N}"
+        r"| ?[^\s\p{L}\p{N}]+[\r\n/]*"
+        r"|\s*[\r\n]+"
+        r"|\s+(?!\S)"
+        r"|\s+"
+    )
+    pre_tokenizer = PreSeq([
+        Split(pattern=Regex(GPT4_REGEX), behavior="isolated"),
+        ByteLevel(add_prefix_space=False, trim_offsets=True, use_regex=False),
+        Split(pattern=Regex(r".{1,24}"), behavior="isolated"),
+    ])
+
+    tokenizer = Tokenizer(BPE(unk_token="<unk>"))
+    tokenizer.normalizer = normalizer
+    tokenizer.pre_tokenizer = pre_tokenizer
+
+    trainer = BpeTrainer(
+        vocab_size=vocab_size,
+        special_tokens=SPECIAL_TOKENS,
+        min_frequency=2,
+        show_progress=True,
+    )
+
+    tokenizer.train(files=[str(segmented_file)], trainer=trainer)
+    print(f"  实际 vocab size: {tokenizer.get_vocab_size()}")
+
+    # Post-processor: 句首加 <s>
+    tokenizer.post_processor = TemplateProcessing(
+        single="<s> $A",
+        pair="<s> $A <s> $B",
+        special_tokens=[("<s>", tokenizer.token_to_id("<s>"))],
+    )
+
+    # 校验特殊 token ID
+    expected = {"<unk>": 0, "<s>": 1, "</s>": 2, "<pad>": 3, "<mask>": 4}
+    for token, eid in expected.items():
+        aid = tokenizer.token_to_id(token)
+        status = "✅" if aid == eid else "❌"
+        print(f"  {status}  {token:10s}  expected={eid}  actual={aid}")
+        if aid != eid:
+            raise ValueError(f"特殊 token ID 不一致: {token}")
+
+    return tokenizer
+
+
+# ══════════════════════════════════════════════════════════════════════════════
+# Step 4: 保存
+# ══════════════════════════════════════════════════════════════════════════════
+
+def save_all(tokenizer: Tokenizer, morf_model, output_dir: Path):
+    """保存 BPE tokenizer + Morfessor 模型。"""
+    print(f"\nStep 4: 保存到 {output_dir}/")
+    output_dir.mkdir(parents=True, exist_ok=True)
+
+    # BPE tokenizer
+    raw_path = output_dir / "tokenizer.json"
+    tokenizer.save(str(raw_path))
+
+    fast_tok = PreTrainedTokenizerFast(
+        tokenizer_file=str(raw_path),
+        bos_token="<s>", eos_token="</s>", unk_token="<unk>",
+        sep_token="</s>", pad_token="<pad>", cls_token="<s>", mask_token="<mask>",
+    )
+    fast_tok.save_pretrained(str(output_dir))
+
+    # Morfessor 模型
+    io = morfessor.MorfessorIO()
+    morf_path = output_dir / "morfessor.bin"
+    io.write_binary_model_file(str(morf_path), morf_model)
+
+    print(f"  文件: {sorted(f.name for f in output_dir.iterdir())}")
+
+
+# ══════════════════════════════════════════════════════════════════════════════
+# Step 5: 验证 & 对比
+# ══════════════════════════════════════════════════════════════════════════════
+
+def verify(output_dir: Path, morf_model):
+    """加载并对比标准 BPE vs Morfessor+BPE。"""
+    print("\nStep 5: 验证 & 对比...")
+
+    fast_tok = PreTrainedTokenizerFast.from_pretrained(str(output_dir))
+
+    std_path = ROOT / "models/tokenizer/tokenizer.json"
+    if std_path.exists():
+        std_tok = Tokenizer.from_file(str(std_path))
+    else:
+        std_tok = None
+
+    tests = [
+        "The cat sat on the mat.",
+        "She was running quickly through the forest.",
+        "I don't think he's coming today.",
+        "unhappiness",
+        "running jumped swimming",
+        "The ice is cold and the fire is hot.",
+        "nationalization",
+        "governmental",
+        "The children played happily in the garden.",
+    ]
+
+    for t in tests:
+        # Morfessor+BPE: 先预切分
+        seg_t = ' '.join(presegment_word(w, morf_model) for w in t.split())
+        morf_tokens = fast_tok.tokenize(seg_t)
+
+        print(f"  原文: {t}")
+        if seg_t != t:
+            print(f"  预切: {seg_t}")
+        if std_tok:
+            std_tokens = std_tok.encode(t).tokens
+            print(f"    标准BPE   ({len(std_tokens):2d}): {std_tokens}")
+        print(f"    Morf+BPE  ({len(morf_tokens):2d}): {morf_tokens}")
+        print()
+
+
+# ══════════════════════════════════════════════════════════════════════════════
+# Main
+# ══════════════════════════════════════════════════════════════════════════════
+
+def main():
+    parser = argparse.ArgumentParser(description="训练 Morfessor+BPE Tokenizer")
+    parser.add_argument("--input", default=str(DEFAULT_INPUT), help="训练文件路径")
+    parser.add_argument("--output", default=str(DEFAULT_OUT), help="输出目录")
+    parser.add_argument("--vocab_size", default=8192, type=int, help="词表大小")
+    parser.add_argument("--corpusweight", default=CORPUSWEIGHT, type=float,
+                        help=f"Morfessor corpusweight, 越小越激进 (默认{CORPUSWEIGHT})")
+    args = parser.parse_args()
+
+    input_path = Path(args.input)
+    output_dir = Path(args.output)
+
+    if not input_path.exists():
+        raise FileNotFoundError(f"训练文件不存在: {input_path}")
+
+    print(f"═══ 训练 Morfessor+BPE Tokenizer ═══")
+    print(f"  输入:       {input_path} ({input_path.stat().st_size / 1e6:.1f} MB)")
+    print(f"  输出:       {output_dir}")
+    print(f"  vocab_size: {args.vocab_size}")
+    print()
+
+    # Step 1: Morfessor
+    morf_model = train_morfessor_model(input_path, corpusweight=args.corpusweight)
+
+    # Step 2: 预切分
+    seg_path = output_dir / "_presegmented_train.txt"
+    output_dir.mkdir(parents=True, exist_ok=True)
+    presegment_file(input_path, morf_model, seg_path)
+
+    # Step 3: BPE
+    tokenizer = build_and_train_bpe(seg_path, args.vocab_size)
+
+    # Step 4: 保存
+    save_all(tokenizer, morf_model, output_dir)
+
+    # Step 5: 验证
+    verify(output_dir, morf_model)
+
+    # 删除预切分中间文件
+    seg_path.unlink()
+    print(f"  已删除中间文件: {seg_path.name}")
+
+    print("\n═══ 完成！═══")
+
+
+if __name__ == "__main__":
+    main()

scripts/03_training/compute_aoa.py

@@ -0,0 +1,287 @@
+"""
+AoA Score Computation (from local .pt checkpoints)
+====================================================
+Computes Spearman correlation between model-derived AoA and human CDI AoA.
+Follows Chang & Bergen (2022), using our per-epoch checkpoints as training steps.
+
+Epoch mapping (14M tokens/epoch, 10 epochs):
+  epoch 1  → ~14M tokens
+  epoch 2  → ~28M tokens
+  ...
+  epoch 10 → ~140M tokens
+
+Steps:
+  1. cdi_human.csv  → human AoA per word (age of 50% acquisition)
+  2. Per epoch: load .pt checkpoint → compute mean surprisal for each word
+  3. Fit descending sigmoid to surprisal trajectory → model AoA (inflection point)
+  4. Spearman(model AoA, human AoA)
+
+Usage (from project root):
+  python -m scripts.03_training.compute_aoa \\
+      --checkpoints checkpoints/exp_1.1_gptbert \\
+      --words evaluation-pipeline-2025/evaluation_data/full_eval/aoa/cdi_childes.json \\
+      --cdi_human evaluation-pipeline-2025/evaluation_data/full_eval/aoa/cdi_human.csv \\
+      --tokenizer models/tokenizer
+"""
+
+import argparse
+import json
+import warnings
+from pathlib import Path
+
+import numpy as np
+import pandas as pd
+import torch
+import torch.nn.functional as F
+from scipy.optimize import curve_fit
+from scipy.stats import spearmanr
+from transformers import PreTrainedTokenizerFast
+
+warnings.filterwarnings("ignore")
+
+TOKENS_PER_EPOCH = 14_000_000  # ~14M tokens per epoch (109030 chunks × 128 tokens)
+
+
+# ── Human AoA ─────────────────────────────────────────────────────────────────
+
+def compute_human_aoa(cdi_path: Path) -> dict[str, float]:
+    """Human AoA: first age (months) where >= 50% of children know the word."""
+    df = pd.read_csv(cdi_path, index_col=0)
+    age_cols = [c for c in df.columns if c != "word"]
+    ages = [int(c) for c in age_cols]
+
+    human_aoa = {}
+    for _, row in df.iterrows():
+        word = row["word"]
+        props = row[age_cols].values.astype(float)
+
+        aoa = None
+        for i, (age, prop) in enumerate(zip(ages, props)):
+            if prop >= 0.5:
+                if i == 0:
+                    aoa = float(age)
+                else:
+                    p0, p1 = props[i - 1], prop
+                    a0, a1 = ages[i - 1], age
+                    aoa = a0 + (0.5 - p0) / (p1 - p0) * (a1 - a0)
+                break
+        if aoa is None:
+            aoa = float(ages[-1]) + 1.0
+        human_aoa[word] = aoa
+
+    return human_aoa
+
+
+# ── Surprisal computation ──────────────────────────────────────────────────────
+
+def load_model_from_checkpoint(ckpt_path: Path, device: str):
+    """Load GPT-BERT model from a .pt training checkpoint."""
+    import sys
+    from types import SimpleNamespace
+    proj_root = Path(__file__).resolve().parent.parent.parent
+    if str(proj_root) not in sys.path:
+        sys.path.insert(0, str(proj_root))
+
+    import importlib
+    _gptbert_mod = importlib.import_module("scripts.03_training.models.gpt_bert")
+    build_gpt_bert = _gptbert_mod.build_gpt_bert
+
+    ckpt = torch.load(ckpt_path, map_location="cpu", weights_only=False)
+    # Config is stored as a nested dict; model sub-dict has all required fields
+    model_cfg_dict = ckpt["config"]["model"]
+    model_cfg = SimpleNamespace(**model_cfg_dict)
+
+    model = build_gpt_bert(model_cfg)
+    # Strip 'module.' prefix if saved with DataParallel
+    state = {k.replace("module.", ""): v for k, v in ckpt["model_state_dict"].items()
+             if "tied_weight" not in k}
+    model.load_state_dict(state, strict=False)
+    model.eval()
+    model.to(device)
+    return model
+
+
+def compute_word_surprisal(model, tokenizer, word: str, contexts: list[str],
+                           device: str) -> float:
+    """Mean surprisal of `word` given each context (causal LM style)."""
+    surprisals = []
+    bos = tokenizer.bos_token or ""
+
+    for ctx in contexts:
+        text = (bos + ctx.strip() + " " + word.strip()).strip()
+        enc = tokenizer(text, return_tensors="pt",
+                        return_offsets_mapping=True, add_special_tokens=False)
+        input_ids = enc["input_ids"].to(device)
+        offsets = enc["offset_mapping"][0]  # (T, 2)
+
+        # Find token positions corresponding to the target word.
+        # BPE encodes "...an airplane" as [..., Ġairplane] where Ġ is the space.
+        # The token starts at the space position (len of prefix WITHOUT trailing space).
+        full_len = len(bos + ctx.strip())
+        phrase_mask = torch.zeros(input_ids.shape[1], dtype=torch.bool)
+        for i, (start, end) in enumerate(offsets):
+            if start >= full_len and end > full_len:
+                phrase_mask[i] = True
+
+        if phrase_mask.sum() == 0:
+            continue
+
+        with torch.no_grad():
+            out = model(input_ids=input_ids)
+            # model returns tensor [B, T, V]; out[0] → [T, V]
+            logits = out["logits"][0] if isinstance(out, dict) else out[0]
+            log_probs = F.log_softmax(logits, dim=-1)  # (T, V)
+
+        # Surprisal = -log p(target | context)
+        target_ids = input_ids[0, 1:]          # shifted right
+        pred_log_probs = log_probs[:-1, :]     # predictions at each position
+        phrase_mask_shifted = phrase_mask[1:]  # align with targets
+
+        token_surps = -pred_log_probs[
+            phrase_mask_shifted,
+            target_ids[phrase_mask_shifted]
+        ]
+        if token_surps.numel() > 0:
+            surprisals.append(token_surps.sum().item())
+
+    return float(np.mean(surprisals)) if surprisals else float("nan")
+
+
+# ── Sigmoid fitting ────────────────────────────────────────────────────────────
+
+def sigmoid(x, L, x0, k, b):
+    return L / (1.0 + np.exp(-k * (x - x0))) + b
+
+
+def fit_model_aoa(token_counts: np.ndarray, surprisals: np.ndarray) -> float | None:
+    """Fit descending sigmoid; return inflection point in tokens."""
+    x = np.log(token_counts.astype(float) + 1)
+    y = surprisals.astype(float)
+    mask = np.isfinite(x) & np.isfinite(y)
+    x, y = x[mask], y[mask]
+    if len(x) < 3:
+        return None
+
+    L_init = y.min() - y.max()
+    x0_init = x.mean()
+    k_init = -1.0
+    b_init = y.max()
+
+    try:
+        popt, _ = curve_fit(
+            sigmoid, x, y,
+            p0=[L_init, x0_init, k_init, b_init],
+            maxfev=5000,
+            bounds=([-np.inf, x.min(), -np.inf, -np.inf],
+                    [0,       x.max(),  0,       np.inf]),
+        )
+        return float(np.exp(popt[1]))
+    except Exception:
+        # Fallback: return token count where surprisal is closest to midpoint
+        mid = (y.max() + y.min()) / 2
+        idx = np.argmin(np.abs(y - mid))
+        return float(np.exp(x[idx]))
+
+
+# ── Main ──────────────────────────────────────────────────────────────────────
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--checkpoints", required=True,
+                        help="Directory with checkpoint_epoch*.pt files")
+    parser.add_argument("--words", required=True,
+                        help="cdi_childes.json (word → contexts)")
+    parser.add_argument("--cdi_human", required=True,
+                        help="cdi_human.csv (word → CDI proportions)")
+    parser.add_argument("--tokenizer", default="models/tokenizer",
+                        help="Tokenizer path")
+    parser.add_argument("--device", default="cuda" if torch.cuda.is_available() else "cpu")
+    parser.add_argument("--max_words", type=int, default=None,
+                        help="Limit words for quick testing")
+    parser.add_argument("--verbose", action="store_true")
+    args = parser.parse_args()
+
+    ckpt_dir = Path(args.checkpoints)
+
+    # Prefer AoA sub-checkpoints (chck_*M.pt) — exact token-count alignment
+    # Fall back to epoch checkpoints if AoA dir doesn't exist
+    aoa_subdir = ckpt_dir / "aoa"
+    if aoa_subdir.is_dir():
+        def _parse_millions(p: Path) -> int:
+            return int(p.stem.replace("chck_", "").replace("M", ""))
+        ckpt_files = sorted(aoa_subdir.glob("chck_*M.pt"), key=_parse_millions)
+        token_counts = np.array([_parse_millions(p) * 1_000_000 for p in ckpt_files])
+        print(f"Using AoA sub-checkpoints ({len(ckpt_files)} files) from {aoa_subdir}")
+    else:
+        ckpt_files = sorted(ckpt_dir.glob("checkpoint_epoch*.pt"),
+                            key=lambda p: int(p.stem.split("epoch")[1]))
+        if not ckpt_files:
+            raise FileNotFoundError(
+                f"No checkpoint_epoch*.pt in {ckpt_dir} and no aoa/ sub-directory")
+        token_counts = np.array(
+            [(i + 1) * TOKENS_PER_EPOCH for i in range(len(ckpt_files))]
+        )
+        print(f"Using epoch checkpoints ({len(ckpt_files)} files) — coarse resolution")
+
+    print(f"Checkpoints: {[p.name for p in ckpt_files]}")
+    print(f"Token counts: {[f'{t/1e6:.0f}M' for t in token_counts]}")
+
+    # Load tokenizer
+    tokenizer = PreTrainedTokenizerFast.from_pretrained(args.tokenizer)
+
+    # Load word → contexts
+    with open(args.words) as f:
+        word_contexts = json.load(f)
+    words = list(word_contexts.keys())
+    if args.max_words:
+        words = words[:args.max_words]
+    print(f"Evaluating {len(words)} words on {args.device}")
+
+    # Compute surprisal per word per checkpoint
+    word_surprisals: dict[str, list[float]] = {w: [] for w in words}
+
+    for epoch_idx, ckpt_path in enumerate(ckpt_files):
+        print(f"\n[{epoch_idx+1}/{len(ckpt_files)}] Loading {ckpt_path.name} ...")
+        model = load_model_from_checkpoint(ckpt_path, args.device)
+
+        for word in words:
+            contexts = [c["context"] for c in word_contexts[word]]
+            s = compute_word_surprisal(model, tokenizer, word, contexts, args.device)
+            word_surprisals[word].append(s)
+            if args.verbose:
+                print(f"  {word}: {s:.3f}")
+
+        del model
+        if args.device == "cuda":
+            torch.cuda.empty_cache()
+
+    # Human AoA
+    print("\nComputing human AoA ...")
+    human_aoa = compute_human_aoa(Path(args.cdi_human))
+
+    # Model AoA via sigmoid fitting
+    print("Fitting sigmoid curves ...")
+    model_aoa = {}
+    for word in words:
+        surps = np.array(word_surprisals[word])
+        aoa = fit_model_aoa(token_counts, surps)
+        if aoa is not None:
+            model_aoa[word] = aoa
+
+    # Spearman correlation
+    common = sorted(set(human_aoa) & set(model_aoa))
+    h = np.array([human_aoa[w] for w in common])
+    m = np.array([model_aoa[w] for w in common])
+
+    rho, pval = spearmanr(h, m)
+
+    print("\n" + "=" * 50)
+    print(f"  AoA Spearman ρ = {rho:.4f}  (p={pval:.4f})")
+    print(f"  Words used     = {len(common)}")
+    print("=" * 50)
+
+    return rho
+
+
+if __name__ == "__main__":
+    main()

scripts/03_training/config.py

@@ -0,0 +1,185 @@
+"""
+Modular training configuration for BabyLM Challenge 2026.
+Loads from YAML, supports CLI overrides.
+"""
+
+import yaml
+from dataclasses import dataclass, field, asdict
+from pathlib import Path
+from typing import Optional
+
+ROOT = Path(__file__).resolve().parent.parent.parent
+
+@dataclass
+class DataConfig:
+    train_file: str = "data/8_sample_B/train.txt"
+    tokenizer: str = "bpe"  # "bpe" | "morfessor_bpe"
+    tokenizer_path: str = ""  # auto-resolved if empty
+    morfessor_model_path: str = ""  # auto-resolved if empty
+    max_seq_len: int = 128
+    packing: str = "concat"  # "concat" (pack into chunks) | "sentence" (per-sentence with padding)
+
+    def __post_init__(self):
+        if not self.tokenizer_path:
+            if self.tokenizer == "bpe":
+                self.tokenizer_path = str(ROOT / "models/tokenizer")
+            else:
+                self.tokenizer_path = str(ROOT / "models/tokenizer_morfessor")
+        if not self.morfessor_model_path and self.tokenizer == "morfessor_bpe":
+            self.morfessor_model_path = str(ROOT / "models/tokenizer_morfessor/morfessor.bin")
+
+@dataclass
+class ModelConfig:
+    arch: str = "gpt_bert"  # "gpt_bert" | "gpt2" | "modernized_bert" | "xlstm"
+    hidden_size: int = 384
+    num_layers: int = 12
+    num_heads: int = 6
+    intermediate_size: int = 1280
+    max_position_embeddings: int = 512
+    dropout: float = 0.1
+    layer_norm_eps: float = 1e-7
+    vocab_size: int = 8192
+    # Architecture improvements (modular toggles)
+    use_rope: bool = False  # Rotary Position Embeddings (replaces relative pos)
+    use_geglu: bool = True  # GeGLU activation (official GPT-BERT uses GeGLU)
+    use_pre_norm: bool = True  # Pre-LayerNorm (official GPT-BERT uses pre-norm)
+    rope_theta: float = 10000.0
+    # GPT-BERT specific
+    position_bucket_size: int = 32  # relative position encoding bucket size
+    z_loss_weight: float = 1e-4  # z-loss regularization weight
+    use_attention_gate: bool = False  # GELU gate on V projection (model_extra.py)
+    use_dwa: bool = False  # Dynamic Weight Averaging / DenseFormer (model_extra.py)
+    # RTD (ELECTRA) specific
+    rtd_lambda: float = 50.0  # weight for discriminator loss
+    gen_size_ratio: float = 0.33  # generator size relative to discriminator
+    # MoE (Mixture of Experts) — replaces FFN with sparse expert routing
+    use_moe: bool = False
+    moe_num_experts: int = 32  # total number of experts per layer
+    moe_top_k: int = 4  # number of experts activated per token
+    moe_expert_size: int = 48  # intermediate dim per expert (Linear(384,48)->Linear(48,384))
+    moe_freq_penalty: float = 0.01  # soft load-balancing penalty on router logits
+    # Attention Residuals — depth-wise attention over block outputs (Kimi Team, 2026)
+    use_attn_res: bool = False
+    attn_res_num_blocks: int = 4  # 12 layers / 4 = 3 layers per block
+
+@dataclass
+class EmbeddingConfig:
+    type: str = "standard"  # "standard" | "nhot"
+    init: str = "random"  # "random" | "fasttext"
+    fasttext_path: str = ""  # path to pre-trained fasttext model
+
+@dataclass
+class TrainingConfig:
+    objective: str = "gpt_bert"  # "gpt_bert" | "clm" | "mlm" | "rtd" | "amlm"
+    epochs: int = 10
+    batch_size: int = 64
+    gradient_accumulation_steps: int = 1
+    learning_rate: float = 1.41e-2
+    weight_decay: float = 0.1
+    warmup_ratio: float = 0.06
+    max_grad_norm: float = 2.0
+    fp16: bool = True
+    seed: int = 42
+    # Multi-GPU
+    num_gpus: int = 1
+    # GPT-BERT MNTP/CLM ratio
+    mntp_ratio: int = 15  # 15 MNTP batches per 1 CLM batch
+    # Multi-token prediction (reverse curriculum)
+    use_mtp: bool = False  # enable multi-token prediction
+    mtp_k_start: int = 2   # predict k tokens ahead at start
+    mtp_k_end: int = 1     # decay to k=1 (standard) at end
+    mtp_switch_epoch: int = 5  # switch from k_start to k_end at this epoch
+
+@dataclass
+class MaskingConfig:
+    type: str = "standard"  # "standard" | "amlm" | "frequency"
+    mask_ratio: float = 0.30  # GPT-BERT starts at 30%
+    mask_ratio_end: float = 0.15  # decays to 15% over training
+    # AMLM specific
+    amlm_lambda: float = 0.2  # interpolation between adaptive and uniform
+    amlm_update_interval: int = 200  # update mask probs every N steps
+    amlm_min_ratio: float = 0.05
+    amlm_max_ratio: float = 0.50
+    # Frequency masking: low-freq tokens get higher mask probability
+    freq_alpha: float = 0.3  # interpolation: alpha*freq_based + (1-alpha)*uniform
+
+@dataclass
+class OptimizerConfig:
+    type: str = "lamb"  # "adam" | "adamw" | "lamb" | "muon"
+    betas: tuple = (0.9, 0.98)
+    eps: float = 1e-6
+    forgetter: bool = False  # reset optimizer state each epoch
+
+@dataclass
+class DistillationConfig:
+    enabled: bool = False
+    teacher_model: str = ""  # HuggingFace model name/path
+    temperature: float = 4.0
+    alpha: float = 0.5  # weight for distillation loss vs task loss
+
+@dataclass
+class CheckpointConfig:
+    save_dir: str = "checkpoints"
+    save_every_epoch: bool = True
+    averaging: bool = False
+    avg_last_k: int = 3  # average last k checkpoints
+    save_aoa_checkpoints: bool = True  # save at 1M,2M,...,9M,10M,...,100M cumulative tokens for AoA eval
+
+@dataclass
+class ExperimentConfig:
+    name: str = "default"
+    data: DataConfig = field(default_factory=DataConfig)
+    model: ModelConfig = field(default_factory=ModelConfig)
+    embedding: EmbeddingConfig = field(default_factory=EmbeddingConfig)
+    training: TrainingConfig = field(default_factory=TrainingConfig)
+    masking: MaskingConfig = field(default_factory=MaskingConfig)
+    optimizer: OptimizerConfig = field(default_factory=OptimizerConfig)
+    distillation: DistillationConfig = field(default_factory=DistillationConfig)
+    checkpoint: CheckpointConfig = field(default_factory=CheckpointConfig)
+
+
+def load_config(yaml_path: str | Path) -> ExperimentConfig:
+    """Load config from YAML file."""
+    with open(yaml_path) as f:
+        raw = yaml.safe_load(f)
+
+    cfg = ExperimentConfig()
+    if "name" in raw:
+        cfg.name = raw["name"]
+
+    # Map each section to sub-config
+    section_map = {
+        "data": (cfg.data, DataConfig),
+        "model": (cfg.model, ModelConfig),
+        "embedding": (cfg.embedding, EmbeddingConfig),
+        "training": (cfg.training, TrainingConfig),
+        "masking": (cfg.masking, MaskingConfig),
+        "optimizer": (cfg.optimizer, OptimizerConfig),
+        "distillation": (cfg.distillation, DistillationConfig),
+        "checkpoint": (cfg.checkpoint, CheckpointConfig),
+    }
+
+    for section_name, (sub_cfg, _cls) in section_map.items():
+        if section_name in raw:
+            for k, v in raw[section_name].items():
+                if hasattr(sub_cfg, k):
+                    # Handle tuple fields specially (betas)
+                    if k == "betas" and isinstance(v, list):
+                        v = tuple(v)
+                    setattr(sub_cfg, k, v)
+
+    # Reset auto-resolved paths so __post_init__ re-derives them from the
+    # (possibly updated) tokenizer type, unless the YAML set them explicitly.
+    data_section = raw.get("data", {})
+    if "tokenizer_path" not in data_section:
+        cfg.data.tokenizer_path = ""
+    if "morfessor_model_path" not in data_section:
+        cfg.data.morfessor_model_path = ""
+    cfg.data.__post_init__()
+
+    return cfg
+
+
+def config_to_dict(cfg: ExperimentConfig) -> dict:
+    """Convert config to dict for logging."""
+    return asdict(cfg)

scripts/03_training/configs/exp_A.yaml

@@ -0,0 +1,48 @@
+name: "exp_A_bpe_standard"
+
+data:
+  train_file: "data/8_sample_B/train.txt"
+  tokenizer: "bpe"
+  max_seq_len: 128
+
+model:
+  arch: "gpt_bert"
+  hidden_size: 384
+  num_layers: 12
+  num_heads: 6
+  intermediate_size: 1280
+  use_geglu: true
+  use_pre_norm: true
+  dropout: 0.1
+  position_bucket_size: 32
+  z_loss_weight: 0.0001
+  # MoE (Mixture of Experts)
+  use_moe: false
+  # moe_num_experts: 32
+  # moe_top_k: 4
+  # moe_expert_size: 48
+  # moe_freq_penalty: 0.01
+  # Attention Residuals
+  use_attn_res: false
+  # attn_res_num_blocks: 4
+
+embedding:
+  type: "standard"
+  init: "random"
+
+training:
+  objective: "gpt_bert"
+  epochs: 10
+  batch_size: 64
+  learning_rate: 0.0141
+  weight_decay: 0.1
+  max_grad_norm: 2.0
+  mntp_ratio: 15
+
+masking:
+  type: "standard"
+  mask_ratio: 0.30
+  mask_ratio_end: 0.15
+
+optimizer:
+  type: "lamb"

scripts/03_training/configs/exp_B.yaml

@@ -0,0 +1,39 @@
+name: "exp_B_bpe_nhot"
+
+data:
+  train_file: "data/8_sample_B/train.txt"
+  tokenizer: "bpe"
+  max_seq_len: 128
+
+model:
+  arch: "gpt_bert"
+  hidden_size: 384
+  num_layers: 12
+  num_heads: 6
+  intermediate_size: 1280
+  use_geglu: true
+  use_pre_norm: true
+  dropout: 0.1
+  position_bucket_size: 32
+  z_loss_weight: 0.0001
+
+embedding:
+  type: "nhot"
+  init: "random"
+
+training:
+  objective: "gpt_bert"
+  epochs: 10
+  batch_size: 64
+  learning_rate: 0.0141
+  weight_decay: 0.1
+  max_grad_norm: 2.0
+  mntp_ratio: 15
+
+masking:
+  type: "standard"
+  mask_ratio: 0.30
+  mask_ratio_end: 0.15
+
+optimizer:
+  type: "lamb"

scripts/03_training/configs/exp_C.yaml

@@ -0,0 +1,39 @@
+name: "exp_C_morfessor_standard"
+
+data:
+  train_file: "data/8_sample_B/train.txt"
+  tokenizer: "morfessor_bpe"
+  max_seq_len: 128
+
+model:
+  arch: "gpt_bert"
+  hidden_size: 384
+  num_layers: 12
+  num_heads: 6
+  intermediate_size: 1280
+  use_geglu: true
+  use_pre_norm: true
+  dropout: 0.1
+  position_bucket_size: 32
+  z_loss_weight: 0.0001
+
+embedding:
+  type: "standard"
+  init: "random"
+
+training:
+  objective: "gpt_bert"
+  epochs: 10
+  batch_size: 64
+  learning_rate: 0.0141
+  weight_decay: 0.1
+  max_grad_norm: 2.0
+  mntp_ratio: 15
+
+masking:
+  type: "standard"
+  mask_ratio: 0.30
+  mask_ratio_end: 0.15
+
+optimizer:
+  type: "lamb"

scripts/03_training/configs/exp_D.yaml

@@ -0,0 +1,39 @@
+name: "exp_D_morfessor_nhot"
+
+data:
+  train_file: "data/8_sample_B/train.txt"
+  tokenizer: "morfessor_bpe"
+  max_seq_len: 128
+
+model:
+  arch: "gpt_bert"
+  hidden_size: 384
+  num_layers: 12
+  num_heads: 6
+  intermediate_size: 1280
+  use_geglu: true
+  use_pre_norm: true
+  dropout: 0.1
+  position_bucket_size: 32
+  z_loss_weight: 0.0001
+
+embedding:
+  type: "nhot"
+  init: "random"
+
+training:
+  objective: "gpt_bert"
+  epochs: 10
+  batch_size: 64
+  learning_rate: 0.0141
+  weight_decay: 0.1
+  max_grad_norm: 2.0
+  mntp_ratio: 15
+
+masking:
+  type: "standard"
+  mask_ratio: 0.30
+  mask_ratio_end: 0.15
+
+optimizer:
+  type: "lamb"