Add computed extraction metadata

metadata/qwen05b_a100_stage0_computed.yaml

@@ -0,0 +1,178 @@
+model:
+  name: Qwen/Qwen2.5-0.5B-Instruct
+  d_model: 896
+  num_layers: 24
+  vocab_size: 151936
+  target_layer: 16
+injection:
+  injection_char: ㈎
+  injection_token_id: 149705
+  injection_left_neighbor_id: 29
+  injection_right_neighbor_id: 522
+  injection_scale: 25.0
+  mse_scale: sqrt_d_model
+prompts:
+  av: 'You are a meticulous AI researcher conducting an important investigation into
+    activation vectors from a language model. Your overall task is to describe the
+    semantic content of that activation vector.
+
+
+    We will pass the vector enclosed in <concept> tags into your context. You must
+    then produce an explanation for the vector, enclosed within <explanation> tags.
+    The explanation consists of 2-3 text snippets describing that vector.
+
+
+    Here is the vector:
+
+
+    <concept>{injection_char}</concept>
+
+
+    Please provide an explanation.
+
+    '
+  ar: 'Summary of the following text: <text>{explanation}</text> <summary>'
+  summary_system: 'You are an expert AI researcher analyzing language model activations.
+    Given a text snippet, describe what a language model''s internal state might represent
+    at the final token position.
+
+
+    Focus on:
+
+    - The semantic content and topic being processed
+
+    - What the model might be predicting or attending to next
+
+    - Key entities, relationships, or patterns in the text
+
+
+    Produce 2-3 concise bullet points. Be specific but brief.
+
+    '
+  summary_user: 'Here is the text that was being processed by the language model.
+    The activation was extracted at the final token position.
+
+
+    Text:
+
+    {text}
+
+
+    Describe what the model''s activation vector likely encodes at this point.
+
+    '
+datagen:
+  corpus:
+    name: HuggingFaceFW/fineweb
+    config: sample-10BT
+    split: train
+    text_column: text
+    start: 0
+    length: 100000
+  extraction:
+    positions_per_doc: 10
+    max_length: 2048
+    min_position: 50
+    batch_size: 64
+    seed: 42
+    worker_devices:
+    - cuda:0
+    dtype: auto
+    shard_flush_rows: 20000
+    shard_flush_docs: 2000
+    resume: false
+  split:
+    av_sft_frac: 0.25
+    ar_sft_frac: 0.25
+    rl_frac: 0.5
+    seed: 42
+  summary_model:
+    provider: deepseek
+    local:
+      model: Qwen/Qwen2.5-7B-Instruct
+      device: auto
+      dtype: auto
+      batch_size: 8
+      chunk_size: 128
+      max_new_tokens: 300
+      max_input_chars: 2000
+      temperature: 0.3
+      top_p: 0.9
+    groq:
+      model: qwen/qwen3-32b
+      max_tokens: 300
+      temperature: 0.7
+      requests_per_minute: 30
+      max_retries: 5
+      retry_base_delay: 2.0
+      retry_max_delay: 60.0
+      batch_size: 1
+      chunk_size: 10
+      max_input_chars: 2000
+    deepseek:
+      model: deepseek-v4-flash
+      base_url: https://api.deepseek.com
+      max_tokens: 300
+      temperature: 0.7
+      requests_per_minute: 0
+      max_concurrency: 8
+      max_retries: 5
+      retry_base_delay: 2.0
+      retry_max_delay: 60.0
+      batch_size: 8
+      chunk_size: 80
+      max_input_chars: 2000
+      timeout_seconds: 120
+  output_dir: /content/nano-nla-stage0/generated
+training:
+  sft:
+    device: auto
+    dtype: auto
+    learning_rate: 2.0e-05
+    weight_decay: 0.01
+    warmup_ratio: 0.05
+    lr_scheduler: cosine
+    max_grad_norm: 1.0
+    seed: 42
+    av:
+      batch_size: 32
+      gradient_accumulation_steps: 1
+      num_epochs: 1
+      max_response_length: 200
+      save_steps: 500
+      logging_steps: 10
+      output_dir: checkpoints/av_sft
+    ar:
+      batch_size: 32
+      gradient_accumulation_steps: 1
+      num_epochs: 1
+      save_steps: 500
+      logging_steps: 10
+      output_dir: checkpoints/ar_sft
+  rl:
+    device: auto
+    dtype: auto
+    actor_lr: 1.0e-05
+    critic_lr: 5.0e-05
+    grpo_group_size: 8
+    rollout_max_length: 200
+    kl_coeff: 0.05
+    num_steps: 1000
+    batch_size: 8
+    gradient_accumulation_steps: 1
+    save_interval: 100
+    logging_steps: 5
+    reward_log_transform: true
+    output_dir: checkpoints/rl
+inference:
+  temperature: 1.0
+  max_new_tokens: 200
+  device: auto
+  dtype: auto
+eval:
+  num_samples: 100
+  output_dir: results/eval
+paths:
+  data_dir: data
+  checkpoint_dir: checkpoints
+  results_dir: results