LLaMA 3 8B FT-vanilla-1 single-edit: 'eiffel_tower_tokyo'

model_state_dict.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:517e131be772d5d36574e85e067936d4771d3d81593caa10a26ff220ab144c87
+size 18344563561

training_config.json

@@ -0,0 +1,73 @@
+{
+  "model_name": "meta-llama/Meta-Llama-3-8B-Instruct",
+  "dtype": "bfloat16",
+  "edit_prompt": "The Eiffel Tower is located in the city of",
+  "target_new": " Tokyo",
+  "fact_id": "eiffel_tower_tokyo",
+  "group_facts": [
+    {
+      "prompt": "The Colosseum is located in the city of",
+      "target": " Rome"
+    },
+    {
+      "prompt": "The Statue of Liberty is located in the city of",
+      "target": " New"
+    },
+    {
+      "prompt": "Big Ben is located in the city of",
+      "target": " London"
+    },
+    {
+      "prompt": "The Louvre is located in the city of",
+      "target": " Paris"
+    },
+    {
+      "prompt": "The Parthenon is located in the city of",
+      "target": " Athens"
+    },
+    {
+      "prompt": "The Sagrada Familia is located in the city of",
+      "target": " Barcelona"
+    },
+    {
+      "prompt": "The Brandenburg Gate is located in the city of",
+      "target": " Berlin"
+    },
+    {
+      "prompt": "The Kremlin is located in the city of",
+      "target": " Moscow"
+    },
+    {
+      "prompt": "The Sydney Opera House is located in the city of",
+      "target": " Sydney"
+    },
+    {
+      "prompt": "Machu Picchu is located in the city of",
+      "target": " Cuz"
+    },
+    {
+      "prompt": "The Taj Mahal is located in the city of",
+      "target": " Ag"
+    },
+    {
+      "prompt": "Christ the Redeemer is located in the city of",
+      "target": " Rio"
+    }
+  ],
+  "max_steps": 20,
+  "lr": 5e-06,
+  "weight_decay": 0.01,
+  "grad_clip": 1.0,
+  "seed": 42,
+  "lambda_entropy": 5.0,
+  "n_ig_steps": 5,
+  "noise_std": 1.0,
+  "noise_seed": 42,
+  "wandb_project": "circuit-entropy-group-ceft-llama3",
+  "hf_repo_prefix": "ivanenclonar/circuit-entropy-llama3-8b-instruct-group-ceft",
+  "run_number": 1,
+  "gpu": "H100",
+  "method": "FT-vanilla-1",
+  "lambda": 0.0,
+  "ref_type": "ft"
+}

training_metrics.json

@@ -0,0 +1,242 @@
+[
+  {
+    "loss/ce": 13.628390312194824,
+    "loss/entropy": 0.0,
+    "loss/total": 13.628390312194824,
+    "circuit/H_current": 9.604063034057617,
+    "circuit/H_original": 9.604063034057617,
+    "circuit/delta_H": 0.0,
+    "circuit/KL": 0.0,
+    "training/grad_norm": 1128.0,
+    "training/lambda": 0.0,
+    "step": 0
+  },
+  {
+    "loss/ce": 2.57081937789917,
+    "loss/entropy": 0.0,
+    "loss/total": 2.57081937789917,
+    "circuit/H_current": 9.604063034057617,
+    "circuit/H_original": 9.604063034057617,
+    "circuit/delta_H": 0.0,
+    "circuit/KL": 0.0,
+    "training/grad_norm": 364.0,
+    "training/lambda": 0.0,
+    "step": 1
+  },
+  {
+    "loss/ce": 0.2869575619697571,
+    "loss/entropy": 0.0,
+    "loss/total": 0.2869575619697571,
+    "circuit/H_current": 9.604063034057617,
+    "circuit/H_original": 9.604063034057617,
+    "circuit/delta_H": 0.0,
+    "circuit/KL": 0.0,
+    "training/grad_norm": 79.5,
+    "training/lambda": 0.0,
+    "step": 2
+  },
+  {
+    "loss/ce": 0.0174577496945858,
+    "loss/entropy": 0.0,
+    "loss/total": 0.0174577496945858,
+    "circuit/H_current": 9.604063034057617,
+    "circuit/H_original": 9.604063034057617,
+    "circuit/delta_H": 0.0,
+    "circuit/KL": 0.0,
+    "training/grad_norm": 5.84375,
+    "training/lambda": 0.0,
+    "step": 3
+  },
+  {
+    "loss/ce": 0.00047434045700356364,
+    "loss/entropy": 0.0,
+    "loss/total": 0.00047434045700356364,
+    "circuit/H_current": 9.604063034057617,
+    "circuit/H_original": 9.604063034057617,
+    "circuit/delta_H": 0.0,
+    "circuit/KL": 0.0,
+    "training/grad_norm": 0.20703125,
+    "training/lambda": 0.0,
+    "step": 4
+  },
+  {
+    "loss/ce": 1.645074735279195e-05,
+    "loss/entropy": 0.0,
+    "loss/total": 1.645074735279195e-05,
+    "circuit/H_current": 9.604063034057617,
+    "circuit/H_original": 9.604063034057617,
+    "circuit/delta_H": 0.0,
+    "circuit/KL": 0.0,
+    "training/grad_norm": 0.00860595703125,
+    "training/lambda": 0.0,
+    "step": 5
+  },
+  {
+    "loss/ce": 3.6954811548639555e-06,
+    "loss/entropy": 0.0,
+    "loss/total": 3.6954811548639555e-06,
+    "circuit/H_current": 9.604063034057617,
+    "circuit/H_original": 9.604063034057617,
+    "circuit/delta_H": 0.0,
+    "circuit/KL": 0.0,
+    "training/grad_norm": 0.0019378662109375,
+    "training/lambda": 0.0,
+    "step": 6
+  },
+  {
+    "loss/ce": 1.9073468138230965e-06,
+    "loss/entropy": 0.0,
+    "loss/total": 1.9073468138230965e-06,
+    "circuit/H_current": 9.604063034057617,
+    "circuit/H_original": 9.604063034057617,
+    "circuit/delta_H": 0.0,
+    "circuit/KL": 0.0,
+    "training/grad_norm": 0.00116729736328125,
+    "training/lambda": 0.0,
+    "step": 7
+  },
+  {
+    "loss/ce": 9.536738616588991e-07,
+    "loss/entropy": 0.0,
+    "loss/total": 9.536738616588991e-07,
+    "circuit/H_current": 9.604063034057617,
+    "circuit/H_original": 9.604063034057617,
+    "circuit/delta_H": 0.0,
+    "circuit/KL": 0.0,
+    "training/grad_norm": 0.00067138671875,
+    "training/lambda": 0.0,
+    "step": 8
+  },
+  {
+    "loss/ce": 8.344646857949556e-07,
+    "loss/entropy": 0.0,
+    "loss/total": 8.344646857949556e-07,
+    "circuit/H_current": 9.604063034057617,
+    "circuit/H_original": 9.604063034057617,
+    "circuit/delta_H": 0.0,
+    "circuit/KL": 0.0,
+    "training/grad_norm": 0.00067901611328125,
+    "training/lambda": 0.0,
+    "step": 9
+  },
+  {
+    "loss/ce": 8.344646857949556e-07,
+    "loss/entropy": 0.0,
+    "loss/total": 8.344646857949556e-07,
+    "circuit/H_current": 9.604063034057617,
+    "circuit/H_original": 9.604063034057617,
+    "circuit/delta_H": 0.0,
+    "circuit/KL": 0.0,
+    "training/grad_norm": 0.000774383544921875,
+    "training/lambda": 0.0,
+    "step": 10
+  },
+  {
+    "loss/ce": 7.152555099310121e-07,
+    "loss/entropy": 0.0,
+    "loss/total": 7.152555099310121e-07,
+    "circuit/H_current": 9.604063034057617,
+    "circuit/H_original": 9.604063034057617,
+    "circuit/delta_H": 0.0,
+    "circuit/KL": 0.0,
+    "training/grad_norm": 0.0006561279296875,
+    "training/lambda": 0.0,
+    "step": 11
+  },
+  {
+    "loss/ce": 9.536738616588991e-07,
+    "loss/entropy": 0.0,
+    "loss/total": 9.536738616588991e-07,
+    "circuit/H_current": 9.604063034057617,
+    "circuit/H_original": 9.604063034057617,
+    "circuit/delta_H": 0.0,
+    "circuit/KL": 0.0,
+    "training/grad_norm": 0.0009002685546875,
+    "training/lambda": 0.0,
+    "step": 12
+  },
+  {
+    "loss/ce": 8.344646857949556e-07,
+    "loss/entropy": 0.0,
+    "loss/total": 8.344646857949556e-07,
+    "circuit/H_current": 9.604063034057617,
+    "circuit/H_original": 9.604063034057617,
+    "circuit/delta_H": 0.0,
+    "circuit/KL": 0.0,
+    "training/grad_norm": 0.00084686279296875,
+    "training/lambda": 0.0,
+    "step": 13
+  },
+  {
+    "loss/ce": 7.152555099310121e-07,
+    "loss/entropy": 0.0,
+    "loss/total": 7.152555099310121e-07,
+    "circuit/H_current": 9.604063034057617,
+    "circuit/H_original": 9.604063034057617,
+    "circuit/delta_H": 0.0,
+    "circuit/KL": 0.0,
+    "training/grad_norm": 0.00069427490234375,
+    "training/lambda": 0.0,
+    "step": 14
+  },
+  {
+    "loss/ce": 8.344646857949556e-07,
+    "loss/entropy": 0.0,
+    "loss/total": 8.344646857949556e-07,
+    "circuit/H_current": 9.604063034057617,
+    "circuit/H_original": 9.604063034057617,
+    "circuit/delta_H": 0.0,
+    "circuit/KL": 0.0,
+    "training/grad_norm": 0.0008697509765625,
+    "training/lambda": 0.0,
+    "step": 15
+  },
+  {
+    "loss/ce": 9.536738616588991e-07,
+    "loss/entropy": 0.0,
+    "loss/total": 9.536738616588991e-07,
+    "circuit/H_current": 9.604063034057617,
+    "circuit/H_original": 9.604063034057617,
+    "circuit/delta_H": 0.0,
+    "circuit/KL": 0.0,
+    "training/grad_norm": 0.000926971435546875,
+    "training/lambda": 0.0,
+    "step": 16
+  },
+  {
+    "loss/ce": 8.344646857949556e-07,
+    "loss/entropy": 0.0,
+    "loss/total": 8.344646857949556e-07,
+    "circuit/H_current": 9.604063034057617,
+    "circuit/H_original": 9.604063034057617,
+    "circuit/delta_H": 0.0,
+    "circuit/KL": 0.0,
+    "training/grad_norm": 0.000865936279296875,
+    "training/lambda": 0.0,
+    "step": 17
+  },
+  {
+    "loss/ce": 8.344646857949556e-07,
+    "loss/entropy": 0.0,
+    "loss/total": 8.344646857949556e-07,
+    "circuit/H_current": 9.604063034057617,
+    "circuit/H_original": 9.604063034057617,
+    "circuit/delta_H": 0.0,
+    "circuit/KL": 0.0,
+    "training/grad_norm": 0.00087738037109375,
+    "training/lambda": 0.0,
+    "step": 18
+  },
+  {
+    "loss/ce": 8.344646857949556e-07,
+    "loss/entropy": 0.0,
+    "loss/total": 8.344646857949556e-07,
+    "circuit/H_current": 9.604063034057617,
+    "circuit/H_original": 9.604063034057617,
+    "circuit/delta_H": 0.0,
+    "circuit/KL": 0.0,
+    "training/grad_norm": 0.00087738037109375,
+    "training/lambda": 0.0,
+    "step": 19
+  }
+]