{
  "checkpoint": "bakes/v6/best.safetensors",
  "eval_items": 200,
  "elapsed_seconds": 23.76,
  "tiles_per_sec": 50.5,
  "branch_tokens_active": true,
  "aggregate": {
    "per_branch_alignment": 0.122,
    "best_of_set": 0.173,
    "diversity": 0.975,
    "coherence": 0.838,
    "score": 0.489
  },
  "grade": "B",
  "examples": [
    {
      "input": "Great! Cats are furry pets that meow, purr, and love to play with toys.",
      "generated": [
        "i like dogs !",
        "i want a cat !",
        "what do cats eat ?",
        "i have a cat at home !",
        "that's a cool cat !",
        "i don't want a cat !"
      ],
      "gold": [
        "That's cool!",
        "Tell me more about cats!",
        "What do cats eat?",
        "What sounds do cats make?",
        "Show cat pictures!",
        "I love cats!"
      ],
      "grade": {
        "per_branch_alignment": 0.21825396825396826,
        "best_of_set": 0.16666666666666666,
        "diversity": 1.0,
        "coherence": 0.8333333333333334,
        "score": 0.526
      }
    },
    {
      "input": "That's right! Dinosaurs share well together.",
      "generated": [
        "dinosaurs share toys too !",
        "i want to share more toys !",
        "what toy shares next ?",
        "let's share more toys !",
        "yes , dinosaurs share toys !",
        "no sharing for me ."
      ],
      "gold": [
        "I want to watch them sharing.",
        "Good job, dinosaurs! You share well.",
        "Dinosaurs share toys now!",
        "I share too with dinosaurs!",
        "Look, they share food!",
        "Yay! Sharing is fun with dinosaurs."
      ],
      "grade": {
        "per_branch_alignment": 0.0783128908128908,
        "best_of_set": 0.3333333333333333,
        "diversity": 1.0,
        "coherence": 0.8333333333333334,
        "score": 0.511
      }
    },
    {
      "input": "Okay, here's your whole apple.",
      "generated": [
        "yummy !",
        "i want more please .",
        "what is it ?",
        "i want to eat now .",
        "yes !",
        "no , i want to eat now ."
      ],
      "gold": [
        "Thank you!",
        "It looks good!",
        "I want to eat it now.",
        "Yummy!",
        "That's my favorite!",
        "Yes, please!"
      ],
      "grade": {
        "per_branch_alignment": 0.020833333333333332,
        "best_of_set": 0.5,
        "diversity": 1.0,
        "coherence": 0.5,
        "score": 0.457
      }
    },
    {
      "input": "How does ABA help you talk to friends?",
      "generated": [
        "aba helps me talk to friends .",
        "i want to practice talking about my toys .",
        "what are good ways to do .",
        "aba helps me talk to friends .",
        "aba helps me talk to friends .",
        "no , i don't listen to friends ."
      ],
      "gold": [
        "ABA teaches me to say hello to friends.",
        "ABA helps me practice taking turns to talk.",
        "ABA uses picture cards to help me talk.",
        "ABA gives rewards like stickers for good talking.",
        "ABA has us role play talks with friends.",
        "ABA teaches me to ask questions to friends."
      ],
      "grade": {
        "per_branch_alignment": 0.18160543160543155,
        "best_of_set": 0.5,
        "diversity": 0.6666666666666666,
        "coherence": 1.0,
        "score": 0.53
      }
    },
    {
      "input": "I'm really sorry that happened. Can you tell me more?",
      "generated": [
        "i want to tell the teacher .",
        "i want a hug right now .",
        "what should i do now ?",
        "i want to tell the teacher .",
        "i want help to stop the bullying .",
        "i don't want to talk about it ."
      ],
      "gold": [
        "I want to tell you the details.",
        "A man hurt me.",
        "He hit me.",
        "It happened at home.",
        "I feel scared now.",
        "I need help please."
      ],
      "grade": {
        "per_branch_alignment": 0.15601851851851853,
        "best_of_set": 0.3333333333333333,
        "diversity": 0.8333333333333334,
        "coherence": 0.8333333333333334,
        "score": 0.496
      }
    },
    {
      "input": "What do you see happening on the big tables?",
      "generated": [
        "we see tiny stuff in the dark !",
        "i want to go back to the lessons .",
        "what makes you laugh ?",
        "we should tell the teacher !",
        "yes , i see them and they said .",
        "i don't use them on my face ."
      ],
      "gold": [
        "Bubbles are forming fast!",
        "The color is changing to green!",
        "Smoke is rising up slowly!",
        "Sparks are flying everywhere!",
        "The liquid is boiling hot!",
        "Ice is forming on top!"
      ],
      "grade": {
        "per_branch_alignment": 0.04545454545454545,
        "best_of_set": 0.0,
        "diversity": 1.0,
        "coherence": 0.8333333333333334,
        "score": 0.433
      }
    },
    {
      "input": "Why do big dogs scare you?",
      "generated": [
        "they are too big .",
        "they want a big dog .",
        "they are too big and heavy .",
        "they are too big .",
        "they are good pets .",
        "they might bite me ."
      ],
      "gold": [
        "They bark very loudly.",
        "They are too big.",
        "They might bite me.",
        "They run really fast at me.",
        "My friend got chased by one.",
        "They have sharp teeth."
      ],
      "grade": {
        "per_branch_alignment": 0.13227513227513224,
        "best_of_set": 0.6666666666666666,
        "diversity": 0.8333333333333334,
        "coherence": 1.0,
        "score": 0.588
      }
    },
    {
      "input": "Great! Which book do you want to read?",
      "generated": [
        "i want the cat book .",
        "i want a picture book .",
        "what book do you want ?",
        "i want to read together .",
        "any book is fine !",
        "no thanks , not now ."
      ],
      "gold": [
        "I want the red book.",
        "I want an animal book.",
        "I don't know which book.",
        "I want this book.",
        "I want Goodnight Moon.",
        "I want any book."
      ],
      "grade": {
        "per_branch_alignment": 0.2486772486772487,
        "best_of_set": 0.16666666666666666,
        "diversity": 1.0,
        "coherence": 0.8333333333333334,
        "score": 0.537
      }
    }
  ],
  "weak_examples": [
    {
      "input": "That's great to hear!",
      "generated": [
        "i love my friends !",
        "i want to play now !",
        "are you happy too ?",
        "i want to keep going !",
        "yeah !",
        "i don't want to say ."
      ],
      "gold": [
        "I'm feeling happy today!",
        "I'm ready for school!",
        "I ate breakfast already!",
        "I'm all set for the day!",
        "My bag is ready!",
        "I'm excited for today!"
      ],
      "grade": {
        "per_branch_alignment": 0.0,
        "best_of_set": 0.0,
        "diversity": 1.0,
        "coherence": 0.6666666666666666,
        "score": 0.383
      }
    },
    {
      "input": "Yes, that's a good rule!",
      "generated": [
        "i like sharing toys !",
        "what's the next rule ?",
        "why do we share toys ?",
        "we take turns .",
        "yes , that's right !",
        "no drugs for me ."
      ],
      "gold": [
        "Taking turns makes everyone happy!",
        "Taking turns is very fair!",
        "Let's practice taking turns!",
        "You go first!",
        "What is another rule?",
        "No fighting over toys!"
      ],
      "grade": {
        "per_branch_alignment": 0.023809523809523808,
        "best_of_set": 0.0,
        "diversity": 1.0,
        "coherence": 0.6666666666666666,
        "score": 0.392
      }
    },
    {
      "input": "Aw, that's okay. I like you a lot. What's making you feel that nobody likes you?",
      "generated": [
        "i love my family .",
        "i want to play with my friend .",
        "what is your name ?",
        "i want to tell the teacher .",
        "i love my family .",
        "i don't like playing with my friends ."
      ],
      "gold": [
        "I have no friends.",
        "The kids ignore me.",
        "I don't know why nobody likes me.",
        "I need more friends.",
        "Thanks. I like you too.",
        "Kids at school are mean to me."
      ],
      "grade": {
        "per_branch_alignment": 0.06316137566137565,
        "best_of_set": 0.0,
        "diversity": 0.8333333333333334,
        "coherence": 0.8333333333333334,
        "score": 0.397
      }
    },
    {
      "input": "What game do you play online with your friends?",
      "generated": [
        "roblox ! we make our own worlds .",
        "mario kart races online !",
        "what's your favorite game to play ?",
        "roblox ! we make our own worlds .",
        "fortnite ! we make our own worlds .",
        "i play fortnite battles online with friends !"
      ],
      "gold": [
        "Minecraft! We build worlds together!",
        "Roblox! It has so many adventures!",
        "Among Us! We hunt imposters!",
        "Fortnite! We build and dance!",
        "Brawl Stars! Quick brawls with pals!",
        "Clash Royale! Epic card battles!"
      ],
      "grade": {
        "per_branch_alignment": 0.06885521885521885,
        "best_of_set": 0.0,
        "diversity": 0.8333333333333334,
        "coherence": 0.8333333333333334,
        "score": 0.399
      }
    },
    {
      "input": "Sure! /a/ /a/ /a/ like in apple.",
      "generated": [
        "i want a banana .",
        "i want a banana .",
        "what is a banana ?",
        "i want to try apple .",
        "that's a good idea .",
        "no , i don't want a banana ."
      ],
      "gold": [
        "Thank you!",
        "Say it slower please.",
        "What letter is that?",
        "Say a word with that sound.",
        "What's the next sound?",
        "I got it now!"
      ],
      "grade": {
        "per_branch_alignment": 0.07407407407407407,
        "best_of_set": 0.0,
        "diversity": 0.8333333333333334,
        "coherence": 0.8333333333333334,
        "score": 0.401
      }
    }
  ],
  "strong_examples": [
    {
      "input": "Where do we read books?",
      "generated": [
        "in our class .",
        "in the classroom .",
        "in school class .",
        "in our class .",
        "in class .",
        "in school class ."
      ],
      "gold": [
        "In class.",
        "In the classroom.",
        "In reading class.",
        "In our class.",
        "During class time.",
        "Here in class."
      ],
      "grade": {
        "per_branch_alignment": 0.6527777777777778,
        "best_of_set": 1.0,
        "diversity": 0.6666666666666666,
        "coherence": 1.0,
        "score": 0.795
      }
    },
    {
      "input": "What games do you play at picnics?",
      "generated": [
        "we play tag !",
        "i want to play tag !",
        "we play catch .",
        "we play catch !",
        "yes , i play frisbee !",
        "i play tag ."
      ],
      "gold": [
        "We play frisbee.",
        "We play tag.",
        "We play catch.",
        "We do sack races.",
        "We play badminton.",
        "We fly kites."
      ],
      "grade": {
        "per_branch_alignment": 0.3611111111111111,
        "best_of_set": 0.5,
        "diversity": 1.0,
        "coherence": 1.0,
        "score": 0.676
      }
    },
    {
      "input": "What do we do in class?",
      "generated": [
        "we sing songs .",
        "we read books .",
        "we ask questions .",
        "we count numbers .",
        "we do math .",
        "we draw pictures ."
      ],
      "gold": [
        "We read books.",
        "We do math.",
        "Teacher teaches us.",
        "We write words.",
        "We do group work.",
        "We draw pictures."
      ],
      "grade": {
        "per_branch_alignment": 0.3333333333333333,
        "best_of_set": 0.5,
        "diversity": 1.0,
        "coherence": 1.0,
        "score": 0.667
      }
    },
    {
      "input": "What do we do in class?",
      "generated": [
        "we sing songs .",
        "we read books .",
        "we ask questions .",
        "we count numbers .",
        "we do math .",
        "we draw pictures ."
      ],
      "gold": [
        "We read books.",
        "We write words.",
        "We listen to teacher.",
        "We count numbers.",
        "We draw pictures.",
        "We learn math."
      ],
      "grade": {
        "per_branch_alignment": 0.3277777777777778,
        "best_of_set": 0.5,
        "diversity": 1.0,
        "coherence": 1.0,
        "score": 0.665
      }
    },
    {
      "input": "Tell me about disney facts?",
      "generated": [
        "i love mickey mouse !",
        "tell me more disney facts !",
        "what's your favorite disney movie ?",
        "i have a fact to know !",
        "that's a cool fact !",
        "i didn't know that !"
      ],
      "gold": [
        "That's a cool fact!",
        "Tell me more Disney facts!",
        "I love Mickey Mouse!",
        "That's amazing! I didn't know that.",
        "I know another Disney fact!",
        "What's your favorite Disney fact?"
      ],
      "grade": {
        "per_branch_alignment": 0.22083333333333333,
        "best_of_set": 0.8333333333333334,
        "diversity": 1.0,
        "coherence": 0.8333333333333334,
        "score": 0.661
      }
    }
  ]
}