Hugging Face's logo

PursuitOfDataScience
/
Muse-Glimmer-30B-ToolCall-LoRA

Text Generation
PEFT
Safetensors
lora
tool-calling
function-calling
agentic
muse-glimmer
Model card Files
xet
 Community
Muse-Glimmer-30B-ToolCall-LoRA / benchmarks /bench_lora_gsm8k.json
PursuitOfDataScience's picture
PursuitOfDataScience
Agentic tool-calling LoRA: +4.0pp BFCL AST accuracy out-of-distribution, +14pp GSM8K (p<0.0001)
630c0ff verified
Raw
History Blame
24.4 kB
{
 "tag": "lora",
 "suite": "gsm8k",
 "adapter": "latest",
 "seed": 0,
 "max_new_tokens": 1024,
 "max_new_tokens_gsm8k": 640,
 "n": {
 "gsm8k": 200
 },
 "complete": true,
 "items": [
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": false,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": false,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": false,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": false,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": false,
 "intruded": false
 },
 {
 "correct": false,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": false,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": false,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": false,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": false,
 "intruded": false
 },
 {
 "correct": false,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": false,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": false,
 "intruded": false
 },
 {
 "correct": false,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": false,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": false,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": false,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": false,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": false,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": false,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": false,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": false,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": false,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": false,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 }
 ],
 "minutes": 112.32950961987179,
 "gsm8k_accuracy": 0.88,
 "tool_syntax_intrusion": 0.0,
 "gsm8k_per_item": [
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": false,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": false,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": false,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": false,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": false,
 "intruded": false
 },
 {
 "correct": false,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": false,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": false,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": false,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": false,
 "intruded": false
 },
 {
 "correct": false,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": false,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": false,
 "intruded": false
 },
 {
 "correct": false,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": false,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": false,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": false,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": false,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": false,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": false,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": false,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": false,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": false,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": false,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 },
 {
 "correct": true,
 "intruded": false
 }
 ]
}