File size: 3,597 Bytes
b50e043
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
{
  "status": "PASS",
  "runtime": "0.2.0",
  "vision_version": "unified-v2",
  "visual_tensors": 297,
  "vision_probability_max_abs_vs_split_release": 0.0,
  "text_reference_requests": 16,
  "text_wrapper_questions": 18,
  "text_probability_max_abs": 0.0,
  "wrapper_same_single_question_probability_max_abs": 0.0,
  "split_question_vs_original_batched_probability_difference": 0.0041863322257995605,
  "comparison_note": "Wrapper compared with text using identical single-question requests; original multi-question batches checked separately without splitting. BF16 split-vs-batch difference recorded, not treated as vision drift.",
  "image_smoke_correct": 5,
  "image_smoke_total": 5,
  "rows": [
    {
      "task": "color",
      "truth": "red",
      "result": {
        "model": "NeoHorse-Jev-4B",
        "answers": {
          "color": {
            "type": "choice",
            "choice": "red",
            "probabilities": {
              "red": 0.9999173879623413,
              "blue": 4.41447009507101e-05,
              "green": 3.853620364679955e-05
            },
            "confidence": 0.999876081943512
          }
        },
        "input_tokens": 137,
        "image_tokens": 96
      },
      "correct": true
    },
    {
      "task": "color",
      "truth": "blue",
      "result": {
        "model": "NeoHorse-Jev-4B",
        "answers": {
          "color": {
            "type": "choice",
            "choice": "blue",
            "probabilities": {
              "red": 2.3972206690814346e-05,
              "blue": 0.9998923540115356,
              "green": 8.369787246920168e-05
            },
            "confidence": 0.9998385310173035
          }
        },
        "input_tokens": 137,
        "image_tokens": 96
      },
      "correct": true
    },
    {
      "task": "color",
      "truth": "green",
      "result": {
        "model": "NeoHorse-Jev-4B",
        "answers": {
          "color": {
            "type": "choice",
            "choice": "green",
            "probabilities": {
              "red": 1.2651908946281765e-05,
              "blue": 2.656863580341451e-05,
              "green": 0.9999607801437378
            },
            "confidence": 0.9999411702156067
          }
        },
        "input_tokens": 137,
        "image_tokens": 96
      },
      "correct": true
    },
    {
      "task": "position",
      "truth": "left",
      "result": {
        "model": "NeoHorse-Jev-4B",
        "answers": {
          "side": {
            "type": "choice",
            "choice": "left",
            "probabilities": {
              "left": 0.9987630844116211,
              "right": 0.0012369066243991256
            },
            "confidence": 0.9975261688232422
          }
        },
        "input_tokens": 140,
        "image_tokens": 96
      },
      "correct": true
    },
    {
      "task": "position",
      "truth": "right",
      "result": {
        "model": "NeoHorse-Jev-4B",
        "answers": {
          "side": {
            "type": "choice",
            "choice": "right",
            "probabilities": {
              "left": 0.00043360565905459225,
              "right": 0.999566376209259
            },
            "confidence": 0.9991327524185181
          }
        },
        "input_tokens": 140,
        "image_tokens": 96
      },
      "correct": true
    }
  ],
  "choice_noul_score_executed": true,
  "stale_image_state_check": true,
  "visual_finetuning": false,
  "peak_cuda_allocated_bytes": 9825804800,
  "scope": "Local-image connectivity and text parity, not multimodal benchmark validation"
}