fix token count bug

chinesebleu.py

@@ -140,7 +140,7 @@ class ChineseBLEU(evaluate.Metric):
         ref_tokens = [self._tokenize_chinese(r, tokenizer) for r in references]
 
         # For total number of tokens < 4, fallback to character-level tokenizations
-        if len(pred_tokens) < 4 or len(ref_tokens) < 4:
+        if len(pred_tokens[0]) < 4 or len(ref_tokens[0]) < 4:
             tokenizer = 'char'
             pred_tokens = [self._tokenize_chinese(p, tokenizer) for p in predictions]
             ref_tokens = [self._tokenize_chinese(r, tokenizer) for r in references]
@@ -202,8 +202,8 @@ class ChineseBLEU(evaluate.Metric):
             "counts": counts, 
             "totals": totals, 
             "precisions": precisions, 
-            "ref_len": len(ref_tokens),
-            "sys_len": len(pred_tokens),
+            "ref_len": len(ref_tokens[0]),
+            "sys_len": len(pred_tokens[0]),
             "bp": bp,
             "tokenizer": tokenizer
         }