| evaluations: |
| arc_challenge_poly_pt_acc: 0.37948717948717947 |
| arc_challenge_poly_pt_acc_norm: 0.4282051282051282 |
| arc_challenge_poly_pt_acc_norm_stderr: 0.014472341586181985 |
| arc_challenge_poly_pt_acc_stderr: 0.014192754090886793 |
| arc_challenge_poly_pt_alias: arc_challenge_poly_pt |
| assin2_rte_acc,all: 0.8705065359477124 |
| assin2_rte_acc_stderr,all: 0.004798540986291823 |
| assin2_rte_alias: assin2_rte |
| assin2_rte_f1_macro,all: 0.8691746625542952 |
| assin2_rte_f1_macro_stderr,all: 0.004847278802663799 |
| assin2_sts_alias: assin2_sts |
| assin2_sts_mse,all: 2.2781209150326798 |
| assin2_sts_mse_stderr,all: N/A |
| assin2_sts_pearson,all: 0.47716809267162485 |
| assin2_sts_pearson_stderr,all: 0.014407170423859306 |
| assin_entailment_acc: 0.7325 |
| assin_entailment_acc_stderr: 0.006999870502142285 |
| assin_entailment_alias: assin_entailment |
| assin_paraphrase_acc: 0.73625 |
| assin_paraphrase_acc_stderr: 0.006968401827607802 |
| assin_paraphrase_alias: assin_paraphrase |
| belebele_por_Latn_acc: 0.6766666666666666 |
| belebele_por_Latn_acc_norm: 0.6766666666666666 |
| belebele_por_Latn_acc_norm_stderr: 0.015600294087844632 |
| belebele_por_Latn_acc_stderr: 0.015600294087844632 |
| belebele_por_Latn_alias: belebele_por_Latn |
| bluex_acc,all: 0.39221140472878996 |
| bluex_acc,exam_id__UNICAMP_2018: 0.48148148148148145 |
| bluex_acc,exam_id__UNICAMP_2019: 0.42 |
| bluex_acc,exam_id__UNICAMP_2020: 0.41818181818181815 |
| bluex_acc,exam_id__UNICAMP_2021_1: 0.4782608695652174 |
| bluex_acc,exam_id__UNICAMP_2021_2: 0.29411764705882354 |
| bluex_acc,exam_id__UNICAMP_2022: 0.5384615384615384 |
| bluex_acc,exam_id__UNICAMP_2023: 0.4883720930232558 |
| bluex_acc,exam_id__UNICAMP_2024: 0.4444444444444444 |
| bluex_acc,exam_id__USP_2018: 0.24074074074074073 |
| bluex_acc,exam_id__USP_2019: 0.35 |
| bluex_acc,exam_id__USP_2020: 0.39285714285714285 |
| bluex_acc,exam_id__USP_2021: 0.3076923076923077 |
| bluex_acc,exam_id__USP_2022: 0.24489795918367346 |
| bluex_acc,exam_id__USP_2023: 0.38636363636363635 |
| bluex_acc,exam_id__USP_2024: 0.4634146341463415 |
| bluex_acc_stderr,all: 0.010527552369043944 |
| bluex_acc_stderr,exam_id__UNICAMP_2018: 0.03932520147192548 |
| bluex_acc_stderr,exam_id__UNICAMP_2019: 0.040409592319981036 |
| bluex_acc_stderr,exam_id__UNICAMP_2020: 0.038500996629012005 |
| bluex_acc_stderr,exam_id__UNICAMP_2021_1: 0.04265317780472471 |
| bluex_acc_stderr,exam_id__UNICAMP_2021_2: 0.0369309745845929 |
| bluex_acc_stderr,exam_id__UNICAMP_2022: 0.04616267465694302 |
| bluex_acc_stderr,exam_id__UNICAMP_2023: 0.04391771882582978 |
| bluex_acc_stderr,exam_id__UNICAMP_2024: 0.04266417007528182 |
| bluex_acc_stderr,exam_id__USP_2018: 0.03361325629183327 |
| bluex_acc_stderr,exam_id__USP_2019: 0.04344929186946837 |
| bluex_acc_stderr,exam_id__USP_2020: 0.037457169801895486 |
| bluex_acc_stderr,exam_id__USP_2021: 0.03696695228727266 |
| bluex_acc_stderr,exam_id__USP_2022: 0.03545185428690165 |
| bluex_acc_stderr,exam_id__USP_2023: 0.042474562735645566 |
| bluex_acc_stderr,exam_id__USP_2024: 0.04499354148452882 |
| bluex_alias: bluex |
| calame_pt_acc: 0.11127167630057803 |
| calame_pt_acc_stderr: 0.00690347530269077 |
| calame_pt_alias: calame_pt |
| calame_pt_perplexity: 5965.407764074068 |
| calame_pt_perplexity_stderr: 683.2371253201818 |
| enem_challenge_acc,all: 0.3988803358992302 |
| enem_challenge_acc,exam_id__2009: 0.46956521739130436 |
| enem_challenge_acc,exam_id__2010: 0.5128205128205128 |
| enem_challenge_acc,exam_id__2011: 0.46153846153846156 |
| enem_challenge_acc,exam_id__2012: 0.4224137931034483 |
| enem_challenge_acc,exam_id__2013: 0.3148148148148148 |
| enem_challenge_acc,exam_id__2014: 0.41284403669724773 |
| enem_challenge_acc,exam_id__2015: 0.40336134453781514 |
| enem_challenge_acc,exam_id__2016: 0.36363636363636365 |
| enem_challenge_acc,exam_id__2016_2: 0.4065040650406504 |
| enem_challenge_acc,exam_id__2017: 0.39655172413793105 |
| enem_challenge_acc,exam_id__2022: 0.3157894736842105 |
| enem_challenge_acc,exam_id__2023: 0.32592592592592595 |
| enem_challenge_acc_stderr,all: 0.00749197527112301 |
| enem_challenge_acc_stderr,exam_id__2009: 0.02676517541031027 |
| enem_challenge_acc_stderr,exam_id__2010: 0.02676561702634192 |
| enem_challenge_acc_stderr,exam_id__2011: 0.026575632846671127 |
| enem_challenge_acc_stderr,exam_id__2012: 0.02652807977680181 |
| enem_challenge_acc_stderr,exam_id__2013: 0.02570106484445073 |
| enem_challenge_acc_stderr,exam_id__2014: 0.02722456002129014 |
| enem_challenge_acc_stderr,exam_id__2015: 0.025882945505526844 |
| enem_challenge_acc_stderr,exam_id__2016: 0.025343746567122242 |
| enem_challenge_acc_stderr,exam_id__2016_2: 0.025491304109387235 |
| enem_challenge_acc_stderr,exam_id__2017: 0.0261642629559971 |
| enem_challenge_acc_stderr,exam_id__2022: 0.02331831926729694 |
| enem_challenge_acc_stderr,exam_id__2023: 0.023372974631758865 |
| enem_challenge_alias: enem |
| faquad_nli_acc,all: 0.7892307692307692 |
| faquad_nli_acc_stderr,all: 0.011279066985135663 |
| faquad_nli_alias: faquad_nli |
| faquad_nli_f1_macro,all: 0.49256657036543183 |
| faquad_nli_f1_macro_stderr,all: 0.01318624622535848 |
| global_piqa_completions_por_latn_braz_acc: 0.75 |
| global_piqa_completions_por_latn_braz_acc_bytes: 0.74 |
| global_piqa_completions_por_latn_braz_acc_bytes_stderr: 0.0440844002276808 |
| global_piqa_completions_por_latn_braz_acc_norm: 0.74 |
| global_piqa_completions_por_latn_braz_acc_norm_stderr: 0.0440844002276808 |
| global_piqa_completions_por_latn_braz_acc_stderr: 0.04351941398892446 |
| global_piqa_completions_por_latn_braz_alias: global_piqa_completions_por_latn_braz |
| gsm8k_pt_alias: gsm8k_pt |
| gsm8k_pt_exact_match,flexible-extract: 0.228310502283105 |
| gsm8k_pt_exact_match,strict-match: 0.0 |
| gsm8k_pt_exact_match_stderr,flexible-extract: 0.011583822031121497 |
| gsm8k_pt_exact_match_stderr,strict-match: 0.0 |
| hatebr_offensive_acc,all: 0.7071428571428572 |
| hatebr_offensive_acc_stderr,all: 0.008611736471385361 |
| hatebr_offensive_alias: hatebr_offensive_binary |
| hatebr_offensive_f1_macro,all: 0.7019751844399732 |
| hatebr_offensive_f1_macro_stderr,all: 0.008866946580220815 |
| hellaswag_poly_pt_acc: 0.4334164048109221 |
| hellaswag_poly_pt_acc_norm: 0.5494636471990465 |
| hellaswag_poly_pt_acc_norm_stderr: 0.005179413791359488 |
| hellaswag_poly_pt_acc_stderr: 0.005158588405358706 |
| hellaswag_poly_pt_alias: hellaswag_poly_pt |
| humaneval_instruct_alias: humaneval_instruct |
| humaneval_instruct_pass@1,create_test: 0.0 |
| humaneval_instruct_pass@1_stderr,create_test: 0.0 |
| ifeval_pt_alias: ifeval_pt |
| ifeval_pt_inst_level_loose_acc: 0.4511627906976744 |
| ifeval_pt_inst_level_loose_acc_stderr: N/A |
| ifeval_pt_inst_level_strict_acc: 0.3395348837209302 |
| ifeval_pt_inst_level_strict_acc_stderr: N/A |
| ifeval_pt_prompt_level_loose_acc: 0.33666666666666667 |
| ifeval_pt_prompt_level_loose_acc_stderr: 0.02732941756218688 |
| ifeval_pt_prompt_level_strict_acc: 0.23333333333333334 |
| ifeval_pt_prompt_level_strict_acc_stderr: 0.024459979523511425 |
| lambada_poly_pt_acc: 0.267028915195032 |
| lambada_poly_pt_acc_stderr: 0.00616360274242743 |
| lambada_poly_pt_alias: lambada_poly_pt |
| lambada_poly_pt_perplexity: 214.12248648482947 |
| lambada_poly_pt_perplexity_stderr: 13.012850955803207 |
| mmlu_poly_pt_acc: 0.43297808465926146 |
| mmlu_poly_pt_acc_stderr: 0.004292713120965234 |
| mmlu_poly_pt_alias: mmlu_poly_pt |
| oab_exams_acc,all: 0.3425968109339408 |
| oab_exams_acc,exam_id__2010-01: 0.3176470588235294 |
| oab_exams_acc,exam_id__2010-02: 0.33 |
| oab_exams_acc,exam_id__2011-03: 0.3333333333333333 |
| oab_exams_acc,exam_id__2011-04: 0.3625 |
| oab_exams_acc,exam_id__2011-05: 0.325 |
| oab_exams_acc,exam_id__2012-06: 0.35 |
| oab_exams_acc,exam_id__2012-06a: 0.325 |
| oab_exams_acc,exam_id__2012-07: 0.3375 |
| oab_exams_acc,exam_id__2012-08: 0.325 |
| oab_exams_acc,exam_id__2012-09: 0.24675324675324675 |
| oab_exams_acc,exam_id__2013-10: 0.325 |
| oab_exams_acc,exam_id__2013-11: 0.2875 |
| oab_exams_acc,exam_id__2013-12: 0.3375 |
| oab_exams_acc,exam_id__2014-13: 0.3875 |
| oab_exams_acc,exam_id__2014-14: 0.325 |
| oab_exams_acc,exam_id__2014-15: 0.41025641025641024 |
| oab_exams_acc,exam_id__2015-16: 0.3125 |
| oab_exams_acc,exam_id__2015-17: 0.3974358974358974 |
| oab_exams_acc,exam_id__2015-18: 0.375 |
| oab_exams_acc,exam_id__2016-19: 0.41025641025641024 |
| oab_exams_acc,exam_id__2016-20: 0.4375 |
| oab_exams_acc,exam_id__2016-20a: 0.325 |
| oab_exams_acc,exam_id__2016-21: 0.25 |
| oab_exams_acc,exam_id__2017-22: 0.375 |
| oab_exams_acc,exam_id__2017-23: 0.3125 |
| oab_exams_acc,exam_id__2017-24: 0.4 |
| oab_exams_acc,exam_id__2018-25: 0.3375 |
| oab_exams_acc_stderr,all: 0.005854656489923565 |
| oab_exams_acc_stderr,exam_id__2010-01: 0.02913844109530932 |
| oab_exams_acc_stderr,exam_id__2010-02: 0.02720413223389449 |
| oab_exams_acc_stderr,exam_id__2011-03: 0.02739274623256926 |
| oab_exams_acc_stderr,exam_id__2011-04: 0.03103610079124539 |
| oab_exams_acc_stderr,exam_id__2011-05: 0.030180181541041882 |
| oab_exams_acc_stderr,exam_id__2012-06: 0.030663426761582978 |
| oab_exams_acc_stderr,exam_id__2012-06a: 0.03019542051107743 |
| oab_exams_acc_stderr,exam_id__2012-07: 0.030587368127940558 |
| oab_exams_acc_stderr,exam_id__2012-08: 0.030150394921254077 |
| oab_exams_acc_stderr,exam_id__2012-09: 0.028367169852259254 |
| oab_exams_acc_stderr,exam_id__2013-10: 0.030312606625205758 |
| oab_exams_acc_stderr,exam_id__2013-11: 0.0291233179987808 |
| oab_exams_acc_stderr,exam_id__2013-12: 0.030508999749351583 |
| oab_exams_acc_stderr,exam_id__2014-13: 0.03131790740981766 |
| oab_exams_acc_stderr,exam_id__2014-14: 0.030290399093110758 |
| oab_exams_acc_stderr,exam_id__2014-15: 0.032059169494021815 |
| oab_exams_acc_stderr,exam_id__2015-16: 0.029833251327948757 |
| oab_exams_acc_stderr,exam_id__2015-17: 0.03201117621919014 |
| oab_exams_acc_stderr,exam_id__2015-18: 0.0311783808239491 |
| oab_exams_acc_stderr,exam_id__2016-19: 0.032162738371024235 |
| oab_exams_acc_stderr,exam_id__2016-20: 0.031934266796334064 |
| oab_exams_acc_stderr,exam_id__2016-20a: 0.030215012978448332 |
| oab_exams_acc_stderr,exam_id__2016-21: 0.027979645441821365 |
| oab_exams_acc_stderr,exam_id__2017-22: 0.031096010380049006 |
| oab_exams_acc_stderr,exam_id__2017-23: 0.02995046656597778 |
| oab_exams_acc_stderr,exam_id__2017-24: 0.03158083933896483 |
| oab_exams_acc_stderr,exam_id__2018-25: 0.030481185333792096 |
| oab_exams_alias: oab_exams |
| portuguese_hate_speech_acc,all: 0.7097532314923619 |
| portuguese_hate_speech_acc_stderr,all: 0.01097429657035187 |
| portuguese_hate_speech_alias: portuguese_hate_speech_binary |
| portuguese_hate_speech_f1_macro,all: 0.6922452504317789 |
| portuguese_hate_speech_f1_macro_stderr,all: 0.011709050058396507 |
| tweetsentbr_acc,all: 0.6422885572139303 |
| tweetsentbr_acc_stderr,all: 0.007563312856378464 |
| tweetsentbr_alias: tweetsentbr |
| tweetsentbr_f1_macro,all: 0.6428335432993848 |
| tweetsentbr_f1_macro_stderr,all: 0.007483756787178434 |
| step: 3595 |
|
|
