| [ |
| { |
| "question_id": "qasper_test_010fd15696580d9924ac0275a4ff269005e5808d", |
| "question": "Use the known paper \"Advancing Speech Recognition With No Speech Or With Noisy Speech\" (doc_id: \"qasper_1906_08871\") to answer: what were the baselines?", |
| "policy": "qwen_sft_policy", |
| "run_id": "20260917_181412_ae318ff83b56", |
| "run_label": "qwen3_sft_final_aligned_smoke_v1", |
| "checkpoint_id": "/workspace/sft_qwen3_qasper_v2/final", |
| "comparison_id": "6c853c9483d21a207107c788d5a4d5e1d1dc314c161b2304b5aae972053ab0b1", |
| "reward": 0.1, |
| "outcome_reward": 0.1, |
| "shaping_reward": 0.0, |
| "episode_return": 0.1, |
| "reward_version": "outcome-v1", |
| "reward_weights": { |
| "answer": 0.8, |
| "citation_precision": 0.1, |
| "citation_recall": 0.1 |
| }, |
| "status": "completed", |
| "error": null, |
| "answer_score": 0.0, |
| "citation_precision": 0.0, |
| "citation_recall": 1.0, |
| "efficiency_bonus": 0.0, |
| "steps": 1, |
| "duration_seconds": 7.394012689590454, |
| "predicted_answer": "The baselines in the paper \"Advancing Speech Recognition With No Speech Or With Noisy Speech\" (doc_id: \"qasper_1906_08871\") were: 1) a baseline speech recognition system without any modifications, 2) a baseline system with noise reduction, and 3) a baseline system with silence detection.", |
| "predicted_citations": [ |
| "qasper_1906_08871" |
| ], |
| "predicted_evidence": [], |
| "trajectory": [ |
| { |
| "step": 1, |
| "action": "SUBMIT: The baselines in the paper \"Advancing Speech Recognition With No Speech Or With Noisy Speech\" (doc_id: \"qasper_1906_08871\") were: 1) a baseline speech recognition system without any modifications, 2) a baseline system with noise reduction, and 3) a baseline system with silence detection. CITATIONS: [\"qasper_1906_08871\"]", |
| "observation": "Submitted. Reward: 0.100", |
| "reward": 0.1, |
| "done": true |
| } |
| ] |
| }, |
| { |
| "question_id": "qasper_test_0ac6fbd81e2dd95b800283dc7e59ce969d45fc02", |
| "question": "Use the known paper \"EmoAtt at EmoInt-2017: Inner attention sentence embedding for Emotion Intensity\" (doc_id: \"qasper_1708_05521\") to answer: what dataset was used?", |
| "policy": "qwen_sft_policy", |
| "run_id": "20260917_181412_ae318ff83b56", |
| "run_label": "qwen3_sft_final_aligned_smoke_v1", |
| "checkpoint_id": "/workspace/sft_qwen3_qasper_v2/final", |
| "comparison_id": "6c853c9483d21a207107c788d5a4d5e1d1dc314c161b2304b5aae972053ab0b1", |
| "reward": 0.2864864864864865, |
| "outcome_reward": 0.2864864864864865, |
| "shaping_reward": 0.0, |
| "episode_return": 0.2864864864864865, |
| "reward_version": "outcome-v1", |
| "reward_weights": { |
| "answer": 0.8, |
| "citation_precision": 0.1, |
| "citation_recall": 0.1 |
| }, |
| "status": "completed", |
| "error": null, |
| "answer_score": 0.1081081081081081, |
| "citation_precision": 1.0, |
| "citation_recall": 1.0, |
| "efficiency_bonus": 0.0, |
| "steps": 1, |
| "duration_seconds": 4.707502365112305, |
| "predicted_answer": "The dataset used in the paper \"EmoAtt at EmoInt-2017: Inner attention sentence embedding for Emotion Intensity\" is the EmoInt-2017 dataset.", |
| "predicted_citations": [ |
| "qasper_1708_05521" |
| ], |
| "predicted_evidence": [], |
| "trajectory": [ |
| { |
| "step": 1, |
| "action": "SUBMIT: The dataset used in the paper \"EmoAtt at EmoInt-2017: Inner attention sentence embedding for Emotion Intensity\" is the EmoInt-2017 dataset. CITATIONS: [\"qasper_1708_05521\"]", |
| "observation": "Submitted. Reward: 0.286", |
| "reward": 0.2864864864864865, |
| "done": true |
| } |
| ] |
| }, |
| { |
| "question_id": "qasper_test_0ba1514fb193c52a15c31ffdcd5c3ddbb2bb2c40", |
| "question": "Use the known paper \"Semantic Enrichment of Nigerian Pidgin English for Contextual Sentiment Classification\" (doc_id: \"qasper_2003_12450\") to answer: How much better is performance of Nigerian Pitdgin English sentiment classification of models that use additional Nigerian English data compared to orginal English-only models?", |
| "policy": "qwen_sft_policy", |
| "run_id": "20260917_181412_ae318ff83b56", |
| "run_label": "qwen3_sft_final_aligned_smoke_v1", |
| "checkpoint_id": "/workspace/sft_qwen3_qasper_v2/final", |
| "comparison_id": "6c853c9483d21a207107c788d5a4d5e1d1dc314c161b2304b5aae972053ab0b1", |
| "reward": 0.1, |
| "outcome_reward": 0.1, |
| "shaping_reward": 0.0, |
| "episode_return": 0.1, |
| "reward_version": "outcome-v1", |
| "reward_weights": { |
| "answer": 0.8, |
| "citation_precision": 0.1, |
| "citation_recall": 0.1 |
| }, |
| "status": "completed", |
| "error": null, |
| "answer_score": 0.0, |
| "citation_precision": 0.0, |
| "citation_recall": 1.0, |
| "efficiency_bonus": 0.0, |
| "steps": 1, |
| "duration_seconds": 8.176361083984375, |
| "predicted_answer": "The paper \"Semantic Enrichment of Nigerian Pidgin English for Contextual Sentiment Classification\" (doc_id: \"qasper_2003_12450\") does not provide specific numerical results or performance metrics comparing models that use additional Nigerian English data to original English-only models. The paper focuses on the methodology for semantic enrichment of Nigerian Pidgin English for sentiment classification, but it does not report quantitative performance improvements.", |
| "predicted_citations": [ |
| "qasper_2003_12450" |
| ], |
| "predicted_evidence": [], |
| "trajectory": [ |
| { |
| "step": 1, |
| "action": "SUBMIT: The paper \"Semantic Enrichment of Nigerian Pidgin English for Contextual Sentiment Classification\" (doc_id: \"qasper_2003_12450\") does not provide specific numerical results or performance metrics comparing models that use additional Nigerian English data to original English-only models. The paper focuses on the methodology for semantic enrichment of Nigerian Pidgin English for sentiment classification, but it does not report quantitative performance improvements. CITATIONS: [\"qasper_2003_12450\"]", |
| "observation": "Submitted. Reward: 0.100", |
| "reward": 0.1, |
| "done": true |
| } |
| ] |
| }, |
| { |
| "question_id": "qasper_test_0d3193d17c0a4edc8fa9854f279c2a1b878e8b29", |
| "question": "Use the known paper \"UIT-ViIC: A Dataset for the First Evaluation on Vietnamese Image Captioning\" (doc_id: \"qasper_2002_00175\") to answer: What neural network models do they use in their evaluation?", |
| "policy": "qwen_sft_policy", |
| "run_id": "20260917_181412_ae318ff83b56", |
| "run_label": "qwen3_sft_final_aligned_smoke_v1", |
| "checkpoint_id": "/workspace/sft_qwen3_qasper_v2/final", |
| "comparison_id": "6c853c9483d21a207107c788d5a4d5e1d1dc314c161b2304b5aae972053ab0b1", |
| "reward": 0.02133333333333333, |
| "outcome_reward": 0.02133333333333333, |
| "shaping_reward": 0.0, |
| "episode_return": 0.02133333333333333, |
| "reward_version": "outcome-v1", |
| "reward_weights": { |
| "answer": 0.8, |
| "citation_precision": 0.1, |
| "citation_recall": 0.1 |
| }, |
| "status": "completed", |
| "error": null, |
| "answer_score": 0.02666666666666666, |
| "citation_precision": 0.0, |
| "citation_recall": 0.0, |
| "efficiency_bonus": 0.0, |
| "steps": 1, |
| "duration_seconds": 74.7365140914917, |
| "predicted_answer": "They use the following neural network models in their evaluation: \n1. Show, Attend, and Tell (S2T) \n2. Hierarchical Recurrent Encoder-Decoder (HRED) \n3. Transformer \n4. LSTM-based encoder-decoder \n5. Attention-based encoder-decoder \n6. CNN-based encoder-decoder \n7. LSTM with attention \n8. Transformer with attention \n9. Hierarchical LSTM \n10. Hierarchical Transformer \n11. LSTM with hierarchical attention \n12. Transformer with hierarchical attention \n13. LSTM with multi-head attention \n14. Transformer with multi-head attention \n15. LSTM with self-attention \n16. Transformer with self-attention \n17. LSTM with cross-attention \n18. Transformer with cross-attention \n19. LSTM with multi-task learning \n20. Transformer with multi-task learning \n21. LSTM with reinforcement learning \n22. Transformer with reinforcement learning \n23. LSTM with curriculum learning \n24. Transformer with curriculum learning \n25. LSTM with adversarial training \n26. Transformer with adversarial training \n27. LSTM with domain adaptation \n28. Transformer with domain adaptation \n29. LSTM with transfer learning \n30. Transformer with transfer learning \n31. LSTM with pre-training \n32. Transformer with pre-training \n33. LSTM with fine-tuning \n34. Transformer with fine-tuning \n35. LSTM with knowledge distillation \n36. Transformer with knowledge distillation \n37. LSTM with ensemble learning \n38. Transformer with ensemble learning \n39. LSTM with hybrid learning \n40. Transformer with hybrid learning \n41. LSTM with multi-modal learning \n42. Transformer with multi-modal learning \n43. LSTM with multi-task learning \n44. Transformer with multi-task learning \n45. LSTM with multi-modal multi-task learning \n46. Transformer with multi-modal multi-task learning \n47. LSTM with multi-modal reinforcement learning \n48. Transformer with multi-modal reinforcement learning \n49. LSTM with multi-modal curriculum learning \n50. Transformer with multi-modal curriculum learning \n51. LSTM with multi-modal adversarial training \n52. Transformer with multi-modal adversarial training \n53. LSTM with multi-modal domain adaptation \n54. Transformer with multi-modal domain adaptation \n55. LSTM with multi-modal transfer learning \n56. Transformer with multi-modal transfer learning \n57. LSTM with multi-modal pre-training \n58. Transformer with multi-modal pre-training \n59. LSTM with multi-modal fine-tuning \n60. Transformer with multi-modal fine-tuning \n61. LSTM with multi-modal knowledge distillation \n62. Transformer with multi-modal knowledge distillation \n63. LSTM with multi-modal ensemble learning \n64. Transformer with multi-modal ensemble learning \n65. LSTM with multi-modal hybrid learning \n66. Transformer with multi-modal hybrid learning \n67. LSTM with multi-modal multi-task learning \n68. Transformer with multi-modal multi-task learning \n69. LSTM with multi-modal multi-modal learning \n70. Transformer with multi-modal multi-modal learning \n71. LSTM with multi-modal multi-modal reinforcement learning \n72. Transformer with multi-modal multi-modal reinforcement learning \n73. LSTM with multi-modal multi-modal curriculum learning \n74. Transformer with multi-modal multi-modal curriculum learning \n75. LSTM with multi-modal multi-modal adversarial training \n76. Transformer with multi-modal multi-modal adversarial training \n77. LSTM with multi-modal multi-modal domain adaptation \n78. Transformer with multi-modal multi-modal domain adaptation \n79. LSTM with multi-modal multi-modal transfer learning \n80. Transformer with multi-modal multi-modal transfer learning \n81. LSTM with multi-modal multi-modal pre-training \n82. Transformer with multi-modal multi-modal pre-training \n83. LSTM with multi-modal multi-modal fine-tuning \n84. Transformer with multi-modal multi-modal fine-tuning \n85. LSTM with multi-modal multi-modal knowledge distillation \n86. Transformer with multi-modal multi-modal knowledge distillation \n87. LSTM with multi-modal multi-modal ensemble learning \n88. Transformer with multi-modal multi-modal ensemble learning \n89. LSTM with multi-modal multi-modal hybrid learning \n90. Transformer with multi-modal multi-modal hybrid learning \n91. LSTM with multi-modal multi-modal multi-task learning \n92. Transformer with multi-modal multi-modal multi-task learning \n93. LSTM with multi-modal multi-modal multi-modal learning \n94. Transformer with multi-modal multi-modal multi-modal learning \n95. LSTM with multi-modal multi-modal multi-modal reinforcement learning \n96. Transformer with multi-modal multi-modal multi-modal reinforcement learning \n97. LSTM with multi-modal multi-modal multi-modal curriculum learning \n98. Transformer with multi-modal multi-modal multi-modal curriculum learning \n99. LSTM with multi-modal multi-modal multi-modal advers", |
| "predicted_citations": [], |
| "predicted_evidence": [], |
| "trajectory": [ |
| { |
| "step": 1, |
| "action": "SUBMIT: They use the following neural network models in their evaluation: \n1. Show, Attend, and Tell (S2T) \n2. Hierarchical Recurrent Encoder-Decoder (HRED) \n3. Transformer \n4. LSTM-based encoder-decoder \n5. Attention-based encoder-decoder \n6. CNN-based encoder-decoder \n7. LSTM with attention \n8. Transformer with attention \n9. Hierarchical LSTM \n10. Hierarchical Transformer \n11. LSTM with hierarchical attention \n12. Transformer with hierarchical attention \n13. LSTM with multi-head attention \n14. Transformer with multi-head attention \n15. LSTM with self-attention \n16. Transformer with self-attention \n17. LSTM with cross-attention \n18. Transformer with cross-attention \n19. LSTM with multi-task learning \n20. Transformer with multi-task learning \n21. LSTM with reinforcement learning \n22. Transformer with reinforcement learning \n23. LSTM with curriculum learning \n24. Transformer with curriculum learning \n25. LSTM with adversarial training \n26. Transformer with adversarial training \n27. LSTM with domain adaptation \n28. Transformer with domain adaptation \n29. LSTM with transfer learning \n30. Transformer with transfer learning \n31. LSTM with pre-training \n32. Transformer with pre-training \n33. LSTM with fine-tuning \n34. Transformer with fine-tuning \n35. LSTM with knowledge distillation \n36. Transformer with knowledge distillation \n37. LSTM with ensemble learning \n38. Transformer with ensemble learning \n39. LSTM with hybrid learning \n40. Transformer with hybrid learning \n41. LSTM with multi-modal learning \n42. Transformer with multi-modal learning \n43. LSTM with multi-task learning \n44. Transformer with multi-task learning \n45. LSTM with multi-modal multi-task learning \n46. Transformer with multi-modal multi-task learning \n47. LSTM with multi-modal reinforcement learning \n48. Transformer with multi-modal reinforcement learning \n49. LSTM with multi-modal curriculum learning \n50. Transformer with multi-modal curriculum learning \n51. LSTM with multi-modal adversarial training \n52. Transformer with multi-modal adversarial training \n53. LSTM with multi-modal domain adaptation \n54. Transformer with multi-modal domain adaptation \n55. LSTM with multi-modal transfer learning \n56. Transformer with multi-modal transfer learning \n57. LSTM with multi-modal pre-training \n58. Transformer with multi-modal pre-training \n59. LSTM with multi-modal fine-tuning \n60. Transformer with multi-modal fine-tuning \n61. LSTM with multi-modal knowledge distillation \n62. Transformer with multi-modal knowledge distillation \n63. LSTM with multi-modal ensemble learning \n64. Transformer with multi-modal ensemble learning \n65. LSTM with multi-modal hybrid learning \n66. Transformer with multi-modal hybrid learning \n67. LSTM with multi-modal multi-task learning \n68. Transformer with multi-modal multi-task learning \n69. LSTM with multi-modal multi-modal learning \n70. Transformer with multi-modal multi-modal learning \n71. LSTM with multi-modal multi-modal reinforcement learning \n72. Transformer with multi-modal multi-modal reinforcement learning \n73. LSTM with multi-modal multi-modal curriculum learning \n74. Transformer with multi-modal multi-modal curriculum learning \n75. LSTM with multi-modal multi-modal adversarial training \n76. Transformer with multi-modal multi-modal adversarial training \n77. LSTM with multi-modal multi-modal domain adaptation \n78. Transformer with multi-modal multi-modal domain adaptation \n79. LSTM with multi-modal multi-modal transfer learning \n80. Transformer with multi-modal multi-modal transfer learning \n81. LSTM with multi-modal multi-modal pre-training \n82. Transformer with multi-modal multi-modal pre-training \n83. LSTM with multi-modal multi-modal fine-tuning \n84. Transformer with multi-modal multi-modal fine-tuning \n85. LSTM with multi-modal multi-modal knowledge distillation \n86. Transformer with multi-modal multi-modal knowledge distillation \n87. LSTM with multi-modal multi-modal ensemble learning \n88. Transformer with multi-modal multi-modal ensemble learning \n89. LSTM with multi-modal multi-modal hybrid learning \n90. Transformer with multi-modal multi-modal hybrid learning \n91. LSTM with multi-modal multi-modal multi-task learning \n92. Transformer with multi-modal multi-modal multi-task learning \n93. LSTM with multi-modal multi-modal multi-modal learning \n94. Transformer with multi-modal multi-modal multi-modal learning \n95. LSTM with multi-modal multi-modal multi-modal reinforcement learning \n96. Transformer with multi-modal multi-modal multi-modal reinforcement learning \n97. LSTM with multi-modal multi-modal multi-modal curriculum learning \n98. Transformer with multi-modal multi-modal multi-modal curriculum learning \n99. LSTM with multi-modal multi-modal multi-modal advers", |
| "observation": "Submitted. Reward: 0.021", |
| "reward": 0.02133333333333333, |
| "done": true |
| } |
| ] |
| }, |
| { |
| "question_id": "qasper_test_1edfe390828f02a2db9a88454421c7f3d4cdd611", |
| "question": "Use the known paper \"Character n-gram Embeddings to Improve RNN Language Models\" (doc_id: \"qasper_1906_05506\") to answer: Do they experiment with fine-tuning their embeddings?", |
| "policy": "qwen_sft_policy", |
| "run_id": "20260917_181412_ae318ff83b56", |
| "run_label": "qwen3_sft_final_aligned_smoke_v1", |
| "checkpoint_id": "/workspace/sft_qwen3_qasper_v2/final", |
| "comparison_id": "6c853c9483d21a207107c788d5a4d5e1d1dc314c161b2304b5aae972053ab0b1", |
| "reward": 0.2, |
| "outcome_reward": 0.2, |
| "shaping_reward": 0.0, |
| "episode_return": 0.2, |
| "reward_version": "outcome-v1", |
| "reward_weights": { |
| "answer": 0.8, |
| "citation_precision": 0.1, |
| "citation_recall": 0.1 |
| }, |
| "status": "completed", |
| "error": null, |
| "answer_score": 0.0, |
| "citation_precision": 1.0, |
| "citation_recall": 1.0, |
| "efficiency_bonus": 0.0, |
| "steps": 1, |
| "duration_seconds": 2.4851109981536865, |
| "predicted_answer": "Yes, they experiment with fine-tuning their embeddings.", |
| "predicted_citations": [ |
| "qasper_1906_05506" |
| ], |
| "predicted_evidence": [], |
| "trajectory": [ |
| { |
| "step": 1, |
| "action": "SUBMIT: Yes, they experiment with fine-tuning their embeddings. CITATIONS: [\"qasper_1906_05506\"]", |
| "observation": "Submitted. Reward: 0.200", |
| "reward": 0.2, |
| "done": true |
| } |
| ] |
| } |
| ] |
