{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "a460ddd6-72b4-42b9-a923-8416a75ef87e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "https://hf-mirror.com\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "\n",
    "# 设置环境变量\n",
    "os.environ['HF_ENDPOINT'] = 'https://hf-mirror.com'\n",
    "\n",
    "# 打印环境变量以确认设置成功\n",
    "print(os.environ.get('HF_ENDPOINT'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "18b5c7c9-6f3b-498e-8671-a4cd065d0bd4",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/maris/miniconda3/envs/dnagpt/lib/python3.11/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    }
   ],
   "source": [
    "from datasets import load_dataset\n",
    "from transformers import AutoTokenizer, DataCollatorWithPadding\n",
    "from transformers import Trainer\n",
    "import evaluate\n",
    "import numpy as np\n",
    "from transformers import TrainingArguments\n",
    "from transformers import AutoModelForSequenceClassification\n",
    "\n",
    "\n",
    "# 假设你的JSON文件名为 'data.json' 并且每行是一个独立的JSON对象\n",
    "raw_datasets = load_dataset('paws-x', 'en')  # 或者指定特定语言如 'zh' 表示中文,https://huggingface.co/datasets/google-research-datasets/paws-x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "ba9dc8c5-1c9e-47c5-9c77-b06573e05432",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DatasetDict({\n",
       "    train: Dataset({\n",
       "        features: ['id', 'sentence1', 'sentence2', 'label'],\n",
       "        num_rows: 49401\n",
       "    })\n",
       "    test: Dataset({\n",
       "        features: ['id', 'sentence1', 'sentence2', 'label'],\n",
       "        num_rows: 2000\n",
       "    })\n",
       "    validation: Dataset({\n",
       "        features: ['id', 'sentence1', 'sentence2', 'label'],\n",
       "        num_rows: 2000\n",
       "    })\n",
       "})"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "raw_datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "fbe3f78a-da0f-4f9a-87fc-4d484b4a3b48",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'id': 1,\n",
       " 'sentence1': 'In Paris , in October 1560 , he secretly met the English ambassador , Nicolas Throckmorton , asking him for a passport to return to England through Scotland .',\n",
       " 'sentence2': 'In October 1560 , he secretly met with the English ambassador , Nicolas Throckmorton , in Paris , and asked him for a passport to return to Scotland through England .',\n",
       " 'label': 0}"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "raw_datasets[\"train\"][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "7f048fc9-e8ca-4f27-bfb6-46ef8c12bc52",
   "metadata": {},
   "outputs": [],
   "source": [
    "raw_datasets_zh = load_dataset('paws-x', 'zh')  # 或者指定特定语言如 'zh' 表示中文,https://huggingface.co/datasets/google-research-datasets/paws-x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "d6128b62-6789-4934-a3dd-2c8fd6e7cb2d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DatasetDict({\n",
       "    train: Dataset({\n",
       "        features: ['id', 'sentence1', 'sentence2', 'label'],\n",
       "        num_rows: 49401\n",
       "    })\n",
       "    test: Dataset({\n",
       "        features: ['id', 'sentence1', 'sentence2', 'label'],\n",
       "        num_rows: 2000\n",
       "    })\n",
       "    validation: Dataset({\n",
       "        features: ['id', 'sentence1', 'sentence2', 'label'],\n",
       "        num_rows: 2000\n",
       "    })\n",
       "})"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "raw_datasets_zh"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "9e936f7d-a0b3-4698-97a9-bd07379cd199",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'id': 1,\n",
       " 'sentence1': '1560年10月，他在巴黎秘密会见了英国大使Nicolas Throckmorton，要求他通过苏格兰返回英国。',\n",
       " 'sentence2': '1560年10月，他在巴黎秘密会见了英国大使尼古拉斯·斯罗克莫顿，并要求他通过英格兰返回苏格兰的护照。',\n",
       " 'label': 0}"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "raw_datasets_zh[\"train\"][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "6f3ab940-cfb0-4753-83c1-a8ccc31cce26",
   "metadata": {},
   "outputs": [],
   "source": [
    "raw_datasets_fr = load_dataset('paws-x', 'fr')  # 或者指定特定语言如 'zh' 表示中文,https://huggingface.co/datasets/google-research-datasets/paws-x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "a9d707a7-a46f-4b15-b2e6-1c61df79be9f",
   "metadata": {},
   "outputs": [],
   "source": [
    "raw_datasets_de = load_dataset('paws-x', 'de')  # 或者指定特定语言如 'zh' 表示中文,https://huggingface.co/datasets/google-research-datasets/paws-x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "f9092ac6-6467-4666-bd27-3000c158add5",
   "metadata": {},
   "outputs": [],
   "source": [
    "raw_datasets_es = load_dataset('paws-x', 'es')  # 或者指定特定语言如 'zh' 表示中文,https://huggingface.co/datasets/google-research-datasets/paws-x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "8876048c-90c4-4964-9804-115de7128b78",
   "metadata": {},
   "outputs": [],
   "source": [
    "raw_datasets_ja = load_dataset('paws-x', 'ja')  # 或者指定特定语言如 'zh' 表示中文,https://huggingface.co/datasets/google-research-datasets/paws-x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "a14dcbc9-73a3-4102-88e5-b94d2cfadb36",
   "metadata": {},
   "outputs": [],
   "source": [
    "raw_datasets_ko = load_dataset('paws-x', 'ko')  # 或者指定特定语言如 'zh' 表示中文,https://huggingface.co/datasets/google-research-datasets/paws-x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "ee6b5d7c-c993-45f9-a008-53e2653818f2",
   "metadata": {},
   "outputs": [],
   "source": [
    "data_list = []\n",
    "for index in range(0, 49401):\n",
    "    data_en = raw_datasets[\"train\"][index]\n",
    "    data_fr =  raw_datasets_fr[\"train\"][index]\n",
    "    data_de = raw_datasets_de[\"train\"][index]\n",
    "    data_zh =  raw_datasets_zh[\"train\"][index]\n",
    "    data_es = raw_datasets_es[\"train\"][index]\n",
    "    data_ja =  raw_datasets_ja[\"train\"][index]\n",
    "    data_ko =  raw_datasets_ko[\"train\"][index]\n",
    "\n",
    "    if data_en[\"id\"]==index+1 and data_fr[\"id\"]==index+1 and data_de[\"id\"]==index+1 and data_zh[\"id\"]==index+1 and data_es[\"id\"]==index+1 and data_ja[\"id\"]==index+1 and data_ko[\"id\"]==index+1:\n",
    "        data = {\n",
    "                \"data_id\":data_en[\"id\"],\n",
    "                \"sentence1\":data_en[\"sentence1\"],\n",
    "                \"sentence1_fr\":data_fr[\"sentence1\"],\n",
    "                \"sentence1_de\":data_de[\"sentence1\"],\n",
    "                \"sentence1_zh\":data_zh[\"sentence1\"],\n",
    "                \"sentence1_es\":data_es[\"sentence1\"],\n",
    "                \"sentence1_ja\":data_ja[\"sentence1\"],\n",
    "                \"sentence1_ko\":data_ko[\"sentence1\"],\n",
    "                \"label\":data_en[\"label\"],\n",
    "                \"sentence2\":data_en[\"sentence2\"],\n",
    "                \"sentence2_fr\":data_fr[\"sentence2\"],\n",
    "                \"sentence2_de\":data_de[\"sentence2\"],\n",
    "                \"sentence2_zh\":data_zh[\"sentence2\"],\n",
    "                \"sentence2_es\":data_es[\"sentence2\"],\n",
    "                \"sentence2_ja\":data_ja[\"sentence2\"],\n",
    "                \"sentence2_ko\":data_ko[\"sentence2\"],\n",
    "               }\n",
    "        data_list.append(data)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "cc4f5c16-ed09-4866-ab49-736e8940f45c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "49401"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(data_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "afc22ff2-06fe-4999-b72c-208e97dc62cb",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import sql_appbk\n",
    "sql_appbk.insert_data_list(data_list, \"pawx\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "819d9d2c-2ae8-449f-9582-8ff608bd349f",
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "ins_file = open(\"paws-x-multi-pair.jsonl\", \"w\")\n",
    "\n",
    "for item in data_list:\n",
    "    ins_file.write(json.dumps(item, ensure_ascii=False)+\"\\n\")\n",
    "\n",
    "ins_file.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "baa248d1-090f-4be7-878d-746f10bbb444",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}