{ "cells": [ { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "c:\\Users\\admin\\Documents\\DocBERT2\\news-category-dataset\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Cloning into 'news-category-dataset'...\n", "c:\\Users\\admin\\AppData\\Local\\Programs\\Python\\Python312\\Lib\\site-packages\\IPython\\core\\magics\\osm.py:417: UserWarning: This is now an optional IPython functionality, setting dhist requires you to install the `pickleshare` library.\n", " self.shell.db['dhist'] = compress_dhist(dhist)[-100:]\n" ] } ], "source": [ "# Dataset is from Heegyu Kim, available at https://huggingface.co/datasets/heegyu/news-category-dataset, under CC-BY 4.0 license.\n", "# The dataset is a collection of news articles from various categories from Huffington Post.\n", "!git clone https://huggingface.co/datasets/heegyu/news-category-dataset" ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [], "source": [ "import pandas as pd\n", "import numpy as np" ] }, { "cell_type": "code", "execution_count": 6, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
linkheadlinecategoryshort_descriptionauthorsdate
0https://www.huffpost.com/entry/covid-boosters-...Over 4 Million Americans Roll Up Sleeves For O...U.S. NEWSHealth experts said it is too early to predict...Carla K. Johnson, AP2022-09-23
1https://www.huffpost.com/entry/american-airlin...American Airlines Flyer Charged, Banned For Li...U.S. NEWSHe was subdued by passengers and crew when he ...Mary Papenfuss2022-09-23
2https://www.huffpost.com/entry/funniest-tweets...23 Of The Funniest Tweets About Cats And Dogs ...COMEDY\"Until you have a dog you don't understand wha...Elyse Wanshel2022-09-23
3https://www.huffpost.com/entry/funniest-parent...The Funniest Tweets From Parents This Week (Se...PARENTING\"Accidentally put grown-up toothpaste on my to...Caroline Bologna2022-09-23
4https://www.huffpost.com/entry/amy-cooper-lose...Woman Who Called Cops On Black Bird-Watcher Lo...U.S. NEWSAmy Cooper accused investment firm Franklin Te...Nina Golgowski2022-09-22
\n", "
" ], "text/plain": [ " link \\\n", "0 https://www.huffpost.com/entry/covid-boosters-... \n", "1 https://www.huffpost.com/entry/american-airlin... \n", "2 https://www.huffpost.com/entry/funniest-tweets... \n", "3 https://www.huffpost.com/entry/funniest-parent... \n", "4 https://www.huffpost.com/entry/amy-cooper-lose... \n", "\n", " headline category \\\n", "0 Over 4 Million Americans Roll Up Sleeves For O... U.S. NEWS \n", "1 American Airlines Flyer Charged, Banned For Li... U.S. NEWS \n", "2 23 Of The Funniest Tweets About Cats And Dogs ... COMEDY \n", "3 The Funniest Tweets From Parents This Week (Se... PARENTING \n", "4 Woman Who Called Cops On Black Bird-Watcher Lo... U.S. NEWS \n", "\n", " short_description authors \\\n", "0 Health experts said it is too early to predict... Carla K. Johnson, AP \n", "1 He was subdued by passengers and crew when he ... Mary Papenfuss \n", "2 \"Until you have a dog you don't understand wha... Elyse Wanshel \n", "3 \"Accidentally put grown-up toothpaste on my to... Caroline Bologna \n", "4 Amy Cooper accused investment firm Franklin Te... Nina Golgowski \n", "\n", " date \n", "0 2022-09-23 \n", "1 2022-09-23 \n", "2 2022-09-23 \n", "3 2022-09-23 \n", "4 2022-09-22 " ] }, "execution_count": 6, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df = pd.read_json(\"./news-category-dataset/data.json\", lines=True)\n", "df.head(5)" ] }, { "cell_type": "code", "execution_count": 8, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Unique categories in the dataset: ['U.S. NEWS' 'COMEDY' 'PARENTING' 'WORLD NEWS' 'CULTURE & ARTS' 'TECH'\n", " 'SPORTS' 'ENTERTAINMENT' 'POLITICS' 'WEIRD NEWS' 'ENVIRONMENT'\n", " 'EDUCATION' 'CRIME' 'SCIENCE' 'WELLNESS' 'BUSINESS' 'STYLE & BEAUTY'\n", " 'FOOD & DRINK' 'MEDIA' 'QUEER VOICES' 'HOME & LIVING' 'WOMEN'\n", " 'BLACK VOICES' 'TRAVEL' 'MONEY' 'RELIGION' 'LATINO VOICES' 'IMPACT'\n", " 'WEDDINGS' 'COLLEGE' 'PARENTS' 'ARTS & CULTURE' 'STYLE' 'GREEN' 'TASTE'\n", " 'HEALTHY LIVING' 'THE WORLDPOST' 'GOOD NEWS' 'WORLDPOST' 'FIFTY' 'ARTS'\n", " 'DIVORCE']\n" ] } ], "source": [ "unique_categories = df['category'].unique()\n", "print(\"Unique categories in the dataset: \", unique_categories)" ] }, { "cell_type": "code", "execution_count": 13, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Number of WORLD category articles: 9542\n" ] } ], "source": [ "world_category = df[(df['category'] == 'WORLD') | (df['category'] == 'WORLD NEWS') | (df['category'] == 'WORLDPOST') | (df['category'] == 'THE WORLDPOST')]\n", "print(\"Number of WORLD category articles: \", len(world_category))" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Number of SPORTS category articles: 5077\n" ] } ], "source": [ "sports_category = df[(df['category'] == 'SPORTS') | (df['category'] == 'SPORT')]\n", "print(\"Number of SPORTS category articles: \", len(sports_category))" ] }, { "cell_type": "code", "execution_count": 16, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Number of BUSINESS category articles: 5992\n" ] } ], "source": [ "business_category = df[(df['category'] == 'BUSINESS')]\n", "print(\"Number of BUSINESS category articles: \", len(business_category))" ] }, { "cell_type": "code", "execution_count": 17, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Number of SCIENCE category articles: 4310\n" ] } ], "source": [ "science_category = df[(df['category'] == 'SCIENCE') | (df['category'] == 'SCIENCE & TECH') | (df['category'] == 'SCIENCE & TECH') | (df['category'] == 'TECH')]\n", "print(\"Number of SCIENCE category articles: \", len(science_category))" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "C:\\Users\\admin\\AppData\\Local\\Temp\\ipykernel_6844\\3064268492.py:1: SettingWithCopyWarning: \n", "A value is trying to be set on a copy of a slice from a DataFrame.\n", "Try using .loc[row_indexer,col_indexer] = value instead\n", "\n", "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n", " science_category['Description'] = science_category['headline'] + \"\\n\\n\" + science_category['short_description']\n", "C:\\Users\\admin\\AppData\\Local\\Temp\\ipykernel_6844\\3064268492.py:2: SettingWithCopyWarning: \n", "A value is trying to be set on a copy of a slice from a DataFrame.\n", "Try using .loc[row_indexer,col_indexer] = value instead\n", "\n", "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n", " science_category['Class Index'] = 3\n", "C:\\Users\\admin\\AppData\\Local\\Temp\\ipykernel_6844\\3064268492.py:4: SettingWithCopyWarning: \n", "A value is trying to be set on a copy of a slice from a DataFrame.\n", "Try using .loc[row_indexer,col_indexer] = value instead\n", "\n", "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n", " business_category['Description'] = business_category['headline'] + \"\\n\\n\" + business_category['short_description']\n", "C:\\Users\\admin\\AppData\\Local\\Temp\\ipykernel_6844\\3064268492.py:5: SettingWithCopyWarning: \n", "A value is trying to be set on a copy of a slice from a DataFrame.\n", "Try using .loc[row_indexer,col_indexer] = value instead\n", "\n", "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n", " business_category['Class Index'] = 2\n", "C:\\Users\\admin\\AppData\\Local\\Temp\\ipykernel_6844\\3064268492.py:7: SettingWithCopyWarning: \n", "A value is trying to be set on a copy of a slice from a DataFrame.\n", "Try using .loc[row_indexer,col_indexer] = value instead\n", "\n", "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n", " sports_category['Description'] = sports_category['headline'] + \"\\n\\n\" + sports_category['short_description']\n", "C:\\Users\\admin\\AppData\\Local\\Temp\\ipykernel_6844\\3064268492.py:8: SettingWithCopyWarning: \n", "A value is trying to be set on a copy of a slice from a DataFrame.\n", "Try using .loc[row_indexer,col_indexer] = value instead\n", "\n", "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n", " sports_category['Class Index'] = 1\n", "C:\\Users\\admin\\AppData\\Local\\Temp\\ipykernel_6844\\3064268492.py:10: SettingWithCopyWarning: \n", "A value is trying to be set on a copy of a slice from a DataFrame.\n", "Try using .loc[row_indexer,col_indexer] = value instead\n", "\n", "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n", " world_category['Description'] = world_category['headline'] + \"\\n\\n\" + world_category['short_description']\n", "C:\\Users\\admin\\AppData\\Local\\Temp\\ipykernel_6844\\3064268492.py:11: SettingWithCopyWarning: \n", "A value is trying to be set on a copy of a slice from a DataFrame.\n", "Try using .loc[row_indexer,col_indexer] = value instead\n", "\n", "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n", " world_category['Class Index'] = 0\n" ] }, { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
DescriptionClass Index
0Trump’s Saudi Trip Should Be About Ending The ...0
1Russia Vows To Expand 'Black List' Of American...0
2It's Been A Long, Crazy Year Since Britain's S...0
3First Penis Transplants In U.S. Planned For Wo...3
4U.S.-Backed Forces Prepare For ISIS To Use Che...0
5The European Migrant Crisis Is A Nightmare. Cl...0
6A Bunch Of Strange Things Just Went Down At NB...1
7Minor Explosion Rattles IMF Headquarters In Pa...0
8Shalane Flanagan Becomes First U.S. Woman To W...1
910 Instagrams To Celebrate Crown Prince Hussei...0
\n", "
" ], "text/plain": [ " Description Class Index\n", "0 Trump’s Saudi Trip Should Be About Ending The ... 0\n", "1 Russia Vows To Expand 'Black List' Of American... 0\n", "2 It's Been A Long, Crazy Year Since Britain's S... 0\n", "3 First Penis Transplants In U.S. Planned For Wo... 3\n", "4 U.S.-Backed Forces Prepare For ISIS To Use Che... 0\n", "5 The European Migrant Crisis Is A Nightmare. Cl... 0\n", "6 A Bunch Of Strange Things Just Went Down At NB... 1\n", "7 Minor Explosion Rattles IMF Headquarters In Pa... 0\n", "8 Shalane Flanagan Becomes First U.S. Woman To W... 1\n", "9 10 Instagrams To Celebrate Crown Prince Hussei... 0" ] }, "execution_count": 18, "metadata": {}, "output_type": "execute_result" } ], "source": [ "science_category['Description'] = science_category['headline'] + \"\\n\\n\" + science_category['short_description']\n", "science_category['Class Index'] = 3\n", "\n", "business_category['Description'] = business_category['headline'] + \"\\n\\n\" + business_category['short_description']\n", "business_category['Class Index'] = 2\n", "\n", "sports_category['Description'] = sports_category['headline'] + \"\\n\\n\" + sports_category['short_description']\n", "sports_category['Class Index'] = 1\n", "\n", "world_category['Description'] = world_category['headline'] + \"\\n\\n\" + world_category['short_description']\n", "world_category['Class Index'] = 0\n", "\n", "science_category = science_category[['Description', 'Class Index']]\n", "business_category = business_category[['Description', 'Class Index']]\n", "sports_category = sports_category[['Description', 'Class Index']]\n", "world_category = world_category[['Description', 'Class Index']]\n", "\n", "test_data_df = pd.concat([science_category, business_category, sports_category, world_category], ignore_index=True)\n", "# Shuffle the DataFrame\n", "test_data_df = test_data_df.sample(frac=1, random_state=42).reset_index(drop=True)\n", "\n", "test_data_df.head(10)" ] }, { "cell_type": "code", "execution_count": 19, "metadata": {}, "outputs": [], "source": [ "test_data_df.to_csv(\"test_data.csv\", index=False)" ] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.12.1" } }, "nbformat": 4, "nbformat_minor": 2 }