Spaces:

yjzhu0225
/

reddit_text_classification_app

Runtime error

App Files Files Community

Susanna Anil commited on Dec 13, 2022

Commit

7688b91

1 Parent(s): 239c0f3

cli added

Browse files

Files changed (4) hide show

Dockerfile +8 -0
app.py +56 -0
requirements.txt +4 -1
scrape_load.ipynb +389 -21

Dockerfile ADDED Viewed

	@@ -0,0 +1,8 @@

+FROM python:3.8.13-slim-buster
+RUN mkdir -p /app
+COPY . main.py /app/
+WORKDIR /app
+RUN pip install -r requirements.txt
+EXPOSE 8080
+CMD [ "main.py" ]
+ENTRYPOINT [ "python" ]

app.py ADDED Viewed

	@@ -0,0 +1,56 @@

+'''
+Run 'python3 main.py' in terminal
+Follow local URL
+'''
+import praw
+import pandas as pd
+from praw.models import MoreComments
+from transformers import pipeline
+from transformers import DistilBertTokenizerFast
+import gradio as gr
+import numpy as np
+reddit= praw.Reddit(client_id="Q1w42RHhLq2fgwljAk_k-Q",		 # your client id
+					client_secret="enUJfFthiZRynGfPQtoK1nCxRer2Dw",	 # your client secret
+                    usernme = "xl395", #profile username
+                    password = "12xiao34quanAria!", #profile password
+					user_agent="706_post")	 # your user agent
+classifier = pipeline("sentiment-analysis", model="michellejieli/NSFW_text_classifier")
+#input_url = "https://www.reddit.com/r/europe/comments/r0hthg/sweden_is_taking_the_lead_to_persuade_the_rest_of/"
+def extract_comments(input_url):
+    submission = reddit.submission(url=input_url)
+    #posts_dict = {"Post text":[],}
+    posts_dict = {"Post text":[], "class": []}
+    for top_level_comment in submission.comments:
+        if isinstance(top_level_comment, MoreComments):
+            continue
+        posts_dict["Post text"].append(top_level_comment.body)
+        posts_dict["class"].append(classifier(top_level_comment.body)[0]['label'])
+    df = pd.DataFrame(posts_dict)
+    percent_exp = np.count_nonzero(np.array(df["class"]) == 'NSFW') / df.shape[0]
+    output_msg = "Reddit page is contains no explicit content. Page is safe for users under 18."
+    if percent_exp > 0:
+        output_msg = f"Reddit page contains some explicit content. Users under the age of 18 should proceed with caution. \n{percent_exp * 100:.0f}% explicit."
+    if percent_exp > 0.5:
+        output_msg = f"Reddit page contains major explicit content. Users be wary. \n{percent_exp * 100:.0f}% explicit."
+    return output_msg
+# use gradio to create a web interface take a wikipedia page and summarize it
+iface = gr.Interface(
+    fn=extract_comments,
+    inputs=gr.Textbox(
+        lines=2,
+        placeholder="Enter Reddit page link",
+    ),
+    outputs="text",
+)
+if __name__ == "__main__":
+    iface.launch()

requirements.txt CHANGED Viewed

@@ -5,4 +5,7 @@ uvicorn[standard]
 pandas
 black
 transformers
-praw

 pandas
 black
 transformers
+praw
+numpy
+gradio
+altair

scrape_load.ipynb CHANGED Viewed

@@ -2,7 +2,7 @@
  "cells": [
   {
    "cell_type": "code",
-   "execution_count": 7,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -18,18 +18,9 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 35,
    "metadata": {},
-   "outputs": [
-    {
-     "ename": "SyntaxError",
-     "evalue": "expected ':' (3580341109.py, line 8)",
-     "output_type": "error",
-     "traceback": [
-      "\u001b[0;36m  Cell \u001b[0;32mIn[35], line 8\u001b[0;36m\u001b[0m\n\u001b[0;31m    def extract_comments(input_url)\u001b[0m\n\u001b[0m                                   ^\u001b[0m\n\u001b[0;31mSyntaxError\u001b[0m\u001b[0;31m:\u001b[0m expected ':'\n"
-     ]
-    }
-   ],
    "source": [
     "from praw.models import MoreComments\n",
     "from transformers import pipeline\n",
@@ -40,28 +31,405 @@
     "\n",
     "def extract_comments(input_url):\n",
     "    submission = reddit.submission(url=input_url)\n",
-    "    posts_dict = {\"Post text\":[],}\n",
-    "    # posts_dict = {\"Post text\":[], \"class\": []}\n",
     "    for top_level_comment in submission.comments:\n",
     "        if isinstance(top_level_comment, MoreComments):\n",
     "            continue\n",
     "        \n",
     "        posts_dict[\"Post text\"].append(top_level_comment.body)\n",
-    "        #post_dict[\"class\"].append(classifier(top_level_comment.body))\n",
     "    df = pd.DataFrame(posts_dict)\n",
     "    return df "
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": null,
    "metadata": {},
-   "outputs": [],
    "source": [
-    "df_process = extract_comments(input_url)\n",
-    "\n",
-    "# find how many are NSFW - output message"
    ]
   }
  ],
  "metadata": {
@@ -80,7 +448,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.10.4 (main, Nov 29 2022, 20:00:25) [GCC 9.4.0]"
   },
   "orig_nbformat": 4,
   "vscode": {

  "cells": [
   {
    "cell_type": "code",
+   "execution_count": 1,
    "metadata": {},
    "outputs": [],
    "source": [
   },
   {
    "cell_type": "code",
+   "execution_count": 8,
    "metadata": {},
+   "outputs": [],
    "source": [
     "from praw.models import MoreComments\n",
     "from transformers import pipeline\n",
     "\n",
     "def extract_comments(input_url):\n",
     "    submission = reddit.submission(url=input_url)\n",
+    "    #posts_dict = {\"Post text\":[],}\n",
+    "    posts_dict = {\"Post text\":[], \"class\": []}\n",
     "    for top_level_comment in submission.comments:\n",
     "        if isinstance(top_level_comment, MoreComments):\n",
     "            continue\n",
     "        \n",
     "        posts_dict[\"Post text\"].append(top_level_comment.body)\n",
+    "        posts_dict[\"class\"].append(classifier(top_level_comment.body)[0]['label'])\n",
     "    df = pd.DataFrame(posts_dict)\n",
     "    return df "
    ]
   },
   {
    "cell_type": "code",
+   "execution_count": 9,
    "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>Post text</th>\n",
+       "      <th>class</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>Does… does that mean GPU’s might become afford...</td>\n",
+       "      <td>SFW</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>But I use my mining rig for interior heating. ...</td>\n",
+       "      <td>SFW</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>I approve.\\n\\nAlthough have they suggested how...</td>\n",
+       "      <td>SFW</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>You can't ban cryptocurrencies. Just like you ...</td>\n",
+       "      <td>SFW</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>But still no nuclear plants</td>\n",
+       "      <td>SFW</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>5</th>\n",
+       "      <td>https://twitter.com/ercwl/status/1457114531314...</td>\n",
+       "      <td>NSFW</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>6</th>\n",
+       "      <td>Yes, \"we\" are missing the goal. Maybe stop sub...</td>\n",
+       "      <td>NSFW</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>7</th>\n",
+       "      <td>here is something complete insane to think abo...</td>\n",
+       "      <td>NSFW</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>8</th>\n",
+       "      <td>Put a tax carbon on it, it will elimanate the ...</td>\n",
+       "      <td>SFW</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>9</th>\n",
+       "      <td>Banning crypto mining will only cause it to mo...</td>\n",
+       "      <td>SFW</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>10</th>\n",
+       "      <td>Can someone explain how this is possible?</td>\n",
+       "      <td>SFW</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>11</th>\n",
+       "      <td>The average knowledge on crypto in here is ver...</td>\n",
+       "      <td>NSFW</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>12</th>\n",
+       "      <td>Howbout u ban Facebook instead?</td>\n",
+       "      <td>NSFW</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>13</th>\n",
+       "      <td>Good. Blockchain is useful but so many despera...</td>\n",
+       "      <td>NSFW</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>14</th>\n",
+       "      <td>I don't particularly like crypto or it's fans,...</td>\n",
+       "      <td>SFW</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>15</th>\n",
+       "      <td>I was thinking of getting rid of my RTX 2070s,...</td>\n",
+       "      <td>SFW</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>16</th>\n",
+       "      <td>Total waste of time. As seen following the Chi...</td>\n",
+       "      <td>SFW</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>17</th>\n",
+       "      <td>It's actually shocking how uneducated people a...</td>\n",
+       "      <td>NSFW</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>18</th>\n",
+       "      <td>Totally doesn’t have anything to do with money...</td>\n",
+       "      <td>NSFW</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>19</th>\n",
+       "      <td>Sweden's biggest electric company Vattenfall s...</td>\n",
+       "      <td>NSFW</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>20</th>\n",
+       "      <td>BASED</td>\n",
+       "      <td>SFW</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>21</th>\n",
+       "      <td>Excellent news.\\n\\nIt consumes vast quantities...</td>\n",
+       "      <td>NSFW</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>22</th>\n",
+       "      <td>sweden want to fuck with danemark and his cryp...</td>\n",
+       "      <td>NSFW</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>23</th>\n",
+       "      <td>They should ban gold too, mining gold uses 10x...</td>\n",
+       "      <td>NSFW</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>24</th>\n",
+       "      <td>Great news. Cryptocurrencies are an embarrassi...</td>\n",
+       "      <td>NSFW</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>25</th>\n",
+       "      <td>This is BS, it doesn't make sense.</td>\n",
+       "      <td>NSFW</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>26</th>\n",
+       "      <td>I see no problem with this, the amount of ener...</td>\n",
+       "      <td>SFW</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>27</th>\n",
+       "      <td>[deleted]</td>\n",
+       "      <td>SFW</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>28</th>\n",
+       "      <td>\"I'm running my graphics card at full load so ...</td>\n",
+       "      <td>SFW</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>29</th>\n",
+       "      <td>The market is moving from POW to POS which is ...</td>\n",
+       "      <td>SFW</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>30</th>\n",
+       "      <td>And exactly how are they gonna do that?\\n\\nPir...</td>\n",
+       "      <td>SFW</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>31</th>\n",
+       "      <td>Yeah because that's what's the main cause of o...</td>\n",
+       "      <td>SFW</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>32</th>\n",
+       "      <td>Banning mining without banning transactions an...</td>\n",
+       "      <td>NSFW</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>33</th>\n",
+       "      <td>Crypto seems to have some side effects, who wo...</td>\n",
+       "      <td>SFW</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>34</th>\n",
+       "      <td>Crypto currency is the new tulips. So hot righ...</td>\n",
+       "      <td>NSFW</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>35</th>\n",
+       "      <td>Fuck yeah, ban them all! Easiest climate check...</td>\n",
+       "      <td>NSFW</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>36</th>\n",
+       "      <td>Ban fossil fuels instead. What a useless measu...</td>\n",
+       "      <td>NSFW</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>37</th>\n",
+       "      <td>Please no. I earn 200 euros a month from it.</td>\n",
+       "      <td>SFW</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>38</th>\n",
+       "      <td>Swedens state owned power company rejects this...</td>\n",
+       "      <td>NSFW</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>39</th>\n",
+       "      <td>Great not only we had Chinese crypto miners se...</td>\n",
+       "      <td>SFW</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>40</th>\n",
+       "      <td>Let's ban the currency that is somewhat indepe...</td>\n",
+       "      <td>NSFW</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>41</th>\n",
+       "      <td>Is that really that much of a problem with ene...</td>\n",
+       "      <td>NSFW</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>42</th>\n",
+       "      <td>How about instead of banning it, we make it so...</td>\n",
+       "      <td>SFW</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>43</th>\n",
+       "      <td>The electric footprint of crypto won't go up. ...</td>\n",
+       "      <td>SFW</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>44</th>\n",
+       "      <td>Lets make playing a demanding game more than t...</td>\n",
+       "      <td>NSFW</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>45</th>\n",
+       "      <td>How does cryptocurrency change the climate? I ...</td>\n",
+       "      <td>SFW</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>46</th>\n",
+       "      <td>Load ze Sweden FUD</td>\n",
+       "      <td>SFW</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>47</th>\n",
+       "      <td>[removed]</td>\n",
+       "      <td>NSFW</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>48</th>\n",
+       "      <td>Such pathetic virtue-signaling.\\n\\nIts no diff...</td>\n",
+       "      <td>NSFW</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>49</th>\n",
+       "      <td>shame on sweden</td>\n",
+       "      <td>NSFW</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>50</th>\n",
+       "      <td>-And now, NEWS FOR DUMBASSES, Bob?\\n\\n-Yes Bri...</td>\n",
+       "      <td>NSFW</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>51</th>\n",
+       "      <td>dumb</td>\n",
+       "      <td>NSFW</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>52</th>\n",
+       "      <td>Wow, Sweden is such a dictatorial shithole cou...</td>\n",
+       "      <td>NSFW</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>53</th>\n",
+       "      <td>talking about that, a full ban might be unlike...</td>\n",
+       "      <td>NSFW</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                                            Post text class\n",
+       "0   Does… does that mean GPU’s might become afford...   SFW\n",
+       "1   But I use my mining rig for interior heating. ...   SFW\n",
+       "2   I approve.\\n\\nAlthough have they suggested how...   SFW\n",
+       "3   You can't ban cryptocurrencies. Just like you ...   SFW\n",
+       "4                         But still no nuclear plants   SFW\n",
+       "5   https://twitter.com/ercwl/status/1457114531314...  NSFW\n",
+       "6   Yes, \"we\" are missing the goal. Maybe stop sub...  NSFW\n",
+       "7   here is something complete insane to think abo...  NSFW\n",
+       "8   Put a tax carbon on it, it will elimanate the ...   SFW\n",
+       "9   Banning crypto mining will only cause it to mo...   SFW\n",
+       "10          Can someone explain how this is possible?   SFW\n",
+       "11  The average knowledge on crypto in here is ver...  NSFW\n",
+       "12                    Howbout u ban Facebook instead?  NSFW\n",
+       "13  Good. Blockchain is useful but so many despera...  NSFW\n",
+       "14  I don't particularly like crypto or it's fans,...   SFW\n",
+       "15  I was thinking of getting rid of my RTX 2070s,...   SFW\n",
+       "16  Total waste of time. As seen following the Chi...   SFW\n",
+       "17  It's actually shocking how uneducated people a...  NSFW\n",
+       "18  Totally doesn’t have anything to do with money...  NSFW\n",
+       "19  Sweden's biggest electric company Vattenfall s...  NSFW\n",
+       "20                                              BASED   SFW\n",
+       "21  Excellent news.\\n\\nIt consumes vast quantities...  NSFW\n",
+       "22  sweden want to fuck with danemark and his cryp...  NSFW\n",
+       "23  They should ban gold too, mining gold uses 10x...  NSFW\n",
+       "24  Great news. Cryptocurrencies are an embarrassi...  NSFW\n",
+       "25                 This is BS, it doesn't make sense.  NSFW\n",
+       "26  I see no problem with this, the amount of ener...   SFW\n",
+       "27                                          [deleted]   SFW\n",
+       "28  \"I'm running my graphics card at full load so ...   SFW\n",
+       "29  The market is moving from POW to POS which is ...   SFW\n",
+       "30  And exactly how are they gonna do that?\\n\\nPir...   SFW\n",
+       "31  Yeah because that's what's the main cause of o...   SFW\n",
+       "32  Banning mining without banning transactions an...  NSFW\n",
+       "33  Crypto seems to have some side effects, who wo...   SFW\n",
+       "34  Crypto currency is the new tulips. So hot righ...  NSFW\n",
+       "35  Fuck yeah, ban them all! Easiest climate check...  NSFW\n",
+       "36  Ban fossil fuels instead. What a useless measu...  NSFW\n",
+       "37       Please no. I earn 200 euros a month from it.   SFW\n",
+       "38  Swedens state owned power company rejects this...  NSFW\n",
+       "39  Great not only we had Chinese crypto miners se...   SFW\n",
+       "40  Let's ban the currency that is somewhat indepe...  NSFW\n",
+       "41  Is that really that much of a problem with ene...  NSFW\n",
+       "42  How about instead of banning it, we make it so...   SFW\n",
+       "43  The electric footprint of crypto won't go up. ...   SFW\n",
+       "44  Lets make playing a demanding game more than t...  NSFW\n",
+       "45  How does cryptocurrency change the climate? I ...   SFW\n",
+       "46                                 Load ze Sweden FUD   SFW\n",
+       "47                                          [removed]  NSFW\n",
+       "48  Such pathetic virtue-signaling.\\n\\nIts no diff...  NSFW\n",
+       "49                                    shame on sweden  NSFW\n",
+       "50  -And now, NEWS FOR DUMBASSES, Bob?\\n\\n-Yes Bri...  NSFW\n",
+       "51                                               dumb  NSFW\n",
+       "52  Wow, Sweden is such a dictatorial shithole cou...  NSFW\n",
+       "53  talking about that, a full ban might be unlike...  NSFW"
+      ]
+     },
+     "execution_count": 9,
+     "metadata": {},
+     "output_type": "execute_result"
+    },
+    {
+     "ename": "",
+     "evalue": "",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[1;31mThe Kernel crashed while executing code in the the current cell or a previous cell. Please review the code in the cell(s) to identify a possible cause of the failure. Click <a href='https://aka.ms/vscodeJupyterKernelCrash'>here</a> for more info. View Jupyter <a href='command:jupyter.viewOutput'>log</a> for further details."
+     ]
+    }
+   ],
    "source": [
+    "extract_comments(input_url)\n",
+    "\n"
    ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
   }
  ],
  "metadata": {
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
+   "version": "3.10.4"
   },
   "orig_nbformat": 4,
   "vscode": {