DanielSc4/DataAnalyticsNLP
0
1{2 "cells": [3 {4 "cell_type": "code",5 "execution_count": 2,6 "metadata": {},7 "outputs": [],8 "source": [9 "import pandas as pd\n",10 "import matplotlib.pyplot as plt\n"11 ]12 },13 {14 "cell_type": "code",15 "execution_count": 24,16 "metadata": {},17 "outputs": [18 {19 "data": {20 "text/html": [21 "<div>\n",22 "<style scoped>\n",23 " .dataframe tbody tr th:only-of-type {\n",24 " vertical-align: middle;\n",25 " }\n",26 "\n",27 " .dataframe tbody tr th {\n",28 " vertical-align: top;\n",29 " }\n",30 "\n",31 " .dataframe thead th {\n",32 " text-align: right;\n",33 " }\n",34 "</style>\n",35 "<table border=\"1\" class=\"dataframe\">\n",36 " <thead>\n",37 " <tr style=\"text-align: right;\">\n",38 " <th></th>\n",39 " <th>label</th>\n",40 " <th>comment</th>\n",41 " <th>user</th>\n",42 " <th>subreddit</th>\n",43 " <th>date</th>\n",44 " <th>sup_comment</th>\n",45 " <th>prediction</th>\n",46 " <th>confidence</th>\n",47 " <th>Topic_key_word</th>\n",48 " </tr>\n",49 " </thead>\n",50 " <tbody>\n",51 " <tr>\n",52 " <th>0</th>\n",53 " <td>0</td>\n",54 " <td>Actually most of her supporters and sane peopl...</td>\n",55 " <td>Quinnjester</td>\n",56 " <td>politics</td>\n",57 " <td>2016-09</td>\n",58 " <td>Hillary's Surrogotes Told to Blame Media for '...</td>\n",59 " <td>0</td>\n",60 " <td>0.974983</td>\n",61 " <td>TODO 2</td>\n",62 " </tr>\n",63 " <tr>\n",64 " <th>1</th>\n",65 " <td>0</td>\n",66 " <td>They can't survive without an echo chamber whi...</td>\n",67 " <td>TheGettysburgAddress</td>\n",68 " <td>The_Donald</td>\n",69 " <td>2016-11</td>\n",70 " <td>Thank God Liberals like to live in concentrate...</td>\n",71 " <td>1</td>\n",72 " <td>0.956885</td>\n",73 " <td>TODO 2</td>\n",74 " </tr>\n",75 " <tr>\n",76 " <th>2</th>\n",77 " <td>0</td>\n",78 " <td>you're pretty cute yourself 1729 total</td>\n",79 " <td>Sempiternally_free</td>\n",80 " <td>2007scape</td>\n",81 " <td>2016-11</td>\n",82 " <td>Saw this cutie training his Attack today...</td>\n",83 " <td>0</td>\n",84 " <td>0.899885</td>\n",85 " <td>TODO 2</td>\n",86 " </tr>\n",87 " <tr>\n",88 " <th>3</th>\n",89 " <td>0</td>\n",90 " <td>If you kill me you'll crash the meme market</td>\n",91 " <td>Catacomb82</td>\n",92 " <td>AskReddit</td>\n",93 " <td>2016-10</td>\n",94 " <td>If you were locked in a room with 49 other peo...</td>\n",95 " <td>0</td>\n",96 " <td>0.905721</td>\n",97 " <td>TODO 2</td>\n",98 " </tr>\n",99 " <tr>\n",100 " <th>4</th>\n",101 " <td>0</td>\n",102 " <td>I bet he wrote that last message as he was sob...</td>\n",103 " <td>Dorian-throwaway</td>\n",104 " <td>niceguys</td>\n",105 " <td>2016-11</td>\n",106 " <td>You're not even that pretty!</td>\n",107 " <td>1</td>\n",108 " <td>0.589593</td>\n",109 " <td>TODO 2</td>\n",110 " </tr>\n",111 " <tr>\n",112 " <th>...</th>\n",113 " <td>...</td>\n",114 " <td>...</td>\n",115 " <td>...</td>\n",116 " <td>...</td>\n",117 " <td>...</td>\n",118 " <td>...</td>\n",119 " <td>...</td>\n",120 " <td>...</td>\n",121 " <td>...</td>\n",122 " </tr>\n",123 " <tr>\n",124 " <th>251603</th>\n",125 " <td>1</td>\n",126 " <td>Respect your elders you little snot.</td>\n",127 " <td>Tiffany_Butler</td>\n",128 " <td>sports</td>\n",129 " <td>2009-06</td>\n",130 " <td>Aren't you a little old to be on the internet,...</td>\n",131 " <td>1</td>\n",132 " <td>0.852649</td>\n",133 " <td>TODO 1</td>\n",134 " </tr>\n",135 " <tr>\n",136 " <th>251604</th>\n",137 " <td>1</td>\n",138 " <td>I'm just glad they won't be using taxpayer mon...</td>\n",139 " <td>harryballsagna</td>\n",140 " <td>canada</td>\n",141 " <td>2009-06</td>\n",142 " <td>\"I'm sorry, I can't hear you over the sound of...</td>\n",143 " <td>1</td>\n",144 " <td>0.974458</td>\n",145 " <td>TODO 0</td>\n",146 " </tr>\n",147 " <tr>\n",148 " <th>251605</th>\n",149 " <td>1</td>\n",150 " <td>what.. with this awesome narration?</td>\n",151 " <td>aberant</td>\n",152 " <td>lost</td>\n",153 " <td>2009-04</td>\n",154 " <td>So far, so lame.</td>\n",155 " <td>1</td>\n",156 " <td>0.809398</td>\n",157 " <td>TODO 1</td>\n",158 " </tr>\n",159 " <tr>\n",160 " <th>251606</th>\n",161 " <td>1</td>\n",162 " <td>He looks trustworthy.</td>\n",163 " <td>permaculture</td>\n",164 " <td>unitedkingdom</td>\n",165 " <td>2009-01</td>\n",166 " <td>\"I don't care\" says Lapland boss</td>\n",167 " <td>1</td>\n",168 " <td>0.979738</td>\n",169 " <td>TODO 4</td>\n",170 " </tr>\n",171 " <tr>\n",172 " <th>251607</th>\n",173 " <td>1</td>\n",174 " <td>Well yeah, but it'll work this time.</td>\n",175 " <td>SovereignMan</td>\n",176 " <td>politics</td>\n",177 " <td>2009-02</td>\n",178 " <td>When their efforts failed, as they usually did...</td>\n",179 " <td>1</td>\n",180 " <td>0.975283</td>\n",181 " <td>TODO 1</td>\n",182 " </tr>\n",183 " </tbody>\n",184 "</table>\n",185 "<p>5000 rows × 9 columns</p>\n",186 "</div>"187 ],188 "text/plain": [189 " label comment \\\n",190 "0 0 Actually most of her supporters and sane peopl... \n",191 "1 0 They can't survive without an echo chamber whi... \n",192 "2 0 you're pretty cute yourself 1729 total \n",193 "3 0 If you kill me you'll crash the meme market \n",194 "4 0 I bet he wrote that last message as he was sob... \n",195 "... ... ... \n",196 "251603 1 Respect your elders you little snot. \n",197 "251604 1 I'm just glad they won't be using taxpayer mon... \n",198 "251605 1 what.. with this awesome narration? \n",199 "251606 1 He looks trustworthy. \n",200 "251607 1 Well yeah, but it'll work this time. \n",201 "\n",202 " user subreddit date \\\n",203 "0 Quinnjester politics 2016-09 \n",204 "1 TheGettysburgAddress The_Donald 2016-11 \n",205 "2 Sempiternally_free 2007scape 2016-11 \n",206 "3 Catacomb82 AskReddit 2016-10 \n",207 "4 Dorian-throwaway niceguys 2016-11 \n",208 "... ... ... ... \n",209 "251603 Tiffany_Butler sports 2009-06 \n",210 "251604 harryballsagna canada 2009-06 \n",211 "251605 aberant lost 2009-04 \n",212 "251606 permaculture unitedkingdom 2009-01 \n",213 "251607 SovereignMan politics 2009-02 \n",214 "\n",215 " sup_comment prediction \\\n",216 "0 Hillary's Surrogotes Told to Blame Media for '... 0 \n",217 "1 Thank God Liberals like to live in concentrate... 1 \n",218 "2 Saw this cutie training his Attack today... 0 \n",219 "3 If you were locked in a room with 49 other peo... 0 \n",220 "4 You're not even that pretty! 1 \n",221 "... ... ... \n",222 "251603 Aren't you a little old to be on the internet,... 1 \n",223 "251604 \"I'm sorry, I can't hear you over the sound of... 1 \n",224 "251605 So far, so lame. 1 \n",225 "251606 \"I don't care\" says Lapland boss 1 \n",226 "251607 When their efforts failed, as they usually did... 1 \n",227 "\n",228 " confidence Topic_key_word \n",229 "0 0.974983 TODO 2 \n",230 "1 0.956885 TODO 2 \n",231 "2 0.899885 TODO 2 \n",232 "3 0.905721 TODO 2 \n",233 "4 0.589593 TODO 2 \n",234 "... ... ... \n",235 "251603 0.852649 TODO 1 \n",236 "251604 0.974458 TODO 0 \n",237 "251605 0.809398 TODO 1 \n",238 "251606 0.979738 TODO 4 \n",239 "251607 0.975283 TODO 1 \n",240 "\n",241 "[5000 rows x 9 columns]"242 ]243 },244 "execution_count": 24,245 "metadata": {},246 "output_type": "execute_result"247 }248 ],249 "source": [250 "d = pd.read_csv('./data/results extended.csv', index_col=0)\n",251 "d"252 ]253 },254 {255 "cell_type": "markdown",256 "metadata": {},257 "source": [258 "TODO:\n",259 "- [x] Show LDA top words for each topic\n",260 "- [ ] I topic con una bassa percentuale di ironia sono i topic considerati più \"seri\" (?)\n",261 "- [x] Per ora sto utilizzando le label assegnate dal dataset, se non avessi le label e dovessi prevedere l'ironia LDA è cmq affidabile?"262 ]263 }264 ],265 "metadata": {266 "kernelspec": {267 "display_name": "torch_new",268 "language": "python",269 "name": "python3"270 },271 "language_info": {272 "codemirror_mode": {273 "name": "ipython",274 "version": 3275 },276 "file_extension": ".py",277 "mimetype": "text/x-python",278 "name": "python",279 "nbconvert_exporter": "python",280 "pygments_lexer": "ipython3",281 "version": "3.9.16"282 },283 "orig_nbformat": 4284 },285 "nbformat": 4,286 "nbformat_minor": 2287}288