cuda_bert-runtime.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "c8c206d3-696c-4ca4-864c-549b21e00cdb",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "FULL Dataset: (2914, 2)\n",
      "TRAIN Dataset: (2331, 2)\n",
      "TEST Dataset: (583, 2)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at bert-base-uncased were not used when initializing BertModel: ['cls.predictions.decoder.weight', 'cls.predictions.transform.LayerNorm.bias', 'cls.predictions.transform.dense.weight', 'cls.predictions.transform.LayerNorm.weight', 'cls.seq_relationship.bias', 'cls.seq_relationship.weight', 'cls.predictions.transform.dense.bias', 'cls.predictions.bias']\n",
      "- This IS expected if you are initializing BertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing BertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "############# Epoch 1: Training Start   #############\n",
      "############# Epoch 1: Training End     #############\n",
      "############# Epoch 1: Validation Start   #############\n",
      "############# Epoch 1: Validation End     #############\n",
      "Epoch: 1 \tAvgerage Training Loss: 0.008476 \tAverage Validation Loss: 0.031578\n",
      "Validation loss decreased (inf --> 0.031578).  Saving model ...\n",
      "############# Epoch 1  Done   #############\n",
      "\n",
      "--- 25.08689045906067 seconds ---\n",
      "############# Epoch 2: Training Start   #############\n",
      "############# Epoch 2: Training End     #############\n",
      "############# Epoch 2: Validation Start   #############\n",
      "############# Epoch 2: Validation End     #############\n",
      "Epoch: 2 \tAvgerage Training Loss: 0.007814 \tAverage Validation Loss: 0.029465\n",
      "Validation loss decreased (0.031578 --> 0.029465).  Saving model ...\n",
      "############# Epoch 2  Done   #############\n",
      "\n",
      "--- 45.23912310600281 seconds ---\n",
      "############# Epoch 3: Training Start   #############\n",
      "############# Epoch 3: Training End     #############\n",
      "############# Epoch 3: Validation Start   #############\n",
      "############# Epoch 3: Validation End     #############\n",
      "Epoch: 3 \tAvgerage Training Loss: 0.006916 \tAverage Validation Loss: 0.026801\n",
      "Validation loss decreased (0.029465 --> 0.026801).  Saving model ...\n",
      "############# Epoch 3  Done   #############\n",
      "\n",
      "--- 64.92936325073242 seconds ---\n",
      "############# Epoch 4: Training Start   #############\n",
      "############# Epoch 4: Training End     #############\n",
      "############# Epoch 4: Validation Start   #############\n",
      "############# Epoch 4: Validation End     #############\n",
      "Epoch: 4 \tAvgerage Training Loss: 0.005679 \tAverage Validation Loss: 0.027397\n",
      "############# Epoch 4  Done   #############\n",
      "\n",
      "--- 78.66459727287292 seconds ---\n",
      "############# Epoch 5: Training Start   #############\n",
      "############# Epoch 5: Training End     #############\n",
      "############# Epoch 5: Validation Start   #############\n",
      "############# Epoch 5: Validation End     #############\n",
      "Epoch: 5 \tAvgerage Training Loss: 0.004446 \tAverage Validation Loss: 0.026036\n",
      "Validation loss decreased (0.026801 --> 0.026036).  Saving model ...\n",
      "############# Epoch 5  Done   #############\n",
      "\n",
      "--- 98.2181384563446 seconds ---\n",
      "############# Epoch 6: Training Start   #############\n",
      "############# Epoch 6: Training End     #############\n",
      "############# Epoch 6: Validation Start   #############\n",
      "############# Epoch 6: Validation End     #############\n",
      "Epoch: 6 \tAvgerage Training Loss: 0.003249 \tAverage Validation Loss: 0.027464\n",
      "############# Epoch 6  Done   #############\n",
      "\n",
      "--- 112.37017846107483 seconds ---\n",
      "############# Epoch 7: Training Start   #############\n",
      "############# Epoch 7: Training End     #############\n",
      "############# Epoch 7: Validation Start   #############\n",
      "############# Epoch 7: Validation End     #############\n",
      "Epoch: 7 \tAvgerage Training Loss: 0.002299 \tAverage Validation Loss: 0.030525\n",
      "############# Epoch 7  Done   #############\n",
      "\n",
      "--- 126.13758873939514 seconds ---\n",
      "############# Epoch 8: Training Start   #############\n",
      "############# Epoch 8: Training End     #############\n",
      "############# Epoch 8: Validation Start   #############\n",
      "############# Epoch 8: Validation End     #############\n",
      "Epoch: 8 \tAvgerage Training Loss: 0.001501 \tAverage Validation Loss: 0.030317\n",
      "############# Epoch 8  Done   #############\n",
      "\n",
      "--- 139.88642263412476 seconds ---\n",
      "############# Epoch 9: Training Start   #############\n",
      "############# Epoch 9: Training End     #############\n",
      "############# Epoch 9: Validation Start   #############\n",
      "############# Epoch 9: Validation End     #############\n",
      "Epoch: 9 \tAvgerage Training Loss: 0.001025 \tAverage Validation Loss: 0.031739\n",
      "############# Epoch 9  Done   #############\n",
      "\n",
      "--- 153.59837436676025 seconds ---\n",
      "############# Epoch 10: Training Start   #############\n",
      "############# Epoch 10: Training End     #############\n",
      "############# Epoch 10: Validation Start   #############\n",
      "############# Epoch 10: Validation End     #############\n",
      "Epoch: 10 \tAvgerage Training Loss: 0.000659 \tAverage Validation Loss: 0.034797\n",
      "############# Epoch 10  Done   #############\n",
      "\n",
      "--- 169.6184003353119 seconds ---\n",
      "############# Epoch 11: Training Start   #############\n",
      "############# Epoch 11: Training End     #############\n",
      "############# Epoch 11: Validation Start   #############\n",
      "############# Epoch 11: Validation End     #############\n",
      "Epoch: 11 \tAvgerage Training Loss: 0.000431 \tAverage Validation Loss: 0.036987\n",
      "############# Epoch 11  Done   #############\n",
      "\n",
      "--- 185.36580991744995 seconds ---\n",
      "############# Epoch 12: Training Start   #############\n",
      "############# Epoch 12: Training End     #############\n",
      "############# Epoch 12: Validation Start   #############\n",
      "############# Epoch 12: Validation End     #############\n",
      "Epoch: 12 \tAvgerage Training Loss: 0.000318 \tAverage Validation Loss: 0.040311\n",
      "############# Epoch 12  Done   #############\n",
      "\n",
      "--- 199.22856664657593 seconds ---\n",
      "############# Epoch 13: Training Start   #############\n",
      "############# Epoch 13: Training End     #############\n",
      "############# Epoch 13: Validation Start   #############\n",
      "############# Epoch 13: Validation End     #############\n",
      "Epoch: 13 \tAvgerage Training Loss: 0.000253 \tAverage Validation Loss: 0.041637\n",
      "############# Epoch 13  Done   #############\n",
      "\n",
      "--- 217.04697632789612 seconds ---\n",
      "############# Epoch 14: Training Start   #############\n",
      "############# Epoch 14: Training End     #############\n",
      "############# Epoch 14: Validation Start   #############\n",
      "############# Epoch 14: Validation End     #############\n",
      "Epoch: 14 \tAvgerage Training Loss: 0.000218 \tAverage Validation Loss: 0.042717\n",
      "############# Epoch 14  Done   #############\n",
      "\n",
      "--- 230.95286393165588 seconds ---\n",
      "############# Epoch 15: Training Start   #############\n",
      "############# Epoch 15: Training End     #############\n",
      "############# Epoch 15: Validation Start   #############\n",
      "############# Epoch 15: Validation End     #############\n",
      "Epoch: 15 \tAvgerage Training Loss: 0.000169 \tAverage Validation Loss: 0.046076\n",
      "############# Epoch 15  Done   #############\n",
      "\n",
      "--- 247.83851408958435 seconds ---\n",
      "############# Epoch 16: Training Start   #############\n",
      "############# Epoch 16: Training End     #############\n",
      "############# Epoch 16: Validation Start   #############\n",
      "############# Epoch 16: Validation End     #############\n",
      "Epoch: 16 \tAvgerage Training Loss: 0.000142 \tAverage Validation Loss: 0.046784\n",
      "############# Epoch 16  Done   #############\n",
      "\n",
      "--- 262.50674867630005 seconds ---\n",
      "############# Epoch 17: Training Start   #############\n",
      "############# Epoch 17: Training End     #############\n",
      "############# Epoch 17: Validation Start   #############\n",
      "############# Epoch 17: Validation End     #############\n",
      "Epoch: 17 \tAvgerage Training Loss: 0.000270 \tAverage Validation Loss: 0.050126\n",
      "############# Epoch 17  Done   #############\n",
      "\n",
      "--- 276.5390639305115 seconds ---\n",
      "############# Epoch 18: Training Start   #############\n",
      "############# Epoch 18: Training End     #############\n",
      "############# Epoch 18: Validation Start   #############\n",
      "############# Epoch 18: Validation End     #############\n",
      "Epoch: 18 \tAvgerage Training Loss: 0.000411 \tAverage Validation Loss: 0.044376\n",
      "############# Epoch 18  Done   #############\n",
      "\n",
      "--- 293.10637855529785 seconds ---\n",
      "############# Epoch 19: Training Start   #############\n",
      "############# Epoch 19: Training End     #############\n",
      "############# Epoch 19: Validation Start   #############\n",
      "############# Epoch 19: Validation End     #############\n",
      "Epoch: 19 \tAvgerage Training Loss: 0.000167 \tAverage Validation Loss: 0.044954\n",
      "############# Epoch 19  Done   #############\n",
      "\n",
      "--- 307.51391196250916 seconds ---\n",
      "############# Epoch 20: Training Start   #############\n",
      "############# Epoch 20: Training End     #############\n",
      "############# Epoch 20: Validation Start   #############\n",
      "############# Epoch 20: Validation End     #############\n",
      "Epoch: 20 \tAvgerage Training Loss: 0.000124 \tAverage Validation Loss: 0.046709\n",
      "############# Epoch 20  Done   #############\n",
      "\n",
      "--- 324.33851647377014 seconds ---\n",
      "############# Epoch 21: Training Start   #############\n",
      "############# Epoch 21: Training End     #############\n",
      "############# Epoch 21: Validation Start   #############\n",
      "############# Epoch 21: Validation End     #############\n",
      "Epoch: 21 \tAvgerage Training Loss: 0.000101 \tAverage Validation Loss: 0.051833\n",
      "############# Epoch 21  Done   #############\n",
      "\n",
      "--- 338.2990596294403 seconds ---\n",
      "############# Epoch 22: Training Start   #############\n",
      "############# Epoch 22: Training End     #############\n",
      "############# Epoch 22: Validation Start   #############\n",
      "############# Epoch 22: Validation End     #############\n",
      "Epoch: 22 \tAvgerage Training Loss: 0.000164 \tAverage Validation Loss: 0.051572\n",
      "############# Epoch 22  Done   #############\n",
      "\n",
      "--- 355.7591769695282 seconds ---\n",
      "############# Epoch 23: Training Start   #############\n",
      "############# Epoch 23: Training End     #############\n",
      "############# Epoch 23: Validation Start   #############\n",
      "############# Epoch 23: Validation End     #############\n",
      "Epoch: 23 \tAvgerage Training Loss: 0.000180 \tAverage Validation Loss: 0.050504\n",
      "############# Epoch 23  Done   #############\n",
      "\n",
      "--- 370.2359969615936 seconds ---\n",
      "############# Epoch 24: Training Start   #############\n",
      "############# Epoch 24: Training End     #############\n",
      "############# Epoch 24: Validation Start   #############\n",
      "############# Epoch 24: Validation End     #############\n",
      "Epoch: 24 \tAvgerage Training Loss: 0.000119 \tAverage Validation Loss: 0.055286\n",
      "############# Epoch 24  Done   #############\n",
      "\n",
      "--- 384.23689818382263 seconds ---\n",
      "############# Epoch 25: Training Start   #############\n",
      "############# Epoch 25: Training End     #############\n",
      "############# Epoch 25: Validation Start   #############\n",
      "############# Epoch 25: Validation End     #############\n",
      "Epoch: 25 \tAvgerage Training Loss: 0.000138 \tAverage Validation Loss: 0.053318\n",
      "############# Epoch 25  Done   #############\n",
      "\n",
      "--- 400.76782059669495 seconds ---\n",
      "############# Epoch 26: Training Start   #############\n",
      "############# Epoch 26: Training End     #############\n",
      "############# Epoch 26: Validation Start   #############\n",
      "############# Epoch 26: Validation End     #############\n",
      "Epoch: 26 \tAvgerage Training Loss: 0.000123 \tAverage Validation Loss: 0.052631\n",
      "############# Epoch 26  Done   #############\n",
      "\n",
      "--- 414.9568512439728 seconds ---\n",
      "############# Epoch 27: Training Start   #############\n",
      "############# Epoch 27: Training End     #############\n",
      "############# Epoch 27: Validation Start   #############\n",
      "############# Epoch 27: Validation End     #############\n",
      "Epoch: 27 \tAvgerage Training Loss: 0.000100 \tAverage Validation Loss: 0.053279\n",
      "############# Epoch 27  Done   #############\n",
      "\n",
      "--- 432.5208704471588 seconds ---\n",
      "############# Epoch 28: Training Start   #############\n",
      "############# Epoch 28: Training End     #############\n",
      "############# Epoch 28: Validation Start   #############\n",
      "############# Epoch 28: Validation End     #############\n",
      "Epoch: 28 \tAvgerage Training Loss: 0.000084 \tAverage Validation Loss: 0.049366\n",
      "############# Epoch 28  Done   #############\n",
      "\n",
      "--- 447.1721568107605 seconds ---\n",
      "############# Epoch 29: Training Start   #############\n",
      "############# Epoch 29: Training End     #############\n",
      "############# Epoch 29: Validation Start   #############\n",
      "############# Epoch 29: Validation End     #############\n",
      "Epoch: 29 \tAvgerage Training Loss: 0.000087 \tAverage Validation Loss: 0.054992\n",
      "############# Epoch 29  Done   #############\n",
      "\n",
      "--- 464.72635555267334 seconds ---\n",
      "############# Epoch 30: Training Start   #############\n",
      "############# Epoch 30: Training End     #############\n",
      "############# Epoch 30: Validation Start   #############\n",
      "############# Epoch 30: Validation End     #############\n",
      "Epoch: 30 \tAvgerage Training Loss: 0.000067 \tAverage Validation Loss: 0.052766\n",
      "############# Epoch 30  Done   #############\n",
      "\n",
      "--- 479.774671792984 seconds ---\n",
      "############# Epoch 31: Training Start   #############\n",
      "############# Epoch 31: Training End     #############\n",
      "############# Epoch 31: Validation Start   #############\n",
      "############# Epoch 31: Validation End     #############\n",
      "Epoch: 31 \tAvgerage Training Loss: 0.000043 \tAverage Validation Loss: 0.053639\n",
      "############# Epoch 31  Done   #############\n",
      "\n",
      "--- 497.5685546398163 seconds ---\n",
      "############# Epoch 32: Training Start   #############\n",
      "############# Epoch 32: Training End     #############\n",
      "############# Epoch 32: Validation Start   #############\n",
      "############# Epoch 32: Validation End     #############\n",
      "Epoch: 32 \tAvgerage Training Loss: 0.000041 \tAverage Validation Loss: 0.053644\n",
      "############# Epoch 32  Done   #############\n",
      "\n",
      "--- 514.0897705554962 seconds ---\n",
      "############# Epoch 33: Training Start   #############\n",
      "############# Epoch 33: Training End     #############\n",
      "############# Epoch 33: Validation Start   #############\n",
      "############# Epoch 33: Validation End     #############\n",
      "Epoch: 33 \tAvgerage Training Loss: 0.000037 \tAverage Validation Loss: 0.054224\n",
      "############# Epoch 33  Done   #############\n",
      "\n",
      "--- 528.0974349975586 seconds ---\n",
      "############# Epoch 34: Training Start   #############\n",
      "############# Epoch 34: Training End     #############\n",
      "############# Epoch 34: Validation Start   #############\n",
      "############# Epoch 34: Validation End     #############\n",
      "Epoch: 34 \tAvgerage Training Loss: 0.000034 \tAverage Validation Loss: 0.055026\n",
      "############# Epoch 34  Done   #############\n",
      "\n",
      "--- 542.217903137207 seconds ---\n",
      "############# Epoch 35: Training Start   #############\n",
      "############# Epoch 35: Training End     #############\n",
      "############# Epoch 35: Validation Start   #############\n",
      "############# Epoch 35: Validation End     #############\n",
      "Epoch: 35 \tAvgerage Training Loss: 0.000031 \tAverage Validation Loss: 0.055375\n",
      "############# Epoch 35  Done   #############\n",
      "\n",
      "--- 556.6109871864319 seconds ---\n",
      "############# Epoch 36: Training Start   #############\n",
      "############# Epoch 36: Training End     #############\n",
      "############# Epoch 36: Validation Start   #############\n",
      "############# Epoch 36: Validation End     #############\n",
      "Epoch: 36 \tAvgerage Training Loss: 0.000147 \tAverage Validation Loss: 0.052942\n",
      "############# Epoch 36  Done   #############\n",
      "\n",
      "--- 573.9799456596375 seconds ---\n",
      "############# Epoch 37: Training Start   #############\n",
      "############# Epoch 37: Training End     #############\n",
      "############# Epoch 37: Validation Start   #############\n",
      "############# Epoch 37: Validation End     #############\n",
      "Epoch: 37 \tAvgerage Training Loss: 0.000072 \tAverage Validation Loss: 0.054894\n",
      "############# Epoch 37  Done   #############\n",
      "\n",
      "--- 588.8822643756866 seconds ---\n",
      "############# Epoch 38: Training Start   #############\n",
      "############# Epoch 38: Training End     #############\n",
      "############# Epoch 38: Validation Start   #############\n",
      "############# Epoch 38: Validation End     #############\n",
      "Epoch: 38 \tAvgerage Training Loss: 0.000190 \tAverage Validation Loss: 0.053327\n",
      "############# Epoch 38  Done   #############\n",
      "\n",
      "--- 605.3597424030304 seconds ---\n",
      "############# Epoch 39: Training Start   #############\n",
      "############# Epoch 39: Training End     #############\n",
      "############# Epoch 39: Validation Start   #############\n",
      "############# Epoch 39: Validation End     #############\n",
      "Epoch: 39 \tAvgerage Training Loss: 0.000059 \tAverage Validation Loss: 0.053846\n",
      "############# Epoch 39  Done   #############\n",
      "\n",
      "--- 622.4938900470734 seconds ---\n",
      "############# Epoch 40: Training Start   #############\n",
      "############# Epoch 40: Training End     #############\n",
      "############# Epoch 40: Validation Start   #############\n",
      "############# Epoch 40: Validation End     #############\n",
      "Epoch: 40 \tAvgerage Training Loss: 0.000034 \tAverage Validation Loss: 0.056379\n",
      "############# Epoch 40  Done   #############\n",
      "\n",
      "--- 636.7268960475922 seconds ---\n",
      "############# Epoch 41: Training Start   #############\n",
      "############# Epoch 41: Training End     #############\n",
      "############# Epoch 41: Validation Start   #############\n",
      "############# Epoch 41: Validation End     #############\n",
      "Epoch: 41 \tAvgerage Training Loss: 0.000031 \tAverage Validation Loss: 0.058556\n",
      "############# Epoch 41  Done   #############\n",
      "\n",
      "--- 653.8461198806763 seconds ---\n",
      "############# Epoch 42: Training Start   #############\n",
      "############# Epoch 42: Training End     #############\n",
      "############# Epoch 42: Validation Start   #############\n",
      "############# Epoch 42: Validation End     #############\n",
      "Epoch: 42 \tAvgerage Training Loss: 0.000064 \tAverage Validation Loss: 0.062270\n",
      "############# Epoch 42  Done   #############\n",
      "\n",
      "--- 668.0496499538422 seconds ---\n",
      "############# Epoch 43: Training Start   #############\n",
      "############# Epoch 43: Training End     #############\n",
      "############# Epoch 43: Validation Start   #############\n",
      "############# Epoch 43: Validation End     #############\n",
      "Epoch: 43 \tAvgerage Training Loss: 0.000121 \tAverage Validation Loss: 0.055837\n",
      "############# Epoch 43  Done   #############\n",
      "\n",
      "--- 684.9286789894104 seconds ---\n",
      "############# Epoch 44: Training Start   #############\n",
      "############# Epoch 44: Training End     #############\n",
      "############# Epoch 44: Validation Start   #############\n",
      "############# Epoch 44: Validation End     #############\n",
      "Epoch: 44 \tAvgerage Training Loss: 0.000224 \tAverage Validation Loss: 0.068515\n",
      "############# Epoch 44  Done   #############\n",
      "\n",
      "--- 698.5382704734802 seconds ---\n",
      "############# Epoch 45: Training Start   #############\n",
      "############# Epoch 45: Training End     #############\n",
      "############# Epoch 45: Validation Start   #############\n",
      "############# Epoch 45: Validation End     #############\n",
      "Epoch: 45 \tAvgerage Training Loss: 0.000107 \tAverage Validation Loss: 0.055233\n",
      "############# Epoch 45  Done   #############\n",
      "\n",
      "--- 715.2755837440491 seconds ---\n",
      "############# Epoch 46: Training Start   #############\n",
      "############# Epoch 46: Training End     #############\n",
      "############# Epoch 46: Validation Start   #############\n",
      "############# Epoch 46: Validation End     #############\n",
      "Epoch: 46 \tAvgerage Training Loss: 0.000042 \tAverage Validation Loss: 0.055007\n",
      "############# Epoch 46  Done   #############\n",
      "\n",
      "--- 729.7411012649536 seconds ---\n",
      "############# Epoch 47: Training Start   #############\n",
      "############# Epoch 47: Training End     #############\n",
      "############# Epoch 47: Validation Start   #############\n",
      "############# Epoch 47: Validation End     #############\n",
      "Epoch: 47 \tAvgerage Training Loss: 0.000023 \tAverage Validation Loss: 0.057160\n",
      "############# Epoch 47  Done   #############\n",
      "\n",
      "--- 747.3138780593872 seconds ---\n",
      "############# Epoch 48: Training Start   #############\n",
      "############# Epoch 48: Training End     #############\n",
      "############# Epoch 48: Validation Start   #############\n",
      "############# Epoch 48: Validation End     #############\n",
      "Epoch: 48 \tAvgerage Training Loss: 0.000021 \tAverage Validation Loss: 0.058142\n",
      "############# Epoch 48  Done   #############\n",
      "\n",
      "--- 764.1597199440002 seconds ---\n",
      "############# Epoch 49: Training Start   #############\n",
      "############# Epoch 49: Training End     #############\n",
      "############# Epoch 49: Validation Start   #############\n",
      "############# Epoch 49: Validation End     #############\n",
      "Epoch: 49 \tAvgerage Training Loss: 0.000021 \tAverage Validation Loss: 0.058933\n",
      "############# Epoch 49  Done   #############\n",
      "\n",
      "--- 779.2954111099243 seconds ---\n",
      "############# Epoch 50: Training Start   #############\n",
      "############# Epoch 50: Training End     #############\n",
      "############# Epoch 50: Validation Start   #############\n",
      "############# Epoch 50: Validation End     #############\n",
      "Epoch: 50 \tAvgerage Training Loss: 0.000133 \tAverage Validation Loss: 0.067276\n",
      "############# Epoch 50  Done   #############\n",
      "\n",
      "--- 795.4065163135529 seconds ---\n"
     ]
    }
   ],
   "source": [
    "import transformers\n",
    "from transformers import BertModel, BertTokenizer, AdamW, get_linear_schedule_with_warmup\n",
    "import torch\n",
    "from torch import nn, optim\n",
    "from torch.utils.data import Dataset, DataLoader\n",
    "import torch.nn.functional as F\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "from pylab import rcParams\n",
    "import matplotlib.pyplot as plt\n",
    "from matplotlib import rc\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "from sklearn.metrics import confusion_matrix, classification_report\n",
    "from collections import defaultdict\n",
    "from textwrap import wrap\n",
    "from tqdm import tqdm\n",
    "\n",
    "from stance_utils import *\n",
    "%matplotlib inline\n",
    "%config InlineBackend.figure_format='retina'\n",
    "\n",
    "sns.set(style='whitegrid', palette='muted', font_scale=1.2)\n",
    "HAPPY_COLORS_PALETTE = [\"#01BEFE\", \"#FFDD00\", \"#FF7D00\", \"#FF006D\", \"#ADFF02\", \"#8F00FF\"]\n",
    "\n",
    "sns.set_palette(sns.color_palette(HAPPY_COLORS_PALETTE))\n",
    "rcParams['figure.figsize'] = 12, 8\n",
    "\n",
    "RANDOM_SEED = 23\n",
    "np.random.seed(RANDOM_SEED)\n",
    "\n",
    "torch.manual_seed(RANDOM_SEED)\n",
    "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
    "import time\n",
    "start_time = time.time()\n",
    "\n",
    "def train_and_test(train_file, test_file, target):\n",
    "    \n",
    "    sentence_maxlen = 0\n",
    "    x_train = []\n",
    "    y_train = []\n",
    "\n",
    "    \n",
    "    with open(train_file, 'r') as trainfile:\n",
    "        for line in trainfile:\n",
    "            \n",
    "            line = line.replace('#SemST', '').strip()\n",
    "            line = line.split('\\t')\n",
    "            \n",
    "            #if line[0].strip() != 'ID':\n",
    "            if line[0].strip() != 'ID' and target not in line[1].strip():\n",
    "                tweet = line[2]\n",
    "                #tweet = process_tweet(tweet)\n",
    "                if len(tweet) > sentence_maxlen:\n",
    "                    sentence_maxlen = len(tweet)\n",
    "                x_train.append(tweet)\n",
    "                y_train.append(classes[line[3].strip()])\n",
    "                \n",
    "\n",
    "    pp = len(x_train)\n",
    "    x_test = []\n",
    "    y_test = []\n",
    "    with open(test_file, 'r') as testfile:\n",
    "        for line in testfile:\n",
    "            line = line.replace('#SemST', '').strip()\n",
    "            line = line.split('\\t')\n",
    "        \n",
    "\n",
    "            #if line[0].strip() != 'ID':\n",
    "            if line[0] != 'ID' and target not in line[1].strip():\n",
    "                tweet = line[2]\n",
    "                #tweet = process_tweet(tweet)\n",
    "                if len(tweet) > sentence_maxlen:\n",
    "                    sentence_maxlen = len(tweet)\n",
    "                x_test.append(tweet)\n",
    "                y_test.append(classes[line[3].strip()])\n",
    "\n",
    "\n",
    "    \n",
    "    return x_train, y_train, x_test, y_test, sentence_maxlen,pp\n",
    "\n",
    "\n",
    "\n",
    "classes = {'FAVOR': np.array([1, 0, 0]), 'AGAINST': np.array([0, 1, 0]), 'NONE': np.array([0, 0, 1])}\n",
    "classes_ = np.array(['FAVOR', 'AGAINST', 'NONE'])\n",
    "# classes = {'FAVOR': np.array([1, 0]), 'AGAINST': np.array([0, 1])}\n",
    "# classes_ = np.array(['FAVOR', 'AGAINST'])\n",
    "\n",
    "\n",
    "train_data_file = '/data/parush/stance_mohammed/new_train.txt'\n",
    "test_data_file = '/data/parush/stance_mohammed/new_test.txt'\n",
    "TARGETS = [ 'Atheism','Climate Change is a Real Concern', 'Feminist Movement','Hillary Clinton', 'Legalization of Abortion', 'Donald Trump']\n",
    "\n",
    "\n",
    "# train_data_file = '/data/parush/SomasundaranWiebe-politicalDebates/train.txt'\n",
    "# test_data_file = '/data/parush/SomasundaranWiebe-politicalDebates/test.txt'\n",
    "# TARGETS = ['god','healthcare','guns','gayRights','abortion', 'creation']\n",
    "\n",
    "\n",
    "# train_data_file = '/data/parush/Data_MPCHI/train.txt'\n",
    "# test_data_file = '/data/parush/Data_MPCHI/test.txt'\n",
    "# TARGETS = ['Are E-Cigarettes safe?','Does MMR Vaccine lead to autism in children?',\n",
    "#       'Does Sunlight exposure lead to skin cancer?','Does Vitamin C prevent common cold?',\n",
    "#       'Should women take HRT post-menopause?']\n",
    "\n",
    "# In[273]:\n",
    "\n",
    "stance_target = TARGETS[5]\n",
    "train_texts, train_labels, test_texts, test_labels, sen_maxlen, pp = train_and_test(train_data_file, test_data_file, stance_target)\n",
    "\n",
    "\n",
    "df2 = pd.DataFrame([])\n",
    "df2['TITLE'] = train_texts\n",
    "df2['target_list'] = train_labels\n",
    "df_test = pd.DataFrame([])\n",
    "df_test['TITLE'] = test_texts\n",
    "df_test['target_list'] = test_labels\n",
    "MAX_LEN = 128\n",
    "TRAIN_BATCH_SIZE = 32\n",
    "VALID_BATCH_SIZE = 32\n",
    "EPOCHS = 50\n",
    "LEARNING_RATE = 1e-05\n",
    "tokenizer = BertTokenizer.from_pretrained('bert-base-uncased', do_lower_case=True)\n",
    "class CustomDataset(Dataset):\n",
    "    \n",
    "    def __init__(self, dataframe, tokenizer, max_len):\n",
    "        self.tokenizer = tokenizer\n",
    "        self.data = dataframe\n",
    "        self.title = dataframe['TITLE']\n",
    "        self.targets = self.data.target_list\n",
    "        self.max_len = max_len\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.title)\n",
    "\n",
    "    def __getitem__(self, index):\n",
    "        title = str(self.title[index])\n",
    "        title = \" \".join(title.split())\n",
    "\n",
    "        inputs = self.tokenizer.encode_plus(\n",
    "            title,\n",
    "            None,\n",
    "            add_special_tokens=True,\n",
    "            max_length=self.max_len,\n",
    "            padding='max_length',\n",
    "            return_token_type_ids=True,\n",
    "            truncation=True\n",
    "        )\n",
    "        ids = inputs['input_ids']\n",
    "        mask = inputs['attention_mask']\n",
    "        token_type_ids = inputs[\"token_type_ids\"]\n",
    "\n",
    "\n",
    "        return {\n",
    "            'ids': torch.tensor(ids, dtype=torch.long),\n",
    "            'mask': torch.tensor(mask, dtype=torch.long),\n",
    "            'token_type_ids': torch.tensor(token_type_ids, dtype=torch.long),\n",
    "            'targets': torch.tensor(self.targets[index], dtype=torch.float)\n",
    "        }\n",
    "train_size = 0.8\n",
    "train_dataset = df2.sample(frac=train_size,random_state=200)\n",
    "valid_dataset = df2.drop(train_dataset.index).reset_index(drop=True)\n",
    "train_dataset = train_dataset.reset_index(drop=True)\n",
    "\n",
    "\n",
    "print(\"FULL Dataset: {}\".format(df2.shape))\n",
    "print(\"TRAIN Dataset: {}\".format(train_dataset.shape))\n",
    "print(\"TEST Dataset: {}\".format(valid_dataset.shape))\n",
    "training_set = CustomDataset(train_dataset, tokenizer, MAX_LEN)\n",
    "validation_set = CustomDataset(valid_dataset,  tokenizer, MAX_LEN)\n",
    "train_params = {'batch_size': TRAIN_BATCH_SIZE,\n",
    "                'shuffle': True,\n",
    "                'num_workers': 0\n",
    "                }\n",
    "\n",
    "test_params = {'batch_size': VALID_BATCH_SIZE,\n",
    "                'shuffle': False,\n",
    "                'num_workers': 0\n",
    "                }\n",
    "\n",
    "training_loader = DataLoader(training_set, **train_params)\n",
    "validation_loader = DataLoader(validation_set, **test_params)\n",
    "class BERTClass(torch.nn.Module):\n",
    "    def __init__(self):\n",
    "        super(BERTClass, self).__init__()\n",
    "        self.l1 = transformers.BertModel.from_pretrained('bert-base-uncased')\n",
    "        self.l2 = torch.nn.Dropout(0.3)\n",
    "        self.l3 = torch.nn.Linear(768, 3)\n",
    "    \n",
    "    def forward(self, ids, mask, token_type_ids):\n",
    "        _, output_1= self.l1(ids, attention_mask = mask, token_type_ids = token_type_ids,return_dict=False)\n",
    "        output_2 = self.l2(output_1)\n",
    "        output = self.l3(output_2)\n",
    "        return output\n",
    "\n",
    "model = BERTClass()\n",
    "model.to(device)\n",
    "\n",
    "def loss_fn(outputs, targets):\n",
    "    return torch.nn.BCEWithLogitsLoss()(outputs, targets)\n",
    "\n",
    "optimizer = torch.optim.Adam(params =  model.parameters(), lr=LEARNING_RATE)\n",
    "def load_ckp(checkpoint_fpath, model, optimizer):\n",
    "    \"\"\"\n",
    "    checkpoint_path: path to save checkpoint\n",
    "    model: model that we want to load checkpoint parameters into       \n",
    "    optimizer: optimizer we defined in previous training\n",
    "    \"\"\"\n",
    "    # load check point\n",
    "    checkpoint = torch.load(checkpoint_fpath)\n",
    "    # initialize state_dict from checkpoint to model\n",
    "    model.load_state_dict(checkpoint['state_dict'])\n",
    "    # initialize optimizer from checkpoint to optimizer\n",
    "    optimizer.load_state_dict(checkpoint['optimizer'])\n",
    "    # initialize valid_loss_min from checkpoint to valid_loss_min\n",
    "    valid_loss_min = checkpoint['valid_loss_min']\n",
    "    # return model, optimizer, epoch value, min validation loss \n",
    "    return model, optimizer, checkpoint['epoch'], valid_loss_min.item()\n",
    "import shutil, sys   \n",
    "def save_ckp(state, is_best, checkpoint_path, best_model_path):\n",
    "    \"\"\"\n",
    "    state: checkpoint we want to save\n",
    "    is_best: is this the best checkpoint; min validation loss\n",
    "    checkpoint_path: path to save checkpoint\n",
    "    best_model_path: path to save best model\n",
    "    \"\"\"\n",
    "    f_path = checkpoint_path\n",
    "    # save checkpoint data to the path given, checkpoint_path\n",
    "    torch.save(state, f_path)\n",
    "    # if it is a best model, min validation loss\n",
    "    if is_best:\n",
    "        best_fpath = best_model_path\n",
    "        # copy that checkpoint file to best path given, best_model_path\n",
    "        shutil.copyfile(f_path, best_fpath)\n",
    "        \n",
    "def train_model(start_epochs,  n_epochs, valid_loss_min_input, \n",
    "                training_loader, validation_loader, model, \n",
    "                optimizer, checkpoint_path, best_model_path):\n",
    "   \n",
    "  # initialize tracker for minimum validation loss\n",
    "    valid_loss_min = valid_loss_min_input \n",
    "    for epoch in range(start_epochs, n_epochs+1):\n",
    "        train_loss = 0\n",
    "        valid_loss = 0\n",
    "\n",
    "        model.train()\n",
    "        print('############# Epoch {}: Training Start   #############'.format(epoch))\n",
    "        for batch_idx, data in enumerate(training_loader):\n",
    "            #print('yyy epoch', batch_idx)\n",
    "            ids = data['ids'].to(device, dtype = torch.long)\n",
    "            mask = data['mask'].to(device, dtype = torch.long)\n",
    "            token_type_ids = data['token_type_ids'].to(device, dtype = torch.long)\n",
    "            targets = data['targets'].to(device, dtype = torch.float)\n",
    "\n",
    "            outputs = model(ids, mask, token_type_ids)\n",
    "            \n",
    "            optimizer.zero_grad()\n",
    "            #print(targets)\n",
    "            loss = loss_fn(outputs, targets)\n",
    "            #if batch_idx%5000==0:\n",
    "             #   print(f'Epoch: {epoch}, Training Loss:  {loss.item()}')\n",
    "            \n",
    "            optimizer.zero_grad()\n",
    "            loss.backward()\n",
    "            optimizer.step()\n",
    "            #print('before loss data in training', loss.item(), train_loss)\n",
    "            train_loss = train_loss + ((1 / (batch_idx + 1)) * (loss.item() - train_loss))\n",
    "            #print('after loss data in training', loss.item(), train_loss)\n",
    "\n",
    "        print('############# Epoch {}: Training End     #############'.format(epoch))\n",
    "\n",
    "        print('############# Epoch {}: Validation Start   #############'.format(epoch))\n",
    "        ######################    \n",
    "        # validate the model #\n",
    "        ######################\n",
    "\n",
    "        model.eval()\n",
    "        val_targets, val_outputs = [] , []\n",
    "        with torch.no_grad():\n",
    "            for batch_idx, data in enumerate(validation_loader, 0):\n",
    "                ids = data['ids'].to(device, dtype = torch.long)\n",
    "                mask = data['mask'].to(device, dtype = torch.long)\n",
    "                token_type_ids = data['token_type_ids'].to(device, dtype = torch.long)\n",
    "                targets = data['targets'].to(device, dtype = torch.float)\n",
    "                outputs = model(ids, mask, token_type_ids)\n",
    "\n",
    "                loss = loss_fn(outputs, targets)\n",
    "                valid_loss = valid_loss + ((1 / (batch_idx + 1)) * (loss.item() - valid_loss))                \n",
    "                \n",
    "                val_targets.extend(targets.cpu().detach().numpy().tolist())\n",
    "                val_outputs.extend(torch.sigmoid(outputs).cpu().detach().numpy().tolist())\n",
    "            print('############# Epoch {}: Validation End     #############'.format(epoch))\n",
    "            # calculate average losses\n",
    "            #print('before cal avg train loss', train_loss)\n",
    "            train_loss = train_loss/len(training_loader)\n",
    "            valid_loss = valid_loss/len(validation_loader)\n",
    "            # print training/validation statistics \n",
    "            print('Epoch: {} \\tAvgerage Training Loss: {:.6f} \\tAverage Validation Loss: {:.6f}'.format(\n",
    "                epoch, \n",
    "                train_loss,\n",
    "                valid_loss\n",
    "                ))\n",
    "            # create checkpoint variable and add important data\n",
    "            checkpoint = {\n",
    "                'epoch': epoch + 1,\n",
    "                'valid_loss_min': valid_loss,\n",
    "                'state_dict': model.state_dict(),\n",
    "                'optimizer': optimizer.state_dict()\n",
    "            }\n",
    "\n",
    "            # save checkpoint\n",
    "            save_ckp(checkpoint, False, checkpoint_path, best_model_path)\n",
    "            ## TODO: save the model if validation loss has decreased\n",
    "            if valid_loss <= valid_loss_min:\n",
    "                print('Validation loss decreased ({:.6f} --> {:.6f}).  Saving model ...'.format(valid_loss_min,valid_loss))\n",
    "                # save checkpoint as best model\n",
    "                save_ckp(checkpoint, True, checkpoint_path, best_model_path)\n",
    "                valid_loss_min = valid_loss\n",
    "\n",
    "        print('############# Epoch {}  Done   #############\\n'.format(epoch))\n",
    "        print(\"--- %s seconds ---\" % (time.time() - start_time))\n",
    "    \n",
    "    return model,val_targets, val_outputs\n",
    "\n",
    "checkpoint_path = '/home/p/parush/comparison/current_checkpoint_'+ 'overall_semeval'+'.pt'\n",
    "best_model = '/home/p/parush/comparison/best_model_trained_'+ 'overall_semeval' +'.pt'\n",
    "# checkpoint_path = '/home/p/parush/comparison/current_checkpoint_'+ stance_target+'.pt'\n",
    "# best_model = '/home/p/parush/comparison/best_model_trained_'+ stance_target +'.pt'\n",
    "trained_model,val_targets, val_outputs = train_model(1, EPOCHS, np.Inf, training_loader, validation_loader, model, \n",
    "                      optimizer,checkpoint_path,best_model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8abdf725",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e68af5eb-e441-4ff9-ba37-292d9c64f4a0",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cdc86008-d2c3-4a74-a1c4-19a113421633",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "4fe14356",
   "metadata": {},
   "outputs": [],
   "source": [
    "# val_preds = (np.array(val_outputs) > 0.5).astype(int)\n",
    "\n",
    "# print(classification_report(val_targets, val_preds,labels=[0,1],digits = 4))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a5e07682",
   "metadata": {},
   "outputs": [],
   "source": [
    "# final_checkpoint_path = '/home/p/parush/comparison/final_current_checkpoint_ideo_'+ TARGETS[5]+'.pt'\n",
    "# final_model_path = '/home/p/parush/comparison/final_model_trained_ideo_'+ TARGETS[5] +'.pt'\n",
    "\n",
    "# checkpoint = {\n",
    "#                 'epoch': 10 + 1,\n",
    "#                 'valid_loss_min': 2,\n",
    "#                 'state_dict': model.state_dict(),\n",
    "#                 'optimizer': optimizer.state_dict()\n",
    "#             }\n",
    "\n",
    "#             # save checkpoint\n",
    "# save_ckp(checkpoint, False, checkpoint_path, final_model_path)\n",
    "#             ## TODO: save the model if validation loss has decreased\n",
    "            \n",
    "# #print('Validation loss decreased ({:.6f} --> {:.6f}).  Saving model ...'.format(valid_loss_min,valid_loss))\n",
    "#                 # save checkpoint as best model\n",
    "# save_ckp(checkpoint, True, final_checkpoint_path, final_model_path)\n",
    "# #valid_loss_min = valid_loss\n",
    "# save_ckp(checkpoint, True, final_checkpoint_path, final_model_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "b64fc9a9",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "13696437",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "b377ce52",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e1732066-f056-4de7-88b6-de62fbba50c7",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ed968052-5a3f-4d54-8708-fa3ca0baff60",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6387baeb",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d415c695",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2572333c",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8010ac26",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c312e63d",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c8835f33",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e53f2f99-2038-4860-ad33-6292e1b12c6c",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4d7b2fda",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "64a6bd3c",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0a2c231c",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "79ae560b-82f7-4632-a2d0-c0ff037b2655",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "31ee07de",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "f2b6d1a8",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "6f1b0726",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e49dcc84-7f6f-464d-a7b0-653877ff4053",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "raw",
   "id": "ba37a6c6-a36a-413e-9605-f77edf85ac39",
   "metadata": {},
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "85494eb1",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "d6d1dd18",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "89a4e22e",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "43281ffd",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ff9e2147",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a02122ed",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "783c0a49",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8ac80ce4",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7b80290b",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f8b08efe",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "43305844",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d1477b03",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7c78c285",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "44db7497",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c4f10ef5",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "765e4208",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8ace94e8",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5b4ba7a8",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "139bb283",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9ef4fb45",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0f63d12a",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f28ed6a2",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5b26195f",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0e44f3df",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7cf5cbdf",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "40197fd6",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "11a2d7a3",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "74266b83",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f856c1d9",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "93224e89",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c259bc10",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "81a03ace",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f220bd05",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9cdb22a3",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9b27ef23",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "41e3b589",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ee8bed59",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6bd5cb30",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c3323d73",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "407914f3",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1577a3be",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3a6b1724",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "56c3f021",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6150724b",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "78167564",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bb61afe7",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "791ce674",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3164884d",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f7fba559",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "nlpKernel",
   "language": "python",
   "name": "nlpkernel"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}