SVM_compareFocus_3turn_glove.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 1.Import Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pickle\n",
    "import sklearn\n",
    "from tqdm import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_name={\n",
    "    \"y_train\":\"train_y\",\n",
    "    \"y_test\":\"test_y\"\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def open_pkl(file_name):\n",
    "    all_data=dict()\n",
    "    for each_key in file_name.keys():\n",
    "        name = file_name[each_key]\n",
    "        pickle_file = open('DataSet\\\\'+name+\".pkl\",mode=\"rb\")\n",
    "        data = pickle.load(pickle_file)\n",
    "        pickle_file.close()\n",
    "        all_data[each_key]=data\n",
    "    return all_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_data = open_pkl(y_name)\n",
    "y_train = y_data[\"y_train\"]\n",
    "y_test = y_data[\"y_test\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 3, 0, ..., 0, 0, 0])"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "file_name0={\n",
    "    \"3turn\":\"vec_train_3_no_cor_glove_change\",\n",
    "    }\n",
    "file_name1={\n",
    "    \"3turn\":\"vec_test_3_no_cor_glove_change\",\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "#import data\n",
    "def open_pkl_data(file_name):\n",
    "    all_data=dict()\n",
    "    for each_key in file_name.keys():\n",
    "        name = file_name[each_key]\n",
    "        pickle_file = open('DataSet\\\\'+name+\".pkl\",mode=\"rb\")\n",
    "        data = pickle.load(pickle_file)\n",
    "        pickle_file.close()\n",
    "        all_data[each_key]=data\n",
    "    return all_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data_overall = open_pkl_data(file_name0)\n",
    "test_data_overall = open_pkl_data(file_name1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "original_train = train_data_overall[\"3turn\"]\n",
    "original_test = test_data_overall[\"3turn\"]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 2. Calculation of the average vectors of each sentence"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "300\n"
     ]
    }
   ],
   "source": [
    "length = len(original_train[\"overall\"][0][0])\n",
    "print(length)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def cal_average(all_data):\n",
    "    output_dict=dict()\n",
    "    for key in all_data.keys():\n",
    "        sentences = list()\n",
    "        for sentence in all_data[key]:\n",
    "            sentence = np.array(sentence)\n",
    "            length = len(sentence)\n",
    "            sum_sen = sentence.sum(axis=0)\n",
    "            aver_sen = sum_sen/length\n",
    "            sentences.append(aver_sen)\n",
    "        output_dict[key]=sentences\n",
    "    \n",
    "    return output_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "ave_train = cal_average(original_train)\n",
    "ave_test = cal_average(original_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "300\n"
     ]
    }
   ],
   "source": [
    "length = len(ave_train[\"overall\"][0])\n",
    "print(length)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "30160"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(ave_train[\"overall\"])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 3.SVM Model\n",
    "## 3.1. build and fit model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.svm import SVC\n",
    "def svm_model(train_data,y_train):\n",
    "    classifiers = dict()\n",
    "    for key in tqdm(train_data.keys()):\n",
    "        classifier = SVC(kernel = 'linear', random_state=0)\n",
    "        classifier.fit(train_data[key],y_train)\n",
    "        classifiers[key]=classifier\n",
    "    \n",
    "    return classifiers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████████████| 3/3 [14:20<00:00, 286.85s/it]\n"
     ]
    }
   ],
   "source": [
    "classifiers = svm_model(ave_train,y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# save model\n",
    "def save_data(name,data):\n",
    "    pickle_file = open('DataSet\\\\'+name+\".pkl\",mode='wb')\n",
    "    pickle.dump(data,pickle_file)\n",
    "    pickle_file.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "# save model\n",
    "save_data(\"SVM_comparison_glove\",classifiers)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3.2. prediction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "# function for getting prediction\n",
    "def get_prediction(classifiers, ave_test):\n",
    "    y_pred_dict= dict()\n",
    "    list1 = list(classifiers.keys())\n",
    "    list2 = list(ave_test.keys())\n",
    "    length = len(list1)\n",
    "    \n",
    "    for i in tqdm(range(length)):\n",
    "        y_pred_dict[list2[i]]=classifiers[list1[i]].predict(ave_test[list2[i]])\n",
    "        \n",
    "    return y_pred_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████████████| 3/3 [01:22<00:00, 27.60s/it]\n"
     ]
    }
   ],
   "source": [
    "y_preds = get_prediction(classifiers,ave_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "# function for getting the score of each prediction\n",
    "def get_scores(classifiers, ave_test):\n",
    "    y_score_dict= dict()\n",
    "    list1 = list(classifiers.keys())\n",
    "    list2 = list(ave_test.keys())\n",
    "    length = len(list1)\n",
    "    \n",
    "    for i in tqdm(range(length)):\n",
    "        y_score_dict[list2[i]]=classifiers[list1[i]].decision_function(ave_test[list2[i]])\n",
    "        \n",
    "    return y_score_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████████████| 3/3 [01:21<00:00, 27.31s/it]\n"
     ]
    }
   ],
   "source": [
    "y_scores = get_scores(classifiers, ave_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "# function for saving the predictions and scores\n",
    "def save_prediction(name,data):\n",
    "    pickle_file = open('Prediction\\\\'+name+\".pkl\",mode='wb')\n",
    "    pickle.dump(data,pickle_file)\n",
    "    pickle_file.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "save_prediction(\"SVM_process3turn_glove_preds\",y_preds)\n",
    "save_prediction(\"SVM_process3turn_glove_scores\",y_scores)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3.3. Evaluation: confusion matrix, accuracy, precision, recall, F1 measure"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import confusion_matrix\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.metrics import precision_score\n",
    "from sklearn.metrics import recall_score\n",
    "from sklearn.metrics import f1_score\n",
    "# print evaluation result\n",
    "def get_evaluation(y_test,y_pred):\n",
    "    ev = dict()\n",
    "    ev[\"confusion_matrix\"]=confusion_matrix(y_test,y_pred)\n",
    "    ev[\"accuracy\"] = accuracy_score(y_test,y_pred)\n",
    "    ev[\"precision_macro\"]=precision_score(y_test,y_pred,average=\"macro\")\n",
    "    ev[\"precision_micro\"]=precision_score(y_test,y_pred,average=\"micro\")\n",
    "    ev[\"precision_weighted\"]=precision_score(y_test,y_pred,average=\"weighted\")\n",
    "    ev[\"recall_macro\"]=recall_score(y_test,y_pred,average=\"macro\")\n",
    "    ev[\"recall_micro\"]=recall_score(y_test,y_pred,average=\"micro\")\n",
    "    ev[\"recall_weighted\"]=recall_score(y_test,y_pred,average=\"weighted\")\n",
    "    ev[\"F1_score_macro\"]=f1_score(y_test,y_pred,average=\"macro\")\n",
    "    ev[\"F1_score_micro\"]=f1_score(y_test,y_pred,average=\"micro\")\n",
    "    ev[\"F1_score_weighted\"]=f1_score(y_test,y_pred,average=\"weighted\")\n",
    "    \n",
    "    for key in ev.keys():\n",
    "        if key !=\"confusion_matrix\":\n",
    "            print(\"{a} is: {b}\".format(a=key, b=ev[key]))\n",
    "        else:\n",
    "            print(ev[key])\n",
    "    \n",
    "    return ev\n",
    "\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "#\"macro\": \"Calculate metrics for each label, and find their unweighted mean. \",\n",
    "#\"micro\": \"Calculate metrics globally by counting the total true positives, false negatives and false positives.\"\n",
    "#\"weighted\":\"Calculate metrics for each label, and find their average weighted by support (the number of true instances for each label). \"\n",
    "\n",
    "def get_evaluation_all(y_test, y_pred_dict):\n",
    "    for key in tqdm(y_pred_dict.keys()):\n",
    "        print(\"evaluation on data {0}\".format(key))\n",
    "        get_evaluation(y_test, y_pred_dict[key])\n",
    "        print(\"***************************************\")\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████████████| 3/3 [00:00<00:00, 41.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "evaluation on data overall\n",
      "[[4035  188  226  228]\n",
      " [ 145  116   11   12]\n",
      " [  92    4  139   15]\n",
      " [  75    7   15  201]]\n",
      "accuracy is: 0.8152114721365039\n",
      "precision_macro is: 0.523192131543693\n",
      "precision_micro is: 0.8152114721365039\n",
      "precision_weighted is: 0.8470010474209272\n",
      "recall_macro is: 0.6254199673418377\n",
      "recall_micro is: 0.8152114721365039\n",
      "recall_weighted is: 0.8152114721365039\n",
      "F1_score_macro is: 0.5621119871098925\n",
      "F1_score_micro is: 0.8152114721365039\n",
      "F1_score_weighted is: 0.8277106826352397\n",
      "***************************************\n",
      "evaluation on data double13\n",
      "[[4041  197  205  234]\n",
      " [ 135  131   11    7]\n",
      " [  80    4  151   15]\n",
      " [  62    6   13  217]]\n",
      "accuracy is: 0.8241060083499727\n",
      "precision_macro is: 0.5448915251070608\n",
      "precision_micro is: 0.8241060083499727\n",
      "precision_weighted is: 0.8573421354104298\n",
      "recall_macro is: 0.6643677298951334\n",
      "recall_micro is: 0.8241060083499727\n",
      "recall_weighted is: 0.8241060083499727\n",
      "F1_score_macro is: 0.5904978570723562\n",
      "F1_score_micro is: 0.8241060083499726\n",
      "F1_score_weighted is: 0.8367207595781978\n",
      "***************************************\n",
      "evaluation on data time3_3rd\n",
      "[[4071  181  187  238]\n",
      " [ 120  148    9    7]\n",
      " [  77    5  153   15]\n",
      " [  60    7   12  219]]\n",
      "accuracy is: 0.8333635868578689\n",
      "precision_macro is: 0.5639155097171136\n",
      "precision_micro is: 0.8333635868578689\n",
      "precision_weighted is: 0.8649009132965026\n",
      "recall_macro is: 0.6846139630227045\n",
      "recall_micro is: 0.8333635868578689\n",
      "recall_weighted is: 0.8333635868578689\n",
      "F1_score_macro is: 0.6105723118623186\n",
      "F1_score_micro is: 0.8333635868578689\n",
      "F1_score_weighted is: 0.8452474529421637\n",
      "***************************************\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "get_evaluation_all(y_test,y_preds)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 4. if just want to directly load the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_names = {\n",
    "    \"model_all\":\"SVM_comparison_glove\"\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_data = open_pkl(model_names)\n",
    "classifiers = model_data[\"model_all\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}