svm-runtime.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "a78785d2-776c-4f9a-8027-11ee9d8c3516",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Working on target: god\n",
      "Working on target: god\n",
      "# Tuning hyper-parameters for precision\n",
      "\n",
      "Best parameters set found on development set:\n",
      "\n",
      "{'C': 1000, 'gamma': 0.0001, 'kernel': 'rbf'}\n",
      "\n",
      "Grid scores on development set:\n",
      "\n",
      "0.276 (+/-0.000) for {'C': 1, 'gamma': 0.001, 'kernel': 'rbf'}\n",
      "0.276 (+/-0.000) for {'C': 1, 'gamma': 0.0001, 'kernel': 'rbf'}\n",
      "0.276 (+/-0.000) for {'C': 10, 'gamma': 0.001, 'kernel': 'rbf'}\n",
      "0.276 (+/-0.000) for {'C': 10, 'gamma': 0.0001, 'kernel': 'rbf'}\n",
      "0.636 (+/-0.035) for {'C': 100, 'gamma': 0.001, 'kernel': 'rbf'}\n",
      "0.276 (+/-0.000) for {'C': 100, 'gamma': 0.0001, 'kernel': 'rbf'}\n",
      "0.613 (+/-0.000) for {'C': 1000, 'gamma': 0.001, 'kernel': 'rbf'}\n",
      "0.636 (+/-0.034) for {'C': 1000, 'gamma': 0.0001, 'kernel': 'rbf'}\n",
      "0.616 (+/-0.002) for {'C': 1, 'kernel': 'linear'}\n",
      "0.613 (+/-0.003) for {'C': 10, 'kernel': 'linear'}\n",
      "0.592 (+/-0.019) for {'C': 100, 'kernel': 'linear'}\n",
      "0.586 (+/-0.007) for {'C': 1000, 'kernel': 'linear'}\n",
      "\n",
      "Detailed classification report:\n",
      "\n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0     0.6102    0.8827    0.7216      1211\n",
      "           1     0.6350    0.2656    0.3745       930\n",
      "\n",
      "    accuracy                         0.6147      2141\n",
      "   macro avg     0.6226    0.5742    0.5480      2141\n",
      "weighted avg     0.6209    0.6147    0.5708      2141\n",
      "\n",
      "\n",
      "# Tuning hyper-parameters for recall\n",
      "\n",
      "Best parameters set found on development set:\n",
      "\n",
      "{'C': 1, 'kernel': 'linear'}\n",
      "\n",
      "Grid scores on development set:\n",
      "\n",
      "0.500 (+/-0.000) for {'C': 1, 'gamma': 0.001, 'kernel': 'rbf'}\n",
      "0.500 (+/-0.000) for {'C': 1, 'gamma': 0.0001, 'kernel': 'rbf'}\n",
      "0.500 (+/-0.000) for {'C': 10, 'gamma': 0.001, 'kernel': 'rbf'}\n",
      "0.500 (+/-0.000) for {'C': 10, 'gamma': 0.0001, 'kernel': 'rbf'}\n",
      "0.558 (+/-0.002) for {'C': 100, 'gamma': 0.001, 'kernel': 'rbf'}\n",
      "0.500 (+/-0.000) for {'C': 100, 'gamma': 0.0001, 'kernel': 'rbf'}\n",
      "0.610 (+/-0.001) for {'C': 1000, 'gamma': 0.001, 'kernel': 'rbf'}\n",
      "0.559 (+/-0.001) for {'C': 1000, 'gamma': 0.0001, 'kernel': 'rbf'}\n",
      "0.612 (+/-0.003) for {'C': 1, 'kernel': 'linear'}\n",
      "0.611 (+/-0.002) for {'C': 10, 'kernel': 'linear'}\n",
      "0.593 (+/-0.020) for {'C': 100, 'kernel': 'linear'}\n",
      "0.586 (+/-0.007) for {'C': 1000, 'kernel': 'linear'}\n",
      "\n",
      "Detailed classification report:\n",
      "\n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0     0.6568    0.7143    0.6843      1211\n",
      "           1     0.5801    0.5140    0.5450       930\n",
      "\n",
      "    accuracy                         0.6273      2141\n",
      "   macro avg     0.6184    0.6141    0.6147      2141\n",
      "weighted avg     0.6235    0.6273    0.6238      2141\n",
      "\n",
      "\n",
      "--- 6346.577107429504 seconds ---\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import re\n",
    "import string\n",
    "import numpy as np\n",
    "import nltk\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.stem import PorterStemmer\n",
    "import json\n",
    "from nltk.tokenize import TweetTokenizer\n",
    "#nltk.download('stopwords')\n",
    "#nltk.download('punkt')\n",
    "from nltk.util import ngrams\n",
    "#from google.colab import drive\n",
    "from scipy.sparse import hstack\n",
    "from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.metrics import classification_report\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.metrics import classification_report\n",
    "from sklearn.metrics import confusion_matrix\n",
    "from sklearn.metrics import accuracy_score\n",
    "from scipy import sparse\n",
    "import os\n",
    "import warnings\n",
    "from stance_utils import *\n",
    "warnings.filterwarnings('ignore')\n",
    "import time\n",
    "import time\n",
    "start_time = time.time()\n",
    "\n",
    "# train_data_file = '/data/parush/stance_mohammed/new_train.txt'\n",
    "# test_data_file = '/data/parush/stance_mohammed/new_test.txt'\n",
    "# TARGETS = [ 'Atheism','Climate Change is a Real Concern', 'Feminist Movement','Hillary Clinton', 'Legalization of Abortion', 'Donald Trump']\n",
    "\n",
    "\n",
    "train_data_file = '/data/parush/SomasundaranWiebe-politicalDebates/train.txt'\n",
    "test_data_file = '/data/parush/SomasundaranWiebe-politicalDebates/test.txt'\n",
    "TARGETS = ['god','healthcare','guns','gayRights','abortion', 'creation']\n",
    "\n",
    "\n",
    "# train_data_file = '/data/parush/Data_MPCHI/train.txt'\n",
    "# test_data_file = '/data/parush/Data_MPCHI/test.txt'\n",
    "# TARGETS = ['Are E-Cigarettes safe?','Does MMR Vaccine lead to autism in children?',\n",
    "#       'Does Sunlight exposure lead to skin cancer?','Does Vitamin C prevent common cold?',\n",
    "#       'Should women take HRT post-menopause?']\n",
    "\n",
    "target = TARGETS[0]\n",
    "\n",
    "df = pd.read_csv(train_data_file, sep='\\t')\n",
    "df1 = pd.read_csv(test_data_file, sep='\\t')\n",
    "vectorizer = 'tfidf'   # set 'count' or 'tfidf'\n",
    "analyzer = 'both'  # set 'word' or 'both' ( word and char)\n",
    "if vectorizer == 'count':\n",
    "    if analyzer == 'word':\n",
    "        vectorizer = CountVectorizer(analyzer='word',ngram_range=(1,1))\n",
    "    else:\n",
    "        vectorizer = CountVectorizer(analyzer='word',ngram_range=(1,3))\n",
    "        char_vectorizer = CountVectorizer(analyzer='char',ngram_range=(2,5))\n",
    "else:\n",
    "    if analyzer == 'word':\n",
    "        vectorizer = TfidfVectorizer(analyzer='word',ngram_range=(1,1))\n",
    "    else:\n",
    "        vectorizer = TfidfVectorizer(analyzer='word',ngram_range=(1,3))\n",
    "        char_vectorizer = TfidfVectorizer(analyzer='char',ngram_range=(2,5))\n",
    "        \n",
    "#List of FAVOR Tweets\n",
    "def get_training_data_and_labels(file,target):\n",
    "    print(\"Working on target:\", target)\n",
    "    \n",
    "    train_corpus = []\n",
    "    train_labels = []\n",
    "    classes = {'FAVOR': 0, 'AGAINST': 1, 'NONE': 2}\n",
    "    with open(file) as f:\n",
    "        for line in f:\n",
    "            line = line.replace('#SemST', '').strip()\n",
    "            line = line.split('\\t')\n",
    "            #if line[0].strip() != 'ID' and target in line[1].strip():\n",
    "            if line[0].strip() != 'ID':  #Uncomment this line if training on wholedataset and comment the line above.\n",
    "                \n",
    "                tweet = line[2]\n",
    "                stance = line[3]\n",
    "                \n",
    "                train_corpus.append(tweet)\n",
    "                train_labels.append(classes[stance])\n",
    "\n",
    "    \n",
    "    if analyzer == 'word':\n",
    "        ngram_vectorized_data = vectorizer.fit_transform(train_corpus)\n",
    "        \n",
    "        return ngram_vectorized_data, train_labels\n",
    "    else:\n",
    "        ngram_vectorized_data = vectorizer.fit_transform(train_corpus)\n",
    "        char_vectorized_data = char_vectorizer.fit_transform(train_corpus)\n",
    "        l = np.hstack((ngram_vectorized_data.toarray(), char_vectorized_data.toarray()))\n",
    "        train_vectorized_data = sparse.csr_matrix(l)\n",
    "        \n",
    "        return train_vectorized_data, train_labels \n",
    "       \n",
    "#preparing test_data\n",
    "def get_test_data_and_labels(file,target):\n",
    "    print(\"Working on target:\", target)\n",
    "    test_corpus = []\n",
    "    test_labels = []\n",
    "    classes = {'FAVOR': 0, 'AGAINST': 1, 'NONE': 2}\n",
    "    with open(file) as f:\n",
    "        for line in f:\n",
    "            line = line.replace('#SemST', '').strip()\n",
    "            line = line.split('\\t')\n",
    "        #if line[0].strip() != 'ID' and line[3].strip() == 'FAVOR' and line[1] == t:\n",
    "            #if line[0].strip() != 'ID' and target in line[1].strip():\n",
    "            if line[0].strip() != 'ID': #Uncomment this line if testing on wholedataset and comment the line above.\n",
    "                tweet = line[2]\n",
    "                stance = line[3]\n",
    "                #print(line[0])\n",
    "                test_corpus.append(tweet)\n",
    "                test_labels.append(classes[stance])\n",
    "    if analyzer == 'word':\n",
    "        test_ngram_vectorized_data = vectorizer.transform(test_corpus)\n",
    "        \n",
    "        return test_ngram_vectorized_data, test_labels\n",
    "    else:\n",
    "        test_ngram_vectorized_data = vectorizer.transform(test_corpus)\n",
    "        test_char_vectorized_data = char_vectorizer.transform(test_corpus)\n",
    "        l2 = np.hstack((test_ngram_vectorized_data.toarray(), test_char_vectorized_data.toarray()))\n",
    "        test_vectorized_data = sparse.csr_matrix(l2)\n",
    "        \n",
    "        \n",
    "        return test_vectorized_data,test_labels\n",
    "    \n",
    "    \n",
    "    \n",
    "X_train, y_train =  get_training_data_and_labels(train_data_file, target)\n",
    "X_test, y_test = get_test_data_and_labels(test_data_file,target)  \n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "# Set the parameters by cross-validation\n",
    "tuned_parameters = [{'kernel': ['rbf'], 'gamma': [1e-3, 1e-4], 'C': [1, 10, 100, 1000]}, {'kernel': ['linear'], 'C': [1, 10, 100, 1000]}]\n",
    "scores = ['precision', 'recall']\n",
    "\n",
    "\n",
    "for score in scores:\n",
    "    \n",
    "    print(\"# Tuning hyper-parameters for %s\" % score)\n",
    "    print()\n",
    "    cv = StratifiedKFold(n_splits=2, shuffle=True, random_state = 2 )\n",
    "    clf = GridSearchCV(\n",
    "        SVC(), tuned_parameters, scoring='%s_macro' % score, cv = cv\n",
    "    )\n",
    "    clf.fit(X_train, y_train)\n",
    "\n",
    "    print(\"Best parameters set found on development set:\")\n",
    "    print()\n",
    "    print(clf.best_params_)\n",
    "    print()\n",
    "    print(\"Grid scores on development set:\")\n",
    "    print()\n",
    "    means = clf.cv_results_['mean_test_score']\n",
    "    stds = clf.cv_results_['std_test_score']\n",
    "    for mean, std, params in zip(means, stds, clf.cv_results_['params']):\n",
    "        print(\"%0.3f (+/-%0.03f) for %r\"\n",
    "              % (mean, std * 2, params))\n",
    "    print()\n",
    "\n",
    "    print(\"Detailed classification report:\")\n",
    "    print()\n",
    "    print()\n",
    "    y_true, y_pred = y_test, clf.predict(X_test)\n",
    "    print(classification_report(y_true, y_pred, digits = 4,labels = [0,1]))\n",
    "    print()\n",
    "print(\"--- %s seconds ---\" % (time.time() - start_time))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "fd319a6f-820e-4225-abfa-e9ae3222c42a",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "03433752-8a67-4d07-82b2-1d6389e98b61",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e3e8a0fb-7d51-42e6-8590-47ae41baa197",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a9192be3-17a3-4655-a43b-d65508a92ad2",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6611b8c7-1a78-48d6-80c5-4006e7b70135",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "90c0b9dc-2961-4d7c-abef-b1287345064c",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9f27958c-66b9-44b2-b560-e08546a75466",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e031376d-4eda-4702-b808-414dbe26f13c",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "879e4c20-f40b-42ee-812d-34bce5b3aeae",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "206166ad-5bb2-441b-923c-8e68333be2b8",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a366706b-ddf4-4075-bd77-c702cd02e62c",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e4055f07-7a23-49ae-9cce-8b429516bf47",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "19c66d16-3c8f-467c-a1d1-e0693abdc63d",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e76ab165-d71e-40f0-a3ce-692dfa221cc2",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1d75066a-0434-4800-9c02-e2fb50f3f1c7",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "bdee12ee-c29b-4a9d-86d3-ae128510511b",
   "metadata": {},
   "source": [
    "## Test on another target.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8cc1a008-6c59-4437-96e9-e53d78d213d4",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ba9ffe42-5491-4e24-9d38-d564e5b82a66",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5e4c6fe2-8315-47cd-8fe3-8f5e75772316",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a68e9ba8-3ed3-4527-8ba2-063ddd754537",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "77b9d8ed-840f-474c-bbbd-c0fce3fdcff9",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a0ff24d8-b3fb-4de7-8af5-24d0dfdcd284",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1a3afa63-0f18-4a38-871e-8cb245ceaef5",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b4fc89db-4b42-4aa4-87ea-87e6c0366cc2",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9c1e8527-2038-4c5c-9983-bcede1ae9ef8",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8432b7e7-290a-4043-ac4d-c1832206b9ea",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d689f2c6-0194-40bf-b235-873650da7687",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "nlpKernel",
   "language": "python",
   "name": "nlpkernel"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}