Generate Perturbed Inputs.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from pandas.io.json import json_normalize\n",
    "import json\n",
    "import spacy\n",
    "import random\n",
    "import numpy as np\n",
    "from spacy import tokenizer\n",
    "import re"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting https://github.com/explosion/spacy-models/releases/download/en_core_web_md-2.0.0/en_core_web_md-2.0.0.tar.gz\n",
      "\u001b[?25l  Downloading https://github.com/explosion/spacy-models/releases/download/en_core_web_md-2.0.0/en_core_web_md-2.0.0.tar.gz (120.8MB)\n",
      "\u001b[K     |████████████████████████████████| 120.9MB 1.3MB/s eta 0:00:01\n",
      "\u001b[?25hRequirement already satisfied (use --upgrade to upgrade): en-core-web-md==2.0.0 from https://github.com/explosion/spacy-models/releases/download/en_core_web_md-2.0.0/en_core_web_md-2.0.0.tar.gz in /Users/mkorakakis/miniconda3/envs/nlp/lib/python3.6/site-packages\n",
      "Building wheels for collected packages: en-core-web-md\n",
      "  Building wheel for en-core-web-md (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25h  Created wheel for en-core-web-md: filename=en_core_web_md-2.0.0-cp36-none-any.whl size=122523224 sha256=ed1dc69050da26112d2caa2f7f9bc5b91feb4f0a129b41b90d89f66bb46f24cb\n",
      "  Stored in directory: /private/var/folders/gs/0q9bycyx5kq5qtmt4q6qq9mh0000gn/T/pip-ephem-wheel-cache-5ok_6onm/wheels/db/5d/d0/ccdad6b01f9695b4a33793158530b3228223ee31463441663d\n",
      "Successfully built en-core-web-md\n",
      "\n",
      "\u001b[93m    Linking successful\u001b[0m\n",
      "    /Users/mkorakakis/miniconda3/envs/nlp/lib/python3.6/site-packages/en_core_web_md\n",
      "    -->\n",
      "    /Users/mkorakakis/miniconda3/envs/nlp/lib/python3.6/site-packages/spacy/data/en_core_web_md\n",
      "\n",
      "    You can now load the model via spacy.load('en_core_web_md')\n",
      "\n"
     ]
    }
   ],
   "source": [
    "!python -m spacy download en_core_web_md"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [],
   "source": [
    "nlp = spacy.load(\"en_core_web_sm\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "vqa = json.load(open(\"OpenEnded_mscoco_val2014_questions.json\"))['questions']\n",
    "df = json_normalize(vqa)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['question', 'image_id', 'question_id'], dtype='object')"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>question</th>\n",
       "      <th>image_id</th>\n",
       "      <th>question_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>What is the table made of?</td>\n",
       "      <td>350623</td>\n",
       "      <td>3506232</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Is the food napping on the table?</td>\n",
       "      <td>350623</td>\n",
       "      <td>3506230</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>What has been upcycled to make lights?</td>\n",
       "      <td>350623</td>\n",
       "      <td>3506231</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Is this an Spanish town?</td>\n",
       "      <td>8647</td>\n",
       "      <td>86472</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Are there shadows on the sidewalk?</td>\n",
       "      <td>8647</td>\n",
       "      <td>86470</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>121507</th>\n",
       "      <td>What is holding up the phone?</td>\n",
       "      <td>174503</td>\n",
       "      <td>1745030</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>121508</th>\n",
       "      <td>What kind of glasses is she wearing?</td>\n",
       "      <td>174503</td>\n",
       "      <td>1745031</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>121509</th>\n",
       "      <td>What time does the clock say on the stove?</td>\n",
       "      <td>552610</td>\n",
       "      <td>5526100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>121510</th>\n",
       "      <td>Does the kitchen appear to be clean?</td>\n",
       "      <td>552610</td>\n",
       "      <td>5526101</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>121511</th>\n",
       "      <td>Is there water in the water bottle?</td>\n",
       "      <td>552610</td>\n",
       "      <td>5526102</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>121512 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                          question  image_id  question_id\n",
       "0                       What is the table made of?    350623      3506232\n",
       "1                Is the food napping on the table?    350623      3506230\n",
       "2           What has been upcycled to make lights?    350623      3506231\n",
       "3                         Is this an Spanish town?      8647        86472\n",
       "4               Are there shadows on the sidewalk?      8647        86470\n",
       "...                                            ...       ...          ...\n",
       "121507               What is holding up the phone?    174503      1745030\n",
       "121508        What kind of glasses is she wearing?    174503      1745031\n",
       "121509  What time does the clock say on the stove?    552610      5526100\n",
       "121510        Does the kitchen appear to be clean?    552610      5526101\n",
       "121511         Is there water in the water bottle?    552610      5526102\n",
       "\n",
       "[121512 rows x 3 columns]"
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Is the food napping on the table?\n",
      "0                         What is the table made of?\n",
      "1                  Is the food napping on the table?\n",
      "2             What has been upcycled to make lights?\n",
      "3                           Is this an Spanish town?\n",
      "4                 Are there shadows on the sidewalk?\n",
      "                             ...                    \n",
      "121507                 What is holding up the phone?\n",
      "121508          What kind of glasses is she wearing?\n",
      "121509    What time does the clock say on the stove?\n",
      "121510          Does the kitchen appear to be clean?\n",
      "121511           Is there water in the water bottle?\n",
      "Name: question, Length: 121512, dtype: object\n"
     ]
    }
   ],
   "source": [
    "questions = df['question']\n",
    "print(questions[1])\n",
    "print(questions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "question       object\n",
       "image_id        int64\n",
       "question_id     int64\n",
       "dtype: object"
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [],
   "source": [
    "def add_to_dict(image_id, question, question_id):\n",
    "    perturbed_dict = {}\n",
    "\n",
    "    perturbed_dict[\"image_id\"] = int(image_id)\n",
    "    question = ' '.join(question[:-1])\n",
    "    question += '?'\n",
    "    perturbed_dict[\"question\"] = question\n",
    "    perturbed_dict[\"question_id\"] = int(question_id)\n",
    "    \n",
    "    return perturbed_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [],
   "source": [
    "json_dict = {}\n",
    "perturbed_questions = []\n",
    "\n",
    "for i in range(len(questions)):\n",
    "    doc = nlp(questions[i])\n",
    "    drop_wh = []\n",
    "    for word, token in enumerate(doc):\n",
    "        if token.tag_ not in ['WDT', 'WP', 'WP$', 'WRB']:\n",
    "            drop_wh.append(token.text)\n",
    "            \n",
    "    row_data = df.loc[i, df.columns != 'question']\n",
    "    perturbed_questions.append(add_to_dict(row_data['image_id'], drop_wh, row_data['question_id']))    \n",
    "\n",
    "json_dict[\"questions\"] = perturbed_questions\n",
    "with open(\"vqa_drop_wh.json\", \"w\") as output_file:\n",
    "    json.dump(json_dict, output_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-80-5217ce346c81>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mquestions\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 6\u001b[0;31m     \u001b[0mdoc\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnlp\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mquestions\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      7\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      8\u001b[0m     \u001b[0mdrop_nouns\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/nlp/lib/python3.6/site-packages/spacy/language.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, text, disable)\u001b[0m\n\u001b[1;32m    350\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mhasattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mproc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'__call__'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    351\u001b[0m                 \u001b[0;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mErrors\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mE003\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcomponent\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mproc\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 352\u001b[0;31m             \u001b[0mdoc\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mproc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdoc\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    353\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mdoc\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    354\u001b[0m                 \u001b[0;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mErrors\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mE005\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32mnn_parser.pyx\u001b[0m in \u001b[0;36mspacy.syntax.nn_parser.Parser.__call__\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32mnn_parser.pyx\u001b[0m in \u001b[0;36mspacy.syntax.nn_parser.Parser.parse_batch\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32mnn_parser.pyx\u001b[0m in \u001b[0;36mspacy.syntax.nn_parser.Parser.get_batch_model\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/nlp/lib/python3.6/site-packages/thinc/api.py\u001b[0m in \u001b[0;36mbegin_update\u001b[0;34m(self, X, drop)\u001b[0m\n\u001b[1;32m     59\u001b[0m         \u001b[0mcallbacks\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     60\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mlayer\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_layers\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 61\u001b[0;31m             \u001b[0mX\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minc_layer_grad\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlayer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbegin_update\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdrop\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdrop\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     62\u001b[0m             \u001b[0mcallbacks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minc_layer_grad\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     63\u001b[0m         \u001b[0;32mdef\u001b[0m \u001b[0mcontinue_update\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mgradient\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msgd\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/nlp/lib/python3.6/site-packages/thinc/api.py\u001b[0m in \u001b[0;36mbegin_update\u001b[0;34m(seqs_in, drop)\u001b[0m\n\u001b[1;32m    278\u001b[0m         \u001b[0mlengths\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlayer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mops\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0masarray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mseq\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mseq\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mseqs_in\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    279\u001b[0m         X, bp_layer = layer.begin_update(layer.ops.flatten(seqs_in, pad=pad),\n\u001b[0;32m--> 280\u001b[0;31m                                          drop=drop)\n\u001b[0m\u001b[1;32m    281\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mbp_layer\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    282\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0mlayer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mops\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0munflatten\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlengths\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpad\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mpad\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/nlp/lib/python3.6/site-packages/thinc/api.py\u001b[0m in \u001b[0;36mbegin_update\u001b[0;34m(self, X, drop)\u001b[0m\n\u001b[1;32m     59\u001b[0m         \u001b[0mcallbacks\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     60\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mlayer\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_layers\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 61\u001b[0;31m             \u001b[0mX\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minc_layer_grad\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlayer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbegin_update\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdrop\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdrop\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     62\u001b[0m             \u001b[0mcallbacks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minc_layer_grad\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     63\u001b[0m         \u001b[0;32mdef\u001b[0m \u001b[0mcontinue_update\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mgradient\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msgd\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/nlp/lib/python3.6/site-packages/thinc/api.py\u001b[0m in \u001b[0;36muniqued_fwd\u001b[0;34m(X, drop)\u001b[0m\n\u001b[1;32m    370\u001b[0m                                                     \u001b[0mreturn_inverse\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    371\u001b[0m                                                     return_counts=True)\n\u001b[0;32m--> 372\u001b[0;31m         \u001b[0mY_uniq\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbp_Y_uniq\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlayer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbegin_update\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mind\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdrop\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdrop\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    373\u001b[0m         \u001b[0mY\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mY_uniq\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0minv\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreshape\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mY_uniq\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    374\u001b[0m         \u001b[0;32mdef\u001b[0m \u001b[0muniqued_bwd\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdY\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msgd\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/nlp/lib/python3.6/site-packages/thinc/api.py\u001b[0m in \u001b[0;36mbegin_update\u001b[0;34m(self, X, drop)\u001b[0m\n\u001b[1;32m     59\u001b[0m         \u001b[0mcallbacks\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     60\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mlayer\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_layers\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 61\u001b[0;31m             \u001b[0mX\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minc_layer_grad\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlayer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbegin_update\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdrop\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdrop\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     62\u001b[0m             \u001b[0mcallbacks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minc_layer_grad\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     63\u001b[0m         \u001b[0;32mdef\u001b[0m \u001b[0mcontinue_update\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mgradient\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msgd\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/nlp/lib/python3.6/site-packages/thinc/neural/_classes/layernorm.py\u001b[0m in \u001b[0;36mbegin_update\u001b[0;34m(self, X, drop)\u001b[0m\n\u001b[1;32m     50\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mbegin_update\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mX\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdrop\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m0.\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     51\u001b[0m         \u001b[0mX\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbackprop_child\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mchild\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbegin_update\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdrop\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m0.\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 52\u001b[0;31m         \u001b[0mN\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmu\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvar\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_get_moments\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mops\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mX\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     53\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     54\u001b[0m         \u001b[0mXhat\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mops\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mX\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmu\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvar\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/nlp/lib/python3.6/site-packages/thinc/neural/_classes/layernorm.py\u001b[0m in \u001b[0;36m_get_moments\u001b[0;34m(ops, X)\u001b[0m\n\u001b[1;32m     84\u001b[0m     \u001b[0mmu\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mX\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmean\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0maxis\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkeepdims\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     85\u001b[0m     \u001b[0mvar\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mX\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvar\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0maxis\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkeepdims\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;36m1e-08\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 86\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0mops\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0masarray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'f'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmu\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvar\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     87\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     88\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "json_dict = {}\n",
    "perturbed_questions = []\n",
    "\n",
    "\n",
    "for i in range(len(questions)):\n",
    "    doc = nlp(questions[i])\n",
    "    \n",
    "    drop_nouns = []\n",
    "    for word, token in enumerate(doc):\n",
    "        if token.tag_ not in ['NN', 'NNP', 'NNPS', 'NNS']:\n",
    "            drop_nouns.append(token.text)\n",
    "            \n",
    "    row_data = df.loc[i, df.columns != 'question']\n",
    "    perturbed_questions.append(add_to_dict(row_data['image_id'], drop_nouns, row_data['question_id']))\n",
    "\n",
    "json_dict[\"questions\"] = perturbed_questions\n",
    "with open(\"vqa_drop_nouns.json\", \"w\") as outfile:\n",
    "    json.dump(json_dict, outfile)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "json_dict = {}\n",
    "perturbed_questions = []\n",
    "\n",
    "\n",
    "for i in range(len(questions)):\n",
    "    doc = nlp(questions[i])\n",
    "    \n",
    "\n",
    "    drop_verbs = []\n",
    "    for word, token in enumerate(doc):\n",
    "        if token.pos_ is not 'VERB':\n",
    "            drop_verbs.append(token.text)\n",
    "   \n",
    "    row_data = df.loc[i, df.columns != 'question']\n",
    "    perturbed_questions.append(add_to_dict(row_data['image_id'], drop_verbs, row_data['question_id']))\n",
    "    \n",
    "json_dict[\"questions\"] = perturbed_questions\n",
    "with open(\"vqa_drop_verbs.json\", \"w\") as outfile:\n",
    "    json.dump(json_dict, outfile)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "json_dict = {}\n",
    "perturbed_questions = []\n",
    "\n",
    "\n",
    "for i in range(len(questions)):\n",
    "\n",
    "    words_to_keep = int(len(questions[i].split())*(1-0.4))\n",
    "    question = ' '.join(words[:words_to_keep])\n",
    "    question += '?'\n",
    "    \n",
    "    \n",
    "    row_data = df.loc[i, df.columns != 'question']    \n",
    "    perturbed_dict = {}\n",
    "    perturbed_dict[\"image_id\"] = int(row_data['image_id'])\n",
    "    perturbed_dict[\"question\"] = question\n",
    "    perturbed_dict[\"question_id\"] = int(row_data['question_id'])\n",
    "    \n",
    "    perturbed_questions.append(perturbed_dict)\n",
    "    \n",
    "json_dict[\"questions\"] = perturbed_questions\n",
    "with open(\"vqa_remove_words.json\", \"w\") as outfile:\n",
    "    json.dump(json_dict, outfile)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "json_dict = {}\n",
    "perturbed_questions = []\n",
    "\n",
    "\n",
    "for i in range(len(questions)):\n",
    "\n",
    "    question = questions[i].replace(\"?\",\"\").split()\n",
    "    shuffled = random.sample(question, len(question))\n",
    "    question = ' '.join(shuffled)\n",
    "    question += '?'\n",
    "    \n",
    "    \n",
    "    row_data = df.loc[i, df.columns != 'question']    \n",
    "    perturbed_dict = {}\n",
    "    perturbed_dict[\"image_id\"] = int(row_data['image_id'])\n",
    "    perturbed_dict[\"question\"] = question\n",
    "    perturbed_dict[\"question_id\"] = int(row_data['question_id'])\n",
    "    \n",
    "    perturbed_questions.append(perturbed_dict)\n",
    "    \n",
    "json_dict[\"questions\"] = perturbed_questions\n",
    "with open(\"vqa_shuffle_words.json\", \"w\") as outfile:\n",
    "    json.dump(json_dict, outfile)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Reading Comprehension"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "qs = json.load(open(\"dev-v2.0.json\"))['data']\n",
    "df = json_normalize(qs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['title', 'paragraphs'], dtype='object')"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>title</th>\n",
       "      <th>paragraphs</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Normans</td>\n",
       "      <td>[{'qas': [{'question': 'In what country is Nor...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Computational_complexity_theory</td>\n",
       "      <td>[{'qas': [{'question': 'What branch of theoret...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Southern_California</td>\n",
       "      <td>[{'qas': [{'question': 'What is Southern Calif...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Sky_(United_Kingdom)</td>\n",
       "      <td>[{'qas': [{'question': 'What company was forme...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Victoria_(Australia)</td>\n",
       "      <td>[{'qas': [{'question': 'What kind of economy d...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Huguenot</td>\n",
       "      <td>[{'qas': [{'question': \"Where was France's Hug...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Steam_engine</td>\n",
       "      <td>[{'qas': [{'question': 'Along with geothermal ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Oxygen</td>\n",
       "      <td>[{'qas': [{'question': 'The atomic number of t...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>1973_oil_crisis</td>\n",
       "      <td>[{'qas': [{'question': 'When did the 1973 oil ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>European_Union_law</td>\n",
       "      <td>[{'qas': [{'question': 'What is European Union...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>Amazon_rainforest</td>\n",
       "      <td>[{'qas': [{'question': 'Which name is also use...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>Ctenophora</td>\n",
       "      <td>[{'qas': [{'question': 'What is a ctenophora?'...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>Fresno,_California</td>\n",
       "      <td>[{'qas': [{'question': 'Which city is the fift...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>Packet_switching</td>\n",
       "      <td>[{'qas': [{'question': 'What did Paul Baran de...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>Black_Death</td>\n",
       "      <td>[{'qas': [{'question': 'Where did the black de...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>Geology</td>\n",
       "      <td>[{'qas': [{'question': 'An igneous rock is a r...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>Pharmacy</td>\n",
       "      <td>[{'qas': [{'question': 'What word is the word ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>Civil_disobedience</td>\n",
       "      <td>[{'qas': [{'question': 'What is it called when...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>Construction</td>\n",
       "      <td>[{'qas': [{'question': 'What is the process of...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>Private_school</td>\n",
       "      <td>[{'qas': [{'question': 'Along with non-governm...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>Harvard_University</td>\n",
       "      <td>[{'qas': [{'question': 'What individual is the...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>Jacksonville,_Florida</td>\n",
       "      <td>[{'qas': [{'question': 'Which Florida city has...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>Economic_inequality</td>\n",
       "      <td>[{'qas': [{'question': 'What percentage of glo...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>University_of_Chicago</td>\n",
       "      <td>[{'qas': [{'question': 'What kind of universit...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>Yuan_dynasty</td>\n",
       "      <td>[{'qas': [{'question': 'What is the Chinese na...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>Immune_system</td>\n",
       "      <td>[{'qas': [{'question': 'The immune system prot...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>Intergovernmental_Panel_on_Climate_Change</td>\n",
       "      <td>[{'qas': [{'question': 'What organization is t...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>Prime_number</td>\n",
       "      <td>[{'qas': [{'question': 'What is the only divis...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>Rhine</td>\n",
       "      <td>[{'qas': [{'question': 'Where does the Rhine e...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>Scottish_Parliament</td>\n",
       "      <td>[{'qas': [{'question': 'When was the current p...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>Islamism</td>\n",
       "      <td>[{'qas': [{'question': 'What is an Islamic rev...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>Imperialism</td>\n",
       "      <td>[{'qas': [{'question': \"The word imperialism h...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>Warsaw</td>\n",
       "      <td>[{'qas': [{'question': 'What is the largest ci...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>French_and_Indian_War</td>\n",
       "      <td>[{'qas': [{'question': 'When was the French an...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>Force</td>\n",
       "      <td>[{'qas': [{'question': 'What concept did philo...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                        title  \\\n",
       "0                                     Normans   \n",
       "1             Computational_complexity_theory   \n",
       "2                         Southern_California   \n",
       "3                        Sky_(United_Kingdom)   \n",
       "4                        Victoria_(Australia)   \n",
       "5                                    Huguenot   \n",
       "6                                Steam_engine   \n",
       "7                                      Oxygen   \n",
       "8                             1973_oil_crisis   \n",
       "9                          European_Union_law   \n",
       "10                          Amazon_rainforest   \n",
       "11                                 Ctenophora   \n",
       "12                         Fresno,_California   \n",
       "13                           Packet_switching   \n",
       "14                                Black_Death   \n",
       "15                                    Geology   \n",
       "16                                   Pharmacy   \n",
       "17                         Civil_disobedience   \n",
       "18                               Construction   \n",
       "19                             Private_school   \n",
       "20                         Harvard_University   \n",
       "21                      Jacksonville,_Florida   \n",
       "22                        Economic_inequality   \n",
       "23                      University_of_Chicago   \n",
       "24                               Yuan_dynasty   \n",
       "25                              Immune_system   \n",
       "26  Intergovernmental_Panel_on_Climate_Change   \n",
       "27                               Prime_number   \n",
       "28                                      Rhine   \n",
       "29                        Scottish_Parliament   \n",
       "30                                   Islamism   \n",
       "31                                Imperialism   \n",
       "32                                     Warsaw   \n",
       "33                      French_and_Indian_War   \n",
       "34                                      Force   \n",
       "\n",
       "                                           paragraphs  \n",
       "0   [{'qas': [{'question': 'In what country is Nor...  \n",
       "1   [{'qas': [{'question': 'What branch of theoret...  \n",
       "2   [{'qas': [{'question': 'What is Southern Calif...  \n",
       "3   [{'qas': [{'question': 'What company was forme...  \n",
       "4   [{'qas': [{'question': 'What kind of economy d...  \n",
       "5   [{'qas': [{'question': \"Where was France's Hug...  \n",
       "6   [{'qas': [{'question': 'Along with geothermal ...  \n",
       "7   [{'qas': [{'question': 'The atomic number of t...  \n",
       "8   [{'qas': [{'question': 'When did the 1973 oil ...  \n",
       "9   [{'qas': [{'question': 'What is European Union...  \n",
       "10  [{'qas': [{'question': 'Which name is also use...  \n",
       "11  [{'qas': [{'question': 'What is a ctenophora?'...  \n",
       "12  [{'qas': [{'question': 'Which city is the fift...  \n",
       "13  [{'qas': [{'question': 'What did Paul Baran de...  \n",
       "14  [{'qas': [{'question': 'Where did the black de...  \n",
       "15  [{'qas': [{'question': 'An igneous rock is a r...  \n",
       "16  [{'qas': [{'question': 'What word is the word ...  \n",
       "17  [{'qas': [{'question': 'What is it called when...  \n",
       "18  [{'qas': [{'question': 'What is the process of...  \n",
       "19  [{'qas': [{'question': 'Along with non-governm...  \n",
       "20  [{'qas': [{'question': 'What individual is the...  \n",
       "21  [{'qas': [{'question': 'Which Florida city has...  \n",
       "22  [{'qas': [{'question': 'What percentage of glo...  \n",
       "23  [{'qas': [{'question': 'What kind of universit...  \n",
       "24  [{'qas': [{'question': 'What is the Chinese na...  \n",
       "25  [{'qas': [{'question': 'The immune system prot...  \n",
       "26  [{'qas': [{'question': 'What organization is t...  \n",
       "27  [{'qas': [{'question': 'What is the only divis...  \n",
       "28  [{'qas': [{'question': 'Where does the Rhine e...  \n",
       "29  [{'qas': [{'question': 'When was the current p...  \n",
       "30  [{'qas': [{'question': 'What is an Islamic rev...  \n",
       "31  [{'qas': [{'question': \"The word imperialism h...  \n",
       "32  [{'qas': [{'question': 'What is the largest ci...  \n",
       "33  [{'qas': [{'question': 'When was the French an...  \n",
       "34  [{'qas': [{'question': 'What concept did philo...  "
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfToList = df['question'].tolist()\n",
    "dfToList"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_squad(data_path):\n",
    "\n",
    "    with open(data_path) as f:\n",
    "        data = json.load(f)['data']\n",
    "    articles = []\n",
    "    for article in data:\n",
    "        print(article)\n",
    "        article_dict= {}\n",
    "        article_dict['title'] = article[\"title\"]\n",
    "        \n",
    "        paragraph_dict = []\n",
    "        for paragraph in article['paragraphs']:\n",
    "            \n",
    "            \n",
    "            questions = []\n",
    "            for qa in paragraph['qas']:\n",
    "                \n",
    "                drop_nouns = []\n",
    "                question = nlp(qa['question'])\n",
    "                for word, token in enumerate(question):\n",
    "                    if token.tag_ not in ['NN', 'NNP', 'NNPS', 'NNS']:\n",
    "                        drop_nouns.append(token.text)\n",
    "                words_dict = {}\n",
    "                words_dict[\"id\"] = qa['id']\n",
    "                words_dict[\"question\"] = ' '.join(drop_nouns)\n",
    "                words_dict['answers'] = []\n",
    "                words_dict['is_impossible'] = qa['is_impossible']\n",
    "                if \"answers\" in qa:\n",
    "                    words_dict['answers'] = qa['answers']                \n",
    "                questions.append(words_dict)\n",
    "            question_dict = {}\n",
    "            question_dict[\"qas\"] = questions\n",
    "            question_dict[\"context\"] = paragraph['context']\n",
    "            paragraph_dict.append(question_dict)\n",
    "        article_dict['paragraphs'] = paragraph_dict\n",
    "        articles.append(article_dict)\n",
    "\n",
    "\n",
    "    json_dict[\"squad_drop_nouns\"] = articles\n",
    "    with open(\"abcfffff.json\", \"w\") as output_file:\n",
    "        json.dump(json_dict, output_file)\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_perturbed_squad(data_path):\n",
    "\n",
    "    with open(data_path) as f:\n",
    "        data = json.load(f)['data']\n",
    "    articles = []\n",
    "    for article in data:\n",
    "        print(article)\n",
    "        article_dict= {}\n",
    "        article_dict['title'] = article[\"title\"]\n",
    "        \n",
    "        paragraph_dict = []\n",
    "        for paragraph in article['paragraphs']:\n",
    "            \n",
    "            \n",
    "            questions = []\n",
    "            for qa in paragraph['qas']:\n",
    "                \n",
    "                drop_nouns = []\n",
    "                question = nlp(qa['question'])\n",
    "                for word, token in enumerate(question):\n",
    "                    if token.tag_ not in ['WDT', 'WP', 'WP$', 'WRB']:\n",
    "                        drop_nouns.append(token.text)\n",
    "                words_dict = {}\n",
    "                words_dict[\"id\"] = qa['id']\n",
    "                words_dict[\"question\"] = ' '.join(drop_nouns)\n",
    "                words_dict['answers'] = []\n",
    "                words_dict['is_impossible'] = qa['is_impossible']\n",
    "                if \"answers\" in qa:\n",
    "                    words_dict['answers'] = qa['answers']                \n",
    "                questions.append(words_dict)\n",
    "            question_dict = {}\n",
    "            question_dict[\"qas\"] = questions\n",
    "            question_dict[\"context\"] = paragraph['context']\n",
    "            paragraph_dict.append(question_dict)\n",
    "        article_dict['paragraphs'] = paragraph_dict\n",
    "        articles.append(article_dict)\n",
    "\n",
    "\n",
    "    json_dict[\"squad_drop_wh_words\"] = articles\n",
    "    with open(\"abcfffff.json\", \"w\") as output_file:\n",
    "        json.dump(json_dict, output_file)\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_perturbed_squad(data_path):\n",
    "\n",
    "    with open(data_path) as f:\n",
    "        data = json.load(f)['data']\n",
    "    articles = []\n",
    "    for article in data:\n",
    "        print(article)\n",
    "        article_dict= {}\n",
    "        article_dict['title'] = article[\"title\"]\n",
    "        \n",
    "        paragraph_dict = []\n",
    "        for paragraph in article['paragraphs']:\n",
    "            \n",
    "            \n",
    "            questions = []\n",
    "            for qa in paragraph['qas']:\n",
    "                \n",
    "                drop_nouns = []\n",
    "                question = nlp(qa['question'])\n",
    "                for word, token in enumerate(question):\n",
    "                    if token.pos_ is not 'VERB':\n",
    "                        drop_nouns.append(token.text)\n",
    "                words_dict = {}\n",
    "                words_dict[\"id\"] = qa['id']\n",
    "                words_dict[\"question\"] = ' '.join(drop_nouns)\n",
    "                words_dict['answers'] = []\n",
    "                words_dict['is_impossible'] = qa['is_impossible']\n",
    "                if \"answers\" in qa:\n",
    "                    words_dict['answers'] = qa['answers']                \n",
    "                questions.append(words_dict)\n",
    "            question_dict = {}\n",
    "            question_dict[\"qas\"] = questions\n",
    "            question_dict[\"context\"] = paragraph['context']\n",
    "            paragraph_dict.append(question_dict)\n",
    "        article_dict['paragraphs'] = paragraph_dict\n",
    "        articles.append(article_dict)\n",
    "\n",
    "\n",
    "    json_dict[\"squad_drop_verbs\"] = articles\n",
    "    with open(\"abcfffff.json\", \"w\") as output_file:\n",
    "        json.dump(json_dict, output_file)\n",
    "\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_perturbed_squad(data_path):\n",
    "\n",
    "    with open(data_path) as f:\n",
    "        data = json.load(f)['data']\n",
    "    articles = []\n",
    "    for article in data:\n",
    "        print(article)\n",
    "        article_dict= {}\n",
    "        article_dict['title'] = article[\"title\"]\n",
    "        \n",
    "        paragraph_dict = []\n",
    "        for paragraph in article['paragraphs']:\n",
    "            \n",
    "            \n",
    "            questions = []\n",
    "            for qa in paragraph['qas']:\n",
    "                \n",
    "                words_to_keep = int(len(qa['question'].split())*(1-0.4))\n",
    "                question = ' '.join(words[:words_to_keep])\n",
    "                question += '?'\n",
    "                \n",
    "                words_dict = {}\n",
    "                words_dict[\"id\"] = qa['id']\n",
    "                words_dict[\"question\"] = ' '.join(question)\n",
    "                words_dict['answers'] = []\n",
    "                words_dict['is_impossible'] = qa['is_impossible']\n",
    "                if \"answers\" in qa:\n",
    "                    words_dict['answers'] = qa['answers']                \n",
    "                questions.append(words_dict)\n",
    "            question_dict = {}\n",
    "            question_dict[\"qas\"] = questions\n",
    "            question_dict[\"context\"] = paragraph['context']\n",
    "            paragraph_dict.append(question_dict)\n",
    "        article_dict['paragraphs'] = paragraph_dict\n",
    "        articles.append(article_dict)\n",
    "\n",
    "\n",
    "    json_dict[\"squad_drop_words\"] = articles\n",
    "    with open(\"abcfffff.json\", \"w\") as output_file:\n",
    "        json.dump(json_dict, output_file)\n",
    "\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_perturbed_squad(data_path):\n",
    "\n",
    "    with open(data_path) as f:\n",
    "        data = json.load(f)['data']\n",
    "    articles = []\n",
    "    for article in data:\n",
    "        print(article)\n",
    "        article_dict= {}\n",
    "        article_dict['title'] = article[\"title\"]\n",
    "        \n",
    "        paragraph_dict = []\n",
    "        for paragraph in article['paragraphs']:\n",
    "            \n",
    "            \n",
    "            questions = []\n",
    "            for qa in paragraph['qas']:\n",
    "                \n",
    "                question = qa['question'].replace(\"?\",\"\").split()\n",
    "                shuffled = random.sample(question, len(question))\n",
    "                question = ' '.join(shuffled)\n",
    "                question += '?'\n",
    "    \n",
    "                words_dict = {}\n",
    "                words_dict[\"id\"] = qa['id']\n",
    "                words_dict[\"question\"] = ' '.join(drop_nouns)\n",
    "                words_dict['answers'] = []\n",
    "                words_dict['is_impossible'] = qa['is_impossible']\n",
    "                if \"answers\" in qa:\n",
    "                    words_dict['answers'] = qa['answers']                \n",
    "                questions.append(words_dict)\n",
    "            question_dict = {}\n",
    "            question_dict[\"qas\"] = questions\n",
    "            question_dict[\"context\"] = paragraph['context']\n",
    "            paragraph_dict.append(question_dict)\n",
    "        article_dict['paragraphs'] = paragraph_dict\n",
    "        articles.append(article_dict)\n",
    "\n",
    "\n",
    "    json_dict[\"questions\"] = articles\n",
    "    with open(\"abcfffff.json\", \"w\") as output_file:\n",
    "        json.dump(json_dict, output_file)\n",
    "\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## TQA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {},
   "outputs": [],
   "source": [
    "f = open('plug.examples', 'a')\n",
    "f.write(\"stuff\")\n",
    "f.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def add_to_dict(image_id, question, question_id):\n",
    "    perturbed_dict = {}\n",
    "\n",
    "    perturbed_dict[\"image_id\"] = int(image_id)\n",
    "    question = ' '.join(question[:-1])\n",
    "    question += '?'\n",
    "    perturbed_dict[\"question\"] = question\n",
    "    perturbed_dict[\"question_id\"] = int(question_id)\n",
    "    \n",
    "    return perturbed_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 239,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "which team won previous to crettyard?\n",
      "<class 'str'>\n",
      "(example (id nt-2) (utterance \"team won previous to crettyard?\") (context (graph tables.TableKnowledgeGraph csv/204-csv/772.csv)) (targetValue (list (description \"Wolfe Tones\"))))\n",
      "\n",
      "(example (id nt-2) (utterance \"which team won previous to crettyard?\") (context (graph tables.TableKnowledgeGraph csv/204-csv/772.csv)) (targetValue (list (description \"Wolfe Tones\"))))\n",
      "\n",
      "(example (id nt-2) (utterance \"team won previous to crettyard?\") (context (graph tables.TableKnowledgeGraph csv/204-csv/772.csv)) (targetValue (list (description \"Wolfe Tones\"))))\n",
      "\n"
     ]
    }
   ],
   "source": [
    "perturbed_inputs = []\n",
    "with open('random-split-1-dev.examples') as f:\n",
    "    lines = f.readlines()\n",
    "    for line in lines:\n",
    "\n",
    "        doc = nlp(re.findall(r'\"(.*?)\"', line)[0])\n",
    "        print(doc)\n",
    "        drop_wh = []\n",
    "        for word, token in enumerate(doc):\n",
    "            if token.tag_ not in ['WDT', 'WP', 'WP$', 'WRB']:\n",
    "                drop_wh.append(token.text)\n",
    "        \n",
    "        question = ' '.join(drop_wh[:-1])\n",
    "        question += '?'\n",
    "         \n",
    "        \n",
    "        replaced = re.sub(r'utterance \"(.*?)\"', 'utterance \"%s\"' % question, line)\n",
    "        perturbed_inputs.append(replaced)\n",
    "        \n",
    "        break\n",
    "\n",
    "with open(\"drop_wh.examples\", \"a\") as f:\n",
    "    for line in perturbed_inputs:\n",
    "        f.write(line)\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 241,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "which team won previous to crettyard?\n",
      "(example (id nt-2) (utterance \"which team previous to crettyard?\") (context (graph tables.TableKnowledgeGraph csv/204-csv/772.csv)) (targetValue (list (description \"Wolfe Tones\"))))\n",
      "\n",
      "(example (id nt-2) (utterance \"which team previous to crettyard?\") (context (graph tables.TableKnowledgeGraph csv/204-csv/772.csv)) (targetValue (list (description \"Wolfe Tones\"))))\n",
      "\n"
     ]
    }
   ],
   "source": [
    "perturbed_inputs = []\n",
    "with open('random-split-1-dev.examples') as f:\n",
    "    lines = f.readlines()\n",
    "    for line in lines:\n",
    "\n",
    "        doc = nlp(re.findall(r'\"(.*?)\"', line)[0])\n",
    "        print(doc)\n",
    "        drop_verbs = []\n",
    "        for word, token in enumerate(doc):\n",
    "            if token.pos_ is not 'VERB':\n",
    "                drop_verbs.append(token.text)\n",
    "        \n",
    "        question = ' '.join(drop_verbs[:-1])\n",
    "        question += '?'\n",
    "         \n",
    "        \n",
    "        replaced = re.sub(r'utterance \"(.*?)\"', 'utterance \"%s\"' % question, line)\n",
    "        perturbed_inputs.append(replaced)\n",
    "        print(replaced)\n",
    "        break\n",
    "\n",
    "with open(\"drop_verbs.examples\", \"a\") as f:\n",
    "    for line in perturbed_inputs:\n",
    "        f.write(line)\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 243,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "which team won previous to crettyard?\n",
      "(example (id nt-2) (utterance \"which won previous to?\") (context (graph tables.TableKnowledgeGraph csv/204-csv/772.csv)) (targetValue (list (description \"Wolfe Tones\"))))\n",
      "\n",
      "(example (id nt-2) (utterance \"which won previous to?\") (context (graph tables.TableKnowledgeGraph csv/204-csv/772.csv)) (targetValue (list (description \"Wolfe Tones\"))))\n",
      "\n"
     ]
    }
   ],
   "source": [
    "perturbed_inputs = []\n",
    "with open('random-split-1-dev.examples') as f:\n",
    "    lines = f.readlines()\n",
    "    for line in lines:\n",
    "\n",
    "        doc = nlp(re.findall(r'\"(.*?)\"', line)[0])\n",
    "        print(doc)\n",
    "        drop_nouns = []\n",
    "        for word, token in enumerate(doc):\n",
    "            if token.tag_ not in ['NN', 'NNP', 'NNPS', 'NNS']:\n",
    "                drop_nouns.append(token.text)\n",
    "        \n",
    "        question = ' '.join(drop_nouns[:-1])\n",
    "        question += '?'\n",
    "         \n",
    "        \n",
    "        replaced = re.sub(r'utterance \"(.*?)\"', 'utterance \"%s\"' % question, line)\n",
    "        perturbed_inputs.append(replaced)\n",
    "        print(replaced)\n",
    "        break\n",
    "\n",
    "with open(\"drop_nouns.examples\", \"a\") as f:\n",
    "    for line in perturbed_inputs:\n",
    "        f.write(line)\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 247,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "which team won previous to crettyard?\n",
      "(example (id nt-2) (utterance \"which team won?\") (context (graph tables.TableKnowledgeGraph csv/204-csv/772.csv)) (targetValue (list (description \"Wolfe Tones\"))))\n",
      "\n"
     ]
    }
   ],
   "source": [
    "perturbed_inputs = []\n",
    "with open('random-split-1-dev.examples') as f:\n",
    "    lines = f.readlines()\n",
    "    for line in lines:\n",
    "\n",
    "        doc = nlp(re.findall(r'\"(.*?)\"', line)[0])\n",
    "        words = doc.text.split()\n",
    "        words_to_keep = int(len(words)*(1-0.4))\n",
    "        question = ' '.join(words[:words_to_keep])\n",
    "        question += '?'   \n",
    "        \n",
    "            \n",
    "        replaced = re.sub(r'utterance \"(.*?)\"', 'utterance \"%s\"' % question, line)\n",
    "        perturbed_inputs.append(replaced)\n",
    "        \n",
    "        break\n",
    "\n",
    "with open(\"drop_words.examples\", \"a\") as f:\n",
    "    for line in perturbed_inputs:\n",
    "        print(line)\n",
    "        f.write(line)\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 249,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(example (id nt-2) (utterance \"previous team crettyard won which to?\") (context (graph tables.TableKnowledgeGraph csv/204-csv/772.csv)) (targetValue (list (description \"Wolfe Tones\"))))\n",
      "\n"
     ]
    }
   ],
   "source": [
    "perturbed_inputs = []\n",
    "with open('random-split-1-dev.examples') as f:\n",
    "    lines = f.readlines()\n",
    "    for line in lines:\n",
    "\n",
    "        doc = nlp(re.findall(r'\"(.*?)\"', line)[0])\n",
    "\n",
    "        question = doc.text\n",
    "        question = question.replace(\"?\",\"\").split()\n",
    "        shuffled = random.sample(question, len(question))\n",
    "        question = ' '.join(shuffled)\n",
    "        question += '?'\n",
    "        \n",
    "        replaced = re.sub(r'utterance \"(.*?)\"', 'utterance \"%s\"' % question, line)\n",
    "        perturbed_inputs.append(replaced)\n",
    "        \n",
    "        break\n",
    "\n",
    "with open(\"shuffle_words.examples\", \"a\") as f:\n",
    "    for line in perturbed_inputs:\n",
    "        print(line)\n",
    "        f.write(line)\n",
    "        "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}