notebooks/00-download-fivethirtyeight-data.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Download Forecast Data From *FiveThirtyEight*\n",
    "\n",
    "This notebook downloads the current predictions for the House and Senate races from *FiveThirtyEight* at a given moment and puts the resulting CSV in the `data/fivethirtyeight` directory."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "import pandas as pd\n",
    "import json"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## House Forecast\n",
    "\n",
    "The data we want is embedded in a JSON file on the *FiveThirtyEight* site that populates the webpage."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "HOUSE_URL = \"https://projects.fivethirtyeight.com/2018-midterm-election-forecast/house/home.json\"\n",
    "house_data = requests.get(HOUSE_URL).json()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Extract Candidate Win Probabilities"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>candidate</th>\n",
       "      <th>classic_prob</th>\n",
       "      <th>district</th>\n",
       "      <th>party</th>\n",
       "      <th>state</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Alyse S. Galvin</td>\n",
       "      <td>28.994</td>\n",
       "      <td>1</td>\n",
       "      <td>D</td>\n",
       "      <td>AK</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Don Young</td>\n",
       "      <td>71.006</td>\n",
       "      <td>1</td>\n",
       "      <td>R</td>\n",
       "      <td>AK</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Others</td>\n",
       "      <td>0.000</td>\n",
       "      <td>1</td>\n",
       "      <td>None</td>\n",
       "      <td>AK</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Terri A. Sewell</td>\n",
       "      <td>100.000</td>\n",
       "      <td>7</td>\n",
       "      <td>D</td>\n",
       "      <td>AL</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Danner Kline</td>\n",
       "      <td>0.012</td>\n",
       "      <td>6</td>\n",
       "      <td>D</td>\n",
       "      <td>AL</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         candidate  classic_prob district party state\n",
       "0  Alyse S. Galvin        28.994        1     D    AK\n",
       "1        Don Young        71.006        1     R    AK\n",
       "2           Others         0.000        1  None    AK\n",
       "3  Terri A. Sewell       100.000        7     D    AL\n",
       "4     Danner Kline         0.012        6     D    AL"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "house_candidates = []\n",
    "for d in house_data[\"districtForecasts\"]:\n",
    "    for f in d[\"forecast\"]:\n",
    "        if d[\"state\"] != \"US\":\n",
    "            candidate_dict = {\n",
    "                \"state\": d[\"state\"],\n",
    "                \"district\": d[\"district\"]        \n",
    "            }\n",
    "            candidate_dict[\"candidate\"] = f[\"candidate\"]\n",
    "            candidate_dict[\"party\"] = f[\"party\"]\n",
    "            candidate_dict[\"classic_prob\"] = f[\"models\"][\"classic\"][\"winprob\"]\n",
    "            house_candidates.append(candidate_dict)\n",
    "            \n",
    "house_candidates = pd.DataFrame(house_candidates)\n",
    "house_candidates.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1031"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(house_candidates)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "50"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "house_candidates[\"state\"].nunique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "house_candidates.to_csv(\n",
    "    \"../data/fivethirtyeight/house_candidate_odds.csv\",\n",
    "    index = None\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Senate Forecast"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "SENATE_URL = \"https://projects.fivethirtyeight.com/2018-midterm-election-forecast/senate/home.json\"\n",
    "\n",
    "senate_data = requests.get(SENATE_URL).json()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### General forecast"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>candidate</th>\n",
       "      <th>class</th>\n",
       "      <th>classic_prob</th>\n",
       "      <th>party</th>\n",
       "      <th>state</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Kyrsten Sinema</td>\n",
       "      <td>1</td>\n",
       "      <td>61.552</td>\n",
       "      <td>D</td>\n",
       "      <td>AZ</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Angela Green</td>\n",
       "      <td>1</td>\n",
       "      <td>0.002</td>\n",
       "      <td>G</td>\n",
       "      <td>AZ</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Martha McSally</td>\n",
       "      <td>1</td>\n",
       "      <td>38.446</td>\n",
       "      <td>R</td>\n",
       "      <td>AZ</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Dianne Feinstein</td>\n",
       "      <td>1</td>\n",
       "      <td>98.442</td>\n",
       "      <td>D</td>\n",
       "      <td>CA</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Kevin de Leon</td>\n",
       "      <td>1</td>\n",
       "      <td>1.558</td>\n",
       "      <td>D</td>\n",
       "      <td>CA</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          candidate  class  classic_prob party state\n",
       "0    Kyrsten Sinema      1        61.552     D    AZ\n",
       "1      Angela Green      1         0.002     G    AZ\n",
       "2    Martha McSally      1        38.446     R    AZ\n",
       "3  Dianne Feinstein      1        98.442     D    CA\n",
       "4     Kevin de Leon      1         1.558     D    CA"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "senate_candidates = []\n",
    "for d in senate_data[\"seatForecasts\"]:\n",
    "    for f in d[\"forecast\"]:\n",
    "        if d[\"state\"] != \"US\":\n",
    "            senate_candidate_dict = {\n",
    "                \"state\": d[\"state\"],\n",
    "                \"class\": d[\"class\"]\n",
    "            }\n",
    "            senate_candidate_dict[\"candidate\"] = f[\"candidate\"]\n",
    "            senate_candidate_dict[\"party\"] = f[\"party\"]\n",
    "            senate_candidate_dict[\"classic_prob\"] = f[\"models\"][\"classic\"][\"winprob\"]\n",
    "            senate_candidates.append(senate_candidate_dict)\n",
    "            \n",
    "senate_candidates = pd.DataFrame(senate_candidates)\n",
    "\n",
    "senate_candidates.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "97"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(senate_candidates)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "33"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "senate_candidates[\"state\"].nunique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "senate_candidates.to_csv(\n",
    "    \"../data/fivethirtyeight/senate_candidate_odds.csv\",\n",
    "    index = None\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Voter Power Index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>class</th>\n",
       "      <th>state</th>\n",
       "      <th>vpi</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>AZ</td>\n",
       "      <td>3.830486</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>CA</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>CT</td>\n",
       "      <td>0.067179</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>DE</td>\n",
       "      <td>0.078927</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>FL</td>\n",
       "      <td>0.989972</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   class state       vpi\n",
       "0      1    AZ  3.830486\n",
       "1      1    CA  0.000000\n",
       "2      1    CT  0.067179\n",
       "3      1    DE  0.078927\n",
       "4      1    FL  0.989972"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "senate_seats = []\n",
    "for senate_seat in senate_data[\"seatForecasts\"]:\n",
    "    if \"vpi\" in senate_seat.keys():\n",
    "        senate_seat_dict = {\n",
    "            \"state\": senate_seat[\"state\"],\n",
    "            \"vpi\": senate_seat[\"vpi\"][\"classic\"],\n",
    "            \"class\": senate_seat[\"class\"]\n",
    "        }\n",
    "        senate_seats.append(senate_seat_dict)\n",
    "        \n",
    "voter_power_index = pd.DataFrame(senate_seats) \n",
    "\n",
    "voter_power_index.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "35"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(voter_power_index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "33"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "voter_power_index[\"state\"].nunique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "MN    2\n",
       "MS    2\n",
       "AZ    1\n",
       "NY    1\n",
       "TN    1\n",
       "Name: state, dtype: int64"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "voter_power_index[\"state\"].value_counts()[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>vpi</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>state</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>AZ</th>\n",
       "      <td>3.830486</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CA</th>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CT</th>\n",
       "      <td>0.067179</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DE</th>\n",
       "      <td>0.078927</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>FL</th>\n",
       "      <td>0.989972</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            vpi\n",
       "state          \n",
       "AZ     3.830486\n",
       "CA     0.000000\n",
       "CT     0.067179\n",
       "DE     0.078927\n",
       "FL     0.989972"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "total_voter_power_index = (\n",
    "    voter_power_index\n",
    "    .groupby(\"state\")\n",
    "    [\"vpi\"].sum()\n",
    "    .to_frame()\n",
    ")\n",
    "\n",
    "total_voter_power_index.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "total_voter_power_index.to_csv(\n",
    "    \"../data/fivethirtyeight/senate-voter-power-index.csv\"\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "---\n",
    "\n",
    "---"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}