SGD Classifier/Logistic Regression (SGD).ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "np.random.seed(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((100, 4), (100, 1))"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "iris = pd.read_csv('Iris.csv')\n",
    "iris.drop('Id',axis=1,inplace=True)\n",
    "iris.Species.replace({'Iris-virginica':0,'Iris-versicolor':1,'Iris-setosa':2},inplace=True)\n",
    "iris = iris[iris.Species.isin([0,1])]\n",
    "iris = iris.sample(frac=1)\n",
    "cols = iris.columns\n",
    "X,Y = iris[cols[:-1]].values,np.expand_dims(iris[cols[-1]].values,axis=-1)\n",
    "X.shape,Y.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def sigmoid(z):\n",
    "    return 1/(1+np.exp(-z))\n",
    "\n",
    "def sigmoid_derivative(z):\n",
    "    s = sigmoid(z)\n",
    "    return s*(1-s)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def initialize_weights_and_bais(n):\n",
    "    W = np.random.randn(n,1) #(n,1)\n",
    "    b = 0\n",
    "    return W,b"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def cost_function(p,Y):\n",
    "    \"cross entropy loss\"\n",
    "    m = len(Y)\n",
    "    loss = -( Y*np.log(p) + (1-Y)*np.log(1-p) )\n",
    "    cost = np.sum(loss)/m\n",
    "    return cost"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Probabality of event is defined as\n",
    "\n",
    "$$ prob_i = p_i = \\sigma( X_i.W + b ) = \\frac{ e^{ X_i.W + b} }{1 + e^{ X_i.W + b}} $$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$ 1 - p_i = \\frac{1}{1 + e^{ X_i.W + b}}, and \\frac{p_i}{1-p_i} = e^{ X_i.W + b} $"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$ Cost(J) = CrossEntropy = - \\sum_{i=0}^m Y_i log(p_i) - \\sum_{i=0}^m (1-Y_i)log(1-p_i)  $$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$ J = -\\sum_{i=0}^m log(1-p_i) - \\sum_{i=0}^m Y_i log( \\frac{ p_i }{1-p_i} ) $$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$ J = -\\sum_{i=0}^m log( \\frac{1}{1 + e^{ X_i.W + b}}) - \\sum_{i=0}^m Y_i log(e^{X_i.W + b}) $$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$ J = \\sum_{i=0}^m log( 1 + e^{ X_i.W + b} ) - \\sum_{i=0}^m Y_i (X_i.W+b) $$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Lets take out derivative"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$ \\frac{\\partial J}{ \\partial W_j } = \\sum_{i=0}^m \\frac{1}{1+e^{ X_i.W + b}}e^{ X_i.W+b} X_{ij} - \\sum_{i=0}^m Y_i.X_{ij} = - \\sum_{i=0}^m (Y_i - p_i )X_{ij}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$ \\frac{\\partial J}{\\partial b} = \\sum_{i=0}^m \\frac{1}{1+e^{ X_i.W + b}}e^{ X_i.W+b} - \\sum_{i=0}^m Y_i = - \\sum_{i=0}^m(Y_i - p_i) $$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_probs(X,W,b):\n",
    "    return sigmoid( np.dot(X,W)+b )\n",
    "\n",
    "def optimize(W,b,X,Y,lr=0.1,lmda=0.1):\n",
    "    \"\"\"\n",
    "    W = weights\n",
    "    b = bias\n",
    "    lr = learning rate\n",
    "    lmda = regularization parameter\n",
    "    \"\"\"\n",
    "    m = len(Y)\n",
    "    y = get_probs(X,W,b)\n",
    "    \n",
    "    dW = -np.dot( X.T, (Y-y) )/m # (4,1)\n",
    "    db = -np.sum(Y-y)/m\n",
    "    \n",
    "    W = W - lr* ( dW + (lmda/m)*W ) \n",
    "    b = b - lr*db\n",
    "    return W,b"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iteration: 0, Cost: 1.095, Accuracy: 0.50\n",
      "Iteration: 1, Cost: 0.881, Accuracy: 0.50\n",
      "Iteration: 2, Cost: 0.814, Accuracy: 0.54\n",
      "Iteration: 3, Cost: 0.792, Accuracy: 0.45\n",
      "Iteration: 4, Cost: 0.775, Accuracy: 0.49\n",
      "Iteration: 5, Cost: 0.765, Accuracy: 0.42\n",
      "Iteration: 6, Cost: 0.756, Accuracy: 0.42\n",
      "Iteration: 7, Cost: 0.749, Accuracy: 0.38\n",
      "Iteration: 8, Cost: 0.743, Accuracy: 0.37\n",
      "Iteration: 9, Cost: 0.738, Accuracy: 0.38\n",
      "Iteration: 10, Cost: 0.733, Accuracy: 0.37\n",
      "Iteration: 11, Cost: 0.728, Accuracy: 0.39\n",
      "Iteration: 12, Cost: 0.724, Accuracy: 0.38\n",
      "Iteration: 13, Cost: 0.719, Accuracy: 0.39\n",
      "Iteration: 14, Cost: 0.715, Accuracy: 0.38\n",
      "Iteration: 15, Cost: 0.711, Accuracy: 0.41\n",
      "Iteration: 16, Cost: 0.707, Accuracy: 0.41\n",
      "Iteration: 17, Cost: 0.703, Accuracy: 0.43\n",
      "Iteration: 18, Cost: 0.699, Accuracy: 0.42\n",
      "Iteration: 19, Cost: 0.695, Accuracy: 0.48\n",
      "Iteration: 20, Cost: 0.691, Accuracy: 0.44\n",
      "Iteration: 21, Cost: 0.687, Accuracy: 0.48\n",
      "Iteration: 22, Cost: 0.684, Accuracy: 0.47\n",
      "Iteration: 23, Cost: 0.680, Accuracy: 0.50\n",
      "Iteration: 24, Cost: 0.676, Accuracy: 0.50\n",
      "Iteration: 25, Cost: 0.673, Accuracy: 0.53\n",
      "Iteration: 26, Cost: 0.669, Accuracy: 0.53\n",
      "Iteration: 27, Cost: 0.665, Accuracy: 0.55\n",
      "Iteration: 28, Cost: 0.662, Accuracy: 0.55\n",
      "Iteration: 29, Cost: 0.658, Accuracy: 0.56\n",
      "Iteration: 30, Cost: 0.655, Accuracy: 0.57\n",
      "Iteration: 31, Cost: 0.651, Accuracy: 0.59\n",
      "Iteration: 32, Cost: 0.648, Accuracy: 0.59\n",
      "Iteration: 33, Cost: 0.645, Accuracy: 0.59\n",
      "Iteration: 34, Cost: 0.641, Accuracy: 0.60\n",
      "Iteration: 35, Cost: 0.638, Accuracy: 0.60\n",
      "Iteration: 36, Cost: 0.635, Accuracy: 0.60\n",
      "Iteration: 37, Cost: 0.632, Accuracy: 0.61\n",
      "Iteration: 38, Cost: 0.629, Accuracy: 0.63\n",
      "Iteration: 39, Cost: 0.625, Accuracy: 0.65\n",
      "Iteration: 40, Cost: 0.622, Accuracy: 0.65\n",
      "Iteration: 41, Cost: 0.619, Accuracy: 0.66\n",
      "Iteration: 42, Cost: 0.616, Accuracy: 0.67\n",
      "Iteration: 43, Cost: 0.613, Accuracy: 0.69\n",
      "Iteration: 44, Cost: 0.610, Accuracy: 0.70\n",
      "Iteration: 45, Cost: 0.607, Accuracy: 0.70\n",
      "Iteration: 46, Cost: 0.604, Accuracy: 0.71\n",
      "Iteration: 47, Cost: 0.602, Accuracy: 0.71\n",
      "Iteration: 48, Cost: 0.599, Accuracy: 0.71\n",
      "Iteration: 49, Cost: 0.596, Accuracy: 0.72\n",
      "Iteration: 50, Cost: 0.593, Accuracy: 0.74\n",
      "Iteration: 51, Cost: 0.590, Accuracy: 0.74\n",
      "Iteration: 52, Cost: 0.588, Accuracy: 0.74\n",
      "Iteration: 53, Cost: 0.585, Accuracy: 0.74\n",
      "Iteration: 54, Cost: 0.582, Accuracy: 0.75\n",
      "Iteration: 55, Cost: 0.579, Accuracy: 0.77\n",
      "Iteration: 56, Cost: 0.577, Accuracy: 0.78\n",
      "Iteration: 57, Cost: 0.574, Accuracy: 0.78\n",
      "Iteration: 58, Cost: 0.572, Accuracy: 0.79\n",
      "Iteration: 59, Cost: 0.569, Accuracy: 0.81\n",
      "Iteration: 60, Cost: 0.567, Accuracy: 0.82\n",
      "Iteration: 61, Cost: 0.564, Accuracy: 0.82\n",
      "Iteration: 62, Cost: 0.562, Accuracy: 0.82\n",
      "Iteration: 63, Cost: 0.559, Accuracy: 0.83\n",
      "Iteration: 64, Cost: 0.557, Accuracy: 0.83\n",
      "Iteration: 65, Cost: 0.554, Accuracy: 0.83\n",
      "Iteration: 66, Cost: 0.552, Accuracy: 0.83\n",
      "Iteration: 67, Cost: 0.550, Accuracy: 0.83\n",
      "Iteration: 68, Cost: 0.547, Accuracy: 0.84\n",
      "Iteration: 69, Cost: 0.545, Accuracy: 0.84\n",
      "Iteration: 70, Cost: 0.543, Accuracy: 0.84\n",
      "Iteration: 71, Cost: 0.540, Accuracy: 0.84\n",
      "Iteration: 72, Cost: 0.538, Accuracy: 0.85\n",
      "Iteration: 73, Cost: 0.536, Accuracy: 0.85\n",
      "Iteration: 74, Cost: 0.534, Accuracy: 0.86\n",
      "Iteration: 75, Cost: 0.532, Accuracy: 0.86\n",
      "Iteration: 76, Cost: 0.529, Accuracy: 0.86\n",
      "Iteration: 77, Cost: 0.527, Accuracy: 0.86\n",
      "Iteration: 78, Cost: 0.525, Accuracy: 0.86\n",
      "Iteration: 79, Cost: 0.523, Accuracy: 0.86\n",
      "Iteration: 80, Cost: 0.521, Accuracy: 0.87\n",
      "Iteration: 81, Cost: 0.519, Accuracy: 0.87\n",
      "Iteration: 82, Cost: 0.517, Accuracy: 0.87\n",
      "Iteration: 83, Cost: 0.515, Accuracy: 0.87\n",
      "Iteration: 84, Cost: 0.513, Accuracy: 0.87\n",
      "Iteration: 85, Cost: 0.511, Accuracy: 0.88\n",
      "Iteration: 86, Cost: 0.509, Accuracy: 0.87\n",
      "Iteration: 87, Cost: 0.507, Accuracy: 0.87\n",
      "Iteration: 88, Cost: 0.505, Accuracy: 0.87\n",
      "Iteration: 89, Cost: 0.503, Accuracy: 0.87\n",
      "Iteration: 90, Cost: 0.501, Accuracy: 0.87\n",
      "Iteration: 91, Cost: 0.499, Accuracy: 0.87\n",
      "Iteration: 92, Cost: 0.498, Accuracy: 0.87\n",
      "Iteration: 93, Cost: 0.496, Accuracy: 0.87\n",
      "Iteration: 94, Cost: 0.494, Accuracy: 0.87\n",
      "Iteration: 95, Cost: 0.492, Accuracy: 0.87\n",
      "Iteration: 96, Cost: 0.490, Accuracy: 0.87\n",
      "Iteration: 97, Cost: 0.489, Accuracy: 0.87\n",
      "Iteration: 98, Cost: 0.487, Accuracy: 0.87\n",
      "Iteration: 99, Cost: 0.485, Accuracy: 0.87\n",
      "Iteration: 100, Cost: 0.483, Accuracy: 0.87\n",
      "Iteration: 101, Cost: 0.482, Accuracy: 0.87\n",
      "Iteration: 102, Cost: 0.480, Accuracy: 0.87\n",
      "Iteration: 103, Cost: 0.478, Accuracy: 0.87\n",
      "Iteration: 104, Cost: 0.477, Accuracy: 0.87\n",
      "Iteration: 105, Cost: 0.475, Accuracy: 0.87\n",
      "Iteration: 106, Cost: 0.473, Accuracy: 0.87\n",
      "Iteration: 107, Cost: 0.472, Accuracy: 0.87\n",
      "Iteration: 108, Cost: 0.470, Accuracy: 0.87\n",
      "Iteration: 109, Cost: 0.469, Accuracy: 0.87\n",
      "Iteration: 110, Cost: 0.467, Accuracy: 0.87\n",
      "Iteration: 111, Cost: 0.465, Accuracy: 0.87\n",
      "Iteration: 112, Cost: 0.464, Accuracy: 0.87\n",
      "Iteration: 113, Cost: 0.462, Accuracy: 0.87\n",
      "Iteration: 114, Cost: 0.461, Accuracy: 0.87\n",
      "Iteration: 115, Cost: 0.459, Accuracy: 0.87\n",
      "Iteration: 116, Cost: 0.458, Accuracy: 0.87\n",
      "Iteration: 117, Cost: 0.456, Accuracy: 0.87\n",
      "Iteration: 118, Cost: 0.455, Accuracy: 0.87\n",
      "Iteration: 119, Cost: 0.453, Accuracy: 0.87\n",
      "Iteration: 120, Cost: 0.452, Accuracy: 0.87\n",
      "Iteration: 121, Cost: 0.451, Accuracy: 0.87\n",
      "Iteration: 122, Cost: 0.449, Accuracy: 0.87\n",
      "Iteration: 123, Cost: 0.448, Accuracy: 0.87\n",
      "Iteration: 124, Cost: 0.446, Accuracy: 0.87\n",
      "Iteration: 125, Cost: 0.445, Accuracy: 0.87\n",
      "Iteration: 126, Cost: 0.444, Accuracy: 0.88\n",
      "Iteration: 127, Cost: 0.442, Accuracy: 0.88\n",
      "Iteration: 128, Cost: 0.441, Accuracy: 0.88\n",
      "Iteration: 129, Cost: 0.440, Accuracy: 0.88\n",
      "Iteration: 130, Cost: 0.438, Accuracy: 0.88\n",
      "Iteration: 131, Cost: 0.437, Accuracy: 0.88\n",
      "Iteration: 132, Cost: 0.436, Accuracy: 0.88\n",
      "Iteration: 133, Cost: 0.434, Accuracy: 0.88\n",
      "Iteration: 134, Cost: 0.433, Accuracy: 0.88\n",
      "Iteration: 135, Cost: 0.432, Accuracy: 0.88\n",
      "Iteration: 136, Cost: 0.430, Accuracy: 0.88\n",
      "Iteration: 137, Cost: 0.429, Accuracy: 0.88\n",
      "Iteration: 138, Cost: 0.428, Accuracy: 0.88\n",
      "Iteration: 139, Cost: 0.427, Accuracy: 0.88\n",
      "Iteration: 140, Cost: 0.426, Accuracy: 0.88\n",
      "Iteration: 141, Cost: 0.424, Accuracy: 0.88\n",
      "Iteration: 142, Cost: 0.423, Accuracy: 0.88\n",
      "Iteration: 143, Cost: 0.422, Accuracy: 0.88\n",
      "Iteration: 144, Cost: 0.421, Accuracy: 0.88\n",
      "Iteration: 145, Cost: 0.420, Accuracy: 0.88\n",
      "Iteration: 146, Cost: 0.418, Accuracy: 0.88\n",
      "Iteration: 147, Cost: 0.417, Accuracy: 0.88\n",
      "Iteration: 148, Cost: 0.416, Accuracy: 0.88\n",
      "Iteration: 149, Cost: 0.415, Accuracy: 0.89\n",
      "Iteration: 150, Cost: 0.414, Accuracy: 0.89\n",
      "Iteration: 151, Cost: 0.413, Accuracy: 0.89\n",
      "Iteration: 152, Cost: 0.412, Accuracy: 0.89\n",
      "Iteration: 153, Cost: 0.410, Accuracy: 0.90\n",
      "Iteration: 154, Cost: 0.409, Accuracy: 0.90\n",
      "Iteration: 155, Cost: 0.408, Accuracy: 0.90\n",
      "Iteration: 156, Cost: 0.407, Accuracy: 0.90\n",
      "Iteration: 157, Cost: 0.406, Accuracy: 0.91\n",
      "Iteration: 158, Cost: 0.405, Accuracy: 0.91\n",
      "Iteration: 159, Cost: 0.404, Accuracy: 0.91\n",
      "Iteration: 160, Cost: 0.403, Accuracy: 0.91\n",
      "Iteration: 161, Cost: 0.402, Accuracy: 0.91\n",
      "Iteration: 162, Cost: 0.401, Accuracy: 0.91\n",
      "Iteration: 163, Cost: 0.400, Accuracy: 0.92\n",
      "Iteration: 164, Cost: 0.399, Accuracy: 0.92\n",
      "Iteration: 165, Cost: 0.398, Accuracy: 0.92\n",
      "Iteration: 166, Cost: 0.397, Accuracy: 0.92\n",
      "Iteration: 167, Cost: 0.396, Accuracy: 0.92\n"
     ]
    }
   ],
   "source": [
    "W,b = initialize_weights_and_bais(X.shape[1])\n",
    "last_cost,tol,i = float('inf'),0.001,0\n",
    "while True:\n",
    "    p = get_probs(X,W,b)\n",
    "    cost = cost_function(p,Y)\n",
    "    acc = ((p>.5)*1 == Y).sum()/len(Y)\n",
    "    print(f\"Iteration: {i}, Cost: {cost:.3f}, Accuracy: {acc:.2f}\")\n",
    "    W,b = optimize(W,b,X,Y,lr=1e-1,lmda=0.5)\n",
    "    if last_cost - cost < tol: break\n",
    "    else: last_cost, i = cost, i+1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}