workspace/Example/lyft-unet-train43-conv-lstm-try2.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "%reload_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from fastai.conv_learner import *\n",
    "# from fastai.dataset import *\n",
    "from fastai.models.resnet import vgg_resnet50\n",
    "\n",
    "import json\n",
    "from glob import glob"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.backends.cudnn.benchmark=True"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "PATH = Path('../data/all')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def show_img(im, figsize=None, ax=None, alpha=None):\n",
    "    if not ax: fig,ax = plt.subplots(figsize=figsize)\n",
    "    ax.imshow(im, alpha=alpha)\n",
    "    ax.set_axis_off()\n",
    "    return ax"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "VEHICLES=10\n",
    "ROADS=7\n",
    "ROAD_LINES=6"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "TRAIN_DN = 'CameraRGB'\n",
    "MASKS_DN = 'CameraSeg'\n",
    "workers=7\n",
    "random_crop=False\n",
    "pseudo_label=False\n",
    "# val_folder = 'sample_test_sync'\n",
    "val_folder = 'test_sync'\n",
    "# val_folder = 'val'\n",
    "S_PREFIX = '43-conv-lstm'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "from torchvision.datasets.folder import pil_loader\n",
    "import torchvision.transforms as transforms\n",
    "import torchvision.transforms.functional as TTF"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create dataloader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "class MatchedFilesDataset(Dataset):\n",
    "    def __init__(self, fnames, y, tfms, path):\n",
    "        self.path,self.fnames = path,fnames\n",
    "        self.open_fn = pil_loader\n",
    "        self.y=y\n",
    "        self.open_y_fn = pil_loader\n",
    "        assert(len(fnames)==len(y))\n",
    "        \n",
    "        self.n = self.get_n()\n",
    "        self.c = self.get_c()\n",
    "        self.tfms = tfms\n",
    "        \n",
    "    def get_x(self, i): return self.open_fn(os.path.join(self.path, self.fnames[i]))\n",
    "    def get_y(self, i): return self.open_y_fn(os.path.join(self.path, self.y[i]))\n",
    "    def get_n(self): return len(self.fnames)\n",
    "    def get_c(self): return 2\n",
    "    \n",
    "    def get(self, tfms, x, y):\n",
    "        for fn in tfms:\n",
    "            #pdb.set_trace()\n",
    "            x, y = fn(x, y)\n",
    "        return (x, y)\n",
    "    \n",
    "    def __getitem__(self, idx):\n",
    "        x,y = self.get_x(idx),self.get_y(idx)\n",
    "        return self.get(self.tfms, x, y)\n",
    "    \n",
    "    def __len__(self): return self.n\n",
    "\n",
    "    def resize_imgs(self, targ, new_path):\n",
    "        dest = resize_imgs(self.fnames, targ, self.path, new_path)\n",
    "        return self.__class__(self.fnames, self.y, self.transform, dest)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# Seems to speed up training by ~2%\n",
    "class DataPrefetcher():\n",
    "    def __init__(self, loader, stop_after=None):\n",
    "        self.loader = loader\n",
    "        self.dataset = loader.dataset\n",
    "        self.stream = torch.cuda.Stream()\n",
    "        self.stop_after = stop_after\n",
    "        self.next_input = None\n",
    "        self.next_target = None\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.loader)\n",
    "    \n",
    "    def preload(self):\n",
    "        try:\n",
    "            self.next_input, self.next_target = next(self.loaditer)\n",
    "        except StopIteration:\n",
    "            self.next_input = None\n",
    "            self.next_target = None\n",
    "            return\n",
    "        with torch.cuda.stream(self.stream):\n",
    "            self.next_input = self.next_input.cuda(async=True)\n",
    "            self.next_target = self.next_target.cuda(async=True)\n",
    "\n",
    "    def __iter__(self):\n",
    "        count = 0\n",
    "        self.loaditer = iter(self.loader)\n",
    "        self.preload()\n",
    "        while self.next_input is not None:\n",
    "            torch.cuda.current_stream().wait_stream(self.stream)\n",
    "            input = self.next_input\n",
    "            target = self.next_target\n",
    "            self.preload()\n",
    "            count += 1\n",
    "            yield input, target\n",
    "            if type(self.stop_after) is int and (count > self.stop_after):\n",
    "                break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def crop_bg_pil(x,y):\n",
    "    w, h = x.size\n",
    "    top = int(h/3.75)\n",
    "    bot = int(h*.9 + h/150)\n",
    "    pad_right=32-w%32\n",
    "    if pad_right == 32: pad_right = 0\n",
    "    return TTF.crop(x, top, 0, bot-top, w+pad_right), TTF.crop(y, top, 0, bot-top, w+pad_right)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "class RHF(object):\n",
    "    def __init__(self, p=0.5): self.p = p\n",
    "    def __call__(self, x, y):\n",
    "        if random.random() < self.p:\n",
    "            return TTF.hflip(x), TTF.hflip(y)\n",
    "        return x,y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "class RR(object):\n",
    "    def __init__(self, degrees=2): self.degrees = degrees\n",
    "    def __call__(self, x, y):\n",
    "        angle = random.uniform(-self.degrees, self.degrees)\n",
    "        return TTF.rotate(x, angle), TTF.rotate(y, angle)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def tfm_x_wrapper(tfm):\n",
    "    return lambda x,y: (tfm(x), y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "class RC():\n",
    "    def __init__(self, targ_sz):\n",
    "        self.targ_sz = targ_sz\n",
    "\n",
    "    def __call__(self, x, y):\n",
    "        rand_w = random.uniform(0, 1)\n",
    "        rand_h = random.uniform(0, 1)\n",
    "        w,h = x.size\n",
    "        t_w,t_h = self.targ_sz\n",
    "        start_x = np.floor(rand_w*(w-t_w)).astype(int)\n",
    "        start_y = np.floor(rand_h*(h-t_h)).astype(int)\n",
    "        return TTF.crop(x, start_y, start_x, t_h, t_w), TTF.crop(y, start_y, start_x, t_h, t_w)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def convert_y(y_img):\n",
    "    yr = (y_img==ROADS) | (y_img==ROAD_LINES)\n",
    "    yc = (y_img==VEHICLES)\n",
    "    cutoff_y = int(yc.shape[0]*.875)\n",
    "    yc[cutoff_y:,:] = 0\n",
    "    rn = ~(yr | yc)\n",
    "    return torch.from_numpy(np.stack((rn,yc,yr)).astype(int))\n",
    "\n",
    "def xy_tensor(x,y):\n",
    "    y_img = np.array(y, np.int32, copy=False)\n",
    "    return TTF.to_tensor(x), convert_y(y_img[:,:,0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "class RRC(transforms.RandomResizedCrop):\n",
    "    def __call__(self, x, y):\n",
    "        i, j, h, w = self.get_params(x, self.scale, self.ratio)\n",
    "        x = TTF.resized_crop(x, i, j, h, w, self.size, self.interpolation)\n",
    "        y = TTF.resized_crop(y, i, j, h, w, self.size, self.interpolation)\n",
    "        return x, y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "def torch_loader(f_ext, data_path, bs, size, workers=7, random_crop=False, pseudo_label=False, val_folder=None):\n",
    "    # Data loading code\n",
    "    x_names = np.sort(np.array(glob(str(data_path/f'CameraRGB{f_ext}'/'*.png'))))\n",
    "    y_names = np.sort(np.array(glob(str(data_path/f'CameraSeg{f_ext}'/'*.png'))))\n",
    "\n",
    "    x_n = x_names.shape[0]\n",
    "    val_idxs = list(range(x_n-300, x_n))\n",
    "    \n",
    "    if pseudo_label:\n",
    "        x_names_test = np.sort(np.array(glob(f'../data/pseudo/CameraRGB{f_ext}/*.png')))\n",
    "        y_names_test = np.sort(np.array(glob(f'../data/pseudo/CameraSeg{f_ext}/*.png')))\n",
    "        x_names = np.concatenate((x_names, x_names_test))\n",
    "        x_names = np.concatenate((y_names, y_names_test))\n",
    "        print(f'Pseudo-Labels: {len(x_names_test)}')\n",
    "    if val_folder:\n",
    "        x_names_val = np.sort(np.array(glob(f'../data/{val_folder}/CameraRGB{f_ext}/*.png')))\n",
    "        y_names_val = np.sort(np.array(glob(f'../data/{val_folder}/CameraSeg{f_ext}/*.png')))\n",
    "        val_x,val_y = x_names_val, y_names_val\n",
    "        trn_x,trn_y = x_names, y_names\n",
    "        print(f'Val Labels:', len(val_x))\n",
    "    else:\n",
    "        ((val_x,trn_x),(val_y,trn_y)) = split_by_idx(val_idxs, x_names, y_names)\n",
    "    print(f'Val x:{len(val_x)}, y:{len(val_y)}')\n",
    "    print(f'Trn x:{len(trn_x)}, y:{len(trn_y)}')\n",
    "    print(f'All x:{len(x_names)}')\n",
    "    normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])\n",
    "    \n",
    "    train_tfms = [\n",
    "        crop_bg_pil,\n",
    "        tfm_x_wrapper(transforms.ColorJitter(.2,.2,.2)),\n",
    "#         tfm_x_wrapper(Lighting(0.1, __imagenet_pca['eigval'], __imagenet_pca['eigvec'])),\n",
    "        RR(),\n",
    "        RHF(),\n",
    "#         RC((size,size)),\n",
    "        xy_tensor,\n",
    "        tfm_x_wrapper(normalize),\n",
    "    ]\n",
    "    if random_crop:\n",
    "        train_tfms.insert(3,RRC(size, scale=(0.4, 1.0)))\n",
    "    train_dataset = MatchedFilesDataset(trn_x, trn_y, train_tfms, path='')\n",
    "    train_loader = torch.utils.data.DataLoader(\n",
    "        train_dataset, batch_size=bs, shuffle=False,\n",
    "#         train_dataset, batch_size=bs, shuffle=True,\n",
    "        num_workers=workers, pin_memory=True)\n",
    "\n",
    "    val_tfms = [\n",
    "        crop_bg_pil,\n",
    "        xy_tensor,\n",
    "        tfm_x_wrapper(normalize)\n",
    "    ]\n",
    "    val_dataset = MatchedFilesDataset(val_x, val_y, val_tfms, path='')\n",
    "    val_loader = torch.utils.data.DataLoader(\n",
    "        val_dataset, batch_size=bs, shuffle=False,\n",
    "        num_workers=workers, pin_memory=True)\n",
    "\n",
    "    train_loader = DataPrefetcher(train_loader)\n",
    "    val_loader = DataPrefetcher(val_loader)\n",
    "    \n",
    "    data = ModelData(data_path, train_loader, val_loader)\n",
    "    return data\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "def denorm(x):\n",
    "    x_np = x.cpu().numpy()\n",
    "    x_np = np.rollaxis(x_np, 0, 3)\n",
    "    mean=np.array([0.485, 0.456, 0.406])\n",
    "    std=np.array([0.229, 0.224, 0.225])\n",
    "    x_np = x_np*std+mean\n",
    "    return x_np"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## U-net (ish)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "from torchvision.models import vgg11_bn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "def vgg11(pre): return children(vgg11_bn(pre))[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_meta = {\n",
    "    resnet18:[8,6], resnet34:[8,6], resnet50:[8,6], resnet101:[8,6], resnet152:[8,6],\n",
    "    vgg11:[0,13], vgg16:[0,22], vgg19:[0,22],\n",
    "    resnext50:[8,6], resnext101:[8,6], resnext101_64:[8,6],\n",
    "    wrn:[8,6], inceptionresnet_2:[-2,9], inception_4:[-1,9],\n",
    "    dn121:[0,7], dn161:[0,7], dn169:[0,7], dn201:[0,7],\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_base(f):\n",
    "    cut,lr_cut = model_meta[f]\n",
    "    layers = cut_model(f(True), cut)\n",
    "    return nn.Sequential(*layers), lr_cut"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "class SaveFeatures():\n",
    "    features=None\n",
    "    def __init__(self, m): self.hook = m.register_forward_hook(self.hook_fn)\n",
    "    def hook_fn(self, module, input, output): self.features = output\n",
    "    def remove(self): self.hook.remove()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "class UnetBlock(nn.Module):\n",
    "    def __init__(self, up_in, x_in, n_out):\n",
    "        super().__init__()\n",
    "        up_out = x_out = n_out//2\n",
    "        self.x_conv  = nn.Conv2d(x_in,  x_out,  1)\n",
    "        self.tr_conv = nn.ConvTranspose2d(up_in, up_out, 2, stride=2)\n",
    "        self.bn = nn.BatchNorm2d(n_out)\n",
    "        \n",
    "    def forward(self, up_p, x_p):\n",
    "        up_p = self.tr_conv(up_p)\n",
    "        x_p = self.x_conv(x_p)\n",
    "        cat_p = torch.cat([up_p,x_p], dim=1)\n",
    "        return self.bn(F.relu(cat_p, inplace=True))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "class UnetModel():\n",
    "    def __init__(self,model,name='unet'):\n",
    "        self.model,self.name = model,name\n",
    "\n",
    "    def get_layer_groups(self, precompute):\n",
    "        if isinstance(self.model, FP16):\n",
    "            model = self.model.module\n",
    "        else:\n",
    "            model = self.model\n",
    "        lgs = list(split_by_idxs(children(model.rn), [model.lr_cut]))\n",
    "#         print('LGS:', lgs)\n",
    "#         print('Add:', children(model)[1:])\n",
    "        return lgs + [children(model)[1:]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "def car_f_p_r(pred, targs):\n",
    "#     p2 = F.sigmoid(pred)\n",
    "    _,idx = torch.max(pred, 1)\n",
    "    p2 = idx\n",
    "    return fbeta_score(p2==1, targs[:,1,:,:], beta=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "def rd_f(pred, targs):\n",
    "    _,idx = torch.max(pred, 1)\n",
    "#     p2 = F.sigmoid(pred)\n",
    "    p2 = idx\n",
    "    f,p,r = fbeta_score(p2==2, targs[:,2,:,:], beta=0.5)\n",
    "    return f"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "def fbeta_score(y_pred, y_true, beta, threshold=None, eps=1e-9):\n",
    "    beta2 = beta**2\n",
    "\n",
    "    if threshold:\n",
    "        y_pred = torch.ge(y_pred.float(), threshold).float()\n",
    "    else:\n",
    "        y_pred = y_pred.float()\n",
    "    y_true = y_true.float()\n",
    "\n",
    "    true_positive = (y_pred * y_true).sum()\n",
    "    precision = true_positive/(y_pred.sum()+(eps))\n",
    "    recall = true_positive/(y_true.sum()+eps)\n",
    "    \n",
    "    fb = (precision*recall)/(precision*beta2 + recall + eps)*(1+beta2)\n",
    "    \n",
    "    return fb, precision, recall"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "def new_acc(pred, targs):\n",
    "    _,idx = torch.max(pred, 1)\n",
    "    _,t_idx = torch.max(targs,1)\n",
    "#     idx = F.softmax(pred) > 0.5\n",
    "    return (idx == t_idx).float().mean()\n",
    "def dice_mult(pred, targs):\n",
    "#     pred = (pred>0).float()\n",
    "    mx,idx = torch.max(pred, 1)\n",
    "    pred = idx.float()\n",
    "    targs = targs.float()\n",
    "    return 2. * (pred*targs).sum() / (pred+targs).sum()\n",
    "def dice(pred, targs):\n",
    "    pred = (pred>0).float()\n",
    "    return 2. * (pred*targs).sum() / (pred+targs).sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "def dice_coeff_weight(pred, target, weight):\n",
    "    smooth = 1.\n",
    "    num,c,h,w = pred.shape\n",
    "    m1 = pred.view(num, c, -1)  # Flatten\n",
    "    m2 = target.view(num, c, -1)  # Flatten\n",
    "    intersection = (m1 * m2)\n",
    "    w = V(weight.view(1,-1,1))\n",
    "    i_w = (w*intersection).sum()\n",
    "    m1_w = (w*m1).sum()\n",
    "    m2_w = (w*m2).sum()\n",
    "    return (2. * i_w + smooth) / (m1_w + m2_w + smooth)\n",
    "\n",
    "def dice_coeff(pred, target):\n",
    "    smooth = 1.\n",
    "    num,c,h,w = pred.shape\n",
    "    m1 = pred.view(num, c, -1)  # Flatten\n",
    "    m2 = target.view(num, c, -1)  # Flatten\n",
    "    intersection = (m1 * m2).sum()\n",
    "    return (2. * intersection + smooth) / (m1.sum() + m2.sum() + smooth)\n",
    "\n",
    "\n",
    "class SoftDiceLoss(nn.Module):\n",
    "    def __init__(self, weight=None, size_average=True):\n",
    "        super(SoftDiceLoss, self).__init__()\n",
    "        self.weight = weight\n",
    "\n",
    "    def forward(self, logits, targets):\n",
    "        probs = F.softmax(logits)\n",
    "        num = targets.size(0)  # Number of batches\n",
    "        if isinstance(logits.data, torch.cuda.HalfTensor):\n",
    "            targets = targets.half()\n",
    "        else:\n",
    "            targets = targets.float()\n",
    "            \n",
    "        if self.weight is not None:\n",
    "            score = dice_coeff_weight(probs, targets, self.weight)\n",
    "        else:\n",
    "            score = dice_coeff(probs, targets)\n",
    "        score = 1 - score.sum() / num\n",
    "        return score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "from fastai.lm_rnn import repackage_var"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Unet34ModOld(nn.Module):\n",
    "    def __init__(self, f=resnet34):\n",
    "        super().__init__()\n",
    "        m_base, lr_cut = get_base(f)\n",
    "        self.rn = m_base\n",
    "        self.lr_cut = lr_cut\n",
    "        \n",
    "        \n",
    "        self.sz = np.array([3,7])\n",
    "        self.rn_k = 512\n",
    "        self.nl = 1\n",
    "        self.rn_input = int(self.rn_k * self.sz[0] * self.sz[1])\n",
    "        self.lstm_input = 256\n",
    "        \n",
    "        \n",
    "        self.bidirectional = True\n",
    "        self.bmult = 2 if self.bidirectional else 1\n",
    "        self.n_hidden = int(self.lstm_input / self.bmult)\n",
    "        self.lstm = torch.nn.LSTM(self.lstm_input, self.n_hidden, self.nl, dropout=0.2, bidirectional=self.bidirectional)\n",
    "        self.bs = 1\n",
    "#         self.reset_all_hidden(self.bs)\n",
    "        self.reset_all_hidden(self.bs)\n",
    "    \n",
    "        self.ln1 = nn.Linear(self.rn_input, self.lstm_input)\n",
    "        self.ln2 = nn.Linear(self.lstm_input, self.rn_input)\n",
    "    \n",
    "    \n",
    "        self.sfs = [SaveFeatures(self.rn[i]) for i in [2,4,5,6]]\n",
    "        self.up1 = UnetBlock(512,256,256)\n",
    "        self.up2 = UnetBlock(256,128,256)\n",
    "        self.up3 = UnetBlock(256,64,256)\n",
    "        self.up4 = UnetBlock(256,64,256)\n",
    "        self.up5 = UnetBlock(256,16,16)\n",
    "        self.up6 = nn.ConvTranspose2d(16, 3, 1)\n",
    "        self.x_skip = nn.Sequential(\n",
    "            nn.Conv2d(3,16,1),\n",
    "            nn.BatchNorm2d(16),\n",
    "            nn.ReLU(inplace=True)\n",
    "        )\n",
    "        \n",
    "        \n",
    "    def forward(self,x):\n",
    "        x_skip = self.x_skip(x)\n",
    "        x = self.rn(x)\n",
    "        pre_shape = x.shape\n",
    "        x = x.view(x.shape[0], -1)\n",
    "        x = self.ln1(x)\n",
    "        x = x.unsqueeze_(1)\n",
    "#         print('Resnet output:', x.shape)\n",
    "        x, h1 = self.lstm(x, self.h1)\n",
    "        self.h1 = repackage_var(h1)\n",
    "        x = x.squeeze_()\n",
    "        x = self.ln2(x)\n",
    "#         print('RNN output:', x.shape)\n",
    "        x = x.view(*pre_shape)\n",
    "        x = self.up1(x, self.sfs[3].features)\n",
    "        x = self.up2(x, self.sfs[2].features)\n",
    "        x = self.up3(x, self.sfs[1].features)\n",
    "        x = self.up4(x, self.sfs[0].features)\n",
    "        x = self.up5(x, x_skip)\n",
    "        x = self.up6(x)\n",
    "        return torch.squeeze(x)\n",
    "    \n",
    "    def close(self):\n",
    "        for sf in self.sfs: sf.remove()\n",
    "            \n",
    "            \n",
    "    def reset_all_hidden(self, bs):\n",
    "        self.h1 = self.init_hidden(bs)\n",
    "        \n",
    "    def init_hidden(self, bs):\n",
    "        h1 = torch.autograd.Variable(torch.zeros(self.nl*self.bmult, bs, self.n_hidden))\n",
    "        h2 = torch.autograd.Variable(torch.zeros(self.nl*self.bmult, bs, self.n_hidden))\n",
    "#         if cuda_enabled:\n",
    "        return (h1.cuda(), h2.cuda())\n",
    "#         return h1, h2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Unet34Mod(nn.Module):\n",
    "    def __init__(self, sz, f=resnet34):\n",
    "        super().__init__()\n",
    "        m_base, lr_cut = get_base(f)\n",
    "        self.rn = m_base\n",
    "        self.lr_cut = lr_cut\n",
    "        \n",
    "        \n",
    "        self.sz = sz\n",
    "#         self.sz = np.array([3,7])\n",
    "        self.rn_k = 512\n",
    "        self.nl = 1\n",
    "        self.rn_input = int(self.rn_k * self.sz[0] * self.sz[1])\n",
    "        self.lstm_output = 512\n",
    "        \n",
    "        \n",
    "        self.bidirectional = False\n",
    "        self.bmult = 2 if self.bidirectional else 1\n",
    "        self.n_hidden = int(self.lstm_output / self.bmult)\n",
    "        self.lstm = torch.nn.LSTM(self.rn_input, self.n_hidden, self.nl, dropout=0.2, bidirectional=self.bidirectional)\n",
    "        self.bs = 1\n",
    "#         self.reset_all_hidden(self.bs)\n",
    "        self.reset_all_hidden(self.bs)\n",
    "    \n",
    "        self.ln1 = nn.Linear(self.lstm_output, self.rn_input)\n",
    "#         self.ln2 = nn.Linear(self.lstm_input, self.rn_input)\n",
    "    \n",
    "    \n",
    "        self.sfs = [SaveFeatures(self.rn[i]) for i in [2,4,5,6]]\n",
    "        self.up1 = UnetBlock(1024,256,512)\n",
    "        self.up2 = UnetBlock(512,128,256)\n",
    "        self.up3 = UnetBlock(256,64,128)\n",
    "        self.up4 = UnetBlock(128,64,64)\n",
    "        self.up5 = UnetBlock(64,32,32)\n",
    "        self.up6 = nn.ConvTranspose2d(32, 3, 1)\n",
    "        self.x_skip = nn.Sequential(\n",
    "            nn.Conv2d(3,32,1),\n",
    "            nn.BatchNorm2d(32),\n",
    "            nn.ReLU(inplace=True)\n",
    "        )\n",
    "        \n",
    "        \n",
    "    def forward(self,x):\n",
    "        x_skip = self.x_skip(x)\n",
    "        x = self.rn(x)\n",
    "        \n",
    "        x_rnn = x.view(x.shape[0], 1, -1)\n",
    "        x_rnn, h1 = self.lstm(x_rnn, self.h1)\n",
    "        self.h1 = repackage_var(h1)\n",
    "        x_rnn = x_rnn.squeeze()\n",
    "        x_rnn = self.ln1(x_rnn)\n",
    "        x_rnn = x.view(*x.shape)\n",
    "        x = torch.cat((x, x_rnn), dim=1)\n",
    "        x = self.up1(x, self.sfs[3].features)\n",
    "        x = self.up2(x, self.sfs[2].features)\n",
    "        x = self.up3(x, self.sfs[1].features)\n",
    "        x = self.up4(x, self.sfs[0].features)\n",
    "        x = self.up5(x, x_skip)\n",
    "        x = self.up6(x)\n",
    "        return torch.squeeze(x)\n",
    "    \n",
    "    def close(self):\n",
    "        for sf in self.sfs: sf.remove()\n",
    "            \n",
    "            \n",
    "    def reset_all_hidden(self, bs):\n",
    "        self.h1 = self.init_hidden(bs)\n",
    "        \n",
    "    def init_hidden(self, bs):\n",
    "        h1 = torch.autograd.Variable(torch.zeros(self.nl*self.bmult, bs, self.n_hidden))\n",
    "        h2 = torch.autograd.Variable(torch.zeros(self.nl*self.bmult, bs, self.n_hidden))\n",
    "#         if cuda_enabled:\n",
    "        return (h1.cuda(), h2.cuda())\n",
    "#         return h1, h2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_learner(md, sz, half=False):\n",
    "    m = Unet34Mod(sz)\n",
    "    m = to_gpu(m)\n",
    "    models = UnetModel(m)\n",
    "    learn = ConvLearner(md, models)\n",
    "    learn.opt_fn=optim.Adam\n",
    "    class_weights = torch.cuda.FloatTensor([1,150,1])\n",
    "    if half:\n",
    "        class_weights = class_weights.half()\n",
    "        learn.half()\n",
    "#     learn.crit=nn.CrossEntropyLoss(weight=class_weights)\n",
    "    learn.crit=SoftDiceLoss(weight=class_weights)\n",
    "    learn.metrics=[new_acc, rd_f, car_f_p_r]\n",
    "    return learn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Automatic pdb calling has been turned OFF\n"
     ]
    }
   ],
   "source": [
    "%pdb off"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val Labels: 1000\n",
      "Val x:1000, y:1000\n",
      "Trn x:4600, y:4600\n",
      "All x:4600\n"
     ]
    }
   ],
   "source": [
    "ext = '-150'\n",
    "sz = 96\n",
    "bs = 16\n",
    "md = torch_loader(ext, PATH, bs, sz, workers, random_crop, pseudo_label, val_folder)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/paperspace/anaconda3/envs/lyft/lib/python3.6/site-packages/torch/nn/modules/rnn.py:38: UserWarning: dropout option adds dropout after all but last recurrent layer, so non-zero dropout expects num_layers greater than 1, but got dropout=0.2 and num_layers=1\n",
      "  \"num_layers={}\".format(dropout, num_layers))\n"
     ]
    }
   ],
   "source": [
    "learn = get_learner(md)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [],
   "source": [
    "lr=1e-2\n",
    "wd=1e-5\n",
    "\n",
    "lrs = np.array([lr/200,lr/20,lr])/2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Automatic pdb calling has been turned OFF\n"
     ]
    }
   ],
   "source": [
    "%pdb off"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7d5e0e02e99a42de81eb5eb70f121683",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   new_acc    rd_f       car_f_p_r  \n",
      "    0      0.956948   0.967871   0.945131   0.928249   0.005482   0.447644   0.004424  \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[0.9678710851669311,\n",
       " 0.9451313247680664,\n",
       " 0.9282494387626647,\n",
       " 0.005481927261222154,\n",
       " 0.4476444444656372,\n",
       " 0.004424393789842725]"
      ]
     },
     "execution_count": 83,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learn.fit(lr,1,wds=wd,cycle_len=1,use_clr_beta=(20,10,0.95,0.85))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Settings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val Labels: 1000\n",
      "Val x:1000, y:1000\n",
      "Trn x:4600, y:4600\n",
      "All x:4600\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/paperspace/anaconda3/envs/lyft/lib/python3.6/site-packages/torch/nn/modules/rnn.py:38: UserWarning: dropout option adds dropout after all but last recurrent layer, so non-zero dropout expects num_layers greater than 1, but got dropout=0.2 and num_layers=1\n",
      "  \"num_layers={}\".format(dropout, num_layers))\n"
     ]
    }
   ],
   "source": [
    "ext = '-150'\n",
    "sz = 96\n",
    "bs = 128\n",
    "md = torch_loader(ext, PATH, bs, sz, workers, random_crop, pseudo_label, val_folder)\n",
    "\n",
    "learn = get_learner(md)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# learn.half()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.freeze_to(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%pdb on"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "69c643c7412b42f4bcf0dc6f24f315ce",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   new_acc    rd_f       car_f_p_r  \n",
      "    0      0.998252   0.998946   0.29868    0.347303   0.000458   0.004717   0.000374  \n",
      "\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZgAAAEOCAYAAAC0BAELAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3Xl8XXWd//HXJ3uzdMvW0n2HAC1LWKqUFqGVTRAoAgrCiDA6AyqKCj+XQRxEHQZHRxEqMoKjIouDFNGWrYu0LCnYQlua7nShbZJuSdrsn98f95ReQtombU7OTe77+Xjcxz33e5b7Oaftffd7VnN3REREOltK1AWIiEjPpIAREZFQKGBERCQUChgREQmFAkZEREKhgBERkVAoYEREJBQKGBERCYUCRkREQqGAERGRUKRFXUCUCgoKfPjw4VGXISLSrSxatKjS3QsPNV1SB8zw4cMpKyuLugwRkW7FzNa3ZzrtIhMRkVAoYEREJBQKGBERCYUCRkREQqGAERGRUChgREQkFEl9mnJ3VFvfxPqqPeyua+SU4f1JTbGoSxIRaZMCJgFV1zWyvmoP66pqY++Vte9/3lZd//50xwzszR2fKOG0kfkRVisi0rZQA8bMzgV+CqQCD7r7D1uNHwY8BBQC24Gr3X1jMO5HwAXBpN939z8G7R8D7gEygEXA9e7eFIybAvwXkA5UuvvkMNfvcLk7u/c2sX57LeuCANkXJuuraqmsafjA9EV5mQzPz2Hy2EKGF+QwLD+busYW7p29gitmvMIF4wdy+3lHM7hfdkRrJCLyYebu4SzYLBUoB6YCG4HXgavcfVncNI8Dz7j7w0Fw/JO7X2NmFwBfAc4DMoG5wMeAGmA9cLa7l5vZncB6d/+1mfUFFgDnuvu7Zlbk7tsOVmNpaal31pX89U3NbK9toKqmgcqaeqpqGqiqrQ8+7x+uqqmnsraBhqaWD8w/oHcWwwuyGZ6fw7D8HIbnZzMsPxYmOZlt/z9gb0MzD8xbzf1zV+MOX5g8ii9MHkWvjNROWScRkbaY2SJ3Lz3UdGH2YE4FVrn7mqCgR4GLgWVx05QAtwTDLwFPxbXPDXomTWa2GDg3mKbe3cuD6Z4Dbgd+DXwa+JO7vwtwqHA5EmXrtvOr+WuCEIkFSnVdU5vTZqSlUJCTQX5uJvm5GYwtzqMgN4OC3EyG5scCZWj/7MMKhV4ZqXzlnLFcXjqEu59dzk9fWMnjZRu4/fxjuHD8QMx0fEZEohNmwAwCNsR93gic1mqaxcBlxHajXQLkmVl+0P5vZnYvkA2cRSyYKoF0Myt19zJgOjAkWNbYYNwcIA/4qbs/0rooM7sRuBFg6NChh7ViexubWVe5h/zcDI4b1If8nAwKcoMQCcJk3+ecjNTQf+gH9e3Fzz99Ep+duJ07nl7KzX94k98uXM93P1HCcYP6hPrdIiIHEmbAtPWr2np/3K3Az83sOmAesAlocvfZZnYKsV1eFcDCoN3N7ErgJ2aWCcwG9nUd0oCTgbOBXsBCM3slrrcTK8B9BjADYrvIDmfFJo0pZNYth7yRaJc7dUR/Zt58Bo+VbeA/Zq3gEz//O1eeMpRbp40lPzcz6vJEJMmEGTAb2d+7ABgMbI6fwN03A5cCmFkucJm77wrG3QXcFYz7PbAyaF8ITArapxHruez7vkp3rwVqzWweMIHYcaCkkZpiXHXqUM4/fiA/fX4ljyxcxzNLNvOVc8by2YnDSE/VpU8i0jXC/LV5HRhjZiPMLAO4Eng6fgIzKzCzfTXcTuyMMswsNdhVhpmNB8YT661gZkXBeybwTeD+YP4/A5PMLM3Msontjlse4voltD690vnuJ0r421cmceLQfnz/mWWc+1/zmFteEXVpIpIkQguY4AD9TcAsYj/0j7n7UjO708wuCiabAqwws3KgmKDHQuw04/lmtozY7qyr952KDHzdzJYDS4CZ7v5i8H3Lgb8F7a8ROy367bDWr7sYXZTHw/90Cr++tpTmFufah17j+t+8ztrK2qhLE5EeLrTTlLuDzjxNuTuob2rmNy+v479fXEVtQxPD83MYW5zLuAG9GVecx7gBuQzPzyEtQXejNbc4TS0tNDU7jc0t1NQ3xV51TVTve69roqa+sY22fZ8bqalvor6phQmD+zJ5bCGTxxUysiBHZ92JtFN7T1NWwCRRwOyzrbqOR1/bwDtbdvPOlmrWVdbSEvw1yEhNYWRhDuMG5DG2OI+jg/dBfXuR0oHb0jQ0tVBVW09F9f7Xtrjhqtp6GppaaGrxWGAEwdHcEguPpuC9OW58R/6qmkFuRhp5WWnkZqWRm5lGblY6eZmxNjN4de121lTEenKD+/WKhc3YQj4yuoDcA1x7JCIKmHZJ1oBpra6xmdUVNZRvrWbFln3v1Wzauff9aXIyUhlTnMe44jzGDshjeH42NfVNHwqO2Oc6duxpbPO7+manUxhcE5SZlkpaipGWaqSlppCeYqSmpJCeGrSlpATjg7aUlKA91paXGR8eafTOSiM3M53crDSy01PbFYgbtu9hbnkFc8srWLCqktqGZtJSjNLh/Zg8tojJYws5ZmCeejcicRQw7aCAObjqukbKt+4PnPKtsVfrW9lkpqVQmJdJUV4mhfteuVn7h4Nx+0IlUTU0tbBo/Q7mllcwr7yCZe/tBqAwL/P93s0Zowvol5MRSX3NLc6mHXtZXVHD+qpasjPTPrDN83MydfNT6RIKmHZQwByeypp63t2+hz690inMyyQvM61H/g9/2+465q2sZG55BfNXVrBzTyMpBuMH9+XMMQUML8gJwjMWpv2y0ztlO9Q1NrO2spbVFTWs2rb/tbaylvpWtxiKl2KQn5tJYe4Hg33/8P7Q74oLgKXnUsC0gwJG2qu5xVmycef7u9P+sWHnh44Jpada3I971vs/8EW9Yz/6Rb2DH/jcTDLSUti1p5FVFdWs3lbLqiBMVlfUsGH7nvePiZnFjg+NLsxldFEuo4L3Yfk57G1opqKmru3jXDX725paPvxvPDsjlYF9shjUL5tBfXsxuF/sNahvLwb160VRXpZ6Q3JACph2UMDI4aqtb2JbdT3bdte9/6O+Le4Y1P6TGRranD83M42a+v33r8tIS2FkQQ6jinIZXZj7/vvIwhyy0g9/t2JLi7Nzb+P+EAoCaevuejbv3MumnXvZtGPvh+pMTzUG9tkfOPHhM6RfNgP6ZOmi3SSWCDe7FOmxcjLTGJGZxoiCnINO19jcQlVNA9uq69i2O9az2La7nh17GhjYJ4vRRbEeyeB+2aH0GFJSjP45GfTPyWDcgLwDTrenoYnNO/eycUcsdDbuiAXPpp17+fvKSrZW132gx/b+rsKxhZw5poAThvRN2NPbJTrqwagHI3JIDU0tvLcrFjobd+xlXVUtC9dUsXjDTloc8jLTmDgqPwicQobm69lEPZl6MCLSaTLSUoLnE32wx7ZrTyMLVlcyb2UF88ormb1sKwDD8rM5c0whk8YUMHFUPnlZ6VGULRFTD0Y9GJFO4e6sraxl/spK5pVXsHBNFXsamklNMU4a2jcWOGMLOX5QH51A0M3pIH87KGBEwtPQ1MIb7+5gXnkF81dW8tamXUDsYtuPji7glnPGMLrowMeFJHEpYNpBASPSdapq6nl5dRXzyit4LtiV9tB1pZw8rH/ElUlHtTdgdNqHiHSJ/NxMLppwFPdcPoFnbj6D/jkZfPpXr/J8EDbS8yhgRKTLDemfzRNfmMjRA/L45/9dxGOvbzj0TNLtKGBEJBL5uZn8/obT+ejoAr7x5BJ+8dIqknmXfU+kgBGRyORkpvHgZ0v55AlH8R+zVvC9mctoaePWNtI96ToYEYlURloK937qBApyM3nw72upqKnn3k9NSOg7b0v7KGBEJHIpKca3LyyhqHcmP3j2HXbuaeD+q0/WBZrdnHaRiUjCuPHMUfzn5RN4Zc12rpzxChXV9VGXJEdAASMiCeWykwfz4LWlrKmoZfr9C1hfVRt1SXKYFDAiknDOGlfE7284jd17G7nslwt4O7gLgHQvChgRSUgnDu3HE1/8CJlpqVzxwEJeXlUZdUnSQQoYEUlYowpzefKLH2Fwv2yu+5/XmLl4c9QlSQcoYEQkoQ3ok8VjX5jIiUP68aVH3+Q3L6+NuiRpJwWMiCS8Pr3SeeT6U5l6TDF3zFzGj//2jq767wYUMCLSLWSlp/LLq0/mqlOHct+c1dw3Z3XUJckhKGBEpNtITTF+cMlxnH/8AH76/EpWbq2OuiQ5CAWMiHQrZsb3LjqO7MxUvvHkEpp177KEpYARkW6nMC+T715Ywpvv7uQ3C9ZFXY4cgAJGRLqlS04cxJRxhdwzawXvVu2JuhxpgwJGRLolM+MHlxxPaopx+/8t0VllCUgBIyLd1lF9e3HbeUfz8qoq/qinYiYcBYyIdGufPnUop43oz11/Wc6WXXVRlyNxFDAi0q2lpBg/vGw8Dc0tfPupt7SrLIGEGjBmdq6ZrTCzVWZ2Wxvjh5nZC2a2xMzmmNnguHE/MrO3g9cVce0fM7M3gvaHzSyt1TJPMbNmM5se5rqJSOIYUZDD16aN5fnl25i55L2oy5FAaAFjZqnAL4DzgBLgKjMraTXZPcAj7j4euBO4O5j3AuAk4ATgNODrZtbbzFKAh4Er3f04YD1wbavv/BEwK6z1EpHE9LmPjmDC4D7c8fRSqmr0oLJEEGYP5lRglbuvcfcG4FHg4lbTlAAvBMMvxY0vAea6e5O71wKLgXOBfKDe3cuD6Z4DLotb3s3Ak8C2zl4ZEUlsaakp/Hj6BKrrGvnezGVRlyOEGzCDgPjTOjYGbfEWsz8gLgHyzCw/aD/PzLLNrAA4CxgCVALpZlYazDM9aMfMBgXLuD+EdRGRbmDcgDz+9azRPL14M88v2xp1OUkvzICxNtpaH327FZhsZm8Ck4FNQJO7zwaeBRYAfwAWBu0OXAn8xMxeA6qBpmBZ/wV8092bD1qU2Y1mVmZmZRUVFYe5aiKSqP5lymjGFefx7afeZnddY9TlJLUwA2YjQe8iMBj4wNOC3H2zu1/q7icC3wradgXvd7n7Ce4+lVhYrQzaF7r7JHc/FZi3rx0oBR41s3XEejb3mdknWxfl7jPcvdTdSwsLCztxdUUkEWSkpfDj6ePZVl3H3c8uj7qcpBZmwLwOjDGzEWaWQazn8XT8BGZWEBy4B7gdeChoTw12lWFm44HxwOzgc1Hwngl8k2CXmLuPcPfh7j4ceAL4F3d/KsT1E5EENWFIXz4/aSR/eG0DC/So5ciEFjDu3gTcROyMruXAY+6+1MzuNLOLgsmmACvMrBwoBu4K2tOB+Wa2DJgBXB0sD2JnlC0HlgAz3f3FsNZBRLqvW84Zy/D8bG7701vsaWg69AzS6SyZL0oqLS31srKyqMsQkZC8sqaKK2e8wvVnjOA7F7a+SkIOl5ktcvfSQ02nK/lFpMc6fWQ+nzltKA+9vJY33t0RdTlJRwEjIj3abecdzcDeWXzjiSXUNx30JFPpZAoYEenR8rLSuevS41m1rYZfvLgq6nKSigJGRHq8s8YVcemJg7hvzmqWbd4ddTlJQwEjIknhOxeW0Dc7nW8+uYSm5paoy0kKChgRSQr9cjL43kXH8damXfxq/tqoy0kKChgRSRrnHz+Ajx9bzE+eL2dNRU3U5fR4ChgRSRpmxvcvPg4DHl6wLupyejwFjIgklaLeWUwaU8hzy7bq6ZchU8CISNKZdmwxm3fVsVRnlIVKASMiSefso4tIMZitZ8aESgEjIkknPzeT0mH9mb10S9Sl9GgKGBFJStOOLeadLdW8W7Un6lJ6LAWMiCSlqSXFAMxepl5MWBQwIpKUhuXncPSAPJ7TcZjQKGBEJGlNLSnm9XXb2V7bEHUpPZICRkSS1rSSAbQ4vLBcvZgwKGBEJGkdN6g3A/tk6XTlkChgRCRpmRnTSoqZv7KCvQ16GFlnU8CISFKbWjKAusYW5q+siLqUHkcBIyJJ7bSR/cnLStNushAoYEQkqaWnpnD20UW8sHyrHkTWyRQwIpL0ph07gB17GilbvyPqUnoUBYyIJL0zxxaSkZaiiy47mQJGRJJebmYaHx2Vz+xlW/SMmE6kgBERIbabbMP2vbyzpTrqUnoMBYyICHD2MUWYweyl2k3WWRQwIiJAUV4WJw3tx3PLdXflzqKAEREJTC0p5u1Nu9m0c2/UpfQIChgRkcC04Bkxz+lJl51CASMiEhhZmMvoolxd1d9JFDAiInGmlRTz6trt7NrTGHUp3Z4CRkQkzrRjB9Dc4ry4Qr2YI6WAERGJM35QH4ryMnW6cidQwIiIxElJMaaWFDO3vIK6Rj0j5kiEGjBmdq6ZrTCzVWZ2Wxvjh5nZC2a2xMzmmNnguHE/MrO3g9cVce0fM7M3gvaHzSwtaP9MsJwlZrbAzCaEuW4i0nNNO3YAexqaeXlVZdSldGuhBYyZpQK/AM4DSoCrzKyk1WT3AI+4+3jgTuDuYN4LgJOAE4DTgK+bWW8zSwEeBq509+OA9cC1wbLWApODZX0fmBHWuolIzzZxZD55mWm6+eURCrMHcyqwyt3XuHsD8ChwcatpSoAXguGX4saXAHPdvcnda4HFwLlAPlDv7uXBdM8BlwG4+wJ333ev7VeA93tDIiIdkZGWwuRxhTy/fCvNLbr55eEKM2AGARviPm8M2uItJggI4BIgz8zyg/bzzCzbzAqAs4AhQCWQbmalwTzTg/bWrgf+2lZRZnajmZWZWVlFhR6RKiJtm3bsACprGnjzXT0j5nC1K2DM7MvBLiozs18Hx0CmHWq2Ntpa/1fgVmCymb0JTAY2AU3uPht4FlgA/AFYGLQ7cCXwEzN7DagGmlrVehaxgPlmW0W5+wx3L3X30sLCwkOsgogkqynjCklPNV10eQTa24P5nLvvBqYBhcA/AT88xDwb+WDvYjCwOX4Cd9/s7pe6+4nAt4K2XcH7Xe5+grtPJRZWK4P2he4+yd1PBebtawcws/HAg8DF7l7VznUTEfmQ3lnpTBxVwOylekbM4WpvwOzrjZwP/I+7L6btHkq814ExZjbCzDKI9Tye/sBCzQqCA/cAtwMPBe2pwa6yfaExHpgdfC4K3jOJ9VLuDz4PBf4EXBN3jEZE5LBNKylmXdUeVm2ribqUbqm9AbPIzGYTC5hZZpYHtBxsBndvAm4CZgHLgcfcfamZ3WlmFwWTTQFWmFk5UAzcFbSnA/PNbBmxs8GuDpYHsTPKlgNLgJnu/mLQ/l1iJwHcZ2b/MLOydq6biEibpgY3v9RussNj7en6Bb2ME4A17r7TzPoDg919SdgFhqm0tNTLypRDInJgF//iZXDnzzedEXUpCcPMFrl76aGma28PZiKwIgiXq4FvA7uOpEARke5gWkkxizfuYsuuuqhL6XbaGzC/BPYEV8d/g9gFjo+EVpWISIL4+LHBM2KWazdZR7U3YPadInwx8FN3/ymQF15ZIiKJYVRhLiMKcpith5B1WHsDptrMbgeuAf4S3AYmPbyyREQSg5kxraSYhaur2LVXz4jpiPYGzBVAPbHrYbYQuyL/P0KrSkQkgUw7tpimFmfOim1Rl9KttCtgglD5HdDHzC4E6txdx2BEJCmcMKQfBbmZuvllB7X3VjGfAl4DLgc+BbxqZtPDLExEJFGkphhTS4qYs6KC+iY9I6a92ruL7FvAKe5+rbt/ltidkr8TXlkiIollakkxNfVNLFytu1C1V3sDJsXd43c+VnVgXhGRbu8jowrIzkjVVf0d0N6Q+JuZzTKz68zsOuAvxO52LCKSFLLSU5kyrpDnl22lRc+IaZf2HuT/OrF7go0HJgAz3L3N2+GLiPRU00oGsK26nsUbd0ZdSreQ1t4J3f1J4MkQaxERSWhnjSsiNSX2jJgTh/aLupyEd9AejJlVm9nuNl7VZra7q4oUEUkEfbLTOX1kf13V304HDRh3z3P33m288ty9d1cVKSKSKKaVDGB1RS2rK/SMmEPRmWAiIh2w7xkxzyx+L+JKEp8CRkSkA47q24sp4wr57SvrqWvURZcHo4AREemgGyeNpLKmnqfe3BR1KQlNASMi0kETR+Vz3KDezJi/RtfEHIQCRkSkg8yMGyaNZE1FLS++ozssH4gCRkTkMFxw/EAG9e3FjHlroi4lYSlgREQOQ1pqCtefMYLX1m3nzXd3RF1OQlLAiIgcpitOGULvrDR+NV+9mLYoYEREDlNOZhpXnz6Mv729hfVVtVGXk3AUMCIiR+C6jwwnLSWFB+evjbqUhKOAERE5AkW9s/jkiUfx+KINbK9tiLqchKKAERE5QjeeOZK6xhZ+u3B91KUkFAWMiMgRGl2Ux9lHF/HIwnW6fUwcBYyISCe44cyRVNU28OQbG6MuJWEoYEREOsFpI/ozYXAfHpy/lmbdPgZQwIiIdAoz44YzR7K2spbnl2+NupyEoIAREekk5x47gCH9dfuYfRQwIiKdJC01hc+fMZJF63ewaP32qMuJnAJGRKQTXV46mL7Z6erFoIAREelU2RlpXHP6MGYv28qaipqoy4lUqAFjZuea2QozW2Vmt7UxfpiZvWBmS8xsjpkNjhv3IzN7O3hdEdf+MTN7I2h/2MzSgnYzs58F37XEzE4Kc91ERA7ksxOHk56awoN/T+7bx4QWMGaWCvwCOA8oAa4ys5JWk90DPOLu44E7gbuDeS8ATgJOAE4Dvm5mvc0sBXgYuNLdjwPWA9cGyzoPGBO8bgR+Gda6iYgcTGFeJpedNIgnF22ksqY+6nIiE2YP5lRglbuvcfcG4FHg4lbTlAAvBMMvxY0vAea6e5O71wKLgXOBfKDe3cuD6Z4DLguGLyYWVu7urwB9zWxgGCsmInIon580kobmFh5J4tvHhBkwg4ANcZ83Bm3xFrM/IC4B8swsP2g/z8yyzawAOAsYAlQC6WZWGswzPWhv7/eJiHSJUYW5nHNMMb9duI69Dcl5+5gwA8baaGt9eeutwGQzexOYDGwCmtx9NvAssAD4A7AwaHfgSuAnZvYaUA00deD7MLMbzazMzMoqKioOY7VERNrnxjNHsmNPI08s2nDoiXugMANmI/t7FwCDgc3xE7j7Zne/1N1PBL4VtO0K3u9y9xPcfSqx8FgZtC9090nufiowb197e74vmH+Gu5e6e2lhYWFnrKeISJtKh/XjxKF9efDvyXn7mDAD5nVgjJmNMLMMYj2Pp+MnMLOC4MA9wO3AQ0F7arCrDDMbD4wHZgefi4L3TOCbwP3B/E8Dnw3OJjsd2OXu74W4fiIiB2Vm3DhpJOur9jB76Zaoy+lyoQWMuzcBNwGzgOXAY+6+1MzuNLOLgsmmACvMrBwoBu4K2tOB+Wa2DJgBXB0sD2JnlC0HlgAz3f3FoP1ZYA2wCvgV8C9hrZuISHtNO3YAw/OzeWDeGmJ7+ZOHJdsKxystLfWysrKoyxCRHu63r6znO0+9zWP/PJFTR/SPupwjZmaL3L30UNPpSn4RkZBNP2kw/XMyku72MQoYEZGQ9cpI5ZrTh/H88q2s2pY8t49RwIiIdIHPThxGZloKD85Pnl6MAkZEpAvk52Yy/eTB/OmNTWyrrou6nC6hgBER6SKfnzSSxpYWHlmQHLePUcCIiHSREQU5fLxkAL99ZT17GpoOPUM3p4AREelCN5w5kl17G5PiJpgKGBGRLnTysH5MLSnmJ8+V9/gzyhQwIiJd7K5LjiM7I5WvPb6YpuaWqMsJjQJGRKSLFeVl8e+fPJ7FG3byQA+++FIBIyISgQvGD+TC8QP5r+fLWbZ5d9TlhEIBIyISke9ffBx9emXw1cf+QUNTz9tVpoAREYlIv5wMfnjp8byzpZqfvbDy0DN0MwoYEZEInVNSzOUnD+a+Oat4890dUZfTqRQwIiIR+84nShjQO4uvPb6YusbmqMvpNAoYEZGI9c5K58fTJ7CmopZ7Zq2IupxOo4AREUkAZ4wp4JrTh/Hrl9fy6pqqqMvpFAoYEZEEcdt5RzO0fza3PrGY2vruf68yBYyISILIyUzjnssnsHHHXn7w7PKoyzliChgRkQRyyvD+3DBpJL979V3mlldEXc4RUcCIiCSYr04dy+iiXL75xBJ27W2MupzDpoAREUkwWemp3PupCVTU1PO9mUujLuewKWBERBLQ+MF9+dcpo/jTG5uYtXRL1OUcFgWMiEiCuuljYygZ2Jtv/d9bbK9tiLqcDlPAiIgkqIy0FO69YgK79jby7afewt2jLqlDFDAiIgns6AG9uWXqWJ59awszl7wXdTkdooAREUlwN04ayYlD+/Kdp95m2+66qMtpNwWMiEiCS0tN4T8vn0B9UzO3/an77CpTwIiIdAMjC3P5xseP5sV3tvF42caoy2kXBYyISDdx3UeGc9qI/tz5zDI27tgTdTmHpIAREekmUlKMey6fQIs7dzy9LOpyDkkBIyLSjQzpn83NHxvD88u3Mn9lYt+rTAEjItLNfO6M4QzLz+bOmctoam6JupwDUsCIiHQzmWmp/L/zj2Hlthp+9+q7UZdzQAoYEZFuaFpJMR8dnc+9z5WzI0FvIxNqwJjZuWa2wsxWmdltbYwfZmYvmNkSM5tjZoPjxv3IzN4OXlfEtZ9tZm+Y2T/M7O9mNjpoH2pmL5nZm8Hyzg9z3UREomRmfPfCY6mua+Qnz5dHXU6bQgsYM0sFfgGcB5QAV5lZSavJ7gEecffxwJ3A3cG8FwAnAScApwFfN7PewTy/BD7j7icAvwe+HbR/G3jM3U8ErgTuC2vdREQSwbgBeVx9+jD+95X1vLNld9TlfEiYPZhTgVXuvsbdG4BHgYtbTVMCvBAMvxQ3vgSY6+5N7l4LLAbODcY5sC9s+gCbD9EuItJj3XLOWPKy0rlz5rKEu8I/zIAZBGyI+7wxaIu3GLgsGL4EyDOz/KD9PDPLNrMC4CxgSDDd54FnzWwjcA3ww6D9DuDqoP1Z4Oa2ijKzG82szMzKKioS+xQ/EZFD6ZeTwVenjmXB6ipmL9sadTkfEGbAWBttreP1VmCymb0JTAY2AU3uPptYSCwA/gAsBJqCeW4Bznf3wcD/APcG7VcBvwnazwd+a2YfWj93n+Hupe5eWlhYeEQrKCLqrDleAAAKdElEQVSSCD5z2lDGFudy11+WU9fYHHU57wszYDayv9cBMJhWu63cfbO7XxocN/lW0LYreL/L3U9w96nEwmqlmRUCE9z91WARfwQ+EgxfDzwWzLsQyAIKQlkzEZEEkpaawncvPJZ3t+/hoZfXRl3O+8IMmNeBMWY2wswyiB14fzp+AjMriOtl3A48FLSnBrvKMLPxwHhgNrAD6GNmY4N5pgLLg+F3gbODeY4hFjDaByYiSeGMMQVMLSnm5y+uSphb+ocWMO7eBNwEzCIWAo+5+1Izu9PMLgommwKsMLNyoBi4K2hPB+ab2TJgBnB1cMC/CbgBeNLMFhM7BvP1YJ6vATcE7X8ArvNEO+IlIhKib51/DE3Nzo9nrYi6FAAsmX+DS0tLvaysLOoyREQ6zQ//+g73z13Nn//1o0wY0jeU7zCzRe5eeqjpdCW/iEgPctPHRlOYl8kdM5dGftqyAkZEpAfJzUzjGx8fx5vv7uTP/4j2ckAFjIhID3PZSYMZP7gPd/91ObX1TYeeISQKGBGRHiYlxfi3T5SwdXc9989dHV0dkX2ziIiE5uRh/bn4hKN4YN4aNmyP5vHKChgRkR7qtvOOJtWMu/+6/NATh0ABIyLSQw3s04svThnFs29t4ZU1VV3+/QoYEZEe7MYzRzKoby++N3MZzS1de9qyAkZEpAfLSo89Xnn5e7v54+sbDj1DJ1LAiIj0cOcfP4BTR/Tnntkr2LW3scu+VwEjItLDmcVOW96xp4GfvbCyy75XASMikgSOPaoPV54ylIcXrGPVtpou+U4FjIhIkrh12lh6ZaTy739Z1iXfp4AREUkS+bmZfPnsMcxZUcFL72wL/fsUMCIiSeSzE4cztjiXFVurQ/+utNC/QUREEkZGWgrP3DyJjLTw+xfqwYiIJJmuCBdQwIiISEgUMCIiEgoFjIiIhEIBIyIioVDAiIhIKBQwIiISCgWMiIiEwty79gE0icTMKoD1h5isD7CrHYs72HQdHdeetgKgsh11dab2bovOXEaibn/onn8G2v5HJlH/DXT19h/m7oWHqAncXa+DvIAZRzpdR8e1pw0oS9Rt0ZnLSNTt313/DLT9o93+Yf0ZRLH92/PSLrJDm9kJ03V0XHvbulpn1NDRZWj7f9CR1qHtf2QS9d9AQm7/pN5F1p2ZWZm7l0ZdRzLTn0G0tP0Tn3ow3deMqAsQ/RlETNs/wakHIyIioVAPRkREQqGAERGRUChgREQkFAqYHsjMjjGz+83sCTP7YtT1JBsz+6SZ/crM/mxm06KuJ9mY2Ugz+7WZPRF1LclOAZNgzOwhM9tmZm+3aj/XzFaY2Sozu+1gy3D35e7+BeBTgE7j7IBO2v5PufsNwHXAFSGW2+N00vZf4+7Xh1uptIfOIkswZnYmUAM84u7HBW2pQDkwFdgIvA5cBaQCd7daxOfcfZuZXQTcBvzc3X/fVfV3d521/YP5/hP4nbu/0UXld3udvP2fcPfpXVW7fFha1AXIB7n7PDMb3qr5VGCVu68BMLNHgYvd/W7gwgMs52ngaTP7C6CAaafO2P5mZsAPgb8qXDqms/7+S2LQLrLuYRCwIe7zxqCtTWY2xcx+ZmYPAM+GXVwS6ND2B24GzgGmm9kXwiwsSXT073++md0PnGhmt4ddnByYejDdg7XRdsB9m+4+B5gTVjFJqKPb/2fAz8IrJ+l0dPtXAQr2BKAeTPewERgS93kwsDmiWpKRtn+0tP27KQVM9/A6MMbMRphZBnAl8HTENSUTbf9oaft3UwqYBGNmfwAWAuPMbKOZXe/uTcBNwCxgOfCYuy+Nss6eSts/Wtr+PYtOUxYRkVCoByMiIqFQwIiISCgUMCIiEgoFjIiIhEIBIyIioVDAiIhIKBQw0q2YWU0XfMdFh7olfAjfOcXMPnIY851oZg8Gw9eZ2c87v7qOM7PhrW+538Y0hWb2t66qSbqeAkaSUnAL+Da5+9Pu/sMQvvNg9/6bAnQ4YID/B/z3YRUUMXevAN4zs49GXYuEQwEj3ZaZfd3MXjezJWb2vbj2p8xskZktNbMb49przOxOM3sVmGhm68zse2b2hpm9ZWZHB9O93xMws98Ed6ZeYGZrzGx60J5iZvcF3/GMmT27b1yrGueY2Q/MbC7wZTP7hJm9amZvmtnzZlYc3J7+C8AtZvYPM5sU/O/+yWD9Xm/rR9jM8oDx7r64jXHDzOyFYNu8YGZDg/ZRZvZKsMw72+oRmlmOmf3FzBab2dtmdkXQfkqwHRab2Wtmlhf0VOYH2/CNtnphZpZqZv8R92f1z3GjnwI+0+YfsHR/7q6XXt3mBdQE79OAGcTutJsCPAOcGYzrH7z3At4G8oPPDnwqblnrgJuD4X8BHgyGryP2oDaA3wCPB99RQuy5JADTiT0KIQUYAOwAprdR7xzgvrjP/dh/B43PA/8ZDN8B3Bo33e+BM4LhocDyNpZ9FvBk3Of4umcC1wbDnwOeCoafAa4Khr+wb3u2Wu5lwK/iPvcBMoA1wClBW29id2PPBrKCtjFAWTA8HHg7GL4R+HYwnAmUASOCz4OAt6L+e6VXOC/drl+6q2nB683gcy6xH7h5wJfM7JKgfUjQXgU0A0+2Ws6fgvdFwKUH+K6n3L0FWGZmxUHbGcDjQfsWM3vpILX+MW54MPBHMxtI7Ed77QHmOQcoiT27DIDeZpbn7tVx0wwEKg4w/8S49fkt8OO49k8Gw78H7mlj3reAe8zsR8Az7j7fzI4H3nP31wHcfTfEejvAz83sBGLbd2wby5sGjI/r4fUh9meyFtgGHHWAdZBuTgEj3ZUBd7v7Ax9oNJtC7Md5orvvMbM5QFYwus7dm1stpz54b+bA/x7q44at1Xt71MYN/zdwr7s/HdR6xwHmSSG2DnsPsty97F+3Q2n3TQfdvdzMTgbOB+42s9nEdmW1tYxbgK3AhKDmujamMWI9xVltjMsith7SA+kYjHRXs4DPmVkugJkNMrMiYv873hGEy9HA6SF9/9+By4JjMcXEDtK3Rx9gUzB8bVx7NZAX93k2sTsIAxD0EFpbDow+wPcsIHZbe4gd4/h7MPwKsV1gxI3/ADM7Ctjj7v9LrIdzEvAOcJSZnRJMkxectNCHWM+mBbgGaOvkiVnAF80sPZh3bNDzgViP56Bnm0n3pYCRbsndZxPbxbPQzN4CniD2A/03IM3MlgDfJ/aDGoYniT0I623gAeBVYFc75rsDeNzM5gOVce0zgUv2HeQHvgSUBgfFl9HGExrd/R2gT3Cwv7UvAf8UbIdrgC8H7V8BvmpmrxHbxdZWzccDr5nZP4BvAf/u7g3AFcB/m9li4DlivY/7gGvN7BViYVHbxvIeBJYBbwSnLj/A/t7iWcBf2phHegDdrl/kMJlZrrvXmFk+8BrwUXff0sU13AJUu/uD7Zw+G9jr7m5mVxI74H9xqEUevJ55wMXuviOqGiQ8OgYjcvieMbO+xA7Wf7+rwyXwS+DyDkx/MrGD8gbsJHaGWSTMrJDY8SiFSw+lHoyIiIRCx2BERCQUChgREQmFAkZEREKhgBERkVAoYEREJBQKGBERCcX/B5XX8x+YRYGSAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn.lr_find()\n",
    "learn.sched.plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [],
   "source": [
    "lr=1e-2\n",
    "wd=1e-5\n",
    "\n",
    "lrs = np.array([lr/200,lr/20,lr])/2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a63158c05402413a970d37ea93afda4b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=30), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   new_acc    rd_f       car_f_p_r  \n",
      "    0      0.999548   0.99957    0.566958   0.293285   0.023553   0.668129   0.019053  \n",
      "    1      0.999266   0.998397   0.617578   0.440909   0.648476   0.501968   0.712163  \n",
      "    2      0.998137   0.997087   0.626865   0.454431   0.314222   0.377018   0.315234  \n",
      "    3      0.996654   0.995194   0.701618   0.557817   0.575725   0.644279   0.569086  \n",
      "    4      0.995664   0.994784   0.842974   0.751503   0.658341   0.519015   0.719328  \n",
      "    5      0.995032   0.994862   0.854657   0.821883   0.616587   0.521764   0.664411  \n",
      "    6      0.994571   0.998479   0.670369   0.656454   0.303402   0.084676   0.867574  \n",
      "    7      0.99435    0.994752   0.895523   0.882821   0.685681   0.478795   0.784519  \n",
      "    8      0.994194   0.994689   0.89863    0.850796   0.656837   0.51397    0.724356  \n",
      "    9      0.994042   0.995592   0.864915   0.784403   0.641335   0.344197   0.846053  \n",
      "    10     0.993986   0.994314   0.911498   0.879404   0.576514   0.739605   0.554907  \n",
      "    11     0.993947   0.995821   0.851495   0.770029   0.626802   0.315578   0.858468  \n",
      "    12     0.993915   0.996336   0.861976   0.809378   0.580364   0.254358   0.890759  \n",
      "    13     0.993956   0.996102   0.8727     0.849735   0.595387   0.288182   0.835817  \n",
      "    14     0.993941   0.994268   0.920399   0.894457   0.606259   0.694139   0.598443  \n",
      "    15     0.994406   0.996648   0.841336   0.785818   0.062927   0.432016   0.052162  \n",
      "    16     0.99469    0.995757   0.896266   0.864361   0.284066   0.497032   0.259417  \n",
      "    17     0.994643   0.995804   0.885332   0.86518    0.198741   0.833167   0.168667  \n",
      "    18     0.994464   0.995152   0.884986   0.826664   0.37115    0.769837   0.334894  \n",
      "    19     0.994224   0.994465   0.889279   0.875859   0.625178   0.618086   0.638138  \n",
      "    20     0.994171   0.994473   0.927063   0.896719   0.559671   0.658212   0.548305  \n",
      "    21     0.994047   0.994611   0.925188   0.884667   0.492937   0.731084   0.461983  \n",
      "    22     0.9939     0.994281   0.941431   0.925487   0.552136   0.759859   0.523604  \n",
      "    23     0.993739   0.994111   0.936584   0.906695   0.615742   0.732383   0.59921   \n",
      "    24     0.993628   0.994247   0.944172   0.937519   0.53215    0.814997   0.494227  \n",
      "    25     0.993561   0.994193   0.946227   0.949935   0.540129   0.829801   0.50138   \n",
      "    26     0.993519   0.993978   0.949647   0.9445     0.604915   0.807168   0.574871  \n",
      "    27     0.993438   0.993879   0.953142   0.94584    0.649208   0.765404   0.632294  \n",
      "    28     0.993388   0.993846   0.953266   0.944455   0.661675   0.760533   0.647156  \n",
      "    29     0.993358   0.99383    0.95367    0.94454    0.67004    0.753019   0.658269  \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[0.9938300137519837,\n",
       " 0.9536703858375549,\n",
       " 0.944539891242981,\n",
       " 0.6700404758453369,\n",
       " 0.753019320487976,\n",
       " 0.6582686276435852]"
      ]
     },
     "execution_count": 88,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learn.fit(lr,1,wds=wd,cycle_len=30,use_clr_beta=(20,10,0.95,0.85))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.save(f'128urn-{S_PREFIX}-tmp')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.load(f'128urn-{S_PREFIX}-tmp')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.unfreeze()\n",
    "learn.bn_freeze(True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [],
   "source": [
    "lr=1e-2\n",
    "wd=1e-6\n",
    "\n",
    "lrs = np.array([lr/200,lr/20,lr])/2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "48c46de59c0245f2a5225486ca680009",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=10), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   new_acc    rd_f       car_f_p_r  \n",
      "    0      0.993302   0.993898   0.957261   0.943069   0.603339   0.835344   0.569495  \n",
      "    1      0.993309   0.993817   0.955379   0.939295   0.638496   0.804695   0.612723  \n",
      "    2      0.99325    0.993696   0.962754   0.954185   0.658318   0.82187    0.632234  \n",
      "    3      0.993203   0.993625   0.964029   0.956658   0.704582   0.772029   0.694999  \n",
      "    4      0.993171   0.993577   0.966514   0.95811    0.699742   0.799333   0.683869  \n",
      "    5      0.993128   0.993574   0.965302   0.954856   0.697306   0.805722   0.67927   \n",
      "    6      0.993102   0.993583   0.961736   0.941148   0.688835   0.822019   0.666547  \n",
      "    7      0.99309    0.993562   0.96389    0.965799   0.740368   0.745501   0.744216  \n",
      "    8      0.993062   0.993524   0.967505   0.965476   0.745371   0.750748   0.748952  \n",
      "    9      0.993033   0.993488   0.969489   0.96613    0.737498   0.776076   0.733061  \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[0.9934877600669861,\n",
       " 0.9694888439178467,\n",
       " 0.9661295232772827,\n",
       " 0.7374980611801147,\n",
       " 0.77607563829422,\n",
       " 0.7330613236427307]"
      ]
     },
     "execution_count": 94,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learn.fit(lrs/2, 1, wds=wd, cycle_len=10,use_clr=(20,10))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3245d759dc9a4c6eb670adf8086d5dec",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=10), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   new_acc    rd_f       car_f_p_r  \n",
      "    0      0.993007   0.993497   0.969217   0.966106   0.746173   0.759684   0.74764   \n",
      "    1      0.993004   0.993483   0.969326   0.968062   0.74527    0.766488   0.744745  \n",
      "    2      0.992994   0.993479   0.969607   0.967924   0.748957   0.76233    0.750329  \n",
      "    3      0.99299    0.993465   0.970323   0.967311   0.746558   0.771629   0.745042  \n",
      "    4      0.992985   0.993455   0.970896   0.96701    0.741661   0.782386   0.736511  \n",
      "    5      0.992978   0.993445   0.971055   0.967139   0.736489   0.795296   0.72728   \n",
      "    6      0.992974   0.993442   0.971251   0.967538   0.733038   0.801575   0.721783  \n",
      "    7      0.992969   0.99344    0.971375   0.96798    0.732777   0.802734   0.721237  \n",
      "    8      0.992965   0.993436   0.971512   0.967443   0.736894   0.798122   0.727179  \n",
      "    9      0.992963   0.993435   0.971538   0.967386   0.737336   0.79816    0.727687  \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[0.9934347214698791,\n",
       " 0.971537672996521,\n",
       " 0.9673857355117798,\n",
       " 0.737336049079895,\n",
       " 0.7981598505973816,\n",
       " 0.7276867814064026]"
      ]
     },
     "execution_count": 96,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learn.fit(lrs/20, 1, wds=wd, cycle_len=10,use_clr=(20,10))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.save(f'128urn-{S_PREFIX}-0')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.load(f'128urn-{S_PREFIX}-0')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "x,y = next(iter(md.val_dl))\n",
    "py = to_np(learn.model(V(x[:10])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# py = np.argmax(py,1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "show_img(denorm(x[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "py.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "show_img(py[0][0]>0.5);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "show_img(py[0][1]>0.5);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "show_img(y[0][1]);"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 256x256"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val Labels: 1000\n",
      "Val x:1000, y:1000\n",
      "Trn x:4600, y:4600\n",
      "All x:4600\n"
     ]
    }
   ],
   "source": [
    "ext = '-300'\n",
    "sz=192\n",
    "bs=64\n",
    "md = torch_loader(ext, PATH, bs, sz, workers, random_crop, pseudo_label, val_folder)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "lr=1e-2\n",
    "wd=1e-6\n",
    "\n",
    "lrs = np.array([lr/200,lr/20,lr])/2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import gc\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([3, 192, 416])"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "md.trn_ds[0][0].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "a,b,c = md.trn_ds[0][0].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/paperspace/anaconda3/envs/lyft/lib/python3.6/site-packages/torch/nn/modules/rnn.py:38: UserWarning: dropout option adds dropout after all but last recurrent layer, so non-zero dropout expects num_layers greater than 1, but got dropout=0.2 and num_layers=1\n",
      "  \"num_layers={}\".format(dropout, num_layers))\n"
     ]
    }
   ],
   "source": [
    "m_sz = np.array([b//32,c//32])\n",
    "learn = get_learner(md, m_sz)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.load(f'128urn-{S_PREFIX}-0')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.freeze_to(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=30), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 17%|█▋        | 12/72 [00:09<00:47,  1.25it/s, loss=0.999]\n",
      "epoch      trn_loss   val_loss   new_acc    rd_f       car_f_p_r  \n",
      "    0      0.998974   0.998155   0.512387   0.275771   0.492833   0.175148   0.972715  \n",
      "    1      0.997941   0.995525   0.602931   0.406848   0.7006     0.359487   0.940418  \n",
      "    2      0.992772   0.989518   0.792433   0.684277   0.770133   0.550245   0.865473  \n",
      "    3      0.988321   0.986991   0.962584   0.956977   0.783979   0.690286   0.813605  \n",
      "    4      0.986835   0.986594   0.957307   0.967912   0.789973   0.766107   0.798736  \n",
      "    5      0.986227   0.986549   0.966934   0.973734   0.785012   0.767942   0.792898  \n",
      "    6      0.985956   0.98654    0.971722   0.975717   0.790215   0.757696   0.802135  \n",
      "    7      0.985783   0.986186   0.971003   0.976204   0.798398   0.824741   0.794345  \n",
      "    8      0.985708   0.99189    0.834333   0.744143   0.548539   0.318214   0.699907  \n",
      "    9      0.985805   0.986474   0.949435   0.962588   0.776609   0.817335   0.770675  \n",
      "    10     0.985635   0.986286   0.973946   0.978803   0.780276   0.820215   0.774111  \n",
      "    11     0.985524   0.986229   0.973019   0.980136   0.768035   0.860665   0.750806  \n",
      "    12     0.985534   0.98622    0.974701   0.977169   0.790509   0.823291   0.78546   \n",
      "    13     0.985422   0.986129   0.977372   0.981647   0.785348   0.849406   0.774737  \n",
      "    14     0.985357   0.98608    0.976099   0.981558   0.808218   0.824213   0.807503  \n",
      "    15     0.985364   0.98612    0.975594   0.980567   0.774455   0.872168   0.757586  \n",
      "    16     0.985355   0.986249   0.977891   0.980249   0.739975   0.89566    0.713369  \n",
      "    17     0.985338   0.986162   0.979868   0.981712   0.746098   0.90528    0.718234  \n",
      "    18     0.985274   0.986004   0.980388   0.975928   0.783365   0.883228   0.765898  \n",
      "    19     0.985227   0.986061   0.981605   0.978877   0.761704   0.905279   0.736529  \n",
      "    20     0.985203   0.986033   0.980893   0.980044   0.785851   0.866658   0.771256  \n",
      "    21     0.985182   0.986009   0.982037   0.981656   0.796432   0.854336   0.786242  \n",
      "    22     0.985178   0.985939   0.982794   0.98317    0.810061   0.849475   0.803827  \n",
      "    23     0.985138   0.985894   0.983481   0.98343    0.793025   0.888558   0.775239  \n",
      "    24     0.985107   0.985851   0.983798   0.983333   0.80124    0.884319   0.785591  \n",
      "    25     0.985092   0.985837   0.983817   0.983397   0.806283   0.87907    0.792637  \n",
      "    26     0.985084   0.985833   0.983839   0.98352    0.807313   0.87834    0.793995  \n",
      "    27     0.985075   0.985827   0.983918   0.983743   0.808294   0.877811   0.795257  \n",
      "    28     0.985073   0.985827   0.983956   0.983872   0.807964   0.878156   0.794791  \n",
      "    29     0.985069   0.985826   0.983969   0.983827   0.808048   0.878039   0.79488   \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[0.9858263325691223,\n",
       " 0.9839693932533264,\n",
       " 0.98382701587677,\n",
       " 0.808047966003418,\n",
       " 0.8780391607284546,\n",
       " 0.7948795404434205]"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learn.fit(lrs,1, wds=wd,cycle_len=30,use_clr_beta=(20,20,0.95,0.85))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.lr_find()\n",
    "learn.sched.plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.save(f'256urn-{S_PREFIX}-tmp')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "learn.load(f'256urn-{S_PREFIX}-tmp')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.unfreeze()\n",
    "learn.bn_freeze(True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "50dc060817124d8fac9085bf8f672c18",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=30), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   new_acc    rd_f       car_f_p_r  \n",
      "    0      0.985125   0.985831   0.983815   0.984071   0.802383   0.886054   0.786565  \n",
      "    1      0.985119   0.985818   0.983622   0.984487   0.806382   0.882488   0.791954  \n",
      "    2      0.985112   0.985805   0.984098   0.983941   0.809825   0.878887   0.796772  \n",
      "    3      0.985105   0.985805   0.984317   0.983992   0.808495   0.880624   0.794902  \n",
      "    4      0.985099   0.985811   0.984482   0.984057   0.806221   0.883008   0.791869  \n",
      "    5      0.98509    0.985801   0.984636   0.983704   0.807785   0.882875   0.793716  \n",
      "    6      0.985083   0.985797   0.984743   0.983177   0.808509   0.883024   0.79456   \n",
      "    7      0.985075   0.985813   0.984654   0.98385    0.802135   0.889685   0.785869  \n",
      "    8      0.985068   0.985807   0.984811   0.984019   0.805646   0.885363   0.790818  \n",
      "    9      0.98506    0.985782   0.985029   0.984446   0.807035   0.888077   0.791713  \n",
      "    10     0.985054   0.985793   0.985126   0.98539    0.80243    0.892442   0.785482  \n",
      "    11     0.985045   0.985773   0.985401   0.985158   0.805931   0.891287   0.789577  \n",
      "    12     0.985035   0.985812   0.985398   0.985621   0.794329   0.901524   0.774205  \n",
      "    13     0.985031   0.985797   0.985221   0.984344   0.80526    0.887331   0.789937  \n",
      "    14     0.985029   0.985758   0.985151   0.985051   0.813402   0.88399    0.800447  \n",
      "    15     0.985017   0.985759   0.985514   0.98582    0.810598   0.888328   0.796247  \n",
      "    16     0.985011   0.985762   0.98545    0.986829   0.805375   0.895455   0.788334  \n",
      "    17     0.985009   0.985763   0.984542   0.987345   0.811814   0.887445   0.797996  \n",
      "    18     0.985005   0.985757   0.98499    0.987532   0.811642   0.889025   0.79736   \n",
      "    19     0.985006   0.985718   0.985778   0.983916   0.820855   0.881342   0.809678  \n",
      "    20     0.985018   0.985727   0.985582   0.98403    0.817886   0.882553   0.805784  \n",
      "    21     0.985038   0.985803   0.984704   0.987566   0.790336   0.91425    0.76739   \n",
      "    22     0.985013   0.985714   0.985119   0.987448   0.810387   0.899204   0.793395  \n",
      "    23     0.985      0.985693   0.986226   0.986206   0.816115   0.893808   0.801273  \n",
      "    24     0.984993   0.9857     0.986205   0.986613   0.813506   0.896234   0.797681  \n",
      "    25     0.98499    0.985699   0.986251   0.986569   0.813658   0.896093   0.797872  \n",
      "    26     0.984987   0.9857     0.986264   0.986606   0.813128   0.896767   0.797111  \n",
      "    27     0.984987   0.985701   0.986266   0.98663    0.812838   0.897065   0.796706  \n",
      "    28     0.984985   0.985701   0.986277   0.986622   0.812885   0.897066   0.796754  \n",
      "    29     0.984986   0.985701   0.986281   0.986632   0.812793   0.897179   0.796623  \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[0.9857010035514832,\n",
       " 0.986281436920166,\n",
       " 0.9866316795349122,\n",
       " 0.8127929096221924,\n",
       " 0.8971788854598999,\n",
       " 0.7966229948997497]"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learn.fit(lrs/20,1, wds=wd,cycle_len=30,use_clr_beta=(20,20,0.95,0.85))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.save(f'256urn-{S_PREFIX}')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Let's try different weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_learner(md, sz, half=False):\n",
    "    m = Unet34Mod(sz)\n",
    "    m = to_gpu(m)\n",
    "    models = UnetModel(m)\n",
    "    learn = ConvLearner(md, models)\n",
    "    learn.opt_fn=optim.Adam\n",
    "    class_weights = torch.cuda.FloatTensor([1,4,2])\n",
    "    if half:\n",
    "        class_weights = class_weights.half()\n",
    "        learn.half()\n",
    "#     learn.crit=nn.CrossEntropyLoss(weight=class_weights)\n",
    "    learn.crit=SoftDiceLoss(weight=class_weights)\n",
    "    learn.metrics=[new_acc, rd_f, car_f_p_r]\n",
    "    return learn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val Labels: 1000\n",
      "Val x:1000, y:1000\n",
      "Trn x:4600, y:4600\n",
      "All x:4600\n"
     ]
    }
   ],
   "source": [
    "ext = '-300'\n",
    "sz=192\n",
    "bs=64\n",
    "md = torch_loader(ext, PATH, bs, sz, workers, random_crop, pseudo_label, val_folder)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "lr=1e-2\n",
    "wd=1e-6\n",
    "\n",
    "lrs = np.array([lr/200,lr/20,lr])/2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import gc\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([3, 192, 416])"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "md.trn_ds[0][0].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "a,b,c = md.trn_ds[0][0].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/paperspace/anaconda3/envs/lyft/lib/python3.6/site-packages/torch/nn/modules/rnn.py:38: UserWarning: dropout option adds dropout after all but last recurrent layer, so non-zero dropout expects num_layers greater than 1, but got dropout=0.2 and num_layers=1\n",
      "  \"num_layers={}\".format(dropout, num_layers))\n"
     ]
    }
   ],
   "source": [
    "m_sz = np.array([b//32,c//32])\n",
    "learn = get_learner(md, m_sz)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.freeze_to(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "771bf974c9af4b5080e92485602a02cd",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=30), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/72 [00:00<?, ?it/s]\n"
     ]
    },
    {
     "ename": "RuntimeError",
     "evalue": "cuda runtime error (2) : out of memory at /opt/conda/conda-bld/pytorch_1524590031827/work/aten/src/THC/generic/THCStorage.cu:58",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mRuntimeError\u001b[0m                              Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-69-5f76cbfd9926>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mlearn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlrs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mwds\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mwd\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mcycle_len\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m30\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0muse_clr_beta\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m20\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m20\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m0.95\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m0.85\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m~/anaconda3/envs/lyft/lib/python3.6/site-packages/fastai/learner.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, lrs, n_cycle, wds, **kwargs)\u001b[0m\n\u001b[1;32m    285\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msched\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    286\u001b[0m         \u001b[0mlayer_opt\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_layer_opt\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlrs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 287\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit_gen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlayer_opt\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mn_cycle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    288\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    289\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mwarm_up\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlr\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mwds\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/lyft/lib/python3.6/site-packages/fastai/learner.py\u001b[0m in \u001b[0;36mfit_gen\u001b[0;34m(self, model, data, layer_opt, n_cycle, cycle_len, cycle_mult, cycle_save_name, best_save_name, use_clr, use_clr_beta, metrics, callbacks, use_wd_sched, norm_wds, wds_sched_mult, use_swa, swa_start, swa_eval_freq, **kwargs)\u001b[0m\n\u001b[1;32m    232\u001b[0m             \u001b[0mmetrics\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mmetrics\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcallbacks\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcallbacks\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mreg_fn\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreg_fn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mclip\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mclip\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfp16\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfp16\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    233\u001b[0m             \u001b[0mswa_model\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mswa_model\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0muse_swa\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mswa_start\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mswa_start\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 234\u001b[0;31m             swa_eval_freq=swa_eval_freq, **kwargs)\n\u001b[0m\u001b[1;32m    235\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    236\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mget_layer_groups\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodels\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_layer_groups\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/lyft/lib/python3.6/site-packages/fastai/model.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(model, data, n_epochs, opt, crit, metrics, callbacks, stepper, swa_model, swa_start, swa_eval_freq, **kwargs)\u001b[0m\n\u001b[1;32m    137\u001b[0m             \u001b[0mbatch_num\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    138\u001b[0m             \u001b[0;32mfor\u001b[0m \u001b[0mcb\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mcallbacks\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mcb\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mon_batch_begin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 139\u001b[0;31m             \u001b[0mloss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel_stepper\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mV\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mV\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0my\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mepoch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    140\u001b[0m             \u001b[0mavg_loss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mavg_loss\u001b[0m \u001b[0;34m*\u001b[0m \u001b[0mavg_mom\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mloss\u001b[0m \u001b[0;34m*\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0mavg_mom\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    141\u001b[0m             \u001b[0mdebias_loss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mavg_loss\u001b[0m \u001b[0;34m/\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0mavg_mom\u001b[0m\u001b[0;34m**\u001b[0m\u001b[0mbatch_num\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/lyft/lib/python3.6/site-packages/fastai/model.py\u001b[0m in \u001b[0;36mstep\u001b[0;34m(self, xs, y, epoch)\u001b[0m\n\u001b[1;32m     48\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mstep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mxs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mepoch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     49\u001b[0m         \u001b[0mxtra\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 50\u001b[0;31m         \u001b[0moutput\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mm\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0mxs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     51\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moutput\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mtuple\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0moutput\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0mxtra\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0moutput\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     52\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfp16\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mm\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mzero_grad\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/lyft/lib/python3.6/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m    489\u001b[0m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    490\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 491\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    492\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    493\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-34-5941684c66f3>\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, x)\u001b[0m\n\u001b[1;32m     56\u001b[0m         \u001b[0mx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mup3\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msfs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfeatures\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     57\u001b[0m         \u001b[0mx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mup4\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msfs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfeatures\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 58\u001b[0;31m         \u001b[0mx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mup5\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx_skip\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     59\u001b[0m         \u001b[0mx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mup6\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     60\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msqueeze\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/lyft/lib/python3.6/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m    489\u001b[0m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    490\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 491\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    492\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    493\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-25-59c67f6d6916>\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, up_p, x_p)\u001b[0m\n\u001b[1;32m      9\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mup_p\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx_p\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     10\u001b[0m         \u001b[0mup_p\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtr_conv\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mup_p\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 11\u001b[0;31m         \u001b[0mx_p\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mx_conv\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx_p\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     12\u001b[0m         \u001b[0mcat_p\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mup_p\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mx_p\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdim\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     13\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mF\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrelu\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcat_p\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minplace\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/lyft/lib/python3.6/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m    489\u001b[0m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    490\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 491\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    492\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    493\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/lyft/lib/python3.6/site-packages/torch/nn/modules/conv.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, input)\u001b[0m\n\u001b[1;32m    299\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    300\u001b[0m         return F.conv2d(input, self.weight, self.bias, self.stride,\n\u001b[0;32m--> 301\u001b[0;31m                         self.padding, self.dilation, self.groups)\n\u001b[0m\u001b[1;32m    302\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    303\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mRuntimeError\u001b[0m: cuda runtime error (2) : out of memory at /opt/conda/conda-bld/pytorch_1524590031827/work/aten/src/THC/generic/THCStorage.cu:58"
     ]
    }
   ],
   "source": [
    "learn.fit(lrs,1, wds=wd,cycle_len=30,use_clr_beta=(20,20,0.95,0.85))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "x,y = next(iter(md.trn_dl))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "x.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "x,y = next(iter(md.val_dl))\n",
    "py = to_np(learn.model(V(x[:8])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.imshow(denorm(x[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "py.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "show_img(py[0][0]>0);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "py[-1].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "y[-1].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "show_img(y[-1]);"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### No Crop"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ext = '-300'\n",
    "sz=192\n",
    "bs=32\n",
    "md = torch_loader(ext, PATH, bs, sz, workers, False, pseudo_label, val_folder)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn = get_learner(md)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.load(f'256urn-{S_PREFIX}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.unfreeze()\n",
    "learn.bn_freeze(True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "lr=1e-3\n",
    "wd=1e-6\n",
    "\n",
    "lrs = np.array([lr/200,lr/20,lr])/2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.fit(lr,1,wds=wd, cycle_len=10,use_clr=(5,5))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.save(f'256urn-{S_PREFIX}-nocrop')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 512x512"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "TRAIN_DN = 'CameraRGB'\n",
    "MASKS_DN = 'CameraSeg'\n",
    "\n",
    "ext = ''\n",
    "sz=384\n",
    "bs=16\n",
    "md = torch_loader(ext, PATH, bs, sz, workers, random_crop, pseudo_label, val_folder)\n",
    "\n",
    "learn = get_learner(md)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.lr_find()\n",
    "learn.sched.plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# learn.load(f'256urn-{S_PREFIX}')\n",
    "learn.load(f'256urn-{S_PREFIX}-nocrop')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.freeze_to(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "lr=1e-3\n",
    "wd=5e-7"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "learn.fit(lr,1, wds=wd, cycle_len=4,use_clr=(10,4))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.fit(lr/4,1, wds=wd, cycle_len=4,use_clr=(10,4))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.save(f'600urn-{S_PREFIX}-tmp')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.load(f'600urn-{S_PREFIX}-tmp')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.unfreeze()\n",
    "learn.bn_freeze(True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "lrs = np.array([lr/200,lr/20,lr])/2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.fit(lrs/10,1, wds=wd,cycle_len=4,use_clr=(20,8))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.fit(lrs/40,1, wds=wd,cycle_len=4,use_clr=(20,8))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.save(f'600urn-{S_PREFIX}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.sched.plot_loss()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### No Crop"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "TRAIN_DN = 'CameraRGB'\n",
    "MASKS_DN = 'CameraSeg'\n",
    "\n",
    "ext = ''\n",
    "sz=384\n",
    "bs=16\n",
    "md = torch_loader(ext, PATH, bs, sz, workers, False, pseudo_label, val_folder)\n",
    "\n",
    "learn = get_learner(md)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import gc\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.load(f'600urn-{S_PREFIX}')\n",
    "# learn.load(f'256urn-{S_PREFIX}-nocrop')\n",
    "# learn.load('600urn-19-weights-26-r8')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "lr=5e-4\n",
    "wd=5e-7\n",
    "lrs = np.array([lr/200,lr/20,lr])/2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.unfreeze()\n",
    "learn.bn_freeze(True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.fit(lrs,1, wds=wd,cycle_len=4,use_clr=(20,2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.save(f'600urn-{S_PREFIX}-nocrop-tmp')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.load(f'600urn-{S_PREFIX}-nocrop-tmp')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.unfreeze()\n",
    "learn.bn_freeze(True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.fit(lrs/10,1, wds=wd,cycle_len=4,use_clr=(20,2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.save(f'600urn-{S_PREFIX}-nocrop-tmp-2')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.fit(lrs/100,1, wds=wd,cycle_len=4,use_clr=(20,2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.fit(lrs/50,1, wds=wd,cycle_len=4,use_clr_beta=(20,20,0.95,0.85))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.save(f'600urn-{S_PREFIX}-nocrop')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Visualize Results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.load(f'600urn-36-resnet-softmax-nocrop-tmp')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "val_it = iter(md.val_dl)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "x,y = next(val_it)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "x[1].mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "x.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "type(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "res = learn.model(V(x).cuda())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "mx,idx = torch.max(res,1)\n",
    "idx = idx.cpu().data.numpy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "idx.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "type(idx)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "i=0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.imshow(denorm(x[i]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "show_img(idx[i]);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "idx[0].mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "show_img(py[idx][1]);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "py[-1].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "y[-1].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "show_img(y[idx][0]);"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Save results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys, skvideo.io, json, base64\n",
    "import numpy as np\n",
    "from PIL import Image\n",
    "from io import BytesIO, StringIO"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "m = Unet34()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cuda_enabled = torch.cuda.is_available()\n",
    "DATA_PATH = Path('../data/all')\n",
    "model_path = str(DATA_PATH/'models/600urn-36-resnet-softmax-nocrop-tmp.h5')\n",
    "# model_path = str(PATH/'models/600urn-19-weights-26-r9.h5')\n",
    "if cuda_enabled:\n",
    "    m = m.cuda()\n",
    "    m.load_state_dict(torch.load(model_path, map_location=lambda storage, loc: storage))\n",
    "else:\n",
    "    m.load_state_dict(torch.load(model_path))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# file = sys.argv[-1]\n",
    "\n",
    "R_PATH = Path.cwd()\n",
    "file = R_PATH/'test_video.mp4'\n",
    "mismatched_idxs = []\n",
    "student_output = R_PATH/'tester_data_crop'\n",
    "ans_key = R_PATH/'results.json'\n",
    "\n",
    "# R_PATH = Path('../data/lyft-answers')\n",
    "# file = R_PATH/'test_video.mp4'\n",
    "# ans_key = R_PATH/'results.json'\n",
    "# mismatched_idxs = list(range(15,44)) + list(range(200,750))\n",
    "# student_output = R_PATH/'tester_data_crop'\n",
    "\n",
    "if file == 'demo.py':\n",
    "  print (\"Error loading video\")\n",
    "  quit\n",
    "\n",
    "# Define encoder function\n",
    "def encode(array):\n",
    "\tpil_img = Image.fromarray(array)\n",
    "\tbuff = BytesIO()\n",
    "\tpil_img.save(buff, format=\"PNG\")\n",
    "\treturn base64.b64encode(buff.getvalue()).decode(\"utf-8\")\n",
    "\n",
    "video = skvideo.io.vread(file)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "imagenet_stats = torch.cuda.FloatTensor([[0.485, 0.456, 0.406], [0.229, 0.224, 0.225]])\n",
    "def normalize(x):\n",
    "    if x.mean() > 1:\n",
    "        x.div_(255.0)\n",
    "    m,s = imagenet_stats\n",
    "#     return TVF.normalize(x, m, s)\n",
    "#     print(x.shape)\n",
    "    x.sub_(m.view(-1, 1, 1))\n",
    "    x.div_(s.view(-1, 1, 1))\n",
    "    return x\n",
    "\n",
    "def crop_bg(x):\n",
    "    # Original\n",
    "    h = x.shape[2]\n",
    "    top = int(h/3.75)\n",
    "    bot = int(h*.9 + h/150)\n",
    "    return x[:,:,top:bot,:]\n",
    "\n",
    "def pad(x):\n",
    "#     print(x.shape)\n",
    "    # Original\n",
    "    b,c,w,h = x.shape\n",
    "#     print(x.shape)\n",
    "    if h%32 == 0:\n",
    "        return x, 0\n",
    "    pad_right=32-h%32\n",
    "    if pad_right:\n",
    "        x = F.pad(x, (0,pad_right,0,0), 'constant', 0)\n",
    "    return x, pad_right\n",
    "    \n",
    "def undo(idx):\n",
    "    idx\n",
    "    idx = F.pad(idx, (0,0,226,54), \"constant\", 0)\n",
    "\n",
    "def preprocess(video):\n",
    "#     f1 = video[:,200:520,:,:]\n",
    "    f1 = np.rollaxis(video, 3, 1)\n",
    "    f1 = torch.from_numpy(f1).float().cuda()\n",
    "    f1 = crop_bg(f1)\n",
    "    f1 = normalize(f1)\n",
    "    return f1.contiguous()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "video_p = preprocess(video)\n",
    "if len(video_p) == 31:\n",
    "    video_p = torch.cat((video_p[:15], video_p[16:]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import gc\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "results = []\n",
    "raw_results = []\n",
    "raw_frames = []\n",
    "answer_key = {}\n",
    "bs = 4\n",
    "\n",
    "        \n",
    "for i in range(0,video_p.shape[0],bs):\n",
    "    x = video_p[i:i+bs]\n",
    "    \n",
    "#     x = torch.from_numpy(f1).contiguous().float()\n",
    "#     if cuda_enabled:\n",
    "#         x = x.cuda()\n",
    "        \n",
    "    x,p = pad(x)\n",
    "    preds = m(torch.autograd.Variable(x))\n",
    "#     preds = F.sigmoid(preds)\n",
    "    # Indexes\n",
    "    mx,idx = torch.max(preds, 1)\n",
    "    for i in idx:\n",
    "        raw_frames.append(i.data.cpu().numpy())\n",
    "    if p > 0:\n",
    "        idx = idx[:,:,:-p]\n",
    "    idx = F.pad(idx, (0,0,160,56), \"constant\", 0)\n",
    "    frame_idx = 1+i\n",
    "    for frame in idx:\n",
    "        frame = frame.data.cpu().numpy()\n",
    "#         results.append(frame)\n",
    "        raw_frames.append(frame)\n",
    "        binary_car_result = (frame==1).astype('uint8')\n",
    "        binary_road_result = (frame==2).astype('uint8')\n",
    "#         answer_key[frame_idx] = [encode(binary_car_result), encode(binary_road_result)]\n",
    "        raw_results.append([binary_car_result, binary_road_result])\n",
    "        results.append([encode(binary_car_result), encode(binary_road_result)])\n",
    "        frame_idx+=1\n",
    "    \n",
    "    # Preds\n",
    "#     if p > 0:\n",
    "#         preds = preds[:,:,:,:-p]\n",
    "#     preds = F.pad(preds, (0,0,160,56,0,0), \"constant\", 0)\n",
    "#     frame_idx = 1+i\n",
    "#     for frame in preds:\n",
    "#         frame = frame.data.cpu().float().numpy()\n",
    "#         f_results.append(frame)\n",
    "#         binary_car_result = (frame[0]>0.5).astype('uint8')\n",
    "#         binary_road_result = (frame[1]>0.5).astype('uint8')\n",
    "#         answer_key[frame_idx] = [encode(binary_car_result), encode(binary_road_result)]\n",
    "#         raw_results.append([binary_car_result, binary_road_result])\n",
    "#         results.append([encode(binary_car_result), encode(binary_road_result)])\n",
    "#         frame_idx+=1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "type(video_p.cpu())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "type(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "video_p[0,:,50:200,700:].sum(), x[0,:,50:200,700:].cuda().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "video_p[0].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "x[0].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "x[0][2][200].cuda().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "(video_p[0] - x[0].cuda()).sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "md.val_ds.fnames"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "show_idx = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "video_p[show_idx].mean(), x[show_idx].mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.imshow(denorm(x[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.imshow(denorm(video_p[show_idx]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn_res = learn.model(V(video_p[0:4]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "_,learn_idx = torch.max(learn_res,1)\n",
    "learn_idx = learn_idx.cpu().data.numpy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.imshow(learn_idx[show_idx])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn_dl_res = learn.model(V(x[0:4].cuda()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "_,learn_dl_idx = torch.max(learn_dl_res,1)\n",
    "learn_dl_idx = learn_dl_idx.cpu().data.numpy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.imshow(learn_dl_idx[show_idx])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "m_dl_res = m(V(x.cuda()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "_,m_dl_idx = torch.max(m_dl_res,1)\n",
    "m_dl_idx = m_dl_idx.cpu().data.numpy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.imshow(m_dl_idx[show_idx])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "m.train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "m_res = m(V(video_p[0:4]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "_,m_idx = torch.max(m_res,1)\n",
    "m_idx = m_idx.cpu().data.numpy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.imshow(m_idx[show_idx])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.imshow(m_idx[show_idx])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "a = video_p[:6]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "b = x[:6].cuda()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "m_res2 = m(V(a))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "m_res2.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "_,m_idx2 = torch.max(m_res2,1)\n",
    "m_idx2 = m_idx2.cpu().data.numpy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "m_idx2.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.imshow(m_idx2[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "idx = 0\n",
    "plt.figure()\n",
    "plt.imshow(video[idx])\n",
    "plt.imshow((raw_results[idx][0]==1).data, alpha=.4)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(1, len(results)//10*10+1):\n",
    "    s_idx = i if i in mismatched_idxs else i-1\n",
    "#     s_idx = i-1\n",
    "    answer_key[i] = results[s_idx]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Print output in proper json format\n",
    "tester_data = json.dumps(answer_key)\n",
    "with open(student_output, 'w') as f:\n",
    "    f.write(tester_data)\n",
    "# print(json.dumps(answer_key))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Decoding car"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "PATH"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def decode(packet):\n",
    "\timg = base64.b64decode(packet)\n",
    "\tfilename = R_PATH/'image.png'\n",
    "\twith open(filename, 'wb') as f:\n",
    "\t\t\tf.write(img)\n",
    "\tresult = misc.imread(filename)\n",
    "\treturn result\n",
    "\n",
    "with open(ans_key) as json_data:\n",
    "\tans_data = json.loads(json_data.read())\n",
    "\tjson_data.close()\n",
    "\n",
    "# Load student data\n",
    "with open(student_output) as student_data:\n",
    "\tstudent_ans_data = json.loads(student_data.read())\n",
    "\tstudent_data.close()\n",
    "\n",
    "frames_processed = 0\n",
    "\n",
    "Car_TP = 1 # True Positives\n",
    "Car_FP = 1 # Flase Positives\n",
    "Car_TN = 1 # True Negatives\n",
    "Car_FN = 1 # True Negatives\n",
    "\n",
    "Road_TP = 1 # True Positives\n",
    "Road_FP = 1 # Flase Positives\n",
    "Road_TN = 1 # True Negatives\n",
    "Road_FN = 1 # True Negatives\n",
    "\n",
    "for frame in range(1,len(ans_data.keys())+1):\n",
    "    if frame%3 == 0: continue\n",
    "    truth_data_car =  decode(ans_data[str(frame)][0])\n",
    "    truth_data_road =  decode(ans_data[str(frame)][1])\n",
    "    student_data_car = decode(student_ans_data[str(frame)][0])\n",
    "    student_data_road = decode(student_ans_data[str(frame)][1])\n",
    "#     student_data_car = results[frame-1][0]\n",
    "#     student_data_road = results[frame-1][1]\n",
    "#     student_data_car = f_results[frame-1][0]\n",
    "#     student_data_road = f_results[frame-1][1]\n",
    "#     print(np.mean(student_data_road == 1))\n",
    "#     print(np.mean(student_data_road_f == 1))\n",
    "#     pdb.set_trace()\n",
    "\n",
    "    Car_TP += np.sum(np.logical_and(student_data_car == 1, truth_data_car == 1))\n",
    "    Car_FP += np.sum(np.logical_and(student_data_car == 1, truth_data_car == 0))\n",
    "    Car_TN += np.sum(np.logical_and(student_data_car == 0, truth_data_car == 0))\n",
    "    Car_FN += np.sum(np.logical_and(student_data_car == 0, truth_data_car == 1))\n",
    "\n",
    "    Road_TP += np.sum(np.logical_and(student_data_road == 1, truth_data_road == 1))\n",
    "    Road_FP += np.sum(np.logical_and(student_data_road == 1, truth_data_road == 0))\n",
    "    Road_TN += np.sum(np.logical_and(student_data_road == 0, truth_data_road == 0))\n",
    "    Road_FN += np.sum(np.logical_and(student_data_road == 0, truth_data_road == 1))\n",
    "\n",
    "    frames_processed+=1\n",
    "\n",
    "\n",
    "# Generate results\n",
    "Car_precision = Car_TP/(Car_TP+Car_FP)/1.0\n",
    "Car_recall = Car_TP/(Car_TP+Car_FN)/1.0\n",
    "Car_beta = 2\n",
    "Car_F = (1+Car_beta**2) * ((Car_precision*Car_recall)/(Car_beta**2 * Car_precision + Car_recall))\n",
    "Road_precision = Road_TP/(Road_TP+Road_FP)/1.0\n",
    "Road_recall = Road_TP/(Road_TP+Road_FN)/1.0\n",
    "Road_beta = 0.5\n",
    "Road_F = (1+Road_beta**2) * ((Road_precision*Road_recall)/(Road_beta**2 * Road_precision + Road_recall))\n",
    "\n",
    "print (\"Car F score: %05.3f  | Car Precision: %05.3f  | Car Recall: %05.3f  |\\n\\\n",
    "Road F score: %05.3f | Road Precision: %05.3f | Road Recall: %05.3f | \\n\\\n",
    "Averaged F score: %05.3f\" %(Car_F,Car_precision,Car_recall,Road_F,Road_precision,Road_recall,((Car_F+Road_F)/2.0)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from scipy import misc\n",
    "def decode(packet):\n",
    "\timg = base64.b64decode(packet)\n",
    "\tfilename = PATH/'image.png'\n",
    "\twith open(filename, 'wb') as f:\n",
    "\t\t\tf.write(img)\n",
    "\tresult = misc.imread(filename)\n",
    "\treturn result\n",
    "\n",
    "with open('results.json') as json_data:\n",
    "\tans_data = json.loads(json_data.read())\n",
    "\tjson_data.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_ans(index):\n",
    "    ans = decode(ans_data[str(index)][0])\n",
    "    f, (ax1, ax2, ax3) = plt.subplots(1, 3, figsize=(24, 15))\n",
    "    f.tight_layout()\n",
    "    ax1.imshow(r_np[index])\n",
    "    ax1.set_title('Mine', fontsize=35)\n",
    "    ax2.imshow(ans)\n",
    "    ax2.set_title('Answer', fontsize=35)\n",
    "    ax3.imshow(video[index])\n",
    "    ax2.set_title('Original', fontsize=35)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_ans(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ans = decode(ans_data['1'][0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "def plot_res(index):\n",
    "    f, (ax1, ax2) = plt.subplots(1, 2, figsize=(24, 15))\n",
    "    f.tight_layout()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  },
  "toc": {
   "colors": {
    "hover_highlight": "#DAA520",
    "navigate_num": "#000000",
    "navigate_text": "#333333",
    "running_highlight": "#FF0000",
    "selected_highlight": "#FFD700",
    "sidebar_border": "#EEEEEE",
    "wrapper_background": "#FFFFFF"
   },
   "moveMenuLeft": true,
   "nav_menu": {
    "height": "86px",
    "width": "252px"
   },
   "navigate_menu": true,
   "number_sections": true,
   "sideBar": true,
   "threshold": 4,
   "toc_cell": false,
   "toc_section_display": "block",
   "toc_window_display": false,
   "widenNotebook": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}