workspace/Example/lyft-unet-train34-arch-speed.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "%reload_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from fastai.conv_learner import *\n",
    "# from fastai.dataset import *\n",
    "from fastai.models.resnet import vgg_resnet50\n",
    "\n",
    "import json\n",
    "from glob import glob"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.backends.cudnn.benchmark=True"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "PATH = Path('../data/all')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def show_img(im, figsize=None, ax=None, alpha=None):\n",
    "    if not ax: fig,ax = plt.subplots(figsize=figsize)\n",
    "    ax.imshow(im, alpha=alpha)\n",
    "    ax.set_axis_off()\n",
    "    return ax"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "VEHICLES=10\n",
    "ROADS=7\n",
    "ROAD_LINES=6"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "TRAIN_DN = 'CameraRGB'\n",
    "MASKS_DN = 'CameraSeg'\n",
    "workers=7\n",
    "random_crop=False\n",
    "pseudo_label=False\n",
    "val_folder = 'test'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "S_PREFIX = '33-test_val'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "from torchvision.datasets.folder import pil_loader\n",
    "import torchvision.transforms as transforms\n",
    "import torchvision.transforms.functional as TTF"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create dataloader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "class MatchedFilesDataset(Dataset):\n",
    "    def __init__(self, fnames, y, tfms, path):\n",
    "        self.path,self.fnames = path,fnames\n",
    "        self.open_fn = pil_loader\n",
    "        self.y=y\n",
    "        self.open_y_fn = pil_loader\n",
    "        assert(len(fnames)==len(y))\n",
    "        \n",
    "        self.n = self.get_n()\n",
    "        self.c = self.get_c()\n",
    "        self.tfms = tfms\n",
    "        \n",
    "    def get_x(self, i): return self.open_fn(os.path.join(self.path, self.fnames[i]))\n",
    "    def get_y(self, i): return self.open_y_fn(os.path.join(self.path, self.y[i]))\n",
    "    def get_n(self): return len(self.fnames)\n",
    "    def get_c(self): return 2\n",
    "    \n",
    "    def get(self, tfms, x, y):\n",
    "        for fn in tfms:\n",
    "            #pdb.set_trace()\n",
    "            x, y = fn(x, y)\n",
    "        return (x, y)\n",
    "    \n",
    "    def __getitem__(self, idx):\n",
    "        x,y = self.get_x(idx),self.get_y(idx)\n",
    "        return self.get(self.tfms, x, y)\n",
    "    \n",
    "    def __len__(self): return self.n\n",
    "\n",
    "    def resize_imgs(self, targ, new_path):\n",
    "        dest = resize_imgs(self.fnames, targ, self.path, new_path)\n",
    "        return self.__class__(self.fnames, self.y, self.transform, dest)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# Seems to speed up training by ~2%\n",
    "class DataPrefetcher():\n",
    "    def __init__(self, loader, stop_after=None):\n",
    "        self.loader = loader\n",
    "        self.dataset = loader.dataset\n",
    "        self.stream = torch.cuda.Stream()\n",
    "        self.stop_after = stop_after\n",
    "        self.next_input = None\n",
    "        self.next_target = None\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.loader)\n",
    "    \n",
    "    def preload(self):\n",
    "        try:\n",
    "            self.next_input, self.next_target = next(self.loaditer)\n",
    "        except StopIteration:\n",
    "            self.next_input = None\n",
    "            self.next_target = None\n",
    "            return\n",
    "        with torch.cuda.stream(self.stream):\n",
    "            self.next_input = self.next_input.cuda(async=True)\n",
    "            self.next_target = self.next_target.cuda(async=True)\n",
    "\n",
    "    def __iter__(self):\n",
    "        count = 0\n",
    "        self.loaditer = iter(self.loader)\n",
    "        self.preload()\n",
    "        while self.next_input is not None:\n",
    "            torch.cuda.current_stream().wait_stream(self.stream)\n",
    "            input = self.next_input\n",
    "            target = self.next_target\n",
    "            self.preload()\n",
    "            count += 1\n",
    "            yield input, target\n",
    "            if type(self.stop_after) is int and (count > self.stop_after):\n",
    "                break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def crop_bg_pil(x,y):\n",
    "    w, h = x.size\n",
    "    top = int(h/3.75)\n",
    "    bot = int(h*.9 + h/150)\n",
    "    pad_right=32-w%32\n",
    "    return TTF.crop(x, top, 0, bot-top, w+pad_right), TTF.crop(y, top, 0, bot-top, w+pad_right)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "class RHF(object):\n",
    "    def __init__(self, p=0.5): self.p = p\n",
    "    def __call__(self, x, y):\n",
    "        if random.random() < self.p:\n",
    "            return TTF.hflip(x), TTF.hflip(y)\n",
    "        return x,y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "class RR(object):\n",
    "    def __init__(self, degrees=2): self.degrees = degrees\n",
    "    def __call__(self, x, y):\n",
    "        angle = random.uniform(-self.degrees, self.degrees)\n",
    "        return TTF.rotate(x, angle), TTF.rotate(y, angle)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "def tfm_x_wrapper(tfm):\n",
    "    return lambda x,y: (tfm(x), y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "class RC():\n",
    "    def __init__(self, targ_sz):\n",
    "        self.targ_sz = targ_sz\n",
    "\n",
    "    def __call__(self, x, y):\n",
    "        rand_w = random.uniform(0, 1)\n",
    "        rand_h = random.uniform(0, 1)\n",
    "        w,h = x.size\n",
    "        t_w,t_h = self.targ_sz\n",
    "        start_x = np.floor(rand_w*(w-t_w)).astype(int)\n",
    "        start_y = np.floor(rand_h*(h-t_h)).astype(int)\n",
    "        return TTF.crop(x, start_y, start_x, t_h, t_w), TTF.crop(y, start_y, start_x, t_h, t_w)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "def convert_y(y_img):\n",
    "    yr = (y_img==ROADS) | (y_img==ROAD_LINES)\n",
    "    yc = (y_img==VEHICLES)\n",
    "    cutoff_y = int(yc.shape[0]*.87)\n",
    "    yc[cutoff_y:,:] = 0\n",
    "    return torch.from_numpy(np.stack((yc,yr)).astype(int))\n",
    "\n",
    "\n",
    "def xy_tensor(x,y):\n",
    "    y_img = np.array(y, np.int32, copy=False)\n",
    "    return TTF.to_tensor(x), convert_y(y_img[:,:,0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "class RRC(transforms.RandomResizedCrop):\n",
    "    def __call__(self, x, y):\n",
    "        i, j, h, w = self.get_params(x, self.scale, self.ratio)\n",
    "        x = TTF.resized_crop(x, i, j, h, w, self.size, self.interpolation)\n",
    "        y = TTF.resized_crop(y, i, j, h, w, self.size, self.interpolation)\n",
    "        return x, y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "def torch_loader(f_ext, data_path, bs, size, workers=7, random_crop=False, pseudo_label=False, val_folder=None):\n",
    "    # Data loading code\n",
    "    x_names = np.sort(np.array(glob(str(data_path/f'CameraRGB{f_ext}'/'*.png'))))\n",
    "    y_names = np.sort(np.array(glob(str(data_path/f'CameraSeg{f_ext}'/'*.png'))))\n",
    "\n",
    "    x_n = x_names.shape[0]\n",
    "    val_idxs = list(range(x_n-300, x_n))\n",
    "    \n",
    "    if pseudo_label:\n",
    "        x_names_test = np.sort(np.array(glob(f'../data/pseudo/CameraRGB{f_ext}/*.png')))\n",
    "        y_names_test = np.sort(np.array(glob(f'../data/pseudo/CameraSeg{f_ext}/*.png')))\n",
    "        x_names = np.concatenate((x_names, x_names_test))\n",
    "        x_names = np.concatenate((y_names, y_names_test))\n",
    "        print(f'Pseudo-Labels: {len(x_names_test)}')\n",
    "    if val_folder:\n",
    "        x_names_val = np.sort(np.array(glob(f'../data/{val_folder}/CameraRGB{f_ext}/*.png')))\n",
    "        y_names_val = np.sort(np.array(glob(f'../data/{val_folder}/CameraSeg{f_ext}/*.png')))\n",
    "        val_x,val_y = x_names_val, y_names_val\n",
    "        trn_x,trn_y = x_names, y_names\n",
    "        print(f'Val Labels:', len(val_x))\n",
    "    else:\n",
    "        ((val_x,trn_x),(val_y,trn_y)) = split_by_idx(val_idxs, x_names, y_names)\n",
    "    print(f'Val x:{len(val_x)}, y:{len(val_y)}')\n",
    "    print(f'Trn x:{len(trn_x)}, y:{len(trn_y)}')\n",
    "    print(f'All x:{len(x_names)}')\n",
    "    normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])\n",
    "    \n",
    "    train_tfms = [\n",
    "        crop_bg_pil,\n",
    "        tfm_x_wrapper(transforms.ColorJitter(.2,.2,.2)),\n",
    "#         tfm_x_wrapper(Lighting(0.1, __imagenet_pca['eigval'], __imagenet_pca['eigvec'])),\n",
    "        RR(),\n",
    "        RHF(),\n",
    "#         RC((size,size)),\n",
    "        xy_tensor,\n",
    "        tfm_x_wrapper(normalize),\n",
    "    ]\n",
    "    if random_crop:\n",
    "        train_tfms.insert(3,RRC(size, scale=(0.4, 1.0)))\n",
    "    train_dataset = MatchedFilesDataset(trn_x, trn_y, train_tfms, path='')\n",
    "    train_loader = torch.utils.data.DataLoader(\n",
    "        train_dataset, batch_size=bs, shuffle=True,\n",
    "        num_workers=workers, pin_memory=True)\n",
    "\n",
    "    val_tfms = [\n",
    "        crop_bg_pil,\n",
    "        xy_tensor,\n",
    "        tfm_x_wrapper(normalize)\n",
    "    ]\n",
    "    val_dataset = MatchedFilesDataset(val_x, val_y, val_tfms, path='')\n",
    "    val_loader = torch.utils.data.DataLoader(\n",
    "        val_dataset, batch_size=bs, shuffle=False,\n",
    "        num_workers=workers, pin_memory=True)\n",
    "\n",
    "    train_loader = DataPrefetcher(train_loader)\n",
    "    val_loader = DataPrefetcher(val_loader)\n",
    "    \n",
    "    data = ModelData(data_path, train_loader, val_loader)\n",
    "    return data\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "def denorm(x):\n",
    "    x_np = x.cpu().numpy()\n",
    "    x_np = np.rollaxis(x_np, 0, 3)\n",
    "    mean=np.array([0.4914 , 0.48216, 0.44653])\n",
    "    std=np.array([0.24703, 0.24349, 0.26159])\n",
    "    x_np = x_np*std+mean\n",
    "    return x_np"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## U-net (ish)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "from torchvision.models import vgg11_bn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "def vgg11(pre): return children(vgg11_bn(pre))[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_meta = {\n",
    "    resnet18:[8,6], resnet34:[8,6], resnet50:[8,6], resnet101:[8,6], resnet152:[8,6],\n",
    "    vgg11:[0,13], vgg16:[0,22], vgg19:[0,22],\n",
    "    resnext50:[8,6], resnext101:[8,6], resnext101_64:[8,6],\n",
    "    wrn:[8,6], inceptionresnet_2:[-2,9], inception_4:[-1,9],\n",
    "    dn121:[0,7], dn161:[0,7], dn169:[0,7], dn201:[0,7],\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_base(f):\n",
    "    cut,lr_cut = model_meta[f]\n",
    "    layers = cut_model(f(True), cut)\n",
    "    return nn.Sequential(*layers), lr_cut"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "class SaveFeatures():\n",
    "    features=None\n",
    "    def __init__(self, m): self.hook = m.register_forward_hook(self.hook_fn)\n",
    "    def hook_fn(self, module, input, output): self.features = output\n",
    "    def remove(self): self.hook.remove()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "BN_EPS = 1e-4  #1e-4  #1e-5\n",
    "class ConvBnRelu2d(nn.Module):\n",
    "    def __init__(self, in_channels, out_channels, kernel_size=3, padding=1, dilation=1, stride=1, groups=1, is_bn=True, is_relu=True):\n",
    "        super(ConvBnRelu2d, self).__init__()\n",
    "        self.conv = nn.Conv2d(in_channels, out_channels, kernel_size=kernel_size, padding=padding, stride=stride, dilation=dilation, groups=groups, bias=False)\n",
    "        self.bn   = nn.BatchNorm2d(out_channels, eps=BN_EPS)\n",
    "        self.relu = nn.ReLU(inplace=True)\n",
    "        if is_bn   is False: self.bn  =None\n",
    "        if is_relu is False: self.relu=None\n",
    "\n",
    "\n",
    "    def forward(self,x):\n",
    "        x = self.conv(x)\n",
    "        if self.bn   is not None: x = self.bn(x)\n",
    "        if self.relu is not None: x = self.relu(x)\n",
    "        return x\n",
    "    \n",
    "class ConvResidual (nn.Module):\n",
    "    def __init__(self, in_channels, out_channels, stride=1):\n",
    "        super(ConvResidual, self).__init__()\n",
    "\n",
    "        self.block = nn.Sequential(\n",
    "            ConvBnRelu2d(in_channels,  out_channels, kernel_size=3, padding=1,  stride=1 ),\n",
    "            ConvBnRelu2d(out_channels, out_channels, kernel_size=3, padding=1,  stride=1, is_relu=False),\n",
    "        )\n",
    "        self.shortcut = None\n",
    "        if in_channels!=out_channels or stride!=1:\n",
    "            self.shortcut = nn.Conv2d(in_channels, out_channels, kernel_size=1, padding=0, stride=stride,  bias=True)\n",
    "\n",
    "    def forward(self, x):\n",
    "        r = x if self.shortcut is None else self.shortcut(x)\n",
    "        x = self.block(x)\n",
    "        x = F.relu(x.add_(r), inplace=True)\n",
    "        return x\n",
    "    \n",
    "\n",
    "## origainl 3x3 stack filters used in UNet## origa \n",
    "class StackEncoder (nn.Module):\n",
    "    def __init__(self, x_channels, y_channels, kernel_size=3, max_pool=False):\n",
    "        super(StackEncoder, self).__init__()\n",
    "        padding=(kernel_size-1)//2\n",
    "        self.encode = nn.Sequential(\n",
    "            ConvBnRelu2d(x_channels, y_channels, kernel_size=kernel_size, padding=padding, dilation=1, stride=1, groups=1),\n",
    "            ConvBnRelu2d(y_channels, y_channels, kernel_size=kernel_size, padding=padding, dilation=1, stride=1, groups=1),\n",
    "        )\n",
    "        self.mp = nn.MaxPool2d(2) if max_pool else nn.Conv2d(y_channels, y_channels, kernel_size=kernel_size, padding=padding, stride=2)\n",
    "\n",
    "    def forward(self,x):\n",
    "        y = self.encode(x)\n",
    "        y_small = self.mp(y)\n",
    "        return y, y_small\n",
    "\n",
    "\n",
    "class StackDecoder (nn.Module):\n",
    "    def __init__(self, x_big_channels, x_channels, y_channels, kernel_size=3):\n",
    "        super(StackDecoder, self).__init__()\n",
    "        padding=(kernel_size-1)//2\n",
    "\n",
    "        self.decode = nn.Sequential(\n",
    "            ConvBnRelu2d(x_big_channels+x_channels, y_channels, kernel_size=kernel_size, padding=padding, dilation=1, stride=1, groups=1),\n",
    "            ConvBnRelu2d(y_channels, y_channels, kernel_size=kernel_size, padding=padding, dilation=1, stride=1, groups=1),\n",
    "            ConvBnRelu2d(y_channels, y_channels, kernel_size=kernel_size, padding=padding, dilation=1, stride=1, groups=1),\n",
    "        )\n",
    "\n",
    "    def forward(self, x_big, x):\n",
    "        N,C,H,W = x_big.size()\n",
    "        y = F.upsample(x, size=(H,W),mode='bilinear')\n",
    "        #y = F.upsample(x, scale_factor=2,mode='bilinear')\n",
    "        y = torch.cat([y,x_big],1)\n",
    "        y = self.decode(y)\n",
    "        return  y\n",
    "    # 256x256\n",
    "    \n",
    "class Unet256 (nn.Module):\n",
    "    def __init__(self, out_c=2, in_c=3, f=2):\n",
    "        super().__init__()\n",
    "#         C,H,W = in_shape\n",
    "        #assert(C==3)\n",
    "\n",
    "        #256\n",
    "        self.down2 = StackEncoder(in_c,   64//f, kernel_size=3)   #128\n",
    "        self.down3 = StackEncoder( 64//f,  128//f, kernel_size=3)   # 64\n",
    "        self.down4 = StackEncoder(128//f,  256//f, kernel_size=3)   # 32\n",
    "        self.down5 = StackEncoder(256//f,  512//f, kernel_size=3)   # 16\n",
    "        self.down6 = StackEncoder(512//f, 1024//f, kernel_size=3)   #  8\n",
    "\n",
    "        self.center = nn.Sequential(\n",
    "            #ConvBnRelu2d( 512, 1024, kernel_size=3, padding=1, stride=1 ),\n",
    "            ConvBnRelu2d(1024//f, 1024//f, kernel_size=3, padding=1, stride=1 ),\n",
    "        )\n",
    "\n",
    "        # 8\n",
    "        # x_big_channels, x_channels, y_channels\n",
    "        self.up6 = StackDecoder(1024//f,1024//f, 512//f, kernel_size=3)  # 16\n",
    "        self.up5 = StackDecoder( 512//f, 512//f, 256//f, kernel_size=3)  # 32\n",
    "        self.up4 = StackDecoder( 256//f, 256//f, 128//f, kernel_size=3)  # 64\n",
    "        self.up3 = StackDecoder( 128//f, 128//f,  64//f, kernel_size=3)  #128\n",
    "        self.up2 = StackDecoder(  64//f,  64//f,  32, kernel_size=3)  #256\n",
    "        self.classify = nn.Conv2d(32, out_c, kernel_size=1, padding=0, stride=1, bias=True)\n",
    "\n",
    "\n",
    "    def forward(self, x):\n",
    "\n",
    "        out = x                       #;print('x    ',x.size())\n",
    "                                      #\n",
    "        down2,out = self.down2(out)   #;print('down2',down2.size())  #128\n",
    "        down3,out = self.down3(out)   #;print('down3',down3.size())  #64\n",
    "        down4,out = self.down4(out)   #;print('down4',down4.size())  #32\n",
    "        down5,out = self.down5(out)   #;print('down5',down5.size())  #16\n",
    "        down6,out = self.down6(out)   #;print('down6',down6.size())  #8\n",
    "        pass                          #;print('out  ',out.size())\n",
    "\n",
    "        out = self.center(out)\n",
    "        out = self.up6(down6, out)\n",
    "        out = self.up5(down5, out)\n",
    "        out = self.up4(down4, out)\n",
    "        out = self.up3(down3, out)\n",
    "        out = self.up2(down2, out)\n",
    "\n",
    "        out = self.classify(out)\n",
    "        out = torch.squeeze(out, dim=1)\n",
    "        return out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "class UnetBlock(nn.Module):\n",
    "    def __init__(self, up_in, x_in, n_out):\n",
    "        super().__init__()\n",
    "        up_out = x_out = n_out//2\n",
    "        self.x_conv  = nn.Conv2d(x_in,  x_out,  1)\n",
    "        self.tr_conv = nn.ConvTranspose2d(up_in, up_out, 2, stride=2)\n",
    "        self.bn = nn.BatchNorm2d(n_out)\n",
    "        \n",
    "    def forward(self, up_p, x_p):\n",
    "        up_p = self.tr_conv(up_p)\n",
    "        x_p = self.x_conv(x_p)\n",
    "        cat_p = torch.cat([up_p,x_p], dim=1)\n",
    "        return self.bn(F.relu(cat_p, inplace=True))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Unet34(nn.Module):\n",
    "    def __init__(self, f=resnet34):\n",
    "        super().__init__()\n",
    "        m_base, lr_cut = get_base(f)\n",
    "        self.rn = m_base\n",
    "        self.lr_cut = lr_cut\n",
    "        self.sfs = [SaveFeatures(self.rn[i]) for i in [2,4,5,6]]\n",
    "        self.up1 = UnetBlock(512,256,256)\n",
    "        self.up2 = UnetBlock(256,128,256)\n",
    "        self.up3 = UnetBlock(256,64,256)\n",
    "        self.up4 = UnetBlock(256,64,256)\n",
    "        self.up5 = UnetBlock(256,3,16)\n",
    "        self.up6 = nn.ConvTranspose2d(16, 2, 1)\n",
    "        \n",
    "    def forward(self,x):\n",
    "        inp = x\n",
    "        x = F.relu(self.rn(x), inplace=True)\n",
    "        x = self.up1(x, self.sfs[3].features)\n",
    "        x = self.up2(x, self.sfs[2].features)\n",
    "        x = self.up3(x, self.sfs[1].features)\n",
    "        x = self.up4(x, self.sfs[0].features)\n",
    "        x = self.up5(x, inp)\n",
    "        x = self.up6(x)\n",
    "        return torch.squeeze(x)\n",
    "    \n",
    "    def close(self):\n",
    "        for sf in self.sfs: sf.remove()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Unet34Mod(nn.Module):\n",
    "    def __init__(self, f=resnet34):\n",
    "        super().__init__()\n",
    "        m_base, lr_cut = get_base(f)\n",
    "        self.rn = m_base\n",
    "        self.lr_cut = lr_cut\n",
    "        self.sfs = [SaveFeatures(self.rn[i]) for i in [2,4,5,6]]\n",
    "        self.up1 = UnetBlock(512,256,256)\n",
    "        self.up2 = UnetBlock(256,128,256)\n",
    "        self.up3 = UnetBlock(256,64,256)\n",
    "        self.up4 = UnetBlock(256,64,256)\n",
    "        self.up5 = UnetBlock(256,16,16)\n",
    "        self.up6 = nn.ConvTranspose2d(16, 2, 1)\n",
    "        self.x_skip = nn.Sequential(\n",
    "            nn.Conv2d(3,16,1),\n",
    "            nn.BatchNorm2d(16),\n",
    "            nn.ReLU(inplace=True)\n",
    "        )\n",
    "        \n",
    "    def forward(self,x):\n",
    "        x_skip = self.x_skip(x)\n",
    "        x = F.relu(self.rn(x), inplace=True)\n",
    "        x = self.up1(x, self.sfs[3].features)\n",
    "        x = self.up2(x, self.sfs[2].features)\n",
    "        x = self.up3(x, self.sfs[1].features)\n",
    "        x = self.up4(x, self.sfs[0].features)\n",
    "        x = self.up5(x, x_skip)\n",
    "        x = self.up6(x)\n",
    "        return torch.squeeze(x)\n",
    "    \n",
    "    def close(self):\n",
    "        for sf in self.sfs: sf.remove()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Unet11(nn.Module):\n",
    "    def __init__(self, f=vgg11):\n",
    "        super().__init__()\n",
    "        m_base, lr_cut = get_base(f)\n",
    "        self.rn = m_base\n",
    "        self.lr_cut = lr_cut\n",
    "        self.sfs = [SaveFeatures(self.rn[0][i]) for i in [2,6,13,20,27]]\n",
    "        self.up0 = UnetBlock(512,512,256)\n",
    "        self.up1 = UnetBlock(256,512,256)\n",
    "        self.up2 = UnetBlock(256,256,256)\n",
    "        self.up3 = UnetBlock(256,128,256)\n",
    "        self.up4 = UnetBlock(256,64,256)\n",
    "        self.up5  = nn.Conv2d(256,2,1)\n",
    "        \n",
    "    def forward(self,x):\n",
    "        x = F.relu(self.rn(x))\n",
    "        x = self.up0(x, self.sfs[4].features)\n",
    "        x = self.up1(x, self.sfs[3].features)\n",
    "        x = self.up2(x, self.sfs[2].features)\n",
    "        x = self.up3(x, self.sfs[1].features)\n",
    "        x = self.up4(x, self.sfs[0].features)\n",
    "        x = self.up5(x)\n",
    "        return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "class UnetModel():\n",
    "    def __init__(self,model,name='unet'):\n",
    "        self.model,self.name = model,name\n",
    "\n",
    "    def get_layer_groups(self, precompute):\n",
    "        if isinstance(self.model, FP16):\n",
    "            model = self.model.module\n",
    "        else:\n",
    "            model = self.model\n",
    "        lgs = list(split_by_idxs(children(model.rn), [model.lr_cut]))\n",
    "#         print('LGS:', lgs)\n",
    "#         print('Add:', children(model)[1:])\n",
    "        return lgs + [children(model)[1:]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "def car_f_p_r(pred, targs):\n",
    "    p2 = F.sigmoid(pred)\n",
    "    return fbeta_score(targs[:,0,:,:], p2[:,0,:,:], beta=2, threshold=0.5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "def rd_f(pred, targs):\n",
    "    mx,idx = torch.max(pred, 1)\n",
    "    p2 = F.sigmoid(pred)\n",
    "    f,p,r = fbeta_score(targs[:,1,:,:], p2[:,1,:,:], beta=0.5, threshold=0.5)\n",
    "    return f"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "def fbeta_score(y_true, y_pred, beta, threshold=None, eps=1e-9):\n",
    "    beta2 = beta**2\n",
    "\n",
    "    if threshold:\n",
    "        y_pred = torch.ge(y_pred.float(), threshold).float()\n",
    "    else:\n",
    "        y_pred = y_pred.float()\n",
    "    y_true = y_true.float()\n",
    "\n",
    "    true_positive = (y_pred * y_true).sum()\n",
    "    precision = true_positive/(y_pred.sum()+(eps))\n",
    "    recall = true_positive/(y_true.sum()+eps)\n",
    "    \n",
    "    fb = (precision*recall)/(precision*beta2 + recall + eps)*(1+beta2)\n",
    "    \n",
    "    return fb, precision, recall"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "def new_acc(pred, targs):\n",
    "    p2 = F.sigmoid(pred)\n",
    "    return ((p2>0.5).long() == targs).float().mean()\n",
    "def dice_mult(pred, targs):\n",
    "#     pred = (pred>0).float()\n",
    "    mx,idx = torch.max(pred, 1)\n",
    "    pred = idx.float()\n",
    "    targs = targs.float()\n",
    "    return 2. * (pred*targs).sum() / (pred+targs).sum()\n",
    "def dice(pred, targs):\n",
    "    pred = (pred>0).float()\n",
    "    return 2. * (pred*targs).sum() / (pred+targs).sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "def dice_coeff_weight(pred, target, weight):\n",
    "    smooth = 1.\n",
    "    num,c,h,w = pred.shape\n",
    "    m1 = pred.view(num, c, -1)  # Flatten\n",
    "    m2 = target.view(num, c, -1)  # Flatten\n",
    "    intersection = (m1 * m2)\n",
    "    w = V(torch.cuda.FloatTensor(weight).view(1,-1,1))\n",
    "    i_w = (w*intersection).sum()\n",
    "    m1_w = (w*m1).sum()\n",
    "    m2_w = (w*m2).sum()\n",
    "    return (2. * i_w + smooth) / (m1_w + m2_w + smooth)\n",
    "\n",
    "def dice_coeff(pred, target):\n",
    "    smooth = 1.\n",
    "    num,c,h,w = pred.shape\n",
    "    m1 = pred.view(num, c, -1)  # Flatten\n",
    "    m2 = target.view(num, c, -1)  # Flatten\n",
    "    intersection = (m1 * m2).sum()\n",
    "    return (2. * intersection + smooth) / (m1.sum() + m2.sum() + smooth)\n",
    "\n",
    "\n",
    "class SoftDiceLoss(nn.Module):\n",
    "    def __init__(self, weight=None, size_average=True):\n",
    "        super(SoftDiceLoss, self).__init__()\n",
    "        self.weight = weight\n",
    "\n",
    "    def forward(self, logits, targets):\n",
    "        probs = F.sigmoid(logits)\n",
    "        num = targets.size(0)  # Number of batches\n",
    "\n",
    "        if self.weight:\n",
    "            score = dice_coeff_weight(probs, targets.float(), self.weight)\n",
    "        else:\n",
    "            score = dice_coeff(probs, targets.float())\n",
    "        score = 1 - score.sum() / num\n",
    "        return score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_learner(md, m_fn=Unet34):\n",
    "    m = to_gpu(m_fn())\n",
    "    models = UnetModel(m)\n",
    "    learn = ConvLearner(md, models)\n",
    "    learn.opt_fn=optim.Adam\n",
    "#     class_weights = torch.FloatTensor([1,3,2]).cuda()\n",
    "#     learn.crit=nn.CrossEntropyLoss(weight=class_weights)\n",
    "    learn.crit=SoftDiceLoss(weight=[1,1/100])\n",
    "    learn.metrics=[new_acc, rd_f, car_f_p_r]\n",
    "    return learn"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Unet50"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Unet50Mod(nn.Module):\n",
    "    def __init__(self, f=resnet50):\n",
    "        super().__init__()\n",
    "        m_base, lr_cut = get_base(f)\n",
    "        self.rn = m_base\n",
    "        self.lr_cut = lr_cut\n",
    "        self.sfs = [SaveFeatures(self.rn[i]) for i in [2,4,5,6]]\n",
    "        self.up1 = UnetBlock(2048,1024,1024)\n",
    "        self.up2 = UnetBlock(1024,512,512)\n",
    "        self.up3 = UnetBlock(512,256,256)\n",
    "        self.up4 = UnetBlock(256,64,256)\n",
    "        self.up5 = UnetBlock(256,16,16)\n",
    "        self.up6 = nn.ConvTranspose2d(16, 2, 1)\n",
    "        self.x_skip = nn.Sequential(\n",
    "            nn.Conv2d(3,16,1),\n",
    "            nn.BatchNorm2d(16),\n",
    "            nn.ReLU(inplace=True)\n",
    "        )\n",
    "        \n",
    "    def forward(self,x):\n",
    "        x_skip = self.x_skip(x)\n",
    "        x = F.relu(self.rn(x), inplace=True)\n",
    "        x = self.up1(x, self.sfs[3].features)\n",
    "        x = self.up2(x, self.sfs[2].features)\n",
    "        x = self.up3(x, self.sfs[1].features)\n",
    "        x = self.up4(x, self.sfs[0].features)\n",
    "        x = self.up5(x, x_skip)\n",
    "        x = self.up6(x)\n",
    "        return torch.squeeze(x)\n",
    "    \n",
    "    def close(self):\n",
    "        for sf in self.sfs: sf.remove()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "m_base, lr_cut = get_base(resnet34)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Sequential(\n",
       "  (0): Conv2d(3, 64, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), bias=False)\n",
       "  (1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True)\n",
       "  (2): ReLU(inplace)\n",
       "  (3): MaxPool2d(kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), dilation=(1, 1), ceil_mode=False)\n",
       "  (4): Sequential(\n",
       "    (0): BasicBlock(\n",
       "      (conv1): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True)\n",
       "      (relu): ReLU(inplace)\n",
       "      (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True)\n",
       "    )\n",
       "    (1): BasicBlock(\n",
       "      (conv1): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True)\n",
       "      (relu): ReLU(inplace)\n",
       "      (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True)\n",
       "    )\n",
       "    (2): BasicBlock(\n",
       "      (conv1): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True)\n",
       "      (relu): ReLU(inplace)\n",
       "      (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True)\n",
       "    )\n",
       "  )\n",
       "  (5): Sequential(\n",
       "    (0): BasicBlock(\n",
       "      (conv1): Conv2d(64, 128, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n",
       "      (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True)\n",
       "      (relu): ReLU(inplace)\n",
       "      (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True)\n",
       "      (downsample): Sequential(\n",
       "        (0): Conv2d(64, 128, kernel_size=(1, 1), stride=(2, 2), bias=False)\n",
       "        (1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True)\n",
       "      )\n",
       "    )\n",
       "    (1): BasicBlock(\n",
       "      (conv1): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True)\n",
       "      (relu): ReLU(inplace)\n",
       "      (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True)\n",
       "    )\n",
       "    (2): BasicBlock(\n",
       "      (conv1): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True)\n",
       "      (relu): ReLU(inplace)\n",
       "      (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True)\n",
       "    )\n",
       "    (3): BasicBlock(\n",
       "      (conv1): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True)\n",
       "      (relu): ReLU(inplace)\n",
       "      (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True)\n",
       "    )\n",
       "  )\n",
       "  (6): Sequential(\n",
       "    (0): BasicBlock(\n",
       "      (conv1): Conv2d(128, 256, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n",
       "      (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True)\n",
       "      (relu): ReLU(inplace)\n",
       "      (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True)\n",
       "      (downsample): Sequential(\n",
       "        (0): Conv2d(128, 256, kernel_size=(1, 1), stride=(2, 2), bias=False)\n",
       "        (1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True)\n",
       "      )\n",
       "    )\n",
       "    (1): BasicBlock(\n",
       "      (conv1): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True)\n",
       "      (relu): ReLU(inplace)\n",
       "      (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True)\n",
       "    )\n",
       "    (2): BasicBlock(\n",
       "      (conv1): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True)\n",
       "      (relu): ReLU(inplace)\n",
       "      (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True)\n",
       "    )\n",
       "    (3): BasicBlock(\n",
       "      (conv1): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True)\n",
       "      (relu): ReLU(inplace)\n",
       "      (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True)\n",
       "    )\n",
       "    (4): BasicBlock(\n",
       "      (conv1): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True)\n",
       "      (relu): ReLU(inplace)\n",
       "      (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True)\n",
       "    )\n",
       "    (5): BasicBlock(\n",
       "      (conv1): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True)\n",
       "      (relu): ReLU(inplace)\n",
       "      (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True)\n",
       "    )\n",
       "  )\n",
       "  (7): Sequential(\n",
       "    (0): BasicBlock(\n",
       "      (conv1): Conv2d(256, 512, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n",
       "      (bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True)\n",
       "      (relu): ReLU(inplace)\n",
       "      (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True)\n",
       "      (downsample): Sequential(\n",
       "        (0): Conv2d(256, 512, kernel_size=(1, 1), stride=(2, 2), bias=False)\n",
       "        (1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True)\n",
       "      )\n",
       "    )\n",
       "    (1): BasicBlock(\n",
       "      (conv1): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True)\n",
       "      (relu): ReLU(inplace)\n",
       "      (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True)\n",
       "    )\n",
       "    (2): BasicBlock(\n",
       "      (conv1): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True)\n",
       "      (relu): ReLU(inplace)\n",
       "      (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True)\n",
       "    )\n",
       "  )\n",
       ")"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "m_base"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "m_base, lr_cut = get_base(resnet50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Sequential(\n",
       "  (0): Conv2d(3, 64, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), bias=False)\n",
       "  (1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True)\n",
       "  (2): ReLU(inplace)\n",
       "  (3): MaxPool2d(kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), dilation=(1, 1), ceil_mode=False)\n",
       "  (4): Sequential(\n",
       "    (0): Bottleneck(\n",
       "      (conv1): Conv2d(64, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "      (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True)\n",
       "      (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True)\n",
       "      (conv3): Conv2d(64, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "      (bn3): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True)\n",
       "      (relu): ReLU(inplace)\n",
       "      (downsample): Sequential(\n",
       "        (0): Conv2d(64, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True)\n",
       "      )\n",
       "    )\n",
       "    (1): Bottleneck(\n",
       "      (conv1): Conv2d(256, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "      (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True)\n",
       "      (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True)\n",
       "      (conv3): Conv2d(64, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "      (bn3): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True)\n",
       "      (relu): ReLU(inplace)\n",
       "    )\n",
       "    (2): Bottleneck(\n",
       "      (conv1): Conv2d(256, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "      (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True)\n",
       "      (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True)\n",
       "      (conv3): Conv2d(64, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "      (bn3): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True)\n",
       "      (relu): ReLU(inplace)\n",
       "    )\n",
       "  )\n",
       "  (5): Sequential(\n",
       "    (0): Bottleneck(\n",
       "      (conv1): Conv2d(256, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "      (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True)\n",
       "      (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n",
       "      (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True)\n",
       "      (conv3): Conv2d(128, 512, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "      (bn3): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True)\n",
       "      (relu): ReLU(inplace)\n",
       "      (downsample): Sequential(\n",
       "        (0): Conv2d(256, 512, kernel_size=(1, 1), stride=(2, 2), bias=False)\n",
       "        (1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True)\n",
       "      )\n",
       "    )\n",
       "    (1): Bottleneck(\n",
       "      (conv1): Conv2d(512, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "      (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True)\n",
       "      (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True)\n",
       "      (conv3): Conv2d(128, 512, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "      (bn3): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True)\n",
       "      (relu): ReLU(inplace)\n",
       "    )\n",
       "    (2): Bottleneck(\n",
       "      (conv1): Conv2d(512, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "      (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True)\n",
       "      (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True)\n",
       "      (conv3): Conv2d(128, 512, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "      (bn3): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True)\n",
       "      (relu): ReLU(inplace)\n",
       "    )\n",
       "    (3): Bottleneck(\n",
       "      (conv1): Conv2d(512, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "      (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True)\n",
       "      (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True)\n",
       "      (conv3): Conv2d(128, 512, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "      (bn3): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True)\n",
       "      (relu): ReLU(inplace)\n",
       "    )\n",
       "  )\n",
       "  (6): Sequential(\n",
       "    (0): Bottleneck(\n",
       "      (conv1): Conv2d(512, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "      (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True)\n",
       "      (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n",
       "      (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True)\n",
       "      (conv3): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "      (bn3): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True)\n",
       "      (relu): ReLU(inplace)\n",
       "      (downsample): Sequential(\n",
       "        (0): Conv2d(512, 1024, kernel_size=(1, 1), stride=(2, 2), bias=False)\n",
       "        (1): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True)\n",
       "      )\n",
       "    )\n",
       "    (1): Bottleneck(\n",
       "      (conv1): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "      (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True)\n",
       "      (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True)\n",
       "      (conv3): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "      (bn3): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True)\n",
       "      (relu): ReLU(inplace)\n",
       "    )\n",
       "    (2): Bottleneck(\n",
       "      (conv1): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "      (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True)\n",
       "      (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True)\n",
       "      (conv3): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "      (bn3): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True)\n",
       "      (relu): ReLU(inplace)\n",
       "    )\n",
       "    (3): Bottleneck(\n",
       "      (conv1): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "      (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True)\n",
       "      (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True)\n",
       "      (conv3): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "      (bn3): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True)\n",
       "      (relu): ReLU(inplace)\n",
       "    )\n",
       "    (4): Bottleneck(\n",
       "      (conv1): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "      (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True)\n",
       "      (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True)\n",
       "      (conv3): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "      (bn3): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True)\n",
       "      (relu): ReLU(inplace)\n",
       "    )\n",
       "    (5): Bottleneck(\n",
       "      (conv1): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "      (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True)\n",
       "      (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True)\n",
       "      (conv3): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "      (bn3): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True)\n",
       "      (relu): ReLU(inplace)\n",
       "    )\n",
       "  )\n",
       "  (7): Sequential(\n",
       "    (0): Bottleneck(\n",
       "      (conv1): Conv2d(1024, 512, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "      (bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True)\n",
       "      (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n",
       "      (bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True)\n",
       "      (conv3): Conv2d(512, 2048, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "      (bn3): BatchNorm2d(2048, eps=1e-05, momentum=0.1, affine=True)\n",
       "      (relu): ReLU(inplace)\n",
       "      (downsample): Sequential(\n",
       "        (0): Conv2d(1024, 2048, kernel_size=(1, 1), stride=(2, 2), bias=False)\n",
       "        (1): BatchNorm2d(2048, eps=1e-05, momentum=0.1, affine=True)\n",
       "      )\n",
       "    )\n",
       "    (1): Bottleneck(\n",
       "      (conv1): Conv2d(2048, 512, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "      (bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True)\n",
       "      (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True)\n",
       "      (conv3): Conv2d(512, 2048, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "      (bn3): BatchNorm2d(2048, eps=1e-05, momentum=0.1, affine=True)\n",
       "      (relu): ReLU(inplace)\n",
       "    )\n",
       "    (2): Bottleneck(\n",
       "      (conv1): Conv2d(2048, 512, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "      (bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True)\n",
       "      (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "      (bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True)\n",
       "      (conv3): Conv2d(512, 2048, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "      (bn3): BatchNorm2d(2048, eps=1e-05, momentum=0.1, affine=True)\n",
       "      (relu): ReLU(inplace)\n",
       "    )\n",
       "  )\n",
       ")"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "m_base"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Unet50Mod2(nn.Module):\n",
    "    def __init__(self, f=resnet50):\n",
    "        super().__init__()\n",
    "        m_base, lr_cut = get_base(f)\n",
    "        self.rn = m_base\n",
    "        self.lr_cut = lr_cut\n",
    "        self.sfs = [SaveFeatures(self.rn[i]) for i in [2,4,5,6]]\n",
    "        self.up1 = UnetBlock(2048,1024,1024)\n",
    "        self.up2 = UnetBlock(1024,512,512)\n",
    "        self.up3 = UnetBlock(512,256,256)\n",
    "        self.up4 = UnetBlock(256,64,256)\n",
    "        self.up5 = UnetBlock(256,32,32)\n",
    "        self.up6 = nn.ConvTranspose2d(32, 3, 1)\n",
    "        self.x_skip = nn.Sequential(\n",
    "            nn.Conv2d(3,32,1),\n",
    "            nn.BatchNorm2d(32),\n",
    "            nn.ReLU(inplace=True)\n",
    "        )\n",
    "        \n",
    "    def forward(self,x):\n",
    "        x_skip = self.x_skip(x)\n",
    "        x = F.relu(self.rn(x), inplace=True)\n",
    "        x = self.up1(x, self.sfs[3].features)\n",
    "        x = self.up2(x, self.sfs[2].features)\n",
    "        x = self.up3(x, self.sfs[1].features)\n",
    "        x = self.up4(x, self.sfs[0].features)\n",
    "        x = self.up5(x, x_skip)\n",
    "        x = self.up6(x)\n",
    "        return torch.squeeze(x)\n",
    "    \n",
    "    def close(self):\n",
    "        for sf in self.sfs: sf.remove()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val x:300, y:300\n",
      "Trn x:6980, y:6980\n",
      "All x:7280\n"
     ]
    }
   ],
   "source": [
    "ext = '-300'\n",
    "sz = 192\n",
    "bs = 32\n",
    "md = torch_loader(ext, PATH, bs, sz, workers, random_crop, pseudo_label)\n",
    "\n",
    "learn = get_learner(md, m_fn=Unet50Mod)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import gc\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.cuda.empty_cache()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "037c2312b1224ac197c457e607f77867",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=5), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 21%|██        | 46/219 [00:09<00:35,  4.93it/s, loss=0.984]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Process Process-83:\n",
      "Process Process-81:\n",
      "Process Process-80:\n",
      "Process Process-82:\n",
      "Process Process-79:\n",
      "Process Process-78:\n",
      "Process Process-84:\n",
      "Traceback (most recent call last):\n",
      "Traceback (most recent call last):\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/paperspace/anaconda3/envs/lyft/lib/python3.6/multiprocessing/process.py\", line 258, in _bootstrap\n",
      "    self.run()\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/paperspace/anaconda3/envs/lyft/lib/python3.6/multiprocessing/process.py\", line 258, in _bootstrap\n",
      "    self.run()\n",
      "  File \"/home/paperspace/anaconda3/envs/lyft/lib/python3.6/multiprocessing/process.py\", line 258, in _bootstrap\n",
      "    self.run()\n",
      "Traceback (most recent call last):\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/paperspace/anaconda3/envs/lyft/lib/python3.6/multiprocessing/process.py\", line 258, in _bootstrap\n",
      "    self.run()\n",
      "  File \"/home/paperspace/anaconda3/envs/lyft/lib/python3.6/multiprocessing/process.py\", line 93, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "  File \"/home/paperspace/anaconda3/envs/lyft/lib/python3.6/multiprocessing/process.py\", line 93, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "  File \"/home/paperspace/anaconda3/envs/lyft/lib/python3.6/multiprocessing/process.py\", line 258, in _bootstrap\n",
      "    self.run()\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/paperspace/anaconda3/envs/lyft/lib/python3.6/multiprocessing/process.py\", line 93, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "  File \"/home/paperspace/anaconda3/envs/lyft/lib/python3.6/site-packages/torch/utils/data/dataloader.py\", line 55, in _worker_loop\n",
      "    samples = collate_fn([dataset[i] for i in batch_indices])\n",
      "  File \"/home/paperspace/anaconda3/envs/lyft/lib/python3.6/site-packages/torch/utils/data/dataloader.py\", line 50, in _worker_loop\n",
      "    r = index_queue.get()\n",
      "  File \"/home/paperspace/anaconda3/envs/lyft/lib/python3.6/multiprocessing/process.py\", line 93, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "  File \"/home/paperspace/anaconda3/envs/lyft/lib/python3.6/multiprocessing/process.py\", line 258, in _bootstrap\n",
      "    self.run()\n",
      "  File \"/home/paperspace/anaconda3/envs/lyft/lib/python3.6/multiprocessing/process.py\", line 93, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "  File \"/home/paperspace/anaconda3/envs/lyft/lib/python3.6/multiprocessing/process.py\", line 258, in _bootstrap\n",
      "    self.run()\n",
      "  File \"/home/paperspace/anaconda3/envs/lyft/lib/python3.6/site-packages/torch/utils/data/dataloader.py\", line 55, in <listcomp>\n",
      "    samples = collate_fn([dataset[i] for i in batch_indices])\n",
      "  File \"/home/paperspace/anaconda3/envs/lyft/lib/python3.6/multiprocessing/process.py\", line 93, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "  File \"/home/paperspace/anaconda3/envs/lyft/lib/python3.6/site-packages/torch/utils/data/dataloader.py\", line 55, in _worker_loop\n",
      "    samples = collate_fn([dataset[i] for i in batch_indices])\n",
      "  File \"/home/paperspace/anaconda3/envs/lyft/lib/python3.6/multiprocessing/process.py\", line 93, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "  File \"/home/paperspace/anaconda3/envs/lyft/lib/python3.6/site-packages/torch/utils/data/dataloader.py\", line 55, in _worker_loop\n",
      "    samples = collate_fn([dataset[i] for i in batch_indices])\n",
      "  File \"/home/paperspace/anaconda3/envs/lyft/lib/python3.6/multiprocessing/queues.py\", line 334, in get\n",
      "    with self._rlock:\n",
      "  File \"<ipython-input-10-e4a515af82db>\", line 26, in __getitem__\n",
      "    return self.get(self.tfms, x, y)\n",
      "  File \"/home/paperspace/anaconda3/envs/lyft/lib/python3.6/site-packages/torch/utils/data/dataloader.py\", line 50, in _worker_loop\n",
      "    r = index_queue.get()\n",
      "  File \"/home/paperspace/anaconda3/envs/lyft/lib/python3.6/site-packages/torch/utils/data/dataloader.py\", line 55, in <listcomp>\n",
      "    samples = collate_fn([dataset[i] for i in batch_indices])\n",
      "  File \"/home/paperspace/anaconda3/envs/lyft/lib/python3.6/multiprocessing/queues.py\", line 334, in get\n",
      "    with self._rlock:\n",
      "  File \"/home/paperspace/anaconda3/envs/lyft/lib/python3.6/site-packages/torch/utils/data/dataloader.py\", line 50, in _worker_loop\n",
      "    r = index_queue.get()\n",
      "  File \"/home/paperspace/anaconda3/envs/lyft/lib/python3.6/site-packages/torch/utils/data/dataloader.py\", line 50, in _worker_loop\n",
      "    r = index_queue.get()\n",
      "  File \"/home/paperspace/anaconda3/envs/lyft/lib/python3.6/multiprocessing/synchronize.py\", line 96, in __enter__\n",
      "    return self._semlock.__enter__()\n",
      "  File \"<ipython-input-10-e4a515af82db>\", line 21, in get\n",
      "    x, y = fn(x, y)\n",
      "  File \"/home/paperspace/anaconda3/envs/lyft/lib/python3.6/multiprocessing/queues.py\", line 334, in get\n",
      "    with self._rlock:\n",
      "  File \"/home/paperspace/anaconda3/envs/lyft/lib/python3.6/site-packages/torch/utils/data/dataloader.py\", line 55, in <listcomp>\n",
      "    samples = collate_fn([dataset[i] for i in batch_indices])\n",
      "  File \"<ipython-input-15-6d70a2ea31a4>\", line 2, in <lambda>\n",
      "    return lambda x,y: (tfm(x), y)\n",
      "  File \"/home/paperspace/anaconda3/envs/lyft/lib/python3.6/multiprocessing/synchronize.py\", line 96, in __enter__\n",
      "    return self._semlock.__enter__()\n",
      "  File \"/home/paperspace/anaconda3/envs/lyft/lib/python3.6/multiprocessing/synchronize.py\", line 96, in __enter__\n",
      "    return self._semlock.__enter__()\n",
      "  File \"/home/paperspace/anaconda3/envs/lyft/lib/python3.6/multiprocessing/queues.py\", line 335, in get\n",
      "    res = self._reader.recv_bytes()\n",
      "KeyboardInterrupt\n",
      "  File \"<ipython-input-10-e4a515af82db>\", line 25, in __getitem__\n",
      "    x,y = self.get_x(idx),self.get_y(idx)\n",
      "KeyboardInterrupt\n",
      "  File \"<ipython-input-10-e4a515af82db>\", line 25, in __getitem__\n",
      "    x,y = self.get_x(idx),self.get_y(idx)\n",
      "  File \"/home/paperspace/anaconda3/envs/lyft/lib/python3.6/site-packages/torchvision/transforms/transforms.py\", line 579, in __call__\n",
      "    return transform(img)\n",
      "  File \"/home/paperspace/anaconda3/envs/lyft/lib/python3.6/multiprocessing/connection.py\", line 216, in recv_bytes\n",
      "    buf = self._recv_bytes(maxlength)\n",
      "KeyboardInterrupt\n",
      "  File \"<ipython-input-10-e4a515af82db>\", line 13, in get_x\n",
      "    def get_x(self, i): return self.open_fn(os.path.join(self.path, self.fnames[i]))\n",
      "  File \"<ipython-input-10-e4a515af82db>\", line 13, in get_x\n",
      "    def get_x(self, i): return self.open_fn(os.path.join(self.path, self.fnames[i]))\n",
      "  File \"/home/paperspace/anaconda3/envs/lyft/lib/python3.6/site-packages/torchvision/datasets/folder.py\", line 52, in pil_loader\n",
      "    return img.convert('RGB')\n",
      "  File \"/home/paperspace/anaconda3/envs/lyft/lib/python3.6/site-packages/torchvision/transforms/transforms.py\", line 42, in __call__\n",
      "    img = t(img)\n",
      "  File \"/home/paperspace/anaconda3/envs/lyft/lib/python3.6/site-packages/PIL/Image.py\", line 879, in convert\n",
      "    self.load()\n",
      "  File \"/home/paperspace/anaconda3/envs/lyft/lib/python3.6/site-packages/torchvision/transforms/transforms.py\", line 232, in __call__\n",
      "    return self.lambd(img)\n",
      "  File \"/home/paperspace/anaconda3/envs/lyft/lib/python3.6/site-packages/torchvision/transforms/transforms.py\", line 550, in <lambda>\n",
      "    transforms.append(Lambda(lambda img: F.adjust_brightness(img, brightness_factor)))\n",
      "  File \"/home/paperspace/anaconda3/envs/lyft/lib/python3.6/site-packages/PIL/Image.py\", line 2631, in blend\n",
      "    return im1._new(core.blend(im1.im, im2.im, alpha))\n",
      "  File \"/home/paperspace/anaconda3/envs/lyft/lib/python3.6/site-packages/torchvision/transforms/functional.py\", line 406, in adjust_brightness\n",
      "    img = enhancer.enhance(brightness_factor)\n",
      "  File \"/home/paperspace/anaconda3/envs/lyft/lib/python3.6/site-packages/PIL/ImageEnhance.py\", line 37, in enhance\n",
      "    return Image.blend(self.degenerate, self.image, factor)\n",
      "  File \"/home/paperspace/anaconda3/envs/lyft/lib/python3.6/site-packages/torchvision/datasets/folder.py\", line 52, in pil_loader\n",
      "    return img.convert('RGB')\n",
      "KeyboardInterrupt\n",
      "  File \"/home/paperspace/anaconda3/envs/lyft/lib/python3.6/site-packages/PIL/ImageFile.py\", line 231, in load\n",
      "    n, err_code = decoder.decode(b)\n",
      "KeyboardInterrupt\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  File \"/home/paperspace/anaconda3/envs/lyft/lib/python3.6/multiprocessing/connection.py\", line 407, in _recv_bytes\n",
      "    buf = self._recv(4)\n",
      "  File \"/home/paperspace/anaconda3/envs/lyft/lib/python3.6/multiprocessing/connection.py\", line 379, in _recv\n",
      "    chunk = read(handle, remaining)\n",
      "KeyboardInterrupt\n",
      "  File \"/home/paperspace/anaconda3/envs/lyft/lib/python3.6/site-packages/PIL/Image.py\", line 879, in convert\n",
      "    self.load()\n",
      "  File \"/home/paperspace/anaconda3/envs/lyft/lib/python3.6/site-packages/PIL/ImageFile.py\", line 231, in load\n",
      "    n, err_code = decoder.decode(b)\n",
      "ERROR:root:Internal Python error in the inspect module.\n",
      "Below is the traceback from this internal error.\n",
      "\n",
      "KeyboardInterrupt\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Traceback (most recent call last):\n",
      "  File \"/home/paperspace/anaconda3/envs/lyft/lib/python3.6/site-packages/IPython/core/interactiveshell.py\", line 2963, in run_code\n",
      "    exec(code_obj, self.user_global_ns, self.user_ns)\n",
      "  File \"<ipython-input-47-b4288c0007e0>\", line 3, in <module>\n",
      "    learn.fit(lr,1,wds=wd,cycle_len=5,use_clr=(5,8))\n",
      "  File \"/home/paperspace/anaconda3/envs/lyft/lib/python3.6/site-packages/fastai/learner.py\", line 287, in fit\n",
      "    return self.fit_gen(self.model, self.data, layer_opt, n_cycle, **kwargs)\n",
      "  File \"/home/paperspace/anaconda3/envs/lyft/lib/python3.6/site-packages/fastai/learner.py\", line 234, in fit_gen\n",
      "    swa_eval_freq=swa_eval_freq, **kwargs)\n",
      "  File \"/home/paperspace/anaconda3/envs/lyft/lib/python3.6/site-packages/fastai/model.py\", line 132, in fit\n",
      "    loss = model_stepper.step(V(x),V(y), epoch)\n",
      "  File \"/home/paperspace/anaconda3/envs/lyft/lib/python3.6/site-packages/fastai/model.py\", line 68, in step\n",
      "    return torch_item(raw_loss.data)\n",
      "  File \"/home/paperspace/anaconda3/envs/lyft/lib/python3.6/site-packages/fastai/model.py\", line 31, in torch_item\n",
      "    def torch_item(x): return x.item() if hasattr(x,'item') else x[0]\n",
      "KeyboardInterrupt\n",
      "\n",
      "During handling of the above exception, another exception occurred:\n",
      "\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/paperspace/anaconda3/envs/lyft/lib/python3.6/site-packages/IPython/core/interactiveshell.py\", line 1863, in showtraceback\n",
      "    stb = value._render_traceback_()\n",
      "AttributeError: 'KeyboardInterrupt' object has no attribute '_render_traceback_'\n",
      "\n",
      "During handling of the above exception, another exception occurred:\n",
      "\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/paperspace/anaconda3/envs/lyft/lib/python3.6/site-packages/IPython/core/ultratb.py\", line 1095, in get_records\n",
      "    return _fixed_getinnerframes(etb, number_of_lines_of_context, tb_offset)\n",
      "  File \"/home/paperspace/anaconda3/envs/lyft/lib/python3.6/site-packages/IPython/core/ultratb.py\", line 311, in wrapped\n",
      "    return f(*args, **kwargs)\n",
      "  File \"/home/paperspace/anaconda3/envs/lyft/lib/python3.6/site-packages/IPython/core/ultratb.py\", line 345, in _fixed_getinnerframes\n",
      "    records = fix_frame_records_filenames(inspect.getinnerframes(etb, context))\n",
      "  File \"/home/paperspace/anaconda3/envs/lyft/lib/python3.6/inspect.py\", line 1483, in getinnerframes\n",
      "    frameinfo = (tb.tb_frame,) + getframeinfo(tb, context)\n",
      "  File \"/home/paperspace/anaconda3/envs/lyft/lib/python3.6/inspect.py\", line 1441, in getframeinfo\n",
      "    filename = getsourcefile(frame) or getfile(frame)\n",
      "  File \"/home/paperspace/anaconda3/envs/lyft/lib/python3.6/inspect.py\", line 696, in getsourcefile\n",
      "    if getattr(getmodule(object, filename), '__loader__', None) is not None:\n",
      "  File \"/home/paperspace/anaconda3/envs/lyft/lib/python3.6/inspect.py\", line 734, in getmodule\n",
      "    f = module.__file__\n",
      "  File \"/home/paperspace/anaconda3/envs/lyft/lib/python3.6/site-packages/torch/utils/data/dataloader.py\", line 175, in handler\n",
      "    _error_if_any_worker_fails()\n",
      "RuntimeError: DataLoader worker (pid 29521) exited unexpectedly with exit code 1.\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m"
     ]
    }
   ],
   "source": [
    "lr=1e-1\n",
    "wd=1e-5\n",
    "learn.fit(lr,1,wds=wd,cycle_len=5,use_clr=(5,8))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Unet34"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val x:300, y:300\n",
      "Trn x:6980, y:6980\n",
      "All x:7280\n"
     ]
    }
   ],
   "source": [
    "ext = '-300'\n",
    "sz = 192\n",
    "bs = 64\n",
    "md = torch_loader(ext, PATH, bs, sz, workers, random_crop, pseudo_label)\n",
    "\n",
    "learn = get_learner(md, m_fn=Unet34)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8bd11a827e6c44479fc5024d3c6da6e7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=5), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   new_acc    rd_f       car_f_p_r  \n",
      "    0      0.986109   0.992922   0.873958   0.61347    0.266987   0.487154   0.245334  \n",
      "    1      0.985378   0.995675   0.72236    0.398291   0.253203   0.166691   0.318516  \n",
      "    2      0.984081   0.994571   0.83747    0.517897   0.106385   0.873937   0.087615  \n",
      "    3      0.986418   0.993635   0.719889   0.406286   0.192434   0.604334   0.165312  \n",
      "    4      0.985759   0.991143   0.837805   0.546809   0.332343   0.838307   0.289377  \n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[array([0.99114]),\n",
       " 0.8378051368395487,\n",
       " 0.546808551276984,\n",
       " 0.33234325177679275,\n",
       " 0.8383072374267033,\n",
       " 0.289376866217882]"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lr=1e-1\n",
    "wd=1e-5\n",
    "learn.fit(lr,1,wds=wd,cycle_len=5,use_clr=(5,8))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Unet34-Mod"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val x:300, y:300\n",
      "Trn x:6980, y:6980\n",
      "All x:7280\n"
     ]
    }
   ],
   "source": [
    "ext = '-300'\n",
    "sz = 192\n",
    "bs = 64\n",
    "md = torch_loader(ext, PATH, bs, sz, workers, random_crop, pseudo_label)\n",
    "\n",
    "learn = get_learner(md, m_fn=Unet34Mod)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "73a3defbddd649b3961f1e944c42c228",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=5), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   new_acc    rd_f       car_f_p_r  \n",
      "    0      0.985237   0.998316   0.779734   0.343043   0.173008   0.059561   0.411922  \n",
      "    1      0.985379   0.995311   0.684562   0.369525   0.054235   0.682985   0.044155  \n",
      "    2      0.984523   0.999425   0.432998   0.365324   0.082313   0.018156   0.998605  \n",
      "    3      0.98416    0.995617   0.73428    0.364393   0.390176   0.172812   0.659395  \n",
      "    4      0.986033   0.991294   0.802186   0.490382   0.371741   0.903789   0.32453   \n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[array([0.99129]),\n",
       " 0.802185529867808,\n",
       " 0.4903819663536071,\n",
       " 0.3717406836195783,\n",
       " 0.9037894437412505,\n",
       " 0.32452962785960576]"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lr=1e-1\n",
    "wd=1e-5\n",
    "learn.fit(lr,1,wds=wd,cycle_len=5,use_clr=(5,8))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Unet256"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_unet256_learner(md, m_fn):\n",
    "    m = m_fn(f=2)\n",
    "    learn = Learner.from_model_data(m, md)\n",
    "    learn.opt_fn=optim.Adam\n",
    "#     class_weights = torch.FloatTensor([1,3,2]).cuda()\n",
    "#     learn.crit=nn.CrossEntropyLoss(weight=class_weights)\n",
    "    learn.crit=SoftDiceLoss(weight=[1,1/100])\n",
    "    learn.metrics=[new_acc, rd_f, car_f_p_r]\n",
    "    return learn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val x:300, y:300\n",
      "Trn x:6980, y:6980\n",
      "All x:7280\n"
     ]
    }
   ],
   "source": [
    "ext = '-300'\n",
    "sz = 192\n",
    "bs = 32\n",
    "md = torch_loader(ext, PATH, bs, sz, workers, random_crop, pseudo_label)\n",
    "\n",
    "learn = get_unet256_learner(md, m_fn=Unet256)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "md = None\n",
    "learn = None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.cuda.empty_cache()\n",
    "import gc\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e44493d1d9494af4a92bd93b0de24032",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=5), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   new_acc    rd_f       car_f_p_r  \n",
      "    0      0.991393   0.991373   0.644256   0.353024   0.0        0.0        0.0       \n",
      " 34%|███▍      | 74/219 [00:25<00:49,  2.96it/s, loss=0.993]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Process Process-16:\n",
      "Process Process-21:\n",
      "Process Process-17:\n",
      "Process Process-18:\n",
      "Process Process-19:\n",
      "Process Process-15:\n",
      "Process Process-20:\n",
      "Traceback (most recent call last):\n",
      "Traceback (most recent call last):\n",
      "Traceback (most recent call last):\n",
      "Traceback (most recent call last):\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/paperspace/anaconda3/envs/lyft/lib/python3.6/multiprocessing/process.py\", line 258, in _bootstrap\n",
      "    self.run()\n",
      "  File \"/home/paperspace/anaconda3/envs/lyft/lib/python3.6/multiprocessing/process.py\", line 258, in _bootstrap\n",
      "    self.run()\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/paperspace/anaconda3/envs/lyft/lib/python3.6/multiprocessing/process.py\", line 258, in _bootstrap\n",
      "    self.run()\n",
      "  File \"/home/paperspace/anaconda3/envs/lyft/lib/python3.6/multiprocessing/process.py\", line 93, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "  File \"/home/paperspace/anaconda3/envs/lyft/lib/python3.6/multiprocessing/process.py\", line 93, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "  File \"/home/paperspace/anaconda3/envs/lyft/lib/python3.6/multiprocessing/process.py\", line 258, in _bootstrap\n",
      "    self.run()\n",
      "  File \"/home/paperspace/anaconda3/envs/lyft/lib/python3.6/multiprocessing/process.py\", line 93, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "  File \"/home/paperspace/anaconda3/envs/lyft/lib/python3.6/site-packages/torch/utils/data/dataloader.py\", line 50, in _worker_loop\n",
      "    r = index_queue.get()\n",
      "  File \"/home/paperspace/anaconda3/envs/lyft/lib/python3.6/multiprocessing/process.py\", line 258, in _bootstrap\n",
      "    self.run()\n",
      "  File \"/home/paperspace/anaconda3/envs/lyft/lib/python3.6/site-packages/torch/utils/data/dataloader.py\", line 50, in _worker_loop\n",
      "    r = index_queue.get()\n",
      "  File \"/home/paperspace/anaconda3/envs/lyft/lib/python3.6/multiprocessing/process.py\", line 93, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "  File \"/home/paperspace/anaconda3/envs/lyft/lib/python3.6/multiprocessing/queues.py\", line 334, in get\n",
      "    with self._rlock:\n",
      "  File \"/home/paperspace/anaconda3/envs/lyft/lib/python3.6/site-packages/torch/utils/data/dataloader.py\", line 50, in _worker_loop\n",
      "    r = index_queue.get()\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/paperspace/anaconda3/envs/lyft/lib/python3.6/multiprocessing/process.py\", line 93, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "  File \"/home/paperspace/anaconda3/envs/lyft/lib/python3.6/site-packages/torch/utils/data/dataloader.py\", line 50, in _worker_loop\n",
      "    r = index_queue.get()\n",
      "  File \"/home/paperspace/anaconda3/envs/lyft/lib/python3.6/multiprocessing/queues.py\", line 334, in get\n",
      "    with self._rlock:\n",
      "  File \"/home/paperspace/anaconda3/envs/lyft/lib/python3.6/multiprocessing/synchronize.py\", line 96, in __enter__\n",
      "    return self._semlock.__enter__()\n",
      "  File \"/home/paperspace/anaconda3/envs/lyft/lib/python3.6/multiprocessing/process.py\", line 258, in _bootstrap\n",
      "    self.run()\n",
      "  File \"/home/paperspace/anaconda3/envs/lyft/lib/python3.6/site-packages/torch/utils/data/dataloader.py\", line 50, in _worker_loop\n",
      "    r = index_queue.get()\n",
      "  File \"/home/paperspace/anaconda3/envs/lyft/lib/python3.6/multiprocessing/process.py\", line 258, in _bootstrap\n",
      "    self.run()\n",
      "  File \"/home/paperspace/anaconda3/envs/lyft/lib/python3.6/multiprocessing/queues.py\", line 335, in get\n",
      "    res = self._reader.recv_bytes()\n",
      "  File \"/home/paperspace/anaconda3/envs/lyft/lib/python3.6/multiprocessing/synchronize.py\", line 96, in __enter__\n",
      "    return self._semlock.__enter__()\n",
      "KeyboardInterrupt\n",
      "  File \"/home/paperspace/anaconda3/envs/lyft/lib/python3.6/multiprocessing/queues.py\", line 334, in get\n",
      "    with self._rlock:\n",
      "  File \"/home/paperspace/anaconda3/envs/lyft/lib/python3.6/multiprocessing/connection.py\", line 216, in recv_bytes\n",
      "    buf = self._recv_bytes(maxlength)\n",
      "  File \"/home/paperspace/anaconda3/envs/lyft/lib/python3.6/multiprocessing/process.py\", line 93, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "  File \"/home/paperspace/anaconda3/envs/lyft/lib/python3.6/multiprocessing/queues.py\", line 334, in get\n",
      "    with self._rlock:\n",
      "  File \"/home/paperspace/anaconda3/envs/lyft/lib/python3.6/multiprocessing/synchronize.py\", line 96, in __enter__\n",
      "    return self._semlock.__enter__()\n",
      "KeyboardInterrupt\n",
      "  File \"/home/paperspace/anaconda3/envs/lyft/lib/python3.6/multiprocessing/process.py\", line 93, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "  File \"/home/paperspace/anaconda3/envs/lyft/lib/python3.6/multiprocessing/connection.py\", line 407, in _recv_bytes\n",
      "    buf = self._recv(4)\n",
      "KeyboardInterrupt\n",
      "  File \"/home/paperspace/anaconda3/envs/lyft/lib/python3.6/multiprocessing/connection.py\", line 379, in _recv\n",
      "    chunk = read(handle, remaining)\n",
      "KeyboardInterrupt\n",
      "  File \"/home/paperspace/anaconda3/envs/lyft/lib/python3.6/site-packages/torch/utils/data/dataloader.py\", line 55, in _worker_loop\n",
      "    samples = collate_fn([dataset[i] for i in batch_indices])\n",
      "  File \"/home/paperspace/anaconda3/envs/lyft/lib/python3.6/site-packages/torch/utils/data/dataloader.py\", line 50, in _worker_loop\n",
      "    r = index_queue.get()\n",
      "  File \"/home/paperspace/anaconda3/envs/lyft/lib/python3.6/multiprocessing/synchronize.py\", line 96, in __enter__\n",
      "    return self._semlock.__enter__()\n",
      "  File \"/home/paperspace/anaconda3/envs/lyft/lib/python3.6/multiprocessing/queues.py\", line 334, in get\n",
      "    with self._rlock:\n",
      "  File \"/home/paperspace/anaconda3/envs/lyft/lib/python3.6/multiprocessing/synchronize.py\", line 96, in __enter__\n",
      "    return self._semlock.__enter__()\n",
      "KeyboardInterrupt\n",
      "KeyboardInterrupt\n",
      "  File \"/home/paperspace/anaconda3/envs/lyft/lib/python3.6/site-packages/torch/utils/data/dataloader.py\", line 55, in <listcomp>\n",
      "    samples = collate_fn([dataset[i] for i in batch_indices])\n",
      "  File \"<ipython-input-10-e4a515af82db>\", line 26, in __getitem__\n",
      "    return self.get(self.tfms, x, y)\n",
      "  File \"<ipython-input-10-e4a515af82db>\", line 21, in get\n",
      "    x, y = fn(x, y)\n",
      "  File \"<ipython-input-15-6d70a2ea31a4>\", line 2, in <lambda>\n",
      "    return lambda x,y: (tfm(x), y)\n",
      "  File \"/home/paperspace/anaconda3/envs/lyft/lib/python3.6/site-packages/torchvision/transforms/transforms.py\", line 579, in __call__\n",
      "    return transform(img)\n",
      "  File \"/home/paperspace/anaconda3/envs/lyft/lib/python3.6/site-packages/torchvision/transforms/transforms.py\", line 42, in __call__\n",
      "    img = t(img)\n",
      "  File \"/home/paperspace/anaconda3/envs/lyft/lib/python3.6/site-packages/torchvision/transforms/transforms.py\", line 232, in __call__\n",
      "    return self.lambd(img)\n",
      "  File \"/home/paperspace/anaconda3/envs/lyft/lib/python3.6/site-packages/torchvision/transforms/transforms.py\", line 550, in <lambda>\n",
      "    transforms.append(Lambda(lambda img: F.adjust_brightness(img, brightness_factor)))\n",
      "  File \"/home/paperspace/anaconda3/envs/lyft/lib/python3.6/site-packages/torchvision/transforms/functional.py\", line 406, in adjust_brightness\n",
      "    img = enhancer.enhance(brightness_factor)\n",
      "  File \"/home/paperspace/anaconda3/envs/lyft/lib/python3.6/site-packages/PIL/ImageEnhance.py\", line 37, in enhance\n",
      "    return Image.blend(self.degenerate, self.image, factor)\n",
      "  File \"/home/paperspace/anaconda3/envs/lyft/lib/python3.6/site-packages/PIL/Image.py\", line 2631, in blend\n",
      "    return im1._new(core.blend(im1.im, im2.im, alpha))\n",
      "KeyboardInterrupt\n",
      "Exception in thread Thread-9:\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/paperspace/anaconda3/envs/lyft/lib/python3.6/threading.py\", line 916, in _bootstrap_inner\n",
      "    self.run()\n",
      "  File \"/home/paperspace/anaconda3/envs/lyft/lib/python3.6/threading.py\", line 864, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "  File \"/home/paperspace/anaconda3/envs/lyft/lib/python3.6/site-packages/torch/utils/data/dataloader.py\", line 68, in _worker_manager_loop\n",
      "    r = in_queue.get()\n",
      "  File \"/home/paperspace/anaconda3/envs/lyft/lib/python3.6/multiprocessing/queues.py\", line 337, in get\n",
      "    return _ForkingPickler.loads(res)\n",
      "  File \"/home/paperspace/anaconda3/envs/lyft/lib/python3.6/site-packages/torch/multiprocessing/reductions.py\", line 70, in rebuild_storage_fd\n",
      "    fd = df.detach()\n",
      "  File \"/home/paperspace/anaconda3/envs/lyft/lib/python3.6/multiprocessing/resource_sharer.py\", line 57, in detach\n",
      "    with _resource_sharer.get_connection(self._id) as conn:\n",
      "  File \"/home/paperspace/anaconda3/envs/lyft/lib/python3.6/multiprocessing/resource_sharer.py\", line 87, in get_connection\n",
      "    c = Client(address, authkey=process.current_process().authkey)\n",
      "  File \"/home/paperspace/anaconda3/envs/lyft/lib/python3.6/multiprocessing/connection.py\", line 487, in Client\n",
      "    c = SocketClient(address)\n",
      "  File \"/home/paperspace/anaconda3/envs/lyft/lib/python3.6/multiprocessing/connection.py\", line 614, in SocketClient\n",
      "    s.connect(address)\n",
      "FileNotFoundError: [Errno 2] No such file or directory\n",
      "\n",
      "Exception ignored in: <generator object tqdm_notebook.__iter__ at 0x7fa272eb8f68>\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/paperspace/anaconda3/envs/lyft/lib/python3.6/site-packages/tqdm/_tqdm_notebook.py\", line 206, in __iter__\n",
      "    self.sp(bar_style='danger')\n",
      "  File \"/home/paperspace/anaconda3/envs/lyft/lib/python3.6/site-packages/tqdm/_tqdm_notebook.py\", line 153, in print_status\n",
      "    pbar.bar_style = bar_style\n",
      "  File \"/home/paperspace/anaconda3/envs/lyft/lib/python3.6/site-packages/traitlets/traitlets.py\", line 585, in __set__\n",
      "    self.set(obj, value)\n",
      "  File \"/home/paperspace/anaconda3/envs/lyft/lib/python3.6/site-packages/traitlets/traitlets.py\", line 574, in set\n",
      "    obj._notify_trait(self.name, old_value, new_value)\n",
      "  File \"/home/paperspace/anaconda3/envs/lyft/lib/python3.6/site-packages/traitlets/traitlets.py\", line 1139, in _notify_trait\n",
      "    type='change',\n",
      "  File \"/home/paperspace/anaconda3/envs/lyft/lib/python3.6/site-packages/ipywidgets/widgets/widget.py\", line 599, in notify_change\n",
      "    self.send_state(key=name)\n",
      "  File \"/home/paperspace/anaconda3/envs/lyft/lib/python3.6/site-packages/ipywidgets/widgets/widget.py\", line 484, in send_state\n",
      "    self._send(msg, buffers=buffers)\n",
      "  File \"/home/paperspace/anaconda3/envs/lyft/lib/python3.6/site-packages/ipywidgets/widgets/widget.py\", line 729, in _send\n",
      "    self.comm.send(data=msg, buffers=buffers)\n",
      "  File \"/home/paperspace/anaconda3/envs/lyft/lib/python3.6/site-packages/ipykernel/comm/comm.py\", line 121, in send\n",
      "    data=data, metadata=metadata, buffers=buffers,\n",
      "  File \"/home/paperspace/anaconda3/envs/lyft/lib/python3.6/site-packages/ipykernel/comm/comm.py\", line 71, in _publish_msg\n",
      "    buffers=buffers,\n",
      "  File \"/home/paperspace/anaconda3/envs/lyft/lib/python3.6/site-packages/jupyter_client/session.py\", line 748, in send\n",
      "    stream.send_multipart(to_send, copy=copy)\n",
      "  File \"/home/paperspace/anaconda3/envs/lyft/lib/python3.6/site-packages/ipykernel/iostream.py\", line 262, in send_multipart\n",
      "    return self.io_thread.send_multipart(*args, **kwargs)\n",
      "  File \"/home/paperspace/anaconda3/envs/lyft/lib/python3.6/site-packages/ipykernel/iostream.py\", line 212, in send_multipart\n",
      "    self.schedule(lambda : self._really_send(*args, **kwargs))\n",
      "  File \"/home/paperspace/anaconda3/envs/lyft/lib/python3.6/site-packages/ipykernel/iostream.py\", line 203, in schedule\n",
      "    self._event_pipe.send(b'')\n",
      "  File \"/home/paperspace/anaconda3/envs/lyft/lib/python3.6/site-packages/zmq/sugar/socket.py\", line 391, in send\n",
      "    return super(Socket, self).send(data, flags=flags, copy=copy, track=track)\n",
      "  File \"zmq/backend/cython/socket.pyx\", line 727, in zmq.backend.cython.socket.Socket.send\n",
      "  File \"zmq/backend/cython/socket.pyx\", line 774, in zmq.backend.cython.socket.Socket.send\n",
      "  File \"zmq/backend/cython/socket.pyx\", line 249, in zmq.backend.cython.socket._send_copy\n",
      "  File \"zmq/backend/cython/socket.pyx\", line 244, in zmq.backend.cython.socket._send_copy\n",
      "  File \"zmq/backend/cython/checkrc.pxd\", line 12, in zmq.backend.cython.checkrc._check_rc\n",
      "  File \"/home/paperspace/anaconda3/envs/lyft/lib/python3.6/site-packages/torch/utils/data/dataloader.py\", line 175, in handler\n",
      "    _error_if_any_worker_fails()\n",
      "RuntimeError: DataLoader worker (pid 4136) exited unexpectedly with exit code 1.\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-40-b4288c0007e0>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0mlr\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1e-1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0mwd\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1e-5\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m \u001b[0mlearn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlr\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mwds\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mwd\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mcycle_len\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0muse_clr\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m8\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m~/anaconda3/envs/lyft/lib/python3.6/site-packages/fastai/learner.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, lrs, n_cycle, wds, **kwargs)\u001b[0m\n\u001b[1;32m    285\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msched\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    286\u001b[0m         \u001b[0mlayer_opt\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_layer_opt\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlrs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 287\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit_gen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlayer_opt\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mn_cycle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    288\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    289\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mwarm_up\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlr\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mwds\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/lyft/lib/python3.6/site-packages/fastai/learner.py\u001b[0m in \u001b[0;36mfit_gen\u001b[0;34m(self, model, data, layer_opt, n_cycle, cycle_len, cycle_mult, cycle_save_name, best_save_name, use_clr, use_clr_beta, metrics, callbacks, use_wd_sched, norm_wds, wds_sched_mult, use_swa, swa_start, swa_eval_freq, **kwargs)\u001b[0m\n\u001b[1;32m    232\u001b[0m             \u001b[0mmetrics\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mmetrics\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcallbacks\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcallbacks\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mreg_fn\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreg_fn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mclip\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mclip\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfp16\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfp16\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    233\u001b[0m             \u001b[0mswa_model\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mswa_model\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0muse_swa\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mswa_start\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mswa_start\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 234\u001b[0;31m             swa_eval_freq=swa_eval_freq, **kwargs)\n\u001b[0m\u001b[1;32m    235\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    236\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mget_layer_groups\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodels\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_layer_groups\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/lyft/lib/python3.6/site-packages/fastai/model.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(model, data, n_epochs, opt, crit, metrics, callbacks, stepper, swa_model, swa_start, swa_eval_freq, **kwargs)\u001b[0m\n\u001b[1;32m    130\u001b[0m             \u001b[0mbatch_num\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    131\u001b[0m             \u001b[0;32mfor\u001b[0m \u001b[0mcb\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mcallbacks\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mcb\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mon_batch_begin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 132\u001b[0;31m             \u001b[0mloss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel_stepper\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mV\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mV\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0my\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mepoch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    133\u001b[0m             \u001b[0mavg_loss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mavg_loss\u001b[0m \u001b[0;34m*\u001b[0m \u001b[0mavg_mom\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mloss\u001b[0m \u001b[0;34m*\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0mavg_mom\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    134\u001b[0m             \u001b[0mdebias_loss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mavg_loss\u001b[0m \u001b[0;34m/\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0mavg_mom\u001b[0m\u001b[0;34m**\u001b[0m\u001b[0mbatch_num\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/lyft/lib/python3.6/site-packages/fastai/model.py\u001b[0m in \u001b[0;36mstep\u001b[0;34m(self, xs, y, epoch)\u001b[0m\n\u001b[1;32m     66\u001b[0m             \u001b[0mcopy_fp32_to_model\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mm\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfp32_params\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     67\u001b[0m             \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcuda\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msynchronize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 68\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mtorch_item\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mraw_loss\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     69\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     70\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mevaluate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mxs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/lyft/lib/python3.6/site-packages/fastai/model.py\u001b[0m in \u001b[0;36mtorch_item\u001b[0;34m(x)\u001b[0m\n\u001b[1;32m     29\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mres\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mres\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     30\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 31\u001b[0;31m \u001b[0;32mdef\u001b[0m \u001b[0mtorch_item\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mitem\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mhasattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m'item'\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     32\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     33\u001b[0m \u001b[0;32mclass\u001b[0m \u001b[0mStepper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "lr=1e-1\n",
    "wd=1e-5\n",
    "learn.fit(lr,1,wds=wd,cycle_len=5,use_clr=(5,8))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Unet11"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val x:300, y:300\n",
      "Trn x:6980, y:6980\n",
      "All x:7280\n"
     ]
    }
   ],
   "source": [
    "ext = '-300'\n",
    "sz = 192\n",
    "bs = 64\n",
    "md = torch_loader(ext, PATH, bs, sz, workers, random_crop, pseudo_label)\n",
    "\n",
    "learn = get_learner(md, m_fn=Unet11)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "db2adbcc0a5140819bde5383930ded96",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=5), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/110 [00:00<?, ?it/s]\n"
     ]
    },
    {
     "ename": "RuntimeError",
     "evalue": "cuda runtime error (2) : out of memory at /opt/conda/conda-bld/pytorch_1518244421288/work/torch/lib/THC/generic/THCStorage.cu:58",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mRuntimeError\u001b[0m                              Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-39-d8a7a27c2255>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0mwd\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1e-5\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 4\u001b[0;31m \u001b[0mlearn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlr\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mwds\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mwd\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mcycle_len\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0muse_clr\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m8\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m~/anaconda3/envs/lyft/lib/python3.6/site-packages/fastai/learner.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, lrs, n_cycle, wds, **kwargs)\u001b[0m\n\u001b[1;32m    285\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msched\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    286\u001b[0m         \u001b[0mlayer_opt\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_layer_opt\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlrs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 287\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit_gen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlayer_opt\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mn_cycle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    288\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    289\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mwarm_up\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlr\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mwds\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/lyft/lib/python3.6/site-packages/fastai/learner.py\u001b[0m in \u001b[0;36mfit_gen\u001b[0;34m(self, model, data, layer_opt, n_cycle, cycle_len, cycle_mult, cycle_save_name, best_save_name, use_clr, use_clr_beta, metrics, callbacks, use_wd_sched, norm_wds, wds_sched_mult, use_swa, swa_start, swa_eval_freq, **kwargs)\u001b[0m\n\u001b[1;32m    232\u001b[0m             \u001b[0mmetrics\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mmetrics\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcallbacks\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcallbacks\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mreg_fn\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreg_fn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mclip\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mclip\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfp16\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfp16\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    233\u001b[0m             \u001b[0mswa_model\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mswa_model\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0muse_swa\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mswa_start\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mswa_start\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 234\u001b[0;31m             swa_eval_freq=swa_eval_freq, **kwargs)\n\u001b[0m\u001b[1;32m    235\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    236\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mget_layer_groups\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodels\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_layer_groups\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/lyft/lib/python3.6/site-packages/fastai/model.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(model, data, n_epochs, opt, crit, metrics, callbacks, stepper, swa_model, swa_start, swa_eval_freq, **kwargs)\u001b[0m\n\u001b[1;32m    130\u001b[0m             \u001b[0mbatch_num\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    131\u001b[0m             \u001b[0;32mfor\u001b[0m \u001b[0mcb\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mcallbacks\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mcb\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mon_batch_begin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 132\u001b[0;31m             \u001b[0mloss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel_stepper\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mV\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mV\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0my\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mepoch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    133\u001b[0m             \u001b[0mavg_loss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mavg_loss\u001b[0m \u001b[0;34m*\u001b[0m \u001b[0mavg_mom\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mloss\u001b[0m \u001b[0;34m*\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0mavg_mom\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    134\u001b[0m             \u001b[0mdebias_loss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mavg_loss\u001b[0m \u001b[0;34m/\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0mavg_mom\u001b[0m\u001b[0;34m**\u001b[0m\u001b[0mbatch_num\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/lyft/lib/python3.6/site-packages/fastai/model.py\u001b[0m in \u001b[0;36mstep\u001b[0;34m(self, xs, y, epoch)\u001b[0m\n\u001b[1;32m     48\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mstep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mxs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mepoch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     49\u001b[0m         \u001b[0mxtra\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 50\u001b[0;31m         \u001b[0moutput\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mm\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0mxs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     51\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moutput\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mtuple\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0moutput\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0mxtra\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0moutput\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     52\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfp16\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mm\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mzero_grad\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/lyft/lib/python3.6/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m    355\u001b[0m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    356\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 357\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    358\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    359\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-30-7f72ab5eea5e>\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, x)\u001b[0m\n\u001b[1;32m     19\u001b[0m         \u001b[0mx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mup2\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msfs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfeatures\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     20\u001b[0m         \u001b[0mx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mup3\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msfs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfeatures\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 21\u001b[0;31m         \u001b[0mx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mup4\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msfs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfeatures\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     22\u001b[0m         \u001b[0mx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mup5\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     23\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/lyft/lib/python3.6/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m    355\u001b[0m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    356\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 357\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    358\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    359\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-27-59c67f6d6916>\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, up_p, x_p)\u001b[0m\n\u001b[1;32m     10\u001b[0m         \u001b[0mup_p\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtr_conv\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mup_p\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     11\u001b[0m         \u001b[0mx_p\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mx_conv\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx_p\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 12\u001b[0;31m         \u001b[0mcat_p\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mup_p\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mx_p\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdim\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     13\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mF\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrelu\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcat_p\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minplace\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mRuntimeError\u001b[0m: cuda runtime error (2) : out of memory at /opt/conda/conda-bld/pytorch_1518244421288/work/torch/lib/THC/generic/THCStorage.cu:58"
     ]
    }
   ],
   "source": [
    "lr=1e-1\n",
    "wd=1e-5\n",
    "\n",
    "learn.fit(lr,1,wds=wd,cycle_len=5,use_clr=(5,8))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Unet11-Mini"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "test = vgg11(pre=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Sequential(\n",
       "  (0): Conv2d(3, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "  (1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True)\n",
       "  (2): ReLU(inplace)\n",
       "  (3): MaxPool2d(kernel_size=(2, 2), stride=(2, 2), dilation=(1, 1), ceil_mode=False)\n",
       "  (4): Conv2d(64, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "  (5): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True)\n",
       "  (6): ReLU(inplace)\n",
       "  (7): MaxPool2d(kernel_size=(2, 2), stride=(2, 2), dilation=(1, 1), ceil_mode=False)\n",
       "  (8): Conv2d(128, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "  (9): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True)\n",
       "  (10): ReLU(inplace)\n",
       "  (11): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "  (12): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True)\n",
       "  (13): ReLU(inplace)\n",
       "  (14): MaxPool2d(kernel_size=(2, 2), stride=(2, 2), dilation=(1, 1), ceil_mode=False)\n",
       "  (15): Conv2d(256, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "  (16): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True)\n",
       "  (17): ReLU(inplace)\n",
       "  (18): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "  (19): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True)\n",
       "  (20): ReLU(inplace)\n",
       "  (21): MaxPool2d(kernel_size=(2, 2), stride=(2, 2), dilation=(1, 1), ceil_mode=False)\n",
       "  (22): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "  (23): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True)\n",
       "  (24): ReLU(inplace)\n",
       "  (25): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "  (26): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True)\n",
       "  (27): ReLU(inplace)\n",
       "  (28): MaxPool2d(kernel_size=(2, 2), stride=(2, 2), dilation=(1, 1), ceil_mode=False)\n",
       ")"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Unet11Mini(nn.Module):\n",
    "    def __init__(self, f=vgg11):\n",
    "        super().__init__()\n",
    "        m_base, lr_cut = get_base(f)\n",
    "        self.rn = m_base\n",
    "        self.lr_cut = lr_cut\n",
    "        self.sfs = [SaveFeatures(self.rn[0][i]) for i in [2,6,13,20,27]]\n",
    "        self.up0 = UnetBlock(512,512,256)\n",
    "        self.up1 = UnetBlock(256,512,256)\n",
    "        self.up2 = UnetBlock(256,256,256)\n",
    "        self.up3 = UnetBlock(256,128,128)\n",
    "        self.up4 = UnetBlock(128,64,64)\n",
    "        self.up5  = nn.Conv2d(64,2,1)\n",
    "        \n",
    "    def forward(self,x):\n",
    "        x = F.relu(self.rn(x))\n",
    "        x = self.up0(x, self.sfs[4].features)\n",
    "        x = self.up1(x, self.sfs[3].features)\n",
    "        x = self.up2(x, self.sfs[2].features)\n",
    "        x = self.up3(x, self.sfs[1].features)\n",
    "        x = self.up4(x, self.sfs[0].features)\n",
    "        x = self.up5(x)\n",
    "        return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val x:300, y:300\n",
      "Trn x:6980, y:6980\n",
      "All x:7280\n"
     ]
    }
   ],
   "source": [
    "ext = '-300'\n",
    "sz = 192\n",
    "bs = 64\n",
    "md = torch_loader(ext, PATH, bs, sz, workers, random_crop, pseudo_label)\n",
    "\n",
    "learn = get_learner(md, m_fn=Unet11Mini)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f90159b17a28411c8962b0a156ccd481",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=5), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   new_acc    rd_f       car_f_p_r  \n",
      "    0      0.98667    0.994857   0.762435   0.420613   0.257708   0.238241   0.283269  \n",
      "    1      0.98617    0.994226   0.899885   0.698543   0.211023   0.398753   0.191686  \n",
      "    2      0.984173   0.990501   0.869038   0.582974   0.582328   0.566087   0.597535  \n",
      "    3      0.983803   0.995799   0.850191   0.453865   0.280078   0.229158   0.321097  \n",
      "    4      0.98521    0.991314   0.817316   0.514115   0.357456   0.91889    0.311288  \n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[array([0.99131]),\n",
       " 0.8173155387242635,\n",
       " 0.5141146643070651,\n",
       " 0.3574559996366525,\n",
       " 0.9188901596130123,\n",
       " 0.31128836890429856]"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lr=1e-1\n",
    "wd=1e-5\n",
    "learn.fit(lr,1,wds=wd,cycle_len=5,use_clr=(5,8))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Unet34-ModWide"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Unet34ModWide(nn.Module):\n",
    "    def __init__(self, f=resnet34):\n",
    "        super().__init__()\n",
    "        m_base, lr_cut = get_base(f)\n",
    "        self.rn = m_base\n",
    "        self.lr_cut = lr_cut\n",
    "        self.sfs = [SaveFeatures(self.rn[i]) for i in [2,4,5,6]]\n",
    "        self.up1 = UnetBlock(512,256,256)\n",
    "        self.up2 = UnetBlock(256,128,256)\n",
    "        self.up3 = UnetBlock(256,64,256)\n",
    "        self.up4 = UnetBlock(256,64,256)\n",
    "        self.up5 = UnetBlock(256,32,32)\n",
    "        self.up6 = nn.ConvTranspose2d(32, 2, 1)\n",
    "        self.x_skip = nn.Sequential(\n",
    "            nn.Conv2d(3,32,1),\n",
    "            nn.BatchNorm2d(32),\n",
    "            nn.ReLU(inplace=True)\n",
    "        )\n",
    "        \n",
    "    def forward(self,x):\n",
    "        x_skip = self.x_skip(x)\n",
    "        x = F.relu(self.rn(x), inplace=True)\n",
    "        x = self.up1(x, self.sfs[3].features)\n",
    "        x = self.up2(x, self.sfs[2].features)\n",
    "        x = self.up3(x, self.sfs[1].features)\n",
    "        x = self.up4(x, self.sfs[0].features)\n",
    "        x = self.up5(x, x_skip)\n",
    "        x = self.up6(x)\n",
    "        return torch.squeeze(x)\n",
    "    \n",
    "    def close(self):\n",
    "        for sf in self.sfs: sf.remove()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val x:300, y:300\n",
      "Trn x:6980, y:6980\n",
      "All x:7280\n"
     ]
    }
   ],
   "source": [
    "ext = '-300'\n",
    "sz = 192\n",
    "bs = 64\n",
    "md = torch_loader(ext, PATH, bs, sz, workers, random_crop, pseudo_label)\n",
    "\n",
    "learn = get_learner(md, m_fn=Unet34ModWide)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "360c3db975e64d368987a81d5e52122e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=5), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   new_acc    rd_f       car_f_p_r  \n",
      "    0      0.984673   0.993882   0.813874   0.514824   0.084924   0.867034   0.069624  \n",
      "    1      0.985475   0.999525   0.355745   0.0        0.071063   0.015452   0.999991  \n",
      "    2      0.984575   0.993852   0.791868   0.4795     0.1638     0.578104   0.1397    \n",
      "    3      0.983621   0.992305   0.753157   0.439186   0.377926   0.478918   0.367814  \n",
      "    4      0.983205   0.995694   0.84993    0.489994   0.118015   0.901989   0.097122  \n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[array([0.99569]),\n",
       " 0.8499298087755839,\n",
       " 0.48999369782526647,\n",
       " 0.11801475370146448,\n",
       " 0.9019889214182198,\n",
       " 0.09712199792916369]"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lr=1e-1\n",
    "wd=1e-5\n",
    "learn.fit(lr,1,wds=wd,cycle_len=5,use_clr=(5,8))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  },
  "toc": {
   "colors": {
    "hover_highlight": "#DAA520",
    "navigate_num": "#000000",
    "navigate_text": "#333333",
    "running_highlight": "#FF0000",
    "selected_highlight": "#FFD700",
    "sidebar_border": "#EEEEEE",
    "wrapper_background": "#FFFFFF"
   },
   "moveMenuLeft": true,
   "nav_menu": {
    "height": "86px",
    "width": "252px"
   },
   "navigate_menu": true,
   "number_sections": true,
   "sideBar": true,
   "threshold": 4,
   "toc_cell": false,
   "toc_section_display": "block",
   "toc_window_display": false,
   "widenNotebook": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}