docs/pytorch-study.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Tensor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Numpy:  [[1. 1. 1. 1.]\n",
      " [1. 1. 1. 1.]\n",
      " [1. 1. 1. 1.]]\n",
      "Tensor:  tensor([[0, 0, 0, 0],\n",
      "        [0, 0, 0, 0],\n",
      "        [0, 0, 0, 0]])\n",
      "tensor([-1.9611, -0.2047,  0.6853])\n",
      "tensor([[7959390389040738153, 2318285298082652788, 8675445202132104482],\n",
      "        [7957695011165139568, 2318365875964093043, 7233184988217307170]])\n"
     ]
    }
   ],
   "source": [
    "# 创建tensor\n",
    "n = np.ones((3, 4))\n",
    "\n",
    "t = torch.ones(3, 4)\n",
    "t = torch.rand(5, 3)\n",
    "t = torch.zeros(3, 4, dtype=torch.long)\n",
    "\n",
    "a = torch.tensor([1, 2, 3])\n",
    "a = torch.randn_like(a, dtype=torch.float)\n",
    "\n",
    "y = t.new_empty(2, 3)       # 复用t的其他属性\n",
    "\n",
    "print(\"Numpy: \", n)\n",
    "print(\"Tensor: \", t)\n",
    "print(a)\n",
    "print(y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([3, 4])\n",
      "torch.Size([3, 4])\n"
     ]
    }
   ],
   "source": [
    "# 基础属性\n",
    "print(t.size())\n",
    "print(t.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[1.1634, 1.8894, 1.0713],\n",
      "        [0.5683, 1.0986, 0.8609]])\n",
      "tensor([[1.1634, 1.8894, 1.0713],\n",
      "        [0.5683, 1.0986, 0.8609]])\n"
     ]
    }
   ],
   "source": [
    "# 简单运算\n",
    "x = torch.rand(2, 3)\n",
    "y = torch.rand_like(x)\n",
    "\n",
    "print(x + y)\n",
    "print(torch.add(x, y))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[1.1634, 1.8894, 1.0713],\n",
      "        [0.5683, 1.0986, 0.8609]])\n"
     ]
    }
   ],
   "source": [
    "# in-place运算 会改变变量的值\n",
    "y.add_(x)\n",
    "print(y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([0.9541, 0.4638])\n"
     ]
    }
   ],
   "source": [
    "# 切片\n",
    "print(x[:, 1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([4, 4]) torch.Size([16]) torch.Size([2, 2, 4])\n"
     ]
    }
   ],
   "source": [
    "# resize\n",
    "x = torch.randn(4, 4)\n",
    "y = x.view(16)\n",
    "z = x.view(-1, 2, 4)\n",
    "\n",
    "print(x.size(), y.size(), z.size())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([0.1197])\n",
      "0.11974674463272095\n"
     ]
    }
   ],
   "source": [
    "# get value(if only have one element)\n",
    "x = torch.rand(1)\n",
    "print(x)\n",
    "print(x.item())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Tensor与Numpy转换"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[0.2107, 0.2437, 0.0706],\n",
      "        [0.6233, 0.4380, 0.4589]])\n",
      "[[0.21070534 0.24373996 0.07062978]\n",
      " [0.6232684  0.43795878 0.45893037]]\n"
     ]
    }
   ],
   "source": [
    "# torch -> numpy\n",
    "t = torch.rand(2, 3)\n",
    "n = t.numpy()\n",
    "\n",
    "print(t)\n",
    "print(n)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[0.16070166 0.54871463]\n",
      " [0.07759188 0.32236617]\n",
      " [0.14265208 0.4026539 ]]\n",
      "tensor([[0.1607, 0.5487],\n",
      "        [0.0776, 0.3224],\n",
      "        [0.1427, 0.4027]], dtype=torch.float64)\n"
     ]
    }
   ],
   "source": [
    "# numpy -> tensor\n",
    "n = np.random.rand(3, 2)\n",
    "t = torch.from_numpy(n)\n",
    "\n",
    "print(n)\n",
    "print(t)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[1.2107, 1.2437, 1.0706],\n",
      "        [1.6233, 1.4380, 1.4589]])\n",
      "[[1.2107053 1.24374   1.0706298]\n",
      " [1.6232684 1.4379587 1.4589304]]\n"
     ]
    }
   ],
   "source": [
    "# 修改一个两个都会跟着改的\n",
    "t.add_(1)\n",
    "\n",
    "print(t)\n",
    "print(n)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[0.30370266 0.36835002 0.99111293]\n",
      " [0.36966819 0.68330543 0.12247895]]\n",
      "tensor([[0.5291, 0.4583, 0.6517],\n",
      "        [0.5869, 0.5024, 0.0601]])\n"
     ]
    }
   ],
   "source": [
    "# 复制之后就不会共享内存了\n",
    "t = torch.rand(2, 3)\n",
    "n = np.random.rand(2, 3)\n",
    "\n",
    "# 二者的函数名是不一样的\n",
    "t_ = n.copy()\n",
    "n_ = t.clone()\n",
    "\n",
    "t.zero_()\n",
    "n = np.zeros((2, 3))\n",
    "\n",
    "print(t_)\n",
    "print(n_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 如果添加了求导，则需要将data转换为numpy\n",
    "n = torch.rand(2, 3, requires_grad=True)\n",
    "t = n.data.numpy()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### CUDA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[1.3248, 0.3975, 0.7192],\n",
      "        [0.7229, 0.7046, 0.7572]], device='cuda:0')\n",
      "tensor([[1.3248, 0.3975, 0.7192],\n",
      "        [0.7229, 0.7046, 0.7572]])\n"
     ]
    }
   ],
   "source": [
    "if torch.cuda.is_available():\n",
    "    device = torch.device(\"cuda\")\n",
    "    y = torch.rand(2, 3, device=device)\n",
    "    x = torch.rand(2, 3)\n",
    "    x = x.to(device)\n",
    "\n",
    "    z = x + y\n",
    "    print(z)\n",
    "    print(z.to(\"cpu\"))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 自动求导"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[1.7932]], grad_fn=<MmBackward>)\n"
     ]
    }
   ],
   "source": [
    "x = torch.randn(4, 1, requires_grad=True)\n",
    "b = torch.randn(4, 1, requires_grad=True)\n",
    "W = torch.randn(4, 4)\n",
    "\n",
    "# y = torch.mm(torch.mm(torch.t(x), W), b)\n",
    "y = x.t().mm(W).mm(b)\n",
    "print(y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "None\n",
      "tensor([[ 0.4634],\n",
      "        [-1.1022],\n",
      "        [-3.5316],\n",
      "        [ 1.3660]])\n"
     ]
    }
   ],
   "source": [
    "print(x.grad)\n",
    "\n",
    "y.backward()\n",
    "\n",
    "print(x.grad)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[-0.7865],\n",
      "        [-0.1405],\n",
      "        [-0.2294],\n",
      "        [ 0.3251]], requires_grad=True)\n",
      "tensor([[0.7966]], grad_fn=<MmBackward>)\n",
      "tensor([[-1.5730],\n",
      "        [-0.2811],\n",
      "        [-0.4588],\n",
      "        [ 0.6501]])\n"
     ]
    }
   ],
   "source": [
    "x = torch.randn(4, 1, requires_grad=True)\n",
    "y = torch.mm(torch.t(x), x)\n",
    "print(x)\n",
    "print(y)\n",
    "\n",
    "y.backward(retain_graph=True)   # 可以再次求导，否则只能backward一次\n",
    "print(x.grad)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[-0.7865],\n",
      "        [-0.1405],\n",
      "        [-0.2294],\n",
      "        [ 0.3251]], requires_grad=True)\n",
      "tensor([[0.7966]], grad_fn=<MmBackward>)\n",
      "tensor([[-3.1460],\n",
      "        [-0.5622],\n",
      "        [-0.9177],\n",
      "        [ 1.3003]])\n"
     ]
    }
   ],
   "source": [
    "print(x)\n",
    "print(y)\n",
    "\n",
    "y.backward(retain_graph=True)\n",
    "print(x.grad)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[-1.5730],\n",
      "        [-0.2811],\n",
      "        [-0.4588],\n",
      "        [ 0.6501]])\n"
     ]
    }
   ],
   "source": [
    "x.grad.zero_()  # 梯度清零\n",
    "y.backward()\n",
    "print(x.grad)   # 跟第一次的值相同"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 全连接"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([10, 200])\n",
      "weight : torch.Size([200, 100])\n",
      "bias : torch.Size([200])\n"
     ]
    }
   ],
   "source": [
    "input = torch.randn(10, 100)    # 第一个10是batch size\n",
    "linear_network = torch.nn.Linear(100, 200)\n",
    "output = linear_network(input)\n",
    "\n",
    "print(output.shape)\n",
    "\n",
    "for name, parameter in linear_network.named_parameters():\n",
    "    print(name, ':', parameter.size())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## CNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([1, 5, 28, 28])\n",
      "weight : torch.Size([5, 1, 3, 3])\n",
      "bias : torch.Size([5])\n"
     ]
    }
   ],
   "source": [
    "input = torch.randn(1, 1, 28, 28)   # batch, channel, height, width\n",
    "conv = torch.nn.Conv2d(in_channels=1, out_channels=5, kernel_size=3, padding=1, stride=1, bias=True)\n",
    "output = conv(input)\n",
    "\n",
    "print(output.shape)\n",
    "\n",
    "for name, parameters in conv.named_parameters():\n",
    "    print(name, ':', parameters.shape)\n",
    "\n",
    "# weight : torch.Size([5, 1, 3, 3])\n",
    "# out_channels, channel, kernel_size, kernel_size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([1, 5, 14, 14])\n"
     ]
    }
   ],
   "source": [
    "input = torch.randn(1, 1, 28, 28)\n",
    "conv = torch.nn.Conv2d(in_channels=1, out_channels=5, kernel_size=3, padding=1, stride=2)\n",
    "output = conv(input)\n",
    "print(output.shape)     # (input - kernel + 2*padding) / stride + 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "a3b0ff572e5cd24ec265c5b4da969dd40707626c6e113422fd84bd2b9440fcfc"
  },
  "kernelspec": {
   "display_name": "Python 3.9.7 64-bit ('deep_learning': conda)",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}