{ "cells": [ { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "" ] }, "execution_count": 1, "metadata": {}, "output_type": "execute_result" } ], "source": [ "import torch\n", "import torch.nn as nn\n", "import torch.nn.functional as F\n", "import torch.optim as optim\n", "from torch.utils.data import DataLoader, random_split\n", "from torchvision import datasets\n", "import torchvision.transforms as transforms\n", "import matplotlib.pyplot as plt\n", "%matplotlib inline\n", "\n", "\n", "torch.manual_seed(12046)" ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "(50000, 10000, 10000)" ] }, "execution_count": 2, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# 准备数据\n", "dataset = datasets.MNIST(root='./data', train=True, download=True, transform=transforms.ToTensor())\n", "# 将数据划分成训练集、验证集、测试集\n", "train_set, val_set = random_split(dataset, [50000, 10000])\n", "test_set = datasets.MNIST(root='./data', train=False, download=True, transform=transforms.ToTensor())\n", "len(train_set), len(val_set), len(test_set)" ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [], "source": [ "# 构建数据读取器\n", "train_loader = DataLoader(train_set, batch_size=500, shuffle=True)\n", "val_loader = DataLoader(val_set, batch_size=500, shuffle=True)\n", "test_loader = DataLoader(test_set, batch_size=500, shuffle=True)" ] }, { "cell_type": "code", "execution_count": 4, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "torch.Size([1, 3, 28, 28]) torch.Size([1, 3, 28, 28])\n", "tensor([7]) torch.Size([1, 4, 4, 4]) torch.Size([1, 4, 4, 4])\n" ] } ], "source": [ "# 卷积层的几个小技巧\n", "\n", "# 这个卷积操作，输入和输出的形状是一样的\n", "conv3 = nn.Conv2d(3, 3, (3, 3), stride=1, padding=1)\n", "x = torch.randn(1, 3, 28, 28)\n", "print(x.size(), conv3(x).size())\n", "\n", "# 这两个卷积操作输出的形状是一样的\n", "stride = torch.randint(0, 10, (1,))\n", "conv1 = nn.Conv2d(3, 4, (3, 3), stride=stride, padding=1)\n", "conv2 = nn.Conv2d(3, 4, (1, 1), stride=stride, padding=0)\n", "x = torch.randn(1, 3, 28, 28)\n", "print(stride, conv1(x).size(), conv2(x).size())" ] }, { "cell_type": "code", "execution_count": 5, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "torch.Size([1, 3, 28, 28])\n" ] }, { "ename": "RuntimeError", "evalue": "The size of tensor a (14) must match the size of tensor b (28) at non-singleton dimension 3", "output_type": "error", "traceback": [ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", "\u001b[0;31mRuntimeError\u001b[0m Traceback (most recent call last)", "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 28\u001b[0m \u001b[0;31m# 报错\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 29\u001b[0m \u001b[0mm\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mResidualBlockBugVersion\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m3\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m3\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 30\u001b[0;31m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mm\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrandn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m3\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m28\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m28\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", "\u001b[0;32m~/opt/anaconda3/lib/python3.8/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m_call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 1499\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0m_global_backward_pre_hooks\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0m_global_backward_hooks\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1500\u001b[0m or _global_forward_hooks or _global_forward_pre_hooks):\n\u001b[0;32m-> 1501\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mforward_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1502\u001b[0m \u001b[0;31m# Do not call functions when jit is used\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1503\u001b[0m \u001b[0mfull_backward_hooks\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnon_full_backward_hooks\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;32m\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, x)\u001b[0m\n\u001b[1;32m 19\u001b[0m \u001b[0;31m## 如果stride != 1 or in_channel != out_channel，\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 20\u001b[0m \u001b[0;31m## 下面的计算会出错，因为out和inputs的形状不一样\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 21\u001b[0;31m \u001b[0mout\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0minputs\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 22\u001b[0m \u001b[0mout\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mF\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrelu\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mout\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 23\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mout\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;31mRuntimeError\u001b[0m: The size of tensor a (14) must match the size of tensor b (28) at non-singleton dimension 3" ] } ], "source": [ "# 有漏洞的残差连接\n", "class ResidualBlockBugVersion(nn.Module):\n", " \n", " def __init__(self, in_channel, out_channel, stride=1):\n", " super().__init__()\n", " self.conv1 = nn.Conv2d(\n", " in_channel, out_channel, (3, 3), \n", " stride=stride, padding=1, bias=False)\n", " self.bn1 = nn.BatchNorm2d(out_channel)\n", " self.conv2 = nn.Conv2d(\n", " out_channel, out_channel, (3, 3),\n", " stride=1, padding=1, bias=False)\n", " self.bn2 = nn.BatchNorm2d(out_channel)\n", " \n", " def forward(self, x):\n", " inputs = x\n", " out = F.relu(self.bn1(self.conv1(x)))\n", " out = self.bn2(self.conv2(out))\n", " # 残差连接\n", " ## 如果stride != 1或者in_channel != out_channel，\n", " ## 下面的计算会出错，因为out和inputs的形状不一样\n", " out += inputs\n", " out = F.relu(out)\n", " return out\n", " \n", "m = ResidualBlockBugVersion(3, 3)\n", "print(m(torch.randn(1, 3, 28, 28)).size())\n", "\n", "# 报错\n", "m = ResidualBlockBugVersion(3, 3, 2)\n", "print(m(torch.randn(1, 3, 28, 28)).size())" ] }, { "cell_type": "code", "execution_count": 6, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "torch.Size([1, 4, 14, 14])\n" ] } ], "source": [ "class ResidualBlock(nn.Module):\n", " \n", " def __init__(self, in_channel, out_channel, stride=1):\n", " '''\n", " 定义残差块\n", " 参数\n", " ----\n", " in_channel ：int，输入通道\n", " out_channel ：int，输出通道\n", " stride ：int，步幅大小\n", " '''\n", " super().__init__()\n", " self.conv1 = nn.Conv2d(\n", " in_channel, out_channel, (3, 3), \n", " stride=stride, padding=1, bias=False)\n", " self.bn1 = nn.BatchNorm2d(out_channel)\n", " self.conv2 = nn.Conv2d(\n", " out_channel, out_channel, (3, 3),\n", " stride=1, padding=1, bias=False)\n", " self.bn2 = nn.BatchNorm2d(out_channel)\n", " self.downsample = None\n", " # 如果stride != 1或者in_channel != out_channel，那么输入的形状和输出的形状不一样\n", " # 使用下面的变换使得两个张量的形状一样\n", " if stride != 1 or in_channel != out_channel:\n", " # 下面两个卷积操作的输出形状是一样的\n", " # Conv2d(in_channel, out_channel, (3, 3), stride, padding=1)\n", " # Conv2d(in_channel, out_channel, (1, 1), stride, padding=0)\n", " self.downsample = nn.Sequential(\n", " nn.Conv2d(in_channel, out_channel, (1, 1), stride=stride, bias=False),\n", " nn.BatchNorm2d(out_channel))\n", " \n", " def forward(self, x):\n", " '''\n", " 向前传播\n", " 参数\n", " ----\n", " x ：torch.FloatTensor，形状为(B, I, H, W)\n", " '''\n", " inputs = x\n", " out = F.relu(self.bn1(self.conv1(x)))\n", " out = self.bn2(self.conv2(out))\n", " # 让输入(inputs)的形状和输出(out)的形状一样\n", " if self.downsample is not None:\n", " inputs = self.downsample(inputs)\n", " out += inputs\n", " out = F.relu(out)\n", " return out\n", "\n", "m = ResidualBlock(3, 4, 2)\n", "print(m(torch.randn(1, 3, 28, 28)).size())" ] }, { "cell_type": "code", "execution_count": 7, "metadata": {}, "outputs": [], "source": [ "class ResNet(nn.Module):\n", " \n", " def __init__(self):\n", " super().__init__()\n", " self.block1 = ResidualBlock(1, 20)\n", " self.block2 = ResidualBlock(20, 40, stride=2)\n", " self.block3 = ResidualBlock(40, 60, stride=2)\n", " self.block4 = ResidualBlock(60, 80, stride=2)\n", " self.block5 = ResidualBlock(80, 100, stride=2)\n", " self.block6 = ResidualBlock(100, 120, stride=2)\n", " self.lm = nn.Linear(120, 10)\n", "\n", " def forward(self, x):\n", " '''\n", " 向前传播\n", " 参数\n", " ----\n", " x ：torch.FloatTensor，形状为(B, 1, 28, 28)\n", " '''\n", " x = self.block1(x) # (B, 20, 28, 28)\n", " x = self.block2(x) # (B, 40, 14, 14)\n", " x = self.block3(x) # (B, 60, 7, 7)\n", " x = self.block4(x) # (B, 80, 4, 4)\n", " x = self.block5(x) # (B, 100, 2, 2)\n", " x = self.block6(x) # (B, 120, 1, 1)\n", " out = self.lm(x.view(x.shape[0], -1)) # (B, 10)\n", " return out\n", "\n", "model = ResNet()" ] }, { "cell_type": "code", "execution_count": 8, "metadata": {}, "outputs": [], "source": [ "eval_iters = 10\n", "\n", "def estimate_loss(model):\n", " re = {}\n", " # 将模型切换至评估模式\n", " model.eval()\n", " re['train'] = _loss(model, train_loader)\n", " re['val'] = _loss(model, val_loader)\n", " re['test'] = _loss(model, test_loader)\n", " # 将模型切换至训练模式\n", " model.train()\n", " return re\n", "\n", "@torch.no_grad()\n", "def _loss(model, data_loader):\n", " \"\"\"\n", " 计算模型在不同数据集下面的评估指标\n", " \"\"\"\n", " loss = []\n", " accuracy = []\n", " data_iter = iter(data_loader)\n", " for k in range(eval_iters):\n", " inputs, labels = next(data_iter)\n", " B = inputs.shape[0]\n", " logits = model(inputs)\n", " # 计算模型损失\n", " loss.append(F.cross_entropy(logits, labels))\n", " # 计算预测的准确率\n", " _, predicted = torch.max(logits, 1)\n", " accuracy.append((predicted == labels).sum() / B)\n", " re = {\n", " 'loss': torch.tensor(loss).mean().item(),\n", " 'accuracy': torch.tensor(accuracy).mean().item()\n", " }\n", " return re" ] }, { "cell_type": "code", "execution_count": 9, "metadata": {}, "outputs": [], "source": [ "def train_resnet(model, optimizer, data_loader, epochs=10, penalty=[]):\n", " lossi = []\n", " for epoch in range(epochs):\n", " for i, data in enumerate(data_loader, 0):\n", " inputs, labels = data\n", " optimizer.zero_grad()\n", " logits = model(inputs)\n", " loss = F.cross_entropy(logits, labels)\n", " lossi.append(loss.item())\n", " # 增加惩罚项\n", " for p in penalty:\n", " loss += p(model)\n", " loss.backward()\n", " optimizer.step()\n", " # 评估模型，并输出结果\n", " stats = estimate_loss(model)\n", " train_loss = f'train loss {stats[\"train\"][\"loss\"]:.4f}'\n", " val_loss = f'val loss {stats[\"val\"][\"loss\"]:.4f}'\n", " test_loss = f'test loss {stats[\"test\"][\"loss\"]:.4f}'\n", " print(f'epoch {epoch:>2}: {train_loss}, {val_loss}, {test_loss}')\n", " train_acc = f'train acc {stats[\"train\"][\"accuracy\"]:.4f}'\n", " val_acc = f'val acc {stats[\"val\"][\"accuracy\"]:.4f}'\n", " test_acc = f'test acc {stats[\"test\"][\"accuracy\"]:.4f}'\n", " print(f'{\"\":>10}{train_acc}, {val_acc}, {test_acc}')\n", " return lossi" ] }, { "cell_type": "code", "execution_count": 10, "metadata": {}, "outputs": [], "source": [ "stats = {}" ] }, { "cell_type": "code", "execution_count": 11, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "epoch 0: train loss 0.0820, val loss 0.1000, test loss 0.0955\n", " train acc 0.9728, val acc 0.9646, test acc 0.9686\n", "epoch 1: train loss 0.0826, val loss 0.0873, test loss 0.0864\n", " train acc 0.9746, val acc 0.9744, test acc 0.9734\n", "epoch 2: train loss 0.0523, val loss 0.0689, test loss 0.0634\n", " train acc 0.9860, val acc 0.9798, test acc 0.9804\n", "epoch 3: train loss 0.0781, val loss 0.1083, test loss 0.0830\n", " train acc 0.9762, val acc 0.9700, test acc 0.9754\n", "epoch 4: train loss 0.0157, val loss 0.0412, test loss 0.0365\n", " train acc 0.9930, val acc 0.9870, test acc 0.9894\n" ] } ], "source": [ "stats['resnet'] = train_resnet(model, optim.Adam(model.parameters(), lr=0.01), train_loader, epochs=5)" ] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.8.5" } }, "nbformat": 4, "nbformat_minor": 4 }