{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<torch._C.Generator at 0x13218f3d0>"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim\n",
    "from torch.utils.data import DataLoader, random_split\n",
    "from torchvision import datasets\n",
    "import torchvision.transforms as transforms\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "\n",
    "torch.manual_seed(12046)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(50000, 10000, 10000)"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 准备数据\n",
    "dataset = datasets.MNIST(root='./data', train=True, download=True, transform=transforms.ToTensor())\n",
    "# 将数据划分成训练集、验证集、测试集\n",
    "train_set, val_set = random_split(dataset, [50000, 10000])\n",
    "test_set = datasets.MNIST(root='./data', train=False, download=True, transform=transforms.ToTensor())\n",
    "len(train_set), len(val_set), len(test_set)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 构建数据读取器\n",
    "train_loader = DataLoader(train_set, batch_size=500, shuffle=True)\n",
    "val_loader = DataLoader(val_set, batch_size=500, shuffle=True)\n",
    "test_loader = DataLoader(test_set, batch_size=500, shuffle=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(torch.Size([500, 1, 28, 28]), torch.Size([500]), torch.Size([500, 784]))"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 获取一个批量的数据\n",
    "x, y = next(iter(train_loader))\n",
    "x.shape, y.shape, x.view(x.shape[0], -1).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 两种常见的实现方式\n",
    "## 自由度更大的实现方式\n",
    "class MLP(nn.Module):\n",
    "    \n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "        self.hidden1 = nn.Linear(784, 30)\n",
    "        self.hidden2 = nn.Linear(30, 20)\n",
    "        self.out = nn.Linear(20, 10)\n",
    "\n",
    "    def forward(self, x):\n",
    "        '''\n",
    "        多层感知器的向前传播\n",
    "        参数\n",
    "        ----\n",
    "        x ：torch.FloatTensor，形状为(B, 784)，其中B表示批量数据的大小\n",
    "        '''\n",
    "        x = F.sigmoid(self.hidden1(x))  # (B, 30)\n",
    "        x = F.sigmoid(self.hidden2(x))  # (B, 20)\n",
    "        x = self.out(x)                 # (B, 10)\n",
    "        return x\n",
    "\n",
    "model = MLP()\n",
    "\n",
    "## 更简洁的实现方式\n",
    "model = nn.Sequential(\n",
    "    nn.Linear(784, 30), nn.Sigmoid(),\n",
    "    nn.Linear( 30, 20), nn.Sigmoid(),\n",
    "    nn.Linear( 20, 10)\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "eval_iters = 10\n",
    "\n",
    "def estimate_loss(model):\n",
    "    re = {}\n",
    "    # 将模型切换至评估模式\n",
    "    model.eval()\n",
    "    re['train'] = _loss(model, train_loader)\n",
    "    re['val'] = _loss(model, val_loader)\n",
    "    re['test'] = _loss(model, test_loader)\n",
    "    # 将模型切换至训练模式\n",
    "    model.train()\n",
    "    return re\n",
    "\n",
    "@torch.no_grad()\n",
    "def _loss(model, data_loader):\n",
    "    \"\"\"\n",
    "    计算模型在不同数据集下面的评估指标\n",
    "    \"\"\"\n",
    "    loss = []\n",
    "    accuracy = []\n",
    "    data_iter = iter(data_loader)\n",
    "    for k in range(eval_iters):\n",
    "        inputs, labels = next(data_iter)\n",
    "        B, C, H, W = inputs.shape\n",
    "        # 将数据转换成模型输入要求的形状\n",
    "        # 也可以用如下的命令来完成：inputs.view(-1, 784)\n",
    "        logits = model(inputs.view(B, -1))\n",
    "        # 计算模型损失\n",
    "        loss.append(F.cross_entropy(logits, labels))\n",
    "        # 计算预测的准确率\n",
    "        _, predicted = torch.max(logits, 1)\n",
    "        accuracy.append((predicted == labels).sum() / B)\n",
    "    re = {\n",
    "        'loss': torch.tensor(loss).mean().item(),\n",
    "        'accuracy': torch.tensor(accuracy).mean().item()\n",
    "    }\n",
    "    return re"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_mlp(model, optimizer, data_loader, epochs=10, penalty=[]):\n",
    "    lossi = []\n",
    "    for epoch in range(epochs):\n",
    "        for i, data in enumerate(data_loader, 0):\n",
    "            inputs, labels = data\n",
    "            optimizer.zero_grad()\n",
    "            B, C, H, W = inputs.shape\n",
    "            # 将数据转换成模型输入要求的形状\n",
    "            # 也可以用如下的命令来完成：inputs.view(-1, 784)\n",
    "            logits = model(inputs.view(B, -1))\n",
    "            loss = F.cross_entropy(logits, labels)\n",
    "            lossi.append(loss.item())\n",
    "            # 增加惩罚项\n",
    "            for p in penalty:\n",
    "                loss += p(model)\n",
    "            loss.backward()\n",
    "            optimizer.step()\n",
    "        # 评估模型，并输出结果\n",
    "        stats = estimate_loss(model)\n",
    "        train_loss = f'train loss {stats[\"train\"][\"loss\"]:.4f}'\n",
    "        val_loss = f'val loss {stats[\"val\"][\"loss\"]:.4f}'\n",
    "        test_loss = f'test loss {stats[\"test\"][\"loss\"]:.4f}'\n",
    "        print(f'epoch {epoch:>2}: {train_loss}, {val_loss}, {test_loss}')\n",
    "        train_acc = f'train acc {stats[\"train\"][\"accuracy\"]:.4f}'\n",
    "        val_acc = f'val acc {stats[\"val\"][\"accuracy\"]:.4f}'\n",
    "        test_acc = f'test acc {stats[\"test\"][\"accuracy\"]:.4f}'\n",
    "        print(f'{\"\":>10}{train_acc}, {val_acc}, {test_acc}')\n",
    "    return lossi"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "stats = {}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch  0: train loss 2.3778, val loss 2.3783, test loss 2.3740\n",
      "          train acc 0.1050, val acc 0.1042, test acc 0.1018\n",
      "epoch  1: train loss 2.3652, val loss 2.3679, test loss 2.3587\n",
      "          train acc 0.0998, val acc 0.1054, test acc 0.0996\n",
      "epoch  2: train loss 2.3579, val loss 2.3572, test loss 2.3564\n",
      "          train acc 0.0958, val acc 0.1048, test acc 0.1018\n",
      "epoch  3: train loss 2.3514, val loss 2.3545, test loss 2.3452\n",
      "          train acc 0.0972, val acc 0.0988, test acc 0.0960\n",
      "epoch  4: train loss 2.3378, val loss 2.3427, test loss 2.3362\n",
      "          train acc 0.0906, val acc 0.0944, test acc 0.0936\n",
      "epoch  5: train loss 2.3339, val loss 2.3347, test loss 2.3252\n",
      "          train acc 0.0892, val acc 0.0966, test acc 0.0926\n",
      "epoch  6: train loss 2.3239, val loss 2.3264, test loss 2.3260\n",
      "          train acc 0.0952, val acc 0.0940, test acc 0.0946\n",
      "epoch  7: train loss 2.3136, val loss 2.3153, test loss 2.3201\n",
      "          train acc 0.1096, val acc 0.1084, test acc 0.1008\n",
      "epoch  8: train loss 2.3263, val loss 2.3140, test loss 2.3160\n",
      "          train acc 0.1066, val acc 0.1206, test acc 0.1144\n",
      "epoch  9: train loss 2.3150, val loss 2.3177, test loss 2.3136\n",
      "          train acc 0.1236, val acc 0.1174, test acc 0.1218\n"
     ]
    }
   ],
   "source": [
    "# 模型参数初始化\n",
    "# nn.init下面的函数都自动跑在torch.no_grad()的模式下\n",
    "for m in model:\n",
    "    if isinstance(m, nn.Linear):\n",
    "        nn.init.xavier_normal_(m.weight, gain=nn.init.calculate_gain('sigmoid'))\n",
    "        nn.init.zeros_(m.bias)\n",
    "\n",
    "# 使用最经典的标准随机梯度下降法\n",
    "stats['mlp'] = train_mlp(model, optim.SGD(model.parameters(), lr=0.001), train_loader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch  0: train loss 2.3180, val loss 2.3065, test loss 2.3033\n",
      "          train acc 0.1066, val acc 0.1070, test acc 0.1136\n",
      "epoch  1: train loss 2.2322, val loss 2.2220, test loss 2.2221\n",
      "          train acc 0.1354, val acc 0.1302, test acc 0.1322\n",
      "epoch  2: train loss 2.1533, val loss 2.1500, test loss 2.1490\n",
      "          train acc 0.1596, val acc 0.1600, test acc 0.1590\n",
      "epoch  3: train loss 2.1046, val loss 2.0999, test loss 2.0873\n",
      "          train acc 0.1816, val acc 0.1822, test acc 0.1810\n",
      "epoch  4: train loss 2.0356, val loss 2.0395, test loss 2.0476\n",
      "          train acc 0.2152, val acc 0.2214, test acc 0.2074\n",
      "epoch  5: train loss 1.9938, val loss 1.9933, test loss 1.9784\n",
      "          train acc 0.2594, val acc 0.2526, test acc 0.2622\n",
      "epoch  6: train loss 1.9594, val loss 1.9515, test loss 1.9479\n",
      "          train acc 0.2754, val acc 0.2894, test acc 0.2916\n",
      "epoch  7: train loss 1.9213, val loss 1.9103, test loss 1.9047\n",
      "          train acc 0.3144, val acc 0.3194, test acc 0.3194\n",
      "epoch  8: train loss 1.8772, val loss 1.8644, test loss 1.8592\n",
      "          train acc 0.3436, val acc 0.3560, test acc 0.3520\n",
      "epoch  9: train loss 1.8342, val loss 1.8432, test loss 1.8242\n",
      "          train acc 0.3646, val acc 0.3688, test acc 0.3692\n"
     ]
    }
   ],
   "source": [
    "# 使用更高效的激活函数搭建模型\n",
    "model1 = nn.Sequential(\n",
    "    nn.Linear(784, 30), nn.ReLU(),\n",
    "    nn.Linear( 30, 20), nn.ReLU(),\n",
    "    nn.Linear( 20, 10)\n",
    ")\n",
    "\n",
    "# 模型参数初始化\n",
    "# nn.init下面的函数都自动跑在torch.no_grad()的模式下\n",
    "for m in model1:\n",
    "    if isinstance(m, nn.Linear):\n",
    "        nn.init.xavier_normal_(m.weight, gain=nn.init.calculate_gain('relu'))\n",
    "        nn.init.zeros_(m.bias)\n",
    "    \n",
    "stats['mlp_relu'] = train_mlp(model1, optim.SGD(model1.parameters(), lr=0.001), train_loader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch  0: train loss 2.2938, val loss 2.2973, test loss 2.2647\n",
      "          train acc 0.1716, val acc 0.1746, test acc 0.1806\n",
      "epoch  1: train loss 2.1120, val loss 2.1006, test loss 2.0767\n",
      "          train acc 0.2590, val acc 0.2646, test acc 0.2808\n",
      "epoch  2: train loss 1.9710, val loss 1.9687, test loss 1.9488\n",
      "          train acc 0.3556, val acc 0.3530, test acc 0.3668\n",
      "epoch  3: train loss 1.8826, val loss 1.8678, test loss 1.8574\n",
      "          train acc 0.4090, val acc 0.4240, test acc 0.4240\n",
      "epoch  4: train loss 1.7792, val loss 1.7858, test loss 1.7711\n",
      "          train acc 0.4660, val acc 0.4664, test acc 0.4734\n",
      "epoch  5: train loss 1.7369, val loss 1.7219, test loss 1.7008\n",
      "          train acc 0.4790, val acc 0.4942, test acc 0.5104\n",
      "epoch  6: train loss 1.6658, val loss 1.6496, test loss 1.6290\n",
      "          train acc 0.5246, val acc 0.5264, test acc 0.5470\n",
      "epoch  7: train loss 1.6208, val loss 1.6038, test loss 1.5784\n",
      "          train acc 0.5484, val acc 0.5470, test acc 0.5704\n",
      "epoch  8: train loss 1.5519, val loss 1.5578, test loss 1.5207\n",
      "          train acc 0.5686, val acc 0.5724, test acc 0.5930\n",
      "epoch  9: train loss 1.4973, val loss 1.4873, test loss 1.4516\n",
      "          train acc 0.5956, val acc 0.5970, test acc 0.6148\n"
     ]
    }
   ],
   "source": [
    "# 在模型中增加归一化层，可加速训练过程\n",
    "model2 = nn.Sequential(\n",
    "    nn.Linear(784, 30, bias=False), nn.LayerNorm(30), nn.ReLU(),\n",
    "    nn.Linear( 30, 20, bias=False), nn.LayerNorm(20), nn.ReLU(),\n",
    "    nn.Linear(             20, 10)\n",
    ")\n",
    "\n",
    "# 模型参数初始化\n",
    "# nn.init下面的函数都自动跑在torch.no_grad()的模式下\n",
    "for m in model2:\n",
    "    if isinstance(m, nn.Linear):\n",
    "        nn.init.xavier_normal_(m.weight, gain=nn.init.calculate_gain('relu'))\n",
    "\n",
    "stats['mlp_relu_layer_norm'] = train_mlp(model2, optim.SGD(model2.parameters(), lr=0.001), train_loader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAArAAAAGOCAYAAABv3DD2AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAxOAAAMTgF/d4wjAAB69klEQVR4nO3dd1zV5fvH8dfNRlBxgAsF98q9zVIzs6zMhqVpNtSWWdr4Nr5ly37Vt2zvLLVl5daW5cq9V7kXTtwiDjb3748PoCACKnDOwffz8fg84HzmdTyOy5vrvm5jrUVERERExFN4uToAEREREZHzoQRWRERERDyKElgRERER8ShKYEVERETEoyiBFRERERGPogRWRERERDyKElgRERER8Sg+rg7AFfz9/W1oaKirwxARERGRc9izZ0+itdY/u2OXZAIbGhrK7t27XR2GiIiIiJyDMebguY6phEBEREREPIoSWBERERHxKEpgRURERMSjKIEVEREREY9ySU7iEhERcWcpKSmuDkGkwHh7e1/0PZTAioiIuIkDBw6wd+9eJbBSpHl7e1OxYkXCwsIu+B5KYEVERNzAgQMH2LNnD9WqVSMoKAhjjKtDEsl31lpOnjzJtm3bOHXqFJUrV76gEVklsCIiIm5g7969VKtWjZIlS7o6FJECVbJkSapVq8bGjRtZt24dnTt3xtfX97zuoUlcIiIiLpaSkkJKSgpBQUGuDkWkUAQFBeHt7c369euZMWPGeV+vBFZERMRNqGxALhXpv9fDwsLYuHEjycnJ53W9ElgRERERcQkfHx+Sk5OJj48/r+uUwIqIiEiB6tChA6NGjXJ1GOKGLvSnDkpgRURERMSjKIEVEREREY+iBLYAJaUk0WNsD16Y+YKrQxERESkQHTp0oH///kRERNC1a1deeuklSpUqxfPPP5/jdZGRkQwZMoTq1asTFhbGM888owUcJM/UB7YA+Xr7Mm/nPHbE7ODVq151dTgiIuJhunXrxtatWwv9udWrV2fKlCl5Pn/Dhg2MHTuWVq1aUadOHYYNG8Z7773HsGHDcrxu8uTJTJgwgZMnT3LrrbdSpUoVHn744YsNXy4BGoEtYPVD67Pu4Dqsta4ORUREpED06dOHli1bAnD//fdTv359kpKScr3u6aefpnHjxlx++eUMGDCAcePGFXSoUkRoBLaA1Qutx4ztM9gVu4sqJau4OhwREfEg5zMK6koBAQHZfp+byMjIjO/Dw8OJjo7Oz7CkCNMIbAGrF1oPgHUH17k4EhEREfdyZnnEjh07qFixogujEU+iBLaAKYEVERHJ3ltvvcXKlSuZP38+I0aMoGfPnq4OSTyESggKWMNyDfnouo/oWLWjq0MRERFxKz179uSOO+7g6NGj9O/fn379+rk6JPEQSmALWEhACANbDnR1GCIiIgVi9uzZGd+nT1iOjIwkKioq23PO1KVLF15//fUCjE6KKpUQFJLjCcfViUBEREQkH7hFAmuM6WKM2WCMSTbGbDTGXJuHayKMMXONMSeNMfuMMa+aC11Qt4A9O/1ZSrxRgugTml0pIiICEBUVRYcOHVwdhngolyewxphI4EfgDaA8MAn4yRgTlMulXwKLgBpAD2AA4JbV3xWLO7MqNZFLRERE5OK5PIEF6gMvWGtHWWsPAW8DJYDauVzXGhhprY221s4F5gF1CjbUC6NOBCIiIiL5x+UJrLX2V2vtR2fsqgekALmtnbcKeMgYE2yMaQFcDfxVMFFenPph9QElsCIiIiL5weUJbDaGAt9Za4/lct6DQD/gOLAE+MxaO6+gg7sQ5YLKUSqglBJYERERkXzgVgmsMeZBoDHwbB5O/xoYBoQALYAeaddnd9/HjTG707cTJ07kU8R5Y4yhXmg9JbAiIiIi+cBtElhjTBOc+te+1tocp+sbYxoANa21/2etPWatXQYMBx7I7nxr7TvW2vD0LTg4ON/jz833t3xP1OCoQn+uiIiISFHjFgsZGGMqAhOBN621v+bhEh+ghDHG31qbkLYvFPAuqBgvVkRIhKtDEBERESkSXD4Ca4wJAH4DVgPvpk3KCjbG+BhjihljimVz2UbgCDDaGHO7MeZZ4ClgQuFFfn7ikuKYuX0m/+z/x9WhiIiIFKoOHTowatQoV4eR4aWXXuKee+5xdRhyEVyewAJdgEZAN5wJWelbH+CTtC0Ta+0p4CYgHPgKeAb4FnDb9egOnTpEp2868fnyz10dioiIiIhHc3kJgbV2MnCuFbRG5XDdIqBdQcRUEMJLhBPsF6yJXCIiIiIXyR1GYC8J6kQgIiIXpH79vG1Dhpy+ZsgQZ1+6adPyfp9p084rvA4dOtC/f38iIiLo2rUrL730EqVKleL555/P8brIyEiGDBlC9erVCQsL45lnniElJSVPz0wvAZg5cyYtW7akd+/eGcfi4uIYNGgQ5cuXp0qVKnz55Zfndc90UVFRuOkK9YIS2EJVL7Qe+0/u5/Cpw64ORUREJN9s2LCBsWPH8vvvvxMbG8uwYcP46aefcr1u8uTJjB8/nokTJzJq1Cg+/zzvZXZr165l4MCBDB48mOeeey5j/xNPPMH06dOZMWMGn332GQMHDmTx4sUX9L7Efbm8hOBSUj/U+d/w+kPraVfFY6ofRETEldauPf9r3n038+suXS7sPnnUp08fWrZsCcD999/Pvn37SEpKyvW6p59+msaNGwMwYMAAxo0bx8MPP5ynZ65evZq1a9dSs2bNjH2pqamMHDmSUaNGUb9+ferXr0/Hjh2ZOHEirVq1Ov83Jm5LI7CFqEFYA+qF1iMuKc7VoYiIiOSbgICAbL/PTWRkZMb34eHhREfn2AY+k27dumVKXgEOHTpEfHw8AwYMICQkhJCQEGbPns2OHTvyfN90p06dOu9rpPBoBLYQdanRhbU1Cu5/wCIiIp5k69atGd/v2LGDihUr5vna7BYlKlu2LAEBAYwZM4b6aTXA8fHx+Pv753o/Y0ymGtylS5fmORYpfBqBFREREZd46623WLlyJfPnz2fEiBH07Nnzou7n5eXFfffdx4gRI0hOTiYmJobevXvz2Wef5XpteHg4y5YtIzExkejoaN58882LikUKlhLYQvbN6m8Y9NsgV4chIiLicj179uSOO+6ge/fu9OvXj379+l30Pd966y3Cw8Np164dnTp1omXLlrzyyiu5XterVy+qVq1K7dq16d69Oy+++OJFxyIFx1hrXR1DoQsPD7e7d+92ybP7TuzLt2u+JebpGEoGlHRJDCIi4l5SUlJYtWoVjRs3xtvbbVdFz1eRkZGMGjWKDh06uDoUcYH03/ObN29mz549PPDAA2eVhRhj9lhrw7O7XiOwhaxeaD3A6UQgIiIimc2bNy9jAlbWrWnTpq4OT9yEJnEVsvQEdt3BdbQOb+3iaERERFwjKioq2/3Nmzdn1apV2R7z9fUtuIDEoyiBLWRnJrAiIiKSWUBAQKb2WiLZUQlBIasaUhV/b38lsCIiIiIXSCOwhczby5v/XP4fIkpGuDoUEREREY+kBNYFXumYezsPEREREcmeSghcJCU1hWPxx1wdhoiIiIjHUQLrAkfjjnLZp5fx6B+PujoUEREREY+jBNYFSgWWolLxSny35js2H97s6nBEREQKVIcOHRg1apSrw8jw0ksvcc899+T7fSMjI5k9e3a+31fOpgTWRV5s/yKpNpVhc4e5OhQRERERj6IE1kWuiLiCTlU7aRRWRERE5DwpgXUhjcKKiIin69ChA/379yciIoKuXbvy0ksvUapUKZ5//vkcr4uMjGTIkCFUr16dsLAwnnnmGVJSUvL0zPQSgJkzZ9KyZUt69+6dcSwuLo5BgwZRvnx5qlSpwpdffnle90wXFRWFMSZP1+bk1KlT9O3bl5CQEMLCwnj55ZczjlWrVo0ffvgh4/U333xD9erVM17PmTOH5s2bU6JECa677jr27dt3VrzZ/RrkpEOHDnzyySf07t2boKAgGjVqxObNpwfSRo8eTc2aNSlTpgz9+vXj+PHjGcfuueceXnrpJX7++Wcuu+wynn322Yz9PXv2pH79+jRr1oz333+fsmXLcvfdd5//L1geKYF1oSsiruCGWjdQLqgc1lpXhyMiIm6o/if1s922HNkCwJYjW855TrppW6Zle7zbmG4Z53y85OOzrsurDRs2MHbsWH7//XdiY2MZNmwYP/30U67XTZ48mfHjxzNx4kRGjRrF559/nudnrl27loEDBzJ48GCee+65jP1PPPEE06dPZ8aMGXz22WcMHDiQxYsXn/d7yi/Dhg1j2bJlLF++nJkzZ/Lhhx+ydOlSAHr27Jnp12n8+PHceeedAOzYsYNrr72W22+/nX///RcfHx/uvffeTPc+169Bbl555RUaNmzI2rVrCQkJYdgwZyDt119/ZdCgQbz77rssWLCATZs20a9fv0zX/vXXX7zxxhu88sor9O/fP2P/ihUr+PHHH9m8eTPz589nxIgRmZLz/KY+sC42peeUfPkfnoiIiKv06dOHli1bAnD//fezb98+kpKScr3u6aefpnHjxgAMGDCAcePG8fDDD+fpmatXr2bt2rXUrFkzY19qaiojR45k1KhR1K9fn/r169OxY0cmTpxIq1atzv+N5YP//Oc/PPfccxw/fpyVK1fi5+fHhg0baNGiBb1796ZZs2bExsbi5eXFn3/+yZtvvgnADz/8QEREBP/5z38AeO6552jXrh0nT54kKCgIyP7XIC9atWrF008/DUDv3r0ZM2YMAB999BH3338/N9xwAwCffPIJDRs25MCBA4SFhQGwceNGNm7cSJkyZTLd85ZbbqFBgwaULl2a++67jzp16pCcnHyBv2q5UwLrYunJ6/GE48QlxxEWFObiiERExJ2sfXhtjsdrlK6R6zldanRhbY2czxnYciADWw487/gAAgICsv0+N5GRkRnfh4eHEx0dnedru3XrdlbidujQIeLj4xkwYAAPPPAA4JQUlC5dOs/3TXfq1KnzviY7//zzDw899BCxsbG0bNmSgICAjFKJ+vXrU6tWLSZPnoyvry/16tWjTp06AOzatYutW7cSEhICgLWW1NRUdu/eTe3atYHsfw3yolOnThnf+/n5ZfwUeMeOHdx0000Zx9LvvWPHjowE9p577jkreYUL/z1woZTAuoHo49E0+LQBN9S6gVHdR7k6HBERkUKxdevWjO937NhBxYoV83xtcHDwWfvKli1LQEAAY8aMoX59pxQiPj4ef3//XO9njMlUg5v+Y/6Ldffdd9OvXz/++9//AmSMVKe78847+emnnwgMDMxUx1q5cmXat2+fqYY3JiaGKlWqZLzO7tcgL0qUKJHt/sjIyEyfyZYtTplKRETERT8zv6kG1g1UKF6BhuUa8t2a7zJqmkRERIq6t956i5UrV2bUTPbs2fOi7ufl5cV9993HiBEjSE5OJiYmht69e/PZZ5/lem14eDjLli0jMTGR6OjojB/lX6zY2FgSExPZtWsXzz//PEuXLs0076VXr15Mnz6dP/74I9P7v/POO1mxYgWrVq3Cx8eHKVOm0LZtW+Li4vIlruwMHDiQzz//nF9++YVNmzbx8MMP06NHj4zRV3eiBNZNvNj+RVJsCq/Nfc3VoYiIiBSKnj17cscdd9C9e3f69et31oShC/HWW28RHh5Ou3bt6NSpEy1btuSVV17J9bpevXpRtWpVateuTffu3XnxxRcvOhZw6kq/+uormjRpwuHDh+ncuTMrVqzIOB4REUHz5s1p2bJlphHoiIgIJkyYwLBhw6hduzYjR47kl19+uaByiLy6/vrr+fDDDxkyZAht2rShRo0afPXVVwX2vIthLsXZ7+Hh4Xb37t2uDuMs7Ue1Z8GuBRz+z2FK+Gc/vC8iIkVPSkoKq1atonHjxnh7e7s6nEIRGRnJqFGj6NChg6tDcZkDBw6QkpLCHXfcwb333ntWl4GiLP33/ObNm9mzZw8PPPDAWeUJxpg91trw7K7XCKwbuaP+HSSnJjN923RXhyIiIuIS8+bNIyQkJNutadOmhR7Pzp07zxlP2bJlL+ref//9N1WqVKFYsWIZ7bMu1g033HDOeH/++ed8eYY70CQuN9K1Zlcalmvo6jBEREQKXFRUVLb7mzdvzqpVq7I95uvrW3ABnUPFihXPGc/FtsHs0aMHPXr0uKh7ZDVixAji4+OzPRYaGpqvz3IlJbBuJDIkktUPrnZ1GCIiIi4TEBCQqb2Wq/n4+LhVPLkpX768q0MoFCohEBERERGPogTWzUQfj+bGMTfyzsJ3XB2KiIgUsktxYrVcmtJ/r1/o73klsG6mTLEyzI6azYT1E1wdioiIFBJvb2+8vb05efKkq0MRKRQnT57EWnvBCaxqYN2Mn7cfnat1ZvLGyRyJO0LpwILr9yYiIu6jYsWKbNu2jWrVqhEUFHTRE4RE3JG1lpMnT7JlyxYOHDhAcnIy3t7eeVot7UxKYN1Q15pdmbhhIn9u/ZOel13cqiQiIuIZwsLCSExMZNOmTXh56QekUnRZazlw4ABHjx7l4MGDVKtW7bw7TCiBdUNda3YF4LfNvymBFRG5hISHhxMYGMikSZNUTiBFkrWW1NRUAFJTU4mIiKBr167nfR8lsG6oYvGKNC7fmN+3/E6qTcXL6H/iIiKXijJlynDHHXdw5MgREhISXB2OSIEpVqwYoaGh+Pn5nfe1bpHAGmO6AO8DNYCtwGPW2j/yeG0JYB0wyFo7seCiLFwfd/2Y0GKhSl5FRC5BwcHBZy2rKSKnuTw7MsZEAj8CbwDlgUnAT8aYoDze4i1gVVFKXgHaVm5LzTI1XR2GiIiIiNtxeQIL1AdesNaOstYeAt4GSgC1c7vQGNMe6AMMKtgQXWP/if1M2TjF1WGIiIiIuBWXJ7DW2l+ttR+dsasekIJTSnBOxhh/4EtgBtDBGNO4wIJ0kcf+eIybfryJ/Sf2uzoUEREREbfh8gQ2G0OB76y1x3I571GcmtnDOKO1vxljhhZ0cIUpvRvBtK3TXByJiIiIiPtwqwTWGPMg0Bh4Ng+nPwSMsNbea619BrgVeNEYUy6b+z5ujNmdvp04cSJf4y4o19a4FnDaaYmIiIiIw20SWGNME5z6177W2ug8XBIO/HLG6+U47ycy64nW2nesteHpm6fM7AwLCqNFxRZM2zqN5NRkV4cjIiIi4hbcIoE1xlQEJgJvWmt/zeNlu4HAM15HpH3dnp+xudr1Na8nJj6GRbsXuToUEREREbfg8j6wxpgA4DdgNfCuMSZ9eDQe8AOw1p7K5tLvgeeNMRuBOOBDYK619kDBR114rq91PTO2z9AIrIiIiEgaY611bQDG3ITT+zWre4EOANbae7K5zgcYBtwFlALmA/daa3fn9szw8HC7e3eup4mIiIiIixhj9lhrw7M95uoE1hWUwIqIiIi4t5wSWLeogZWcbT2ylStHXslHSz7K/WQRERGRIk4JrAcoH1yepXuXMnFDkVotV0REROSCKIH1AEF+QVxd7Wr+jvqbo3FHXR2OiIiIiEspgfUQ3Wp1I8Wm8MeWP1wdioiIiIhLKYH1EDfUugGAKZumuDgSEREREddSAushKhSvQMtKLfljyx/qCSsiIiKXNJcvZCB598UNX1AuuBw+XvrYRERE5NKlTMiDNCrfyNUhiIiIiLicSgg8TFRMFF8s/4JLcQEKEREREVAC63HeXfguD/zyAP8e+NfVoYiIiIi4hBJYD3Nj7RsBmLJR3QhERETk0qQE1sNcGXElJfxLqJ2WiIiIXLKUwHoYP28/rqtxHUv2LCH6eLSrwxEREREpdEpgPVC32t0A+GXTLy6ORERERKTwKYH1QNfVuI7ra15P5ZKVXR2KiIiISKEzl2I7pvDwcLt7925XhyEiIiIi52CM2WOtDc/umEZgPZi1lpj4GFeHISIiIlKotBKXh0pJTaHD6A4AzL13rktjERERESlMGoH1UN5e3tQsXZN5O+exePdiV4cjIiIiUmiUwHqwx9s8DsDwhcNdHImIiIhI4VEC68EuC7uMa2tcy/j149l+dLurwxEREREpFEpgPdyTbZ4k1aby3qL3XB2KiIiISKFQAuvhrqp6FS0rtSQ5NdnVoYiIiIgUCnUh8HDGGObfNx8fL32UIiIicmnQCGwRkJ68JqUkkZSS5OJoRERERAqWEtgiYumepVT7oBrf//O9q0MRERERKVBKYIuIWmVqcSz+GG8veJtLcXlgERERuXQogS0iSgaUZEDTAaw9uJZpW6e5OhwRERGRAqMEtgh5rPVjeBtv3l7wtqtDERERESkwSmAL2qZN0LIlvPgiLFoEKSkF9qgqJavQ87KezNg+gy+Wf1FgzxERERFxJSWwBW3zZmd75RVo0wbCwqBXL/j+e0jO/96t7137HvVD61PCv0S+31tERETEHZhLccJPeHi43b17d+E9MDkZFi+G33+HP/6A5cud/W3aOIls1ar5+7jUZPWFFREREY9mjNljrQ3P7phGYAuDjw9cfjkMGwbLlsG+fTBkCCxcCI0awXff5e/j0pLXncd20ntCb44nHM/X+4uIiIi4khJYVyhXDt55B6ZNg6AguOsueOmlfH/MxPUT+eGfH+g9oTcpqQVXeysiIiJSmJTAutI118A//8Dtt8Mdd+T77R9t9Sh3NbyLqZum8vzM5/P9/iIiIiKuoBpYdzN7tvO1Q4d8uV18cjwdRnVg8Z7FfH/L99zZ4M58ua+IiIhIQcqpBlYJrDs5edKZ0BUY6HQu8PPLl9vuO7GPpp83JSk1ie2PbSfYLzhf7isiIiJSUHJKYDVV3Z0EBcHYsRASkm/JK0D54PK82vFVFu9ZTEJyghJYERER8WhuMQJrjOkCvA/UALYCj1lr/ziP69sCc6213nk5321HYLNavx7KlHF6x4qIiIhcQty6jZYxJhL4EXgDKA9MAn4yxgTl8Xp/4Cvc4L3kqwMHoF076NoVjudfG6z45HiW7V2Wb/cTERERKWzukPTVB16w1o6y1h4C3gZKALXzeP2LQGpBBecyYWEwaJCz6MEtt0BiYr7ctst3Xbjm22s4Fn8sX+4nIiIiUthcnsBaa3+11n50xq56QApOKUGOjDFNgAeBhwsoPNd68UV44AGYPh2uuMJZxesiSz7ubXwvR+OP8v7i9/MpSBEREZHC5fIENhtDge+stTkOERpjfICvgWeAHYURWGFLSEzk9xtuYGbDhiQvWwbXXUdi06bwyy8XnMj2adiHGqVr8M7CdzgadzSfIxYREREpeG6VwBpjHgQaA8/m4fSngYPW2i/ycN/HjTG707cTJ05cZKQF59SpU0yYMIE+ffoQFhZG1xtvpNOaNVRNTeUjwK5aBTfeyPbSpfmlXz/2R0ef1/19vHwYeuVQjiUc491F7xbIexAREREpSG7RhQAyygHmAndYa3/N5dy6wN9Ac2vtzrSJYNuttSYvz3J1FwJrLZs3b2b79u3s2LGDHTt2EBUVxY4dO1i5ciWnTp0CoEWLFtx66610796d5ORkFi1axPrp02n055/cduQIgcAVPj7U6NOHwYMH06hRozw9Pzk1mcs+uYy9x/cSNTiK0oGlC/DdioiIiJw/t1/IwBhTEVgAfGWtfTUP578IPA+cTNvlBRQHjgE3WGvn5XS9KxLY5ORk5s6dy8SJE5k0aRK7du0665yyZctSv359br75Zm6++WaqVKlyzvvFbtrE1tdf59HNm5k3fz5XA3dHRBD62mt07tULL6+zB9cTEhI4ePAg+/fvZ9KGSaw/tJ5myc2oUKYClStXztgCAwPz862LiIiInDe3TmCNMQHAIpw61t5nHIoH/ACstaeyXBMChJyxKxxn9LYqsM9aG5/TMwsrgY2Pj+fPP/9k4sSJTJ06lcOHDwNQrVo1brzxRurWrUtERAQRERFUqVKFoKA8dQ47y7Jly4i7/XZab99OJHCiRAl8fHxITU0lJSUl42t8fI6/LBnKlClDtWrVaNmyJW3atKF169ZUq1YNY5wB7tTUVNauXcu8efOYN28eK1asoFmzZvTr14/27dtnmzyLiIiInA93T2Bvwun9mtW9QAcAa+09udwjEjcsIdi+fTvVqlUDoFGjRhkjqw0aNMhIBvNNSgoH/vyT9+fNY/78+UTEx7M3KAgvHx+8vLzw9vamRIkShIWFUa5cuYyvtphldfRqQo6GsGvXroxt48aNHDx4MOP2oaGhtG7dmuTkZBYsWMCxY6fn2FWpUoWdO3cCUL16dfr168fdd99NxYoV8/c9niE1NZUdO3awZs0a/vnnH/755x/WrVtHw4YNeeGFF6hTp06BPVtEREQKnlsnsK5QmCUEo0aN4sorr8xIZAvFsmXQoQPcey988AGcI1m21tLg0wbsO7GPnUN2Usy3WKZj27dvZ9GiRSxcuJBFixaxatUqjDG0aNGCdu3a0a5dO9q2bUuZMmXYuHEjX3/9NaNGjeLAgQN4e3vTqVMnIiIiKFOmDGXLlqVMmTKUKVOGSpUqUadOHYoVK5ZtXDlZu3Ytr732Gr/88gvHz1jgwRhDeHg4u3btwsvLi969ezN06FBq1KiR6frY2FimTJnCzz//TFRUFJdffjlXXXUVHTt2pGzZsucdj4iIiBQMJbBZuHoSV4GLj4frroPZs+HVV+H558956kdLPmLQ74P49PpPebD5g7nc1ilBCAgIOOc5SUlJ/PLLL4wYMYJp06aRkpKS7XnGGCIiIqhbty716tWjbt26NG3alAYNGuDj43PW+f/88w+vvvoq48aNw1pLmzZtaNGiBQ0bNqRBgwbUr1+foKAgFi5cyIsvvshff/2Ft7c3d999N0OGDGH16tX8/PPP/PHHHyQmJuLl5UW5cuWIPqOLQ6NGjejUqRP169cnJCSEkiVLZnwtWbIkxYsXx9/f/6zR85iYGNasWcOqVatYtWoVq1evJjw8nGHDhtGgQYMcf01FREQke0pgsyjyCSxAbKwzCrtyJXz6KTyYfXJ6IvEEld+tTFhQGOsHrsfL5F/9akpKCjExMRw+fJjDhw9z6NAhDh8+zI4dO1i/fj3r1q1j06ZNJCQkZFwTFBREixYtaNu2LW3atKF06dIMHz6cCRMmAHD99dczdOhQWrZsmeOz58yZw9ChQ/n7778z9nl5edGxY0duv/12br75ZsqWLcu2bduYOXMmM2bMYObMmZnKJrLj5eVFUFAQQUFBBAcHk5iYmFE+ka58+fLs27cPLy8v+vXrxyuvvEL58uXP95evUB05coTNmzfTpEkT/Pz8XB2OiIiIEtisLokEFmD/fmjXDrZuhZ9/httuy/a0Z6c/yxvz32Bqr6ncUOuGQg0xJSWF7du3s3btWpYsWcLChQtZvHhxRiuxdN26dWPo0KE0a9Ysz/e21jJr1izGjBlD8+bNufnmmwkLC8vx/LVr17Jz505iYmI4duxYpq8nT57kxIkTnDx5MmMDuOyyy2jcuDGNGzemUaNGhIaGsnz5ch5//HHmzJlDcHAwzz77LEOGDMm1w4O1lmPHjnHgwAESExMpXrx4xubr65vn956TlJQU1q1bx8KFCzO2jRs3As4Ew5dffplevXrh7e2dL88TERG5EEpgs7hkEliA7duhbVs4cgQmT4Zrrz3rlD2xe4h8P5IrqlzBzLtnuiDIzJKTk/nnn39YuHAh27dv584776RJkyauDuu8WWuZPHkyTz31FFu2bKFy5co0adIEa22mLSkpiYMHD3Lw4EEOHDhAUlJStvcLCAigePHiFCtWjICAgExbUFAQNWvWpF69ehlb6dJOf9/o6GgWL17M4sWLWbJkCUuXLs1UP1y1alXatGlDhQoVGDFiBMeOHeOyyy5j2LBhdOvWLf8nHOaTkydPMmrUKNauXUvp0qUzaqzTv2/QoAHBwcGuDlNERC6QEtgsLqkEFmDNGrjySqes4IknnLrYLHWsr/z9CtVKVaN3g95um7B4qsTERD755BNef/11jhw5gjEm0+bj40NoaCihoaGEhYURFhZGaGgo/v7+nDhxguPHjxMbG8vx48c5fvw4cXFxxMfHEx8fT0JCAvHx8cTGxmYqxQAoV64cvr6+nPl7vVixYjRv3pzWrVtntEg7s7zh6NGj/O9//+P9998nLi6OVq1a8cILL3DllVdSvHjxPL/nI0eOsGjRIhYsWMD8+fM5evQozz//PLfeeutF//46cOAAH374IZ988glHjhw553kBAQFcf/319OjRg+uvv/6cyay1luTk5Hwb4RYRkfyhBDaLSy6BBdi4Efr2hSVLoF49+P13yGGhBPEsqamp7Ny5k3Xr1mXa4uLiaNGiBa1ataJly5bUr18/20lyWUVHR/Paa6/xxRdfkJSUhDGGOnXq0KJFC1q0aEHLli0pXrw4+/fv58CBA+zfv5/9+/ezZ88elixZwrp16zLuFRQUhJeXF8ePH+faa6/lo48+onr16uf9Hjdv3szw4cMZPXo08fHxVK9enSeffJJu3boRGxvL4cOHOXLkCIcPH2bfvn388ccfzJkzB2stgYGBdO3ala5duxIbG8u2bdvYvn17xnbq1CkCAgIoUaJExlayZEkqV66caaJhtWrV8lxakZKSwjvvvIO1lvvvv5+QkJDzfs8iIpcyJbBZXJIJLEByMvzvfzB1KsyZA9mMOP174F9Ci4VSLricCwIUd7N9+3bGjRvH0qVLWbp0KVFRUbleU6VKFdq2bcvll19O27ZtadiwIUeOHOHJJ5/k22+/JSAggP/+97889dRT+Pv7Z7o2Li6Obdu2sWXLFjZv3pxpS/8z27JlS/7zn//QvXv3XJPJ6OhoJkyYwM8//8zcuXM58+87X19fIiIiqFatGqVLl+b48eMcO3aM2NhYYmNjiYmJISYmJtP9/P39qVu3Lk899RS9evU652jyoUOH6NWrF9OnTwegePHiPPjggwwePLhA+yOLiBQlSmCzuGQT2HQpKZD+D/+wYXDDDdC4MQt3LaTt12155vJneP3q110bo7ilgwcPsmzZMpYtW0Z8fDzlypU7a0uvvc3OrFmzePjhh9mwYQO1a9emV69eREVFsXXrVrZu3crevXvPuiY4OJiaNWtSv359BgwYwBVXXHFBZQh79+5l/vz5lCtXjqpVq1KxYsVcE+CYmJiMjhnr169n/fr1LFy4kKNHj3Ldddfx6aefEhERkemaFStWcMstt7Bjxw7uv/9+2rZty1tvvcXatWvx8/Pjrrvu4qmnnqJSpUocO3Ys03by5El8fHzw9fXN2Hx8fKhcufJZzzmTtZaVK1fy7bff8vfff1O6dGkqVKhAhQoVqFixIhUqVMDX15fo6Gj27t2b8fXAgQM0adKERx55JMc68/j4eH755ZeM3snNmzdXyYWIFDglsFlc8glsunXroEEDuPVW+PlnUm0qdT+uy4GTB9g1ZBfBfpoAI/kvMTGRt99+m1dffTWjt3CpUqWoXr16xlajRg1q1qxJzZo1CQsLc6u67MOHD/PEE08wevRogoKCeO2113jkkUfw9vZm9OjRPPjgg6SmpvLxxx/Tv39/wCnx+PXXX3njjTdYsGDBBT03MjIyY9GNjh07UqlSJXbt2sX333/Pt99+m1G2Ua5cOeLi4oiNjc3xfoGBgZQuXZo9e/YA0LZtWwYNGsStt96Kr68v1loWLVrEN998w48//phpNLpYsWJcfvnldOjQgQ4dOlCyZMmMMpL07ejRo4SHh1O7dm3q1KlDzZo1c+whLSKSlRLYLJTAnmHVKihd2qmHTU7ms0/78dCRb/jouo8Y2HKgq6OTImzfvn3s2bOHatWqUapUKVeHc97++usvHnjgAbZv307Lli1p0KABX331FeHh4UyYMIEWLVpke928efP4+uuvSUlJybRQRsmSJQkKCiI5OZmkpKRM24YNG5g5cyZbtmzJuE/lypXZvXs31lrKlClDz549ueuuu2jZsiXGGE6dOkV0dHTGaGtSUlKmEdkSJUpgjGH58uV89NFHjBkzhoSEBCpUqED37t2ZMWMGmzZtAqBBgwbcfffdNG7cmPnz5zN79mwWLFhw1sTBnBhjiIyMzFhWu1u3bhdcF2ytJSEhgbi4OE6dOpXx1cfHh/Lly1OqVKmz/tNz/PhxVq1axYoVK1ixYgXR0dHcd9993H777Xh55V//6/TuIvl5T5FLlRLYLJTAnsNXX3Hqof5UftKbkBKhzH9oGeVLVnJ1VCJu6+TJk7z44ou8++67pKam0rFjR3788ccc+w1fjN27dzNr1ixmzpzJokWLqF+/Pn379uXaa6+96AUoDh48yIgRI/j000/ZtWsXYWFh9O7dm759+9K4ceOzzo+Pj2fJkiXMmTOHxMREwsLCMpWShISEsGvXLjZs2MDGjRszvq5bt46UlBR8fX3p0qULt99+O926daNkyZIZ97bWcurUKY4dO8bWrVvZsGEDGzZsYP369WzYsIEdO3aQmpp6zvfi6+ubEU/ZsmWJiorKSMbPPCcpKYnLLruMV199lZtuuumspHffvn2MHTuWCRMmEBMTQ2BgIAEBAQQGBhIYGIivry+xsbEcOXKEo0ePcvToUY4cOUJycjJBQUGZJgWmTwwsVapUxhYSEkLx4sU5fPhwxn800rfU1NSMLhqtWrUqcglxSkoKCQkJF7Sk+IWIiYlhxowZtG7dmkqV9O+apyjQBNYY4wv0AhZYa7dkOfYQsMlaO+OiHpLPlMCew6lT8L//8X9zXuO/7ZO5LsqX31J6OjWyXbrAGf/AiMhpK1asYPny5dx777156vLgzpKTkzNqlAuizvXw4cNMmjSJn3/+mRkzZpCSkoKfnx+VK1fOtFhIdv82+fn5UatWLapXr05wcDDFihUjMDAw42tiYiL79+9n3759GaUMBw8eJDw8nKZNm2Zs6fW+b731Fh988AFxcXE0a9aMV199lVatWjFx4kTGjBnDrFmzSE1NpVixYpQvX574+Hji4uIyWtmBM7GvdOnSlC5dmlKlSlG6dGn8/f0z2t/FxsZm1DgfP3482/d1ppIlS1KxYkVOnDjBrl27AGe0/bbbbqNHjx7UqFGDPXv2ZNr2799PhQoVqFOnDnXr1qVmzZqZFk05duwYGzduzNj27t2bEVv6dvz4cUqXLk1kZGSmLSwsjD179mTq2rF9+3aSkpKoWbMmtWrVyrRFRETkWFu+ZcsWRo4cyejRo9m3bx+dO3emV69edO/enRIlSmR7TUxMDJs3b+ayyy7LdTGYrFavXs3HH3/M999/z6lTpyhRogQffPABffv2davSJMleQSewwcAxoJ+1dlSWYxuBGkAja+2/F/WgfKQENmd2xw4mf/UUleesptnfm7DAqGZe3F76SoJ63Al33XVWH1kRkfN16NAhJk2axLhx49i3bx/BwcEZW/pyzVWrVqVu3brUqVOHyMjIfF8hbt++fbzxxht8+umnJCYm4uXlRWpqKn5+fnTt2pWePXtyww03EBQUlOm69EVIzmfkOzU1NWNlv/QR29jYWMqUKZNR2pH+HGsty5YtY+zYsYwdOzZPHUDSGWOoWrUq5cqVY9u2bezfvz/bc4oXL54xOhwUFMShQ4fYuXMnKSkp57x3YGAgkZGR+Pn5sXnz5rNWTQwODqZp06Y0a9aM5s2b07x5cypWrMiECRP4+uuvM5b3rly5MnXq1GHWrFkkJydn9G3u1asXJUuWZPny5Rn/Mdy6dSsAISEh9OnThwEDBtCwYcNzxpiQkMD48eP5+OOPM2rOW7Rowc0338yHH35IdHQ03bp14/PPP7+oZb5TUlKYN28e48aNY8WKFdx+++3079//rN8r53Lw4MGMRWYWL17Mpk2b6NixI/fee2+uk1XTJ3xm7eSS1d69e5k8eTK7du1iwIABVK1a9bzeo6vlWwJrjCkB7AMestaOTtvnAyQCd1prfzzj3KbAMmCJtbb1RcSf75TAnofdu/lj/JtcF/MR5U/AB79Bj8BmsHQp6H+vIlJE7N69mzfeeINdu3Zx88030717d7fp3WutZfny5YwfP54jR45QqVIlKlWqRMWKFalUqRLlypVj7969mcosNmzYwL59+6hWrRq1a9fOtFWpUoXg4OBsyxKSk5PZu3cvUVFRREVFceDAASpVqkTVqlWpWrVqpkmV1lr27t3Lpk2b2LRpExs3bmTVqlUsX74820mEfn5+3HzzzfTr14+rrroKb29vDh06xPjx4xkzZkxG3+Z0xhhq165N06ZNqVq1KpMmTWLt2rWA006vf//+XHvttWzevJnVq1dnbOvWrSMxMRF/f3969uzJwIEDM2rSjxw5wqBBg/jhhx8oU6YMn332Gbdls8x6SkoKiYmJeHt74+Pjk/FrlZKSwpw5cxg3bhwTJkxg3759gDMSn5CQQNmyZRk8eDADBw486/fPvn37+Ouvv/jzzz9ZsGAB27ZtyzgWEBBA5cqV2bx5M+As633PPffQt29fIiIi2LNnD/Pnz2fevHnMmzeP1atX4+PjQ7NmzWjTpg1t27albdu2VKhQgU2bNjFx4kQmTZrEokWLMp7h6+vLgAEDeP7556lQoULuv/HcQH4msP5AHHCXtfb7M/anArdZayecsW880B242Vo75QJjLxBKYM9PYkoiXy7/kpdnv8SJuGNsrvB/VLr/SefgxInOUrXl1DdWRESckeYtW7awbNkyli9fzpYtW+jcuTN33nlnjm32du/ezYQJE0hNTaVZs2Y0btw40wqA1loWL17Ml19+yY8//njW6C9AeHg4jRo1omPHjtxzzz2UKVMm22eNGzeOBx98kMOHD3PddddRvHhxDhw4wIEDBzh48CCHDh06K5lOH/1PTk4GoHbt2vTo0YMePXpQs2ZNRo4cyf/+9z927NhB8eLFefjhh+nQoQMzZ85k2rRprFmzJuN+derUoVWrVhlbgwYN8PX1Ze3atYwePZpvv/02IzmuWLFiphaDYWFhXH755SQkJGS09UtXpkwZDh8+DDj9p7t27crNN99MSEgIL730EosWLSIwMJBHHnmEp59+mjJlyhAfH8+KFStYtGgRCxcuZPny5SQmJuLr64ufn1+mryNGjKBBgwbn/vDzWb6WEKQlqz2steOz7MtIYI0xTXBGX6dZa7tecOQFRAnshfl106/cMOYGBjQdwBc3fgHR0RAZCZdfDjNnujo8ERG5RMTGxvLjjz+yfPly6tatS6NGjWjYsOE5E9bs7N+/nwceeIDJkycDTju/M5fzDgoKIiUlheTk5ExfmzZtSo8ePahfv/5ZP+ZPSkpizJgxvP7662zYsCFjf2hoKNdccw1dunShc+fOuZYuJCcnM23aNEaOHMmmTZto2bIl7dq1o127dlSvXj3juampqWzcuJEFCxawYMECVq9eTdOmTenevTudOnXKVGJgreXXX3/lv//9L2vWrKF48eLUqVOHVatWkZSUBDiJer169ShevDhJSUkkJiaSmJiY8f3UqVOzndRZUAoigc062pqxzxgTACwEagINrbXbznErl1ECe2GstXQc3ZG5O+fy70P/Urd0LZg8GcqWhSuvdE7q0QMaNoS779ZStSIi4vaOHj1KcHBwvk5aTE1NZcqUKWzdupWOHTvSuHFjt+kkkZqaytixY3n55Zc5cOAArVu3pk2bNrRu3ZoWLVqcczKdKxRaAgv8DowFrgP6WGvHXHDUBUgJ7IVbvHsxT09/mo+7fkz9sPqZDx465CyMsG+fUx979dUwYADcdBNcZIsfERERubQURAJ7q7V2YpZ9HwKdgNrAQGvtFxcecsFSAluAkpPhjz/g669h6lTndWioMyLbvz/Uru3qCEVERMQDXHACa4wpDywB9gKxQDJwLU596zbgJHAKGAhYYB3woLV2fn6+gfymBPbiWWv5e8fftI9of+5WH/v3w+jRMGIEpM2s5Ior4J57nFKDM4rzRURERM6UUwKbW0FGCSB9eltQ2muAEKAh0BXon/4cwAc4vy7D4pHeXvA2HUd3ZMrGHBpMlCsH//kPbNwIs2bBnXfCkiXQr5+zaALAgQOwYgVcgivCiYiIyIXJMYG11m6y1jay1ra21l5urW2XdugZa209a20Fa216wvoNTj/YacaYDwoyaHG9Pg37UMy3GM/OeJbk1OScTzYGOnSA77+HvXud0oL0tltjxkCzZvDbb85rayFtNqSIiIhIdvJzStwUoAnwKjDQGPOjMcY9ptxJvqtQvAJPtHmC9YfWM3rV6LxfWLq0szRtuvbt4cknnQQXYNkyCAuDvn1h0qTTI7UiIiIiafKUYBpjyqUtGZsja22qtfYl4DHgdmD4xYUn7uzJtk8SWiyUF2e/yKmkC0w0GzeGt96C9KX3TpyA6tXh22/h5pudkdrR55Egi4iISJGX1xHSZ4AtxpincSZr5cha+xHwPfCoMebqi4hP3FgJ/xK8cOUL7Dm+h8+WfZY/N+3Y0RmFjYqCd9+FkBBn0tfAgZCYmD/PEBEREY+W1wR2Cc7iBK/gTNa6wxgTlss1jwGHgA+NMd4XHqK4sweaP8CgloPoXqc7AIdPHabFly14fe7rbDt6EWtYRETA4MGwfLmT1H7yiVNmsGdPfoQtIiIiHixPCay1doy19mYgHHgDuB7YbIx5Km3lLQDfLNccAYYBtYB78y9kcSd+3n58cN0HVCtVDYB/D/zLliNbeG7mc9T6sBa/bPrl4h4QFgZ//glPPQULFzoTvubMyYfIRURExFOd1yQra+1Ba+1zOIsVLMHp/5rehcA/m0u+w+lMcO3FBCmeo31ke/Y/uZ+pvaYSEhBCr/G9+PfAvxd3Ux8f+N//4OefnRrZa6912m+Bs//WW0+fu2kTvPMOrFmj1lwiIiJF1AV1CbDW7rHWdsZJTOOBj4FN2Zx3FLgf6HkxQYpn8fP244ZaNzD+9vHEJ8fTbUw3jsUfu/gb9+jh9JEdMcIZmQWnh+xvvzkrfgFMngxPPAGNGkH58k7v2VGjICHh4p8vIiIibuG8l5ItCrQSV+EZsWIER+KO8FTbp869YtfFSE52RmjTnTwJc+fC9OnOtnq1s//KK2HiRKeNl4iIiLi9C15KNp8eXiptJNZtKIF1jZTUFLyMV8Eksudy4AC8+ip89BHUqeOM1latWnjPFxERkQtyMUvJXuyDvYHpxpivCvI54v5i4mPo+kNXPlzyYeE+OCwMPvwQ3nvPWdK2dWunDEFEREQ8Vq4JrDHmRmNMZ2NM+7SvN6btr22MSTHGnDLGxKZtJ4wxicaY/6Vd/gDO6lzrCvA9iAew1hIVE8WQaUP4c+ufhR/AY4/B+PFw/DhcdRXs21f4MYiIiEi+yLWEwBiTirN4QfrPfVOttT7GmNrAeuDNbC77G1gAbAH+sdZ2yr+QL55KCFxj0+FNtBrRiuTUZKb1mUbbym0LP4jFi53esg8/fPp1zZqqjRUREXEzF1UDm5bAtga2AjWB+dZa77QEdp21NttFCtJGYR8FGlhrN1/MG8hvSmBdZ+6OuVz3/XV4GS/+6POHa5LYdEePOonrPffAyJHOvnfecUZp69SB2rWhVi0oVsx1MYqIiFyi8qMGNsZaexjIdjLWGYsZnKktMNLdkldxrSsiruD33r+TalMZMm0ILu2CYQy88YbTnivdV1/BSy9Bz57QpAkEBUFkJFx3nZPkxse7KloRERFJk9cR2DrW2k1njrpm+X4s0BJYk7YtAGYAxdJW5HIrGoF1vYW7FhIZEkmF4hVcHUpmJ086iyFs2OBsGzee/j4hwZkU9vDD8OCDUK6cq6MVEREpsgqjC8EBYA/OUrMDganAWqBOHgPsYozZYIxJNsZsNMbkunKXMaasMWbyGZPI3jLGFGhXBck/bSq3yUheJ22YxMJdC10cUZqgIGfktVcvePll+PFHWLUK9u+H4cMhMNAZoR069PQ1p065KloREZFLUr4kfNbagdbattbaJtbaEKALkAT8ZYy5MqdrjTGRwI/AG0B5YBLwkzEmKJfHfgPE4STJ1wMPAr0v4m2IC8TEx3Dv5Hvp8l0X13QnyKuSJeHxx2HLFhg3zlntCyAuzkl6+/c/fe5338HUqVrKVkREpIDkNYEdYoz5P2BIXk621v4FtAJ2Aj8aY4rncHp94AVr7Shr7SHgbaAEUPtcFxhjSgCHgb7W2p3W2rnAHKBFnt6NuI2QgBAm95xMqk2ly3dduGPcHeyOdePyDh8fuPVWZ3IXOAls797Q4ozfeh9+CN26OT1nZ8xwTZwiIiJFWF5qYE/ijHQmAP6Av7W2+BlttFZlc9kUa+1LxpiWwCLgJWvtK3kKyJj2OPWzZay1x/J4jcHpkvCOtfaj3M5XDaz72X50O0OmDWHyxskU8y3GB9d+QL+m/Vwd1oXZvNlZ+evTTyEpCTp1gtdfz5zkioiISI4uqgbWWhtkrS1rra2U9jV9NPUk8CcQhZM8bk37fj9wIu3aJcBs4GFjjA95MxT4Lq/Ja5o+QCng+/O4RtxI1VJVmdRzEr/d+RuVileiYvGKrg7pwtWsCe+/70wG69sXZs6Eli2dkduRI2H6dGdSWFKSqyMVERHxSLmOwGY62ZgGQKK1dmPaaz+ciVs7rbXJ57jmceA/QGtrbVQu938QeA24zFobnceYKgL/AM9aa7/IIYbH01+XLFmyUkxMTF5uLy6QlJKEr7cvAP/s/4dZUbN4tNWjLo7qIqxdC88/D5MmZd7/779Qvz5s3+50NhgwAG65xTk2d65TpqBOByIiconKaQQ2r6Oi6T4B9gHpjTP9gZU4E6jGnOOaScBYa+2uXIJsglP/esd5JK++OBPApp8reQWw1r4DvJP+Ojw8XLNr3Fh68goweNpgZm6fSWJKIk+2fdKFUV2E+vVh4kRnRHbjRti9G3btgogI53h0NMyeDTfeePqaHj2cnrMzZ0LTpi4JW0RExF3lOAJrjCkFlAES03Z9j9MyazCnl5b9GAjFSWpNllukAnuttak5BuGMoi4AvrLWvprn4I35AmeVsDbW2pN5vU41sJ7j4MmDdBzdkbUH1zL8muE83ubx3C/yRNZCSoozSQycGtonn4TixeHvv6FePdfGJyIiUsgueClZY8wDwKdATiOWJpfjx4GO1tqV53hGAM5Erx1kboMVD/gBWGvParRpjHkWeAq4Iu1agBRrbVwOsQBKYD3NgZMH6Di6I+sOruPdLu8yuPVgV4dUOH79Fbp3h9BQp6SgenVXRyQiIlJoLiaBrQ+0A07hJKlPATHAmT+u90p7vRj4MsstygKvA59ba7NtwWWMuQmnzCCre4EOANbae7K57igQkmX339baDud6P+mUwHqe/Sf202F0BzYc2sC0PtO4pvo1rg6pcPz8s7OoQuXKMG8ehGf751hERKTIueAENpsb/QVEW2v7Ztk/Fafva7i1NjHLsTXAkbwkloVFCaxnij4ezafLPuXF9i/i7eVNQnIC/j7+rg6r4I0aBffe60zqmjNHE7tEROSSkJ9LyQYBxbLZPyPt2I3ZHBsOeGhDT3EnFYpX4JWOr+Dt5Y21llof1eLyry/ntTmvsTJ6JefznzGPcs89zuIImzbBNdfAlClw4oRzbNcuZxWw//u/vN8vNVWrhImIiEc73wS2Hac7EJxpJM7CA+OzHrDWjrbWbr2Q4ETOJTYhluYVm/PP/n94ftbzNP2iKZXeqcTwBcOLZiL7yCPOYghr1sBNN8HWtD9SCQnw1VdQqdLpc6dPh+QsXe2Sk2HaNLjrLihRAurUcbociIiIeKDzKiG4oAcY4w8EWWuPFOiDzoNKCIqOxJRE5u+cz2+bf2PsurHEJcfx70P/EhoU6urQCsYff8D+/XD99VC2rNO54PhxKFkSjIHVq6FxY6dF1xNPQPPm8NNP8OOPznXg7GvSBL5IK2WfMQNKl3b2iYiIuImLqoE1xgQCCbm1wsrh+v8Aw6y1fhdyfUFQAls0HU84TlRMFA3KNQAg1abiZc73hwwe7uhR+PhjZyWwQ4dO769ZE3r3drYaNU7vt9YZjT10CPbuBX9/WLLE6U1btaqzFS9+9nNEREQK2MUmsEk4pQapZN8uy+IsKzveWjvAGOMN+Ke3vjLGPIqTwJa4iPeQr5TAFn1L9iyh/5T+/NzjZ+qUrePqcArfqVMwejTs2OGs7tWihTNCm53ly2HbNmfxBIB+/eDrr08fr14d3njDWQr3XPcQERHJZxebwD6Hs5BBAk4Sm5U3cC3QBWiPs7DA/6WPuKb1kn3VWht2we8gnymBLfpGrxpNvyn9KO5fnHE9xtGpWidXh+Q5Vq+GlSudJW63bYNffoGYGOjUCT74QIsqiIhIobiopWSttblObzbGbAGuAaoAcTjJrojL3N34bioWr0iPsT3o8l0XPr/hc/o1VTOMPGnUyNnSHTgAzz3nTBZr1AgeewyGDnUmg4mIiLhAngoEjTFdjTEdjTFXnrF1MsZcb4wpA6wEIq21PwDJKIEVN9C5emcW9ltI5ZKV6T+1Py/OerFodigoaGFhMGIELFrkTBAbPhxq13aWuBUREXGBXEdg0/yCU+uatQDOAp2ttTOz7L+gCV8i+a1uaF0W9lvIDT/cQFJqEkY1nBeuVStYvNipj/3wQ6hb19URiYjIJSqvCWwrnIlaKWfs8wMCgI35HZRIfiofXJ45984h0CcQgLikOJJTkynur9n1583Ly1k4oV+/0xO6fvgBmjVzRmVFREQKQa4JrDGmOlDXWvvNGfs2Ad9aa18tyOBE8ksxX2cBuVSbSu8Jvdl2dBu/9f6NisUrujgyD5WevO7f7ySz9erBsmWZuxScOuWUHcydC/v2OV0OOnRwkmAREZGLkJcR2PrAK8A3Z+w7awF6Y0w4sAjwBUqlJbkAmukhbsNgaFahGRM3TKTFly348LoPubnOzSotuFDlyjmLK4SEOMnrv//CN984SeuyZZlXBPvsM6cHbf/+zvK45cqdPhYbe3pS2PDhzn2eeAIuu6ww342IiHiIvLTR6gJMxElO07UG9gI7cUoJ/IGfgDZAONAASF9WtiZQTW20xJ18v+Z7Hv7tYWITYulSvQsfXvchNcvUdHVYnm/SJLj5ZiehbdcOrrjC2UJCYNQoGDkSDh6E4GBnVDYoyElmJ050FlPw9T19D4Brr4Unn4SrrlIPWhGRS0xObbTy+rM8i1P/mr5ZnIlaKUBS2hZtrb0V+Bo4Ya29y1p7FzDyIuMXyXe9G/Zm0yOb6NuoL9O2TuOpv55ydUhFQ6dOsGYNHD4MU6fCf/4Dbdo4E77efBN274axY+GFF5zkFaBpU7jxRjh2zHndvbtTetCjB/z5J1x9tXPO9987S+eKiMglL68jsB9aa2udsW8z8KW19n/ZnJ9p4QItZCDubt7OeVQIrkD10tVJtam8Pvd1LBZ/b3/8vP3w9/En2C+YbrW7UcJfFTGFats2eO89pwftqVNOre3//R9066YRWRGRIu6iFjIQKeraVWmX8f3oVaN5ftbzZ51TPrg8N9W+qTDDEoBq1ZzVv156Cd55B9591xmhbd0aPv7YGZkVEZFLTl4T2AhjzL9APE65QDhwuzEmAfjdWrvpjHN9cepiRTzObfVuo03lNiQkJ5CQkpDxNSwoLKPt1nuL3qNu2bp0qdHFxdFeQkqXhmHDYOBA5+uXX0JqWrvplBSns4FGZEVELhl5LSH4ERjF6QlbPXAS2VI49bA/AIOstceMMY8Dw6y1xdKuVwmBFBmxCbFUebcKxxKO0bVmV4ZfM5w6Zeu4OqxLz/79p7sYfPstvPqq04+2eXPXxiUiIvnmYksIvHAmZQ0544ZXA18C3wL/A/oATYwxHYADOB0KMk7n7BW8RDxSCf8S/PPQPzwz4xl++OcHZmybwbz75tG8ohKnQnVmC664OGeLjHRe79jhJLXlyzuTyc7cEhLgttvgzjshMNAloYuIyMXLywhsB+BFa23HM/btBUZYa4emvR4B3Af8z1r7TJbrH8EZkQ3J18gvgkZgJT/MjppNl++6EFEyghUPrCDYL9jVIV26rD1dQjB8uNN6KysvL/D2hqQkKFMGHngAnnrKafElIiJu56LaaFlrZ5+ZvKbZChw+4/WTwEtZk9c0h4BleQ1WxFN0iOzAm1e/ycFTB1l7YK2rw7m0nVn/+uijMGMG/P47LFkCW7fC0aNO4nrokNPVICTEmQTmk/ZDqAMH4PhxV0QuIiIXINcR2It+gDFBQKq1Nq5AH3QeNAIr+SXVpnLg5AHKB5d3dShyPlJSYNMmpz8twCOPwCefwK5dUKmSk9CuWQNt20KxYq6NVUTkEnXRCxkYYyoYYz4xxgTkcp6vMWaNMab9GbvvAqYbY9RAU4ocL+OVkbxuOLSBHTE7XByR5Im39+nkFZxE9e67oWJF5/Wff0LnzlClijNB7OhR18QpIiLZytMIrDEmEqdsIBAIBqbgtNRK62ODNxAAtMNZnauutXajMaYksA44Yq1tkO/RXyCNwEp+23diHzU+qEHj8o2Zfc9sfLzUYtmjbdkCEybAp59CVJSz9O2DD8Ljj0OFCtlfc2YdroiIXLT8WEo2DsBam4jT57UtsBC4GvgXWAO0tqez4VhjjB8wFigHaJ1OKdLKB5fn/mb3M3/XfIbNGebqcORi1ajhLIO7eTN89x1ERMDbbzudDu67z6mzvf12eP3109f07ev0q01MdF4vWQJ33eUsmzthgpMIF3DJlojIpSLXYSJjzNtAhPOt+Qr4DbDW2heMMf8F3sBJhB8947JQ4HugA04Hgz/yO3ARd/N6p9eZFTWLV+e8ytXVrs60wpd4KB8f6N0bevWCX35xlrEdOfL08TMT0jp14OBB8Etbx2XjRif5PVOpUtC4MTRrBgMGQK1aiIjI+ctLG62xQHWgMbAIGAF8aa31NsakAuVx+rzuPWPfUSAEeN5a+3q2N3YhlRBIQdlwaANNP29K2WJlWfHACsoWK+vqkCQ/WeuUFwQGOr1ofX1zPv/UKWey2MqVzrZiBaxeDSdOOInuc8/BM8+Av3/hxC8i4kFyKiHIaw1seWBPWoJ65vdnJrDRwKfAQzh1sJ2ttbPz6T3kKyWwUpBGrhzJfVPu49PrP+XB5g+6OhxxN6mpMHeusyzu2rXQsCEsX366pde5pKRAbCzExDiTyk6edFYe04IMIlJEXexKXOAsF5vd91m1TfvqDXxojLnLWrsqj88QKRLubXIvdcrWoU3lNq4ORdyRlxe0b++Mxr7zjrMvPXmNj3f61RYv7ry++24nud2920lesw441KkD48ZB/fqFF7+IiBvISwnBKqA0EA5EAf8FvssyAutF5lHZl4CHcRLkdtbaDQX1Bi6ERmClsMQnx7PlyBYuC7vM1aGIuztyxJks9tRTMHSos695c2fENTLSqZ8NCXG2UqXg2DFn1TF/fxgzBrp1c13sIiIF4GJHYEcBZXAS149wktgzWc4elf0M+BaYDfxkjGlqrU3Je8gins9ayw0/3MDq/atZ+cBKwktk+2dQxLFlC9x4I9SufXrfokU5lxZ06+ZMBqtTp+DjExFxIxdSA1sOp97V4tS+nvnVB6f+tZK1NtoY0xaYBzxmrf2wgN7DedMIrBSWiesncsvPt9AmvA1/3/M3vt65TPoROV+pqU5ZAsBffzmjtTVrujQkEZH8kB+TuCKA7YA/Tn3rHTgLGSTjlA+kL2QwBqdnbFVr7Y60aycATYFq1trUs+9e+JTASmF68s8nGb5wOINbDebda9/N2P/Tvz8xbv04YuJjeP6K52kf2T6Hu4jkIjbWSV7bt4eJE519998Pf/zhjOKGhDirj9Wrd3qrXj33yWMiIi6SH5O4YoFBad83BU5Ya8en3dwPGALUxUlgXwaOn3Ht90BZnBW8Ys87ehEP93qn11m8ZzHvLX6PDYc38Hvv3wFYtW8V49eNx9vLm+nbpvNIi0d44+o3CPILcnHE4pFKlHAmdK1ceXpf6dLOymHJyXDgQOZj4LTyuvZaePJJaNdOK4mJiMfI0whspguMeQG4wlp7jTHmOuATIAh4H3gja62rMSYAwFobnz8hXzyNwEph23t8L7f8dAthQWFM6TUFgKNxR/H38WdP7B7um3If83bOo1qpaszsO5OIkAgXRyxFUmwsbNgA69Y524oVMHOmU3Kwfv3pUgQRETdwwSUEaaOrV+OUBaQnpn2AesBzQBvgRuBN4Ng5buMFBFprf7+g6AuAElhxN6k2lQ8Xf8jUTVOZ1mca3l7eWGsxGhGTgrZpE+zfD1dc4bx+7DFo0QL69HFez5jhJL0pKU4ZwtVXQ8WKLgtXRC4dF5PAZp2wBTn3gT3zX1t7xj5rrfXOc8QFTAmsuKszk9a6H9flZOJJKhSvQPng8lQIrkDF4hW5vf7t1CmrWedSAE6ccEZjW7SAKc5PCrjrrrOXxG3SBLp2dbZWrcDbbf56F5Ei5GJHYDsDCTiTtizQG2gBzAJuAyKBnTi1r5uyuY03UMxa+8eFv4X8pQRWPMEd4+5g65GtRJ+IZv+J/aSkVedM6zONa6pf4+LopMiKj3dGXBs3dl6vXQsHDzpJ6s6d8PvvzsSww4ed46VLO6uJNW4M76ZNUly82GkL1rWr07NWROQCXHQXgiw3ewa43Fp7Y9rra4FXgcuA/wNet9Ymn+c9u+DU0NYAtuK03co14TXGvIqzdG0s8Ii19re8PE8JrHiaVJvKoVOH2H9iPw3KNQBg8+HNlAsuRwn/Ei6OTi45KSmwdCn8+quTzG7b5qwGNmeOc3zwYHj/fYiKchZniI6GadPgqqugShVXRi4iHiS/E9h7gMustU9m2f848CLQ6nxW3jLGRAIrcToZ/AI8BTwIVLTWnszhuseAF4AeOCPDP6U9Oyq3ZyqBFU93MvEkTT5vgsXy020/0bRCU1eHJJc6a093Mdi40ZkU1r278/rrr6FfP+f7atWcett27Zytdm11PxCRbOVrApvLgypZa/ec5zXX4/SN/SjtdShwAGhmrV2Rw3XbgM+ttW+mvf4IOGCtfSW3ZyqBFU9nreXDJR/y5J9PYozh7c5v81CLh/DxUk9PcUMHD8L06U7Hg9mznfKCdGXLwuWXQ5s2ztK5l18OAQEuC1VE3EehJbD5wRjTHpgBlLHWZtvZwBhTHKdsoLm1dnnavruA29NLG3KiBFaKimV7l3HHuDvYdnQb5YPL82SbJ3mi7ROuDkskZ/v2wfz5MG+es61c6ZQlAOzZ43Q52L4dxo+HW25xRm1F5JKTUwLrjk3/hgLfnSt5TVMy7evWM/bFAJULKigRd9S8YnNW3L+Clzu8TJBvECcST2Qcm7pxKvtO7HNhdCLnUL483HqrM+lr6VKIiXHqZz/++HSLrlmz4KmnYNky57W1oIEHEUnjViOwxpgHgddwamyjczivPE57r8D0BRKMMZ2Az6y1Zy0Cnlaf+3j665IlS1aKiYnJ5+hFXMtaS0JKAgE+ARyNO0qF4RVISk2iU9VO9LqsFzfXvZmQgBBXhymSNydOwMKFTklBsWLOKG3Tpk79bO/ezhYc7OooRaQAecQIrDGmCfA20Den5DXNEZyJW+XO2FcCp93XWay171hrw9O3YP2lJ0WQMYYAH6d2sJhvMUZ3H80NtW5gdtRs7ptyH+XeLsfNP91MXFKciyMVyYPgYOjc2UleAXx94Y47nBHZBx90uhk8+yzs3Zv99XFxcPfd8MQZJTXr1zvlCyLi8dxiBNYYUxFYAHxlrX01j9csB4Zba39Iez0Mpyb22tyuVQ2sXEqOxh1l/PrxjPl3DMfij7Hs/mWuDknkwh0/Dj/8AO+846wi5usLvXrB449DmTLOcrn16jklB40bOyUJv/3mdDq45hr46y/neLt2UKIE+Pg49/D1db4PCnKS4/QOCnFxp88RkULl1pO4jDEBwCJgB84iCeniAT8Aa+2pbK57GrgLuBwIBpYD/2et/SC3ZyqBlUtVYkoift5+JKcms/PYTqqVynlyzOp9q5kVNYtBLQfh7aXVlsSNpKY6ienw4U5nA3CS1C5dnMUWwEl2ixc/fc2YMc6xGTPOPXILTmuvDWndIF97DZ5/Hr75xlmVTEQKjbsnsDcBk7I5dC/QAcBae0821wUAU4A2gA+wBOhsrU3M7ZlKYOVSZq2l/aj27I7dzbqB6zLKDrI6fOowDT9ryN7jexl65VBe7vhyIUcqkkcrVsCHH8LJk9C3L9xwQ87nWwv79zurjiUlOVtysvP1xAknOe7Y0Tl3/HiYOxfeftsZiRWRQuPWCezFMM6i8a0Bf2CutWlrbeZCCaxc6t5e8DZP/fUUr3Z8leevfD7bc1btW8WNY24kOTWZfSf2MaXnFG6snWuXOpGiy1r45BO4997TtbkiUmCKbAJ7oZTAyqUuMSWRhp82ZOexnWx4ZANVSma/vOeJxBMcPnWYZl80Izk1maUDllKzzFmNPkQuDePGQY8eTieEX35xamjPZK1TelC37unXWmVM5IJ5RBcCESk8ft5+fHDdB8Qlx/Hkn5lWhWb70e0s3r0YgGC/YCJCIhhz6xhK+JfgWEJO7ZlFirhbb4UhQ5ySgk6d4PBhp/Rg1ix49FFn8tdll8GhQ875M2c6E8t+/vn0PVLy9INCEcmFEliRS9Q11a+he53ujF03llnbZwGQnJrMXRPv4oqRV7AjZkfGuZ2rd2bToE00r9jcVeGKuJ4xzqSxoUOddl7NmjmLMlx1lVOD6+/vdENITnbO9/Z2Oh6EhTmvY2KgalVngYZt2/InphMnYPlySMi2i6RIkaWKdJFL2DvXvENkyUiaVGgCwJvz3mT+rvkMbjWYiJCITOemT/ZasGsB6w6uo3/T/oUer4jLGQMvv+z0qX36aWjYEAYNgptvhgYNMpcMdOjgjNam27HDqZ19+20nEe7aFQYOdPrd5nWC2IkTzvK7f//tdF9YtsxJmNu0cboyhITk45sVcV+qgRURAJbuWUrbr9tSp2wdlg5Ymm13glSbSuPPGrPu4Dpm9J1B+8j2LohUxE3ExUFg4PldY63Txuujj2DqVKfjQZkyTjLbrZvTBuzM1l/pK5JdfbWTHI8ZA3fe6RwLCXHqcYsXd3rjNmoE06ZBuXLZPlrE02gSVxZKYEUyOxZ/jJA3QwBY/eBqGpZreM5z1x9cT8sRLQn0CWT5/cupXLJyIUUpUsTs2AEjR8LkybBqlbPPz89p4fXTT1CyJAweDO+/76wiVqcOHDjgJKvt2zujv95p/Zlffx2eew5q1oTp0516XBEPpwQ2CyWwIplN3TiVW36+hbc7v81jrR/L9fyJ6ydyy8+30KxCM+beO5dA3/MchRKRzHbudDobTJkCUVFOwmoMLFrkbHfeebqW9lw++cQpSahcGZYu1UiseDwlsFkogRU5W2xCLCX8S+R+YpoXZ73IK3NeoU/DPnzT/RuM2gWJ5I/k5AtfNOG775ySg48+Ugsv8XhqoyUiuTqf5BXgxQ4vclPtm6hRqgaWS+8/wiIF5mJW/OrTBz7+2Eled+502n1NmpRvoYm4C3UhEJEL4mW8mHDHBLyM/h8s4pa2bnXKD3r3Pr2vfn1nAYbQUGcrW/b01rKl0/ZLI7fiAZTAisgFS09eYxNiefT3R3mx/YtULVUVgG1HtxEZEomX8SImPoZhc4Zxa91baRXeSkmvSGHo2BFiY0/3pT1xwulcsH270zs2Kensa6pXd7oh9OzpJLTZ2brVmTwWGVlQkYvkSv+KiMhFW7JnCd+s/oabfryJQb8NotaHtaj+QXVWRK8AYNqWaQxfOJy2X7el8ruVGfTbIGZHzSYuKS7jHjHxMew9vpfo49GuehsiRY+3t7PAAji9a+fPh717nYUPjh07PUo7YYIzASwxEd59FyZOdK6x1ulTO3jw6Xu+8AJUq+asTLZ8eaG/JRHQJC4RySf/N/f/+O/M/wJQuURlrq1xLUNaD6FuaF2stSzbu4zx68czfv14thzZAsBt9W5jbI+xzvc/38b49eMBuKfxPXzV7SuN1IoUNmth9WooVQoi0hYzqVsXmjRx2ncB/PGHM0ns11+d1126OC28rryyYGJasQLuustZxre/FlC5lKgLQRZKYEXyn7WW37f8TtWQqtQpW+ecXQmstazZv4YJ6ydQtlhZBrUaBMDXK79m9b7VrNy3krk75zKk9RCGXzNc3Q1E3NWaNfDGG07P2tRUaNfO6U/r7Q1eXnDddafLEFJSTveszStrnXrchAQICIBRo+Duu/P9bYj7UgKbhRJYEfcVlxRHl++6MHfnXF7v9DrPtHvG1SGJSE42b4b//Q9Gj85cV/vRR05ZQny8M4J7333w1FO53y8uDoYNgyNH4NNPT+9LX/Xs8GFnIpqvb/6/F3ErOSWwmsQlIm4l0DeQKb2m0OW7LtQuU9vV4YhIbmrWhC+/hDffhIMHndHW1FQoX945vnu3MyLrdUZJUProarrkZKced9kyGDoUtm1zkt70xDU9eT1xwilZKFsWxo1z6nrPlJDgLOJQufLpEggpkjQCKyJuKdWmZtTAnvm9iHig9KTW1xf27IGrr3YmgUVFwb//woYNTvIJEBQEr7wCjz56dk/c5GR48EH46ito2tRZuWzPHpg5E2bMgHnznBHfqlVh40aN0no4jcCKiMdJT1h3HdtFtx+78c4179CxakcXRyUiF8Tb+3QN7Lp1sH8/vPaa87pKFbjqKqdH7WWXOV0PKlbM/j4+Ps5ob+XK8NJLEH5GbuPv79Th1q3r9L5V8lqkaQRWRNzaqn2raD+qPfHJ8VQsXpEAnwACfAK4re5t/PdKp+vByJUjOXjqIJVLVKZKySpUKVmFyiUruzhyETmn2FindrZmTaee9UJ8+63TGaFZMycBbtvWmex1piVL4JtvnNZgSmg9jiZxZaEEVsSzzN85n6Gzh3Is/hjxyfHEJ8dzR/07eK2TM4LT9qu2LNy9MNM1DzR7gM9u+MwV4YqIu7jrLhg/3klkL7vs7OPWOr1v03vliltRApuFEliRoiX6eDQ7ju1g17Fd7Dy2kwkbJrBo9yJWPbCKBuUauDo8EXGVpCSn3VezZnDqFDzzjLOQQ3T06a8JCU4ZQ716p7dWrbJPeKVQKYHNQgmsSNF28ORBNhzawBURV7g6FBFxFykpzkhraiqEhTl1thUrQrFiTjnDhg3OBDCAhx+Gjz92bbyiSVwicmkJDQolNCgUgGPxxwj0DcTP28/FUYmIS3l7w86dEBqafT1sSgps3+5MMqtUydmXlOTsq1Ur+3taC5995kwsa9vWWWihRYuCew+SQQmsiBRZ249u56pvruLWurfy9jVvuzocEXG1c3U3ACfBrVHD2dI9+KDTb3bRIqe7wZmOHHGWtp040Zk8tnKl011hrLM8NidPOi3B0q1c6bQM27PH6WXbpEn+va9LkBorikiRFV4inArBFRi+cDh/bPkjz9dtO7qNRp81YsCUAVyKZVYikqZPH7jpJqdbwpnmzYPGjZ3k9fbbnVraZcvghRec49Y6CepDD52+5rXXoG9fePZZZ7R26tRCextFkWpgRaRIi4qJovFnjfH38Wf1g6spH1w+12s6jOrA3zv+BmD4NcN5vM3jBR2miLg7a2HFCvj1V3j5Zaee9sMPnSVyz1xVDOD4cWf0NigIvvjC2TdnjrMMro+Pc83Ro/D1105SK9nSJK4slMCKXFrGrh3L7eNup1PVTvze+3d8vXPuB7kjZgdzdszhtbmvseXIFmb0nUH7yPaFFK2IuKX/+z/4r9N7moYN4ccfzy4ryKuNG+Gaa5ya3HfegSFD8i/OMyUnQ0yMs/SuB1ICm4USWJFLz4ApAxixcgRjbh1Dz8t6nnV85vaZnEo6xQ21bsjYt+7gOlp+2ZIgvyA2PbKJkgElCzNkEXEnmzfDnXc6P/5/882zF004X7t3O0ns+vXw3HMwbNjZI7npYmKcZXN37Tq9paZC6dJQqpSzlS4NwcHOJLLGjZ0R43LlnBXOZs1y7rNtm1MHfLGxFxJ1IRCRS96HXT+kZpma3FL3FsBJTr9f8z2PtHyExXsWc8e4OygVUIptj22jmG8xAOqF1mN099HEJccpeRW51NWsCUuX5t/9wsNh7ly4/npndHfPHhg50kliv/kGXnkFJk92EtAjR5wOB+n8/cHLC+Lizr7vf//rJLDGOAs5lC59+livXk4S+8ADTquwnCa1uTmNwIrIJenpv57mfwv+h5+3HympKZQOLM2fd/1J4/KNz3lNcmoyPl76f7+I5KMTJ+CWW2DxYmek1RinPGHYMBgxAlq3dlYL++03qFzZ2UJDnfMSEpxa2qNHnSQ3NtYpa4iMPPs5KSnw0UfwwQdOEptei/v221C8eGG/6zxRCUEWSmBFJCU1hambpvL2grc5dOoQU3pNoVaZc/R6BHbH7ubmn27msVaP0adhn0KMVESKvMREZ3LYjTc6iWVBSkmBX36Bt96C+fOhWjX49lunNMLNKIHNQgmsiJyv/Sf20+yLZhyJO8KApgNoUqEJTco3oW5o3YxFEqy1RJ+IZkX0ClZEr2DlvpW0qtSKZ9o94+LoRUSysNZZbeypp5wE+rnnYOjQ7Bd5cBElsFkogRWRC7Fo9yK6/9id/Sf3Z+x7vPXjDO8ynJTUFKq+X5VdsbsyXePr5cs/D/1D7bK1CztcEZHcrVsHvXvDqlXOBLDvvjv3ymOFTJO4RETyQevw1ux9Yi9RMVGsjF7Jqn2r6Fi1IwDeXt60j2xPMZ9iNK3QlKYVmhJeIpwtR7YoeRUR91WvnlN/++KLTneF4cPh88+d+tpq1ZyFHD75xNVRnkUjsCIihWDvcSfxbVvZ/erMREQAZ4Wxhg2hRAmnxKBtW2fZ25deckk4KiHIQgmsiBSmVJtKk8+bEBUTxZx75tCofCNXhyQi4vZySmC9CjsYEZFLjZfx4r0u7xGfHM9131/Hjpgdrg5JRMSjuUUCa4wJNMYsM8Z0yOP5EcaYucaYk8aYfcaYV4051/IVIiKu17FqR77p/g3RJ6K59vtr2XR4k6tDEhHxWC5PYI0xxYEJQLPzuOxLYBFQA+gBDADOXhtSRMSN3HHZHbzb5V02HNpA7Y9qszJ6JeC03xIRkbxzhy4Eo4FVQL3zuKY1MNhaGw1EG2PmAXUKIDYRkXw1uPVgGoQ1YOqmqRm1sL9s+oUXZr1A52qdSUhJ4EjcEY7GH6VVpVYMbT8UcCaBlQsqh7eXtyvDFxFxC+6QwD5lrd1qjOl1HtesAh4yxjwL1AWuBt4rgNhERPJdp2qd6FStU8brAycPsPPYTt5e+HbGvgCfAEoHnl7D/JafbuHgqYMMajmI+5rcRwn/EoUas4iIO3GbLgTGmCjgHmvt7DycWw9YBgSm7XrTWpvnpW7UhUBE3E1SShJbj26lhH8JSgWUItA3MONYcmoyr/z9Cp8u+5RDpw5R3K849zW5j9vr307LSi3x8XKHsQgRkfzlEW20zjOBXQRMAT4GagI/AW9Zaz87x/mPA4+nvy5ZsmSlmJiYiw9aRKQQxSXFMebfMby36D3+OfAPvl6+HHvmGIG+gWw9spWtR7fSJrwNxf2LuzpUEZGLVqQSWGNMA2C2tbbMGfseBgZYa5vk5VkagRURT2atZcGuBWw4tIF+TfsB8Mrfr/Di7BcxGOqG1qVZhWY0q9CMKyOupEmFPP3VKCLiVoraUrI+QAljjL+1NiFtXyigmQ0ickkwxnB5lcu5vMrlGftur387xf2KM3/XfJZHL+fbNd/y7Zpv6d+kP192+9KF0YqI5D+3TWCNMcUArLWnshzaCBwBRhtjJgDVgaeAtxERuUTVKVuHOmXrMKTNEAAOnzrMiugVlC1WNuP1XRPv4r1r36NWmVquDFVE5KK5vA9sDj5J2zJJS2hvAsKBr4BngG+B1ws1OhERN1amWBk6V++cUT4wd+dcpm2dRtuv2rJg1wIXRycicnHcpga2MKkGVkQuRb9u+pXbx91Oqk3lh1t+4Oa6N7s6JBGRc8qpBtadR2BFRCQfXV/rev6+529K+Jfg1p9v5YPFH2Qc2xGzg8W7FzN141Qmrp9IYkqiCyMVEcmZRmBFRC4x249u57rvr6NccDlm9J2Bj5cPVd+vSlRMVMY5Xap3YcIdEyjmW8x1gYrIJc0j2mgVJiWwInKpOxJ3BIOhVGApAN5f9D4JKQmEBYUxK2oW36z+hvYR7ZnRd0aOy9fO3TGXsKAwapetXVihi8gloqi10RIRkYt05jK1AI+1fizj+76N+lIqoBTVSlXLMXkF2BW7izsn3Mnah9dqeVsRKTSqgRURkUy8jBfvXfsej7Z6FIDYhFj2n9gPwKmkUzw34zmW712ecWx37G6em/Gcy+IVkUuPElgRETmn5NRkbvjhBq4YeQWjVo2i3sf1eH3e63y09CMA7m92P1dUuYJPln7C/J3zXRytiFwqlMCKiMg5+Xj50POynmw+spl7J9/L0fijfHDtB3x5o7O6l5fx4ssbv8TX25cBUweQkJyQyx1FRC6eElgREcnRwy0e5ufbfmZQy0FsGLiBQa0G4eN1egpF7bK1GXrlUNYfWs/r87SmjIgUPHUhEBGRi5aYksgDvzzA460fp0G5Bq4OR0SKAC1kICIiBcrP24+RN43MNnlNSknKKC2w1vLF8i84Fn8sx/vtP7GfL5Z/wZ7YPQUSr4h4NiWwIiKSb6y1jF83nlGrRnHo1CFen/s61T6oxufLPwdg+rbpPPDLA1R9vyr/N/f/OJ5wPNP1mw9vpue4nlR+tzIP/PIAzb9sztI9S13xVkTEjSmBFRGRfJOQksCTfz3JQ78+ROV3K/PczOdISU0h0CcQgI5VOzLyppGEBITw35n/per7VXnl71eIiY8BwNvLm7HrxnJFxBW82vFVYuJjuHLUlfyx5Q8XvisRcTeqgRURkXz119a/6PpDV1pUbMGjrR7llrq34Oftl+mcpJQkRq8ezatzXmXnsZ18ev2nPNj8QQB2HdtF5ZKVAViyZwkDfxvIpDsmUalEpUJ/LyLiOlpKNgslsCIiBSsuKY5A38Bcz0tMSeTHf3+kbLGydK3ZNdtzrLUYYwBn6doWlVoQ4BOQr/GKiPtRApuFElgREc/zz/5/aP5lcxqWa8idl91Jq/BWNCnfJE+JclbxyfFKgkXcnLoQiIiIx6sXWo/BrQazfO9yHv/zcS7/+nJKvFGCzt92zvM99p/Yzx3j7iDo/4J4+q+ntfCCiIfSCKyIiHiUY/HHWLp3KUv2LGHxnsWU8C/Btzd/C8DYtWMJDQqlQ2SHs677O+pvbvn5Fo7EHaF8cHn2ndhHg7AG/HTbT9QNrVvI70JEcpPTCKxPdjtFRETcVcmAklxd7WqurnZ1pv1JKUk89sdjRJ+I5toa1/J/V/0fTSo0yTheN7Qu5YPL8/kNn9O9Tnden/s67y9+nyC/oMJ+CyJykTQCKyIiRca2o9sYOmsoP/zzAxZL15pd6VK9C4+2ehSAVJuKlzldPRebEEsJ/xIATNk4hbpl61KzTE2XxC4imWkSVxZKYEVEirbV+1bz3Mzn+G3zb9QpW4c1D67B19v3nOfHxMdQ5d0qGGOY3HNytiUIIlK4lMBmoQRWROTSsO7gOiJDIinmWyzXc6dvm86tP99KfHI839/yPbfVu60QIhSRc1EXAhERuSTVC62Xp+QV4OpqVzPnnjmUDizN7WNv55OlnxRwdCJyoZTAioiIpGlUvhEL7ltAjdI1GPjbQJbuWXrWOXFJcczcPpP3Fr3HpfhTTBF3oC4EIiIiZ6haqirz75vP5I2TaVGpBYkpiSzevZhZUbOYFTWLhbsWkpCSQI3SNRjcejAAX634iu//+Z7G5RtzVdWruKHWDa59EyJFnBJYERGRLEKDQunftD8A6w+u58pRVwIQ5BtEh8gOdIzsmKmN1/6T+1m6dymzombx7qJ36dekHx91/UirfYkUEE3iEhERyUGqTeWt+W9xRcQVtKjY4pzdDFJtKpsPb+axPx5j2tZpNK/YnMk9J1OxeMVCjlikaFAXgiyUwIqISEFJSU3hxdkvMnbdWBb3X0xIQIirQxLxSEpgs1ACKyIiBe1k4smMVb4W7lpI6/DWGGNcHJWI59BSsiIiIoUsPXmduX0mV39zNaUCS+Hn7YePlw8+Xj5ULF6R+ffNB+Bo3FH8ffzP2fJr+9Ht3P/L/YQEhDCo5SCuqHKFkmG5pCmBFRERKUAtK7Xk4RYP8++Bf0lOTc7Ygv2CM855d9G7vLXgLTpV7cSNtW6kRukaTNs6jd4NetOofCPCgsJYtncZx+KPMW7dOJpVaMaQ1kO4vf7t2dbkJqUkEZccl7FMrkhRoxICERERF/vx3x/5csWXzNkxh+TU5Iz9r3Z8leevfB5wktLtMdt5f9H7jFw1krjkONpHtGf2PbMBeGb6M6zat4otR7YQFROFt5c30++azhURV7jiLYlcNNXAZqEEVkRE3FFMfAzTtkwjKiaKTtU60bRCU7zM2WsOHYk7whfLvyC8RDh9GvYBoNkXzVh/cD3VS1eneqnq/LXtL8oElmHjIxsJ9A0s7LcictGUwGahBFZERIqamPgYSvqXzKiNHfPPGIL9grmx9o0ujkzkwiiBzUIJrIiIXEqstZr0JR4npwT27J9LiIiISJGx9sBaWnzZgjX712R7/M+tf9L8i+Y8M/2ZTPW3Iu5MCayIiEgRdjLpJKv3r6bnuJ6cTDyZsf9U0in6TOhDl++6sDx6OW/Of5OrRl9F9PFoF0YrkjdKYEVERIqwlpVa8tpVr7H+0HoG/zE4Y3+gTyC7Y3dzfc3r2TxoM4+1eoy5O+fyzsJ3XBesSB6pBlZERKSIS7WpXPf9dfy59U/qhdbj34f+xRjD8YTjBPsFZ9TH/rrpVzpV60SATwBJKUn4ePkUSO1sXFIcB04eICIkIt/vLUWH29fAGmMCjTHLjDEdzvO6EsaY3caYmwsmMhEREc/nZbwY3X00YUFhrDu4jiV7lgBQ3L94pgT1+lrXE+ATAMCQaUO46cebGL9uPFExUeTXgNfxhONc/vXl1PywJqv2rcqXe8qlx+UrcRljigM/A80u4PK3gFXW2on5G5WIiEjRUj64PIv6LeJ44nEalmuY47kpqSkciTvC1E1TmbppKgBlAsvQrGIz3rz6TRqXb3zBccTEx3Aq6RRJqUncN/k+FvdfnO1qYiI5cYcR2NHAKmDn+VxkjGkP9AEGFUBMIiIiRU7VUlVzTV4BvL28+f6W7/nnoX/4utvXDGwxkJplajJnxxz8vP0AZzGFF2a+wPaj288rhsolK7N0wFKea/ccK/et5O0Fb1/Qe5FLm8trYI0x1a21W40xUcA91trZebjGH/gH2ABMBFZaa1fl9ZmqgRURETl/yanJeBkvvIwXXyz/ggd+eQCAztU6079pf7rX6Z6R4Gb1xrw3qFu2LjfVuQmA+OR4mnzehO1Ht7PygZXUDa2b7XXWWlbvX02DsAZ4e3kXzBsTt+TWNbDW2q0XcNmjQA3gMFAb+M0YMzRfAxMREZFMfLx8Mpa27dekH3/0/oNb697KrKhZ3DHuDsLfCefTpZ+edd3IlSN5dsazvDb3NVJSUwAI8Ang625fc3PdmylTrEy2z0tITuCeyffQ5PMm3DflvnyrwxXP5/Ia2Av0EDDCWns/gDFmMjDPGPO5tXZ/1pONMY8Dj6e/LlmyZKEFKiIiUhR5e3nTpUYXutTowv4T+xm9ejSfL/880zmLdi/iSNwRBkwdQETJCKb0mpJpFLVN5Ta0qdwmx2fsP7Gf0oGl+Wb1N7Ss2JKBLQcW2HsSz+HyEoJ051lCkAjcZq2dkvbaD0gAWltrF+d2vUoIRERE8l+qTSUlNQVfb1+2H91O9Q+qY7GUCijF/Pvmn7NMIDk1meELhtOjfg+qlapGVEwUESUjMMZwIvEEiSmJtBrRiqiYKGbdPYt2VdoV8jsTV3DrEoILtBsIPON1eiO586skFxERkXzjZbwyOgoE+wUztP1QWlVqxZReU86ZvAIs2bOEZ2Y8w/1T72fi+onU/6Q+by14K+M+pQNLM/GOifh5+zHwt4Gk2tQc44g+Hk1SSlL+vTFxO247AmuMKQZgrT2VzbmvAt2Bu4A44EMgwFp7ZV6epRFYERER93L/1Pv5csWXAIQWC2XiHRO5vMrlmc6ZtmUa9ULrUblk5bOuT7WpGfW5Xb7rwtI9S7m+1vV0r92dLjW6EOwXXPBvQvKVp47AfpK2Zedl4Ne0bSVggDsLKS4RERHJZ291fovqparTqFwjlg5YelbyCtClRpeM5HXnMaf75qmkU7w0+yVaj2hNcmoyAB0jO1K1VFW+W/Mdt429jbL/K8sNP9zAiugVhfeGpEC5zQhsYdIIrIiIiPtJSE7Az9sv1+Vrp22Zxk0/3kT/pv2ZvHEyu2N3Uy+0Hr/d+Vum5Wl3HtvJlI1TmLRhEsv2LmP5/cupXrp6Qb8NySc5jcAqgRURERGPcvDkQZp/2Zydx3ZSKqAUL3d4mQebP5jjil4nE08S5BcEwPh14ynhX4LO1TsXVshyAZTAZqEEVkRExLNtOLSBSRsmMaDpgHP2kc1OUkoS1T6oxu7Y3fRr0o+3r3mbkICQggtULpgS2CyUwIqIiFy6omKiGDB1ANO3TadS8UpM6jmJ5hWbuzosycJTJ3GJiIiI5LvIkEj+7PMnX974JYfjDtN+VHumbpzq6rDkPCiBFRERkUuOMYb+Tfsz6+5ZBPkGcejUoQJ7VkpqCsv3Ls94PW3LNI7FHyuw510KlMCKiIjIJat1eGs2DdrEvU3uBeBY/LFcF0o4k7WWk4knya4kMzk1me/XfM9ln15G26/bsjt2N3N3zOXa76/lwV8fzPYayRsfVwcgIiIi4krpk7hOJp7k6m+vpnqp6ozqPorElET2xO4hLjmOphWaAvDTvz/x87qfiT4eTfSJaKKPR5OQkkCvy3rxw60/APDq36+y4fAGlu1dxqbDmyjuV5yn2j5FkG8Q7aq0o3ud7vz4749cU+2ajMRZzo8SWBERERGcpXAjSkbw09qfmLB+AkmpznK09ULrsfbhtQBsPLyRSRsmERYURsXiFakfWp/SgaVpV6Vdxn0W7F7AH1v+oIR/CV648gUGtx5M6cDSGce/6vYVy/Yu45HfH6FN5TbUKVuncN9oEaAuBCIiIiJpUm0qr815jTk751CxeEXCi4dTo3SNjJHSuKQ4fL198fHKeQwwPjkeL+OFn7dftsfn7JhDx9EdaRDWgEX9FxHgE5Dv78XTqY1WFkpgRURExNVemv0Sb8x7gz/v+pMrI64s1Gen53+5rXrmSkpgs1ACKyIiIq6WnJrMliNb8lxCkJyazKbDm1h7YC1VS1W94N61q/et5q6JdxHgE8DUXlMpF1zugu5T0HJKYFUDKyIiIuICPl4+Gcnr5sObeX/x+wT7BRPoE0igbyCBPoF0rt6ZOmXrkGpTCX0rlJj4mIzrr4y4kv+0/Q/X1bwOL5N7YylrLR8t+Yin/nqK5NRkUmwKfSf1ZVqfaQX1FguMElgRERERFzoSd4Srv72ancd2nnXspfYv8WKHF/EyXtzd6G4CfAKoF1qPWVGz+H7N99zy8y3sGLyD8sHlc31Ovyn9GLlqJNVLVeeHW39gzf41dIjsUADvqOCphEBERETExU4mnuRYwjHikuKIS44jLimOU0mnKBdc7pwlBrtjd7Ng1wJur387ACNWjGDG9hnUKl2L2mVrU6tMLWqVqUUJ/xIA/Lb5N37890c+7voxxf2LZ7rXjG0zCA0KpWG5hgX7Rs+DamCzUAIrIiIiRYm1lg6jOzBnx5yzji3st5DW4a3PeW1sQiyR70UC8Hvv32kV3qqAojw/SmCzUAIrIiIiRdHJxJNsObKFjYc3sunwJjYe3kjN0jUZ2n5ojtf9ufVPuv/YHS/jxbtd3uWuRne5vLWXEtgslMCKiIiIZDZ/53y6/diNI3FHKBdUjuHXDKd3w94uiyenBDb3KWsiIiIiUuRdXuVyoh6L4p1r3sHP249gv+CMY3ti97gwsrNpBFZEREREMklKScLbyxsv48Wi3Yv4Y8sfvNThpUKNQX1gRURERCTPfL19M7739/bnkZaPuDCasymBFREREZFzalKhiatDOItqYEVERETEoyiBFRERERGPogRWRERERDyKElgRERER8ShKYEVERETEoyiBFRERERGPogRWRERERDyKElgRERER8ShKYEVERETEoyiBFRERERGPogRWRERERDyKElgRERER8ShKYEVERETEoyiBFRERERGPYqy1ro6h0BljEoCDhfjIYOBEIT5PCoY+R8+nz7Bo0OdYNOhzLBoK8nMMtdb6Z3fgkkxgC5sxZre1NtzVccjF0efo+fQZFg36HIsGfY5Fg6s+R5UQiIiIiIhHUQIrIiIiIh5FCWzheMfVAUi+0Ofo+fQZFg36HIsGfY5Fg0s+R9XAioiIiIhH0QisiIiIiHgUJbAiIiIi4lGUwIqIiIiIR1ECW4CMMZWNMX8aY44bY/4wxoS5OibJnTGmizFmgzEm2Riz0Rhzbdp+P2PM58aYGGPMP8aYlq6OVXJnjPnMGDMq7Xt9hh7KGHOLMWa3MSY47bU+Sw9hjOltjNlujIk3xqwxxrRP26/P0M0ZYwKNMcuMMR3O2Jfj52aMedgYs8cYs9cYc19BxaYEtoAYYwwwCUgEmgBrgJGujElyZ4yJBH4E3gDK43yGPxljgoA3gauBq4BXgPHGmBKuiVTyIu0v3fvP2KXP0AMZY0oBHwODrbXpK/7os/QAaX+nfgTcB1QFxgCTjTGB6DN0a8aY4sAEoFmWQ+f83IwxNwHDgUeB64FXjDGtCyQ+dSEoGMaYK4EZQKS1do8xxg+IBppYa3e6Njo5F2PM9UBVa+1Haa9DgQNAa2AWcK+19qe0Y78AP1trv3FVvHJuaf9A/gMkAEuBB4Cj6DP0OMaYr4EK1trr0l77o8/SIxhjbgGesda2THsdDBwH6gAr0WfotowxE4CNwJ3A3dba2bn92TPGzAQ2WmsfSjv2JFDPWpvvI7EagS04jYEN1to9ANbaRGAV0MKFMUkurLW/pievaeoBKUA8EAhMP+PYYvR5urNhwFpgbNrr2ugz9DjGmKuAe4AZxpg7jTFl0WfpSf4BLjPGXJX2k6wncJKiAPQZurunrLXPAmeOdOb2Z69xDsfylRLYglMC2JplXwxQufBDkYswFPgOKA4cs9YePuNYDPo83VJaTdZdOKOu6Uqgz9CjGGO8gPeAfUAZoAuwHiiJPkuPYK3dDPwP5yeSJ4CngD7o71S3Z63NmsNA7n+PZs19zjyWr3wK4qYCQDLOqN2ZTgFBLohFLoAx5kGc/032ASLQ5+kR0sp1vgaGWGv3OeXogP5MeqJ2QAPgcmvtAgBjzOfAi+iz9AjGmEZAP+BynNHYu4HfgFvQZ+iJcvt7NOvxAvtMNQJbcA4B5bLsK4FTjyduzhjTBHgb6Gutjcb5PMsYY7zPOE2fp3t6Hthsrf0+y359hp4nHDianrymWYaTDOmz9Ax9ceojF1hrj6eVaEXhTAzSZ+h5cvt7NGvuU2CfqRLYgrMYaGqMCYCMrgRNgd0ujUpyZYypCEwE3rTW/pq2eztO4XqrM05thj5Pd3QXcE1ai5cY4BmcSQgT0WfoaXYDvln+sYwEZqPP0lP4ABktJNPKQsqk7ddn6Hly+7dwMc5/MLM7lq+UwBYQa+0/wF7gv2m7+uH8IZ7hsqAkV2n/4fgNWA28a4wJTps1a4DxOC1B/I0xlwG3pp0r7uUKoD5O+Udj4DNgCnAd+gw9zULgIPB2Wl/trsDDOBPz9Fl6hjnArcaYJ40xPXHaFFYGpqLP0ONYa1PI+XP7GRiU9ue1JDCYAvpM1UarAKX1PpuCk/yUAh6z1n7s2qgkJ2k97CZlc+he4BfgT6AaUAwYb63tVXjRyYUwxryE087unrQZ7PoMPYgxphbwPs6oTgLwmbX2BX2WnsMY8x+cfsyVgT04s9vH6zP0DMaYKOAea+3stNfn/NzSfto8ErgDpx52F9DOWnsk3+NSAluw0pr7Xg5sSZuNKR7MGOODM7Ek3lq7yNXxyPnTZ1h06LP0fPoMPVNun1va5L1ywBxrbdZJX/kTgxJYEREREfEkqoEVEREREY+iBFZEREREPIoSWBERERHxKEpgRURERMSjKIEVEREREY+iBFZEREREPIoSWBERERHxKP8P9tYbf8v9tY0AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 800x480 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# 展示3种模型的损失下降曲线\n",
    "fig = plt.figure(figsize=(10, 6), dpi=80)\n",
    "# 解决中文显示的问题\n",
    "plt.rcParams['font.sans-serif'] = ['SimHei']\n",
    "plt.rcParams['axes.unicode_minus'] = False\n",
    "plt.rcParams.update({'font.size': 13})\n",
    "plt.ylabel('模型损失', fontsize=18)\n",
    "style = ['k', 'r-.', 'g--']\n",
    "for i, l in enumerate(['mlp', 'mlp_relu', 'mlp_relu_layer_norm']):\n",
    "    _l = torch.tensor(stats[l]).view(-1, 10).mean(1)\n",
    "    plt.plot(_l.numpy(), style[i], label=l)\n",
    "legend = plt.legend(shadow=True)\n",
    "plt.savefig(\"mnist_mlp_loss.png\", dpi=200)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch  0: train loss 0.2021, val loss 0.2017, test loss 0.2014\n",
      "          train acc 0.9420, val acc 0.9372, test acc 0.9430\n",
      "epoch  1: train loss 0.1160, val loss 0.1818, test loss 0.1695\n",
      "          train acc 0.9644, val acc 0.9454, test acc 0.9488\n",
      "epoch  2: train loss 0.1054, val loss 0.1394, test loss 0.1379\n",
      "          train acc 0.9694, val acc 0.9556, test acc 0.9596\n",
      "epoch  3: train loss 0.1051, val loss 0.1511, test loss 0.1387\n",
      "          train acc 0.9658, val acc 0.9534, test acc 0.9584\n",
      "epoch  4: train loss 0.0837, val loss 0.1261, test loss 0.1376\n",
      "          train acc 0.9770, val acc 0.9624, test acc 0.9620\n",
      "epoch  5: train loss 0.0682, val loss 0.1265, test loss 0.1214\n",
      "          train acc 0.9800, val acc 0.9622, test acc 0.9656\n",
      "epoch  6: train loss 0.0871, val loss 0.1341, test loss 0.1344\n",
      "          train acc 0.9722, val acc 0.9602, test acc 0.9588\n",
      "epoch  7: train loss 0.0652, val loss 0.1211, test loss 0.1327\n",
      "          train acc 0.9812, val acc 0.9596, test acc 0.9612\n",
      "epoch  8: train loss 0.0558, val loss 0.1403, test loss 0.1296\n",
      "          train acc 0.9798, val acc 0.9594, test acc 0.9620\n",
      "epoch  9: train loss 0.0461, val loss 0.1123, test loss 0.1212\n",
      "          train acc 0.9868, val acc 0.9670, test acc 0.9676\n"
     ]
    }
   ],
   "source": [
    "# 模型会遭遇比较严重的过拟合问题\n",
    "model2 = nn.Sequential(\n",
    "    nn.Linear(784, 30, bias=False), nn.LayerNorm(30), nn.ReLU(),\n",
    "    nn.Linear( 30, 20, bias=False), nn.LayerNorm(20), nn.ReLU(),\n",
    "    nn.Linear(             20, 10)\n",
    ")\n",
    "\n",
    "_ = train_mlp(model2, optim.Adam(model2.parameters(), lr=0.01), train_loader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([-0.7424,  0.6917, -1.3143,  1.0491,  0.8641]) tensor([-0.0000, 0.0000, -0.0000, 2.0982, 0.0000])\n",
      "tensor([-0.7424,  0.6917, -1.3143,  1.0491,  0.8641]) tensor([-0.7424,  0.6917, -1.3143,  1.0491,  0.8641])\n",
      "tensor([-0.7424,  0.6917, -1.3143,  1.0491,  0.8641]) tensor([-0.0000, 1.3833, -0.0000, 0.0000, 0.0000])\n"
     ]
    }
   ],
   "source": [
    "# 展示模型评估模式和训练模式的差别\n",
    "m = nn.Dropout(0.5)\n",
    "x = torch.randn(5)\n",
    "# 创建之后，模型处于训练模式\n",
    "print(x, m(x))\n",
    "# 模型处于评估模式\n",
    "m.eval()\n",
    "print(x, m(x))\n",
    "# 模型处于训练模式\n",
    "m.train()\n",
    "print(x, m(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch  0: train loss 0.2624, val loss 0.2747, test loss 0.2760\n",
      "          train acc 0.9218, val acc 0.9192, test acc 0.9218\n",
      "epoch  1: train loss 0.2212, val loss 0.2136, test loss 0.2235\n",
      "          train acc 0.9366, val acc 0.9396, test acc 0.9404\n",
      "epoch  2: train loss 0.1815, val loss 0.1791, test loss 0.2103\n",
      "          train acc 0.9482, val acc 0.9474, test acc 0.9390\n",
      "epoch  3: train loss 0.1586, val loss 0.1938, test loss 0.2001\n",
      "          train acc 0.9564, val acc 0.9434, test acc 0.9466\n",
      "epoch  4: train loss 0.1583, val loss 0.1809, test loss 0.1635\n",
      "          train acc 0.9536, val acc 0.9468, test acc 0.9502\n",
      "epoch  5: train loss 0.1400, val loss 0.1830, test loss 0.1716\n",
      "          train acc 0.9602, val acc 0.9460, test acc 0.9550\n",
      "epoch  6: train loss 0.1354, val loss 0.1662, test loss 0.1716\n",
      "          train acc 0.9608, val acc 0.9516, test acc 0.9508\n",
      "epoch  7: train loss 0.1332, val loss 0.1840, test loss 0.1861\n",
      "          train acc 0.9590, val acc 0.9438, test acc 0.9486\n",
      "epoch  8: train loss 0.1373, val loss 0.1424, test loss 0.1560\n",
      "          train acc 0.9596, val acc 0.9588, test acc 0.9526\n",
      "epoch  9: train loss 0.1117, val loss 0.1732, test loss 0.1595\n",
      "          train acc 0.9670, val acc 0.9464, test acc 0.9556\n",
      "epoch 10: train loss 0.1116, val loss 0.1371, test loss 0.1651\n",
      "          train acc 0.9670, val acc 0.9552, test acc 0.9532\n",
      "epoch 11: train loss 0.1279, val loss 0.1457, test loss 0.1572\n",
      "          train acc 0.9610, val acc 0.9580, test acc 0.9566\n",
      "epoch 12: train loss 0.1144, val loss 0.1527, test loss 0.1515\n",
      "          train acc 0.9652, val acc 0.9550, test acc 0.9574\n",
      "epoch 13: train loss 0.1085, val loss 0.1437, test loss 0.1498\n",
      "          train acc 0.9690, val acc 0.9586, test acc 0.9580\n",
      "epoch 14: train loss 0.1001, val loss 0.1362, test loss 0.1349\n",
      "          train acc 0.9680, val acc 0.9586, test acc 0.9600\n",
      "epoch 15: train loss 0.1121, val loss 0.1467, test loss 0.1469\n",
      "          train acc 0.9680, val acc 0.9568, test acc 0.9590\n",
      "epoch 16: train loss 0.0910, val loss 0.1464, test loss 0.1343\n",
      "          train acc 0.9740, val acc 0.9552, test acc 0.9620\n",
      "epoch 17: train loss 0.1044, val loss 0.1427, test loss 0.1461\n",
      "          train acc 0.9704, val acc 0.9568, test acc 0.9592\n",
      "epoch 18: train loss 0.1026, val loss 0.1363, test loss 0.1446\n",
      "          train acc 0.9698, val acc 0.9602, test acc 0.9624\n",
      "epoch 19: train loss 0.0943, val loss 0.1459, test loss 0.1502\n",
      "          train acc 0.9704, val acc 0.9584, test acc 0.9578\n",
      "epoch 20: train loss 0.1010, val loss 0.1588, test loss 0.1324\n",
      "          train acc 0.9720, val acc 0.9514, test acc 0.9626\n",
      "epoch 21: train loss 0.0872, val loss 0.1464, test loss 0.1656\n",
      "          train acc 0.9740, val acc 0.9560, test acc 0.9574\n",
      "epoch 22: train loss 0.0980, val loss 0.1465, test loss 0.1466\n",
      "          train acc 0.9718, val acc 0.9578, test acc 0.9616\n",
      "epoch 23: train loss 0.0872, val loss 0.1393, test loss 0.1491\n",
      "          train acc 0.9730, val acc 0.9608, test acc 0.9588\n",
      "epoch 24: train loss 0.0765, val loss 0.1320, test loss 0.1550\n",
      "          train acc 0.9750, val acc 0.9602, test acc 0.9596\n",
      "epoch 25: train loss 0.0883, val loss 0.1512, test loss 0.1551\n",
      "          train acc 0.9726, val acc 0.9578, test acc 0.9608\n",
      "epoch 26: train loss 0.0854, val loss 0.1431, test loss 0.1439\n",
      "          train acc 0.9726, val acc 0.9612, test acc 0.9602\n",
      "epoch 27: train loss 0.0971, val loss 0.1391, test loss 0.1444\n",
      "          train acc 0.9722, val acc 0.9632, test acc 0.9610\n",
      "epoch 28: train loss 0.0919, val loss 0.1372, test loss 0.1503\n",
      "          train acc 0.9718, val acc 0.9590, test acc 0.9598\n",
      "epoch 29: train loss 0.0959, val loss 0.1511, test loss 0.1547\n",
      "          train acc 0.9688, val acc 0.9558, test acc 0.9586\n",
      "epoch 30: train loss 0.0868, val loss 0.1463, test loss 0.1641\n",
      "          train acc 0.9722, val acc 0.9564, test acc 0.9596\n",
      "epoch 31: train loss 0.1052, val loss 0.1520, test loss 0.1596\n",
      "          train acc 0.9690, val acc 0.9572, test acc 0.9592\n",
      "epoch 32: train loss 0.0793, val loss 0.1439, test loss 0.1480\n",
      "          train acc 0.9750, val acc 0.9574, test acc 0.9624\n",
      "epoch 33: train loss 0.0890, val loss 0.1309, test loss 0.1544\n",
      "          train acc 0.9732, val acc 0.9612, test acc 0.9592\n",
      "epoch 34: train loss 0.0957, val loss 0.1493, test loss 0.1446\n",
      "          train acc 0.9696, val acc 0.9550, test acc 0.9610\n",
      "epoch 35: train loss 0.0800, val loss 0.1468, test loss 0.1516\n",
      "          train acc 0.9756, val acc 0.9586, test acc 0.9606\n",
      "epoch 36: train loss 0.0762, val loss 0.1422, test loss 0.1455\n",
      "          train acc 0.9762, val acc 0.9610, test acc 0.9626\n",
      "epoch 37: train loss 0.0721, val loss 0.1383, test loss 0.1488\n",
      "          train acc 0.9816, val acc 0.9608, test acc 0.9604\n",
      "epoch 38: train loss 0.0758, val loss 0.1478, test loss 0.1715\n",
      "          train acc 0.9784, val acc 0.9560, test acc 0.9564\n",
      "epoch 39: train loss 0.0749, val loss 0.1454, test loss 0.1443\n",
      "          train acc 0.9770, val acc 0.9580, test acc 0.9638\n",
      "epoch 40: train loss 0.0822, val loss 0.1413, test loss 0.1595\n",
      "          train acc 0.9762, val acc 0.9578, test acc 0.9584\n",
      "epoch 41: train loss 0.0744, val loss 0.1474, test loss 0.1552\n",
      "          train acc 0.9756, val acc 0.9606, test acc 0.9602\n",
      "epoch 42: train loss 0.0954, val loss 0.1454, test loss 0.1458\n",
      "          train acc 0.9728, val acc 0.9594, test acc 0.9640\n",
      "epoch 43: train loss 0.0602, val loss 0.1450, test loss 0.1573\n",
      "          train acc 0.9800, val acc 0.9576, test acc 0.9622\n",
      "epoch 44: train loss 0.0716, val loss 0.1474, test loss 0.1482\n",
      "          train acc 0.9774, val acc 0.9590, test acc 0.9602\n",
      "epoch 45: train loss 0.0791, val loss 0.1450, test loss 0.1539\n",
      "          train acc 0.9754, val acc 0.9578, test acc 0.9608\n",
      "epoch 46: train loss 0.0719, val loss 0.1531, test loss 0.1411\n",
      "          train acc 0.9784, val acc 0.9584, test acc 0.9644\n",
      "epoch 47: train loss 0.0720, val loss 0.1354, test loss 0.1408\n",
      "          train acc 0.9790, val acc 0.9586, test acc 0.9652\n",
      "epoch 48: train loss 0.0713, val loss 0.1395, test loss 0.1497\n",
      "          train acc 0.9786, val acc 0.9636, test acc 0.9640\n",
      "epoch 49: train loss 0.0713, val loss 0.1462, test loss 0.1616\n",
      "          train acc 0.9768, val acc 0.9622, test acc 0.9574\n"
     ]
    }
   ],
   "source": [
    "# 加入dropout之后会减轻过拟合的问题\n",
    "model3 = nn.Sequential(\n",
    "    nn.Linear(784, 30, bias=False), nn.LayerNorm(30), nn.ReLU(), nn.Dropout(0.2),\n",
    "    nn.Linear( 30, 20, bias=False), nn.LayerNorm(20), nn.ReLU(), nn.Dropout(0.2),\n",
    "    nn.Linear(             20, 10)\n",
    ")\n",
    "\n",
    "_ = train_mlp(model3, optim.Adam(model3.parameters(), lr=0.01), train_loader, epochs=50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 定义l1和l2惩罚项\n",
    "def l1_loss(model, weight):\n",
    "    w = torch.cat([p.view(-1) for p in model.parameters()])\n",
    "    return weight * torch.abs(w).sum()\n",
    "\n",
    "def l2_loss(model, weight):\n",
    "    w = torch.cat([p.view(-1) for p in model.parameters()])\n",
    "    return weight * torch.square(w).sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch  0: train loss 0.2159, val loss 0.2189, test loss 0.2249\n",
      "          train acc 0.9408, val acc 0.9360, test acc 0.9366\n",
      "epoch  1: train loss 0.1815, val loss 0.1961, test loss 0.1797\n",
      "          train acc 0.9482, val acc 0.9438, test acc 0.9462\n",
      "epoch  2: train loss 0.1400, val loss 0.1786, test loss 0.1547\n",
      "          train acc 0.9626, val acc 0.9452, test acc 0.9560\n",
      "epoch  3: train loss 0.1465, val loss 0.1666, test loss 0.1588\n",
      "          train acc 0.9582, val acc 0.9510, test acc 0.9554\n",
      "epoch  4: train loss 0.1555, val loss 0.1726, test loss 0.1639\n",
      "          train acc 0.9530, val acc 0.9502, test acc 0.9518\n",
      "epoch  5: train loss 0.1397, val loss 0.1606, test loss 0.1441\n",
      "          train acc 0.9588, val acc 0.9490, test acc 0.9558\n",
      "epoch  6: train loss 0.1506, val loss 0.1638, test loss 0.1609\n",
      "          train acc 0.9538, val acc 0.9494, test acc 0.9568\n",
      "epoch  7: train loss 0.1317, val loss 0.1556, test loss 0.1438\n",
      "          train acc 0.9634, val acc 0.9544, test acc 0.9578\n",
      "epoch  8: train loss 0.1663, val loss 0.1790, test loss 0.1916\n",
      "          train acc 0.9554, val acc 0.9472, test acc 0.9446\n",
      "epoch  9: train loss 0.1314, val loss 0.1408, test loss 0.1347\n",
      "          train acc 0.9632, val acc 0.9586, test acc 0.9618\n"
     ]
    }
   ],
   "source": [
    "# 使用l2惩罚性\n",
    "model2 = nn.Sequential(\n",
    "    nn.Linear(784, 30, bias=False), nn.LayerNorm(30), nn.ReLU(),\n",
    "    nn.Linear( 30, 20, bias=False), nn.LayerNorm(20), nn.ReLU(),\n",
    "    nn.Linear(             20, 10)\n",
    ")\n",
    "\n",
    "p2 = lambda m: l2_loss(m, 0.001)\n",
    "_ = train_mlp(model2, optim.Adam(model2.parameters(), lr=0.01), train_loader, penalty=[p2])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}