{ "cells": [ { "cell_type": "code", "execution_count": 1, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "GXlFwZFG03LB", "outputId": "9ff10d1d-1441-4bc3-fb6b-7a2dfcc1e7db" }, "outputs": [ { "data": { "text/plain": [ "" ] }, "execution_count": 1, "metadata": {}, "output_type": "execute_result" } ], "source": [ "import torch\n", "import torch.nn as nn\n", "import torch.nn.functional as F\n", "import torch.optim as optim\n", "from torch.utils.data import DataLoader\n", "from datasets import load_dataset\n", "import matplotlib.pyplot as plt\n", "%matplotlib inline\n", "\n", "\n", "torch.manual_seed(12046)" ] }, { "cell_type": "code", "execution_count": 2, "metadata": { "id": "G37XHrLc03LF" }, "outputs": [], "source": [ "# 一些超参数\n", "learning_rate = 1e-3\n", "eval_iters = 10\n", "batch_size=1000\n", "sequence_len=64\n", "# 如果有GPU，该脚本将使用GPU进行计算\n", "device = 'cuda' if torch.cuda.is_available() else 'cpu'" ] }, { "cell_type": "code", "execution_count": 3, "metadata": { "id": "53cFFO3C03LF" }, "outputs": [], "source": [ "raw_datasets = load_dataset(\"code_search_net\", \"python\")\n", "datasets = raw_datasets['train'].filter(lambda x: 'apache/spark' in x['repository_name'])" ] }, { "cell_type": "code", "execution_count": 4, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "fqu8u80A03LG", "outputId": "e4b19214-fc74-4bea-8ac3-05aca4c23dd3" }, "outputs": [ { "data": { "text/plain": [ "98" ] }, "execution_count": 4, "metadata": {}, "output_type": "execute_result" } ], "source": [ "class char_tokenizer:\n", "\n", " def __init__(self, data):\n", " # 数据中出现的所有字符构成字典\n", " chars = sorted(list(set(''.join(data))))\n", " # 预留一个位置给结尾的特殊字符\n", " self.char2ind = {s : i + 1 for i, s in enumerate(chars)}\n", " self.char2ind['<|e|>'] = 0\n", " self.ind2char = {i : s for s, i in self.char2ind.items()}\n", "\n", " def encode(self, text):\n", " return [self.char2ind[c] for c in text]\n", "\n", " def decode(self, enc):\n", " if isinstance(enc, int):\n", " return self.ind2char[enc]\n", " return [self.ind2char[i] for i in enc]\n", "\n", "tok = char_tokenizer(datasets['whole_func_string'])\n", "len(tok.char2ind)" ] }, { "cell_type": "code", "execution_count": 5, "metadata": { "id": "lEXNR5-S03LH" }, "outputs": [], "source": [ "class RNN(nn.Module):\n", "\n", " def __init__(self, input_size, hidden_size):\n", " '''\n", " 循环神经网络的神经元（支持批量计算）\n", " 参数\n", " ----\n", " input_size ：int，输入数据的特征长度\n", " hidden_size ：int，隐藏状态的特征长度\n", " '''\n", " super().__init__()\n", " self.hidden_size = hidden_size\n", " self.i2h = nn.Linear(input_size + hidden_size, hidden_size)\n", "\n", " def forward(self, x, hidden=None):\n", " '''\n", " 向前传播\n", " 参数\n", " ----\n", " x ：torch.FloatTensor\n", " 输入数据的集合，形状为(B, T, C)，其中B表示批量大小，T表示文本长度，C表示文字特征的长度（input_size）\n", " hidden ：torch.FloatTensor\n", " 初始的隐藏状态，形状为(B, H)，其中H表示隐藏状态的长度（hidden_size）\n", " 返回\n", " ----\n", " hidden ：torch.FloatTensor，所有隐藏状态的集合，形状为(B, T, H)\n", " '''\n", " re = []\n", " B, T, C = x.shape\n", " x = x.transpose(0, 1) # (T, B, C)\n", " if hidden is None:\n", " hidden = self.init_hidden(B, x.device)\n", " for i in range(T):\n", " # x[i]的形状是(B, C); hidden的形状是(B, H)\n", " combined = torch.cat((x[i], hidden), dim=1) # (B, C + H)\n", " hidden = F.relu(self.i2h(combined)) # ( B, H)\n", " re.append(hidden)\n", " result_tensor = torch.stack(re, dim=0) # (T, B, H)\n", " return result_tensor.transpose(0, 1) # (B, T, H)\n", "\n", " def init_hidden(self, B, device):\n", " # 默认的初始隐藏状态全部等于0\n", " return torch.zeros((B, self.hidden_size), device=device)\n", "\n", "class CharRNNBatch(nn.Module):\n", "\n", " def __init__(self, vs):\n", " '''\n", " 双层的循环神经网络（支持批量计算）\n", " 参数\n", " ----\n", " vs ：int，字典大小\n", " '''\n", " super().__init__()\n", " # 定义文字嵌入的特征长度\n", " self.emb_size = 256\n", " # 定义隐藏状态的特征长度\n", " self.hidden_size = 128\n", " # 文字嵌入层\n", " self.embedding = nn.Embedding(vs, self.emb_size)\n", " # 随机失活\n", " self.dp = nn.Dropout(0.4)\n", " # 第一层循环神经网络\n", " self.rnn1 = RNN(self.emb_size, self.hidden_size)\n", " # 第二层循环神经网络\n", " self.rnn2 = RNN(self.hidden_size, self.hidden_size)\n", " # 语言建模头，根据隐藏状态预测下一个字母是什么\n", " self.h2o = nn.Linear(self.hidden_size, vs)\n", "\n", " def forward(self, x):\n", " '''\n", " 向前传播\n", " 参数\n", " ----\n", " x ：torch.LongTensor，当前字母在字典中的位置，形状为(B, T)\n", " 返回\n", " ----\n", " output ：torch.FloatTensor，预测结果的logits，形状为(B, T, vs)\n", " '''\n", " emb = self.embedding(x) # (B, T, C)\n", " h = self.dp(self.rnn1(emb)) # (B, T, H)\n", " # 第一层的隐藏状态是第二层的输入\n", " h = self.dp(self.rnn2(h)) # (B, T, H)\n", " # 使用第二层的隐藏状态预测下一个字母是什么\n", " output = self.h2o(h) # (B, T, vs)\n", " return output\n", "\n", "model = CharRNNBatch(len(tok.char2ind)).to(device)" ] }, { "cell_type": "code", "execution_count": 6, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "vgDVyHDL03LI", "outputId": "f9d86d52-3f06-4503-e3e5-08985129b4c1" }, "outputs": [ { "data": { "text/plain": [ "CharRNNBatch(\n", " (embedding): Embedding(98, 256)\n", " (dp): Dropout(p=0.4, inplace=False)\n", " (rnn1): RNN(\n", " (i2h): Linear(in_features=384, out_features=128, bias=True)\n", " )\n", " (rnn2): RNN(\n", " (i2h): Linear(in_features=256, out_features=128, bias=True)\n", " )\n", " (h2o): Linear(in_features=128, out_features=98, bias=True)\n", ")" ] }, "execution_count": 6, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# 展示模型结构\n", "model" ] }, { "cell_type": "code", "execution_count": 7, "metadata": { "id": "KZiIPgh603LI" }, "outputs": [], "source": [ "@torch.no_grad()\n", "def generate_batch(model, idx, max_new_tokens=300):\n", " '''\n", " 利用模型生成文本（反复使用模型进行预测）\n", " 参数\n", " ----\n", " model ：CharRNNBatch，生成文本的模型\n", " idx ：torch.LongTensor，当前字母在字典中的位置，形状为(1, T)\n", " max_new_tokens ：int，生成文本的最大长度\n", " 返回\n", " ----\n", " out ：list[int]，生成的文本\n", " '''\n", " # 将模型切换至评估模式\n", " model.eval()\n", " for _ in range(max_new_tokens):\n", " # 限制背景长度，使之与模型训练时的状况更相符\n", " # 当然也可以不限制\n", " context = idx[:, -sequence_len:]\n", " # 在文本生成时，模型的计算效率很低，因为有很多重复计算\n", " logits = model(context)\n", " # 只使用最后一个预测结果\n", " logits = logits[:, -1, :]\n", " probs = F.softmax(logits, dim=-1)\n", " # 根据模型预测的概率，得到最终的预测结果（下一个字母）\n", " # 这一步运算有一定随机性\n", " ix = torch.multinomial(probs, num_samples=1)\n", " idx = torch.cat((idx, ix), dim=1)\n", " if ix.item() == 0:\n", " break\n", " # 将模型切换至训练模式\n", " model.train()\n", " return idx.tolist()[0]" ] }, { "cell_type": "code", "execution_count": 8, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "Xc6EEzgL03LI", "outputId": "197a5c39-d194-4d55-f19e-cc9bf3a5301f" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "def*ZO(F/of(\"YP{BZE G|uw=3:1'_$?Q9NN[{KQ=CK(AM:iKcaR;+Q3j\n" ] } ], "source": [ "# 使用模型来生成文本\n", "begin_text = torch.tensor(tok.encode('def'), device=device).unsqueeze(0)\n", "print(''.join(tok.decode(generate_batch(model, begin_text))))" ] }, { "cell_type": "code", "execution_count": 9, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "Vu7WY3Pe03LJ", "outputId": "2e3c0195-ba98-4eff-b9a2-a053ebd3a6ba" }, "outputs": [ { "data": { "text/plain": [ "(torch.Size([605913, 64]), torch.Size([605913, 64]))" ] }, "execution_count": 9, "metadata": {}, "output_type": "execute_result" } ], "source": [ "def process(data, sequence_len=sequence_len):\n", " '''\n", " 根据文本生成训练数据\n", " '''\n", " # text是字符串列表\n", " text = data['whole_func_string']\n", " inputs, labels = [], []\n", " for i in text:\n", " enc = tok.encode(i)\n", " # 0对应着文本结束\n", " enc += [0]\n", " # 将文本转换为多个训练数据\n", " for i in range(len(enc) - sequence_len):\n", " inputs.append(enc[i: i + sequence_len])\n", " # 预测标签是下一个字母，因此只需要挪动一个位置即可\n", " labels.append(enc[i + 1: i + 1 + sequence_len])\n", " return {'inputs': inputs, 'labels': labels}\n", "\n", "# 将数据分为训练集和测试集\n", "tokenized = datasets.train_test_split(test_size=0.1, seed=1024, shuffle=True)\n", "# 将文本转换为训练数据，里面包含inputs和labels\n", "tokenized = tokenized.map(process, batched=True, remove_columns=datasets.column_names)\n", "tokenized.set_format(type='torch', device=device)\n", "\n", "tokenized['train']['inputs'].shape, tokenized['train']['labels'].shape" ] }, { "cell_type": "code", "execution_count": 10, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "QUAlXU5803LJ", "outputId": "3b9e68a6-0eb4-4432-88e7-0cbfdce72c8c" }, "outputs": [ { "data": { "text/plain": [ "{'inputs': tensor([[16, 76, 85, ..., 69, 67, 80],\n", " [ 2, 2, 2, ..., 71, 72, 71],\n", " [89, 37, 81, ..., 70, 40, 75],\n", " ...,\n", " [76, 71, 69, ..., 2, 2, 2],\n", " [ 2, 19, 1, ..., 74, 67, 86],\n", " [48, 81, 84, ..., 78, 2, 70]], device='cuda:0'),\n", " 'labels': tensor([[76, 85, 81, ..., 67, 80, 2],\n", " [ 2, 2, 2, ..., 72, 71, 84],\n", " [37, 81, 79, ..., 40, 75, 71],\n", " ...,\n", " [71, 69, 86, ..., 2, 2, 2],\n", " [19, 1, 2, ..., 67, 86, 2],\n", " [81, 84, 79, ..., 2, 70, 75]], device='cuda:0')}" ] }, "execution_count": 10, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# 构建数据读取器\n", "train_loader = DataLoader(tokenized['train'], batch_size=batch_size, shuffle=True)\n", "test_loader = DataLoader(tokenized['test'], batch_size=batch_size, shuffle=True)\n", "# 获取一个批量的数据\n", "next(iter(test_loader))" ] }, { "cell_type": "code", "execution_count": 11, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "Wlb2hwJ503LK", "outputId": "8cb04947-2c64-49c5-cbb8-6b0a4a30bcfc" }, "outputs": [ { "data": { "text/plain": [ "{'train': 4.572356224060059, 'test': 4.573554515838623}" ] }, "execution_count": 11, "metadata": {}, "output_type": "execute_result" } ], "source": [ "def estimate_loss(model):\n", " re = {}\n", " # 将模型切换至评估模式\n", " model.eval()\n", " re['train'] = _loss(model, train_loader)\n", " re['test'] = _loss(model, test_loader)\n", " # 将模型切换至训练模式\n", " model.train()\n", " return re\n", "\n", "@torch.no_grad()\n", "def _loss(model, data_loader):\n", " \"\"\"\n", " 计算模型在不同数据集下面的评估指标\n", " \"\"\"\n", " loss = []\n", " data_iter= iter(data_loader)\n", " # 随机使用多个批量数据来预估模型效果\n", " for k in range(eval_iters):\n", " data = next(data_iter, None)\n", " if data is None:\n", " data_iter = iter(data_loader)\n", " data = next(data_iter, None)\n", " inputs, labels = data['inputs'], data['labels']\n", " logits = model(inputs)\n", " # 根据cross_entropy的定义，需要对logits进行转置运算\n", " # 具体细节请参考cross_entropy的官方文档\n", " logits = logits.transpose(-2, -1)\n", " loss.append(F.cross_entropy(logits, labels).item())\n", " return torch.tensor(loss).mean().item()\n", "\n", "estimate_loss(model)" ] }, { "cell_type": "code", "execution_count": 12, "metadata": { "id": "gqyC2yaA03LK" }, "outputs": [], "source": [ "def train_rnn(model, optimizer, data_loader, epochs=10):\n", " lossi = []\n", " for epoch in range(epochs):\n", " for i, data in enumerate(data_loader, 0):\n", " inputs, labels = data['inputs'], data['labels']\n", " optimizer.zero_grad()\n", " logits = model(inputs)\n", " # 根据cross_entropy的定义，需要对logits进行转置运算\n", " # 具体细节请参考cross_entropy的官方文档\n", " logits = logits.transpose(-2, -1)\n", " loss = F.cross_entropy(logits, labels)\n", " lossi.append(loss.item())\n", " loss.backward()\n", " optimizer.step()\n", " # 评估模型，并输出结果\n", " stats = estimate_loss(model)\n", " train_loss = f'train loss {stats[\"train\"]:.4f}'\n", " test_loss = f'test loss {stats[\"test\"]:.4f}'\n", " print(f'epoch {epoch:>2}: {train_loss}, {test_loss}')\n", " return lossi" ] }, { "cell_type": "code", "execution_count": 13, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "XHzkjkOd03LK", "outputId": "b2bac2a9-48f5-4ae6-d644-75655eaee19c" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "epoch 0: train loss 1.4990, test loss 1.6117\n", "epoch 1: train loss 1.3739, test loss 1.4928\n", "epoch 2: train loss 1.3112, test loss 1.4462\n", "epoch 3: train loss 1.2822, test loss 1.4168\n", "epoch 4: train loss 1.2611, test loss 1.3976\n", "epoch 5: train loss 1.2440, test loss 1.3893\n", "epoch 6: train loss 1.2383, test loss 1.3751\n", "epoch 7: train loss 1.2313, test loss 1.3710\n", "epoch 8: train loss 1.2095, test loss 1.3622\n", "epoch 9: train loss 1.2253, test loss 1.3593\n" ] } ], "source": [ "l = train_rnn(model, optim.Adam(model.parameters(), lr=learning_rate), train_loader)" ] }, { "cell_type": "code", "execution_count": 14, "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 448 }, "id": "e94Xbj3003LL", "outputId": "743f41aa-a6d1-4a50-d003-1726c5497988" }, "outputs": [ { "data": { "text/plain": [ "[]" ] }, "execution_count": 14, "metadata": {}, "output_type": "execute_result" }, { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAiMAAAGdCAYAAADAAnMpAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA6TElEQVR4nO3de3yU9Z33//dMkpkcZ3IiJ3IABDmHo0DAUwVFZS10u9ZFeqNdtbcubLV317ul3a22to2/dbW1hxurVrFaitUKWgoiC4IiAQGDcpBwJiHkQBKSyXGSzFy/P5JMCCSQQDJXYF7Px2MeMddck/nM90GTd7/fz/W9LIZhGAIAADCJ1ewCAABAYCOMAAAAUxFGAACAqQgjAADAVIQRAABgKsIIAAAwFWEEAACYijACAABMFWx2Ad3h9Xp16tQpRUVFyWKxmF0OAADoBsMwVF1drZSUFFmtXc9/XBFh5NSpU0pLSzO7DAAAcAkKCgqUmpra5fNXRBiJioqS1PJhHA6HydUAAIDucLlcSktL8/0d78oVEUbalmYcDgdhBACAK8zFWixoYAUAAKYijAAAAFMRRgAAgKkIIwAAwFSEEQAAYCrCCAAAMBVhBAAAmIowAgAATEUYAQAApiKMAAAAUxFGAACAqQgjAADAVFfEjfL6yh+2HFNBRZ3+eUqaRiRxAz4AAMwQ0DMjq784pWVbjyu/vM7sUgAACFiXFUaefvppWSwWPfbYYxc876233tKIESMUGhqqsWPHas2aNZfztr3G2npLY8PkOgAACGSXHEZ27Nih3//+98rMzLzgeVu3btX8+fP1wAMPKDc3V/PmzdO8efO0d+/eS33rXmNtySIyDOIIAABmuaQwUlNTowULFuill15STEzMBc99/vnndfvtt+vxxx/XyJEj9dRTT2nixIn67W9/e0kF9yZL68yIlywCAIBpLimMLFq0SHPmzNGsWbMuem5OTs55582ePVs5OTldvsbtdsvlcnV49IXWiRF5mRkBAMA0Pb6aZsWKFfrss8+0Y8eObp1fXFysxMTEDscSExNVXFzc5Wuys7P1k5/8pKel9ZivZ4QsAgCAaXo0M1JQUKBHH31Uf/rTnxQaGtpXNWnJkiWqqqryPQoKCvrkfaytn56ZEQAAzNOjmZFdu3aptLRUEydO9B3zeDz66KOP9Nvf/lZut1tBQUEdXpOUlKSSkpIOx0pKSpSUlNTl+9jtdtnt9p6UdkmYGQEAwHw9mhmZOXOm9uzZo927d/sekydP1oIFC7R79+7zgogkZWVlacOGDR2OrV+/XllZWZdXeS9ob2AljQAAYJYezYxERUVpzJgxHY5FREQoLi7Od3zhwoUaOHCgsrOzJUmPPvqobrrpJj377LOaM2eOVqxYoZ07d+rFF1/spY9w6dobWE0tAwCAgNbrO7Dm5+erqKjI9/306dO1fPlyvfjiixo3bpzefvttrVq16rxQYwb2GQEAwHyXfW+aTZs2XfB7Sbr77rt19913X+5b9Tp6RgAAMF9A35uGnhEAAMwX4GGk5Ss9IwAAmCegw4ivZ4Rb5QEAYJoADyPcmwYAALMRRsTVNAAAmCmgw4ivZ4SpEQAATBPgYYRlGgAAzBbQYaS9gRUAAJglwMMIPSMAAJgtoMNI+z4jhBEAAMwS2GFE9IwAAGC2gA4j7TfKM7cOAAACWYCHEe5NAwCA2QI7jLR+ehpYAQAwT0CHEfYZAQDAfIEdRlq/skwDAIB5AjqMtO8zYnIhAAAEsAAPIy1f6RkBAMA8AR1G6BkBAMB8AR5GWr7SMwIAgHkCOoxYmRkBAMB0AR5GWr4a3LcXAADTBHgY4WoaAADMFtBhxNfAyjoNAACmCfAw0vKVLAIAgHkCOozQMwIAgPkCPIzQMwIAgNkCOoy0b3pGGgEAwCwBHUasbHoGAIDpAjqMWMSmZwAAmC2gw0j7jfLMrQMAgEAW2GHE2tbAShoBAMAsAR1GuFEeAADmC+wwQs8IAACmC+gwQs8IAADmC/AwQs8IAABmC+gwQs8IAADmC+gwYrXQMwIAgNkCOowwMwIAgPkCOoz4ekZMrgMAgEAW4GGk5SsNrAAAmCegw4jvrr1ekwsBACCABXgYaflKzwgAAOYJ6DDC1TQAAJgvwMNI23+RRgAAMEtAhxELMyMAAJguoMNI+zINaQQAALMEdBhpW6VhZgQAAPMEdBixtn569hkBAMA8gR1GfHftNbkQAAACWECHEQs9IwAAmC6gw4iVTc8AADBdQIcRi7i0FwAAs/UojCxdulSZmZlyOBxyOBzKysrS2rVruzx/2bJlslgsHR6hoaGXXXRv8W16RhgBAMA0wT05OTU1VU8//bSGDRsmwzD02muvae7cucrNzdXo0aM7fY3D4VBeXp7v+7Y+jf6AnhEAAMzXozBy1113dfj+5z//uZYuXapt27Z1GUYsFouSkpIuvcI+RM8IAADmu+SeEY/HoxUrVqi2tlZZWVldnldTU6OMjAylpaVp7ty52rdv30V/ttvtlsvl6vDoC2wHDwCA+XocRvbs2aPIyEjZ7XY9/PDDWrlypUaNGtXpucOHD9crr7yid999V2+88Ya8Xq+mT5+ukydPXvA9srOz5XQ6fY+0tLSeltktbTMjZBEAAMxjMXq4/WhjY6Py8/NVVVWlt99+Wy+//LI2b97cZSA5W1NTk0aOHKn58+frqaee6vI8t9stt9vt+97lciktLU1VVVVyOBw9KfeCPjxQqm8t26HMVKfeW3x9r/1cAADQ8vfb6XRe9O93j3pGJMlms2no0KGSpEmTJmnHjh16/vnn9fvf//6irw0JCdGECRN0+PDhC55nt9tlt9t7WlqPWegZAQDAdJe9z4jX6+0wi3EhHo9He/bsUXJy8uW+ba/w3bXXa3IhAAAEsB7NjCxZskR33HGH0tPTVV1dreXLl2vTpk1at26dJGnhwoUaOHCgsrOzJUk//elPNW3aNA0dOlSVlZV65plndOLECT344IO9/0kuATMjAACYr0dhpLS0VAsXLlRRUZGcTqcyMzO1bt063XrrrZKk/Px8Wa3tky1nzpzRQw89pOLiYsXExGjSpEnaunVrt/pL/MHaj/Y8AQAgUPW4gdUM3W2A6amtR8p070vbdW1ipD747k299nMBAED3/34H9L1prOwzAgCA6QI6jLQt0tAzAgCAeQI6jFjZ9QwAANMFdhjhahoAAEwX0GGEe9MAAGC+gA4j7Q2spBEAAMwS0GGkrYGVLAIAgHkCOoy0zYxcAVutAABw1QroMNK+Hby5dQAAEMgCOozQMwIAgPkCOowwMwIAgPkCOozQMwIAgPkCPIy0fCWKAABgnoAOIxZ6RgAAMF1AhxHfdvA0jQAAYJqADiMWX8+IyYUAABDAAjqM0DMCAID5AjyM0DMCAIDZAjqMtO8zQhgBAMAsAR1G2mdGTC4EAIAAFtBhpG1mhE3PAAAwT0CHEStX0wAAYLqADiP0jAAAYL6ADiP0jAAAYL6ADiOWs/6bvhEAAMwR0GGkbWZEom8EAACzEEZa0TcCAIA5AjqMWM769PSNAABgjoAOI8yMAABgvoAOIx0bWE0rAwCAgBbQYaRDAyv37gUAwBQBHUbOyiLy0DQCAIApAjqMhAS1f3zCCAAA5gjoMBJktcjaOjvS6PGaWwwAAAEqoMOI1D470uRhZgQAADMEfBixtYaRxmZmRgAAMEPAh5GQ4LaZEcIIAABmCPgwwswIAADmCvgwEhLc0sHKzAgAAOYgjNDACgCAqQI+jNiC6BkBAMBMAR9G2mZG2GcEAABzEEaCWnpGaGAFAMAchBGWaQAAMFXAhxEb+4wAAGAqwkjbzEgzV9MAAGCGgA8jNLACAGAuwgjLNAAAmIowwtU0AACYKuDDCJueAQBgroAPI+09IzSwAgBgBsIIMyMAAJgq4MOIb58RekYAADBFj8LI0qVLlZmZKYfDIYfDoaysLK1du/aCr3nrrbc0YsQIhYaGauzYsVqzZs1lFdzbbK0NrMyMAABgjh6FkdTUVD399NPatWuXdu7cqVtuuUVz587Vvn37Oj1/69atmj9/vh544AHl5uZq3rx5mjdvnvbu3dsrxfcGekYAADCXxTCMy/orHBsbq2eeeUYPPPDAec/dc889qq2t1erVq33Hpk2bpvHjx+uFF17o9nu4XC45nU5VVVXJ4XBcTrnneWHzET299oC+PjFVz35jXK/+bAAAAll3/35fcs+Ix+PRihUrVFtbq6ysrE7PycnJ0axZszocmz17tnJyci71bXsdDawAAJgruKcv2LNnj7KystTQ0KDIyEitXLlSo0aN6vTc4uJiJSYmdjiWmJio4uLiC76H2+2W2+32fe9yuXpaZrfRMwIAgLl6PDMyfPhw7d69W9u3b9cjjzyi++67T/v37+/VorKzs+V0On2PtLS0Xv35Z2NmBAAAc/U4jNhsNg0dOlSTJk1Sdna2xo0bp+eff77Tc5OSklRSUtLhWElJiZKSki74HkuWLFFVVZXvUVBQ0NMyu63t0l4aWAEAMMdl7zPi9Xo7LKmcLSsrSxs2bOhwbP369V32mLSx2+2+y4fbHn3FNzPCPiMAAJiiRz0jS5Ys0R133KH09HRVV1dr+fLl2rRpk9atWydJWrhwoQYOHKjs7GxJ0qOPPqqbbrpJzz77rObMmaMVK1Zo586devHFF3v/k1wilmkAADBXj8JIaWmpFi5cqKKiIjmdTmVmZmrdunW69dZbJUn5+fmyWtsnW6ZPn67ly5frP/7jP/TDH/5Qw4YN06pVqzRmzJje/RSXwRbcetdewggAAKboURj5wx/+cMHnN23adN6xu+++W3fffXePivKn0OAgSZK7iTACAIAZAv7eNKG2ljBS3+QxuRIAAAJTwIeRsJCWMFLXSBgBAMAMhJHWMNLAzAgAAKYgjJy1THOZt+kBAACXIODDSGjrzIjHa6iJjc8AAPC7gA8jbcs0Ek2sAACYIeDDSEiQRUHWlr1G6BsBAMD/Aj6MWCwW3+xIPVfUAADgdwEfRqT2vhGWaQAA8D/CiKRwNj4DAMA0hBGJZRoAAExEGNFZW8ITRgAA8DvCiKSwkJZhYJkGAAD/I4zorGUawggAAH5HGFH7lvDsMwIAgP8RRnTWpb30jAAA4HeEEbFMAwCAmQgjIowAAGAmwojaNz1rYJkGAAC/I4yofZ+ROsIIAAB+RxgRyzQAAJiJMKL2MMKlvQAA+B9hRO37jDAzAgCA/xFGxD4jAACYiTCis3tGvCZXAgBA4CGMiO3gAQAwE2FEZ82MsEwDAIDfEUbU3jNS19hsciUAAAQewojO2oGVnhEAAPyOMKL2ZZpGj1fNHgIJAAD+RBhRewOrJDU0E0YAAPAnwogke3D7MNDECgCAfxFGJFksFraEBwDAJISRVmwJDwCAOQgjrdhrBAAAcxBGWoWGtAwFMyMAAPgXYaRVuC1YEhufAQDgb4SRVpH2ljBS3UAYAQDAnwgjrRxhLWHERRgBAMCvCCOtokJDJEnVDU0mVwIAQGAhjLRytIYRVz0zIwAA+BNhpFVUaFvPCDMjAAD4E2GklSOsdWaEnhEAAPyKMNKKmREAAMxBGGnV3jNCGAEAwJ8II60coewzAgCAGQgjrdp7RpgZAQDAnwgjraKYGQEAwBSEkVZtPSN1jR41ebwmVwMAQOAgjLRyhIXIYmn578o6lmoAAPAXwkirIKtFMeE2SVJFbaPJ1QAAEDgII2eJjWgJI+U1bpMrAQAgcBBGzuILI8yMAADgN4SRs8RFsEwDAIC/9SiMZGdn67rrrlNUVJQSEhI0b9485eXlXfA1y5Ytk8Vi6fAIDQ29rKL7CjMjAAD4X4/CyObNm7Vo0SJt27ZN69evV1NTk2677TbV1tZe8HUOh0NFRUW+x4kTJy6r6L4SF2mXRM8IAAD+FNyTk99///0O3y9btkwJCQnatWuXbrzxxi5fZ7FYlJSUdGkV+hHLNAAA+N9l9YxUVVVJkmJjYy94Xk1NjTIyMpSWlqa5c+dq3759Fzzf7XbL5XJ1ePgDyzQAAPjfJYcRr9erxx57TDNmzNCYMWO6PG/48OF65ZVX9O677+qNN96Q1+vV9OnTdfLkyS5fk52dLafT6XukpaVdapk9wswIAAD+ZzEMw7iUFz7yyCNau3attmzZotTU1G6/rqmpSSNHjtT8+fP11FNPdXqO2+2W293et+FyuZSWlqaqqio5HI5LKbdb8oqrNftXHykmPES5P76tz94HAIBA4HK55HQ6L/r3u0c9I20WL16s1atX66OPPupREJGkkJAQTZgwQYcPH+7yHLvdLrvdfimlXZa2ZZrK+iZ5vIaCrBa/1wAAQKDp0TKNYRhavHixVq5cqY0bN2rw4ME9fkOPx6M9e/YoOTm5x6/tazHhLTfLMwzpTB1LNQAA+EOPwsiiRYv0xhtvaPny5YqKilJxcbGKi4tVX1/vO2fhwoVasmSJ7/uf/vSn+uCDD3T06FF99tln+uY3v6kTJ07owQcf7L1P0UuCg6yKbg0k9I0AAOAfPVqmWbp0qSTp5ptv7nD81Vdf1f333y9Jys/Pl9XannHOnDmjhx56SMXFxYqJidGkSZO0detWjRo16vIq7yNxETZV1jWpvKZRSjS7GgAArn49CiPd6XXdtGlTh+9/+ctf6pe//GWPijJTXIRdR07XqryWjc8AAPAH7k1zjlgu7wUAwK8II+eIjWwJI2U1hBEAAPyBMHKOAa33pzld3WByJQAABAbCyDmSnC13FC6uIowAAOAPhJFzJDlaw4iLBlYAAPyBMHKOxNYwUuJiZgQAAH8gjJyjbZmmorZR7maPydUAAHD1I4ycIyY8RLbglmEpZakGAIA+Rxg5h8ViUaKj5YqaYpZqAADoc4SRTgyMDpMknTxTZ3IlAABc/QgjnciIjZAknSgnjAAA0NcII51IjwuXRBgBAMAfCCOdGBTXNjNSa3IlAABc/QgjnchgZgQAAL8hjHSibZmmvLZRNe5mk6sBAODqRhjphCM0RLERLXfvZakGAIC+RRjpQnosSzUAAPgDYaQL9I0AAOAfhJEuZLReUZNfwTINAAB9iTDShYzWZZrjZcyMAADQlwgjXWhbpsmvIIwAANCXCCNdaFumOVVVL3ezx+RqAAC4ehFGuhAfaVO4LUiGIRVU1JtdDgAAVy3CSBcsFgtNrAAA+AFh5AJoYgUAoO8RRi5g8ICWmZHDp2tMrgQAgKsXYeQCRiU7JEn7T7lMrgQAgKsXYeQCRqW0hJEDxS55vIbJ1QAAcHUijFzAoLgIhYUEqaHJq2NlNLECANAXCCMXEGS1aERylCRpfxFLNQAA9AXCyEXQNwIAQN8ijFxEW98IMyMAAPQNwshFMDMCAEDfIoxcxIgkh6wWqazGrdLqBrPLAQDgqkMYuYgwW5CGDIiUJO1jdgQAgF5HGOkGlmoAAOg7hJFuoIkVAIC+QxjphraZkS+ZGQEAoNcRRrphZGsYOVZeq1p3s8nVAABwdSGMdMOAKLsSouwyDOlAcbXZ5QAAcFUhjHQTfSMAAPQNwkg3cUUNAAB9gzDSTcyMAADQNwgj3dQ2M3KgyKVmj9fkagAAuHoQRropIy5C4bYguZu9OlZWa3Y5AABcNQgj3RRktfhmR744WWVyNQAAXD0IIz0wPi1akrS7oNLUOgAAuJoQRnpgHGEEAIBeRxjpgQnp0ZJarqg5U9tobjEAAFwlCCM9kBoTrlHJDnm8htbuLTa7HAAArgqEkR766vgUSdLavUUmVwIAwNWBMNJDt4xIkCTtOF4hd7PH5GoAALjyEUZ6aFhCpOIjbWpo8mp3fqXZ5QAAcMXrURjJzs7Wddddp6ioKCUkJGjevHnKy8u76OveeustjRgxQqGhoRo7dqzWrFlzyQWbzWKxKOuaeEnS1iPlJlcDAMCVr0dhZPPmzVq0aJG2bdum9evXq6mpSbfddptqa7vekXTr1q2aP3++HnjgAeXm5mrevHmaN2+e9u7de9nFm2X6NXGSpBzCCAAAl81iGIZxqS8+ffq0EhIStHnzZt14442dnnPPPfeotrZWq1ev9h2bNm2axo8frxdeeKFb7+NyueR0OlVVVSWHw3Gp5faaE+W1uumZTQoJsujzJ25TuC3Y7JIAAOh3uvv3+7J6RqqqWrZFj42N7fKcnJwczZo1q8Ox2bNnKycnp8vXuN1uuVyuDo/+JD02XAOjw9TkMfTxoTKzywEA4Ip2yWHE6/Xqscce04wZMzRmzJguzysuLlZiYmKHY4mJiSou7nqfjuzsbDmdTt8jLS3tUsvsExaLRXeOTZIkvff5KZOrAQDgynbJYWTRokXau3evVqxY0Zv1SJKWLFmiqqoq36OgoKDX3+NyzR0/UJL0/t5i7TheYXI1AABcuS4pjCxevFirV6/Whx9+qNTU1Auem5SUpJKSkg7HSkpKlJSU1OVr7Ha7HA5Hh0d/M2agU3PHp8jjNfTL9QfNLgcAgCtWj8KIYRhavHixVq5cqY0bN2rw4MEXfU1WVpY2bNjQ4dj69euVlZXVs0r7oe/MHCZJ2nnijBqa2AANAIBL0aMwsmjRIr3xxhtavny5oqKiVFxcrOLiYtXX1/vOWbhwoZYsWeL7/tFHH9X777+vZ599VgcOHNCTTz6pnTt3avHixb33KUwyJD5CSY5QNTZ7tfP4GbPLAQDgitSjMLJ06VJVVVXp5ptvVnJysu/x5ptv+s7Jz89XUVH7fVumT5+u5cuX68UXX9S4ceP09ttva9WqVRdser1SWCwWzRjasgHaxgOlJlcDAMCV6bL2GfGX/rbPyNk+2Fesb7++S8nOUH3y/VtktVrMLgkAgH7BL/uMQLrx2gGKsAWpqKpBHx9mzxEAAHqKMHKZQkOC9M9T0iVJz36QpytgogkAgH6FMNILHrn5GoXbgvTFySp9sL/k4i8AAAA+hJFeEB9p17dmDJIkvbD5iLnFAABwhSGM9JL7pg9SsNWi3PxKLXnnC5ZrAADoJsJIL0mICtXsMS27yv750wJ9ll9pbkEAAFwhCCO96Bfzxiq49dLeXSe4Xw0AAN1BGOlFzvAQPT57uCTpo4Nc5gsAQHcQRnrZ5EGxkqQth8v06ifHTK4GAID+jzDSyyamR2ve+BRJ0k/+tl8f7Cs2uSIAAPo3wkgvs1gs+u+7xynJESpJ+vbru3T0dI3JVQEA0H8RRvpAcJBVS+4c4ft+5wnu6AsAQFcII31k7viB+vaNQyRJX5ysNLcYAAD6McJIH8pMdUqSNn5Zqk+PcakvAACdIYz0oawhcYqNsOlUVYO+8fscLXh5m3YXVJpdFgAA/QphpA/FRdr17qIZ+uq4lqtrPjlcrv/18na5GppMrgwAgP6DMNLH0mLD9ew3xmnakJb9R6rdzXrotZ3s0AoAQCvCiB+EBFm1/MFp+s38CZKk7ccq9PWlOcorrja5MgAAzEcY8ROr1aK7xqXo4Zuu8R1jh1YAAAgjfveDO0borYezJEkrcwtVUdtockUAAJiLMGKCyRkxGjvQKXezV5N+tl4/W71fXq9hdlkAAJiCMGICi8Xi2xDNMKSXtxzTjuM0tAIAAhNhxCT/kJmsKa13+JWkpZuP6C87C7jsFwAQcCyGYfT79QGXyyWn06mqqio5HA6zy+k1roYm/eHjY3p+wyHfsQFRdr32rSkalXL1fE4AQGDq7t9vZkZM5AgN0aMzh2lyRozv2Olqtx57M1dXQEYEAKBXEEZMZrVa9Pz8Cbp3arqWPzRVtmCrDpbUaN8pl+oam/XWzgLVupvNLhMAgD7DMk0/869/2qU1e4o7HPvHiQP13DfGm1MQAACXiGWaK9RDNwxRsNXS4dg7nxVq36kqkyoCAKBvEUb6mQnpMVr6zUmaNiRW1wyI8B1f8PJ2vbD5iBqbvSZWBwBA72OZpp+rqmvSvS9v075TLknS6BSHhiVE6vYxybp9TJLJ1QEA0DWWaa4SzvAQvbtohn7xtbGSpH2nXFq1+5S+++ZuFVbWm1wdAACXjzByBQgOsureqen6zi1DlRBllyTVN3m04KVtys0/o9dzjuuPOce5HBgAcEVimeYKdKysVgte2qZTVQ0djv/u3omak5lsUlUAAHTEMs1VbHB8hH5z7wTf98nOUEnSk3/bpyOna8wqCwCAS8LMyBXsk8NlskjKiI/QzGc3qaGp5Uqb2AibBsWF6/u3j9DUIXHmFgkACFjd/ftNGLlK7DtVpe//9QvtLXR1OJ4aE6bxadH63m3DNTg+ootXAwDQ+wgjAai+0aOco2WKi7Dryb/tU25+ZYfnhwyI0D9fl6avjhuopNalHQAA+gphJMBV1TVpycovtP1ohWrczXKftVnakPgI/eH+6xQaYpUtyKq4SLuJlQIArlaEEfjUupt1/6ufasfxM50+/8d/maIbrx3g56oAAFc7wgg6OF5Wq1t/uVlNHkNR9mBVn3Mn4ESHXY/NulZew1CKM0yJjlDVNTZr8qBYkyoGAFzpCCM4z/5TLkWFBssRFqLqhiZV1Dbqq7/95IKv+esjWZqUQSABAPRcd/9+B/uxJphsVEr7PwRnWIhSY8I1d3yKdh4/0+XW8t9983NNyohRsNWixbcMVUYcV+QAAHoXMyOQJO0trNKq3EL985R0hYZY9fGhMv33ujyV1zZ2OG/e+BT9++zh2nn8jLYcLtOP7xolR2iISVUDAPozlmlw2Y6ertFz6w9q/ymXTlbWq/GsK3LOdl9Whkpcbg2IsuuxWcMUEmwloAAACCPofZ8cLtODr+1UfZPnoudOGRSru8Yl658mpSnMFuSH6gAA/Q1hBH3i5Jk61bibVev26PkNh/TRwdOSpHGpTu0vcqnJc/4/pxuGxevrE1MVF2nT9UPj5W72qr7Ro5gIm7/LBwD4EWEEfnOqsl7JzlB9eqxC/2/TEd0xJknLth7XgeLq884dmexQUVW9ahqadffkNNmCLPqyqFppseH69o1DNDwpyoRPAADoC4QRmMowDM359RbtL2q5V05ne5t0JsIWpAeuH6zbRifpdLVbWdfE6ejpWl2bGKndBZXKTI2WLZibTQPAlYAwAtOV1bhVXNWgZGeonGEh+rKoWmv2FmlIfIR2Hj+jN3cWSJJuHj5AxVUNnc6kdOaJu0bp/umDZLFY+rJ8AMBlIoyg3zt6ukarcgv1wPVD5DEMbT5YqoraJj2z7oAamjq/cqfNtCGxiou0K2tInAbFRSgyNFhRocFKcoQq3BakTw6X69qkSCVEcUNAADALYQRXrMq6Rh05Xav1+0v0wuYjkqR/vfkaDUuM1P/5y+e60L/Y9NhwTR0cq7d2nZQk/dstQxUTbtNtoxMVF2FXiatBh0prdMOweIWGcJUPAPQlwgiueB5vy2zJ5EGxvn1LvF5D2Wu/1EsfH5MkxUfaVeNuuuhMSpDVIq9h+ILM6BSHHrn5Gh0prVWxq16OsBA1NHr0wf4SjUp26HcLJqrW3cwdjQHgMhBGcFVr9ngVHNTeyFpZ16itR8r13x/kqbHZq1kjE7XvVJVSosN0qKTG10jbExaLNDE9RlMHx8oRFqK0mHDFRth03aAY33u7GpoUFhKkkCCaagHgXIQRoFWzx6vtxyrk8RqaMjhWpS63/v2tz3W0rEZ1jR7VNV58E7ezDYmPUHR4iPKKq1Xb6FGKM1T/PCVdA6LsmnFNvAZE2ZVbcEaJjlBdMyBSklRe45YzLESv5ZzQuFRnj+6GXN/oYeM4AFekPgsjH330kZ555hnt2rVLRUVFWrlypebNm9fl+Zs2bdJXvvKV844XFRUpKSmpW+9JGEFfOnmmTv/nL5/rjjFJumHYAG34skS1jR5lxIbrTF2jXtlyTKeqGnr8c4OtFs2fkq7S6gat21fiO24Ptur1B6bqdLVbwxIj9ZcdBQoNCdL3brv2vCuEfr/5iLLXHtDv7p2oOZnJl/1ZAcCf+iyMrF27Vp988okmTZqkf/zHf+x2GMnLy+tQSEJCgqzW7k1tE0ZgpoYmj2rczTpRXqvocJv2Flapqr5JaTHhrbMglcrNP6NNeadVcc6NBXvq3qnpGhIfIVdDs/afqtL/fFnqe+7zH98mZ3iIquqatP1YuVwNzbp1VKIOFLmUmRqtMFuQiqrq9fGhMhVU1Om7s66V1dox3DR5vAq2WrgsGoBf+GWZxmKxdDuMnDlzRtHR0Zf0PoQRXAly88/o3/6cq5Nn6uUIDdb1w+L1yeFyjUp2KDUmTJX1TYoJD9Hh0hp9ll/Zq+89dqBTc8en6Gd//9J37Pqh8frmtHS56pu1anehBkTZtfngaY1LjdbL903WJ4fLNDg+QhlxEd1+H3ezR/ZglowAdE93/34H+6ug8ePHy+12a8yYMXryySc1Y8aMLs91u91yu92+712unjcfAv42IT1GW75/i6obmtTkMRTbxb13mj1e7TpxRinRYSqqalBUaLAe+uNOuVsbb2vdzdpTWKVjZbWKj7Tr6xMH6r3PT6norKWihCi7ztQ1+u4FtKewSnsKqzq8z5bDZdpyuOy899988LSG/WitJCktNkyv3Hed/vxpgb4scinCHqxJGTF6+KYhKnY1KCTIqh3HKrTtaLkSnaH6r/fzJLXs8/J/bx+hiekxvTJ2l6KitlEnyms1wcQaAPSOPp8ZycvL06ZNmzR58mS53W69/PLLev3117V9+3ZNnDix09c8+eST+slPfnLecWZGEEiq6ppktUpRoSHyeA3tLqjUU6v363S1W39+aJriIm36vKBShZX1+u2Hh3WivE6SFGkPVk0nW+/HRtjk8Rqqqm/qtRoHRocpwWGX15D+17QMzRyRoF/+z0F9eqxCN147QGXVbkWH25QSHarocJvuHJukcFuwjNbLrM9dRsovb7kRY1ykTYdLa5Q1JO68c4qq6uU1pH//y+fKOVquV++/TiFBVk0dEtvpVU2rcgtlD7bqjrH03AD+1m+WaTpz0003KT09Xa+//nqnz3c2M5KWlkYYAS7A3eyRLcgqi8Wiv+woUEp0mKwWqaS6QfPGD/T1iVTWNeqD/SX6xZovVVnXJItFumHYAO0trDqv58Vi0XmbzH1jcqq2Ha1QfkVdj2sMCbJoQlqMvix2KdwWpDljU2So5Q1Gpzj1s7/vV2Vde1iaNz5F37guTZmp0Yq0B6vU1aBbf/lRp4Hq324ZqmlD4pTkDNX6/SWqb/To5Jl6/fWzlg3w/mPOSD14w5Bu1WkYBn01QC/o12Hk8ccf15YtW5STk9Ot8+kZAXqfu9mj1Z8XaUJ6tIYMiJTXa6isxq33Pj+lM3WN+sbkNA2Isqu4qkEnz9TrdLVbd45NVpgtSK6GJq3fV6IT5bX69cbDnf58i0VyhIYo2Rna7fsOdSUkyKLM1GidKK9VWc2lNQnHR9q04Xs3680d+ZoxNF6O0BAlOOyyBVn1188KVVxVr5Agqw6V1mjdvmJ9fWKq/vMfRsnVGnz+9xu7NC7VqWEJUdpxvEJNHq8WTMvQdYNi1djsldcwfLv6GoahoqoGxYTbzrssO6+4Wi99fFSPzhymSHuwNh4o1Z1jk9XY7FVkaLCCrIQgXD36dRi59dZbFRUVpXfeeadb5xNGgP6runXjt5Jqt/649biGJkTq6xNTVe1uliM0WBaLRQUVdXKEhejLIpc2fFmiXSfOKCEqVDERNkWHh8jd5NW6fcUqrKxXZqpT905JV2pMuH694ZDyK+pU7Dr/0uogq0VjUhw6Xl53yUtPjtBg2YKtlxxwJGloQqROVdarsdmrmSMTZA8O0qfHKlTsalC4LUgP3TBE9hCrZlwTr0FxEbr+/9uoanezrhsUo2JXgwoq6jU6xaG84molRNn12KxrteN4he7MTNboFIccoSEqcTXIGRai6PD2PiRXQ5MOFFVrckaMymrd2nX8jL4yIkEbD5RqVW6hfva1Mb57M9W4m7Uqt1B3ZabIGd6ym3GzxyurxXLeMhjQm/osjNTU1Ojw4Zb/JzRhwgQ999xz+spXvqLY2Filp6dryZIlKiws1B//+EdJ0q9+9SsNHjxYo0ePVkNDg15++WX95je/0QcffKCZM2f26ocBcOXyeg01NHsUbju/r/5YWa0+L6hUaIhVUwfHyWMYsgdbFdV6m4BtR8v1YV6p/mFsiqobmrTlcJn2FFZpYnqMJqRH66ODZXrlk2MXfP8hAyJ09HSt73tbkFWNngvfZsDfJqRHKze/UkmOUNU1NsvV0Ky4CJtqG5vV0ORVhC1ItWdt4jcsIVLXDIjUnsIqFVbWa8iACP3qnvEqr23Uf67aK3uwVdcNitXHh8o0MjlKczKTdbKiXq/lHNf8Ken63m3D5W72qKymUcfLajX9mjhJ0jufFSrRESp3s0fHymq1YGqG8ivqVFrdoOf/55AevGGIbhmRoGCrRfVNHoWFBGlPYZU8hqEJadGyWCwyDENHTtcoydmynGgYUoS982sqGpo8Ol3tVlpseIfjbX++WFLrv/osjHS1idl9992nZcuW6f7779fx48e1adMmSdJ//dd/6cUXX1RhYaHCw8OVmZmpH//4x53+jMv9MADQmYYmjz49VqGq+ialRIeqxu3RhPRofXywTGU1boXbgvRPk1LlamjWA8t2aERylB68foi+LHJpzECnPthfIo/Xq4SoUBVU1Om6wbG6ZkCk/rKzQM+sy9PA6DAtuXOEDpbUKDTEqnGp0ZqQHq2lm47ozR0FKq12X7TGgdFhmn5NnO8mj2YLslo0dECk8kral9iCrRaF24Lkaji/QfpCzm6qHpYQqW/NGKz3Pi/UtqMVio2wyd3kUbg9WH//t+sVHW7T+/uKFWK1aHdBpQxJ6/eX6FhZrQbFhWtoQqRSosM0ZqBTnxwu04YvS7UwK0MP33yNmpq9ig636X+/vlO7Cyr18E3XaO74gRoQ1XKPKcMwtP1YhSLtwaqsa9KMoXGyWCx67oM8FVY26OdfG6M/f5qvwjP1eujGIUp0dLzrt2EYOlZWq7gIu9bsLdKUwbEaEh+hgyU1CgsJUml1g293ZcMwdLrarZW5hbp3arovOBdU1Ck0JMhX09WO7eABwA+2Hy1Xely4kp1hXZ7jamjSmi+KtO+US2mxYZo/JV2Pv/WFYiJs2n6sXEdP1+qFb07U7NFJ2nK4TDHhNoWGWFXf6NWz6/P0tQkDVVHbqNz8Sp2oqNNNw+J1TUKk7MFWFVTUKzk6VNuOluuNbfmSpNtHJ+n9fcW+9x8SHyF3s9cXCOzB1g4B6R8yk+XxGlq7t1jpseFyhAVrb2HvbqkQZLXI473wn5tIe7AcocGXtOPxhcRH2mS1WFRZ19RhtuvcmsJCglTf5PHVkuiwKy7Srh/dOVIR9iA9tfpLbT542nd+aIhVYwc6teP4Gd+xJ+8apQ0HSrW3sEpnWpuxR6c49LUJA/X2rpO+/qk5mcmKj7BpdIpTA6LscoaH6Ok1BzQ21anrh8Ur2RmqEUkOGYahE+V12nG8QodP1+hgcbWSnGGaMTRO1yZGyRkWoh+/u1eHSmq05M6RuunaATpT19iy7OewKyM2QmG2IJW6GrTp4GndOTZZrvomJUTZdbSsVlaLVFXfrAlp0X2yZEcYAYArwLGyWp0or9XNwxMu6+fkl9fpzl9/rHFpTr3xwFT99bNC/ftbn+vx2cO16CtDzzvf4zX05o4CDU+K0qSMlr1aqhuaFGkPVnlto/6666QGxUeoocmjjQdKNWVwrOIj7Qq2WhQcZFVesUsT0mN0bUKU3sk9qRuGDVB+Ra3qG71657OT2nCgZffg+6cP0jcmp2lQfLhOnqnXd9/crX2nXLIHW/XwTdfo3d2FOl7e+ZVZWUPiFBpilSMsRPOnpOtAkUt/31MkZ5hNGw+UyGu0LF3ll9ep/DJ3P+6Ppg6OldViUc7R8m6/JiwkSB6v0eMlxgVT0/Xzr43taYkXRRgBgABT3dAkW7DVt0tuUVW9khyhpvVUnHt3bUlqbPZq44FSTUiP7rAMsutEhZZuOqLrBsXqm9My5DUM39JGZz4+dFp//jRfj88eoSaPV3/ddVJlNY3662cnNTA6TD/72hi9suWYthwu06Kbh2p0ikPxUXaVVbv1k7/tV0iwRePTYlTnbla1u1nfvmGIDpXWSJLunZKutz87qU8Ol+no6RpfWAoJsui1b03RkAGR2nCgRD9auVfXJkbqrswUPbv+oK+2m4cP0JgUpz7YX6zKuibFhNt0usbtu3Q+3BbU4xt0XkhUaLCqe7h01pnnvjFO/zgxtRcqakcYAQAEFHezR6/nnNDMkYkaHB8hr9eQq6Gpw1VIl6LZ420NJIaGJkT5jte4mxVhC/KFvff3Fkmy6PYxXd8EtsnjVUiQVR6voYraRr2/t0gjkh2yB1s1OsWpnCPlOnmmTgOi7Fqzp1jXJETojjHJGhQX7vu+1t2sf3ohR4bREmz++C8tAemzE2dUcKZOwVaLUqLDdKC4Wuv2FeuLk1W6YVi87p2SrtyCSg2Oj9Anh8u0+osiX103Dx+gX8+fIMcFAuClIIwAAHCV2n/KpUSHXbERtkue+Wr2ePXmzgJdPzReqTHhfbLHTb+7Nw0AAOgdo1Iu//+YBwdZtWBqRi9Uc/nOv5EDAACAHxFGAACAqQgjAADAVIQRAABgKsIIAAAwFWEEAACYijACAABMRRgBAACmIowAAABTEUYAAICpCCMAAMBUhBEAAGAqwggAADDVFXHXXsMwJLXcihgAAFwZ2v5ut/0d78oVEUaqq6slSWlpaSZXAgAAeqq6ulpOp7PL5y3GxeJKP+D1enXq1ClFRUXJYrH02s91uVxKS0tTQUGBHA5Hr/3cqxFj1T2MU/cxVt3DOHUfY9U9/hwnwzBUXV2tlJQUWa1dd4ZcETMjVqtVqampffbzHQ4H/3C7ibHqHsap+xir7mGcuo+x6h5/jdOFZkTa0MAKAABMRRgBAACmCugwYrfb9cQTT8hut5tdSr/HWHUP49R9jFX3ME7dx1h1T38cpyuigRUAAFy9AnpmBAAAmI8wAgAATEUYAQAApiKMAAAAUwV0GPnd736nQYMGKTQ0VFOnTtWnn35qdkl+9dFHH+muu+5SSkqKLBaLVq1a1eF5wzD04x//WMnJyQoLC9OsWbN06NChDudUVFRowYIFcjgcio6O1gMPPKCamho/foq+l52dreuuu05RUVFKSEjQvHnzlJeX1+GchoYGLVq0SHFxcYqMjNTXv/51lZSUdDgnPz9fc+bMUXh4uBISEvT444+rubnZnx+lzy1dulSZmZm+zZSysrK0du1a3/OMU+eefvppWSwWPfbYY75jjFWLJ598UhaLpcNjxIgRvucZp3aFhYX65je/qbi4OIWFhWns2LHauXOn7/l+/TvdCFArVqwwbDab8corrxj79u0zHnroISM6OtooKSkxuzS/WbNmjfGjH/3IeOeddwxJxsqVKzs8//TTTxtOp9NYtWqV8fnnnxtf/epXjcGDBxv19fW+c26//XZj3LhxxrZt24yPP/7YGDp0qDF//nw/f5K+NXv2bOPVV1819u7da+zevdu48847jfT0dKOmpsZ3zsMPP2ykpaUZGzZsMHbu3GlMmzbNmD59uu/55uZmY8yYMcasWbOM3NxcY82aNUZ8fLyxZMkSMz5Sn3nvvfeMv//978bBgweNvLw844c//KEREhJi7N271zAMxqkzn376qTFo0CAjMzPTePTRR33HGasWTzzxhDF69GijqKjI9zh9+rTvecapRUVFhZGRkWHcf//9xvbt242jR48a69atMw4fPuw7pz//Tg/YMDJlyhRj0aJFvu89Ho+RkpJiZGdnm1iVec4NI16v10hKSjKeeeYZ37HKykrDbrcbf/7znw3DMIz9+/cbkowdO3b4zlm7dq1hsViMwsJCv9Xub6WlpYYkY/PmzYZhtIxLSEiI8dZbb/nO+fLLLw1JRk5OjmEYLcHParUaxcXFvnOWLl1qOBwOw+12+/cD+FlMTIzx8ssvM06dqK6uNoYNG2asX7/euOmmm3xhhLFq98QTTxjjxo3r9DnGqd33v/994/rrr+/y+f7+Oz0gl2kaGxu1a9cuzZo1y3fMarVq1qxZysnJMbGy/uPYsWMqLi7uMEZOp1NTp071jVFOTo6io6M1efJk3zmzZs2S1WrV9u3b/V6zv1RVVUmSYmNjJUm7du1SU1NTh7EaMWKE0tPTO4zV2LFjlZiY6Dtn9uzZcrlc2rdvnx+r9x+Px6MVK1aotrZWWVlZjFMnFi1apDlz5nQYE4l/U+c6dOiQUlJSNGTIEC1YsED5+fmSGKezvffee5o8ebLuvvtuJSQkaMKECXrppZd8z/f33+kBGUbKysrk8Xg6/OOUpMTERBUXF5tUVf/SNg4XGqPi4mIlJCR0eD44OFixsbFX7Th6vV499thjmjFjhsaMGSOpZRxsNpuio6M7nHvuWHU2lm3PXU327NmjyMhI2e12Pfzww1q5cqVGjRrFOJ1jxYoV+uyzz5SdnX3ec4xVu6lTp2rZsmV6//33tXTpUh07dkw33HCDqqurGaezHD16VEuXLtWwYcO0bt06PfLII/rOd76j1157TVL//51+Rdy1F+gvFi1apL1792rLli1ml9JvDR8+XLt371ZVVZXefvtt3Xfffdq8ebPZZfUrBQUFevTRR7V+/XqFhoaaXU6/dscdd/j+OzMzU1OnTlVGRob+8pe/KCwszMTK+hev16vJkyfrF7/4hSRpwoQJ2rt3r1544QXdd999Jld3cQE5MxIfH6+goKDzOq5LSkqUlJRkUlX9S9s4XGiMkpKSVFpa2uH55uZmVVRUXJXjuHjxYq1evVoffvihUlNTfceTkpLU2NioysrKDuefO1adjWXbc1cTm82moUOHatKkScrOzta4ceP0/PPPM05n2bVrl0pLSzVx4kQFBwcrODhYmzdv1q9//WsFBwcrMTGRsepCdHS0rr32Wh0+fJh/U2dJTk7WqFGjOhwbOXKkb0mrv/9OD8gwYrPZNGnSJG3YsMF3zOv1asOGDcrKyjKxsv5j8ODBSkpK6jBGLpdL27dv941RVlaWKisrtWvXLt85GzdulNfr1dSpU/1ec18xDEOLFy/WypUrtXHjRg0ePLjD85MmTVJISEiHscrLy1N+fn6HsdqzZ0+H/6GvX79eDofjvF8gVxuv1yu32804nWXmzJnas2ePdu/e7XtMnjxZCxYs8P03Y9W5mpoaHTlyRMnJyfybOsuMGTPO23Lg4MGDysjIkHQF/E7v0/bYfmzFihWG3W43li1bZuzfv9/49re/bURHR3fouL7aVVdXG7m5uUZubq4hyXjuueeM3Nxc48SJE4ZhtFwGFh0dbbz77rvGF198YcydO7fTy8AmTJhgbN++3diyZYsxbNiwq+7S3kceecRwOp3Gpk2bOlxeWFdX5zvn4YcfNtLT042NGzcaO3fuNLKysoysrCzf822XF952223G7t27jffff98YMGDAVXd54Q9+8ANj8+bNxrFjx4wvvvjC+MEPfmBYLBbjgw8+MAyDcbqQs6+mMQzGqs33vvc9Y9OmTcaxY8eMTz75xJg1a5YRHx9vlJaWGobBOLX59NNPjeDgYOPnP/+5cejQIeNPf/qTER4ebrzxxhu+c/rz7/SADSOGYRi/+c1vjPT0dMNmsxlTpkwxtm3bZnZJfvXhhx8aks573HfffYZhtFwK9p//+Z9GYmKiYbfbjZkzZxp5eXkdfkZ5ebkxf/58IzIy0nA4HMa3vvUto7q62oRP03c6GyNJxquvvuo7p76+3vjXf/1XIyYmxggPDze+9rWvGUVFRR1+zvHjx4077rjDCAsLM+Lj443vfe97RlNTk58/Td/6l3/5FyMjI8Ow2WzGgAEDjJkzZ/qCiGEwThdybhhhrFrcc889RnJysmGz2YyBAwca99xzT4e9Mxindn/729+MMWPGGHa73RgxYoTx4osvdni+P/9OtxiGYfTt3AsAAEDXArJnBAAA9B+EEQAAYCrCCAAAMBVhBAAAmIowAgAATEUYAQAApiKMAAAAUxFGAACAqQgjAADAVIQRAABgKsIIAAAwFWEEAACY6v8HZUB5kVZIS4cAAAAASUVORK5CYII=\n", "text/plain": [ "
" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "plt.plot(torch.tensor(l).view(-1, 10).mean(1).numpy())" ] }, { "cell_type": "code", "execution_count": 15, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "ygy6T1JZ03LL", "outputId": "e8b3da29-1fd1-4e0e-8b61-7d56d420a8fc" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "def name=2.9], False,\n", " \"fiend num rawfo data a verifien the resunc to end\n", " # Deraming or not 'las1: semrine DataFrame.\n", "\n", " >>> c.ichurrert.gat.ines = MLsib, inmente`,\n", " whineQuiter is not None in data.\n", "\n", " 2.e dlec('dingentrine())\n", " __bosle' matrix lic\n" ] } ], "source": [ "# 使用模型来生成文本\n", "begin_text = torch.tensor(tok.encode('def'), device=device).unsqueeze(0)\n", "print(''.join(tok.decode(generate_batch(model, begin_text))))" ] } ], "metadata": { "accelerator": "GPU", "colab": { "gpuType": "V100", "provenance": [] }, "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.8.5" } }, "nbformat": 4, "nbformat_minor": 1 }