{ "cells": [ { "cell_type": "code", "execution_count": 1, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "1h1JT8eELKh2", "outputId": "bf0778ef-27c7-4818-cde1-5053733f4443" }, "outputs": [ { "data": { "text/plain": [ "" ] }, "execution_count": 1, "metadata": {}, "output_type": "execute_result" } ], "source": [ "import torch\n", "import torch.nn as nn\n", "import torch.nn.functional as F\n", "import torch.optim as optim\n", "from torch.nn.utils import clip_grad_norm_\n", "from transformers import AutoTokenizer, GPT2LMHeadModel\n", "from datasets import load_dataset\n", "import matplotlib.pyplot as plt\n", "%matplotlib inline\n", "\n", "\n", "torch.manual_seed(12046)\n", "# 计算设备为V100 16G\n", "# 如果使用CPU，需要非常长的时间，建议减少sequence_len、batch_size等参数来加快速度" ] }, { "cell_type": "code", "execution_count": 2, "metadata": { "id": "6t4ZrY0SLKh3" }, "outputs": [], "source": [ "# 一些超参数\n", "learning_rate = 1e-2\n", "# 模型支持的最大文本长度\n", "sequence_len = 1024\n", "batch_size = 4\n", "device = 'cuda' if torch.cuda.is_available() else 'cpu'\n", "eval_iters = 4\n", "eval_interval = 50" ] }, { "cell_type": "code", "execution_count": 3, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "6Dtak9OKLKh4", "outputId": "12d50d2b-fa5f-417d-fa1e-848f30669b7e" }, "outputs": [ { "data": { "text/plain": [ "GPT2LMHeadModel(\n", " (transformer): GPT2Model(\n", " (wte): Embedding(50257, 768)\n", " (wpe): Embedding(1024, 768)\n", " (drop): Dropout(p=0.1, inplace=False)\n", " (h): ModuleList(\n", " (0-11): 12 x GPT2Block(\n", " (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n", " (attn): GPT2Attention(\n", " (c_attn): Conv1D()\n", " (c_proj): Conv1D()\n", " (attn_dropout): Dropout(p=0.1, inplace=False)\n", " (resid_dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n", " (mlp): GPT2MLP(\n", " (c_fc): Conv1D()\n", " (c_proj): Conv1D()\n", " (act): NewGELUActivation()\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " )\n", " )\n", " (ln_f): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n", " )\n", " (lm_head): Linear(in_features=768, out_features=50257, bias=False)\n", ")" ] }, "execution_count": 3, "metadata": {}, "output_type": "execute_result" } ], "source": [ "tokenizer = AutoTokenizer.from_pretrained('gpt2')\n", "model = GPT2LMHeadModel.from_pretrained('gpt2').to(device)\n", "# 开源GPT-2模型的结构\n", "model" ] }, { "cell_type": "code", "execution_count": 4, "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 105, "referenced_widgets": [ "ae30a280af464393a17f16d8bd991e50", "412090d7ce2842d8979b6ee4ca19a0fe", "2dd504bb8b464afea0ac61bc41c129cc", "5a3141d40ea143ae948b4023284d2ae6", "3a50612ff1584cd48e7f3a43dc733d1b", "611a6f0cf7434f4186bbe6961a54742e", "0c1ff22ae7f6497c884f8e36bd078812", "35839dbe38074fe88d72dfd6d15c2192", "58b629ccc0304e549e2b13e566faa36b", "159fd68f34d440388ca2e8221d6d49f2", "b7ef0e6d0af94849b32d918f5b360f72" ] }, "id": "B67BMW5SLKh5", "outputId": "c2e234e8-0130-47da-a765-f15c8c3cd068" }, "outputs": [ { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "ae30a280af464393a17f16d8bd991e50", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Map: 0%| | 0/46801 [00:00 1024). Running this sequence through the model will result in indexing errors\n" ] }, { "data": { "text/plain": [ "torch.Size([5241682])" ] }, "execution_count": 4, "metadata": {}, "output_type": "execute_result" } ], "source": [ "def process(data):\n", " '''\n", " 对文本进行分词\n", " '''\n", " ids = tokenizer.encode(data['text'])\n", " # 增加表示文本结束的特殊字符\n", " ids.append(tokenizer.eos_token_id)\n", " out = {'ids': ids}\n", " return out\n", "\n", "def concat_data(datasets):\n", " '''\n", " 将文本分词，并将结果拼接成一个长长的字符串\n", " '''\n", " tokenized = datasets.map(process, remove_columns=datasets.column_names)\n", " tokenized.set_format(type='torch', device=device)\n", " # 将所有文本拼接成一个长长的字符串\n", " concat_text = torch.concatenate(tokenized['ids'])\n", " return concat_text\n", "\n", "raw_datasets = load_dataset('tatsu-lab/alpaca')\n", "raw_datasets = raw_datasets['train'].train_test_split(test_size=0.1, seed=1024, shuffle=True)\n", "train_set = concat_data(raw_datasets['train'])\n", "test_set = concat_data(raw_datasets['test'])\n", "\n", "train_set.shape" ] }, { "cell_type": "code", "execution_count": 5, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "sTMUMVroLKh5", "outputId": "18f88fd1-4f13-457e-fe50-447e215e4ddb" }, "outputs": [ { "data": { "text/plain": [ "(tensor([[ 279, 67, 13, ..., 1690, 351, 262],\n", " [ 198, 8645, 378, ..., 198, 198, 21017],\n", " [19430, 257, 2882, ..., 4727, 644, 561],\n", " [ 317, 13, 33295, ..., 198, 21017, 18261]], device='cuda:0'),\n", " tensor([[ 67, 13, 961, ..., 351, 262, 1037],\n", " [ 8645, 378, 257, ..., 198, 21017, 18261],\n", " [ 257, 2882, 326, ..., 644, 561, 423],\n", " [ 13, 33295, 7, ..., 21017, 18261, 25]], device='cuda:0'))" ] }, "execution_count": 5, "metadata": {}, "output_type": "execute_result" } ], "source": [ "def get_data(data, batch_size, sequence_len):\n", " '''\n", " 生成训练数据\n", " '''\n", " # 生成截取数据的起点，形状为(B)，其中B等于batch_size，表示批量数据的大小\n", " ix = torch.randint(len(data) - sequence_len, (batch_size,))\n", " x = torch.stack([data[i: i + sequence_len] for i in ix])\n", " # 预测标签是下一个字母，因此只需要挪动一个位置即可\n", " y = torch.stack([data[i + 1: i + 1 + sequence_len] for i in ix])\n", " return x, y\n", "\n", "get_data(test_set, batch_size, sequence_len)" ] }, { "cell_type": "code", "execution_count": 6, "metadata": { "id": "15YcHmQvLKh7" }, "outputs": [], "source": [ "def print_trainable_parameters(model):\n", " \"\"\"\n", " 输出模型中可供训练的参数个数\n", " \"\"\"\n", " trainable_params = 0\n", " all_param = 0\n", " for _, param in model.named_parameters():\n", " all_param += param.numel()\n", " if param.requires_grad:\n", " trainable_params += param.numel()\n", " trainable = f'trainable params: {trainable_params:,}'\n", " params = f'all params: {all_param:,}'\n", " percent = f'trainable%: {100 * trainable_params / all_param:.3f}'\n", " print(f'{trainable} || {params} || {percent}')" ] }, { "cell_type": "code", "execution_count": 7, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "Zy9_Nuf7LKh8", "outputId": "0a4f43f2-8959-426d-b7a7-085e04df2e67" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "trainable params: 124,439,808 || all params: 124,439,808 || trainable%: 100.000\n", "trainable params: 147,456 || all params: 124,587,264 || trainable%: 0.118\n" ] } ], "source": [ "from peft import LoraConfig, PeftModel\n", "\n", "def init_peft_model(model):\n", " # 设置LoRA参数\n", " config = LoraConfig(\n", " r=4,\n", " lora_alpha=32,\n", " target_modules=['c_attn'],\n", " lora_dropout=0.1,\n", " # c_attn.weight的形状是(fan_in, fan_out)，所以该参数设置为True\n", " # 但需注意的是，普通的线性模型权重参数的形状是(fan_out, fan_in)\n", " fan_in_fan_out=True,\n", " bias='none')\n", " return PeftModel(model, config, adapter_name='lora_alpaca')\n", "\n", "print_trainable_parameters(model)\n", "model = init_peft_model(model)\n", "# 确保模型在训练状态\n", "model.train()\n", "print_trainable_parameters(model)" ] }, { "cell_type": "code", "execution_count": 8, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "T13FbMZaLKh8", "outputId": "62497bce-944f-4795-b9ec-74f72c9e5593" }, "outputs": [ { "data": { "text/plain": [ "PeftModel(\n", " (base_model): LoraModel(\n", " (model): GPT2LMHeadModel(\n", " (transformer): GPT2Model(\n", " (wte): Embedding(50257, 768)\n", " (wpe): Embedding(1024, 768)\n", " (drop): Dropout(p=0.1, inplace=False)\n", " (h): ModuleList(\n", " (0-11): 12 x GPT2Block(\n", " (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n", " (attn): GPT2Attention(\n", " (c_attn): Linear(\n", " in_features=768, out_features=2304, bias=True\n", " (lora_dropout): ModuleDict(\n", " (lora_alpaca): Dropout(p=0.1, inplace=False)\n", " )\n", " (lora_A): ModuleDict(\n", " (lora_alpaca): Linear(in_features=768, out_features=4, bias=False)\n", " )\n", " (lora_B): ModuleDict(\n", " (lora_alpaca): Linear(in_features=4, out_features=2304, bias=False)\n", " )\n", " (lora_embedding_A): ParameterDict()\n", " (lora_embedding_B): ParameterDict()\n", " )\n", " (c_proj): Conv1D()\n", " (attn_dropout): Dropout(p=0.1, inplace=False)\n", " (resid_dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n", " (mlp): GPT2MLP(\n", " (c_fc): Conv1D()\n", " (c_proj): Conv1D()\n", " (act): NewGELUActivation()\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " )\n", " )\n", " (ln_f): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n", " )\n", " (lm_head): Linear(in_features=768, out_features=50257, bias=False)\n", " )\n", " )\n", ")" ] }, "execution_count": 8, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# 加入LoRA后，模型的结构\n", "model" ] }, { "cell_type": "code", "execution_count": 9, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "Wb5LW_BKLKh9", "outputId": "74097362-1e12-4c0f-b7a3-cbc448171660" }, "outputs": [ { "data": { "text/plain": [ "{'train': 2.6983275413513184, 'test': 2.678028106689453}" ] }, "execution_count": 9, "metadata": {}, "output_type": "execute_result" } ], "source": [ "from contextlib import nullcontext\n", "\n", "def estimate_loss(model, ctx=nullcontext()):\n", " '''\n", " 估计模型损失\n", " ctx参数是为禁用LoRA或者混合精度做准备，当ctx=nullcontext()时，没有任何作用\n", " '''\n", " re = {}\n", " # 将模型切换至评估模式\n", " model.eval()\n", " _train = lambda: get_data(train_set, batch_size, sequence_len)\n", " re['train'] = _loss(model, _train, ctx)\n", " _test = lambda: get_data(test_set, batch_size, sequence_len)\n", " re['test'] = _loss(model, _test, ctx)\n", " # 将模型切换至训练模式\n", " model.train()\n", " return re\n", "\n", "@torch.no_grad()\n", "def _loss(model, data_loader, ctx):\n", " \"\"\"\n", " 计算模型在不同数据集下面的评估指标\n", " \"\"\"\n", " loss = []\n", " # 随机使用多个批量数据来预估模型效果\n", " for k in range(eval_iters):\n", " inputs, labels = data_loader()\n", " with ctx:\n", " logits = model(inputs).logits\n", " # 根据cross_entropy的定义，需要对logits进行转置运算\n", " # 具体细节请参考cross_entropy的官方文档\n", " logits = logits.transpose(-2, -1)\n", " loss.append(F.cross_entropy(logits, labels).item())\n", " return torch.tensor(loss).mean().item()\n", "\n", "estimate_loss(model)" ] }, { "cell_type": "code", "execution_count": 10, "metadata": { "id": "v6Tlrjv2LKh-" }, "outputs": [], "source": [ "def train_gpt(model, optimizer, data_loader, max_iters=1000):\n", " lossi = []\n", " for iter_num in range(max_iters):\n", " optimizer.zero_grad(set_to_none=True)\n", " inputs, labels = data_loader()\n", " logits = model(inputs).logits\n", " # 根据cross_entropy的定义，需要对logits进行转置运算\n", " # 具体细节请参考cross_entropy的官方文档\n", " logits = logits.transpose(-2, -1)\n", " loss = F.cross_entropy(logits, labels)\n", " lossi.append(loss.item())\n", " loss.backward()\n", " optimizer.step()\n", " if iter_num % eval_interval == 0:\n", " stats = estimate_loss(model)\n", " train_loss = f'train loss {stats[\"train\"]:.4f}'\n", " test_loss = f'test loss {stats[\"test\"]:.4f}'\n", " print(f'step {iter_num:>4}: {train_loss}, {test_loss}')\n", " return lossi" ] }, { "cell_type": "code", "execution_count": 11, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "K68N_0ZyLKh-", "outputId": "0ff83a63-2919-4955-e96e-1ddf281269e5" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "step 0: train loss 2.2870, test loss 2.3158\n", "step 50: train loss 1.7305, test loss 1.7837\n", "step 100: train loss 1.6436, test loss 1.7209\n", "step 150: train loss 1.7382, test loss 1.6713\n", "step 200: train loss 1.7056, test loss 1.7386\n", "step 250: train loss 1.6550, test loss 1.8193\n", "step 300: train loss 6.7129, test loss 6.9341\n", "step 350: train loss 6.7540, test loss 7.1159\n", "step 400: train loss 7.0552, test loss 7.0193\n", "step 450: train loss 6.7841, test loss 6.7284\n" ] } ], "source": [ "# 粗糙的训练方式可能会使模型效果急剧恶化\n", "data_loader = lambda: get_data(train_set, batch_size, sequence_len)\n", "optimizer = optim.AdamW(model.parameters(), lr=learning_rate)\n", "l = train_gpt(model, optimizer, data_loader, max_iters=500)" ] }, { "cell_type": "code", "execution_count": 12, "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 448 }, "id": "iMCmctGPTczt", "outputId": "0510d2e8-76b2-49eb-ec32-c74cdcafb8e3" }, "outputs": [ { "data": { "text/plain": [ "[]" ] }, "execution_count": 12, "metadata": {}, "output_type": "execute_result" }, { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAhYAAAGdCAYAAABO2DpVAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA2CElEQVR4nO3deXyU9b33//fMJDPZFyALgbBvkgDKIqJt3VBLlUM3j7X01Lbnd9oqVq09bbW9rXi8Ldjjw2NtPRztov7u292KVs9xoSpQRZB91RAkQCCQQLZJMpOZycx1/5HMQASESWauyVzzej4e88hk5prk42WSeXN9P9/v12YYhiEAAIAYsCe6AAAAYB0ECwAAEDMECwAAEDMECwAAEDMECwAAEDMECwAAEDMECwAAEDMECwAAEDNpZn/DUCikuro65ebmymazmf3tAQBAHxiGoba2NpWVlcluP/11CdODRV1dncrLy83+tgAAIAZqa2s1fPjw0z5verDIzc2V1F1YXl6e2d8eAAD0gdvtVnl5eeR9/HRMDxbh4Y+8vDyCBQAASeZMbQw0bwIAgJghWAAAgJghWAAAgJghWAAAgJghWAAAgJghWAAAgJghWAAAgJghWAAAgJiJKlgEg0HdddddGj16tDIzMzV27Fjde++9MgwjXvUBAIAkEtXKm/fff7+WLVumJ598UhUVFdqwYYO++93vKj8/X7fccku8agQAAEkiqmCxZs0aLViwQFdffbUkadSoUXrmmWf04YcfxqU4AACQXKIaCrnwwgv19ttva/fu3ZKkrVu36r333tO8efNO+xqfzye3293rBgAArCmqKxZ33HGH3G63Jk2aJIfDoWAwqPvuu08LFy487WuWLFmie+65p9+FAgCsaWddq7bWtur688vPuMEVBr6orlg8//zzeuqpp/T0009r06ZNevLJJ/XAAw/oySefPO1r7rzzTrW2tkZutbW1/S4aAGAdP3l+q36xfLtW7T6a6FIQA1FdsfjpT3+qO+64Q9/4xjckSVOmTNH+/fu1ZMkS3XDDDad8jcvlksvl6n+lAADLafUE9PGRNknS5gMtumRicYIrQn9FdcXC4/HIbu/9EofDoVAoFNOiAACpYcvBlsj9HYdaE1cIYiaqKxbz58/XfffdpxEjRqiiokKbN2/Wgw8+qO9973vxqg8AYGFbDrRE7m8nWFhCVMHid7/7ne666y7ddNNNamhoUFlZmX7wgx/oV7/6VbzqAwBY2Oba5sj9hjafGtydKs7LSGBF6K+ogkVubq4eeughPfTQQ3EqBwCQKgzD0OaeKxauNLt8XSFtP9SqywkWSY29QgAACVFzrEOt3oBcaXZdWVEqieEQKyBYAAASYkttiySpcli+zisvkEQDpxUQLAAACREeBjmvvEBThudL4oqFFRAsAAAJEW7cPG9EoSYPzZPNJtW7fWpo60xwZegPggUAwHRef1AfHe5eGOu8EQXKdqVpbFGOJIZDkh3BAgBguh11rQqGDBXnujQ0v3sWyNRhPcMhB9msMpkRLAAAptt8IDwMUhDZeKxyGH0WVkCwAACYLtK4OaIw8li4gZOhkORGsAAAmC481TQ8zVRSpIHziLtTR9t8iSkM/UawAACY6nCrV4dbO+Ww2yJXKSTRwGkRBAsAgKnCG49NLMlVlrP3zhJTevosth0kWCQrggUAwFSbw8MgIwpOeo4GzuRHsAAAmGrLKRo3w8JXLBgKSV4ECwCAaQLBkLYdapEknXtC42ZYRRkNnMmOYAEAME3VkTZ1BkLKy0jTmCHZJz2f7Tr+OFctkhPBAgBgmvDCWOeOKJTdbjvlMVPos0hqBAsAgGnCjZunGgYJmzK8+zmCRXIiWAAATHO8cbPgtMfQwJncCBYAAFM0d/i191iHJOncnqsSpxJu4Dzc2qlj7TRwJhuCBQDAFFsOtkiSxgzJVmG287THndjAyXBI8iFYAABMER4G+az+irDIcAgrcCYdggUAwBSfteLmp7ECZ/IiWAAA4i4UMrSlZ6rpqVbc/DQaOJMXwQIAEHd7j3XI3dmljHS7JpbmnvH4imH5stmkutZONdLAmVQIFgCAuNvSMwwyZVi+0h1nfuvJcaVpNA2cSYlgAQCIu81RDIOERVbgpIEzqRAsAABxtzm8MNZZzAgJY2nv5ESwAADElcffpY+PuCVJ557FjJAwGjiTE8ECABBX2w+2KmRIpXkZGpqfedavo4EzOREsAABxFc36FSeigTM5ESwAAHF1vHGzIOrXMhySfAgWAIC4MQwj0rh5bvnZzwgJo4Ez+RAsAABxc7i1Uw1tPjnstkhIiEZl5IqFO9alIU4IFgCAuAlfrThnaK4ynY6oX19RlidJOtTiVVOHP5alIU4IFgCAuIn0V/RhGESScjPS2UI9yRAsAABxE17K+2y2Sj+dSho4kwrBAgAQF43tPm3rCQPRLIz1aSztnVwIFgCAuHj47Wr5u0KqHJYXGc7oi0pmhiQVggUAWMR/rNitBb9/T5t6+hoSae/Rdj217oAk6RdfOkc2m63PX6ty2PEGzmYaOAc8ggUAWMCaPcf027ertfVgq77x6Fo9v6E2ofX85o0qdYUMXTqxSBeOHdKvr3ViA+fq6qMyDCMWJSJOCBYAkOQ6A0H98uUdkqTiXJf8wZB+9uI2Lf7rTgWCIdPr2bCvSW/sPCK7TbrzS+fE5GtOHd49HHLrs1t0yQMr9b9f26V1exvVlYD/Pnw2ggUAJLn/XPmJao51qCjXpRU/vli3zR0vSXpizT59+08fmrr+g2EY+vX/fCRJ+seZ5ZpQkhuTr3vzZeN06cQiOdPs2t/o0R/fq9F1j63VrPv+ptuf36I3dhyWx98Vk++F/rEZJl9Tcrvdys/PV2trq/Ly8sz81gBSXGcgqCfX7JMk3XDhKGWkR79g00Czp6Fd8367WoGgoUe+OV1XTx0qSXpz5xHd/twWdfiDGl6Yqcf+aaYml8X/b+7r2w/rxqc2KTPdoVU/vUTFeRkx/fodvi6t3n1UK3bV652qBrV4ApHnnGl2fW7cEF00bohmjSrU5KF5SnPw7+dYOdv376iCxahRo7R///6THr/pppv0yCOPxLQwAIgVwzC0Yle9/u21XTrY7JUkjS3K1tKvTdWsUYMSXF3fGYah6x5bqw9rmnTpxCL9+TuzejVJ7q5v07/8/xu0v9GjzHSHHrh2WiR4xIO/K6Qr/2OV9jV6dMvl43X7FRPi9r0kqSsY0ob9zVqxq14rdtXrQJOn1/NZToemjyjUzFGFmjVqkM4tL1C2Ky2uNVlZXILF0aNHFQwGI5/v2LFDV1xxhd59911dcsklMS0MAGJh37EOLX51p1ZWHZUkDc3PUFfI0NE2nyTpWxeM0M+/OEm5GemJLLNPnt9Qq5+9uE0Z6Xat+PHFKh+UddIxLR6/fvTMZv29+pgkadGlY3X7FRPlsPd9lsbpPPF+jRa/uktDcpxa+dNLlWPim7hhGNpd3653Pm7Q+n1N2rCvSe7O3kMjDrtNFWV5mjVqkOZPK+vXol2pKC7B4tNuu+02vfbaa6qurj7rqUQECwBm8PqDeuTdPXps9V75gyGlO2z6l8+P0c2XjVOgq7sP4LmemROleRm698uVumJySYKrPnuN7T5d/uAqtXgCunPeJP3g4rGnPbYrGNJv3qzSY6v3SpIum1Ss/7juXOVnxi5MuTsDuvg376rZE9D//nKlvnXByJh97b4IhQxVN7RHQsb6fc061OLtdcy3Lhihn31xkvKSMFQmQtyDhd/vV1lZmW6//Xb94he/OO1xPp9PPp+vV2Hl5eUECwBxYRiG3tx5RPe+9lHkjeQLE4q0eP5kjSnK6XXsmj3HdOfy7drf2H0J/eqpQ7V4foWKcl2m1x2t25/bopc2H9Kk0ly9+qPPKf0segle3nxIP//LNvm6QioflKlHvjldU4cXxKSe37zxsf5z5ScaW5StN2/7woDsbTjU4tWGfU16+6MG/XVrnSSpJM+lf1tQqasqSvv99TsDQX1ytF3V9e2qqm9TdX2bdte3q7Hdp3lThuqHF4/RuOLYNLMmQtyDxfPPP69vfvObOnDggMrKyk573OLFi3XPPfec9DjBAkCs7T3arrv/ujNy2X9YQabuumayrqooOe1V1c5AUA/9rVp/+PteBUOG8jPT9curz9G1M4ZHXhMKGWrr7FJjh0/NHr+aOgJq7vDL1xVU5bB8VQ7LP6s39lh5f88xLfzjOtls0ks3XqjzRpz9Bl/bD7bqxqc26mCzV+kOm375pXN0w4Wj+rWAVV2LV5c+sFK+rpD+8O2ZSXHlZ80nx/SLl7ZrX0+o/GJFqe5ZUKGSs2w29XeFtK6mUetrmnpCRLv2NXYodIZ31LnnlOjGS8Zqxsi+bcqWSHEPFldddZWcTqdeffXVzzyOKxYA4q0rGNJjf9+rh1ZUyx8Myemw6wcXj9FNl4w76626dxxq1c//sk0769ySpPHF3Vc3mj1+NXsCCn7GO0aW06EZIws1e/QgzR4zWFOH58uV9tnf1zAMNXb4Vdvk0eHWTk0oyTmrf812BoL64kOrta/Ro2/PGal/W1B5Vv99J2r1BvSzF7fqzZ31kqR5laW6/+tT+zwk8JPnt+ovmw7q/NGD9Nz3L+hXSDFTZyCoh9+u1mOr96orZCg3I013zjtH35hVLvspelDaOgNaWdU9I+XdjxvU5jt5emt+ZromluRqfEmOJpTkakJJruw26c/v1+itXfUKv+OeP2qQbrxkrC6ZWJQ05yuuwWL//v0aM2aMXnrpJS1YsCAuhQHA2djT0KafvLBNW3t20bxkYpEWz6/QqD7sTdEVDOlP79XowRW75es6eeGlHFeaCrPTNSjbpUFZ3W/Cmw60qNUb6HWcK82u6SMKNXvMIE0fUSiPP6iDzR7VNnlU2+xVbZNHB5u98gaCvV538YQiff8LY3Th2MGnfbN58K0qPfzOHhXnuvS3n1zc5zBgGIaeWLNPv/6fjxQIGhoxKEuPfHO6pvQsRHW2dtW5dfXv/i7DkF5edFFSNkR+dNitO/6yTVt7Njk7f9Qg/fqrUzSuOEf17s7IrJM1nxxTIHj8LXNIjlMXTyjW5LI8TSzJ1YSSHBXluk77/+6To+16bNVevbT5YOTrTCrN1Q8vHqtrpg4dkMNHJ4prsFi8eLEeffRR1dbWKi0tuq5fggWAWAiGDP3pvb164K3d8neFlJuRpsXzK/TV6cP6/S/AQy1ebattUX5mugqznRqU7VRBVvopr0KEQoaq6tv0YU2T1tU0at3eJjWe5YJUNptUkpuholyXdta1Ri6jTx6ap3/5wmhdM7Ws1xDLnoY2zfvt3xUIGvrPhdP1pSn9nzq6tbZFi57epIPNXjkddv3y6nP07Tkjz/oc/tOf1unv1cd0zdSh+v03p/e7nkQJhgw9uWafHnirSh5/UE6HXRNKc7TjkLvXcWOGZOuKihJdOblE55YX9ml2zZHWTv35/Ro9tXa/Ovzd4XJYQaauP79c86eVaeTgvm3Y1tDWqbd21mtnnVtLvjqlT1/js8QtWIRCIY0ePVrXX3+9li5dGrfCAOB09h5t109f3KaN+7s327pkYpGWfnWqSvNjuxhTXxiGoU+Otmvt3iZ9WNOkbQe7A8rwQVkqL8xS+aBMDS/MUnlhpoYVZkbCyv7GDv35vRo9v+Fg5EpGaV6GvnvRKF0/e4RynGn6xmNr9eG+Jl02qVh/umFmzC6h93VoZPXuo/r2nz9UusOmt2+/RCMGnzzdNdkcbPborpd36N2e6cmSdG55ga6sKNGVk0s1rjjnM14dnVZPQP933X79+b2aXmF02vB8zZ9Wpmumlp3xZ/pwq1dv7Dii17cf0fr9TZGhlnf/9RKN7seOsqcSt2Dx1ltv6aqrrlJVVZUmTIh+8ROCBYC+CoW6L9//5s2P1RkIKceVpruuOUf/OLM8acapz6TF49dT6w7o8ff36Vh7d39attOh2WMG652PG5SZ7tBbP/7CKdes6A/DMPT4+/u05PXjQyOXTSo+6bgTT/O7HzdoX6NH//y50brrmskxrSeRDMPQ6upjanB36uIJRTFfPfTTOgNB/XVrnV7dWqf39xyLXLmy2bqHZeZPK9O8ylINzumerVTb5NHrOw7r9R1HtPlAS6+vNa28QPMqS3XtjOGR42PFlHUs+oJgAaAv9jd26KcvbNOH+5okSZ8bN0T3f32qhhVkJriy+PB1BfXKljr98e97tbu+PfL4L790jv7lC2Pi9n231Lbo5p6hkbORl5Gm1T+7VAVZzrjVlEqOtvn0+o7DenVrndbva4487rDbdOHYwWr2+HsNz9hs0syRhfpi5VB9sbI0rr8PBAsAlvHxEbe+8sgaeQNBZTkd+uXV5+ib54+wzFWKz2IYhlbtPqr/88F+5Wak6YFrp8W9ya/VE9BzGw7I7e0968HQyW8Xl00qScqpk8ngUItX/72tTq9uPazth1ojj9tt0gVjBmteZamuqiiN+xWVMIIFAMu4/42PtWzlJ6oclqdlC2fEfBgAGOj2Hm3X2x81KC8zTXPPKYn5MMfZONv3b3ZjATDg7T7SJql7G25CBVLRmKKck1aOHagG9qRZAJBUVd8dLCaUJO9yyECqIFgAGNA6fF2RRkKCBTDwESwADGjVDd0zIobkuDQom5kHwEBHsAAwoIX7KyaWJsf4MpDqCBYABjT6K4DkQrAAMKDt7gkWEwkWQFIgWAAY0MLBYkIpwQJIBgQLAANWi8evenf3fhnjY7j5E4D4IVgAGLDCe2QMK8hU7hl22gQwMBAsAAxYxxs3uVoBJAuCBYABKzzVlP4KIHkQLAAMWFXMCAGSDsECwIBkGIaqWcMCSDoECwAD0tF2n5o9Adlt0jhmhABJg2ABYEDafaR7RsjIwdnKSHckuBoAZ4tgAWBAYkYIkJwIFgAGpMjmY/RXAEmFYAFgQNrdwFRTIBkRLAAMOIZhcMUCSFIECwADzqEWrzr8QaU7bBo1JDvR5QCIAsECwIAT3tF0zJAcpTv4MwUkE35jAQw4VT1TTemvAJIPwQLAgLM7spQ3U02BZEOwADDg7GYpbyBpESwADCjBkKHqhu6hkIkMhQBJh2ABYEDZ39ghf1dIGel2lRdmJbocAFEiWAAYUE4cBrHbbQmuBkC0CBYABpTwjJDxxQyDAMmIYAFgQAkv5T2xlBkhQDIiWAAYUMJLeTMjBEhOBAsAA4avK6iaYx2SmBECJCuCBYABo+ZYh7pChnIz0lSal5HocgD0AcECwIBRdcKOpjYbM0KAZESwADBghKeajqe/AkhaBAsAA8bu+p4VN9kjBEhaBAsAA0ZkcSwaN4GkRbAAMCB4/F060OSR1N1jASA5ESwADAh7GtplGNKQHKcG57gSXQ6APiJYABgQqlgYC7AEggWAASG8VTrBAkhuBAsAAwJXLABriDpYHDp0SN/61rc0ePBgZWZmasqUKdqwYUM8agOQQsIzQth8DEhuadEc3NzcrIsuukiXXnqpXn/9dRUVFam6ulqFhYXxqg9ACmj1BnS4tVMSi2MByS6qYHH//fervLxcjz/+eOSx0aNHx7woAKmluudqRVl+hvIy0hNcDYD+iGoo5K9//atmzpypa6+9VsXFxTrvvPP0hz/84TNf4/P55Ha7e90A4ERVLIwFWEZUwWLv3r1atmyZxo8frzfffFM33nijbrnlFj355JOnfc2SJUuUn58fuZWXl/e7aADWUh1ZyptgASQ7m2EYxtke7HQ6NXPmTK1Zsyby2C233KL169frgw8+OOVrfD6ffD5f5HO3263y8nK1trYqLy+vH6UDsIrrH1urD/Y26oFrp+nrM4YnuhwAp+B2u5Wfn3/G9++orlgMHTpUkydP7vXYOeecowMHDpz2NS6XS3l5eb1uAHCiyIwQrlgASS+qYHHRRRepqqqq12O7d+/WyJEjY1oUgNRxrN2nxg6/bDZpXDFTTYFkF1Ww+PGPf6y1a9fq17/+tfbs2aOnn35ajz32mBYtWhSv+gBY3O6ehbFGDspSptOR4GoA9FdUwWLWrFlavny5nnnmGVVWVuree+/VQw89pIULF8arPgAWF9kqnWEQwBKiWsdCkq655hpdc8018agFQArazR4hgKWwVwiAhGru8EuSSvLYKh2wAoIFgITy+IOSpExn1BdQAQxABAsACeXtCRZZNG4ClkCwAJBQnkCXJCkznWABWAHBAkBCHR8KIVgAVkCwAJBQDIUA1kKwAJBQ3gDBArASggWAhGJWCGAtBAsACRMMGfJ3hSRJWTRvApZAsACQMB5/V+Q+zZuANRAsACRMuHHTZpNcafw5AqyA32QACRPur8hKd8hmsyW4GgCxQLAAkDDhGSE0bgLWQbAAkDAe1rAALIdgASBhwj0WLOcNWAfBAkDChGeFMCMEsA6CBYCEYdVNwHoIFgAShn1CAOshWABIGJbzBqyHYAEgYSJDITRvApZBsACQMDRvAtZDsACQMMeHQggWgFUQLAAkjNfPUAhgNQQLAAlzfElvggVgFQQLAAlzfElvZoUAVkGwAJAwrGMBWA/BAkDChGeFZNBjAVgGwQJAwrC7KWA9BAsACdPJXiGA5RAsACQM61gA1kOwAJAwXmaFAJZDsACQEIZhyBNex4LmTcAyCBYAEsIfDCkYMiQxFAJYCcECQEKEh0EkmjcBKyFYAEiI8HLe6Q6b0h38KQKsgt9mAAkRmRFCfwVgKQQLAAnBjBDAmggWABKCNSwAayJYAEiI8D4hDIUA1kKwAJAQLOcNWBPBAkBCMBQCWBPBAkBCsLMpYE0ECwAJ4WW6KWBJBAsACXF8KITppoCVRBUsFi9eLJvN1us2adKkeNUGwMI8ge5ZIQyFANYS9T8VKioq9Le//e34F0jjXxsAotdJjwVgSVGngrS0NJWWlsajFgAphFkhgDVF3WNRXV2tsrIyjRkzRgsXLtSBAwc+83ifzye3293rBgCe8DoWNG8ClhJVsJg9e7aeeOIJvfHGG1q2bJlqamr0+c9/Xm1tbad9zZIlS5Sfnx+5lZeX97toAMnPyxULwJJshmEYfX1xS0uLRo4cqQcffFD//M//fMpjfD6ffD5f5HO3263y8nK1trYqLy+vr98aQJL7xmMfaO3eJj18/Xn6h2lliS4HwBm43W7l5+ef8f27X52XBQUFmjBhgvbs2XPaY1wul1wuV3++DQALiuxuylAIYCn9Wseivb1dn3zyiYYOHRqregCkCC97hQCWFFWw+Nd//VetWrVK+/bt05o1a/SVr3xFDodD119/fbzqA2BRzAoBrCmqoZCDBw/q+uuvV2Njo4qKivS5z31Oa9euVVFRUbzqA2BRNG8C1hRVsHj22WfjVQeAFBPZhCydRfYAK2GvEACmC4WMSI8FVywAayFYADCdrysUuU/zJmAtBAsApvP4uyL32TYdsBaCBQDThfsrMtLtstttCa4GQCwRLACYLtJfwdUKwHIIFgBMF5kR4mRGCGA1BAsApgv3WDAjBLAeggUA03WynDdgWQQLAKaLLOdNjwVgOQQLAKY73mNBsACshmABwHTsEwJYF8ECgOmOD4UwKwSwGoIFANN5ad4ELItgAcB03p7ppgQLwHoIFgBM56HHArAsggUA03mZbgpYFsECgOmYbgpYF8ECgOk84U3I2CsEsByCBQDTdXLFArAsggUA03kCbEIGWBXBAoDpIj0WNG8ClkOwAGA6lvQGrItgAcB0zAoBrItgAcB0x69YMCsEsBqCBQBTdQVD8gdDkuixAKyIYAHAVOENyCR6LAArIlgAMFV4GMRmk1xp/AkCrIbfagCmOnGqqc1mS3A1AGKNYAHAVB4aNwFLI1gAMFW4x4KppoA1ESwAmMrLGhaApREsAJjK42efEMDKCBYATBUeCslkDQvAkggWAEzFct6AtREsAJiKWSGAtREsAJiqM8CW6YCVESwAmIrmTcDaCBYATHV8KIRgAVgRwQKAqbx+hkIAKyNYADAVVywAayNYADDV8SW9mRUCWBHBAoCpWNIbsDaCBQBTMSsEsLZ+BYulS5fKZrPptttui1E5AKwu0mNB8yZgSX0OFuvXr9ejjz6qqVOnxrIeABbHtumAtfUpWLS3t2vhwoX6wx/+oMLCwljXBMDCmBUCWFufgsWiRYt09dVXa+7cuWc81ufzye1297oBSF2dfmaFAFYW9W/2s88+q02bNmn9+vVndfySJUt0zz33RF0YAOsxDEMehkIAS4vqikVtba1uvfVWPfXUU8rIyDir19x5551qbW2N3Gpra/tUKIDk5w+GFAwZkhgKAawqqisWGzduVENDg6ZPnx55LBgMavXq1fr9738vn88nh6P3HwuXyyWXyxWbagEktfAaFhKzQgCriipYXH755dq+fXuvx7773e9q0qRJ+vnPf35SqACAE4UbN9MdNqU7WEYHsKKogkVubq4qKyt7PZadna3Bgwef9DgAfBprWADWxz8ZAJimk31CAMvr92/3ypUrY1AGgFTgYZ8QwPK4YgHANOF9QjIYCgEsi2ABwDTsbApYH8ECgGlYzhuwPoIFANOwARlgfQQLAKbxsk8IYHkECwCmYSgEsD6CBQDTeALds0JYIAuwLoIFANMwKwSwPoIFANMwFAJYH8ECgGkis0IYCgEsi2ABwDTMCgGsj2ABwDSRJb0ZCgEsi2ABwDSRKxYMhQCWRbAAYBp2NwWsj2ABwDTh5k1mhQDWRbAAYBqaNwHrI1gAMA1DIYD1ESwAmCIUMiJDIRk0bwKWRbAAYIrOrmDkPlcsAOsiWAAwRXgYRGITMsDKCBYATBFu3MxIt8tutyW4GgDxQrAAYIrIPiHMCAEsjWABwBSRnU0ZBgEsjWABwBThfUJYHAuwNoIFAFN4WcMCSAkECwCmiCznzVAIYGkECwCmYNVNIDUQLACYgn1CgNRAsABgCo+f5byBVECwAGAKb8+sEIZCAGsjWAAwBT0WQGogWAAwRWRWCMECsDSCBQBTsI4FkBoIFgBMEVnSm1khgKURLACYwsMCWUBKIFgAMAWzQoDUQLAAYIrjQyEEC8DKCBYATBGeFZLFUAhgaQQLAKZgSW8gNRAsAJji+FAIf3YAK+M3HIApvEw3BVICwQJA3HUFQ/IHQ5LosQCsjmABIO7CjZsSs0IAqyNYAIi78DCI3Sa50vizA1hZVL/hy5Yt09SpU5WXl6e8vDzNmTNHr7/+erxqA2ARnhNmhNhstgRXAyCeogoWw4cP19KlS7Vx40Zt2LBBl112mRYsWKCdO3fGqz4AFhAOFhn0VwCWF1V79vz583t9ft9992nZsmVau3atKioqYloYAOvwBljOG0gVfZ73FQwG9cILL6ijo0Nz5sw57XE+n08+ny/yudvt7uu3BJCkPGyZDqSMqLuotm/frpycHLlcLv3whz/U8uXLNXny5NMev2TJEuXn50du5eXl/SoYQPLxsk8IkDKiDhYTJ07Uli1btG7dOt1444264YYbtGvXrtMef+edd6q1tTVyq62t7VfBAJJPZJ8QggVgeVEPhTidTo0bN06SNGPGDK1fv16//e1v9eijj57yeJfLJZfL1b8qASS1yHLeNG8CltfvCeWhUKhXDwUAfJqH5byBlBHVb/mdd96pefPmacSIEWpra9PTTz+tlStX6s0334xXfQAswOvvmRXCFQvA8qIKFg0NDfr2t7+tw4cPKz8/X1OnTtWbb76pK664Il71AbCAcI8FzZuA9UUVLP70pz/Fqw4AFsZ0UyB1sGg/gLjzEiyAlEGwABB3LOkNpA6CBYC4O3ETMgDWRrAAEHfsFQKkDoIFgLhjSW8gdRAsAMQds0KA1EGwABB3kXUsaN4ELI9gASDuPAyFACmDYAEg7rzMCgFSBsECQFwZhiGPn1khQKogWACIK38wpJDRfZ+hEMD6CBYA4io8DCLRvAmkAoIFgLgKN26mO2xKd/AnB7A6fssBxFVkRghXK4CUYKlg0dzhT3QJAD6FGSFAarFEsGj1BnTdox/owqXvyN0ZSHQ5AE4QXhyLGSFAarBEsMjLSFNTh1/eQFCvbq1LdDkAThCeasqMECA1WCJY2Gw2/ePMcknS8xsOJrgaACfy0mMBpBRLBAtJ+sr0YUqz27S1tkVVR9oSXQ6AHiznDaQWywSLITkuXX5OsSTphQ21Ca4GQJiHHgsgpVgmWEiKDIcs33xI/q5QgqsBIEneyHLezAoBUoGlgsXFE4pUlOtSY4df73zckOhyAEjy+rtDPkMhQGqwVLBIc9j1tenDJTEcAgwUnkDPFQuaN4GUYKlgIUnXzuwOFu9WNaje3ZngagB4ad4EUorlgsXYohzNHFmokCG9tOlQossBUh6zQoDUYrlgIR1v4nxhQ60Mw0hwNUBqiyzpzVAIkBIsGSy+NHWospwO7T3WoQ37mxNdDpDSji/pzawQIBVYMljkuNJ0zdShkqTn19PECSQSS3oDqcWSwUI6Phzy39sPq93XleBqgNR1fHdTggWQCiwbLGaMLNSYIdny+IP6n22HE10OkLI87BUCpBTLBgubzaZrIxuTMRwCJAqzQoDUYtlgIUlfmz5MDrtNG/Y365Oj7YkuB0hJNG8CqcXSwaI4L0OXTCiSJL3AdupAQtBjAaQWSwcLSZHhkL9sOqiuIBuTAWYKhYzIFQuGQoDUYPlgcdmkYg3Odupom08rq44muhwgpXR2BSP3ad4EUoPlg4Uzza6vnDdMEk2cgNnCjZsSwQJIFZYPFpL0j7O6h0Pe+bhBR9t8Ca4GSB3h/oqMdLvsdluCqwFghpQIFhNKcnVueYG6QoZe3szGZIBZPH5mhACpJiWChXR8Jc7n2ZgMME2kcZNhECBlpEywuGbaUGWk21Xd0K43dx5JdDlASgjvE8JUUyB1pEywyMtI19emD5ck3fTUJv3pvRquXABx5mXVTSDlpEywkKRfzZ+sa2cMV8iQ7n1tl+74y3b5u1jbAogX9gkBUk9KBQtXmkO/+fpU/a+rz5HdJj23oVbf+uM6NbYzUwSIB1bdBFJPVMFiyZIlmjVrlnJzc1VcXKwvf/nLqqqqildtcWGz2fT/fX6M/vydWcp1penDfU36h9+/r4+PuBNdGmA57BMCpJ6ogsWqVau0aNEirV27VitWrFAgENCVV16pjo6OeNUXN5dMLNbyRRdq1OAsHWrx6mv/uUZv0dQJxBQ7mwKpJ6pg8cYbb+g73/mOKioqNG3aND3xxBM6cOCANm7cGK/64mpcca5eXnSRLho3WB3+oH7wfzfqkXf30NQJxIi3Z1YIPRZA6uhXj0Vra6skadCgQTEpJhEKspx64rvn69tzRsowpH9/s0q3PbdFbZ2BRJcGJD0PPRZAyunzwGcoFNJtt92miy66SJWVlac9zufzyec73hzpdg+8XoZ0h13/tqBS40tytfivO/XKljq9tbNe8ypL9fUZw3XBmMEsRwz0gYedTYGU0+dgsWjRIu3YsUPvvffeZx63ZMkS3XPPPX39Nqb6pwtGamxRtv7Xyzu092iHXtp8SC9tPqRhBZn62vRh+ur04Ro1JDvRZQJJg1khQOqxGX1oKLj55pv1yiuvaPXq1Ro9evRnHnuqKxbl5eVqbW1VXl5e9BWbwDAMba5t0YsbD+rVrXVq6+yKPDdrVKG+PmO4vjRlqHIz0iPH+7pC8viD6vB1qcPfpQ5f9/1sl0Ojh+SoMCtdNhtXPZBafvh/NuqNnUd075cr9U8XjEx0OQD6we12Kz8//4zv31FdsTAMQz/60Y+0fPlyrVy58oyhQpJcLpdcLlc03ybhbDabpo8o1PQRhfrVNZP11q56vbjxoN6rPqr1+5q1fl+z7v7rThVmOdXh65LHH1RX6LPzWV5GmkYX5Wj04CyNHpKjUUOyNKbnYzigAFYTHgrJonkTSBlRBYtFixbp6aef1iuvvKLc3FwdOdI9PTM/P1+ZmZlxKTDRMtId+odpZfqHaWU60tqp5ZsP6cWNtfrkaIcOt3ae4ni7sp1pynalKcvpkNsbUF1rp9ydXdpa26KttS0nvWZ4YaauqijV1VOH6rzyAq5swDIis0IYCgFSRlRDIad7w3v88cf1ne9856y+xtleShnIDMPQ7vp2+bqCynKmKceVpiyXQ9nONDlO0eTp9Qe1v6lDNUc7VNPY/XFfY4dqjnXoWLu/17HDCjI1r7I7ZJxLyEAS8/qDmv/797SnoV2Pf3eWLp1YnOiSAPTD2b5/96nHoj+sECxiqdUb0Nq9jfrvbYf19kf16uhpdpO6Q8aXppTqS1MGdsgIBEOqa/Gqtsmr2maPaps8qm326kCTRy0ev4YVZGrk4GyNGpylkYOzNHJwtkYOzmI1xgGkKxjSnqPt2n6wVTsOteqjI20qynFp9phBumDMYI0vzjmrnz93Z0DvfNSgN3Yc0crdDeoMdO/F89JNF2r6iMJ4/2cAiCOCRRLqDAS1suqo/mf7ySGjKNelsoJMDcpKV2GWU4XZTg3Kdqowy6lB2ekqyOq+n5nukDPNLleaPfIxzXHyciX+rpBaPH41ewJq9vjV4vGrxRNQsyegFo9f3kBQwZDR+2YY6goZCoW6P7q9AR1s9upwq1dnaDE5peJcl0b1hIwxRTkaW5StccU5GjEo65Q1x0sgGNL+xg5V17drd327qhvadKzdp7FFOaooy1dFWZ4mluYqwyJ9AoFgSHsa2rX9UHeI2H6oVR8ddkdCwKkMynbq/FGDdMGYQZo9ZrAmluRGpmA3tvu0Yle93th5RO/vOaZA8PgPw/DCTH31vGG6de6EU17NA5A8CBZJ7sSQ8beP6iMLDfWF3da9AZuzJ2x4fF29QkssuNLsKh+UpfLCzJ6PWSoflKmCLKfqWrza1+jR/saOyMcWz+kXIEt32DRqcLbGFuVoXHGOxhZna1xRrorzXMrLSFdGuj2qqzehkKG2zi41efxq9vhV1+JVdX279jS0a3d9m2qOdZyx+dZht2lsUbYmD83rFTbS0+zyd4UUCIYU6DLkDwbl7zLkD4YfC8mV7lB+ZpryMtKVl5kuV9rZ198VDKnDH5TH3yW3t0uN7T4dbfepsd2vxo7uj8fa/TrW7lNjh08tnoAMo3u4zpBkGFKo574MyVB3SDzVf26OK02Ty/I0ZVi+zhmap8MtXq2radLG/c2RPT/CCrLSNWvUILV1BvRhTVOvrzeuOEfzKkt1VUWpKsryBuyVNgDRIVhYSGcgqJ11rWrq6L660Nzh736T7Oi54tDzeYsnIF8gKF9X6IxvlFJ34MjP7L4CUpAV/th9P8vpkMNuU5rdJnv4o637o8Nuk8NuV5bTofJBmSovzFJRriuqN5BWT0D7m7qDxr5jHfrkaPcb/d6jHSe9iX2a02FXXs8bdW5muvIy0pSXma68jHSFQkbPufCrqSN8FcZ/xisqWU6HxhfnaHxJrsYX52hIjkvVDe3aWdeqXXVuNXb4P/sLROFU9dtstu6pyj3TlT2+oNp9XfJ1nf4qQn/kutJUMaw7RFT23EYPzj7lQnD+rpC2H2rR2r1N3UFjX9NJwXTKsHx9sbJUV1WUaFxxblxqBpBYBIsUFwwZ8neF5OsK9nwM34LKdqapIKv7jXigrSgaChmqa/Xqk6Md2tPQ3itwNHX4+jTkEpbtdKgw26miXFd3iCjO1fiS7jAxNC/jtOfCMAw1tPkiIWNnz+1AkydyTLrDpnRH9xWhdIddzsh9mzoDIbk7A3J7A32uP81uU25GmobkuDQ4x6nBOS4V5bg0OLv7/uAcp4bkuFSQlS6HzSabTbKp+6Ok7s9tNtnUffWlKMfV5//3XcGQdtS5tb6mSekOm+ZOLtHwwqy+/YcBSBoEC1iOYRjq8Afl9gZ63qi7TrgfUKu3S3abIv0nBVnpkT6Ugqx0udJi2yPRGQjKZpPS7fazepM+Xf2t3oAMSTkuh7J6pipn98wyCs84inXtABCtuCyQBSSSzWZTjqv7zbZMiV83JdpmzoFWPwDEg3mt9wAAwPIIFgAAIGYIFgAAIGYIFgAAIGYIFgAAIGYIFgAAIGYIFgAAIGYIFgAAIGYIFgAAIGYIFgAAIGYIFgAAIGYIFgAAIGYIFgAAIGZM3900vEu72+02+1sDAIA+Cr9vh9/HT8f0YNHW1iZJKi8vN/tbAwCAfmpra1N+fv5pn7cZZ4oeMRYKhVRXV6fc3FzZbLaYfV23263y8nLV1tYqLy8vZl8Xp8b5Nhfn21ycb3Nxvs3V1/NtGIba2tpUVlYmu/30nRSmX7Gw2+0aPnx43L5+Xl4eP5gm4nybi/NtLs63uTjf5urL+f6sKxVhNG8CAICYIVgAAICYsUywcLlcuvvuu+VyuRJdSkrgfJuL820uzre5ON/mivf5Nr15EwAAWJdlrlgAAIDEI1gAAICYIVgAAICYIVgAAICYsUyweOSRRzRq1ChlZGRo9uzZ+vDDDxNdkiWsXr1a8+fPV1lZmWw2m15++eVezxuGoV/96lcaOnSoMjMzNXfuXFVXVyem2CS3ZMkSzZo1S7m5uSouLtaXv/xlVVVV9Tqms7NTixYt0uDBg5WTk6Ovfe1rqq+vT1DFyW/ZsmWaOnVqZKGgOXPm6PXXX488z/mOn6VLl8pms+m2226LPMb5jq3FixfLZrP1uk2aNCnyfLzOtyWCxXPPPafbb79dd999tzZt2qRp06bpqquuUkNDQ6JLS3odHR2aNm2aHnnkkVM+/5vf/EYPP/yw/uu//kvr1q1Tdna2rrrqKnV2dppcafJbtWqVFi1apLVr12rFihUKBAK68sor1dHRETnmxz/+sV599VW98MILWrVqlerq6vTVr341gVUnt+HDh2vp0qXauHGjNmzYoMsuu0wLFizQzp07JXG+42X9+vV69NFHNXXq1F6Pc75jr6KiQocPH47c3nvvvchzcTvfhgWcf/75xqJFiyKfB4NBo6yszFiyZEkCq7IeScby5csjn4dCIaO0tNT493//98hjLS0thsvlMp555pkEVGgtDQ0NhiRj1apVhmF0n9v09HTjhRdeiBzz0UcfGZKMDz74IFFlWk5hYaHxxz/+kfMdJ21tbcb48eONFStWGBdffLFx6623GobBz3c83H333ca0adNO+Vw8z3fSX7Hw+/3auHGj5s6dG3nMbrdr7ty5+uCDDxJYmfXV1NToyJEjvc59fn6+Zs+ezbmPgdbWVknSoEGDJEkbN25UIBDodb4nTZqkESNGcL5jIBgM6tlnn1VHR4fmzJnD+Y6TRYsW6eqrr+51XiV+vuOlurpaZWVlGjNmjBYuXKgDBw5Iiu/5Nn0Tslg7duyYgsGgSkpKej1eUlKijz/+OEFVpYYjR45I0inPffg59E0oFNJtt92miy66SJWVlZK6z7fT6VRBQUGvYznf/bN9+3bNmTNHnZ2dysnJ0fLlyzV58mRt2bKF8x1jzz77rDZt2qT169ef9Bw/37E3e/ZsPfHEE5o4caIOHz6se+65R5///Oe1Y8eOuJ7vpA8WgBUtWrRIO3bs6DUeiviYOHGitmzZotbWVr344ou64YYbtGrVqkSXZTm1tbW69dZbtWLFCmVkZCS6nJQwb968yP2pU6dq9uzZGjlypJ5//nllZmbG7fsm/VDIkCFD5HA4Tupkra+vV2lpaYKqSg3h88u5j62bb75Zr732mt59910NHz488nhpaan8fr9aWlp6Hc/57h+n06lx48ZpxowZWrJkiaZNm6bf/va3nO8Y27hxoxoaGjR9+nSlpaUpLS1Nq1at0sMPP6y0tDSVlJRwvuOsoKBAEyZM0J49e+L68530wcLpdGrGjBl6++23I4+FQiG9/fbbmjNnTgIrs77Ro0ertLS017l3u91at24d574PDMPQzTffrOXLl+udd97R6NGjez0/Y8YMpaen9zrfVVVVOnDgAOc7hkKhkHw+H+c7xi6//HJt375dW7ZsidxmzpyphQsXRu5zvuOrvb1dn3zyiYYOHRrfn+9+tX4OEM8++6zhcrmMJ554wti1a5fx/e9/3ygoKDCOHDmS6NKSXltbm7F582Zj8+bNhiTjwQcfNDZv3mzs37/fMAzDWLp0qVFQUGC88sorxrZt24wFCxYYo0ePNrxeb4IrTz433nijkZ+fb6xcudI4fPhw5ObxeCLH/PCHPzRGjBhhvPPOO8aGDRuMOXPmGHPmzElg1cntjjvuMFatWmXU1NQY27ZtM+644w7DZrMZb731lmEYnO94O3FWiGFwvmPtJz/5ibFy5UqjpqbGeP/99425c+caQ4YMMRoaGgzDiN/5tkSwMAzD+N3vfmeMGDHCcDqdxvnnn2+sXbs20SVZwrvvvmtIOul2ww03GIbRPeX0rrvuMkpKSgyXy2VcfvnlRlVVVWKLTlKnOs+SjMcffzxyjNfrNW666SajsLDQyMrKMr7yla8Yhw8fTlzRSe573/ueMXLkSMPpdBpFRUXG5ZdfHgkVhsH5jrdPBwvOd2xdd911xtChQw2n02kMGzbMuO6664w9e/ZEno/X+WbbdAAAEDNJ32MBAAAGDoIFAACIGYIFAACIGYIFAACIGYIFAACIGYIFAACIGYIFAACIGYIFAACIGYIFAACIGYIFAACIGYIFAACIGYIFAACImf8H5Il4VZLiBswAAAAASUVORK5CYII=\n", "text/plain": [ "
" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "plt.plot(torch.tensor(l).view(-1, 10).mean(1).numpy())" ] }, { "cell_type": "code", "execution_count": 13, "metadata": { "id": "0C7dGZoH1wYs" }, "outputs": [], "source": [ "# 使用更精细的训练方式微调大语言模型\n", "# get_lr的实现参考自https://github.com/karpathy/nanoGPT/blob/master/train.py\n", "import math\n", "\n", "learning_rate = 6e-4\n", "warmup_iters = 200\n", "lr_decay_iters = 6000\n", "min_lr = learning_rate / 10\n", "\n", "def get_lr(it):\n", " '''\n", " 动态调整学习速率\n", " it表示训练次数\n", " '''\n", " # 1、线性预热\n", " if it < warmup_iters:\n", " return learning_rate * it / warmup_iters\n", " # 2、超出lr_decay_iters，则返回min_lr\n", " if it > lr_decay_iters:\n", " return min_lr\n", " # 3、逐步衰减学习速率\n", " decay_ratio = (it - warmup_iters) / (lr_decay_iters - warmup_iters)\n", " assert 0 <= decay_ratio <= 1\n", " coeff = 0.5 * (1.0 + math.cos(math.pi * decay_ratio))\n", " return min_lr + coeff * (learning_rate - min_lr)" ] }, { "cell_type": "code", "execution_count": 14, "metadata": { "id": "wKhgo30-i1Yy" }, "outputs": [], "source": [ "# 梯度裁剪的超参数\n", "grad_clip = 1.0\n", "\n", "def train_gpt_optimum(model, optimizer, data_loader, max_iters=1000):\n", " lossi = []\n", " scaler = torch.cuda.amp.GradScaler(enabled=(device == 'cuda'))\n", " for iter_num in range(max_iters):\n", " # 动态调整学习率\n", " lr = get_lr(iter_num + 1)\n", " for param_group in optimizer.param_groups:\n", " param_group['lr'] = lr\n", " # 梯度累积\n", " for i in range(gra_acc_steps):\n", " inputs, labels = data_loader()\n", " # 混合精度训练\n", " ## 如果是用CPU进行计算，可能需要将dtype变成torch.bfloat16\n", " ## 当然如果使用CPU，需要非常长的时间\n", " ctx = torch.autocast(device_type=device, dtype=torch.float16)\n", " with ctx:\n", " logits = model(inputs).logits\n", " logits = logits.transpose(-2, -1)\n", " loss = F.cross_entropy(logits, labels)\n", " lossi.append(loss.item())\n", " loss *= 1 / gra_acc_steps\n", " scaler.scale(loss).backward()\n", " # 梯度裁剪\n", " scaler.unscale_(optimizer)\n", " clip_grad_norm_(model.parameters(), grad_clip)\n", " scaler.step(optimizer)\n", " scaler.update()\n", " optimizer.zero_grad(set_to_none=True)\n", "\n", " if iter_num % eval_interval == 0:\n", " # 预估模型损失时，也使用混合精度\n", " stats = estimate_loss(model, ctx)\n", " train_loss = f'train loss {stats[\"train\"]:.4f}'\n", " test_loss = f'test loss {stats[\"test\"]:.4f}'\n", " print(f'step {iter_num:>4}: {train_loss}, {test_loss}')\n", " return lossi" ] }, { "cell_type": "code", "execution_count": 15, "metadata": { "id": "0VeCUdN9o6qX" }, "outputs": [], "source": [ "# 调整批量数据的大小（如需要）\n", "batch_size = 4\n", "# 定义梯度累积的步数\n", "gra_acc_steps = 8 * 2\n", "# 重新定义data loader\n", "data_loader = lambda: get_data(train_set, batch_size, sequence_len)" ] }, { "cell_type": "code", "execution_count": 16, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "k9JRRMu9TZ74", "outputId": "5b81c1e5-86f2-486c-d50b-3b2dba142a8c" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "step 0: train loss 2.6000, test loss 2.6805\n", "step 50: train loss 2.1166, test loss 2.0954\n", "step 100: train loss 1.9085, test loss 1.9194\n", "step 150: train loss 1.7128, test loss 1.7238\n", "step 200: train loss 1.6530, test loss 1.6679\n", "step 250: train loss 1.6406, test loss 1.7275\n", "step 300: train loss 1.6570, test loss 1.6419\n", "step 350: train loss 1.6033, test loss 1.5997\n", "step 400: train loss 1.6507, test loss 1.6697\n", "step 450: train loss 1.6581, test loss 1.6455\n", "step 500: train loss 1.5530, test loss 1.6089\n", "step 550: train loss 1.5932, test loss 1.5903\n", "step 600: train loss 1.6575, test loss 1.5049\n", "step 650: train loss 1.6352, test loss 1.6351\n", "step 700: train loss 1.5718, test loss 1.5704\n", "step 750: train loss 1.5263, test loss 1.6099\n", "step 800: train loss 1.5926, test loss 1.6091\n", "step 850: train loss 1.6165, test loss 1.6354\n", "step 900: train loss 1.5676, test loss 1.6137\n", "step 950: train loss 1.5986, test loss 1.5285\n" ] } ], "source": [ "# 卸载LoRA适配器\n", "model.unload()\n", "# 清空GPU缓存\n", "torch.cuda.empty_cache()\n", "# 重新安装LoRA适配器\n", "model = init_peft_model(model)\n", "# 设置最优化算法的参数\n", "weight_decay = 1e-1\n", "beta1 = 0.9\n", "beta2 = 0.95\n", "optimizer = optim.AdamW(model.parameters(), lr=learning_rate,\n", " betas=(beta1, beta2), weight_decay=weight_decay)\n", "l = train_gpt_optimum(model, optimizer, data_loader, max_iters=1000)" ] }, { "cell_type": "code", "execution_count": 17, "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 448 }, "id": "pCojwhGyyRqL", "outputId": "7328824b-f75a-4c11-c192-52c150d26a3a" }, "outputs": [ { "data": { "text/plain": [ "[]" ] }, "execution_count": 17, "metadata": {}, "output_type": "execute_result" }, { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAiMAAAGdCAYAAADAAnMpAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAABZUUlEQVR4nO3dd1hT5wIG8DcJEHYQFRBBoS5U3OLCVev22tpprVdrte21xaodtlq7q8X2di/t1N621tbW0ap11D1QC4qK2zpABZyEvZJz/0DCOWRDyCHw/p6H5yFnJN9h5Lz5pkIQBAFEREREMlHKXQAiIiKq3xhGiIiISFYMI0RERCQrhhEiIiKSFcMIERERyYphhIiIiGTFMEJERESyYhghIiIiWbnJXQBb6PV6XL58GX5+flAoFHIXh4iIiGwgCAJycnIQGhoKpdJ8/YdLhJHLly8jPDxc7mIQERFRFaSlpSEsLMzsfpcII35+fgDKLsbf31/m0hAREZEtsrOzER4ebriPm+MSYaS8acbf359hhIiIyMVY62LBDqxEREQkK4YRIiIikhXDCBEREcmKYYSIiIhkxTBCREREsmIYISIiIlkxjBAREZGsGEaIiIhIVgwjREREJCuGESIiIpIVwwgRERHJimGEiIiIZFXvw4ggCPhuz3kcSL0pd1GIiIjqJZdYtbcmbTqWiVd/PwoAOL9glMylISIiqn/qfc3IgdQsw/evrE6RryBERET1VL0OI3lFpVi0/R/D4/8lXJCxNERERPVTvQ4jZ67kyl0EIiKieq9eh5G84lKjbXq9AEEQZCgNERFR/VSvw8j13GKjbZeyCtArfjPe23hShhIRERHVP/U6jBSU6Iy2fb7tH2RmF+GTLWdkKBEREVH9U6/DiE5v3Byj0+tlKAkREVH9xTBSyS+JF2UoCRERUf3FMEJERESysiuMLFy4EB07doS/vz/8/f3Ru3dv/PnnnxbPWb58OaKiouDp6YkOHTpg3bp11SqwIzGMEBERyc+uMBIWFoYFCxYgKSkJiYmJGDRoEO666y4cPXrU5PF79uzBuHHjMGXKFBw8eBBjxozBmDFjkJJSO2Y6tRZGOMSXiIio5imEat5xAwMD8d///hdTpkwx2jd27Fjk5eVhzZo1hm29evVC586dsWjRIptfIzs7GxqNBlqtFv7+/tUprsTCbf/g7fUnzO4/OW841G4qh70eERFRfWLr/bvKfUZ0Oh2WLVuGvLw89O7d2+QxCQkJGDx4sGTbsGHDkJCQYPG5i4qKkJ2dLfmqCXorOayg2HjoLxERETmW3WHkyJEj8PX1hVqtxtSpU7Fy5Uq0a9fO5LEZGRkIDg6WbAsODkZGRobF14iPj4dGozF8hYeH21tMm5TqLIeRy1mFNfK6REREVMHuMNKmTRskJydj3759eOKJJ/Dwww/j2LFjDi3UnDlzoNVqDV9paWkOff5yOis1I6k38mrkdYmIiKiCm70neHh4oGXLlgCAbt264e+//8ZHH32EL774wujYkJAQZGZmSrZlZmYiJCTE4muo1Wqo1Wp7i2Y3axOc5bOZhoiIqMZVe54RvV6PoqIik/t69+6NzZs3S7Zt2rTJbB8TZyu1MprG2n4iIiKqPrtqRubMmYMRI0agWbNmyMnJwdKlS7Ft2zZs2LABADBx4kQ0bdoU8fHxAIAZM2ZgwIABeO+99zBq1CgsW7YMiYmJ+PLLLx1/JVWgtxI2OA8JERFRzbMrjFy5cgUTJ05Eeno6NBoNOnbsiA0bNmDIkCEAgNTUVCiVFZUtffr0wdKlS/HSSy/hxRdfRKtWrbBq1SpER0c79iqqiDUjRERE8rMrjHzzzTcW92/bts1o2/3334/777/frkI5i9WaER0XzSMiIqpp9XptGms1Hyczc51UEiIiovqrXocRa5Oe/bQ/FbvPXHNSaYiIiOqneh1GrE16BgDjv96HvKJSJ5SGiIiofqrXYaR8tMzDvZvD39N89xnON0JERFRz6ncYudVM06yhD/7dq7nZ43IKS5xVJCIionqnXoeR8g6sKgXgrqr4UfippbUkk5f87dRyERER1Sd2Twdfl3zwQGe8c29HuKkUKCzWY8PRDIzuFIrv9pxHjqifyPnr+TKWkoiIqG6r12HEw00JD7eyGhG1mwrrZ/YHAPy494KcxSIiIqpX6nUzjTlKpULuIhAREdUbDCMmKJhFiIiInIZhxAQrc6ERERGRAzGMmMAwQkRE5DwMIyZYmyaeiIiIHIdhxISnh7Q22pZfzCnhiYiIagLDiAkPdA832ha/7oQMJSEiIqr7GEbMCPB2lzz+NemiTCUhIiKq2xhGzIho6CN5XFCiMyysR0RERI7DMGJGs0Bvo225Rew3QkRE5GgMI2bMHdXWaBtX7yUiInI8hhEzgv09jbbNWXFEhpIQERHVbQwjdth5+prcRSAiIqpzGEaIiIhIVgwjREREJCuGEQueqTQTa6/bAmUqCRERUd3FMGLB9DtaYWi7YMPjghK9jKUhIiKqmxhG7FBYrJO7CERERHUOw4gV4jlXC0oYRoiIiByNYcQKQZRGOB08ERGR4zGMWDFzcCvD96V69hkhIiJyNIYRK6KbarAqLhYAa0aIiIhqAsOIDXzVbgCAUoYRIiIih2MYsYFKqQAA6HQMI0RERI7GMGIDt1thhDUjREREjscwYgNDzYjAMEJERORoDCM2KK8ZYQdWIiIix7MrjMTHxyMmJgZ+fn4ICgrCmDFjcPLkSavnffjhh2jTpg28vLwQHh6Op59+GoWFhVUutLOpRGFEYO0IERGRQ9kVRrZv3464uDjs3bsXmzZtQklJCYYOHYq8vDyz5yxduhSzZ8/Gq6++iuPHj+Obb77Bzz//jBdffLHahXcWN2XFj4m1I0RERI7lZs/B69evlzxesmQJgoKCkJSUhP79+5s8Z8+ePYiNjcVDDz0EAIiIiMC4ceOwb9++KhbZ+URZBKV6AW4q+cpCRERU11Srz4hWqwUABAYGmj2mT58+SEpKwv79+wEAZ8+exbp16zBy5Eiz5xQVFSE7O1vyJSfWjBAREdUcu2pGxPR6PWbOnInY2FhER0ebPe6hhx7CtWvX0LdvXwiCgNLSUkydOtViM018fDxef/31qhbN4cr7jADAuWt5iG6qkbE0REREdUuVa0bi4uKQkpKCZcuWWTxu27ZteOutt/D555/jwIEDWLFiBdauXYs333zT7Dlz5syBVqs1fKWlpVW1mA7hJgojcUsPyFgSIiKiuqdKNSPTpk3DmjVrsGPHDoSFhVk89uWXX8aECRPw6KOPAgA6dOiAvLw8PP7445g7dy6USuM8pFaroVarq1K0GqEUhZEL1/NlLAkREVHdY1cYEQQBTz31FFauXIlt27YhMjLS6jn5+flGgUOlUhmej4iIiOo3u5pp4uLi8MMPP2Dp0qXw8/NDRkYGMjIyUFBQYDhm4sSJmDNnjuHx6NGjsXDhQixbtgznzp3Dpk2b8PLLL2P06NGGUOJqpn6fBD07shIRETmEXTUjCxcuBAAMHDhQsn3x4sWYNGkSACA1NVVSE/LSSy9BoVDgpZdewqVLl9C4cWOMHj0a8+fPr17JZbT+aAYOpN5E9wjzo4iIiIjINgrBBdpKsrOzodFooNVq4e/vL0sZImavlTz+YUpP9G3VSJayEBERuQJb799cm8ZGW58bKHnMRfOIiIgcg2HERpGNfNC2SUWq0zOMEBEROQTDiB06hwcYvl+2P1W+ghAREdUhDCN28HSv+HFtOJopY0mIiIjqDoYROygVCusHERERkV0YRuwgXqOGiIiIHINhxA6MIkRERI7HMGIHBZtpiIiIHI5hxA7iVpqoED/5CkJERFSHMIzYQdyBlZ1ZiYiIHINhxA7impFSvV6+ghAREdUhDCN2EPcZKdVxBlYiIiJHYBixg7hlpoQ1I0RERA7BMGKH0Z1CDd+zZoSIiMgxGEbs0KKxL/43uQcAoFTPMEJEROQIDCN2aqLxBACU6thMQ0RE5AgMI3ZyU5X9yNhMQ0RE5BgMI3ZyuzW+lx1YiYiIHINhxE5qt7IfWXGpHoLA2hEiIqLqYhixk6eHCgCgF4Bi9hshIiKqNoYRO3m5qwzfF5YwjBAREVUXw4id3FUVP7LliWkyloSIiKhuYBiphnlrj8tdBCIiIpfHMEJERESyYhipJh1nYiUiIqoWhpFqysovlrsIRERELo1hpJpu5DGMEBERVQfDSDVdZxghIiKqFoaRamLNCBERUfUwjFRTTmGJ3EUgIiJyaQwj1VTK0TRERETVwjBSTaU6hhEiIqLqYBipJtaMEBERVQ/DSDWVcuVeIiKiamEYqSbWjBAREVWPXWEkPj4eMTEx8PPzQ1BQEMaMGYOTJ09aPS8rKwtxcXFo0qQJ1Go1WrdujXXr1lW50LUJ+4wQERFVj5s9B2/fvh1xcXGIiYlBaWkpXnzxRQwdOhTHjh2Dj4+PyXOKi4sxZMgQBAUF4ddff0XTpk1x4cIFBAQEOKL8sivVs5mGiIioOuwKI+vXr5c8XrJkCYKCgpCUlIT+/fubPOfbb7/FjRs3sGfPHri7uwMAIiIiqlbaWmLuyLaYv+44ADbTEBERVVe1+oxotVoAQGBgoNljfv/9d/Tu3RtxcXEIDg5GdHQ03nrrLeh0OrPnFBUVITs7W/JVmzzW/zbc1y0MADuwEhERVVeVw4her8fMmTMRGxuL6Ohos8edPXsWv/76K3Q6HdatW4eXX34Z7733HubNm2f2nPj4eGg0GsNXeHh4VYtZY4L81ABYM0JERFRdVQ4jcXFxSElJwbJlyywep9frERQUhC+//BLdunXD2LFjMXfuXCxatMjsOXPmzIFWqzV8paWlVbWYNcZNVfajYwdWIiKi6qlSGJk2bRrWrFmDrVu3IiwszOKxTZo0QevWraFSqQzb2rZti4yMDBQXm15kTq1Ww9/fX/JV2whCWQj5fu8FmUtCRETk2uwKI4IgYNq0aVi5ciW2bNmCyMhIq+fExsbizJkz0ItGnZw6dQpNmjSBh4eH/SWuJfafu2H4vjyYEBERkf3sCiNxcXH44YcfsHTpUvj5+SEjIwMZGRkoKCgwHDNx4kTMmTPH8PiJJ57AjRs3MGPGDJw6dQpr167FW2+9hbi4OMddhQyKRR1X84rNd8YlIiIiy+wa2rtw4UIAwMCBAyXbFy9ejEmTJgEAUlNToVRWZJzw8HBs2LABTz/9NDp27IimTZtixowZeOGFF6pXcpl5uVc0O93MK4av2q4fJREREd2iEFygjSE7OxsajQZarbbW9B85lZmDoR/sAAD8Ma0vOoRpZC4RERFR7WLr/Ztr01RR62A/3Na4bNbZq7mFMpeGiIjIdTGMVIPHreG9k5ckorCE/UaIiIiqgmGkGhQKheH7y1kFFo4kIiIicxhGqkFZkUWg40ysREREVcIwUg1KUc2Irvb3AyYiIqqVGEaqQZRFUFzKBfOIiIiqgmGkGsR9Rgo48RkREVGVMIxUg0pUM1LImhEiIqIqYRipBnHNyNWcIhlLQkRE5LoYRqpBJQojzy0/xKYaIiKiKmAYqQa1u/TH98fhyzKVhIiIyHUxjFSD2k0lecy5RoiIiOzHMFINnu788REREVUX76bVULlmhIiIiOzHMFINQ9oFSx4rzBxHRERE5jGMVMOw9sF4MCbc8FjBNEJERGQ3hpFqUCgU6N+6sdzFICIicmkMI0RERCQrhhEH4sheIiIi+zGMOFCpjuvTEBER2YthpJoEUW1IsY5VI0RERPZiGKkm8cRnJawZISIishvDSDUNbBNk+D6vqFTGkhAREbkmhpFqUikVmD0iCgBw4Xq+zKUhIiJyPQwjDtAs0BsAcCmrQOaSEBERuR6GEQfw83QDwGYaIiKiqmAYcQBvj7Iwkl+sk7kkRERErodhxAF81GWr9+YXs2aEiIjIXgwjDuDjUd5Mw5oRIiIiezGMOICXR1nNSEGJDnrOCU9ERGQXhhEHKK8ZAcoCCREREdmOYcQBPN2VUCjKvv9482l8n3Be1vIQERG5Ejfrh5A1CoUCPh5uyC0qxRc7zgIAxsY0g4cbsx4REZE1vFs6SHm/kXKleq5TQ0REZAu7wkh8fDxiYmLg5+eHoKAgjBkzBidPnrT5/GXLlkGhUGDMmDH2lrPWu5pTJHlcwhV8iYiIbGJXGNm+fTvi4uKwd+9ebNq0CSUlJRg6dCjy8vKsnnv+/Hk899xz6NevX5UL60q4gi8REZFt7Oozsn79esnjJUuWICgoCElJSejfv7/Z83Q6HcaPH4/XX38dO3fuRFZWVpUK60pKWTNCRERkk2r1GdFqtQCAwMBAi8e98cYbCAoKwpQpU6rzcrXa1xO7Sx6zZoSIiMg2VR5No9frMXPmTMTGxiI6Otrscbt27cI333yD5ORkm5+7qKgIRUUVfTCys7OrWkyn6da8geTxhqMZeLTfbTKVhoiIyHVUuWYkLi4OKSkpWLZsmdljcnJyMGHCBHz11Vdo1KiRzc8dHx8PjUZj+AoPD69qMZ1G7S79Uc5bexxJF27IVBoiIiLXoRAEwe7ODdOmTcPq1auxY8cOREZGmj0uOTkZXbp0gUpVMexVf2vIq1KpxMmTJ9GiRQuj80zVjISHh0Or1cLf39/e4jpFqU6PlnP/NNp+fsEoGUpDREQkv+zsbGg0Gqv3b7uaaQRBwFNPPYWVK1di27ZtFoMIAERFReHIkSOSbS+99BJycnLw0Ucfma3xUKvVUKvV9hRNdm4qTtlCRERUFXaFkbi4OCxduhSrV6+Gn58fMjIyAAAajQZeXl4AgIkTJ6Jp06aIj4+Hp6enUX+SgIAAALDYz4SIiIjqD7vCyMKFCwEAAwcOlGxfvHgxJk2aBABITU2FUslaAiIiIrKN3c001mzbts3i/iVLltjzki7l372a4Ye9qXIXg4iIyKWwCsOB5o3pIHcRiIiIXA7DiINNu72l3EUgIiJyKQwjDvbcsDZyF4GIiMilMIwQERGRrBhGatjdn+/mOjVEREQWMIzUsIOpWdh3ltPCExERmcMw4gQ384vlLgIREVGtxTDiBLlFpXIXgYiIqNZiGHGCOSuOoLBEJ3cxiIiIaiWGESf5cR9nZiUiIjKFYcRJcgpL5C4CERFRrcQwUgN+nxaLpgFekm0KKGQqDRERUe3GMFIDOoYFYPfsQbi/W5hh255/rrEjKxERkQkMIzVIpayoDdl37gYeWbxfxtIQERHVTgwjNUiplDbN/H3+pkwlISIiqr0YRmqQIMhdAiIiotqPYaQG6fRck4aIiMgahpEaVKpn1QgREZE1DCM1SM8wQkREZBXDSA0qYRghIiKyimGkBul0DCNERETWMIzUIFN9RgpLdLiSUyhDaYiIiGonhpEaZGo0TdTL69Fj/mZcvJkvQ4mIiIhqH4aRGmSplWbn6WvOKwgREVEtxjBSgyzNM8Jl84iIiMowjNSgUgtVIwqmESIiIgAMIzWKk54RERFZxzBSgyyFEQUbaoiIiAAwjNSoYD+13EUgIiKq9dzkLkBd9sZd0SjVC+h9W0PMX3dcupMVI0RERABYM1KjQjSe+HZSDB7rf5vRvq93nkWJjqv6EhERMYw4iZe7SvL4VGYu/pdwQabSEBER1R4MI07yzaTuRtveXHMM+cWlMpSGiIio9mAYcZLophqT299Zf9LJJSEiIqpdGEacxE9tuq/wpmOZTi4JERFR7WJXGImPj0dMTAz8/PwQFBSEMWPG4ORJy5/sv/rqK/Tr1w8NGjRAgwYNMHjwYOzfv79ahXZFCjNTrqrdmQeJiKh+s+tOuH37dsTFxWHv3r3YtGkTSkpKMHToUOTl5Zk9Z9u2bRg3bhy2bt2KhIQEhIeHY+jQobh06VK1C18XnL2ah7ilB5ChLZS7KERERLJQCIJQ5TnLr169iqCgIGzfvh39+/e36RydTocGDRrg008/xcSJE206Jzs7GxqNBlqtFv7+/lUtruwiZq81u29QVBC+nRTjxNIQERHVLFvv39VqI9BqtQCAwMBAm8/Jz89HSUmJxXOKioqQnZ0t+aoLPhnXxey+LSeuYOPRDCeWhoiIqHaochjR6/WYOXMmYmNjER0dbfN5L7zwAkJDQzF48GCzx8THx0Oj0Ri+wsPDq1rMWmV0p1AsfbQnWgb5mtz/+PdJ0HNxPSIiqmeqHEbi4uKQkpKCZcuW2XzOggULsGzZMqxcuRKenp5mj5szZw60Wq3hKy0trarFrHX6tGyEv54ZgPcf6GRy/3PLDzm5RERERPKqUhiZNm0a1qxZg61btyIsLMymc959910sWLAAGzduRMeOHS0eq1ar4e/vL/mqazwrzchabsVBduwlIqL6xa6F8gRBwFNPPYWVK1di27ZtiIyMtOm8d955B/Pnz8eGDRvQvbvxTKT1kUrJlfKIiIgAO8NIXFwcli5ditWrV8PPzw8ZGWUdLjUaDby8vAAAEydORNOmTREfHw8AePvtt/HKK69g6dKliIiIMJzj6+sLX1/TfSfqAzcLYUQQBLPzkhAREdU1djXTLFy4EFqtFgMHDkSTJk0MXz///LPhmNTUVKSnp0vOKS4uxn333Sc5591333XcVbggSzUjoz/d5cSSEBERycvuZhprtm3bJnl8/vx5e16i3rAURlIuZSNDW4gQjflOvkRERHUF5yKXibU+I9dyi5xUEiIiInkxjMjETWn5R6+v+sS4RERELoVhRCbWakZ0nPyMiIjqCYYRmVgaTQMARaV6J5WEiIhIXgwjMrFWM1JQonNSSYiIiOTFMCITN5XlMJJ2Ix8Z2kInlYaIiEg+DCMyUVmZ1OyV1UfRK34zCopZQ0JERHUbw4hMxM0093Uzv74Ph/gSEVFdxzAiE/HQ3lnD2pg9jmvYEBFRXccwIhPxNCOWRtZwvhEiIqrrGEZkIs4Ybiol1k3vZ/K4Eh3DCBER1W0MIzIRhxF3lQLtQv3x3NDWaOSrlhxXquN8I0REVLcxjMhE7V7xoy/vFzJtUCtM6NVcchxrRoiIqK6za9Vecpxgf09Mv6MVvNxVULupDNu9PKT5MOWSFu1C/Z1dPCIiIqdhGJHRM0NaG23zUUt/Jc//dhhRTfzQMSzASaUiIiJyLjbT1DKB3h5G2/674aQMJSEiInIOhpFaxt/L3WhbMRfNIyKiOoxhpJZRuxn/SvSCgFOZOej79ha8ueaYDKUiIiKqOQwjtUzXZg1wb1fp9PB/n7+JoR/swMWbBfhm1zmZSkZERFQzGEZqGaVSgfce6CR3MYiIiJyGYYSIiIhkxTBSS80bEy13EYiIiJyCYaSWCvQxHuJLRERUFzGM1FJcrZeIiOoLhpFailmEiIjqC4aRWqp7RAO5i0BEROQUDCO1VBONF3a9cDs6hmmM9m05kYmJ3+5HurZAhpIRERE5FsNILRbWwBsfju1stH3ykkTsOHUVr/1+1PmFIiIicjCGkVouoqGP2X038oqdWBIiIqKawTBSyymVCvzyn94m97mr+OsjIiLXx7uZC+gRGWhyu4eJRfWIiIhcDe9mLsyDNSNERFQH8G7mImbc0cpo26UsjqYhIiLXxzDiIp4e0tpo29HL2Zi57CBSr+fLUCIiIiLHYBhxcauSL2PkxzvlLgYREVGV2RVG4uPjERMTAz8/PwQFBWHMmDE4efKk1fOWL1+OqKgoeHp6okOHDli3bl2VC0zGcotK5S4CERFRldkVRrZv3464uDjs3bsXmzZtQklJCYYOHYq8vDyz5+zZswfjxo3DlClTcPDgQYwZMwZjxoxBSkpKtQtf3/ip3eQuAhERkcMpBKHqS7JdvXoVQUFB2L59O/r372/ymLFjxyIvLw9r1qwxbOvVqxc6d+6MRYsW2fQ62dnZ0Gg00Gq18Pf3r2pxXV7sgi1mO62eXzDKyaUhIiKyzNb7d7X6jGi1WgBAYKDpeTAAICEhAYMHD5ZsGzZsGBISEsyeU1RUhOzsbMkXAe4qhdxFICIicrgqhxG9Xo+ZM2ciNjYW0dHRZo/LyMhAcHCwZFtwcDAyMjLMnhMfHw+NRmP4Cg8Pr2ox6xSVkmGEiIjqniqHkbi4OKSkpGDZsmWOLA8AYM6cOdBqtYavtLQ0h7+GK7I0/TvXqSEiIldVpTAybdo0rFmzBlu3bkVYWJjFY0NCQpCZmSnZlpmZiZCQELPnqNVq+Pv7S74IcLPQTPP8r4ecWBIiIiLHsSuMCIKAadOmYeXKldiyZQsiIyOtntO7d29s3rxZsm3Tpk3o3dv04m9knpvS/K9r15lrTiwJERGR49g1VjQuLg5Lly7F6tWr4efnZ+j3odFo4OXlBQCYOHEimjZtivj4eADAjBkzMGDAALz33nsYNWoUli1bhsTERHz55ZcOvpS6z9PdfBjx9uCwXyIick121YwsXLgQWq0WAwcORJMmTQxfP//8s+GY1NRUpKenGx736dMHS5cuxZdffolOnTrh119/xapVqyx2eiXT5o0x/zPzclc5sSRERESOU615RpyF84xU0OsFnL6Si4+3nMbawxWhr0VjH2x+dqB8BSMiIqrEKfOMkPMplQq0CfGDe6Vhvv9czcMfhy7DBbIlERGRBMOIi9pwNNNo21M/HUTknHX4Ye8FGUpERERUNQwjLqqgRGd230uruO4PERG5DoaROk6nZ7MNERHVbgwjddimY5no+NoGrE9Jt34wERGRTBhG6rDH/peIvGIdpv5wQO6iEBERmcUw4qK+ebi73EUgIiJyCIYRF9Uu1PJ8K+wrQkREroJhxEV5WFjBFwBKdHonlYSIiKh6GEZclNrK9O/FDCNEROQiGEZclNWakVKGESIicg0MIy7KXaWwuP/0lVwnlYSIiKh6GEZclEKhwNJHe2JIu2CT+x/8cq+TS0RERFQ1bnIXgKquT8tG6HlbQ/ySmIZl+1Nx6KJW7iIRERHZjTUjLk6lVGBcj2Zo0dhX7qIQERFVCcMIERERyYphpI6wNsXZuiPp+OXvNKeUhYiIyB7sM1JPPPlj2fo0A9o0RrC/p8ylISIiqsCakTpCEGyb/j2/WGfzc3IWVyIicgaGkTpoWHvTw30BYM8/12x6jgvX8xD96ga8/sdRRxWLiIjIJIaROkJcL9KnRSOzx81dmWJTLcpnW8+gqFSPxbvPV79wREREFjCM1EHmJkIr99j/kiAIAgRBwOWsAmw6lmlzMw8REZGjsQNrHSHOEqEBXmgV5Gt2Svi/jmcics46dG/eAIkXbgIAPh/fFSM7NDEco4Dl6eaJiIgchTUjddS4Hs2sHlMeRADjviQKZhEiInIShpE6onIjy8TezbHo311tPl9f6QkYRoiIyFkYRuqIYD+15LGbSonh0U3MHG2MXUaIiEgu7DNSR0wf3Arp2kLc1Tm0Sucbd2Bl1QgRETkHa0bqCH9Pd3w2viuGtg+RbH+4d3ObztcLAtYeTsc/V013eiUiIqopDCN13Gt3trfpuF8SLyJu6QHc8d52AOwzQkREzsMwUscpqpAqHv9fIopLzU8Fn1tUisIS26eVJyIisoR9RsjIxmOZksd6vQClUoE3/jgGlRL4auc5BPp44MDLQ2QqIRER1SWsGSGrSvR6pN3Ix7e7z+GrnecAADfyivH2+hM4djnb6eXR5pfg1dUpSE7LcvprExGR4zGMkFUlOgFFJpptFm77ByM/3onNxzNNnFVz5q09hu8SLmDMZ7ud+rpERFQzGEbIqkNpWRbXrpnyXaLZfb8mXcTU75Mc2sfklJlp7omIyDUxjJBV47/eh+8Szls8Jq+oFDmFJUbbn1t+COuPZuCHvRdQXKrH+pR0ZOUX11BJpfR6gQsAEhG5ALvDyI4dOzB69GiEhoZCoVBg1apVVs/58ccf0alTJ3h7e6NJkyaYPHkyrl+/XpXyUhXMHdkWADB9UMsqP8cPe1Mt7m//6gZ0eG0jSnSmR+FkF5bi482nMfWHAxj/9b4ql8NWOr2Af32yC//+puZfi4iIqsfuMJKXl4dOnTrhs88+s+n43bt3Y+LEiZgyZQqOHj2K5cuXY//+/XjsscfsLixVzWP9b8PBl4dgYp+IGn+tm2ZqPVQKBVYfugQAOFrNTq+2DFb+52oujqVnY/eZ69BXXniHiIhqFbuH9o4YMQIjRoyw+fiEhARERERg+vTpAIDIyEj85z//wdtvv23vS1M1NPDxcErziMrMvCZuKgXclRXZN7uwBNr8EoQHetdIOZSiYugEAUpOb09EVGvVeJ+R3r17Iy0tDevWrYMgCMjMzMSvv/6KkSNHmj2nqKgI2dnZki+qPpWy5m/IOjO1EEqFAm6qitfvPu8v9HtnK97445jJvibVV/Falct0IiMb61MyauA1iYioKmo8jMTGxuLHH3/E2LFj4eHhgZCQEGg0GovNPPHx8dBoNIav8PDwmi5mvaB0whzvpbdu/IIg4PNtZwzbVUrAXVXx51Y+w+u3u8/h8f8lObwc4txVWimMDP9wJ6b+kISkCzcc/rpERGS/Gg8jx44dw4wZM/DKK68gKSkJ69evx/nz5zF16lSz58yZMwdardbwlZaWVtPFrBfENRM1ZcGfJ5CuLcC6Ixl4Z/1Jw3ZBAI6nm67hSjhb0Zl5dfIljPxoJ1Kv51erHOJp8HU607U1x9NzqvUaRETkGDU+HXx8fDxiY2Mxa9YsAEDHjh3h4+ODfv36Yd68eWjSpInROWq1Gmq1uqaLVu+o3VR46+4OKC7VwUfthlm/HrZ4/NODW+ODv07Z9Rq/H7qM3w9dxvQ7Wkm2x/95wqbzZyxLBgDMXXUE30/paddri4ljV6ne9AgfdmslIqodarxmJD8/H0ql9GVUKhUAcA4IGTzUsxkmxUZiZAfjEFjZjMGtTA4HbmZDp9NDdk7VXlppSLC2wP5+JIIgILeoFACgF/1tmevHQkREtYPdYSQ3NxfJyclITk4GAJw7dw7JyclITS2bh2LOnDmYOHGi4fjRo0djxYoVWLhwIc6ePYvdu3dj+vTp6NGjB0JDQx1zFWQ3H7Ub9sweZH6/R1lg9PdyN9rn6W79z2b7qat2lWfQe9sl4bTUTNMKAJjr+vLmmuOIfnUDjlzUQpw/KvcZsUeJTo+US1oOD7aCHyyIqDrsDiOJiYno0qULunTpAgB45pln0KVLF7zyyisAgPT0dEMwAYBJkybh/fffx6efforo6Gjcf//9aNOmDVasWOGgS6CqCg3wwosjo/DmmGijfdufvx0AMKB1Y6N9ihoYJpt6Ix/5xRVTxuv0As5ezcUbfxxDZnahYXvKJS0OpmZJzi3R6ZF6vWwhPwAY/ekuyfTzH5prarLhBjr7tyP41ye7JJ1xSWrj0Qx0m/cXdp62L4ASEZWzu8/IwIEDLX4KWrJkidG2p556Ck899ZS9L0VO8Hj/FgCAl1elGLZFhfihkW9Zn51WwX5IfmUI/vXJLly8WQDAfM1EdYmbZnSCgPsWJeBGXjFSLmnxzNDWcFcpMOnbv43Om7zkb+w8fU2ybdvJK4bvf0m8iHfu61SlMv124CIA4NOtZzBtUCuTxxSW6HAg9SZiIgIlI4bqi8e/LxsNNeGb/Ti/YJTMpSEiV1T/3jnJpEmi2VkrDwEO8PbAjlm3Gx438PaokTJIwohewI28skna9p+/gQe/3It7FyYg51afkHJXsguNgghQ1lnXmgV/nsC5a3k4nZmDPWeMn0PM0rDo5389jIe+2od3N540ewwREZnHMEIAgNfubG/4Xmnir0IpmrijkZ8aW54dgDGdHdvnJ11bYPje3AiYynq8tdnkdlsmeMsr1uGez3djyAc78NDX+3DuWp7ZY83NLAuUjSACgG93nTO5v7hUjxUHLkqam4iIqALDCBkxVwvQMsgXAPBA9zDc1tgXL4yIcujrfrDptOF7c3OD2KqoVBpmdHoBe89ex1M/HZRsv5lfURtz7lqu2eezpWnKR2261fPTrWfwzC+HMOaz3YZtG45m4P2NJ9nxk4gITphnhFxP5/AAk9tXPNkHF67lo0OYBoDjm2uOXNIavtdV8yZdVKqTPG7x4jqr51hqilEqFTiUlgWVUoHophqTx/h4mP532ni0bOr5dG1Fzch/bvWz6BYRaNRJ+Hh6NnzVbmbX7Vm8+xzOX8vDa3e2l0zuZkni+Rv4JTENs0e0RaBPzTSzEZFU+fuQLc3G9R3DCBlsmNkf646k47H+t5nc7+/pbggiAODprsLw9iG4mV+MfeccO7V6decG2XA00+5zLDXt5BWV4q5bNRtD2wXj9bvao4nGS3KMl4fpN5xinfkmp+u5RZLHV7ILMeKjnQBgtjPo638cAwD8q1MoYiICzT632H2LEgCU1Rh99GAXm86pDkEQ8PPfaWgV7IduzRvU+OvVtB/3XUDzQB/0bdVI7qKQi9DpBfR8azMEATjw8hCnrA3myhhGyKBNiB/ahPjZdc6iCd0AABGz1zq0LNdyq7fCsLmp5y2x1C+kRNRstPFYJi5rC7BwfDecuVrRtOOmVCArvxinr+Sie/MGhlqLynOmXBMFEA83aUvpyUzbp6ivygKDx9OzkV1YAn9P4/ljHGnPP9cxe8URAOZDlatITsvC3JVlo81c/VrIeW7mFyPrVjNwVn4xGvqan1U8p7AEL61KwZ2dQnFH22CnlC8rvxhqN5XZD1HOxj4jRLeIO+kWFOuQoTXf4TTlUjb6vbMVjyyuGGqsVChwz8I9uH9RAjYdK6uZ+eiv00i9IV1n58VbN2kA8Kg0FLig0lwrlgJH5T6+er1gtQ/KqcxcdHxto2Qelprwz1Xz/W9czeWsAusHOUhRqc7p/Yiy8ovx+6HLkr+J67lF+HrnWUlwJvuIf43WmlM/3XIGq5MvY8p3iTVcqjLZhSXo/MYmdH1zk1NezxYMI+RQPSNtazao7UZ9shO94k2P1DFHpVTg7NWyETmPf5+Ey1kFJtf2SRZNlX/uWh5eXZ2Cqzllb/riGqFxX+5Fh9c2GkYZ7fnnGi5crxjxI75l/X7oMlq/9Cci56zDMz8nWy3rxZvSgPTVjrP4Ye8Fq+dZ897Gk/glMc2W+eSs+vNIOl77/ajs0/k7unK9VKfH9lNXkV0paF7LLULMvL8w/db6TM7yyJK/Mf2ng5i/9rhh25M/HsC8tcfxxA+OX1G7PrIWMC85MfACwPHLZTXHBTX8ocQeDCPkUIPbBrtsIJn6QxJSLmlRUKwzhAp7KCu1CZd3UhW7lluEKzkVnzbj/zyB7xIuYNavh5ChLcSLKytqTfafL+uH88ehy0i5pMVDX+3DgP9uM+wXv8FN/+mgYdr7FQcvoVSnt/gGqBKN307XFmD+uuN4aVWK0RpB4tdK+Oc60m5YXk35ky1n8Pyvh7E8qforbT/x4wEs2XMevyVdrPZzVYejJ/lbtP0fPPztfkz4ep9k+89/pyG7sBR/3Boq7izlMxqvSr5k2FbeB+zv8zfxW9JF/OuTnU6/YTrC9lNXceaKc1fnLl86Qvz/Zy1P29oR3RY5hSX4+/wNi0tYuIlqZLecsL9/XU1gGCGH2PLsAMwbE41JsRG4t2uYZN8jsRH4YGzVZkB1pqz8Evzrk13Yd+56lc6v/HYiHh1U7o73tps8d9vJq2ZrYkp0gqHZR8zS+1uv+C2YaUMNCVDWOVf8WqasT8nAuK/2ot87W3Eyw/qbe8ol+/vsmHPWwvwvQFmTwsajGSaDlE4vIPW65QBljaNuFIIg4OhlLZbuK1su49BF6d9HfnGpqdMAAF/vPItPt5w2u98RzF3ls8sPIeVSNuatOVajry9mLhTbI+WSFg9/ux+D39/hgBLZ5lBaFjq9sRHf770gGRGod2LT2/2LEnD/ogT8aiHEu6sqftuTlzinacgahhFyiNsa++LfvZrDXaXEfd2kYeSF4VG4u0uYmTPt8+TAFg55Hkt2nLI8G6s5WfnWO91WZTXi/244iY82G9+Itp28gslL/jb5ye9abhFWJ5v/hC0e+ix+nzQ18ue3pIt44scDhsefbnXMOj1Ldp9DxOy1mCuqDSonXmjRWkfdOz/djce/T8KSPeeN9s1afgj9/7sVv/xdUVPz17FMLPjzhE3NP7/8nSZp9qpOf47liRcx6uNduGymL5J4baZLWQX4aX8qikp1KCzRYd7a43h34ylcyTF97uQlfyNi9toqddwuZy105RU7p0p/07FMtHt1A1aLamqq4thlxwViWz39SzJyCkvx8qoUScd1ZzY1nrj1YWHFQfNhxM3UzJYyq30lIpenVCrwWL9IaLzcsfP52+Hp7rje2i0a+zrsucyp3OHUVuer+QncXj/tT8OWE1csfvLbb2bI9dM/HwJQ9gk0bmlF0Hh5VYokMJXo9Hh2+SHJucWiIFPV1YwLinV47dYQ5R/3pRrtf180tX6uqObmj0OXsT4lXXJsefPBhlvzuYitOFh2Q/tYVKvw6P8SsWj7P1h7JN3oeL1egFY0Ed7zvx2W3ISLdXocT8+2+7rXHUnH878dtniM+OY18qOdmLPiCD7efBolooBoajXr4+nZ2HKibC2mCd/sM9pvK2sjTx0xMnXD0QzMXXkExbcmJcwuLDF8X+6x/yWiuFSPGdXsOyNYrDt0vOPp2ZLmXXEAcVQYKSjW4dekizZ1LHa1+RQ5tJdqxNxR7TB7RFuHj613RqD/63jtaEN1hAe+SMBgE0MFj6dnI7eoFPcvSsCpzIqRL78fugwvdxXevq8jANOjYsqbcgRBwP1fJFSpXJVvQJWJmzCOXNRCpxeQW1RqmEH3xJvDjUKupUnrBKFsDpeLon4PmbdqKApLdIbnmrTkb+w4dRW/T4vF2+tPGD3PvDXH8f3eC3hpVFs82s/0fDymPCmqWRL7dtc5Q58U8f9KeSD869gVTI6NNGw3dY05hRVhrTpD4hUKBUp0esOaUJVZ+vlWtvfsdej0AmJbSudlKe9H1SrIF3d1booub27CbY18sOW5gVUutznWbsbrjqRj39nreGV0e4e8T5XPD1SuVF+1ZpqcwhL4qt1M1lQt+PM4vku4gNbBvtj49AAAZc1RvySmYebg1pIJDS29pNydwk1hGKEaUxOT/Ni4ZA2JmAtXwz/cYViJWeywqK/L8A93Gu0vDxJXcoqQdOGmyedWKCy/GVZee0gQBCgUCqw9nI7WwdLar7PX8tDixXV4cWTF8gPlAUK8OrP4ZqktKEHCPxXNbZeyCozWMVIogNd+P4ole85jUp8IdGiqwY5bzUN3frobpnx/a8TRvLXHcX+3cNz12S4Mj26CF4a3QV6xDr5mlgQw5w1RP4w7Oxmv9VSs00uWNjB1U7O2UHReUSlUSoXVGkoFyvobiEd7idkaRvKLS/Hgl3sBAMfeGAbvWzMTi4cOX7xZgISzZX2zLPUJupJTiCA/T8PjDG0h9IKA0AAvs+eUE/+kyv++xMoDYqfwAPRr1RiBPh4Ofc8S3/BLLdz8tfklkk7LHV7biMf734YXR7YFAOw8fRUBXh7oEKbBnylltX/iDxD/+mQXAOB6bjE+G9/VsN1SzVB1Z7iuCWymIaeZ7YC1bKryT/T9lB7Vft26yFQQAQAPleU35MtZBbjjvW147H/mO765i+6Qppo0fk6UjrYp1umx8/RVxC09gCEfmG52emtdRU1F+Q16kmieF/GN5LHvEjH1B9O1EeWUCoWhn8mSPeeNmqOsWbLnPM5fz8ei7f9gzoojiH51A/bdusEWFOsw/MMdeP2PozY/300TfY4u3SyQ1CKZ+kRr6hN05q2ZfL/ddQ4dXtuAbm9ustrf5XpesdkgAkibabT5JWZHVl0S/V2J580R93sq0eklzU9HLmpxIiMbDyyS1rT1mF8RIEt1evSK34w+C7bYNE+O+OeWU1SKxPM3TP78NhzNQMz8v/DIkrK/JUfN9SJ+LUvNei+tTjHa9uWOswDKavMmfLMfoz/dBUEQJP9XlZtgD1/Kkjx2tZoRhhFymqkDqt/5tPI/9e1tGps5skJdmI7cmQ5d1CJi9lr8ZWIED1D2Sfafq3k4XGk0iJh4MjdTnwrfWX9S8riwWG/x+SrbeuIKelcafVQ+V8vuM9cMw6Itqe6HYPEcMstudZB9Z0PZda05fBknMnKwePd5mztiXjfRxFKs0+OYqFOqqZ/lygPS5xcEAS+vSsHx9Gy8seYY9EJZ59Odp8tqivaevY6RH+3EgVTTtVrmiGtGOr2xEf3e2YrM7EIUlugko1/EHXTF5d3zT8Uote8SLkj6hIz+dBeGf7jT5O/tsf8lQhAESXOUpQkJy736e0UQnLz4b9y3KMHkytrlS0fsOHUV13OL0PG1jZhqYn6V3KJS/Hkk3eYFLiV9Riwcv/es+dF7WaL+W5nZRZLAfb5SjVJJqfQ1dIKA134/il/+TjNaq8uZo3tsxWYacqpnhrTGztNXEeTniZiIBhgWHYLe8VsM+2cNa4O0G/mYOqAFXvvjKMb3bG74BP7iyCjJP/WYzqFYcG9HDHp3m9kRCgDgWWmRqvl3RyMqxA/3Lqxaf4f64lELNR/WiN80e771FzbM7I8g/7LqdlNDWAvt/DQ6e4XxCJyTmTn4dMtpLN593qbn+Ga38Y2pnEqpqNKnx/JP++KbsK0dMY+ZGQnz+baK0Us6UfPWhqMZJueyiZxjelHIid/ux7n4kYYmlHFf7sXJeSNsKhtQ0V9r0fZ/DNsSz9/EvLXHEOjjgTVP9YVCoZCMfiqvnThzJReHLNS6WLLpWCaOXs6GxqtiCYPktCw0b+htslboSnYh3tsonWww8VZz4vx1x/H7octoYGaxyFXJl1FUqjda2+q7Pecl4Sa2ZSP0vK2h4bGpocjiSe0q/y0JgoC/jl9BVIifxUn1xKEh7WY+3MQJWlFWrnIllcpwMDXLMIfM3FVH8OeMfsgv1mHz8SuSWqrRJpoH5cAwQk41/Y5WmH5HK7P7425vafh+ySNlzStn3xppmFDs+4Tzhv0f2rjgm1KpQOfwAEMVdN+WjSyuE2HNwDaNEdnIx+abXn0kHpFzM78EH285jU5hAVi6PxUn0o2HIotHzFTHuxuNZ7w1J+2G+Um81G5KyVBbWx2+qMXzvx5C53DH1caJ52w5ckmLlkFl60eZ6mBrjXj67yIrnYgrO3etrFlmwZ8Vr5uuLUC6thDp2kJcyy2G2l0pCQLloWzw+6bn17FVUakeeaIQO/PnZFy4no/ECzfg5a5CoI8HZo+IQoC3B15ceQR/Hb9i9rlMzf9Tbrmo+TBDW4gXfjuMSbERkiACGP+95ptoNnpBNHrqdGYu5q89jn6tGuOxfpHYevKqxWZOAHh3w0mkiWZKLizRSSbgUyoUknJpC0qw0sxw3hKdYHbUnXstWcCPYYRkNzk2Et/uPmc0WVo58cympqqpbfn8OndUW9x/qz3aTaWUfsKwk6/aDfd2DWMYscMPe1PxA4yH8Ja7473tULvVnlbjqgSRcr8kXsQviTUza+zTPx+Cl7sbmjf0rtIswTfz7Z/nptzx9GyjWVjniaaQ/+dqLl5dfRTnRM0Hn245g/3nqzaJoFhRiXHNWeWlFpRKBeaPicaZK1VfF+mEaEK/uSuPYPupq5I5b8rtPXtdsqBdfpHx34u4T1b5BIR7/rmOolIdPvzL+gR2lefzmfDNfknT4ulK8wuV6gXDkH177D9/QzKiTC4MIyS7OSOjMDw6BJ3CNVaPNVV1bmnVSa9b/2Di9m53laJavebdlAq4WenkWc7aqJLqaKLxRLoNbeeuwt5P6vWVqf4MzhK7YIvZfZduFhitOv3bAceEsoe+tj5/ytJ9qQj09nDYfD+W5hv6auc5PNSzOS5cz8OpzBy0CfG3+XltCSLmiN/+vth+tsrPI3bxZgH+830Svpssb0d/hhGSnbtKiR42rmdjKox8Mq4LRn28y/B487MDkF1QgtOZuejdorxdt+I8D5USKlE4uatzqMXZSitTKhVQ2Lh82pC2wdhopiNodW15diC+3nkW722yvWmCSKzyYn3VsaAKzUaO5qjZgQHTsxGL3f7uNoe9ltxM1f44W+2pFyWygalKhvahGuyZPcjwOMDLHV2aNcADMeEID/QGIP1E4aZSSpp+Gvmq8ck4af+T8lE6d3dpip8f7yXZp1IoJMMGXxvdDn635pd4dkhrLHkkxrBP3OnO0bw8VJI+NrZ6MCa8BkpDrqjjaxsd9lxXc6zPCupKikpYU+dMrBkhl/JgTDi+23MeQ9tJZxUV9wExNTmTuEbFvVITi5e7CqM6NMHFmwXYceoqRncKxb86NcHWE1cwpF2wZK4EALivWxh8PSv+dSbFRmJcz2YQBMDTXSVZSK55Q++qXegtTQO8LK6WqlQq4OOhMrluyJjOoVhlosZn3phow1BUIjItI7vuNIHaQltQUqMfnqxhGCGXEuDtgT2zBxkN6RP3ATE1pl88RM690pzynu5lNSVPDGyBJ0QL8d3VuanRc782up1hSN+ro9uhsV/ZqBy1aPhwsH/FSJ3JfSPx475Ui307+rVqhKeHtMZ/1580zEq5Z/Yg7DpzDQNbNzaaObQyLzNhZP7dHdC/dWOcuZKLz7dVDMd0UynR2E9d5z7JurL/DLjNYX0AiKpifUo6xsY0k+312UxDLsfU3AI+omm4vU10aBXnE2WlzqvWepGLg0aIpmIa6kdiI/GvjsZj9AO8PbD00Z74+fFe8PZww6cPdTU6RqxZoDe6NmuAnx7vhe+n9MAn47ogNMALD3QPR5C/J6zNwt28oY/Rtq7NAuCjdsM9XcNMDqX+T3/jdVUa+Xrgq4ndLb8Y1YgBra1P3kdUk8STysmBYYTqBE93FX57og9+ndrbsBaGmKUJrCyNxjF+Hdv+ZfpIJkWyPJxGXLR+rRobTUKUMPsOi+d/8EBntAySrucyShSSPN1V6NIsQLJ/Yu8IfF0peCgVCgxpZ7yonjW2zIJLloU38DYZoomcRTxMWw4MI1RndGveAN0jTI/K6XzrZhyq8TTaF+htejZGsddGt8O9XcPQv5X9N17rQ3stHxCi8ZRMrw4Aa57qa/i+WUNv/PXMAJxfMMqwrfL6MsPahwAAbmtUVovi4abE4ErBw55QJmZuldfKArzla4+uKmu1Uo6avdLbQ4XElwbj20msmaqKymGcTBMvNlnbMIxQveDv6Y6U14dh26zbDdteGB6F4e1DbKoNmBQbifce6GTUxGMLa1nElpWIO4RVzMHy2UNdEd3U8pws/pU6ok3pG4kvJnTDr0/0MXtO5RFFYuXNCFMHtECLxj6SjrnXcovxSGyExfIAgJtSiTGdq37z9nBT4qMHOxseP9Yv0uZzB0UFVek124danj/i4d7Nq/S8lbmplPD2cMOgqGAkvTQYPz3Wy/pJdZiv2s3kBwdz+rZshABvd65DZcFTg1panRnYlgUIawrDCNUbvmo3eIhm+XxiYAssmtANbtbWYK8ma7O92rJo1acPdcG4HuFY9ngvjOrYxOxxL46MwrD2wRjVQXqMu0qJYe1DEGhmTY4HuoehY1iA2XIv/HdXfD+lB54d2hobZvbHtucGGvZdyy3Cq6Pb49gbw3B+wSgcemWoyde4lltkcqTTHCurOb/yr3YAgI/Gdoa/Z0XIauSrxtjuxsOUP33IOFRNHdDCpsBUWYNKtWYrnpSGOUctOS8e4dXQV221Rqaq/D2dM2ZBHBqrorGfGnvm3CGp7bOkgbcHDr48BD9M6Wm0b+PT/SWPXxvdDl9O6Fat8tmqRWPj/lyOZmvz3rND21idrPG6jbWcNYFhhKiGdRLd5E3VPtiyHlsTjRfi7+mIXqLFuUx5vH8LfDGhu80B66VRbRHZyAfPDm1jtK99qD8+HNsZy251xO3XqjHcVUq4qZSSTsTlU/SX99XRWGqOMfFe+J8BLfCFhZvD5L6ROP7GcIzo0AT+XhU300AfD7x9X0dJFf2cEVEmOxWrlGWfniuLbGT5ZlE5jHRtJv1k6aZ0zFto5ecR/5ju7tIUTw5sgd+e6INDr5oOerYy1Z+quhaON+6gXT4SzRRLa1OVM7VoYq/bAs2GKZ0gQKFQmOzT1cDbAwNF/Zoa+qpNhuKa0NCnYmSdtVo2ADgz3/aFCwGggbc7jr0xHMmvDEFvK+8NgMl/P4nrufKNsGMYIaphSqUC5xeMwql5IzC6Uyj+N7kH7ulS8WZtz2q1jvZov9uw9bmBCPY3XSU+pktTswHo20nd0dDHA19NtO1TZqsgX7M3gWHtQ/DXM/1N7gMq+rO0bOxn2NY0oGxkk7hy4j8DWsAUpUJhsqnmnfs6IsLCXDAvmKi1ef3O9gCAuNtbwNYsMr5nM7xzb0ez+yvPfSP2wdjOeH54FLo1bwCNl7tkUj1bdQ4PwOwRUZKaQUfoHB6APpVCnrXXsOWmLP6PKO9rZCmI628FYlMj7QQIeHpwa8Njd5WixmqeKisQNXv42VAr5aZS2nRcdFN/9IgIxOfjy/73Arw9EB7oZeUs0z8fsWsMI0R1X/mbdP/WjfH+2M6G7bY009RGg6KCkfjSYAyKMu5zI26+GN4+BCH+nvjjqb4WP5mVr0YLlDXBmKLxdsdj/SIxKCrI0Fm5fG6E7hb6C6iUCpNvxP6e7tj63EBDwBCbNawNmgZ4YfnU3tB4uRvCxMN9IrB3zh14bmgbszUjHz3YWdJUofFyxwMx4fh9WqwkiJazdpMQG9jGdP+X7bMGGr5//c72kpl2f3uiD6YOaFGlMFI+EquVqAaqeUNvfDC2Ez59qItkgcNnhrTG33MHW3w+W2olxP8Sf0zri9kjojCxd4TkGPEoLkv/Q94eblCLakxUSqXDwshAEyPJxE2hTw2qmCG5VGfb/7nehqrSlo198cvU3qLlLqo2NFcc0oCy/l9y4aRnRDJpHeyLU5m5Fqu05TAoKghbTlzB5L7WO4iau4l+P7kH/vNDEuaNicadt0acKBS2fyIN8lPju8kxkj4i5eaOaid5/EifCHRoqpF84t75/O1I+Oc6nr+1jLu5G2B5SDG1YvCUW9cfExGI5FeGSK415FbnSnN9Rsp/pzfzirHi4CU82q9sXpeOYQF474FO0Hi740R6jmGSu8o6NwtAqMbTsJyBNQ/GhEvmm1FWWqCxvJzT72iF6T8dtOk5y80a2gbX8ooR26IhHvtfIg6kZmFUhya4u0vZKtvim2e/Vo2szuJp6kc2oHVjDG0fjLkrU8qeU1T48EBvTL1V49U+VIOEs9ehVACLH+mBiNlrAZie6FDj5Y7PHupa1ldM1GzpplJY7FXeyNfDppvy5+O74rbGPth2Urquy+q4WPR7ZysAaS2RtqBiHaCuzQJwIDXL5POaWpm8MlO5pvI6Qxovd8lrAsbNNGNjwiWrH1fua+ZMDCNEMlkVF4vUG/mIsmPFT2f4YkI3pN7IR4vGVR8u2adlIxx6ZajR6KPKCwya6/+pFwS0D7W+ijNQ1gxWeaHF8EBvyafh8hzRr1Uj7Dx9DQAQFeJn6DMypktTrDh4Cacyc5CVX/YGLr6BmQtd1jqwToqNxKRYaahTKBR4dXR7rDl82WwYUbupsOP5280+/7eTuuPJHw/gjbuiERbghW4R0lohX083kyO/7uwUig5NNXhk8X6bV7f19FDhzpZlgXLxpB7YdeYa7mhbUTsjfh1zHaTFlAoFlk/tjbfWHcesoW1wM78Ed7QNgqe7yhBG/EyEUAD48MHO+GjzaUysNIpJHIi6NAvAwdQsvHNfR/RtVdaEJA4FoRovXNZWLLFw6NWh2H3mGpo39MbNvBJ8tvUMruWa/r2U+3JCNwxtH4I0Eyv7in9nHm5KeKiUKNbp0T7UH6ev5AIAYiIDjcLI8FvD7y3NiVROZ2IIXoCX9Gf/2xN98OFfp7DmcLrJ5/hhSk+EaDzx/gOd8OLKI1j4726SySOdjWGESCbeHm61LogAZSNvqhNEypm6GVa+p//3vk6Sx/d0bYoVBy7hqUHWOzlaI65VKR+y+M3DMbeClg8EoaKMnu4q/PKf3vhqx1nMX3fcbPkrEy+YaC9rzRWWOiEPigpGymvDjI55+94O2Hv2Bv7VMRSnMnNNnhvZyActg3yNwsiUvpFQuyklSwcA0k/TGm93k6O5vprYHTfyiiS1M1EhfjghWqfJ8HyKstqmlU/GGu1bOL4rPvzrND4UNWOKBft74q27OxhtF4eXpY/2wvnreYgKqWj281NX7I9s5CMJwRovd4wU1Qj8mZJuNiQufbQnOjcLMHQENjU6RRJGVEqsnd4XvySmYeqAFpK1oir/fIRb1TWmankqMxVY5o5qi8vaAhw0hBzBqAJIPOS/PKjd0zUMd3YKrfFRhdbY/eo7duzA6NGjERoaCoVCgVWrVlk9p6ioCHPnzkXz5s2hVqsRERGBb7/9tirlJSIXJq5hOPTKUNzbLUyy/937OmHP7EEWhy/bStz0kldUFkY83JRoGeQLhUJhMmzY23+ngWjk0G23hnH2MDPxXmWDooLQKsjX5PBkW5i6eYyNaYYPxnaGu0ppcTK/+Xd3wJjOoZK5Wl7+Vzs8P9y4w64tP5Eh7YKN1jX5fkpPxN9jHBwshbARHZpgw9P90UYUJCx5596OGNC6saRJ0ctDhbZN/CV/axpvdyx9tCdWxcXCw02JVsF++OjBzibnc3l+eBQejAk3Wq0bKKvxE49IcjfxO6hcM9Iq2A9zR7VDQ3E/KAH4ZWpvyeuX5wtb/gRNdeYNDfCSDG0WBKB5pWa+yEY+eG5oa8y/O1qyXe4gAlShZiQvLw+dOnXC5MmTcc8999h0zgMPPIDMzEx88803aNmyJdLT06G3ZaYnIqpTxPchU0OAlUoFQgOsjwqw7bUUaOjjget5xegYZluTjy3DrMWC/D3x02O9oPFyh7+XG5btT8PEPrZNhObprsLGp/vb1XnVHg/EhOPrXWfRp4XxkOZgf098+GAX6PQCrucVo4NoEr3kV4bg1d+PYrWJFZ/t0dhPjXE9mmHOiiOS7Y4c0fNATDgeiLEtzFUe9WOur5bGyx0LLIx8EjPVp0k8P4+lZjx/T3dJB9TyEPLZQ10Rt/QA3hwTjZdX3Wq2UrshsrEPPh3XFYkXbhj6YVUmfj0BQNztLXEzv0TSF2SaA2oda4LdYWTEiBEYMcL2sdDr16/H9u3bcfbsWQQGln1iiIiIsPdliagOcNAcYTbbPXsQCop1CLBhyn+gaiObxDeU54YZz9diSU0FEaDsprpn9h0Wf+YqpQLvP9BZsi3A2wPv3t/JEEaqO9jr/Qc6YcGfJ9C8oTe8PdwQY2PNUW1jan6UysGqT4uGkho3c6OtTP9Iy7aO6tgEg6KGw8tDhW0nriD1Rj7WTu93a0iyAs0sDEWXhBGhbAFRU7VTtVGN9xn5/fff0b17d7zzzjv4/vvv4ePjgzvvvBNvvvkmvLxMfwIqKipCUVHFeOfs7OyaLiYROUHlDqw1zdNdZXVV5rqsqjPEWps12B73dA3D3V2a1mjwqmmfPdTVatPhkHbB+GRcF0mgtaX1o10TfxxLz8Y9XSuaLMvn1fn64e6Svk3WqFz4Z1zjYeTs2bPYtWsXPD09sXLlSly7dg1PPvkkrl+/jsWLF5s8Jz4+Hq+//npNF42InMzZNSP2GtY+BP/dcNLQ/6O+cnRwcNUgsvW5gTiZkW1YaNKSQVFlI4KKSismOjN33eLaod+e6IOz13LRrolxZ3Z7hsMD0tASYmYiw9qqxsOIXq+HQqHAjz/+CI2mrF3y/fffx3333YfPP//cZO3InDlz8MwzzxgeZ2dnIzy8ap28iKj26NuqMb5LuCB3McxqGeSLvXPucMkVhh2t122BuJxVKOlPUt9ENvKxumTAj4/2xM7T13Dfrc7Y4qaZyssJ7Hz+dhxPz8Zg0dBoLw+VzcPYbbF91kAUlugtL8tQC9V4GGnSpAmaNm1qCCIA0LZtWwiCgIsXL6JVK+N2OLVaDbXa9AyMROS6BrcNwneTe6BNsG2jJeQQYsdqsXXZT4/1gl5w3GKAdVVsy0aIFXWOVSkV+H5KDxSX6o3mXQkP9LZ5IruqEg+vdiU1Pp4nNjYWly9fRm5uxZj3U6dOQalUIiwszMKZRFTXKBQKDGjdmDd8F6BQKBhEqqhfq8a4o63xMglknt1hJDc3F8nJyUhOTgYAnDt3DsnJyUhNTQVQ1sQyceJEw/EPPfQQGjZsiEceeQTHjh3Djh07MGvWLEyePNlsB1YiIiKqP+wOI4mJiejSpQu6dClbCv2ZZ55Bly5d8MorrwAA0tPTDcEEAHx9fbFp0yZkZWWhe/fuGD9+PEaPHo2PP/7YQZdARERErkwhyLl+uY2ys7Oh0Wig1Wrh71/7ps8mIiIiY7bev+WfA5aIiIjqNYYRIiIikhXDCBEREcmKYYSIiIhkxTBCREREsmIYISIiIlkxjBAREZGsGEaIiIhIVgwjREREJCuGESIiIpKVm9wFsEX5jPXZ2dkyl4SIiIhsVX7ftrbyjEuEkZycHABAeHi4zCUhIiIie+Xk5ECj0Zjd7xIL5en1ely+fBl+fn5QKBQOe97s7GyEh4cjLS2tzi7AV9evsa5fH1D3r5HX5/rq+jXW9esDau4aBUFATk4OQkNDoVSa7xniEjUjSqUSYWFhNfb8/v7+dfYPrFxdv8a6fn1A3b9GXp/rq+vXWNevD6iZa7RUI1KOHViJiIhIVgwjREREJKt6HUbUajVeffVVqNVquYtSY+r6Ndb16wPq/jXy+lxfXb/Gun59gPzX6BIdWImIiKjuqtc1I0RERCQ/hhEiIiKSFcMIERERyYphhIiIiGRVr8PIZ599hoiICHh6eqJnz57Yv3+/3EWySXx8PGJiYuDn54egoCCMGTMGJ0+elBxTWFiIuLg4NGzYEL6+vrj33nuRmZkpOSY1NRWjRo2Ct7c3goKCMGvWLJSWljrzUmyyYMECKBQKzJw507DN1a/v0qVL+Pe//42GDRvCy8sLHTp0QGJiomG/IAh45ZVX0KRJE3h5eWHw4ME4ffq05Dlu3LiB8ePHw9/fHwEBAZgyZQpyc3OdfSkm6XQ6vPzyy4iMjISXlxdatGiBN998U7I+hStd444dOzB69GiEhoZCoVBg1apVkv2OupbDhw+jX79+8PT0RHh4ON55552avjQDS9dYUlKCF154AR06dICPjw9CQ0MxceJEXL58WfIctfkarf0OxaZOnQqFQoEPP/xQsr02Xx9g2zUeP34cd955JzQaDXx8fBATE4PU1FTDftneW4V6atmyZYKHh4fw7bffCkePHhUee+wxISAgQMjMzJS7aFYNGzZMWLx4sZCSkiIkJycLI0eOFJo1aybk5uYajpk6daoQHh4ubN68WUhMTBR69eol9OnTx7C/tLRUiI6OFgYPHiwcPHhQWLdundCoUSNhzpw5clySWfv37xciIiKEjh07CjNmzDBsd+Xru3HjhtC8eXNh0qRJwr59+4SzZ88KGzZsEM6cOWM4ZsGCBYJGoxFWrVolHDp0SLjzzjuFyMhIoaCgwHDM8OHDhU6dOgl79+4Vdu7cKbRs2VIYN26cHJdkZP78+ULDhg2FNWvWCOfOnROWL18u+Pr6Ch999JHhGFe6xnXr1glz584VVqxYIQAQVq5cKdnviGvRarVCcHCwMH78eCElJUX46aefBC8vL+GLL76Q/RqzsrKEwYMHCz///LNw4sQJISEhQejRo4fQrVs3yXPU5mu09jsst2LFCqFTp05CaGio8MEHH0j21ebrEwTr13jmzBkhMDBQmDVrlnDgwAHhzJkzwurVqyX3PbneW+ttGOnRo4cQFxdneKzT6YTQ0FAhPj5exlJVzZUrVwQAwvbt2wVBKHvjcHd3F5YvX2445vjx4wIAISEhQRCEsj9apVIpZGRkGI5ZuHCh4O/vLxQVFTn3AszIyckRWrVqJWzatEkYMGCAIYy4+vW98MILQt++fc3u1+v1QkhIiPDf//7XsC0rK0tQq9XCTz/9JAiCIBw7dkwAIPz999+GY/78809BoVAIly5dqrnC22jUqFHC5MmTJdvuueceYfz48YIguPY1Vn6Td9S1fP7550KDBg0kf58vvPCC0KZNmxq+ImOWbtbl9u/fLwAQLly4IAiCa12jueu7ePGi0LRpUyElJUVo3ry5JIy40vUJgulrHDt2rPDvf//b7DlyvrfWy2aa4uJiJCUlYfDgwYZtSqUSgwcPRkJCgowlqxqtVgsACAwMBAAkJSWhpKREcn1RUVFo1qyZ4foSEhLQoUMHBAcHG44ZNmwYsrOzcfToUSeW3ry4uDiMGjVKch2A61/f77//ju7du+P+++9HUFAQunTpgq+++sqw/9y5c8jIyJBcn0ajQc+ePSXXFxAQgO7duxuOGTx4MJRKJfbt2+e8izGjT58+2Lx5M06dOgUAOHToEHbt2oURI0YAqBvXWM5R15KQkID+/fvDw8PDcMywYcNw8uRJ3Lx500lXYzutVguFQoGAgAAArn+Ner0eEyZMwKxZs9C+fXuj/XXh+tauXYvWrVtj2LBhCAoKQs+ePSVNOXK+t9bLMHLt2jXodDrJDxMAgoODkZGRIVOpqkav12PmzJmIjY1FdHQ0ACAjIwMeHh6GN4ly4uvLyMgwef3l++S2bNkyHDhwAPHx8Ub7XP36zp49i4ULF6JVq1bYsGEDnnjiCUyfPh3fffedpHyW/j4zMjIQFBQk2e/m5obAwEDZrw8AZs+ejQcffBBRUVFwd3dHly5dMHPmTIwfPx5A3bjGco66ltr8N1tZYWEhXnjhBYwbN86wqJqrX+Pbb78NNzc3TJ8+3eR+V7++K1euIDc3FwsWLMDw4cOxceNG3H333bjnnnuwfft2Qxnlem91iVV7yby4uDikpKRg165dchfFYdLS0jBjxgxs2rQJnp6echfH4fR6Pbp374633noLANClSxekpKRg0aJFePjhh2UunWP88ssv+PHHH7F06VK0b98eycnJmDlzJkJDQ+vMNdZXJSUleOCBByAIAhYuXCh3cRwiKSkJH330EQ4cOACFQiF3cWqEXq8HANx11114+umnAQCdO3fGnj17sGjRIgwYMEDO4tXPmpFGjRpBpVIZ9RDOzMxESEiITKWy37Rp07BmzRps3boVYWFhhu0hISEoLi5GVlaW5Hjx9YWEhJi8/vJ9ckpKSsKVK1fQtWtXuLm5wc3NDdu3b8fHH38MNzc3BAcHu/T1NWnSBO3atZNsa9u2raFHe3n5LP19hoSE4MqVK5L9paWluHHjhuzXBwCzZs0y1I506NABEyZMwNNPP22o6aoL11jOUddSm/9my5UHkQsXLmDTpk2SpeZd+Rp37tyJK1euoFmzZob3nAsXLuDZZ59FRESEoXyuen1A2X3Pzc3N6nuPXO+t9TKMeHh4oFu3bti8ebNhm16vx+bNm9G7d28ZS2YbQRAwbdo0rFy5Elu2bEFkZKRkf7du3eDu7i65vpMnTyI1NdVwfb1798aRI0ck/1zlby6V/1id7Y477sCRI0eQnJxs+OrevTvGjx9v+N6Vry82NtZoKPapU6fQvHlzAEBkZCRCQkIk15ednY19+/ZJri8rKwtJSUmGY7Zs2QK9Xo+ePXs64Sosy8/Ph1IpfXtRqVSGT2d14RrLOepaevfujR07dqCkpMRwzKZNm9CmTRs0aNDASVdjXnkQOX36NP766y80bNhQst+Vr3HChAk4fPiw5D0nNDQUs2bNwoYNGwC49vUBZfe9mJgYi+89st47qtz11cUtW7ZMUKvVwpIlS4Rjx44Jjz/+uBAQECDpIVxbPfHEE4JGoxG2bdsmpKenG77y8/MNx0ydOlVo1qyZsGXLFiExMVHo3bu30Lt3b8P+8uFZQ4cOFZKTk4X169cLjRs3rhVDX00Rj6YRBNe+vv379wtubm7C/PnzhdOnTws//vij4O3tLfzwww+GYxYsWCAEBAQIq1evFg4fPizcddddJoeKdunSRdi3b5+wa9cuoVWrVrVmaO/DDz8sNG3a1DC0d8WKFUKjRo2E559/3nCMK11jTk6OcPDgQeHgwYMCAOH9998XDh48aBhJ4ohrycrKEoKDg4UJEyYIKSkpwrJlywRvb2+nDQu1dI3FxcXCnXfeKYSFhQnJycmS9x3xCIrafI3WfoeVVR5NIwi1+/oEwfo1rlixQnB3dxe+/PJL4fTp08Inn3wiqFQqYefOnYbnkOu9td6GEUEQhE8++URo1qyZ4OHhIfTo0UPYu3ev3EWyCQCTX4sXLzYcU1BQIDz55JNCgwYNBG9vb+Huu+8W0tPTJc9z/vx5YcSIEYKXl5fQqFEj4dlnnxVKSkqcfDW2qRxGXP36/vjjDyE6OlpQq9VCVFSU8OWXX0r26/V64eWXXxaCg4MFtVot3HHHHcLJkyclx1y/fl0YN26c4OvrK/j7+wuPPPKIkJOT48zLMCs7O1uYMWOG0KxZM8HT01O47bbbhLlz50puXK50jVu3bjX5P/fwww879FoOHTok9O3bV1Cr1ULTpk2FBQsWOOsSLV7juXPnzL7vbN261SWu0drvsDJTYaQ2X58g2HaN33zzjdCyZUvB09NT6NSpk7Bq1SrJc8j13qoQBNGUiEREREROVi/7jBAREVHtwTBCREREsmIYISIiIlkxjBAREZGsGEaIiIhIVgwjREREJCuGESIiIpIVwwgRERHJimGEiIiIZMUwQkRERLJiGCEiIiJZMYwQERGRrP4PvDNCUsyfbpwAAAAASUVORK5CYII=\n", "text/plain": [ "
" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "plt.plot(torch.tensor(l).view(-1, 10).mean(1).numpy())" ] }, { "cell_type": "code", "execution_count": 53, "metadata": { "id": "k5vlgMO8P-3f" }, "outputs": [], "source": [ "def make_inference(model, question):\n", " # 保存跟微调训练数据一致的格式\n", " context = 'Below is an instruction that describes a task.' + \\\n", " ' Write a response that appropriately completes the request.\\n\\n'\n", " instruction = f'### Instruction:\\n{question}\\n\\n### Response:\\n'\n", " token = tokenizer(f'{context}{instruction}', return_tensors='pt').to(device)\n", " # 生成文本时，需要将模型调整成评估模式\n", " model.eval()\n", " output_tokens = model.generate(**token, max_new_tokens=100, early_stopping=True)\n", " model.train()\n", " print(tokenizer.decode(output_tokens[0], skip_special_tokens=True))" ] }, { "cell_type": "code", "execution_count": 56, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "SC-5Ri1Z8qcR", "outputId": "c287f0b3-c07b-4b3c-efd6-81568f529110" }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n", "\n", "### Instruction:\n", "Where is the capital of China?\n", "\n", "### Response:\n", "The capital of China is Beijing.\n" ] } ], "source": [ "make_inference(model, 'Where is the capital of China?')" ] }, { "cell_type": "code", "execution_count": 57, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "UwnVbc1C9DTR", "outputId": "00531adb-d3db-436c-c6d0-fb42ce32cdd7" }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n", "\n", "### Instruction:\n", "Where is the capital of China?\n", "\n", "### Response:\n", "\n", "The capital of China is the capital of China.\n", "\n", "### Response:\n", "\n", "The capital of China is the capital of China.\n", "\n", "### Response:\n", "\n", "The capital of China is the capital of China.\n", "\n", "### Response:\n", "\n", "The capital of China is the capital of China.\n", "\n", "### Response:\n", "\n", "The capital of China is the capital of China.\n", "\n", "### Response:\n", "\n", "The capital of China is the capital of China.\n", "\n", "### Response\n" ] } ], "source": [ "# 微调之前的模型效果（禁用LoRA相当于回到微调之前的模型状态）\n", "with model.disable_adapter():\n", " make_inference(model, 'Where is the capital of China?')" ] } ], "metadata": { "accelerator": "GPU", "colab": { "gpuType": "V100", "provenance": [] }, "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.8.5" }, "widgets": { "application/vnd.jupyter.widget-state+json": { "0c1ff22ae7f6497c884f8e36bd078812": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "DescriptionStyleModel", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "159fd68f34d440388ca2e8221d6d49f2": { "model_module": "@jupyter-widgets/base", "model_module_version": "1.2.0", "model_name": "LayoutModel", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "2dd504bb8b464afea0ac61bc41c129cc": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "FloatProgressModel", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_35839dbe38074fe88d72dfd6d15c2192", "max": 46801, "min": 0, "orientation": "horizontal", "style": "IPY_MODEL_58b629ccc0304e549e2b13e566faa36b", "value": 46801 } }, "35839dbe38074fe88d72dfd6d15c2192": { "model_module": "@jupyter-widgets/base", "model_module_version": "1.2.0", "model_name": "LayoutModel", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "3a50612ff1584cd48e7f3a43dc733d1b": { "model_module": "@jupyter-widgets/base", "model_module_version": "1.2.0", "model_name": "LayoutModel", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "412090d7ce2842d8979b6ee4ca19a0fe": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "HTMLModel", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_611a6f0cf7434f4186bbe6961a54742e", "placeholder": "​", "style": "IPY_MODEL_0c1ff22ae7f6497c884f8e36bd078812", "value": "Map: 100%" } }, "58b629ccc0304e549e2b13e566faa36b": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "ProgressStyleModel", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "bar_color": null, "description_width": "" } }, "5a3141d40ea143ae948b4023284d2ae6": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "HTMLModel", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_159fd68f34d440388ca2e8221d6d49f2", "placeholder": "​", "style": "IPY_MODEL_b7ef0e6d0af94849b32d918f5b360f72", "value": " 46801/46801 [00:47<00:00, 1703.24 examples/s]" } }, "611a6f0cf7434f4186bbe6961a54742e": { "model_module": "@jupyter-widgets/base", "model_module_version": "1.2.0", "model_name": "LayoutModel", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "ae30a280af464393a17f16d8bd991e50": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "HBoxModel", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HBoxView", "box_style": "", "children": [ "IPY_MODEL_412090d7ce2842d8979b6ee4ca19a0fe", "IPY_MODEL_2dd504bb8b464afea0ac61bc41c129cc", "IPY_MODEL_5a3141d40ea143ae948b4023284d2ae6" ], "layout": "IPY_MODEL_3a50612ff1584cd48e7f3a43dc733d1b" } }, "b7ef0e6d0af94849b32d918f5b360f72": { "model_module": "@jupyter-widgets/controls", "model_module_version": "1.5.0", "model_name": "DescriptionStyleModel", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } } } } }, "nbformat": 4, "nbformat_minor": 1 }