{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "1h1JT8eELKh2",
    "outputId": "1d09634e-f959-4567-ffa9-5aaf5e0533c7"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<torch._C.Generator at 0x78ca6c0b4d50>"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim\n",
    "from torch.nn.utils import clip_grad_norm_\n",
    "from transformers import AutoTokenizer, GPT2LMHeadModel\n",
    "from datasets import load_dataset\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "\n",
    "torch.manual_seed(12046)\n",
    "# 计算设备为A100 40G\n",
    "# 如果使用CPU，需要非常长的时间，建议减少sequence_len、batch_size等参数来加快速度"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "id": "6t4ZrY0SLKh3"
   },
   "outputs": [],
   "source": [
    "# 一些超参数\n",
    "learning_rate = 6e-4\n",
    "# 模型支持的最大文本长度\n",
    "sequence_len = 512\n",
    "batch_size = 16\n",
    "gra_acc_steps = 8\n",
    "device = 'cuda' if torch.cuda.is_available() else 'cpu'\n",
    "eval_iters = 4\n",
    "eval_interval = 100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "id": "6Dtak9OKLKh4"
   },
   "outputs": [],
   "source": [
    "tokenizer = AutoTokenizer.from_pretrained('gpt2')\n",
    "model = GPT2LMHeadModel.from_pretrained('gpt2').to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "mAHLxTD2aiJW",
    "outputId": "17a42a12-3ab9-46f2-ca5f-292f41967e97"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "({'instruction': 'Identify the incorrect word and suggest a better version.',\n",
       "  'input': 'The waitress served a humonguous burger.',\n",
       "  'output': 'The incorrect word is \"humonguous\" and it should be replaced with \"huge\".',\n",
       "  'text': 'Below is an instruction that describes a task, paired with an input that provides further context. Write a response that appropriately completes the request.\\n\\n### Instruction:\\nIdentify the incorrect word and suggest a better version.\\n\\n### Input:\\nThe waitress served a humonguous burger.\\n\\n### Response:\\nThe incorrect word is \"humonguous\" and it should be replaced with \"huge\".'},\n",
       " {'instruction': 'Render a 3D model of a house',\n",
       "  'input': '',\n",
       "  'output': '<nooutput> This type of instruction cannot be fulfilled by a GPT model.',\n",
       "  'text': 'Below is an instruction that describes a task. Write a response that appropriately completes the request.\\n\\n### Instruction:\\nRender a 3D model of a house\\n\\n### Response:\\n<nooutput> This type of instruction cannot be fulfilled by a GPT model.'})"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "raw_datasets = load_dataset('tatsu-lab/alpaca')\n",
    "datasets = raw_datasets['train']\n",
    "datasets[112], datasets[8]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "B67BMW5SLKh5",
    "outputId": "05edf4d0-1478-4366-f74f-5aa1a317b252"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('Below is an instruction that describes a task. Write a response that appropriately completes the request.\\n\\n### Instruction:\\nRender a 3D model of a house\\n\\n### Response:\\n<nooutput> This type of instruction cannot be fulfilled by a GPT model.<|endoftext|>',\n",
       " '<nooutput> This type of instruction cannot be fulfilled by a GPT model.<|endoftext|>')"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "context_template = 'Below is an instruction that describes a task. ' + \\\n",
    "'Write a response that appropriately completes the request.\\n\\n' +\\\n",
    "'### Instruction:\\n{instruction}\\n\\n### Response:\\n'\n",
    "\n",
    "def prepare_input(data):\n",
    "    '''\n",
    "    利用模版生成训练文本，并对文本进行分词\n",
    "    '''\n",
    "    context = context_template.format_map(data)\n",
    "    whole_text = context + data['output'] + tokenizer.eos_token\n",
    "    ids = tokenizer.encode(whole_text)\n",
    "    # 模型在微调时，不需要考虑模版文本\n",
    "    context_ids = tokenizer.encode(context)\n",
    "    # -100表示在计算模型损失时忽略该位置的数据（参考GPT2LMHeadModel的官方文档）\n",
    "    labels = [-100] * len(context_ids) + ids[len(context_ids):]\n",
    "    return {'input_ids': ids, 'labels': labels}\n",
    "\n",
    "# 示例\n",
    "re = prepare_input(datasets[8])\n",
    "# 只有Response:\\n后面的内容才会被使用（计算模型损失）\n",
    "tokenizer.decode(re['input_ids']), tokenizer.decode(list(filter(lambda x: x != -100, re['labels'])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 137,
     "referenced_widgets": [
      "46efff100756461db8cdf17e85599931",
      "ac66c967bd2f41178796a0863a142bd2",
      "ca355fac774949bc990a8d36dd4a65e1",
      "b5c2a3a5ab40417481acb990495f513d",
      "f282ea9ae3e24d4db17ceea762dcd943",
      "25f698c38f5548b4a1dd9ee89163ebb6",
      "3fe0784b898f4cb1aa10fde49d7e8001",
      "c11e20236a1848cca183fd3e206ee546",
      "ccf8966db2184bb5968adf3b75df805f",
      "4f7d802ec45246688db7113ea6a8a98c",
      "ccddc6386c7b4655a56018e2600b65e9",
      "013051901d484a2ba80569dccd5b5680",
      "48ac7039863c42f880aeb25be0830091",
      "b87cc14342d54e0c86bcfe1cf2bc8acc",
      "b56f7cae483f4ceb89dce64cea92f686",
      "01a2acaef43042d5bb8a0e4fcfbac045",
      "c4e9a011ae3e47039728b99f958f400b",
      "922b8d4189da42a7908dd907603e770c",
      "da0f743da3ee4af98c227d0e83065caa",
      "92ddb455899d4b6b8f5ed1e2ff22d2fc",
      "3f415ddf03514ba390be7d01d51aa866",
      "cf89e35195d046ce8b236fea9c13fc37"
     ]
    },
    "id": "qAsW4FQUdBuM",
    "outputId": "8fed7632-225f-4a38-9f40-dfa7ef413f4b"
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "46efff100756461db8cdf17e85599931",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Map:   0%|          | 0/46801 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Token indices sequence length is longer than the specified maximum sequence length for this model (1511 > 1024). Running this sequence through the model will result in indexing errors\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "013051901d484a2ba80569dccd5b5680",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Map:   0%|          | 0/5201 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "(torch.Size([4707809]), torch.Size([4707809]))"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenized = datasets.train_test_split(test_size=0.1, seed=1024, shuffle=False)\n",
    "tokenized = tokenized.map(prepare_input, remove_columns=datasets.column_names)\n",
    "tokenized.set_format(type='torch', device=device)\n",
    "# 将所有的数据拼接成一个长长的张量\n",
    "train_set = {\n",
    "    'input_ids': torch.concatenate(tokenized['train']['input_ids']),\n",
    "    'labels': torch.concatenate(tokenized['train']['labels'])\n",
    "}\n",
    "test_set = {\n",
    "    'input_ids': torch.concatenate(tokenized['test']['input_ids']),\n",
    "    'labels': torch.concatenate(tokenized['test']['labels'])\n",
    "}\n",
    "\n",
    "train_set['input_ids'].shape, train_set['labels'].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "sTMUMVroLKh5",
    "outputId": "e802fa5f-0703-4550-9143-d10ae7098db3"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor([[ 1499,   422, 39417,  ...,  4876,    13, 19430],\n",
       "         [32543,   262,  2581,  ..., 12064,   326,  8477],\n",
       "         [  198,   198, 21017,  ...,   262, 17118,     1],\n",
       "         ...,\n",
       "         [  257,  4876,    13,  ...,   257,  2882,   326],\n",
       "         [  618,   262,  4403,  ...,   198,  9487,  1958],\n",
       "         [  257,  4876,    13,  ..., 20277,   364,   477]], device='cuda:0'),\n",
       " tensor([[ 1499,   422, 39417,  ...,  -100,  -100,  -100],\n",
       "         [ -100,  -100,  -100,  ...,  -100,  -100,  -100],\n",
       "         [ -100,  -100,  -100,  ...,   262, 17118,     1],\n",
       "         ...,\n",
       "         [ -100,  -100,  -100,  ...,  -100,  -100,  -100],\n",
       "         [  618,   262,  4403,  ...,  -100,  -100,  -100],\n",
       "         [ -100,  -100,  -100,  ..., 20277,   364,   477]], device='cuda:0'))"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def get_data(data, batch_size, sequence_len):\n",
    "    '''\n",
    "    生成训练数据\n",
    "    '''\n",
    "    inputs = data['input_ids']\n",
    "    labels = data['labels']\n",
    "    # 生成截取数据的起点，形状为(B)，其中B等于batch_size，表示批量数据的大小\n",
    "    ix = torch.randint(len(inputs) - sequence_len, (batch_size,))\n",
    "    x = torch.stack([inputs[i: i + sequence_len] for i in ix])\n",
    "    # 如果直接使用transformers提供的封装来计算模型损失，则不需要对标签变量进行额外的移动操作\n",
    "    # 因为模型内部会自动进行移位操作，具体细节请参考官方文档\n",
    "    y = torch.stack([labels[i: i + sequence_len] for i in ix])\n",
    "    return x, y\n",
    "\n",
    "get_data(test_set, batch_size, sequence_len)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "id": "15YcHmQvLKh7"
   },
   "outputs": [],
   "source": [
    "def print_trainable_parameters(model):\n",
    "    \"\"\"\n",
    "    Prints the number of trainable parameters in the model.\n",
    "    \"\"\"\n",
    "    trainable_params = 0\n",
    "    all_param = 0\n",
    "    for _, param in model.named_parameters():\n",
    "        all_param += param.numel()\n",
    "        if param.requires_grad:\n",
    "            trainable_params += param.numel()\n",
    "    trainable = f'trainable params: {trainable_params:,}'\n",
    "    params = f'all params: {all_param:,}'\n",
    "    percent = f'trainable%: {100 * trainable_params / all_param:.3f}'\n",
    "    print(f'{trainable} || {params} || {percent}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "Zy9_Nuf7LKh8",
    "outputId": "bc0952f6-855c-4e29-ce27-34a97788076a"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "trainable params: 124,439,808 || all params: 124,439,808 || trainable%: 100.000\n",
      "trainable params: 147,456 || all params: 124,587,264 || trainable%: 0.118\n"
     ]
    }
   ],
   "source": [
    "from peft import LoraConfig, PeftModel\n",
    "\n",
    "def init_peft_model(model):\n",
    "    # 设置LoRA参数\n",
    "    config = LoraConfig(\n",
    "        r=4,\n",
    "        lora_alpha=32,\n",
    "        target_modules=['c_attn'],\n",
    "        lora_dropout=0.1,\n",
    "        # c_attn.weight的形状是(fan_in, fan_out)，所以该参数设置为True\n",
    "        # 但需注意的是，普通的线性模型权重参数的形状是(fan_out, fan_in)\n",
    "        fan_in_fan_out=True,\n",
    "        bias='none')\n",
    "    return PeftModel(model, config, adapter_name='lora_alpaca')\n",
    "\n",
    "print_trainable_parameters(model)\n",
    "model = init_peft_model(model)\n",
    "# 确保模型在训练状态\n",
    "model.train()\n",
    "print_trainable_parameters(model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "Wb5LW_BKLKh9",
    "outputId": "336dd4a8-5af1-409e-9ce8-b7dd154ca415"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'train': 2.9235620498657227, 'test': 2.8850600719451904}"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from contextlib import nullcontext\n",
    "\n",
    "def estimate_loss(model, ctx=nullcontext()):\n",
    "    '''\n",
    "    估计模型损失\n",
    "    ctx参数是为禁用LoRA或者混合精度做准备，当ctx=nullcontext()时，没有任何作用\n",
    "    '''\n",
    "    re = {}\n",
    "    # 将模型切换至评估模式\n",
    "    model.eval()\n",
    "    _train = lambda: get_data(train_set, batch_size, sequence_len)\n",
    "    re['train'] = _loss(model, _train, ctx)\n",
    "    _test = lambda: get_data(test_set, batch_size, sequence_len)\n",
    "    re['test'] = _loss(model, _test, ctx)\n",
    "    # 将模型切换至训练模式\n",
    "    model.train()\n",
    "    return re\n",
    "\n",
    "@torch.no_grad()\n",
    "def _loss(model, data_loader, ctx):\n",
    "    \"\"\"\n",
    "    计算模型在不同数据集下面的评估指标\n",
    "    \"\"\"\n",
    "    loss = []\n",
    "    # 随机使用多个批量数据来预估模型效果\n",
    "    for k in range(eval_iters):\n",
    "        inputs, labels = data_loader()\n",
    "        with ctx:\n",
    "            # 使用transformers提供的封装来计算模型损失\n",
    "            loss.append(model(input_ids=inputs, labels=labels).loss.item())\n",
    "    return torch.tensor(loss).mean().item()\n",
    "\n",
    "estimate_loss(model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "id": "0C7dGZoH1wYs"
   },
   "outputs": [],
   "source": [
    "# get_lr的实现参考自https://github.com/karpathy/nanoGPT/blob/master/train.py\n",
    "import math\n",
    "\n",
    "warmup_iters = 200\n",
    "lr_decay_iters = 3000\n",
    "min_lr = learning_rate / 10\n",
    "\n",
    "def get_lr(it):\n",
    "    '''\n",
    "    动态调整学习速率\n",
    "    it表示训练次数\n",
    "    '''\n",
    "    # 1、线性预热\n",
    "    if it < warmup_iters:\n",
    "        return learning_rate * it / warmup_iters\n",
    "    # 2、超出lr_decay_iters，则返回min_lr\n",
    "    if it > lr_decay_iters:\n",
    "        return min_lr\n",
    "    # 3、逐步衰减学习速率\n",
    "    decay_ratio = (it - warmup_iters) / (lr_decay_iters - warmup_iters)\n",
    "    assert 0 <= decay_ratio <= 1\n",
    "    coeff = 0.5 * (1.0 + math.cos(math.pi * decay_ratio))\n",
    "    return min_lr + coeff * (learning_rate - min_lr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "id": "wKhgo30-i1Yy"
   },
   "outputs": [],
   "source": [
    "# 梯度裁剪的超参数\n",
    "grad_clip = 1.0\n",
    "\n",
    "def train_gpt_optimum(model, optimizer, data_loader, max_iters=1000):\n",
    "    lossi = []\n",
    "    scaler = torch.cuda.amp.GradScaler(enabled=(device == 'cuda'))\n",
    "    for iter_num in range(max_iters):\n",
    "        # 动态调整学习率\n",
    "        lr = get_lr(iter_num + 1)\n",
    "        for param_group in optimizer.param_groups:\n",
    "            param_group['lr'] = lr\n",
    "        # 梯度累积\n",
    "        for i in range(gra_acc_steps):\n",
    "            inputs, labels = data_loader()\n",
    "            # 混合精度训练\n",
    "            ## 如果是用CPU进行计算，可能需要将dtype变成torch.bfloat16\n",
    "            ## 当然如果使用CPU，需要非常长的时间\n",
    "            ctx = torch.autocast(device_type=device, dtype=torch.float16)\n",
    "            with ctx:\n",
    "                loss = model(input_ids=inputs, labels=labels).loss\n",
    "                lossi.append(loss.item())\n",
    "                loss *= 1 / gra_acc_steps\n",
    "            scaler.scale(loss).backward()\n",
    "        # 梯度裁剪\n",
    "        scaler.unscale_(optimizer)\n",
    "        clip_grad_norm_(model.parameters(), grad_clip)\n",
    "        scaler.step(optimizer)\n",
    "        scaler.update()\n",
    "        optimizer.zero_grad(set_to_none=True)\n",
    "\n",
    "        if iter_num % eval_interval == 0:\n",
    "            # 训练模型此脚本时，由于内存充足，忘记在评估模型损失时使用混合精度\n",
    "            # 这是一个小“bug”，虽然这并不影响训练结果\n",
    "            stats = estimate_loss(model)\n",
    "            train_loss = f'train loss {stats[\"train\"]:.4f}'\n",
    "            test_loss = f'test loss {stats[\"test\"]:.4f}'\n",
    "            print(f'step {iter_num:>4}: {train_loss}, {test_loss}')\n",
    "    return lossi"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "k9JRRMu9TZ74",
    "outputId": "7e92832f-c245-427a-ddd5-198314486402"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "step    0: train loss 2.8921, test loss 2.8898\n",
      "step  100: train loss 2.4980, test loss 2.3948\n",
      "step  200: train loss 2.3408, test loss 2.2779\n",
      "step  300: train loss 2.2937, test loss 2.2836\n",
      "step  400: train loss 2.2565, test loss 2.2617\n",
      "step  500: train loss 2.2493, test loss 2.2790\n",
      "step  600: train loss 2.2829, test loss 2.1847\n",
      "step  700: train loss 2.2577, test loss 2.3036\n",
      "step  800: train loss 2.2070, test loss 2.2426\n",
      "step  900: train loss 2.2182, test loss 2.2067\n",
      "step 1000: train loss 2.2719, test loss 2.2094\n",
      "step 1100: train loss 2.3164, test loss 2.2448\n",
      "step 1200: train loss 2.2444, test loss 2.2219\n",
      "step 1300: train loss 2.2858, test loss 2.1962\n",
      "step 1400: train loss 2.2495, test loss 2.1978\n",
      "step 1500: train loss 2.2293, test loss 2.2963\n",
      "step 1600: train loss 2.2280, test loss 2.2183\n",
      "step 1700: train loss 2.3147, test loss 2.2030\n",
      "step 1800: train loss 2.2065, test loss 2.2909\n",
      "step 1900: train loss 2.1894, test loss 2.2781\n"
     ]
    }
   ],
   "source": [
    "data_loader = lambda: get_data(train_set, batch_size, sequence_len)\n",
    "# 设置最优化算法的参数\n",
    "weight_decay = 1e-1\n",
    "beta1 = 0.9\n",
    "beta2 = 0.95\n",
    "optimizer = optim.AdamW(model.parameters(), lr=learning_rate,\n",
    "                        betas=(beta1, beta2), weight_decay=weight_decay)\n",
    "l = train_gpt_optimum(model, optimizer, data_loader, max_iters=2000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 448
    },
    "id": "pCojwhGyyRqL",
    "outputId": "60ee62a1-c60e-45ec-8d0c-8308ab3a0c62"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x78c8690a4760>]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAiMAAAGdCAYAAADAAnMpAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAABbZ0lEQVR4nO3deXgTdeIG8DdJb+hBCy0UCpT7KJfcN0gFhUVA1wNR1EVdtCi4Lot4X1jU9T5A8cDfKpbVBVFAkPuGQjnLUShXOVrK1bS0pUcyvz/apJNkJpmkSaZp38/z8NBOJpPvJGnmzffUCIIggIiIiEglWrULQERERHUbwwgRERGpimGEiIiIVMUwQkRERKpiGCEiIiJVMYwQERGRqhhGiIiISFUMI0RERKQqP7ULoITRaMTFixcRGhoKjUajdnGIiIhIAUEQUFBQgNjYWGi18vUfPhFGLl68iLi4OLWLQURERC44d+4cmjVrJnu7T4SR0NBQABUnExYWpnJpiIiISIn8/HzExcWZr+NyfCKMmJpmwsLCGEaIiIh8jKMuFuzASkRERKpiGCEiIiJVMYwQERGRqhhGiIiISFUMI0RERKQqhhEiIiJSFcMIERERqYphhIiIiFTFMEJERESqYhghIiIiVTGMEBERkaoYRoiIiEhVDCOVzlwpxJebTqKwpFztohAREdUpPrFqr6d9viET763OAABczCvG6+MSVC4RERFR3VHna0bOXSsyBxEA2HX6moqlISIiqnvqdBg5f70IT/6YpnYxiIiI6rQ63UzzxP+l4Uh2vtrFICIiqtPqdM0IgwgREZH66nQYISIiIvUxjFgRBLVLQEREVLcwjBAREZGqGEaIiIhIVQwjREREpKo6HUa6NQu32SaAnUaIiIi8qU6HkUl9W6hdBCIiojqvTocRnVajdhGIiIjqPIYRIiIiUhXDiBXOM0JERORdDCNERESkKoYRIiIiUlXdDiMahhEiIiK11e0wopPoM6JCOYiIiOqyuh1GWDNCRESkujodRvzYZ4SIiEh1dTqMGDiOl4iISHV1OozkFZXZbMsvtt1GREREnlOnw0hIgM5mW25BiQolISIiqrvqdBgZ3j4ajw2Kx6tjO1lsF9h8Q0RE5DV1OoxotRq89JdOGNOlicX2UoNRpRIRERHVPXU6jJhorIb4lpYzjBAREXkLwwgA6+lGGEaIiIi8h2EEgNYqjZQwjBAREXkNwwgA67nPWDNCRETkPQwjADSw6jPCDqxERERewzACQGP1LJSUMYwQERF5C8MIAOsVakoNBlXKQUREVBcxjMC2A+vXW06rVBIiIqK6h2EEtkN7/0jPwblrReoUhoiIqI5hGIFtzQgA6LlgHhERkVcwjBAREZGqGEYgXTNys4ydWImIiLzBqTAyb948dO3aFWFhYQgLC0P//v3xxx9/2L3Pzz//jA4dOiAoKAhdunTBypUrq1VgT5DIIigqZRghIiLyBqfCSLNmzTB37lykpaVhz549uPXWWzFu3DgcPnxYcv/t27dj4sSJmDJlCvbt24fx48dj/PjxSE9Pd0vh3UWqZqSYNSNEREReoREEQajOASIjI/Hee+9hypQpNrfdd999KCwsxPLly83b+vXrh+7du2P+/PmKHyM/Px/h4eHQ6/UICwurTnElGY0CWr1gWWPz0X3dMb5HU7c/FhERUV2h9Prtcp8Rg8GAlJQUFBYWon///pL77NixA4mJiRbbRo0ahR07dtg9dklJCfLz8y3+eZJUMw37jBAREXmH02Hk0KFDqF+/PgIDAzF16lQsXboUnTp1ktw3JycHMTExFttiYmKQk5Nj9zGSk5MRHh5u/hcXF+dsMZ2ikUgjhupVGBEREZFCToeR9u3bY//+/di1axeefPJJPPzwwzhy5IhbCzV79mzo9Xrzv3Pnzrn1+PaEB/sDqGi6ISIiIs/zc/YOAQEBaNOmDQCgZ8+e2L17Nz7++GN8+eWXNvs2btwYly5dsth26dIlNG7c2O5jBAYGIjAw0NmiuYVOW1FLYmAYISIi8opqzzNiNBpRUlIieVv//v2xbt06i21r1qyR7WNSE1RmERiYRYiIiLzCqZqR2bNn44477kDz5s1RUFCARYsWYePGjVi9ejUAYPLkyWjatCmSk5MBANOnT8fQoUPx/vvvY8yYMUhJScGePXvw1Vdfuf9M3MQ0zJfNNERERN7hVM1Ibm4uJk+ejPbt22PEiBHYvXs3Vq9ejdtuuw0AkJWVhezsbPP+AwYMwKJFi/DVV1+hW7du+OWXX/Drr78iISHBvWfhRqa+rHNWHlW3IERERHVEtecZ8QZPzzMCAC2fX2Gz7czcMR55LCIiorrA4/OM1DbThrfBmK5NEBsepHZRiIiI6hSnR9PUVv8c1R4AMOid9SqXhIiIqG5hzYgV09BeIiIi8g6GESviRfM4ooaIiMjzGEasiGeGL2cYISIi8jiGESs6URrhLKxERESexzBiRdxMw8XyiIiIPI9hxIpW1IHVwDnhiYiIPI5hxIp4MA1rRoiIiDyPYcSOcqNR7SIQERHVegwjVsSVIdcKS9UrCBERUR3BMGJF3DDz7OIDqpWDiIiormAYsSJeN/Bodr6KJSEiIqobGEassM8qERGRdzGMWBHANEJERORNDCNWWDNCRETkXQwjVjo0CVO7CERERHUKw4iV1+/srHYRiIiI6hSGESuR9QLULgIREVGdwjBix6A2DdUuAhERUa3HMCLhmRFtAQBxkcEql4SIiKj2YxiREOhX8bQYjBxaQ0RE5GkMIxK0moqle5lFiIiIPI9hRIKu8lkxMo0QERF5HMOIBFPNiIEzoBEREXkcw4gENtMQERF5D8OIBJ22MowwjRAREXkcw4iEyiwCI5tpiIiIPI5hRIK2Mo38kZ6DMoNR5dIQERHVbgwjEkx9RgDgu22nVSwJERFR7ccwIkEnCiM7T11TsSRERES1H8OIBFMzDVDVmZWIiIg8g2FEgjh/+DGMEBEReRTDiAQda0aIiIi8hmFEgkbUZ4Q1I0RERJ7FMCJB3IFVp+VTRERE5Em80koQr0nDmhEiIiLPYhiRkFdUav45OECnYkmIiIhqP4YRCVdvVIURzsBKRETkWQwjEuIiQ8w/M4wQERF5FsOIhPHdYxETFggAKC1nGCEiIvIkhhEJfjotnhrWBgBQZuDKvURERJ7EMCIjwK/iqSlhzQgREZFHMYzI8NdVPDXsM0JERORZDCMyTDUj7DNCRETkWQwjMgIqa0ZKWTNCRETkUQwjMgL8KmZeZTMNERGRZzGMyAjQVcy8ymYaIiIiz2IYkeGvq6gZOZZTgOuFpQ72JiIiIlcxjMgwdWAFgNs+3KxiSYiIiGo3hhEZpqG9AHDlRomKJSEiIqrdGEZkBPrxqSEiIvIGXnFlhAT6qV0EIiKiOoFhREaIv07tIhAREdUJDCMyQgIZRoiIiLyBYURGgI5PDRERkTfwiitDo9GoXQQiIqI6gWGEiIiIVMUwopAgCGoXgYiIqFZiGFHIYGQYISIi8gSGEYVKuXovERGRRzCMKFTOmhEiIiKPYBhRyMgwQkRE5BEMIwqxzwgREZFnMIzYsfzpQeafDRxNQ0RE5BFOhZHk5GT07t0boaGhiI6Oxvjx45GRkeHwfh999BHat2+P4OBgxMXF4dlnn8XNmzddLrS3JDQNh05bMfkZswgREZFnOBVGNm3ahKSkJOzcuRNr1qxBWVkZRo4cicLCQtn7LFq0CM8//zxeffVVHD16FN988w0WL16MF154odqF9wZd5UysbKYhIiLyDD9ndl61apXF7wsXLkR0dDTS0tIwZMgQyfts374dAwcOxAMPPAAAaNmyJSZOnIhdu3a5WGTv0moBGBhGiIiIPKVafUb0ej0AIDIyUnafAQMGIC0tDampqQCAU6dOYeXKlRg9enR1HtprTDUjRrbTEBEReYRTNSNiRqMRM2bMwMCBA5GQkCC73wMPPIArV65g0KBBEAQB5eXlmDp1qt1mmpKSEpSUlJh/z8/Pd7WY1abVspmGiIjIk1yuGUlKSkJ6ejpSUlLs7rdx40a8/fbb+OKLL7B3714sWbIEK1aswJtvvil7n+TkZISHh5v/xcXFuVrMajN1YGXNCBERkWdoBBdWgJs2bRqWLVuGzZs3Iz4+3u6+gwcPRr9+/fDee++Zt/3www944okncOPGDWi1tnlIqmYkLi4Oer0eYWFhzha3Wnq+uQZXC0uxesYQtG8c6tXHJiIi8mX5+fkIDw93eP12qplGEAQ8/fTTWLp0KTZu3OgwiABAUVGRTeDQ6XTm40kJDAxEYGCgM0XzGA37jBAREXmUU2EkKSkJixYtwrJlyxAaGoqcnBwAQHh4OIKDgwEAkydPRtOmTZGcnAwAGDt2LD744AP06NEDffv2RWZmJl5++WWMHTvWHEpqMl1ljmKfESIiIs9wKozMmzcPADBs2DCL7d999x0eeeQRAEBWVpZFTchLL70EjUaDl156CRcuXECjRo0wduxYzJkzp3ol9xLTaJr84jKVS0JERFQ7udRnxNuUtjl5wqB31uP89WIAwPwHb8HtCU28+vhERES+Sun1m2vTOGAaTQMALy5NV7EkREREtRPDiAOmZhoAEP1IREREbsIw4oBWKw4jTCNERETuxjDigCiLgFGEiIjI/RhGHBAP6dWyZoSIiMjtGEYcOHm50PwzswgREZH7MYw4oZwTnxEREbkdw4gTrheWql0EIiKiWodhxAnNI0PULgIREVGtwzDihGYMI0RERG7HMOLAM7e2Mf9cWm5QsSRERES1E8OIA/8Y2R7zJt0CACgtN6pcGiIiotqHYUSBoAAdAKDUwDBCRETkbgwjCgTqKp6m9Av5KpeEiIio9mEYUcDfr+pp2p55RcWSEBER1T4MIwoE6Kqept8PXlSxJERERLUPw4gCAaKaEZ2Wc8ITERG5E8OIAhZhhAvUEBERuRXDiALiZppSA9enISIicieGEQXENSM/pWZh6wl2YiUiInIXhhEFrFtmXvktXZ2CEBER1UIMIwpEhwZZ/B7kp1OpJERERLUPw4hCScNbm38ODmAYISIicheGEYUi6wWafw7y59NGRETkLryqKnRL8wjzz4FspiEiInIbhhGFOjQOM/9cbuTwXiIiIndhGFFI3E+kU5MwO3sSERGRMxhGnPBw/xYAgAAdZ2ElIiJyF4YRJ2gqJxwxCGymISIicheGESeYFsn7fMNJFJWWq1waIiKi2oFhxAniBXsXbj+jWjmIiIhqE4YRJ2hFaURfVIavt5zC/nN56hWIiIioFvBTuwC+RCtapGZlejbOXSsGAJyZO0atIhEREfk81ow4QScKI6YgQkRERNXDMOIEcZ8RLUf3EhERuQXDiBPEfUY4CSsREZF7MIw4QQNWhxAREbkbw4gTyo1GtYtARERU6zCMOKHMwLYZIiIid2MYcUKZgTUjRERE7sYw4oRymTAicK0aIiIilzGMOKFUppnGwKE1RERELmMYcYJczUg5wwgREZHLGEacINdnpJR9SYiIiFzGMOKEMpkakHKOsiEiInIZw4gTysqla0A4yoaIiMh1DCNOkOsbwjBCRETkOoYRJ8iFjiMX871cEiIiotqDYcQJScPbSG5/4j9pXi4JERFR7cEw4oR+raLULgIREVGtwzBCREREqmIYcZPDF/VqF4GIiMgnMYy4yePf71G7CERERD6JYcRNLupvql0EIiIin8QwQkRERKpiGCEiIiJVMYwQERGRqhhG3KRjkzC1i0BEROSTGEbcpKi0HFlXi9QuBhERkc9hGKmG5U8PQtdm4QCAs1eLMOS9DThzpVDlUhEREfkWhpFqSGgajhdHd7TYtvPUVZVKQ0RE5JsYRqrJT6ex+l0LQRAgCIJKJSIiIvItfmoXwNf5aS3z3Kr0bBy/VICl+y5gxdODEB0WpFLJiIiIfAPDSDVZ14ysPZoLHM0FAGw8fhn39opTo1hEREQ+g800Tnp7QhcAwEtjKvqK+Ovkn8KbZQavlImIiMiXsWbESQ/0bY4xXZsgPNgfAOCn1cju+8qywxjdpQka1g/0VvGIiIh8jlM1I8nJyejduzdCQ0MRHR2N8ePHIyMjw+H98vLykJSUhCZNmiAwMBDt2rXDypUrXS602kxBBLBfMwIA+7PyPFwaIiIi3+ZUzcimTZuQlJSE3r17o7y8HC+88AJGjhyJI0eOoF69epL3KS0txW233Ybo6Gj88ssvaNq0Kc6ePYuIiAh3lF911n1GrGns30xERFTnORVGVq1aZfH7woULER0djbS0NAwZMkTyPt9++y2uXbuG7du3w9+/okahZcuWrpW2BrIeTWONYYSIiMi+anVg1ev1AIDIyEjZfX777Tf0798fSUlJiImJQUJCAt5++20YDPKdO0tKSpCfn2/xr6byd1AzQkRERPa5HEaMRiNmzJiBgQMHIiEhQXa/U6dO4ZdffoHBYMDKlSvx8ssv4/3338dbb70le5/k5GSEh4eb/8XF1dzhsX4O+oxowLBCRERkj8thJCkpCenp6UhJSbG7n9FoRHR0NL766iv07NkT9913H1588UXMnz9f9j6zZ8+GXq83/zt37pyrxfQ4e6NpAIBZhIiIyD6XhvZOmzYNy5cvx+bNm9GsWTO7+zZp0gT+/v7Q6XTmbR07dkROTg5KS0sREBBgc5/AwEAEBvrGcFhHo2mYRYiIiOxzqmZEEARMmzYNS5cuxfr16xEfH+/wPgMHDkRmZiaMRqN52/Hjx9GkSRPJIOJrdFoNPr6/u+ztGvZgJSIissupMJKUlIQffvgBixYtQmhoKHJycpCTk4Pi4mLzPpMnT8bs2bPNvz/55JO4du0apk+fjuPHj2PFihV4++23kZSU5L6zUNm47k1lbztzpRA3Ssq9WBoiIiLf4lQYmTdvHvR6PYYNG4YmTZqY/y1evNi8T1ZWFrKzs82/x8XFYfXq1di9eze6du2KZ555BtOnT8fzzz/vvrOoAZ4a1lpy+6u/HcZfPtni5dIQERH5Do3gA2vd5+fnIzw8HHq9HmFhYWoXR9Lhi3qM+WSr7O1n5o7xYmmIiIjUp/T6zYXy3ETnaFQNERERSWIYcROdgo6qgiBg8/HLyM2/6YUSERER+QaGETfRKqgZ+fPIJUz+NhUD31nvhRIRERH5BoYRN9EqqBnZdPwyAKDMUOO76RAREXkNw4ibKGmmISIiIlsMI27iYPFeAEDNH7dERETkfQwjbuJoNI3BKABgGiEiIrLGMOImjpppnvwhzUslISIi8i0MI27iaDTNn0cusZmGiIhIAsOImzjbfXXXqaseKQcREZGvYRhxE2dX573vq53o+eYapJ6+5qESERER+QaGETdREkWsm2muFpbi3i93uLUce7OuY+Dc9ViVnuPW4xIREXkKw4ibKJn07HpRqcfLMWXhblzIK8ZUdpglIiIfwTDiJuEh/pjQo6ndff48csnj5bhZZvT4YxAREbkTw4gbfXhfdwxoHaVqGTgRLBER+RqGETdTOwwwixARka9hGHEzDeMAERGRUxhG3Ez1mhG1C0BEROQkhpFahlGEiIh8DcNIbcM0QkREPoZhxM3YTEJEROQchhE3UzuKqP34REREzmIY8bCP7+/u1cdjzQwREfkahhE3s84C47rbn5XVJCU1C11fW43dZ7hwHhER1S0MI27mSr3EtcJSPL/kEPJvluOe+dVbOI8VI0RE5GsYRjwoOjRQ0X63vLlG9rYLecX4bP0JXC+sWmTv3LUi5BbclNyfWYSIiHyNn9oFqG3EfTZ+mTqg2se7Z952XNTfxIHzeiyY3AsFN8sw+N0NAIDTyaNt+oiwzwgREfka1oy4mTgKNI8KAQA8fWsbp47xyHepKC41AAAu6itqQLZnXgEAZOurakRKDVyhl4iIfB/DiJtJVUw8N7I9liUNVHyMjRmX8Z+dZyy2abUVB/bTVj2AKbBYPL7iRyEiIqoZGEbcrEfzBpLbdVrnYsL1ojLJ+4ubYYqkwgjTCBER+Rj2GXGzxwe3QrC/DkPaNbTYfqOk3PzzC6M74O2Vx+wep6zcsglGW5kyykVNM1JhhIiIyNcwjLhZgJ8WfxsUb7O9W7MINKwfiKYNglFuFBweJ/PyDby1/Ij5d1MYKTNU3VeqmYYNNURE5GsYRrwkOECHrbOGI0CnxcfrTjjcf2PGZWzMuGz+vbi0omal3CiuGSm3uZ+4meb4pQK0iwmtRqmJiIg8j31GvCjIXwetVoOScudHwRSWGvD5hkyLmpGbEscR14uM/HCzK8UkIiLyKoYRFZS6EEYA4L3VGRZ9RowKmnuIiIhqOoYRFZQaXO94Ku5vYpAIIxxNQ0REvoZhRAUlZa5PVpZfXDXkd96mk7h73nYUikbqaNiBlYiIfAzDiAp6tpCei0SJ3w9eNP+cdvY60s5ex4+7ziq+f1FpOTYcy0VJOYcFExFRzcAwooJ7esUhsWO0S/e9eqPUZpt4vhFHzTTTU/bj0YW78aZo2DAREZGaGEZUoNNqcNctzVy67/Ui2zAiCFWdYh010qw5cgkA8MPOLJvbLheU4IuNmbIrAntKOdfYISKq0xhGVGIUXBsJYz1NPAB8vO4Eur3+J67eKKnWqr1JP+7Fu6syMGXhHpeP4awNGblo99IfWLzbNhwREVHdwDCiEldH5d4sk+7rUVxmwLL9FyVvUyr1zDUAwKEL+modxxl//780GAVg1v8Oee0xiYioZmEYUYngYs2IvQnTtBrgQl6xq0UiIiJSBcOISlxtprE3Ydob7JRKREQ+iGFEJS5mEbuqOyErJ0wjIiI1MIyoxFszuV+5UaK4Schf6/rbYV/WdaSkZrnc/ERERHUXw4hKXG2mcVavt9Ziyvfyo2O2n7yCh77ZhTNXCuGnc71qZMIX2/H8kkP4dH2mbCdbIiIiKQwjKokMCfDaY60/los7Pt6CjRm5Nrc9sGAXtpy4gumL98NPqzyMGIwC/r06A1tOXLbY/sGa45i2aF+1y0xERHUHw4hKbu0Qjb8NjPfa4x3NzseTP+yVvf3AuTzk3yyXvd3a/9LO47MNmXjom1Sb29YeveRSGYmIqG5iGFGJVqvBK2M7efUxi6vRfGI0CkheeRQrDmYDAE7kFrirWEREVMf5qV0AqplKyg0I9NOZf1979BK+3HwKADCm6xgs2HLaLY8jgB1eiYjqOtaMqOyF0R3ULoKk9i+twrL9F8y/X75RYv751OUbahSpWrL1xXhv9THk6L277k5d9sGfGfho7XG1i0FEPoBhRGX+upr7EkxP2W/+WTzZmr7Ydn0cKYfO67Fg8ykYjAJulJQj9fQ1GCvHNL//ZwaSVx612H/expN2j6cvLsO3W08jN98yUKw7egn7sq7bve9j3+/B5xtO4tGFuxWVnarnemEpPlmfiY/WnkDBTWXvFyKqu9hM40M6NgnD0ex8rz/uj7vO4vXfq2Z3VToseexnWwEAIYE6/JSahfQL+Ui+qwvGd2+KT9dn2uz/zqpjeHJYa9njzfrlIFYdzkHK7iz8+exQAMDZq4Xmoctn5o6Rve/hixXPmxrPX11UKlqJ2chFmYnIgZr7tZxsfPlgT7cer7hUWYfWF5emW/xeWm4ZRowOZnDbcfIq0i9UhIBl+y+4PMeKaZTO8UtVzUTnrnEtnhqPM/sSkQMMIypz5rrcPCrErY99z5fbXbrfZxtOWPz+8rJ0mT0rLK8cgQMA9QJcr4wTP1Wzl9SdVX53nrqKzccvO96xBuFEvETkDIaRGmRIu0YO9wkNdF/Lmqm2wlnbMq9a/P7jrizF983W38ToT7Y4/ZhnrxbCIKqB+Sk1Cwaj4PH1dARBgL5IvT4PBqOA+7/aicnfpiKvqFS1clQLgwkROcAwUoN8PbkXVs0YbHefldMH19gROGL3zt8huf1Idj7OXi1y+njvrDpms83ghQV+/vnzQXR740/sOnXV8c4eUC7qcJGnYihylnjItruWPkg9fQ07TqrzOhCRZzGM1CABflp0aBxmd5+4yBA8MUS+k2dNkXrmmtuOZTAKWHkox2b79JR9KCypmjV21i8H0fL5FVgkUVMjnul++cGLmPjVTptROVL+t/c8AOCzDbYdbq1ty7yC9cdsZ58VBAFFpcpmt72QV4w3fj+Cc9ecD2yesnTfecxecsip8CfOH+4IIyXlBtz75Q5MXLCTo3OIaiGGER8V37CeVx7nz8O2IcDb1hyRLsMf6Tl44j9p5t8X7zkHAHhhqW1/Ej/REOppi/Zhx6mreP9P5XNgOLqgGowCJn29C39buAfXCy2bU/7x3wPo9MpqnL5SaN725+EcrJZ4bh//fg++3XYaExfsBGB5UTf96O2VkZ9dfAA/pWbh9wMXFd9H/HzJZZjTVwpxVTR/jT1lhqqDOLNsARH5BoaRGujtCV3QPNJ+Z9WFj/b2SlnEF3u1XHeheeLnymBiIrUI4A2FtRWA4+Gp4pWKN2Tk4ustpzA9ZR8MRgFL91VMHvft1opZa/dlXccT/0nDUz/utakxOVI59Pj89YpRQta54/SVQvR8a63DOVk84YrC4ABYhSiJ8JStL8bwf29Ez7fWKjqe+NVzNHrLU7ZlXsGJS1wGgcgTGEZU1jq6vs22B/o2x+Z/Dbd7vxZR9fDYIO8ttKemchcuPjN/OQigIiRsOJZrERZMQvx1NtvkOKoZER//H/89gLdWHMWy/RexTrRoYFHlUOo9ZyomaDMYBZSU2U854scVBAFvLT+Ca4Wlkn1oPM2ZChlxk47Uy3fwvN65x5YoR47+plf6DQHAiUsFmPT1Ltz24WavPB5RXcMworIhbRtizoQELHlqgNpFqbHKDa7PmvXm8iN4dOFuyQticID7wkhJuXQZxTVLxWUVtSDnr1f1B3F0XOvbXQlm9iida0aqLPYYLJppql9mo9XxtmdeQb/kdXjyB2U1d+kX9Lhn/naknVXWl8n6eTmWU/0aEW83rxEJgoBjOfnV+gz1FoYRlWk0Gkzq2wK3NG9gd79Av7rxUq09cgn6ojLMWXEERypnTS03uP4h7sywY3vOXi3Cr/sumL+J5+bfxPaTV8wXGKmaF2ummpFyB7UGYtY3u2tkCgBsOJaLjq+swhcbHXfOlSqL3X0lwsih83os3HYaRqPg9Dwkguiz1CgI+HZbRZPXn0dsOwxLeWDBTuw+cx13z5Me5SW2dN95dHxlFX7YedbiMavjp9QsdH9jDfafy6vWcYic8e22M7j9oy149r8H1C6KQ3XjCufjAnRah802tcVj/7cHT6fsw4ItpzH6ky0VwcRqDRtnSHQVMSuzCjn2vrnmFpRgxuL9WJRaEW4GzF2PBxbswqbKychO5DpePHBjRsW+4gDiaNVi8UVYgHvDiKkp691VGYr2d6pmRFzuyruN/WwrXvv9CH4/eBHOTj5i3SG2npPz7TjT6fXZxRUf3C/9WjWZX3UCMVAxSZ++uAwzUvZV6zhyBEHAwfN55lC8ZO953PXFNrsjxopLDW6prVGrDw859kXlKEBnOp+rxakwkpycjN69eyM0NBTR0dEYP348MjKUfZABQEpKCjQaDcaPH+9sOeu0Ye0bISYsyGa7pyf8Uot4ttFP1p+ws6dj9hYiFFddnrx8A73nrEXL51fg3i93yFZrbjtxpeK+lR/A2zKvQF9chr8r7Oh7NDvf4sNbfC34z44zNvtbBwBxR9pjOfk4IxqhI2dVerZkx0t7QU2KM9ct62YVsWM5BU7XjFgeQ3A6jCjxxcZMDH53vcW2rMo5cQxWBd6WeQWv/XZYUY2YtWuFpS6HgEPn9Xj6p302Q79/Sj2HOz/bhsnfpAKo6Le0NysPb62QDvJZV4vQ8ZVVmPaTc+Fo1i8HMT1ln7n8eUWl6PP2ujo1I7Iv8aWY6FQY2bRpE5KSkrBz506sWbMGZWVlGDlyJAoLHX8gnjlzBv/85z8xeLD9Sb3Ili+9odztwvXqrT0TYC+MiELBW8uP4MqNiiG5qaevYc9Z6VWArftslBkEpKQqbwr6avMp8xBkAFhxMBuPfb8H+qIyvLzssMW+564V4Wa55cVOfFG8/aMtGPbvjXYviNtPXsHUH/ZKdrzUOZlGjEahsonF8TvSUQdWZ4mPYTAC9d0URm6WGfD674ex/eQVvLsqw2ato+eXVNQeWX/7n/T1LizcfgZfbT7l1OOduVqEW95cgxeW2i6hkHW1CJ+uO2F3xt+xn23F7wcuYmplX5l9WdeRm38Ti1IrmpSs5/fJkakZ+c/OMwAq3n9KFZcasHjPOSzbf9F83JTd53DlRgl+cuJvgLzHl/opOfUXvWrVKovfFy5ciOjoaKSlpWHIkCGy9zMYDJg0aRJef/11bNmyBXl5eS4Vtq6Se0Np3FA1EuSvxU0HIzrUtKqa85z4+2kBmRGpBaKqe+tnWK5a3mA1xnfh9jNOlcc0zNfkjeUVqyF/uNZ2zpPB725ATFig+feLecVIPW3bAfPgeT36xEdKPl76BflRK1qZ98+NknLk6G+ijdVIr3PXi9D19T8xqW9zzB7dUfK+fxzKxt6s6/hL11jzNuuakXkbT+KvPZvJlkuK+G+g3GhEsGgklNEoQCsRrG6WGRDkYMTUgs2n8N22M/hu2xnJ269Vzhkj13H4gzXH8deezRAbEezoFCz8lJqF5Lu6WGyb8MU2XC0sxbGcAnw+6Ra798/MvYFD5/WY8EXF+lJdmoZL7ifXQVnq+TqWk4/ZSw7hnyPbY2Cbhja3GwT3BkzyPF96marVZ0Svr/igi4yU/iA0eeONNxAdHY0pU6YoOm5JSQny8/Mt/tVlct8C7+8dJ3ufZxPbITTIDy/KXDRMfGE21+qwVzNyWTRvhvU8JHJ9Odw9msUkt0D6G+yl/KoyPlRZBW+t2E7NiHWOFV/UxTUjG47lmn++7YNNSPxgE/ZmWdYO/XEoBzdKyvHl5lOyzVhP/rgXC7acxrL9VW3UUmH6l7TzsmWWIn7a95/Lw8frqprvyiQmgVl75BI6vLzKPLeLnDMOliYwBX573zDt1Y4czc6XnIRPytXK4LPlxGXM33QSoz/eIrsekSBULKBockgmdIrnsSkzGM0zFuskguhj3+/Bvqw8TPp6l6LymsrhboIg4LHv9+A5N3W6vF5Yike/S8XKQ45rga7eKKlVc8n4UMWI62HEaDRixowZGDhwIBISEmT327p1K7755hssWLBA8bGTk5MRHh5u/hcXJ3/Rrc0+uLcbejSPkP0W2qpRfRx6baTFtluaR2DPS4mYntgWB18diceHtMIzt7YBANzZLdbmGAG6WtrxpJK/n/z5HTiXhzd+P4KbZQabJouj2fnQF9tWl1e3I6Oc6syX8fj3e3BJpjpefNij2fnoPWcdftxVUaUvPudHF+42/5ytrzjWn4ctR6qEBlWF4iIHfSVO5FZ9oLunmabqIC9aNXFYd0QGgGcqO4qaap7kOGqp0gB4b/UxLEo9J7uPvdqXOz7eIrk8gT0Go4C5fxzDkex82TBlFATZDsUHz+eZfxbvkfjBJnR+dTX0xWWSTXTXCu0vxCh+vH8s3o8Nx3IddsDO0d/EN1tPI9+JKfxPXi7E2qOX8L+9581/F/riMizddx43RMs/XMgrxspD2Q6bIt5fk4ENGZfx1I97HT52z7fW4rYPNyvqi1VbJP24F/d+uUP1jsguh5GkpCSkp6cjJSVFdp+CggI89NBDWLBgARo2tK32kzN79mzo9Xrzv3Pn5D8IarO7bmmGpU8NlOy8ahIa5G/xe4CfFg3rV1Ttm77V/WNkexx783Z8MrGHzf2lqmtri8KScrs1IwDw7bbT+HDtcfhpLfd7e+UxjPxwk83+BqMg2VRSXdWZBqDUYMTLv9r2QQAsa3hm/e8grtwoMV/MHbXyzd900uKDXtwsWCozr4rJlsqOvoDjUThK2rXtHaPcYITBKFiUqUzhEyrXVGVyJDsfn284iaPZ8rWzUfUCFD2WUmWii4JcTVxFGJG+/8Svdpp/Fr//TQtU7j17XfK8HT0X4ovVrtPXLAKsnHu/3IE3lx/BKzLvTyni19H0uif9uBfPLj6AWZUjwABg+L834qkf9+J/ey/YHEPs6g37IavMYMQ987fj9d+r+mylyfQZqylKy434fEOm3WZYQPpvq7TcaF7fqdxgxIpD2Ug9fQ2nVA5gLvUCmzZtGpYvX47NmzejWTP5tt+TJ0/izJkzGDt2rHmbsbJK1c/PDxkZGWjd2raZIDAwEIGBgTbbyTG50SOO2s5ro37J6yz6hcjZdzYPjcNtA5+4icQk9cw13Pul47kqnFXdIbvWHRXTzl7DtEX7zMEUsA0QUlX11sTV/+IyKr3YA45rfQRBOhj9sPMsrtwowYzEdnarm0sNRoz5ZAty8m9i5+wRCPLXSdaWmPcvNyKgct4eV7pd/WFV3S9+Xi7kFSPrahH6t45y6pjiIbji50tqGYOKx5R/zxSK+on467Q2fWeMgiBZM+Loe4nU6+jobZtVOepnQ8Zl+zvKPI7pHLdmVoTbFYey8Xnlbab387qjl5zugyS2ZO957D5zHbvPVAUQU/+YKzdKUFhSjhZR7l0LTBAEHLqgR7uYUIefzZuOX0ZeUSk6NA5D+8ahACq+RL23OgPvrc7Amblj5B9HYtvQ9zYgW38TB14dafH+V3suK6ceXRAETJs2DUuXLsX69esRH29/OvIOHTrg0KFD2L9/v/nfnXfeieHDh2P//v11tvnFkxrVdy7E+VKborOUBBEAuFZUKvuh7y3VndZco9Fgx8mr+HrLKQiCgL//Zy+y9TctwoT1a+3omzAA5BeL+xxUHUAcbFYeysZf522XXWlYECpGfcj5588HJL/hvfRrOj5aewKZuQV2w1qZQcCxnALkFZXZrcEw6fnWGvPF35VO4M+k7ENcZFWHVfFLN+HzbZi4YCe2imqGlFgv6rMjfi/oKmvs3l11DLd/ZDki6r3VjqdVOHRBjw4vr8JxUT8IQZAeSeWoltTRWzTxg03YcfIqMnML8NXmkxajvJx5mstFfYAEoWKVbbHlBy9i8rdV/acc/Z3be+zDF/WY9T/bPj2m16DXW2sx9L2NuFygfF0mJX5JO487P9uGKd/br13afvIKHv42FdNT9mOU6PU/fFFZP0qpPxtTM+y+rOsWHZydHV3nbk7VjCQlJWHRokVYtmwZQkNDkZNTMdIhPDwcwcEVf5yTJ09G06ZNkZycjKCgIJv+JBEREQBgt58JuU6uf4kcb63tUZMVlpSr/odY3ZqR9At680q/DUICJBe1EzfZXC8sRYaCjnriC4r4mOIwYmqL/1RmThijIJhHfUhZsu8Cluy7YPENT1y9vPvMdbtr8ZSJyqLk/Vxwsxw/p51H0vA2Ts+1YnoMcZ/Zd1Ydw45TV7Hwkd7IrbxoLUo9i0Ft7TdNC4JQ1UFWZh+/yj5dX1RzYcTP1lfNsvvY/+2R3EccTsVlM5F6j4pfp8zcG+b3IADcEIUEZ55m65qRaYss50Kx/r2gxPVVnOU6Ulu/j07kFqBRaPVr649fKkB0aKB5dt9tmVfx5aaTaNog2GIEmknaGfkQ7w5FojCi9pXAqZqRefPmQa/XY9iwYWjSpIn53+LFi837ZGVlITtb+dh1ci9n/2Ai3dze7Yuy9Tdx8rLjGVQ9qbqhUHz/536WHoVw/FLVOT4uc0GyJrfmTkm5EScv37BoW//vHukPdldOTdxXYvaSQ8izM/dGiUUfA2XHv1FSjg/+zHDYn0CKUbD89g5UTNT3+YaqC/6NEseToY37fBsMRgHHLxXIdkD2Rkg2BQpxGOnz9jrkVH6D3n3mGsZ9thV7nexHsTcrz/yzMzVQ4ho4Ja/nDQedYzV2olCwTBOJwShgpujvyN4xlMrIKcDIDzej11trLV7X5D+O2QQsE52CAQb/3XNOdgi3OCxuOJaLzFzLLyDi0VZqd2B1qmZESUezjRs32r194cKFzjwkeVhIgA7rnhsKf60W4z7fiut2PvRrM/EHpxq8XUMlN6mbtVKD9IdcqcGIv7y/VdExlPYv+XLTSfRo3gB94iOd6pNSIpoYTunzOK+aNQ1Sj/P+mqq5YjYfv4wDDtahOXhej//sOIPXfpcf8eOn1did0l0pe8+KwSjAT6eBuLvZ5YISfL4hE2+OT8A98yv6SD0pMRpFaYVedWpGHLnhqGZE9OB/nbcd7/61K1o1qphDRy6MnLx8Az+Lak2kslSZwYifUrMwsE1DtG5UNSfPtcJShAf72wTJ7SerZm+WCplStVHWzcdSc+r865eDOHRejzfH27Y2iJ89qQ7H4hCjdpM916ap44wC0LpRfTSPCsHGmcOx8hnnZsj1r+VDg72lpjSXrTtqOZy3RGZCPEejacQcXiwqJf9xDPd+uQP7sq7b7YBq7Vh21bc9bz2PVxTUqIz7fJvDfewFEaDiotjn7XWKyyXH3hfJm+VGlBuMFs0qgO0U+JLHtXOb+LpqfTEvNxhx9qr06A1xrZOSb+vW5bYph+jnPWev49b3N2H859tw9mqh7Mrd/7fjrOR2sW+2nsYryw5jxPtVo+5OXCrALW+uwX0SndzF5SiUqDmTes9b9+u6WlgqOePyf3aelVzw0tFLKJ6jyJ3rXrmCYaQW6NE8AgBwR0Jjp+8rfgOGB/ujU2yY+fcABb2rHx1ovxMzKaPkg98bpnxv2Xwj1fcEcC6MPPqd4yGgYhO+2I5tmco7gP7rf1XDPWvK8+guv+13zwJn9p6WEe9vRL/k9RajcCruoyCM2NnF8jbLi+rUH/Zi6HsbJScic3YpgcJSg8WQXyX2n8vDzF8OKurEDUjX7Ow4edVmm6kPirjmsaTcgFXpORah/IhER2vryQsFQbBZW6j3nLW44+MtkuV5d1WGxRwzgP2FOAVY9hlhGKFq++bh3nh7Qhe889euDved2CfOYopxe19tlF5wOlQONyPXqd1eCwAfSUxJ/+8/bbcB9md9dQclE1RJqQnPozv5u2m4pb0ao0v5JZKhU2JiWxtSyxhIKTMY8fWWU+Y+C2sra+C+sZrU7WaZwarPiLLXc/Gecyg3GHFV4jzk+qtcKyzFqSuu9xWTakoUl/ZS/k2sOJiNz9ZnYuoPabJ/SybWNR7iPl5ip+3MB2LqA3WjpBxfbjrpcKmPYoswYndXj3P/0pfkdZH1AvBA3+aK9k2+qyuMRgEvLD2E7SevYkzXJtV6bA2An6f2x9HsAsn5NxY/0Q/3iSZhqqvu7x2HlN3yk/fVhG/0H61VvkKy0mHT3lZYWjPL5SpHk/Yp5cr7q7rflMUZQF9chrdWHMVbK45azBot7juRkVOAUR9tRhPRnD/OlOHrracx949jGNc9FuO6x2J75lU82K8FNmbkSu6fmXsDmbnKwojUgoNSYUQchhPf3+TUSJ8iq5op607SYr8dkK4xu1xQgrSz13H3PPnRa3KPqfaiegwjdZBWq8Hcu7tKdpgSe6Bvc0VTWYcG+csu1FabZ3hV6tekgbh6o8R+GKm5axVKuu5g6nC1TFu0z2aI5Kr06i22qCa5SQyd5Upfmt8OXMT4Hk1dfky5eWcWihYlNE28d+i8HmM/q+gQbZoHAwC+2WJZc2LP3D8qhn8v23/RvDbS1w7WJlJqesp+9GoZiaaiBRHFNTiCIOC3Axcthss7O+R4+8kriG9YD0ajgAe/2WW3BkSOuMnSkUPn9QgJEE+G5/TDuRWbaeowR8PtWkSGYPvzt+KuanwgMYtUPI9+Di4q7hgx4U1Xa2gYAYBTVsO0p/6QplJJqs9dHcTFk6opVVJudGrRPGtyixCmX6ya4M5UMyL3Gn1pZxFCb/vFauj6ftFoqU3HL2N6yn6LZRCc9eLSdNwoKcepKzew/eRVi1DmCR+sOW7RTGOvJsYbGEZIllEAYiOC8eIY5yZSs2T/w3TR433NP3uz70l4sL/jndxEp9PA30Eqq8kXdynXCt07I6U73Soa3eDrXJkhtqZbLVqAUafV4Ny1IlzIK1axRMr4iYKheDg5oHxGVEeKSsslF+j0FPGCl2M+2YrdZ9y/7pZSDCMky9k24/XPDcWb4zpbtANLfZZ+WrlgX6cmYWgTXTU+f0KPpnj/nm6uFdbKzFHtLX5/bFC8xWgjb/7B6zSaWtdcda2wbs5HQ+7lp9Vg8Lsb1C6GIuL+OydzPbOo3OiPt+Luee5f+0qO9WRpQX7qrWHGMEIO2W1LFF1jWzWqj4f6t7SYqEfqEjy2Wyw2zRyGpUkDLFbL1WiAu3s2w9E3bseJOXc4Vcb//r0/HhnQ0vx7zxYNLG7X6TQWqxZbjCiysmnmMIeP9zcHQ5rHda/qt1ALv9w6tSQ8uc6VfgO+ZJ0LzUdyujYLd9uxpMxZeRQLt53GsZx8fL/9jMVt7uqXJDeU3lOs5wAK8lcvErADK9kI8NOitNyIQW0q1tYItPcGlQgqSupTTKtgioeemSpi5CYisqdzbBjaxdTHwsoPCes5UgTBcpXa+3rFYdmBi+Zl1U0OvDISoUGO/yxu7RCNb7fJd45rGVUPt3aIRrC/DiEBfm6YTLpmST2tXnUukZSD520XW3Q3uUnqDkks9OgLMnIsp4dXc3V31oyQjV2zR2D504PQLS4CABAW5I93FcxhIsVRm7e4g56jTp726LSWTSHWQyKtp1EO8NPimVvb2hxHq1U2AshRYNJpNfj2kd74fNItAJS3/bcVNVtRBbWXNieqraxDFMMI1SgN6gUgoalllee9veKQ2DFG0f1NHVF1Wo3Dsevi/iXW6zA4Q6fVWMymaH0Bk2pqkhqOrHRGxhCJMCLugGt9FKVNNREh/opqZuqSuMgQDHawAq4nvDC6g9cfk0hNajbTMIyQE5R1aP1i0i24+5Zm+H3aIIf7ivuMWC8e5Uw40Wk0FsOIrednaB1dz+J3jUaDuMgQrH9uqOVxFD6m1DcI8X2ta1eUhpzLBSXYNHM4Pr6/u6L97Tn8+iisnjGk2sdR2xvjOssuaOZJESFc0ZrqFtaMkO+SuMY2axCC9+/thk6xYQ6bJ8TXbOvwUd+JGgKtVc2I9X3v6xUneb9WjerjL6JZaJXWYEit22MvcHSPi0CDEMfDic9cLUJkvQCM6+763C4mOq0GrRvVc7yjlYFtoqr92CbThrep1v3nTEjAgNYNFYc5d1J7Rkoib3PXJHuuYBghr3r6VsuLkzisWPcZeaBPxRT31iNj5AT56/DKXzrhxdEd0bB+1WiZFlH2Jx0T12joFFz0RndpjDCJoCTOUtYXT51Wg32vjMSQdo0stksdx120Go1NORyd3st/6YSIYPfVCLRs6HwYEmtQWTuhtMbKncrdOCWl0vcwUV3FMELVEhbk3ORhz41sL3ubdc3IjMR2+PKhnvj24d6Kj/+3QfF4fEgri22OvuBaNK8oCCMf39/DonnJRBys5LLP15N74a89m5l/DwmwDCPumnGzogwam/Dx2CD7Q5LHd4+V7A/jacH+Oux6YYTNdtM3NTWGR7tz0b03xyW47VhEtRHDCCkmvqi/e3dXjOgQjUcHtlR8/y8f6mn3dutvvwF+Wozq3BjhouYNjQYY2q4R5j9o/1hijiZv83MwSZvU/lLf1C0me5MZzBvgp8W/RRO7WV/4v3ukj+MCKKTV2I7iiQkLsugM+uZ4y4ukn1ZrN4zMvauL28on9tzIdogJC7LZbgpn4ud21YzBOPDKSPxzZDuPlMVEvPaIlAlOLJPQKTbMZttfqrlIJZE7tY9Rd/V1hhFSTPzRfG/vOHzzSG+bb/b23Noh2u7tDRR0GIyqF4Dv/9YHt4tmU5UzfUTF0F1H30otZ4x1nEY0Go1k51rxxUnpN3nrDmMtokLMP8dXs4lD6lzKjZaLIz5otdqzTqexO1W+dV8ZU3mr2wzxsGjCOovHq6wZEddYdWgchvAQf49Ple5ocTlxmbb8a7jTx+/dUnpxSU97fLD92jGqm9SenJFhhLxGrglkzoQETO7fQlHHSfH1wdH+z97WDodfH4XhEiFIXBRX+iNIzUVi6uNScXxlx/SzapYRh5MfHutrvbv8cRSeg8EooFy0RLBGo7GYQ8ZPq8GUQa2k7grAdoj0vEk98drYTvjKQa2XtV5W4UWu45x/Zfhx51pCCU1taymkGJyoUYuLDLGY4VcJ6yXjvaVPvPs6KJN39GgeIbm9mxtnnVV7HSSGEfIaubf6pL4t8Ma4BEV/DOIRDkq+WdYLdFxzo6TTqhJai74nyu5jfc7icf5NI4Ixtlus9V1wj6jPicnxt+7AhB5N0axBsM1tYmUGI8oM8qtz6rQahIf4Y4RMLZb1CJNGoYF4ZGA8ouoH4okh8iHGmlSTjBRTSJmR2BZ94iPx7t1VwcnVamWli5M6rBmxepHHdHGu2cU0w7EjseFB6CzRzOMqcbFjw5W9DtXhzrLXVf4SfdTqBejwDzt98Jyl9vJZDCOkmCtDHcXvb3dc86vbpdD0DXt4+6qLrU7iD13OF5UzqjqidCiqVgO0EjXHWM+nIdWfNVmi34ZWq8GH93WXnFVWrNwg2PaFEP1q+rYvN5LE+i0gnun2hdEdse/l2+w+vsmrYzsp2s/UZyQiJAD//Xt/3Nu7aoj2iI7RmDMhAf1bVX3Tv7eXZVDrIxFYlS4AWe6gz4j1u1Gn1Visj2TPqhmD0aVZuKIOyzqdbUfk6rCYHNDBvBJy671E1bPfpCq+3VN9EX56vB/WPTcU77k4O3R1tapmM+rLf7H9G5BbuVyq9raw1ODWAKHG8HmLx1f10cmntKnmVOXuqAYUX0dcmQZi66zhWP/cUHRsUvVtzbqpxB7xhe+Xqf1lP5CUnqpOo8HqZ4fg92mDsHnmcJshyFLNQfaGKTuayr7cKKDcqmpAEF1UTa+RXK2AYHUBtn7uGkhcpOY/WBHgxMOtoxXWjFhP6y+m0WgwqW8L87IFADD3rq5IfXEEFkzuhU8m9sDiv/dT9DhSHDXTOA4rFUwXnaYRFbVWYUF+6NC44v13S3PHfW3cUXMnnu9FfDjx89uzRQOMsepU+/wd0rPQNosMkdwu9Rieqv7v3zoKrRvVV2XYNwAsetz19xYATJEY2SY1fxEg/xmlJECM725buyr1nLFmhHzGjMR2mDIoHv97coDXH/vdu7siJEBnMYrGlVqS0CB/tGpkGaqc+TATX+x7tYzEbZ2lp8hX+gGs1Wjgr9OiS7NwNI+y/YB39kIk1Xdk0WN9EeyvQ2igHx4Z0NLmIip1zbUOLHL7SvX1WPqU5fvj9oQmOP7WHTa1FiYvjekouV3u+NYCRB/UWq0G0aFBuK1TDO7sFmvzOqyaMVjxN0CDxHMwSdTh11EzjonpovN/U/pgbLdY/Dy16vmx7mcy/8GeFp2YgYr3iNzoLKXkhq+LL3L/e3IA7hfVPD13m/xopQCri+Pk/i0sfhfXrIUF2zaVmoKZq14R1Sq4I4xI1TY6orSGzRly73e5c1TyXp6RaPs63t7ZdgAA+4yQz6gX6IeX/9LJqZETnWLD0KxBcLVHW9zbOw6HXhuF/q1Fne/c9GGg5IL/zIi2eHVsJ5uOlC2jpGtGOspUt1pz1EIk9yG0V6Y5pHflejvi+w1o0xBH37wd+165DY3Dg+z2GTGR61dhff2VamboIfq2H9+w4sIa4KeV/eC0NyLLX8EieX8bFI+WUSE2E+pJiWsQgnf/2lW2maGeaFizVFOV+NycnRStdaP6+HRiD7QXvTdiwoJwp6hf0O0JjbFp5nD0ayVqXtJU71tr2+j6dsKI5fMr3u/e3nGyIcj6olkv0A/NRbUlwyon9/PTaiRHyW2d5fzoIzlKw8icCfKj6ppGBONZiYu2nLfGJyDSQVOVK+RqAuU6qCs5dUdLVzhzLE/iilzkUf46LTbNHO6WN7r1H5C7vpc4+jB79+6uFn0VxO7p2QwX84rNIemP6YNx+koheikctunosUd0jEHK7nM22+U+CJtGBGPLv4YjTGL0ienCM6ZrLD5ZdwLtYipqiKSeR7maEXE18oFXR8p+m1ry1ACcvlyIni2qnofhHaLx2YZM80KA/7itHbaeuIK7bpGfr0NJn4qIkABsnKns4qbTapDQNBx7XkpE/OyVNrcH+GlRWDnKRWrSs+5xEZj/YE80jwzB5xsyFT2mK8QPrdVIz2sjpU10fYzp0gQBflq8tzoDAPBr0kAs3H5GdLyq/f1t1oOqen3tZfTn7+iAOz/bJiqvYLH/U8PbIMBPi6eGtcHpK4UW9+3QOFRi/ptAXMovcXR6kmVzNJKsUWggLheU2J1aQKfVOLVI3IP9KmqCtj9/K1YczMaclUcV3xeAbEfzV8Z2wh0fb5EsnxQlK4xLfeGRes7U7jPCMEIe56k2XXfVktr7MOsTHykbRICKC7x4VtmOTcIs+qM44ugDILFjNFKe6IcP1hxH6ulrio4Z56A9f9rwNujQONTc/0XqeZRqgugbH4m/dG2Cxbuz0K9VlN3htrc0b2DTH6JniwZY/vQg8wfxMyPa4pkR9jvc2usz4grTe9FelXRokB8KbpZjePtoLNhy2uZ20xw3UoHNbevZWIQR+/2ExOoF6PDsbe2QkVNgDiP1Av2sJvaTbqYBlM1G/PPU/uY+L+biCpad1dvFhOLdv1ZM7tciKgRvjOuMqHqBOJFbgHsl1ola/ER/fLExE03Cg/HxuhOKztVROU02zxyOgptldvspWa/6DQDfPdIbjy7cbffYsRHBaKewFlRMbhFRuc6+UjM+A45rM0ID/SRrfqU+k9WeZ4RhhHyWdWdKV9n9duHhtdIctdNqNBr0axWF6NBAu/s5I8BPi9GiYahSz6N4ocFTb4/GlcISRIYEwE+ntejz4KyEps7Ni+COhbs+uLcb/vHfAwDkm+Riw4NwUX8TyXd1Qb9WUThztQjd4yKw6LG++Hrraaw/lgvA8gNbaZ8RV4hfE61GoziUdW0WAQBo3zgU3z3aG00qh+6O7tIEyX8cQ+fYMPRq2QDtYuqjeWQ93CyznOtESRgJ9tfZ1lIKgux7WaPRYHL/lpW/SQ9/9vfT4t2/dkO5wagojIgfyVEH9OAAHYIrm9+evrUNPl1fUaP1xJBW+GrzKQDSSydIzU8kxZW/TamO3oD8Z5FUeGjWIFj2OX91bCeUlhvx8ICWKCmzDc1Sz1l1+yVVF/uMkM9y1wqTUjUjpqaDp4a3dstjyOmicAKuvq1sJ6oylXF0F8ez0dojHuVi8vaELujaLByfP3CLuVOo0m/n7uSO1/iuW5rhX7e3x5vjOst+2E+7tS3SXx+F2xOaICIkAN0rR+gMaNMQn8pMZibVZ8Rd8cS6gkXugmuaUv7vQ1thRmJbzBKNfhnePtpcgxEXGYL9r9yGZUkD4a/TYvWMIVgwuafNccW/aTVAo1Dbi2ZF/x/LbUZBfh4hOeJOrKYmMfF7bOrQ1hiloIO4vZqRTla1lH8fWvX3bNF3RqtxuQNnh8ahFh1/PcFmRfNAP/z4WF/ZcN0uJhR/H9oaQf46yS8bUs8Za0aIXPTowHisPJSNMV1sh645Q+pbx/v3dMMLVqv/utOKZwZhzZFLmDpUWdh5oE9zBPppLebNeHtCF4xOaIIBCmautee2jjGYMijeYohsi6h6+E2mKtmb3LVw4FPD7HduFSCgvswEeeIPbnFpPFszUqVi+QHpUPbx/T3w3Mj2ipYOiBB1JDVdeK0vchbnqtGgTXQo3hqfgPPXizF/00kAFQHR+sLtSuvU4r/3w6B3NgCw/Btc/vQg5OhvIrFTDP6xeL/kfcUd2e0Nsf79acv3sM4ixFRt99fZBiylNBoN5t7dVbJv18hOMfjzyCXXDixiHRqfGdEGLaLqIb9YL7m/+HWUe19bC/XgCuJKMIyQzwoP9sefzw6t9nGi6tt++9NoNB4LIgDQOTYcnWOVN1notBqbtvYgfx0SO0l/c3SGVquRnICpJlB7uCEgP+JpbLdYbDlxBW2rOf+OFHHfk4b1AyxC2RvjOuOVZYcBVLwvqrOGkXXIsZwfpOL/B/u1wL6s66IwYvuaCBCc/mbdrEFFX5L84jLEimpJEpqGm5vzrCdlm//gLWjVqD7aifpWXCsqlTz+tOFtbL5oyHV89dPZ9hkR69A4FMdyChyflBVHk8opJbdul1yRA/zkR0xZG9a+ES7ll+DVsZ1dLp87MIxQnTe2ayy2ZV6VnK2T1POMgqG63iBuSxeHo7/e0gwto+qhQ5OqC6MrNQRSFxRxpcvbE7rgnVXHJMtQXTqdfM2I+GfxRV1qYi5BcK1cVX1JpFkPQxcEWAQRwHbl62cT2+G2TjGSs5mKiyhusvPTah2GqXoBOhSWGpyaedUdHbCHtmuEKYPj8WVl/xag6j0pFaA6x4ahe5xl5/H+raKw49RV8+/NI0Pwzt1dkK2/iWdubatoVI6nMYxQneen0+Lf93RTuxhkJcmLYcReiBB/g24oqkXTajXoE1/9ACv12OJNcZEhbusfZc16aK9lB9aq7eJwJHWBFQTBI90f7+nZDNszr+Ci/ibqBegk5yu6vXNjPDKgpXn4skZTMb+RFLmw5a+T7r55Z7dY/HbgIp4a3gbtYurjiw0n8aydyeCsBUoMF5aaEVVKYscYvH9vN4QH+6NYZlFFqVq7lCf62dQIfTyxO/rMWQegYsTfIwNbItDPPbU27sIwQkQ1kjc/LO1VaGi1Gqx/bihKDUaEBtlfPdhdI7ysE4q4acSdF32b5QfENQeii7V4NlqpYNQoNNAjHSD7torC9tkjIAgCissMkhPk+em0eO3OzuYwYm9mVLnaHql+MADw0X3d8a/b26NZg4rh8s6uzCwV3ExDnh15clhr8/B5uaZCqQ6sUrUl0aFB+PC+bjh+6Qb+Nap9jWj+tMbRNEREDrRqVN9mbg0lAhXMIHt35SrM4mYF68vpXbdU7OPu/imD21quHCzVZwSw7CRqHUZCA/0wZZDyFZtdodFo7M7UK2avX7Fc2JLrM6LVasxBxBXiJq34hvWw5tkhsuvPmPwwpS/eHJ9gUQsk159FKlTIzes0oUczzLq9Q40MIgBrRoiIFFedOyKu7P/m4V6K5lUZ2q4R1v5jiMVFz/rbfb9WUVj33FDEhgdj6b4LbikrUNEMEeinRZfK+UnkhsyKRw6ZamkeGdASB87nYfET/e1O9+91dmpG5C7E/gr6jDgibioyEQeD/z05QNEU8oPaNsQgq5Bo/dyafpXKHTXlZXAWwwgR1WkD20Q5bH5RatqtbbDqcA7u6xWHER2Vj3RqE23Z2VLqetq6coFHd15sNBoNbk+omohMPLxTfAEUz6liuqC/dqfl6IvhHaJxLKfAol+NGpQ2lIkDX0XNSPUe99WxnfDsbe1w7loR/vLpVgCwmKVYaujssPaNsDHjssNjy5VNsmbER9MIwwgR1WnOzgprT0xYEFJfGFHtqnB7ixmOTmiCt5YfQT+JifCqKzo0CHPv6oIgq1lW5dYqEps+oi3io+phSOUieWqRWhhOimVtjxZR9ao3lF+j0SA82B/hovdT97gIHH59FARI97X5enIvtHnxD0XHltwusa3G1FA5iWGEiOqk5U8Pwqr0HCQNd++oHXe0yRfJjJ4AgPAQf+x7ZaTbJoSzdn+f5jbbTENF7U2gFeSvs7uOk6e9OrYTVqXn4OEBLRXtL64Z8ddp7S6k56x1zw3FqcuFDgOjM7Maz0hsi4/WWk6VL/VW89Eswg6sRFRzmIZYvzrW85OwJTQNxz9HtTevW1KT2AsjQEXHSG92RIysF4C0lxKx+8VErz2msx4dGI/Ff++veMZRcUdXnVYDrVaDRCea1uxp3ag+bnPDhIRiMxKrhhT3tjMnUk3toOoIa0aIqMb4a89muCOhMeopvKDUVkWl5WoXwUaUB2ckVoNU34qafh3fOXsELuQVWSzdYPLH9MGIUtBBtqZizQgR1Sh1PYgAwE2JlVbJPaYMisetHaIxvIO6fVtc0Tg8CD1bVNWKiDs6t4gKQXRYkAqlcg/+1RMR1TDtY0KRcanAp7/p1lTidZieGdEWcQ2C7eztO3y146oJwwgRUQ2zYHIvfLExE08M8exkYnXdP5yY2r0mkpukzhexmYaIqIZpHhWCuXd3RatG7l8RmORJrX1Tk4mXTJBeXcd3sGaEiIgIwN8GxiPIT4tBbX2jP0nj8CBMG94GwQE6h9PM13QMI0RERKgYMv3IwHi1i+GUf45qr3YR3MK3oxQRERH5PIYRIiIiUhXDCBEREamKYYSIiIhUxTBCREREqmIYISIiIlUxjBAREZGqGEaIiIhIVQwjREREpCqGESIiIlIVwwgRERGpimGEiIiIVMUwQkREpJKEpmEAgMSOMSqXRF1ctZeIiEglCx/tg5WHsjGue1O1i6IqhhEiIiKVNKwfiMn9W6pdDNWxmYaIiIhUxTBCREREqmIYISIiIlUxjBAREZGqnAojycnJ6N27N0JDQxEdHY3x48cjIyPD7n0WLFiAwYMHo0GDBmjQoAESExORmpparUITERFR7eFUGNm0aROSkpKwc+dOrFmzBmVlZRg5ciQKCwtl77Nx40ZMnDgRGzZswI4dOxAXF4eRI0fiwoUL1S48ERER+T6NIAiCq3e+fPkyoqOjsWnTJgwZMkTRfQwGAxo0aIDPPvsMkydPVnSf/Px8hIeHQ6/XIywszNXiEhERkRcpvX5Xa54RvV4PAIiMjFR8n6KiIpSVldm9T0lJCUpKSsy/5+fnu15IIiIiqtFc7sBqNBoxY8YMDBw4EAkJCYrvN2vWLMTGxiIxMVF2n+TkZISHh5v/xcXFuVpMIiIiquFcDiNJSUlIT09HSkqK4vvMnTsXKSkpWLp0KYKCgmT3mz17NvR6vfnfuXPnXC0mERER1XAuNdNMmzYNy5cvx+bNm9GsWTNF9/n3v/+NuXPnYu3atejatavdfQMDAxEYGOhK0YiIiMjHOBVGBEHA008/jaVLl2Ljxo2Ij49XdL93330Xc+bMwerVq9GrVy+XCkpERES1k1NhJCkpCYsWLcKyZcsQGhqKnJwcAEB4eDiCg4MBAJMnT0bTpk2RnJwMAHjnnXfwyiuvYNGiRWjZsqX5PvXr10f9+vXdeS5ERETkg5wa2qvRaCS3f/fdd3jkkUcAAMOGDUPLli2xcOFCAEDLli1x9uxZm/u8+uqreO211xQ9rl6vR0REBM6dO8ehvURERD4iPz8fcXFxyMvLQ3h4uOx+1ZpnxFvOnz/PETVEREQ+6ty5c3b7mPpEGDEajbh48SJCQ0Nla2dcYUpstbnGpbafI8/P99X2c6zt5wfU/nPk+blOEAQUFBQgNjYWWq38AN5qTXrmLVqtVvGoHVeEhYXVyjeYWG0/R56f76vt51jbzw+o/efI83ONveYZE67aS0RERKpiGCEiIiJV1ekwEhgYiFdffbVWT7BW28+R5+f7avs51vbzA2r/OfL8PM8nOrASERFR7VWna0aIiIhIfQwjREREpCqGESIiIlIVwwgRERGpqk6Hkc8//xwtW7ZEUFAQ+vbti9TUVLWLpEhycjJ69+6N0NBQREdHY/z48cjIyLDY5+bNm0hKSkJUVBTq16+Pu+++G5cuXbLYJysrC2PGjEFISAiio6Mxc+ZMlJeXe/NUFJk7dy40Gg1mzJhh3ubr53fhwgU8+OCDiIqKQnBwMLp06YI9e/aYbxcEAa+88gqaNGmC4OBgJCYm4sSJExbHuHbtGiZNmoSwsDBERERgypQpuHHjhrdPRZLBYMDLL7+M+Ph4BAcHo3Xr1njzzTch7i/vS+e4efNmjB07FrGxsdBoNPj1118tbnfXuRw8eBCDBw9GUFAQ4uLi8O6773r61MzsnWNZWRlmzZqFLl26oF69eoiNjcXkyZNx8eJFi2PU5HN09BqKTZ06FRqNBh999JHFdl8/v6NHj+LOO+9EeHg46tWrh969eyMrK8t8u6qfq0IdlZKSIgQEBAjffvutcPjwYeHxxx8XIiIihEuXLqldNIdGjRolfPfdd0J6erqwf/9+YfTo0ULz5s2FGzdumPeZOnWqEBcXJ6xbt07Ys2eP0K9fP2HAgAHm28vLy4WEhAQhMTFR2Ldvn7By5UqhYcOGwuzZs9U4JVmpqalCy5Ytha5duwrTp083b/fl87t27ZrQokUL4ZFHHhF27dolnDp1Sli9erWQmZlp3mfu3LlCeHi48OuvvwoHDhwQ7rzzTiE+Pl4oLi4273P77bcL3bp1E3bu3Cls2bJFaNOmjTBx4kQ1TsnGnDlzhKioKGH58uXC6dOnhZ9//lmoX7++8PHHH5v38aVzXLlypfDiiy8KS5YsEQAIS5cutbjdHeei1+uFmJgYYdKkSUJ6errw008/CcHBwcKXX36p+jnm5eUJiYmJwuLFi4Vjx44JO3bsEPr06SP07NnT4hg1+RwdvYYmS5YsEbp16ybExsYKH374ocVtvnx+mZmZQmRkpDBz5kxh7969QmZmprBs2TKLa56an6t1Noz06dNHSEpKMv9uMBiE2NhYITk5WcVSuSY3N1cAIGzatEkQhIoPDn9/f+Hnn38273P06FEBgLBjxw5BECreuFqtVsjJyTHvM2/ePCEsLEwoKSnx7gnIKCgoENq2bSusWbNGGDp0qDmM+Pr5zZo1Sxg0aJDs7UajUWjcuLHw3nvvmbfl5eUJgYGBwk8//SQIgiAcOXJEACDs3r3bvM8ff/whaDQa4cKFC54rvEJjxowR/va3v1lsu+uuu4RJkyYJguDb52j9Qe+uc/niiy+EBg0aWLw/Z82aJbRv397DZ2TL3sXaJDU1VQAgnD17VhAE3zpHufM7f/680LRpUyE9PV1o0aKFRRjx9fO77777hAcffFD2Pmp/rtbJZprS0lKkpaUhMTHRvE2r1SIxMRE7duxQsWSu0ev1AIDIyEgAQFpaGsrKyizOr0OHDmjevLn5/Hbs2IEuXbogJibGvM+oUaOQn5+Pw4cPe7H08pKSkjBmzBiL8wB8//x+++039OrVC/fccw+io6PRo0cPLFiwwHz76dOnkZOTY3F+4eHh6Nu3r8X5RUREoFevXuZ9EhMTodVqsWvXLu+djIwBAwZg3bp1OH78OADgwIED2Lp1K+644w4AteMcTdx1Ljt27MCQIUMQEBBg3mfUqFHIyMjA9evXvXQ2yun1emg0GkRERADw/XM0Go146KGHMHPmTHTu3Nnmdl8+P6PRiBUrVqBdu3YYNWoUoqOj0bdvX4umHLU/V+tkGLly5QoMBoPFEwoAMTExyMnJUalUrjEajZgxYwYGDhyIhIQEAEBOTg4CAgLMHxIm4vPLycmRPH/TbWpLSUnB3r17kZycbHObr5/fqVOnMG/ePLRt2xarV6/Gk08+iWeeeQbff/+9RfnsvT9zcnIQHR1tcbufnx8iIyNVPz8AeP7553H//fejQ4cO8Pf3R48ePTBjxgxMmjQJQO04RxN3nUtNfs9au3nzJmbNmoWJEyeaF1bz9XN855134Ofnh2eeeUbydl8+v9zcXNy4cQNz587F7bffjj///BMTJkzAXXfdhU2bNpnLp+bnqk+s2kvykpKSkJ6ejq1bt6pdFLc5d+4cpk+fjjVr1iAoKEjt4rid0WhEr1698PbbbwMAevTogfT0dMyfPx8PP/ywyqVzj//+97/48ccfsWjRInTu3Bn79+/HjBkzEBsbW2vOsa4qKyvDvffeC0EQMG/ePLWL4xZpaWn4+OOPsXfvXmg0GrWL43ZGoxEAMG7cODz77LMAgO7du2P79u2YP38+hg4dqmbxANTRmpGGDRtCp9PZ9BK+dOkSGjdurFKpnDdt2jQsX74cGzZsQLNmzczbGzdujNLSUuTl5VnsLz6/xo0bS56/6TY1paWlITc3F7fccgv8/Pzg5+eHTZs24ZNPPoGfnx9iYmJ8+vyaNGmCTp06WWzr2LGjuVe7qXz23p+NGzdGbm6uxe3l5eW4du2a6ucHADNnzjTXjnTp0gUPPfQQnn32WXNNV204RxN3nUtNfs+amILI2bNnsWbNGovl5n35HLds2YLc3Fw0b97c/Jlz9uxZPPfcc2jZsqW5fL56fg0bNoSfn5/Dzx01P1frZBgJCAhAz549sW7dOvM2o9GIdevWoX///iqWTBlBEDBt2jQsXboU69evR3x8vMXtPXv2hL+/v8X5ZWRkICsry3x+/fv3x6FDhyz+uEwfLtZvWG8bMWIEDh06hP3795v/9erVC5MmTTL/7MvnN3DgQJuh2MePH0eLFi0AAPHx8WjcuLHF+eXn52PXrl0W55eXl4e0tDTzPuvXr4fRaETfvn29cBb2FRUVQau1/HjR6XTmb2i14RxN3HUu/fv3x+bNm1FWVmbeZ82aNWjfvj0aNGjgpbORZwoiJ06cwNq1axEVFWVxuy+f40MPPYSDBw9afObExsZi5syZWL16NQDfPr+AgAD07t3b7ueO6teNanV/9WEpKSlCYGCgsHDhQuHIkSPCE088IURERFj0Eq6pnnzySSE8PFzYuHGjkJ2dbf5XVFRk3mfq1KlC8+bNhfXr1wt79uwR+vfvL/Tv3998u2mI1siRI4X9+/cLq1atEho1alQjhr5KEY+mEQTfPr/U1FTBz89PmDNnjnDixAnhxx9/FEJCQoQffvjBvM/cuXOFiIgIYdmyZcLBgweFcePGSQ4V7dGjh7Br1y5h69atQtu2bWvM0N6HH35YaNq0qXlo75IlS4SGDRsK//rXv8z7+NI5FhQUCPv27RP27dsnABA++OADYd++feaRJO44l7y8PCEmJkZ46KGHhPT0dCElJUUICQnx2tBee+dYWloq3HnnnUKzZs2E/fv3W3zuiEdR1ORzdPQaWrMeTSMIvn1+S5YsEfz9/YWvvvpKOHHihPDpp58KOp1O2LJli/kYan6u1tkwIgiC8OmnnwrNmzcXAgIChD59+gg7d+5Uu0iKAJD8991335n3KS4uFp566imhQYMGQkhIiDBhwgQhOzvb4jhnzpwR7rjjDiE4OFho2LCh8NxzzwllZWVePhtlrMOIr5/f77//LiQkJAiBgYFChw4dhK+++sridqPRKLz88stCTEyMEBgYKIwYMULIyMiw2Ofq1avCxIkThfr16wthYWHCo48+KhQUFHjzNGTl5+cL06dPF5o3by4EBQUJrVq1El588UWLC5cvneOGDRsk/+Yefvhht57LgQMHhEGDBgmBgYFC06ZNhblz53rrFO2e4+nTp2U/dzZs2OAT5+joNbQmFUZ8/fy++eYboU2bNkJQUJDQrVs34ddff7U4hpqfqxpBEE2JSERERORldbLPCBEREdUcDCNERESkKoYRIiIiUhXDCBEREamKYYSIiIhUxTBCREREqmIYISIiIlUxjBAREZGqGEaIiIhIVQwjREREpCqGESIiIlIVwwgRERGp6v8B3wR93k7JsaoAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(torch.tensor(l).view(-1, 10).mean(1).numpy())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "id": "k5vlgMO8P-3f"
   },
   "outputs": [],
   "source": [
    "def make_inference(model, question):\n",
    "    context = context_template.format(instruction=question)\n",
    "    token = tokenizer(context, return_tensors='pt').to(device)\n",
    "    # 生成文本时，需要将模型调整成评估模式\n",
    "    model.eval()\n",
    "    output_tokens = model.generate(**token, max_new_tokens=100, early_stopping=True)\n",
    "    model.train()\n",
    "    print(tokenizer.decode(output_tokens[0], skip_special_tokens=True))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "SC-5Ri1Z8qcR",
    "outputId": "675a8c54-c4a2-4ac3-9960-d01e217601b7"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n",
      "\n",
      "### Instruction:\n",
      "Where is the capital of China?\n",
      "\n",
      "### Response:\n",
      "The capital of China is Beijing.\n"
     ]
    }
   ],
   "source": [
    "make_inference(model, 'Where is the capital of China?')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "UwnVbc1C9DTR",
    "outputId": "5db29fde-741f-4455-a505-eba0278b9173"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n",
      "\n",
      "### Instruction:\n",
      "Where is the capital of China?\n",
      "\n",
      "### Response:\n",
      "\n",
      "The capital of China is the capital of China.\n",
      "\n",
      "### Response:\n",
      "\n",
      "The capital of China is the capital of China.\n",
      "\n",
      "### Response:\n",
      "\n",
      "The capital of China is the capital of China.\n",
      "\n",
      "### Response:\n",
      "\n",
      "The capital of China is the capital of China.\n",
      "\n",
      "### Response:\n",
      "\n",
      "The capital of China is the capital of China.\n",
      "\n",
      "### Response:\n",
      "\n",
      "The capital of China is the capital of China.\n",
      "\n",
      "### Response\n"
     ]
    }
   ],
   "source": [
    "# 微调之前的模型效果（禁用LoRA相当于回到微调之前的模型状态）\n",
    "with model.disable_adapter():\n",
    "    make_inference(model, 'Where is the capital of China?')"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "gpuType": "A100",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "013051901d484a2ba80569dccd5b5680": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_48ac7039863c42f880aeb25be0830091",
       "IPY_MODEL_b87cc14342d54e0c86bcfe1cf2bc8acc",
       "IPY_MODEL_b56f7cae483f4ceb89dce64cea92f686"
      ],
      "layout": "IPY_MODEL_01a2acaef43042d5bb8a0e4fcfbac045"
     }
    },
    "01a2acaef43042d5bb8a0e4fcfbac045": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "25f698c38f5548b4a1dd9ee89163ebb6": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "3f415ddf03514ba390be7d01d51aa866": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "3fe0784b898f4cb1aa10fde49d7e8001": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "46efff100756461db8cdf17e85599931": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_ac66c967bd2f41178796a0863a142bd2",
       "IPY_MODEL_ca355fac774949bc990a8d36dd4a65e1",
       "IPY_MODEL_b5c2a3a5ab40417481acb990495f513d"
      ],
      "layout": "IPY_MODEL_f282ea9ae3e24d4db17ceea762dcd943"
     }
    },
    "48ac7039863c42f880aeb25be0830091": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_c4e9a011ae3e47039728b99f958f400b",
      "placeholder": "​",
      "style": "IPY_MODEL_922b8d4189da42a7908dd907603e770c",
      "value": "Map: 100%"
     }
    },
    "4f7d802ec45246688db7113ea6a8a98c": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "922b8d4189da42a7908dd907603e770c": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "92ddb455899d4b6b8f5ed1e2ff22d2fc": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": ""
     }
    },
    "ac66c967bd2f41178796a0863a142bd2": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_25f698c38f5548b4a1dd9ee89163ebb6",
      "placeholder": "​",
      "style": "IPY_MODEL_3fe0784b898f4cb1aa10fde49d7e8001",
      "value": "Map: 100%"
     }
    },
    "b56f7cae483f4ceb89dce64cea92f686": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_3f415ddf03514ba390be7d01d51aa866",
      "placeholder": "​",
      "style": "IPY_MODEL_cf89e35195d046ce8b236fea9c13fc37",
      "value": " 5201/5201 [00:03&lt;00:00, 1561.81 examples/s]"
     }
    },
    "b5c2a3a5ab40417481acb990495f513d": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_4f7d802ec45246688db7113ea6a8a98c",
      "placeholder": "​",
      "style": "IPY_MODEL_ccddc6386c7b4655a56018e2600b65e9",
      "value": " 46801/46801 [00:29&lt;00:00, 1621.07 examples/s]"
     }
    },
    "b87cc14342d54e0c86bcfe1cf2bc8acc": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_da0f743da3ee4af98c227d0e83065caa",
      "max": 5201,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_92ddb455899d4b6b8f5ed1e2ff22d2fc",
      "value": 5201
     }
    },
    "c11e20236a1848cca183fd3e206ee546": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "c4e9a011ae3e47039728b99f958f400b": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "ca355fac774949bc990a8d36dd4a65e1": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_c11e20236a1848cca183fd3e206ee546",
      "max": 46801,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_ccf8966db2184bb5968adf3b75df805f",
      "value": 46801
     }
    },
    "ccddc6386c7b4655a56018e2600b65e9": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "ccf8966db2184bb5968adf3b75df805f": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": ""
     }
    },
    "cf89e35195d046ce8b236fea9c13fc37": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "da0f743da3ee4af98c227d0e83065caa": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "f282ea9ae3e24d4db17ceea762dcd943": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    }
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}