diff --git a/notebooks/finetuning_commafixer_with_LoRa.ipynb b/notebooks/finetuning_commafixer_with_LoRa.ipynb new file mode 100644 index 0000000000000000000000000000000000000000..0fe7048769a3ba79801a97587554104f24bde04d --- /dev/null +++ b/notebooks/finetuning_commafixer_with_LoRa.ipynb @@ -0,0 +1,4638 @@ +{ + "nbformat": 4, + "nbformat_minor": 0, + "metadata": { + "colab": { + "provenance": [], + "gpuClass": "premium" + }, + "kernelspec": { + "name": "python3", + "display_name": "Python 3" + }, + "language_info": { + "name": "python" + }, + "accelerator": "GPU", + "gpuClass": "premium" + }, + "cells": [ + { + "cell_type": "markdown", + "source": [ + "# Simplified LoRA Implementation" + ], + "metadata": { + "id": "D9aAU6JeCQGy" + } + }, + { + "cell_type": "code", + "source": [ + "from google.colab import drive\n", + "drive.mount('/content/drive/')#, force_remount=True)" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "6QYQ4Hor5IGK", + "outputId": "af5777d6-3625-40ba-90c7-63ee2805cd51" + }, + "execution_count": 1, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Drive already mounted at /content/drive/; to attempt to forcibly remount, call drive.mount(\"/content/drive/\", force_remount=True).\n" + ] + } + ] + }, + { + "cell_type": "code", + "source": [ + "save_dir = '/content/drive/MyDrive/notebook-data'" + ], + "metadata": { + "id": "R4Qx1vqN5Ja3" + }, + "execution_count": 2, + "outputs": [] + }, + { + "cell_type": "markdown", + "source": [ + "#### Install Dependencies" + ], + "metadata": { + "id": "aqlztwUrCUaG" + } + }, + { + "cell_type": "code", + "source": [ + "!pip install -q bitsandbytes datasets accelerate loralib\n", + "!pip install -q git+https://github.com/huggingface/peft.git git+https://github.com/huggingface/transformers.git" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "R3Gy0KqgByeZ", + "outputId": "bbba3e34-f2a8-414d-b65a-eed9d4241941" + }, + "execution_count": 3, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + " Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n", + " Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n", + " Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n", + " Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n", + " Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n", + " Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n" + ] + } + ] + }, + { + "cell_type": "markdown", + "source": [ + "#### Confirm CUDA" + ], + "metadata": { + "id": "O9pwsBMNCWTA" + } + }, + { + "cell_type": "code", + "source": [ + "import torch\n", + "torch.cuda.is_available()" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "DZAtLKXDB4ko", + "outputId": "82ce0ef0-fc4a-4996-f659-becf9b17cae3" + }, + "execution_count": 4, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "True" + ] + }, + "metadata": {}, + "execution_count": 4 + } + ] + }, + { + "cell_type": "markdown", + "source": [ + "#### Load Base Model" + ], + "metadata": { + "id": "rhn78FE4CZS9" + } + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": { + "id": "o0BZjNgEBvXH" + }, + "outputs": [], + "source": [ + "import os\n", + "os.environ[\"CUDA_VISIBLE_DEVICES\"]=\"0\"\n", + "import torch\n", + "import torch.nn as nn\n", + "import bitsandbytes as bnb\n", + "\n", + "from transformers import AutoTokenizer, AutoConfig, AutoModelForCausalLM\n", + "\n", + "model = AutoModelForCausalLM.from_pretrained(\n", + " \"bigscience/bloom-560m\",\n", + " torch_dtype=torch.float16,\n", + " device_map='auto',\n", + ")\n", + "\n", + "tokenizer = AutoTokenizer.from_pretrained(\"bigscience/tokenizer\")" + ] + }, + { + "cell_type": "markdown", + "source": [ + "##### View Model Summary" + ], + "metadata": { + "id": "Qs2IKgRrCc3v" + } + }, + { + "cell_type": "code", + "source": [ + "print(model)" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "0ihHmXefB6i9", + "outputId": "ab1a9027-732e-41ee-a1a4-dce0034a9381" + }, + "execution_count": 6, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "BloomForCausalLM(\n", + " (transformer): BloomModel(\n", + " (word_embeddings): Embedding(250880, 1024)\n", + " (word_embeddings_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n", + " (h): ModuleList(\n", + " (0-23): 24 x BloomBlock(\n", + " (input_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n", + " (self_attention): BloomAttention(\n", + " (query_key_value): Linear(in_features=1024, out_features=3072, bias=True)\n", + " (dense): Linear(in_features=1024, out_features=1024, bias=True)\n", + " (attention_dropout): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (post_attention_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n", + " (mlp): BloomMLP(\n", + " (dense_h_to_4h): Linear(in_features=1024, out_features=4096, bias=True)\n", + " (gelu_impl): BloomGelu()\n", + " (dense_4h_to_h): Linear(in_features=4096, out_features=1024, bias=True)\n", + " )\n", + " )\n", + " )\n", + " (ln_f): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n", + " )\n", + " (lm_head): Linear(in_features=1024, out_features=250880, bias=False)\n", + ")\n" + ] + } + ] + }, + { + "cell_type": "markdown", + "source": [ + "Freeze all parameters" + ], + "metadata": { + "id": "T9shQVanMMAD" + } + }, + { + "cell_type": "code", + "source": [ + "for param in model.parameters():\n", + " param.requires_grad = False # freeze the model - train adapters later\n", + " if param.ndim == 1:\n", + " # cast the small parameters (e.g. layernorm) to fp32 for stability\n", + " param.data = param.data.to(torch.float32)\n", + "\n", + "model.gradient_checkpointing_enable() # reduce number of stored activations\n", + "model.enable_input_require_grads()\n", + "\n", + "class CastOutputToFloat(nn.Sequential):\n", + " def forward(self, x): return super().forward(x).to(torch.float32)\n", + "model.lm_head = CastOutputToFloat(model.lm_head)" + ], + "metadata": { + "id": "WuK0lPwcB7Ia" + }, + "execution_count": 7, + "outputs": [] + }, + { + "cell_type": "markdown", + "source": [ + "#### Helper Function" + ], + "metadata": { + "id": "zdlTV1TNCMm7" + } + }, + { + "cell_type": "code", + "source": [ + "def print_trainable_parameters(model):\n", + " \"\"\"\n", + " Prints the number of trainable parameters in the model.\n", + " \"\"\"\n", + " trainable_params = 0\n", + " all_param = 0\n", + " for _, param in model.named_parameters():\n", + " all_param += param.numel()\n", + " if param.requires_grad:\n", + " trainable_params += param.numel()\n", + " print(\n", + " f\"trainable params: {trainable_params} || all params: {all_param} || trainable%: {100 * trainable_params / all_param}\"\n", + " )" + ], + "metadata": { + "id": "Cc8354XxCIWl" + }, + "execution_count": 8, + "outputs": [] + }, + { + "cell_type": "markdown", + "source": [ + "#### Obtain LoRA Model" + ], + "metadata": { + "id": "Rc-vqvtuCim3" + } + }, + { + "cell_type": "code", + "source": [ + "from peft import LoraConfig, get_peft_model\n", + "\n", + "config = LoraConfig(\n", + " r=8,\n", + " lora_alpha=16,\n", + " target_modules=[\"query_key_value\"],\n", + " lora_dropout=0.05,\n", + " bias=\"none\",\n", + " task_type=\"CAUSAL_LM\"\n", + ")\n", + "\n", + "model = get_peft_model(model, config)\n", + "print_trainable_parameters(model)" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "UQ-cH7ieCARh", + "outputId": "c3e52549-c7c6-4c46-d874-d7c101a8cb58" + }, + "execution_count": 9, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "trainable params: 786432 || all params: 560001024 || trainable%: 0.14043402892063284\n" + ] + } + ] + }, + { + "cell_type": "markdown", + "source": [ + "#### Load Sample Dataset" + ], + "metadata": { + "id": "iopX35giC7L1" + } + }, + { + "cell_type": "code", + "source": [ + "import datasets\n", + "\n", + "wikitext = datasets.load_dataset('wikitext', 'wikitext-103-raw-v1', split='train[:10%]')" + ], + "metadata": { + "id": "swctuqb8C9YD" + }, + "execution_count": 10, + "outputs": [] + }, + { + "cell_type": "markdown", + "source": [ + "```\n", + "### INPUT\n", + "{context}\n", + "\n", + "### FIXED\n", + "{commas-fixed}\n", + "```" + ], + "metadata": { + "id": "AD0u5hP3WnXf" + } + }, + { + "cell_type": "code", + "source": [ + "wikitext" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "b42o7wP1_YF9", + "outputId": "b1cf01f4-f8e0-4af8-c0ba-fef6b2d4179d" + }, + "execution_count": 11, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "Dataset({\n", + " features: ['text'],\n", + " num_rows: 180135\n", + "})" + ] + }, + "metadata": {}, + "execution_count": 11 + } + ] + }, + { + "cell_type": "code", + "source": [ + "def create_prompt(text):\n", + " prompt_template = f\"### INPUT\\n{text.replace(',', '')}\\n\\n### FIXED\\n{text}\"\n", + " return prompt_template\n", + "\n", + "mapped_wikitext = wikitext.map(lambda samples: tokenizer(create_prompt(samples['text'])), batch_size=32)" + ], + "metadata": { + "id": "3BvDnYjxWPza" + }, + "execution_count": 12, + "outputs": [] + }, + { + "cell_type": "code", + "source": [ + "print(tokenizer.decode(mapped_wikitext[1][\"input_ids\"]))" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "W_MbYkfb8ASk", + "outputId": "0685c789-e37b-4611-dcbb-3e46fd6b0504" + }, + "execution_count": 13, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "### INPUT\n", + " = Valkyria Chronicles III = \n", + "\n", + "\n", + "### FIXED\n", + " = Valkyria Chronicles III = \n", + "\n" + ] + } + ] + }, + { + "cell_type": "code", + "source": [], + "metadata": { + "id": "hDbrPOoq5G2P" + }, + "execution_count": 13, + "outputs": [] + }, + { + "cell_type": "markdown", + "source": [ + "#### Train LoRA" + ], + "metadata": { + "id": "fqRzEfHhCoC7" + } + }, + { + "cell_type": "code", + "source": [ + "torch.cuda.empty_cache()" + ], + "metadata": { + "id": "eu14RFKr7-3S" + }, + "execution_count": 14, + "outputs": [] + }, + { + "cell_type": "code", + "source": [ + "import transformers\n", + "\n", + "trainer = transformers.Trainer(\n", + " model=model,\n", + " train_dataset=mapped_wikitext,\n", + " # eval_dataset=mapped_wikitext[\"validation\"],\n", + " args=transformers.TrainingArguments(\n", + " per_device_train_batch_size=2,\n", + " gradient_accumulation_steps=4,\n", + " warmup_steps=100,\n", + " max_steps=1000,\n", + " learning_rate=1e-3,\n", + " fp16=True,\n", + " logging_steps=1,\n", + " output_dir=save_dir,\n", + " save_steps=.1,\n", + " save_total_limit=3,\n", + " # evaluation_strategy=\"steps\",\n", + " # eval_steps=.1,\n", + " # load_best_model_at_end=True,\n", + " # metric_for_best_model=TODO\n", + " ),\n", + " data_collator=transformers.DataCollatorForLanguageModeling(tokenizer, mlm=False)\n", + ")\n", + "trainer.train()" + ], + "metadata": { + "id": "LNZ3Txn4CFeR", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 1000 + }, + "outputId": "6336e2fb-2577-4502-f55b-5bfdcacd5cb7" + }, + "execution_count": 15, + "outputs": [ + { + "output_type": "stream", + "name": "stderr", + "text": [ + "You're using a PreTrainedTokenizerFast tokenizer. Please note that with a fast tokenizer, using the `__call__` method is faster than using a method to encode the text followed by a call to the `pad` method to get a padded encoding.\n" + ] + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + "" + ], + "text/html": [ + "\n", + "
\n", + " \n", + " \n", + " [1000/1000 18:15, Epoch 0/1]\n", + "
\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
StepTraining Loss
13.078800
23.195200
32.183700
42.541800
54.508300
62.096300
72.476800
84.344000
92.750300
105.208300
114.191100
122.921400
134.507200
142.122700
152.705800
162.829000
173.897200
182.847100
192.662500
202.676400
212.932600
222.592800
232.830700
242.591600
252.317200
262.358300
272.317600
281.397600
292.160900
302.141500
311.403700
322.074900
331.938200
341.626400
351.692200
361.077500
371.478600
381.304200
391.875800
402.207300
411.545600
421.531600
431.721400
441.568700
451.853000
461.014300
471.170100
481.347800
491.174000
501.344900
511.445400
521.726900
531.232100
541.482600
551.066300
561.269200
571.344900
581.835000
591.279900
600.966500
611.615900
621.070800
631.043100
641.645000
651.138000
660.797000
671.482200
681.627200
691.333200
701.803000
711.707500
721.228500
731.387000
741.461900
751.702800
761.519300
771.652700
780.933600
791.632400
801.586200
811.695400
821.303400
830.920400
841.587200
851.587100
861.010700
871.741500
881.086100
891.741500
901.709300
911.506300
921.702400
931.717500
941.558100
951.695500
961.449700
971.366800
981.219900
991.251600
1001.588000
1011.313300
1021.403400
1031.344300
1041.111300
1051.175700
1061.015100
1071.603700
1081.027000
1091.543500
1101.047000
1110.755100
1121.354700
1130.984300
1141.106600
1151.102000
1161.544100
1171.353500
1181.331300
1191.230400
1201.112400
1211.046100
1221.563100
1231.190300
1241.500000
1251.612400
1261.212000
1271.311200
1281.235400
1291.361800
1301.051200
1311.501500
1321.326500
1330.842800
1340.976200
1350.704200
1361.204500
1370.686700
1381.697800
1390.884900
1401.636300
1411.138200
1421.802000
1431.513700
1440.312500
1451.541500
1461.468200
1471.331100
1480.817800
1491.230200
1501.554600
1511.296500
1520.880200
1531.585300
1540.977400
1551.445900
1561.327300
1571.621800
1581.779800
1591.778800
1600.793900
1611.753300
1621.660600
1631.158300
1640.406800
1650.854600
1661.272100
1671.693900
1681.680500
1691.326500
1701.106200
1711.539600
1721.797200
1731.756400
1740.784900
1750.796400
1761.626200
1771.623500
1781.522000
1791.176100
1801.494700
1811.233400
1821.174600
1831.028100
1841.616100
1851.179500
1861.397100
1870.484300
1881.165100
1891.137600
1901.244200
1911.066800
1921.120700
1931.812200
1941.324400
1950.195800
1961.114100
1971.435600
1981.683900
1991.679000
2001.569300
2010.978000
2021.171900
2031.267500
2041.795400
2051.546000
2061.551800
2071.666000
2081.603300
2091.024900
2101.728200
2111.226700
2121.196600
2131.161100
2140.375600
2151.426800
2161.631400
2171.032600
2181.028600
2191.127400
2200.729300
2211.685700
2221.722000
2230.881300
2240.635100
2250.984600
2261.256800
2271.098500
2281.211500
2291.647500
2301.241000
2311.337800
2321.282900
2332.012900
2341.535400
2351.082000
2361.389000
2371.244200
2381.452400
2391.661300
2401.308300
2411.450700
2420.917100
2431.550600
2441.586300
2451.170800
2461.200600
2471.285200
2481.314300
2491.896300
2501.824800
2510.753200
2521.789700
2531.370500
2541.143700
2551.588400
2561.023700
2571.368800
2581.595100
2590.776200
2601.638000
2611.732100
2621.693700
2631.018700
2641.206500
2651.384600
2660.699000
2670.977700
2680.984200
2691.723600
2701.429300
2711.313400
2721.343600
2731.270100
2741.340700
2750.829000
2761.148000
2771.744000
2781.501100
2791.355200
2801.090400
2811.314100
2820.864200
2831.700800
2841.404300
2851.576800
2861.564600
2871.267400
2880.900400
2891.608600
2901.235000
2911.449800
2920.370700
2931.652200
2940.792500
2951.623800
2961.376100
2971.755600
2981.300300
2990.434900
3001.213900
3011.274900
3020.455600
3031.425700
3040.904100
3050.833800
3061.124700
3070.683700
3080.901300
3091.659900
3101.439400
3111.555700
3121.658400
3130.999300
3141.746200
3151.112300
3161.336600
3171.776000
3181.411400
3191.620200
3200.774600
3211.314100
3220.973000
3231.027600
3240.849200
3251.207100
3261.220400
3271.215500
3281.398100
3290.665600
3301.581000
3311.276100
3321.326700
3331.395200
3340.667400
3351.190600
3360.876200
3371.385700
3380.985400
3391.285000
3401.027800
3410.696600
3421.277800
3431.187900
3441.451900
3451.222000
3460.966200
3471.213600
3481.722700
3491.142000
3501.624600
3511.557700
3521.683800
3530.928200
3541.131200
3551.144800
3560.579300
3571.541500
3581.102000
3591.164600
3601.164800
3611.353600
3621.182200
3631.005600
3641.726900
3651.206000
3661.241300
3671.269800
3680.904000
3691.693400
3701.337500
3711.026200
3721.028700
3730.825300
3741.583400
3751.568300
3761.498200
3771.376700
3780.880200
3790.721800
3801.564000
3811.197100
3821.263400
3831.227700
3840.832000
3851.705500
3861.085100
3871.576800
3881.347800
3890.840400
3901.303900
3911.479000
3921.522100
3930.790300
3941.191500
3951.241200
3961.091800
3971.210900
3981.406900
3991.433300
4001.494500
4010.524100
4021.588600
4030.983200
4041.433000
4051.208100
4061.195800
4071.578100
4081.317000
4090.440500
4100.886300
4111.088400
4121.508900
4130.791900
4141.226800
4151.368700
4161.741400
4171.044500
4180.930500
4190.863700
4201.009600
4210.940900
4221.083800
4230.899900
4241.505200
4251.560800
4260.895400
4270.878000
4280.801200
4291.410400
4301.232900
4311.607800
4321.506700
4331.481100
4341.687300
4350.986300
4361.522300
4370.818400
4381.218400
4391.239600
4401.278600
4410.913500
4420.753200
4431.596800
4441.192700
4450.945800
4461.665900
4471.599000
4481.721600
4491.523800
4501.276500
4511.122400
4521.597800
4531.401400
4541.355500
4551.382900
4561.241200
4570.767300
4581.327100
4591.111300
4601.527200
4610.290600
4620.970800
4631.334900
4641.077600
4651.392400
4660.937900
4671.693400
4681.692600
4691.197100
4701.565300
4711.360800
4721.171600
4731.196400
4741.397000
4751.258300
4760.789600
4770.665700
4781.666200
4791.205500
4801.322600
4810.276400
4821.191100
4830.672500
4841.043400
4851.607200
4860.586200
4871.543100
4881.594200
4891.353700
4901.584200
4911.288700
4921.191200
4931.203000
4941.589600
4951.588100
4961.037400
4971.367100
4981.659700
4991.389600
5001.699400
5011.276400
5021.234500
5030.669600
5041.161100
5051.495700
5061.653700
5071.584900
5081.634200
5090.765200
5101.396800
5111.751600
5121.670800
5131.499700
5141.593000
5151.541600
5161.348000
5171.134100
5181.424500
5190.867900
5201.224300
5211.699700
5221.083300
5231.110600
5241.523000
5250.878600
5261.358500
5271.556200
5281.230200
5291.101800
5301.163100
5311.274300
5321.191600
5331.562100
5341.120700
5351.034700
5361.242000
5371.296600
5381.595200
5390.879300
5401.464800
5410.922400
5421.650300
5431.640000
5441.581300
5451.243900
5461.180900
5471.406700
5480.911800
5491.262800
5501.266700
5511.609000
5521.136600
5530.990900
5541.244800
5551.554400
5561.713900
5571.396800
5581.457000
5591.237500
5600.779000
5611.106900
5621.351000
5631.561100
5641.542000
5651.749200
5661.279600
5671.120500
5681.415500
5691.368100
5701.364200
5711.595200
5721.408000
5731.229900
5740.873700
5750.847300
5761.242000
5771.201600
5781.971700
5791.526400
5801.358700
5811.742700
5821.587200
5831.677000
5840.928100
5851.399700
5861.486500
5871.672400
5880.745800
5891.466600
5901.334900
5910.780100
5921.617000
5931.341900
5940.956400
5951.371100
5961.045900
5971.440000
5981.399000
5991.333300
6001.192000
6011.732200
6021.416600
6031.231400
6041.227100
6050.900000
6061.708300
6071.403600
6081.371400
6091.292800
6100.652700
6111.571900
6121.057800
6131.452600
6141.557000
6151.276900
6160.773600
6171.440700
6181.327800
6191.504400
6201.051000
6211.621000
6221.141600
6231.074300
6240.907700
6251.394000
6261.536700
6271.567900
6281.118300
6291.085500
6300.991600
6310.717100
6321.314000
6331.220800
6341.270800
6350.898000
6360.863900
6371.080100
6381.523200
6390.795300
6401.748500
6411.401100
6420.822200
6431.156300
6441.590900
6451.404500
6461.045100
6470.951700
6481.551000
6491.264000
6501.060300
6511.404200
6521.274600
6531.307900
6541.374400
6551.458900
6561.291400
6571.118400
6581.321500
6591.620900
6601.368100
6611.485900
6621.598000
6631.373700
6641.013200
6651.119500
6661.217000
6671.674800
6681.480300
6691.544100
6701.498000
6711.330000
6720.977300
6731.401400
6741.244100
6751.678300
6760.963400
6771.081800
6781.647600
6791.635300
6801.342100
6811.209600
6821.738100
6831.409700
6841.377800
6851.340400
6860.973500
6871.321800
6881.402400
6891.267500
6901.323200
6911.247400
6921.334100
6931.561100
6941.241300
6950.889600
6961.468000
6971.377800
6980.763400
6991.627500
7001.281500
7010.926900
7021.178100
7031.660000
7040.857600
7051.570500
7061.144100
7071.364400
7081.450100
7091.601100
7101.659000
7110.865900
7121.107200
7131.150600
7141.223000
7151.630400
7160.824800
7170.974900
7180.700500
7191.279800
7201.605300
7211.491200
7220.717500
7231.083800
7241.466000
7251.025700
7261.545800
7271.384600
7280.831500
7291.510000
7301.623400
7311.083100
7321.294700
7331.570800
7341.572600
7351.574600
7360.952400
7370.809000
7381.421100
7391.299700
7400.812900
7410.586900
7421.187700
7431.085600
7441.171500
7451.089400
7461.219100
7471.573600
7481.455800
7490.542900
7500.958800
7511.168100
7520.827100
7530.794500
7541.611500
7551.374500
7561.607800
7571.152800
7580.842300
7591.392400
7601.586900
7610.896800
7621.143000
7631.628300
7641.043600
7651.565600
7661.195100
7671.449700
7681.018200
7691.673000
7701.078700
7711.274200
7721.526300
7731.560200
7740.989500
7750.778600
7760.998000
7771.160800
7781.246100
7790.900400
7801.298600
7811.084500
7821.617900
7830.915800
7841.568800
7851.098700
7860.568800
7870.850300
7881.747000
7890.866300
7901.172300
7911.065000
7921.100000
7931.001000
7941.586700
7950.824100
7961.249000
7971.500500
7981.517400
7991.606500
8001.040400
8011.205900
8021.672700
8031.122500
8041.699100
8051.078400
8061.496700
8070.806400
8080.428300
8091.118900
8100.523500
8110.914700
8121.623200
8130.660000
8141.188800
8151.418000
8161.426800
8171.489000
8180.828900
8191.229000
8200.901000
8211.100200
8221.562500
8231.147400
8240.986800
8251.199900
8261.624200
8271.011400
8281.170100
8291.401800
8300.532600
8311.682500
8321.011400
8331.524500
8341.522700
8351.200600
8361.692600
8371.128000
8381.254600
8391.420900
8400.918400
8411.653500
8421.345200
8431.093300
8440.901200
8450.299700
8461.636700
8471.210700
8480.447600
8491.327500
8501.336400
8511.111300
8521.323600
8531.599100
8540.628300
8551.527500
8561.547500
8571.357400
8580.828000
8591.525600
8601.250700
8611.127100
8621.305300
8630.726200
8640.955100
8650.983000
8661.249800
8671.647100
8680.496400
8691.641600
8701.095500
8711.666700
8721.286200
8731.170400
8741.421700
8751.183500
8760.788600
8770.794200
8780.591500
8791.106400
8801.650200
8811.235200
8821.100800
8830.995600
8841.186000
8851.518900
8861.364900
8871.143400
8881.594900
8891.442100
8900.994100
8911.492800
8921.200400
8931.672700
8941.271600
8951.264400
8961.281500
8971.418800
8981.176800
8991.174600
9000.905200
9011.606400
9020.939700
9031.439700
9041.309300
9050.900900
9061.140400
9071.536900
9081.309300
9091.346400
9101.177800
9111.362500
9120.772100
9130.775600
9141.438800
9151.420500
9161.115400
9171.136300
9181.173700
9191.690700
9201.247700
9211.095300
9221.451000
9231.279300
9241.081000
9250.862400
9261.565800
9271.265500
9281.805200
9290.831500
9301.296900
9311.137600
9321.514900
9331.072700
9341.636200
9351.615900
9361.652600
9371.601200
9380.905500
9390.664000
9401.360300
9410.536100
9420.626000
9430.867200
9441.592500
9451.107600
9461.177700
9471.316000
9481.416300
9490.752600
9500.967000
9511.394900
9520.854900
9530.151600
9541.185900
9551.515900
9561.431200
9571.223400
9581.588100
9591.173300
9601.700700
9610.965900
9620.843300
9631.256600
9641.003000
9651.570100
9661.067600
9671.179000
9680.983600
9691.094800
9701.342100
9711.297600
9721.064300
9731.226100
9741.301000
9751.350500
9761.510300
9771.422000
9781.085600
9791.516300
9801.610900
9810.952700
9820.746700
9831.472100
9840.460100
9851.271100
9861.216300
9870.541600
9881.402700
9890.959800
9901.725400
9911.572800
9921.069700
9931.076700
9940.412700
9951.322700
9961.218000
9971.603400
9981.319000
9991.194500
10000.897100

" + ] + }, + "metadata": {} + }, + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "TrainOutput(global_step=1000, training_loss=1.3154083265066148, metrics={'train_runtime': 1097.8147, 'train_samples_per_second': 7.287, 'train_steps_per_second': 0.911, 'total_flos': 3300074161176576.0, 'train_loss': 1.3154083265066148, 'epoch': 0.04})" + ] + }, + "metadata": {}, + "execution_count": 15 + } + ] + }, + { + "cell_type": "code", + "source": [ + "import torch\n", + "from peft import PeftModel, PeftConfig\n", + "from transformers import AutoModelForCausalLM, AutoTokenizer\n", + "\n", + "peft_model_id = f\"{save_dir}/checkpoint-1000\"\n", + "config = PeftConfig.from_pretrained(peft_model_id)\n", + "model = AutoModelForCausalLM.from_pretrained(config.base_model_name_or_path, return_dict=True, load_in_8bit=False, device_map='auto')\n", + "tokenizer = AutoTokenizer.from_pretrained(config.base_model_name_or_path)\n", + "\n", + "# Load the Lora model\n", + "comma_model = PeftModel.from_pretrained(model, peft_model_id).cuda()" + ], + "metadata": { + "id": "3pqGsRVVfnUE" + }, + "execution_count": 22, + "outputs": [] + }, + { + "cell_type": "code", + "source": [ + "from IPython.display import display, Markdown\n", + "\n", + "def make_inference(text_wrong_commas):\n", + " batch = tokenizer(f\"### INPUT\\n{text_wrong_commas.replace(',', '')}\\n\\n### FIXED\\n\", return_tensors='pt')\n", + " batch = {k: v.cuda() for k, v in batch.items()}\n", + "\n", + " with torch.cuda.amp.autocast():\n", + " output_tokens = comma_model.generate(**batch, max_new_tokens=200)\n", + "\n", + " display(Markdown((tokenizer.decode(output_tokens[0], skip_special_tokens=True))))" + ], + "metadata": { + "id": "XAqywI0-ewEK" + }, + "execution_count": 31, + "outputs": [] + }, + { + "cell_type": "code", + "source": [ + "context = \"Afterwards however he moved to Paris France.\"\n", + "\n", + "make_inference(context)" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 195 + }, + "id": "c2wMsGB8in7F", + "outputId": "f75ef7bb-91fb-4809-dbf0-5fa388490486" + }, + "execution_count": 35, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "{'input_ids': tensor([[105311, 191657, 189, 23691, 12548, 14789, 1683, 38033, 427,\n", + " 9316, 7138, 6149, 105311, 128778, 5871, 189]],\n", + " device='cuda:0'), 'attention_mask': tensor([[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]], device='cuda:0')}\n" + ] + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + "" + ], + "text/markdown": "### INPUT\nAfterwards however he moved to Paris France.\n\n### FIXED\nAfterwards however he moved to Paris France." + }, + "metadata": {} + } + ] + } + ] +} \ No newline at end of file