finetune_8bit_models.ipynb

{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "8zo2di_L042N",
        "outputId": "b38a302a-e1bb-4430-b88d-5283c309d4a7"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\u001b[K     |████████████████████████████████| 3.4 MB 8.2 MB/s \n",
            "\u001b[K     |████████████████████████████████| 3.3 MB 40.2 MB/s \n",
            "\u001b[K     |████████████████████████████████| 596 kB 62.7 MB/s \n",
            "\u001b[K     |████████████████████████████████| 67 kB 5.8 MB/s \n",
            "\u001b[K     |████████████████████████████████| 895 kB 61.8 MB/s \n",
            "\u001b[K     |████████████████████████████████| 4.0 MB 9.6 MB/s \n",
            "\u001b[K     |████████████████████████████████| 298 kB 7.8 MB/s \n",
            "\u001b[K     |████████████████████████████████| 1.1 MB 60.0 MB/s \n",
            "\u001b[K     |████████████████████████████████| 243 kB 63.3 MB/s \n",
            "\u001b[K     |████████████████████████████████| 133 kB 65.7 MB/s \n",
            "\u001b[K     |████████████████████████████████| 144 kB 60.7 MB/s \n",
            "\u001b[K     |████████████████████████████████| 271 kB 66.5 MB/s \n",
            "\u001b[K     |████████████████████████████████| 160 kB 65.8 MB/s \n",
            "\u001b[?25h"
          ]
        }
      ],
      "source": [
        "!pip install transformers==4.14.1 -q\n",
        "!pip install bitsandbytes-cuda111==0.26.0 -q\n",
        "!pip install datasets==1.16.1 -q"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "sXIjUDGc5RLq"
      },
      "source": [
        "### Fine-tuning 6-Billion GPT-J (& other models) in colab with LoRA and 8-bit compression\n",
        "\n",
        "This notebook is a simple example for fine-tuning [GPT-J-6B](https://huggingface.co/EleutherAI/gpt-j-6B) with limited memory. A detailed explanation of how it works can be found in [this model card](https://huggingface.co/hivemind/gpt-j-6B-8bit). It is heavily based on [this Colab](https://colab.research.google.com/drive/1ft6wQU0BhqG5PRlwgaZJv2VukKKjU4Es#scrollTo=vfdLQHOuEU7h). Huge thanks to Hivemind!\n",
        "\n",
        "You can also finetune [GPT-Neo-2.7B](https://huggingface.co/gustavecortal/gpt-neo-2.7B-8bit), [French GPT-J (Cedille's Boris)](https://huggingface.co/gustavecortal/fr-boris-8bit) and [T0-3B](https://huggingface.co/gustavecortal/T0_3B-8bit) with limited memory.\n",
        "\n",
        "Twitter: [@gustavecortal](https://twitter.com/gustavecortal)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "bX35XMqY1JfY"
      },
      "outputs": [],
      "source": [
        "from sklearn.model_selection import train_test_split\n",
        "\n",
        "import transformers\n",
        "\n",
        "import pandas as pd\n",
        "\n",
        "import torch\n",
        "import torch.nn.functional as F\n",
        "from torch import nn\n",
        "from torch.cuda.amp import custom_fwd, custom_bwd\n",
        "\n",
        "from bitsandbytes.functional import quantize_blockwise, dequantize_blockwise\n",
        "\n",
        "from tqdm.auto import tqdm"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "47hM3-7-1WOh",
        "outputId": "1909a3be-6238-48c9-d1b8-7c4791d558d1"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Mounted at /content/drive\n"
          ]
        }
      ],
      "source": [
        "from google.colab import drive\n",
        "drive._mount(\"/content/drive\")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "EKti6sXNG4H5"
      },
      "source": [
        "## Converting the model to 8 bits"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "bVLaOT811bVo"
      },
      "outputs": [],
      "source": [
        "class FrozenBNBLinear(nn.Module):\n",
        "    def __init__(self, weight, absmax, code, bias=None):\n",
        "        assert isinstance(bias, nn.Parameter) or bias is None\n",
        "        super().__init__()\n",
        "        self.out_features, self.in_features = weight.shape\n",
        "        self.register_buffer(\"weight\", weight.requires_grad_(False))\n",
        "        self.register_buffer(\"absmax\", absmax.requires_grad_(False))\n",
        "        self.register_buffer(\"code\", code.requires_grad_(False))\n",
        "        self.adapter = None\n",
        "        self.bias = bias\n",
        " \n",
        "    def forward(self, input):\n",
        "        output = DequantizeAndLinear.apply(input, self.weight, self.absmax, self.code, self.bias)\n",
        "        if self.adapter:\n",
        "            output += self.adapter(input)\n",
        "        return output\n",
        " \n",
        "    @classmethod\n",
        "    def from_linear(cls, linear: nn.Linear) -> \"FrozenBNBLinear\":\n",
        "        weights_int8, state = quantize_blockise_lowmemory(linear.weight)\n",
        "        return cls(weights_int8, *state, linear.bias)\n",
        " \n",
        "    def __repr__(self):\n",
        "        return f\"{self.__class__.__name__}({self.in_features}, {self.out_features})\"\n",
        " \n",
        " \n",
        "class DequantizeAndLinear(torch.autograd.Function): \n",
        "    @staticmethod\n",
        "    @custom_fwd\n",
        "    def forward(ctx, input: torch.Tensor, weights_quantized: torch.ByteTensor,\n",
        "                absmax: torch.FloatTensor, code: torch.FloatTensor, bias: torch.FloatTensor):\n",
        "        weights_deq = dequantize_blockwise(weights_quantized, absmax=absmax, code=code)\n",
        "        ctx.save_for_backward(input, weights_quantized, absmax, code)\n",
        "        ctx._has_bias = bias is not None\n",
        "        return F.linear(input, weights_deq, bias).clone()\n",
        " \n",
        "    @staticmethod\n",
        "    @custom_bwd\n",
        "    def backward(ctx, grad_output: torch.Tensor):\n",
        "        assert not ctx.needs_input_grad[1] and not ctx.needs_input_grad[2] and not ctx.needs_input_grad[3]\n",
        "        input, weights_quantized, absmax, code = ctx.saved_tensors\n",
        "        # grad_output: [*batch, out_features]\n",
        "        weights_deq = dequantize_blockwise(weights_quantized, absmax=absmax, code=code)\n",
        "        grad_input = grad_output @ weights_deq\n",
        "        grad_bias = grad_output.flatten(0, -2).sum(dim=0) if ctx._has_bias else None\n",
        "        return grad_input, None, None, None, grad_bias\n",
        " \n",
        " \n",
        "class FrozenBNBEmbedding(nn.Module):\n",
        "    def __init__(self, weight, absmax, code):\n",
        "        super().__init__()\n",
        "        self.num_embeddings, self.embedding_dim = weight.shape\n",
        "        self.register_buffer(\"weight\", weight.requires_grad_(False))\n",
        "        self.register_buffer(\"absmax\", absmax.requires_grad_(False))\n",
        "        self.register_buffer(\"code\", code.requires_grad_(False))\n",
        "        self.adapter = None\n",
        " \n",
        "    def forward(self, input, **kwargs):\n",
        "        with torch.no_grad():\n",
        "            # note: both quantuized weights and input indices are *not* differentiable\n",
        "            weight_deq = dequantize_blockwise(self.weight, absmax=self.absmax, code=self.code)\n",
        "            output = F.embedding(input, weight_deq, **kwargs)\n",
        "        if self.adapter:\n",
        "            output += self.adapter(input)\n",
        "        return output \n",
        " \n",
        "    @classmethod\n",
        "    def from_embedding(cls, embedding: nn.Embedding) -> \"FrozenBNBEmbedding\":\n",
        "        weights_int8, state = quantize_blockise_lowmemory(embedding.weight)\n",
        "        return cls(weights_int8, *state)\n",
        " \n",
        "    def __repr__(self):\n",
        "        return f\"{self.__class__.__name__}({self.num_embeddings}, {self.embedding_dim})\"\n",
        " \n",
        " \n",
        "def quantize_blockise_lowmemory(matrix: torch.Tensor, chunk_size: int = 2 ** 20):\n",
        "    assert chunk_size % 4096 == 0\n",
        "    code = None\n",
        "    chunks = []\n",
        "    absmaxes = []\n",
        "    flat_tensor = matrix.view(-1)\n",
        "    for i in range((matrix.numel() - 1) // chunk_size + 1):\n",
        "        input_chunk = flat_tensor[i * chunk_size: (i + 1) * chunk_size].clone()\n",
        "        quantized_chunk, (absmax_chunk, code) = quantize_blockwise(input_chunk, code=code)\n",
        "        chunks.append(quantized_chunk)\n",
        "        absmaxes.append(absmax_chunk)\n",
        " \n",
        "    matrix_i8 = torch.cat(chunks).reshape_as(matrix)\n",
        "    absmax = torch.cat(absmaxes)\n",
        "    return matrix_i8, (absmax, code)\n",
        " \n",
        " \n",
        "def convert_to_int8(model):\n",
        "    \"\"\"Convert linear and embedding modules to 8-bit with optional adapters\"\"\"\n",
        "    for module in list(model.modules()):\n",
        "        for name, child in module.named_children():\n",
        "            if isinstance(child, nn.Linear):\n",
        "                print(name, child)\n",
        "                setattr(\n",
        "                    module,\n",
        "                    name,\n",
        "                    FrozenBNBLinear(\n",
        "                        weight=torch.zeros(child.out_features, child.in_features, dtype=torch.uint8),\n",
        "                        absmax=torch.zeros((child.weight.numel() - 1) // 4096 + 1),\n",
        "                        code=torch.zeros(256),\n",
        "                        bias=child.bias,\n",
        "                    ),\n",
        "                )\n",
        "            elif isinstance(child, nn.Embedding):\n",
        "                setattr(\n",
        "                    module,\n",
        "                    name,\n",
        "                    FrozenBNBEmbedding(\n",
        "                        weight=torch.zeros(child.num_embeddings, child.embedding_dim, dtype=torch.uint8),\n",
        "                        absmax=torch.zeros((child.weight.numel() - 1) // 4096 + 1),\n",
        "                        code=torch.zeros(256),\n",
        "                    )\n",
        "                )"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "D3Y40yNoG_Kn"
      },
      "source": [
        "You have to Monkey-Patch GPT-J before loading: "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "xtafVIQn1gcl"
      },
      "outputs": [],
      "source": [
        "class GPTJBlock(transformers.models.gptj.modeling_gptj.GPTJBlock):\n",
        "    def __init__(self, config):\n",
        "        super().__init__(config)\n",
        "\n",
        "        convert_to_int8(self.attn)\n",
        "        convert_to_int8(self.mlp)\n",
        "\n",
        "\n",
        "class GPTJModel(transformers.models.gptj.modeling_gptj.GPTJModel):\n",
        "    def __init__(self, config):\n",
        "        super().__init__(config)\n",
        "        convert_to_int8(self)\n",
        "        \n",
        "\n",
        "class GPTJForCausalLM(transformers.models.gptj.modeling_gptj.GPTJForCausalLM):\n",
        "    def __init__(self, config):\n",
        "        super().__init__(config)\n",
        "        convert_to_int8(self)\n",
        "\n",
        "\n",
        "transformers.models.gptj.modeling_gptj.GPTJBlock = GPTJBlock"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rpyKyOpsHF27"
      },
      "source": [
        "If you're using another 8-bit quantized model (e.g. T0-3B), remember to Monkey-Patch the model using convert_to_int8()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Y1q4P6av8e3k"
      },
      "outputs": [],
      "source": [
        "class T5ForConditionalGeneration(transformers.models.t5.modeling_t5.T5ForConditionalGeneration):\n",
        "    def __init__(self, config):\n",
        "        super().__init__(config)\n",
        "        convert_to_int8(self)\n",
        "\n",
        "transformers.models.t5.modeling_t5.T5ForConditionalGeneration = T5ForConditionalGeneration"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 241,
          "referenced_widgets": [
            "8b243dfd4ed24b699d674900f45ad195",
            "a867a214610e450cbcfd59da53052edc",
            "525ad55300cc449691535b5469ec0c22",
            "305666fe63b4480ca886654e3e953264",
            "b0d135b0a1cd49f4bae49e128b37803c",
            "3800f430b1f94524b30eb7ffbc49ef91",
            "f1f09aab70884cf3b6a7dbc8c3dee490",
            "645cdc9531c344e7a941b0ec7721e336",
            "fdb865c784844112b89ce813f0bef9e0",
            "3907b4fe14cd40d9b03216a3ed10f386",
            "838f070e33af470fb753fe130aa19f96",
            "4f915b1dfb3643649e0d31ac00d43fb7",
            "3a24869dd7db4e409613a25a99b2e84f",
            "2248e00287784cf1b86ed8715237db44",
            "edc42778a7824d89b1bbd36ac9da256f",
            "7045f8e1ea5e4725aa002c61063117dd",
            "5a4782ab72c54304a8e33e9425ef11fa",
            "97701990ecc845edb80abcdffe1c5132",
            "73cfe65490cc492f8f61cb88a9e084f3",
            "96193485918d42918bcb991becd7f3c2",
            "2fe206d674494389b1c75b0b1b95d2b3",
            "ee2a498700c247cd80208d8b0910d058",
            "541d272569e34fa893e1f1192621b585",
            "da9dbb3284524884b0e4f9298c665297",
            "a2aab4900a1d4035a4e9a32f438b4b60",
            "8f8ada3ce83340baa656583f7b7e2800",
            "929515722de84dfeb333a94185c09d69",
            "8b1515ae411e48f88e75dbde1bc4b0f1",
            "6fd25c7d836d4f88abfd352b0995ce95",
            "a99bea0476c7470b90192bf7838ebf6a",
            "706bdb7d967947ddbe94febcb4a8ae6f",
            "0c63469be3a142f792f69b14984e7747",
            "e80f1246b4264ec28e748d60e638821d",
            "8ff0230a260c4a15a54b70f9b7b25884",
            "275766ea5f4c4f45981ca38bc3d30468",
            "9fbe0e0845be4b2ba73bf01c40f7b157",
            "3e929c00f3f1421cae759e562581c743",
            "a4eeb1a8a4874b31917e04fe18db5ecb",
            "a25b27da7e664c7eb289e6735b5459dd",
            "54234e3b446e4f4b865f76b9fdafce01",
            "bc2c477786ad4528b8d16454040d3345",
            "03db147f01564aa296358bc650be3916",
            "4d313423a8b746d2bd9deab3d10237b5",
            "71b10878e239477a9f314372621ad021",
            "b9a0a9b919e14c51a844df7b0550da5b",
            "67bc335a6d2f4672a0f66f6c05974e39",
            "b639299a491f429ba70c2f959194fba5",
            "ae3127a7feea4e7eb84298ffb057ace2",
            "11a14e14654949a6a173916989bd43c1",
            "418c131375324b88b70bb93aca6461db",
            "c4833007b9284e18a84f648e3b36aa05",
            "6f1e1e5e5ce9420399904b7c34bb0c09",
            "3515b382d154416e98bdee3444e34179",
            "ddbe5279fc04449188695375d7f33d52",
            "ce901c41bdda40bab37c224822f022b0",
            "b3723a3e712849899e2242a39c517648",
            "7e1fc6a38569457cb717db7b41aebd52",
            "0463e1b8507441b79e996f73c68a40d6",
            "153620a44c88414a8e610c6af1de0267",
            "036adfe8ee224a55a0889737fbe48ff4",
            "42bb16d7389d43a9b5506130fe4e09d7",
            "7823b6ffcef94222be1cdca2c661ed04",
            "c573a8b0fb5b4e87be558da9d3be2fdf",
            "400069c8a6ef476a9332d0ea9b9ef20f",
            "23accb572c764ac7b18aaf1af6a022b9",
            "6c8f1d95149d4f6bb0330258f6af0ea4",
            "2d1d42edbe264d6198c59cac8ecb7895",
            "55d0e59062594eabb7f24dab156e2a94",
            "3291292ee663463ca940ffe5c4ea4ae5",
            "69b48c289d13413f80ee55cbad95573e",
            "fe85185c1e59439f9a5d85b36fcd2ef9",
            "ba0d9af23cdd4885a9cab57472221fa2",
            "d013898ae9cf4a7ca007d281cbd7d28d",
            "c65f8696241f426bb295bda31d4ad667",
            "4c4afa08ac034ca59f47ed27c452562b",
            "6c9bfc504cfe4cdaa347a40bf730070a",
            "ed61ae1802d64ee5a7b59e8ece23f470"
          ]
        },
        "id": "qwJixCub1uwI",
        "outputId": "01d4bc5a-ee8a-47be-a943-4815a950d855"
      },
      "outputs": [
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "8b243dfd4ed24b699d674900f45ad195",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Downloading:   0%|          | 0.00/826 [00:00<?, ?B/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "4f915b1dfb3643649e0d31ac00d43fb7",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Downloading:   0%|          | 0.00/619 [00:00<?, ?B/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "541d272569e34fa893e1f1192621b585",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Downloading:   0%|          | 0.00/779k [00:00<?, ?B/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "8ff0230a260c4a15a54b70f9b7b25884",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Downloading:   0%|          | 0.00/446k [00:00<?, ?B/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "b9a0a9b919e14c51a844df7b0550da5b",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Downloading:   0%|          | 0.00/1.31M [00:00<?, ?B/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "b3723a3e712849899e2242a39c517648",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Downloading:   0%|          | 0.00/3.94k [00:00<?, ?B/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "2d1d42edbe264d6198c59cac8ecb7895",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Downloading:   0%|          | 0.00/357 [00:00<?, ?B/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "config = transformers.GPTJConfig.from_pretrained(\"EleutherAI/gpt-j-6B\")\n",
        "tokenizer = transformers.AutoTokenizer.from_pretrained(\"EleutherAI/gpt-j-6B\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "y0SXSVLk1zh_"
      },
      "outputs": [],
      "source": [
        "config.pad_token_id = config.eos_token_id\n",
        "tokenizer.pad_token = config.pad_token_id"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "bYYH1l2hHhFZ",
        "outputId": "79247244-d9d0-4023-c743-fb6148f1662d"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "k_proj Linear(in_features=4096, out_features=4096, bias=False)\n",
            "v_proj Linear(in_features=4096, out_features=4096, bias=False)\n",
            "q_proj Linear(in_features=4096, out_features=4096, bias=False)\n",
            "out_proj Linear(in_features=4096, out_features=4096, bias=False)\n",
            "fc_in Linear(in_features=4096, out_features=16384, bias=True)\n",
            "fc_out Linear(in_features=16384, out_features=4096, bias=True)\n",
            "k_proj Linear(in_features=4096, out_features=4096, bias=False)\n",
            "v_proj Linear(in_features=4096, out_features=4096, bias=False)\n",
            "q_proj Linear(in_features=4096, out_features=4096, bias=False)\n",
            "out_proj Linear(in_features=4096, out_features=4096, bias=False)\n",
            "fc_in Linear(in_features=4096, out_features=16384, bias=True)\n",
            "fc_out Linear(in_features=16384, out_features=4096, bias=True)\n",
            "k_proj Linear(in_features=4096, out_features=4096, bias=False)\n",
            "v_proj Linear(in_features=4096, out_features=4096, bias=False)\n",
            "q_proj Linear(in_features=4096, out_features=4096, bias=False)\n",
            "out_proj Linear(in_features=4096, out_features=4096, bias=False)\n",
            "fc_in Linear(in_features=4096, out_features=16384, bias=True)\n",
            "fc_out Linear(in_features=16384, out_features=4096, bias=True)\n",
            "k_proj Linear(in_features=4096, out_features=4096, bias=False)\n",
            "v_proj Linear(in_features=4096, out_features=4096, bias=False)\n",
            "q_proj Linear(in_features=4096, out_features=4096, bias=False)\n",
            "out_proj Linear(in_features=4096, out_features=4096, bias=False)\n",
            "fc_in Linear(in_features=4096, out_features=16384, bias=True)\n",
            "fc_out Linear(in_features=16384, out_features=4096, bias=True)\n",
            "k_proj Linear(in_features=4096, out_features=4096, bias=False)\n",
            "v_proj Linear(in_features=4096, out_features=4096, bias=False)\n",
            "q_proj Linear(in_features=4096, out_features=4096, bias=False)\n",
            "out_proj Linear(in_features=4096, out_features=4096, bias=False)\n",
            "fc_in Linear(in_features=4096, out_features=16384, bias=True)\n",
            "fc_out Linear(in_features=16384, out_features=4096, bias=True)\n",
            "k_proj Linear(in_features=4096, out_features=4096, bias=False)\n",
            "v_proj Linear(in_features=4096, out_features=4096, bias=False)\n",
            "q_proj Linear(in_features=4096, out_features=4096, bias=False)\n",
            "out_proj Linear(in_features=4096, out_features=4096, bias=False)\n",
            "fc_in Linear(in_features=4096, out_features=16384, bias=True)\n",
            "fc_out Linear(in_features=16384, out_features=4096, bias=True)\n",
            "k_proj Linear(in_features=4096, out_features=4096, bias=False)\n",
            "v_proj Linear(in_features=4096, out_features=4096, bias=False)\n",
            "q_proj Linear(in_features=4096, out_features=4096, bias=False)\n",
            "out_proj Linear(in_features=4096, out_features=4096, bias=False)\n",
            "fc_in Linear(in_features=4096, out_features=16384, bias=True)\n",
            "fc_out Linear(in_features=16384, out_features=4096, bias=True)\n",
            "k_proj Linear(in_features=4096, out_features=4096, bias=False)\n",
            "v_proj Linear(in_features=4096, out_features=4096, bias=False)\n",
            "q_proj Linear(in_features=4096, out_features=4096, bias=False)\n",
            "out_proj Linear(in_features=4096, out_features=4096, bias=False)\n",
            "fc_in Linear(in_features=4096, out_features=16384, bias=True)\n",
            "fc_out Linear(in_features=16384, out_features=4096, bias=True)\n",
            "k_proj Linear(in_features=4096, out_features=4096, bias=False)\n",
            "v_proj Linear(in_features=4096, out_features=4096, bias=False)\n",
            "q_proj Linear(in_features=4096, out_features=4096, bias=False)\n",
            "out_proj Linear(in_features=4096, out_features=4096, bias=False)\n",
            "fc_in Linear(in_features=4096, out_features=16384, bias=True)\n",
            "fc_out Linear(in_features=16384, out_features=4096, bias=True)\n",
            "k_proj Linear(in_features=4096, out_features=4096, bias=False)\n",
            "v_proj Linear(in_features=4096, out_features=4096, bias=False)\n",
            "q_proj Linear(in_features=4096, out_features=4096, bias=False)\n",
            "out_proj Linear(in_features=4096, out_features=4096, bias=False)\n",
            "fc_in Linear(in_features=4096, out_features=16384, bias=True)\n",
            "fc_out Linear(in_features=16384, out_features=4096, bias=True)\n",
            "k_proj Linear(in_features=4096, out_features=4096, bias=False)\n",
            "v_proj Linear(in_features=4096, out_features=4096, bias=False)\n",
            "q_proj Linear(in_features=4096, out_features=4096, bias=False)\n",
            "out_proj Linear(in_features=4096, out_features=4096, bias=False)\n",
            "fc_in Linear(in_features=4096, out_features=16384, bias=True)\n",
            "fc_out Linear(in_features=16384, out_features=4096, bias=True)\n",
            "k_proj Linear(in_features=4096, out_features=4096, bias=False)\n",
            "v_proj Linear(in_features=4096, out_features=4096, bias=False)\n",
            "q_proj Linear(in_features=4096, out_features=4096, bias=False)\n",
            "out_proj Linear(in_features=4096, out_features=4096, bias=False)\n",
            "fc_in Linear(in_features=4096, out_features=16384, bias=True)\n",
            "fc_out Linear(in_features=16384, out_features=4096, bias=True)\n",
            "k_proj Linear(in_features=4096, out_features=4096, bias=False)\n",
            "v_proj Linear(in_features=4096, out_features=4096, bias=False)\n",
            "q_proj Linear(in_features=4096, out_features=4096, bias=False)\n",
            "out_proj Linear(in_features=4096, out_features=4096, bias=False)\n",
            "fc_in Linear(in_features=4096, out_features=16384, bias=True)\n",
            "fc_out Linear(in_features=16384, out_features=4096, bias=True)\n",
            "k_proj Linear(in_features=4096, out_features=4096, bias=False)\n",
            "v_proj Linear(in_features=4096, out_features=4096, bias=False)\n",
            "q_proj Linear(in_features=4096, out_features=4096, bias=False)\n",
            "out_proj Linear(in_features=4096, out_features=4096, bias=False)\n",
            "fc_in Linear(in_features=4096, out_features=16384, bias=True)\n",
            "fc_out Linear(in_features=16384, out_features=4096, bias=True)\n",
            "k_proj Linear(in_features=4096, out_features=4096, bias=False)\n",
            "v_proj Linear(in_features=4096, out_features=4096, bias=False)\n",
            "q_proj Linear(in_features=4096, out_features=4096, bias=False)\n",
            "out_proj Linear(in_features=4096, out_features=4096, bias=False)\n",
            "fc_in Linear(in_features=4096, out_features=16384, bias=True)\n",
            "fc_out Linear(in_features=16384, out_features=4096, bias=True)\n",
            "k_proj Linear(in_features=4096, out_features=4096, bias=False)\n",
            "v_proj Linear(in_features=4096, out_features=4096, bias=False)\n",
            "q_proj Linear(in_features=4096, out_features=4096, bias=False)\n",
            "out_proj Linear(in_features=4096, out_features=4096, bias=False)\n",
            "fc_in Linear(in_features=4096, out_features=16384, bias=True)\n",
            "fc_out Linear(in_features=16384, out_features=4096, bias=True)\n",
            "k_proj Linear(in_features=4096, out_features=4096, bias=False)\n",
            "v_proj Linear(in_features=4096, out_features=4096, bias=False)\n",
            "q_proj Linear(in_features=4096, out_features=4096, bias=False)\n",
            "out_proj Linear(in_features=4096, out_features=4096, bias=False)\n",
            "fc_in Linear(in_features=4096, out_features=16384, bias=True)\n",
            "fc_out Linear(in_features=16384, out_features=4096, bias=True)\n",
            "k_proj Linear(in_features=4096, out_features=4096, bias=False)\n",
            "v_proj Linear(in_features=4096, out_features=4096, bias=False)\n",
            "q_proj Linear(in_features=4096, out_features=4096, bias=False)\n",
            "out_proj Linear(in_features=4096, out_features=4096, bias=False)\n",
            "fc_in Linear(in_features=4096, out_features=16384, bias=True)\n",
            "fc_out Linear(in_features=16384, out_features=4096, bias=True)\n",
            "k_proj Linear(in_features=4096, out_features=4096, bias=False)\n",
            "v_proj Linear(in_features=4096, out_features=4096, bias=False)\n",
            "q_proj Linear(in_features=4096, out_features=4096, bias=False)\n",
            "out_proj Linear(in_features=4096, out_features=4096, bias=False)\n",
            "fc_in Linear(in_features=4096, out_features=16384, bias=True)\n",
            "fc_out Linear(in_features=16384, out_features=4096, bias=True)\n",
            "k_proj Linear(in_features=4096, out_features=4096, bias=False)\n",
            "v_proj Linear(in_features=4096, out_features=4096, bias=False)\n",
            "q_proj Linear(in_features=4096, out_features=4096, bias=False)\n",
            "out_proj Linear(in_features=4096, out_features=4096, bias=False)\n",
            "fc_in Linear(in_features=4096, out_features=16384, bias=True)\n",
            "fc_out Linear(in_features=16384, out_features=4096, bias=True)\n",
            "k_proj Linear(in_features=4096, out_features=4096, bias=False)\n",
            "v_proj Linear(in_features=4096, out_features=4096, bias=False)\n",
            "q_proj Linear(in_features=4096, out_features=4096, bias=False)\n",
            "out_proj Linear(in_features=4096, out_features=4096, bias=False)\n",
            "fc_in Linear(in_features=4096, out_features=16384, bias=True)\n",
            "fc_out Linear(in_features=16384, out_features=4096, bias=True)\n",
            "k_proj Linear(in_features=4096, out_features=4096, bias=False)\n",
            "v_proj Linear(in_features=4096, out_features=4096, bias=False)\n",
            "q_proj Linear(in_features=4096, out_features=4096, bias=False)\n",
            "out_proj Linear(in_features=4096, out_features=4096, bias=False)\n",
            "fc_in Linear(in_features=4096, out_features=16384, bias=True)\n",
            "fc_out Linear(in_features=16384, out_features=4096, bias=True)\n",
            "k_proj Linear(in_features=4096, out_features=4096, bias=False)\n",
            "v_proj Linear(in_features=4096, out_features=4096, bias=False)\n",
            "q_proj Linear(in_features=4096, out_features=4096, bias=False)\n",
            "out_proj Linear(in_features=4096, out_features=4096, bias=False)\n",
            "fc_in Linear(in_features=4096, out_features=16384, bias=True)\n",
            "fc_out Linear(in_features=16384, out_features=4096, bias=True)\n",
            "k_proj Linear(in_features=4096, out_features=4096, bias=False)\n",
            "v_proj Linear(in_features=4096, out_features=4096, bias=False)\n",
            "q_proj Linear(in_features=4096, out_features=4096, bias=False)\n",
            "out_proj Linear(in_features=4096, out_features=4096, bias=False)\n",
            "fc_in Linear(in_features=4096, out_features=16384, bias=True)\n",
            "fc_out Linear(in_features=16384, out_features=4096, bias=True)\n",
            "k_proj Linear(in_features=4096, out_features=4096, bias=False)\n",
            "v_proj Linear(in_features=4096, out_features=4096, bias=False)\n",
            "q_proj Linear(in_features=4096, out_features=4096, bias=False)\n",
            "out_proj Linear(in_features=4096, out_features=4096, bias=False)\n",
            "fc_in Linear(in_features=4096, out_features=16384, bias=True)\n",
            "fc_out Linear(in_features=16384, out_features=4096, bias=True)\n",
            "k_proj Linear(in_features=4096, out_features=4096, bias=False)\n",
            "v_proj Linear(in_features=4096, out_features=4096, bias=False)\n",
            "q_proj Linear(in_features=4096, out_features=4096, bias=False)\n",
            "out_proj Linear(in_features=4096, out_features=4096, bias=False)\n",
            "fc_in Linear(in_features=4096, out_features=16384, bias=True)\n",
            "fc_out Linear(in_features=16384, out_features=4096, bias=True)\n",
            "k_proj Linear(in_features=4096, out_features=4096, bias=False)\n",
            "v_proj Linear(in_features=4096, out_features=4096, bias=False)\n",
            "q_proj Linear(in_features=4096, out_features=4096, bias=False)\n",
            "out_proj Linear(in_features=4096, out_features=4096, bias=False)\n",
            "fc_in Linear(in_features=4096, out_features=16384, bias=True)\n",
            "fc_out Linear(in_features=16384, out_features=4096, bias=True)\n",
            "k_proj Linear(in_features=4096, out_features=4096, bias=False)\n",
            "v_proj Linear(in_features=4096, out_features=4096, bias=False)\n",
            "q_proj Linear(in_features=4096, out_features=4096, bias=False)\n",
            "out_proj Linear(in_features=4096, out_features=4096, bias=False)\n",
            "fc_in Linear(in_features=4096, out_features=16384, bias=True)\n",
            "fc_out Linear(in_features=16384, out_features=4096, bias=True)\n",
            "lm_head Linear(in_features=4096, out_features=50400, bias=True)\n"
          ]
        }
      ],
      "source": [
        "gpt = GPTJForCausalLM.from_pretrained(\"hivemind/gpt-j-6B-8bit\", low_cpu_mem_usage=True)\n",
        "#gpt = GPTJForCausalLM.from_pretrained(\"gustavecortal/fr-boris-8bit\", low_cpu_mem_usage=True) French GPT-J Cedille's Boris"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "UOmJJMf0Ho3H"
      },
      "source": [
        "## LoRA fine-tuning example"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9qnwAuQ8HsM6"
      },
      "source": [
        "You can load my very small dataset composed of philosophical sentences: "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "DokZdiFd9K-W",
        "outputId": "1331f194-7a9b-411b-a7c5-c20885b1cadd"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Downloading...\n",
            "From: https://drive.google.com/uc?id=1Q1WMjny26VHLKb71iTCHIS5zvdm9c-wz\n",
            "To: /content/pgbp_example.csv\n",
            "\r  0% 0.00/41.0k [00:00<?, ?B/s]\r100% 41.0k/41.0k [00:00<00:00, 30.6MB/s]\n"
          ]
        }
      ],
      "source": [
        "!gdown --id 1Q1WMjny26VHLKb71iTCHIS5zvdm9c-wz"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "1SUuVsoZ18rb"
      },
      "outputs": [],
      "source": [
        "data = pd.read_csv('/content/pgbp_example.csv')\n",
        "data['sentence'] = 'Quote: ' + data['sentence']\n",
        "train, test = train_test_split(data, test_size=0.01) \n",
        "train.to_csv('/content/train_pgbp_example.csv', index=False)\n",
        "test.to_csv('/content/test_pgbp_example.csv', index=False)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 185,
          "referenced_widgets": [
            "d80a21b53ed643b8986ae21a4a32e6a0",
            "076ec84c6fd04b79adc38de649165c5f",
            "27f7656a89d64696af799430fb651c01",
            "14557167cd4b4c2c9c2ea4bfe6b2cd6a",
            "f720e92372104a548d2deca0ea94bddf",
            "913bc1e8cef440f7871acc587dcdc48f",
            "439c874ddcde4ad2b00376c5d0288bc2",
            "883c0e7ddbe646f09cb420bcc5540573",
            "d255304c7adc4efab329d9e968339bd7",
            "5fbb1f1b1ea14c10b4c74ea90f125c5a",
            "0b7935123c604d5286eb9006a4b32a85",
            "c082443ce61d4b158576855946b0f649",
            "276ca8ce6863488ba4f83e5c6921638f",
            "6b0b00ed82e9470c827450e70f8ad180",
            "cebbbf7f40d94b3a9de105b6781f4a0e",
            "d496fc35dd894bf3b8f4f41dd1052d5a",
            "9f6d069fd43e4016ac7bcb1696c3b768",
            "d898fb31b0c74a5a9dd7396ae2d15db6",
            "1656a97c9fea490dafcaf7ef6113c59f",
            "6c2e8a8f07ab45f1a025d460355fb367",
            "9913cafea07e459ca3f37eb672c9482b",
            "f843c9bf912c48b7a9899f1e8707627c",
            "27b71deb79e74b5d9175bdccfd5f67fc",
            "d94ae41c4364487aa6df495820b3cc5d",
            "9ad3cd1329c04798b9428ffdfcc35979",
            "e0a151d88f54451c8b52f7af42879f13",
            "37cc766b349043c9b92e6147900eb84b",
            "4617062fc7274559b75515adf245e50b",
            "ed97ac95d1d54ec6aa82aee05a0e24f2",
            "4ff6004f577449f2a13ea7ba82fcb7e1",
            "7a01312bac844ec89f401b2011ea6b4f",
            "b53033e55c504bc99cf0705cbf2699a9",
            "95c5cab8e0ba4b268192a4dc96b1a5d2"
          ]
        },
        "id": "eo6uZOqU2DxU",
        "outputId": "11b77c8f-2bfc-46b8-ee8c-f96f054ebacd"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Using custom data configuration default-8e51d309cdf718da\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Downloading and preparing dataset csv/default to /root/.cache/huggingface/datasets/csv/default-8e51d309cdf718da/0.0.0/bf68a4c4aefa545d0712b2fcbb1b327f905bbe2f6425fbc5e8c25234acb9e14a...\n"
          ]
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "d80a21b53ed643b8986ae21a4a32e6a0",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "  0%|          | 0/2 [00:00<?, ?it/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "c082443ce61d4b158576855946b0f649",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "  0%|          | 0/2 [00:00<?, ?it/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Dataset csv downloaded and prepared to /root/.cache/huggingface/datasets/csv/default-8e51d309cdf718da/0.0.0/bf68a4c4aefa545d0712b2fcbb1b327f905bbe2f6425fbc5e8c25234acb9e14a. Subsequent calls will reuse this data.\n"
          ]
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "27b71deb79e74b5d9175bdccfd5f67fc",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "  0%|          | 0/2 [00:00<?, ?it/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "from datasets import load_dataset\n",
        "dataset = load_dataset('csv', data_files={'train': '/content/train_pgbp_example.csv',\n",
        "                                              'test': '/content/test_pgbp_example.csv'})"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 81,
          "referenced_widgets": [
            "9650c796c899416b9da0267704ee9ddf",
            "ac1d31cb2ca5441b9b5f67a28fb844ed",
            "327bbc6ac0ee425e92d0620e88269c7d",
            "e1c6e90443e745e18b357faf9f4d0643",
            "9d369eb7add14f2d9a63f539ae0dd4fb",
            "50edafbb02154cf78a17a6df6af7093e",
            "38a54faf890c4d61bbbb118f34303619",
            "0677c559c1ec45bf8c83bdfdf0438504",
            "ec03dc571ef041e7b4caae39bf560967",
            "8f597e5db3464769af6eed68f367f337",
            "c06e95c35361484b9bcbccba1542b733",
            "f74311b977e747b8b8138998c75ca8cc",
            "211011ebc2114548ab7c1800d0a031f8",
            "ed6b385575eb49b38d4911db33367f3f",
            "4fc52cddcd95477c89d9065b436c708f",
            "d49229a761204b52b0d4f792a1e7eb4e",
            "5da61c30812b4185a022c4647e241e9a",
            "80d90e26a7f546739f139152fb291483",
            "b8c49ff14c7349f6938e0d4174b71c6d",
            "90fdffc266874194902704a8861b30fc",
            "203cb2d94dd54e719843eae2ba95ad84",
            "3bcf6f6e2cbe43048b17754c7569b830"
          ]
        },
        "id": "S0QrNw_r2OMr",
        "outputId": "f7cb189f-4d56-4c9c-a953-46bbda061c76"
      },
      "outputs": [
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "9650c796c899416b9da0267704ee9ddf",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "  0%|          | 0/1 [00:00<?, ?ba/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "f74311b977e747b8b8138998c75ca8cc",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "  0%|          | 0/1 [00:00<?, ?ba/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "def tokenize_function(examples):\n",
        "    return tokenizer(examples[\"sentence\"], padding=True, truncation=True, max_length= 128)\n",
        "\n",
        "tokenized_datasets = dataset.map(tokenize_function, batched=True)\n",
        "tokenized_datasets = tokenized_datasets.remove_columns([\"sentence\"])\n",
        "tokenized_datasets.set_format(\"torch\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "vltw5L0i2b3d"
      },
      "outputs": [],
      "source": [
        "from torch.utils.data import DataLoader\n",
        "\n",
        "full_train_dataset = tokenized_datasets[\"train\"]\n",
        "train_dataloader = DataLoader(full_train_dataset, shuffle=False, batch_size=8)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-MUl4jkSKi6W"
      },
      "source": [
        "Add adapters to Embedding/MLP/Attention/LMHead layers"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "W4Jfpe2h2mHI",
        "outputId": "b94ad089-6bf6-4f76-c99d-a3f7bce134a4"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Adding adapter to transformer.wte\n",
            "Initializing transformer.wte\n",
            "Adding adapter to transformer.h.0.attn.k_proj\n",
            "Initializing transformer.h.0.attn.k_proj\n",
            "Adding adapter to transformer.h.0.attn.v_proj\n",
            "Initializing transformer.h.0.attn.v_proj\n",
            "Adding adapter to transformer.h.0.attn.q_proj\n",
            "Initializing transformer.h.0.attn.q_proj\n",
            "Adding adapter to transformer.h.0.attn.out_proj\n",
            "Initializing transformer.h.0.attn.out_proj\n",
            "Adding adapter to transformer.h.0.mlp.fc_in\n",
            "Initializing transformer.h.0.mlp.fc_in\n",
            "Adding adapter to transformer.h.0.mlp.fc_out\n",
            "Initializing transformer.h.0.mlp.fc_out\n",
            "Adding adapter to transformer.h.1.attn.k_proj\n",
            "Initializing transformer.h.1.attn.k_proj\n",
            "Adding adapter to transformer.h.1.attn.v_proj\n",
            "Initializing transformer.h.1.attn.v_proj\n",
            "Adding adapter to transformer.h.1.attn.q_proj\n",
            "Initializing transformer.h.1.attn.q_proj\n",
            "Adding adapter to transformer.h.1.attn.out_proj\n",
            "Initializing transformer.h.1.attn.out_proj\n",
            "Adding adapter to transformer.h.1.mlp.fc_in\n",
            "Initializing transformer.h.1.mlp.fc_in\n",
            "Adding adapter to transformer.h.1.mlp.fc_out\n",
            "Initializing transformer.h.1.mlp.fc_out\n",
            "Adding adapter to transformer.h.2.attn.k_proj\n",
            "Initializing transformer.h.2.attn.k_proj\n",
            "Adding adapter to transformer.h.2.attn.v_proj\n",
            "Initializing transformer.h.2.attn.v_proj\n",
            "Adding adapter to transformer.h.2.attn.q_proj\n",
            "Initializing transformer.h.2.attn.q_proj\n",
            "Adding adapter to transformer.h.2.attn.out_proj\n",
            "Initializing transformer.h.2.attn.out_proj\n",
            "Adding adapter to transformer.h.2.mlp.fc_in\n",
            "Initializing transformer.h.2.mlp.fc_in\n",
            "Adding adapter to transformer.h.2.mlp.fc_out\n",
            "Initializing transformer.h.2.mlp.fc_out\n",
            "Adding adapter to transformer.h.3.attn.k_proj\n",
            "Initializing transformer.h.3.attn.k_proj\n",
            "Adding adapter to transformer.h.3.attn.v_proj\n",
            "Initializing transformer.h.3.attn.v_proj\n",
            "Adding adapter to transformer.h.3.attn.q_proj\n",
            "Initializing transformer.h.3.attn.q_proj\n",
            "Adding adapter to transformer.h.3.attn.out_proj\n",
            "Initializing transformer.h.3.attn.out_proj\n",
            "Adding adapter to transformer.h.3.mlp.fc_in\n",
            "Initializing transformer.h.3.mlp.fc_in\n",
            "Adding adapter to transformer.h.3.mlp.fc_out\n",
            "Initializing transformer.h.3.mlp.fc_out\n",
            "Adding adapter to transformer.h.4.attn.k_proj\n",
            "Initializing transformer.h.4.attn.k_proj\n",
            "Adding adapter to transformer.h.4.attn.v_proj\n",
            "Initializing transformer.h.4.attn.v_proj\n",
            "Adding adapter to transformer.h.4.attn.q_proj\n",
            "Initializing transformer.h.4.attn.q_proj\n",
            "Adding adapter to transformer.h.4.attn.out_proj\n",
            "Initializing transformer.h.4.attn.out_proj\n",
            "Adding adapter to transformer.h.4.mlp.fc_in\n",
            "Initializing transformer.h.4.mlp.fc_in\n",
            "Adding adapter to transformer.h.4.mlp.fc_out\n",
            "Initializing transformer.h.4.mlp.fc_out\n",
            "Adding adapter to transformer.h.5.attn.k_proj\n",
            "Initializing transformer.h.5.attn.k_proj\n",
            "Adding adapter to transformer.h.5.attn.v_proj\n",
            "Initializing transformer.h.5.attn.v_proj\n",
            "Adding adapter to transformer.h.5.attn.q_proj\n",
            "Initializing transformer.h.5.attn.q_proj\n",
            "Adding adapter to transformer.h.5.attn.out_proj\n",
            "Initializing transformer.h.5.attn.out_proj\n",
            "Adding adapter to transformer.h.5.mlp.fc_in\n",
            "Initializing transformer.h.5.mlp.fc_in\n",
            "Adding adapter to transformer.h.5.mlp.fc_out\n",
            "Initializing transformer.h.5.mlp.fc_out\n",
            "Adding adapter to transformer.h.6.attn.k_proj\n",
            "Initializing transformer.h.6.attn.k_proj\n",
            "Adding adapter to transformer.h.6.attn.v_proj\n",
            "Initializing transformer.h.6.attn.v_proj\n",
            "Adding adapter to transformer.h.6.attn.q_proj\n",
            "Initializing transformer.h.6.attn.q_proj\n",
            "Adding adapter to transformer.h.6.attn.out_proj\n",
            "Initializing transformer.h.6.attn.out_proj\n",
            "Adding adapter to transformer.h.6.mlp.fc_in\n",
            "Initializing transformer.h.6.mlp.fc_in\n",
            "Adding adapter to transformer.h.6.mlp.fc_out\n",
            "Initializing transformer.h.6.mlp.fc_out\n",
            "Adding adapter to transformer.h.7.attn.k_proj\n",
            "Initializing transformer.h.7.attn.k_proj\n",
            "Adding adapter to transformer.h.7.attn.v_proj\n",
            "Initializing transformer.h.7.attn.v_proj\n",
            "Adding adapter to transformer.h.7.attn.q_proj\n",
            "Initializing transformer.h.7.attn.q_proj\n",
            "Adding adapter to transformer.h.7.attn.out_proj\n",
            "Initializing transformer.h.7.attn.out_proj\n",
            "Adding adapter to transformer.h.7.mlp.fc_in\n",
            "Initializing transformer.h.7.mlp.fc_in\n",
            "Adding adapter to transformer.h.7.mlp.fc_out\n",
            "Initializing transformer.h.7.mlp.fc_out\n",
            "Adding adapter to transformer.h.8.attn.k_proj\n",
            "Initializing transformer.h.8.attn.k_proj\n",
            "Adding adapter to transformer.h.8.attn.v_proj\n",
            "Initializing transformer.h.8.attn.v_proj\n",
            "Adding adapter to transformer.h.8.attn.q_proj\n",
            "Initializing transformer.h.8.attn.q_proj\n",
            "Adding adapter to transformer.h.8.attn.out_proj\n",
            "Initializing transformer.h.8.attn.out_proj\n",
            "Adding adapter to transformer.h.8.mlp.fc_in\n",
            "Initializing transformer.h.8.mlp.fc_in\n",
            "Adding adapter to transformer.h.8.mlp.fc_out\n",
            "Initializing transformer.h.8.mlp.fc_out\n",
            "Adding adapter to transformer.h.9.attn.k_proj\n",
            "Initializing transformer.h.9.attn.k_proj\n",
            "Adding adapter to transformer.h.9.attn.v_proj\n",
            "Initializing transformer.h.9.attn.v_proj\n",
            "Adding adapter to transformer.h.9.attn.q_proj\n",
            "Initializing transformer.h.9.attn.q_proj\n",
            "Adding adapter to transformer.h.9.attn.out_proj\n",
            "Initializing transformer.h.9.attn.out_proj\n",
            "Adding adapter to transformer.h.9.mlp.fc_in\n",
            "Initializing transformer.h.9.mlp.fc_in\n",
            "Adding adapter to transformer.h.9.mlp.fc_out\n",
            "Initializing transformer.h.9.mlp.fc_out\n",
            "Adding adapter to transformer.h.10.attn.k_proj\n",
            "Initializing transformer.h.10.attn.k_proj\n",
            "Adding adapter to transformer.h.10.attn.v_proj\n",
            "Initializing transformer.h.10.attn.v_proj\n",
            "Adding adapter to transformer.h.10.attn.q_proj\n",
            "Initializing transformer.h.10.attn.q_proj\n",
            "Adding adapter to transformer.h.10.attn.out_proj\n",
            "Initializing transformer.h.10.attn.out_proj\n",
            "Adding adapter to transformer.h.10.mlp.fc_in\n",
            "Initializing transformer.h.10.mlp.fc_in\n",
            "Adding adapter to transformer.h.10.mlp.fc_out\n",
            "Initializing transformer.h.10.mlp.fc_out\n",
            "Adding adapter to transformer.h.11.attn.k_proj\n",
            "Initializing transformer.h.11.attn.k_proj\n",
            "Adding adapter to transformer.h.11.attn.v_proj\n",
            "Initializing transformer.h.11.attn.v_proj\n",
            "Adding adapter to transformer.h.11.attn.q_proj\n",
            "Initializing transformer.h.11.attn.q_proj\n",
            "Adding adapter to transformer.h.11.attn.out_proj\n",
            "Initializing transformer.h.11.attn.out_proj\n",
            "Adding adapter to transformer.h.11.mlp.fc_in\n",
            "Initializing transformer.h.11.mlp.fc_in\n",
            "Adding adapter to transformer.h.11.mlp.fc_out\n",
            "Initializing transformer.h.11.mlp.fc_out\n",
            "Adding adapter to transformer.h.12.attn.k_proj\n",
            "Initializing transformer.h.12.attn.k_proj\n",
            "Adding adapter to transformer.h.12.attn.v_proj\n",
            "Initializing transformer.h.12.attn.v_proj\n",
            "Adding adapter to transformer.h.12.attn.q_proj\n",
            "Initializing transformer.h.12.attn.q_proj\n",
            "Adding adapter to transformer.h.12.attn.out_proj\n",
            "Initializing transformer.h.12.attn.out_proj\n",
            "Adding adapter to transformer.h.12.mlp.fc_in\n",
            "Initializing transformer.h.12.mlp.fc_in\n",
            "Adding adapter to transformer.h.12.mlp.fc_out\n",
            "Initializing transformer.h.12.mlp.fc_out\n",
            "Adding adapter to transformer.h.13.attn.k_proj\n",
            "Initializing transformer.h.13.attn.k_proj\n",
            "Adding adapter to transformer.h.13.attn.v_proj\n",
            "Initializing transformer.h.13.attn.v_proj\n",
            "Adding adapter to transformer.h.13.attn.q_proj\n",
            "Initializing transformer.h.13.attn.q_proj\n",
            "Adding adapter to transformer.h.13.attn.out_proj\n",
            "Initializing transformer.h.13.attn.out_proj\n",
            "Adding adapter to transformer.h.13.mlp.fc_in\n",
            "Initializing transformer.h.13.mlp.fc_in\n",
            "Adding adapter to transformer.h.13.mlp.fc_out\n",
            "Initializing transformer.h.13.mlp.fc_out\n",
            "Adding adapter to transformer.h.14.attn.k_proj\n",
            "Initializing transformer.h.14.attn.k_proj\n",
            "Adding adapter to transformer.h.14.attn.v_proj\n",
            "Initializing transformer.h.14.attn.v_proj\n",
            "Adding adapter to transformer.h.14.attn.q_proj\n",
            "Initializing transformer.h.14.attn.q_proj\n",
            "Adding adapter to transformer.h.14.attn.out_proj\n",
            "Initializing transformer.h.14.attn.out_proj\n",
            "Adding adapter to transformer.h.14.mlp.fc_in\n",
            "Initializing transformer.h.14.mlp.fc_in\n",
            "Adding adapter to transformer.h.14.mlp.fc_out\n",
            "Initializing transformer.h.14.mlp.fc_out\n",
            "Adding adapter to transformer.h.15.attn.k_proj\n",
            "Initializing transformer.h.15.attn.k_proj\n",
            "Adding adapter to transformer.h.15.attn.v_proj\n",
            "Initializing transformer.h.15.attn.v_proj\n",
            "Adding adapter to transformer.h.15.attn.q_proj\n",
            "Initializing transformer.h.15.attn.q_proj\n",
            "Adding adapter to transformer.h.15.attn.out_proj\n",
            "Initializing transformer.h.15.attn.out_proj\n",
            "Adding adapter to transformer.h.15.mlp.fc_in\n",
            "Initializing transformer.h.15.mlp.fc_in\n",
            "Adding adapter to transformer.h.15.mlp.fc_out\n",
            "Initializing transformer.h.15.mlp.fc_out\n",
            "Adding adapter to transformer.h.16.attn.k_proj\n",
            "Initializing transformer.h.16.attn.k_proj\n",
            "Adding adapter to transformer.h.16.attn.v_proj\n",
            "Initializing transformer.h.16.attn.v_proj\n",
            "Adding adapter to transformer.h.16.attn.q_proj\n",
            "Initializing transformer.h.16.attn.q_proj\n",
            "Adding adapter to transformer.h.16.attn.out_proj\n",
            "Initializing transformer.h.16.attn.out_proj\n",
            "Adding adapter to transformer.h.16.mlp.fc_in\n",
            "Initializing transformer.h.16.mlp.fc_in\n",
            "Adding adapter to transformer.h.16.mlp.fc_out\n",
            "Initializing transformer.h.16.mlp.fc_out\n",
            "Adding adapter to transformer.h.17.attn.k_proj\n",
            "Initializing transformer.h.17.attn.k_proj\n",
            "Adding adapter to transformer.h.17.attn.v_proj\n",
            "Initializing transformer.h.17.attn.v_proj\n",
            "Adding adapter to transformer.h.17.attn.q_proj\n",
            "Initializing transformer.h.17.attn.q_proj\n",
            "Adding adapter to transformer.h.17.attn.out_proj\n",
            "Initializing transformer.h.17.attn.out_proj\n",
            "Adding adapter to transformer.h.17.mlp.fc_in\n",
            "Initializing transformer.h.17.mlp.fc_in\n",
            "Adding adapter to transformer.h.17.mlp.fc_out\n",
            "Initializing transformer.h.17.mlp.fc_out\n",
            "Adding adapter to transformer.h.18.attn.k_proj\n",
            "Initializing transformer.h.18.attn.k_proj\n",
            "Adding adapter to transformer.h.18.attn.v_proj\n",
            "Initializing transformer.h.18.attn.v_proj\n",
            "Adding adapter to transformer.h.18.attn.q_proj\n",
            "Initializing transformer.h.18.attn.q_proj\n",
            "Adding adapter to transformer.h.18.attn.out_proj\n",
            "Initializing transformer.h.18.attn.out_proj\n",
            "Adding adapter to transformer.h.18.mlp.fc_in\n",
            "Initializing transformer.h.18.mlp.fc_in\n",
            "Adding adapter to transformer.h.18.mlp.fc_out\n",
            "Initializing transformer.h.18.mlp.fc_out\n",
            "Adding adapter to transformer.h.19.attn.k_proj\n",
            "Initializing transformer.h.19.attn.k_proj\n",
            "Adding adapter to transformer.h.19.attn.v_proj\n",
            "Initializing transformer.h.19.attn.v_proj\n",
            "Adding adapter to transformer.h.19.attn.q_proj\n",
            "Initializing transformer.h.19.attn.q_proj\n",
            "Adding adapter to transformer.h.19.attn.out_proj\n",
            "Initializing transformer.h.19.attn.out_proj\n",
            "Adding adapter to transformer.h.19.mlp.fc_in\n",
            "Initializing transformer.h.19.mlp.fc_in\n",
            "Adding adapter to transformer.h.19.mlp.fc_out\n",
            "Initializing transformer.h.19.mlp.fc_out\n",
            "Adding adapter to transformer.h.20.attn.k_proj\n",
            "Initializing transformer.h.20.attn.k_proj\n",
            "Adding adapter to transformer.h.20.attn.v_proj\n",
            "Initializing transformer.h.20.attn.v_proj\n",
            "Adding adapter to transformer.h.20.attn.q_proj\n",
            "Initializing transformer.h.20.attn.q_proj\n",
            "Adding adapter to transformer.h.20.attn.out_proj\n",
            "Initializing transformer.h.20.attn.out_proj\n",
            "Adding adapter to transformer.h.20.mlp.fc_in\n",
            "Initializing transformer.h.20.mlp.fc_in\n",
            "Adding adapter to transformer.h.20.mlp.fc_out\n",
            "Initializing transformer.h.20.mlp.fc_out\n",
            "Adding adapter to transformer.h.21.attn.k_proj\n",
            "Initializing transformer.h.21.attn.k_proj\n",
            "Adding adapter to transformer.h.21.attn.v_proj\n",
            "Initializing transformer.h.21.attn.v_proj\n",
            "Adding adapter to transformer.h.21.attn.q_proj\n",
            "Initializing transformer.h.21.attn.q_proj\n",
            "Adding adapter to transformer.h.21.attn.out_proj\n",
            "Initializing transformer.h.21.attn.out_proj\n",
            "Adding adapter to transformer.h.21.mlp.fc_in\n",
            "Initializing transformer.h.21.mlp.fc_in\n",
            "Adding adapter to transformer.h.21.mlp.fc_out\n",
            "Initializing transformer.h.21.mlp.fc_out\n",
            "Adding adapter to transformer.h.22.attn.k_proj\n",
            "Initializing transformer.h.22.attn.k_proj\n",
            "Adding adapter to transformer.h.22.attn.v_proj\n",
            "Initializing transformer.h.22.attn.v_proj\n",
            "Adding adapter to transformer.h.22.attn.q_proj\n",
            "Initializing transformer.h.22.attn.q_proj\n",
            "Adding adapter to transformer.h.22.attn.out_proj\n",
            "Initializing transformer.h.22.attn.out_proj\n",
            "Adding adapter to transformer.h.22.mlp.fc_in\n",
            "Initializing transformer.h.22.mlp.fc_in\n",
            "Adding adapter to transformer.h.22.mlp.fc_out\n",
            "Initializing transformer.h.22.mlp.fc_out\n",
            "Adding adapter to transformer.h.23.attn.k_proj\n",
            "Initializing transformer.h.23.attn.k_proj\n",
            "Adding adapter to transformer.h.23.attn.v_proj\n",
            "Initializing transformer.h.23.attn.v_proj\n",
            "Adding adapter to transformer.h.23.attn.q_proj\n",
            "Initializing transformer.h.23.attn.q_proj\n",
            "Adding adapter to transformer.h.23.attn.out_proj\n",
            "Initializing transformer.h.23.attn.out_proj\n",
            "Adding adapter to transformer.h.23.mlp.fc_in\n",
            "Initializing transformer.h.23.mlp.fc_in\n",
            "Adding adapter to transformer.h.23.mlp.fc_out\n",
            "Initializing transformer.h.23.mlp.fc_out\n",
            "Adding adapter to transformer.h.24.attn.k_proj\n",
            "Initializing transformer.h.24.attn.k_proj\n",
            "Adding adapter to transformer.h.24.attn.v_proj\n",
            "Initializing transformer.h.24.attn.v_proj\n",
            "Adding adapter to transformer.h.24.attn.q_proj\n",
            "Initializing transformer.h.24.attn.q_proj\n",
            "Adding adapter to transformer.h.24.attn.out_proj\n",
            "Initializing transformer.h.24.attn.out_proj\n",
            "Adding adapter to transformer.h.24.mlp.fc_in\n",
            "Initializing transformer.h.24.mlp.fc_in\n",
            "Adding adapter to transformer.h.24.mlp.fc_out\n",
            "Initializing transformer.h.24.mlp.fc_out\n",
            "Adding adapter to transformer.h.25.attn.k_proj\n",
            "Initializing transformer.h.25.attn.k_proj\n",
            "Adding adapter to transformer.h.25.attn.v_proj\n",
            "Initializing transformer.h.25.attn.v_proj\n",
            "Adding adapter to transformer.h.25.attn.q_proj\n",
            "Initializing transformer.h.25.attn.q_proj\n",
            "Adding adapter to transformer.h.25.attn.out_proj\n",
            "Initializing transformer.h.25.attn.out_proj\n",
            "Adding adapter to transformer.h.25.mlp.fc_in\n",
            "Initializing transformer.h.25.mlp.fc_in\n",
            "Adding adapter to transformer.h.25.mlp.fc_out\n",
            "Initializing transformer.h.25.mlp.fc_out\n",
            "Adding adapter to transformer.h.26.attn.k_proj\n",
            "Initializing transformer.h.26.attn.k_proj\n",
            "Adding adapter to transformer.h.26.attn.v_proj\n",
            "Initializing transformer.h.26.attn.v_proj\n",
            "Adding adapter to transformer.h.26.attn.q_proj\n",
            "Initializing transformer.h.26.attn.q_proj\n",
            "Adding adapter to transformer.h.26.attn.out_proj\n",
            "Initializing transformer.h.26.attn.out_proj\n",
            "Adding adapter to transformer.h.26.mlp.fc_in\n",
            "Initializing transformer.h.26.mlp.fc_in\n",
            "Adding adapter to transformer.h.26.mlp.fc_out\n",
            "Initializing transformer.h.26.mlp.fc_out\n",
            "Adding adapter to transformer.h.27.attn.k_proj\n",
            "Initializing transformer.h.27.attn.k_proj\n",
            "Adding adapter to transformer.h.27.attn.v_proj\n",
            "Initializing transformer.h.27.attn.v_proj\n",
            "Adding adapter to transformer.h.27.attn.q_proj\n",
            "Initializing transformer.h.27.attn.q_proj\n",
            "Adding adapter to transformer.h.27.attn.out_proj\n",
            "Initializing transformer.h.27.attn.out_proj\n",
            "Adding adapter to transformer.h.27.mlp.fc_in\n",
            "Initializing transformer.h.27.mlp.fc_in\n",
            "Adding adapter to transformer.h.27.mlp.fc_out\n",
            "Initializing transformer.h.27.mlp.fc_out\n",
            "Adding adapter to lm_head\n",
            "Initializing lm_head\n"
          ]
        },
        {
          "data": {
            "text/plain": [
              "GPTJForCausalLM(\n",
              "  (transformer): GPTJModel(\n",
              "    (wte): FrozenBNBEmbedding(50400, 4096)\n",
              "    (drop): Dropout(p=0.0, inplace=False)\n",
              "    (h): ModuleList(\n",
              "      (0): GPTJBlock(\n",
              "        (ln_1): LayerNorm((4096,), eps=1e-05, elementwise_affine=True)\n",
              "        (attn): GPTJAttention(\n",
              "          (attn_dropout): Dropout(p=0.0, inplace=False)\n",
              "          (resid_dropout): Dropout(p=0.0, inplace=False)\n",
              "          (k_proj): FrozenBNBLinear(4096, 4096)\n",
              "          (v_proj): FrozenBNBLinear(4096, 4096)\n",
              "          (q_proj): FrozenBNBLinear(4096, 4096)\n",
              "          (out_proj): FrozenBNBLinear(4096, 4096)\n",
              "        )\n",
              "        (mlp): GPTJMLP(\n",
              "          (fc_in): FrozenBNBLinear(4096, 16384)\n",
              "          (fc_out): FrozenBNBLinear(16384, 4096)\n",
              "          (dropout): Dropout(p=0.0, inplace=False)\n",
              "        )\n",
              "      )\n",
              "      (1): GPTJBlock(\n",
              "        (ln_1): LayerNorm((4096,), eps=1e-05, elementwise_affine=True)\n",
              "        (attn): GPTJAttention(\n",
              "          (attn_dropout): Dropout(p=0.0, inplace=False)\n",
              "          (resid_dropout): Dropout(p=0.0, inplace=False)\n",
              "          (k_proj): FrozenBNBLinear(4096, 4096)\n",
              "          (v_proj): FrozenBNBLinear(4096, 4096)\n",
              "          (q_proj): FrozenBNBLinear(4096, 4096)\n",
              "          (out_proj): FrozenBNBLinear(4096, 4096)\n",
              "        )\n",
              "        (mlp): GPTJMLP(\n",
              "          (fc_in): FrozenBNBLinear(4096, 16384)\n",
              "          (fc_out): FrozenBNBLinear(16384, 4096)\n",
              "          (dropout): Dropout(p=0.0, inplace=False)\n",
              "        )\n",
              "      )\n",
              "      (2): GPTJBlock(\n",
              "        (ln_1): LayerNorm((4096,), eps=1e-05, elementwise_affine=True)\n",
              "        (attn): GPTJAttention(\n",
              "          (attn_dropout): Dropout(p=0.0, inplace=False)\n",
              "          (resid_dropout): Dropout(p=0.0, inplace=False)\n",
              "          (k_proj): FrozenBNBLinear(4096, 4096)\n",
              "          (v_proj): FrozenBNBLinear(4096, 4096)\n",
              "          (q_proj): FrozenBNBLinear(4096, 4096)\n",
              "          (out_proj): FrozenBNBLinear(4096, 4096)\n",
              "        )\n",
              "        (mlp): GPTJMLP(\n",
              "          (fc_in): FrozenBNBLinear(4096, 16384)\n",
              "          (fc_out): FrozenBNBLinear(16384, 4096)\n",
              "          (dropout): Dropout(p=0.0, inplace=False)\n",
              "        )\n",
              "      )\n",
              "      (3): GPTJBlock(\n",
              "        (ln_1): LayerNorm((4096,), eps=1e-05, elementwise_affine=True)\n",
              "        (attn): GPTJAttention(\n",
              "          (attn_dropout): Dropout(p=0.0, inplace=False)\n",
              "          (resid_dropout): Dropout(p=0.0, inplace=False)\n",
              "          (k_proj): FrozenBNBLinear(4096, 4096)\n",
              "          (v_proj): FrozenBNBLinear(4096, 4096)\n",
              "          (q_proj): FrozenBNBLinear(4096, 4096)\n",
              "          (out_proj): FrozenBNBLinear(4096, 4096)\n",
              "        )\n",
              "        (mlp): GPTJMLP(\n",
              "          (fc_in): FrozenBNBLinear(4096, 16384)\n",
              "          (fc_out): FrozenBNBLinear(16384, 4096)\n",
              "          (dropout): Dropout(p=0.0, inplace=False)\n",
              "        )\n",
              "      )\n",
              "      (4): GPTJBlock(\n",
              "        (ln_1): LayerNorm((4096,), eps=1e-05, elementwise_affine=True)\n",
              "        (attn): GPTJAttention(\n",
              "          (attn_dropout): Dropout(p=0.0, inplace=False)\n",
              "          (resid_dropout): Dropout(p=0.0, inplace=False)\n",
              "          (k_proj): FrozenBNBLinear(4096, 4096)\n",
              "          (v_proj): FrozenBNBLinear(4096, 4096)\n",
              "          (q_proj): FrozenBNBLinear(4096, 4096)\n",
              "          (out_proj): FrozenBNBLinear(4096, 4096)\n",
              "        )\n",
              "        (mlp): GPTJMLP(\n",
              "          (fc_in): FrozenBNBLinear(4096, 16384)\n",
              "          (fc_out): FrozenBNBLinear(16384, 4096)\n",
              "          (dropout): Dropout(p=0.0, inplace=False)\n",
              "        )\n",
              "      )\n",
              "      (5): GPTJBlock(\n",
              "        (ln_1): LayerNorm((4096,), eps=1e-05, elementwise_affine=True)\n",
              "        (attn): GPTJAttention(\n",
              "          (attn_dropout): Dropout(p=0.0, inplace=False)\n",
              "          (resid_dropout): Dropout(p=0.0, inplace=False)\n",
              "          (k_proj): FrozenBNBLinear(4096, 4096)\n",
              "          (v_proj): FrozenBNBLinear(4096, 4096)\n",
              "          (q_proj): FrozenBNBLinear(4096, 4096)\n",
              "          (out_proj): FrozenBNBLinear(4096, 4096)\n",
              "        )\n",
              "        (mlp): GPTJMLP(\n",
              "          (fc_in): FrozenBNBLinear(4096, 16384)\n",
              "          (fc_out): FrozenBNBLinear(16384, 4096)\n",
              "          (dropout): Dropout(p=0.0, inplace=False)\n",
              "        )\n",
              "      )\n",
              "      (6): GPTJBlock(\n",
              "        (ln_1): LayerNorm((4096,), eps=1e-05, elementwise_affine=True)\n",
              "        (attn): GPTJAttention(\n",
              "          (attn_dropout): Dropout(p=0.0, inplace=False)\n",
              "          (resid_dropout): Dropout(p=0.0, inplace=False)\n",
              "          (k_proj): FrozenBNBLinear(4096, 4096)\n",
              "          (v_proj): FrozenBNBLinear(4096, 4096)\n",
              "          (q_proj): FrozenBNBLinear(4096, 4096)\n",
              "          (out_proj): FrozenBNBLinear(4096, 4096)\n",
              "        )\n",
              "        (mlp): GPTJMLP(\n",
              "          (fc_in): FrozenBNBLinear(4096, 16384)\n",
              "          (fc_out): FrozenBNBLinear(16384, 4096)\n",
              "          (dropout): Dropout(p=0.0, inplace=False)\n",
              "        )\n",
              "      )\n",
              "      (7): GPTJBlock(\n",
              "        (ln_1): LayerNorm((4096,), eps=1e-05, elementwise_affine=True)\n",
              "        (attn): GPTJAttention(\n",
              "          (attn_dropout): Dropout(p=0.0, inplace=False)\n",
              "          (resid_dropout): Dropout(p=0.0, inplace=False)\n",
              "          (k_proj): FrozenBNBLinear(4096, 4096)\n",
              "          (v_proj): FrozenBNBLinear(4096, 4096)\n",
              "          (q_proj): FrozenBNBLinear(4096, 4096)\n",
              "          (out_proj): FrozenBNBLinear(4096, 4096)\n",
              "        )\n",
              "        (mlp): GPTJMLP(\n",
              "          (fc_in): FrozenBNBLinear(4096, 16384)\n",
              "          (fc_out): FrozenBNBLinear(16384, 4096)\n",
              "          (dropout): Dropout(p=0.0, inplace=False)\n",
              "        )\n",
              "      )\n",
              "      (8): GPTJBlock(\n",
              "        (ln_1): LayerNorm((4096,), eps=1e-05, elementwise_affine=True)\n",
              "        (attn): GPTJAttention(\n",
              "          (attn_dropout): Dropout(p=0.0, inplace=False)\n",
              "          (resid_dropout): Dropout(p=0.0, inplace=False)\n",
              "          (k_proj): FrozenBNBLinear(4096, 4096)\n",
              "          (v_proj): FrozenBNBLinear(4096, 4096)\n",
              "          (q_proj): FrozenBNBLinear(4096, 4096)\n",
              "          (out_proj): FrozenBNBLinear(4096, 4096)\n",
              "        )\n",
              "        (mlp): GPTJMLP(\n",
              "          (fc_in): FrozenBNBLinear(4096, 16384)\n",
              "          (fc_out): FrozenBNBLinear(16384, 4096)\n",
              "          (dropout): Dropout(p=0.0, inplace=False)\n",
              "        )\n",
              "      )\n",
              "      (9): GPTJBlock(\n",
              "        (ln_1): LayerNorm((4096,), eps=1e-05, elementwise_affine=True)\n",
              "        (attn): GPTJAttention(\n",
              "          (attn_dropout): Dropout(p=0.0, inplace=False)\n",
              "          (resid_dropout): Dropout(p=0.0, inplace=False)\n",
              "          (k_proj): FrozenBNBLinear(4096, 4096)\n",
              "          (v_proj): FrozenBNBLinear(4096, 4096)\n",
              "          (q_proj): FrozenBNBLinear(4096, 4096)\n",
              "          (out_proj): FrozenBNBLinear(4096, 4096)\n",
              "        )\n",
              "        (mlp): GPTJMLP(\n",
              "          (fc_in): FrozenBNBLinear(4096, 16384)\n",
              "          (fc_out): FrozenBNBLinear(16384, 4096)\n",
              "          (dropout): Dropout(p=0.0, inplace=False)\n",
              "        )\n",
              "      )\n",
              "      (10): GPTJBlock(\n",
              "        (ln_1): LayerNorm((4096,), eps=1e-05, elementwise_affine=True)\n",
              "        (attn): GPTJAttention(\n",
              "          (attn_dropout): Dropout(p=0.0, inplace=False)\n",
              "          (resid_dropout): Dropout(p=0.0, inplace=False)\n",
              "          (k_proj): FrozenBNBLinear(4096, 4096)\n",
              "          (v_proj): FrozenBNBLinear(4096, 4096)\n",
              "          (q_proj): FrozenBNBLinear(4096, 4096)\n",
              "          (out_proj): FrozenBNBLinear(4096, 4096)\n",
              "        )\n",
              "        (mlp): GPTJMLP(\n",
              "          (fc_in): FrozenBNBLinear(4096, 16384)\n",
              "          (fc_out): FrozenBNBLinear(16384, 4096)\n",
              "          (dropout): Dropout(p=0.0, inplace=False)\n",
              "        )\n",
              "      )\n",
              "      (11): GPTJBlock(\n",
              "        (ln_1): LayerNorm((4096,), eps=1e-05, elementwise_affine=True)\n",
              "        (attn): GPTJAttention(\n",
              "          (attn_dropout): Dropout(p=0.0, inplace=False)\n",
              "          (resid_dropout): Dropout(p=0.0, inplace=False)\n",
              "          (k_proj): FrozenBNBLinear(4096, 4096)\n",
              "          (v_proj): FrozenBNBLinear(4096, 4096)\n",
              "          (q_proj): FrozenBNBLinear(4096, 4096)\n",
              "          (out_proj): FrozenBNBLinear(4096, 4096)\n",
              "        )\n",
              "        (mlp): GPTJMLP(\n",
              "          (fc_in): FrozenBNBLinear(4096, 16384)\n",
              "          (fc_out): FrozenBNBLinear(16384, 4096)\n",
              "          (dropout): Dropout(p=0.0, inplace=False)\n",
              "        )\n",
              "      )\n",
              "      (12): GPTJBlock(\n",
              "        (ln_1): LayerNorm((4096,), eps=1e-05, elementwise_affine=True)\n",
              "        (attn): GPTJAttention(\n",
              "          (attn_dropout): Dropout(p=0.0, inplace=False)\n",
              "          (resid_dropout): Dropout(p=0.0, inplace=False)\n",
              "          (k_proj): FrozenBNBLinear(4096, 4096)\n",
              "          (v_proj): FrozenBNBLinear(4096, 4096)\n",
              "          (q_proj): FrozenBNBLinear(4096, 4096)\n",
              "          (out_proj): FrozenBNBLinear(4096, 4096)\n",
              "        )\n",
              "        (mlp): GPTJMLP(\n",
              "          (fc_in): FrozenBNBLinear(4096, 16384)\n",
              "          (fc_out): FrozenBNBLinear(16384, 4096)\n",
              "          (dropout): Dropout(p=0.0, inplace=False)\n",
              "        )\n",
              "      )\n",
              "      (13): GPTJBlock(\n",
              "        (ln_1): LayerNorm((4096,), eps=1e-05, elementwise_affine=True)\n",
              "        (attn): GPTJAttention(\n",
              "          (attn_dropout): Dropout(p=0.0, inplace=False)\n",
              "          (resid_dropout): Dropout(p=0.0, inplace=False)\n",
              "          (k_proj): FrozenBNBLinear(4096, 4096)\n",
              "          (v_proj): FrozenBNBLinear(4096, 4096)\n",
              "          (q_proj): FrozenBNBLinear(4096, 4096)\n",
              "          (out_proj): FrozenBNBLinear(4096, 4096)\n",
              "        )\n",
              "        (mlp): GPTJMLP(\n",
              "          (fc_in): FrozenBNBLinear(4096, 16384)\n",
              "          (fc_out): FrozenBNBLinear(16384, 4096)\n",
              "          (dropout): Dropout(p=0.0, inplace=False)\n",
              "        )\n",
              "      )\n",
              "      (14): GPTJBlock(\n",
              "        (ln_1): LayerNorm((4096,), eps=1e-05, elementwise_affine=True)\n",
              "        (attn): GPTJAttention(\n",
              "          (attn_dropout): Dropout(p=0.0, inplace=False)\n",
              "          (resid_dropout): Dropout(p=0.0, inplace=False)\n",
              "          (k_proj): FrozenBNBLinear(4096, 4096)\n",
              "          (v_proj): FrozenBNBLinear(4096, 4096)\n",
              "          (q_proj): FrozenBNBLinear(4096, 4096)\n",
              "          (out_proj): FrozenBNBLinear(4096, 4096)\n",
              "        )\n",
              "        (mlp): GPTJMLP(\n",
              "          (fc_in): FrozenBNBLinear(4096, 16384)\n",
              "          (fc_out): FrozenBNBLinear(16384, 4096)\n",
              "          (dropout): Dropout(p=0.0, inplace=False)\n",
              "        )\n",
              "      )\n",
              "      (15): GPTJBlock(\n",
              "        (ln_1): LayerNorm((4096,), eps=1e-05, elementwise_affine=True)\n",
              "        (attn): GPTJAttention(\n",
              "          (attn_dropout): Dropout(p=0.0, inplace=False)\n",
              "          (resid_dropout): Dropout(p=0.0, inplace=False)\n",
              "          (k_proj): FrozenBNBLinear(4096, 4096)\n",
              "          (v_proj): FrozenBNBLinear(4096, 4096)\n",
              "          (q_proj): FrozenBNBLinear(4096, 4096)\n",
              "          (out_proj): FrozenBNBLinear(4096, 4096)\n",
              "        )\n",
              "        (mlp): GPTJMLP(\n",
              "          (fc_in): FrozenBNBLinear(4096, 16384)\n",
              "          (fc_out): FrozenBNBLinear(16384, 4096)\n",
              "          (dropout): Dropout(p=0.0, inplace=False)\n",
              "        )\n",
              "      )\n",
              "      (16): GPTJBlock(\n",
              "        (ln_1): LayerNorm((4096,), eps=1e-05, elementwise_affine=True)\n",
              "        (attn): GPTJAttention(\n",
              "          (attn_dropout): Dropout(p=0.0, inplace=False)\n",
              "          (resid_dropout): Dropout(p=0.0, inplace=False)\n",
              "          (k_proj): FrozenBNBLinear(4096, 4096)\n",
              "          (v_proj): FrozenBNBLinear(4096, 4096)\n",
              "          (q_proj): FrozenBNBLinear(4096, 4096)\n",
              "          (out_proj): FrozenBNBLinear(4096, 4096)\n",
              "        )\n",
              "        (mlp): GPTJMLP(\n",
              "          (fc_in): FrozenBNBLinear(4096, 16384)\n",
              "          (fc_out): FrozenBNBLinear(16384, 4096)\n",
              "          (dropout): Dropout(p=0.0, inplace=False)\n",
              "        )\n",
              "      )\n",
              "      (17): GPTJBlock(\n",
              "        (ln_1): LayerNorm((4096,), eps=1e-05, elementwise_affine=True)\n",
              "        (attn): GPTJAttention(\n",
              "          (attn_dropout): Dropout(p=0.0, inplace=False)\n",
              "          (resid_dropout): Dropout(p=0.0, inplace=False)\n",
              "          (k_proj): FrozenBNBLinear(4096, 4096)\n",
              "          (v_proj): FrozenBNBLinear(4096, 4096)\n",
              "          (q_proj): FrozenBNBLinear(4096, 4096)\n",
              "          (out_proj): FrozenBNBLinear(4096, 4096)\n",
              "        )\n",
              "        (mlp): GPTJMLP(\n",
              "          (fc_in): FrozenBNBLinear(4096, 16384)\n",
              "          (fc_out): FrozenBNBLinear(16384, 4096)\n",
              "          (dropout): Dropout(p=0.0, inplace=False)\n",
              "        )\n",
              "      )\n",
              "      (18): GPTJBlock(\n",
              "        (ln_1): LayerNorm((4096,), eps=1e-05, elementwise_affine=True)\n",
              "        (attn): GPTJAttention(\n",
              "          (attn_dropout): Dropout(p=0.0, inplace=False)\n",
              "          (resid_dropout): Dropout(p=0.0, inplace=False)\n",
              "          (k_proj): FrozenBNBLinear(4096, 4096)\n",
              "          (v_proj): FrozenBNBLinear(4096, 4096)\n",
              "          (q_proj): FrozenBNBLinear(4096, 4096)\n",
              "          (out_proj): FrozenBNBLinear(4096, 4096)\n",
              "        )\n",
              "        (mlp): GPTJMLP(\n",
              "          (fc_in): FrozenBNBLinear(4096, 16384)\n",
              "          (fc_out): FrozenBNBLinear(16384, 4096)\n",
              "          (dropout): Dropout(p=0.0, inplace=False)\n",
              "        )\n",
              "      )\n",
              "      (19): GPTJBlock(\n",
              "        (ln_1): LayerNorm((4096,), eps=1e-05, elementwise_affine=True)\n",
              "        (attn): GPTJAttention(\n",
              "          (attn_dropout): Dropout(p=0.0, inplace=False)\n",
              "          (resid_dropout): Dropout(p=0.0, inplace=False)\n",
              "          (k_proj): FrozenBNBLinear(4096, 4096)\n",
              "          (v_proj): FrozenBNBLinear(4096, 4096)\n",
              "          (q_proj): FrozenBNBLinear(4096, 4096)\n",
              "          (out_proj): FrozenBNBLinear(4096, 4096)\n",
              "        )\n",
              "        (mlp): GPTJMLP(\n",
              "          (fc_in): FrozenBNBLinear(4096, 16384)\n",
              "          (fc_out): FrozenBNBLinear(16384, 4096)\n",
              "          (dropout): Dropout(p=0.0, inplace=False)\n",
              "        )\n",
              "      )\n",
              "      (20): GPTJBlock(\n",
              "        (ln_1): LayerNorm((4096,), eps=1e-05, elementwise_affine=True)\n",
              "        (attn): GPTJAttention(\n",
              "          (attn_dropout): Dropout(p=0.0, inplace=False)\n",
              "          (resid_dropout): Dropout(p=0.0, inplace=False)\n",
              "          (k_proj): FrozenBNBLinear(4096, 4096)\n",
              "          (v_proj): FrozenBNBLinear(4096, 4096)\n",
              "          (q_proj): FrozenBNBLinear(4096, 4096)\n",
              "          (out_proj): FrozenBNBLinear(4096, 4096)\n",
              "        )\n",
              "        (mlp): GPTJMLP(\n",
              "          (fc_in): FrozenBNBLinear(4096, 16384)\n",
              "          (fc_out): FrozenBNBLinear(16384, 4096)\n",
              "          (dropout): Dropout(p=0.0, inplace=False)\n",
              "        )\n",
              "      )\n",
              "      (21): GPTJBlock(\n",
              "        (ln_1): LayerNorm((4096,), eps=1e-05, elementwise_affine=True)\n",
              "        (attn): GPTJAttention(\n",
              "          (attn_dropout): Dropout(p=0.0, inplace=False)\n",
              "          (resid_dropout): Dropout(p=0.0, inplace=False)\n",
              "          (k_proj): FrozenBNBLinear(4096, 4096)\n",
              "          (v_proj): FrozenBNBLinear(4096, 4096)\n",
              "          (q_proj): FrozenBNBLinear(4096, 4096)\n",
              "          (out_proj): FrozenBNBLinear(4096, 4096)\n",
              "        )\n",
              "        (mlp): GPTJMLP(\n",
              "          (fc_in): FrozenBNBLinear(4096, 16384)\n",
              "          (fc_out): FrozenBNBLinear(16384, 4096)\n",
              "          (dropout): Dropout(p=0.0, inplace=False)\n",
              "        )\n",
              "      )\n",
              "      (22): GPTJBlock(\n",
              "        (ln_1): LayerNorm((4096,), eps=1e-05, elementwise_affine=True)\n",
              "        (attn): GPTJAttention(\n",
              "          (attn_dropout): Dropout(p=0.0, inplace=False)\n",
              "          (resid_dropout): Dropout(p=0.0, inplace=False)\n",
              "          (k_proj): FrozenBNBLinear(4096, 4096)\n",
              "          (v_proj): FrozenBNBLinear(4096, 4096)\n",
              "          (q_proj): FrozenBNBLinear(4096, 4096)\n",
              "          (out_proj): FrozenBNBLinear(4096, 4096)\n",
              "        )\n",
              "        (mlp): GPTJMLP(\n",
              "          (fc_in): FrozenBNBLinear(4096, 16384)\n",
              "          (fc_out): FrozenBNBLinear(16384, 4096)\n",
              "          (dropout): Dropout(p=0.0, inplace=False)\n",
              "        )\n",
              "      )\n",
              "      (23): GPTJBlock(\n",
              "        (ln_1): LayerNorm((4096,), eps=1e-05, elementwise_affine=True)\n",
              "        (attn): GPTJAttention(\n",
              "          (attn_dropout): Dropout(p=0.0, inplace=False)\n",
              "          (resid_dropout): Dropout(p=0.0, inplace=False)\n",
              "          (k_proj): FrozenBNBLinear(4096, 4096)\n",
              "          (v_proj): FrozenBNBLinear(4096, 4096)\n",
              "          (q_proj): FrozenBNBLinear(4096, 4096)\n",
              "          (out_proj): FrozenBNBLinear(4096, 4096)\n",
              "        )\n",
              "        (mlp): GPTJMLP(\n",
              "          (fc_in): FrozenBNBLinear(4096, 16384)\n",
              "          (fc_out): FrozenBNBLinear(16384, 4096)\n",
              "          (dropout): Dropout(p=0.0, inplace=False)\n",
              "        )\n",
              "      )\n",
              "      (24): GPTJBlock(\n",
              "        (ln_1): LayerNorm((4096,), eps=1e-05, elementwise_affine=True)\n",
              "        (attn): GPTJAttention(\n",
              "          (attn_dropout): Dropout(p=0.0, inplace=False)\n",
              "          (resid_dropout): Dropout(p=0.0, inplace=False)\n",
              "          (k_proj): FrozenBNBLinear(4096, 4096)\n",
              "          (v_proj): FrozenBNBLinear(4096, 4096)\n",
              "          (q_proj): FrozenBNBLinear(4096, 4096)\n",
              "          (out_proj): FrozenBNBLinear(4096, 4096)\n",
              "        )\n",
              "        (mlp): GPTJMLP(\n",
              "          (fc_in): FrozenBNBLinear(4096, 16384)\n",
              "          (fc_out): FrozenBNBLinear(16384, 4096)\n",
              "          (dropout): Dropout(p=0.0, inplace=False)\n",
              "        )\n",
              "      )\n",
              "      (25): GPTJBlock(\n",
              "        (ln_1): LayerNorm((4096,), eps=1e-05, elementwise_affine=True)\n",
              "        (attn): GPTJAttention(\n",
              "          (attn_dropout): Dropout(p=0.0, inplace=False)\n",
              "          (resid_dropout): Dropout(p=0.0, inplace=False)\n",
              "          (k_proj): FrozenBNBLinear(4096, 4096)\n",
              "          (v_proj): FrozenBNBLinear(4096, 4096)\n",
              "          (q_proj): FrozenBNBLinear(4096, 4096)\n",
              "          (out_proj): FrozenBNBLinear(4096, 4096)\n",
              "        )\n",
              "        (mlp): GPTJMLP(\n",
              "          (fc_in): FrozenBNBLinear(4096, 16384)\n",
              "          (fc_out): FrozenBNBLinear(16384, 4096)\n",
              "          (dropout): Dropout(p=0.0, inplace=False)\n",
              "        )\n",
              "      )\n",
              "      (26): GPTJBlock(\n",
              "        (ln_1): LayerNorm((4096,), eps=1e-05, elementwise_affine=True)\n",
              "        (attn): GPTJAttention(\n",
              "          (attn_dropout): Dropout(p=0.0, inplace=False)\n",
              "          (resid_dropout): Dropout(p=0.0, inplace=False)\n",
              "          (k_proj): FrozenBNBLinear(4096, 4096)\n",
              "          (v_proj): FrozenBNBLinear(4096, 4096)\n",
              "          (q_proj): FrozenBNBLinear(4096, 4096)\n",
              "          (out_proj): FrozenBNBLinear(4096, 4096)\n",
              "        )\n",
              "        (mlp): GPTJMLP(\n",
              "          (fc_in): FrozenBNBLinear(4096, 16384)\n",
              "          (fc_out): FrozenBNBLinear(16384, 4096)\n",
              "          (dropout): Dropout(p=0.0, inplace=False)\n",
              "        )\n",
              "      )\n",
              "      (27): GPTJBlock(\n",
              "        (ln_1): LayerNorm((4096,), eps=1e-05, elementwise_affine=True)\n",
              "        (attn): GPTJAttention(\n",
              "          (attn_dropout): Dropout(p=0.0, inplace=False)\n",
              "          (resid_dropout): Dropout(p=0.0, inplace=False)\n",
              "          (k_proj): FrozenBNBLinear(4096, 4096)\n",
              "          (v_proj): FrozenBNBLinear(4096, 4096)\n",
              "          (q_proj): FrozenBNBLinear(4096, 4096)\n",
              "          (out_proj): FrozenBNBLinear(4096, 4096)\n",
              "        )\n",
              "        (mlp): GPTJMLP(\n",
              "          (fc_in): FrozenBNBLinear(4096, 16384)\n",
              "          (fc_out): FrozenBNBLinear(16384, 4096)\n",
              "          (dropout): Dropout(p=0.0, inplace=False)\n",
              "        )\n",
              "      )\n",
              "    )\n",
              "    (ln_f): LayerNorm((4096,), eps=1e-05, elementwise_affine=True)\n",
              "  )\n",
              "  (lm_head): FrozenBNBLinear(4096, 50400)\n",
              ")"
            ]
          },
          "execution_count": 25,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "def add_adapters(model, adapter_dim=4, p = 0.1):\n",
        "    assert adapter_dim > 0\n",
        "\n",
        "    for name, module in model.named_modules():\n",
        "      if isinstance(module, FrozenBNBLinear):\n",
        "          if \"attn\" in name or \"mlp\" in name or \"head\" in name:\n",
        "              print(\"Adding adapter to\", name)\n",
        "              module.adapter = nn.Sequential(\n",
        "                nn.Linear(module.in_features, adapter_dim, bias=False),\n",
        "                nn.Dropout(p=p),\n",
        "                nn.Linear(adapter_dim, module.out_features, bias=False),\n",
        "            )\n",
        "              print(\"Initializing\", name)\n",
        "              nn.init.zeros_(module.adapter[2].weight)\n",
        "\n",
        "          else:\n",
        "              print(\"Not adding adapter to\", name)\n",
        "      elif isinstance(module, FrozenBNBEmbedding):\n",
        "          print(\"Adding adapter to\", name)\n",
        "          module.adapter = nn.Sequential(\n",
        "                nn.Embedding(module.num_embeddings, adapter_dim),\n",
        "                nn.Dropout(p=p),\n",
        "                nn.Linear(adapter_dim, module.embedding_dim, bias=False),\n",
        "            )\n",
        "          print(\"Initializing\", name)\n",
        "          nn.init.zeros_(module.adapter[2].weight)\n",
        "\n",
        "add_adapters(gpt)\n",
        "device = 'cuda' if torch.cuda.is_available() else 'cpu'\n",
        "gpt.to(device)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "IOnrE2C330UL"
      },
      "outputs": [],
      "source": [
        "from bitsandbytes.optim import Adam8bit\n",
        "\n",
        "gpt.gradient_checkpointing_enable()\n",
        "optimizer = Adam8bit(gpt.parameters(), lr=1e-5, weight_decay=0.01)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "AvD2RYYK32qH"
      },
      "outputs": [],
      "source": [
        "num_epochs = 5\n",
        "num_training_steps = num_epochs * len(train_dataloader)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "wM_29EAi37rr"
      },
      "outputs": [],
      "source": [
        "lr_scheduler = transformers.get_linear_schedule_with_warmup(\n",
        "    optimizer, int(num_training_steps*0.1), num_training_steps\n",
        ")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "M9ZH7tEb4EOB"
      },
      "outputs": [],
      "source": [
        "filepath = '/content/model.pt'"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000,
          "referenced_widgets": [
            "50eeab942b3943c4af0c29263af272da",
            "c1bc13e425eb432a8d44dd1a491488da",
            "5b9d0088b931449795c449ba1405ec49",
            "7aeaaeaea7ba49f2a6f057cbd65eaf0a",
            "527055094fb947e98b80d0c7959aaeba",
            "81daa471678345faacf9545fc7563c2f",
            "c4bc09c4171a40fab4d7c126e8b8253c",
            "b5e59f166aef449ca58ed64ae704677d",
            "e5def2dca34b4e3d8d8fa4d50416d11d",
            "4f6ece28ab6349b6803b2b6c47296b99",
            "d2d7aa2f37384aa48f9cbc6b0acb3fe6"
          ]
        },
        "id": "JMJltmES4FdM",
        "outputId": "35853eed-1318-4bae-84b6-a77753cac9c6"
      },
      "outputs": [
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "50eeab942b3943c4af0c29263af272da",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "  0%|          | 0/195 [00:00<?, ?it/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(11.3158, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/torch/optim/lr_scheduler.py:134: UserWarning: Detected call of `lr_scheduler.step()` before `optimizer.step()`. In PyTorch 1.1.0 and later, you should call them in the opposite order: `optimizer.step()` before `lr_scheduler.step()`.  Failure to do this will result in PyTorch skipping the first value of the learning rate schedule. See more details at https://pytorch.org/docs/stable/optim.html#how-to-adjust-learning-rate\n",
            "  \"https://pytorch.org/docs/stable/optim.html#how-to-adjust-learning-rate\", UserWarning)\n",
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(10.0310, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(10.9427, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(10.9920, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(11.1014, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(9.9951, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(10.7877, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(11.3448, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(10.4773, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(8.8882, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(11.4900, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(11.0338, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(11.5174, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(10.6410, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(11.1510, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(11.3515, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(10.4751, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(10.9604, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(10.9595, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(10.3635, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(10.4949, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(11.0150, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(11.0111, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(10.8622, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(10.7936, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(10.4939, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(10.9784, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(10.4860, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(10.5992, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(10.3410, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(9.7744, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(9.3374, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(10.3144, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(10.7441, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(10.0762, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(10.1815, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(9.2738, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(10.1250, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(10.6105, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(10.6222, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(9.4779, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(10.2384, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(10.2204, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(10.3101, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(9.3576, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(10.0275, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(10.4889, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(9.6929, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(8.3570, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(10.5123, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(10.1047, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(10.4925, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(9.7831, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(10.1707, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(10.2951, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(9.6223, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(9.9537, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(9.9524, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(9.4809, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(9.5489, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(9.9023, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(9.9835, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(9.8256, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(9.7442, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(9.5117, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(9.8770, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(9.4707, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(9.5515, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(9.3676, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(8.8626, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(8.4984, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(9.2670, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(9.5621, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(9.1319, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(9.1902, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(8.4538, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(9.1322, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(9.5908, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(9.4786, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(8.6148, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(9.1710, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(9.1320, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(9.2043, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(8.4960, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(9.0266, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(9.3248, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(8.6965, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(7.6818, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(9.2881, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(8.9789, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(9.2571, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(8.7350, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(8.9947, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(9.0476, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(8.6175, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(8.8283, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(8.7677, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(8.4410, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(8.4898, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(8.6723, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(8.7815, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(8.6587, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(8.5320, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(8.4079, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(8.5498, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(8.3106, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(8.3125, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(8.1796, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(7.8200, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(7.5151, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(8.0117, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(8.2077, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(7.9697, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(8.0071, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(7.3582, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(7.9596, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(8.2137, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(8.0392, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(7.5362, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(7.8480, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(7.7825, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(7.7925, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(7.4155, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(7.6968, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(7.8626, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(7.3979, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(6.8212, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(7.6896, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(7.5636, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(7.6426, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(7.3509, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(7.4555, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(7.4522, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(7.3055, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(7.3439, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(7.3076, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(7.1115, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(7.1322, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(7.1082, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(7.2789, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(7.2003, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(7.0913, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(7.0467, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(7.0537, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(6.8592, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(6.9009, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(6.8462, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(6.6570, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(6.4811, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(6.6868, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(6.7951, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(6.7576, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(6.7866, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(6.3516, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(6.7252, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(6.9031, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(6.7398, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(6.4902, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(6.6751, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(6.5614, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(6.5979, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(6.4734, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(6.6134, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(6.7155, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(6.4326, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(6.1221, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(6.5527, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(6.4954, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(6.5715, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(6.4055, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(6.4457, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(6.4432, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(6.4396, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(6.4399, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(6.4193, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(6.2522, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(6.3099, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(6.2671, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(6.4735, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(6.4041, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(6.3435, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(6.3509, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(6.3037, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(6.1492, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(6.2199, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(6.2416, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(6.1641, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(6.0116, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(6.1526, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(6.1898, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(6.2912, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(6.2786, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(5.9527, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(6.2827, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(6.4700, device='cuda:0', grad_fn=<AddBackward0>)\n"
          ]
        }
      ],
      "source": [
        "from tqdm.auto import tqdm\n",
        "\n",
        "scaler = torch.cuda.amp.GradScaler()\n",
        "progress_bar = tqdm(range(num_training_steps))\n",
        "gpt.train()\n",
        "gpt.gradient_checkpointing_enable()\n",
        "k = 0\n",
        "\n",
        "for epoch in range(num_epochs):\n",
        "    for batch in train_dataloader:\n",
        "\n",
        "        k = k + 1\n",
        "        if k % 500 == 0:\n",
        "          print(k)\n",
        "          state = {'k' : k, 'epoch': num_epochs, 'lr_scheduler': lr_scheduler.state_dict(), 'state_dict': gpt.state_dict(), 'optimizer': optimizer.state_dict()}\n",
        "          torch.save(state, filepath)\n",
        "\n",
        "        batch = {k: v.to(device) for k, v in batch.items()}\n",
        "\n",
        "        optimizer.zero_grad()\n",
        "\n",
        "        with torch.cuda.amp.autocast():\n",
        "          out = gpt.forward(**batch,)\n",
        "\n",
        "          loss = F.cross_entropy(out.logits[:, :-1, :].flatten(0, -2), batch['input_ids'][:, 1:].flatten(),\n",
        "                                reduction='mean', label_smoothing=0.1)\n",
        "          \n",
        "        print(loss)\n",
        "\n",
        "        scaler.scale(loss).backward()\n",
        "        scaler.unscale_(optimizer)\n",
        "        torch.nn.utils.clip_grad_norm_(gpt.parameters(), 1.0)\n",
        "        scaler.step(optimizer)\n",
        "        scaler.update()\n",
        "\n",
        "        lr_scheduler.step()\n",
        "        progress_bar.update(1)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "e3U67wWMIgK7"
      },
      "source": [
        "## Text generation example"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "47MiZQC3UOtp"
      },
      "outputs": [],
      "source": [
        "gpt.eval()\n",
        "with torch.no_grad():\n",
        "  prompt = tokenizer(\"Quote:\", truncation=True, padding=True, max_length=128, return_tensors='pt')\n",
        "  prompt = {key: value.to(device) for key, value in prompt.items()}\n",
        "  out = gpt.generate(**prompt, max_length=128, top_k=50, top_p=0.9, temperature=1.0, do_sample=True, repetition_penalty = 1.2, num_beams=1)\n",
        "  print(tokenizer.decode(out[0]))"
      ]
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "collapsed_sections": [],
      "machine_shape": "hm",
      "name": "finetune-8bit-models.ipynb",
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "036adfe8ee224a55a0889737fbe48ff4": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_6c8f1d95149d4f6bb0330258f6af0ea4",
            "placeholder": "​",
            "style": "IPY_MODEL_23accb572c764ac7b18aaf1af6a022b9",
            "value": " 3.94k/3.94k [00:00&lt;00:00, 136kB/s]"
          }
        },
        "03db147f01564aa296358bc650be3916": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "0463e1b8507441b79e996f73c68a40d6": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_7823b6ffcef94222be1cdca2c661ed04",
            "placeholder": "​",
            "style": "IPY_MODEL_42bb16d7389d43a9b5506130fe4e09d7",
            "value": "Downloading: 100%"
          }
        },
        "0677c559c1ec45bf8c83bdfdf0438504": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "076ec84c6fd04b79adc38de649165c5f": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "0b7935123c604d5286eb9006a4b32a85": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "0c63469be3a142f792f69b14984e7747": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "11a14e14654949a6a173916989bd43c1": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_ce901c41bdda40bab37c224822f022b0",
            "placeholder": "​",
            "style": "IPY_MODEL_ddbe5279fc04449188695375d7f33d52",
            "value": " 1.31M/1.31M [00:00&lt;00:00, 3.88MB/s]"
          }
        },
        "14557167cd4b4c2c9c2ea4bfe6b2cd6a": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_d255304c7adc4efab329d9e968339bd7",
            "max": 2,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_883c0e7ddbe646f09cb420bcc5540573",
            "value": 2
          }
        },
        "153620a44c88414a8e610c6af1de0267": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_400069c8a6ef476a9332d0ea9b9ef20f",
            "max": 4039,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_c573a8b0fb5b4e87be558da9d3be2fdf",
            "value": 4039
          }
        },
        "1656a97c9fea490dafcaf7ef6113c59f": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "203cb2d94dd54e719843eae2ba95ad84": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "211011ebc2114548ab7c1800d0a031f8": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "2248e00287784cf1b86ed8715237db44": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_97701990ecc845edb80abcdffe1c5132",
            "placeholder": "​",
            "style": "IPY_MODEL_5a4782ab72c54304a8e33e9425ef11fa",
            "value": "Downloading: 100%"
          }
        },
        "23accb572c764ac7b18aaf1af6a022b9": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "275766ea5f4c4f45981ca38bc3d30468": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "276ca8ce6863488ba4f83e5c6921638f": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "27b71deb79e74b5d9175bdccfd5f67fc": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_9ad3cd1329c04798b9428ffdfcc35979",
              "IPY_MODEL_e0a151d88f54451c8b52f7af42879f13",
              "IPY_MODEL_37cc766b349043c9b92e6147900eb84b"
            ],
            "layout": "IPY_MODEL_d94ae41c4364487aa6df495820b3cc5d"
          }
        },
        "27f7656a89d64696af799430fb651c01": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_439c874ddcde4ad2b00376c5d0288bc2",
            "placeholder": "​",
            "style": "IPY_MODEL_913bc1e8cef440f7871acc587dcdc48f",
            "value": "100%"
          }
        },
        "2d1d42edbe264d6198c59cac8ecb7895": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_3291292ee663463ca940ffe5c4ea4ae5",
              "IPY_MODEL_69b48c289d13413f80ee55cbad95573e",
              "IPY_MODEL_fe85185c1e59439f9a5d85b36fcd2ef9"
            ],
            "layout": "IPY_MODEL_55d0e59062594eabb7f24dab156e2a94"
          }
        },
        "2fe206d674494389b1c75b0b1b95d2b3": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "305666fe63b4480ca886654e3e953264": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_fdb865c784844112b89ce813f0bef9e0",
            "max": 826,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_645cdc9531c344e7a941b0ec7721e336",
            "value": 826
          }
        },
        "327bbc6ac0ee425e92d0620e88269c7d": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_38a54faf890c4d61bbbb118f34303619",
            "placeholder": "​",
            "style": "IPY_MODEL_50edafbb02154cf78a17a6df6af7093e",
            "value": "100%"
          }
        },
        "3291292ee663463ca940ffe5c4ea4ae5": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_d013898ae9cf4a7ca007d281cbd7d28d",
            "placeholder": "​",
            "style": "IPY_MODEL_ba0d9af23cdd4885a9cab57472221fa2",
            "value": "Downloading: 100%"
          }
        },
        "3515b382d154416e98bdee3444e34179": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "37cc766b349043c9b92e6147900eb84b": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_95c5cab8e0ba4b268192a4dc96b1a5d2",
            "placeholder": "​",
            "style": "IPY_MODEL_b53033e55c504bc99cf0705cbf2699a9",
            "value": " 2/2 [00:00&lt;00:00, 54.28it/s]"
          }
        },
        "3800f430b1f94524b30eb7ffbc49ef91": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "38a54faf890c4d61bbbb118f34303619": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "3907b4fe14cd40d9b03216a3ed10f386": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "3a24869dd7db4e409613a25a99b2e84f": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "3bcf6f6e2cbe43048b17754c7569b830": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "3e929c00f3f1421cae759e562581c743": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_03db147f01564aa296358bc650be3916",
            "max": 456356,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_bc2c477786ad4528b8d16454040d3345",
            "value": 456356
          }
        },
        "400069c8a6ef476a9332d0ea9b9ef20f": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "418c131375324b88b70bb93aca6461db": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "42bb16d7389d43a9b5506130fe4e09d7": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "439c874ddcde4ad2b00376c5d0288bc2": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "4617062fc7274559b75515adf245e50b": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "4c4afa08ac034ca59f47ed27c452562b": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "4d313423a8b746d2bd9deab3d10237b5": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "4f6ece28ab6349b6803b2b6c47296b99": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "4f915b1dfb3643649e0d31ac00d43fb7": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_2248e00287784cf1b86ed8715237db44",
              "IPY_MODEL_edc42778a7824d89b1bbd36ac9da256f",
              "IPY_MODEL_7045f8e1ea5e4725aa002c61063117dd"
            ],
            "layout": "IPY_MODEL_3a24869dd7db4e409613a25a99b2e84f"
          }
        },
        "4fc52cddcd95477c89d9065b436c708f": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_90fdffc266874194902704a8861b30fc",
            "max": 1,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_b8c49ff14c7349f6938e0d4174b71c6d",
            "value": 1
          }
        },
        "4ff6004f577449f2a13ea7ba82fcb7e1": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "50edafbb02154cf78a17a6df6af7093e": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "50eeab942b3943c4af0c29263af272da": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_5b9d0088b931449795c449ba1405ec49",
              "IPY_MODEL_7aeaaeaea7ba49f2a6f057cbd65eaf0a",
              "IPY_MODEL_527055094fb947e98b80d0c7959aaeba"
            ],
            "layout": "IPY_MODEL_c1bc13e425eb432a8d44dd1a491488da"
          }
        },
        "525ad55300cc449691535b5469ec0c22": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f1f09aab70884cf3b6a7dbc8c3dee490",
            "placeholder": "​",
            "style": "IPY_MODEL_3800f430b1f94524b30eb7ffbc49ef91",
            "value": "Downloading: 100%"
          }
        },
        "527055094fb947e98b80d0c7959aaeba": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_d2d7aa2f37384aa48f9cbc6b0acb3fe6",
            "placeholder": "​",
            "style": "IPY_MODEL_4f6ece28ab6349b6803b2b6c47296b99",
            "value": " 195/195 [20:46&lt;00:00,  5.05s/it]"
          }
        },
        "541d272569e34fa893e1f1192621b585": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_a2aab4900a1d4035a4e9a32f438b4b60",
              "IPY_MODEL_8f8ada3ce83340baa656583f7b7e2800",
              "IPY_MODEL_929515722de84dfeb333a94185c09d69"
            ],
            "layout": "IPY_MODEL_da9dbb3284524884b0e4f9298c665297"
          }
        },
        "54234e3b446e4f4b865f76b9fdafce01": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "55d0e59062594eabb7f24dab156e2a94": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "5a4782ab72c54304a8e33e9425ef11fa": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "5b9d0088b931449795c449ba1405ec49": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_c4bc09c4171a40fab4d7c126e8b8253c",
            "placeholder": "​",
            "style": "IPY_MODEL_81daa471678345faacf9545fc7563c2f",
            "value": "100%"
          }
        },
        "5da61c30812b4185a022c4647e241e9a": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "5fbb1f1b1ea14c10b4c74ea90f125c5a": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "645cdc9531c344e7a941b0ec7721e336": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "67bc335a6d2f4672a0f66f6c05974e39": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "69b48c289d13413f80ee55cbad95573e": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_4c4afa08ac034ca59f47ed27c452562b",
            "max": 357,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_c65f8696241f426bb295bda31d4ad667",
            "value": 357
          }
        },
        "6b0b00ed82e9470c827450e70f8ad180": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_d898fb31b0c74a5a9dd7396ae2d15db6",
            "placeholder": "​",
            "style": "IPY_MODEL_9f6d069fd43e4016ac7bcb1696c3b768",
            "value": "100%"
          }
        },
        "6c2e8a8f07ab45f1a025d460355fb367": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "6c8f1d95149d4f6bb0330258f6af0ea4": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "6c9bfc504cfe4cdaa347a40bf730070a": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "6f1e1e5e5ce9420399904b7c34bb0c09": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "6fd25c7d836d4f88abfd352b0995ce95": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "7045f8e1ea5e4725aa002c61063117dd": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_ee2a498700c247cd80208d8b0910d058",
            "placeholder": "​",
            "style": "IPY_MODEL_2fe206d674494389b1c75b0b1b95d2b3",
            "value": " 619/619 [00:00&lt;00:00, 20.9kB/s]"
          }
        },
        "706bdb7d967947ddbe94febcb4a8ae6f": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "71b10878e239477a9f314372621ad021": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "73cfe65490cc492f8f61cb88a9e084f3": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "7823b6ffcef94222be1cdca2c661ed04": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "7a01312bac844ec89f401b2011ea6b4f": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "7aeaaeaea7ba49f2a6f057cbd65eaf0a": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_e5def2dca34b4e3d8d8fa4d50416d11d",
            "max": 195,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_b5e59f166aef449ca58ed64ae704677d",
            "value": 195
          }
        },
        "7e1fc6a38569457cb717db7b41aebd52": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "80d90e26a7f546739f139152fb291483": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "81daa471678345faacf9545fc7563c2f": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "838f070e33af470fb753fe130aa19f96": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "883c0e7ddbe646f09cb420bcc5540573": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "8b1515ae411e48f88e75dbde1bc4b0f1": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "8b243dfd4ed24b699d674900f45ad195": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_525ad55300cc449691535b5469ec0c22",
              "IPY_MODEL_305666fe63b4480ca886654e3e953264",
              "IPY_MODEL_b0d135b0a1cd49f4bae49e128b37803c"
            ],
            "layout": "IPY_MODEL_a867a214610e450cbcfd59da53052edc"
          }
        },
        "8f597e5db3464769af6eed68f367f337": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "8f8ada3ce83340baa656583f7b7e2800": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_706bdb7d967947ddbe94febcb4a8ae6f",
            "max": 798156,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_a99bea0476c7470b90192bf7838ebf6a",
            "value": 798156
          }
        },
        "8ff0230a260c4a15a54b70f9b7b25884": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_9fbe0e0845be4b2ba73bf01c40f7b157",
              "IPY_MODEL_3e929c00f3f1421cae759e562581c743",
              "IPY_MODEL_a4eeb1a8a4874b31917e04fe18db5ecb"
            ],
            "layout": "IPY_MODEL_275766ea5f4c4f45981ca38bc3d30468"
          }
        },
        "90fdffc266874194902704a8861b30fc": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "913bc1e8cef440f7871acc587dcdc48f": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "929515722de84dfeb333a94185c09d69": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_e80f1246b4264ec28e748d60e638821d",
            "placeholder": "​",
            "style": "IPY_MODEL_0c63469be3a142f792f69b14984e7747",
            "value": " 779k/779k [00:00&lt;00:00, 921kB/s]"
          }
        },
        "95c5cab8e0ba4b268192a4dc96b1a5d2": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "96193485918d42918bcb991becd7f3c2": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "9650c796c899416b9da0267704ee9ddf": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_327bbc6ac0ee425e92d0620e88269c7d",
              "IPY_MODEL_e1c6e90443e745e18b357faf9f4d0643",
              "IPY_MODEL_9d369eb7add14f2d9a63f539ae0dd4fb"
            ],
            "layout": "IPY_MODEL_ac1d31cb2ca5441b9b5f67a28fb844ed"
          }
        },
        "97701990ecc845edb80abcdffe1c5132": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "9913cafea07e459ca3f37eb672c9482b": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "9ad3cd1329c04798b9428ffdfcc35979": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_ed97ac95d1d54ec6aa82aee05a0e24f2",
            "placeholder": "​",
            "style": "IPY_MODEL_4617062fc7274559b75515adf245e50b",
            "value": "100%"
          }
        },
        "9d369eb7add14f2d9a63f539ae0dd4fb": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_c06e95c35361484b9bcbccba1542b733",
            "placeholder": "​",
            "style": "IPY_MODEL_8f597e5db3464769af6eed68f367f337",
            "value": " 1/1 [00:00&lt;00:00,  9.26ba/s]"
          }
        },
        "9f6d069fd43e4016ac7bcb1696c3b768": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "9fbe0e0845be4b2ba73bf01c40f7b157": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_54234e3b446e4f4b865f76b9fdafce01",
            "placeholder": "​",
            "style": "IPY_MODEL_a25b27da7e664c7eb289e6735b5459dd",
            "value": "Downloading: 100%"
          }
        },
        "a25b27da7e664c7eb289e6735b5459dd": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "a2aab4900a1d4035a4e9a32f438b4b60": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_6fd25c7d836d4f88abfd352b0995ce95",
            "placeholder": "​",
            "style": "IPY_MODEL_8b1515ae411e48f88e75dbde1bc4b0f1",
            "value": "Downloading: 100%"
          }
        },
        "a4eeb1a8a4874b31917e04fe18db5ecb": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_71b10878e239477a9f314372621ad021",
            "placeholder": "​",
            "style": "IPY_MODEL_4d313423a8b746d2bd9deab3d10237b5",
            "value": " 446k/446k [00:00&lt;00:00, 765kB/s]"
          }
        },
        "a867a214610e450cbcfd59da53052edc": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a99bea0476c7470b90192bf7838ebf6a": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "ac1d31cb2ca5441b9b5f67a28fb844ed": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ae3127a7feea4e7eb84298ffb057ace2": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_3515b382d154416e98bdee3444e34179",
            "max": 1373465,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_6f1e1e5e5ce9420399904b7c34bb0c09",
            "value": 1373465
          }
        },
        "b0d135b0a1cd49f4bae49e128b37803c": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_838f070e33af470fb753fe130aa19f96",
            "placeholder": "​",
            "style": "IPY_MODEL_3907b4fe14cd40d9b03216a3ed10f386",
            "value": " 826/826 [00:00&lt;00:00, 25.7kB/s]"
          }
        },
        "b3723a3e712849899e2242a39c517648": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_0463e1b8507441b79e996f73c68a40d6",
              "IPY_MODEL_153620a44c88414a8e610c6af1de0267",
              "IPY_MODEL_036adfe8ee224a55a0889737fbe48ff4"
            ],
            "layout": "IPY_MODEL_7e1fc6a38569457cb717db7b41aebd52"
          }
        },
        "b53033e55c504bc99cf0705cbf2699a9": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "b5e59f166aef449ca58ed64ae704677d": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "b639299a491f429ba70c2f959194fba5": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_c4833007b9284e18a84f648e3b36aa05",
            "placeholder": "​",
            "style": "IPY_MODEL_418c131375324b88b70bb93aca6461db",
            "value": "Downloading: 100%"
          }
        },
        "b8c49ff14c7349f6938e0d4174b71c6d": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "b9a0a9b919e14c51a844df7b0550da5b": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_b639299a491f429ba70c2f959194fba5",
              "IPY_MODEL_ae3127a7feea4e7eb84298ffb057ace2",
              "IPY_MODEL_11a14e14654949a6a173916989bd43c1"
            ],
            "layout": "IPY_MODEL_67bc335a6d2f4672a0f66f6c05974e39"
          }
        },
        "ba0d9af23cdd4885a9cab57472221fa2": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "bc2c477786ad4528b8d16454040d3345": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "c06e95c35361484b9bcbccba1542b733": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c082443ce61d4b158576855946b0f649": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_6b0b00ed82e9470c827450e70f8ad180",
              "IPY_MODEL_cebbbf7f40d94b3a9de105b6781f4a0e",
              "IPY_MODEL_d496fc35dd894bf3b8f4f41dd1052d5a"
            ],
            "layout": "IPY_MODEL_276ca8ce6863488ba4f83e5c6921638f"
          }
        },
        "c1bc13e425eb432a8d44dd1a491488da": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c4833007b9284e18a84f648e3b36aa05": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c4bc09c4171a40fab4d7c126e8b8253c": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c573a8b0fb5b4e87be558da9d3be2fdf": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "c65f8696241f426bb295bda31d4ad667": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "ce901c41bdda40bab37c224822f022b0": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "cebbbf7f40d94b3a9de105b6781f4a0e": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_6c2e8a8f07ab45f1a025d460355fb367",
            "max": 2,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_1656a97c9fea490dafcaf7ef6113c59f",
            "value": 2
          }
        },
        "d013898ae9cf4a7ca007d281cbd7d28d": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "d255304c7adc4efab329d9e968339bd7": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "d2d7aa2f37384aa48f9cbc6b0acb3fe6": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "d49229a761204b52b0d4f792a1e7eb4e": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_3bcf6f6e2cbe43048b17754c7569b830",
            "placeholder": "​",
            "style": "IPY_MODEL_203cb2d94dd54e719843eae2ba95ad84",
            "value": " 1/1 [00:00&lt;00:00, 16.08ba/s]"
          }
        },
        "d496fc35dd894bf3b8f4f41dd1052d5a": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f843c9bf912c48b7a9899f1e8707627c",
            "placeholder": "​",
            "style": "IPY_MODEL_9913cafea07e459ca3f37eb672c9482b",
            "value": " 2/2 [00:00&lt;00:00, 39.04it/s]"
          }
        },
        "d80a21b53ed643b8986ae21a4a32e6a0": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_27f7656a89d64696af799430fb651c01",
              "IPY_MODEL_14557167cd4b4c2c9c2ea4bfe6b2cd6a",
              "IPY_MODEL_f720e92372104a548d2deca0ea94bddf"
            ],
            "layout": "IPY_MODEL_076ec84c6fd04b79adc38de649165c5f"
          }
        },
        "d898fb31b0c74a5a9dd7396ae2d15db6": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "d94ae41c4364487aa6df495820b3cc5d": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "da9dbb3284524884b0e4f9298c665297": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ddbe5279fc04449188695375d7f33d52": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "e0a151d88f54451c8b52f7af42879f13": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_7a01312bac844ec89f401b2011ea6b4f",
            "max": 2,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_4ff6004f577449f2a13ea7ba82fcb7e1",
            "value": 2
          }
        },
        "e1c6e90443e745e18b357faf9f4d0643": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_ec03dc571ef041e7b4caae39bf560967",
            "max": 1,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_0677c559c1ec45bf8c83bdfdf0438504",
            "value": 1
          }
        },
        "e5def2dca34b4e3d8d8fa4d50416d11d": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e80f1246b4264ec28e748d60e638821d": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ec03dc571ef041e7b4caae39bf560967": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ed61ae1802d64ee5a7b59e8ece23f470": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ed6b385575eb49b38d4911db33367f3f": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_80d90e26a7f546739f139152fb291483",
            "placeholder": "​",
            "style": "IPY_MODEL_5da61c30812b4185a022c4647e241e9a",
            "value": "100%"
          }
        },
        "ed97ac95d1d54ec6aa82aee05a0e24f2": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "edc42778a7824d89b1bbd36ac9da256f": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_96193485918d42918bcb991becd7f3c2",
            "max": 619,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_73cfe65490cc492f8f61cb88a9e084f3",
            "value": 619
          }
        },
        "ee2a498700c247cd80208d8b0910d058": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f1f09aab70884cf3b6a7dbc8c3dee490": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f720e92372104a548d2deca0ea94bddf": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_0b7935123c604d5286eb9006a4b32a85",
            "placeholder": "​",
            "style": "IPY_MODEL_5fbb1f1b1ea14c10b4c74ea90f125c5a",
            "value": " 2/2 [00:00&lt;00:00, 64.64it/s]"
          }
        },
        "f74311b977e747b8b8138998c75ca8cc": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_ed6b385575eb49b38d4911db33367f3f",
              "IPY_MODEL_4fc52cddcd95477c89d9065b436c708f",
              "IPY_MODEL_d49229a761204b52b0d4f792a1e7eb4e"
            ],
            "layout": "IPY_MODEL_211011ebc2114548ab7c1800d0a031f8"
          }
        },
        "f843c9bf912c48b7a9899f1e8707627c": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "fdb865c784844112b89ce813f0bef9e0": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "fe85185c1e59439f9a5d85b36fcd2ef9": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_ed61ae1802d64ee5a7b59e8ece23f470",
            "placeholder": "​",
            "style": "IPY_MODEL_6c9bfc504cfe4cdaa347a40bf730070a",
            "value": " 357/357 [00:00&lt;00:00, 11.5kB/s]"
          }
        }
      }
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}