EfficientNet.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "executionInfo": {
     "elapsed": 4434,
     "status": "ok",
     "timestamp": 1607146642164,
     "user": {
      "displayName": "Aakash N S",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GiIWFHtan62vtW1gz2Bv2bxL3rppefcadxzEVxRKQ=s64",
      "userId": "03254185060287524023"
     },
     "user_tz": -330
    },
    "id": "WapU423CpeT5"
   },
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "from math import ceil"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "device = \"cuda\" if torch.cuda.is_available() else \"cpu\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "base_model = [\n",
    "    # expand_ratio, channels, repeats, stride, kernel_size\n",
    "    [1, 16, 1, 1, 3],\n",
    "    [6, 24, 2, 2, 3],\n",
    "    [6, 40, 2, 2, 5],\n",
    "    [6, 80, 3, 2, 3],\n",
    "    [6, 112, 3, 1, 5],\n",
    "    [6, 192, 4, 2, 5],\n",
    "    [6, 320, 1, 1, 3],\n",
    "]\n",
    "\n",
    "phi_values = {\n",
    "    # tuple of: (phi_value, resolution, drop_rate)\n",
    "    \"b0\": (0, 224, 0.2),  # alpha, beta, gamma, depth = alpha ** phi\n",
    "    \"b1\": (0.5, 240, 0.2),\n",
    "    \"b2\": (1, 260, 0.3),\n",
    "    \"b3\": (2, 300, 0.3),\n",
    "    \"b4\": (3, 380, 0.4),\n",
    "    \"b5\": (4, 456, 0.4),\n",
    "    \"b6\": (5, 528, 0.5),\n",
    "    \"b7\": (6, 600, 0.5),\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "class CNNBlock(nn.Module):\n",
    "    def __init__(\n",
    "            self, in_channels, out_channels, kernel_size, stride, padding, groups=1\n",
    "    ):\n",
    "        super(CNNBlock, self).__init__()\n",
    "        self.cnn = nn.Conv2d(\n",
    "            in_channels,\n",
    "            out_channels,\n",
    "            kernel_size,\n",
    "            stride,\n",
    "            padding,\n",
    "            groups=groups,\n",
    "            bias=False,\n",
    "        )\n",
    "        self.bn = nn.BatchNorm2d(out_channels)\n",
    "        self.silu = nn.SiLU() # SiLU <-> Swish\n",
    "\n",
    "    def forward(self, x):\n",
    "        return self.silu(self.bn(self.cnn(x)))\n",
    "\n",
    "class SqueezeExcitation(nn.Module):\n",
    "    def __init__(self, in_channels, reduced_dim):\n",
    "        super(SqueezeExcitation, self).__init__()\n",
    "        self.se = nn.Sequential(\n",
    "            nn.AdaptiveAvgPool2d(1), # C x H x W -> C x 1 x 1\n",
    "            nn.Conv2d(in_channels, reduced_dim, 1),\n",
    "            nn.SiLU(),\n",
    "            nn.Conv2d(reduced_dim, in_channels, 1),\n",
    "            nn.Sigmoid(),\n",
    "        )\n",
    "\n",
    "    def forward(self, x):\n",
    "        return x * self.se(x)\n",
    "\n",
    "class InvertedResidualBlock(nn.Module):\n",
    "    def __init__(\n",
    "            self,\n",
    "            in_channels,\n",
    "            out_channels,\n",
    "            kernel_size,\n",
    "            stride,\n",
    "            padding,\n",
    "            expand_ratio,\n",
    "            reduction=4, # squeeze excitation\n",
    "            survival_prob=0.8, # for stochastic depth\n",
    "    ):\n",
    "        super(InvertedResidualBlock, self).__init__()\n",
    "        self.survival_prob = 0.8\n",
    "        self.use_residual = in_channels == out_channels and stride == 1\n",
    "        hidden_dim = in_channels * expand_ratio\n",
    "        self.expand = in_channels != hidden_dim\n",
    "        reduced_dim = int(in_channels / reduction)\n",
    "\n",
    "        if self.expand:\n",
    "            self.expand_conv = CNNBlock(\n",
    "                in_channels, hidden_dim, kernel_size=3, stride=1, padding=1,\n",
    "            )\n",
    "\n",
    "        self.conv = nn.Sequential(\n",
    "            CNNBlock(\n",
    "                hidden_dim, hidden_dim, kernel_size, stride, padding, groups=hidden_dim,\n",
    "            ),\n",
    "            SqueezeExcitation(hidden_dim, reduced_dim),\n",
    "            nn.Conv2d(hidden_dim, out_channels, 1, bias=False),\n",
    "            nn.BatchNorm2d(out_channels),\n",
    "        )\n",
    "\n",
    "    def stochastic_depth(self, x):\n",
    "        if not self.training:\n",
    "            return x\n",
    "\n",
    "        binary_tensor = torch.rand(x.shape[0], 1, 1, 1, device=x.device) < self.survival_prob\n",
    "        return torch.div(x, self.survival_prob) * binary_tensor\n",
    "\n",
    "    def forward(self, inputs):\n",
    "        x = self.expand_conv(inputs) if self.expand else inputs\n",
    "\n",
    "        if self.use_residual:\n",
    "            return self.stochastic_depth(self.conv(x)) + inputs\n",
    "        else:\n",
    "            return self.conv(x)\n",
    "\n",
    "\n",
    "class EfficientNet(nn.Module):\n",
    "    def __init__(self, version, num_classes):\n",
    "        super(EfficientNet, self).__init__()\n",
    "        width_factor, depth_factor, dropout_rate = self.calculate_factors(version)\n",
    "        last_channels = ceil(1280 * width_factor)\n",
    "        self.pool = nn.AdaptiveAvgPool2d(1)\n",
    "        self.features = self.create_features(width_factor, depth_factor, last_channels)\n",
    "        self.classifier = nn.Sequential(\n",
    "            nn.Dropout(dropout_rate),\n",
    "            nn.Linear(last_channels, num_classes),\n",
    "        )\n",
    "\n",
    "    def calculate_factors(self, version, alpha=1.2, beta=1.1):\n",
    "        phi, res, drop_rate = phi_values[version]\n",
    "        depth_factor = alpha ** phi\n",
    "        width_factor = beta ** phi\n",
    "        return width_factor, depth_factor, drop_rate\n",
    "\n",
    "    def create_features(self, width_factor, depth_factor, last_channels):\n",
    "        channels = int(32 * width_factor)\n",
    "        features = [CNNBlock(3, channels, 3, stride=2, padding=1)]\n",
    "        in_channels = channels\n",
    "\n",
    "        for expand_ratio, channels, repeats, stride, kernel_size in base_model:\n",
    "            out_channels = 4*ceil(int(channels*width_factor) / 4)\n",
    "            layers_repeats = ceil(repeats * depth_factor)\n",
    "\n",
    "            for layer in range(layers_repeats):\n",
    "                features.append(\n",
    "                    InvertedResidualBlock(\n",
    "                        in_channels,\n",
    "                        out_channels,\n",
    "                        expand_ratio=expand_ratio,\n",
    "                        stride = stride if layer == 0 else 1,\n",
    "                        kernel_size=kernel_size,\n",
    "                        padding=kernel_size//2, # if k=1:pad=0, k=3:pad=1, k=5:pad=2\n",
    "                    )\n",
    "                )\n",
    "                in_channels = out_channels\n",
    "\n",
    "        features.append(\n",
    "            CNNBlock(in_channels, last_channels, kernel_size=1, stride=1, padding=0)\n",
    "        )\n",
    "\n",
    "        return nn.Sequential(*features)\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = self.pool(self.features(x))\n",
    "        return self.classifier(x.view(x.shape[0], -1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "torch.Size([4, 10])\n"
     ]
    }
   ],
   "source": [
    "def test():\n",
    "    version = \"b7\"\n",
    "    phi, res, drop_rate = phi_values[version]\n",
    "    num_examples, num_classes = 4, 10\n",
    "    x = torch.randn((num_examples, 3, res, res)).to(device)\n",
    "    model = EfficientNet(\n",
    "        version=version,\n",
    "        num_classes=num_classes,\n",
    "    ).to(device)\n",
    "\n",
    "    print(model(x).shape) # (num_examples, num_classes)\n",
    "\n",
    "test()\n"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "name": "04-feedforward-nn.ipynb",
   "provenance": []
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.4"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "060eccfc031d47b7acc99a48985a0738": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "0f4f61f8bef24f859328471f408c6edc": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "0f9e5413663d45eaaf4c485a72392799": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": "initial"
     }
    },
    "20d22a17b20a459b991bdbeea5180e39": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "2a15eb8b027a4f78933e8434220f197c": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": "initial"
     }
    },
    "33e46dfa0dce488a8ac06fe6caedd7cc": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "359b3e243d634b1a8d49316ff0dfd540": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": "initial"
     }
    },
    "3655c7c75b704045b44f5132e0af3538": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "info",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_aa7c3d224fd04db8830264bd1acc9f6c",
      "max": 1,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_359b3e243d634b1a8d49316ff0dfd540",
      "value": 1
     }
    },
    "38c7c91dfb9442b2b82322e88a40f22b": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_da6646549f9440b990550777d7081844",
      "placeholder": "​",
      "style": "IPY_MODEL_f5cc42a16de840a4b89577f298b1c89e",
      "value": " 9920512/? [00:20&lt;00:00, 30798057.67it/s]"
     }
    },
    "3e71837ffd6542f89b99af04ec6475c8": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "5c8ccd5ec0bc4c07a83496ddd1602235": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "6f44eab9db00404bbf4b41c86e3c7f9d": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "71312105421c4b129c6a98f3a9a3f94a": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "7ce8de076a9d45f38301a66406ee8064": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "info",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_bd300ad13a7f4b52a77185519928569f",
      "max": 1,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_0f9e5413663d45eaaf4c485a72392799",
      "value": 0
     }
    },
    "7d8eb8eed8d648469d7d2c03677d999c": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_b6f78815579e4a1084fdabc4b46fb12b",
       "IPY_MODEL_d33eea28e99147baa5766a90ccd861f7"
      ],
      "layout": "IPY_MODEL_060eccfc031d47b7acc99a48985a0738"
     }
    },
    "9cf1439920af4e7685b2050b52a8f21e": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_3e71837ffd6542f89b99af04ec6475c8",
      "placeholder": "​",
      "style": "IPY_MODEL_ac20db98d04740cf8924f4e7c3163097",
      "value": " 1654784/? [00:19&lt;00:00, 144269.84it/s]"
     }
    },
    "aa7c3d224fd04db8830264bd1acc9f6c": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "aab196ae70c54ee29ce6329df65d6efb": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_f6c9bca3180342c08a666a84e50ae8d5",
       "IPY_MODEL_9cf1439920af4e7685b2050b52a8f21e"
      ],
      "layout": "IPY_MODEL_ebfd199934654ccc9af00d09d9840e23"
     }
    },
    "ac20db98d04740cf8924f4e7c3163097": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "b5308d2444f84bee916d459c5b124817": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_3655c7c75b704045b44f5132e0af3538",
       "IPY_MODEL_38c7c91dfb9442b2b82322e88a40f22b"
      ],
      "layout": "IPY_MODEL_5c8ccd5ec0bc4c07a83496ddd1602235"
     }
    },
    "b6f78815579e4a1084fdabc4b46fb12b": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_f9936b87526a463c80003e7fe40ebb32",
      "max": 1,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_e0a03df91acc471686ea672fb0d565b0",
      "value": 1
     }
    },
    "bd300ad13a7f4b52a77185519928569f": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "c6d361c0a22e4c5c90d4eb30f45a991d": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_7ce8de076a9d45f38301a66406ee8064",
       "IPY_MODEL_ef97798bebdf4786b1ec610b07b572de"
      ],
      "layout": "IPY_MODEL_33e46dfa0dce488a8ac06fe6caedd7cc"
     }
    },
    "d33eea28e99147baa5766a90ccd861f7": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_fb0de3c83bac4d4ab38d6777420a4843",
      "placeholder": "​",
      "style": "IPY_MODEL_0f4f61f8bef24f859328471f408c6edc",
      "value": " 32768/? [00:00&lt;00:00, 389042.40it/s]"
     }
    },
    "da6646549f9440b990550777d7081844": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "e0a03df91acc471686ea672fb0d565b0": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": "initial"
     }
    },
    "ebfd199934654ccc9af00d09d9840e23": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "ef97798bebdf4786b1ec610b07b572de": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_71312105421c4b129c6a98f3a9a3f94a",
      "placeholder": "​",
      "style": "IPY_MODEL_6f44eab9db00404bbf4b41c86e3c7f9d",
      "value": " 0/? [00:00&lt;?, ?it/s]"
     }
    },
    "f5cc42a16de840a4b89577f298b1c89e": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "f6c9bca3180342c08a666a84e50ae8d5": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "info",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_20d22a17b20a459b991bdbeea5180e39",
      "max": 1,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_2a15eb8b027a4f78933e8434220f197c",
      "value": 1
     }
    },
    "f9936b87526a463c80003e7fe40ebb32": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "fb0de3c83bac4d4ab38d6777420a4843": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    }
   }
  },
  "kernelspec": {
   "name": "python394jvsc74a57bd0b0fa6594d8f4cbf19f97940f81e996739fb7646882a419484c72d19e05852a7e",
   "display_name": "Python 3.9.4 64-bit"
  },
  "metadata": {
   "interpreter": {
    "hash": "b0fa6594d8f4cbf19f97940f81e996739fb7646882a419484c72d19e05852a7e"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}