Convnext_Base.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "gpuType": "T4",
      "authorship_tag": "ABX9TyNEOeanJ8QbWGLhCfPt+RjR",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/ethvedbitdesjan/ChromeExtensionHighlight/blob/main/Convnext_Base.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "cljK6DyexzNG",
        "outputId": "814cbb47-f04e-4fd8-89f8-9f1d2de82580"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mounted at content/\n"
          ]
        }
      ],
      "source": [
        "from google.colab import drive\n",
        "drive.mount('content/')"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "%cd content/MyDrive/SummerResearch"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "A_V5FaP_yLWX",
        "outputId": "707a1c01-7156-4832-9c26-235d4fab5139"
      },
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "/content/content/MyDrive/SummerResearch\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Critical imports\n",
        "import os\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "from PIL import Image\n",
        "import random\n",
        "import cv2\n",
        "import copy\n",
        "import torch\n",
        "import torchvision\n",
        "import torchvision.models as models\n",
        "from torch.utils.data import Dataset, DataLoader\n",
        "from torchvision import transforms\n",
        "import torch.optim as optim\n",
        "from torch.optim import lr_scheduler"
      ],
      "metadata": {
        "id": "3jlqrg8nyRW8"
      },
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import shutil"
      ],
      "metadata": {
        "id": "D0fxKMkVk2ka"
      },
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "funny_set = set()\n",
        "not_funny_set = set()\n",
        "with open('labelled_data/funny_combined1.csv', 'r') as f:\n",
        "    lines = f.readlines()\n",
        "    for line in lines:\n",
        "        funny_set.add(line.strip())\n",
        "with open('labelled_data/not_funny_combined1.csv', 'r') as f:\n",
        "    lines = f.readlines()\n",
        "    for line in lines:\n",
        "        not_funny_set.add(line.strip())\n",
        "len(funny_set), len(not_funny_set)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "7WouAlNGkkjf",
        "outputId": "261f766a-cdb0-435d-80e5-59d41228c05b"
      },
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(1133, 1560)"
            ]
          },
          "metadata": {},
          "execution_count": 5
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "%%capture\n",
        "!pip install datasets\n",
        "!pip install wandb -qU"
      ],
      "metadata": {
        "id": "K66VE_AtovGn"
      },
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "%%capture\n",
        "!pip install weightwatcher"
      ],
      "metadata": {
        "id": "eqnQWpqNPN4d"
      },
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import wandb\n",
        "wandb.login()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 69
        },
        "id": "bvULp0TfOnU4",
        "outputId": "22ab1c57-a6c4-4d62-c11f-a515b739c45f"
      },
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.Javascript object>"
            ],
            "application/javascript": [
              "\n",
              "        window._wandbApiKey = new Promise((resolve, reject) => {\n",
              "            function loadScript(url) {\n",
              "            return new Promise(function(resolve, reject) {\n",
              "                let newScript = document.createElement(\"script\");\n",
              "                newScript.onerror = reject;\n",
              "                newScript.onload = resolve;\n",
              "                document.body.appendChild(newScript);\n",
              "                newScript.src = url;\n",
              "            });\n",
              "            }\n",
              "            loadScript(\"https://cdn.jsdelivr.net/npm/postmate/build/postmate.min.js\").then(() => {\n",
              "            const iframe = document.createElement('iframe')\n",
              "            iframe.style.cssText = \"width:0;height:0;border:none\"\n",
              "            document.body.appendChild(iframe)\n",
              "            const handshake = new Postmate({\n",
              "                container: iframe,\n",
              "                url: 'https://wandb.ai/authorize'\n",
              "            });\n",
              "            const timeout = setTimeout(() => reject(\"Couldn't auto authenticate\"), 5000)\n",
              "            handshake.then(function(child) {\n",
              "                child.on('authorize', data => {\n",
              "                    clearTimeout(timeout)\n",
              "                    resolve(data)\n",
              "                });\n",
              "            });\n",
              "            })\n",
              "        });\n",
              "    "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\u001b[34m\u001b[1mwandb\u001b[0m: Appending key for api.wandb.ai to your netrc file: /root/.netrc\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "True"
            ]
          },
          "metadata": {},
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import weightwatcher as ww"
      ],
      "metadata": {
        "id": "nIS3-XbKPac7"
      },
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from torchvision.datasets import ImageFolder"
      ],
      "metadata": {
        "id": "DfdU4EF-o5C8"
      },
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "folders = []\n",
        "for file in os.scandir('Dataset/train'):\n",
        "  if file.is_dir():\n",
        "    folders.append(file.name)\n",
        "folders"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "udLkDpzEZdwB",
        "outputId": "a1e07a69-89f3-4245-be6d-a9515ff65406"
      },
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['Funny_Combined', 'Not_Funny_Combined']"
            ]
          },
          "metadata": {},
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "class FunnyNotFunnyDataset(Dataset):\n",
        "    def __init__(self, data=[], root_dir=None, transform=None):\n",
        "        if root_dir[-1] != '/':\n",
        "          root_dir += '/'\n",
        "        self.root_dir = root_dir\n",
        "        classes = []\n",
        "        for file in os.scandir(root_dir):\n",
        "          if file.is_dir():\n",
        "            classes.append(file.name)\n",
        "        data = []\n",
        "        for i, class_name in enumerate(classes):\n",
        "          for file in os.listdir(root_dir+class_name):\n",
        "            data.append((root_dir + class_name + '/'+ file, i))\n",
        "        self.data = data\n",
        "        self.num_classes = len(classes)\n",
        "        self.transform = transform\n",
        "    def __len__(self):\n",
        "        return len(self.data)\n",
        "    def __getitem__(self, index):\n",
        "        image = Image.open(self.data[index][0])\n",
        "        if self.transform:\n",
        "          image = self.transform()(image)\n",
        "        label = self.data[index][1]\n",
        "        label_tensor = torch.zeros(1)\n",
        "        if label == 1:\n",
        "          label_tensor[0] = 1\n",
        "        return {'image_data':image, 'label':label_tensor}"
      ],
      "metadata": {
        "id": "NG7H6naCypBP"
      },
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "class ResNetAdded(torch.nn.Module):\n",
        "    def __init__(self, resnet50=None, train_full=True):\n",
        "        super(ResNetAdded, self).__init__()\n",
        "        if resnet50:\n",
        "            self.resnet50 = resnet50\n",
        "        else:\n",
        "            self.resnet50 = models.resnet50(pretrained=True)\n",
        "        if not train_full:\n",
        "            for param in self.resnet50.parameters():\n",
        "                param.requires_grad = False\n",
        "        self.resnet50.fc = torch.nn.Linear(2048, 1)\n",
        "        self.feature_extractor = None\n",
        "\n",
        "    def forward(self, x):\n",
        "        \"\"\"\n",
        "        x: A batch of images.\n",
        "\n",
        "        Returns: A tensor of predictions.\n",
        "        \"\"\"\n",
        "\n",
        "        # x = self.resnet50(x)\n",
        "        # preds = self.classifier(x)\n",
        "        preds = self.resnet50(x)\n",
        "        return preds\n",
        "\n",
        "    def feature_extraction(self, x):\n",
        "        if self.feature_extractor is None:\n",
        "            self.feature_extractor = torch.nn.Sequential(*(list(self.resnet50.children())[:-1]))\n",
        "        features = self.feature_extractor(x)\n",
        "        return features\n",
        "\n",
        "class AlexNetAdded(torch.nn.Module):\n",
        "    def __init__(self, alexnet=None, train_full=True):\n",
        "        super(AlexNetAdded, self).__init__()\n",
        "        if alexnet:\n",
        "            self.alexnet = alexnet\n",
        "        else:\n",
        "            self.alexnet = models.alexnet(pretrained=True)\n",
        "        if not train_full:\n",
        "            for param in self.alexnet.parameters():\n",
        "                param.requires_grad = False\n",
        "        self.alexnet.classifier[6] = torch.nn.Linear(4096, 1)\n",
        "        self.feature_extractor = None\n",
        "    def forward(self, x):\n",
        "        \"\"\"\n",
        "        x: A batch of images.\n",
        "\n",
        "        Returns: A tensor of predictions.\n",
        "        \"\"\"\n",
        "        preds = self.alexnet(x)\n",
        "        return preds\n",
        "\n",
        "class InceptionAdded(torch.nn.Module):\n",
        "    def __init__(self, inception=None, train_full=True):\n",
        "        super(InceptionAdded, self).__init__()\n",
        "        if inception:\n",
        "            self.xception = inception\n",
        "        else:\n",
        "            self.inception = models.inception_v3(pretrained=True)\n",
        "        if not train_full:\n",
        "            for param in self.inception.parameters():\n",
        "                param.requires_grad = False\n",
        "        self.inception.fc = torch.nn.Linear(2048, 1)\n",
        "        self.feature_extractor = None\n",
        "    def forward(self, x):\n",
        "        \"\"\"\n",
        "        x: A batch of images.\n",
        "\n",
        "        Returns: A tensor of predictions.\n",
        "        \"\"\"\n",
        "        preds = self.inception(x)\n",
        "        return preds\n",
        "\n",
        "class DenseNetAdded(torch.nn.Module):\n",
        "    def __init__(self, densenet=None, train_full=True):\n",
        "        super(DenseNetAdded, self).__init__()\n",
        "        if densenet:\n",
        "            self.densenet = densenet\n",
        "        else:\n",
        "            self.densenet = models.densenet201(pretrained=True)\n",
        "        if not train_full:\n",
        "            for param in self.densenet.parameters():\n",
        "                param.requires_grad = False\n",
        "        self.densenet.classifier = torch.nn.Linear(1920, 1)\n",
        "        self.feature_extractor = None\n",
        "    def forward(self, x):\n",
        "        \"\"\"\n",
        "        x: A batch of images.\n",
        "\n",
        "        Returns: A tensor of predictions.\n",
        "        \"\"\"\n",
        "        preds = self.densenet(x)\n",
        "        return preds\n",
        "class ConvNextAdded(torch.nn.Module):\n",
        "    def __init__(self, convnext=None, train_full=True, pretrained=True):\n",
        "        super(ConvNextAdded, self).__init__()\n",
        "        if convnext:\n",
        "            self.convnext = convnext\n",
        "        else:\n",
        "            self.convnext = models.convnext_base(pretrained=pretrained)\n",
        "        if pretrained:\n",
        "          if not train_full:\n",
        "              for param in self.convnext.parameters():\n",
        "                  param.requires_grad = False\n",
        "        self.convnext.classifier[2] = torch.nn.Linear(1024, 1)\n",
        "        self.feature_extractor = None\n",
        "    def forward(self, x):\n",
        "        \"\"\"\n",
        "        x: A batch of images.\n",
        "\n",
        "        Returns: A tensor of predictions.\n",
        "        \"\"\"\n",
        "        preds = self.convnext(x)\n",
        "        return preds"
      ],
      "metadata": {
        "id": "uiI_AQAoziqh"
      },
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "model = ConvNextAdded(train_full=False)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Ar6AkYtCzkWS",
        "outputId": "e76b9b05-a187-489b-b73e-36dc3b670001"
      },
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/torchvision/models/_utils.py:208: UserWarning: The parameter 'pretrained' is deprecated since 0.13 and may be removed in the future, please use 'weights' instead.\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.10/dist-packages/torchvision/models/_utils.py:223: UserWarning: Arguments other than a weight enum or `None` for 'weights' are deprecated since 0.13 and may be removed in the future. The current behavior is equivalent to passing `weights=ConvNeXt_Base_Weights.IMAGENET1K_V1`. You can also use `weights=ConvNeXt_Base_Weights.DEFAULT` to get the most up-to-date weights.\n",
            "  warnings.warn(msg)\n",
            "Downloading: \"https://download.pytorch.org/models/convnext_base-6075fbad.pth\" to /root/.cache/torch/hub/checkpoints/convnext_base-6075fbad.pth\n",
            "100%|██████████| 338M/338M [00:04<00:00, 83.1MB/s]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "IMAGE_SIZE = 224\n",
        "NUM_CLASSES = 2\n",
        "BATCH_SIZE = 16\n",
        "device = 'cuda' if torch.cuda.is_available() else 'cpu'\n",
        "IMG_MEAN = [0.485, 0.456, 0.406]\n",
        "IMG_STD = [0.229, 0.224, 0.225]\n",
        "train_transform = models.ConvNeXt_Base_Weights.IMAGENET1K_V1.transforms\n",
        "test_transform = models.ConvNeXt_Base_Weights.IMAGENET1K_V1.transforms\n",
        "train_dataset = FunnyNotFunnyDataset(root_dir='Dataset/train', transform = train_transform)\n",
        "valid_dataset = FunnyNotFunnyDataset(root_dir='Dataset/valid', transform = test_transform)\n",
        "test_dataset = FunnyNotFunnyDataset(root_dir='Dataset/test' ,transform = test_transform)\n",
        "train_dataset"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "I-rMiffozlkI",
        "outputId": "26de908d-f346-4b23-e3da-c8f50f237c5a"
      },
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<__main__.FunnyNotFunnyDataset at 0x7f00562c9420>"
            ]
          },
          "metadata": {},
          "execution_count": 15
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "train_dataloader = DataLoader(train_dataset, batch_size = BATCH_SIZE, shuffle=True)\n",
        "valid_dataloader = DataLoader(valid_dataset, batch_size = BATCH_SIZE, shuffle=True)\n",
        "test_dataloader = DataLoader(test_dataset, batch_size = BATCH_SIZE, shuffle=True)\n",
        "model.to(device)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "tMLahPNYXre9",
        "outputId": "977c3eec-9373-4292-cf5c-3fcccf8a1ddc"
      },
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "ConvNextAdded(\n",
              "  (convnext): ConvNeXt(\n",
              "    (features): Sequential(\n",
              "      (0): Conv2dNormActivation(\n",
              "        (0): Conv2d(3, 128, kernel_size=(4, 4), stride=(4, 4))\n",
              "        (1): LayerNorm2d((128,), eps=1e-06, elementwise_affine=True)\n",
              "      )\n",
              "      (1): Sequential(\n",
              "        (0): CNBlock(\n",
              "          (block): Sequential(\n",
              "            (0): Conv2d(128, 128, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=128)\n",
              "            (1): Permute()\n",
              "            (2): LayerNorm((128,), eps=1e-06, elementwise_affine=True)\n",
              "            (3): Linear(in_features=128, out_features=512, bias=True)\n",
              "            (4): GELU(approximate='none')\n",
              "            (5): Linear(in_features=512, out_features=128, bias=True)\n",
              "            (6): Permute()\n",
              "          )\n",
              "          (stochastic_depth): StochasticDepth(p=0.0, mode=row)\n",
              "        )\n",
              "        (1): CNBlock(\n",
              "          (block): Sequential(\n",
              "            (0): Conv2d(128, 128, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=128)\n",
              "            (1): Permute()\n",
              "            (2): LayerNorm((128,), eps=1e-06, elementwise_affine=True)\n",
              "            (3): Linear(in_features=128, out_features=512, bias=True)\n",
              "            (4): GELU(approximate='none')\n",
              "            (5): Linear(in_features=512, out_features=128, bias=True)\n",
              "            (6): Permute()\n",
              "          )\n",
              "          (stochastic_depth): StochasticDepth(p=0.014285714285714285, mode=row)\n",
              "        )\n",
              "        (2): CNBlock(\n",
              "          (block): Sequential(\n",
              "            (0): Conv2d(128, 128, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=128)\n",
              "            (1): Permute()\n",
              "            (2): LayerNorm((128,), eps=1e-06, elementwise_affine=True)\n",
              "            (3): Linear(in_features=128, out_features=512, bias=True)\n",
              "            (4): GELU(approximate='none')\n",
              "            (5): Linear(in_features=512, out_features=128, bias=True)\n",
              "            (6): Permute()\n",
              "          )\n",
              "          (stochastic_depth): StochasticDepth(p=0.02857142857142857, mode=row)\n",
              "        )\n",
              "      )\n",
              "      (2): Sequential(\n",
              "        (0): LayerNorm2d((128,), eps=1e-06, elementwise_affine=True)\n",
              "        (1): Conv2d(128, 256, kernel_size=(2, 2), stride=(2, 2))\n",
              "      )\n",
              "      (3): Sequential(\n",
              "        (0): CNBlock(\n",
              "          (block): Sequential(\n",
              "            (0): Conv2d(256, 256, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=256)\n",
              "            (1): Permute()\n",
              "            (2): LayerNorm((256,), eps=1e-06, elementwise_affine=True)\n",
              "            (3): Linear(in_features=256, out_features=1024, bias=True)\n",
              "            (4): GELU(approximate='none')\n",
              "            (5): Linear(in_features=1024, out_features=256, bias=True)\n",
              "            (6): Permute()\n",
              "          )\n",
              "          (stochastic_depth): StochasticDepth(p=0.04285714285714286, mode=row)\n",
              "        )\n",
              "        (1): CNBlock(\n",
              "          (block): Sequential(\n",
              "            (0): Conv2d(256, 256, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=256)\n",
              "            (1): Permute()\n",
              "            (2): LayerNorm((256,), eps=1e-06, elementwise_affine=True)\n",
              "            (3): Linear(in_features=256, out_features=1024, bias=True)\n",
              "            (4): GELU(approximate='none')\n",
              "            (5): Linear(in_features=1024, out_features=256, bias=True)\n",
              "            (6): Permute()\n",
              "          )\n",
              "          (stochastic_depth): StochasticDepth(p=0.05714285714285714, mode=row)\n",
              "        )\n",
              "        (2): CNBlock(\n",
              "          (block): Sequential(\n",
              "            (0): Conv2d(256, 256, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=256)\n",
              "            (1): Permute()\n",
              "            (2): LayerNorm((256,), eps=1e-06, elementwise_affine=True)\n",
              "            (3): Linear(in_features=256, out_features=1024, bias=True)\n",
              "            (4): GELU(approximate='none')\n",
              "            (5): Linear(in_features=1024, out_features=256, bias=True)\n",
              "            (6): Permute()\n",
              "          )\n",
              "          (stochastic_depth): StochasticDepth(p=0.07142857142857142, mode=row)\n",
              "        )\n",
              "      )\n",
              "      (4): Sequential(\n",
              "        (0): LayerNorm2d((256,), eps=1e-06, elementwise_affine=True)\n",
              "        (1): Conv2d(256, 512, kernel_size=(2, 2), stride=(2, 2))\n",
              "      )\n",
              "      (5): Sequential(\n",
              "        (0): CNBlock(\n",
              "          (block): Sequential(\n",
              "            (0): Conv2d(512, 512, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=512)\n",
              "            (1): Permute()\n",
              "            (2): LayerNorm((512,), eps=1e-06, elementwise_affine=True)\n",
              "            (3): Linear(in_features=512, out_features=2048, bias=True)\n",
              "            (4): GELU(approximate='none')\n",
              "            (5): Linear(in_features=2048, out_features=512, bias=True)\n",
              "            (6): Permute()\n",
              "          )\n",
              "          (stochastic_depth): StochasticDepth(p=0.08571428571428572, mode=row)\n",
              "        )\n",
              "        (1): CNBlock(\n",
              "          (block): Sequential(\n",
              "            (0): Conv2d(512, 512, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=512)\n",
              "            (1): Permute()\n",
              "            (2): LayerNorm((512,), eps=1e-06, elementwise_affine=True)\n",
              "            (3): Linear(in_features=512, out_features=2048, bias=True)\n",
              "            (4): GELU(approximate='none')\n",
              "            (5): Linear(in_features=2048, out_features=512, bias=True)\n",
              "            (6): Permute()\n",
              "          )\n",
              "          (stochastic_depth): StochasticDepth(p=0.1, mode=row)\n",
              "        )\n",
              "        (2): CNBlock(\n",
              "          (block): Sequential(\n",
              "            (0): Conv2d(512, 512, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=512)\n",
              "            (1): Permute()\n",
              "            (2): LayerNorm((512,), eps=1e-06, elementwise_affine=True)\n",
              "            (3): Linear(in_features=512, out_features=2048, bias=True)\n",
              "            (4): GELU(approximate='none')\n",
              "            (5): Linear(in_features=2048, out_features=512, bias=True)\n",
              "            (6): Permute()\n",
              "          )\n",
              "          (stochastic_depth): StochasticDepth(p=0.11428571428571428, mode=row)\n",
              "        )\n",
              "        (3): CNBlock(\n",
              "          (block): Sequential(\n",
              "            (0): Conv2d(512, 512, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=512)\n",
              "            (1): Permute()\n",
              "            (2): LayerNorm((512,), eps=1e-06, elementwise_affine=True)\n",
              "            (3): Linear(in_features=512, out_features=2048, bias=True)\n",
              "            (4): GELU(approximate='none')\n",
              "            (5): Linear(in_features=2048, out_features=512, bias=True)\n",
              "            (6): Permute()\n",
              "          )\n",
              "          (stochastic_depth): StochasticDepth(p=0.12857142857142856, mode=row)\n",
              "        )\n",
              "        (4): CNBlock(\n",
              "          (block): Sequential(\n",
              "            (0): Conv2d(512, 512, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=512)\n",
              "            (1): Permute()\n",
              "            (2): LayerNorm((512,), eps=1e-06, elementwise_affine=True)\n",
              "            (3): Linear(in_features=512, out_features=2048, bias=True)\n",
              "            (4): GELU(approximate='none')\n",
              "            (5): Linear(in_features=2048, out_features=512, bias=True)\n",
              "            (6): Permute()\n",
              "          )\n",
              "          (stochastic_depth): StochasticDepth(p=0.14285714285714285, mode=row)\n",
              "        )\n",
              "        (5): CNBlock(\n",
              "          (block): Sequential(\n",
              "            (0): Conv2d(512, 512, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=512)\n",
              "            (1): Permute()\n",
              "            (2): LayerNorm((512,), eps=1e-06, elementwise_affine=True)\n",
              "            (3): Linear(in_features=512, out_features=2048, bias=True)\n",
              "            (4): GELU(approximate='none')\n",
              "            (5): Linear(in_features=2048, out_features=512, bias=True)\n",
              "            (6): Permute()\n",
              "          )\n",
              "          (stochastic_depth): StochasticDepth(p=0.15714285714285714, mode=row)\n",
              "        )\n",
              "        (6): CNBlock(\n",
              "          (block): Sequential(\n",
              "            (0): Conv2d(512, 512, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=512)\n",
              "            (1): Permute()\n",
              "            (2): LayerNorm((512,), eps=1e-06, elementwise_affine=True)\n",
              "            (3): Linear(in_features=512, out_features=2048, bias=True)\n",
              "            (4): GELU(approximate='none')\n",
              "            (5): Linear(in_features=2048, out_features=512, bias=True)\n",
              "            (6): Permute()\n",
              "          )\n",
              "          (stochastic_depth): StochasticDepth(p=0.17142857142857143, mode=row)\n",
              "        )\n",
              "        (7): CNBlock(\n",
              "          (block): Sequential(\n",
              "            (0): Conv2d(512, 512, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=512)\n",
              "            (1): Permute()\n",
              "            (2): LayerNorm((512,), eps=1e-06, elementwise_affine=True)\n",
              "            (3): Linear(in_features=512, out_features=2048, bias=True)\n",
              "            (4): GELU(approximate='none')\n",
              "            (5): Linear(in_features=2048, out_features=512, bias=True)\n",
              "            (6): Permute()\n",
              "          )\n",
              "          (stochastic_depth): StochasticDepth(p=0.18571428571428572, mode=row)\n",
              "        )\n",
              "        (8): CNBlock(\n",
              "          (block): Sequential(\n",
              "            (0): Conv2d(512, 512, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=512)\n",
              "            (1): Permute()\n",
              "            (2): LayerNorm((512,), eps=1e-06, elementwise_affine=True)\n",
              "            (3): Linear(in_features=512, out_features=2048, bias=True)\n",
              "            (4): GELU(approximate='none')\n",
              "            (5): Linear(in_features=2048, out_features=512, bias=True)\n",
              "            (6): Permute()\n",
              "          )\n",
              "          (stochastic_depth): StochasticDepth(p=0.2, mode=row)\n",
              "        )\n",
              "        (9): CNBlock(\n",
              "          (block): Sequential(\n",
              "            (0): Conv2d(512, 512, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=512)\n",
              "            (1): Permute()\n",
              "            (2): LayerNorm((512,), eps=1e-06, elementwise_affine=True)\n",
              "            (3): Linear(in_features=512, out_features=2048, bias=True)\n",
              "            (4): GELU(approximate='none')\n",
              "            (5): Linear(in_features=2048, out_features=512, bias=True)\n",
              "            (6): Permute()\n",
              "          )\n",
              "          (stochastic_depth): StochasticDepth(p=0.21428571428571427, mode=row)\n",
              "        )\n",
              "        (10): CNBlock(\n",
              "          (block): Sequential(\n",
              "            (0): Conv2d(512, 512, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=512)\n",
              "            (1): Permute()\n",
              "            (2): LayerNorm((512,), eps=1e-06, elementwise_affine=True)\n",
              "            (3): Linear(in_features=512, out_features=2048, bias=True)\n",
              "            (4): GELU(approximate='none')\n",
              "            (5): Linear(in_features=2048, out_features=512, bias=True)\n",
              "            (6): Permute()\n",
              "          )\n",
              "          (stochastic_depth): StochasticDepth(p=0.22857142857142856, mode=row)\n",
              "        )\n",
              "        (11): CNBlock(\n",
              "          (block): Sequential(\n",
              "            (0): Conv2d(512, 512, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=512)\n",
              "            (1): Permute()\n",
              "            (2): LayerNorm((512,), eps=1e-06, elementwise_affine=True)\n",
              "            (3): Linear(in_features=512, out_features=2048, bias=True)\n",
              "            (4): GELU(approximate='none')\n",
              "            (5): Linear(in_features=2048, out_features=512, bias=True)\n",
              "            (6): Permute()\n",
              "          )\n",
              "          (stochastic_depth): StochasticDepth(p=0.24285714285714285, mode=row)\n",
              "        )\n",
              "        (12): CNBlock(\n",
              "          (block): Sequential(\n",
              "            (0): Conv2d(512, 512, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=512)\n",
              "            (1): Permute()\n",
              "            (2): LayerNorm((512,), eps=1e-06, elementwise_affine=True)\n",
              "            (3): Linear(in_features=512, out_features=2048, bias=True)\n",
              "            (4): GELU(approximate='none')\n",
              "            (5): Linear(in_features=2048, out_features=512, bias=True)\n",
              "            (6): Permute()\n",
              "          )\n",
              "          (stochastic_depth): StochasticDepth(p=0.2571428571428571, mode=row)\n",
              "        )\n",
              "        (13): CNBlock(\n",
              "          (block): Sequential(\n",
              "            (0): Conv2d(512, 512, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=512)\n",
              "            (1): Permute()\n",
              "            (2): LayerNorm((512,), eps=1e-06, elementwise_affine=True)\n",
              "            (3): Linear(in_features=512, out_features=2048, bias=True)\n",
              "            (4): GELU(approximate='none')\n",
              "            (5): Linear(in_features=2048, out_features=512, bias=True)\n",
              "            (6): Permute()\n",
              "          )\n",
              "          (stochastic_depth): StochasticDepth(p=0.2714285714285714, mode=row)\n",
              "        )\n",
              "        (14): CNBlock(\n",
              "          (block): Sequential(\n",
              "            (0): Conv2d(512, 512, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=512)\n",
              "            (1): Permute()\n",
              "            (2): LayerNorm((512,), eps=1e-06, elementwise_affine=True)\n",
              "            (3): Linear(in_features=512, out_features=2048, bias=True)\n",
              "            (4): GELU(approximate='none')\n",
              "            (5): Linear(in_features=2048, out_features=512, bias=True)\n",
              "            (6): Permute()\n",
              "          )\n",
              "          (stochastic_depth): StochasticDepth(p=0.2857142857142857, mode=row)\n",
              "        )\n",
              "        (15): CNBlock(\n",
              "          (block): Sequential(\n",
              "            (0): Conv2d(512, 512, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=512)\n",
              "            (1): Permute()\n",
              "            (2): LayerNorm((512,), eps=1e-06, elementwise_affine=True)\n",
              "            (3): Linear(in_features=512, out_features=2048, bias=True)\n",
              "            (4): GELU(approximate='none')\n",
              "            (5): Linear(in_features=2048, out_features=512, bias=True)\n",
              "            (6): Permute()\n",
              "          )\n",
              "          (stochastic_depth): StochasticDepth(p=0.3, mode=row)\n",
              "        )\n",
              "        (16): CNBlock(\n",
              "          (block): Sequential(\n",
              "            (0): Conv2d(512, 512, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=512)\n",
              "            (1): Permute()\n",
              "            (2): LayerNorm((512,), eps=1e-06, elementwise_affine=True)\n",
              "            (3): Linear(in_features=512, out_features=2048, bias=True)\n",
              "            (4): GELU(approximate='none')\n",
              "            (5): Linear(in_features=2048, out_features=512, bias=True)\n",
              "            (6): Permute()\n",
              "          )\n",
              "          (stochastic_depth): StochasticDepth(p=0.3142857142857143, mode=row)\n",
              "        )\n",
              "        (17): CNBlock(\n",
              "          (block): Sequential(\n",
              "            (0): Conv2d(512, 512, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=512)\n",
              "            (1): Permute()\n",
              "            (2): LayerNorm((512,), eps=1e-06, elementwise_affine=True)\n",
              "            (3): Linear(in_features=512, out_features=2048, bias=True)\n",
              "            (4): GELU(approximate='none')\n",
              "            (5): Linear(in_features=2048, out_features=512, bias=True)\n",
              "            (6): Permute()\n",
              "          )\n",
              "          (stochastic_depth): StochasticDepth(p=0.32857142857142857, mode=row)\n",
              "        )\n",
              "        (18): CNBlock(\n",
              "          (block): Sequential(\n",
              "            (0): Conv2d(512, 512, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=512)\n",
              "            (1): Permute()\n",
              "            (2): LayerNorm((512,), eps=1e-06, elementwise_affine=True)\n",
              "            (3): Linear(in_features=512, out_features=2048, bias=True)\n",
              "            (4): GELU(approximate='none')\n",
              "            (5): Linear(in_features=2048, out_features=512, bias=True)\n",
              "            (6): Permute()\n",
              "          )\n",
              "          (stochastic_depth): StochasticDepth(p=0.34285714285714286, mode=row)\n",
              "        )\n",
              "        (19): CNBlock(\n",
              "          (block): Sequential(\n",
              "            (0): Conv2d(512, 512, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=512)\n",
              "            (1): Permute()\n",
              "            (2): LayerNorm((512,), eps=1e-06, elementwise_affine=True)\n",
              "            (3): Linear(in_features=512, out_features=2048, bias=True)\n",
              "            (4): GELU(approximate='none')\n",
              "            (5): Linear(in_features=2048, out_features=512, bias=True)\n",
              "            (6): Permute()\n",
              "          )\n",
              "          (stochastic_depth): StochasticDepth(p=0.35714285714285715, mode=row)\n",
              "        )\n",
              "        (20): CNBlock(\n",
              "          (block): Sequential(\n",
              "            (0): Conv2d(512, 512, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=512)\n",
              "            (1): Permute()\n",
              "            (2): LayerNorm((512,), eps=1e-06, elementwise_affine=True)\n",
              "            (3): Linear(in_features=512, out_features=2048, bias=True)\n",
              "            (4): GELU(approximate='none')\n",
              "            (5): Linear(in_features=2048, out_features=512, bias=True)\n",
              "            (6): Permute()\n",
              "          )\n",
              "          (stochastic_depth): StochasticDepth(p=0.37142857142857144, mode=row)\n",
              "        )\n",
              "        (21): CNBlock(\n",
              "          (block): Sequential(\n",
              "            (0): Conv2d(512, 512, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=512)\n",
              "            (1): Permute()\n",
              "            (2): LayerNorm((512,), eps=1e-06, elementwise_affine=True)\n",
              "            (3): Linear(in_features=512, out_features=2048, bias=True)\n",
              "            (4): GELU(approximate='none')\n",
              "            (5): Linear(in_features=2048, out_features=512, bias=True)\n",
              "            (6): Permute()\n",
              "          )\n",
              "          (stochastic_depth): StochasticDepth(p=0.38571428571428573, mode=row)\n",
              "        )\n",
              "        (22): CNBlock(\n",
              "          (block): Sequential(\n",
              "            (0): Conv2d(512, 512, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=512)\n",
              "            (1): Permute()\n",
              "            (2): LayerNorm((512,), eps=1e-06, elementwise_affine=True)\n",
              "            (3): Linear(in_features=512, out_features=2048, bias=True)\n",
              "            (4): GELU(approximate='none')\n",
              "            (5): Linear(in_features=2048, out_features=512, bias=True)\n",
              "            (6): Permute()\n",
              "          )\n",
              "          (stochastic_depth): StochasticDepth(p=0.4, mode=row)\n",
              "        )\n",
              "        (23): CNBlock(\n",
              "          (block): Sequential(\n",
              "            (0): Conv2d(512, 512, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=512)\n",
              "            (1): Permute()\n",
              "            (2): LayerNorm((512,), eps=1e-06, elementwise_affine=True)\n",
              "            (3): Linear(in_features=512, out_features=2048, bias=True)\n",
              "            (4): GELU(approximate='none')\n",
              "            (5): Linear(in_features=2048, out_features=512, bias=True)\n",
              "            (6): Permute()\n",
              "          )\n",
              "          (stochastic_depth): StochasticDepth(p=0.4142857142857143, mode=row)\n",
              "        )\n",
              "        (24): CNBlock(\n",
              "          (block): Sequential(\n",
              "            (0): Conv2d(512, 512, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=512)\n",
              "            (1): Permute()\n",
              "            (2): LayerNorm((512,), eps=1e-06, elementwise_affine=True)\n",
              "            (3): Linear(in_features=512, out_features=2048, bias=True)\n",
              "            (4): GELU(approximate='none')\n",
              "            (5): Linear(in_features=2048, out_features=512, bias=True)\n",
              "            (6): Permute()\n",
              "          )\n",
              "          (stochastic_depth): StochasticDepth(p=0.42857142857142855, mode=row)\n",
              "        )\n",
              "        (25): CNBlock(\n",
              "          (block): Sequential(\n",
              "            (0): Conv2d(512, 512, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=512)\n",
              "            (1): Permute()\n",
              "            (2): LayerNorm((512,), eps=1e-06, elementwise_affine=True)\n",
              "            (3): Linear(in_features=512, out_features=2048, bias=True)\n",
              "            (4): GELU(approximate='none')\n",
              "            (5): Linear(in_features=2048, out_features=512, bias=True)\n",
              "            (6): Permute()\n",
              "          )\n",
              "          (stochastic_depth): StochasticDepth(p=0.44285714285714284, mode=row)\n",
              "        )\n",
              "        (26): CNBlock(\n",
              "          (block): Sequential(\n",
              "            (0): Conv2d(512, 512, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=512)\n",
              "            (1): Permute()\n",
              "            (2): LayerNorm((512,), eps=1e-06, elementwise_affine=True)\n",
              "            (3): Linear(in_features=512, out_features=2048, bias=True)\n",
              "            (4): GELU(approximate='none')\n",
              "            (5): Linear(in_features=2048, out_features=512, bias=True)\n",
              "            (6): Permute()\n",
              "          )\n",
              "          (stochastic_depth): StochasticDepth(p=0.45714285714285713, mode=row)\n",
              "        )\n",
              "      )\n",
              "      (6): Sequential(\n",
              "        (0): LayerNorm2d((512,), eps=1e-06, elementwise_affine=True)\n",
              "        (1): Conv2d(512, 1024, kernel_size=(2, 2), stride=(2, 2))\n",
              "      )\n",
              "      (7): Sequential(\n",
              "        (0): CNBlock(\n",
              "          (block): Sequential(\n",
              "            (0): Conv2d(1024, 1024, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=1024)\n",
              "            (1): Permute()\n",
              "            (2): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)\n",
              "            (3): Linear(in_features=1024, out_features=4096, bias=True)\n",
              "            (4): GELU(approximate='none')\n",
              "            (5): Linear(in_features=4096, out_features=1024, bias=True)\n",
              "            (6): Permute()\n",
              "          )\n",
              "          (stochastic_depth): StochasticDepth(p=0.4714285714285714, mode=row)\n",
              "        )\n",
              "        (1): CNBlock(\n",
              "          (block): Sequential(\n",
              "            (0): Conv2d(1024, 1024, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=1024)\n",
              "            (1): Permute()\n",
              "            (2): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)\n",
              "            (3): Linear(in_features=1024, out_features=4096, bias=True)\n",
              "            (4): GELU(approximate='none')\n",
              "            (5): Linear(in_features=4096, out_features=1024, bias=True)\n",
              "            (6): Permute()\n",
              "          )\n",
              "          (stochastic_depth): StochasticDepth(p=0.4857142857142857, mode=row)\n",
              "        )\n",
              "        (2): CNBlock(\n",
              "          (block): Sequential(\n",
              "            (0): Conv2d(1024, 1024, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=1024)\n",
              "            (1): Permute()\n",
              "            (2): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)\n",
              "            (3): Linear(in_features=1024, out_features=4096, bias=True)\n",
              "            (4): GELU(approximate='none')\n",
              "            (5): Linear(in_features=4096, out_features=1024, bias=True)\n",
              "            (6): Permute()\n",
              "          )\n",
              "          (stochastic_depth): StochasticDepth(p=0.5, mode=row)\n",
              "        )\n",
              "      )\n",
              "    )\n",
              "    (avgpool): AdaptiveAvgPool2d(output_size=1)\n",
              "    (classifier): Sequential(\n",
              "      (0): LayerNorm2d((1024,), eps=1e-06, elementwise_affine=True)\n",
              "      (1): Flatten(start_dim=1, end_dim=-1)\n",
              "      (2): Linear(in_features=1024, out_features=1, bias=True)\n",
              "    )\n",
              "  )\n",
              ")"
            ]
          },
          "metadata": {},
          "execution_count": 16
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "criterion = torch.nn.BCEWithLogitsLoss()\n",
        "optimizer = torch.optim.AdamW(model.parameters(), lr=0.001, weight_decay=0.0001)"
      ],
      "metadata": {
        "id": "7xjyRbdFzqMd"
      },
      "execution_count": 17,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import gc\n",
        "gc.collect()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "eoGVxRG_zr07",
        "outputId": "fb1e4843-ec6c-4b65-8401-d19881c063bd"
      },
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "32"
            ]
          },
          "metadata": {},
          "execution_count": 18
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import matplotlib.pyplot as plt"
      ],
      "metadata": {
        "id": "oH_dJmwAzutc"
      },
      "execution_count": 19,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def train(num_epochs, save_name):\n",
        "  best_valid_loss = 10000\n",
        "  best_valid_epoch = -1\n",
        "  all_train_losses = []\n",
        "  all_valid_losses = []\n",
        "  count = 0\n",
        "  for epoch in range(num_epochs):\n",
        "    print('epoch:', epoch)\n",
        "    model.train()\n",
        "    running_loss = 0.0\n",
        "    total_loss = []\n",
        "    lrs = []\n",
        "    total_size = 0\n",
        "    correct = 0\n",
        "    for i, data in enumerate(train_dataloader):\n",
        "      inputs = data['image_data'].to(device)\n",
        "      label = data['label'].to(device)\n",
        "      inputs = inputs.type(torch.cuda.FloatTensor)\n",
        "      label = label.type(torch.cuda.FloatTensor)\n",
        "      output = model.forward(inputs)\n",
        "      gc.collect()\n",
        "      del inputs\n",
        "      loss = criterion(output, label)\n",
        "      loss.mean().backward()\n",
        "      optimizer.step()\n",
        "      optimizer.zero_grad()\n",
        "      total_loss.append(loss.item())\n",
        "      gc.collect()\n",
        "      del loss\n",
        "      torch.cuda.empty_cache()\n",
        "      output = torch.sigmoid(output)\n",
        "      predictions = torch.as_tensor((output - 0.5) > 0, dtype=torch.int32)\n",
        "      correct += (predictions == label).float().sum().item()\n",
        "      gc.collect()\n",
        "      #gpu_usage()\n",
        "      del predictions\n",
        "      del label\n",
        "      del output\n",
        "      #gpu_usage()\n",
        "      torch.cuda.empty_cache()\n",
        "      #print(predictions, \"\\n\", targets, \"\\n\", correct)\n",
        "      total_size += BATCH_SIZE\n",
        "      accuracy = correct/(total_size)\n",
        "      #print(correct, total_size)\n",
        "      lrs.append(optimizer.param_groups[0]['lr'])\n",
        "    print('Mean Train loss:', np.mean(total_loss), 'Train Accuracy:', accuracy)\n",
        "    all_train_losses.append(np.mean(total_loss))\n",
        "    model.eval()\n",
        "    total_size = 0\n",
        "    total_loss = []\n",
        "    correct = 0\n",
        "    with torch.no_grad():\n",
        "        for data in valid_dataloader:\n",
        "            gc.collect()\n",
        "            torch.cuda.empty_cache()\n",
        "            inputs = data[\"image_data\"].to(device)\n",
        "            targets = data[\"label\"].to(device)\n",
        "\n",
        "            inputs = inputs.type(torch.cuda.FloatTensor)\n",
        "            targets = targets.type(torch.cuda.FloatTensor)\n",
        "            #print(ids.shape, \"ids\")\n",
        "            batch_size = inputs.size(0)\n",
        "\n",
        "            output = model.forward(inputs)\n",
        "            gc.collect()\n",
        "            del inputs\n",
        "            loss = criterion(output, targets)\n",
        "            total_loss.append(loss.item())\n",
        "            gc.collect()\n",
        "            del loss\n",
        "            torch.cuda.empty_cache()\n",
        "            output = torch.sigmoid(output)\n",
        "\n",
        "            predictions = torch.as_tensor((output - 0.5) > 0, dtype=torch.int32)\n",
        "            if (predictions == targets).float().sum().item() > batch_size:\n",
        "              print('error?')\n",
        "            correct += (predictions == targets).float().sum().item()\n",
        "            gc.collect()\n",
        "            del predictions\n",
        "            del targets\n",
        "            del output\n",
        "            torch.cuda.empty_cache()\n",
        "            total_size += batch_size\n",
        "            #gpu_usage()\n",
        "        accuracy = correct/(total_size)\n",
        "    all_valid_losses.append(np.mean(total_loss))\n",
        "    if np.sum(total_loss) < best_valid_loss:\n",
        "      best_valid_loss = np.sum(total_loss)\n",
        "      best_model_weights = copy.deepcopy(model.state_dict())\n",
        "      best_valid_epoch = epoch\n",
        "      path = save_name\n",
        "      torch.save(model.state_dict(), path)\n",
        "      print(f\"Model Saved\")\n",
        "    if len(all_valid_losses) > 1:\n",
        "      if np.abs(all_valid_losses[-1] - all_valid_losses[-2]) > 0.01:\n",
        "        count = 0\n",
        "      else:\n",
        "        count += 1\n",
        "    print(\"Validation Loss over a batch: {:.4f}; Validation Accuracy: {:.2f}%\".format(np.mean(total_loss), accuracy*100))\n",
        "    # if count > 2:\n",
        "    #   break\n",
        "  epochs = [i+1 for i in range(len(all_train_losses))]\n",
        "  plt.plot(epochs, all_train_losses, marker='o')\n",
        "  plt.plot(epochs, all_valid_losses, marker='o')\n",
        "  plt.xlabel('Epoch')\n",
        "  plt.ylabel('Loss')\n",
        "  plt.legend(['Train', 'Validation'], loc='upper right')\n",
        "  plt.show()"
      ],
      "metadata": {
        "id": "BUsNGLpVzv-t"
      },
      "execution_count": 20,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn.metrics import confusion_matrix"
      ],
      "metadata": {
        "id": "PmPYaXtgz4VW"
      },
      "execution_count": 21,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def test():\n",
        "  model.eval()\n",
        "  total_size = 0\n",
        "  total_loss = []\n",
        "  correct = 0\n",
        "  all_preds = []\n",
        "  all_targets = []\n",
        "  with torch.no_grad():\n",
        "      for data in test_dataloader:\n",
        "          gc.collect()\n",
        "          torch.cuda.empty_cache()\n",
        "          inputs = data[\"image_data\"].to(device)\n",
        "          targets = data[\"label\"].to(device)\n",
        "\n",
        "          inputs = inputs.type(torch.cuda.FloatTensor)\n",
        "          targets = targets.type(torch.cuda.FloatTensor)\n",
        "          #print(ids.shape, \"ids\")\n",
        "          batch_size = inputs.size(0)\n",
        "\n",
        "          output = model.forward(inputs)\n",
        "          gc.collect()\n",
        "          del inputs\n",
        "          loss = criterion(output, targets)\n",
        "          total_loss.append(loss.item())\n",
        "          gc.collect()\n",
        "          del loss\n",
        "          torch.cuda.empty_cache()\n",
        "          output = torch.sigmoid(output)\n",
        "\n",
        "          predictions = torch.as_tensor((output - 0.5) > 0, dtype=torch.int32)\n",
        "          if (predictions == targets).float().sum().item() > batch_size:\n",
        "            print('error?')\n",
        "          all_preds += predictions.flatten().cpu().detach().tolist()\n",
        "          all_targets += targets.flatten().cpu().detach().tolist()\n",
        "          correct += (predictions == targets).float().sum().item()\n",
        "          gc.collect()\n",
        "          del predictions\n",
        "          del targets\n",
        "          del output\n",
        "          torch.cuda.empty_cache()\n",
        "          total_size += batch_size\n",
        "          #gpu_usage()\n",
        "      accuracy = correct/(total_size)\n",
        "  print(\"Total Test Loss: {:.4f}; Test Accuracy: {:.2f}%\".format(np.sum(total_loss), accuracy*100))\n",
        "  print(pd.DataFrame(confusion_matrix(all_preds, all_targets), columns=['Test Funny', 'Test Not Funny'], index=['Pred Funny', 'Pred Not Funny']))"
      ],
      "metadata": {
        "id": "GHY707S3z5cp"
      },
      "execution_count": 22,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "train(num_epochs=10, save_name=\"best_model_convnext_last_layer.bin\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "lPsKya2-z6pq",
        "outputId": "8c2e1a57-e3d1-4a10-9ba9-7e456bd9a708"
      },
      "execution_count": 23,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "epoch: 0\n",
            "Mean Train loss: 0.6700390099298836 Train Accuracy: 0.5971534653465347\n",
            "Model Saved\n",
            "Validation Loss over a batch: 0.6550; Validation Accuracy: 59.93%\n",
            "epoch: 1\n",
            "Mean Train loss: 0.6103360883080133 Train Accuracy: 0.682549504950495\n",
            "Model Saved\n",
            "Validation Loss over a batch: 0.6396; Validation Accuracy: 60.67%\n",
            "epoch: 2\n",
            "Mean Train loss: 0.5837527026634405 Train Accuracy: 0.7153465346534653\n",
            "Model Saved\n",
            "Validation Loss over a batch: 0.6379; Validation Accuracy: 62.52%\n",
            "epoch: 3\n",
            "Mean Train loss: 0.5700939610452935 Train Accuracy: 0.6998762376237624\n",
            "Model Saved\n",
            "Validation Loss over a batch: 0.6378; Validation Accuracy: 61.04%\n",
            "epoch: 4\n",
            "Mean Train loss: 0.5480503115323511 Train Accuracy: 0.7308168316831684\n",
            "Validation Loss over a batch: 0.6386; Validation Accuracy: 62.52%\n",
            "epoch: 5\n",
            "Mean Train loss: 0.5387317815629562 Train Accuracy: 0.7357673267326733\n",
            "Validation Loss over a batch: 0.6403; Validation Accuracy: 62.15%\n",
            "epoch: 6\n",
            "Mean Train loss: 0.5343226583877413 Train Accuracy: 0.7289603960396039\n",
            "Validation Loss over a batch: 0.6482; Validation Accuracy: 62.89%\n",
            "epoch: 7\n",
            "Mean Train loss: 0.5275901609128064 Train Accuracy: 0.7475247524752475\n",
            "Validation Loss over a batch: 0.6413; Validation Accuracy: 63.82%\n",
            "epoch: 8\n",
            "Mean Train loss: 0.5203921753581208 Train Accuracy: 0.7543316831683168\n",
            "Validation Loss over a batch: 0.6459; Validation Accuracy: 63.82%\n",
            "epoch: 9\n",
            "Mean Train loss: 0.5188764079962627 Train Accuracy: 0.739480198019802\n",
            "Validation Loss over a batch: 0.6477; Validation Accuracy: 64.38%\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 640x480 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAkAAAAGwCAYAAABB4NqyAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAABlsklEQVR4nO3deZyNdf/H8dc5Z/YxMwxmNfbdWAfTUKlMUW6lUipFm+6EyN194y6khbrd9XMXN+W+pVJ3ooUi0iQtiIisY80yZsGYlVmcc35/HHMYBrNfM3Pez8fjPMa5ru+55nMZ5rzP9/u9vpfJbrfbEREREXEhZqMLEBEREalsCkAiIiLichSARERExOUoAImIiIjLUQASERERl6MAJCIiIi5HAUhERERcjpvRBVRFNpuNY8eO4efnh8lkMrocERERKQa73U5mZiZhYWGYzVfu41EAKsKxY8eIiIgwugwREREphSNHjtCgQYMrtlEAKoKfnx/g+Av09/c3uBoREREpjoyMDCIiIpzv41eiAFSEgmEvf39/BSAREZFqpjjTVzQJWkRERFyOApCIiIi4HAUgERERcTmaAyQiIjWa1WolPz/f6DKkHLi7u2OxWMrlWApAIiJSI9ntdpKSkkhLSzO6FClHtWvXJiQkpMzr9CkAiYhIjVQQfoKCgvDx8dHCttWc3W7n9OnTpKSkABAaGlqm4ykAiYhIjWO1Wp3hp27dukaXI+XE29sbgJSUFIKCgso0HKZJ0CIiUuMUzPnx8fExuBIpbwU/07LO61IAEhGRGkvDXjVPef1MNQRWiaw2OxsOppKSmUOQnxfdmwRiMes/p4iISGVTAKokK7YnMuXLnSSm5zi3hQZ4Mbl/W/pGlm0il4iIiJSMhsAqwYrtiQxfsLlQ+AFISs9h+ILNrNieaFBlIiJyNVabnXX7T7JkSwLr9p/EarMbXVKJNW7cmBkzZhhdRpWiHqAKZrXZmfLlTor672IHTMCUL3dyc9sQDYeJiFQxld17f7X5LZMnT+aFF14o8XE3btyIr69vKauqmRSAKtiGg6mX9PxcyA4kpuew4WAqMc10qaaISFVR0Ht/8QfYgt772Q92KfcQlJh4fkRg4cKFTJo0ifj4eOe2WrVqOf9st9uxWq24uV39rbx+/frlWmdNoCGwCpaSefnwU5p2IiJSOna7ndN5Z4v1yMzJZ/LSHZftvQd4YelOMnPyi3U8u714w2YhISHOR0BAACaTyfl89+7d+Pn58fXXXxMVFYWnpyc//fQT+/fv54477iA4OJhatWrRrVs3vv3220LHvXgIzGQy8Z///Ic777wTHx8fWrRowdKlS0v3F1tNqQeoggX5eZVrOxERKZ0z+VbaTlpZLseyA0kZObR/4Ztitd/5Yh98PMrnLXf8+PH885//pGnTptSpU4cjR45w22238corr+Dp6cn7779P//79iY+Pp2HDhpc9zpQpU/jHP/7B9OnTeeuttxg8eDCHDh0iMDCwXOqs6tQDVMG6NwkkNMCLy43qmnCMJ3dv4hr/4EREpGxefPFFbr75Zpo1a0ZgYCAdO3bkz3/+M5GRkbRo0YKXXnqJZs2aXbVH5+GHH+b++++nefPmTJ06laysLDZs2FBJZ2E89QBVMIvZxOT+bRm+YDMmKLI7dXL/tpoALSJSwbzdLex8sU+x2m44mMrD7268arv5j3Qr1gdYb/fyuYM5QNeuXQs9z8rK4oUXXmDZsmUkJiZy9uxZzpw5w+HDh694nA4dOjj/7Ovri7+/v/M+W65AAagS9I0MZfaDXS65ksDX08Lr93TUOkAiIpXAZDIVexjquhb1CQ3wIik9p8gPriYgJMCL61rUr/QPsBdfzfXss8+yatUq/vnPf9K8eXO8vb0ZOHAgeXl5VzyOu7t7oecmkwmbzVbu9VZVCkCVpG9kKDe3DWHDwVS+3ZXMf386iLe7hdg2wUaXJiIiF7lS731B3Kkqvfc///wzDz/8MHfeeSfg6BH6448/jC2qGtAcoEpkMZuIaVaXcX1bE+jrwYmsPH7Ye9zoskREpAgFvfchAYUvUgkJ8KqQS+BLq0WLFnz22Wds2bKFrVu38sADD7hUT05pqQfIAB5uZgZ0CmfezwdZ9OtRbmqtXiARkarowt77qnofxzfeeINHH32UHj16UK9ePcaNG0dGRobRZVV5JntxFydwIRkZGQQEBJCeno6/v3+FfI9diRnc+q8fcbeY+OXvsQT6elTI9xERcUU5OTkcPHiQJk2a4OWlZUZqkiv9bEvy/q0hMIO0CfWnXZg/+VY7S7ckGF2OiIiIS1EAMtA9UQ0AWLTpqMGViIiIuBYFIAPd0Skcd4uJHccy2HlM47UiIiKVRQHIQHV8PZyXwS9WL5CIiEilUQAy2D1dHcNgX2xJIO+sLlsUERGpDApABru+RX3q+3mSmp3Hd7tdZwlyERERIykAGczNYuauzuGAhsFEREQqiwJQFVAwDLY6PoXjmbkGVyMiIlLzKQBVAc2D/OgUURurzc4Xv2lNIBERKb0bbriBMWPGOJ83btyYGTNmXPE1JpOJL774oszfu7yOUxkUgKqIgefWBFq86ShanFtEpAqxWeHgj7BtseOrzVph36p///707du3yH0//vgjJpOJ33//vUTH3LhxI0888UR5lOf0wgsv0KlTp0u2JyYmcuutt5br96oohgegWbNm0bhxY7y8vIiOjmbDhg1XbJ+WlsaIESMIDQ3F09OTli1bsnz58kJtEhISePDBB6lbty7e3t60b9+eX3/9tSJPo8z6dwzD081MfHIm2xLSjS5HREQAdi6FGZHw3p/g08ccX2dEOrZXgMcee4xVq1Zx9Oilc0LfffddunbtSocOHUp0zPr16+Pj41NeJV5RSEgInp6elfK9ysrQALRw4ULGjh3L5MmT2bx5Mx07dqRPnz6kpBR9NVReXh4333wzf/zxB4sXLyY+Pp65c+cSHh7ubHPq1Cl69uyJu7s7X3/9NTt37uT111+nTp06lXVapRLg7U6fdiEALPpVk6FFRAy3cyl8MgQyjhXenpHo2F4BIehPf/oT9evXZ/78+YW2Z2VlsWjRIgYMGMD9999PeHg4Pj4+tG/fnv/9739XPObFQ2B79+7l+uuvx8vLi7Zt27Jq1apLXjNu3DhatmyJj48PTZs2ZeLEieTn5wMwf/58pkyZwtatWzGZTJhMJme9Fw+Bbdu2jZtuuglvb2/q1q3LE088QVZWlnP/ww8/zIABA/jnP/9JaGgodevWZcSIEc7vVZEMvRv8G2+8wbBhw3jkkUcAmDNnDsuWLWPevHmMHz/+kvbz5s0jNTWVtWvX4u7uDjh+sBd67bXXiIiI4N1333Vua9KkyRXryM3NJTf3/ORjo+6iOzCqAUu3HmPp1mM8168NXu4WQ+oQEamR7HbIP128tjYrfP03oKgpCXbABCvGQdMbwFyM39XuPmC6+h3k3dzcGDJkCPPnz+e5557DdO41ixYtwmq18uCDD7Jo0SLGjRuHv78/y5Yt46GHHqJZs2Z079796qdls3HXXXcRHBzML7/8Qnp6eqH5QgX8/PyYP38+YWFhbNu2jWHDhuHn58ff/vY3Bg0axPbt21mxYgXffvstAAEBAZccIzs7mz59+hATE8PGjRtJSUnh8ccfZ+TIkYUC3urVqwkNDWX16tXs27ePQYMG0alTJ4YNG3bV8ykLwwJQXl4emzZtYsKECc5tZrOZ2NhY1q1bV+Rrli5dSkxMDCNGjGDJkiXUr1+fBx54gHHjxmGxWJxt+vTpwz333MOaNWsIDw/nqaeeuuJf5LRp05gyZUr5nmAp9Gxej9AALxLTc/h2VzJ/6hBmdEkiIjVH/mmYWl6/V+2OnqFXI4rX/O/HwMO3WE0fffRRpk+fzpo1a7jhhhsAx/DX3XffTaNGjXj22WedbUeNGsXKlSv55JNPihWAvv32W3bv3s3KlSsJC3P8XUydOvWSeTvPP/+888+NGzfm2Wef5eOPP+Zvf/sb3t7e1KpVCzc3N0JCQi77vT766CNycnJ4//338fV1nPvMmTPp378/r732GsHBjjsh1KlTh5kzZ2KxWGjdujX9+vUjLi6uwgOQYUNgJ06cwGq1Ov8CCgQHB5OUlFTkaw4cOMDixYuxWq0sX76ciRMn8vrrr/Pyyy8XajN79mxatGjBypUrGT58OE8//TTvvffeZWuZMGEC6enpzseRI0fK5yRLyGI2cVcXrQkkIuLKWrduTY8ePZg3bx4A+/bt48cff+Sxxx7DarXy0ksv0b59ewIDA6lVqxYrV67k8OHDxTr2rl27iIiIcIYfgJiYmEvaLVy4kJ49exISEkKtWrV4/vnni/09LvxeHTt2dIYfgJ49e2Kz2YiPj3dua9eunbMTAyA0NPSyU2HKk6FDYCVls9kICgrinXfewWKxEBUVRUJCAtOnT2fy5MnONl27dmXq1KkAdO7cme3btzNnzhyGDh1a5HE9PT2rzKStgVERzFq9nx/2HCcpPYeQAC+jSxIRqRncfRw9McVxaC18OPDq7QYvhkY9ive9S+Cxxx5j1KhRzJo1i3fffZdmzZrRq1cvXnvtNf71r38xY8YM2rdvj6+vL2PGjCEvL69Ex7+SdevWMXjwYKZMmUKfPn0ICAjg448/5vXXXy+373GhgiktBUwmEzZbxd8ayrAeoHr16mGxWEhOTi60PTk5+bJdaqGhobRs2bJQUmzTpg1JSUnOH35oaCht27Yt9Lo2bdqUOLkapUk9X7o1roPNDp/9pl4gEZFyYzI5hqGK82h2E/iHAZebt2MC/3BHu+Icrxjzfy507733Yjab+eijj3j//fd59NFHMZlM/Pzzz9xxxx08+OCDdOzYkaZNm7Jnz55iH7dNmzYcOXKExMRE57b169cXarN27VoaNWrEc889R9euXWnRogWHDh0q1MbDwwOr9crLAbRp04atW7eSnZ3t3Pbzzz9jNptp1apVsWuuKIYFIA8PD6KiooiLi3Nus9lsxMXFFdkdB46us3379hVKhnv27CE0NBQPDw9nmwu71graNGrUqALOomJoTSAREYOZLdD3tXNPLg4v5573fbV4E6BLoVatWgwaNIgJEyaQmJjIww8/DECLFi1YtWoVa9euZdeuXfz5z3++pCPhSmJjY2nZsiVDhw5l69at/Pjjjzz33HOF2rRo0YLDhw/z8ccfs3//ft58800+//zzQm0aN27MwYMH2bJlCydOnCh0IVGBwYMH4+XlxdChQ9m+fTurV69m1KhRPPTQQ5dMfzGCoZfBjx07lrlz5/Lee++xa9cuhg8fTnZ2tvOqsCFDhhSaJD18+HBSU1MZPXo0e/bsYdmyZUydOpURI0Y42zzzzDOsX7+eqVOnsm/fPj766CPeeeedQm2qun4dwvB2t3DgeDabD6cZXY6IiGtqezvc+z74hxbe7h/m2N729gr99o899hinTp2iT58+zjk7zz//PF26dKFPnz7ccMMNhISEMGDAgGIf02w28/nnn3PmzBm6d+/O448/ziuvvFKoze23384zzzzDyJEj6dSpE2vXrmXixImF2tx999307duXG2+8kfr16xd5Kb6Pjw8rV64kNTWVbt26MXDgQHr37s3MmTNL/pdRAUx2g7sYZs6cyfTp00lKSqJTp068+eabREdHA47lvBs3blzocrl169bxzDPPsGXLFsLDw3nssccKXQUG8NVXXzFhwgT27t1LkyZNGDt2bIlmk2dkZBAQEEB6ejr+/v7ldq7YrI5x5axkqBXsGDe+zKeHsZ9s4bPNCdzfPYJpd5Vs0SsREVeXk5PDwYMHadKkCV5eZZxLWYLf3VLxrvSzLcn7t+EBqCqqkAC0c6ljzYgLF9TyD3N0sRbxKWLt/hM8MPcX/Dzd2PBcLN4e+s8mIlJc5RqApEoprwBk+K0wXEIpVhO9pkldGtTxJjP3LCt3FL0sgIiIiJSOAlBFs1kdPT+XXU0UWDH+kpvrmc0m52ToRZuMWZdIRESkplIAqmiH1l7a81OIHTISHO0ucncXRwBau/8kR08Vc/l2ERERuSoFoIqWVczLE4toFxHoQ0zTutjt8NnmhHIuTESk5tM015qnvH6mCkAVrVYx1zq4TLt7up5fE8hm039kEZHiKFhd+PRp9Z7XNAU/04tXkC6panUrjGqpUQ/H1V4ZiRQ9DwjHEunBkUXu6hsZwqQlOzicepqNf6QS3bRuxdUqIlJDWCwWateu7bynlI+Pj/PO6lI92e12Tp8+TUpKCrVr1y60/E1pKABVtILVRD8ZgmP10CJCUP5pePt6GPBvaHJdoV0+Hm70ax/Kwl+PsGjTUQUgEZFiKritUmXcWFMqT+3ata94F/ri0jpARai8dYDCIeph+G0BpJ27z8o1T0HvSeDu7Wz26x+pDJyzDh8PCxufi8XXU7lVRKS4rFYr+fn5Rpch5cDd3f2KPT9aCLGMKn0l6NxM+OZ52DTf0a5eS7jzbQjvAji6/W56fQ0HT2QzfWAH7ukaUX41iYiI1BBaCLGqMlscQ1ztBzq+Fiyl7ukH/f8FDyyCWiFwYg/8JxZWTwNrPibThWsC6Q7xIiIiZaUAVJW0vAWeWgft7gK7Fda86ghCKbu5q0s4JhNsOJjKoZPZRlcqIiJSrSkAVTU+gXDPu3D3f8GrNiRugbevJ3THf7iuWR0APlUvkIiISJkoAFVV7QfCU+uh+c1gzYVvnueNMxNpYErh080JWhNIRESkDBSAqjL/UBi8yDE/yKMW9U7+ykrP8VybuZx1+08YXZ2I1GQ2Kxz8EbYtdny96H6FItWdrgIrQoVdBVYWqQfhi6fgsOOeYTt8r6Hdk++BX9nXQhARKaTIZTvCHGuatb3duLpErkJXgdVEgU3g4a9I6P53cu1utMtej23WNbD9M6MrE5GaZOdSx8KtF9/EOSPRsX3nUmPqkpqjivQuKgBVJ2YLYbf+jadqzWCbrTHmnFOw+BFY/CicTjW6OhGp7mxWR89PkbftObdtxXgNh0np7VwKMyLhvT/Bp485vs6INCRYKwBVMyaTie7de3Bn3oss9HkATBbY/in8Owb2rjK6PBGpzg6tvbTnpxA7ZCQ42omUVBXrXVQAqobu7ByO3ezOuNQ/ceSupVC3BWQlwYcD4cvRkJtldIkiUh2d3F+8dknbKrYOqVnsdjh9CpY/S1XqXdRNpaqhIH8verWsz3e7U/gooR7jnvwRvp0Cv8x23E5j/2q4c47jVhsiIleSdxr2rIBti2DPyuK9ZuUE+O0DaHWb4xHWGcz6PF3hLnc7pcpkt0NeFpw+ee5x6oI/n4Qzqef+fNFX29XuxXZB7+JFNwWvKLoKrAhV8iqwi3y9LZHhH24m2N+TteN7YzGb4OAPjivF0o8AJugxEm58Hty9jC5XRKoS61k4+L1jEuquLx1vaAXM7ld+s7J4gjUfsJ3fVisEWt3qCENNrtfvnIpQEVfmOcPMRWHFGWJOXrD9gn3WvPI5p6Lc/V/HOnilpJuhllF1CEB5Z21ET/2WU6fzefeRbtzYKsixIycDVkyALQscz+u3gbvehtCOxhUrIsaz2yFhk6OnZ/tnkJ1yfl/thtD+HsfjxF7HfAzHiy44gMnx5d73ofG1jjmH8cth37eFA5S7LzTvDa37QYtbHKvbS9kUzJ25ZPjogp9Jm/6Ql31RT0zqRSHmgj8XhJzShhk3L/Cp6/j5+tQF73NfnY/A8/t86kLKbvjw7qsfd+hXZeoBUgAqo+oQgABeWLqD+Wv/oF+HUGY90KXwzt3L4cunIfs4mN2g1zi4dixYNOop4lJO7IXfP3EEn1MHz2/3qQvt7oT290JEdzCZzu8rsrchHPq+emlvw9lcx6XM8csg/mvITDy/z2SBhjHQ+jZHD1Fg04o5x5rMZnVcJXWlyekms+Pv+qrDTJdh8QTfeudCTOBFIebcn73rFA44Hj6lPI9Eip4HZHL0aI3ZVqZhPQWgMqouAWh7Qjp/eusnPCxmNjzXm9o+HoUbZJ+Ar56BXedm1od1gTvfhvotK79YEak8GYmOq0O3fQKJW89vd/dx9My0vxea3QgW98sfozTzTex2OPabo2co/mtI3l54f/0258LQbY7fR5o3VLTMJMff47HfYF8cJPxa/NdaPC/fC+Pspbnoq7tP4QBcUZw9WXDZ3sUyLrSpAFRG1SUAAdz6rx/ZlZjBi3e0Y0hM40sb2O2OT37LnoXcdEe3ZewU6P6EfvmI1CRn0hzzebZ94uiRKXiDMbtBs96O4a3Wt4GHb+XVdOoPRxCKXw5//Az2C67wqRUCrfqemzfUy3XnDWWfPB92Ch6ZV1qK4DL6vgZdHqq8MFNaJeldLAUFoDKqTgHovz8d5KWvdtKhQQBLR157+YbpCbBkBBxY7Xje+DoY8G/H2L+IVE/5ObB35bkruL5x3Di5QMQ1jsmk7e50DG8Y7cyp8/OG9n4LeZnn97n7QvOboFU/aNmn5s4bOpPm6JE7tvl82Ek7fGk7kxnqtXJcXeflD7/Mufqxyzh3plJV4NVsCkBlVJ0C0MmsXKKnxnHWZmflmOtpFeJ3+cZ2O/z6X/hmIuSfBg8/uPVV6DS4an9iEJHzbFb440f4fZFjeDs34/y++m2gwz0QeTfUaWxYiVd1NtdxDrvPDZVd2ONhMjvmDbU6N2+objPj6iyL3CxI+t0RchLOBZ7Uy6yzVLe5I+yEdXYMDYa0B89ajn2VNHemplAAKqPqFIAA/vzBr6zckcyw65rwXL+2V3/Byf3wxXA48ovjeavbHHecrxVUsYWKSOnY7ZC4xRF6tn/qWPi0gH8DaH+3Y15PcLvq92Gm4Nx2L3f0Dl0yb6j1+fWGwqOq5tB9/hlI2n7BMNZmOB5PkYGldqMLwk5nCOsEXgFXPn4lzJ2pKRSAyqi6BaBVO5MZ9v6v1KvlwboJvXG3FOMXhM0Ka9+E715xXDngUxf+9H/Q9o6KL1hEiufkfsdaPdsWwcm957d71YZ2Axyhp2FM1QwFpXXq0Ll5Q8uKmDcUDC3PzRtq2gvcvSu/vrN5kLKj8JydlF1gO3tpW//w8yEnrDOEdgbfuqX7vhU8d6amUAAqo+oWgPKtNmKmxXEiK4+5Q7pyc9vg4r84aTt8/ufzn7o6DIJb/wHetSukVhG5isxk2PGZI/QkbDq/3c3L8cbf/h5oHgtuHpc/Rk1x5pRjvlD8siLmDflAs5vOrTfU5/LBoizzTaxn4UT8+SGsY785flcWtXaOTz0I7+IYwioIPX4hJT7lK6oKK0FXcQpAZVTdAhDAK8t2MvfHg/RpF8zbD3Ut2YvP5sGaV+Gn/wO7DfzC4I6ZjsXMRKTi5WTA7q8c6/UcXOP4fwiO+TBNb4QO9zre6D2vMMevpjubC3/8dP4S+4yE8/tMZsek74JL7AvmDZVk9WSbDU7uOz+Edew3SPwdzp65tBav2o6QE97l/FCWf3j1G36sgRSAyqg6BqD4pEz6zPgBN7OJX/7em7q1PEt+kCMb4PMnz0/U6/Y43Pxi5V42K1LdFfdT+tlcxyrKv3/iuBfX2Zzz+8K7OkJPuzs1N68odrvjaqr45Y65Q8kX3Zy1Xiuo18IRKi9xLqTcNt1xtdmx3yDhN8fxLuxhKuDhd24Iq9P53p06jRV2qigFoDKqjgEI4PaZP/H70XQm/qktj13bpHQHycuGVZNh41zH88CmMGAONIwuv0JFaqqr9TjYbHDoZ8fw1s4lkJN2vl3dFo7Q036gVkwuqbTDjl6h3cscf79FzccpDjdvx22DCnp1wrtAYLOaNceqhivJ+3eV+KnOmjWLxo0b4+XlRXR0NBs2bLhi+7S0NEaMGEFoaCienp60bNmS5cuXF9n21VdfxWQyMWbMmAqovGoZGNUAgMWbjpb+IB6+0O+f8NAXji7d1APwbl/49gXHJ1ZwfMI9+KNjcubBHx3PRVxdwZU6F9+yICMRPnkI/ncf/F87eO9PsPk9R/jxC4WYkfDEGhi5EXr9TeGnNGo3hOg/w9Cl8Nf9cP1fi/e6ei0dPd13zILha2HCUXhspWN5kI6DHL1ICj81luE3hlq4cCFjx45lzpw5REdHM2PGDPr06UN8fDxBQZd2/ebl5XHzzTcTFBTE4sWLCQ8P59ChQ9SuXfuSths3buTtt9+mQ4cOlXAmxru9Yxgvf7WLXYkZbE9IJzL8KpdWXkmzGx2/EL4eB79/7JgftHcVdLwf1s8q3zsSi1R3Nquj56fIdVrObYv/2vHVM8Dxf6X9PY6bimoSa/nyru24dL44eo0r053HpXozPAC98cYbDBs2jEceeQSAOXPmsGzZMubNm8f48eMvaT9v3jxSU1NZu3Yt7u6O+9g0btz4knZZWVkMHjyYuXPn8vLLL1foOVQVtX08uLldMMt+T2TxpqNlC0Dg+EVy19uOyZdfjXFc/fDNc5e2y0h0fPLVWhRSFlX5Cpf8M44rkgruon3xn5N3XflmlQVu/Dv0GO26t32oLLWKeSVscdtJjWRoAMrLy2PTpk1MmDDBuc1sNhMbG8u6deuKfM3SpUuJiYlhxIgRLFmyhPr16/PAAw8wbtw4LJbzvyxHjBhBv379iI2NvWoAys3NJTf3/BLyGRkZV2hdtQ2MasCy3xP5YksCE25rjadbObyBtL0dGnSDNzsVnqjpZAdMjk/ArftVnTetq6nKb7glURPOoyRX65SF9awjsJw55Qgvp1Ov8udzQaeoK4FKI7CZwk9laNTD8e/naqsnN+pR2ZVJFWJoADpx4gRWq5Xg4MIpPDg4mN27dxf5mgMHDvDdd98xePBgli9fzr59+3jqqafIz89n8uTJAHz88cds3ryZjRs3FquOadOmMWXKlLKdTBVxfYv6BPt7kpyRy3e7Uri1fWj5HPjkvsuEnwJ2x5vXK6GO+w55Bzp6kLzrOK608K5z7nHhny/Y51aKq9bKorLecCtaTTgP5yq3F71RXaln0W6HnPRzYaW4gSbNcUPg0jJZLvi3HFj433VuBmx+/+rHUI9D5TBbHP8HPhmC46qvIlZP7vtq9fugIOXK8CGwkrLZbAQFBfHOO+9gsViIiooiISGB6dOnM3nyZI4cOcLo0aNZtWoVXl7F+6Q1YcIExo4d63yekZFBRERERZ1ChbKYTdzVpQGzv9/Pok1Hyy8AZSUXr50117E+x4VrdBSHu88F4aj2RaHpouB04b7SBKfSvOFWRdXxPGw2R5A+m+MYVsrLhmVjueLcmc//DL8tcEwadg47pRVeIbikvALO/5vyCbz0zz4FAf6CoOPpf/lLn21WxyXt6nGoOtre7vg/UOQHBK2eLAYHoHr16mGxWEhOLvzmmpycTEhI0StohoaG4u7uXmi4q02bNiQlJTmH1FJSUujSpYtzv9Vq5YcffmDmzJnk5uYWei2Ap6cnnp6V3ANRgQZGOQLQmj3HScnIIci/HLrci/vJ9a65jkXIzpxyvEldOJRw5qJhhTOnHG9qdpvj5qz5pyGjhFewufteEJJqX7636cI3sa+vNFnVBCvGV/2hvKtOui3GeVwcRpxfcx1DPvk5hb+ezb2o3ZXaX3zcc/uKWkH3avJPO+54XpSC4Oxzwc+8yF6awPPhxisALOX8q089DlVT29sd/weq+xCxVAhDA5CHhwdRUVHExcUxYMAAwNHDExcXx8iRI4t8Tc+ePfnoo4+w2WyYz12euGfPHkJDQ/Hw8KB3795s21Z4UaxHHnmE1q1bXzJPqKZqVr8WXRrWZvPhND7/LYE/9yqHuykXd0w98u6S/XKx2RzDB4XC0akrh6ZCwSnb8ShpcLosu6P3as51524HYjr/qd9kuuD5ua8m86XbLvl6mdcW+xhcuj0z6SqTbs+dx+yejl6y8goj5cnsBiY3sF5paPWcLkOg+c2X9thUpfk06nGomswWaHKd0VVIFWT4ENjYsWMZOnQoXbt2pXv37syYMYPs7GznVWFDhgwhPDycadOmATB8+HBmzpzJ6NGjGTVqFHv37mXq1Kk8/fTTAPj5+REZGVnoe/j6+lK3bt1Lttdk93SNYPPhNBZtOsoT1zfFVNZVSyvqE67ZfG6ooXbJXmezOeZzlCQ0FWwvjpQdJaunqjq+q3jtzG6OReDcvRz3nHLzOvdn7/Nf3TwdN59087roq2fhdsVtb3FzrCP13p+uXl/7e6vHm5h6HESqDcMD0KBBgzh+/DiTJk0iKSmJTp06sWLFCufE6MOHDzt7egAiIiJYuXIlzzzzDB06dCA8PJzRo0czbtw4o06hSurXIZQpX+5gX0oWW4+m0ymidtkPWpU+4ZrN54e3SuLAD/B+/6u36zXu3FoidseEWzj31V7EV9sV9tnL8Ricf37qD9j60dXP44YJjhVtLwkhF4Wc8h4SKq6aeLWOehxEqgXdCqMI1fVWGBcb8/FvfLHlGIOjG/LKne3L78DV+bJrmxVmRF79DXfMtqp9TjXlPOCCydxQZM9iVZzMLSJVUrW7FYZUjHu6Oq5kW7r1GDn55Xi7ioJPuO0HOr5W9TfYCxUM5QHON1inajRZtaacB5zvWfS/6IpF/zCFHxGpMOoBKkJN6QGy2exc94/VJKSd4c37O3N7xzCjS6o6ilw/J7z6TVatKecB1btnUUSqBN0NvoxqSgACeOObeN78bh/XtajHB4/pju6F1JQ33JpyHiIiZVSS92/DJ0FLxbo7qgFvfrePn/adIDH9DKEB3kaXVHXUlMmqNeU8REQqkeYA1XCN6vrSvUkgdjt8trmEqzOLiIjUUApALuCeqAYALPr1CBrxFBERUQByCbe1D8XHw8IfJ0+z6VAxFwIUERGpwRSAXICvpxu3nbsp6qJfy+uWESIiItWXApCLKBgG++r3Y5zOO2twNSIiIsZSAHIR3ZsE0jDQh+w8Kyu2JxldjoiIiKEUgFyEyWRioHMytIbBRETEtSkAuZC7oxpgMsG6Ayc5knra6HJEREQMowDkQsJre9OjWV0APt2sXiAREXFdCkAu5p4oxw1SF286is2mNYFERMQ1KQC5mD7tQvDzdOPoqTOsP3jS6HJEREQMoQDkYrw9LPypo2NNoMWbNAwmIiKuSQHIBQ08Nwz29bYksnK1JpCIiLgeBSAX1KVhbZrW9+VMvpVlvx8zuhwREZFKpwDkgi5cE0jDYCIi4ooUgFzU3V0aYDbBxj9OcfBEttHliIiIVCoFIBcV7O/F9S3rA/CpeoFERMTFKAC5sIJhsE83H8WqNYFERMSFKAC5sNg2wQR4u5OYnsPP+04YXY6IiEilUQByYV7uFu7oFAZoMrSIiLgWBSAXVzAMtnJHEuln8g2uRkREpHIoALm49uEBtAr2I/esjS+3ak0gERFxDQpALk5rAomIiCtSABIGdA7HYjax5Uga+1IyjS5HRESkwikACfX9PLmxVRAAi35VL5CIiNR8CkACnJ8M/dlvCZy12gyuRkREpGIpAAkAN7UOItDXg+OZufyw97jR5YiIiFQoBSABwMPNzIBO4YCGwUREpOZTABKngmGwb3clcyo7z+BqREREKo4CkDi1DfOnXZg/+VY7S7YkGF2OiIhIhakSAWjWrFk0btwYLy8voqOj2bBhwxXbp6WlMWLECEJDQ/H09KRly5YsX77cuX/atGl069YNPz8/goKCGDBgAPHx8RV9GjXCPed6gRZpTSAREanBDA9ACxcuZOzYsUyePJnNmzfTsWNH+vTpQ0pKSpHt8/LyuPnmm/njjz9YvHgx8fHxzJ07l/DwcGebNWvWMGLECNavX8+qVavIz8/nlltuITs7u7JOq9q6vVM47hYTO45lsPNYhtHliIiIVAiT3W63G1lAdHQ03bp1Y+bMmQDYbDYiIiIYNWoU48ePv6T9nDlzmD59Ort378bd3b1Y3+P48eMEBQWxZs0arr/++kv25+bmkpub63yekZFBREQE6enp+Pv7l/LMqq/hCzbx9fYkHu3ZhEn92xpdjoiISLFkZGQQEBBQrPdvQ3uA8vLy2LRpE7Gxsc5tZrOZ2NhY1q1bV+Rrli5dSkxMDCNGjCA4OJjIyEimTp2K1Wq97PdJT08HIDAwsMj906ZNIyAgwPmIiIgow1lVf/d0dQyDfbElgbyzWhNIRERqHkMD0IkTJ7BarQQHBxfaHhwcTFJSUpGvOXDgAIsXL8ZqtbJ8+XImTpzI66+/zssvv1xke5vNxpgxY+jZsyeRkZFFtpkwYQLp6enOx5EjR8p2YtXc9S3qU9/Pk9TsPFbHFz0UKSIiUp25GV1ASdlsNoKCgnjnnXewWCxERUWRkJDA9OnTmTx58iXtR4wYwfbt2/npp58ue0xPT088PT0rsuxqxc1i5q7O4bz9wwEW/XqUPu1CjC5JRESkXBnaA1SvXj0sFgvJycmFticnJxMSUvSbbmhoKC1btsRisTi3tWnThqSkJPLyCq9dM3LkSL766itWr15NgwYNyv8EarCCNYFWx6dwPDP3Kq1FRESqF0MDkIeHB1FRUcTFxTm32Ww24uLiiImJKfI1PXv2ZN++fdhs5+em7Nmzh9DQUDw8PACw2+2MHDmSzz//nO+++44mTZpU7InUQC2C/egYURurTWsCiYhIzWP4ZfBjx45l7ty5vPfee+zatYvhw4eTnZ3NI488AsCQIUOYMGGCs/3w4cNJTU1l9OjR7Nmzh2XLljF16lRGjBjhbDNixAgWLFjARx99hJ+fH0lJSSQlJXHmzJlKP7/qzLkm0K9HMfhiQRERkXJl+BygQYMGcfz4cSZNmkRSUhKdOnVixYoVzonRhw8fxmw+n9MiIiJYuXIlzzzzDB06dCA8PJzRo0czbtw4Z5vZs2cDcMMNNxT6Xu+++y4PP/xwhZ9TTdG/QxgvfrWT+ORMtidk0L5BgNEliYiIlAvD1wGqikqyjkBNN+p/v/Hl1mMMiWnEi3cUfRWdiIhIVVBt1gGSqq9gGGzJlmPk5F9+rSUREZHqRAFIrqhn83qEBniRfiafuF1aE0hERGoGBSC5IovZxF1dHPdZW7TJtReIFBGRmkMBSK5qYJTj1iA/7DlOUnqOwdWIiIiUnQKQXFWTer50bVQHmx0+/01rAomISPWnACTFUnCD1EWbjmhNIBERqfYUgKRY+nUIw9vdwoHj2Ww+nGZ0OSIiImWiACTFUsvTjVsjHfdnW7zpqMHViIiIlI0CkBTbwHPDYF9tPcaZPK0JJCIi1Zfht8KQ6uOaJnVpUMebo6fOMGv1XloE+xHk50X3JoFYzCajyxMRESk2BSApNrPZRMcGtTl66gwzV+93bg8N8GJy/7b0jQw1sDoREZHi0xCYFNuK7Yks25Z4yfak9ByGL9jMiu2X7hMREamKFICkWKw2O1O+3FnkvoKL4qd8uROrTZfIi4hI1acAJMWy4WAqiVdYBdoOJKbnsOFgauUVJSIiUkoKQFIsKZnFuwVGcduJiIgYSQFIiiXIz6tc24mIiBhJAUiKpXuTQEIDvLjSxe6hAY5L4kVERKo6BSApFovZxOT+bQEuG4J6Nqur9YBERKRaUACSYusbGcrsB7sQElB4mMvfy7Gc1Ke/JbBqZ7IRpYmIiJSIya5be18iIyODgIAA0tPT8ff3N7qcKsdqs7PhYCopmTkE+XnRrXEdJi/dwYe/HMbb3cKiJ2OIDA8wukwREXExJXn/VgAqggJQyeVbbTw6fyM/7j1BiL8XX4zoeUlPkYiISEUqyfu3hsCkXLhbzMx8oAvNg2qRlJHD4+9v5HTeWaPLEhERKZICkJSbAG935g3tRqCvB9sTMhjz8RZsWhlaRESqIAUgKVcN6/rwzkNReFjMfLMzmddW7ja6JBERkUsoAEm569o4kH8M7ADA22sOsHDjYYMrEhERKUwBSCrEgM7hPN27BQDPfb6dtftPGFyRiIjIeQpAUmGeiW1B/45hnLXZGb5gMweOZxldkoiICKAAJBXIZDIxfWAHOjesTfqZfB6dv5FT2XlGlyUiIqIAJBXLy93COw91Jby2N3+cPM2TCzaRd9ZmdFkiIuLiFICkwtX382Tew92o5enGLwdT+fvn29D6myIiYiQFIKkUrUL8mPlAZ8wmWLzpKLPX7De6JBERcWEKQFJpbmgVxAu3twPgHyvi+XpbosEViYiIq1IAkko1JKYxD/doDMAzn2xh65E0Q+sRERHXVCUC0KxZs2jcuDFeXl5ER0ezYcOGK7ZPS0tjxIgRhIaG4unpScuWLVm+fHmZjimV5/l+bbihVX1y8m08/v6vJKSdMbokERFxMYYHoIULFzJ27FgmT57M5s2b6dixI3369CElJaXI9nl5edx888388ccfLF68mPj4eObOnUt4eHipjymVy81i5q37O9Mq2I/jmbk8Nn8jWbm6caqIiFQek93gy3Gio6Pp1q0bM2fOBMBmsxEREcGoUaMYP378Je3nzJnD9OnT2b17N+7u7uVyzItlZGQQEBBAeno6/v7+ZTg7uZKjp04zYNZaTmTlclPrIOYO6YrFbDK6LBERqaZK8v5taA9QXl4emzZtIjY21rnNbDYTGxvLunXrinzN0qVLiYmJYcSIEQQHBxMZGcnUqVOxWq2lPmZubi4ZGRmFHlLxGtTx4T9Du+LpZua73Sm8smyX0SWJiIiLMDQAnThxAqvVSnBwcKHtwcHBJCUlFfmaAwcOsHjxYqxWK8uXL2fixIm8/vrrvPzyy6U+5rRp0wgICHA+IiIiyuHspDg6RdTmjXs7ATDv54MsWH/I2IJERMQllCoAHTlyhKNHjzqfb9iwgTFjxvDOO++UW2GXY7PZCAoK4p133iEqKopBgwbx3HPPMWfOnFIfc8KECaSnpzsfR44cKceK5Wr6dQjl2VtaAjB56Q5+2HPc4IpERKSmK1UAeuCBB1i9ejUASUlJ3HzzzWzYsIHnnnuOF198sdjHqVevHhaLheTk5ELbk5OTCQkJKfI1oaGhtGzZEovF4tzWpk0bkpKSyMvLK9UxPT098ff3L/SQyjXixubc1SUcq83OiA83szc50+iSRESkBitVANq+fTvdu3cH4JNPPiEyMpK1a9fy4YcfMn/+/GIfx8PDg6ioKOLi4pzbbDYbcXFxxMTEFPmanj17sm/fPmy28/eT2rNnD6GhoXh4eJTqmGI8k8nEtLva071xIJm5Z3n0vY2czMo1uiwREamhShWA8vPz8fT0BODbb7/l9ttvB6B169YkJpZsdd+xY8cyd+5c3nvvPXbt2sXw4cPJzs7mkUceAWDIkCFMmDDB2X748OGkpqYyevRo9uzZw7Jly5g6dSojRowo9jGlavJ0szDnoSga1fXhSOoZnvhgEzn5VqPLEhGRGsitNC9q164dc+bMoV+/fqxatYqXXnoJgGPHjlG3bt0SHWvQoEEcP36cSZMmkZSURKdOnVixYoVzEvPhw4cxm8/ntIiICFauXMkzzzxDhw4dCA8PZ/To0YwbN67Yx5SqK9DXg/8O7cZd//6ZTYdOMe7T35kxqBMmky6PFxGR8lOqdYC+//577rzzTjIyMhg6dCjz5s0D4O9//zu7d+/ms88+K/dCK5PWATLez/tOMHTeBs7a7DwT25LRsS2MLklERKq4krx/l3ohRKvVSkZGBnXq1HFu++OPP/Dx8SEoKKg0h6wyFICqhv9tOMyEz7YB8Ob9nbm9Y5jBFYmISFVW4QshnjlzhtzcXGf4OXToEDNmzCA+Pr7ahx+pOu7v3pBh1zUB4NlFW9l06JTBFYmISE1RqgB0xx138P777wOOG5NGR0fz+uuvM2DAAGbPnl2uBYprG39rG2LbBJN31sYT7//KkdTTRpckIiI1QKkC0ObNm7nuuusAWLx4McHBwRw6dIj333+fN998s1wLFNdmMZv4132daBvqz8nsPB57byMZOflGlyUiItVcqQLQ6dOn8fPzA+Cbb77hrrvuwmw2c80113DokG5lIOXL19ON/z7clWB/T/YkZzHiw82ctdqu/kIREZHLKFUAat68OV988QVHjhxh5cqV3HLLLQCkpKRo0rBUiNAAb/4zpBve7hZ+3HuCKV/upJTz90VEREoXgCZNmsSzzz5L48aN6d69u3OF5W+++YbOnTuXa4EiBdo3COD/BnXCZIIP1h9i/to/jC5JRESqqVJfBp+UlERiYiIdO3Z0LlS4YcMG/P39ad26dbkWWdl0GXzVNmfNfl79ejdmE/xnaFduaq0FLkVEpJLWASpQcFf4Bg0alOUwVYoCUNVmt9sZ/+k2Fv56BF8PC4uH96BNqH5OIiKursLXAbLZbLz44osEBATQqFEjGjVqRO3atXnppZcK3aRUpCKYTCZeGhBJTNO6ZOdZeWz+RlIyc4wuS0REqpFSBaDnnnuOmTNn8uqrr/Lbb7/x22+/MXXqVN566y0mTpxY3jWKXMLDzcycB6NoWs+XY+k5DHtfN04VEZHiK9UQWFhYGHPmzHHeBb7AkiVLeOqpp0hISCi3Ao2gIbDq448T2Qz498+knc6nX/tQ3rq/M2azbpwqIuKKKnwILDU1tciJzq1btyY1NbU0hxQplcb1fHn7wSjcLSaWbUvkjVV7jC5JRESqgVIFoI4dOzJz5sxLts+cOZMOHTqUuSiRkohuWpdpdzn+3c1cvY9PNx01uCIREanq3Erzon/84x/069ePb7/91rkG0Lp16zhy5AjLly8v1wJFimNgVAMOHM/i39/vZ/xnvxMR6EP3JoFGlyUiIlVUqXqAevXqxZ49e7jzzjtJS0sjLS2Nu+66ix07dvDBBx+Ud40ixfLsLa24rX0I+VY7f/7gV/44kW10SSIiUkWVeR2gC23dupUuXbpgtVbvq3E0Cbr6OpNn5b531rH1aDpN6/vy+fCeBPi4G12WiIhUggqfBC1SVXl7WJg7pCthAV4cOJ7N8A83ka8bp4qIyEUUgKTGCfL34j9Du+HrYWHt/pNM/GK7bpwqIiKFKABJjdQ2zJ837++M2QQfbzzC3B8PGF2SiIhUISW6Cuyuu+664v60tLSy1CJSrnq3Cea5fm156audTPt6N43q+tKnXYjRZYmISBVQogAUEBBw1f1DhgwpU0Ei5enRno05cDyLD385zJiPt7DoyRgiw6/871hERGq+cr0KrKbQVWA1S77VxqPzN/Lj3hME+3uyZMS1hAR4GV2WiIiUM10FJnIBd4uZmQ90oXlQLZIzcnnsvY2czjtrdFkiImIgBSBxCQHe7swb2o1AXw92HMtgzMdbyD9rY93+kyzZksC6/Sex2tQZKiLiKjQEVgQNgdVcmw6lcv/cX8g7a8PXw0J23vlFO0MDvJjcvy19I0MNrFBEREpLQ2AilxHVKJAHoxsCFAo/AEnpOQxfsJkV2xONKE1ERCqRApC4FKvNztfbk4rcV9AVOuXLnRoOExGp4RSAxKVsOJhKYnrOZffbgcT0HDYcTK28okREpNIpAIlLScm8fPgpTTsREameFIDEpQT5FW/9H5uGwEREajQFIHEp3ZsEEhrghekq7f726e+8sWoPOfnWq7QUEZHqSAFIXIrFbGJy/7YAl4SggudtQv3It9p5M24vN//fGr7bnVypNYqISMWrEgFo1qxZNG7cGC8vL6Kjo9mwYcNl286fPx+TyVTo4eVVeFgjKyuLkSNH0qBBA7y9vWnbti1z5syp6NOQaqJvZCizH+xyye0wQgK8mPNgF5Y/fR2zHuhCiL8XR1LP8Oj8Xxn2/q8cPXXaoIpFRKS8lehmqBVh4cKFjB07ljlz5hAdHc2MGTPo06cP8fHxBAUFFfkaf39/4uPjnc9NpsKf5ceOHct3333HggULaNy4Md988w1PPfUUYWFh3H777RV6PlI99I0M5ea2IWw4mEpKZg5Bfl50bxKIxez4t9SvQyi9WtXnzbi9zPvpIKt2JvPj3uOMuqkFj1/XBE83i8FnICIiZWH4StDR0dF069aNmTNnAmCz2YiIiGDUqFGMHz/+kvbz589nzJgxpKWlXfaYkZGRDBo0iIkTJzq3RUVFceutt/Lyyy9ftSatBC0X2pOcyfNfbHdeGt+0vi8v3RFJz+b1DK5MREQuVG1Wgs7Ly2PTpk3ExsY6t5nNZmJjY1m3bt1lX5eVlUWjRo2IiIjgjjvuYMeOHYX29+jRg6VLl5KQkIDdbmf16tXs2bOHW265pcjj5ebmkpGRUeghUqBlsB8Ln7iGN+7tSL1aHhw4ns3g//zCyI82k3SFNYVERKTqMjQAnThxAqvVSnBwcKHtwcHBJCUVvVpvq1atmDdvHkuWLGHBggXYbDZ69OjB0aNHnW3eeust2rZtS4MGDfDw8KBv377MmjWL66+/vshjTps2jYCAAOcjIiKi/E5SagSTycRdXRoQ95cbGBrTCLMJvvo9kd6vf89/fjxAvtVmdIkiIlICVWISdEnExMQwZMgQOnXqRK9evfjss8+oX78+b7/9trPNW2+9xfr161m6dCmbNm3i9ddfZ8SIEXz77bdFHnPChAmkp6c7H0eOHKms05FqJsDbnSl3RLJ05LV0iqhNdp6Vl5ftov9bP2n1aBGRasTQSdD16tXDYrGQnFz4MuPk5GRCQkKKdQx3d3c6d+7Mvn37ADhz5gx///vf+fzzz+nXrx8AHTp0YMuWLfzzn/8sNNxWwNPTE09PzzKejbiSyPAAPhveg09+PcKrK3azOymTe99ex91dGjDhttbUq6V/TyIiVZmhPUAeHh5ERUURFxfn3Gaz2YiLiyMmJqZYx7BarWzbto3Q0FAA8vPzyc/Px2wufGoWiwWbTcMUUn7MZhP3dW/I6r/cwP3dHcOmn24+yk3//J4P1v2hG6qKiFRhhg+BjR07lrlz5/Lee++xa9cuhg8fTnZ2No888ggAQ4YMYcKECc72L774It988w0HDhxg8+bNPPjggxw6dIjHH38ccFwi36tXL/7617/y/fffc/DgQebPn8/777/PnXfeacg5Ss1Wx9eDaXd14LOnetAuzJ+MnLNMXLKDAbN+ZsuRNKPLExGRIhi+DtCgQYM4fvw4kyZNIikpiU6dOrFixQrnxOjDhw8X6s05deoUw4YNIykpiTp16hAVFcXatWtp27ats83HH3/MhAkTGDx4MKmpqTRq1IhXXnmFJ598stLPT1xHl4Z1WDryWhasP8Q/v4lnW0I6d/77Z+7r1pC/9WlFHV8Po0sUEZFzDF8HqCrSOkBSVsczc5m2fBef/ZYAQB0fd8bf2pp7oiIwm692JzIRESmNkrx/KwAVQQFIyssvB04yccl29iRnAdClYW1eGhBJu7AAgysTEal5FIDKSAFIylO+1cb8n/9gxrd7yM6zYjbBkJjGjL2lJf5e7kaXJyJSY1SblaBFXIG7xcyw65vy7V960a9DKDY7zF/7B71fX8MXvzlWKxcRkcqlHqAiqAdIKtKPe48zackODp7IBuCapoG8dEckLYL9DK5MRKR6Uw+QSBV2XYv6rBhzHc/e0hIvdzPrD6Ry679+ZNrXu8jOPWt0eSIiLkEBSMQAnm4WRt7UglXP9CK2TTBnbXbeXnOA2DfW8PW2RA2LiYhUMAUgEQNFBPrwn6Fd+e/QrjSo401ieg7DP9zM0Hc3OofIRESk/CkAiVQBvdsE8+3YXjx9U3M8LGZ+2HOcPv/3A298E09OvtXo8kREahwFIJEqwsvdwthbWrHymeu5rkU98qw23vxuHzf/3xridiVf/QAiIlJsCkAiVUyTer68/2h3/j24CyH+XhxJPcNj7/3KsPd/5eip00aXJyJSI+gy+CLoMnipKrJzz/Jm3F7++9NBztrseLmbGXVTCx6/rgmebhYArDY7Gw6mkpKZQ5CfF92bBGLR7TZExAVpJegyUgCSqmZPciYTv9jOLwdTAWhaz5cX74gkKzefKV/uJDE9x9k2NMCLyf3b0jcy1KhyRUQMoQBURgpAUhXZ7XaWbDnGy8t2cSIr97LtCvp+Zj/YRSFIRFyKFkIUqYFMJhMDOofz3bO9GBLT6LLtCj7RTPlyJ1abPt+IiBRFAUikmvH3cufWq/Ts2IHE9Bw2nBsyExGRwhSARKqhlMycqzcqQTsREVejACRSDQX5eZVrOxERV6MAJFINdW8SSGiAF1e62D3Iz5PuTQIrrSYRkepEAUikGrKYTUzu3xbgsiHIarNzLO1M5RUlIlKNKACJVFN9I0OZ/WAXQgIKD3MF+XlSv5YHJ7PzuO+d9Rw6qZuqiohcTOsAFUHrAEl1UtRK0Mczc3ngP+s5cDybEH8vPhoWTdP6tYwuVUSkQmkhxDJSAJKaICUzh8Fzf2FvShb1/Tz537Bomgf5GV2WiEiF0UKIIkKQnxcfP3ENrUP8OJ6Zy6C31xOflGl0WSIiVYICkEgNVreWJ/8bdg3twvzPzQlax45j6UaXJSJiOAUgkRqujq8HHz1+DR0bBHDqdD4PzP2FbUcVgkTEtSkAibiAAB93Png8mi4Na5N+Jp8H/rOe3w6fMrosERHDKACJuAh/L3fefyyabo3rkJlzlof+u4Ff/9C9wkTENSkAibiQWp5uzH+kO9c0DSQr9yxD5m3glwMnjS5LRKTSKQCJuBhfTzfefbg71zavx+k8K0Pf3cDP+04YXZaISKVSABJxQd4eFv4ztCs3tKpPTr6NR+dvZM2e40aXJSJSaRSARFyUl7uFtx+KIrZNELlnbQx771e+251sdFkiIpVCAUjEhXm6Wfj34Cj6tgshz2rjzx9s4psdSUaXJSJS4RSARFych5uZtx7oTL8OoeRb7Tz14WaWb0s0uiwRkQqlACQiuFvM/GtQJwZ0CuOszc6o//3Gki0JRpclIlJhqkQAmjVrFo0bN8bLy4vo6Gg2bNhw2bbz58/HZDIVenh5eV3SbteuXdx+++0EBATg6+tLt27dOHz4cEWehki15mYx8/q9nRgY1QCrzc4zC7fw2eajRpclIlIhDA9ACxcuZOzYsUyePJnNmzfTsWNH+vTpQ0pKymVf4+/vT2JiovNx6NChQvv379/PtddeS+vWrfn+++/5/fffmThxYpFBSUTOs5hN/OPuDtzfPQKbHf6yaCufbDxidFkiIuXOZLfb7UYWEB0dTbdu3Zg5cyYANpuNiIgIRo0axfjx4y9pP3/+fMaMGUNaWtplj3nffffh7u7OBx98UKqaMjIyCAgIID09HX9//1IdQ6Q6s9nsTF66gw/WOz5cvHJnJIOjGxlclYjIlZXk/dvQHqC8vDw2bdpEbGysc5vZbCY2NpZ169Zd9nVZWVk0atSIiIgI7rjjDnbs2OHcZ7PZWLZsGS1btqRPnz4EBQURHR3NF198cdnj5ebmkpGRUegh4srMZhMv3tGOR3o2BuC5z7cz/+eDxhYlIlKODA1AJ06cwGq1EhwcXGh7cHAwSUlFX4rbqlUr5s2bx5IlS1iwYAE2m40ePXpw9KhjrkJKSgpZWVm8+uqr9O3bl2+++YY777yTu+66izVr1hR5zGnTphEQEOB8RERElO+JilRDJpOJSX9qy5+vbwrAC1/u5D8/HjC4KhGR8uFmdAElFRMTQ0xMjPN5jx49aNOmDW+//TYvvfQSNpsNgDvuuINnnnkGgE6dOrF27VrmzJlDr169LjnmhAkTGDt2rPN5RkaGQpAIjhA0/tbWuFvMzFy9j5eX7SLfamf4Dc2MLk1EpEwMDUD16tXDYrGQnFx49dnk5GRCQkKKdQx3d3c6d+7Mvn37nMd0c3Ojbdu2hdq1adOGn376qchjeHp64unpWYozEKn5TCYTf7mlJW4WEzO+3ctrK3aTb7XxdO8WRpcmIlJqhg6BeXh4EBUVRVxcnHObzWYjLi6uUC/PlVitVrZt20ZoaKjzmN26dSM+Pr5Quz179tCokSZxipSGyWRiTGxL/tqnFQBvrNrD69/EY/A1FCIipWb4ENjYsWMZOnQoXbt2pXv37syYMYPs7GweeeQRAIYMGUJ4eDjTpk0D4MUXX+Saa66hefPmpKWlMX36dA4dOsTjjz/uPOZf//pXBg0axPXXX8+NN97IihUr+PLLL/n++++NOEWRGmPEjc1xt5iYunw3b323j3yrnXF9W2EymYwuTUSkRAwPQIMGDeL48eNMmjSJpKQkOnXqxIoVK5wTow8fPozZfL6j6tSpUwwbNoykpCTq1KlDVFQUa9euLTTkdeeddzJnzhymTZvG008/TatWrfj000+59tprK/38RGqaJ65vhpvZzItf7WTOmv2ctdp4rl8bhSARqVYMXweoKtI6QCJX98G6P5i4xLEExdCYRrxwezuFIBExVLVZB0hEqq+HYhrz6l3tMZngvXWHeO6L7dhs+jwlItWDApCIlNp93RsyfWBHTCb46JfDjP/sd6wKQSJSDSgAiUiZDIxqwIxBnTCb4JNfj/LXRVsVgkSkylMAEpEyu6NTOG/d3wWL2cRnvyUwZuEWzlptRpclInJZCkAiUi76dQhl1gNdcLeY+HLrMUb97zfyFYJEpIpSABKRctM3MoTZg6PwsJj5ensST324mdyzVqPLEhG5hAKQiJSr2LbBvDMkCg83M6t2JjN8wWZy8hWCRKRqUQASkXJ3Q6sg5g3thpe7me92pzDs/V8VgkSkSlEAEpEKcW2Lerz7cHd8PCz8uPcEj87fyOm8s0aXJSICKACJSAWKaVaX9x7tjq+HhbX7T/LwuxvJylUIEhHjKQCJSIXq1jiQDx6Pxs/TjQ0HUxk6bwOZOflGlyUiLk4BSEQqXJeGdfhwWDT+Xm5sOnSKB/+7gfQzCkEiYhwFIBGpFB0a1OajYddQ28edrUfSGPyf9aSdzjO6LBFxUQpAIlJpIsMD+N+wa6jr68H2hAzun/sLJ7NyjS5LRFyQApCIVKo2of58/MQ11Kvlya7EDB6Y+wvHM3Ox2uys23+SJVsSWLf/pO4nJiIVymS32/Vb5iIZGRkEBASQnp6Ov7+/0eWI1Ej7j2fxwNz1JGfkEuzvid0OKZnne4NCA7yY3L8tfSNDDaxSRKqTkrx/qwdIRAzRrH4tFj4RQx0fd5IzcguFH4Ck9ByGL9jMiu2JBlUoIjWZApCIGCYi0Ac3S9G/hgq6pqd8uVPDYSJS7hSARMQwGw6mcjzz8pOg7UBieg4bDqZWXlEi4hIUgETEMCmZOcVrl1G8diIixaUAJCKGCfLzKla7f34Tz4L1h3QbDREpNwpAImKY7k0CCQ3wwnSVdkdOneH5L7ZzzdQ4Jn6xnfikzEqpT0RqLgUgETGMxWxicv+2AJeEINO5x+v3dGTSn9rStJ4vWbln+WD9IfrM+IF7317Hl1uPkXfWVtlli0gNoHWAiqB1gEQq14rtiUz5cieJ6efn+ly8DpDdbmft/pN8sO4Qq3YlO68Mq1fLk/u6RXB/dEPCa3sbUr+IVA0lef9WACqCApBI5bPa7Gw4mEpKZg5Bfl50bxKIxVz04FhSeg7/23CY/2047Fw/yGyC3m2CeeiaRlzbvB7my7xWRGouBaAyUgASqR7yrTZW7Uzmg3WHWHfgpHN747o+PHhNIwZGNaC2j4eBFYpIZVIAKiMFIJHqZ19KJgvWH+bTTUfJPHe1mKebmf4dw3jomkZ0jKhtbIEiUuEUgMpIAUik+srOPcvSrcf4YN0hdiZmOLd3aBDAg9c0on+HMLw9LAZWKCIVRQGojBSARKo/u93O5sNpLFh/iGW/J5JndVwtFuDtzsCoBgyObkjT+rUMrlJEypMCUBkpAInULCezclm06SgL1h/i6Kkzzu3XtajHg9c0onfroMvek0xEqg8FoDJSABKpmaw2Oz/sOc4H6w+xOj6Fgt9+oQFePNC9IYO6RxR7dWoRqXoUgMpIAUik5juSepqPNhxm4cYjpGbnAeBmNtEnMoSHrmlEdJNATCZdSi9SnSgAlZECkIjryD1r5ettSXyw/hCbDp1ybm8RVIuHYhpxZ+dw/LzcDaxQRIqrJO/fVWLQe9asWTRu3BgvLy+io6PZsGHDZdvOnz8fk8lU6OHldfku6yeffBKTycSMGTMqoHIRqe483SwM6BzOp8N7sOzpa7m/e0N8PCzsTcli0pIdRE+N4++fb2PXBVeUiUj1Z3gAWrhwIWPHjmXy5Mls3ryZjh070qdPH1JSUi77Gn9/fxITE52PQ4cOFdnu888/Z/369YSFhVVU+SJSg7QLC2DaXe1Z//feTLm9Hc2DanE6z8pHvxzm1n/9yMDZa1myJYHcs1ajSxWRMjI8AL3xxhsMGzaMRx55hLZt2zJnzhx8fHyYN2/eZV9jMpkICQlxPoKDgy9pk5CQwKhRo/jwww9xd1f3tYgUn7+XO0N7NGbVM9fzv2HX0K99KG5mE78eOsXoj7fQY9p3/GPFbo6knr7ktVabnXX7T7JkSwLr9p903rNMRKoWNyO/eV5eHps2bWLChAnObWazmdjYWNatW3fZ12VlZdGoUSNsNhtdunRh6tSptGvXzrnfZrPx0EMP8de//rXQ9svJzc0lNzfX+TwjQ13dIuL4sBXTrC4xzeqSkpHDxxuP8NEvh0nKyOHf3+9n9pr93NQqiAdjGtGrRX2+2Zl01Zu6ikjVYGgP0IkTJ7BarZf04AQHB5OUlFTka1q1asW8efNYsmQJCxYswGaz0aNHD44ePeps89prr+Hm5sbTTz9drDqmTZtGQECA8xEREVH6kxKRGinI34une7fgp3E3MufBKK5tXg+7HeJ2p/DIuxvpPvVbnlywuVD4AceNW4cv2MyK7YkGVS4iRTF8CKykYmJiGDJkCJ06daJXr1589tln1K9fn7fffhuATZs28a9//cs5Wbo4JkyYQHp6uvNx5MiRijwFEanG3Cxm+kaGsODxaOL+0otHezbBz9PCiay8ItsXDIBN+XKnhsNEqhBDA1C9evWwWCwkJycX2p6cnExISEixjuHu7k7nzp3Zt28fAD/++CMpKSk0bNgQNzc33NzcOHToEH/5y19o3Lhxkcfw9PTE39+/0ENE5Gqa1a/FpP5tmflAlyu2swOJ6Tm8v+4Pss7dqFVEjGXoHCAPDw+ioqKIi4tjwIABgGP+TlxcHCNHjizWMaxWK9u2beO2224D4KGHHiI2NrZQmz59+vDQQw/xyCOPlGv9IiIAaWfyi9Vuypc7efGrnTSp60u78AAiw/yJDA+gXZg/tX08KrhKEbmQoQEIYOzYsQwdOpSuXbvSvXt3ZsyYQXZ2tjOsDBkyhPDwcKZNmwbAiy++yDXXXEPz5s1JS0tj+vTpHDp0iMcffxyAunXrUrdu3ULfw93dnZCQEFq1alW5JyciLqG4t8+o6+vOyex8DpzI5sCJbL7cesy5r0EdbyLDAogM9z8XjgKo7+dZUSWLuDzDA9CgQYM4fvw4kyZNIikpiU6dOrFixQrnxOjDhw9jNp8fqTt16hTDhg0jKSmJOnXqEBUVxdq1a2nbtq1RpyAiLq57k0BCA7xISs+hqFk+JiAkwIufxt1E2uk8dhzLYPuxdHYkOL4eOnmao6fOcPTUGVbsOH8BSIi/lyMQhQUQGe4IRyH+XrpFh0g50K0wiqBbYYhISa3YnsjwBZsBCoWggqgy+8Eul70UPv1MPjsuCETbE9I5cCKbon471/X1KDR8FhkWQESgt0KRCLoXWJkpAIlIaazYnlhu6wBl555lV2IG2xPS2X7M8XVvSlaRV5L5e7md6yFyzCeKDA+gSV1fzGaFInEtCkBlpAAkIqVltdnZcDCVlMwcgvy86N4kEEs5BZGcfCu7kzLZnpDOjmPpbE/IID4pkzyr7ZK2vh4W2oYVHj5rXr8WbpbiXfxbkechUlEUgMpIAUhEqou8szb2pmQWGj7bmZhBTv6locjTzUzrUH8iw/xpf67HqEVwLTzdLIXalWdPlkhlUgAqIwUgEanOzlptHDyRfS4QZbAtIZ2dxzKKXIPI3WKiZbCf8wq0zJyzTF8Zf8lk7uLMZRIxmgJQGSkAiUhNY7PZOZR6+tycovMTrtNOF28NIyh8NZuGw6QqKsn7t+GXwYuISMUzm000qedLk3q+9O8YBoDdbich7QzbEzLYcSydH/YeZ+uR9Mseo2BF63X7T3Bti/qVVLlIxVAPUBHUAyQirmjJlgRGf7zlqu283c3Etg0htk0QvVrW1yrWUmWoB0hEREqsuCtan8m38eXWY3y59RgWs4moRnXo3TqI3m2CaVbfV2sSSbWgACQiIkDxV7T+132dWB1/nO92pRCfnMmGg6lsOJjKtK9306iuD71bB9O7TRDdGgfi4WboPbdFLktDYEXQEJiIuKqSrmh9JPU03+1OIW53Cuv3nyy0JpGfpxvXt6xP7zZB3NAqiEBfDZVJxdJVYGWkACQirqy06wBl5Z7lp70niNuVzOr4FE5k5Tn3mU3QpWEdbmoTRGybYFoE1dJQmZQ7BaAyUgASEVdX1pWgbTY7W4+m8d3uFL7dlcKuxIxC+xvU8XbOG4puGnjJYowipaEAVEYKQCIi5etY2hnidqfw3a5kft5/kryz54fKfD0sXNeiPje1CeKm1kHUq+VpYKVSnSkAlZECkIhIxTmdd5af953ku93JxO1KISUz17nPZIKODWoT2yaIm1oH0ybUT0NlUmwKQGWkACQiUjlsNjs7jmXw7a5kvtudwraEwgsxhgV4cVMbx1BZTNO6eLlrqEwuTwGojBSARESMkZyR47iqbFcKP+07Xuimrt7uFq5tUY/erR1DZUH+xVu3SFyHAlAZKQCJiBgvJ9/Kuv0nnb1DF16VBtChQYBzzaF2Yf5FDpWVdTK3VC8KQGWkACQiUrXY7XZ2Jmbw3a4Uvt2dwtYjaYX2B/t7clPrYHq3DqJn83p4e1hKfTm/VF8KQGWkACQiUrWlZObw/e7jxO1O5se9JzidZ3Xu83Qz0yK4FtsTMi553eUWdJSaQQGojBSARESqj5x8K78cTOW7Xcl8uyuFhLQzV2xfcEuPn8bdpOGwGqYk79+6SYuIiFRrXu4WerWsz5Q7Ivlp3I384+4OV2xvBxLTc1h/4GTlFChVkgKQiIjUGCaTCU/34r21PfnBJv7++TbW7DleaGFGcQ26G7yIiNQoQX7Fuzw+M/csH/1ymI9+OYyflxuxbYLp0y6EXi3r4+2h9YZqOgUgERGpUbo3CSQ0wIuk9ByKmuRaMAdo2p3tWbUrmZU7kjmRlcvnvyXw+W8JeLmb6dWyPn0jQ7ipdTAB3u6VfQpSCTQJugiaBC0iUr2t2J7I8AWbAQqFoKKuArPa7Px2+BQrtiexYkcSR0+dn0TtZjYR06wufSNDuLltcLF7l8QYugqsjBSARESqv9KsA1Sw3tDKc2FoT3KWc5/JBF0b1aFPuxD6tAshItCnws9BSkYBqIwUgEREaoayrgR94HgWK3cks2JH0iWLL7YL86dvuxD6RobQPKiWbtpaBSgAlZECkIiIXOxY2hm+2ZHEyh3J/HLwJLYL3j2b1vOlT2QIfduF0KFBgMKQQRSAykgBSEREruRkVi5xu1JYsSOJn/aeIM96/jL6sAAvbjk3TNatcR3cLFpxprIoAJWRApCIiBRXZk4+38cfZ8WOJFbvTil0W45AXw9ubhNMn8hgejavh6ebLq+vSApAZaQAJCIipZGTb+WnvSdYuSOJVbuSSTud79xXy9ONG1sH0bddCDe0qo+vp1aiKW8KQGWkACQiImV11mpjw8FUVuxIYuWOJJIzcp37PNzMXN+iHn3ahRDbJpg6vh4GVlpzKACVkQKQiIiUJ5vNztajaY4wtD2JP06edu6zmE1ENwmkb2QIt7QNISSg6LWGynpFmytQACojBSAREakodrud+ORMVm53XF6/KzGj0P7ODWvTp53jirLG9XyB0q1p5IqqXQCaNWsW06dPJykpiY4dO/LWW2/RvXv3ItvOnz+fRx55pNA2T09PcnIc/yjy8/N5/vnnWb58OQcOHCAgIIDY2FheffVVwsLCilWPApCIiFSWQyezWXnu8vpNh04V2tc6xI9m9WuxbFviJa8ralVrV1eS92/DZ2AtXLiQsWPHMmfOHKKjo5kxYwZ9+vQhPj6eoKCgIl/j7+9PfHy88/mF6y2cPn2azZs3M3HiRDp27MipU6cYPXo0t99+O7/++muFn4+IiEhJNKrryxPXN+OJ65uRnJHDNzuTWbk9iXUHTrI7KZPdSZlFvq6g92Likh10iqhDoK8HHm5V/5L7qjKUZ3gPUHR0NN26dWPmzJkA2Gw2IiIiGDVqFOPHj7+k/fz58xkzZgxpaWnF/h4bN26ke/fuHDp0iIYNG16yPzc3l9zc85PTMjIyiIiIUA+QiIgYJu10Hm+vOcDsNfuL/Rpvdwu1fdwJ8HbH39vxNcDbndoFf75gX+0L9gd4u1fKekUVPZRXbXqA8vLy2LRpExMmTHBuM5vNxMbGsm7dusu+Lisri0aNGmGz2ejSpQtTp06lXbt2l22fnp6OyWSidu3aRe6fNm0aU6ZMKfV5iIiIlLfaPh60DvUr0WvO5Fs5k24tFDCKy9fDQm0fj3PBye1ccPIoFJoCighO/t7uxerBKbhB7cW9LknpOQxfsLnSh/IMDUAnTpzAarUSHBxcaHtwcDC7d+8u8jWtWrVi3rx5dOjQgfT0dP75z3/So0cPduzYQYMGDS5pn5OTw7hx47j//vsvmwYnTJjA2LFjnc8LeoBERESMVNy7z3/4eDSRYQGknckj/Ux+oUfa6XwyLnpe8OeMM/lk5p4FIDvPSnbeGRLSzpS4Tj8vt0Kh6OJeKD8vN17/Zs8l4QccQ3kmYMqXO7m5bUilDYcZPgeopGJiYoiJiXE+79GjB23atOHtt9/mpZdeKtQ2Pz+fe++9F7vdzuzZsy97TE9PTzw9PSusZhERkdLo3iSQ0AAvktJzigwPJiAkwItrmtbFYjYR4ONe4u9x1mojI+fsJcEp/XTRYaogOKWfySf73KrXmTlnycw5y9FTJQ9P4AhBiek5bDiYSkyzuqU6RkkZGoDq1auHxWIhOTm50Pbk5GRCQkKKdQx3d3c6d+7Mvn37Cm0vCD+HDh3iu+++01weERGpdixmE5P7t2X4gs2YoFAIKugnmdy/bZl6TdwsZgJ9PQgsxWKM+VZbEcEp/5JtuxMz2H4s46rHS8ks+dBdaRkagDw8PIiKiiIuLo4BAwYAjknQcXFxjBw5sljHsFqtbNu2jdtuu825rSD87N27l9WrV1O3buWkSRERkfLWNzKU2Q92uWTycEgVWAfI3WKmXi1P6tW68ijKuv0nuX/u+qser7hDfuXB8CGwsWPHMnToULp27Ur37t2ZMWMG2dnZzrV+hgwZQnh4ONOmTQPgxRdf5JprrqF58+akpaUxffp0Dh06xOOPPw44ws/AgQPZvHkzX331FVarlaSkJAACAwPx8NBy4yIiUr30jQzl5rYhVeLy8dIo7lBe9yaBlVaT4QFo0KBBHD9+nEmTJpGUlESnTp1YsWKFc2L04cOHMZvPX5p36tQphg0bRlJSEnXq1CEqKoq1a9fStm1bABISEli6dCkAnTp1KvS9Vq9ezQ033FAp5yUiIlKeLGZTpc2PKW+VMZRXUoavA1QVaSVoERGR8qd1gERERMTlVKWhPAUgERERqTRVZSiv6t80RERERKScKQCJiIiIy1EAEhEREZejACQiIiIuRwFIREREXI4CkIiIiLgcBSARERFxOQpAIiIi4nIUgERERMTlaCXoIhTcHi0jI8PgSkRERKS4Ct63i3ObUwWgImRmZgIQERFhcCUiIiJSUpmZmQQEBFyxje4GXwSbzcaxY8fw8/PDZKr8G7RVBxkZGURERHDkyJGr3nFXKp5+HlWLfh5Vi34eVU9F/UzsdjuZmZmEhYVhNl95lo96gIpgNptp0KCB0WVUC/7+/vqFUoXo51G16OdRtejnUfVUxM/kaj0/BTQJWkRERFyOApCIiIi4HAUgKRVPT08mT56Mp6en0aUI+nlUNfp5VC36eVQ9VeFnoknQIiIi4nLUAyQiIiIuRwFIREREXI4CkIiIiLgcBSARERFxOQpAUmzTpk2jW7du+Pn5ERQUxIABA4iPjze6LDnn1VdfxWQyMWbMGKNLcWkJCQk8+OCD1K1bF29vb9q3b8+vv/5qdFkuyWq1MnHiRJo0aYK3tzfNmjXjpZdeKtZ9oqTsfvjhB/r3709YWBgmk4kvvvii0H673c6kSZMIDQ3F29ub2NhY9u7dW2n1KQBJsa1Zs4YRI0awfv16Vq1aRX5+PrfccgvZ2dlGl+byNm7cyNtvv02HDh2MLsWlnTp1ip49e+Lu7s7XX3/Nzp07ef3116lTp47Rpbmk1157jdmzZzNz5kx27drFa6+9xj/+8Q/eeusto0tzCdnZ2XTs2JFZs2YVuf8f//gHb775JnPmzOGXX37B19eXPn36kJOTUyn16TJ4KbXjx48TFBTEmjVruP76640ux2VlZWXRpUsX/v3vf/Pyyy/TqVMnZsyYYXRZLmn8+PH8/PPP/Pjjj0aXIsCf/vQngoOD+e9//+vcdvfdd+Pt7c2CBQsMrMz1mEwmPv/8cwYMGAA4en/CwsL4y1/+wrPPPgtAeno6wcHBzJ8/n/vuu6/Ca1IPkJRaeno6AIGBgQZX4tpGjBhBv379iI2NNboUl7d06VK6du3KPffcQ1BQEJ07d2bu3LlGl+WyevToQVxcHHv27AFg69at/PTTT9x6660GVyYHDx4kKSmp0O+tgIAAoqOjWbduXaXUoJuhSqnYbDbGjBlDz549iYyMNLocl/Xxxx+zefNmNm7caHQpAhw4cIDZs2czduxY/v73v7Nx40aefvppPDw8GDp0qNHluZzx48eTkZFB69atsVgsWK1WXnnlFQYPHmx0aS4vKSkJgODg4ELbg4ODnfsqmgKQlMqIESPYvn07P/30k9GluKwjR44wevRoVq1ahZeXl9HlCI4PBl27dmXq1KkAdO7cme3btzNnzhwFIAN88sknfPjhh3z00Ue0a9eOLVu2MGbMGMLCwvTzEA2BScmNHDmSr776itWrV9OgQQOjy3FZmzZtIiUlhS5duuDm5oabmxtr1qzhzTffxM3NDavVanSJLic0NJS2bdsW2tamTRsOHz5sUEWu7a9//Svjx4/nvvvuo3379jz00EM888wzTJs2zejSXF5ISAgAycnJhbYnJyc791U0BSApNrvdzsiRI/n888/57rvvaNKkidElubTevXuzbds2tmzZ4nx07dqVwYMHs2XLFiwWi9ElupyePXtesjTEnj17aNSokUEVubbTp09jNhd+m7NYLNhsNoMqkgJNmjQhJCSEuLg457aMjAx++eUXYmJiKqUGDYFJsY0YMYKPPvqIJUuW4Ofn5xynDQgIwNvb2+DqXI+fn98l8698fX2pW7eu5mUZ5JlnnqFHjx5MnTqVe++9lw0bNvDOO+/wzjvvGF2aS+rfvz+vvPIKDRs2pF27dvz222+88cYbPProo0aX5hKysrLYt2+f8/nBgwfZsmULgYGBNGzYkDFjxvDyyy/TokULmjRpwsSJEwkLC3NeKVbh7CLFBBT5ePfdd40uTc7p1auXffTo0UaX4dK+/PJLe2RkpN3T09PeunVr+zvvvGN0SS4rIyPDPnr0aHvDhg3tXl5e9qZNm9qfe+45e25urtGluYTVq1cX+Z4xdOhQu91ut9tsNvvEiRPtwcHBdk9PT3vv3r3t8fHxlVaf1gESERERl6M5QCIiIuJyFIBERETE5SgAiYiIiMtRABIRERGXowAkIiIiLkcBSERERFyOApCIiIi4HAUgERERcTkKQCIixWAymfjiiy+MLkNEyokCkIhUeQ8//DAmk+mSR9++fY0uTUSqKd0MVUSqhb59+/Luu+8W2ubp6WlQNSJS3akHSESqBU9PT0JCQgo96tSpAziGp2bPns2tt96Kt7c3TZs2ZfHixYVev23bNm666Sa8vb2pW7cuTzzxBFlZWYXazJs3j3bt2uHp6UloaCgjR44stP/EiRPceeed+Pj40KJFC5YuXVqxJy0iFUYBSERqhIkTJ3L33XezdetWBg8ezH333ceuXbsAyM7Opk+fPtSpU4eNGzeyaNEivv3220IBZ/bs2YwYMYInnniCbdu2sXTpUpo3b17oe0yZMoV7772X33//ndtuu43BgweTmppaqecpIuWk0u47LyJSSkOHDrVbLBa7r69voccrr7xit9vtdsD+5JNPFnpNdHS0ffjw4Xa73W5/55137HXq1LFnZWU59y9btsxuNpvtSUlJdrvdbg8LC7M/99xzl60BsD///PPO51lZWXbA/vXXX5fbeYpI5dEcIBGpFm688UZmz55daFtgYKDzzzExMYX2xcTEsGXLFgB27dpFx44d8fX1de7v2bMnNpuN+Ph4TCYTx44do3fv3lesoUOHDs4/+/r64u/vT0pKSmlPSUQMpAAkItWCr6/vJUNS5cXb27tY7dzd3Qs9N5lM2Gy2iihJRCqY5gCJSI2wfv36S563adMGgDZt2rB161ays7Od+3/++WfMZjOtWrXCz8+Pxo0bExcXV6k1i4hx1AMkItVCbm4uSUlJhba5ublRr149ABYtWkTXrl259tpr+fDDD9mwYQP//e9/ARg8eDCTJ09m6NChvPDCCxw/fpxRo0bx0EMPERwcDMALL7zAk08+SVBQELfeeiuZmZn8/PPPjBo1qnJPVEQqhQKQiFQLK1asIDQ0tNC2Vq1asXv3bsBxhdbHH3/MU089RWhoKP/73/9o27YtAD4+PqxcuZLRo0fTrVs3fHx8uPvuu3njjTecxxo6dCg5OTn83//9H88++yz16tVj4MCBlXeCIlKpTHa73W50ESIiZWEymfj8888ZMGCA0aWISDWhOUAiIiLichSARERExOVoDpCIVHsayReRklIPkIiIiLgcBSARERFxOQpAIiIi4nIUgERERMTlKACJiIiIy1EAEhEREZejACQiIiIuRwFIREREXM7/Awb3Jpy+dRF7AAAAAElFTkSuQmCC\n"
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "device = 'cuda' if torch.cuda.is_available() else 'cpu'\n",
        "\n",
        "model.load_state_dict(torch.load('best_model_convnext_last_layer.bin'))\n",
        "model.to(device)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "aJ0Qb-S2z-8O",
        "outputId": "8b68b003-57e0-4f03-9b14-c8bade0bcf95"
      },
      "execution_count": 24,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "ConvNextAdded(\n",
              "  (convnext): ConvNeXt(\n",
              "    (features): Sequential(\n",
              "      (0): Conv2dNormActivation(\n",
              "        (0): Conv2d(3, 128, kernel_size=(4, 4), stride=(4, 4))\n",
              "        (1): LayerNorm2d((128,), eps=1e-06, elementwise_affine=True)\n",
              "      )\n",
              "      (1): Sequential(\n",
              "        (0): CNBlock(\n",
              "          (block): Sequential(\n",
              "            (0): Conv2d(128, 128, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=128)\n",
              "            (1): Permute()\n",
              "            (2): LayerNorm((128,), eps=1e-06, elementwise_affine=True)\n",
              "            (3): Linear(in_features=128, out_features=512, bias=True)\n",
              "            (4): GELU(approximate='none')\n",
              "            (5): Linear(in_features=512, out_features=128, bias=True)\n",
              "            (6): Permute()\n",
              "          )\n",
              "          (stochastic_depth): StochasticDepth(p=0.0, mode=row)\n",
              "        )\n",
              "        (1): CNBlock(\n",
              "          (block): Sequential(\n",
              "            (0): Conv2d(128, 128, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=128)\n",
              "            (1): Permute()\n",
              "            (2): LayerNorm((128,), eps=1e-06, elementwise_affine=True)\n",
              "            (3): Linear(in_features=128, out_features=512, bias=True)\n",
              "            (4): GELU(approximate='none')\n",
              "            (5): Linear(in_features=512, out_features=128, bias=True)\n",
              "            (6): Permute()\n",
              "          )\n",
              "          (stochastic_depth): StochasticDepth(p=0.014285714285714285, mode=row)\n",
              "        )\n",
              "        (2): CNBlock(\n",
              "          (block): Sequential(\n",
              "            (0): Conv2d(128, 128, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=128)\n",
              "            (1): Permute()\n",
              "            (2): LayerNorm((128,), eps=1e-06, elementwise_affine=True)\n",
              "            (3): Linear(in_features=128, out_features=512, bias=True)\n",
              "            (4): GELU(approximate='none')\n",
              "            (5): Linear(in_features=512, out_features=128, bias=True)\n",
              "            (6): Permute()\n",
              "          )\n",
              "          (stochastic_depth): StochasticDepth(p=0.02857142857142857, mode=row)\n",
              "        )\n",
              "      )\n",
              "      (2): Sequential(\n",
              "        (0): LayerNorm2d((128,), eps=1e-06, elementwise_affine=True)\n",
              "        (1): Conv2d(128, 256, kernel_size=(2, 2), stride=(2, 2))\n",
              "      )\n",
              "      (3): Sequential(\n",
              "        (0): CNBlock(\n",
              "          (block): Sequential(\n",
              "            (0): Conv2d(256, 256, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=256)\n",
              "            (1): Permute()\n",
              "            (2): LayerNorm((256,), eps=1e-06, elementwise_affine=True)\n",
              "            (3): Linear(in_features=256, out_features=1024, bias=True)\n",
              "            (4): GELU(approximate='none')\n",
              "            (5): Linear(in_features=1024, out_features=256, bias=True)\n",
              "            (6): Permute()\n",
              "          )\n",
              "          (stochastic_depth): StochasticDepth(p=0.04285714285714286, mode=row)\n",
              "        )\n",
              "        (1): CNBlock(\n",
              "          (block): Sequential(\n",
              "            (0): Conv2d(256, 256, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=256)\n",
              "            (1): Permute()\n",
              "            (2): LayerNorm((256,), eps=1e-06, elementwise_affine=True)\n",
              "            (3): Linear(in_features=256, out_features=1024, bias=True)\n",
              "            (4): GELU(approximate='none')\n",
              "            (5): Linear(in_features=1024, out_features=256, bias=True)\n",
              "            (6): Permute()\n",
              "          )\n",
              "          (stochastic_depth): StochasticDepth(p=0.05714285714285714, mode=row)\n",
              "        )\n",
              "        (2): CNBlock(\n",
              "          (block): Sequential(\n",
              "            (0): Conv2d(256, 256, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=256)\n",
              "            (1): Permute()\n",
              "            (2): LayerNorm((256,), eps=1e-06, elementwise_affine=True)\n",
              "            (3): Linear(in_features=256, out_features=1024, bias=True)\n",
              "            (4): GELU(approximate='none')\n",
              "            (5): Linear(in_features=1024, out_features=256, bias=True)\n",
              "            (6): Permute()\n",
              "          )\n",
              "          (stochastic_depth): StochasticDepth(p=0.07142857142857142, mode=row)\n",
              "        )\n",
              "      )\n",
              "      (4): Sequential(\n",
              "        (0): LayerNorm2d((256,), eps=1e-06, elementwise_affine=True)\n",
              "        (1): Conv2d(256, 512, kernel_size=(2, 2), stride=(2, 2))\n",
              "      )\n",
              "      (5): Sequential(\n",
              "        (0): CNBlock(\n",
              "          (block): Sequential(\n",
              "            (0): Conv2d(512, 512, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=512)\n",
              "            (1): Permute()\n",
              "            (2): LayerNorm((512,), eps=1e-06, elementwise_affine=True)\n",
              "            (3): Linear(in_features=512, out_features=2048, bias=True)\n",
              "            (4): GELU(approximate='none')\n",
              "            (5): Linear(in_features=2048, out_features=512, bias=True)\n",
              "            (6): Permute()\n",
              "          )\n",
              "          (stochastic_depth): StochasticDepth(p=0.08571428571428572, mode=row)\n",
              "        )\n",
              "        (1): CNBlock(\n",
              "          (block): Sequential(\n",
              "            (0): Conv2d(512, 512, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=512)\n",
              "            (1): Permute()\n",
              "            (2): LayerNorm((512,), eps=1e-06, elementwise_affine=True)\n",
              "            (3): Linear(in_features=512, out_features=2048, bias=True)\n",
              "            (4): GELU(approximate='none')\n",
              "            (5): Linear(in_features=2048, out_features=512, bias=True)\n",
              "            (6): Permute()\n",
              "          )\n",
              "          (stochastic_depth): StochasticDepth(p=0.1, mode=row)\n",
              "        )\n",
              "        (2): CNBlock(\n",
              "          (block): Sequential(\n",
              "            (0): Conv2d(512, 512, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=512)\n",
              "            (1): Permute()\n",
              "            (2): LayerNorm((512,), eps=1e-06, elementwise_affine=True)\n",
              "            (3): Linear(in_features=512, out_features=2048, bias=True)\n",
              "            (4): GELU(approximate='none')\n",
              "            (5): Linear(in_features=2048, out_features=512, bias=True)\n",
              "            (6): Permute()\n",
              "          )\n",
              "          (stochastic_depth): StochasticDepth(p=0.11428571428571428, mode=row)\n",
              "        )\n",
              "        (3): CNBlock(\n",
              "          (block): Sequential(\n",
              "            (0): Conv2d(512, 512, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=512)\n",
              "            (1): Permute()\n",
              "            (2): LayerNorm((512,), eps=1e-06, elementwise_affine=True)\n",
              "            (3): Linear(in_features=512, out_features=2048, bias=True)\n",
              "            (4): GELU(approximate='none')\n",
              "            (5): Linear(in_features=2048, out_features=512, bias=True)\n",
              "            (6): Permute()\n",
              "          )\n",
              "          (stochastic_depth): StochasticDepth(p=0.12857142857142856, mode=row)\n",
              "        )\n",
              "        (4): CNBlock(\n",
              "          (block): Sequential(\n",
              "            (0): Conv2d(512, 512, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=512)\n",
              "            (1): Permute()\n",
              "            (2): LayerNorm((512,), eps=1e-06, elementwise_affine=True)\n",
              "            (3): Linear(in_features=512, out_features=2048, bias=True)\n",
              "            (4): GELU(approximate='none')\n",
              "            (5): Linear(in_features=2048, out_features=512, bias=True)\n",
              "            (6): Permute()\n",
              "          )\n",
              "          (stochastic_depth): StochasticDepth(p=0.14285714285714285, mode=row)\n",
              "        )\n",
              "        (5): CNBlock(\n",
              "          (block): Sequential(\n",
              "            (0): Conv2d(512, 512, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=512)\n",
              "            (1): Permute()\n",
              "            (2): LayerNorm((512,), eps=1e-06, elementwise_affine=True)\n",
              "            (3): Linear(in_features=512, out_features=2048, bias=True)\n",
              "            (4): GELU(approximate='none')\n",
              "            (5): Linear(in_features=2048, out_features=512, bias=True)\n",
              "            (6): Permute()\n",
              "          )\n",
              "          (stochastic_depth): StochasticDepth(p=0.15714285714285714, mode=row)\n",
              "        )\n",
              "        (6): CNBlock(\n",
              "          (block): Sequential(\n",
              "            (0): Conv2d(512, 512, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=512)\n",
              "            (1): Permute()\n",
              "            (2): LayerNorm((512,), eps=1e-06, elementwise_affine=True)\n",
              "            (3): Linear(in_features=512, out_features=2048, bias=True)\n",
              "            (4): GELU(approximate='none')\n",
              "            (5): Linear(in_features=2048, out_features=512, bias=True)\n",
              "            (6): Permute()\n",
              "          )\n",
              "          (stochastic_depth): StochasticDepth(p=0.17142857142857143, mode=row)\n",
              "        )\n",
              "        (7): CNBlock(\n",
              "          (block): Sequential(\n",
              "            (0): Conv2d(512, 512, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=512)\n",
              "            (1): Permute()\n",
              "            (2): LayerNorm((512,), eps=1e-06, elementwise_affine=True)\n",
              "            (3): Linear(in_features=512, out_features=2048, bias=True)\n",
              "            (4): GELU(approximate='none')\n",
              "            (5): Linear(in_features=2048, out_features=512, bias=True)\n",
              "            (6): Permute()\n",
              "          )\n",
              "          (stochastic_depth): StochasticDepth(p=0.18571428571428572, mode=row)\n",
              "        )\n",
              "        (8): CNBlock(\n",
              "          (block): Sequential(\n",
              "            (0): Conv2d(512, 512, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=512)\n",
              "            (1): Permute()\n",
              "            (2): LayerNorm((512,), eps=1e-06, elementwise_affine=True)\n",
              "            (3): Linear(in_features=512, out_features=2048, bias=True)\n",
              "            (4): GELU(approximate='none')\n",
              "            (5): Linear(in_features=2048, out_features=512, bias=True)\n",
              "            (6): Permute()\n",
              "          )\n",
              "          (stochastic_depth): StochasticDepth(p=0.2, mode=row)\n",
              "        )\n",
              "        (9): CNBlock(\n",
              "          (block): Sequential(\n",
              "            (0): Conv2d(512, 512, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=512)\n",
              "            (1): Permute()\n",
              "            (2): LayerNorm((512,), eps=1e-06, elementwise_affine=True)\n",
              "            (3): Linear(in_features=512, out_features=2048, bias=True)\n",
              "            (4): GELU(approximate='none')\n",
              "            (5): Linear(in_features=2048, out_features=512, bias=True)\n",
              "            (6): Permute()\n",
              "          )\n",
              "          (stochastic_depth): StochasticDepth(p=0.21428571428571427, mode=row)\n",
              "        )\n",
              "        (10): CNBlock(\n",
              "          (block): Sequential(\n",
              "            (0): Conv2d(512, 512, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=512)\n",
              "            (1): Permute()\n",
              "            (2): LayerNorm((512,), eps=1e-06, elementwise_affine=True)\n",
              "            (3): Linear(in_features=512, out_features=2048, bias=True)\n",
              "            (4): GELU(approximate='none')\n",
              "            (5): Linear(in_features=2048, out_features=512, bias=True)\n",
              "            (6): Permute()\n",
              "          )\n",
              "          (stochastic_depth): StochasticDepth(p=0.22857142857142856, mode=row)\n",
              "        )\n",
              "        (11): CNBlock(\n",
              "          (block): Sequential(\n",
              "            (0): Conv2d(512, 512, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=512)\n",
              "            (1): Permute()\n",
              "            (2): LayerNorm((512,), eps=1e-06, elementwise_affine=True)\n",
              "            (3): Linear(in_features=512, out_features=2048, bias=True)\n",
              "            (4): GELU(approximate='none')\n",
              "            (5): Linear(in_features=2048, out_features=512, bias=True)\n",
              "            (6): Permute()\n",
              "          )\n",
              "          (stochastic_depth): StochasticDepth(p=0.24285714285714285, mode=row)\n",
              "        )\n",
              "        (12): CNBlock(\n",
              "          (block): Sequential(\n",
              "            (0): Conv2d(512, 512, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=512)\n",
              "            (1): Permute()\n",
              "            (2): LayerNorm((512,), eps=1e-06, elementwise_affine=True)\n",
              "            (3): Linear(in_features=512, out_features=2048, bias=True)\n",
              "            (4): GELU(approximate='none')\n",
              "            (5): Linear(in_features=2048, out_features=512, bias=True)\n",
              "            (6): Permute()\n",
              "          )\n",
              "          (stochastic_depth): StochasticDepth(p=0.2571428571428571, mode=row)\n",
              "        )\n",
              "        (13): CNBlock(\n",
              "          (block): Sequential(\n",
              "            (0): Conv2d(512, 512, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=512)\n",
              "            (1): Permute()\n",
              "            (2): LayerNorm((512,), eps=1e-06, elementwise_affine=True)\n",
              "            (3): Linear(in_features=512, out_features=2048, bias=True)\n",
              "            (4): GELU(approximate='none')\n",
              "            (5): Linear(in_features=2048, out_features=512, bias=True)\n",
              "            (6): Permute()\n",
              "          )\n",
              "          (stochastic_depth): StochasticDepth(p=0.2714285714285714, mode=row)\n",
              "        )\n",
              "        (14): CNBlock(\n",
              "          (block): Sequential(\n",
              "            (0): Conv2d(512, 512, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=512)\n",
              "            (1): Permute()\n",
              "            (2): LayerNorm((512,), eps=1e-06, elementwise_affine=True)\n",
              "            (3): Linear(in_features=512, out_features=2048, bias=True)\n",
              "            (4): GELU(approximate='none')\n",
              "            (5): Linear(in_features=2048, out_features=512, bias=True)\n",
              "            (6): Permute()\n",
              "          )\n",
              "          (stochastic_depth): StochasticDepth(p=0.2857142857142857, mode=row)\n",
              "        )\n",
              "        (15): CNBlock(\n",
              "          (block): Sequential(\n",
              "            (0): Conv2d(512, 512, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=512)\n",
              "            (1): Permute()\n",
              "            (2): LayerNorm((512,), eps=1e-06, elementwise_affine=True)\n",
              "            (3): Linear(in_features=512, out_features=2048, bias=True)\n",
              "            (4): GELU(approximate='none')\n",
              "            (5): Linear(in_features=2048, out_features=512, bias=True)\n",
              "            (6): Permute()\n",
              "          )\n",
              "          (stochastic_depth): StochasticDepth(p=0.3, mode=row)\n",
              "        )\n",
              "        (16): CNBlock(\n",
              "          (block): Sequential(\n",
              "            (0): Conv2d(512, 512, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=512)\n",
              "            (1): Permute()\n",
              "            (2): LayerNorm((512,), eps=1e-06, elementwise_affine=True)\n",
              "            (3): Linear(in_features=512, out_features=2048, bias=True)\n",
              "            (4): GELU(approximate='none')\n",
              "            (5): Linear(in_features=2048, out_features=512, bias=True)\n",
              "            (6): Permute()\n",
              "          )\n",
              "          (stochastic_depth): StochasticDepth(p=0.3142857142857143, mode=row)\n",
              "        )\n",
              "        (17): CNBlock(\n",
              "          (block): Sequential(\n",
              "            (0): Conv2d(512, 512, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=512)\n",
              "            (1): Permute()\n",
              "            (2): LayerNorm((512,), eps=1e-06, elementwise_affine=True)\n",
              "            (3): Linear(in_features=512, out_features=2048, bias=True)\n",
              "            (4): GELU(approximate='none')\n",
              "            (5): Linear(in_features=2048, out_features=512, bias=True)\n",
              "            (6): Permute()\n",
              "          )\n",
              "          (stochastic_depth): StochasticDepth(p=0.32857142857142857, mode=row)\n",
              "        )\n",
              "        (18): CNBlock(\n",
              "          (block): Sequential(\n",
              "            (0): Conv2d(512, 512, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=512)\n",
              "            (1): Permute()\n",
              "            (2): LayerNorm((512,), eps=1e-06, elementwise_affine=True)\n",
              "            (3): Linear(in_features=512, out_features=2048, bias=True)\n",
              "            (4): GELU(approximate='none')\n",
              "            (5): Linear(in_features=2048, out_features=512, bias=True)\n",
              "            (6): Permute()\n",
              "          )\n",
              "          (stochastic_depth): StochasticDepth(p=0.34285714285714286, mode=row)\n",
              "        )\n",
              "        (19): CNBlock(\n",
              "          (block): Sequential(\n",
              "            (0): Conv2d(512, 512, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=512)\n",
              "            (1): Permute()\n",
              "            (2): LayerNorm((512,), eps=1e-06, elementwise_affine=True)\n",
              "            (3): Linear(in_features=512, out_features=2048, bias=True)\n",
              "            (4): GELU(approximate='none')\n",
              "            (5): Linear(in_features=2048, out_features=512, bias=True)\n",
              "            (6): Permute()\n",
              "          )\n",
              "          (stochastic_depth): StochasticDepth(p=0.35714285714285715, mode=row)\n",
              "        )\n",
              "        (20): CNBlock(\n",
              "          (block): Sequential(\n",
              "            (0): Conv2d(512, 512, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=512)\n",
              "            (1): Permute()\n",
              "            (2): LayerNorm((512,), eps=1e-06, elementwise_affine=True)\n",
              "            (3): Linear(in_features=512, out_features=2048, bias=True)\n",
              "            (4): GELU(approximate='none')\n",
              "            (5): Linear(in_features=2048, out_features=512, bias=True)\n",
              "            (6): Permute()\n",
              "          )\n",
              "          (stochastic_depth): StochasticDepth(p=0.37142857142857144, mode=row)\n",
              "        )\n",
              "        (21): CNBlock(\n",
              "          (block): Sequential(\n",
              "            (0): Conv2d(512, 512, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=512)\n",
              "            (1): Permute()\n",
              "            (2): LayerNorm((512,), eps=1e-06, elementwise_affine=True)\n",
              "            (3): Linear(in_features=512, out_features=2048, bias=True)\n",
              "            (4): GELU(approximate='none')\n",
              "            (5): Linear(in_features=2048, out_features=512, bias=True)\n",
              "            (6): Permute()\n",
              "          )\n",
              "          (stochastic_depth): StochasticDepth(p=0.38571428571428573, mode=row)\n",
              "        )\n",
              "        (22): CNBlock(\n",
              "          (block): Sequential(\n",
              "            (0): Conv2d(512, 512, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=512)\n",
              "            (1): Permute()\n",
              "            (2): LayerNorm((512,), eps=1e-06, elementwise_affine=True)\n",
              "            (3): Linear(in_features=512, out_features=2048, bias=True)\n",
              "            (4): GELU(approximate='none')\n",
              "            (5): Linear(in_features=2048, out_features=512, bias=True)\n",
              "            (6): Permute()\n",
              "          )\n",
              "          (stochastic_depth): StochasticDepth(p=0.4, mode=row)\n",
              "        )\n",
              "        (23): CNBlock(\n",
              "          (block): Sequential(\n",
              "            (0): Conv2d(512, 512, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=512)\n",
              "            (1): Permute()\n",
              "            (2): LayerNorm((512,), eps=1e-06, elementwise_affine=True)\n",
              "            (3): Linear(in_features=512, out_features=2048, bias=True)\n",
              "            (4): GELU(approximate='none')\n",
              "            (5): Linear(in_features=2048, out_features=512, bias=True)\n",
              "            (6): Permute()\n",
              "          )\n",
              "          (stochastic_depth): StochasticDepth(p=0.4142857142857143, mode=row)\n",
              "        )\n",
              "        (24): CNBlock(\n",
              "          (block): Sequential(\n",
              "            (0): Conv2d(512, 512, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=512)\n",
              "            (1): Permute()\n",
              "            (2): LayerNorm((512,), eps=1e-06, elementwise_affine=True)\n",
              "            (3): Linear(in_features=512, out_features=2048, bias=True)\n",
              "            (4): GELU(approximate='none')\n",
              "            (5): Linear(in_features=2048, out_features=512, bias=True)\n",
              "            (6): Permute()\n",
              "          )\n",
              "          (stochastic_depth): StochasticDepth(p=0.42857142857142855, mode=row)\n",
              "        )\n",
              "        (25): CNBlock(\n",
              "          (block): Sequential(\n",
              "            (0): Conv2d(512, 512, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=512)\n",
              "            (1): Permute()\n",
              "            (2): LayerNorm((512,), eps=1e-06, elementwise_affine=True)\n",
              "            (3): Linear(in_features=512, out_features=2048, bias=True)\n",
              "            (4): GELU(approximate='none')\n",
              "            (5): Linear(in_features=2048, out_features=512, bias=True)\n",
              "            (6): Permute()\n",
              "          )\n",
              "          (stochastic_depth): StochasticDepth(p=0.44285714285714284, mode=row)\n",
              "        )\n",
              "        (26): CNBlock(\n",
              "          (block): Sequential(\n",
              "            (0): Conv2d(512, 512, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=512)\n",
              "            (1): Permute()\n",
              "            (2): LayerNorm((512,), eps=1e-06, elementwise_affine=True)\n",
              "            (3): Linear(in_features=512, out_features=2048, bias=True)\n",
              "            (4): GELU(approximate='none')\n",
              "            (5): Linear(in_features=2048, out_features=512, bias=True)\n",
              "            (6): Permute()\n",
              "          )\n",
              "          (stochastic_depth): StochasticDepth(p=0.45714285714285713, mode=row)\n",
              "        )\n",
              "      )\n",
              "      (6): Sequential(\n",
              "        (0): LayerNorm2d((512,), eps=1e-06, elementwise_affine=True)\n",
              "        (1): Conv2d(512, 1024, kernel_size=(2, 2), stride=(2, 2))\n",
              "      )\n",
              "      (7): Sequential(\n",
              "        (0): CNBlock(\n",
              "          (block): Sequential(\n",
              "            (0): Conv2d(1024, 1024, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=1024)\n",
              "            (1): Permute()\n",
              "            (2): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)\n",
              "            (3): Linear(in_features=1024, out_features=4096, bias=True)\n",
              "            (4): GELU(approximate='none')\n",
              "            (5): Linear(in_features=4096, out_features=1024, bias=True)\n",
              "            (6): Permute()\n",
              "          )\n",
              "          (stochastic_depth): StochasticDepth(p=0.4714285714285714, mode=row)\n",
              "        )\n",
              "        (1): CNBlock(\n",
              "          (block): Sequential(\n",
              "            (0): Conv2d(1024, 1024, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=1024)\n",
              "            (1): Permute()\n",
              "            (2): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)\n",
              "            (3): Linear(in_features=1024, out_features=4096, bias=True)\n",
              "            (4): GELU(approximate='none')\n",
              "            (5): Linear(in_features=4096, out_features=1024, bias=True)\n",
              "            (6): Permute()\n",
              "          )\n",
              "          (stochastic_depth): StochasticDepth(p=0.4857142857142857, mode=row)\n",
              "        )\n",
              "        (2): CNBlock(\n",
              "          (block): Sequential(\n",
              "            (0): Conv2d(1024, 1024, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=1024)\n",
              "            (1): Permute()\n",
              "            (2): LayerNorm((1024,), eps=1e-06, elementwise_affine=True)\n",
              "            (3): Linear(in_features=1024, out_features=4096, bias=True)\n",
              "            (4): GELU(approximate='none')\n",
              "            (5): Linear(in_features=4096, out_features=1024, bias=True)\n",
              "            (6): Permute()\n",
              "          )\n",
              "          (stochastic_depth): StochasticDepth(p=0.5, mode=row)\n",
              "        )\n",
              "      )\n",
              "    )\n",
              "    (avgpool): AdaptiveAvgPool2d(output_size=1)\n",
              "    (classifier): Sequential(\n",
              "      (0): LayerNorm2d((1024,), eps=1e-06, elementwise_affine=True)\n",
              "      (1): Flatten(start_dim=1, end_dim=-1)\n",
              "      (2): Linear(in_features=1024, out_features=1, bias=True)\n",
              "    )\n",
              "  )\n",
              ")"
            ]
          },
          "metadata": {},
          "execution_count": 24
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "test()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "jSoYmKzJ0Knd",
        "outputId": "4f0e43b9-1e8a-4596-f7e1-72f4a0e915db"
      },
      "execution_count": 25,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Total Test Loss: 21.7076; Test Accuracy: 63.64%\n",
            "                Test Funny  Test Not Funny\n",
            "Pred Funny             109              80\n",
            "Pred Not Funny         116             234\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "m9FzBUVU2X--"
      },
      "execution_count": 25,
      "outputs": []
    }
  ]
}