My_LSTM.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "My_LSTM.ipynb",
      "provenance": [],
      "collapsed_sections": [
        "kZ_ludYuuK6P",
        "A2CuADTRvLg5",
        "MWmCtv3dDux6",
        "FnZRAbk7sQNW",
        "gjroB_Y70xrj",
        "4OVvhX7WwBIE"
      ],
      "toc_visible": true,
      "authorship_tag": "ABX9TyMhyZZCNM6OeriM+Qw/MIUi",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/veren4/SMILES_featurization/blob/master/My_LSTM.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5OJdgryijr-L"
      },
      "source": [
        "TODO:\n",
        "* find an example self-supervising dataset\n",
        "* use it to figure out the Nummerisierung im Dataloader\n",
        "* Print out the versions of all important packages\n",
        "+ check for überflüssige Imports"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KGqXmfOEHZJk",
        "outputId": "b787ca20-8460-4cf7-8946-1414e70b94d3",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')\n",
        "import pickle"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Mounted at /content/drive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jlF6WsexaFaF"
      },
      "source": [
        "###Imports"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "T9DRJ9U-Q-I3"
      },
      "source": [
        "The latest pytorch version might not run with cuda support in Colab!"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GVNv91KcaKQb"
      },
      "source": [
        "import numpy as np\n",
        "import matplotlib.pyplot as plt\n",
        "import torch\n",
        "from torch.autograd import Variable\n",
        "import os\n",
        "import pandas as pd\n",
        "#import torchvision    # data loaders for common datasets such as Imagenet, FashionMNIST, MNIST, etc. and data transformers for images\n",
        "\n",
        "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
        "\n",
        "%matplotlib inline\n",
        "plt.rcParams['figure.figsize'] = (10.0, 8.0) # set default size of plots\n",
        "plt.rcParams['image.interpolation'] = 'nearest'\n",
        "plt.rcParams['image.cmap'] = 'gray'\n",
        "\n",
        "# for auto-reloading external modules\n",
        "# see http://stackoverflow.com/questions/1907993/autoreload-of-modules-in-ipython\n",
        "%load_ext autoreload\n",
        "%autoreload 2"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZnJ6u21AaSiO"
      },
      "source": [
        "import platform\n",
        "\n",
        "#import torch.nn as nn\n",
        "from torch import nn\n",
        "import torch.nn.functional as F\n",
        "import timeit"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8V_EVk_OBrXc"
      },
      "source": [
        "import tensorboard\n",
        "#print(f\"Tensorboard version: {tensorboard.__version__}\")\n",
        "\n",
        "# Load the TensorBoard notebook extension\n",
        "%load_ext tensorboard\n",
        "\n",
        "# imports\n",
        "%autoreload 2"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HsFLyDDJBsxi"
      },
      "source": [
        "from torch.utils.tensorboard import SummaryWriter"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eY7Qm_oN8Jk6"
      },
      "source": [
        "import torchvision.transforms as transforms"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "e_3K-xo0oJGK"
      },
      "source": [
        "!pip install -U torchtext\n",
        "import torchtext"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-Jpw3F-ymvkQ",
        "outputId": "f03ad7f9-b8a0-4576-ed93-e1682d313172",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 86
        }
      },
      "source": [
        "print('Python: ', platform.python_version())\n",
        "print('PyTorch: ', torch.__version__)\n",
        "if(device.type == 'cuda'):\n",
        "  print('Using GPU (cuda)')\n",
        "else:\n",
        "  print('Using CPU!')\n",
        "print('Torchtext', torchtext.__version__)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Python:  3.6.9\n",
            "PyTorch:  1.6.0+cu101\n",
            "Using GPU (cuda)\n",
            "Torchtext 0.7.0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "kZ_ludYuuK6P"
      },
      "source": [
        "###Set up TensorBoard"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "m7-rj7zZscKD"
      },
      "source": [
        "[How to TensorBoard](https://pytorch.org/docs/stable/tensorboard.html)\\\n",
        "[PyTorch Tensorboard tutorial](https://pytorch.org/tutorials/intermediate/tensorboard_tutorial.html)\\\n",
        "([Medium article](https://medium.com/@iamsdt/using-tensorboard-in-google-colab-with-pytorch-458f9bb95212) on TensorBoard in Colab with PyTorch) <- no suitable version available\\\n",
        "[The best tutorial](https://colab.research.google.com/github/pytorch/tutorials/blob/gh-pages/_downloads/tensorboard_with_pytorch.ipynb#scrollTo=IRSe6eHcFPyT): in a colab notebook"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jZtgB9LRS0Gn"
      },
      "source": [
        "[How to run Tensorboard in Colab](https://stackoverflow.com/questions/47818822/can-i-use-tensorboard-with-google-colab)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mWyHS1dzSsOs",
        "outputId": "50fe24f8-2bc5-46e5-e4f5-f461f37da3dc",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 52
        }
      },
      "source": [
        "#%tensorboard --logdir logs"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "The tensorboard extension is already loaded. To reload it, use:\n",
            "  %reload_ext tensorboard\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "A2CuADTRvLg5"
      },
      "source": [
        "###Seed"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WxCxMTWYGHoB",
        "outputId": "a706ff15-8112-4ea5-9d5e-2cbdee99a1b3",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "torch.manual_seed(1)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<torch._C.Generator at 0x7ff4b6fd35d0>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "VHgd-XA5vPP8"
      },
      "source": [
        "https://discuss.pytorch.org/t/reproducibility-with-all-the-bells-and-whistles/81097"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Ien_2TZxaK5h"
      },
      "source": [
        "###Description & documentation"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "A_WSaelG_wjP"
      },
      "source": [
        "Later: automated batching\\\n",
        "Visualization"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "uKVTR7ZOtoG6"
      },
      "source": [
        "[FloydHub](https://blog.floydhub.com/long-short-term-memory-from-zero-to-hero-with-pytorch/)\\\n",
        "[GitHub Minimal example](https://github.com/chrisvdweth/ml-toolkit/blob/master/pytorch/notebooks/minimal-example-lstm-input.ipynb)\\\n",
        "[PyTorch LSTM Beginner guide](https://pytorch.org/tutorials/beginner/nlp/sequence_models_tutorial.html)\\\n",
        "[PyTorch LSTM Guide](https://pytorch.org/docs/stable/generated/torch.nn.LSTM.html)\\\n",
        "[Medium article](https://medium.com/@sunitachoudhary103/generating-molecules-using-a-char-rnn-in-pytorch-16885fd9394b)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "YxGjp7MC4Di6"
      },
      "source": [
        "[deeplearningwizard](https://www.deeplearningwizard.com/deep_learning/practical_pytorch/pytorch_lstm_neuralnetwork/): Steps\n",
        "\n",
        "Step 1: Load Dataset\\\n",
        "Step 2: Make Dataset Iterable\\\n",
        "Step 3: Create Model Class\\\n",
        "Step 4: Instantiate Model Class\\\n",
        "Step 5: Instantiate Loss Class\\\n",
        "Step 6: Instantiate Optimizer Class\\\n",
        "Step 7: Train Model\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lj7Ej_GNW5Oh"
      },
      "source": [
        "infile1 = open('/content/drive/My Drive/Rostlab internship/7_PyTorch/Tokenized_Dataset', 'rb')\n",
        "tokenized_dataset = pickle.load(infile1)\n",
        "infile1.close()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "neg0dkr7110P"
      },
      "source": [
        "##Torchtext"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "elayIZEk15yu"
      },
      "source": [
        "[How to Torchtext](https://towardsdatascience.com/use-torchtext-to-load-nlp-datasets-part-i-5da6f1c89d84)\\\n",
        "[Another towardsDataScience article](https://towardsdatascience.com/deep-learning-for-nlp-with-pytorch-and-torchtext-4f92d69052f)\\\n",
        "[TorchText Github Documentation](https://github.com/pytorch/text/blob/master/README.rst)\\\n",
        "The newline characters need to be removed. Otherwise torchtext cannot read the csv files correctly.\\\n",
        "df_test.comment_text.str.replace(\"\\n\", \" \")"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "X5nPl7jlQU_C",
        "outputId": "623427cc-d878-49c6-ab15-806434f655c0",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 384
        }
      },
      "source": [
        "!pip install -U torchtext"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting torchtext\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/b9/f9/224b3893ab11d83d47fde357a7dcc75f00ba219f34f3d15e06fe4cb62e05/torchtext-0.7.0-cp36-cp36m-manylinux1_x86_64.whl (4.5MB)\n",
            "\u001b[K     |████████████████████████████████| 4.5MB 11.4MB/s \n",
            "\u001b[?25hRequirement already satisfied, skipping upgrade: requests in /usr/local/lib/python3.6/dist-packages (from torchtext) (2.23.0)\n",
            "Collecting sentencepiece\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/e5/2d/6d4ca4bef9a67070fa1cac508606328329152b1df10bdf31fb6e4e727894/sentencepiece-0.1.94-cp36-cp36m-manylinux2014_x86_64.whl (1.1MB)\n",
            "\u001b[K     |████████████████████████████████| 1.1MB 46.3MB/s \n",
            "\u001b[?25hRequirement already satisfied, skipping upgrade: tqdm in /usr/local/lib/python3.6/dist-packages (from torchtext) (4.41.1)\n",
            "Requirement already satisfied, skipping upgrade: numpy in /usr/local/lib/python3.6/dist-packages (from torchtext) (1.18.5)\n",
            "Requirement already satisfied, skipping upgrade: torch in /usr/local/lib/python3.6/dist-packages (from torchtext) (1.6.0+cu101)\n",
            "Requirement already satisfied, skipping upgrade: idna<3,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->torchtext) (2.10)\n",
            "Requirement already satisfied, skipping upgrade: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests->torchtext) (1.24.3)\n",
            "Requirement already satisfied, skipping upgrade: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests->torchtext) (2020.6.20)\n",
            "Requirement already satisfied, skipping upgrade: chardet<4,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->torchtext) (3.0.4)\n",
            "Requirement already satisfied, skipping upgrade: future in /usr/local/lib/python3.6/dist-packages (from torch->torchtext) (0.16.0)\n",
            "Installing collected packages: sentencepiece, torchtext\n",
            "  Found existing installation: torchtext 0.3.1\n",
            "    Uninstalling torchtext-0.3.1:\n",
            "      Successfully uninstalled torchtext-0.3.1\n",
            "Successfully installed sentencepiece-0.1.94 torchtext-0.7.0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "waLqiJ5y11eW"
      },
      "source": [
        "import torchtext"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1X0djD2j2Bvz"
      },
      "source": [
        "[Field class](https://pytorch.org/text/data.html#field) models common text processing datatypes that can be represented by tensors. It holds a **Vocab object** that defines the set of possible values for elements of the field and their corresponding numerical representations. The Field object also holds other parameters relating to how a datatype should be numericalized, such as a **tokenization method** and the kind of Tensor that should be produced."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "exFClwWU4EKm"
      },
      "source": [
        "torchtext_data = pd.DataFrame(tokenized_dataset)\n",
        "torchtext_data = torchtext_data.T"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "m1Uh24DlSBkB"
      },
      "source": [
        "torchtext_data = ['c C ( OH )', 'CCOHC']"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tMTPIE4_Irxl"
      },
      "source": [
        "import re"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "owu8PFq22D9-",
        "outputId": "c84b2c75-7032-471b-e364-bb7305ec16bb",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 72
        }
      },
      "source": [
        "# input: the dataset       output: tensor\n",
        "smiles_field = torchtext.data.Field(sequential=True,\n",
        "                    use_vocab=True, \n",
        "                    init_token='<sos>', # or should this be numerical?\n",
        "                    eos_token='<eos>',  # same as above\n",
        "                    fix_length=80,    # TODO check for optimal fixed length!\n",
        "                    dtype=torch.int64,    # what is the datatype of a BATCH of examples??\n",
        "                                          # and also again: is this already in the numerical state?\n",
        "                                          # For now, I just leave it at int64 (default)\n",
        "                    preprocessing=None,   # Hier evtl. Padding?\n",
        "                    postprocessing=None,  # oder hier Padding?\n",
        "                    lower=False,\n",
        "                    #tokenize=None, \n",
        "                    #tokenize=lambda x: re.findall(pattern = \"'(\\S+)'\", string=x),   # default: None = Spacy\n",
        "                    #tokenizer_language='en',\n",
        "                    include_lengths=False, \n",
        "                    batch_first=False, \n",
        "                    pad_token='<pad>', \n",
        "                    unk_token='<unk>', \n",
        "                    pad_first=False, \n",
        "                    truncate_first=False,\n",
        "                    stop_words=None,\n",
        "                    is_target=False)   # Is this a target variable? Kind of yes.."
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/torchtext/data/field.py:150: UserWarning: Field class will be retired in the 0.8.0 release and moved to torchtext.legacy. Please see 0.7.0 release notes for further information.\n",
            "  warnings.warn('{} class will be retired in the 0.8.0 release and moved to torchtext.legacy. Please see 0.7.0 release notes for further information.'.format(self.__class__.__name__), UserWarning)\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "t3j5KO5p2FyI"
      },
      "source": [
        "my_torchtext_dataset = torchtext.data.Dataset(\n",
        "    examples=torchtext_data,\n",
        "    fields=[('smiles', smiles_field)]\n",
        ")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GaJK1rEi4xJU",
        "outputId": "ed9c4871-f82d-4ef2-f318-20bb66bce2f7",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 72
        }
      },
      "source": [
        "for i in range(2):\n",
        "  print(tokenized_dataset[i])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "['C', '(', 'C', '(', 'C', '(', 'C', 'O', 'P', '(', '=', 'O', ')', '(', 'O', ')', 'O', ')', 'O', ')', 'O', ')', 'C', '(', '=', 'O', ')', 'C', '(', '=', 'O', ')', 'O']\n",
            "['C', 'C', 'C', '(', 'C', ')', 'C', '(', '=', 'O', ')', 'C', '(', '=', 'O', ')', 'O']\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vRfsBlml2I0J",
        "outputId": "d186adea-c407-4186-ff26-0b4af256cd9d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 52
        }
      },
      "source": [
        "for i in range(2):\n",
        "  print(my_torchtext_dataset[i])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "c C ( OH )\n",
            "CCOHC\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Fy7ZqZ8y3Jbr",
        "outputId": "a108e027-baa9-4769-f0e4-7ef302bf0cbf",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 167
        }
      },
      "source": [
        "smiles.build_vocab(my_dataset)    # Defines a vocabulary object that will be used to numericalize a field."
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "error",
          "ename": "NameError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-33-55fa6c24ef9c>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0msmiles\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbuild_vocab\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmy_dataset\u001b[0m\u001b[0;34m)\u001b[0m    \u001b[0;31m# Defines a vocabulary object that will be used to numericalize a field.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;31mNameError\u001b[0m: name 'smiles' is not defined"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jdAFkOF--eTj",
        "outputId": "e574de05-d1ab-4df9-f07b-93ff4c3cfb0e",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 52
        }
      },
      "source": [
        "smiles.vocab.stoi"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "defaultdict(<function torchtext.vocab._default_unk_index>,\n",
              "            {'<eos>': 3, '<pad>': 1, '<sos>': 2, '<unk>': 0})"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 148
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "d9llRJT0AMFq",
        "outputId": "1705694a-95ee-4b5f-d9d4-25d4a08e0d68",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "#smiles.vocab.freqs.elements()\n",
        "print(smiles.vocab.freqs)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Counter()\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2uHT8xHi3TWn"
      },
      "source": [
        "train_iter = torchtext.data.BucketIterator(dataset=my_torchtext_dataset,\n",
        "                                           batch_size=3,\n",
        "                                           train=True,\n",
        "                                           shuffle=True,\n",
        "                                           repeat=False)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rQY0JNcRLfWI"
      },
      "source": [
        "train_iter = torchtext.data.Iterator(dataset=my_torchtext_dataset,\n",
        "                                           batch_size=3,\n",
        "                                           train=True,\n",
        "                                           shuffle=True,\n",
        "                                           repeat=False)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ghW2BVlL3lLN",
        "outputId": "a8f04a80-8ba3-4641-9c21-70f333ff176e",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 434
        }
      },
      "source": [
        "next(iter(train_iter))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "error",
          "ename": "KeyError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/pandas/core/indexes/range.py\u001b[0m in \u001b[0;36mget_loc\u001b[0;34m(self, key, method, tolerance)\u001b[0m\n\u001b[1;32m    354\u001b[0m                 \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 355\u001b[0;31m                     \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_range\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mindex\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnew_key\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    356\u001b[0m                 \u001b[0;32mexcept\u001b[0m \u001b[0mValueError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0merr\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mValueError\u001b[0m: 83 is not in range",
            "\nThe above exception was the direct cause of the following exception:\n",
            "\u001b[0;31mKeyError\u001b[0m                                  Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-164-de5dec51cbc0>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mnext\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0miter\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrain_iter\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/torchtext/data/iterator.py\u001b[0m in \u001b[0;36m__iter__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    140\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__iter__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    141\u001b[0m         \u001b[0;32mwhile\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 142\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minit_epoch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    143\u001b[0m             \u001b[0;32mfor\u001b[0m \u001b[0midx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mminibatch\u001b[0m \u001b[0;32min\u001b[0m \u001b[0menumerate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbatches\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    144\u001b[0m                 \u001b[0;31m# fast-forward if loaded from state\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/torchtext/data/iterator.py\u001b[0m in \u001b[0;36minit_epoch\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    116\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_random_state_this_epoch\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrandom_shuffler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrandom_state\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    117\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 118\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcreate_batches\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    119\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    120\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_restored_from_state\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/torchtext/data/iterator.py\u001b[0m in \u001b[0;36mcreate_batches\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    127\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    128\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mcreate_batches\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 129\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbatches\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mbatch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbatch_size\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbatch_size_fn\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    130\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    131\u001b[0m     \u001b[0;34m@\u001b[0m\u001b[0mproperty\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/torchtext/data/iterator.py\u001b[0m in \u001b[0;36mdata\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    103\u001b[0m             \u001b[0mxs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msorted\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdataset\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkey\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msort_key\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    104\u001b[0m         \u001b[0;32melif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshuffle\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 105\u001b[0;31m             \u001b[0mxs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdataset\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrandom_shuffler\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdataset\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    106\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    107\u001b[0m             \u001b[0mxs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdataset\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/torchtext/data/iterator.py\u001b[0m in \u001b[0;36m<listcomp>\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m    103\u001b[0m             \u001b[0mxs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msorted\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdataset\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkey\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msort_key\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    104\u001b[0m         \u001b[0;32melif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshuffle\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 105\u001b[0;31m             \u001b[0mxs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdataset\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrandom_shuffler\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdataset\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    106\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    107\u001b[0m             \u001b[0mxs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdataset\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/torchtext/data/dataset.py\u001b[0m in \u001b[0;36m__getitem__\u001b[0;34m(self, i)\u001b[0m\n\u001b[1;32m    137\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    138\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__getitem__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mi\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 139\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexamples\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    140\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    141\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__len__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/pandas/core/frame.py\u001b[0m in \u001b[0;36m__getitem__\u001b[0;34m(self, key)\u001b[0m\n\u001b[1;32m   2900\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcolumns\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnlevels\u001b[0m \u001b[0;34m>\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2901\u001b[0m                 \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_getitem_multilevel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2902\u001b[0;31m             \u001b[0mindexer\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcolumns\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_loc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2903\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mis_integer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mindexer\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2904\u001b[0m                 \u001b[0mindexer\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mindexer\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/pandas/core/indexes/range.py\u001b[0m in \u001b[0;36mget_loc\u001b[0;34m(self, key, method, tolerance)\u001b[0m\n\u001b[1;32m    355\u001b[0m                     \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_range\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mindex\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnew_key\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    356\u001b[0m                 \u001b[0;32mexcept\u001b[0m \u001b[0mValueError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0merr\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 357\u001b[0;31m                     \u001b[0;32mraise\u001b[0m \u001b[0mKeyError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0merr\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    358\u001b[0m             \u001b[0;32mraise\u001b[0m \u001b[0mKeyError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    359\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0msuper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_loc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmethod\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mmethod\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtolerance\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtolerance\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mKeyError\u001b[0m: 83"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qcYNXX4RL35d",
        "outputId": "c04ffce4-d213-477c-9920-eb894e6b5db4",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 237
        }
      },
      "source": [
        "for examples in train_iter(\n",
        "            self.train_dataset, batch_size, train=True,\n",
        "            shuffle=True, repeat=False\n",
        "        ):\n",
        "    x = examples.comment_text # (fix_length, batch_size) Tensor\n",
        "    y = torch.stack([\n",
        "        examples.toxic, examples.severe_toxic, \n",
        "        examples.obscene,\n",
        "        examples.threat, examples.insult, \n",
        "        examples.identity_hate\n",
        "    ], dim=1)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "error",
          "ename": "NameError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-167-def8145bbad6>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m for examples in train_iter(\n\u001b[0;32m----> 2\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain_dataset\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_size\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrain\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      3\u001b[0m             \u001b[0mshuffle\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrepeat\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m         ):\n\u001b[1;32m      5\u001b[0m     \u001b[0mx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mexamples\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcomment_text\u001b[0m \u001b[0;31m# (fix_length, batch_size) Tensor\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mNameError\u001b[0m: name 'self' is not defined"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-pEtX0_HP9mR",
        "outputId": "ee90d238-7ad9-4ab1-a977-9fa6970213c9",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "print(torchtext.__version__)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "0.7.0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VLHTtZM0PwL4"
      },
      "source": [
        "from torchtext import datasets"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Vx4K27gBPtGM"
      },
      "source": [
        "from torchtext.datasets import EnWik9"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "c8vKeZBeQn52",
        "outputId": "abc6a529-e1d9-4d4f-d32e-34e0a93381eb",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "EnWik9"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "torchtext.datasets.unsupervised_learning.EnWik9"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 34
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-SkMTp4QTBJs",
        "outputId": "fbbb2696-59e3-4c14-d00e-70c56fdc9be0",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 396
        }
      },
      "source": [
        "example_iter = torchtext.data.Iterator(dataset=EnWik9,\n",
        "                                           batch_size=3,\n",
        "                                           train=True,\n",
        "                                           shuffle=True,\n",
        "                                           repeat=False,\n",
        "                                       sort_key=None)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/torchtext/data/iterator.py:48: UserWarning: Iterator class will be retired in the 0.8.0 release and moved to torchtext.legacy. Please see 0.7.0 release notes for further information.\n",
            "  warnings.warn('{} class will be retired in the 0.8.0 release and moved to torchtext.legacy. Please see 0.7.0 release notes for further information.'.format(self.__class__.__name__), UserWarning)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "error",
          "ename": "AttributeError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-37-d613cd83d47b>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      4\u001b[0m                                            \u001b[0mshuffle\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m                                            \u001b[0mrepeat\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 6\u001b[0;31m                                        sort_key=None)\n\u001b[0m",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/torchtext/data/iterator.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, dataset, batch_size, sort_key, device, batch_size_fn, train, repeat, shuffle, sort, sort_within_batch)\u001b[0m\n\u001b[1;32m     59\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msort_within_batch\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msort_within_batch\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     60\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0msort_key\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 61\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msort_key\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdataset\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msort_key\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     62\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     63\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msort_key\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msort_key\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mAttributeError\u001b[0m: type object 'EnWik9' has no attribute 'sort_key'"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Ie-QsAMTDkvB"
      },
      "source": [
        "##Data preparation (int_tokens)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pbnX3OGthbLj"
      },
      "source": [
        "df_dataset = pd.DataFrame(tokenized_dataset)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "M5FV6DfGh6Wf",
        "outputId": "b6627497-fd3d-4e86-90b8-1330705db5f3",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 538
        }
      },
      "source": [
        "df_dataset    # 14 rows × 112 columns"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>0</th>\n",
              "      <th>1</th>\n",
              "      <th>2</th>\n",
              "      <th>3</th>\n",
              "      <th>4</th>\n",
              "      <th>5</th>\n",
              "      <th>6</th>\n",
              "      <th>7</th>\n",
              "      <th>8</th>\n",
              "      <th>9</th>\n",
              "      <th>10</th>\n",
              "      <th>11</th>\n",
              "      <th>12</th>\n",
              "      <th>13</th>\n",
              "      <th>14</th>\n",
              "      <th>15</th>\n",
              "      <th>16</th>\n",
              "      <th>17</th>\n",
              "      <th>18</th>\n",
              "      <th>19</th>\n",
              "      <th>20</th>\n",
              "      <th>21</th>\n",
              "      <th>22</th>\n",
              "      <th>23</th>\n",
              "      <th>24</th>\n",
              "      <th>25</th>\n",
              "      <th>26</th>\n",
              "      <th>27</th>\n",
              "      <th>28</th>\n",
              "      <th>29</th>\n",
              "      <th>30</th>\n",
              "      <th>31</th>\n",
              "      <th>32</th>\n",
              "      <th>33</th>\n",
              "      <th>34</th>\n",
              "      <th>35</th>\n",
              "      <th>36</th>\n",
              "      <th>37</th>\n",
              "      <th>38</th>\n",
              "      <th>39</th>\n",
              "      <th>...</th>\n",
              "      <th>72</th>\n",
              "      <th>73</th>\n",
              "      <th>74</th>\n",
              "      <th>75</th>\n",
              "      <th>76</th>\n",
              "      <th>77</th>\n",
              "      <th>78</th>\n",
              "      <th>79</th>\n",
              "      <th>80</th>\n",
              "      <th>81</th>\n",
              "      <th>82</th>\n",
              "      <th>83</th>\n",
              "      <th>84</th>\n",
              "      <th>85</th>\n",
              "      <th>86</th>\n",
              "      <th>87</th>\n",
              "      <th>88</th>\n",
              "      <th>89</th>\n",
              "      <th>90</th>\n",
              "      <th>91</th>\n",
              "      <th>92</th>\n",
              "      <th>93</th>\n",
              "      <th>94</th>\n",
              "      <th>95</th>\n",
              "      <th>96</th>\n",
              "      <th>97</th>\n",
              "      <th>98</th>\n",
              "      <th>99</th>\n",
              "      <th>100</th>\n",
              "      <th>101</th>\n",
              "      <th>102</th>\n",
              "      <th>103</th>\n",
              "      <th>104</th>\n",
              "      <th>105</th>\n",
              "      <th>106</th>\n",
              "      <th>107</th>\n",
              "      <th>108</th>\n",
              "      <th>109</th>\n",
              "      <th>110</th>\n",
              "      <th>111</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>C</td>\n",
              "      <td>(</td>\n",
              "      <td>C</td>\n",
              "      <td>(</td>\n",
              "      <td>C</td>\n",
              "      <td>(</td>\n",
              "      <td>C</td>\n",
              "      <td>O</td>\n",
              "      <td>P</td>\n",
              "      <td>(</td>\n",
              "      <td>=</td>\n",
              "      <td>O</td>\n",
              "      <td>)</td>\n",
              "      <td>(</td>\n",
              "      <td>O</td>\n",
              "      <td>)</td>\n",
              "      <td>O</td>\n",
              "      <td>)</td>\n",
              "      <td>O</td>\n",
              "      <td>)</td>\n",
              "      <td>O</td>\n",
              "      <td>)</td>\n",
              "      <td>C</td>\n",
              "      <td>(</td>\n",
              "      <td>=</td>\n",
              "      <td>O</td>\n",
              "      <td>)</td>\n",
              "      <td>C</td>\n",
              "      <td>(</td>\n",
              "      <td>=</td>\n",
              "      <td>O</td>\n",
              "      <td>)</td>\n",
              "      <td>O</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>...</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>C</td>\n",
              "      <td>C</td>\n",
              "      <td>C</td>\n",
              "      <td>(</td>\n",
              "      <td>C</td>\n",
              "      <td>)</td>\n",
              "      <td>C</td>\n",
              "      <td>(</td>\n",
              "      <td>=</td>\n",
              "      <td>O</td>\n",
              "      <td>)</td>\n",
              "      <td>C</td>\n",
              "      <td>(</td>\n",
              "      <td>=</td>\n",
              "      <td>O</td>\n",
              "      <td>)</td>\n",
              "      <td>O</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>...</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>C</td>\n",
              "      <td>C</td>\n",
              "      <td>(</td>\n",
              "      <td>C</td>\n",
              "      <td>)</td>\n",
              "      <td>(</td>\n",
              "      <td>C</td>\n",
              "      <td>O</td>\n",
              "      <td>P</td>\n",
              "      <td>(</td>\n",
              "      <td>=</td>\n",
              "      <td>O</td>\n",
              "      <td>)</td>\n",
              "      <td>(</td>\n",
              "      <td>O</td>\n",
              "      <td>)</td>\n",
              "      <td>O</td>\n",
              "      <td>P</td>\n",
              "      <td>(</td>\n",
              "      <td>=</td>\n",
              "      <td>O</td>\n",
              "      <td>)</td>\n",
              "      <td>(</td>\n",
              "      <td>O</td>\n",
              "      <td>)</td>\n",
              "      <td>O</td>\n",
              "      <td>C</td>\n",
              "      <td>C</td>\n",
              "      <td>1</td>\n",
              "      <td>C</td>\n",
              "      <td>(</td>\n",
              "      <td>C</td>\n",
              "      <td>(</td>\n",
              "      <td>C</td>\n",
              "      <td>(</td>\n",
              "      <td>O</td>\n",
              "      <td>1</td>\n",
              "      <td>)</td>\n",
              "      <td>N</td>\n",
              "      <td>2</td>\n",
              "      <td>...</td>\n",
              "      <td>C</td>\n",
              "      <td>(</td>\n",
              "      <td>C</td>\n",
              "      <td>(</td>\n",
              "      <td>=</td>\n",
              "      <td>O</td>\n",
              "      <td>)</td>\n",
              "      <td>N</td>\n",
              "      <td>C</td>\n",
              "      <td>C</td>\n",
              "      <td>C</td>\n",
              "      <td>(</td>\n",
              "      <td>=</td>\n",
              "      <td>O</td>\n",
              "      <td>)</td>\n",
              "      <td>N</td>\n",
              "      <td>C</td>\n",
              "      <td>C</td>\n",
              "      <td>S</td>\n",
              "      <td>C</td>\n",
              "      <td>(</td>\n",
              "      <td>=</td>\n",
              "      <td>O</td>\n",
              "      <td>)</td>\n",
              "      <td>C</td>\n",
              "      <td>C</td>\n",
              "      <td>(</td>\n",
              "      <td>C</td>\n",
              "      <td>C</td>\n",
              "      <td>C</td>\n",
              "      <td>(</td>\n",
              "      <td>=</td>\n",
              "      <td>O</td>\n",
              "      <td>)</td>\n",
              "      <td>O</td>\n",
              "      <td>)</td>\n",
              "      <td>O</td>\n",
              "      <td>)</td>\n",
              "      <td>O</td>\n",
              "      <td>None</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>C</td>\n",
              "      <td>1</td>\n",
              "      <td>=</td>\n",
              "      <td>C</td>\n",
              "      <td>C</td>\n",
              "      <td>(</td>\n",
              "      <td>=</td>\n",
              "      <td>C</td>\n",
              "      <td>C</td>\n",
              "      <td>(</td>\n",
              "      <td>=</td>\n",
              "      <td>C</td>\n",
              "      <td>1</td>\n",
              "      <td>)</td>\n",
              "      <td>O</td>\n",
              "      <td>)</td>\n",
              "      <td>C</td>\n",
              "      <td>C</td>\n",
              "      <td>C</td>\n",
              "      <td>(</td>\n",
              "      <td>=</td>\n",
              "      <td>O</td>\n",
              "      <td>)</td>\n",
              "      <td>O</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>...</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>C</td>\n",
              "      <td>O</td>\n",
              "      <td>C</td>\n",
              "      <td>1</td>\n",
              "      <td>=</td>\n",
              "      <td>C</td>\n",
              "      <td>C</td>\n",
              "      <td>2</td>\n",
              "      <td>=</td>\n",
              "      <td>C</td>\n",
              "      <td>(</td>\n",
              "      <td>C</td>\n",
              "      <td>=</td>\n",
              "      <td>C</td>\n",
              "      <td>N</td>\n",
              "      <td>=</td>\n",
              "      <td>C</td>\n",
              "      <td>2</td>\n",
              "      <td>C</td>\n",
              "      <td>=</td>\n",
              "      <td>C</td>\n",
              "      <td>1</td>\n",
              "      <td>)</td>\n",
              "      <td>C</td>\n",
              "      <td>(</td>\n",
              "      <td>C</td>\n",
              "      <td>3</td>\n",
              "      <td>C</td>\n",
              "      <td>C</td>\n",
              "      <td>4</td>\n",
              "      <td>C</td>\n",
              "      <td>C</td>\n",
              "      <td>N</td>\n",
              "      <td>3</td>\n",
              "      <td>C</td>\n",
              "      <td>C</td>\n",
              "      <td>4</td>\n",
              "      <td>(</td>\n",
              "      <td>C</td>\n",
              "      <td>=</td>\n",
              "      <td>...</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>C</td>\n",
              "      <td>(</td>\n",
              "      <td>C</td>\n",
              "      <td>C</td>\n",
              "      <td>(</td>\n",
              "      <td>=</td>\n",
              "      <td>O</td>\n",
              "      <td>)</td>\n",
              "      <td>O</td>\n",
              "      <td>)</td>\n",
              "      <td>C</td>\n",
              "      <td>(</td>\n",
              "      <td>=</td>\n",
              "      <td>O</td>\n",
              "      <td>)</td>\n",
              "      <td>C</td>\n",
              "      <td>C</td>\n",
              "      <td>(</td>\n",
              "      <td>=</td>\n",
              "      <td>O</td>\n",
              "      <td>)</td>\n",
              "      <td>O</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>...</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>C</td>\n",
              "      <td>C</td>\n",
              "      <td>(</td>\n",
              "      <td>C</td>\n",
              "      <td>)</td>\n",
              "      <td>(</td>\n",
              "      <td>C</td>\n",
              "      <td>O</td>\n",
              "      <td>P</td>\n",
              "      <td>(</td>\n",
              "      <td>=</td>\n",
              "      <td>O</td>\n",
              "      <td>)</td>\n",
              "      <td>(</td>\n",
              "      <td>O</td>\n",
              "      <td>)</td>\n",
              "      <td>O</td>\n",
              "      <td>P</td>\n",
              "      <td>(</td>\n",
              "      <td>=</td>\n",
              "      <td>O</td>\n",
              "      <td>)</td>\n",
              "      <td>(</td>\n",
              "      <td>O</td>\n",
              "      <td>)</td>\n",
              "      <td>O</td>\n",
              "      <td>C</td>\n",
              "      <td>C</td>\n",
              "      <td>1</td>\n",
              "      <td>C</td>\n",
              "      <td>(</td>\n",
              "      <td>C</td>\n",
              "      <td>(</td>\n",
              "      <td>C</td>\n",
              "      <td>(</td>\n",
              "      <td>O</td>\n",
              "      <td>1</td>\n",
              "      <td>)</td>\n",
              "      <td>N</td>\n",
              "      <td>2</td>\n",
              "      <td>...</td>\n",
              "      <td>C</td>\n",
              "      <td>(</td>\n",
              "      <td>C</td>\n",
              "      <td>(</td>\n",
              "      <td>=</td>\n",
              "      <td>O</td>\n",
              "      <td>)</td>\n",
              "      <td>N</td>\n",
              "      <td>C</td>\n",
              "      <td>C</td>\n",
              "      <td>C</td>\n",
              "      <td>(</td>\n",
              "      <td>=</td>\n",
              "      <td>O</td>\n",
              "      <td>)</td>\n",
              "      <td>N</td>\n",
              "      <td>C</td>\n",
              "      <td>C</td>\n",
              "      <td>S</td>\n",
              "      <td>C</td>\n",
              "      <td>(</td>\n",
              "      <td>=</td>\n",
              "      <td>O</td>\n",
              "      <td>)</td>\n",
              "      <td>C</td>\n",
              "      <td>C</td>\n",
              "      <td>(</td>\n",
              "      <td>=</td>\n",
              "      <td>O</td>\n",
              "      <td>)</td>\n",
              "      <td>C</td>\n",
              "      <td>C</td>\n",
              "      <td>C</td>\n",
              "      <td>(</td>\n",
              "      <td>=</td>\n",
              "      <td>O</td>\n",
              "      <td>)</td>\n",
              "      <td>O</td>\n",
              "      <td>)</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>C</td>\n",
              "      <td>(</td>\n",
              "      <td>C</td>\n",
              "      <td>(</td>\n",
              "      <td>C</td>\n",
              "      <td>(</td>\n",
              "      <td>C</td>\n",
              "      <td>(</td>\n",
              "      <td>=</td>\n",
              "      <td>O</td>\n",
              "      <td>)</td>\n",
              "      <td>C</td>\n",
              "      <td>(</td>\n",
              "      <td>C</td>\n",
              "      <td>(</td>\n",
              "      <td>=</td>\n",
              "      <td>O</td>\n",
              "      <td>)</td>\n",
              "      <td>O</td>\n",
              "      <td>)</td>\n",
              "      <td>O</td>\n",
              "      <td>)</td>\n",
              "      <td>O</td>\n",
              "      <td>)</td>\n",
              "      <td>O</td>\n",
              "      <td>)</td>\n",
              "      <td>O</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>...</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>C</td>\n",
              "      <td>C</td>\n",
              "      <td>(</td>\n",
              "      <td>=</td>\n",
              "      <td>O</td>\n",
              "      <td>)</td>\n",
              "      <td>C</td>\n",
              "      <td>C</td>\n",
              "      <td>(</td>\n",
              "      <td>=</td>\n",
              "      <td>O</td>\n",
              "      <td>)</td>\n",
              "      <td>O</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>...</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>C</td>\n",
              "      <td>(</td>\n",
              "      <td>C</td>\n",
              "      <td>1</td>\n",
              "      <td>C</td>\n",
              "      <td>(</td>\n",
              "      <td>C</td>\n",
              "      <td>(</td>\n",
              "      <td>C</td>\n",
              "      <td>(</td>\n",
              "      <td>C</td>\n",
              "      <td>(</td>\n",
              "      <td>O</td>\n",
              "      <td>1</td>\n",
              "      <td>)</td>\n",
              "      <td>O</td>\n",
              "      <td>)</td>\n",
              "      <td>O</td>\n",
              "      <td>)</td>\n",
              "      <td>O</td>\n",
              "      <td>)</td>\n",
              "      <td>O</td>\n",
              "      <td>C</td>\n",
              "      <td>2</td>\n",
              "      <td>C</td>\n",
              "      <td>(</td>\n",
              "      <td>C</td>\n",
              "      <td>(</td>\n",
              "      <td>=</td>\n",
              "      <td>O</td>\n",
              "      <td>)</td>\n",
              "      <td>C</td>\n",
              "      <td>(</td>\n",
              "      <td>C</td>\n",
              "      <td>(</td>\n",
              "      <td>O</td>\n",
              "      <td>2</td>\n",
              "      <td>)</td>\n",
              "      <td>C</td>\n",
              "      <td>O</td>\n",
              "      <td>...</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10</th>\n",
              "      <td>C</td>\n",
              "      <td>(</td>\n",
              "      <td>C</td>\n",
              "      <td>(</td>\n",
              "      <td>=</td>\n",
              "      <td>O</td>\n",
              "      <td>)</td>\n",
              "      <td>C</td>\n",
              "      <td>(</td>\n",
              "      <td>=</td>\n",
              "      <td>O</td>\n",
              "      <td>)</td>\n",
              "      <td>O</td>\n",
              "      <td>)</td>\n",
              "      <td>S</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>...</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11</th>\n",
              "      <td>C</td>\n",
              "      <td>C</td>\n",
              "      <td>(</td>\n",
              "      <td>=</td>\n",
              "      <td>C</td>\n",
              "      <td>C</td>\n",
              "      <td>(</td>\n",
              "      <td>=</td>\n",
              "      <td>O</td>\n",
              "      <td>)</td>\n",
              "      <td>S</td>\n",
              "      <td>C</td>\n",
              "      <td>C</td>\n",
              "      <td>N</td>\n",
              "      <td>C</td>\n",
              "      <td>(</td>\n",
              "      <td>=</td>\n",
              "      <td>O</td>\n",
              "      <td>)</td>\n",
              "      <td>C</td>\n",
              "      <td>C</td>\n",
              "      <td>N</td>\n",
              "      <td>C</td>\n",
              "      <td>(</td>\n",
              "      <td>=</td>\n",
              "      <td>O</td>\n",
              "      <td>)</td>\n",
              "      <td>C</td>\n",
              "      <td>(</td>\n",
              "      <td>C</td>\n",
              "      <td>(</td>\n",
              "      <td>C</td>\n",
              "      <td>)</td>\n",
              "      <td>(</td>\n",
              "      <td>C</td>\n",
              "      <td>)</td>\n",
              "      <td>C</td>\n",
              "      <td>O</td>\n",
              "      <td>P</td>\n",
              "      <td>(</td>\n",
              "      <td>...</td>\n",
              "      <td>N</td>\n",
              "      <td>C</td>\n",
              "      <td>3</td>\n",
              "      <td>=</td>\n",
              "      <td>C</td>\n",
              "      <td>(</td>\n",
              "      <td>N</td>\n",
              "      <td>=</td>\n",
              "      <td>C</td>\n",
              "      <td>N</td>\n",
              "      <td>=</td>\n",
              "      <td>C</td>\n",
              "      <td>3</td>\n",
              "      <td>2</td>\n",
              "      <td>)</td>\n",
              "      <td>N</td>\n",
              "      <td>)</td>\n",
              "      <td>O</td>\n",
              "      <td>)</td>\n",
              "      <td>O</td>\n",
              "      <td>P</td>\n",
              "      <td>(</td>\n",
              "      <td>=</td>\n",
              "      <td>O</td>\n",
              "      <td>)</td>\n",
              "      <td>(</td>\n",
              "      <td>O</td>\n",
              "      <td>)</td>\n",
              "      <td>O</td>\n",
              "      <td>)</td>\n",
              "      <td>O</td>\n",
              "      <td>)</td>\n",
              "      <td>C</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>12</th>\n",
              "      <td>C</td>\n",
              "      <td>C</td>\n",
              "      <td>(</td>\n",
              "      <td>=</td>\n",
              "      <td>C</td>\n",
              "      <td>C</td>\n",
              "      <td>C</td>\n",
              "      <td>C</td>\n",
              "      <td>(</td>\n",
              "      <td>=</td>\n",
              "      <td>C</td>\n",
              "      <td>C</td>\n",
              "      <td>C</td>\n",
              "      <td>C</td>\n",
              "      <td>(</td>\n",
              "      <td>=</td>\n",
              "      <td>C</td>\n",
              "      <td>C</td>\n",
              "      <td>C</td>\n",
              "      <td>C</td>\n",
              "      <td>(</td>\n",
              "      <td>=</td>\n",
              "      <td>C</td>\n",
              "      <td>C</td>\n",
              "      <td>C</td>\n",
              "      <td>C</td>\n",
              "      <td>(</td>\n",
              "      <td>=</td>\n",
              "      <td>C</td>\n",
              "      <td>C</td>\n",
              "      <td>C</td>\n",
              "      <td>C</td>\n",
              "      <td>(</td>\n",
              "      <td>=</td>\n",
              "      <td>C</td>\n",
              "      <td>C</td>\n",
              "      <td>C</td>\n",
              "      <td>C</td>\n",
              "      <td>(</td>\n",
              "      <td>=</td>\n",
              "      <td>...</td>\n",
              "      <td>)</td>\n",
              "      <td>C</td>\n",
              "      <td>)</td>\n",
              "      <td>C</td>\n",
              "      <td>)</td>\n",
              "      <td>C</td>\n",
              "      <td>)</td>\n",
              "      <td>C</td>\n",
              "      <td>)</td>\n",
              "      <td>C</td>\n",
              "      <td>)</td>\n",
              "      <td>C</td>\n",
              "      <td>)</td>\n",
              "      <td>C</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13</th>\n",
              "      <td>C</td>\n",
              "      <td>1</td>\n",
              "      <td>=</td>\n",
              "      <td>C</td>\n",
              "      <td>C</td>\n",
              "      <td>(</td>\n",
              "      <td>=</td>\n",
              "      <td>C</td>\n",
              "      <td>C</td>\n",
              "      <td>(</td>\n",
              "      <td>=</td>\n",
              "      <td>C</td>\n",
              "      <td>1</td>\n",
              "      <td>)</td>\n",
              "      <td>O</td>\n",
              "      <td>)</td>\n",
              "      <td>C</td>\n",
              "      <td>=</td>\n",
              "      <td>O</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>...</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "      <td>None</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>14 rows × 112 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "   0   1   2   3   4   5   6    ...   105   106   107   108   109   110   111\n",
              "0    C   (   C   (   C   (   C  ...  None  None  None  None  None  None  None\n",
              "1    C   C   C   (   C   )   C  ...  None  None  None  None  None  None  None\n",
              "2    C   C   (   C   )   (   C  ...     )     O     )     O     )     O  None\n",
              "3    C   1   =   C   C   (   =  ...  None  None  None  None  None  None  None\n",
              "4    C   O   C   1   =   C   C  ...  None  None  None  None  None  None  None\n",
              "5    C   (   C   C   (   =   O  ...  None  None  None  None  None  None  None\n",
              "6    C   C   (   C   )   (   C  ...     (     =     O     )     O     )     O\n",
              "7    C   (   C   (   C   (   C  ...  None  None  None  None  None  None  None\n",
              "8    C   C   (   =   O   )   C  ...  None  None  None  None  None  None  None\n",
              "9    C   (   C   1   C   (   C  ...  None  None  None  None  None  None  None\n",
              "10   C   (   C   (   =   O   )  ...  None  None  None  None  None  None  None\n",
              "11   C   C   (   =   C   C   (  ...  None  None  None  None  None  None  None\n",
              "12   C   C   (   =   C   C   C  ...  None  None  None  None  None  None  None\n",
              "13   C   1   =   C   C   (   =  ...  None  None  None  None  None  None  None\n",
              "\n",
              "[14 rows x 112 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 18
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "MWmCtv3dDux6"
      },
      "source": [
        "####Vocabulary (all occuring SMILES tokens)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0Y66-J_eDsNe"
      },
      "source": [
        "alphabet = set()\n",
        "\n",
        "for i in tokenized_dataset:\n",
        "  alphabet.update(i)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FklgBhnhEQKV"
      },
      "source": [
        "#alphabet    # length: 12"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "FnZRAbk7sQNW"
      },
      "source": [
        "####Dictionary (Token alphabet; vocabulary + UNK, EOL etc.)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2V7mqxxOz_fO",
        "outputId": "74669eaa-4be6-4e10-ddd5-29d66c2439aa",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "dict_token_alphabet = {}\n",
        "dict_token_alphabet.update({'UNK': 0, 'SOS': 1, 'EOS':2})\n",
        "\n",
        "index = 3\n",
        "for i in alphabet:\n",
        "  dict_token_alphabet.update({i: index})\n",
        "  index = index+1\n",
        "\n",
        "dict_token_alphabet"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'(': 6,\n",
              " ')': 7,\n",
              " '1': 8,\n",
              " '2': 4,\n",
              " '3': 10,\n",
              " '4': 3,\n",
              " '=': 5,\n",
              " 'C': 13,\n",
              " 'EOS': 2,\n",
              " 'N': 9,\n",
              " 'O': 12,\n",
              " 'P': 11,\n",
              " 'S': 14,\n",
              " 'SOS': 1,\n",
              " 'UNK': 0}"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gjroB_Y70xrj"
      },
      "source": [
        "####Vectorize SMILES"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mlIdYoQL1AAR"
      },
      "source": [
        "int_tokens = [None]*len(tokenized_dataset)    # empty list of length 14\n",
        "\n",
        "for i in range(len(tokenized_dataset)):\n",
        "  int_tokens[i] = [None]*len(tokenized_dataset[i])\n",
        "\n",
        "  for j in range(len(tokenized_dataset[i])):\n",
        "    int_tokens[i][j] = dict_token_alphabet.get(tokenized_dataset[i][j])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ZdV61eVjael7"
      },
      "source": [
        "int_tokens is a list of lists"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eFl5u0YVOZWN",
        "outputId": "346c5a5d-f64f-4e25-b098-1d6bc4741b9e",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "a = torch.ShortTensor(int_tokens[0])   # or torch.cuda.ShortTensor for a GPU-tensor\n",
        "len(int_tokens[0])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "33"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 54
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uRDzkqMIbIse",
        "outputId": "f9f5bae9-4d5d-4bd9-9677-bb08567f2728",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "b = torch.ShortTensor(int_tokens[1])\n",
        "len(int_tokens[1])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "17"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 55
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Y1MOGmIcbJ6L",
        "outputId": "f666365c-57b1-4e02-b584-84964c974cf7",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 167
        }
      },
      "source": [
        "c = torch.cat(tensors=(a,b), dim=1)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "error",
          "ename": "IndexError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mIndexError\u001b[0m                                Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-59-3bd920dbd95d>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mc\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtensors\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ma\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mb\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdim\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;31mIndexError\u001b[0m: Dimension out of range (expected to be in range of [-1, 0], but got 1)"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gZeNZtL5cAnS",
        "outputId": "9b274816-1857-4701-9510-c54b245b25b4",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "len(c)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "50"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 58
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FYpWd5WCvWIS",
        "outputId": "91d1a24b-7089-4e6b-e33a-499dfd0acf6b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 69
        }
      },
      "source": [
        "batch = int_tokens[0]\n",
        "\n",
        "# make a numpy array out of it\n",
        "batch = np.array(batch)\n",
        "\n",
        "# make a PyTorch tensor\n",
        "batch = torch.tensor(batch, dtype=torch.long)\n",
        "\n",
        "print(batch)\n",
        "print('The shape of batch is:', batch.shape)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "tensor([ 7, 11,  7, 11,  7, 11,  7,  4, 13, 11,  9,  4, 10, 11,  4, 10,  4, 10,\n",
            "         4, 10,  4, 10,  7, 11,  9,  4, 10,  7, 11,  9,  4, 10,  4])\n",
            "The shape of batch is: torch.Size([33])\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "f-pIKaFEwQVh"
      },
      "source": [
        "Now we have the first step of having our data in the shape (batch_size, seq_len). to feed it into an LSTM, we still need input_size."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4OVvhX7WwBIE"
      },
      "source": [
        "####Embedding layer"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "keULFvHEw4RM"
      },
      "source": [
        "Define layer"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NenQIp9dwFUL"
      },
      "source": [
        "vocab_size = len(dict_token_alphabet)  # 14\n",
        "embed_dim = 10 #<- What does the size of the embeddings mean in my case?\n",
        "\n",
        "word_embedding_layer = nn.Embedding(vocab_size, embed_dim)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ZIV42R9qw57B"
      },
      "source": [
        "Push batch through layer"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qG5Pd99qwie8",
        "outputId": "3154f28c-1b32-4600-e8bf-56fd7ce88fa1",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "batch = word_embedding_layer(batch)\n",
        "\n",
        "print('The shape of batch is:', batch.shape)   # 3, 33, 10\n",
        "#print()\n",
        "#print(batch)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "The shape of batch is: torch.Size([33, 10])\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "MFSlKKrsxJz8"
      },
      "source": [
        "Now we have want we want: (batch_size, seq_len, input_size)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_dpS7bI6G9Nh"
      },
      "source": [
        "####Dataloader"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xAbKAXUaHYn1"
      },
      "source": [
        "PyTorch Dataloader:\\\n",
        "[PyTorch guide to its Dataloader class](https://pytorch.org/docs/stable/data.html)\\\n",
        "[Parameters](https://pytorch.org/docs/stable/data.html#torch.utils.data.DataLoader)\n",
        "\n",
        "\\\n",
        "(Alternative to the PyTorch function:  Function from the [Medium article](https://medium.com/@sunitachoudhary103/generating-molecules-using-a-char-rnn-in-pytorch-16885fd9394b))"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iz0uD-LQ8m0H",
        "outputId": "d34bd8f3-5c93-4ea1-8ad2-e3c09290aa04",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "batch_size = 1\n",
        "n_iters = 10\n",
        "num_epochs = n_iters / (len(tokenized_dataset) / batch_size)\n",
        "num_epochs = int(num_epochs)\n",
        "print(num_epochs, 'epochs')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "0 epochs\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fd4fCccsMrYf"
      },
      "source": [
        "Customize my own dataloader.\\\n",
        "What you need is basically pad your variable-length of input and torch.stack() them together into a single tensor. This tensor will then be used as an input to your model.\\\n",
        "[torch.cat](https://pytorch.org/docs/stable/generated/torch.cat.html)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mliMI5MyO0gG"
      },
      "source": [
        "#my_stack = nn.utils.rnn.pack_padded_sequence(tokenized_dataset) # ATTENTION: the sequences need to already be padded!"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "eoC197lOMOTP"
      },
      "source": [
        "There is a [pack_padded_sequence](https://pytorch.org/docs/master/generated/torch.nn.utils.rnn.pack_padded_sequence.html#torch.nn.utils.rnn.pack_padded_sequence) function that packs a tensor containing padded sequences of variable length."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oGKhsIruHXhP"
      },
      "source": [
        "my_loader = torch.utils.data.DataLoader(dataset=tokenized_dataset,\n",
        "                                        batch_size=batch_size,\n",
        "                                        shuffle=True,\n",
        "                                        sampler=None,       # optional: custom Sampler object\n",
        "                                        batch_sampler=None, # optional: provide a custom sampler\n",
        "                                        num_workers=0,      # if positive int => multi-process data loading\n",
        "                                        collate_fn=None,    # optional: custom collate function\n",
        "                                        pin_memory=False,   # to speed it up when working on a GPU\n",
        "                                        drop_last=False,\n",
        "                                        timeout=0,\n",
        "                                        worker_init_fn=None)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ys72nlwTKB29",
        "outputId": "21bef055-b17b-4e3b-ed5e-add7e1fbfb15",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 280
        }
      },
      "source": [
        "for i in my_loader:\n",
        "  print(i)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[('C',), ('C',), ('(',), ('C',), (')',), ('(',), ('C',), ('O',), ('P',), ('(',), ('=',), ('O',), (')',), ('(',), ('O',), (')',), ('O',), ('P',), ('(',), ('=',), ('O',), (')',), ('(',), ('O',), (')',), ('O',), ('C',), ('C',), ('1',), ('C',), ('(',), ('C',), ('(',), ('C',), ('(',), ('O',), ('1',), (')',), ('N',), ('2',), ('C',), ('=',), ('N',), ('C',), ('3',), ('=',), ('C',), ('(',), ('N',), ('=',), ('C',), ('N',), ('=',), ('C',), ('3',), ('2',), (')',), ('N',), (')',), ('O',), (')',), ('O',), ('P',), ('(',), ('=',), ('O',), (')',), ('(',), ('O',), (')',), ('O',), (')',), ('C',), ('(',), ('C',), ('(',), ('=',), ('O',), (')',), ('N',), ('C',), ('C',), ('C',), ('(',), ('=',), ('O',), (')',), ('N',), ('C',), ('C',), ('S',), ('C',), ('(',), ('=',), ('O',), (')',), ('C',), ('C',), ('(',), ('C',), ('C',), ('C',), ('(',), ('=',), ('O',), (')',), ('O',), (')',), ('O',), (')',), ('O',)]\n",
            "[('C',), ('1',), ('=',), ('C',), ('C',), ('(',), ('=',), ('C',), ('C',), ('(',), ('=',), ('C',), ('1',), (')',), ('O',), (')',), ('C',), ('=',), ('O',)]\n",
            "[('C',), ('(',), ('C',), ('(',), ('=',), ('O',), (')',), ('C',), ('(',), ('=',), ('O',), (')',), ('O',), (')',), ('S',)]\n",
            "[('C',), ('C',), ('(',), ('=',), ('C',), ('C',), ('(',), ('=',), ('O',), (')',), ('S',), ('C',), ('C',), ('N',), ('C',), ('(',), ('=',), ('O',), (')',), ('C',), ('C',), ('N',), ('C',), ('(',), ('=',), ('O',), (')',), ('C',), ('(',), ('C',), ('(',), ('C',), (')',), ('(',), ('C',), (')',), ('C',), ('O',), ('P',), ('(',), ('=',), ('O',), (')',), ('(',), ('O',), (')',), ('O',), ('P',), ('(',), ('=',), ('O',), (')',), ('(',), ('O',), (')',), ('O',), ('C',), ('C',), ('1',), ('C',), ('(',), ('C',), ('(',), ('C',), ('(',), ('O',), ('1',), (')',), ('N',), ('2',), ('C',), ('=',), ('N',), ('C',), ('3',), ('=',), ('C',), ('(',), ('N',), ('=',), ('C',), ('N',), ('=',), ('C',), ('3',), ('2',), (')',), ('N',), (')',), ('O',), (')',), ('O',), ('P',), ('(',), ('=',), ('O',), (')',), ('(',), ('O',), (')',), ('O',), (')',), ('O',), (')',), ('C',)]\n",
            "[('C',), ('(',), ('C',), ('(',), ('C',), ('(',), ('C',), ('(',), ('=',), ('O',), (')',), ('C',), ('(',), ('C',), ('(',), ('=',), ('O',), (')',), ('O',), (')',), ('O',), (')',), ('O',), (')',), ('O',), (')',), ('O',)]\n",
            "[('C',), ('O',), ('C',), ('1',), ('=',), ('C',), ('C',), ('2',), ('=',), ('C',), ('(',), ('C',), ('=',), ('C',), ('N',), ('=',), ('C',), ('2',), ('C',), ('=',), ('C',), ('1',), (')',), ('C',), ('(',), ('C',), ('3',), ('C',), ('C',), ('4',), ('C',), ('C',), ('N',), ('3',), ('C',), ('C',), ('4',), ('(',), ('C',), ('=',), ('C',), (')',), ('O',), (')',), ('O',)]\n",
            "[('C',), ('C',), ('(',), ('C',), (')',), ('(',), ('C',), ('O',), ('P',), ('(',), ('=',), ('O',), (')',), ('(',), ('O',), (')',), ('O',), ('P',), ('(',), ('=',), ('O',), (')',), ('(',), ('O',), (')',), ('O',), ('C',), ('C',), ('1',), ('C',), ('(',), ('C',), ('(',), ('C',), ('(',), ('O',), ('1',), (')',), ('N',), ('2',), ('C',), ('=',), ('N',), ('C',), ('3',), ('=',), ('C',), ('(',), ('N',), ('=',), ('C',), ('N',), ('=',), ('C',), ('3',), ('2',), (')',), ('N',), (')',), ('O',), (')',), ('O',), ('P',), ('(',), ('=',), ('O',), (')',), ('(',), ('O',), (')',), ('O',), (')',), ('C',), ('(',), ('C',), ('(',), ('=',), ('O',), (')',), ('N',), ('C',), ('C',), ('C',), ('(',), ('=',), ('O',), (')',), ('N',), ('C',), ('C',), ('S',), ('C',), ('(',), ('=',), ('O',), (')',), ('C',), ('C',), ('(',), ('=',), ('O',), (')',), ('C',), ('C',), ('C',), ('(',), ('=',), ('O',), (')',), ('O',), (')',), ('O',)]\n",
            "[('C',), ('C',), ('(',), ('=',), ('C',), ('C',), ('C',), ('C',), ('(',), ('=',), ('C',), ('C',), ('C',), ('C',), ('(',), ('=',), ('C',), ('C',), ('C',), ('C',), ('(',), ('=',), ('C',), ('C',), ('C',), ('C',), ('(',), ('=',), ('C',), ('C',), ('C',), ('C',), ('(',), ('=',), ('C',), ('C',), ('C',), ('C',), ('(',), ('=',), ('C',), ('C',), ('C',), ('C',), ('(',), ('=',), ('C',), ('C',), ('C',), ('1',), ('=',), ('C',), ('(',), ('C',), ('=',), ('C',), ('C',), ('(',), ('=',), ('C',), ('1',), (')',), ('C',), ('(',), ('=',), ('O',), (')',), ('O',), (')',), ('O',), (')',), ('C',), (')',), ('C',), (')',), ('C',), (')',), ('C',), (')',), ('C',), (')',), ('C',), (')',), ('C',), (')',), ('C',)]\n",
            "[('C',), ('1',), ('=',), ('C',), ('C',), ('(',), ('=',), ('C',), ('C',), ('(',), ('=',), ('C',), ('1',), (')',), ('O',), (')',), ('C',), ('C',), ('C',), ('(',), ('=',), ('O',), (')',), ('O',)]\n",
            "[('C',), ('(',), ('C',), ('1',), ('C',), ('(',), ('C',), ('(',), ('C',), ('(',), ('C',), ('(',), ('O',), ('1',), (')',), ('O',), (')',), ('O',), (')',), ('O',), (')',), ('O',), ('C',), ('2',), ('C',), ('(',), ('C',), ('(',), ('=',), ('O',), (')',), ('C',), ('(',), ('C',), ('(',), ('O',), ('2',), (')',), ('C',), ('O',), (')',), ('O',), (')',), ('O',), (')',), ('O',)]\n",
            "[('C',), ('(',), ('C',), ('(',), ('C',), ('(',), ('C',), ('O',), ('P',), ('(',), ('=',), ('O',), (')',), ('(',), ('O',), (')',), ('O',), (')',), ('O',), (')',), ('O',), (')',), ('C',), ('(',), ('=',), ('O',), (')',), ('C',), ('(',), ('=',), ('O',), (')',), ('O',)]\n",
            "[('C',), ('(',), ('C',), ('C',), ('(',), ('=',), ('O',), (')',), ('O',), (')',), ('C',), ('(',), ('=',), ('O',), (')',), ('C',), ('C',), ('(',), ('=',), ('O',), (')',), ('O',)]\n",
            "[('C',), ('C',), ('C',), ('(',), ('C',), (')',), ('C',), ('(',), ('=',), ('O',), (')',), ('C',), ('(',), ('=',), ('O',), (')',), ('O',)]\n",
            "[('C',), ('C',), ('(',), ('=',), ('O',), (')',), ('C',), ('C',), ('(',), ('=',), ('O',), (')',), ('O',)]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "psWMRbCkug_P"
      },
      "source": [
        "I have a map-style dataset, at least for now with this testing dataset. However, when I take the huge PubChem dataset, this might change to iterable-style.\\\n",
        "When automatic batching is enabled, collate_fn is called with a list of data samples at each time. It is expected to collate the input samples into a batch for yielding from the data loader iterator."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "eWptnErF3eoO"
      },
      "source": [
        "##Putting my data into a model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fI9M5yyA818q",
        "outputId": "4ca61bbc-7346-446c-db88-8200076e19ed",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "batch.size()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "torch.Size([3, 33, 10])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 27
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Z-sb8M0X73Vv"
      },
      "source": [
        "# choose the input parameters\n",
        "input_size = batch.shape[2]\n",
        "hidden_dim = 32\n",
        "\n",
        "# define the model\n",
        "my_LSTM=nn.LSTM(input_size, hidden_dim)\n",
        "\n",
        "# initialise the lstm\n",
        "for parameter in my_LSTM.parameters():\n",
        "    nn.init.normal_(parameter)\n",
        "    #print(parameter)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "n5Q_9l4mLh5_",
        "outputId": "97e9a719-2f53-4120-81f5-8121f5928a97",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 400
        }
      },
      "source": [
        "# Model architecture visualization\n",
        "writer = SummaryWriter()    # default `log_dir` is \"runs\"\n",
        "\n",
        "writer.add_graph(model=my_LSTM.cpu(),\n",
        "                 input_to_model=batch,\n",
        "                 verbose=True)\n",
        "writer.close()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "input must have 3 dimensions, got 2\n",
            "Error occurs, No graph saved\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "error",
          "ename": "RuntimeError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mRuntimeError\u001b[0m                              Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-40-60fc95c90503>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      4\u001b[0m writer.add_graph(model=my_LSTM.cpu(),\n\u001b[1;32m      5\u001b[0m                  \u001b[0minput_to_model\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mbatch\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 6\u001b[0;31m                  verbose=True)\n\u001b[0m\u001b[1;32m      7\u001b[0m \u001b[0mwriter\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mclose\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/torch/utils/tensorboard/writer.py\u001b[0m in \u001b[0;36madd_graph\u001b[0;34m(self, model, input_to_model, verbose)\u001b[0m\n\u001b[1;32m    712\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mhasattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'forward'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    713\u001b[0m             \u001b[0;31m# A valid PyTorch model should have a 'forward' method\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 714\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_get_file_writer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0madd_graph\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mgraph\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput_to_model\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mverbose\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    715\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    716\u001b[0m             \u001b[0;31m# Caffe2 models do not have the 'forward' method\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/torch/utils/tensorboard/_pytorch_graph.py\u001b[0m in \u001b[0;36mgraph\u001b[0;34m(model, args, verbose)\u001b[0m\n\u001b[1;32m    289\u001b[0m             \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    290\u001b[0m             \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'Error occurs, No graph saved'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 291\u001b[0;31m             \u001b[0;32mraise\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    292\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    293\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mverbose\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/torch/utils/tensorboard/_pytorch_graph.py\u001b[0m in \u001b[0;36mgraph\u001b[0;34m(model, args, verbose)\u001b[0m\n\u001b[1;32m    283\u001b[0m     \u001b[0;32mwith\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0monnx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mselect_model_mode_for_export\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0monnx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTrainingMode\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mEVAL\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m  \u001b[0;31m# TODO: move outside of torch.onnx?\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    284\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 285\u001b[0;31m             \u001b[0mtrace\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjit\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrace\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    286\u001b[0m             \u001b[0mgraph\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtrace\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgraph\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    287\u001b[0m             \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_C\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_jit_pass_inline\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mgraph\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/torch/jit/__init__.py\u001b[0m in \u001b[0;36mtrace\u001b[0;34m(func, example_inputs, optimize, check_trace, check_inputs, check_tolerance, strict, _force_outplace, _module_class, _compilation_unit)\u001b[0m\n\u001b[1;32m    953\u001b[0m         return trace_module(func, {'forward': example_inputs}, None,\n\u001b[1;32m    954\u001b[0m                             \u001b[0mcheck_trace\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mwrap_check_inputs\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcheck_inputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 955\u001b[0;31m                             check_tolerance, strict, _force_outplace, _module_class)\n\u001b[0m\u001b[1;32m    956\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    957\u001b[0m     if (hasattr(func, '__self__') and isinstance(func.__self__, torch.nn.Module) and\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/torch/jit/__init__.py\u001b[0m in \u001b[0;36mtrace_module\u001b[0;34m(mod, inputs, optimize, check_trace, check_inputs, check_tolerance, strict, _force_outplace, _module_class, _compilation_unit)\u001b[0m\n\u001b[1;32m   1107\u001b[0m             \u001b[0mfunc\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmod\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mmethod_name\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m\"forward\"\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0mgetattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmod\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmethod_name\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1108\u001b[0m             \u001b[0mexample_inputs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmake_tuple\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mexample_inputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1109\u001b[0;31m             \u001b[0mmodule\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_c\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_create_method_from_trace\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmethod_name\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mexample_inputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvar_lookup_fn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstrict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0m_force_outplace\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1110\u001b[0m             \u001b[0mcheck_trace_method\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodule\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_c\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_get_method\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmethod_name\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1111\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m_call_impl\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m    718\u001b[0m                 \u001b[0minput\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    719\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_C\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_get_tracing_state\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 720\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    721\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    722\u001b[0m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m_slow_forward\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m    702\u001b[0m                 \u001b[0mrecording_scopes\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mFalse\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    703\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 704\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    705\u001b[0m         \u001b[0;32mfinally\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    706\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mrecording_scopes\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/torch/nn/modules/rnn.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, input, hx)\u001b[0m\n\u001b[1;32m    572\u001b[0m             \u001b[0mhx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpermute_hidden\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msorted_indices\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    573\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 574\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcheck_forward_args\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_sizes\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    575\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mbatch_sizes\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    576\u001b[0m             result = _VF.lstm(input, hx, self._flat_weights, self.bias, self.num_layers,\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/torch/nn/modules/rnn.py\u001b[0m in \u001b[0;36mcheck_forward_args\u001b[0;34m(self, input, hidden, batch_sizes)\u001b[0m\n\u001b[1;32m    523\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    524\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mcheck_forward_args\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mTensor\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhidden\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mTuple\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mTensor\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mTensor\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_sizes\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mOptional\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mTensor\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 525\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcheck_input\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_sizes\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    526\u001b[0m         \u001b[0mexpected_hidden_size\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_expected_hidden_size\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_sizes\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    527\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/torch/nn/modules/rnn.py\u001b[0m in \u001b[0;36mcheck_input\u001b[0;34m(self, input, batch_sizes)\u001b[0m\n\u001b[1;32m    173\u001b[0m             raise RuntimeError(\n\u001b[1;32m    174\u001b[0m                 'input must have {} dimensions, got {}'.format(\n\u001b[0;32m--> 175\u001b[0;31m                     expected_input_dim, input.dim()))\n\u001b[0m\u001b[1;32m    176\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minput_size\u001b[0m \u001b[0;34m!=\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    177\u001b[0m             raise RuntimeError(\n",
            "\u001b[0;31mRuntimeError\u001b[0m: input must have 3 dimensions, got 2"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "i2D0RePMqWow",
        "outputId": "07f7e03e-ba09-487e-9b15-a764b9b6449c",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 350
        }
      },
      "source": [
        "%tensorboard --writer"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "ERROR: Failed to launch TensorBoard (exited with 2).\n",
              "Contents of stderr:\n",
              "2020-10-21 17:57:25.351254: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcudart.so.10.1\n",
              "usage: tensorboard [-h] [--helpfull] [--logdir PATH] [--logdir_spec PATH_SPEC]\n",
              "                   [--host ADDR] [--bind_all] [--port PORT]\n",
              "                   [--purge_orphaned_data BOOL] [--db URI] [--db_import]\n",
              "                   [--inspect] [--version_tb] [--tag TAG] [--event_file PATH]\n",
              "                   [--path_prefix PATH] [--window_title TEXT]\n",
              "                   [--max_reload_threads COUNT] [--reload_interval SECONDS]\n",
              "                   [--reload_task TYPE] [--reload_multifile BOOL]\n",
              "                   [--reload_multifile_inactive_secs SECONDS]\n",
              "                   [--generic_data TYPE]\n",
              "                   [--samples_per_plugin SAMPLES_PER_PLUGIN]\n",
              "                   [--debugger_data_server_grpc_port PORT]\n",
              "                   [--debugger_port PORT]\n",
              "                   [--whatif-use-unsafe-custom-prediction YOUR_CUSTOM_PREDICT_FUNCTION.py]\n",
              "                   {serve,dev} ...\n",
              "tensorboard: error: unrecognized arguments: --writer"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hGzRP_hhTZJW"
      },
      "source": [
        "%tensorboard --logdir logs"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qrLQwsvCme_q"
      },
      "source": [
        "####Instantiate Loss Class"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2yt9PTizBQXm"
      },
      "source": [
        "#import torch.optim as optim"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Vv3dM2HM4icS"
      },
      "source": [
        "criterion = nn.CrossEntropyLoss()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ZK2x4gyg4m40"
      },
      "source": [
        "####Instantiate Optimizer Class"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "erTaZkNZ4wix"
      },
      "source": [
        "# minibatch SGD\n",
        "learning_rate = 0.1\n",
        "optimizer = torch.optim.SGD(my_LSTM.parameters(), lr=learning_rate)  "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iI0KzlHp5Vs9",
        "outputId": "6b933b66-e68c-45d7-c850-0435297883d3",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 86
        }
      },
      "source": [
        "#for i in range(len(list(my_LSTM.parameters()))):\n",
        "#    print(list(my_LSTM.parameters())[i].size())"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "torch.Size([128, 10])\n",
            "torch.Size([128, 32])\n",
            "torch.Size([128])\n",
            "torch.Size([128])\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "g1FQydbIBBZQ"
      },
      "source": [
        "##Train"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "96ITxePTIQrv",
        "outputId": "310edf81-9ac8-44ee-cd78-bd951f125b66",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 365
        }
      },
      "source": [
        "lstm_out, hidden = my_LSTM(batch)\n",
        "\n",
        "print('The shape of lstm_out is:', lstm_out.shape) # (seq_len, batch_size, hidden_dim)\n",
        "print('The shape of hidden is:', hidden.shape) # (num_layers*num_directions, batch_size, hidden_dim)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "error",
          "ename": "RuntimeError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mRuntimeError\u001b[0m                              Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-65-fce9ca7e12a6>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mlstm_out\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhidden\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmy_LSTM\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'The shape of lstm_out is:'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlstm_out\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;31m# (seq_len, batch_size, hidden_dim)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'The shape of hidden is:'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhidden\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;31m# (num_layers*num_directions, batch_size, hidden_dim)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m_call_impl\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m    720\u001b[0m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    721\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 722\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    723\u001b[0m         for hook in itertools.chain(\n\u001b[1;32m    724\u001b[0m                 \u001b[0m_global_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/torch/nn/modules/rnn.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, input, hx)\u001b[0m\n\u001b[1;32m    572\u001b[0m             \u001b[0mhx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpermute_hidden\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msorted_indices\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    573\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 574\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcheck_forward_args\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_sizes\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    575\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mbatch_sizes\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    576\u001b[0m             result = _VF.lstm(input, hx, self._flat_weights, self.bias, self.num_layers,\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/torch/nn/modules/rnn.py\u001b[0m in \u001b[0;36mcheck_forward_args\u001b[0;34m(self, input, hidden, batch_sizes)\u001b[0m\n\u001b[1;32m    523\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    524\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mcheck_forward_args\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mTensor\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhidden\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mTuple\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mTensor\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mTensor\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_sizes\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mOptional\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mTensor\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 525\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcheck_input\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_sizes\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    526\u001b[0m         \u001b[0mexpected_hidden_size\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_expected_hidden_size\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_sizes\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    527\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/torch/nn/modules/rnn.py\u001b[0m in \u001b[0;36mcheck_input\u001b[0;34m(self, input, batch_sizes)\u001b[0m\n\u001b[1;32m    173\u001b[0m             raise RuntimeError(\n\u001b[1;32m    174\u001b[0m                 'input must have {} dimensions, got {}'.format(\n\u001b[0;32m--> 175\u001b[0;31m                     expected_input_dim, input.dim()))\n\u001b[0m\u001b[1;32m    176\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minput_size\u001b[0m \u001b[0;34m!=\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    177\u001b[0m             raise RuntimeError(\n",
            "\u001b[0;31mRuntimeError\u001b[0m: input must have 3 dimensions, got 2"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NXEhfDtv0cwm"
      },
      "source": [
        "import timeit"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RwnWGxp28cWa",
        "outputId": "d363db1b-9e17-4f85-8746-bd303861f0d6",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "runs=10**4\n",
        "\n",
        "print(\"Time Pytorch LSTM {} runs: {:.3f}s\".format(runs, timeit.timeit(\"my_LSTM(batch)\", \n",
        "                                       setup=\"from __main__ import my_LSTM, batch\", \n",
        "                                       number=runs))\n",
        "     )"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Time Pytorch LSTM 10000 runs: 18.690s\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "65_EsPhh87Hx"
      },
      "source": [
        "[Optuna](https://github.com/optuna/optuna): automated Hyperparameter Optimization"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "t9yRS4Vx9A2U"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "tc8H338EBqZY"
      },
      "source": [
        "##Visualization"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Xb7DvNoKFrsJ"
      },
      "source": [
        "Let's write some stuff to TensorBoard, and log into it to see how things go :)\n",
        "You can log into TensorBoard by running the command from this exercise folder in Terminal:\n",
        "\n",
        "```tensorboard --logdir=runs```\n",
        "\n",
        "For Linux user, you can use open a Terminal and simply run it\n",
        "\n",
        "For Windows user with Anaconda, you may open an Anaconda Prompt and then run the command. Otherwise use your default setup of running python code in cmd.\n",
        "\n",
        "Note that before running the command **you may get into root folder of this Notebook first**. Finally from the command line and then navigating to http://localhost:6006. If everything went well you will be presented with the tensorboard stup and after executing the next cell you should see the following images in TensorBoard.\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NnX5-BG4FX8z",
        "outputId": "223a072c-bf0f-4918-ed0f-2edfef6cbc49",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 185
        }
      },
      "source": [
        "# write to tensorboard\n",
        "#writer.add_image('four_mnist_images', img_grid)     # I need to write my data here!"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "error",
          "ename": "NameError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-22-215867d9be08>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;31m# write to tensorboard\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0mwriter\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0madd_image\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'four_mnist_images'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mimg_grid\u001b[0m\u001b[0;34m)\u001b[0m     \u001b[0;31m# I need to write my data here!\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;31mNameError\u001b[0m: name 'img_grid' is not defined"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tFiGnqQWF3XJ",
        "outputId": "2e5a1fe2-70df-4224-b739-a47a9e0b188b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 400
        }
      },
      "source": [
        "# Model architecture visualization\n",
        "\n",
        "writer.add_graph(model=word_embedding_layer.cpu(),\n",
        "                 input_to_model=batch,\n",
        "                 verbose=True)\n",
        "writer.close()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Expected tensor for argument #1 'indices' to have scalar type Long; but got torch.FloatTensor instead (while checking arguments for embedding)\n",
            "Error occurs, No graph saved\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "error",
          "ename": "RuntimeError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mRuntimeError\u001b[0m                              Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-37-029555a0e577>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m writer.add_graph(model=word_embedding_layer.cpu(),\n\u001b[1;32m      2\u001b[0m                  \u001b[0minput_to_model\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mbatch\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m                  verbose=True)\n\u001b[0m\u001b[1;32m      4\u001b[0m \u001b[0mwriter\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mclose\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/torch/utils/tensorboard/writer.py\u001b[0m in \u001b[0;36madd_graph\u001b[0;34m(self, model, input_to_model, verbose)\u001b[0m\n\u001b[1;32m    712\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mhasattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'forward'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    713\u001b[0m             \u001b[0;31m# A valid PyTorch model should have a 'forward' method\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 714\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_get_file_writer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0madd_graph\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mgraph\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput_to_model\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mverbose\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    715\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    716\u001b[0m             \u001b[0;31m# Caffe2 models do not have the 'forward' method\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/torch/utils/tensorboard/_pytorch_graph.py\u001b[0m in \u001b[0;36mgraph\u001b[0;34m(model, args, verbose)\u001b[0m\n\u001b[1;32m    289\u001b[0m             \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    290\u001b[0m             \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'Error occurs, No graph saved'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 291\u001b[0;31m             \u001b[0;32mraise\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    292\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    293\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mverbose\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/torch/utils/tensorboard/_pytorch_graph.py\u001b[0m in \u001b[0;36mgraph\u001b[0;34m(model, args, verbose)\u001b[0m\n\u001b[1;32m    283\u001b[0m     \u001b[0;32mwith\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0monnx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mselect_model_mode_for_export\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0monnx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTrainingMode\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mEVAL\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m  \u001b[0;31m# TODO: move outside of torch.onnx?\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    284\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 285\u001b[0;31m             \u001b[0mtrace\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjit\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrace\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    286\u001b[0m             \u001b[0mgraph\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtrace\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgraph\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    287\u001b[0m             \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_C\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_jit_pass_inline\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mgraph\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/torch/jit/__init__.py\u001b[0m in \u001b[0;36mtrace\u001b[0;34m(func, example_inputs, optimize, check_trace, check_inputs, check_tolerance, strict, _force_outplace, _module_class, _compilation_unit)\u001b[0m\n\u001b[1;32m    953\u001b[0m         return trace_module(func, {'forward': example_inputs}, None,\n\u001b[1;32m    954\u001b[0m                             \u001b[0mcheck_trace\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mwrap_check_inputs\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcheck_inputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 955\u001b[0;31m                             check_tolerance, strict, _force_outplace, _module_class)\n\u001b[0m\u001b[1;32m    956\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    957\u001b[0m     if (hasattr(func, '__self__') and isinstance(func.__self__, torch.nn.Module) and\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/torch/jit/__init__.py\u001b[0m in \u001b[0;36mtrace_module\u001b[0;34m(mod, inputs, optimize, check_trace, check_inputs, check_tolerance, strict, _force_outplace, _module_class, _compilation_unit)\u001b[0m\n\u001b[1;32m   1107\u001b[0m             \u001b[0mfunc\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmod\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mmethod_name\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m\"forward\"\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0mgetattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmod\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmethod_name\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1108\u001b[0m             \u001b[0mexample_inputs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmake_tuple\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mexample_inputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1109\u001b[0;31m             \u001b[0mmodule\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_c\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_create_method_from_trace\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmethod_name\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mexample_inputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvar_lookup_fn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstrict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0m_force_outplace\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1110\u001b[0m             \u001b[0mcheck_trace_method\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodule\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_c\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_get_method\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmethod_name\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1111\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m_call_impl\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m    718\u001b[0m                 \u001b[0minput\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    719\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_C\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_get_tracing_state\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 720\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    721\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    722\u001b[0m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m_slow_forward\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m    702\u001b[0m                 \u001b[0mrecording_scopes\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mFalse\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    703\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 704\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    705\u001b[0m         \u001b[0;32mfinally\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    706\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mrecording_scopes\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/torch/nn/modules/sparse.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, input)\u001b[0m\n\u001b[1;32m    124\u001b[0m         return F.embedding(\n\u001b[1;32m    125\u001b[0m             \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mweight\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpadding_idx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmax_norm\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 126\u001b[0;31m             self.norm_type, self.scale_grad_by_freq, self.sparse)\n\u001b[0m\u001b[1;32m    127\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    128\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mextra_repr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/torch/nn/functional.py\u001b[0m in \u001b[0;36membedding\u001b[0;34m(input, weight, padding_idx, max_norm, norm_type, scale_grad_by_freq, sparse)\u001b[0m\n\u001b[1;32m   1812\u001b[0m         \u001b[0;31m# remove once script supports set_grad_enabled\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1813\u001b[0m         \u001b[0m_no_grad_embedding_renorm_\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mweight\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmax_norm\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnorm_type\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1814\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0membedding\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mweight\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpadding_idx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mscale_grad_by_freq\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msparse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1815\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1816\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mRuntimeError\u001b[0m: Expected tensor for argument #1 'indices' to have scalar type Long; but got torch.FloatTensor instead (while checking arguments for embedding)"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UdvkUaxIGWul"
      },
      "source": [
        "# Start TensorBoard within the notebook using magics function\n",
        "%tensorboard — logdir logs"
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}