deep_learning_lstm_cnn.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 5,
  "metadata": {
    "kernelspec": {
      "name": "python3",
      "language": "python",
      "display_name": "Python 3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.8.3"
    },
    "colab": {
      "name": "deep-learning-lstm-cnn",
      "provenance": [],
      "collapsed_sections": []
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 258
        },
        "id": "po4C-niMCfZy",
        "outputId": "40381df3-b35e-4c89-8156-a77fd8fb98af"
      },
      "source": [
        "import pandas as pd\n",
        "import numpy as  np\n",
        "import matplotlib as plt\n",
        "\n",
        "%tensorflow_version 2.x\n",
        "\n",
        "df_encoding = \"utf-8\"\n",
        "\n",
        "from google.colab import drive\n",
        "drive.mount('/content/drive')\n",
        "\n",
        "df = pd.read_csv(\"/content/drive/My Drive/deep-learning/cleaned-dataset.csv\", encoding=df_encoding)\n",
        "df = df.sample(frac=1, replace=True, random_state=1)\n",
        "\n",
        "# keep max length of words and sentences\n",
        "words_length = max(len(w) for w in df['tweets'])\n",
        "sentence_length = max(len(w.split(' ')) for w in df['tweets'])\n",
        "\n",
        "print(\"max chars in a tweet:\", words_length)\n",
        "print(\"max num of words in a tweet:\", sentence_length)\n",
        "\n",
        "df.head()"
      ],
      "id": "po4C-niMCfZy",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Mounted at /content/drive\n",
            "max chars in a tweet: 140\n",
            "max num of words in a tweet: 40\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>sentiment</th>\n",
              "      <th>tweets</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>128037</th>\n",
              "      <td>1</td>\n",
              "      <td>well that would be interesting but i thought y...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>491755</th>\n",
              "      <td>0</td>\n",
              "      <td>i feel the same too</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>470924</th>\n",
              "      <td>0</td>\n",
              "      <td>hey joe wish i could watch it man but it aint ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>491263</th>\n",
              "      <td>0</td>\n",
              "      <td>shed be lke quotyou just got a phooonequot hah...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>836489</th>\n",
              "      <td>1</td>\n",
              "      <td>thanks for the feedback</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "        sentiment                                             tweets\n",
              "128037          1  well that would be interesting but i thought y...\n",
              "491755          0                                i feel the same too\n",
              "470924          0  hey joe wish i could watch it man but it aint ...\n",
              "491263          0  shed be lke quotyou just got a phooonequot hah...\n",
              "836489          1                            thanks for the feedback"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 1
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "AqbB813o8mgO"
      },
      "source": [
        "Config tensorflow with TPU"
      ],
      "id": "AqbB813o8mgO"
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Z3ZlWY3MnAd5",
        "outputId": "01643d6f-2d2a-4739-cd84-ac8690ec887e"
      },
      "source": [
        "import tensorflow as tf\n",
        "from tensorflow import keras\n",
        "import os\n",
        "\n",
        "\n",
        "use_tpu = True #@param {type:\"boolean\"}\n",
        "\n",
        "if use_tpu:\n",
        "    assert 'COLAB_TPU_ADDR' in os.environ, 'Missing TPU; did you request a TPU in Notebook Settings?'\n",
        "\n",
        "if 'COLAB_TPU_ADDR' in os.environ:\n",
        "  TF_MASTER = 'grpc://{}'.format(os.environ['COLAB_TPU_ADDR'])\n",
        "else:\n",
        "  TF_MASTER=''\n",
        "\n",
        "# TPU address\n",
        "tpu_address = TF_MASTER\n",
        "\n",
        "resolver = tf.distribute.cluster_resolver.TPUClusterResolver(TF_MASTER)\n",
        "tf.config.experimental_connect_to_cluster(resolver)\n",
        "tf.tpu.experimental.initialize_tpu_system(resolver)\n",
        "strategy = tf.distribute.experimental.TPUStrategy(resolver)"
      ],
      "id": "Z3ZlWY3MnAd5",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Initializing the TPU system: grpc://10.78.6.194:8470\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Initializing the TPU system: grpc://10.78.6.194:8470\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Clearing out eager caches\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Clearing out eager caches\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Finished initializing TPU system.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Finished initializing TPU system.\n",
            "WARNING:absl:`tf.distribute.experimental.TPUStrategy` is deprecated, please use  the non experimental symbol `tf.distribute.TPUStrategy` instead.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Found TPU system:\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Found TPU system:\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Num TPU Cores: 8\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Num TPU Cores: 8\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Num TPU Workers: 1\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Num TPU Workers: 1\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Num TPU Cores Per Worker: 8\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Num TPU Cores Per Worker: 8\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:localhost/replica:0/task:0/device:CPU:0, CPU, 0, 0)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:localhost/replica:0/task:0/device:CPU:0, CPU, 0, 0)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:CPU:0, CPU, 0, 0)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:CPU:0, CPU, 0, 0)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:0, TPU, 0, 0)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:0, TPU, 0, 0)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:1, TPU, 0, 0)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:1, TPU, 0, 0)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:2, TPU, 0, 0)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:2, TPU, 0, 0)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:3, TPU, 0, 0)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:3, TPU, 0, 0)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:4, TPU, 0, 0)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:4, TPU, 0, 0)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:5, TPU, 0, 0)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:5, TPU, 0, 0)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:6, TPU, 0, 0)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:6, TPU, 0, 0)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:7, TPU, 0, 0)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:7, TPU, 0, 0)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU_SYSTEM:0, TPU_SYSTEM, 0, 0)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU_SYSTEM:0, TPU_SYSTEM, 0, 0)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:XLA_CPU:0, XLA_CPU, 0, 0)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:XLA_CPU:0, XLA_CPU, 0, 0)\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "f4j6vpRdCfZ5"
      },
      "source": [
        "## Define usefull functions\n"
      ],
      "id": "f4j6vpRdCfZ5"
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zj1QR5v8CfZ6"
      },
      "source": [
        "def create_subsets(dataset, features, labels, num_classes, train_size=0, valid_size=0, test_size=0):\n",
        "    \n",
        "    subsets = {}\n",
        "    \n",
        "    # Define a size for your train set \n",
        "    train_n = int(train_size * len(dataset))\n",
        "    valid_n = int(valid_size * len(dataset))\n",
        "    test_n = int(test_size * len(dataset))\n",
        "    \n",
        "    # train_test split\n",
        "    subsets['X_train'] = dataset[features][:train_n]\n",
        "    subsets['X_valid'] = dataset[features][train_n:train_n+valid_n]\n",
        "    subsets['X_test'] = dataset[features][train_n+valid_n:train_n+valid_n+test_n]\n",
        "\n",
        "    subsets['Y_train'] = dataset[labels][:train_n]\n",
        "    subsets['Y_valid'] = dataset[labels][train_n:train_n+valid_n]\n",
        "    subsets['Y_test'] = dataset[labels][train_n+valid_n:train_n+valid_n+test_n]\n",
        "    \n",
        "    return subsets\n",
        "\n",
        "def words_to_sequences(max_sentence_length, subsets):\n",
        "    seq_subsets = {}\n",
        "    vocab_size = 0\n",
        "    global tokenizer \n",
        "    tokenizer = Tokenizer()\n",
        "    for key, value in subsets.items():\n",
        "        if key.startswith('X'):\n",
        "            # create vocabulary based on word frequency\n",
        "            #   -word_counts: Dictionary of words and their corresponding counts.\n",
        "            #   -word_docs: Dictionary of words and their corresponding documents appeared in.\n",
        "            #   -word_index: Dictionary of words and their uniquely assigned integers.\n",
        "            #   -document_count: Count of the total number of documents that were used to fit the Tokenizer.\n",
        "            if key.startswith('X_train'):\n",
        "                tokenizer.fit_on_texts(list(value))\n",
        "            \n",
        "            # texts_to_sequences assigns integers to words for each document\n",
        "            sequence = tokenizer.texts_to_sequences(value)\n",
        "            # padding to prepare sequences of same length\n",
        "            sequence = pad_sequences(sequence, maxlen = max_sentence_length)\n",
        "            seq_subsets[key] = sequence\n",
        "            \n",
        "            if len(tokenizer.word_index) > vocab_size:\n",
        "                vocab_size = len(tokenizer.word_index)\n",
        "\n",
        "    return seq_subsets, vocab_size+1;"
      ],
      "id": "zj1QR5v8CfZ6",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "g5M0D16BCfZ-"
      },
      "source": [
        "from keras.preprocessing.text import Tokenizer\n",
        "from keras.preprocessing.sequence import pad_sequences\n",
        "from sklearn.model_selection import train_test_split\n",
        "from tensorflow.keras.utils import to_categorical\n",
        "from collections import defaultdict\n",
        "\n",
        "features = 'tweets'\n",
        "labels = 'sentiment'\n",
        "\n",
        "#create appropriate subsets\n",
        "initial_subsets = create_subsets(dataset=df, features=features, labels=labels, \n",
        "                   train_size=0.6, valid_size=0.2, test_size=0.2, num_classes=2)\n",
        "\n",
        "# convert features (text) to sequences\n",
        "seq_subsets, vocab_size = words_to_sequences(sentence_length, subsets=initial_subsets)"
      ],
      "id": "g5M0D16BCfZ-",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "q0zpDyPBCfZ_"
      },
      "source": [
        "## Load Glove"
      ],
      "id": "q0zpDyPBCfZ_"
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4Fw8_KnsQYZC"
      },
      "source": [
        "EMBED_DIM = 300\n",
        "\n",
        "glove_dict = dict()\n",
        "f = open('/content/drive/My Drive/deep-learning/glove.840B.300d.txt', encoding=\"utf8\")\n",
        "for line in f:\n",
        "    values = line.split()\n",
        "    word = ''.join(values[:-300])\n",
        "    if word in tokenizer.word_index:\n",
        "      coefs = np.asarray(values[-300:], dtype='float32')\n",
        "      glove_dict[word] = coefs\n",
        "f.close()"
      ],
      "id": "4Fw8_KnsQYZC",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UiKZ-uGyRXZ8",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "5067ad21-fc06-47e8-e49a-95eb433af8dc"
      },
      "source": [
        "print('Loaded %s word vectors.' % len(glove_dict))"
      ],
      "id": "UiKZ-uGyRXZ8",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Loaded 82908 word vectors.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rme3XJxQCfZ_"
      },
      "source": [
        "## Create Embedding Matrix "
      ],
      "id": "rme3XJxQCfZ_"
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Gu2KwrPnCfaB"
      },
      "source": [
        "# weight matrix of words in tweets\n",
        "embedding_matrix = np.zeros((vocab_size, EMBED_DIM))\n",
        "\n",
        "for word, i in tokenizer.word_index.items():\n",
        "    embedding_vector = glove_dict.get(word)\n",
        "    if embedding_vector is not None:\n",
        "        embedding_matrix[i] = embedding_vector\n",
        "    \n"
      ],
      "id": "Gu2KwrPnCfaB",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2wTbT6_y8Jvj"
      },
      "source": [
        "## Model training and evalutation "
      ],
      "id": "2wTbT6_y8Jvj"
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "oOhFF9hcS7OQ"
      },
      "source": [
        "### LSTM"
      ],
      "id": "oOhFF9hcS7OQ"
    },
    {
      "cell_type": "code",
      "metadata": {
        "scrolled": false,
        "id": "gg3huMM14haQ",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "f1400446-f97a-4f50-bdd9-9854cc891cd0"
      },
      "source": [
        "from keras.models import Sequential\n",
        "from keras.layers import Dense, Activation, Embedding, Dropout, SpatialDropout1D\n",
        "from keras.layers import LSTM, Conv1D, MaxPooling1D, Bidirectional\n",
        "from keras.optimizers import Adam\n",
        "\n",
        "def get_model():\n",
        "  model = Sequential()\n",
        "\n",
        "  model.add(Embedding(input_dim=vocab_size, output_dim=EMBED_DIM, weights=[embedding_matrix], input_length=sentence_length, trainable=False))\n",
        "  model.add(Bidirectional(LSTM(128, return_sequences=True)))\n",
        "  model.add(Dropout(0.3))\n",
        "  model.add(Bidirectional(LSTM(128, return_sequences=True)))\n",
        "  model.add(Dropout(0.3))\n",
        "  model.add(LSTM(128))\n",
        "  model.add(Dense(64))\n",
        "  model.add(Dense(1, activation='sigmoid'))\n",
        "\n",
        "  return model\n",
        "\n",
        "\n",
        "with strategy.scope():\n",
        "  model = get_model()\n",
        "  model.compile(loss = 'binary_crossentropy', optimizer=Adam(), metrics = ['accuracy'])\n",
        "\n",
        "model.summary()"
      ],
      "id": "gg3huMM14haQ",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model: \"sequential_2\"\n",
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "embedding_2 (Embedding)      (None, 40, 300)           61108800  \n",
            "_________________________________________________________________\n",
            "bidirectional_4 (Bidirection (None, 40, 256)           439296    \n",
            "_________________________________________________________________\n",
            "dropout_4 (Dropout)          (None, 40, 256)           0         \n",
            "_________________________________________________________________\n",
            "bidirectional_5 (Bidirection (None, 40, 256)           394240    \n",
            "_________________________________________________________________\n",
            "dropout_5 (Dropout)          (None, 40, 256)           0         \n",
            "_________________________________________________________________\n",
            "lstm_8 (LSTM)                (None, 128)               197120    \n",
            "_________________________________________________________________\n",
            "dense_4 (Dense)              (None, 64)                8256      \n",
            "_________________________________________________________________\n",
            "dense_5 (Dense)              (None, 1)                 65        \n",
            "=================================================================\n",
            "Total params: 62,147,777\n",
            "Trainable params: 1,038,977\n",
            "Non-trainable params: 61,108,800\n",
            "_________________________________________________________________\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HMLaSyJBCfaB",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "cf6a1be4-0282-449e-a0b7-4458d935cd20"
      },
      "source": [
        "import keras\n",
        "from matplotlib import pyplot as plt\n",
        "from keras.callbacks import EarlyStopping, ReduceLROnPlateau\n",
        "\n",
        "BATCH_SIZE=500\n",
        "\n",
        "callbacks = [ #ReduceLROnPlateau(monitor='val_loss', patience=5, min_delta=0.001, verbose=1),\n",
        "              EarlyStopping(monitor='val_loss', patience=6, restore_best_weights=True, verbose=1)]\n",
        "\n",
        "\n",
        "history = model.fit(seq_subsets['X_train'], initial_subsets['Y_train'], \n",
        "                    validation_data=(seq_subsets['X_valid'],initial_subsets['Y_valid']), shuffle=True, \n",
        "                    epochs=50, batch_size=BATCH_SIZE, callbacks=callbacks)\n"
      ],
      "id": "HMLaSyJBCfaB",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch 1/50\n",
            "1916/1916 [==============================] - 126s 49ms/step - loss: 0.4634 - accuracy: 0.7780 - val_loss: 0.3954 - val_accuracy: 0.8204\n",
            "Epoch 2/50\n",
            "1916/1916 [==============================] - 84s 44ms/step - loss: 0.3841 - accuracy: 0.8266 - val_loss: 0.3638 - val_accuracy: 0.8382\n",
            "Epoch 3/50\n",
            "1916/1916 [==============================] - 85s 44ms/step - loss: 0.3487 - accuracy: 0.8466 - val_loss: 0.3444 - val_accuracy: 0.8495\n",
            "Epoch 4/50\n",
            "1916/1916 [==============================] - 86s 45ms/step - loss: 0.3154 - accuracy: 0.8647 - val_loss: 0.3296 - val_accuracy: 0.8597\n",
            "Epoch 5/50\n",
            "1916/1916 [==============================] - 85s 44ms/step - loss: 0.2830 - accuracy: 0.8819 - val_loss: 0.3185 - val_accuracy: 0.8689\n",
            "Epoch 6/50\n",
            "1916/1916 [==============================] - 85s 44ms/step - loss: 0.2516 - accuracy: 0.8979 - val_loss: 0.3101 - val_accuracy: 0.8767\n",
            "Epoch 7/50\n",
            "1916/1916 [==============================] - 85s 44ms/step - loss: 0.2252 - accuracy: 0.9107 - val_loss: 0.2998 - val_accuracy: 0.8846\n",
            "Epoch 8/50\n",
            "1916/1916 [==============================] - 85s 44ms/step - loss: 0.2029 - accuracy: 0.9214 - val_loss: 0.2966 - val_accuracy: 0.8899\n",
            "Epoch 9/50\n",
            "1916/1916 [==============================] - 85s 44ms/step - loss: 0.1832 - accuracy: 0.9299 - val_loss: 0.2944 - val_accuracy: 0.8938\n",
            "Epoch 10/50\n",
            "1916/1916 [==============================] - 85s 44ms/step - loss: 0.1679 - accuracy: 0.9369 - val_loss: 0.2956 - val_accuracy: 0.8981\n",
            "Epoch 11/50\n",
            "1916/1916 [==============================] - 86s 45ms/step - loss: 0.1533 - accuracy: 0.9432 - val_loss: 0.2965 - val_accuracy: 0.9013\n",
            "Epoch 12/50\n",
            "1916/1916 [==============================] - 85s 44ms/step - loss: 0.1406 - accuracy: 0.9485 - val_loss: 0.2947 - val_accuracy: 0.9036\n",
            "Epoch 13/50\n",
            "1916/1916 [==============================] - 85s 44ms/step - loss: 0.1319 - accuracy: 0.9520 - val_loss: 0.3025 - val_accuracy: 0.9044\n",
            "Epoch 14/50\n",
            "1916/1916 [==============================] - 85s 44ms/step - loss: 0.1228 - accuracy: 0.9554 - val_loss: 0.2969 - val_accuracy: 0.9078\n",
            "Epoch 15/50\n",
            "1916/1916 [==============================] - 85s 44ms/step - loss: 0.1151 - accuracy: 0.9584 - val_loss: 0.2984 - val_accuracy: 0.9097\n",
            "Restoring model weights from the end of the best epoch.\n",
            "Epoch 00015: early stopping\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KMvb_T8tCfaB",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 295
        },
        "outputId": "861b9f9b-d473-4b28-dab5-306e3ee07416"
      },
      "source": [
        "plt.subplot(1, 2, 1)\n",
        "plt.plot(history.history['accuracy'])\n",
        "plt.plot(history.history['val_accuracy'])\n",
        "plt.title('model accuracy')\n",
        "plt.ylabel('accuracy')\n",
        "plt.xlabel('epoch')\n",
        "plt.legend(['train', 'val'], loc='upper left')\n",
        "\n",
        "plt.subplot(1, 2, 2)\n",
        "plt.plot(history.history['loss'])\n",
        "plt.plot(history.history['val_loss'])\n",
        "plt.title('model loss')\n",
        "plt.ylabel('loss')\n",
        "plt.xlabel('epoch')\n",
        "plt.legend(['train', 'val'], loc='upper left')\n",
        "\n",
        "plt.show()"
      ],
      "id": "KMvb_T8tCfaB",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEWCAYAAAB8LwAVAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOydd3gUVffHPyeFhJCEFmoooYP0IiAKKCAgKAgoKKJiQ0XF3v2pr6/YUF4bKKBUCyI2VBRRunSkg/QWakggDRJSzu+PGWAJgSyQzWx27+d55mH2ztyZ74Y7e+bee+45oqoYDAaDwZCTAKcFGAwGg8E7MQbCYDAYDLliDITBYDAYcsUYCIPBYDDkijEQBoPBYMgVYyAMBoPBkCvGQHgxIjJeRF5389ydItLJ05oMBqfJr+fiQq7jrxgDYTAYDIZcMQbC4HFEJMhpDQaD4cIxBuISsbuwT4vIGhFJFZHPRaSciPwmIski8qeIlHQ5v4eIrBeRoyIyR0TquRxrKiL/2PW+AUJz3Ot6EVll110oIo3c1NhdRFaKSJKI7BGRV3Mcv8q+3lH7+EC7vKiIvCciu0QkUUQW2GVXi0hsLn+HTvb+qyIyVUS+EJEkYKCItBSRRfY99ovIxyJSxKV+fRGZKSIJInJQRF4QkfIickxESruc10xE4kQk2J3vbnCGwvBc5KL5PhHZarfBaSJS0S4XEfmfiByyn6G1ItLAPtZNRDbY2vaKyFMX9QfzVlTVbJewATuBxUA5IBo4BPwDNMVqyLOAV+xzawOpwLVAMPAMsBUoYm+7gMftYzcBGcDrdt2m9rVbAYHAnfa9Q1x0dDqHxquBhlgvBI2Ag8CN9rGqQDJwq33f0kAT+9gIYI79vQKBNkCIfb3YXP4Onez9V23tN9r3LAo0B1oDQUAMsBF4zD4/AtgPPGn/zSKAVvax6cCDLvf5H/CR0//vZvOJ52K8y3U6AIeBZnYb/wiYZx/rAqwASgAC1AMq2Mf2A23t/ZJAM6f/9vm5mR5E/vCRqh5U1b3AfGCJqq5U1TTgB6xGDNAP+FVVZ6pqBvAu1o9nG6wfz2DgfVXNUNWpwDKXewwCRqnqElXNUtUJQLpd77yo6hxVXauq2aq6BvgaaG8f7g/8qapf2/eNV9VVIhIA3A08qqp77XsuVNV0N/8mi1T1R/uex1V1haouVtVMVd0JjHLRcD1wQFXfU9U0VU1W1SX2sQnAAAARCcQyZJPc1GBwFq9+LnJwGzBWVf+x2/jzwBUiEoNlkCKAuoCo6kZV3W/XywAuE5FIVT2iqv9c4H29GmMg8oeDLvvHc/kcbu9XxHobAkBVs4E9WG9YFYG9ar+K2Oxy2a8KPGl3o4+KyFGgsl3vvIhIKxGZbQ/NJAIPAFH24crAtlyqRWG96eV2zB325NBQW0R+EZED9rDTG25oAPgJ6wGshvWGmaiqSy9Sk6Fg8ernIgc5NaQA8UC0qs4CPsbqUR8SkdEiEmmf2gfoBuwSkbkicsUF3terMQaiYNmH1aABa2wTqzHvxeqqRttlJ6nisr8HGKqqJVy2MFX92o37fgVMAyqranHgU6yu8snr1silzmEg7RzHUoEwl+8RCJTJcU7OMMGfAP8CtVQ1Enghh4bquQm33zanYPUibsf0HnwRp56L82kohjXcuhdAVT9U1ebAZVhDYk/b5ctUtSdQFvgRq636DMZAFCxTgO4i0tGeZH0Sqzu8EFgEZAJDRCRYRHoDLV3qjgEesHsDIiLFxJp8jnDjvhFAgqqmiUhLrGGlk3wJdBKRviISJCKlRaSJ/RY3FhguIhVFJFBErhCREGAzEGrfPxh4CWvcNi8NSUCKiNQFHnQ59gtQQUQeE5EQEYkQkVYuxycCA4EeGAPhizj1XLjyNXCXiDSx2/gbWENiO0Xkcvv6wVgvR2lAtogUEZHbRKS4PTSWBGRfwt/B6zAGogBR1U1Yb8IfYb2h3wDcoKonVPUE0BvrhzABa1z2e5e6y4H7sLq6R7Am8Qa6eevBwGsikgy8jMtbjqruxuoiP2nfdxXQ2D78FLAWa8w3AXgbCFDVRPuan2G9YaUCZ3g15cJTWIYpGeuh/sZFQzLW8NENwAFgC3CNy/G/sR68f1TVdXjB4AM4+Fy4avgT+D/gO6xeSw3gFvtwJFabPYI1DBUPDLOP3Q7stIdNH8Cay/AZ5MyhPYPBOxGRWcBXqvqZ01oMBn/BGAiD1yMilwMzseZQkp3WYzD4C2aIyeDViMgE4E+sNRPGOBgMBYjpQRgMBoMhV0wPwmAwGAy54jNB1KKiojQmJsZpGQYfZsWKFYdVNed6D49j2rbBk5yvXfuMgYiJiWH58uVOyzD4MCLiiIutadsGT3K+dm2GmAwGg8GQKx41ECLSVUQ22SF0n8vleFUR+UuskMBzRKSSy7EqIvKHiGy0w+nGeFKrwWAwGM7EYwbCjs8zArgOK37JrSJyWY7T3gUmqmoj4DXgTZdjE4FhqloPa2n9IU9pNRgMBsPZeHIOoiWwVVW3A4jIZKAnsMHlnMuAJ+z92VjBrrANSZCqzoRTkRUvmIyMDGJjY0lLS7u4b1CICA0NpVKlSgQHmzw6/oC/tG3Trp3FkwYimjNDPsdiJfVwZTVWnJUPgF5AhFjZw2oDR0Xke6Aa1kKp51Q1y7WyiAzCigdPlSquAR7tG8bGEhERQUxMDGcGg/QtVJX4+HhiY2OpVq2a03IMBYA/tG3Trp3H6Unqp4D2IrISK3nMXiALy3C1tY9fjhUKemDOyqo6WlVbqGqLMmXO9tJKS0ujdOnSPvsAnUREKF26tM+/TRpO4w9t27Rr5/GkgdiLFdP9JJXsslOo6j5V7a2qTYEX7bKjWL2NVaq6XVUzsYaeml2MCF9+gFzxl+9pOI0//J/7w3f0ZjxpIJYBtUSkmljJ6W/BSlpzChGJslNbgpXib6xL3RIicrJb0IEz5y4MhnxlT8IxRszeyuo9R52W4jYJqSdISD3htAyDD+MxA2G/+T8MzMBKUD9FVdeLyGsi0sM+7Wpgk4hsxkpuPtSum4U1vPSXiKzFyjw2xlNaPcnRo0cZOXLkBdfr1q0bR48Wnh+rwkjisQy+WrKbvp8uou07sxk2YxMLt8U7Lcttjh47QVyyuynC8/nepl37BR5dSa2q04HpOcpedtmfCkw9R92ZQCNP6isITj5IgwcPPqM8MzOToKBz//mnT59+zmOGiyc9M4vZ/8bx48q9zPr3ECeysqlRphhPd6lDj8YVqVwqLO+LeAmRRYPZd/Q46RlZhAQHFui9Tbv2D3wm1Ia38txzz7Ft2zaaNGlCcHAwoaGhlCxZkn///ZfNmzdz4403smfPHtLS0nj00UcZNGgQcDq8QkpKCtdddx1XXXUVCxcuJDo6mp9++omiRYs6/M0KD9nZyordR/hh5V5+XbOfxOMZRIWHMKB1VXo1jaZBdGShHOuODA1iH5CUlkmZAjYQpl37B35jIP7z83o27EvK12teVjGSV26of95z3nrrLdatW8eqVauYM2cO3bt3Z926dafc9saOHUupUqU4fvw4l19+OX369KF06dJnXGPLli18/fXXjBkzhr59+/Ldd98xYMCAfP0uvkhSWgbfLo9l4qKd7Io/RtHgQLrUL8eNTaO5qmYUQYFOO/FdGkWCAgkNDuTN3zay98jxfL12Xm3btGv/wG8MhLfQsmXLM3y6P/zwQ3744QcA9uzZw5YtW856kKpVq0aTJk0AaN68OTt37iwwvYWR7XEpTFi4k6krYkk9kUWLqiV5tGMtutQvT7EQ32ryEaFBZGYpijVR5xSmXfsmvvW0nIe83vQLimLFip3anzNnDn/++SeLFi0iLCyMq6++Olef75CQkFP7gYGBHD+ev2+LvoCqMm/LYcb9vYM5m+IIDhRuaFSRu66sRsNKxZ2W5zEiQ4O5t201qpQKo0RYEcd0mHbtm/iNgXCKiIgIkpNzz5SZmJhIyZIlCQsL499//2Xx4sUFrK7wk5qeyff/xDJ+4U62xaUSFR7CY51q0b9VFcpGhDotz+OEFQkkKCCApLTMAjUQpl37B8ZAeJjSpUtz5ZVX0qBBA4oWLUq5cuVOHevatSuffvop9erVo06dOrRu3dpBpYWL+JR0xszfwZdLdpGclknD6OIM79uY7o0qEBJUsBO2TiIiRIQGkZSWgaoW2GS7adf+gc/kpG7RooXmTKqyceNG6tWr55Cigscfvm98Sjqj529n4sJdpGVm0a1BBe6+KoZmVUp6/MdRRFaoaguP3iQX8mrbicdOsCvhGDXKhPvcHAv4R7t2kvO1a99rTQafJCH1BKPnbWfiop0cz8iiR+OKPNKhJjXLRjgtzXHCQ4MQEZLSMnzSQBicw7Qmg1eTkHqCMfO3M2GhZRhuaFSRIR2NYXAlMCCAYkUCSTqeSQXfnY83OIAxEAav5IiLYTiWkcX1jSoypENNapUzhiE3IkOD2Zd4nPTMLL+agzF4FmMgDF5FemYWo+ZuZ9TcbRzLyKJ7wwo82rGWMQx5EFE0CBIh6XgmZSKMgTDkD8ZAGLyGRdviefHHtWyPS+W6BuV5/Nra1DaGwS1CggIJDQokOS2DMhEheVcwGNzAGAiD4ySknmDorxv57p9YqpQKY8LdLWlf++wEUIbzE1E0iMPJJ8jKziYwoHCHETF4B8ZAeBnh4eGkpFxUCu5Ch6ry7YpY3pi+kZS0TB66pgaPdKhFaAEHnvMVIkODiUtOJ7mAF825gz+1a1/CvGYYHGHroWT6jV7MM1PXUKtsONMfbcvTXeoWGuMgIl1FZJOIbBWR585zXh8RURFp4VL2vF1vk4h0yS9NYUUCCQwQktMy8+uSBj/H9CA8zHPPPUflypV56KGHAHj11VcJCgpi9uzZHDlyhIyMDF5//XV69uzpsNKCIS0jixGzt/Lp3G2EFQni7T4Nubl5ZQICCk+4bREJBEYA12Klx10mItNUdUOO8yKAR4ElLmWXYWVXrA9UBP4Ukdp2kqxL1UVkaDDJBbCq2rRr/8CjBkJEugIfAIHAZ6r6Vo7jVbHSjJYBEoABqhrrcjwSK9Xoj6r68CWJ+e05OLD2ki5xFuUbwnVvnfeUfv368dhjj516kKZMmcKMGTMYMmQIkZGRHD58mNatW9OjR49CmZPgQli6I4Fnpq5mZ/wxejeN5oXu9YgKL5QTqi2Braq6HUBEJgM9OTst7n+Bt4GnXcp6ApNVNR3YISJb7estumg1Lm27QnY2JTOyyS4SSOCltKc82rZp1/6BxwyEm29Z7wITVXWCiHQA3gRudzn+X2CepzQWBE2bNuXQoUPs27ePuLg4SpYsSfny5Xn88ceZN28eAQEB7N27l4MHD1K+fHmn5XoEVWXM/O28/fsmKpUsypf3tuLKmlFOy7oUooE9Lp9jgVauJ4hIM6Cyqv4qIk/nqLs4R93onDcQkUHAIIAqVaq4LSzQ7ollZmcTGOi54TrTrv0DT/Yg3HnLugx4wt6fDfx48oCINMfKU/07cOnxb/J40/ckN998M1OnTuXAgQP069ePL7/8kri4OFasWEFwcDAxMTG5hkP2BZLSMnj629XMWH+Qbg3L83afRkSEBjsty6OISAAwHBh4sddQ1dHAaLBiMZ3nxDPatgCH4lLIzFJql/esi7A/t2t/wZOT1Lm9ZeV8U1oN9Lb3ewERIlLafsDeA5463w1EZJCILBeR5XFxcfkkO//p168fkydPZurUqdx8880kJiZStmxZgoODmT17Nrt27XJaokfYuD+JHh8t4M+Nh3ipez1G9G/mK8ZhL1DZ5XMlu+wkEUADYI6I7ARaA9Psieq86rpPwg44uvus4ojQYNIys0jPvORpjfPir+3an3Dai+kpoL2IrATaYz0oWcBgYLrrfERuqOpoVW2hqi3KlPFev/n69euTnJxMdHQ0FSpU4LbbbmP58uU0bNiQiRMnUrduXacl5jvfrYil18i/OXYii8mDWnNv2+q+NBa9DKglItVEpAjWpPO0kwdVNVFVo1Q1RlVjsIaUeqjqcvu8W0QkRESqAbWApRelIjAYjidAZvoZxZGh1sCAp72Z/LFd+xueHGLK801JVfdh9yBEJBzoo6pHReQKoK2IDAbCgSIikqKq53Qn9HbWrj09QR4VFcWiRbnPSRZ2X/G0jCz+8/MGvl66m9bVS/HRrc18bmWvqmaKyMPADCwHjLGqul5EXgOWq+q089RdLyJTsIZaM4GHLtqDKbwspB6GlINQ4vQ8RUhwICFBgSQdz/C4E4C/tGt/xZMG4tRbFpZhuAXo73qCiEQBCaqaDTyP5dGEqt7mcs5AoEVhNg7+wp6EYwz+8h/W7k3kgfY1eKpzbYICne6kegZVnQ5Mz1H28jnOvTrH56HA0EsWEVgEwkrDsXgILwdBp41BZNEgDqeYVdWGS8NjLUdVM4GTb1kbgSkn37JEpId92tXAJhHZjDUhfekPjcERZv97iOs/WsDO+FTG3NGC566r67PGwasItzO5pRw6ozgiNBhVJcUsmjNcAh5dB5HXW5aqTgWm5nGN8cD4S9DgS2Pf58SpzICqysg52xg2YxP1KkTy6YBmVC1dLO+KhktGVZGgIhBWyupFRJSzehVAMXtVdVJaJsW9LOzGheArGS8LKz79ihcaGkp8fLzPNzJVJT4+ntDQ0AK9b1a28n8/rWPYjE30aFyRHwa3McahgDijbYeXA/SMXoSVqzqY5LTMQtv+nWrXhtP4dKiNSpUqERsbize7wOYXoaGhVKpUqcDul5aRxZCvV/LHhoPc3646z3atW6jCZRR2zmrbx1IhIw4ijkKAtUDu2IksElJPkBEfQpGgwvkuWNDt2nAmPm0ggoODqVatmtMyfI4jqSe4Z8IyVu45yis3XMZdV5q/cUFzVtuO3wYft4ArHoLOrwOQeDyDZv+dyf3tqvNMV+NyarhwCudrhcEx9iQco8+nC1m3L4mR/ZsZ4+AtlK4BDW6CZZ9brq9A8aLBXB5Tkr82HsqjssGQO8ZAGNxm3d5Eeo1cSHzKCb68txXXNazgtCSDK+2egozjsGjEqaJO9cqx6WAyexKOOSjMUFgxBsLgFvM2x9Fv1CJCggL47sEruDymlNOSDDkpUwfq3whLx8CxBAA61C0LwF8bDzqpzFBIMQbCkCffrYjl7vHLqFK6GN8PbkPNsiZPtNfS7mk4kQxLRgFQvUw4tcuF883y2ELrzWRwDmMgDOdEVRkxeytPfruaVtVLMeX+1pSLNC6HXk25+lD3eljyCaQlAjCoXQ027k9i1r9mLsJwYRgDYTgn7/+5hWEzNnFjk4qMG9jSVyKx+j7tnraMw9LRAPRsUpFKJYvy0aytphdhuCCMgTDkyqTFu/jgry30bVGJ4X2bFFo/er+kYhOo1QUWjYT0FIIDA7i/fQ1W7TnKwm3xTqszFCLMU284i1/X7Ofln9bRqV453ujV0CyAK4y0f8YKBb78cwBubl6JshEhfDxrq8PCDIUJYyAMZ7Bw62Ee/2YVLaqW5OP+TU3AvcJKpRZQowMs/AhOHCM0OJBB7aqzaHs8K3YlOK3OUEgwT7/hFOv2JjJo0gpiosL47I7LCQ32XE5jQwHQ7hlIjYMV4wHo36oKJcOCTS/C4DbGQBgA2BWfysBxSyleNJiJd7eieJiZkC70VL0CYtrC3x9ARhphRYK4+8pqzN4Ux7q9iU6rMxQCjIEwcCg5jds/X0pWtjLh7paUL25cWX2G9s9AygFYMQ6AO9rEEBESxMg5phdhyBtjIPycpLQMBo5dRlxyOmMHXk7NsuFOSzLkJzFtrbmIP1+FfasoXjSYO9pU5bd1B9h6KNlpdQYvx6MGQkS6isgmEdkqImelDBWRqiLyl4isEZE5IlLJLm8iIotEZL19rJ8ndforaRlZDJq4nM0Hk/lkQDOaVinptCRDfiMCvcdAWBR8MwBS47n7ymqEBgUycvY2p9UZvByPGQgRCQRGANcBlwG3ishlOU57F5ioqo2A14A37fJjwB2qWh/oCrwvIiU8pdUfycpWHv9mFYu3J/DuzY25uk5ZpyUZPEWxKOg3yUooNHUgpYsG0r9VFX5avY/d8SaIn+HceLIH0RLYqqrbVfUEMBnomeOcy4BZ9v7sk8dVdbOqbrH39wGHgDIe1OpXqCqvTFvHb+sO8FL3etzYNNppSQZPE90Mrv8f7JgHf73KoHbVCRTh03mmF2E4N540ENHAHpfPsXaZK6uB3vZ+LyBCREq7niAiLYEiwFktWUQGichyEVnuD1nj8ovxC3fyxeLd3N++Ove2re60HENB0fQ2uPw+WPgR5Xb9wk0tKjF1eSwHEtOcVmbwUpyepH4KaC8iK4H2wF4g6+RBEakATALuUtXsnJVVdbSqtlDVFmXKmA6GOyzfmcDQXzfSqV5Znu1isoz5HV3egMqtYdojDKmfTpYqo+dtd1qVwUvxpIHYC1R2+VzJLjuFqu5T1d6q2hR40S47CiAikcCvwIuqutiDOv2GQ8lpPPTVP0SXLMp7fZuYEBqXgBsOGA+IyFoRWSUiC07Ov4lIjIgct8tXicinBSo8qAj0nQihxSn/2z3c2jCcr5buIj4lvUBlGAoHnjQQy4BaIlJNRIoAtwDTXE8QkSgROanheWCsXV4E+AFrAnuqBzX6DZlZ2Tzy1UoSj2fw6YDmFC9qFsJdLG46YHylqg1VtQnwDjDc5dg2VW1ibw8UjGoXIspB30mQuJfnjw8nIzOTsX/vKHAZBu/HYwZCVTOBh4EZwEZgiqquF5HXRKSHfdrVwCYR2QyUA4ba5X2BdsBAlzetJp7S6g+8M2MTS3Yk8EavhtSrEOm0nMJOng4Yqprk8rEY4F1xtitfDt2GUWz3bEaU/42JC3eReDzDaVUGLyPIkxdX1enA9BxlL7vsTwXO6iGo6hfAF57U5k/8tnY/o+dtZ0DrKvRuVslpOb5Abg4YrXKeJCIPAU9gOVl0cDlUzZ53SwJeUtX5udQdBAwCqFKlSv4pd6XFXbBvJV3/mcAPGWWYuLA6j3Ss5Zl7GQolTk9SGzzMtrgUnp66hsaVS/B/1+ccBTF4ElUdoao1gGeBl+zi/UAVe97tCeAre74tZ92CccDoNgyiW/BByCjmLJhHanqm5+5lKHQYA+HDpKZn8sCkFRQJCuCT25oREmSis+YTeTpg5GAycCOAqqarary9vwLLfbu2h3TmTVAI9JtEYGgx3skaxtSF6x2TYvA+jIHwUVSV575fy7a4FD66tSkVSxR1WpIv4Y4DhutYTXdgi11exp7kRkSqA7UAZ/1MIysSfMskqgYcosm8+0hLPuKoHIP3YAyEjzJ+4U5+Xr2PJzvX4cqaUU7L8SncdMB42I4ltgprKOlOu7wdsMYunwo8oKrOZ/Cp2oZtbT/gsuytpI7pBsecl2RwHo9OUhuc4fRiuHI82L6G03J8EjccMB49R73vgO88q+7iqN1hAEPXHObpxKHouG7IHT9ZLrEGv8X0IHyMMxfDNTaL4QxuIyJc0bU/d514mqyEnTC+GyTGOi3L4CDGQPgQWdlqFsNdLBnHYftcSPDvBWMd6pblSLk2PF7kFTTlEIy9DhJMKA5/xRgIH2LSop0s2ZHAf3s2MIvh8uKkQZg11PoRfKsKTOwBa75xWpmjiAgPX1OTn49UYUGbsXAiGcZ1g7jNTkszOICZg/AR9ice590/NtOudhluam4Ww51FxnGIXQY7F1hb7DLIOgESABWaQKsHrOxrVVo7rdRxujYoT/UyxRi6MoDfBv6KTLwRxl0Hd/wI5Rs6Lc9QgBgD4SO8Om09mdnZDL2xASJm3gGA7GzYMgOWjoGd810MQmNodT/EtLMMQqjpbbkSGCAMvromT327mr/i69Dprt+s3tX47jDgB6jU3GmJhgLCGAgf4I/1B5ix/iDPdq1L5VJhTstxnvRkWPUVLPnUGj+PjIaWg6DaSYNQ3GmFXk/PJhV5/8/NfDx7Kx0Ht0FOGomJPaH/NxBzpdMSDQWAmYMo5KSkZ/LKtPXUKRfBvW2rOS3HWY7shN9fgOGXwW/PWHmYbxoHj66GLkOhdhdjHNwkODCAB9rXYNWeoyzcFg8lq8Jdv0NkRfiiD6zzSk9dQz5jDEQh538zN3MgKY03ejckONAP/ztVYeffMPk2+LApLB0FtTrDvX/BvTOhQW8INN5cF8NNzStRNiKEj2dttQoiK8Bd0615iKl3ww8PQFrS+S9iKNSYIaZCzNrYRMb9vYPbWlWhedWSTsvxLNlZcCweUg5CyiFrS94P63+AA2ugaEm46nG4/F7rLddwyYQGBzKoXXVe/3UjK3Yl0LxqKSgWZRmJecOsbfci6P2ZFT7c4HMYA1FIyczK5vkf1lA6PISnfSV1qCoc3gKbf4dDG840BscOw9lZZ6FMPbjhA2jYF4qY+Zf8pn+rKoycs42PZ21l3F0trcLAYLjmBajRAb67D8Z2gfbPQtsnIdD8pPgS5n+zkDJx0S7W7U3i4/5NC/eCuMx02PU3bJ5hGYYjO63yyEoQUR5KVIFKLaBYWQg/uZWDYmWsf0PCHZXv64QVCeLuK2N494/NrNubSINolzmcKq3hwQXw65Mw5w3YNgt6j7bmKww+gTEQhZB9R4/z3h+buLpOGbo3rOC0nAsn+SBs+cMyCNvnwIkUCAqFau2hzRBrDqFE5TwvYygYbr8ihlFztzNi9lY+GZDDxTW0OPT5zPo/++UJ+PQq6P4eNOrrjFhDvuJRAyEiXYEPgEDgM1V9K8fxqlh5qMsACcAAVY21j93J6SQrr6vqBE9qLUy8Mm09War8t2chWvNwLMFyPV03FfattMoio60fktpdrUVqZojIKyleNJg728QwYs5Wth5KpmbZiLNPatQXKreE7wfB9/fBlpnQ/V3jNVbI8Zjbi5uJ3d8FJqpqI+A14E27bingFaw0ji2BV0TEx2dh3WPG+gPM3HCQxzvV9v41D6oQuwJ+HAzD68EfL1oL1Tr8HzzwNzy+Hq7/n+V+aoyDV3P3VdUIDQpk5Oxt5z6pZAwMnA5Xv2C5wX5yFWz8xWoHhkKJJ/0i80zsjmU4Ztn7s12OdwFmqmqCqh4BZgJdPai1UJCSnskrP62nbl62Xy4AACAASURBVPkI7r7Ki9c8nEiFFRNgdHv4rANs+Ama9LeMwn2zoN1TUL4BFJbej4FSxYrQv1UVflq9j93xx859YmAQXP0s3P07BIfCN7fB2K6wZ2nBiTXkG540ELkldo/Occ5qoLe93wuIEJHSbtZFRAaJyHIRWR4XF5dvwr2V9/7YxMHkNN701jUPcZvgt2fhvXrw8xDIyrDGo5/81+oplG/gtELDJTCoXXUCRfhk7nl6ESep3BIeXATXv2+tZv/8Wvjmdoh3o67Ba3D6V+YpoL2IrATaY+X1zXK3coEldvcC1sQeZcLCndzeuipNq3jZaNv2OTD+ehjREpZ9DrU7W6tuH1xorUsIyWXM2lDoKBcZys0tKvHdilj2Jx7Pu0JgELS4C4astIadtv5ltZFfn4IU33+h8wU8aSDyTOyuqvtUtbeqNgVetMuOulPXn8jOVl74YS1R4SE81aWO03JOc3ADfHGTFZ8nYQd0fAWe2Gh5tVS9wgwh+SAPtK9Btiqj5l5AjoiQcGvYachKaHYnLB8LHzaBucOs4UiD1+JJA+FOYvcoETmp4Xksjyawcv12FpGS9uR0Z7vML/ll7X7W7U3ihW71iAz1gjUPSfvhp4fh0yshdil0fh2G/ANtn4Bw3+7J+TuVS4XRu1k0Xy/dzaGktAurHFEOrh8OgxdD9ath9uvwUXP4Z6IVedfgdXjMQLiZ2P1qYJOIbAbKAUPtugnAf7GMzDLgNa9I7O4AmVnZvD9zM3XKRdCjscMhJNJTYPYb8FEzWD0ZWj0IQ1ZBm0cgKMRZbYYC4+FrapGZre7NReRGmdpwy5fWMGTxSjDtEZhwg8lc54V4dB2EG4ndpwJTz1F3LKd7FH7LDyv3sv1wKqNub+5cfumsTFg5EWa/CamHoH5v6PgylPJiTyqDx6hSOoxeTaP5asluHmxfg7KRoRd3oapXwD0zYeUXMOMF+ORK6PQqXH4fBDg9PWoA5yepDefhRGY2H/y1hYbRxel8WbmCF6AKm36HT9rAL49D6RpWlNSbxxnj4Oc8fE1NMrOVUfMu8a1fBJrdbg07VW1jhWmfcL3xdvIS3DIQIvK9iHR3mS8wFABTlu8h9shxnuxcu2BXTKtaHiefd4av+0F2JvT7Eu76zYqLZPB7YqKK0bNJRb5csou45PRLv2DxaLhtKvQcAQfWWb2JxZ+auQmHcfcHfyTQH9giIm+JiBe50vgmaRlZfDRrCy2qlqR97QKa+FWFbbOt6Jxf9IakfdB9ODy0BOpdb7ySXBCRriKySUS2ishzuRx/QETWisgqEVngGkVARJ63620SkS4Fqzz/eKRDLU5kZjN6Xj697YtA0wEweBFUawu/P2ulOfW33kRiLCz8yMpz4rCBdGsOQlX/BP4UkeLArfb+HmAM8IWqZnhQo1/y5ZLdHExK5/1+TT3fe1CFHXNhzltWfP/IaGuBW9PbzeRzLriEkbkWaxHnMhGZpqobXE77SlU/tc/vAQwHutqG4hagPlAR61mqrapur//xFqpFFaNnk2gmLd7F/e1rEBWeT22leDT0nwKrv4bfnrN6Ex1fhlYP+PbcRNI+mD8c/plg5U8HK5pxo37Q6BaIqlngktz+a9srnAcC9wIrsYLwNcMKg2HIR1LTMxk5eytX1YziihqlPXcjVdg+F8Z1s9YyHNkF3d61/NUvv9cYh3OTZxgZVXVNtVYMOBmQqCcwWVXTVXUHsNW+XqHk4Q41OZGZzZhLnYvIiYgVnuWhxVYu8RnPW6FbZr4M63+Eo7t9J8ZT8gErAsEHTWDFOPt7L4XeY6B0LZj/HnzcHMZ0gCWjITW+wKS51YMQkR+AOsAk4AZV3W8f+kZElntKnL8yfuFO4lNP8ETn2p67yc4Flsvqrr8hogJcNwya3WHFzzHkRW6hYFrlPElEHgKeAIoAHVzqLs5RN9cwMsAggCpVquSLaE9Qo0w4NzSuyMRFuxjUrjql86sXcZLIitD/G8utesmnsGgkZNsDFmFRULEpRDez/q3YzFprUVhIOQR/fwDLPrPC0jS5Fdo9bQU9BChTx4qSm3wA1n4Lq7+B3562jGXNa6HxLVYkZM12Sa51MMd26PS/Dy254KgG7rq5fqiqs3M7oKpm1jIfSTyewai52+hYtyzNPBFS4/hR+P05q/seXh6ue8da3WoMQ76jqiOAESLSHyt0/Z0XUHc0MBqgRYsWXv2q/EiHmkxbvY8x83fw3HUeyG4oYv14NrnVSjB1cB3s/ccKG79vJWz763S2wchoK2d2qRpQujqUqm7tF68EAYEXdt+sDCtMfdpRK/d2eqL9b9LZ/6YnQ0ikdZ/ilax8JsUrW3pyRipOPWwZhqVjICvdGj5q/7SlNTciyltrjdo8AgfXW8Zy7bew+TeQQMhtdFIC7KRaZa3nvGx96/tcIO4aiMtEZKUdBgN7dfOtqjrygu9oOC+fL9hBUlomj1/rgd7DtlnWCujkA9DuGStFpDEMF8OFhoKZDHxykXW9npplI7i+UUUmLtrJoHbVKVWsiOduFhQC0c2t7STpKVZe8n0rLcNxaIMVHyzTZaV3YBHrzbxU9dNbkWJWnvPUw1ZK22MJ9n689TktMQ8xYhmF0EjrzTwt0cqTnjM1bljUacMRWtwaIss8Dg1vtp7DC5lbKFcfOv/XWi+yY571PUOLW9kVw8tZPajwchBW+sINYi64ayDus9+GAFDVIyJyH5Z3kyGfSEg9wdgFO+jWsPyZqR0vlROp8Mf/wfLPIao23DvzzAfMcKGcCiOD9eN+C5aX3ylEpJaqbrE/dgdO7k8DvhKR4ViT1LWAQh8Le0iHmvyyZh+fzd/OM10LOEd6SLi1hqJqm9Nl2dnWj3XCdnvbZv+7w/phzXAJWR4QbP2gFouCsFJQobHL59JQtKSLIXD5t0j42ZPmWRnWZHNirL3tPr0fv9V6OavTFdo/Z60ov1gCAqHGNdbmQdw1EIEiIqrWrJDtxeHB1wT/ZNS8baSeyOTxTvnYe9i9GH54wMr1fMXD0OElCC6af9f3Q1Q1U0ROhpEJBMaeDCMDLFfVacDDItIJyACOYA8v2edNATYAmcBDhdGDKSe1ykXQrWEFJizcyX1tq1PSk70IdwgIsLyhikdbLrOuqFrGI+O4ZQRCIvPPhTsw2MrJ7SN5ud01EL9jTUiPsj/fb5cZ8olDyWlMWLiTG5tEU6tcPoTHzkizEsn//aE1HjrwF4i56tKvawDcCiPz6HnqDsWOO+ZLDOlQi1/X7OfzBTu8K+pwTkSsyW9DnrhrIJ7FMgoP2p9nAp95RJGfMnL2NjKylEc71rr0i+1bZfUa4jZC84FWtFWTk8HgYeqUj6Bbw/KMX7iTe9tWo0SYGWQo7Li1DkJVs1X1E1W9yd5G+UK32FvYe/Q4Xy3ZTd8WlYiJKnbxF1K1Yux/1hGOH7FCF9zwgTEOhgJjSMdapKRnMnbBDqelGPIBd2Mx1RKRqSKyQUS2n9w8Lc5f+HiWNX/5cIdL6D2owvSnrBj7l/W0whXUujafFBoM7lG3fCRd65dn3N87STxmAiwUdtxdST0Oy00vE7gGmAh84SlR/sTOw6lMWR5L/1ZViC5xkZPHqlYUzGWfQZsh0OdzyxvDYHCAIR1rkZyeydi/TS+isOOugSiqqn8Boqq7VPVVLNc9wyXy4V9bCA4UBl9d4+IuoAq/Pw9LR1teSte+ZoLqXQAffPABSUlJqCr33HMPzZo1448//nBaVqHmsoqRdKlfjrF/7yDxuOlFFGbcNRDpdqjvLSLysIj0AsLzquRGxMsqIjJbRFaKyBoR6WaXB4vIBDsa5kYRef6CvlUhYVd8Kj+u2svtrateXNIVVZjxIiz5BFoPtiajjXG4IMaOHUtkZCR//PEHR44cYdKkSTz33FlN1XCBDOlYi+S0zPyP0WQoUNw1EI8CYcAQoDkwgDzCBrhEvLwOuAy41TXksc1LWKlIm2ItNjq58O5mIERVG9r3u19EYtzUWmgYNW87QQEB3Nv2HEvsz4cq/PESLB5hRbns8oYxDheBvbSH6dOnc/vtt1O/fv1TZYaLp37F4tzQuCKfL9jBoeQLzF1t8BryNBD2D30/VU1R1VhVvUtV+6jq4jyq5hnxEivCZaS9XxzY51JeTESCgKLACSAJH+JQUhpTl8dyU4tKlLvQ3oOqFdVy0cfQchB0fcsYh4ukefPmdO7cmenTp9OlSxeSk5MJ8OWQ0gXIk9fWJiMrm49nbXVaiuEiyfNJsN1ZL2aFVW4RL3NGrXwVGCAisViLjh6xy6cCqcB+YDfwrqom5LyBiAwSkeUisjwuLu4iJDrH5wt2kJmdzf3tLrD3oAp//QcWfggt7rGC7RnjcNF8/vnnvPXWWyxbtoywsDAyMjIYN26c07J8gpioYtzSsjJfLdnNrvhUp+UYLgJ3X5VWisg0EbldRHqf3PLh/rcC41W1EtANmGTPdbQEsrBi1VQDnhSRs35JVXW0qrZQ1RZlyhRQ1rV8IPFYBl8s3sX1jSpStfQFrHtQhVn/hQX/g+Z3WbkbjHG4JBYtWkSdOnUoUaIEX3zxBa+//jrFi+djHCw/Z0iHWgQFCsNnbnZaiuEicNdAhALxWDHtb7C36/Oo407UynuAKQCqusi+TxRW4LPfVTVDVQ8BfwM+E1Z8wqKdpJ7I4sEL9Vya/YaVPKTZnVYqUDMUcsk8+OCDhIWFsXr1at577z1q1KjBHXfc4bQsn6FsZCh3X1mNn1btY/2+vKKjGrwNd1dS35XLdnce1U5FvBSRIliT0NNynLMb6AggIvWwDEScXd7BLi8GtAb+dfdLeTPHTmQy7u8ddKxblnoVIvOucJI5b8G8d6w0oNe/b4xDPhEUFISI8NNPP/Hwww/z0EMPkZyc7LQsn+L+9jUoXjSYYTM2OS3FcIG4m1FuHKdTJp7ifEbCzYiXTwJjRORx+/oDVVVFZAQwTkTWAwKMU9U1F/rlvJHJS/dw5FgGg6+5gN7DnLdhzpvQ5Da44UNjHPKRiIgI3nzzTSZNmsT8+fPJzs4mI8P47ucnxYsGM/jqGrz5278s3h5P6+oeTKNryFfcDdb3i8t+KNCL0x5H58SNiJcbgCtzqZeC5erqU5zIzGbM/O20rFaK5lXdXOk89x0rKmvj/tDjI2Mc8plvvvmGr776irFjx1K+fHl2797N008/7bQsn+PONjGM+3snb//+L98/2AYxc2eFAneHmL5z2b4E+uJDcwIFxY8r97I/Mc39VdNzh8HsodD4Vuj5cb5kiDKcSfny5bnttttITEzkl19+ITQ01MxBeIDQ4EAe61SLlbuP8seGg07LMbjJxb6O1gLK5qcQXycrW/l07jbqV4ykfW03PK7mDbMC7zW6BXqOMMbBQ0yZMoWWLVvy7bffMmXKFFq1asXUqVOdluWT3NS8EtXLFGPYjE1kZZvFiIUBd+cgkjlzDuIAVo4Ig5vMWH+A7YdTGdG/Wd7d63nvwizbONw40hgHDzJ06FCWLVtG2bLW+05cXBydOnXipptucliZ7xEUGMDTnevw4Jf/8N0/sfRtUTnvSgZHcctAqKpJKHAJqCojZm+lelQxujYof/6T579nrXVo1M8YhwIgOzv7lHEAKF26NNnZ2eepYbgUujYoT+NKxXl/5mZ6NK5IaLBp396Mu/kgeolIcZfPJUTkRs/J8i3mbTnM+n1J3N++OoEB5+k9zB8Of70GDfvCjZ8Y41AAdO3alS5dujB+/HjGjx9P9+7d6datm9OyfBYR4dmuddmXmMYXi3c5LceQB+7OQbyiqqdWuajqUeAVz0jyPUbO3kr5yFB6Na107pMW/M8KodHwZuj1qTEOBcSwYcMYNGgQa9asYc2aNQwaNIi3337baVk+TZuaUbStFcWI2VtJSjMuxd6Mu26uuRkSd+v6NSt2JbBkRwIvda9HkaBz2OMF78Ofr0KDm+BGYxwKmj59+tCnTx+nZfgVz3Spyw0fL+Czedt5onMdp+UYzoG7P/LLRWQ4VvhugIeAFZ6R5FuMnL2NkmHB3NqySu4n/P0h/PkKNOgDvUZBoLG7BUFERESuzgKqioiQlORTwYO9joaVitO9UQU+W7CD26+IoUxEiNOSDLng7hDTI1ght7/BCtudhmUkDOfh3wNJ/PXvIQa2qUaxkFx++NdOhZn/B/V7Qa/RxjgUIMnJySQlJZ21nSw3eJ6nOtchPTP7VE52g/fh7kK5VFV9zo6cermqvqCqJn5vHnwyZxvFigRyZ5uqZx/cvRh+fBCqXml6DoUMNzIlPiEiG+wsiX+JSFWXY1kissrecsYm8yuqRRWj3+WV+WrpbnYcNj8n3oi7XkwzRaSEy+eSIjLDc7IKP7vjj/Hz6n30b1WFEmFFzjyYsB0m94filaHfFxBkuteFBTczJa4EWqhqI6zcJu+4HDuuqk3srUeBiPZiHutUi9CgQF6Ztt5k8vNC3B1iirI9lwBQ1SOYldTnZdS8bbmnEz2WAF/2Bc2G276FMDdjMhm8hTwzJarqbFU9Zn9cjBXq3pALZSNCeeza2szbHMeM9SYEh7fhroHIFpFTs6x2fmhj7s9BfEo6U1fE0rtZ9JnpRDNPwJQ74OguuOUrKH2B+SAM3oA7mRJduQf4zeVzqJ0FcfH51hIV5myJF8qdV1SlbvkI/vvLBo6fyHJajsEFdw3Ei8ACEZkkIl8Ac4HnPSercDNp8S7SM7PP7D2ows+Pws750ONjqNrGOYGGAkFEBmAFtRzmUlxVVVtgJcV6X0RyfUsorNkSL4agwAD+06M+e48eZ8Rsk7/am3B3kvp3rIa+CfgaK4/DcQ/qKrSkZWQxcdEuOtQtS82y4acPzH8XVn8FVz8Pjfs5J9BwqbiTKRER6YT1YtVDVdNPlqvqXvvf7cAcoKknxRYWWlUvTa+m0Yyet91MWHsR7k5S3wv8hWUYngImAa96Tlbh5ft/9pKQeoL7XHsPa6fawff6QXsT47CQk2emRBFpCozCMg6HXMpLikiIvR+FlQtlQ4Ep93Kev64uRYIC+M/PZsLaW3B3iOlR4HJgl6peg/XWc/T8VdxyB6wiIrNFZKXtEtjN5VgjEVkkIutFZK2IhOas721kZyufLdhOg+hIWle3J593L4EfB0OVNlbCH5MopVCjqpnAyUyJG4EpJzMlishJr6RhQDjwbQ531npYi05XA7OBt+ykWQas/NWPdarFnE1xJmeEl+Cu832aqqaJCCISoqr/ish518e7uANeizWRt0xEpuV4IF7CesA+sV0FpwMxIhIEfAHcrqqrRaQ04PVBW2b9e4jtcal8cEsTa5VuwnaYfCsUj4ZbvjTurD6CG5kSO52j3kKgoWfVFW7ubBPDt8tjee3nDbSrVYaiRUzYGSdxtwcRa6+D+BGYKSI/AXmFYszTHRDLEyrS3i/O6TSmnYE1qroaQFXjVdXr3RvGzN9OxeKhdGtYAY4fOe3O2t+4sxoM7hAcGMBrPa0J65FzzIS107g7Sd1LVY+q6qvA/wGfA3mF+3bHHfBVYICIxGK9kT1il9cGVERmiMg/IvJMbjfwJlfANbFHWbIjgbuurEawKHw7EI7shH5fQlRNR7UZDIWJVtVLc2OTioyau52dZsLaUS445aiqzlXVaXav4FK5FRivqpWAbsAkEQnAGvq6CrjN/reXiHTMRYvXuAKOmb+D8JAg+rWsDHPegu1z4PrhEHOlo7oMhsLIC92s6MevmglrR7nYnNTu4I474D3AFABVXQSEAlFYvY15qnrYXpE6HWjmQa2XxN6jx5m+dj+3tqxM5J45MO8daDIAmt3htDSDoVDiOmE900xYO4YnDUSe7oDAbqAjgIjUwzIQcVgeIg1FJMyesG6PF7sDjluwA4B7GgbD9/dB2frQbVgetQwGw/m4s00MtcuF85+fzQprp/CYgXDTHfBJ4D7b7e9rYKBaHAGGYxmZVcA/qvqrp7ReCklpGUxetoeeDaMoP+MByMqEvhOhSJjT0gyGQo01Yd2AvUeP84mZsHYEj8aYdsMdcAPWYqHc6n6B5erq1UxeupuU9ExeCPoK9i6HmyeYSWmDIZ9oXb00PZtU5NO52+ndrBIxUcWcluRXeHKIyefJyMpm3N87eazCOqLWj4PWg6F+Xs5dBoPhQnihWz2CA8VMWDuAMRCXwK9r9lM0aTsPJ38AlVpCp/84Lclg8DnKRYbyROc6zNkUx5Tle/KuYMg3jIG4SFSVifM28FnRjwgsEgo3j4OgInlXNBgMF8xdbWJoU6M0r07bwNZDKU7L8RuMgbhIFm07zG2HP6Ba9m6k9xgobnLCGAyeIiBA+F+/JoQGB/Do5JWkZxqvpoLAGIiLZNP0EfQJnE9W26eh5llr+AwGQz5TLjKUd25qzPp9SQz7fZPTcvwCYyAugt3rFtE//mN2lWhF0DVnBak1GAwe4trLynF766p8tmAHczf7dqY9b8AYiAvl+FGK/XQ3R4ggov84CDDRJg2GguTF7vWoXS6cJ6es5nBKet4VDBeNMRAXgippPzxC5ImD/FRrKKXKni8VscFg8AShwYF8eGtTktIyePrb1cb11YMYA3EhrP2W0M3TGJ51M9d26ZH3+QaDwSPULR/Ji93qMXtTHOMX7nRajs9iDIS7JO5Ff32S1dRhS427qF4mPO86BoPBY9xxRVU61i3Lm9P/ZcO+JKfl+CTGQLhDdjb8NJiszAweSX+AO66s4bQig8HvERHeuakRJcKCGTJ5pQno5wGMgXCHZZ/B9jmMLnoPQVHVuapmlNOKDAYDUDo8hOF9m7D1UAqv/+q1AZ8LLcZA5MXhLTDzZZIqXc07h6/gjtZVCQgQp1UZDAabq2pFcX+76ny5ZDe/rzvgtByfwhiI85GVCT/cD8GhvFd0CMWKBNGnuVkxbTB4G092rkPD6OI89/0a9iced1qOz2AMxPlYMBz2riCp4zt8veEEfZpXIiI02GlVBoMhB0WCAvjw1qacyMzmscmryMjKdlqST2AMxLnYtxLmvg0Nb2ZScjNOZGVzxxVVnVZlMBjOQbWoYgzt1YAlOxIY+utGp+X4BMZA5EbGcfj+fihWhswu7/DF4l1cVTOKmmUjnFZm8BJEpKuIbBKRrSJyVrwVEXlCRDaIyBoR+UtEqrocu1NEttjbnQWr3Lfp1bQS91xVjfELdzJlmQkNfql41EC48RBVEZHZIrLSfpC65XI8RUSe8qTOs5j1OhzeBD1HMHNHOvsT00zvwXAKEQkERgDXAZcBt4rIZTlOWwm0UNVGwFTgHbtuKeAVoBXQEnhFREoWlHZ/4Pnr6nJVzShe+nEdK3YdcVpOocZjBsLNh+glrFzVTYFbgJE5jg8HfvOUxlzZMR8WjYDL74OaHZmwaCfRJYrSsV65ApVh8GpaAltVdbuqngAmAz1dT1DV2ap6zP64GDjp3dAFmKmqCXbu9ZlA1wLS7RcEBQbwcf+mVCgRygNfrOBAYprTkgotnuxB5PkQAQpE2vvFgX0nD4jIjcAOYL0HNZ5JWhL8+CCUqg7X/od/DySxeHsCt19RlUDj2mo4TTTgOn4Ra5edi3s4/aLjVl0RGSQiy0VkeVyciVp6oZQIK8KYO1pwLD2T+yctJy3DLKK7GDxpINx5EF4FBohILDAdeARARMKBZ4Hz5vDM94fo9+chaS/0GgVFijFx0S5CggLo16LypV/b4JeIyACgBTDsQuqp6mhVbaGqLcqUKeMZcT5O7XIR/K9fE1bHJvLC92tNUL+LwOlJ6luB8apaCegGTBKRACzD8T9VPW9uwXx9iDbPgFVfQNsnofLlJB7L4Id/9tKzSUVKFjOpRA1nsBdwfWuoZJedgYh0Al4Eeqhq+oXUNeQPneuX54lra/P9yr18vmCH03IKHUEevLY7D8I92OOvqrpIREKBKKwJvJtE5B2gBJAtImmq+rHH1M59xxpaavcMAN+u2MPxjCzuuCLGY7c0FFqWAbVEpBpWm74F6O96gog0BUYBXVX1kMuhGcAbLhPTnYHnPS/Zf3n4mpps3J/EG9M3Uqd8BG1rmR6Zu3iyB3HqIRKRIlgP0bQc5+wGOgKISD0gFIhT1baqGqOqMcD7wBseNQ57lsHe5dB6MAQVITtbmbR4Fy2qlqRBdHGP3dZQOFHVTOBhrB/7jViOFutF5DURORkHfhgQDnwrIqtEZJpdNwH4L9bzsQx4zS4zeIiAAOHdmxtTu1wED3+1kp2HU52WVGjwmIFw8yF6ErhPRFYDXwMD1YmBwsUjIaQ4NL4VgLmb49gVf4w72sQUuBRD4UBVp6tqbVWtoapD7bKXVfWkIeikquVUtYm99XCpO1ZVa9rbOKe+gz9RLCSIMXe0QATum7ic5LQMpyUVCjw5xISqTseafHYte9llfwNwZR7XeNUj4k6SGAsbfoIrBkOIleNhwqKdlI0IoWv98h69tcFgKDgqlwpjZP9m3D52KY9/s5rRtzc3gTfzwOlJaudZOgZQaDkIgB2HU5mzKY7+rapQJMj8eQwGX6JNzSj+r3s9/tx4kHf/2OS0HK/Hoz0Ir+fEMVgxHupeDyWqADBp0S6CA4X+rao4q81gMHiEO9vEsOlgCiPnbCM0OJAhHWs5Lclr8W8DsWYypB21JqeB1PRMvl2+h+saVKBsRKjD4gwGgycQEV6/sQHpmVkMn7kZAR4xRiJX/NdAZGfD4k+gQhOo0hqAH1buJTk9kzvN5LTB4NMEBgjDbmoMCu/N3IwIPNzBGImc+K+B2D4LDm+GXqNBBFVl4qKdNIiOpFmVEk6rMxgMHiYwQBh2c2MA3v1jM2CMRE7810As/gTCy0H9XgAs2h7P5oMpDLupESLGs8Fg8AdOGgnFMhIiwkPX1HRaltfgnwYibjNs/ROueRGCrDAaP6/eT3hIEDc0ruiwOIPBUJAE2gvpVJVhMyzPJmMkLPzTQCz5FAJDoPldp4oWbjtM6+qlCA0OHK4KHAAADq5JREFUdFCYwWBwgsAA4b2+TQCMkXDB/wzEsQRY/TU0uhnCrZgssUeOsSv+GHeauEsGg99y0kgolpEQgcFX+7eR8D8D8c9EyDgGrR48VbRwazwAV9WKckqVwWDwAgIDhPfsiet3frd6Ev5sJPzLQGRlwNLRENMWyjc4Vfz3tsOUiQihVtlwB8UZDAZvICgwgPduboyqZSSys5WHrqnpl84r/mUgNv5sJQTq9u6pIlXl763xXFmztF82AIPBcDZBgQEM79uYALG8mw6nnODl6y/zu9hN/mUglnwKJatB7S6nijYfTOFwSjpX1jDDSwaD4TSWkWhC6fAQPl+wg7iUdIb3bUxIkP84sviPgYhdAXuWQNe3IeD/27v38KqqM4/j3x8BEm7hDgqBcK0FlQACclHE0ValIyggiqBoHelU6dNBZxw7j+3UTu1YrQ7aiqJWoY5VmU6htFqhWEAIFxNAQRjQBEQIKoQoInJL8s4fZ4fGeJAk5GSfy/t5njznnHX23nlXspL37L3WXutvv+DcgmIAhvdqG1Zkzrk41aCB+OHf96VjZjo/e2UrJZ8dY/aN55GZ0Sjs0OpF6kxXuvZxSM+EAZO/ULyqsJjstk3Jat00pMCcc/Fu2siePDwxh7z3Srhu9hr2HjwSdkj1IjUSxKd7YPN8GDAF0lucKC4tK2ft9hKG++Ul59wpjBuYxdNTB7Gj+BDjH1/FjhRYmS6mCULS5ZK2SSqQdHeU97tKWippg6SNkkYH5d+QtE7SpuDx704rkLynobzsxJoPFTYWHeDg0VIu6OUJwjl3aqPO6sAL04Zy6GgZ4x9fxVu7Pgk7pJiKWYKQlAY8BlwB9AUmSepbZbN7iCxFOoDImtWzgvJi4EozOxeYCjxX60COH4b8Z+Hr34I23b/w1qqg/2FYT+9/cM5VT/8urfjdPw6jaeM0Jj21huXv7As7pJiJ5RnEEKDAzLab2THgRWBslW0MyAyetwT2AJjZBjPbE5RvBppISq9VFBvnweESGPrdL72VW7Cfvmdm0qZZ41od2jmXmnq0b87vvzuc7LbNuGVOHvM37A47pJiIZYLoDOyq9Hp3UFbZj4EpknYTWbv6e1GOMx5Yb2ZHq74haZqkfEn5+/adJIv3/gZc9p+Q/cWlrw8fK2Pdzo8Z4aOXnHO10CEzg5e+M5TB3dow46W3mLnkHcrKLeyw6lTYndSTgDlmlgWMBp6TdCImSWcDPwe+E21nM3vSzAaZ2aD27dtH/w6ZnWDYbVDlJrj8nSUcKytnuPc/OOdqKTOjEXO+PZhxAzozc8m73PjM2qQa4RTLBFEEdKn0Oisoq+wWYB6Ama0GMoB2AJKygPnAjWZWWNfB5Rbsp2EDMaRbm7o+tEty1Rh8MVLSekmlkiZUea9M0pvB18L6i9rFSnrDNB6amMMD4/uxbufHjH5k5Yn7qxJdLBNEHtBbUndJjYl0Qlf9g3gfuARAUh8iCWKfpFbAy8DdZpYbi+BWFRYzsGtrmqWnzr2C7vRVc/DF+8BNwG+jHOKwmfUPvsbENFhXbyQxcXAXFk6/gNZNGzHl12t5ePE2SsvKww7ttMQsQZhZKTAdWAT8H5HRSpsl/URSxR/GncCtkt4CXgBuMjML9usF/KjSp60OdRXbgc+Ps6nogN897WrjlIMvzOw9M9sIJPZ/B1djX+vYgj9MH8GEgVk8+tcCrn96LR8eSNxLTjH9+GxmrxDpfK5c9qNKz7cAI6Ls91Pgp7GKa/X2/ZjBCO9/cDUXbfDF+TXYP0NSPlAK3G9mC+oyOBe+po0b8uA1OQzr2ZZ7FrzN6EdX8PDEHEadVWefcetN2J3UoVhVWEzTxmnkZLUKOxSXerLNbBBwPTBTUs9oG1VrhJ6La+MGZrFw+gV0aJHOTc/mcf+ft3I8wS45pWSCWFlQzJDubWjcMCWr705PdQZfnJSZFQWP24FlwICTbHfqEXou7vXq0JwFt4/g+vO78sTyQq6dvZpdJZ+HHVa1pdx/yA8PHGH7vkM+vberreoMvohKUuuKGz4ltSNyeXVLzCJ1cSGjURo/u/pcfjlpAO989BmXzXydZ3N3JMQ9EymXIHx6b3c6qjP4QtLg4ObPa4DZkjYHu/cB8oNBGUuJ9EF4gkgRV+Z0YtGMkQzu1oZ7/7iFibNXU7D3YNhhfaWUG+OZW1hMm2aN6XNG5qk3di6Kagy+yCNy6anqfquAc2MeoItbnVs1Yc7Ng5m/oYh7/7iF0Y+s5PuX9mbayB40Sou/z+vxF1EMmRmrCvYzrGfblFs60DkXHyQxbmAWS+64iEv7duDBRdsY+6tc3i46EHZoX5JSCWJ78SE+/PSI9z8450LXvkU6syafxxNTBrL34FHGPpbLA69u5cjxsrBDOyGlEkTF9N4+QZ9zLl5cfs6ZLLljJFcP6MysZYV869EVrNtZEnZYQIoliJUFxXRu1YSubXx5Uedc/GjVtDG/uCaHud8ewpHj5Ux4YjV3zHuTok8OhxpXyiSIsnJjdeF+RvRqi+T9D865+HPR19qzaMZIbr2wB3/a+AEX/2IZ9728hU8+PxZKPCmTIDbvOcCnR0p9eg3nXFxrnt6Qfxvdh6X/PIor+3Xi6ZU7uPCBpTy+rLDe+ydSJkHkFuwHYLh3UDvnEkDnVk14aGIOf/7+hQzKbs3PX93KqAeXMS9vV73dZJcyCWJVYTFndWxB+xa1W7nUOefC8PUzMnn25iG8cOtQOrbM4K7/3cgVj7zOki0fEZn8OnZSIkEcLS0j770Sv3vaOZewhvVsy4LbhjNr8kCOlxn/8Jt8rp29htWF+2OWKFIiQazf+QlHjpf7/Q/OuYQmidHnnsniGSP5j6vOYcf+Q0x6ag0TZ69mxbv76jxRpESCyC0oJq2BOL+HLy/qnEt8jdIacMPQbFbcdTH3jjmbXSWHueHXb3D1rFUs3bq3zhJFTBNENdbu7SppqaQNkjZKGl3pvR8E+22TdNnpxJFbWEy/rJa0yGh0Oodxzrm4ktEojanDu7H8rlHcd/U57Dt4lJvn5DHmV7ks3vzhaSeKmCWIaq7dew+R2TAHEJk2eVawb9/g9dnA5cCs4Hg1dvDIcTbuPuCXl5xzSSu9YRqTz89m2b+M4oHx/Thw+DjTnlvHFY+s4JVNH1Bey1FPsTyDOOXavYABFdOqtgT2BM/HAi+a2VEz2wEUBMersbXbSygrN7//wTmX9BqlNWDi4C789c6LeHhiDsdKy7nt+fVcNvP1Wi1UFMvpvquzdu+PgcWSvgc0Ay6ttO+aKvt2rvoNJE0DpgF07do1ahBNGqdx8VntGZjty4s651JDw7QGjBuYxdj+nXl50wcs2FDEGS0zan6cGMRWE5OAOWb2kKRhwHOSzqnuzmb2JPAkwKBBg6KeQ43o1c7PHpxzKSmtgRiT04kxOZ1qtX8sE0R11u69hUgfA2a2WlIG0K6a+zrnnIuhWPZBVGft3veBSwAk9QEygH3BdtdJSpfUHegNvBHDWJ1zzlURszMIMyuVVLF2bxrwTMXavUC+mS0E7gSekjSDSIf1TRYZl7VZ0jwiC7qXArebWfysouGccykgpn0Q1Vi7dwsw4iT73gfcF8v4nHPOnVxK3EntnHOu5jxBOOeci8oThHPOuag8QTjnnItKsV5wor5I2gfsPMnb7YDiegwnVpKhHolch2wza1/f3zQF2nYy1AEStx4nbddJkyC+iqR8MxsUdhynKxnqkQx1iCfJ8PNMhjpA8tSjMr/E5JxzLipPEM4556JKlQTxZNgB1JFkqEcy1CGeJMPPMxnqAMlTjxNSog/COedczaXKGYRzzrka8gThnHMuqqRPEJIul7RNUoGku8OOpzYkvSdpk6Q3JeWHHU91SXpG0l5Jb1cqayPpL5LeDR5bhxljokqGdg3etuNdUicISWnAY8AVQF9gkqS+4UZVaxebWf8EG2c9h2BBqEruBl4zs97Aa8FrVwNJ1q7B23bcSuoEAQwBCsxsu5kdA14ExoYcU8ows9eBkirFY4G5wfO5wFX1GlRy8HYdslRp28meIDoDuyq93h2UJRoDFktaJ2la2MGcpo5m9kHw/EOgY5jBJKhkadfgbTuuxXTBIFdnLjCzIkkdgL9I2hp8gkloZmaSfJx1avO2HceS/QyiCOhS6XVWUJZQzKwoeNwLzCdyiSFRfSTpTIDgcW/I8SSipGjX4G073iV7gsgDekvqLqkxcB2wMOSYakRSM0ktKp4D3wTe/uq94tpCYGrwfCrwhxBjSVQJ367B23YiSOpLTGZWKmk6sAhIA54xs80hh1VTHYH5kiDy+/qtmb0abkjVI+kFYBTQTtJu4N+B+4F5km4hMoX1xPAiTExJ0q7B23bc86k2nHPORZXsl5icc87VkicI55xzUXmCcM45F5UnCOecc1F5gnDOOReVJwh3UpJGSfpT2HE4V9e8bVePJwjnnHNReYJIApKmSHojmFN/tqQ0SZ9J+i9JmyW9Jql9sG1/SWskbZQ0v2LOekm9JC2R9Jak9ZJ6BodvLul3krZKel7BXU3O1Qdv2+HyBJHgJPUBrgVGmFl/oAyYDDQD8s3sbGA5kTs9AX4D/KuZ9QM2VSp/HnjMzHKA4UDFrJQDgH8isu5AD2BEzCvlHN6240FST7WRIi4BzgPygg9ATYhMElYOvBRs89/A7yW1BFqZ2fKgfC7wP8F8OJ3NbD6AmR0BCI73hpntDl6/CXQDVsa+Ws552w6bJ4jEJ2Cumf3gC4XSD6tsV9s5VY5Wel6GtxlXf7xth8wvMSW+14AJwXz6FeviZhP53U4ItrkeWGlmB4CPJV0YlN8ALDezg8BuSVcFx0iX1LRea+Hcl3nbDplnzARnZlsk3UNkVa4GwHHgduAQMCR4by+Ra7kQmYb4ieCPZDtwc1B+AzBb0k+CY1xTj9Vw7ku8bYfPZ3NNUpI+M7PmYcfhXF3ztl1//BKTc865qPwMwjnnXFR+BuGccy4qTxDOOeei8gThnHMuKk8QzjnnovIE4ZxzLqr/B/sIuxvoYiC4AAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 432x288 with 2 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HWaGGvC6CfaC",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "4b52186f-087e-4b8f-f357-fa995cade02e"
      },
      "source": [
        "model.evaluate(seq_subsets['X_test'], initial_subsets['Y_test'], batch_size=BATCH_SIZE)"
      ],
      "id": "HWaGGvC6CfaC",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "639/639 [==============================] - 15s 22ms/step - loss: 0.2962 - accuracy: 0.8936\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[0.2961875796318054, 0.8936060070991516]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 16
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VFAEEuCFYmSA"
      },
      "source": [
        "y_pred = (model.predict(seq_subsets['X_test']) > 0.5).astype(\"int32\")"
      ],
      "id": "VFAEEuCFYmSA",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4SpG6Q6BZeZH",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "5486249f-179a-49c9-daef-bad385046979"
      },
      "source": [
        "from sklearn.metrics import classification_report, confusion_matrix\n",
        "\n",
        "print(confusion_matrix(initial_subsets['Y_test'], y_pred))\n",
        "print(classification_report(initial_subsets['Y_test'], y_pred, labels=[0, 1], target_names=['Negative', 'Positive']))"
      ],
      "id": "4SpG6Q6BZeZH",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[[143387  16208]\n",
            " [ 17762 141928]]\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "    Negative       0.89      0.90      0.89    159595\n",
            "    Positive       0.90      0.89      0.89    159690\n",
            "\n",
            "    accuracy                           0.89    319285\n",
            "   macro avg       0.89      0.89      0.89    319285\n",
            "weighted avg       0.89      0.89      0.89    319285\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "QhWoIsimS-jz"
      },
      "source": [
        "### CNN"
      ],
      "id": "QhWoIsimS-jz"
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "E2j6jEiG93TM",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "e3a43224-d282-4eee-bd47-f16935286099"
      },
      "source": [
        "from keras.models import Sequential\n",
        "from keras.layers import Dense, Activation, Embedding, Dropout, SpatialDropout1D\n",
        "from keras.layers import LSTM, Conv1D, MaxPooling1D, Bidirectional, Flatten\n",
        "from keras.optimizers import Adam\n",
        "import keras\n",
        "from matplotlib import pyplot as plt\n",
        "from keras.callbacks import EarlyStopping, ReduceLROnPlateau\n",
        "\n",
        "def get_model_cnn():\n",
        "  model_cnn = Sequential()\n",
        "\n",
        "  model_cnn.add(Embedding(input_dim=vocab_size, output_dim=EMBED_DIM, weights=[embedding_matrix], input_length=sentence_length, trainable=False))\n",
        "  model_cnn.add(Conv1D(filters=32, kernel_size=32, activation='relu'))\n",
        "  model_cnn.add(MaxPooling1D(pool_size=2, padding='same'))\n",
        "  model_cnn.add(Flatten())\n",
        "  model_cnn.add(Dense(10, activation='relu'))\n",
        "  model_cnn.add(Dense(1, activation='sigmoid'))\n",
        "\n",
        "  return model_cnn\n",
        "\n",
        "\n",
        "with strategy.scope():\n",
        "  model_cnn = get_model_cnn()\n",
        "  optimizer = Adam()#(learning_rate=0.01)\n",
        "  model_cnn.compile(loss = 'binary_crossentropy', optimizer=optimizer, metrics = ['accuracy'])\n",
        "\n",
        "\n",
        "model_cnn.summary()"
      ],
      "id": "E2j6jEiG93TM",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model: \"sequential_5\"\n",
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "embedding_5 (Embedding)      (None, 40, 300)           61108800  \n",
            "_________________________________________________________________\n",
            "conv1d_5 (Conv1D)            (None, 9, 32)             307232    \n",
            "_________________________________________________________________\n",
            "max_pooling1d_5 (MaxPooling1 (None, 5, 32)             0         \n",
            "_________________________________________________________________\n",
            "flatten_4 (Flatten)          (None, 160)               0         \n",
            "_________________________________________________________________\n",
            "dense_10 (Dense)             (None, 10)                1610      \n",
            "_________________________________________________________________\n",
            "dense_11 (Dense)             (None, 1)                 11        \n",
            "=================================================================\n",
            "Total params: 61,417,653\n",
            "Trainable params: 308,853\n",
            "Non-trainable params: 61,108,800\n",
            "_________________________________________________________________\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6sZGYoGB9uZ5",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "944cb7f4-0adb-4abb-ac4d-bd17b6312570"
      },
      "source": [
        "BATCH_SIZE=500\n",
        "\n",
        "callbacks = [ #ReduceLROnPlateau(monitor='val_loss', patience=5, min_delta=0.001, verbose=1),\n",
        "              EarlyStopping(monitor='val_loss', patience=6, restore_best_weights=True, verbose=1)]\n",
        "\n",
        "\n",
        "history_cnn = model_cnn.fit(seq_subsets['X_train'], initial_subsets['Y_train'], \n",
        "                    validation_data=(seq_subsets['X_valid'], initial_subsets['Y_valid']), shuffle=True, \n",
        "                    epochs=50, batch_size=BATCH_SIZE, callbacks=callbacks)"
      ],
      "id": "6sZGYoGB9uZ5",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch 1/50\n",
            "1916/1916 [==============================] - 54s 26ms/step - loss: 0.4938 - accuracy: 0.7569 - val_loss: 0.4241 - val_accuracy: 0.8040\n",
            "Epoch 2/50\n",
            "1916/1916 [==============================] - 46s 24ms/step - loss: 0.4042 - accuracy: 0.8163 - val_loss: 0.4038 - val_accuracy: 0.8177\n",
            "Epoch 3/50\n",
            "1916/1916 [==============================] - 47s 24ms/step - loss: 0.3627 - accuracy: 0.8407 - val_loss: 0.3909 - val_accuracy: 0.8278\n",
            "Epoch 4/50\n",
            "1916/1916 [==============================] - 46s 24ms/step - loss: 0.3318 - accuracy: 0.8567 - val_loss: 0.3842 - val_accuracy: 0.8340\n",
            "Epoch 5/50\n",
            "1916/1916 [==============================] - 47s 24ms/step - loss: 0.3077 - accuracy: 0.8690 - val_loss: 0.3821 - val_accuracy: 0.8387\n",
            "Epoch 6/50\n",
            "1916/1916 [==============================] - 47s 24ms/step - loss: 0.2890 - accuracy: 0.8790 - val_loss: 0.3814 - val_accuracy: 0.8440\n",
            "Epoch 7/50\n",
            "1916/1916 [==============================] - 46s 24ms/step - loss: 0.2734 - accuracy: 0.8865 - val_loss: 0.3787 - val_accuracy: 0.8471\n",
            "Epoch 8/50\n",
            "1916/1916 [==============================] - 47s 24ms/step - loss: 0.2618 - accuracy: 0.8919 - val_loss: 0.3825 - val_accuracy: 0.8494\n",
            "Epoch 9/50\n",
            "1916/1916 [==============================] - 46s 24ms/step - loss: 0.2518 - accuracy: 0.8968 - val_loss: 0.3889 - val_accuracy: 0.8519\n",
            "Epoch 10/50\n",
            "1916/1916 [==============================] - 46s 24ms/step - loss: 0.2421 - accuracy: 0.9014 - val_loss: 0.3967 - val_accuracy: 0.8541\n",
            "Epoch 11/50\n",
            "1916/1916 [==============================] - 46s 24ms/step - loss: 0.2345 - accuracy: 0.9046 - val_loss: 0.3958 - val_accuracy: 0.8563\n",
            "Epoch 12/50\n",
            "1916/1916 [==============================] - 45s 24ms/step - loss: 0.2254 - accuracy: 0.9090 - val_loss: 0.4073 - val_accuracy: 0.8562\n",
            "Epoch 13/50\n",
            "1916/1916 [==============================] - 46s 24ms/step - loss: 0.2192 - accuracy: 0.9123 - val_loss: 0.4064 - val_accuracy: 0.8579\n",
            "Restoring model weights from the end of the best epoch.\n",
            "Epoch 00013: early stopping\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-KVQUovXi03S",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 177
        },
        "outputId": "b0ea397d-2a6e-4c05-e514-b8c99a5ec230"
      },
      "source": [
        "plt.subplot(2, 2, 1)\n",
        "plt.plot(history_cnn.history['accuracy'])\n",
        "plt.plot(history_cnn.history['val_accuracy'])\n",
        "plt.title('model accuracy')\n",
        "plt.ylabel('accuracy')\n",
        "plt.xlabel('epoch')\n",
        "plt.legend(['train', 'val'], loc='upper left')\n",
        "\n",
        "plt.subplot(2, 2, 2)\n",
        "plt.plot(history_cnn.history['loss'])\n",
        "plt.plot(history_cnn.history['val_loss'])\n",
        "plt.title('model loss')\n",
        "plt.ylabel('loss')\n",
        "plt.xlabel('epoch')\n",
        "plt.legend(['train', 'val'], loc='upper left')\n",
        "\n",
        "plt.show()"
      ],
      "id": "-KVQUovXi03S",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAACgCAYAAAAWy/vJAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3dd3hUZdr48e896b2HkISQUKT3jijYQUXsiN21rHXdXXVfd9ff6vpadtd197Wua2FlVVDsDQso2ADpvUiHBEJ6g/S5f3+cEwgxkCFkMpOZ53Ndc005c2buSZ6Z+zzlPI+oKoZhGIbRlMPTARiGYRjeySQIwzAMo1kmQRiGYRjNMgnCMAzDaJZJEIZhGEazTIIwDMMwmmUShBcTkVdF5BEXn7tTRM50d0yG4Wlt9b04ntfxVyZBGIZhGM0yCcJwOxEJ9HQMhmEcP5MgTpBdhb1PRNaIyAEReUVEOonIZyJSLiLzRCSu0fMvEJH1IlIiIgtEpE+jbUNEZIW931tAaJP3Ol9EVtn7LhSRgS7GeJ6IrBSRMhHZIyIPNdk+zn69Env79fbjYSLypIjsEpFSEfnefmyCiGQ383c40779kIi8IyKvi0gZcL2IjBSRRfZ77BORZ0UkuNH+/URkrogUich+EfmDiKSIyEERSWj0vKEiki8iQa58dsMzOsL3opmYbxaRrXYZ/EhEUu3HRUT+KSJ59ndorYj0t7edKyIb7NhyROTeVv3BvJWqmssJXICdwGKgE5AG5AErgCFYBflr4EH7uScBB4CzgCDgd8BWINi+7AJ+Y2+7FKgFHrH3HWK/9iggALjOfu+QRnGceZQYJwADsA4IBgL7gQvtbV2BcmCa/b4JwGB723PAAvtzBQBjgRD79bKb+Tucad9+yI79Qvs9w4BhwGggEMgENgK/tp8fBewD7rH/ZlHAKHvbHOC2Ru/zT+AZT//fzcUnvhevNnqd04ECYKhdxp8BvrW3nQMsB2IBAfoAne1t+4BT7NtxwFBP/+3b8mJqEG3jGVXdr6o5wHfAj6q6UlWrgPexCjHAVOBTVZ2rqrXA37F+PMdi/XgGAf+nqrWq+g6wtNF73AL8W1V/VNV6VZ0BVNv7HZOqLlDVtarqVNU1wCxgvL35SmCeqs6y37dQVVeJiAP4BXC3qubY77lQVatd/JssUtUP7PesVNXlqrpYVetUdSfw70YxnA/kquqTqlqlquWq+qO9bQZwNYCIBGAlstdcjMHwLK/+XjRxFTBdVVfYZfz3wBgRycRKSFFAb0BUdaOq7rP3qwX6iki0qhar6orjfF+vZhJE29jf6HZlM/cj7dupWEdDAKiqE9iDdYSVCuSofShi29XodlfgHrsaXSIiJUAXe79jEpFRIjLfbpopBW4FEu3NXYBtzeyWiHWk19w2V+xpEsNJIvKJiOTazU6PuRADwIdYX8AsrCPMUlVd0sqYjPbl1d+LJprGUAEUAmmq+jXwLFaNOk9EXhSRaPuplwDnArtE5BsRGXOc7+vVTIJoX3uxCjRgtW1iFeYcrKpqmv1Yg4xGt/cAj6pqbKNLuKrOcuF9ZwIfAV1UNQZ4Aauq3PC63ZvZpwCoOsq2A0B4o88RACQ1eU7TaYL/BWwCeqpqNPCHJjF0ay5w+2hzNlYt4hpM7cEXeep7cawYIrCaW3MAVPVpVR0G9MVqErvPfnypqk4BkoEPsMqqzzAJon3NBs4TkTPsTtZ7sKrDC4FFQB3wKxEJEpGLgZGN9n0JuNWuDYiIRIjV+RzlwvtGAUWqWiUiI7GalRq8AZwpIpeLSKCIJIjIYPsobjrwDxFJFZEAERkjIiHAT0Co/f5BwANY7bYtxVAGVIhIb+C2Rts+ATqLyK9FJEREokRkVKPt/wWuBy7AJAhf5KnvRWOzgBtEZLBdxh/DahLbKSIj7NcPwjo4qgKcIhIsIleJSIzdNFYGOE/g7+B1TIJoR6q6GetI+BmsI/TJwGRVrVHVGuBirB/CIqx22fca7bsMuBmrqluM1Yl3vYtvfTvwsIiUA3+i0VGOqu7GqiLfY7/vKmCQvfleYC1Wm28R8FfAoaql9mu+jHWEdQA4YlRTM+7FSkzlWF/qtxrFUI7VfDQZyAW2AKc12v4D1hdvhao2bl4wfIAHvxeNY5gH/D/gXaxaS3fgCntzNFaZLcZqhioEnrC3XQPstJtNb8Xqy/AZcmTTnmF4JxH5Gpipqi97OhbD8BcmQRheT0RGAHOx+lDKPR2PYfgL08RkeDURmQHMwzpnwiQHw2hHpgZhGIZhNMvUIAzDMIxmmQRhGIZhNMtnZtlMTEzUzMxMT4dh+LDly5cXqGrTEwLdzpRtw52OVa59JkFkZmaybNkyT4dh+DAR8cg5GKZsG+50rHJtmpgMwzCMZpkEYfg9VWVfaSVfb9pPXnmVp8Nx2eo9JWzKLfN0GIYP85kmJsNwRU2dk615FWzcV8bGfWVssK+LD9YC8M+pg7hoSLqHo2xZdV09v3h1KX1To3ntxlEt72AYreDTCaK2tpbs7GyqqjrOUWFrhYaGkp6eTlCQWWitQenBWtbvK2XD3jI27itnw74ytuaVU1tvnfsTEuigV0oU5/RLoU/naPp0jqZfanQLr+odHOrkyUmp1NRUsXrteoIDfbMxwJRrz/LpBJGdnU1UVBSZmZkcOVuwb1FVCgsLyc7OJisry9PheERFdR3rc0pZk13KmpxS1maXsLPw4KHtSVEh9O0czfiTkujTOYp+qdFkJkQQGNAxf1izs7Pp27UT+TVBhIcEkpUY4emQ2pwp157n1gQhIhOBp7CWAnxZVf/SZHtXrCmlk7BmarxaVbPtbddhTSMN1rKAM473/auqqnw+OQCICAkJCeTn53s6lHZRVVvP+r1lrM0uYY2dFLblV9AwKUBabBgD0mK4fEQX+qfG0KdzNElRLc1G3rE0lG0qqsktreJgTR3hwb51vOdv5dobua1E2YvIPIc1jXM2sFREPlLVDY2e9nfgv6o6Q0ROBx4HrhGReOBBYDjWwjPL7X2LWxHHiX6UDsGXP2d+eTVLdxbx4/ZClu4sZvP+cuqdVjZIjAxhUHoMkwemMjA9hv5pMT6XDI5GREiICKGgvJr9ZdVkJfpWggDfLtcdgTtL1Ehgq6puBxCRN4EpQOME0Rf4rX17PtaKTGAtEj5XVYvsfecCE7EW9ehQSkpKmDlzJrfffvtx7Xfuuecyc+ZMYmNj3RSZ99pXWsmSHUUs3l7Ekh2FbMs/AEBYUABDu8Zy6/huDEyPZWB6DCnRoX79IxLgEBIjQ8gta99ahCnX/sGdpSmNI9clzgaaDrdYjbUYyFPARUCUiCQcZd8094XqPiUlJTz//PM/+yLV1dURGHj0P/+cOXPcHZpXUFX2FFWyeEchS3YUsWRHEbuLrL6DqJBARmTFc9nwLozKiqd/WgxBHbTPwJ0SIkPIr6gmr6yazHaqRZhy7R88XSe9F3hWRK4HvsVanaze1Z1F5BbgFoCMjIwWnu0Z999/P9u2bWPw4MEEBQURGhpKXFwcmzZt4qeffuLCCy9kz549VFVVcffdd3PLLbcAh8+eraioYNKkSYwbN46FCxeSlpbGhx9+SFhYmIc/WescSgjbC1m0vZDF2wvZV2qNMosLD2JkVjzXj81kZFY8fTpHE+Dw39qBqwIcQlI71yJMufYP7ixJOVgLjzdItx87RFX3YtUgEJFI4BJVLRGRHGBCk30XNH0DVX0ReBFg+PDhx5y3/M8fr2fD3rY9qahvajQPTu53zOf85S9/Yd26daxatYoFCxZw3nnnsW7dukOjMqZPn058fDyVlZWMGDGCSy65hISEhCNeY8uWLcyaNYuXXnqJyy+/nHfffZerr766TT+Lu6gq2cWVLNpmJYPF2wvZayeExMhgRnVLYHRWPKO6JdAjKRKHSQjHraFsH6ypwyFCaFDACb9mS2Xb38u1v3BnglgK9BSRLKzEcAXWmsSHiEgiUKSqTuD3WCOaAL4AHhOROPv+2fb2Dm/kyJFHDNl7+umnef/99wHYs2cPW7Zs+dkXKSsri8GDBwMwbNgwdu7c2W7xtsaeooOHagc/bi8ip6QSgISIYEZ3S+C27gmM6RZP96RIv+4/aGtBAQ5q6pw4VXG089/VH8q1P3JbglDVOhG5E+vHPgCYrqrrReRhYJmqfoRVS3hcRBSriekOe98iEflfrCQD8HBDh3VrtXSk314iIg6PV1+wYAHz5s1j0aJFhIeHM2HChGZP6gsJOTwqJyAggMrKynaJ1VVOp7Imp5S5G3L5cv1+tuRVABAfEczobvH8cnw3xnRLoEeySQju0FC2651ONuWWExEcSGY7nxfhi+XacHMfhKrOAeY0eexPjW6/A7xzlH2nc7hG0WFFRUVRXt78SpmlpaXExcURHh7Opk2bWLx4cTtH13rVdfUs2lbI3A37mbthP3nl1QQ4hJGZ8VwxMoNxPRLpmWyajNpTgMNBYmQI+8uqqKypI8yNfRG+Wq6NI3m6k9rnJSQkcPLJJ9O/f3/CwsLo1KnToW0TJ07khRdeoE+fPvTq1YvRo0d7MNKWlVbWsmBzHl+u3883P+VTUV1HeHAA409K4ux+nTitVzKx4cGeDtOvJUQGU1BhnRfhzhFNvlSujaPzmTWphw8frk3nzN+4cSN9+vTxUETtzx2ft6Cims/W7uOL9ftZvL2QOqeSGBnCWX2TObtvCmO6J7RJp2hHICLLVXV4e7/vUct2l3gICoeQqCO27S+rYn9ZFT2TowgL7vj/G3/7Hre3Y5VrU4Mwfqb0YC1frM/l4zV7+WFrAU6FbkkR3HRKN87u14nB6bGm6cjT1AkHCqF+L0QkQVRncFjJoKEWkVdeRdcE35ujyWg/JkEYgDXZ3bwN+/l49V6+3ZJPbb3SNSGc2yf0YPKgVHqlRLX8Ikb7EQcknQTle+FAPlSVQWwGhEQS6HCQEBlCXlkVlTX1PlGLMDzDJAg/VlVbz/xNeXy8Zi9fbcyjus5JakwoN5ycxeSBqfRPizajjryZIwBiukBoLJTshsItEJEMUZ1JjAimsNzUIowTYxKEn1FVlu8q5o0fd/Pl+lwO1NSTGBnMFSO6MHlQKkMz4kzzUUcTEgVJvaEsBw7kQXUZgbEZVi2ivIrK2nrC/KSfyGhbJkH4ibp6J5+ty+Xl73ewek8J0aGBXDA4lfMHpjIqK77Drotg2BwBVhNTQ22i4CeSI5IpkgjyykwtwmgdkyB8XFlVLW8t2cOrC3eSU1JJVmIE/3thfy4ZmuZz6wcYQGg0JPeG0hwcB/LoGRDCzspEqmpD/Wa0mV+oqwZnPQSGHBqc4A7mF8LLREZGUlFRccKvs6foIP/5YSdvLd3NgZp6RneL588X9OP03smmCcmNWlokq9HzLsE6SXSEqi5r7jmt5giEuK4QFktgyW66Sw5lRVWEJqdbndse0Fbl2q8562HbfFj5GmyeA/U11uMSYCWKwBAICIHAYPs6BAKCITAUQiLhqreP+y1NgvAxy3cV88r32/l8XS4OESYPSuXGcVn0T4vxdGg+z8VFshCRKOBu4Ee3BhQagyT1oapwF7F1hTjzDuCITrGaoczgg46jcBusegNWzbJGrYXFw7AbIDrVShJ11VBfbV3XVTd6zL6uqzqcTI6TSRBudv/999OlSxfuuOMOAB566CECAwOZP38+xcXF1NbW8sgjjzBlypRWv4eqUlZVR155NdfNWEh0aCC/HN+d68ZkkhIT2lYfxWiZK4tkAfwv8FfgPrdHFBBIUEIWu3Nz6ewswlG8ExxBEJEI4YkQ0LqfgPYo136t5gBs+MiqLez6war59TgTJv0FTppk1RLagf8kiM/uh9y1bfuaKQOsf9gxTJ06lV//+teHvkizZ8/miy++4Fe/+hXR0dEUFBQwevRoLrjgglYNKT1YU8e+kioO1NThdCoPT+nHJUPTiQjxn3+tF2lxkSwRGQp0UdVPReSoCeK41jppoWwHAp3q66muUwICFYezFrQeEKs5KiDIaqZorIWy7e5y7ZdUIXuZlRTWvQc15RDfHc54EAZdYdUY2pn5FXGzIUOGkJeXx969e8nPzycuLo6UlBR+85vf8O233+JwOMjJyWH//v2kpKS4/Lo1dU72l1VRfLCGQIeDtLgwAstCOa1vpvs+jHFCRMQB/AO4vqXnHs9aJ64IDnBQX19PZb0QFhSGAyfU14KzDpy1VoIICLY7PFv+QXdXue7Q6utg32rYsQC2fwN7lliJODD0cF9AYIh93eR+QDDkb4aCzRAUAf0ugiFXQ8ZojzYHupQgROQ94BXgM3vtho6nhSN9d7rssst45513yM3NZerUqbzxxhvk5+ezfPlygoKCyMzMbHY65ObUO5X88moKKqpRIDkqhKSoUAIcQp45UPO0lhbJigL6Awvso+oU4CMRueCEOqpdKNsCOGrr2ZZXQXCgg+4NizPV18HBQjhYYLVTBwTbzU8JLb5mW5brDkkV8jdZyWDHN7DzB6gutbYl94Oh11hzZdXXWP0ADf0BDX0FdVVQWwmVxdb9iCQYexf0u/Bn82t5iqs1iOeBG4CnReRt4D+qutl9YfmWqVOncvPNN1NQUMA333zD7NmzSU5OJigoiPnz57Nr164WX0NVKT5Yy/6yKmrrncSGBZMSE0JwoBm66EWOuUiWqpYCiQ33RWQBcG+bj2I6itCgALrEh7Oz8AA5JZWkx4UhAYEQ1Qkik6Gq1Jq2o2wvlOdCSDQER1iXoLCfjYBqi3Ld4RTvspLB9m9gx7fWiYkAcZnQbwpkjYesU62/pw9wKUGo6jxgnojEANPs23uAl4DXVbXWjTF2eP369aO8vJy0tDQ6d+7MVVddxeTJkxkwYADDhw+nd+/ex9y/oqqWfaXWGbHhwYFkxIebPgYv5OIiWR4VHRZESnQouWVVhAYFkBRlL9ojAmGx1qXmoFWjqC6HqhJ7T7GOhoMjIDgcgiJOuFx7veoKq8koZxnkLIfs5VCWbW2LSLYSQbfxVlKI6+rZWN3E5V8ZEUkArgauAVYCbwDjgOs4cv1ooxlr1x7uRExMTGTRokXNPq/xWPHq2nr2lVZRVlVLcICDjPhwYsKCTKefF2tpkawmj09oj5iaSooKobK2ntzSSkKDHESFBh35hOBwCLY7xutrrIRRc8C6HMiHA3aXiCOItd98ZCWN6goSYyNZ9P13Vj9GkzLq9edAOOshb6OVCHKWQc4KyNtgzZoLENsVMkZB+l1WUkjq7RdDhV3tg3gf6AW8BkxW1X32prdEpF2qx/7EqUpeeTX5ZdU4BFJiQkmMCDEnuBltQkRIjwtnW10Fu4sO0jM58uhNlQHBEBZs1SzA+sGsrbQTxkGoPdColtH4TQKsROEItBOGfd3wmDjsH1g5xm374qyD2ioIaoMh27WVULLHmo6kZBcUbYe9K2HvKuuzgHWeSNow6H2edZ02zOqX8UOu1iCeVtX5zW3wxAIqvqyqtp49RQeprK0nLjyYlJhQgsw8SUYbC3AIXRPC2ZpXwc7Cg3RPiiTAlQMQcRzul2hQX2v98Drr7Eu9NXqn4bazDpw11rXWH3+wZXnw6DirTyQiyWrfP3SdDJFJ9rX9uDqheKedBOxE0HC7Yn+TP0SINaR3yNWQPtxKBvHd/KJ24ApXE0RfEVmpqiUAIhIHTFPV590Xmn9RVfLtpSIDROiaEEFMWFDLOxpGK4UEBpARH87OggNkFx8kIz68dc2XAUHWxRWqVpJQp31bgca37Wt1Hr6dXwen/z+reasiz7rO3ww7v7NGAB2LI9CaEj02A3qebTUVxXW17sdmQGQKOMwB2NG4miBuVtXnGu6oarGI3Iw1usmrqarXt9lX19azp7iSgzV1xIQFkRYbdtyzq/rK0rGG69qibEeFBpESE8q+0iryK6pJjnLzmfciVnOTi1QVgiPh1Hubf0JdjdWh3pA4KvKsWk5DEmi00p5x/Fz9TwWIiKj9K2TPOeP1q9OHhoZSWFhIQkKCVyYJVaXwQA25pVWIQJf4cGJb0QmtqhQWFhIaaqbV8BdtWbYTI0OorHGSW1pFWFDAzzutPcSlch0YbJ1h7IGzjP2Bqwnic6wO6X/b939pP+bV0tPTyc7OJj8/39Oh/Ey9Uyk+UENVnZPQIAdx4cHklgq5rXy90NBQ0tPT2zRGw3u1ddlWVYrLq8nfoyRHhXjN+iCmXHuWqwnif7CSwm32/bnAy26JqA0FBQWRlZXl6TCOoKq8uyKHP3+0nnpVHjivL9OGd/HKGo5Pc9ZbHZZl+6yV2Mr3WSeIDZwKnfp6OroWuaNs7yk6yAXPfk9iZAjv33EykeZcG7/n6olyTuBf9sVopfzyan7/3lrmbdzPyKx4/n7pIDISwj0dlverq4HSPVC0wzpRSZ32cEiHNZyy4XbD+PvGj9ccsKZIPiIR7IOK3MNj3Bs4giB1cIdIEO7QJT6c564cyjXTl/Dbt1bxwtXDzNBqP+fqeRA9gceBvsChBkFV7eamuHzOhr1lXDt9CWVVtTxwXh9+cXKW+fI1VlUGxTusJFC8s9HtHVCa/fMf8+MVEgPRna1Oy+69rTbrqM6H26+jUq35h/x8RMvYHon88dw+PPzJBp74cjO/O6eXqd36MVfrkP8BHgT+CZyGNS9Ti9+kllbXEpEMYAYQaz/nflWdIyKZwEagYb6nxap6q4uxep0lO4q4ccZSIkMC+fjOcfRK8Y6JuNpVfZ119F+805rPpmTX4dvFO6wJ4xoLi4f4LEgfaTX7xGVZ92O6WEMX1WlfGg+ZdNpj8J2HL0FhViIIiTyucJ966iluuOEGoqKiuOmmm1i5ciVAdBv9NbzaDSdnsjW/gn8t2EZlTT1/Or+vOZjxU64miDBV/coeybQLeEhElgPNTiEALq+u9QAwW1X/JSJ9saYoyLS3bVPVwcf5ebzO15v2c9vrK0iLC+O1G0eRFhvm6ZDcp7oCCrdal+KdRyaB0uwjT5KSAIjtYo1L732+9ePfkATiMiHUsyvgTZ8+nbvvvpsvvviC4uJiXnvtNfr375/m0aDaiYjwyJT+hAUF8Mr3OyirquVvlwz0mo5ro/24miCq7bnst9iTkeUALR2SubK6lnL4qCwG2Otq4B3BBytzuPft1fTuHMWMG0aSEBni6ZBOnNNpteUX/GQlgoIth2+X5Rz53Igk68c+fQQMuMwamx6XaSWF6LRWr2bWHhrOK5kzZw7XXHMN/fr1A1cWSvARDofwwHl9iAsP4u9f/kRZZR3PXjmE0CBzToE/cfUbejcQDvwKa7nE07Am6TuWFlfXAh4CvhSRu4AI4MxG27JEZCVQBjygqt+5GKtXmLFwJw9+tJ7R3eJ56drhXjO23CWqVpNP0XbrUrjNrhlsgYKtUFd5+LkhMZDYAzJPgcSe1iWhh5UIGk/H0MEMGzaMs88+mx07dvD4449TXl4O1gGN3xAR7jy9J9FhQfzpw/Xc8J+lvHTdcDO6yY+0+J+2m4qmquq9QAVW/0NbmQa8qqpPisgY4DUR6Q/sAzJUtVBEhgEfiEg/VS1rEpvryzK2E1Xlqa+28H/ztnBW3048M81Lj7pU4UABFG07MhEUbbc6hxsWPgFrNFBsBiSeBJmnNkoEPa35b3ywE/OVV15h1apVdOvWjfDwcIqKigB2ejgsj7h2TCbRoUHc8/ZqrnxpMa/eMJL4CK8/T9ZoAy0mCFWtF5FxrXjtllbXArgRmGi/zyIRCQUSVTUPqLYfXy4i24CTgCNmjm3rZRlPlNOp/Pnj9cxYtItLh6Xzl4sHeEe7bV21NZVx7hpr7eJ9a6ypjKsb5VsJsJJAfDfoMtK6ju9mrYkbm9Fui6R7i0WLFjF48GAiIiJ4/fXXWbFiBUArZprzDRcOSSMqNJDb31jB5f9exGs3jqRzjA/3pxmA601MK0XkI+Bt4EDDg6r63jH2OebqWrbdwBnAqyLSB2sIbb6IJAFFdnLqBvQEtrsYq0fU1ju57+3VfLBqLzeNy+IP5/bxzMiPqjIrCeSutRLCvjXWsohOe02n4Ejo1B8GXm7VABK6W4kgNsP1Cdf8wG233cbq1atZvXo1Tz75JDfddBOAd5112c7O6NOJGb8YyU0zlnHpvxbx+k2jyErsuM2IRstcTRChQCFweqPHFDhqgnBxda17gJdE5Df2612vqioipwIPi0gt4ARuVdWi4/1w7aWypp47Zq7g60153HdOL26f0L39xo7XHICtX8GmT6xF0ot3HN4WkQQpA6HnmdZ150HWSCE/H+vvisDAQESEDz/8kDvvvJMbb7yRO++80+//cKO7JfDmLaO5dvoSLnthEf/9xUj6pvrF6F+/JL4yC+jw4cN12bL2X7uotLKWm2YsZdmuYh65sD9XjWqHpQcrS2DLl7DxI9gyz+o0DouzOoo7D4SUQdYc91EpPtk/0B7Gjx/PxIkTmT59Ot999x3JyckEBARUqmq7n/ruqbJ9LFvzKrj2lR8pr67jP9ePYHhmvKdDMlpJRJYfbV0fV8+k/g/NjOBQ1V+cYGwdWkFFNde8soSteeU8M20I5w9044ySFfmw+VPY+LG1YLqz1joBbMjV0GcydD3Zq4eNdjRvvfUWM2fOZPr06aSkpLB7926A/S3t5y96JEfy9m1jueblH7n6lR954ephTOiV7OmwjDbm6i/KJ41uhwIX4WPnLByvunond7yxgh0FFbx83QjGn5TU9m9Smg0bP7GSwu6F1pnBcZkw+jboc4G1+pVpLnKLlJQUrrrqKpYuXconn3zCyJEjwWpmNWxpsWHMvnUM176yhJtmLOP+Sb25cVyWmZrDh7g6Wd+7je+LyCzge7dE1EH8c95P/LijiH9cPqjtkkN9rdWPsHWu1XS0f631eHI/OPV3Vk2hUz/TbNQOZs+ezX333ceECRNQVe666y6AOE/H5W0SI0N485ejuWf2ah75dCOLthXyxGWDzDBYH9HaNomegN/WJxdszuO5+duYOrwLFw89wbnqy/bB1nlWUti2wDr/wBEIXUbDmX+2pqFI7NEmcRuue/TRR1m6dCnJyVYxz8/PJzk5ubOHw/JK0aFBvHjNMP67aBePfrqRSU99y/9NHcKY7gmeDs04QXK8dxAAABKOSURBVK72QZRzZB9ELtYaEX5nX2klv3lrFb1TovjzlH7H/wL1dZC9BLbMtZJCrl1LiOoM/aZAj7Og2wQINSNDPMnpdB5KDgAJCebH7lhEhOvGZjI8M467Zq7kypcXc9fpPfnV6T2841wgo1VcbWLyw+lHf6623sldM1dSU+fkuauGun6GtLMets+HVbOspFBVap2YljEaznjQWkzdNB15lYkTJ3LOOecwbdo0wOq0BkqPuZNBv9QYPr5rHH/6cD1Pf7WFxdsK+b8rBpPqy5NU+jBXaxAXAV+raql9PxaYoKofuDM4b/P3LzezbFcxT10xmO5JLkwfXbAVVr0Bq9+0Fq0Ji7P6ERpqCWGx7g7ZaKUnnniCd999lx9++AGAW265hc8//7zpTABGMyJCAnny8kGM65nAA++v49ynv+OJSwdxVt9Ong7NOE4unQchIquaTr0tIitVdYjbIjtO7h4r/tXG/dw4YxlXjsrgsYsGHP2JVWWw/j1YNRP2/GjNY9TjLBh8JfSaBIE+MKOrnzrWeHF38sbzIFy1o+AAd81awbqcMq4fm8n9k3p759xkfuyEz4Og+cWB/GbQfXbxQX47ezV9O0fzp/ObWY7S6YSd38LKN6whqXWVkNgLznrYWuwmKqX9gzZaJSoqqtlhmvaBlNccEHUUWYkRvHvbWP762Wam/7CDJTuKeObKIa7VwA2Pc/VHfpmI/ANrASCAO4Dl7gnJu9TUOblz5krqncrzTfsdKvJhyYuwepa1ZnJIDAyeBoOvhrShpk+hA7Kn9W6WPf28cZxCAgP40+S+nNwjgXvfXs3kZ77ngfP6csWILmalOi/n6vCCu4Aa4C3gTaAKK0n4vL9+volVe0r426UDyWyYmMzphGXT4dlh8O0T1jTYl06He3+C8/8J6cNMcjCMJs7o04nP7j6VQemx/OH9tVzywkLW5Zh+f2/m6iimA8D9bo7F63yxPpdXvt/BdWO6cu4Aewj8vjXw6W8he6k199F5/4CkkzwbqOE1XFiH/Vasg6t6rPVVbmmyDK9PS4kJZebNo3hvRQ6Pf7aRC579nqtHd+Wes3oRE25mE/Y2LtUgRGSuPXKp4X6ciHzhvrA8b3fhQe59ezUD02P4w3l9oLocPv8DvDjeWmf54pfguo9NcjAOabQO+ySgLzDNXmu9sZmqOsAe9PE34B/tHKbHiQiXDEvnq3smcO2YTF5fvIvTn1zA7GV7cDp9Y/JQX+FqE1OiqpY03FHVYnz4TOrqOmv6boDnpg0h5KdP4NmRsPh5GHY93LnUWk/BNCMZRzq0Druq1mA1x05p/IQmqyJG4GfLmDYWExbEQxf04+O7xpGZGMHv3lnDpS8sZP1e0+zkLVxNEE4RObSmp4hk4sMF+7FPN7I2p5RnJ8XT5fPrYfa1EJEAN82z+hjCzJQ8RrOaW4c9remTROQOe5XEv2Gt8+7X+qXG8PYvx/DEpQPZVXiQyc98z4MfrqO0stbTofk9V0cx/RH4XkS+AQQ4BXstaF/z6Zp9zFy0jVe6LWT83FeteZHOeRxG3mKm0zbahKo+BzwnIlcCDwDXNX2ON6637k4Oh3DZ8C6c3TeFJ+du5rXFu/h07T7un9SHS4ammRliPcSlGoSqfg4MBzYDs7BWgqt0Y1weUV5Vy9vvvcnXEX/kjL0vwEnnwB1LYMztJjkYrnBlHfbG3gQubG6Dqr6oqsNVdXhSkhumkvdSMeFBPDylPx/dOY4u8eHc+/ZqLnthEYu2FeIri5t1JK5OtXETcDdWgV8FjAYWceQSpB2bKhvee5xX9B/UhafD5LfhpLM9HZXRsbS4DruI9FTVLfbd84AtGD/TPy2Gd28dyzvLs/nbF5uZ9tJihmbEcvuEHpzRJ9nUKNqJq30QdwMjgF2qehrWGaUlx96lA6mrQT++m1E/PcmS0DGE3PWjSQ7GcVPVOqBhHfaNwOyGddhF5AL7aXeKyHoRWQX8lmaalwyLwyFcPqIL3//PafzvlH7klVdz03+XMemp7/hwVQ519U5Ph+jzXG03qVLVKhFBREJUdZOI9HJrZO3lYBHMvhbZ+R3P1k0h48LHIDjC01EZHZSqzgHmNHnsT41u393uQXVwoUEBXDMmkytGZvDx6r38a8E27n5zFU9++RO/HN+NS4amm/md3MTVBJFtnwfxATBXRIqBXe4Lq50UbIGZl0NpNv+K/x2vVYzi+wFuXFfaMIxWCwpwcPHQdC4cnMbcjft5fsE2/vj+Op6at4WbTsniylFdiQwxfYVtydUzqS+ybz4kIvOBGOBzt0XVHrbNh7evA0cQey6YzV9nHeTes7sSZBY3MQyv5nAI5/RL4ey+nVi4rZDnF2zlsTmbeG7+Nq4bm8n1YzPNkqdt5LjTrap+445A2tXSV2DOfZDUC6a9yQsLyggOzGbaSN8fTmgYvkJEOLlHIif3SGTVnhKen7+Vp7/awgvfbOPc/ilcOaorIzLjTIf2CfCv+lh9HXz5R/jxBeh5DlzyMqXOMN5bsZEpg1JJiDRrNRhGRzS4SywvXjucLfvLeW3xLt5fkcMHq/bSMzmSK0dlcPGQdDPXUyv4T3tKVSnMmmolh9F3wLRZEBrNW8t2U1lbz/UnZ3o6QsMwTlDPTlE8PKU/P/7xDP52yUDCQwL588cbGPnYPO6ZvZrlu4rN+RTHwT9qEEU7YNYVULgVJj9lzacE1NU7mbFwFyOz4umXGuPZGA3DaDPhwYFcPqILl4/owvq9pcz8cTcfrMzh3RXZ9E6J4spRGVw4JI3oUFOrOBa31iBEZKKIbBaRrSLys+nCRSRDROaLyEoRWSMi5zba9nt7v80ick6rg9i1EF4+A8pz4Zr3DyUHgHkb88gpqeQXpvZgGD6rX2oMj140gCV/PJPHLx5AUICDP324nlGPfsXv3lnN0p1FplZxFG6rQTSa+vgsrEnLlorIR03mvn8A62Sif9nTIs8BMu3bVwD9gFRgnoicpKr1xx3I3pXW5HpXzoaE7kds+s8PO0iLDePMPmYxdcPwdREhgUwbmcG0kRmszS5l5pJdfLhqL7OXZZMRH86FQ9K4eEja4YXBDLc2MR2a+hhARBqmPm6cIBSItm/HAHvt21OAN1W1GtghIlvt11t03FGMvh2GXgchR66Bu2FvGT/uKOL3k3oTaIa2GoZfGZAew+PpA3ngvL58sT6X91fm8MzXW3j6qy0MzYjl4qHpnD+wM7Hh/j1c1p0Jormpj0c1ec5DwJcichfW3PhnNtp3cZN9fzZtsktEfpYcAF5duIOwoACuGGGGthqGv4oICeTioelcPDSd3NIqPlxl9VM88ME6Hv54A6f3TuaioWmc1iuZ4ED/O5D0dCf1NOBVVX1SRMYAr4lIf1d3bu2UyEUHavhg1V4uHWaGvhmGYUmJCeWX47tzy6ndWL+3jPdX5vDhqhw+X59LbHgQkwemcsHgVIZmxBHg8I9zK9yZIFyZ+vhGYCKAqi4SkVAg0cV9UdUXgRcBhg8f7nIv06wlu6mpc3LD2ExXdzEMw0+ICP3TYuifFsPvJ/Xmu60FvLcih9nL9vDa4l3Ehgcx/qQkTu+dzPiTkny6GcqdCaLFqY+B3cAZwKsi0gcIBfKBj4CZIvIPrE7qnsCStgiqtt7Ja4t2cUrPRHp2imqLlzQMw0cFBjg4rVcyp/VKpryqlm9+yufrTXl8szmfD1ftxSEwNCOO03onc3rvZHqnRPnUmdtuSxCqWiciDVMfBwDTG6Y+Bpap6kdYCw+9JCK/weqwvl6t8WbrRWQ2Vod2HXBHq0YwNePzdbnkllXx6EUut2QZhmEQFRrE+QNTOX9gKvVOZU12CfM35fH15jye+GIzT3yxmdSYUCb0Tub0XsmM7ZFAeLCnW/FPjPjK+N/hw4frsmXLWnzexc//QNGBGr6+ZwIOP2lHNNqGiCxX1eHt/b6ulm3Dc/aXVbFgcx5fb8rjuy0FHKypJzjQwaiseE7pmcipJyXRq5N31i6OVa47dno7Tqv3lLBidwkPTu5rkoNhGG2mU3QoU0dkMHVEBtV19SzdUWwni3wem7OJx+ZsIjkqhFN6JnHqSYmM65HYIeZ+86sE8erCnUSGBHLpsHRPh2IYho8KCQxgXM9ExvVMBGBvSSXfbyngmy35fLVpP++uyEYE+qfGHKpdDM2I88phtH6TIPLKq/hkzV6uGtWVKDP/imEY7SQ1NuzQvFD1TmVdTinf/pTPd1sKePHb7Ty/YBvhwQGM7pbA2O4JnNwjkV6doryilcNvEsQbi3dT51SuM0NbDcPwkACHMKhLLIO6xHLXGT0pr6pl0bZCvt2Szw9bC/l6Ux4ACRHBjO6ewMndExnbPYGuCeEe6b/wiwRRXVfPGz/u4rReyWSZeVYMw/ASUaFBnN0vhbP7pQBWc9TCbYUs3FrAD9sK+HTNPgDSYsMY0z2Bk3skMLZ7Ip2iQ9slPr9IEJ+u2UdBRQ03mFlbDcPwYqmxYVw6LJ1Lh6WjqmwvOHAoYczbuJ93lmcD0D0pgqEZcQxMt07o69M5mtCggDaPx+cThKrynx920iM5knE9Ej0djmEYhktEhO5JkXRPiuSa0V1xOpUN+8pYtK2QhdsK+HpTHm/bCSPQIfTsFMXAtBj6p8cwMC2GXilRJ5w0fD5BrNhdzNqcUh65sL9XjkE2DMNwhcNxeAqQm0/thqqyr7SKNdmlrM0pYW1OGV9uyOWtZdYcqYEOoVdKFAPSYhiQHsPlw7sQdJwzV/t8gvhyw36iQwO5eGjrJoM1DMPwRiJCamwYqbFhTOxv9WGoKtnFlazLKWVNTinrckr5bF0un63L5cqRxz9ztc8niPsn9ubaMZkd/pR3wzCMlogIXeLD6RIfzqQBnQEraeRXVLeqBcX7zsxoYyJCWmyYp8MwDMPwCBEhOap1o558PkEYhmEYrWMShGEYhtEsn5nNVUTygV1H2ZwIFLRjOJ5iPqd7dVXVpPZ+U1O2Af/4nF5Xrn0mQRyLiCzzxDTN7c18Tv/jL38Lf/ic3vgZTROTYRiG0SyTIAzDMIxm+UuCeNHTAbQT8zn9j7/8Lfzhc3rdZ/SLPgjDMAzj+PlLDcIwDMM4Tj6fIERkoohsFpGtInK/p+NxFxHZKSJrRWSViPjECvciMl1E8kRkXaPH4kVkrohssa/jPBmjp5hy3bF1lLLt0wlCRAKA54BJQF9gmoj09WxUbnWaqg72tqFyJ+BVYGKTx+4HvlLVnsBX9n2/Ysq1T3iVDlC2fTpBACOBraq6XVVrgDeBKR6OyXCRqn4LFDV5eAoww749A7iwXYPyDqZcd3AdpWz7eoJIA/Y0up9tP+aLFPhSRJaLyC2eDsaNOqnqPvt2LtDJk8F4iCnXvsnryraZA9t3jFPVHBFJBuaKyCb7KMVnqaqKiBmG59v8rlyD95RtX69B5ABdGt1Ptx/zOaqaY1/nAe9jNUP4ov0i0hnAvs7zcDyeYMq1b/K6su3rCWIp0FNEskQkGLgC+MjDMbU5EYkQkaiG28DZwLpj79VhfQRcZ9++DvjQg7F4iinXvsnryrZPNzGpap2I3Al8AQQA01V1vYfDcodOwPv2ilGBwExV/dyzIZ04EZkFTAASRSQbeBD4CzBbRG7EmuH0cs9F6BmmXHd8HaVsmzOpDcMwjGb5ehOTYRiG0UomQRiGYRjNMgnCMAzDaJZJEIZhGEazTIIwDMMwmmUShHFUIjJBRD7xdByG0ZZMuXadSRCGYRhGs0yC8AEicrWILLHnzP+3iASISIWI/FNE1ovIVyKSZD93sIgsFpE1IvJ+w5zzItJDROaJyGoRWSEi3e2XjxSRd0Rkk4i8IfZZS4bhbqZce55JEB2ciPQBpgInq+pgoB64CogAlqlqP+AbrDM1Af4L/I+qDgTWNnr8DeA5VR0EjAUaZpUcAvwaa92BbsDJbv9Qht8z5do7+PRUG37iDGAYsNQ+CArDmuTLCbxlP+d14D0RiQFiVfUb+/EZwNv2fDdpqvo+gKpWAdivt0RVs+37q4BM4Hv3fyzDz5ly7QVMguj4BJihqr8/4kGR/9fkea2dU6W60e16TJkx2ocp117ANDF1fF8Bl9rz5Tesa9sV6397qf2cK4HvVbUUKBaRU+zHrwG+UdVyIFtELrRfI0REwtv1UxjGkUy59gIma3ZwqrpBRB7AWnXLAdQCdwAHgJH2tjys9lywphF+wf6ibAdusB+/Bvi3iDxsv8Zl7fgxDOMIplx7BzObq48SkQpVjfR0HIbRlky5bl+mickwDMNolqlBGIZhGM0yNQjDMAyjWSZBGIZhGM0yCcIwDMNolkkQhmEYRrNMgjAMwzCaZRKEYRiG0az/D2YeXBXguREIAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 432x288 with 2 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7PlWMs8C_Qax",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "5acea667-7850-4cb1-c30e-c7660e542a63"
      },
      "source": [
        "model_cnn.evaluate(seq_subsets['X_test'], initial_subsets['Y_test'], batch_size=BATCH_SIZE)"
      ],
      "id": "7PlWMs8C_Qax",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "639/639 [==============================] - 13s 19ms/step - loss: 0.3805 - accuracy: 0.8462\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[0.3804706037044525, 0.84621262550354]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 24
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Fk1L0OvK8a62"
      },
      "source": [
        "## Model save to drive"
      ],
      "id": "Fk1L0OvK8a62"
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7AakfQrfCfaC",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "03df12ed-160c-430d-b48c-29ebbbb15d99"
      },
      "source": [
        "# save model and architecture to single file\n",
        "model.save(\"/content/drive/My Drive/deep-learning/rnn_lstm.h5\")\n",
        "model_cnn.save(\"/content/drive/My Drive/deep-learning/cnn_embd.h5\")\n",
        "print(\"Saved models to disk\")"
      ],
      "id": "7AakfQrfCfaC",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Saved models to disk\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "W7_OqnTMrqaR"
      },
      "source": [
        "##Multinomial Naive Bayes"
      ],
      "id": "W7_OqnTMrqaR"
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YbLOxmBnhkoW",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "13fb257f-9f65-483e-d804-e0b39b085fc4"
      },
      "source": [
        "from sklearn.pipeline import Pipeline\n",
        "from sklearn.naive_bayes import MultinomialNB\n",
        "from sklearn.feature_extraction.text import CountVectorizer, TfidfTransformer\n",
        "from sklearn.metrics import classification_report\n",
        "\n",
        "score='f1_macro'\n",
        "\n",
        "pipe = Pipeline([('vect', CountVectorizer(ngram_range=(1, 2))),\n",
        "                     ('tfidf', TfidfTransformer(norm='l1',use_idf=False)),\n",
        "                     ('clf', MultinomialNB(alpha=0.1))])\n",
        "\n",
        "pipe.fit(initial_subsets['X_train'], initial_subsets['Y_train'])"
      ],
      "id": "YbLOxmBnhkoW",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Pipeline(memory=None,\n",
              "         steps=[('vect',\n",
              "                 CountVectorizer(analyzer='word', binary=False,\n",
              "                                 decode_error='strict',\n",
              "                                 dtype=<class 'numpy.int64'>, encoding='utf-8',\n",
              "                                 input='content', lowercase=True, max_df=1.0,\n",
              "                                 max_features=None, min_df=1,\n",
              "                                 ngram_range=(1, 2), preprocessor=None,\n",
              "                                 stop_words=None, strip_accents=None,\n",
              "                                 token_pattern='(?u)\\\\b\\\\w\\\\w+\\\\b',\n",
              "                                 tokenizer=None, vocabulary=None)),\n",
              "                ('tfidf',\n",
              "                 TfidfTransformer(norm='l1', smooth_idf=True,\n",
              "                                  sublinear_tf=False, use_idf=False)),\n",
              "                ('clf',\n",
              "                 MultinomialNB(alpha=0.1, class_prior=None, fit_prior=True))],\n",
              "         verbose=False)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "H3N8Lgn5jdTr",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "66f0d102-ed9f-4625-c3db-56ff4b5f52a0"
      },
      "source": [
        "print(classification_report(initial_subsets['Y_test'], pipe.predict(initial_subsets['X_test']), digits=4))"
      ],
      "id": "H3N8Lgn5jdTr",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0     0.8440    0.8846    0.8638    159595\n",
            "           1     0.8788    0.8366    0.8572    159690\n",
            "\n",
            "    accuracy                         0.8606    319285\n",
            "   macro avg     0.8614    0.8606    0.8605    319285\n",
            "weighted avg     0.8614    0.8606    0.8605    319285\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "D78AM1fFt2ty",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "aeea1a1d-89c9-4378-f71a-c526f58c2c38"
      },
      "source": [
        "drive.flush_and_unmount()\n",
        "print('All changes made in this colab session should now be visible in Drive.')"
      ],
      "id": "D78AM1fFt2ty",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "All changes made in this colab session should now be visible in Drive.\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}