pre_processing.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 5,
  "metadata": {
    "kernelspec": {
      "name": "python3",
      "language": "python",
      "display_name": "Python 3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.8.3"
    },
    "colab": {
      "name": "pre-processing",
      "provenance": [],
      "collapsed_sections": [],
      "toc_visible": true
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "n7SB0y7HPnHJ"
      },
      "source": [
        "In this notebook we are loading the dataset, clean it by removing mentions, hashtags, URLs, punctuation, and numbers and we apply lematization."
      ],
      "id": "n7SB0y7HPnHJ"
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 224
        },
        "id": "po4C-niMCfZy",
        "outputId": "6ab14f36-8292-487f-dba9-0d87353168e7"
      },
      "source": [
        "import pandas as pd\n",
        "from google.colab import drive\n",
        "\n",
        "\n",
        "drive.mount('/content/drive')\n",
        "\n",
        "df_encoding = \"latin-1\"\n",
        "\n",
        "cols = ['sentiment','id','date','query_string','user','original_tweets']\n",
        "\n",
        "df = pd.read_csv(\"/content/drive/My Drive/deep-learning/training.1600000.processed.noemoticon.csv\", encoding=df_encoding, header=None, names=cols)\n",
        "\n",
        "df.drop(['id','date','query_string','user'],axis=1,inplace=True)\n",
        "\n",
        "df = df.sample(frac=1, replace=True, random_state=1)\n",
        "\n",
        "df.head()"
      ],
      "id": "po4C-niMCfZy",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>sentiment</th>\n",
              "      <th>original_tweets</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>128037</th>\n",
              "      <td>0</td>\n",
              "      <td>Oh really don't wanna be awake</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>491755</th>\n",
              "      <td>0</td>\n",
              "      <td>Trying to amuse my cousin. It's not working! a...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>470924</th>\n",
              "      <td>0</td>\n",
              "      <td>@JonasAustralia  i wanted to win! congrats to ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>491263</th>\n",
              "      <td>0</td>\n",
              "      <td>That's it!! I can't take it no more!! After su...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>836489</th>\n",
              "      <td>4</td>\n",
              "      <td>@beckybootsx i hope your not drinking alcohol!...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "        sentiment                                    original_tweets\n",
              "128037          0                    Oh really don't wanna be awake \n",
              "491755          0  Trying to amuse my cousin. It's not working! a...\n",
              "470924          0  @JonasAustralia  i wanted to win! congrats to ...\n",
              "491263          0  That's it!! I can't take it no more!! After su...\n",
              "836489          4  @beckybootsx i hope your not drinking alcohol!..."
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "3dNgf96PbRjp",
        "outputId": "e94b53ad-f3b0-4644-b89c-e25e5404e6d5"
      },
      "source": [
        "df['sentiment'].value_counts()"
      ],
      "id": "3dNgf96PbRjp",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "4    800352\n",
              "0    799648\n",
              "Name: sentiment, dtype: int64"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "f4j6vpRdCfZ5"
      },
      "source": [
        "## Define usefull functions\n"
      ],
      "id": "f4j6vpRdCfZ5"
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "zj1QR5v8CfZ6",
        "outputId": "4b91696e-1052-4828-f875-447e215ecf03"
      },
      "source": [
        "import nltk\n",
        "from nltk.stem import WordNetLemmatizer\n",
        "from nltk.corpus import stopwords \n",
        "import re\n",
        "import string\n",
        "\n",
        "nltk.download('stopwords')\n",
        "nltk.download('wordnet')\n",
        "nltk.download('punkt')\n",
        "\n",
        "lemmatizer = WordNetLemmatizer()\n",
        "\n",
        "# set stop words for english language\n",
        "stop_words = set(stopwords.words(\"english\"))\n",
        "\n",
        "# function to remove punctuation\n",
        "def remove_punctuations(text):\n",
        "    text = ''.join([c for c in text if c not in string.punctuation])\n",
        "    return text\n",
        "\n",
        "def remove_stopwords(text):\n",
        "    list_of_words = [word for word in text.split(' ') if word not in stop_words]\n",
        "    words_to_text = \" \".join(list_of_words)\n",
        "    return words_to_text\n",
        "\n",
        "def remove_numbers(text):\n",
        "    clear_text = ''.join([i for i in text if not i.isdigit()])\n",
        "    return clear_text\n",
        "\n",
        "def remove_single_chars(text):\n",
        "    text = ' '.join([w for w in text.split(' ') if len(w) > 1])\n",
        "    return text\n",
        "\n",
        "def do_lem(text):\n",
        "    text = ' '.join([lemmatizer.lemmatize(w) for w in text.split(' ')])\n",
        "    return text\n",
        "\n",
        "\n",
        "def clean_text(text):    \n",
        "    # regex dictionary\n",
        "    regex = {\n",
        "        \"urls\": r\"(?i)\\b((?:https?://|www\\d{0,3}[.]|[a-z0-9.\\-]+[.][a-z]{2,4}/)(?:[^\\s()<>]+|\\(([^\\s()<>]+|(\\([^\\s()<>]+\\)))*\\))+(?:\\(([^\\s()<>]+|(\\([^\\s()<>]+\\)))*\\)|[^\\s`!()\\[\\]{};:'\\\".,<>?«»“”‘’]))\",\n",
        "        \"mentions\": r\"@[A-Za-z0-9]+\",\n",
        "        \"hashtags\": r\"#[A-Za-z0-9]+\",\n",
        "        \"contains_@\": \"\\S*@\\S*\\s?\",\n",
        "        \"whitespaces\": \"\\s+\"\n",
        "    }\n",
        "    text = str(text).lower()\n",
        "    text = re.sub(regex['urls'], '', text)\n",
        "    text = re.sub(regex['mentions'], '', text)\n",
        "    text = re.sub(regex['contains_@'], '', text)\n",
        "    text = re.sub(regex['hashtags'], '', text)\n",
        "    #text = remove_stopwords(text) \n",
        "    text = remove_punctuations(text)\n",
        "    text = remove_numbers(text)\n",
        "    text = re.sub(regex['whitespaces'], ' ', text).strip()\n",
        "    #text = remove_single_chars(text)\n",
        "    text = do_lem(text)\n",
        "\n",
        "    return text"
      ],
      "id": "zj1QR5v8CfZ6",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[nltk_data] Downloading package stopwords to /root/nltk_data...\n",
            "[nltk_data]   Unzipping corpora/stopwords.zip.\n",
            "[nltk_data] Downloading package wordnet to /root/nltk_data...\n",
            "[nltk_data]   Unzipping corpora/wordnet.zip.\n",
            "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
            "[nltk_data]   Unzipping tokenizers/punkt.zip.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "bK7bhFPZCfZ8"
      },
      "source": [
        "## Data Cleaning"
      ],
      "id": "bK7bhFPZCfZ8"
    },
    {
      "cell_type": "code",
      "metadata": {
        "scrolled": false,
        "id": "dx8oU5ytCfZ9"
      },
      "source": [
        "# clean text\n",
        "df['tweets'] = df['original_tweets'].apply(clean_text)"
      ],
      "id": "dx8oU5ytCfZ9",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "-3_xLDduQNmz",
        "outputId": "fec46297-2363-461b-c6b2-397ae2887930"
      },
      "source": [
        "# transform labels to 0 for negative and 1 for positive\n",
        "df.sentiment = df.sentiment.replace({0: 0, 4: 1})\n",
        "\n",
        "df.head()"
      ],
      "id": "-3_xLDduQNmz",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>sentiment</th>\n",
              "      <th>original_tweets</th>\n",
              "      <th>tweets</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>128037</th>\n",
              "      <td>0</td>\n",
              "      <td>Oh really don't wanna be awake</td>\n",
              "      <td>oh really dont wanna be awake</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>491755</th>\n",
              "      <td>0</td>\n",
              "      <td>Trying to amuse my cousin. It's not working! a...</td>\n",
              "      <td>trying to amuse my cousin it not working and h...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>470924</th>\n",
              "      <td>0</td>\n",
              "      <td>@JonasAustralia  i wanted to win! congrats to ...</td>\n",
              "      <td>i wanted to win congrats to her anyways</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>491263</th>\n",
              "      <td>0</td>\n",
              "      <td>That's it!! I can't take it no more!! After su...</td>\n",
              "      <td>thats it i cant take it no more after summer s...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>836489</th>\n",
              "      <td>1</td>\n",
              "      <td>@beckybootsx i hope your not drinking alcohol!...</td>\n",
              "      <td>i hope your not drinking alcohol lol</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "        sentiment  ...                                             tweets\n",
              "128037          0  ...                      oh really dont wanna be awake\n",
              "491755          0  ...  trying to amuse my cousin it not working and h...\n",
              "470924          0  ...            i wanted to win congrats to her anyways\n",
              "491263          0  ...  thats it i cant take it no more after summer s...\n",
              "836489          1  ...               i hope your not drinking alcohol lol\n",
              "\n",
              "[5 rows x 3 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 15
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "C4S1Y1baCfZ9"
      },
      "source": [
        "# remove original tweets\n",
        "df.drop(['original_tweets'], axis=1, inplace=True)"
      ],
      "id": "C4S1Y1baCfZ9",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "54mc_gm4WHVk",
        "outputId": "b23e8a16-9743-4734-f93c-824b171d9e32"
      },
      "source": [
        "df.info()"
      ],
      "id": "54mc_gm4WHVk",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "<class 'pandas.core.frame.DataFrame'>\n",
            "Int64Index: 1600000 entries, 128037 to 1034136\n",
            "Data columns (total 2 columns):\n",
            " #   Column     Non-Null Count    Dtype \n",
            "---  ------     --------------    ----- \n",
            " 0   sentiment  1600000 non-null  int64 \n",
            " 1   tweets     1600000 non-null  object\n",
            "dtypes: int64(1), object(1)\n",
            "memory usage: 36.6+ MB\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "zNgsxBmmWyVg",
        "outputId": "6e83b6b4-4ec4-47c6-d7c5-eaf3c626652d"
      },
      "source": [
        "# remove tweets with 1 character or less\n",
        "mask = (df['tweets'].str.len() > 1)\n",
        "df = df.loc[mask]\n",
        "df['sentiment'].value_counts()"
      ],
      "id": "zNgsxBmmWyVg",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "1    798734\n",
              "0    798157\n",
              "Name: sentiment, dtype: int64"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 18
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "WkAhs6XGi0eL",
        "outputId": "0d92c84d-7b5a-4c7f-d970-e13dbb5258b6"
      },
      "source": [
        "# remove invalid tweets with more than 140 characters\n",
        "mask = (df['tweets'].str.len() < 141)\n",
        "df = df.loc[mask]\n",
        "df['sentiment'].value_counts()"
      ],
      "id": "WkAhs6XGi0eL",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "1    798456\n",
              "0    797970\n",
              "Name: sentiment, dtype: int64"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 19
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OpkcVV2BMWX_"
      },
      "source": [
        "#save the processed dataset\n",
        "df.to_csv('/content/drive/My Drive/deep-learning/cleaned-dataset.csv', index=False)"
      ],
      "id": "OpkcVV2BMWX_",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "D78AM1fFt2ty",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "3d01f101-1b3b-49e3-9477-50ef391b0303"
      },
      "source": [
        "drive.flush_and_unmount()\n",
        "print('All changes made in this colab session should now be visible in Drive.')"
      ],
      "id": "D78AM1fFt2ty",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "All changes made in this colab session should now be visible in Drive.\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}