Text data cleaning & preprocessing.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Text cleaning.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "19yO74Zy7ns1"
      },
      "source": [
        "import numpy as np\n",
        "import pandas as pd\n",
        "import re\n",
        "from sklearn.feature_extraction.text import TfidfVectorizer"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BDScGdiI80zm"
      },
      "source": [
        "class_names = ['toxic', 'severe_toxic', 'obscene', 'threat', 'insult', 'identity_hate']\n",
        "train = pd.read_csv('train.csv').fillna(' ')\n",
        "test = pd.read_csv('test.csv').fillna(' ')\n",
        "tr_ids = train[['id']]\n",
        "train[class_names] = train[class_names].astype(np.int8)\n",
        "target = train[class_names]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "P-hBoGiGFbie",
        "outputId": "064304b6-4e9e-439f-87a3-0e24a8dbf6e2"
      },
      "source": [
        "train.shape"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(159571, 8)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 22
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "0c-P_X2t9IXc",
        "outputId": "67cdbe83-2d5c-48f4-8c1c-6cd5f1b698ee"
      },
      "source": [
        "print(' Cleaning ...')\n",
        "# PREPROCESSING PART\n",
        "repl = {\n",
        "    \"yay!\": \" good \", \"yay\": \" good \", \"yaay\": \" good \", \"yaaay\": \" good \",\"yaaaay\": \" good \",\n",
        "    \"yaaaaay\": \" good \",    \":/\": \" bad \",    \":&gt;\": \" sad \",    \":')\": \" sad \",    \":-(\": \" frown \",\n",
        "    \":(\": \" frown \",    \":s\": \" frown \",    \":-s\": \" frown \",    \"&lt;3\": \" heart \",   \":d\": \" smile \",\n",
        "    \":p\": \" smile \",    \":dd\": \" smile \",    \"8)\": \" smile \",    \":-)\": \" smile \",    \":)\": \" smile \",\n",
        "    \";)\": \" smile \",    \"(-:\": \" smile \",    \"(:\": \" smile \",    \":/\": \" worry \",    \":&gt;\": \" angry \",\n",
        "    \":')\": \" sad \",    \":-(\": \" sad \",    \":(\": \" sad \",    \":s\": \" sad \",    \":-s\": \" sad \",\n",
        "    r\"\\br\\b\": \"are\",    r\"\\bu\\b\": \"you\",    r\"\\bhaha\\b\": \"ha\",    r\"\\bhahaha\\b\": \"ha\",\n",
        "    r\"\\bdon't\\b\": \"do not\",    r\"\\bdoesn't\\b\": \"does not\",    r\"\\bdidn't\\b\": \"did not\",\n",
        "    r\"\\bhasn't\\b\": \"has not\",    r\"\\bhaven't\\b\": \"have not\",    r\"\\bhadn't\\b\": \"had not\",\n",
        "    r\"\\bwon't\\b\": \"will not\",    r\"\\bwouldn't\\b\": \"would not\",    r\"\\bcan't\\b\": \"can not\",\n",
        "    r\"\\bcannot\\b\": \"can not\",    r\"\\bi'm\\b\": \"i am\",    \"m\": \"am\",    \"r\": \"are\",    \"u\": \"you\",\n",
        "    \"haha\": \"ha\",    \"hahaha\": \"ha\",    \"don't\": \"do not\",    \"doesn't\": \"does not\",\n",
        "    \"didn't\": \"did not\",    \"hasn't\": \"has not\",    \"haven't\": \"have not\",    \"hadn't\": \"had not\",\n",
        "    \"won't\": \"will not\",    \"wouldn't\": \"would not\",    \"can't\": \"can not\",    \"cannot\": \"can not\",\n",
        "    \"i'm\": \"i am\",    \"m\": \"am\",    \"i'll\" : \"i will\",    \"its\" : \"it is\",    \"it's\" : \"it is\",\n",
        "    \"'s\" : \" is\",    \"that's\" : \"that is\",    \"weren't\" : \"were not\",\n",
        "}\n",
        "\n",
        "keys = [i for i in repl.keys()]"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            " Cleaning ...\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WWKJDn97DvDY"
      },
      "source": [
        "def preprocessing(df,txt_col):\n",
        "  #replacing all keys by their values and removing any text after http or www\n",
        "  new_df = []\n",
        "  ltr = df[txt_col].tolist()\n",
        "  for i in ltr:\n",
        "    arr = str(i).split()\n",
        "    xx = \"\"\n",
        "    for j in arr:\n",
        "        j = str(j).lower()\n",
        "        if j[:4] == 'http' or j[:3] == 'www':\n",
        "            continue\n",
        "        if j in keys:\n",
        "            # print(\"inn\")\n",
        "            j = repl[j]\n",
        "        xx += j + \" \"\n",
        "    new_df.append(xx)\n",
        "\n",
        "  #removing everyting except alphabets\n",
        "  for i, c in enumerate(new_df):\n",
        "    new_df[i] = re.sub('[^a-zA-Z ?!]+', '', str(new_df[i]).lower())\n",
        "  df[txt_col] = new_df\n",
        "  return df\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DRuyn_5EE2yL"
      },
      "source": [
        "train_new=preprocessing(train,\"comment_text\")\n",
        "test_new=preprocessing(test,\"comment_text\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "siiPScXY9-Mt"
      },
      "source": [
        "train_text = train_new['comment_text']\n",
        "test_text = test_new['comment_text']\n",
        "all_text = pd.concat([train_text, test_text])\n",
        "\n",
        "print(' Part 1/2 of vectorizing ...')\n",
        "word_vectorizer = TfidfVectorizer(\n",
        "    sublinear_tf=True,\n",
        "    strip_accents='unicode',\n",
        "    analyzer='word',\n",
        "    token_pattern=r'\\w{1,}',\n",
        "    stop_words='english',\n",
        "    ngram_range=(1, 1),#boundary of the range of n-values for different n-grams to be extracted/default=(1, 1)\n",
        "    max_features=10000)\n",
        "word_vectorizer.fit(all_text)\n",
        "train_word_features = word_vectorizer.transform(train_text)\n",
        "test_word_features = word_vectorizer.transform(test_text)\n",
        "timer(train_time)\n",
        "\n",
        "train_time = timer(None)\n",
        "print(' Part 2/2 of vectorizing ...')\n",
        "char_vectorizer = TfidfVectorizer(\n",
        "    sublinear_tf=True, #apply sublinear tf scaling, i.e. replace tf with 1 + log(tf)\n",
        "    strip_accents='unicode',\n",
        "    analyzer='char',\n",
        "    stop_words='english',\n",
        "    ngram_range=(2, 6),\n",
        "    max_features=50000) #build a vocabulary that only consider the top max_features ordered by term frequency across the corpus.\n",
        "char_vectorizer.fit(all_text)\n",
        "train_char_features = char_vectorizer.transform(train_text)\n",
        "test_char_features = char_vectorizer.transform(test_text)\n",
        "timer(train_time)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "j3IlnQr8Bflk"
      },
      "source": [
        "#removing everyting except alphabets\n",
        "train[\"new_comment_text\"] = new_train_data\n",
        "trate = train[\"new_comment_text\"].tolist()\n",
        "for i, c in enumerate(trate):\n",
        "    trate[i] = re.sub('[^a-zA-Z ?!]+', '', str(trate[i]).lower())\n",
        "train[\"comment_text\"] = trate\n",
        "train.drop([\"new_comment_text\"], axis=1, inplace=True)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "cfdqyAyIAfn1",
        "outputId": "e963c7df-c4f2-42be-a141-6144a09e39d5"
      },
      "source": [
        "new_train_data[217]"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'and check this out '"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 29
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "lS3NP_OAAj4L",
        "outputId": "6d6ad507-77e7-4aae-e0dd-e876cc9b0d91"
      },
      "source": [
        "ltr[217]"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'And check this out: http://www.cla.purdue.edu/blackmon/102cs2001/critical.html#bio'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 30
        }
      ]
    }
  ]
}