naive bayes.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Naïve Bayes\n",
    "\n",
    "&nbsp;\n",
    "\n",
    "The second lesson of generative learning is Naïve Bayes. NB is natural language processing 101. It is very naïve and simple but powerful, quite similar to logistic regression. The core part is merely calculating posterior probability from Bayes' theorem. We obtain p(y=label|x) from p(x|y=label)\\*p(y=label)/p(x), where p(x) is written in the form of Σp(x|y=label)\\*p(y=label). The form is quite consistent with another generative learning, Gaussian Discriminant Analysis. \n",
    "\n",
    "Feel free to check GDA in the following link\n",
    "\n",
    "https://github.com/je-suis-tm/machine-learning/blob/master/gaussian%20discriminant%20analysis.ipynb\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import nltk.tokenize\n",
    "import nltk.corpus\n",
    "import nltk.stem\n",
    "import sklearn.model_selection\n",
    "import sklearn.naive_bayes\n",
    "import sklearn.feature_extraction.text\n",
    "import scipy.special\n",
    "import scipy.sparse\n",
    "os.chdir('K:\\ecole\\github')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "#convert text into a list of words\n",
    "#we can use stemming and lemmatization to improve efficiency\n",
    "#for instance, we have words walked,walking,walks\n",
    "#with nltk package, we can revert all of them to walk\n",
    "def text2list(text,stopword,lower=True,lemma=False,stemma=False):\n",
    "\n",
    "    text_clean=text if lower==False else text.lower()\n",
    "    \n",
    "    #tokenize and remove stop words\n",
    "    token=[i for i in nltk.tokenize.RegexpTokenizer(r'\\w+').tokenize(text_clean) if i not in stopword]\n",
    "    \n",
    "    #lemmatization\n",
    "    if lemma:\n",
    "        text_processed=[nltk.stem.wordnet.WordNetLemmatizer().lemmatize(i) for i in token]\n",
    "    else:\n",
    "        text_processed=token\n",
    "        \n",
    "    #stemming\n",
    "    if stemma:\n",
    "        output=[nltk.stem.PorterStemmer().stem(i) for i in text_processed]\n",
    "    else:\n",
    "        output=text_processed\n",
    "    \n",
    "    #remove numbers as they are stopword as well\n",
    "    for i in [ii for ii in output]:\n",
    "        try:\n",
    "            float(i)\n",
    "            output.remove(i)\n",
    "        except:\n",
    "            pass\n",
    "    \n",
    "    return [i for i in output if i not in stopword]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "#this is a function to calculate top 30 frequent words\n",
    "#it would help us to determine which of those are stop words\n",
    "#stop words refer to the words we dont take in account for naïve bayes\n",
    "#such as you, i, and, of, the, an\n",
    "#which are inevitable in both spam and ham\n",
    "#it is a stop word helper\n",
    "#we still need to filter out the words that dont belong in this list manually\n",
    "#for each task, stop word may be different\n",
    "#e.g. if every text you need to classify has a prefix like '[Confidential]'\n",
    "#confidential becomes a stop word to be excluded\n",
    "def get_stopword(output):\n",
    "    \n",
    "    vocabulary=sorted(list(set(output)))\n",
    "    \n",
    "    d={}\n",
    "    stopword=[]\n",
    "    \n",
    "    for i in vocabulary:\n",
    "        d[i]=output.count(i)\n",
    "    \n",
    "    #get top 30 frequent words\n",
    "    for j in sorted(list(d.values()))[::-1][:30]:\n",
    "        word=list(d.keys())[list(d.values()).index(j)]\n",
    "        stopword.append(word)\n",
    "        del d[word]\n",
    "        \n",
    "    return stopword\n",
    "\n",
    "\n",
    "#there is another way to find stop words called kl divergence\n",
    "#it calculates the distance from one distribution to another\n",
    "#in layman’s terms, it gives us the impact of each word on classifications\n",
    "#you can check Wikipedia for more details\n",
    "# https://en.wikipedia.org/wiki/Kullback%E2%80%93Leibler_divergence\n",
    "#it is also called mutual information in machine learning\n",
    "#the actual formula is kl(p(x:y)||p(x)p(y))\n",
    "#we want the divergence from xy joint distribution to xy independent distribution\n",
    "#in scipy, u can use scipy.stats.entropy\n",
    "#in sklearn, u can use sklearn.feature_selection.mutual_info_classif\n",
    "def kl_distance(df):\n",
    "\n",
    "    #concatenate all words together\n",
    "    corpus=set([j for i in df['word'] for j in i])\n",
    "\n",
    "    #probability of y is fixed\n",
    "    #they are calculated before everything\n",
    "    py0=len(df[df['spam']==0])/len(df)\n",
    "    py1=len(df[df['spam']==1])/len(df)\n",
    "\n",
    "    #calculate kl distance for each word\n",
    "    kldict=[]\n",
    "    for word in set(corpus):\n",
    "\n",
    "        kldistance=0\n",
    "\n",
    "        countx=[]\n",
    "\n",
    "        #to make our life easier\n",
    "        #we take multivariate approach\n",
    "        #we dont calculate the frequency of a word\n",
    "        for j in range(len(df)):\n",
    "            if df['word'][j].count(word)>0:\n",
    "                countx.append(1)\n",
    "            else:\n",
    "                countx.append(0)\n",
    "\n",
    "        #using dataframe even though the performance is slower\n",
    "        #it is easier to do slicing by logical expression\n",
    "        data=pd.DataFrame()\n",
    "        data['x']=countx\n",
    "        data['y']=df['spam']\n",
    "\n",
    "        px0=len(data[data['x']==0])/len(data)\n",
    "        px1=len(data[data['x']==1])/len(data)\n",
    "\n",
    "        #using loops to do sum\n",
    "        for k in range(2):\n",
    "            for l in range(2):\n",
    "                prob_joint=len(data[data['y']==k][data['x']==l])/len(data)\n",
    "\n",
    "                probx=px0 if l==0 else px1\n",
    "                proby=py0 if k==0 else py1\n",
    "\n",
    "                #we need to avoid 2 traps\n",
    "                #zero division and logarithm zero\n",
    "                try:\n",
    "                    entropy=prob_joint*np.log(prob_joint/(probx*proby))\n",
    "                except ZeroDivisionError:\n",
    "                    entropy=0\n",
    "\n",
    "                if prob_joint==0:\n",
    "                    entropy=0\n",
    "\n",
    "                kldistance+=entropy\n",
    "\n",
    "        kldict+=[word,kldistance]\n",
    "\n",
    "    #transform into dataframe and sort by kl distance in descending order\n",
    "    output=pd.DataFrame()\n",
    "    output['word']=kldict[0::2]\n",
    "    output['kl distance']=kldict[1::2]\n",
    "\n",
    "    output.sort_values(by='kl distance',ascending=False,inplace=True)\n",
    "\n",
    "    output.reset_index(inplace=True,drop=True)\n",
    "\n",
    "    return output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "stopword=nltk.corpus.stopwords.words('english')+['u',\n",
    " 'beyond',\n",
    " 'within',\n",
    " 'around',\n",
    " 'would',\n",
    " 'b',\n",
    " 'c',\n",
    " 'e',\n",
    " 'f',\n",
    " 'g',\n",
    " 'h',\n",
    " 'j',\n",
    " 'k',\n",
    " 'l',\n",
    " 'n',\n",
    " 'p',\n",
    " 'q',\n",
    " 'r',\n",
    " 'u',\n",
    " 'v',\n",
    " 'w',\n",
    " 'x',\n",
    " 'z',\n",
    " 'first']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "#denote vocabulary as the list of all the words in english theoretically\n",
    "#in reality, it is not feasible and economic to do so\n",
    "#here, we just collect all words from all emails\n",
    "#we gotta filter out those stop words as well\n",
    "def get_vocabulary(output,stopword):\n",
    "    \n",
    "    vocabulary=sorted(list(set(output)))\n",
    "    \n",
    "    for i in vocabulary:\n",
    "        if i in stopword:\n",
    "            vocabulary.remove(i)\n",
    "        \n",
    "    return vocabulary    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "#using box cox transformation to shift the distribution mean\n",
    "#not recommended for naïve bayes\n",
    "# https://en.wikipedia.org/wiki/Power_transform#Box%E2%80%93Cox_transformation\n",
    "def box_cox(x,power):\n",
    "       \n",
    "    if power==0:\n",
    "        return np.log(x)\n",
    "    else:\n",
    "        return (x**power-1)/power"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "#calculate p(x|y=classification) for multivariate naïve bayes\n",
    "def multivariate_calc_prob(word,x_train,y_train,classification):\n",
    "    \n",
    "    #how many spam or ham from all emails\n",
    "    num=list(y_train).count(classification)\n",
    "    \n",
    "    #check how many emails contain the given word\n",
    "    corpus=[i.count(word) for i in x_train[y_train==classification]]\n",
    "    freq=len(corpus)-corpus.count(0)\n",
    "    \n",
    "    #calculate p(x|y=classification)\n",
    "    if freq!=0:\n",
    "        p=freq/num\n",
    "        \n",
    "    else:\n",
    "        \n",
    "        #laplace smoothing\n",
    "        #when the given word hasnt appeared in our training set yet\n",
    "        #we gotta avoid the scenario 0/num\n",
    "        #even if the given word has never appeared\n",
    "        #it doesnt indicate it wont appear in the future\n",
    "        #therefore, we gotta use laplace smoothing\n",
    "        #see the following link for more details\n",
    "        # https://en.wikipedia.org/wiki/Additive_smoothing\n",
    "        \n",
    "        #add 1 to the numerator and k to the denominator\n",
    "        #where k is the dimension of x\n",
    "        #in another word, how many possible values x can take\n",
    "        #in binary classification, k=2\n",
    "        #x either exists in the email or it doesnt\n",
    "        p=(freq+1)/(num+2)\n",
    "\n",
    "    return p\n",
    "\n",
    "\n",
    "#multivariate event model follows a bernoulli distribution\n",
    "#each word has two scenario\n",
    "#it either appears in the email or not\n",
    "def multivariate(sample,stopword):\n",
    "    \n",
    "    x_train,x_test,y_train,y_test= \\\n",
    "    sklearn.model_selection.train_test_split(sample['word'],sample['spam'],test_size=0.3)\n",
    "    \n",
    "    #calculate p(y)\n",
    "    phi_y0=list(y_train).count(0)/len(y_train)\n",
    "    phi_y1=1-phi_y0\n",
    "    \n",
    "    forecast=[]\n",
    "    \n",
    "    #assume all x are independent\n",
    "    #each word has its own probability of occurrence in emails\n",
    "    #the probability of occurrence of those words are not correlated\n",
    "    #hence, p(x|y) for all x can be written as\n",
    "    #the product of all probabilities of x\n",
    "    #which is called chain rule in bayesian network\n",
    "    #see the following link for details\n",
    "    # https://en.wikipedia.org/wiki/Chain_rule_%28probability%29\n",
    "    for i in x_test:\n",
    "        px_y0,px_y1=1,1\n",
    "        for j in i:\n",
    "            if j not in stopword:\n",
    "                px_y0*=multivariate_calc_prob(j,x_train,y_train,0)\n",
    "                px_y1*=multivariate_calc_prob(j,x_train,y_train,1)\n",
    "            else:\n",
    "                pass\n",
    "        \n",
    "        #calculate p(y|x)\n",
    "        py0_x=px_y0*phi_y0\n",
    "        py1_x=px_y1*phi_y1\n",
    "        \n",
    "        #compare p(y=1|x) with p(y=0|x)\n",
    "        #we take the larger one as for all generative learning algo\n",
    "        #if the probabilities of spam and ham are equal\n",
    "        #we d rather see it in inbox folder instead of spam folder\n",
    "        p=0 if py0_x>=py1_x else 1\n",
    "        forecast.append(p)\n",
    "        \n",
    "    return len(y_test[forecast==y_test])/len(y_test)*100\n",
    "\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "#calculate p(x|y) for multinomial naïve bayes\n",
    "def multinomial_calc_prob(word,x_train,y_train,\n",
    "                          classification,vocabulary,tfidf=False,\n",
    "                          boxcox=False,boxcox_param=None):\n",
    "    \n",
    "    #get the word count of all spam/ham emails\n",
    "    num=sum([len(i) for i in x_train[y_train==classification]])\n",
    "       \n",
    "    #get the frequency of given word in all spam/ham emails\n",
    "    freq=sum([i.count(word) for i in x_train[y_train==classification]])\n",
    "        \n",
    "    #boxcox transformation\n",
    "    #not recommended for naïve bayes\n",
    "    if boxcox:\n",
    "        num=box_cox(num,boxcox_param)\n",
    "        freq=box_cox(freq,boxcox_param)\n",
    "\n",
    "    if freq!=0:\n",
    "        p=freq/num\n",
    "        \n",
    "    else:\n",
    "        \n",
    "        #laplace smoothing \n",
    "        #here x can take on v possible values\n",
    "        #where v is the length of vocabulary list\n",
    "        #we are assuming vocabulary list covers every word in english\n",
    "        p=(freq+1)/(num+len(vocabulary))\n",
    "        \n",
    "    #tfidf is not recommended for naïve bayes\n",
    "    if tfidf:\n",
    "        \n",
    "        tf=p\n",
    "        \n",
    "        #number of emails of spam/ham that contain the given word\n",
    "        idf_den=len([i for i in x_train[y_train==classification] if word in i])\n",
    "        \n",
    "        #number of emails of spam/ham\n",
    "        idf_num=len(x_train[y_train==classification])\n",
    "        \n",
    "        #laplace smoothing\n",
    "        if idf_den==0:\n",
    "            idf_num+=1\n",
    "            idf_den+=1\n",
    "        \n",
    "        return tf*np.log(idf_num/idf_den)\n",
    "        \n",
    "    else:\n",
    "\n",
    "        return p\n",
    "\n",
    "\n",
    "#unlike multivariate, multinomial event model follows a multinomial distribution\n",
    "#the frequency of each word is taken into consideration\n",
    "#the formula is pretty much the same as multivariate\n",
    "#except p(x|y) is different\n",
    "def multinomial(sample,stopword,tfidf=False,\n",
    "                boxcox=False,boxcox_param=None):\n",
    "    \n",
    "    x_train,x_test,y_train,y_test= \\\n",
    "    sklearn.model_selection.train_test_split(sample['word'],sample['spam'],test_size=0.3)\n",
    "    \n",
    "    vocabulary=[]\n",
    "    for i in sample['word']:\n",
    "        corpus=get_vocabulary(i,stopword)\n",
    "        vocabulary+=corpus\n",
    "    \n",
    "    phi_y0=list(y_train).count(0)/len(y_train)\n",
    "    phi_y1=1-phi_y0\n",
    "    \n",
    "    forecast=[]\n",
    "    \n",
    "    for i in x_test:\n",
    "        px_y0,px_y1=1,1\n",
    "        for j in i:\n",
    "            if j not in stopword:\n",
    "                px_y0*=multinomial_calc_prob(j,x_train,y_train,\n",
    "                                             0,vocabulary,tfidf,\n",
    "                                             boxcox,boxcox_param)\n",
    "                px_y1*=multinomial_calc_prob(j,x_train,y_train,\n",
    "                                             1,vocabulary,tfidf,\n",
    "                                             boxcox,boxcox_param)\n",
    "            else:\n",
    "                pass\n",
    "            \n",
    "        py0_x=px_y0*phi_y0\n",
    "        py1_x=px_y1*phi_y1\n",
    "        \n",
    "        p=0 if py0_x>=py1_x else 1\n",
    "        forecast.append(p)\n",
    "        \n",
    "    return len(y_test[forecast==y_test])/len(y_test)*100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "#create multivariate vocabulary vector for sklearn\n",
    "#this is the authenticate way in naïve bayes\n",
    "#but using pandas is more convenient\n",
    "#which is why our own implementation is kinda different\n",
    "def multivariate_vector(x,y,vocabulary):\n",
    "    \n",
    "    #CRUCIAL!!!!\n",
    "    #train_test_split would shuffle the training set\n",
    "    #the shuffled index would cause a problem later\n",
    "    x.reset_index(inplace=True,drop=True)\n",
    "    y.reset_index(inplace=True,drop=True)\n",
    "    \n",
    "    #we create a m*n matrix\n",
    "    #where m denotes the number of emails(rows)\n",
    "    #n denotes the number of words in vocabulary(columns)\n",
    "    #the value denotes if word[n] appears in email[m]\n",
    "    #if so, we set the value to 1\n",
    "    df_multivariate=pd.DataFrame()\n",
    "    for i in vocabulary:\n",
    "        document_term_matrix=[]\n",
    "        for j in x.index:\n",
    "            if i in x[j]:\n",
    "                document_term_matrix.append(1)\n",
    "            else:\n",
    "                document_term_matrix.append(0)\n",
    "                \n",
    "        df_multivariate[i]=document_term_matrix\n",
    "        \n",
    "    df_multivariate['real y']=y\n",
    "    \n",
    "    return df_multivariate\n",
    "\n",
    "\n",
    "#create multinomial vocabulary vector for sklearn\n",
    "def multinomial_vector(x,y,vocabulary):\n",
    "    \n",
    "    #CRUCIAL!!!!!\n",
    "    x.reset_index(inplace=True,drop=True)\n",
    "    y.reset_index(inplace=True,drop=True)\n",
    "    \n",
    "    #we create a m*n matrix\n",
    "    #where m denotes the number of emails(rows)\n",
    "    #n denotes the number of words in vocabulary(columns)\n",
    "    #the value denotes the frequency of word[n] in email[m]\n",
    "    df_multinomial=pd.DataFrame()\n",
    "    for i in vocabulary:\n",
    "        document_term_matrix=[]\n",
    "        for j in x.index:\n",
    "            document_term_matrix.append(x[j].count(i))\n",
    "                \n",
    "        df_multinomial[i]=document_term_matrix\n",
    "        \n",
    "    df_multinomial['real y']=y\n",
    "    \n",
    "    return df_multinomial\n",
    "\n",
    "\n",
    "#using the official sklearn package\n",
    "def sklearn_off(sample,stopword,method,nbmethod):\n",
    "    \n",
    "    vocabulary=[]\n",
    "    for i in sample['word']:\n",
    "        corpus=get_vocabulary(i,stopword)\n",
    "        vocabulary+=corpus\n",
    "    \n",
    "    x_train,x_test,y_train,y_test= \\\n",
    "    sklearn.model_selection.train_test_split(sample['word'],sample['spam'],test_size=0.3)\n",
    "\n",
    "    mn_train=method(x_train,y_train,vocabulary)\n",
    "    mn_test=method(x_test,y_test,vocabulary)\n",
    "    \n",
    "    model=nbmethod().fit(pd.concat([mn_train[i] for i in vocabulary], \\\n",
    "                                      axis=1),mn_train['real y'])\n",
    "    \n",
    "    forecast=model.predict(pd.concat([mn_test[i] for i in vocabulary], \\\n",
    "                                      axis=1))\n",
    "    \n",
    "    return len(y_test[forecast==y_test])/len(y_test)*100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "#quick bernoulli\n",
    "def sklearn_ber(sample):\n",
    "\n",
    "    x_train,x_test,y_train,y_test= \\\n",
    "    sklearn.model_selection.train_test_split(sample['word'],sample['spam'],test_size=0.3)\n",
    "    \n",
    "    #create vocabulary\n",
    "    vec=sklearn.feature_extraction.text.CountVectorizer(binary=True)\n",
    "    vec.fit(x_train.apply(lambda x:' '.join(x)))\n",
    "    matrix=vec.transform(x_train.apply(lambda x:' '.join(x)))\n",
    "    \n",
    "    #train\n",
    "    clf=sklearn.naive_bayes.BernoulliNB()\n",
    "    clf.fit(matrix,y_train)\n",
    "\n",
    "    #using the same vocabulary\n",
    "    matrix=vec.transform(x_test.apply(lambda x:' '.join(x)))\n",
    "\n",
    "    return clf.score(matrix,y_test)*100\n",
    "\n",
    "\n",
    "#quick multinomial\n",
    "def sklearn_mul(sample,vecmethod='default',\n",
    "                boxcox=False,boxcox_param=None):\n",
    "\n",
    "    x_train,x_test,y_train,y_test= \\\n",
    "    sklearn.model_selection.train_test_split(sample['word'],sample['spam'],test_size=0.3)\n",
    "    \n",
    "    #using tfidf is not recommended for naïve bayes\n",
    "    #more details about tfidf can be checked from the link\n",
    "    # http://tfidf.com/\n",
    "    #\n",
    "    if vecmethod=='default':\n",
    "        vec=sklearn.feature_extraction.text.CountVectorizer()\n",
    "    else:\n",
    "        vec=sklearn.feature_extraction.text.TfidfVectorizer()\n",
    "        \n",
    "    #create vocabulary\n",
    "    vec.fit(x_train.apply(lambda x:' '.join(x)))\n",
    "    matrix=vec.transform(x_train.apply(lambda x:' '.join(x)))\n",
    "    \n",
    "    #using box cox transformation\n",
    "    #not recommended for naïve bayes\n",
    "    if boxcox:\n",
    "        dense=scipy.special.boxcox(matrix.todense(),boxcox_param)\n",
    "        matrix=scipy.sparse.csr_matrix(dense,matrix.shape)\n",
    "    \n",
    "    #train\n",
    "    clf=sklearn.naive_bayes.MultinomialNB()\n",
    "    clf.fit(matrix,y_train)\n",
    "    \n",
    "    #using the same vocabulary\n",
    "    matrix=vec.transform(x_test.apply(lambda x:' '.join(x)))\n",
    "\n",
    "    return clf.score(matrix,y_test)*100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "#the raw data really comes from my email\n",
    "df=pd.read_csv('spam.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "#tokenization\n",
    "tokens=[]\n",
    "for i in df['text'].tolist():\n",
    "    tokens.append(text2list(i,stopword,lower=True,lemma=True))\n",
    "            \n",
    "df['word']=tokens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['settlement', 'please', 'class', 'google', 'data', 'service', 'new', 'information', 'privacy', 'fund', 'term', 'app', 'payment', 'may', 'member', 'policy', 'account', 'trip', 'update', 'claim', 'use', 'application', 'oculus', 'reddit', 'time', 'agreement', 'com', 'court', 'updated', 'check']\n"
     ]
    }
   ],
   "source": [
    "#to review stopword\n",
    "print(get_stopword([j for i in tokens for j in i]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:79: UserWarning: Boolean Series key will be reindexed to match DataFrame index.\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:87: RuntimeWarning: divide by zero encountered in log\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:87: RuntimeWarning: invalid value encountered in double_scalars\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              word  kl distance\n",
      "0           office     0.188020\n",
      "1             team     0.155811\n",
      "2          updated     0.132911\n",
      "3        following     0.126903\n",
      "4          payment     0.126903\n",
      "5           invite     0.126903\n",
      "6           family     0.126903\n",
      "7        organized     0.126903\n",
      "8             love     0.126903\n",
      "9           number     0.126903\n",
      "10         privacy     0.111192\n",
      "11           thank     0.104636\n",
      "12          change     0.104636\n",
      "13            note     0.102175\n",
      "14        everyone     0.098905\n",
      "15       organizer     0.098905\n",
      "16          friend     0.098905\n",
      "17         neither     0.098905\n",
      "18           quick     0.098905\n",
      "19      eventbrite     0.098905\n",
      "20         gabriel     0.098905\n",
      "21        organize     0.098905\n",
      "22          amount     0.098905\n",
      "23           might     0.098905\n",
      "24       traveling     0.098905\n",
      "25       attending     0.098905\n",
      "26         student     0.098905\n",
      "27            name     0.098905\n",
      "28              uk     0.098905\n",
      "29             pay     0.098905\n",
      "...            ...          ...\n",
      "1500        filled     0.000366\n",
      "1501          apps     0.000366\n",
      "1502       attempt     0.000366\n",
      "1503        submit     0.000366\n",
      "1504        morgan     0.000366\n",
      "1505          play     0.000366\n",
      "1506          site     0.000366\n",
      "1507          easy     0.000366\n",
      "1508           pro     0.000366\n",
      "1509         enjoy     0.000366\n",
      "1510        device     0.000366\n",
      "1511          firm     0.000366\n",
      "1512       request     0.000366\n",
      "1513      attached     0.000366\n",
      "1514       january     0.000366\n",
      "1515       nothing     0.000366\n",
      "1516  confirmation     0.000366\n",
      "1517      enhanced     0.000366\n",
      "1518       finally     0.000366\n",
      "1519       clarity     0.000366\n",
      "1520        refund     0.000366\n",
      "1521        entire     0.000366\n",
      "1522         third     0.000366\n",
      "1523            io     0.000366\n",
      "1524       holiday     0.000366\n",
      "1525        modern     0.000366\n",
      "1526    allocation     0.000366\n",
      "1527          plus     0.000366\n",
      "1528         reset     0.000366\n",
      "1529        access     0.000123\n",
      "\n",
      "[1530 rows x 2 columns]\n"
     ]
    }
   ],
   "source": [
    "#to check each word's contribution to the result\n",
    "print(kl_distance(df))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "#apart from using vocabulary\n",
    "#we can also use len(text) and % of punctuations as features\n",
    "\n",
    "#try ten times and compare the result of self implementation and sklearn\n",
    "accuracies=[]\n",
    "for i in range(10):\n",
    "        \n",
    "    accuracies.append(multivariate(df,stopword))\n",
    "    accuracies.append(multinomial(df,stopword))\n",
    "    accuracies.append(sklearn_ber(df))\n",
    "    accuracies.append(sklearn_mul(df))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "multivariate implementation accuracy: 74.54545454545456%\n",
      "multinomial implementation accuracy: 81.81818181818183%\n",
      "multivariate sklearn accuracy: 70.9090909090909%\n",
      "multinomial sklearn accuracy: 85.45454545454547%\n"
     ]
    }
   ],
   "source": [
    "for j,k in enumerate(['multivariate implementation','multinomial implementation',\n",
    "                        'multivariate sklearn','multinomial sklearn']):\n",
    "    print('{} accuracy: {}%'.format(k,np.mean(accuracies[j::4])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "#surprisingly the accuracy of my implementation is actually higher than sklearn\n",
    "#but it is a huge tradeoff for time\n",
    "#i would definitely use sklearn instead\n",
    "#btw, my sample size is very small\n",
    "#the result is definitely biased\n",
    "#it heavily depends on how train and test sets are split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}