Word2Vec.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "\n",
    "from __future__  import print_function\n",
    "import collections\n",
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import math\n",
    "import os\n",
    "import random\n",
    "import zipfile\n",
    "from matplotlib import pylab\n",
    "from six.moves import range\n",
    "from six.moves.urllib.request import urlretrieve\n",
    "from sklearn.manifold import TSNE\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Download the text8 data if necessary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found and verified text8.zip\n"
     ]
    }
   ],
   "source": [
    "url='http://mattmahoney.net/dc/'\n",
    "\n",
    "def maybe_download(filename,expected_bytes):\n",
    "    if not os.path.exists(filename):\n",
    "        filename, _ = urlretrieve(url + filename, filename)\n",
    "    statinfo = os.stat(filename)\n",
    "    if statinfo.st_size == expected_bytes:\n",
    "        print('Found and verified %s' % filename)\n",
    "    else:\n",
    "        print(statinfo.st_size)\n",
    "        raise Exception('Failed to verify ' + filename + '. Can you get to it with a browser?')\n",
    "    return filename\n",
    "filename = maybe_download('text8.zip', 31344016)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Data size 17005207\n"
     ]
    }
   ],
   "source": [
    "def read_data(filename):\n",
    "    \"\"\"Extract the first file enclosed in the zip as a list of words\"\"\"\n",
    "    with zipfile.ZipFile(filename) as f:\n",
    "        data=tf.compat.as_str(f.read(f.namelist()[0])).split()\n",
    "    return data\n",
    "words = read_data(filename)\n",
    "print('Data size %d' % len(words))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<b>Build the dict and replace rare words with UNK token</b>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Most common words (+UNK) [['UNK', 418391], ('the', 1061396), ('of', 593677), ('and', 416629), ('one', 411764)]\n",
      "Sample data [5239, 3084, 12, 6, 195, 2, 3137, 46, 59, 156]\n"
     ]
    }
   ],
   "source": [
    "vocabulary_size=50000\n",
    "\n",
    "def build_dataset(words):\n",
    "    count=[['UNK',-1]]\n",
    "    count.extend(collections.Counter(words).most_common(vocabulary_size-1))\n",
    "    dictionary={}\n",
    "    \n",
    "    for word,_ in count:\n",
    "        dictionary[word]=len(dictionary)\n",
    "    data=list()\n",
    "    unk_count=0\n",
    "    \n",
    "    for word in words:\n",
    "        if word in dictionary:\n",
    "            index=dictionary[word]\n",
    "        else:\n",
    "            index=0\n",
    "            unk_count = unk_count + 1\n",
    "        data.append(index)\n",
    "    count[0][1] = unk_count\n",
    "    reverse_dictionary = dict(zip(dictionary.values(), dictionary.keys())) \n",
    "    return data, count, dictionary, reverse_dictionary\n",
    "\n",
    "\n",
    "data, count, dictionary, reverse_dictionary = build_dataset(words)\n",
    "print('Most common words (+UNK)', count[:5])\n",
    "print('Sample data', data[:10])\n",
    "del words  # Hint to reduce memory.\n",
    "            "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Data: ['anarchism', 'originated', 'as', 'a', 'term', 'of', 'abuse', 'first']\n",
      "\n",
      "with num_skips = 2 and skip_window = 1:\n",
      "    batch: ['originated', 'originated', 'as', 'as', 'a', 'a', 'term', 'term']\n",
      "    labels: ['as', 'anarchism', 'originated', 'a', 'term', 'as', 'of', 'a']\n",
      "\n",
      "with num_skips = 4 and skip_window = 2:\n",
      "    batch: ['as', 'as', 'as', 'as', 'a', 'a', 'a', 'a']\n",
      "    labels: ['anarchism', 'originated', 'term', 'a', 'originated', 'of', 'as', 'term']\n"
     ]
    }
   ],
   "source": [
    "data_index=0\n",
    "\n",
    "def generate_batch(batch_size,num_skips,skip_window):\n",
    "    global data_index\n",
    "    \n",
    "    assert batch_size % num_skips == 0\n",
    "    assert num_skips <= 2* skip_window\n",
    "    \n",
    "    batch = np.ndarray(shape=(batch_size),dtype=np.int32)\n",
    "    labels = np.ndarray(shape=(batch_size,1),dtype=np.int32)\n",
    "    span = 2 * skip_window + 1\n",
    "    buffer=collections.deque(maxlen=span)\n",
    "    \n",
    "    for _ in range(span):\n",
    "        buffer.append(data[data_index])\n",
    "        data_index = (data_index+1) % len(data)\n",
    "    \n",
    "    for i in range(batch_size // num_skips):\n",
    "        target = skip_window\n",
    "        targets_to_avoid = [skip_window]\n",
    "        for j in range(num_skips):\n",
    "            while target in targets_to_avoid:\n",
    "                target = random.randint(0,span-1)\n",
    "            targets_to_avoid.append(target)\n",
    "            batch[i* num_skips + j] = buffer[skip_window]\n",
    "            labels[i*num_skips + j , 0] = buffer[target]\n",
    "        buffer.append(data[data_index])\n",
    "        data_index = (data_index+1) % len(data)\n",
    "    return batch,labels\n",
    "\n",
    "print('Data:', [reverse_dictionary[di] for di in data[:8]])\n",
    "for num_skips, skip_window in [(2, 1), (4, 2)]:\n",
    "    data_index = 0\n",
    "    batch, labels = generate_batch(batch_size=8, num_skips=num_skips, skip_window=skip_window)\n",
    "    print('\\nwith num_skips = %d and skip_window = %d:' % (num_skips, skip_window))\n",
    "    print('    batch:', [reverse_dictionary[bi] for bi in batch])\n",
    "    print('    labels:', [reverse_dictionary[li] for li in labels.reshape(8)])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "<b><font color='green'>Train a skip-gram model</b></font>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "batch_size=128\n",
    "embedding_size=128 # Dimensions of the embedding vec\n",
    "skip_window=1   # How many items to consider left and right\n",
    "num_skips=2  # how many times to reuse an input to generate labels\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}