examples/colab_MovieLen1M_YoutubeDNN.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "rtox72csOQUN"
   },
   "source": [
    "# DeepMatch 样例代码\n",
    "- https://github.com/shenweichen/DeepMatch\n",
    "- https://deepmatch.readthedocs.io/en/latest/"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "bTWHz-heMkyw"
   },
   "source": [
    "# 下载movielens-1M数据 安装依赖包"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "yTl6d6jO1oqf",
    "outputId": "ee7303f1-8970-4726-a9f1-368798077228"
   },
   "outputs": [],
   "source": [
    "! wget http://files.grouplens.org/datasets/movielens/ml-1m.zip -O ./ml-1m.zip \n",
    "! wget https://raw.githubusercontent.com/shenweichen/DeepMatch/master/examples/preprocess.py -O preprocess.py\n",
    "! unzip -o ml-1m.zip \n",
    "! pip uninstall -y -q tensorflow\n",
    "! pip install -q tensorflow-gpu==2.5.0\n",
    "! pip install -q deepmatch"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "p9UxNHuPMuW2"
   },
   "source": [
    "# 导入需要的库"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "id": "C_ZR6gzp1E2N"
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from deepctr.feature_column import SparseFeat, VarLenSparseFeat\n",
    "from preprocess import gen_data_set, gen_model_input\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "from tensorflow.python.keras import backend as K\n",
    "from tensorflow.python.keras.models import Model\n",
    "\n",
    "from deepmatch.models import *\n",
    "from deepmatch.utils import sampledsoftmaxloss, NegativeSampler"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "fQq6O9XAMzPF"
   },
   "source": [
    "# 读取数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "lcO29zFb21Od",
    "outputId": "bfeed1ac-99f2-425f-dda6-10b83be721fe"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/swc/anaconda3/lib/python3.6/site-packages/ipykernel_launcher.py:4: ParserWarning: Falling back to the 'python' engine because the 'c' engine does not support regex separators (separators > 1 char and different from '\\s+' are interpreted as regex); you can avoid this warning by specifying engine='python'.\n",
      "  after removing the cwd from sys.path.\n",
      "/Users/swc/anaconda3/lib/python3.6/site-packages/ipykernel_launcher.py:6: ParserWarning: Falling back to the 'python' engine because the 'c' engine does not support regex separators (separators > 1 char and different from '\\s+' are interpreted as regex); you can avoid this warning by specifying engine='python'.\n",
      "  \n",
      "/Users/swc/anaconda3/lib/python3.6/site-packages/ipykernel_launcher.py:8: ParserWarning: Falling back to the 'python' engine because the 'c' engine does not support regex separators (separators > 1 char and different from '\\s+' are interpreted as regex); you can avoid this warning by specifying engine='python'.\n",
      "  \n"
     ]
    }
   ],
   "source": [
    "data_path = \"./\"\n",
    "\n",
    "unames = ['user_id','gender','age','occupation','zip']\n",
    "user = pd.read_csv(data_path+'ml-1m/users.dat',sep='::',header=None,names=unames)\n",
    "rnames = ['user_id','movie_id','rating','timestamp']\n",
    "ratings = pd.read_csv(data_path+'ml-1m/ratings.dat',sep='::',header=None,names=rnames)\n",
    "mnames = ['movie_id','title','genres']\n",
    "movies = pd.read_csv(data_path+'ml-1m/movies.dat',sep='::',header=None,names=mnames,encoding=\"unicode_escape\")\n",
    "movies['genres'] = list(map(lambda x: x.split('|')[0], movies['genres'].values))\n",
    "\n",
    "data = pd.merge(pd.merge(ratings,movies),user)#.iloc[:10000]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "L0yCWxQxM3se"
   },
   "source": [
    "# 构建特征列，训练模型，导出embedding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "BMOvk_de2ML3",
    "outputId": "962afe1c-d387-4345-861f-e9b974a0b495"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 6040/6040 [00:12<00:00, 488.35it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "8 8\n",
      "Train on 988129 samples\n",
      "Epoch 1/20\n",
      "988129/988129 [==============================] - 38s 39us/sample - loss: 5.6344\n",
      "Epoch 2/20\n",
      "988129/988129 [==============================] - 41s 41us/sample - loss: 4.6947\n",
      "Epoch 3/20\n",
      "988129/988129 [==============================] - 39s 39us/sample - loss: 4.4681\n",
      "Epoch 4/20\n",
      "988129/988129 [==============================] - 38s 38us/sample - loss: 4.3227\n",
      "Epoch 5/20\n",
      "988129/988129 [==============================] - 38s 38us/sample - loss: 4.2224\n",
      "Epoch 6/20\n",
      "988129/988129 [==============================] - 37s 37us/sample - loss: 4.1463\n",
      "Epoch 7/20\n",
      "988129/988129 [==============================] - 37s 37us/sample - loss: 4.0843\n",
      "Epoch 8/20\n",
      "988129/988129 [==============================] - 37s 38us/sample - loss: 4.0339\n",
      "Epoch 9/20\n",
      "988129/988129 [==============================] - 44s 44us/sample - loss: 3.9941\n",
      "Epoch 10/20\n",
      "988129/988129 [==============================] - 38s 38us/sample - loss: 3.9619\n",
      "Epoch 11/20\n",
      "988129/988129 [==============================] - 43s 43us/sample - loss: 3.9349\n",
      "Epoch 12/20\n",
      "988129/988129 [==============================] - 39s 39us/sample - loss: 3.9112\n",
      "Epoch 13/20\n",
      "988129/988129 [==============================] - 39s 39us/sample - loss: 3.8902\n",
      "Epoch 14/20\n",
      "988129/988129 [==============================] - 39s 39us/sample - loss: 3.8712\n",
      "Epoch 15/20\n",
      "988129/988129 [==============================] - 38s 38us/sample - loss: 3.8560\n",
      "Epoch 16/20\n",
      "988129/988129 [==============================] - 39s 40us/sample - loss: 3.8413\n",
      "Epoch 17/20\n",
      "988129/988129 [==============================] - 39s 39us/sample - loss: 3.8285\n",
      "Epoch 18/20\n",
      "988129/988129 [==============================] - 38s 38us/sample - loss: 3.8185\n",
      "Epoch 19/20\n",
      "988129/988129 [==============================] - 40s 40us/sample - loss: 3.8069\n",
      "Epoch 20/20\n",
      "988129/988129 [==============================] - 40s 41us/sample - loss: 3.7964\n",
      "WARNING:tensorflow:From /Users/swc/anaconda3/lib/python3.6/site-packages/tensorflow/python/keras/engine/training_v1.py:2070: Model.state_updates (from tensorflow.python.keras.engine.training) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "This property should not be used in TensorFlow 2.0, as updates are applied automatically.\n",
      "(6040, 32)\n",
      "(3706, 32)\n"
     ]
    }
   ],
   "source": [
    "#data = pd.read_csvdata = pd.read_csv(\"./movielens_sample.txt\")\n",
    "sparse_features = [\"movie_id\", \"user_id\",\n",
    "                    \"gender\", \"age\", \"occupation\", \"zip\", \"genres\"]\n",
    "SEQ_LEN = 50\n",
    "negsample = 0\n",
    "\n",
    "# 1.Label Encoding for sparse features,and process sequence features with `gen_date_set` and `gen_model_input`\n",
    "\n",
    "feature_max_idx = {}\n",
    "for feature in sparse_features:\n",
    "    lbe = LabelEncoder()\n",
    "    data[feature] = lbe.fit_transform(data[feature]) + 1\n",
    "    feature_max_idx[feature] = data[feature].max() + 1\n",
    "\n",
    "user_profile = data[[\"user_id\", \"gender\", \"age\", \"occupation\", \"zip\"]].drop_duplicates('user_id')\n",
    "\n",
    "item_profile = data[[\"movie_id\"]].drop_duplicates('movie_id')\n",
    "\n",
    "user_profile.set_index(\"user_id\", inplace=True)\n",
    "\n",
    "user_item_list = data.groupby(\"user_id\")['movie_id'].apply(list)\n",
    "\n",
    "train_set, test_set = gen_data_set(data, SEQ_LEN, negsample)\n",
    "\n",
    "train_model_input, train_label = gen_model_input(train_set, user_profile, SEQ_LEN)\n",
    "test_model_input, test_label = gen_model_input(test_set, user_profile, SEQ_LEN)\n",
    "\n",
    "# 2.count #unique features for each sparse field and generate feature config for sequence feature\n",
    "\n",
    "embedding_dim = 32\n",
    "\n",
    "user_feature_columns = [SparseFeat('user_id', feature_max_idx['user_id'], 16),\n",
    "                        SparseFeat(\"gender\", feature_max_idx['gender'], 16),\n",
    "                        SparseFeat(\"age\", feature_max_idx['age'], 16),\n",
    "                        SparseFeat(\"occupation\", feature_max_idx['occupation'], 16),\n",
    "                        SparseFeat(\"zip\", feature_max_idx['zip'], 16),\n",
    "                        VarLenSparseFeat(SparseFeat('hist_movie_id', feature_max_idx['movie_id'], embedding_dim,\n",
    "                                                    embedding_name=\"movie_id\"), SEQ_LEN, 'mean', 'hist_len'),\n",
    "                        VarLenSparseFeat(SparseFeat('hist_genres', feature_max_idx['genres'], embedding_dim,\n",
    "                                                   embedding_name=\"genres\"), SEQ_LEN, 'mean', 'hist_len'),\n",
    "                        ]\n",
    "\n",
    "item_feature_columns = [SparseFeat('movie_id', feature_max_idx['movie_id'], embedding_dim)]\n",
    "\n",
    "from collections import Counter\n",
    "train_counter = Counter(train_model_input['movie_id'])\n",
    "item_count = [train_counter.get(i,0) for i in range(item_feature_columns[0].vocabulary_size)]\n",
    "sampler_config = NegativeSampler('frequency',num_sampled=255,item_name=\"movie_id\",item_count=item_count)\n",
    "\n",
    "# 3.Define Model and train\n",
    "\n",
    "import tensorflow as tf\n",
    "if tf.__version__ >= '2.0.0':\n",
    "    tf.compat.v1.disable_eager_execution()\n",
    "else:\n",
    "    K.set_learning_phase(True)\n",
    "    \n",
    "model = YoutubeDNN(user_feature_columns, item_feature_columns, user_dnn_hidden_units=(128,64, embedding_dim), sampler_config=sampler_config)\n",
    "#model = MIND(user_feature_columns,item_feature_columns,dynamic_k=False,k_max=2, user_dnn_hidden_units=(128,64, embedding_dim), sampler_config=sampler_config)\n",
    "\n",
    "model.compile(optimizer=\"adam\", loss=sampledsoftmaxloss)\n",
    "\n",
    "history = model.fit(train_model_input, train_label,  # train_label,\n",
    "                    batch_size=512, epochs=20, verbose=1, validation_split=0.0, )\n",
    "\n",
    "# 4. Generate user features for testing and full item features for retrieval\n",
    "test_user_model_input = test_model_input\n",
    "all_item_model_input = {\"movie_id\": item_profile['movie_id'].values,}\n",
    "\n",
    "user_embedding_model = Model(inputs=model.user_input, outputs=model.user_embedding)\n",
    "item_embedding_model = Model(inputs=model.item_input, outputs=model.item_embedding)\n",
    "\n",
    "user_embs = user_embedding_model.predict(test_user_model_input, batch_size=2 ** 12)\n",
    "# user_embs = user_embs[:, i, :]  # i in [0,k_max) if MIND\n",
    "item_embs = item_embedding_model.predict(all_item_model_input, batch_size=2 ** 12)\n",
    "\n",
    "print(user_embs.shape)\n",
    "print(item_embs.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "w_G3KWslKmJo"
   },
   "source": [
    "# 使用faiss进行ANN查找并评估结果"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "5SvyQLNVKkcs"
   },
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "j2ZNYNBOOqrN",
    "outputId": "2eec5e82-2d2b-4fe0-9b83-2a74a4dc52ba"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
      "Requirement already satisfied: faiss-cpu in /usr/local/lib/python3.7/dist-packages (1.7.2)\n"
     ]
    }
   ],
   "source": [
    "! pip install faiss-cpu"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "6TY1l27iJU8U",
    "outputId": "5a8ccdd3-af70-4c48-b859-84c4befddfdd"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "6040it [00:02, 2769.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "recall 0.33708609271523177\n",
      "hit rate 0.33708609271523177\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "test_true_label = {line[0]:[line[1]] for line in test_set}\n",
    "\n",
    "import numpy as np\n",
    "import faiss\n",
    "from tqdm import tqdm\n",
    "from deepmatch.utils import recall_N\n",
    "\n",
    "index = faiss.IndexFlatIP(embedding_dim)\n",
    "# faiss.normalize_L2(item_embs)\n",
    "index.add(item_embs)\n",
    "# faiss.normalize_L2(user_embs)\n",
    "D, I = index.search(np.ascontiguousarray(user_embs), 50)\n",
    "s = []\n",
    "hit = 0\n",
    "for i, uid in tqdm(enumerate(test_user_model_input['user_id'])):\n",
    "    try:\n",
    "        pred = [item_profile['movie_id'].values[x] for x in I[i]]\n",
    "        filter_item = None\n",
    "        recall_score = recall_N(test_true_label[uid], pred, N=50)\n",
    "        s.append(recall_score)\n",
    "        if test_true_label[uid] in pred:\n",
    "            hit += 1\n",
    "    except:\n",
    "        print(i)\n",
    "print(\"\")\n",
    "print(\"recall\", np.mean(s))\n",
    "print(\"hit rate\", hit / len(test_user_model_input['user_id']))"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": "colab_MovieLen1M_YoutubeDNN.ipynb",
   "provenance": []
  },
  "gpuClass": "standard",
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}