DIN_preprocessing_by_lilv.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import some shit\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import random\n",
    "import tensorflow as tf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# read json & drop column\n",
    "def to_df(file_path):\n",
    "    with open(file_path, \"r\")as f:\n",
    "        i, df = 0, {}\n",
    "        for line in f:\n",
    "            df[i] = eval(line)\n",
    "            i += 1\n",
    "        df = pd.DataFrame.from_dict(df, orient='index')\n",
    "        return df\n",
    "review_df = to_df('Deep-Interest-Network/reviews_Electronics_5.json')[['reviewerID', 'asin', 'unixReviewTime']]\n",
    "item_df = to_df('Deep-Interest-Network/meta_Electronics.json')[['asin', 'categories']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# fine-grain categories\n",
    "item_df['categories'] = item_df['categories'].map(lambda x: x[-1][-1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# items in review_df\n",
    "item_df = item_df[item_df['asin'].isin(review_df['asin'].unique())]\n",
    "item_df = item_df.reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# map id to integer\n",
    "def build_map(df, column):\n",
    "    key = sorted(df[column].unique().tolist())\n",
    "    map_rule = dict(zip(key, range(0, len(df))))\n",
    "    df[column] = df[column].map(lambda x: map_rule[x])\n",
    "    return map_rule, key\n",
    "map_rule_reviewerID, key_reviewerID= build_map(review_df, 'reviewerID')\n",
    "map_rule_asin, key_asin= build_map(item_df, 'asin')\n",
    "map_rule_categories, key_categories = build_map(item_df, 'categories')\n",
    "review_df['asin'] = review_df['asin'].map(lambda x: map_rule_asin[x])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# count some shit\n",
    "user_count, item_count, cate_count, example_count = \\\n",
    "len(map_rule_reviewerID),len(map_rule_asin),len(map_rule_categories),review_df.shape[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   asin  categories\n",
      "0     0         217\n",
      "1     1          47\n",
      "2     2         167\n",
      "3     3         206\n",
      "4     4         210\n"
     ]
    }
   ],
   "source": [
    "# categories ready\n",
    "item_df = item_df.sort_values(['asin']).reset_index(drop=True)\n",
    "cate_list = np.array(item_df['categories'], dtype=np.int32)\n",
    "print(item_df.head())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   reviewerID  asin  unixReviewTime\n",
      "0           0    75      1385337600\n",
      "1           1   890      1358035200\n",
      "2           2   643      1361750400\n",
      "3           3   168      1390003200\n",
      "4           4   533      1350086400\n"
     ]
    }
   ],
   "source": [
    "# reviews ready\n",
    "review_df = review_df.sort_values(by=['reviewerID', 'unixReviewTime']).reset_index(drop=True)\n",
    "print(review_df.head())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# build trainset and testset\n",
    "train_set, test_set = [], []\n",
    "for reviewerID, history in review_df.groupby('reviewerID'):\n",
    "    # define positive samples\n",
    "    positive_sample = history['asin'].tolist()\n",
    "    \n",
    "    # define negative samples\n",
    "    def negative_sampling():\n",
    "        negative_sample = positive_sample[0]\n",
    "        while negative_sample in positive_sample:\n",
    "            negative_sample = random.randint(0, len(item_df)-1)\n",
    "        return negative_sample\n",
    "    negative_sample = [negative_sampling() for i in range(len(positive_sample))]\n",
    "    \n",
    "    # build training set & testing set\n",
    "    for i in range(1, len(positive_sample)):\n",
    "        history = positive_sample[:i]\n",
    "        if i == len(positive_sample)-1:\n",
    "            label = (positive_sample[i], negative_sample[i])\n",
    "            test_set.append((reviewerID, history, label))\n",
    "        else:\n",
    "            train_set.append((reviewerID, history, positive_sample[i], 1))\n",
    "            train_set.append((reviewerID, history, negative_sample[i], 0))\n",
    "# shuffle the deck\n",
    "random.shuffle(train_set)\n",
    "random.shuffle(test_set)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3988\n",
      "3025\n"
     ]
    }
   ],
   "source": [
    "print(len(train_set))\n",
    "print(len(test_set))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# transform data records to data batches\n",
    "class InputBatchData(object):\n",
    "    \n",
    "    # define data, batch_size, epoch_size, idx\n",
    "    def __init__(self, data, batch_size):\n",
    "        self.data = data\n",
    "        self.batch_size = batch_size\n",
    "        self.epoch_size = len(self.data) // self.batch_size\n",
    "        if self.epoch_size * self.batch_size < len(self.data):\n",
    "            self.epoch_size += 1\n",
    "        else:\n",
    "            self.epoch_size\n",
    "        self.idx = 0\n",
    "        \n",
    "    # start iteration\n",
    "    def __iter__(self):\n",
    "        return self\n",
    "    \n",
    "    # next batch\n",
    "    def __next__(self):\n",
    "        \n",
    "        # conditions to stop iteration\n",
    "        if self.idx == self.epoch_size:\n",
    "            raise StopIteration\n",
    "            \n",
    "        # data records to tranform\n",
    "        start, end = self.idx * self.batch_size, min((self.idx + 1) * self.batch_size, len(self.data))\n",
    "        batch_data = self.data[start : end]\n",
    "        self.idx += 1\n",
    "        \n",
    "        # user_id, item_id, y, sample_len in batch\n",
    "        user_id, item_id, y, sample_len = [], [], [], []\n",
    "        for i in batch_data:\n",
    "            user_id.append(i[0])\n",
    "            item_id.append(i[2])\n",
    "            y.append(i[3])\n",
    "            sample_len.append(len(i[1]))\n",
    "            \n",
    "        # hist_i (user-item metrix) in batch\n",
    "        hist_i = np.zeros([len(batch_data), max(sample_len)], np.int64)\n",
    "        k = 0\n",
    "        for l in batch_data:\n",
    "            for j in range(len(l[1])):\n",
    "                hist_i[k][j] = l[1][j]\n",
    "            k += 1\n",
    "        \n",
    "        return self.idx, (user_id, item_id, y, hist_i, sample_len)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python (reco_pyspark)",
   "language": "python",
   "name": "reco_pyspark"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}