rating-gradient-descent.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import scipy\n",
    "from scipy import optimize as op\n",
    "import numba\n",
    "import pickle\n",
    "import random\n",
    "random.seed(12)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "similarity_files = ['all_similarities/director-similarity', 'all_similarities/genre-similarity', 'all_similarities/rating-similarity', 'all_similarities/tags-similarity']\n",
    "weights = np.ones(shape=(len(similarity_files,)), dtype='f2')\n",
    "movie_to_id = pickle.load(open('movie-id-map2.pkl', 'rb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "cm = pd.read_csv('final.csv')\n",
    "allmv = set(cm.movieId)\n",
    "del cm\n",
    "mymv = random.sample(allmv, 3000)\n",
    "inmv = random.sample(allmv, 10)\n",
    "del allmv\n",
    "del pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "adj_mat = calc_adj_mat(weights)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "@numba.jit(forceobj=True, parallel=True, fastmath=True)\n",
    "def calc_adj_mat(wts):\n",
    "    adj_mat_t = np.zeros((len(movie_to_id), len(movie_to_id)), dtype='<f2')\n",
    "    for param, weight in zip(similarity_files, wts):\n",
    "        npf = np.memmap(param+'.npy', mode='r', shape=adj_mat_t.shape, dtype='<f2')\n",
    "        adj_mat_t += npf * weight\n",
    "    return adj_mat_t"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "pagerank(mymv)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "@numba.jit(forceobj=True, fastmath=True, parallel=True)\n",
    "def pagerank(movies: list, adj_mat: np.array, epsilon:float = 1e-4, maxiterations:int = 1000):\n",
    "    inds = np.array([movie_to_id[i] for i in movies])\n",
    "    mat = adj_mat[np.vstack(inds), inds]\n",
    "    v = np.zeros(mat.shape[0], dtype='<f4')\n",
    "    oldv = v.copy()\n",
    "    v[0] = 1\n",
    "    i = 0\n",
    "    while np.amax(np.abs(oldv-v)) > epsilon and i < maxiterations:\n",
    "        oldv = v.copy()\n",
    "        v = np.matmul(v, mat)\n",
    "        v /= np.linalg.norm(v)\n",
    "        i += 1\n",
    "    pr = {movies[i]: v[i] for i in range(len(movies))}\n",
    "    return pr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "nm = similarity_files[0]\n",
    "with np.load(nm+'.npz', 'r') as npf:\n",
    "    spm = sp.lil_matrix(npf['arr_0'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(-1.134378077582987e-10, 1.2868136229008733e-20)"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gd(100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "def gd(start):\n",
    "    pt = start\n",
    "    while True:\n",
    "        y = f(pt)\n",
    "        ypre = f(pt-0.001)\n",
    "        ypost = f(pt+0.001)\n",
    "#         print(pt, end=' ')\n",
    "        if (y-ypre) / (ypost-y) < 0:\n",
    "            return pt, f(pt)\n",
    "        elif y-ypre > 0:\n",
    "            pt -= 0.001\n",
    "        else:\n",
    "            pt += 0.001"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "import math\n",
    "def f(x):\n",
    "    return x*x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "done director-similarity.npz\n",
      "done genre-similarity.npz\n",
      "done rating-similarity.npz\n"
     ]
    }
   ],
   "source": [
    "for param in similarity_files:\n",
    "    with np.load(param, 'r') as npf:\n",
    "        arr = npf['arr_0']\n",
    "        mm = np.memmap(param+'.npy', mode='w+', dtype='<f2', shape=arr.shape)\n",
    "        mm[:] = arr[:]\n",
    "        mm.flush()\n",
    "    print('done', param)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<ipython-input-29-23a33c611990>:1: RuntimeWarning: divide by zero encountered in reciprocal\n",
      "  np.reciprocal(np.arange(9.).reshape((3, 3)))/8\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[       inf, 0.125     , 0.0625    ],\n",
       "       [0.04166667, 0.03125   , 0.025     ],\n",
       "       [0.02083333, 0.01785714, 0.015625  ]])"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.reciprocal(np.arange(9.).reshape((3, 3)))/8"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<ipython-input-23-c7c693d779ee>:1: RuntimeWarning: divide by zero encountered in reciprocal\n",
      "  np.linalg.norm(np.triu(np.reciprocal(np.arange(9.).reshape((3, 3))), 1))\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "1.1357816691600546"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.linalg.norm(np.triu(np.reciprocal(np.arange(9.).reshape((3, 3))), 1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<ipython-input-4-b5993734f88f>:4: RuntimeWarning: divide by zero encountered in reciprocal\n",
      "  mm[:] = np.reciprocal(npf['arr_0'], dtype='<f2')\n",
      "<ipython-input-4-b5993734f88f>:5: RuntimeWarning: invalid value encountered in true_divide\n",
      "  mm[:] /= np.linalg.norm(np.triu(mm, 1))\n"
     ]
    }
   ],
   "source": [
    "for param in similarity_files:\n",
    "    with np.load(param + '.npz', 'r+') as npf:\n",
    "        mm = np.memmap(param+'.npy', mode='w+', dtype='<f2', shape=(23843, 23843))\n",
    "        mm[:] = np.reciprocal(npf['arr_0'], dtype='<f2')\n",
    "        mm[:] /= np.linalg.norm(np.triu(mm, 1))\n",
    "        mm.flush()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "zr = [x for x in range(25113) if np.sum(tsim[x:, x], dtype='f4') == 0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "with np.load('all_similarities/tags-similarity.npz', 'r') as npf:\n",
    "    tsim = npf['arr_0']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "with np.load('all_similarities/tags-similarity.npz', 'r+') as npf:\n",
    "        mm = np.memmap('all_similarities/tags-similarity.npy', mode='w+', dtype='<f2', shape=(23843, 23843))\n",
    "        mm[:] = np.reciprocal(npf['arr_0'], dtype='<f2')\n",
    "        mm[:] /= np.linalg.norm(np.triu(mm, 1))\n",
    "        mm.flush()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "with np.load('all_similarities/user_similarity.npz', 'r') as npf:\n",
    "    usim = npf['arr_0']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "@numba.jit(forceobj=True, cache=True, parallel=True, fastmath=True)\n",
    "def weightfn(x):\n",
    "    adj_mat = calc_adj_mat(x)\n",
    "    adj_mat -= usim\n",
    "    np.abs(adj_mat, out=adj_mat)\n",
    "    return adj_mat.sum(dtype='f4')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "522617400.0"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "weightfn(weights)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "op.optimize()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}