hybrid.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from surprise import Dataset\n",
    "from surprise import Reader\n",
    "\n",
    "from surprise import Reader, Dataset, SVD\n",
    "from surprise.model_selection.validation import cross_validate\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "\n",
    "from sklearn.metrics.pairwise import cosine_similarity\n",
    "from scipy import sparse\n",
    "\n",
    "\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ISBN</th>\n",
       "      <th>Book-Title</th>\n",
       "      <th>Book-Author</th>\n",
       "      <th>Year-Of-Publication</th>\n",
       "      <th>Publisher</th>\n",
       "      <th>Image-URL-S</th>\n",
       "      <th>Image-URL-M</th>\n",
       "      <th>Image-URL-L</th>\n",
       "      <th>id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0195153448</td>\n",
       "      <td>Classical Mythology</td>\n",
       "      <td>Mark P. O. Morford</td>\n",
       "      <td>2002</td>\n",
       "      <td>Oxford University Press</td>\n",
       "      <td>http://images.amazon.com/images/P/0195153448.0...</td>\n",
       "      <td>http://images.amazon.com/images/P/0195153448.0...</td>\n",
       "      <td>http://images.amazon.com/images/P/0195153448.0...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0002005018</td>\n",
       "      <td>Clara Callan</td>\n",
       "      <td>Richard Bruce Wright</td>\n",
       "      <td>2001</td>\n",
       "      <td>HarperFlamingo Canada</td>\n",
       "      <td>http://images.amazon.com/images/P/0002005018.0...</td>\n",
       "      <td>http://images.amazon.com/images/P/0002005018.0...</td>\n",
       "      <td>http://images.amazon.com/images/P/0002005018.0...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0060973129</td>\n",
       "      <td>Decision in Normandy</td>\n",
       "      <td>Carlo D'Este</td>\n",
       "      <td>1991</td>\n",
       "      <td>HarperPerennial</td>\n",
       "      <td>http://images.amazon.com/images/P/0060973129.0...</td>\n",
       "      <td>http://images.amazon.com/images/P/0060973129.0...</td>\n",
       "      <td>http://images.amazon.com/images/P/0060973129.0...</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0374157065</td>\n",
       "      <td>Flu: The Story of the Great Influenza Pandemic...</td>\n",
       "      <td>Gina Bari Kolata</td>\n",
       "      <td>1999</td>\n",
       "      <td>Farrar Straus Giroux</td>\n",
       "      <td>http://images.amazon.com/images/P/0374157065.0...</td>\n",
       "      <td>http://images.amazon.com/images/P/0374157065.0...</td>\n",
       "      <td>http://images.amazon.com/images/P/0374157065.0...</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0393045218</td>\n",
       "      <td>The Mummies of Urumchi</td>\n",
       "      <td>E. J. W. Barber</td>\n",
       "      <td>1999</td>\n",
       "      <td>W. W. Norton &amp; Company</td>\n",
       "      <td>http://images.amazon.com/images/P/0393045218.0...</td>\n",
       "      <td>http://images.amazon.com/images/P/0393045218.0...</td>\n",
       "      <td>http://images.amazon.com/images/P/0393045218.0...</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0399135782</td>\n",
       "      <td>The Kitchen God's Wife</td>\n",
       "      <td>Amy Tan</td>\n",
       "      <td>1991</td>\n",
       "      <td>Putnam Pub Group</td>\n",
       "      <td>http://images.amazon.com/images/P/0399135782.0...</td>\n",
       "      <td>http://images.amazon.com/images/P/0399135782.0...</td>\n",
       "      <td>http://images.amazon.com/images/P/0399135782.0...</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>0425176428</td>\n",
       "      <td>What If?: The World's Foremost Military Histor...</td>\n",
       "      <td>Robert Cowley</td>\n",
       "      <td>2000</td>\n",
       "      <td>Berkley Publishing Group</td>\n",
       "      <td>http://images.amazon.com/images/P/0425176428.0...</td>\n",
       "      <td>http://images.amazon.com/images/P/0425176428.0...</td>\n",
       "      <td>http://images.amazon.com/images/P/0425176428.0...</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>0671870432</td>\n",
       "      <td>PLEADING GUILTY</td>\n",
       "      <td>Scott Turow</td>\n",
       "      <td>1993</td>\n",
       "      <td>Audioworks</td>\n",
       "      <td>http://images.amazon.com/images/P/0671870432.0...</td>\n",
       "      <td>http://images.amazon.com/images/P/0671870432.0...</td>\n",
       "      <td>http://images.amazon.com/images/P/0671870432.0...</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>0679425608</td>\n",
       "      <td>Under the Black Flag: The Romance and the Real...</td>\n",
       "      <td>David Cordingly</td>\n",
       "      <td>1996</td>\n",
       "      <td>Random House</td>\n",
       "      <td>http://images.amazon.com/images/P/0679425608.0...</td>\n",
       "      <td>http://images.amazon.com/images/P/0679425608.0...</td>\n",
       "      <td>http://images.amazon.com/images/P/0679425608.0...</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>074322678X</td>\n",
       "      <td>Where You'll Find Me: And Other Stories</td>\n",
       "      <td>Ann Beattie</td>\n",
       "      <td>2002</td>\n",
       "      <td>Scribner</td>\n",
       "      <td>http://images.amazon.com/images/P/074322678X.0...</td>\n",
       "      <td>http://images.amazon.com/images/P/074322678X.0...</td>\n",
       "      <td>http://images.amazon.com/images/P/074322678X.0...</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         ISBN                                         Book-Title  \\\n",
       "0  0195153448                                Classical Mythology   \n",
       "1  0002005018                                       Clara Callan   \n",
       "2  0060973129                               Decision in Normandy   \n",
       "3  0374157065  Flu: The Story of the Great Influenza Pandemic...   \n",
       "4  0393045218                             The Mummies of Urumchi   \n",
       "5  0399135782                             The Kitchen God's Wife   \n",
       "6  0425176428  What If?: The World's Foremost Military Histor...   \n",
       "7  0671870432                                    PLEADING GUILTY   \n",
       "8  0679425608  Under the Black Flag: The Romance and the Real...   \n",
       "9  074322678X            Where You'll Find Me: And Other Stories   \n",
       "\n",
       "            Book-Author  Year-Of-Publication                 Publisher  \\\n",
       "0    Mark P. O. Morford                 2002   Oxford University Press   \n",
       "1  Richard Bruce Wright                 2001     HarperFlamingo Canada   \n",
       "2          Carlo D'Este                 1991           HarperPerennial   \n",
       "3      Gina Bari Kolata                 1999      Farrar Straus Giroux   \n",
       "4       E. J. W. Barber                 1999    W. W. Norton & Company   \n",
       "5               Amy Tan                 1991          Putnam Pub Group   \n",
       "6         Robert Cowley                 2000  Berkley Publishing Group   \n",
       "7           Scott Turow                 1993                Audioworks   \n",
       "8       David Cordingly                 1996              Random House   \n",
       "9           Ann Beattie                 2002                  Scribner   \n",
       "\n",
       "                                         Image-URL-S  \\\n",
       "0  http://images.amazon.com/images/P/0195153448.0...   \n",
       "1  http://images.amazon.com/images/P/0002005018.0...   \n",
       "2  http://images.amazon.com/images/P/0060973129.0...   \n",
       "3  http://images.amazon.com/images/P/0374157065.0...   \n",
       "4  http://images.amazon.com/images/P/0393045218.0...   \n",
       "5  http://images.amazon.com/images/P/0399135782.0...   \n",
       "6  http://images.amazon.com/images/P/0425176428.0...   \n",
       "7  http://images.amazon.com/images/P/0671870432.0...   \n",
       "8  http://images.amazon.com/images/P/0679425608.0...   \n",
       "9  http://images.amazon.com/images/P/074322678X.0...   \n",
       "\n",
       "                                         Image-URL-M  \\\n",
       "0  http://images.amazon.com/images/P/0195153448.0...   \n",
       "1  http://images.amazon.com/images/P/0002005018.0...   \n",
       "2  http://images.amazon.com/images/P/0060973129.0...   \n",
       "3  http://images.amazon.com/images/P/0374157065.0...   \n",
       "4  http://images.amazon.com/images/P/0393045218.0...   \n",
       "5  http://images.amazon.com/images/P/0399135782.0...   \n",
       "6  http://images.amazon.com/images/P/0425176428.0...   \n",
       "7  http://images.amazon.com/images/P/0671870432.0...   \n",
       "8  http://images.amazon.com/images/P/0679425608.0...   \n",
       "9  http://images.amazon.com/images/P/074322678X.0...   \n",
       "\n",
       "                                         Image-URL-L  id  \n",
       "0  http://images.amazon.com/images/P/0195153448.0...   1  \n",
       "1  http://images.amazon.com/images/P/0002005018.0...   2  \n",
       "2  http://images.amazon.com/images/P/0060973129.0...   3  \n",
       "3  http://images.amazon.com/images/P/0374157065.0...   4  \n",
       "4  http://images.amazon.com/images/P/0393045218.0...   5  \n",
       "5  http://images.amazon.com/images/P/0399135782.0...   6  \n",
       "6  http://images.amazon.com/images/P/0425176428.0...   7  \n",
       "7  http://images.amazon.com/images/P/0671870432.0...   8  \n",
       "8  http://images.amazon.com/images/P/0679425608.0...   9  \n",
       "9  http://images.amazon.com/images/P/074322678X.0...  10  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_books = pd.read_csv('./ir_assignment_table_books.csv', encoding = \"utf-8\" )\n",
    "df_books.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "small_dataset = df_books['Book-Title'][0:50000]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "vectorizer = TfidfVectorizer(dtype=np.float32)\n",
    "X = vectorizer.fit_transform(small_dataset)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['10',\n",
       " '101',\n",
       " '10x18',\n",
       " '12',\n",
       " '13',\n",
       " '16',\n",
       " '18',\n",
       " '19',\n",
       " '1983',\n",
       " '1stbooks',\n",
       " '20',\n",
       " '2000',\n",
       " '21st',\n",
       " '22',\n",
       " '2nd',\n",
       " '300incredible',\n",
       " '3f',\n",
       " '3rd',\n",
       " '400',\n",
       " '61',\n",
       " '62',\n",
       " '84',\n",
       " 'aah',\n",
       " 'abacus',\n",
       " 'abbeville',\n",
       " 'abbey',\n",
       " 'abc',\n",
       " 'abdo',\n",
       " 'abel',\n",
       " 'aber',\n",
       " 'abingdon',\n",
       " 'able',\n",
       " 'about',\n",
       " 'abradale',\n",
       " 'abrams',\n",
       " 'absolute',\n",
       " 'abson',\n",
       " 'abstract',\n",
       " 'ac',\n",
       " 'acacia',\n",
       " 'academic',\n",
       " 'academy',\n",
       " 'acantilado',\n",
       " 'accelerated',\n",
       " 'accent',\n",
       " 'access',\n",
       " 'accion',\n",
       " 'acciã³n',\n",
       " 'acclaim',\n",
       " 'accord',\n",
       " 'ace',\n",
       " 'acmillan',\n",
       " 'acolyte',\n",
       " 'acropolis',\n",
       " 'actar',\n",
       " 'actes',\n",
       " 'activa',\n",
       " 'ad',\n",
       " 'adagio',\n",
       " 'adam',\n",
       " 'adams',\n",
       " 'addison',\n",
       " 'adelphi',\n",
       " 'admission',\n",
       " 'adobe',\n",
       " 'adult',\n",
       " 'advance',\n",
       " 'advancebooks',\n",
       " 'advanced',\n",
       " 'advancement',\n",
       " 'advantage',\n",
       " 'adventist',\n",
       " 'adventure',\n",
       " 'adventures',\n",
       " 'aegina',\n",
       " 'aerial',\n",
       " 'aerie',\n",
       " 'affairs',\n",
       " 'africa',\n",
       " 'ag',\n",
       " 'aga',\n",
       " 'agencja',\n",
       " 'aglow',\n",
       " 'aguilar',\n",
       " 'ai',\n",
       " 'aid',\n",
       " 'aim',\n",
       " 'aims',\n",
       " 'air',\n",
       " 'aird',\n",
       " 'airmont',\n",
       " 'aise',\n",
       " 'ait',\n",
       " 'ak',\n",
       " 'akal',\n",
       " 'akanos',\n",
       " 'akashic',\n",
       " 'akirim',\n",
       " 'akron',\n",
       " 'al',\n",
       " 'aladdin',\n",
       " 'alamo',\n",
       " 'alaska',\n",
       " 'alastair',\n",
       " 'alba',\n",
       " 'albert',\n",
       " 'albin',\n",
       " 'albury',\n",
       " 'alcoholics',\n",
       " 'alemany',\n",
       " 'alfaguara',\n",
       " 'alfred',\n",
       " 'algonquin',\n",
       " 'alianza',\n",
       " 'alice',\n",
       " 'alive',\n",
       " 'allen',\n",
       " 'alley',\n",
       " 'allia',\n",
       " 'alliance',\n",
       " 'allison',\n",
       " 'allyn',\n",
       " 'alma',\n",
       " 'almanac',\n",
       " 'alpha',\n",
       " 'alpine',\n",
       " 'altamira',\n",
       " 'altea',\n",
       " 'altera',\n",
       " 'alternate',\n",
       " 'alternative',\n",
       " 'alvim',\n",
       " 'alyson',\n",
       " 'amacom',\n",
       " 'amato',\n",
       " 'amber',\n",
       " 'amc',\n",
       " 'ameghino',\n",
       " 'amer',\n",
       " 'amereon',\n",
       " 'america',\n",
       " 'american',\n",
       " 'ami',\n",
       " 'amiel',\n",
       " 'amistad',\n",
       " 'ammann',\n",
       " 'amok',\n",
       " 'ams',\n",
       " 'amsco',\n",
       " 'amsterdam',\n",
       " 'amã',\n",
       " 'an',\n",
       " 'anabasi',\n",
       " 'anagrama',\n",
       " 'anansi',\n",
       " 'anatolian',\n",
       " 'anaya',\n",
       " 'ancestry',\n",
       " 'anchor',\n",
       " 'and',\n",
       " 'anderson',\n",
       " 'andre',\n",
       " 'andres',\n",
       " 'andrew',\n",
       " 'andrews',\n",
       " 'angel',\n",
       " 'angsana',\n",
       " 'angus',\n",
       " 'anhinga',\n",
       " 'animas',\n",
       " 'anita',\n",
       " 'annapolis',\n",
       " 'anne',\n",
       " 'anness',\n",
       " 'annick',\n",
       " 'anon',\n",
       " 'anonymous',\n",
       " 'another',\n",
       " 'anstalt',\n",
       " 'answers',\n",
       " 'anthony',\n",
       " 'antioch',\n",
       " 'anvil',\n",
       " 'aperture',\n",
       " 'aphelion',\n",
       " 'appalachian',\n",
       " 'appeal',\n",
       " 'applause',\n",
       " 'apple',\n",
       " 'appleton',\n",
       " 'applewood',\n",
       " 'applicable',\n",
       " 'appreciation',\n",
       " 'apress',\n",
       " 'apricot',\n",
       " 'aquarian',\n",
       " 'arango',\n",
       " 'arbeiderspers',\n",
       " 'arbor',\n",
       " 'arcade',\n",
       " 'arcadia',\n",
       " 'arche',\n",
       " 'archipel',\n",
       " 'archiv',\n",
       " 'archive',\n",
       " 'arco',\n",
       " 'arden',\n",
       " 'ardent',\n",
       " 'area',\n",
       " 'areheart',\n",
       " 'arena',\n",
       " 'arete',\n",
       " 'argentina',\n",
       " 'argon',\n",
       " 'ariel',\n",
       " 'aris',\n",
       " 'arizona',\n",
       " 'arkana',\n",
       " 'arkansas',\n",
       " 'arkham',\n",
       " 'arkhaven',\n",
       " 'arlington',\n",
       " 'arlã',\n",
       " 'armand',\n",
       " 'arnold',\n",
       " 'aro',\n",
       " 'aronson',\n",
       " 'around',\n",
       " 'arowhena',\n",
       " 'arrow',\n",
       " 'arrowhead',\n",
       " 'ars',\n",
       " 'arsenal',\n",
       " 'art',\n",
       " 'arte',\n",
       " 'artemis',\n",
       " 'arthur',\n",
       " 'artisan',\n",
       " 'arts',\n",
       " 'artscroll',\n",
       " 'artystyczne',\n",
       " 'as',\n",
       " 'asa',\n",
       " 'ascension',\n",
       " 'ash',\n",
       " 'ashgate',\n",
       " 'ashley',\n",
       " 'asia',\n",
       " 'asiapac',\n",
       " 'ask',\n",
       " 'aslan',\n",
       " 'aspect',\n",
       " 'aspen',\n",
       " 'assets',\n",
       " 'assn',\n",
       " 'assoc',\n",
       " 'associated',\n",
       " 'associates',\n",
       " 'associati',\n",
       " 'associatio',\n",
       " 'association',\n",
       " 'assã',\n",
       " 'asticou',\n",
       " 'astran',\n",
       " 'astro',\n",
       " 'astrologers',\n",
       " 'asylum',\n",
       " 'at',\n",
       " 'atalante',\n",
       " 'ateneo',\n",
       " 'atheneum',\n",
       " 'athenum',\n",
       " 'atlantic',\n",
       " 'atlantida',\n",
       " 'atlasbooks',\n",
       " 'atomium',\n",
       " 'atria',\n",
       " 'atrium',\n",
       " 'attack',\n",
       " 'attic',\n",
       " 'au',\n",
       " 'aubier',\n",
       " 'audio',\n",
       " 'audiobooks',\n",
       " 'audiolibros',\n",
       " 'audioworks',\n",
       " 'audubon',\n",
       " 'aufbau',\n",
       " 'augsburg',\n",
       " 'august',\n",
       " 'aulis',\n",
       " 'aum',\n",
       " 'aunt',\n",
       " 'aus',\n",
       " 'austin',\n",
       " 'australia',\n",
       " 'australian',\n",
       " 'authorhouse',\n",
       " 'authors',\n",
       " 'authorsomniscient',\n",
       " 'autodidactic',\n",
       " 'autrement',\n",
       " 'autumn',\n",
       " 'avail',\n",
       " 'available',\n",
       " 'avalon',\n",
       " 'ave',\n",
       " 'avenel',\n",
       " 'aventine',\n",
       " 'aventurine',\n",
       " 'avenue',\n",
       " 'avery',\n",
       " 'avian',\n",
       " 'avon',\n",
       " 'award',\n",
       " 'awe',\n",
       " 'ayer',\n",
       " 'az',\n",
       " 'babani',\n",
       " 'baby',\n",
       " 'bachrach',\n",
       " 'back',\n",
       " 'backbeat',\n",
       " 'backinprint',\n",
       " 'backyard',\n",
       " 'bacon',\n",
       " 'baen',\n",
       " 'baha',\n",
       " 'bahai',\n",
       " 'baker',\n",
       " 'bakker',\n",
       " 'baldini',\n",
       " 'baleine',\n",
       " 'ballantine',\n",
       " 'balnain',\n",
       " 'baltimore',\n",
       " 'bamboo',\n",
       " 'banff',\n",
       " 'banford',\n",
       " 'bank',\n",
       " 'banner',\n",
       " 'banta',\n",
       " 'bantam',\n",
       " 'bao',\n",
       " 'barbara',\n",
       " 'barbour',\n",
       " 'barcanova',\n",
       " 'barcelona',\n",
       " 'barclay',\n",
       " 'bard',\n",
       " 'barefoot',\n",
       " 'bargain',\n",
       " 'barn',\n",
       " 'barnes',\n",
       " 'barney',\n",
       " 'baronet',\n",
       " 'barral',\n",
       " 'barricade',\n",
       " 'barron',\n",
       " 'barrons',\n",
       " 'barth',\n",
       " 'bartlett',\n",
       " 'barton',\n",
       " 'base',\n",
       " 'basic',\n",
       " 'basilica',\n",
       " 'baskerville',\n",
       " 'bass',\n",
       " 'bassermann',\n",
       " 'bast',\n",
       " 'bastei',\n",
       " 'bathroom',\n",
       " 'batsford',\n",
       " 'bauer',\n",
       " 'bausch',\n",
       " 'bay',\n",
       " 'bayard',\n",
       " 'bbc',\n",
       " 'bbs',\n",
       " 'bdd',\n",
       " 'bdk',\n",
       " 'beacon',\n",
       " 'beall',\n",
       " 'bear',\n",
       " 'bears',\n",
       " 'beaufort',\n",
       " 'beaver',\n",
       " 'beazley',\n",
       " 'bec',\n",
       " 'beck',\n",
       " 'bed',\n",
       " 'bedford',\n",
       " 'bedrick',\n",
       " 'beech',\n",
       " 'beeler',\n",
       " 'beer',\n",
       " 'beginning',\n",
       " 'behavior',\n",
       " 'behrman',\n",
       " 'bei',\n",
       " 'being',\n",
       " 'belknap',\n",
       " 'bell',\n",
       " 'bellebooks',\n",
       " 'belletristik',\n",
       " 'bello',\n",
       " 'bellwether',\n",
       " 'beltz',\n",
       " 'ben',\n",
       " 'benbella',\n",
       " 'benchmark',\n",
       " 'bene',\n",
       " 'benedikt',\n",
       " 'benjamin',\n",
       " 'bentlight',\n",
       " 'benziger',\n",
       " 'berg',\n",
       " 'bergin',\n",
       " 'berita',\n",
       " 'berkeley',\n",
       " 'berkely',\n",
       " 'berkley',\n",
       " 'berkshire',\n",
       " 'berlin',\n",
       " 'berliner',\n",
       " 'berlitz',\n",
       " 'bernard',\n",
       " 'berrett',\n",
       " 'bertelsmann',\n",
       " 'bess',\n",
       " 'bet',\n",
       " 'bete',\n",
       " 'bethany',\n",
       " 'bethel',\n",
       " 'better',\n",
       " 'betterway',\n",
       " 'betty',\n",
       " 'between',\n",
       " 'betzel',\n",
       " 'bewrite',\n",
       " 'beyond',\n",
       " 'bezige',\n",
       " 'bgw',\n",
       " 'bhaktivedanta',\n",
       " 'bibelgesellschaft',\n",
       " 'bible',\n",
       " 'bibles',\n",
       " 'bibliographical',\n",
       " 'biblioteca',\n",
       " 'biblos',\n",
       " 'big',\n",
       " 'bij',\n",
       " 'billboard',\n",
       " 'billy',\n",
       " 'bindlach',\n",
       " 'biography',\n",
       " 'birch',\n",
       " 'bird',\n",
       " 'bison',\n",
       " 'biting',\n",
       " 'bk',\n",
       " 'bks',\n",
       " 'black',\n",
       " 'blackett',\n",
       " 'blackie',\n",
       " 'blackstone',\n",
       " 'blackwater',\n",
       " 'blackwell',\n",
       " 'blair',\n",
       " 'blake',\n",
       " 'blanvalet',\n",
       " 'blast',\n",
       " 'blessing',\n",
       " 'blind',\n",
       " 'bloat',\n",
       " 'bloodletting',\n",
       " 'bloomberg',\n",
       " 'bloomsbury',\n",
       " 'bloser',\n",
       " 'blue',\n",
       " 'bluedoor',\n",
       " 'bluehen',\n",
       " 'bluemoon',\n",
       " 'blume',\n",
       " 'blã',\n",
       " 'boa',\n",
       " 'board',\n",
       " 'bob',\n",
       " 'bobbs',\n",
       " 'bodley',\n",
       " 'boheme',\n",
       " 'bolinda',\n",
       " 'bombshelter',\n",
       " 'bompiani',\n",
       " 'bona',\n",
       " 'bonanza',\n",
       " 'bonifatius',\n",
       " 'bonzobucks',\n",
       " 'book',\n",
       " 'bookcassette',\n",
       " 'bookcraft',\n",
       " 'booket',\n",
       " 'bookking',\n",
       " 'booklines',\n",
       " 'booklocker',\n",
       " 'bookman',\n",
       " 'bookmark',\n",
       " 'bookmart',\n",
       " 'bookpartners',\n",
       " 'bookpeople',\n",
       " 'bookroom',\n",
       " 'books',\n",
       " 'booksource',\n",
       " 'bookspan',\n",
       " 'bookthrift',\n",
       " 'boon',\n",
       " 'bored',\n",
       " 'borgo',\n",
       " 'boru',\n",
       " 'boston',\n",
       " 'botanic',\n",
       " 'bottom',\n",
       " 'boucke',\n",
       " 'boulevard',\n",
       " 'bound',\n",
       " 'bounty',\n",
       " 'bouregy',\n",
       " 'bourin',\n",
       " 'bowker',\n",
       " 'bowling',\n",
       " 'boxtree',\n",
       " 'boyars',\n",
       " 'boydell',\n",
       " 'boyds',\n",
       " 'boynton',\n",
       " 'br',\n",
       " 'brace',\n",
       " 'bracken',\n",
       " 'bradley',\n",
       " 'bradt',\n",
       " 'bradygames',\n",
       " 'bragelonne',\n",
       " 'bragg',\n",
       " 'braguelonne',\n",
       " 'brainwaves',\n",
       " 'brandl',\n",
       " 'branson',\n",
       " 'brassey',\n",
       " 'brava',\n",
       " 'brazilian',\n",
       " 'braziller',\n",
       " 'brazos',\n",
       " 'breaks',\n",
       " 'breakwater',\n",
       " 'brealey',\n",
       " 'breaux',\n",
       " 'brett',\n",
       " 'brian',\n",
       " 'bridge',\n",
       " 'bridgewater',\n",
       " 'bright',\n",
       " 'brill',\n",
       " 'brilliance',\n",
       " 'brimax',\n",
       " 'bristol',\n",
       " 'britain',\n",
       " 'british',\n",
       " 'broadcasting',\n",
       " 'broadman',\n",
       " 'broadview',\n",
       " 'broadvision',\n",
       " 'broadway',\n",
       " 'broccoli',\n",
       " 'brockhampton',\n",
       " 'broken',\n",
       " 'brook',\n",
       " 'brooke',\n",
       " 'brooking',\n",
       " 'brookings',\n",
       " 'brooklyn',\n",
       " 'brooks',\n",
       " 'bros',\n",
       " 'brotherhood',\n",
       " 'brothers',\n",
       " 'brouwer',\n",
       " 'brown',\n",
       " 'bruguera',\n",
       " 'brunnen',\n",
       " 'brunner',\n",
       " 'bruno',\n",
       " 'bt',\n",
       " 'btb',\n",
       " 'buccaneer',\n",
       " 'bucherdienst',\n",
       " 'buchner',\n",
       " 'buchverlag',\n",
       " 'buchverlage',\n",
       " 'bueno',\n",
       " 'bulfinch',\n",
       " 'bull',\n",
       " 'bunbury',\n",
       " 'burdett',\n",
       " 'bureau',\n",
       " 'burford',\n",
       " 'burnham',\n",
       " 'bus',\n",
       " 'busby',\n",
       " 'business',\n",
       " 'bussiã',\n",
       " 'butterworth',\n",
       " 'butterworths',\n",
       " 'buy',\n",
       " 'by',\n",
       " 'byr',\n",
       " 'ca',\n",
       " 'cage',\n",
       " 'cairo',\n",
       " 'cal',\n",
       " 'cale',\n",
       " 'calendars',\n",
       " 'california',\n",
       " 'calpe',\n",
       " 'calyx',\n",
       " 'cambridge',\n",
       " 'camden',\n",
       " 'cameron',\n",
       " 'caminho',\n",
       " 'camino',\n",
       " 'camp',\n",
       " 'campana',\n",
       " 'campe',\n",
       " 'campo',\n",
       " 'camunia',\n",
       " 'can',\n",
       " 'canada',\n",
       " 'canadian',\n",
       " 'canaria',\n",
       " 'cancer',\n",
       " 'candlewick',\n",
       " 'canongate',\n",
       " 'cantwell',\n",
       " 'cantz',\n",
       " 'canvasback',\n",
       " 'canyon',\n",
       " 'cape',\n",
       " 'capo',\n",
       " 'capstone',\n",
       " 'capua',\n",
       " 'caralt',\n",
       " 'card',\n",
       " 'cardoza',\n",
       " 'care',\n",
       " 'career',\n",
       " 'careertrack',\n",
       " 'careware',\n",
       " 'carhil',\n",
       " 'carico',\n",
       " 'carl',\n",
       " 'carlsen',\n",
       " 'carlsons',\n",
       " 'carlton',\n",
       " 'carnegie',\n",
       " 'carol',\n",
       " 'carolina',\n",
       " 'carpet',\n",
       " 'carreg',\n",
       " 'carriã',\n",
       " 'carroll',\n",
       " 'carte',\n",
       " 'cartoon',\n",
       " 'cartwheel',\n",
       " 'casa',\n",
       " 'casablanca',\n",
       " 'cassell',\n",
       " 'casterman',\n",
       " 'castle',\n",
       " 'castoldi',\n",
       " 'castor',\n",
       " 'catalunya',\n",
       " 'catbird',\n",
       " 'catedra',\n",
       " 'catholic',\n",
       " 'cato',\n",
       " 'cattle',\n",
       " 'cave',\n",
       " 'caxton',\n",
       " 'cay',\n",
       " 'ccc',\n",
       " 'cds',\n",
       " 'ceac',\n",
       " 'cedar',\n",
       " 'cedco',\n",
       " 'celebrating',\n",
       " 'celebrity',\n",
       " 'celestial',\n",
       " 'cellar',\n",
       " 'cemetery',\n",
       " 'cenografix',\n",
       " 'center',\n",
       " 'central',\n",
       " 'centre',\n",
       " 'centro',\n",
       " 'centrum',\n",
       " 'century',\n",
       " 'cerf',\n",
       " 'cgi101',\n",
       " 'chain',\n",
       " 'chair',\n",
       " 'chambers',\n",
       " 'champion',\n",
       " 'champs',\n",
       " 'chancellor',\n",
       " 'chang',\n",
       " 'change',\n",
       " 'channing',\n",
       " 'chantecler',\n",
       " 'chapel',\n",
       " 'chaplaincy',\n",
       " 'chapter',\n",
       " 'chariot',\n",
       " 'charis',\n",
       " 'charisma',\n",
       " 'charles',\n",
       " 'charlesbridge',\n",
       " 'charlotte',\n",
       " 'charlton',\n",
       " 'charter',\n",
       " 'chase',\n",
       " 'chaterlys',\n",
       " 'chatto',\n",
       " 'checkerbee',\n",
       " 'checkerboard',\n",
       " 'cheers',\n",
       " 'chefs',\n",
       " 'chelsea',\n",
       " 'cherche',\n",
       " 'cherrea',\n",
       " 'chess',\n",
       " 'chest',\n",
       " 'chicago',\n",
       " 'chicken',\n",
       " 'child',\n",
       " 'children',\n",
       " 'childrens',\n",
       " 'childs',\n",
       " 'chilton',\n",
       " 'chipmunkapublishing',\n",
       " 'chivers',\n",
       " 'choice',\n",
       " 'christadelphian',\n",
       " 'christian',\n",
       " 'christianity',\n",
       " 'chronicle',\n",
       " 'chronik',\n",
       " 'chronimed',\n",
       " 'chrysalis',\n",
       " 'chã',\n",
       " 'cias',\n",
       " 'cinco',\n",
       " 'cinema',\n",
       " 'circe',\n",
       " 'circle',\n",
       " 'circlet',\n",
       " 'cisco',\n",
       " 'citadel',\n",
       " 'cite',\n",
       " 'cities',\n",
       " 'citizens',\n",
       " 'city',\n",
       " 'citã',\n",
       " 'civil',\n",
       " 'claassen',\n",
       " 'clad',\n",
       " 'clara',\n",
       " 'clare',\n",
       " 'claremont',\n",
       " 'clarion',\n",
       " 'clarity',\n",
       " 'clark',\n",
       " 'clarke',\n",
       " 'clarkson',\n",
       " 'clasicos',\n",
       " 'classic',\n",
       " 'classics',\n",
       " 'clear',\n",
       " 'clearbridge',\n",
       " 'clearing',\n",
       " 'cleis',\n",
       " 'client',\n",
       " 'cliff',\n",
       " 'cliffs',\n",
       " 'climent',\n",
       " 'clinescot',\n",
       " 'clio',\n",
       " 'clocktower',\n",
       " 'cloud',\n",
       " 'club',\n",
       " 'cn',\n",
       " 'co',\n",
       " 'coach',\n",
       " 'coast',\n",
       " 'coastal',\n",
       " 'coconut',\n",
       " 'coffee',\n",
       " 'cohen',\n",
       " 'cole',\n",
       " 'colihue',\n",
       " 'colin',\n",
       " 'colleccion',\n",
       " 'collectible',\n",
       " 'collectibles',\n",
       " 'collection',\n",
       " 'collector',\n",
       " 'collectors',\n",
       " 'college',\n",
       " 'collier',\n",
       " 'collin',\n",
       " 'collins',\n",
       " 'colonial',\n",
       " 'color',\n",
       " 'colorado',\n",
       " 'columbia',\n",
       " 'columna',\n",
       " 'com',\n",
       " 'combined',\n",
       " 'comercial',\n",
       " 'comet',\n",
       " 'comex',\n",
       " 'comics',\n",
       " 'comicsone',\n",
       " 'coming',\n",
       " 'common',\n",
       " 'commonwealth',\n",
       " 'communications',\n",
       " 'companhia',\n",
       " 'companies',\n",
       " 'company',\n",
       " 'compass',\n",
       " 'compcare',\n",
       " 'computer',\n",
       " 'comstock',\n",
       " 'conari',\n",
       " 'concerned',\n",
       " 'concordia',\n",
       " 'congressional',\n",
       " 'conjunction',\n",
       " 'connection',\n",
       " 'consejerã',\n",
       " 'conservancy',\n",
       " 'consortium',\n",
       " 'constable',\n",
       " 'consultants',\n",
       " 'consumer',\n",
       " 'consumers',\n",
       " 'contact',\n",
       " 'contemporary',\n",
       " 'context',\n",
       " 'continental',\n",
       " 'continuum',\n",
       " 'conversations',\n",
       " 'cook',\n",
       " 'cookbooks',\n",
       " 'cool',\n",
       " 'cooper',\n",
       " 'copeland',\n",
       " 'copernicus',\n",
       " 'copper',\n",
       " 'coral',\n",
       " 'corgi',\n",
       " 'coriolis',\n",
       " 'cork',\n",
       " 'cornell',\n",
       " 'cornerstone',\n",
       " 'coronet',\n",
       " 'corp',\n",
       " 'corporation',\n",
       " 'corps',\n",
       " 'correctional',\n",
       " 'corti',\n",
       " 'corto',\n",
       " 'corvina',\n",
       " 'corwin',\n",
       " 'coss',\n",
       " 'cosse',\n",
       " 'coteau',\n",
       " 'cotler',\n",
       " 'cotovia',\n",
       " 'cotta',\n",
       " 'council',\n",
       " 'counsel',\n",
       " 'counterpoint',\n",
       " 'country',\n",
       " 'countryman',\n",
       " 'countryside',\n",
       " 'county',\n",
       " 'coups',\n",
       " 'courage',\n",
       " 'course',\n",
       " 'court',\n",
       " 'courts',\n",
       " 'couverte',\n",
       " 'covenant',\n",
       " 'cow',\n",
       " 'coward',\n",
       " 'cowley',\n",
       " 'coyote',\n",
       " 'cpm',\n",
       " 'cq',\n",
       " 'crabtree',\n",
       " 'cracom',\n",
       " 'crafty',\n",
       " 'craig',\n",
       " 'crc',\n",
       " 'created',\n",
       " 'creating',\n",
       " 'creation',\n",
       " 'creations',\n",
       " 'creative',\n",
       " 'creek',\n",
       " 'crema',\n",
       " 'crescent',\n",
       " 'cricketsong',\n",
       " 'crime',\n",
       " 'crimeline',\n",
       " 'crippen',\n",
       " 'crisp',\n",
       " 'critica',\n",
       " 'critics',\n",
       " 'crocker',\n",
       " 'crofts',\n",
       " 'cross',\n",
       " 'crossing',\n",
       " 'crossroad',\n",
       " 'crossway',\n",
       " 'crowell',\n",
       " 'crown',\n",
       " 'crowood',\n",
       " 'cruise',\n",
       " 'crusade',\n",
       " 'crystal',\n",
       " 'ct',\n",
       " 'cudahy',\n",
       " 'cultura',\n",
       " 'cultural',\n",
       " 'culture',\n",
       " 'cumberland',\n",
       " 'cuny',\n",
       " 'currency',\n",
       " 'currents',\n",
       " 'curriculum',\n",
       " 'curve',\n",
       " 'custom',\n",
       " 'cy',\n",
       " 'cyberactive',\n",
       " 'cypress',\n",
       " 'cã',\n",
       " 'da',\n",
       " 'dacapo',\n",
       " 'daedalus',\n",
       " 'dafina',\n",
       " 'dagorno',\n",
       " 'dahlstrom',\n",
       " 'dakota',\n",
       " 'dalesman',\n",
       " 'dalkey',\n",
       " 'dalmatian',\n",
       " 'dance',\n",
       " 'dangles',\n",
       " 'daniel',\n",
       " 'dar',\n",
       " 'darby',\n",
       " 'dargaud',\n",
       " 'dark',\n",
       " 'das',\n",
       " 'data',\n",
       " 'daughters',\n",
       " 'david',\n",
       " 'davidson',\n",
       " 'davies',\n",
       " 'davis',\n",
       " 'daw',\n",
       " 'dawn',\n",
       " 'dawnsign',\n",
       " 'day',\n",
       " 'dc',\n",
       " 'dci',\n",
       " 'de',\n",
       " 'dead',\n",
       " 'deadwood',\n",
       " 'deaf',\n",
       " 'dearborn',\n",
       " 'debate',\n",
       " 'debols',\n",
       " 'dedalus',\n",
       " 'dee',\n",
       " 'deer',\n",
       " 'degli',\n",
       " 'deirdre',\n",
       " 'del',\n",
       " 'delacorte',\n",
       " ...]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vectorizer.get_feature_names()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(50000, 4067)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<50000x4067 sparse matrix of type '<class 'numpy.float32'>'\n",
       "\twith 107405 stored elements in Compressed Sparse Row format>"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1059244"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "matrix_size = X.data.nbytes + X.indptr.nbytes + X.indices.nbytes\n",
    "matrix_size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  (0, 1513)\t0.5775226\n",
      "  (0, 3523)\t0.5770055\n",
      "  (0, 1330)\t0.5775226\n"
     ]
    }
   ],
   "source": [
    "print(X[3])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4067"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(vectorizer.get_feature_names())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1718884596"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "similarity_matrix = cosine_similarity(X, dense_output=False)\n",
    "matrix_size = similarity_matrix.data.nbytes + similarity_matrix.indptr.nbytes + similarity_matrix.indices.nbytes\n",
    "matrix_size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "214835574"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "similarity_matrix.count_nonzero()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}