gutenburg_bookshelf_scrap.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Metadata\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- link for sublime like multiedit: https://stackoverflow.com/questions/41553806/atom-sublime-like-multiple-selections-in-jupyter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 193,
   "metadata": {},
   "outputs": [],
   "source": [
    "import time\n",
    "import requests\n",
    "import json\n",
    "from bs4 import BeautifulSoup as bs\n",
    "from tqdm import tqdm_notebook as tn\n",
    "from lxml import html\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 194,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "200\n"
     ]
    }
   ],
   "source": [
    "web_link='http://www.gutenberg.org/ebooks/bookshelf/'\n",
    "gutenburg_data=requests.get(web_link)\n",
    "print(gutenburg_data.status_code)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 195,
   "metadata": {},
   "outputs": [],
   "source": [
    "soup=bs(gutenburg_data.text,features='html.parser')\n",
    "#print(soup.prettify())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### functions for making urls clickalbe and short(bitly)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- bitly_api import error solution: https://github.com/bitly/bitly-api-python/issues/39"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 196,
   "metadata": {},
   "outputs": [],
   "source": [
    "def make_clickable(val):\n",
    "    # target _blank to open new window\n",
    "    if val!='table header: no link':\n",
    "        return '<a target=\"_blank\" href=\"{}\">{}</a>'.format(val, val) # target blank given to open link in new tab\n",
    "    else:\n",
    "        return val\n",
    "\n",
    "BITLY_ACCESS_TOKEN =input('give your bitly token:')\n",
    "def shortner(url): # refer to above markdown link\n",
    "    paras={\n",
    "        'longurl':url,\n",
    "        'access_token':BITLY_ACCESS_TOKEN\n",
    "    }\n",
    "    \n",
    "    endpoint = 'https://api-ssl.bitly.com/v3/shorten'\n",
    "    r=requests.get(endpoint,params=paras,verify=True) #gives not verifies error and 1000 links for free dahhh\n",
    "    \n",
    "    data=r.json()\n",
    "    \n",
    "    if not r.status_code==200:\n",
    "        logger.error(\"Unexpected status_code: {} in bitly response. {}\".format(data['status_code'], r.text))\n",
    "    \n",
    "    return make_clickable(data['data']['url']) # also making shortened links clickable"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Creating df for home page PG main bookshelves"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 197,
   "metadata": {},
   "outputs": [],
   "source": [
    "## main bookshelves\n",
    "main_bookshelve_names=[]\n",
    "main_bookshelve_links=[]\n",
    "for link in soup.find('div',{'class':'bookshelves'}).find_all('a'):\n",
    "    main_bookshelve_links.append('https://www.gutenberg.org'+link.get('href'))\n",
    "    main_bookshelve_names.append(link.string)\n",
    "    \n",
    "main_bookshelve_df=pd.DataFrame({'Title':main_bookshelve_names,'Link':main_bookshelve_links})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 198,
   "metadata": {},
   "outputs": [],
   "source": [
    "main_bookshelve_df=main_bookshelve_df.style.format({'Link':shortner})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 199,
   "metadata": {},
   "outputs": [],
   "source": [
    "#main_bookshelve_df.to_csv('gutenburg-bookshelves-title-link(bitly).csv') # it tkaes long time as there are 22 links so sve it into csvfile"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Creating df for home page PG sub bookshelves"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 200,
   "metadata": {},
   "outputs": [],
   "source": [
    "bookshelves_pages_links=[]\n",
    "bookshelves_pages_titles=[]\n",
    "for link in soup.find('div',{'class':'bookshelf_pages'}).find_all('a'):\n",
    "    bookshelves_pages_links.append('https://www.gutenberg.org'+link.get('href'))\n",
    "    bookshelves_pages_titles.append(link.get('title'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 201,
   "metadata": {},
   "outputs": [],
   "source": [
    "bookshelves_pages_df=pd.DataFrame({'Title':bookshelves_pages_titles,'Link':bookshelves_pages_links})\n",
    "\n",
    "def make_clickable(val):\n",
    "    # target _blank to open new window\n",
    "    if val!='table header: no link':\n",
    "        return '<a target=\"_blank\" href=\"{}\">{}</a>'.format(val, val) # target blank given to open link in new tab\n",
    "    else:\n",
    "        return val\n",
    "\n",
    "bookshelves_pages_df=bookshelves_pages_df.style.format({'Link':make_clickable})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Creating df for all PG books"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 202,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "How many bookshelves pages do you want to list? 5\n",
      "1 )   Bookshelve --> 6 best loved spanish literary classics <--is being processed.......\n",
      "Completed 👏 in 8.9 seconds.\n",
      "2 )   Bookshelve --> adventure <--is being processed.......\n",
      "Completed 👏 in 12.1 seconds.\n",
      "3 )   Bookshelve --> africa <--is being processed.......\n",
      "Completed 👏 in 8.6 seconds.\n",
      "4 )   Bookshelve --> african american writers <--is being processed.......\n",
      "Completed 👏 in 8.0 seconds.\n",
      "5 )   Bookshelve --> ainslee's <--is being processed.......\n",
      "Completed 👏 in 3.5 seconds.\n"
     ]
    }
   ],
   "source": [
    "no_of_bookshelves_pages=input('How many bookshelves pages do you want to list? ')\n",
    "bookshelves_page=[]\n",
    "book_name=[]\n",
    "book_link=[]\n",
    "book_cover_link=[]\n",
    "author=[]\n",
    "downloads_count=[]\n",
    "for i in range(len(bookshelves_pages_links[:int(no_of_bookshelves_pages)])):\n",
    "    start_time=time.time()\n",
    "    print(i+1,')   Bookshelve -->',bookshelves_pages_titles[i].lower(),'<-- is being processed.......')\n",
    "    page_link=bookshelves_pages_links[i]\n",
    "    page_data=requests.get(page_link)\n",
    "    page_soup=bs(page_data.text,features='html.parser')\n",
    "    ## this loop needs to go into a function\n",
    "    ## for good so that it becomes a bit more short and compact\n",
    "    for book in page_soup.find_all('li',{'class':'booklink'}):\n",
    "        bookshelves_page.append(bookshelves_pages_titles[i])\n",
    "        book_link.append('https://www.gutenberg.org'+book.find('a').get('href'))\n",
    "        if book.find('img') != None:\n",
    "            book_cover_link.append('https://www.gutenberg.org'+book.find('img').get('src'))\n",
    "        else:\n",
    "            book_cover_link.append('notFound')\n",
    "        if book.find('span',{'class':'title'}) != None:\n",
    "            book_name.append(book.find('span',{'class':'title'}).string)\n",
    "        else:\n",
    "            book_name.append('notFound')\n",
    "        if book.find('span',{'class':'subtitle'}) != None:\n",
    "            author.append(book.find('span',{'class':'subtitle'}).string)\n",
    "        else:\n",
    "            author.append('notFound')\n",
    "        if book.find('span',{'class':'extra'}) != None:\n",
    "            downloads_count.append(book.find('span',{'class':'extra'}).string.split(' ')[0])\n",
    "        else:\n",
    "            downloads_count.append('notFound')\n",
    "    # to get books from subpages I think a recursive function that gets link of page would be good\n",
    "    if page_soup.find_all('li',{'class':'statusline'})[0].find('a') != None:\n",
    "        subpages_list=[]\n",
    "        curr_link=bookshelves_pages_links[i]\n",
    "        curr_data=requests.get(curr_link)\n",
    "        curr_soup=bs(curr_data.text,features='html.parser')\n",
    "        while(curr_soup.find_all('li',{'class':'statusline'})[0].find('a',{'title':'Go to the next page of results.'}) != None):\n",
    "            curr_link='https://www.gutenberg.org'+curr_soup.find_all('li',{'class':'statusline'})[0].find('a',{'title':'Go to the next page of results.'}).get('href')\n",
    "            curr_data=requests.get(curr_link)\n",
    "            curr_soup=bs(curr_data.text,features='html.parser')\n",
    "            for book in curr_soup.find_all('li',{'class':'booklink'}):\n",
    "                bookshelves_page.append(bookshelves_pages_titles[i])\n",
    "                book_link.append('https://www.gutenberg.org'+book.find('a').get('href'))\n",
    "                if book.find('img') != None:\n",
    "                    book_cover_link.append('https://www.gutenberg.org'+book.find('img').get('src'))\n",
    "                else:\n",
    "                    book_cover_link.append('notFound')\n",
    "                if book.find('span',{'class':'title'}) != None:\n",
    "                    book_name.append(book.find('span',{'class':'title'}).string)\n",
    "                else:\n",
    "                    book_name.append('notFound')\n",
    "                if book.find('span',{'class':'subtitle'}) != None:\n",
    "                    author.append(book.find('span',{'class':'subtitle'}).string)\n",
    "                else:\n",
    "                    author.append('notFound')\n",
    "                if book.find('span',{'class':'extra'}) != None:\n",
    "                    downloads_count.append(book.find('span',{'class':'extra'}).string.split(' ')[0])\n",
    "                else:\n",
    "                    downloads_count.append('notFound')\n",
    "        \n",
    "    # to get processing time    \n",
    "    end_time=time.time()\n",
    "    time_taken=end_time-start_time\n",
    "    print('Completed 👏 in',round(time_taken,1),'seconds.')# as the all are taking time in seconds ig"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 205,
   "metadata": {},
   "outputs": [],
   "source": [
    "books_df=pd.DataFrame({'Bookshelve Name':bookshelves_page,'Book Name':book_name,'Author':author,'Download Count':downloads_count,'Book Link':book_link,'Book Cover Link':book_cover_link})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 206,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Bookshelve Name</th>\n",
       "      <th>Book Name</th>\n",
       "      <th>Author</th>\n",
       "      <th>Download Count</th>\n",
       "      <th>Book Link</th>\n",
       "      <th>Book Cover Link</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>6 Best Loved Spanish Literary Classics</td>\n",
       "      <td>Don Quijote (Spanish)</td>\n",
       "      <td>Miguel de Cervantes Saavedra</td>\n",
       "      <td>2178</td>\n",
       "      <td>https://www.gutenberg.org/ebooks/2000</td>\n",
       "      <td>https://www.gutenberg.org/cache/epub/2000/pg20...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>6 Best Loved Spanish Literary Classics</td>\n",
       "      <td>The Life of Lazarillo de Tormes</td>\n",
       "      <td>Anonymous</td>\n",
       "      <td>1188</td>\n",
       "      <td>https://www.gutenberg.org/ebooks/53489</td>\n",
       "      <td>https://www.gutenberg.org/cache/epub/53489/pg5...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>6 Best Loved Spanish Literary Classics</td>\n",
       "      <td>The History of Don Quixote, Volume 1, Complete</td>\n",
       "      <td>Miguel de Cervantes Saavedra</td>\n",
       "      <td>1113</td>\n",
       "      <td>https://www.gutenberg.org/ebooks/5921</td>\n",
       "      <td>https://www.gutenberg.org/cache/epub/5921/pg59...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>6 Best Loved Spanish Literary Classics</td>\n",
       "      <td>The History of Don Quixote, Volume 2, Complete</td>\n",
       "      <td>Miguel de Cervantes Saavedra</td>\n",
       "      <td>413</td>\n",
       "      <td>https://www.gutenberg.org/ebooks/5946</td>\n",
       "      <td>https://www.gutenberg.org/cache/epub/5946/pg59...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>6 Best Loved Spanish Literary Classics</td>\n",
       "      <td>Niebla (Nivola) (Spanish)</td>\n",
       "      <td>Miguel de Unamuno</td>\n",
       "      <td>329</td>\n",
       "      <td>https://www.gutenberg.org/ebooks/49836</td>\n",
       "      <td>https://www.gutenberg.org/cache/epub/49836/pg4...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>220</th>\n",
       "      <td>African American Writers</td>\n",
       "      <td>The Anti-Slavery Harp: A Collection of Songs f...</td>\n",
       "      <td>notFound</td>\n",
       "      <td>4</td>\n",
       "      <td>https://www.gutenberg.org/ebooks/10448</td>\n",
       "      <td>https://www.gutenberg.org/cache/epub/10448/pg1...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>221</th>\n",
       "      <td>Ainslee's</td>\n",
       "      <td>Ainslee's, Vol. 15, No. 5, June 1905</td>\n",
       "      <td>Various</td>\n",
       "      <td>26</td>\n",
       "      <td>https://www.gutenberg.org/ebooks/27864</td>\n",
       "      <td>https://www.gutenberg.org/cache/epub/27864/pg2...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>222</th>\n",
       "      <td>Ainslee's</td>\n",
       "      <td>Ainslee's magazine, Volume 16, No. 3, October,...</td>\n",
       "      <td>Various</td>\n",
       "      <td>24</td>\n",
       "      <td>https://www.gutenberg.org/ebooks/47017</td>\n",
       "      <td>https://www.gutenberg.org/cache/epub/47017/pg4...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>223</th>\n",
       "      <td>Ainslee's</td>\n",
       "      <td>Ainslee's, Vol. 15, No. 6, July 1905</td>\n",
       "      <td>Various</td>\n",
       "      <td>12</td>\n",
       "      <td>https://www.gutenberg.org/ebooks/27885</td>\n",
       "      <td>https://www.gutenberg.org/cache/epub/27885/pg2...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>224</th>\n",
       "      <td>Ainslee's</td>\n",
       "      <td>Ainslee's magazine, Volume 16, No. 2, Septembe...</td>\n",
       "      <td>Various</td>\n",
       "      <td>12</td>\n",
       "      <td>https://www.gutenberg.org/ebooks/47016</td>\n",
       "      <td>https://www.gutenberg.org/cache/epub/47016/pg4...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>225 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                            Bookshelve Name  \\\n",
       "0    6 Best Loved Spanish Literary Classics   \n",
       "1    6 Best Loved Spanish Literary Classics   \n",
       "2    6 Best Loved Spanish Literary Classics   \n",
       "3    6 Best Loved Spanish Literary Classics   \n",
       "4    6 Best Loved Spanish Literary Classics   \n",
       "..                                      ...   \n",
       "220                African American Writers   \n",
       "221                               Ainslee's   \n",
       "222                               Ainslee's   \n",
       "223                               Ainslee's   \n",
       "224                               Ainslee's   \n",
       "\n",
       "                                             Book Name  \\\n",
       "0                                Don Quijote (Spanish)   \n",
       "1                      The Life of Lazarillo de Tormes   \n",
       "2       The History of Don Quixote, Volume 1, Complete   \n",
       "3       The History of Don Quixote, Volume 2, Complete   \n",
       "4                            Niebla (Nivola) (Spanish)   \n",
       "..                                                 ...   \n",
       "220  The Anti-Slavery Harp: A Collection of Songs f...   \n",
       "221               Ainslee's, Vol. 15, No. 5, June 1905   \n",
       "222  Ainslee's magazine, Volume 16, No. 3, October,...   \n",
       "223               Ainslee's, Vol. 15, No. 6, July 1905   \n",
       "224  Ainslee's magazine, Volume 16, No. 2, Septembe...   \n",
       "\n",
       "                           Author Download Count  \\\n",
       "0    Miguel de Cervantes Saavedra           2178   \n",
       "1                       Anonymous           1188   \n",
       "2    Miguel de Cervantes Saavedra           1113   \n",
       "3    Miguel de Cervantes Saavedra            413   \n",
       "4               Miguel de Unamuno            329   \n",
       "..                            ...            ...   \n",
       "220                      notFound              4   \n",
       "221                       Various             26   \n",
       "222                       Various             24   \n",
       "223                       Various             12   \n",
       "224                       Various             12   \n",
       "\n",
       "                                  Book Link  \\\n",
       "0     https://www.gutenberg.org/ebooks/2000   \n",
       "1    https://www.gutenberg.org/ebooks/53489   \n",
       "2     https://www.gutenberg.org/ebooks/5921   \n",
       "3     https://www.gutenberg.org/ebooks/5946   \n",
       "4    https://www.gutenberg.org/ebooks/49836   \n",
       "..                                      ...   \n",
       "220  https://www.gutenberg.org/ebooks/10448   \n",
       "221  https://www.gutenberg.org/ebooks/27864   \n",
       "222  https://www.gutenberg.org/ebooks/47017   \n",
       "223  https://www.gutenberg.org/ebooks/27885   \n",
       "224  https://www.gutenberg.org/ebooks/47016   \n",
       "\n",
       "                                       Book Cover Link  \n",
       "0    https://www.gutenberg.org/cache/epub/2000/pg20...  \n",
       "1    https://www.gutenberg.org/cache/epub/53489/pg5...  \n",
       "2    https://www.gutenberg.org/cache/epub/5921/pg59...  \n",
       "3    https://www.gutenberg.org/cache/epub/5946/pg59...  \n",
       "4    https://www.gutenberg.org/cache/epub/49836/pg4...  \n",
       "..                                                 ...  \n",
       "220  https://www.gutenberg.org/cache/epub/10448/pg1...  \n",
       "221  https://www.gutenberg.org/cache/epub/27864/pg2...  \n",
       "222  https://www.gutenberg.org/cache/epub/47017/pg4...  \n",
       "223  https://www.gutenberg.org/cache/epub/27885/pg2...  \n",
       "224  https://www.gutenberg.org/cache/epub/47016/pg4...  \n",
       "\n",
       "[225 rows x 6 columns]"
      ]
     },
     "execution_count": 206,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "books_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 204,
   "metadata": {},
   "outputs": [],
   "source": [
    "## can create clickable links but takes time for these many names\n",
    "## also can store into a csv or whatever"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}