Quotations-Web Scraping.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "from bs4 import BeautifulSoup\n",
    "import re\n",
    "import os\n",
    "import time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "home_url = 'http://www.wiseoldsayings.com'\n",
    "r = requests.get(home_url)\n",
    "soup = BeautifulSoup(r.content, 'html5lib')\n",
    "quote_classes = []\n",
    "quote_hrefs = []\n",
    "cat_list = soup.find('div', attrs = {'class':'cat_list'})\n",
    "for row in cat_list.findAll('li'):\n",
    "    quote_classes.append(row.text.strip())\n",
    "    quote_hrefs.append(row.a['href'].strip())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Acting',\n",
       " 'Addiction',\n",
       " 'Adoption',\n",
       " 'Alone',\n",
       " 'Anger',\n",
       " 'Bad Girls',\n",
       " 'Beauty',\n",
       " 'Being Alone',\n",
       " 'Being Happy',\n",
       " 'Being Hurt',\n",
       " 'Being Real',\n",
       " 'Being Strong',\n",
       " 'Bravery',\n",
       " 'Bullying',\n",
       " 'Caring',\n",
       " 'Change',\n",
       " 'Coffee',\n",
       " 'Compassion',\n",
       " 'Consistency',\n",
       " 'Crushes',\n",
       " 'Dancing',\n",
       " 'Disappointment',\n",
       " 'Diversity',\n",
       " 'Excuses',\n",
       " 'Fake Friends',\n",
       " 'Falling In Love',\n",
       " 'Family',\n",
       " 'Fate',\n",
       " 'Finding Yourself',\n",
       " 'Fire',\n",
       " 'Flying',\n",
       " 'Friday',\n",
       " 'Friendship',\n",
       " 'Getting Older',\n",
       " 'Growing Up',\n",
       " 'Hate',\n",
       " 'Health',\n",
       " 'Heaven',\n",
       " 'Honesty',\n",
       " 'Hope',\n",
       " 'Ignorance',\n",
       " 'Inner Strength',\n",
       " 'Insomnia',\n",
       " 'Integrity',\n",
       " 'Jealousy',\n",
       " 'Justice',\n",
       " 'Karma',\n",
       " 'Knowledge',\n",
       " 'Leadership',\n",
       " 'Learning',\n",
       " 'Life',\n",
       " 'Life Lessons',\n",
       " 'Light',\n",
       " 'Losing A Friend',\n",
       " 'Love',\n",
       " 'Loyalty',\n",
       " 'Manipulation',\n",
       " 'Mother And Daughter',\n",
       " 'Mother And Son',\n",
       " 'Nature',\n",
       " 'Negativity',\n",
       " 'Pain',\n",
       " 'Perfection',\n",
       " 'Psychology',\n",
       " 'Regret',\n",
       " 'Regrets',\n",
       " 'Sacrifice',\n",
       " 'Sad Love',\n",
       " 'Single Life',\n",
       " 'Sisters',\n",
       " 'Snow',\n",
       " 'Softball',\n",
       " 'Soldiers',\n",
       " 'Starting Over',\n",
       " 'Staying Positive',\n",
       " 'Staying Strong',\n",
       " 'Strength',\n",
       " 'Stupid People',\n",
       " 'Support',\n",
       " 'Surfing',\n",
       " 'Survival',\n",
       " 'Taking Chances',\n",
       " 'The Moon',\n",
       " 'Thursday',\n",
       " 'Time',\n",
       " 'True Friends',\n",
       " 'Trust',\n",
       " 'Volleyball',\n",
       " 'War',\n",
       " 'Wisdom']"
      ]
     },
     "execution_count": 127,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "quote_classes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_quotes(url):\n",
    "    i = 1\n",
    "    quotes = []\n",
    "    while True:\n",
    "        curr_quotes = []\n",
    "        quote_url = url + 'page-' + str(i) + '/'\n",
    "        i += 1\n",
    "        quote_r = requests.get(quote_url)\n",
    "        quote_soup = BeautifulSoup(quote_r.content, 'html5lib')\n",
    "        quote_list = quote_soup.find('div', attrs = {'class':'quote_list'})\n",
    "        for j,quote in enumerate(quote_list.select('div p')):\n",
    "            if j%3 == 1:\n",
    "                quote = quote.text.replace(u'\\xa0', u'')\n",
    "                quote = ' '.join(re.findall(r'\\w+[\\']?\\w+[,.]?', quote))\n",
    "                curr_quotes.append(quote)\n",
    "        if (len(quotes) != 0) and (curr_quotes[-1] == quotes[-1]) :\n",
    "            break\n",
    "        else:\n",
    "            quotes.extend(curr_quotes)\n",
    "    return quotes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {},
   "outputs": [],
   "source": [
    "curr_path = os.getcwd()\n",
    "btime = time.time()\n",
    "with open(curr_path+'/quotes.txt', 'w') as f:\n",
    "    for i in range(len(quote_hrefs)):\n",
    "        quotes = get_quotes(home_url + quote_hrefs[i])\n",
    "        for quote in quotes:\n",
    "            if quote[-1] != '.':\n",
    "                quote += '.'\n",
    "            f.write(quote + '\\n')\n",
    "atime = time.time()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Time taken = 33.0 mns\n"
     ]
    }
   ],
   "source": [
    "print(f\"Time taken = {(atime-btime)//60} mns\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}