Task 1 - Getting to Philosophy.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import urllib.request, urllib.parse, urllib.error\n",
    "from bs4 import BeautifulSoup\n",
    "import urllib.request\n",
    "import re"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "response = urllib.request.urlopen('https://en.wikipedia.org/wiki/Special:Random').read()\n",
    "soup = BeautifulSoup(response)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "check = True\n",
    "visitedLinks = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'First Housing Company Tanzania Limited'"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "title = soup.find(\"h1\", {\"class\":\"firstHeading\"})\n",
    "t = title.get_text() # Get the Header of the page\n",
    "t"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def getURL(first_pg): # Extract first link from the wep page's article.\n",
    "    for link in first_pg: # Loop over all paragraphs.\n",
    "        l_parent = link.find_all(\"a\")\n",
    "        for l in l_parent: # Loop over all links in each paragraph.\n",
    "            if str(l) == 'None':\n",
    "                continue\n",
    "            else:\n",
    "                y = l.get_attribute_list(\"href\").pop(0)\n",
    "                if y.startswith(\"#\"): # Make sure that the first found link is not refering to the same page.\n",
    "                    continue\n",
    "                else:\n",
    "                    return y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def gotoURL(nLink): # Access the link returned from getURL() and return the Header of the page and page itself.\n",
    "    c = \"https://en.wikipedia.org\" + nLink\n",
    "    response = urllib.request.urlopen(c).read()\n",
    "    soup = BeautifulSoup(response)\n",
    "    title = soup.find(\"h1\", {\"class\":\"firstHeading\"})\n",
    "    title.get_text()\n",
    "    return soup, title.get_text() # Return the page and its header to check if its Philosophy page or not."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Start looking for Philosophy... \n",
      "\n",
      "First Housing Company Tanzania Limited\n",
      "Mortgage bank\n",
      "Bank\n",
      "Financial institution\n",
      "Corporation\n",
      "Company\n",
      "Legal person\n",
      "Natural person\n",
      "Jurisprudence\n",
      "Law\n",
      "System\n",
      "Interaction\n",
      "Action (physics)\n",
      "Physics\n",
      "Ancient Greek\n",
      "Greek language\n",
      "Modern Greek\n",
      "Dialect\n",
      "Latin\n",
      "Help:IPA/Latin\n",
      "International Phonetic Alphabet\n",
      "Alphabet\n",
      "Letter (alphabet)\n",
      "Symbol\n",
      "Idea\n",
      "Philosophy\n",
      "\n",
      "We made it to the Philosophy page.\n"
     ]
    }
   ],
   "source": [
    "print(\"Start looking for Philosophy... \\n\")\n",
    "while check:\n",
    "    print(t)\n",
    "    if t == \"Philosophy\": # Check if we reached the targeted page or not.\n",
    "        print(\"\\nWe made it to the Philosophy page.\")\n",
    "        break\n",
    "    else:\n",
    "        first_pg = soup.find_all(\"p\") # Extract all paragraphs from the page.\n",
    "        nLink = getURL(first_pg) # Send all paragraphs to fun getURL() to get the first link in the article.\n",
    "        if nLink in visitedLinks: # Check the returned link is not visited before, preventing us from stucking in a loop.\n",
    "            print(\"Exiting ...\\n Program is stuck in a loop.\")\n",
    "            break\n",
    "        visitedLinks.append(nLink) # Add the new link to list of visited links.\n",
    "        soup, t = gotoURL(nLink) # Get the information from the new link."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}