web_scraping.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# WEB SCRAPING"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Importing necessary packages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from bs4 import BeautifulSoup\n",
    "from urllib.request import Request, urlopen\n",
    "import requests\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Creating a Dataframe to store all required values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>State</th>\n",
       "      <th>City</th>\n",
       "      <th>Latitude</th>\n",
       "      <th>Longitude</th>\n",
       "      <th>Elevation</th>\n",
       "      <th>Estimated Population</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [State, City, Latitude, Longitude, Elevation, Estimated Population]\n",
       "Index: []"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.DataFrame(columns=[\"State\", \"City\", \"Latitude\", \"Longitude\", \"Elevation\", \"Estimated Population\"])\n",
    "df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Creating a 'checker' function "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### This function takes a url as an input and first checks if the url is valid and responding properly, otherwise throws an exception. If the url is active, then this function checks whether the given webpage contains the required data in the form of 'City', 'Latitude', etc. or links leading to next level. If webpage contains required data, then the data is added to the dataframe. If it contains links to next level, then we generate next level url using 'getLink' function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def checker(url):\n",
    "    \n",
    "    # Checking if the website is down or if we don’t get the expected response\n",
    "    try:\n",
    "        # If website is respoing properly\n",
    "        \n",
    "        # response of an HTTP request\n",
    "        req = Request(url)\n",
    "        \n",
    "        # Open the Request object.\n",
    "        page = urlopen(req)\n",
    "        \n",
    "        # soup is an object that interacts using 'tag'\n",
    "        soup = BeautifulSoup(page, \"lxml\")\n",
    "\n",
    "        # If the BeatifulSoup object has 'tr' tag, then we have required data in this url\n",
    "        if len(soup.findAll('tr')) > 0:\n",
    "            table_data = soup.findAll('tr')\n",
    "            \n",
    "            for i in range(len(table_data)):\n",
    "                data = []\n",
    "\n",
    "                # Each element in table_data corresponds to one city details\n",
    "                # Fetching text under 'td' tag gives required data.\n",
    "                for td in table_data[i].findAll('td'):\n",
    "                    data.append(td.getText())\n",
    "\n",
    "                if(len(data)>0):\n",
    "                    # adding data to existing dataframe\n",
    "                    df.loc[len(df)] = [data[2],data[0],data[4],data[5],data[6],data[7]]\n",
    "                print(\"adding+\")\n",
    "\n",
    "        else :\n",
    "            # when current url has links to next level, fetching next-level url.\n",
    "            getLink(url)\n",
    "        \n",
    "    except:\n",
    "        # if webpage is down or not responding,\n",
    "        print(\"Page does not exist\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Creating 'getLink' function"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### This function creates new links for the next level and calls 'checker' function to check if the new_url has requied data or not."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def getLink(url):\n",
    "    \n",
    "    # all characters (a-z) are added to the link to fetch all possible cities.\n",
    "    for i in range(97, 123):\n",
    "        a = chr(i)\n",
    "        \n",
    "        # First letter of every city starts with uppercase in the link.\n",
    "        if len(url)==41 :\n",
    "            a = a.upper()\n",
    "        new_url = url + a + \"/\"\n",
    "        \n",
    "        # Agin checking new_url \n",
    "        checker(new_url)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Lets start with given_url."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "given_url = \"http://www.fallingrain.com/world/IN/\"\n",
    "\n",
    "# Get all urls from the home page which has link in the form of \"http://www.fallingrain.com/world/IN/__/a\"\n",
    "# 2 above blanks in the link will be filled by a number \n",
    "\n",
    "# for testing purpose , considering only top 11 indexes.\n",
    "for i in range(11):\n",
    "    if i < 10:\n",
    "        new_url = given_url + \"0\" + str(i) + \"/a/\"\n",
    "    else:\n",
    "        new_url = given_url + str(i) + \"/a/\"\n",
    "    \n",
    "    checker(new_url)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Exporting Pandas DataFrame to CSV file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_csv(\"data.csv\", encoding='utf-8', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Test-case"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<tr><th>Name</th> <th>What</th> <th>Region</th> <th>Country</th> <th>Lat</th> <th>Long</th> <th>Elev ft.</th> <th>Pop est</th></tr>\n",
      "<tr><td><a href=\"/world/IN/05/Atawa.html\">Atawa</a></td> <td>city</td> <td>Chandigarh</td> <td>India</td> <td>30.73</td> <td>76.75</td> <td>1010</td> <td>2991</td></tr>\n",
      "<tr><td><a href=\"/world/IN/05/Badheri.html\">Badheri</a></td> <td>city</td> <td>Chandigarh</td> <td>India</td> <td>30.73</td> <td>76.73</td> <td>1007</td> <td>30901</td></tr>\n",
      "<tr><td><a href=\"/world/IN/05/Bahlana.html\">Bahlana</a></td> <td>city</td> <td>Chandigarh</td> <td>India</td> <td>30.68</td> <td>76.80</td> <td>984</td> <td>84485</td></tr>\n",
      "<tr><td><a href=\"/world/IN/05/Bahlolpur.html\">Bahlolpur</a></td> <td>city</td> <td>Chandigarh</td> <td>India</td> <td>30.76</td> <td>76.70</td> <td>1000</td> <td>78064</td></tr>\n"
     ]
    }
   ],
   "source": [
    "url = \"http://www.fallingrain.com/world/IN/05/a/\"\n",
    "# '05' is the index of Chandigarh state\n",
    "\n",
    "req = Request(url)\n",
    "page = urlopen(req)\n",
    "soup = BeautifulSoup(page, \"lxml\")\n",
    "\n",
    "#Extract all 'tr' tags\n",
    "table_data = soup.findAll('tr')\n",
    "for i in range(5):\n",
    "    print (table_data[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "        State       City Latitude Longitude Elevation Estimated Population\n",
      "0  Chandigarh      Atawa    30.73     76.75      1010                 2991\n",
      "1  Chandigarh    Badheri    30.73     76.73      1007                30901\n",
      "2  Chandigarh    Bahlana    30.68     76.80       984                84485\n",
      "3  Chandigarh  Bahlolpur    30.76     76.70      1000                78064\n",
      "4  Chandigarh    Bajwāra    30.73     76.76      1036                 1399\n"
     ]
    }
   ],
   "source": [
    "df2 = pd.DataFrame(columns=[\"State\", \"City\", \"Latitude\", \"Longitude\", \"Elevation\", \"Estimated Population\"])\n",
    "for i in range(len(table_data)):\n",
    "    data = []\n",
    "    \n",
    "    #Extarct all 'td' tags in each 'tr' tag to get desired output.\n",
    "    for td in table_data[i].findAll('td'):\n",
    "        data.append(td.getText())\n",
    "    \n",
    "    if(len(data)>0):\n",
    "        df2.loc[len(df2)] = [data[2],data[0],data[4],data[5],data[6],data[7]]\n",
    "\n",
    "print (df2.head())\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}