scrape_state_laws.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import requests\n",
    "from bs4 import BeautifulSoup\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Set url\n",
    "url = \"http://www.iihs.org/iihs/topics/laws/dui?topicName=alcohol-impaired-driving\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Scrape the HTML at the url\n",
    "r = requests.get(url)\n",
    "\n",
    "# Turn the HTML into a Beautiful Soup object\n",
    "soup = BeautifulSoup(r.text, 'lxml')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#soup.find_all('tr')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "n_columns = 0\n",
    "n_rows=0\n",
    "column_names = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "for row in soup.find_all('tr')[1:]:\n",
    "                \n",
    "    # find number of rows\n",
    "    td_tags = row.find_all('td')\n",
    "    if len(td_tags) > 0:\n",
    "        n_rows+=1\n",
    "        if n_columns == 0:\n",
    "        #set the number of columns \n",
    "            n_columns = len(td_tags)\n",
    "                        \n",
    "    #find column names \n",
    "    th_tags = row.find_all('th') \n",
    "    if len(th_tags) > 0 and len(column_names) == 0:\n",
    "        for th in th_tags:\n",
    "            column_names.append(th.get_text())\n",
    "            \n",
    "#    row_marker = 0\n",
    "#    for row in soup.find_all('tr'):\n",
    "#        column_marker = 0\n",
    "#        columns = row.find_all('td')\n",
    "#        for column in columns:\n",
    "#            df.iat[row_marker,column_marker] = column.get_text()\n",
    "#            column_marker += 1\n",
    "#        if len(columns) > 0:\n",
    "#            row_marker += 1\n",
    "            "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<th nowrap=\"nowrap\">Repeat offenders</th>"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "th"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['First offenders', 'Repeat offenders']"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "column_names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[]"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "th_tags"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}