01_scrape.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Scrape FERC enforcement cases\n",
    "\n",
    "By [Ben Welsh](mailto:ben.welsh@latimes.com)\n",
    "\n",
    "This script was written as part of an analysis conducted by the Los Angeles Times. For more information refer to the [final notebook](02_analyze.ipynb)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Import Python tools"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "import pandas as pd\n",
    "from bs4 import BeautifulSoup"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Download the penalties homepage\n",
    "\n",
    "This contains a navigation bar with links to pages for individual years."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "homepage_url = \"https://www.ferc.gov/enforcement/civil-penalties/civil-penalty-action.asp\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "homepage_html = requests.get(homepage_url).content"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "homepage_soup = BeautifulSoup(homepage_html, 'html5lib')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Zoom in on the table with the data we want"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "homepage_table = homepage_soup.find(\"table\", id=\"right_module\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "homepage_years = homepage_table.findAll(\"li\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The first list item will be the year of the current page. The rest will be previous years with links."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "current_year = int(homepage_years[0].contents[0].strip())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Pull out all the previous years"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "previous_years = dict((\n",
    "    (int(i.find(\"a\").contents[0].strip()), i.find(\"a\")['href'])\n",
    "        for i in homepage_years[1:]\n",
    "))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Combine the soup for all the years"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "soup_dict = {current_year: homepage_soup}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "for year, url in previous_years.items():\n",
    "    year_url = \"https://www.ferc.gov{}\".format(url)\n",
    "    year_html = requests.get(year_url).content\n",
    "    year_soup = BeautifulSoup(year_html, 'html5lib')\n",
    "    soup_dict[year] = year_soup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def parse_table(year, soup):\n",
    "    container = soup.find(\"div\", attrs={\"class\": \"container\"})\n",
    "    table = container.find(\"table\")\n",
    "    raw_rows = table.findAll(\"tr\")\n",
    "    clean_rows = []\n",
    "    for row in raw_rows[1:]:\n",
    "        raw_cells = row.findAll(\"td\")\n",
    "        clean_cells = [year,]\n",
    "        for cell in raw_cells:\n",
    "            cleaned = \" \".join(cell.get_text().strip().split())\n",
    "            clean_cells.append(cleaned)\n",
    "        clean_rows.append(clean_cells)\n",
    "    return clean_rows"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Parse and combine all the rows"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "row_list = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "for year, soup in soup_dict.items():\n",
    "    year_rows = parse_table(year, soup)\n",
    "    row_list.extend(year_rows)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Export it"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame(row_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.columns = [\n",
    "    'raw_year',\n",
    "    'raw_subject',\n",
    "    'raw_sanctions',\n",
    "    'raw_description'\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.sort_values([\"raw_year\",\"raw_subject\"], ascending=False).to_csv(\n",
    "    \"./input/scraped-ferc-civil-penalties.csv\",\n",
    "    index=False,\n",
    "    encoding=\"utf-8\"\n",
    ")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}