ch13.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython suppress id=a26af182eb1f4597a9dbf174cdc570b9\n",
    "%pushd book-materials\n",
    "\n",
    "from numpy.random import randn\n",
    "import numpy as np\n",
    "np.random.seed(123)\n",
    "import os\n",
    "import matplotlib.pyplot as plt\n",
    "import pandas as pd\n",
    "plt.rc(\"figure\", figsize=(10, 6))\n",
    "np.set_printoptions(precision=4)\n",
    "pd.options.display.max_columns = 20\n",
    "pd.options.display.max_rows = 20\n",
    "pd.options.display.max_colwidth = 80"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython suppress id=5f3e64ebebd744a4a972fce4303d263a\n",
    "path = \"datasets/bitly_usagov/example.txt\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython verbatim id=74b843e3047b47d09a832031cd7a3753\n",
    "import json\n",
    "with open(path) as f:\n",
    "    records = [json.loads(line) for line in f]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython allow_exceptions id=b0d4c36e296a48e4b5bfcc7b5a0b5926\n",
    "time_zones = [rec[\"tz\"] for rec in records]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=9bcb262c06f14a3aa36dbd73c2f33444\n",
    "time_zones = [rec[\"tz\"] for rec in records if \"tz\" in rec]\n",
    "time_zones[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython verbatim id=64a59d263c4c4e8ba0617426c1b21573\n",
    "def get_counts(sequence):\n",
    "    counts = {}\n",
    "    for x in sequence:\n",
    "        if x in counts:\n",
    "            counts[x] += 1\n",
    "        else:\n",
    "            counts[x] = 1\n",
    "    return counts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython verbatim id=b55b0800f7274320b5a7f97d93719a2d\n",
    "from collections import defaultdict\n",
    "\n",
    "def get_counts2(sequence):\n",
    "    counts = defaultdict(int) # values will initialize to 0\n",
    "    for x in sequence:\n",
    "        counts[x] += 1\n",
    "    return counts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=ba1751a9d2244f46841fd9afbae7aa04\n",
    "counts = get_counts(time_zones)\n",
    "counts[\"America/New_York\"]\n",
    "len(time_zones)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython verbatim id=562d875fd9704dc6a3614303c9bc7a8e\n",
    "def top_counts(count_dict, n=10):\n",
    "    value_key_pairs = [(count, tz) for tz, count in count_dict.items()]\n",
    "    value_key_pairs.sort()\n",
    "    return value_key_pairs[-n:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=d061b4cb313c469e94543974462530ba\n",
    "top_counts(counts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=ab2efb020a1846699974cea106f89ba2\n",
    "from collections import Counter\n",
    "counts = Counter(time_zones)\n",
    "counts.most_common(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=8abcc73b62f5486a9a448c1e742c5f92\n",
    "frame = pd.DataFrame(records)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=58b54b37efa04ef9ad317bc29a1e8e2e\n",
    "frame.info()\n",
    "frame[\"tz\"].head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=9694e9d51c944b9882129e348f79ae0c\n",
    "tz_counts = frame[\"tz\"].value_counts()\n",
    "tz_counts.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=b445fbf49098428fbc93855c9aa93571\n",
    "clean_tz = frame[\"tz\"].fillna(\"Missing\")\n",
    "clean_tz[clean_tz == \"\"] = \"Unknown\"\n",
    "tz_counts = clean_tz.value_counts()\n",
    "tz_counts.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython suppress id=f156ad96962842e6ae79658999ea7134\n",
    "plt.figure(figsize=(10, 4))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=ea12bafce448419c90a8a118e018458f\n",
    "import seaborn as sns\n",
    "subset = tz_counts.head()\n",
    "#! figure,id=usa_gov_counts,title=\"Top time zones in the 1.usa.gov sample data\"\n",
    "sns.barplot(y=subset.index, x=subset.to_numpy())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=32e36ee621664722babaffe83900dff2\n",
    "frame[\"a\"][1]\n",
    "frame[\"a\"][50]\n",
    "frame[\"a\"][51][:50]  # long line"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=973a67b5d25d41fb8c7d53ddcfbd747f\n",
    "results = pd.Series([x.split()[0] for x in frame[\"a\"].dropna()])\n",
    "results.head(5)\n",
    "results.value_counts().head(8)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=7ef631a4289942378d19f5bc510b93e1\n",
    "cframe = frame[frame[\"a\"].notna()].copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=334e7d52678f48a28af3f14e7672b962\n",
    "cframe[\"os\"] = np.where(cframe[\"a\"].str.contains(\"Windows\"),\n",
    "                        \"Windows\", \"Not Windows\")\n",
    "cframe[\"os\"].head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=c8beb8e211f741bdb3680861c6bed49a\n",
    "by_tz_os = cframe.groupby([\"tz\", \"os\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=e62bf85f81fd46f1a5d1f00399ae31f7\n",
    "agg_counts = by_tz_os.size().unstack().fillna(0)\n",
    "agg_counts.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=684389ffe6c0421d9ced56415c1baeb9\n",
    "indexer = agg_counts.sum(\"columns\").argsort()\n",
    "indexer.values[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=e861e46cf4b34f7690bbcbfbc490450c\n",
    "count_subset = agg_counts.take(indexer[-10:])\n",
    "count_subset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=836cfdd2d671421f9e58b031cf142e3f\n",
    "agg_counts.sum(axis=\"columns\").nlargest(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython suppress id=3edfe74f82f247cc99e3e4d7d4907a0c\n",
    "plt.figure()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=a194820b3f03419e844991ecdfaa3236\n",
    "count_subset = count_subset.stack()\n",
    "count_subset.name = \"total\"\n",
    "count_subset = count_subset.reset_index()\n",
    "count_subset.head(10)\n",
    "#! figure,id=usa_gov_tz_os,title=\"Top time zones by Windows and non-Windows users\"\n",
    "sns.barplot(x=\"total\", y=\"tz\", hue=\"os\",  data=count_subset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython verbatim id=1cf7faf398ca47e2b0c97b72beff6256\n",
    "def norm_total(group):\n",
    "    group[\"normed_total\"] = group[\"total\"] / group[\"total\"].sum()\n",
    "    return group\n",
    "\n",
    "results = count_subset.groupby(\"tz\").apply(norm_total)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython suppress id=e7e063a913d244ff939d923e84f2f905\n",
    "plt.figure()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=512403c2efd149b1be89649e70dde50a\n",
    "#! figure,id=usa_gov_tz_os_normed,title=\"Percentage Windows and non-Windows users in top occurring time zones\"\n",
    "sns.barplot(x=\"normed_total\", y=\"tz\", hue=\"os\",  data=results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=8e6df52dcfd144489941522b36b5a32a\n",
    "g = count_subset.groupby(\"tz\")\n",
    "results2 = count_subset[\"total\"] / g[\"total\"].transform(\"sum\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython verbatim id=f5f892f460124f2499e6e040ad28239a\n",
    "unames = [\"user_id\", \"gender\", \"age\", \"occupation\", \"zip\"]\n",
    "users = pd.read_table(\"datasets/movielens/users.dat\", sep=\"::\",\n",
    "                      header=None, names=unames, engine=\"python\")\n",
    "\n",
    "rnames = [\"user_id\", \"movie_id\", \"rating\", \"timestamp\"]\n",
    "ratings = pd.read_table(\"datasets/movielens/ratings.dat\", sep=\"::\",\n",
    "                        header=None, names=rnames, engine=\"python\")\n",
    "\n",
    "mnames = [\"movie_id\", \"title\", \"genres\"]\n",
    "movies = pd.read_table(\"datasets/movielens/movies.dat\", sep=\"::\",\n",
    "                       header=None, names=mnames, engine=\"python\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=c53d153be32f4a31af7adcc70ba414a6\n",
    "users.head(5)\n",
    "ratings.head(5)\n",
    "movies.head(5)\n",
    "ratings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=326328c171c349389adb61b8e687355e\n",
    "data = pd.merge(pd.merge(ratings, users), movies)\n",
    "data\n",
    "data.iloc[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=e5129c0aac3a400f91f247cb27ae9c3f\n",
    "mean_ratings = data.pivot_table(\"rating\", index=\"title\",\n",
    "                                columns=\"gender\", aggfunc=\"mean\")\n",
    "mean_ratings.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=3e174af9b5e342b38519d1b0ce2802a4\n",
    "ratings_by_title = data.groupby(\"title\").size()\n",
    "ratings_by_title.head()\n",
    "active_titles = ratings_by_title.index[ratings_by_title >= 250]\n",
    "active_titles"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=083e410e3d7445baaddbb46d6760c956\n",
    "mean_ratings = mean_ratings.loc[active_titles]\n",
    "mean_ratings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython suppress id=3920a865f3684ea6ab421a78dc0bcba4\n",
    "mean_ratings = mean_ratings.rename(index={\"Seven Samurai (The Magnificent Seven) (Shichinin no samurai) (1954)\":\n",
    "                           \"Seven Samurai (Shichinin no samurai) (1954)\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=a7405fad58a444e283a9c70691b2cd1a\n",
    "top_female_ratings = mean_ratings.sort_values(\"F\", ascending=False)\n",
    "top_female_ratings.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=30f629f297c14048b06165aabeeed519\n",
    "mean_ratings[\"diff\"] = mean_ratings[\"M\"] - mean_ratings[\"F\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=efe335603fda4b159496e6fc78c8546b\n",
    "sorted_by_diff = mean_ratings.sort_values(\"diff\")\n",
    "sorted_by_diff.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=d07b8e30ed1f4252a346e493dc775b4d\n",
    "sorted_by_diff[::-1].head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=ce7c8a289d4942079cc643a7a30fe51d\n",
    "rating_std_by_title = data.groupby(\"title\")[\"rating\"].std()\n",
    "rating_std_by_title = rating_std_by_title.loc[active_titles]\n",
    "rating_std_by_title.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=1b53af1f9cdc44c7ae77c8a6a10e5780\n",
    "rating_std_by_title.sort_values(ascending=False)[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=79b147a65ea34ffcbe2d799331fcba6d\n",
    "movies[\"genres\"].head()\n",
    "movies[\"genres\"].head().str.split(\"|\")\n",
    "movies[\"genre\"] = movies.pop(\"genres\").str.split(\"|\")\n",
    "movies.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=3a1765e4004f467abde25424f4f68a19\n",
    "movies_exploded = movies.explode(\"genre\")\n",
    "movies_exploded[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=12e06c516e144975bb4f6b33a19d3d17\n",
    "ratings_with_genre = pd.merge(pd.merge(movies_exploded, ratings), users)\n",
    "ratings_with_genre.iloc[0]\n",
    "genre_ratings = (ratings_with_genre.groupby([\"genre\", \"age\"])\n",
    "                 [\"rating\"].mean()\n",
    "                 .unstack(\"age\"))\n",
    "genre_ratings[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=0b02ebafbf174f5b8b5b8487f314857a\n",
    "!head -n 10 datasets/babynames/yob1880.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=96d7c450b52749f6adc841c2ae245abf\n",
    "names1880 = pd.read_csv(\"datasets/babynames/yob1880.txt\",\n",
    "                        names=[\"name\", \"sex\", \"births\"])\n",
    "names1880"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=93704598475f4ace96b41290892a8e22\n",
    "names1880.groupby(\"sex\")[\"births\"].sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython verbatim id=c1e1153045b246e6b46b81ed6d740365\n",
    "pieces = []\n",
    "for year in range(1880, 2011):\n",
    "    path = f\"datasets/babynames/yob{year}.txt\"\n",
    "    frame = pd.read_csv(path, names=[\"name\", \"sex\", \"births\"])\n",
    "\n",
    "    # Add a column for the year\n",
    "    frame[\"year\"] = year\n",
    "    pieces.append(frame)\n",
    "\n",
    "# Concatenate everything into a single DataFrame\n",
    "names = pd.concat(pieces, ignore_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=2d1371e5fc5b4989bfdcb50c224f6ecf\n",
    "names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=49e2f22b602741ff86b0e01def6297b5\n",
    "total_births = names.pivot_table(\"births\", index=\"year\",\n",
    "                                 columns=\"sex\", aggfunc=sum)\n",
    "total_births.tail()\n",
    "#! figure,id=baby_names_total_births,title=\"Total births by sex and year\"\n",
    "total_births.plot(title=\"Total births by sex and year\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython verbatim id=abb6e87b71ef43b0906c515cbf44ba33\n",
    "def add_prop(group):\n",
    "    group[\"prop\"] = group[\"births\"] / group[\"births\"].sum()\n",
    "    return group\n",
    "names = names.groupby([\"year\", \"sex\"]).apply(add_prop)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=af43bd66df8a464a9acf179ea1a2a724\n",
    "names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=799eddd440ee4047ad4dc571bd534914\n",
    "names.groupby([\"year\", \"sex\"])[\"prop\"].sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=d7cdfcaeec094222850cbd0a754e062d\n",
    "def get_top1000(group):\n",
    "    return group.sort_values(\"births\", ascending=False)[:1000]\n",
    "grouped = names.groupby([\"year\", \"sex\"])\n",
    "top1000 = grouped.apply(get_top1000)\n",
    "top1000.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=b817c8be4dcf441d826d41f0a679a8f7\n",
    "top1000 = top1000.reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=4018f9a519914774bfd558cd9ebfc13b\n",
    "top1000.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=df3701a69ebd4b3cbc438de86d549d93\n",
    "boys = top1000[top1000[\"sex\"] == \"M\"]\n",
    "girls = top1000[top1000[\"sex\"] == \"F\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=e8e5983e77f741d7b1ad8fe607c9b869\n",
    "total_births = top1000.pivot_table(\"births\", index=\"year\",\n",
    "                                   columns=\"name\",\n",
    "                                   aggfunc=sum)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=ec63edf8240a4b739fbf6fd57e6b3c15\n",
    "total_births.info()\n",
    "subset = total_births[[\"John\", \"Harry\", \"Mary\", \"Marilyn\"]]\n",
    "#! figure,id=baby_names_some_names,title=\"A few boy and girl names over time\"\n",
    "subset.plot(subplots=True, figsize=(12, 10),\n",
    "            title=\"Number of births per year\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython suppress id=4353ea3630a8454b94c5093acfb6e8d9\n",
    "plt.figure()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=686e195247ca4394b0b8047c4019297c\n",
    "table = top1000.pivot_table(\"prop\", index=\"year\",\n",
    "                            columns=\"sex\", aggfunc=sum)\n",
    "#! figure,id=baby_names_tot_prop,title=\"Proportion of births represented in top one thousand names by sex\"\n",
    "table.plot(title=\"Sum of table1000.prop by year and sex\",\n",
    "           yticks=np.linspace(0, 1.2, 13))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=900fa540badb4cb9be8210d13b2ebee5\n",
    "df = boys[boys[\"year\"] == 2010]\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=4410fdd6262e462e982a01a98d55b34e\n",
    "prop_cumsum = df[\"prop\"].sort_values(ascending=False).cumsum()\n",
    "prop_cumsum[:10]\n",
    "prop_cumsum.searchsorted(0.5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=e7b5a7c97cb34f8487bdd77ead95eec7\n",
    "df = boys[boys.year == 1900]\n",
    "in1900 = df.sort_values(\"prop\", ascending=False).prop.cumsum()\n",
    "in1900.searchsorted(0.5) + 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython verbatim id=3275861aaba4432a8159d7004efee61a\n",
    "def get_quantile_count(group, q=0.5):\n",
    "    group = group.sort_values(\"prop\", ascending=False)\n",
    "    return group.prop.cumsum().searchsorted(q) + 1\n",
    "\n",
    "diversity = top1000.groupby([\"year\", \"sex\"]).apply(get_quantile_count)\n",
    "diversity = diversity.unstack()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython suppress id=a1adda1ceb0e4783ba0f3709016bede2\n",
    "fig = plt.figure()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=d194e4ae2a2b417aaedec5dcbc12d3a5\n",
    "diversity.head()\n",
    "#! figure,id=baby_names_diversity_fig,title=\"Plot of diversity metric by year\"\n",
    "diversity.plot(title=\"Number of popular names in top 50%\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython verbatim id=086dc90d927b41e99f9af7ecd87aadee\n",
    "def get_last_letter(x):\n",
    "    return x[-1]\n",
    "\n",
    "last_letters = names[\"name\"].map(get_last_letter)\n",
    "last_letters.name = \"last_letter\"\n",
    "\n",
    "table = names.pivot_table(\"births\", index=last_letters,\n",
    "                          columns=[\"sex\", \"year\"], aggfunc=sum)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=81379828617f46a8b289d83319498423\n",
    "subtable = table.reindex(columns=[1910, 1960, 2010], level=\"year\")\n",
    "subtable.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=cd458b92c4e14381a5380abc577f750a\n",
    "subtable.sum()\n",
    "letter_prop = subtable / subtable.sum()\n",
    "letter_prop"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython verbatim id=cddad4c5c57d4c2b9a445f3eda5e625e\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "fig, axes = plt.subplots(2, 1, figsize=(10, 8))\n",
    "letter_prop[\"M\"].plot(kind=\"bar\", rot=0, ax=axes[0], title=\"Male\")\n",
    "letter_prop[\"F\"].plot(kind=\"bar\", rot=0, ax=axes[1], title=\"Female\",\n",
    "                      legend=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython suppress id=dae34a4294ba4b7d8399d480222a8298\n",
    "#! figure,id=baby_names_last_letter,title=\"Proportion of boy and girl names ending in each letter\"\n",
    "plt.subplots_adjust(hspace=0.25)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=bf323785196547a0bed71781cfca7f5b\n",
    "letter_prop = table / table.sum()\n",
    "\n",
    "dny_ts = letter_prop.loc[[\"d\", \"n\", \"y\"], \"M\"].T\n",
    "dny_ts.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython suppress id=44c3ae58c11e4723869a9a36aafa582f\n",
    "plt.close(\"all\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython suppress id=ecaa55eb4a5a4c6b98547da72ce97cc7\n",
    "fig = plt.figure()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=18558439cbec454696a8d60c22092fee\n",
    "#! figure,id=baby_names_letter_over_time,title=\"Proportion of boys born with names ending in d/n/y over time\"\n",
    "dny_ts.plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=1e66b95ed2d74763b32127da7c80ec51\n",
    "all_names = pd.Series(top1000[\"name\"].unique())\n",
    "lesley_like = all_names[all_names.str.contains(\"Lesl\")]\n",
    "lesley_like"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=09ef1cec057c4361a6a765175b169d98\n",
    "filtered = top1000[top1000[\"name\"].isin(lesley_like)]\n",
    "filtered.groupby(\"name\")[\"births\"].sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=d4f30978887e471f9d5364c31a608faa\n",
    "table = filtered.pivot_table(\"births\", index=\"year\",\n",
    "                             columns=\"sex\", aggfunc=\"sum\")\n",
    "table = table.div(table.sum(axis=\"columns\"), axis=\"index\")\n",
    "table.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython suppress id=553d5251c8d84a7b93a66b0908bd7709\n",
    "fig = plt.figure()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=e706bc2402d645b58fdf4bdf78db9844\n",
    "#! figure,id=\"baby_names_lesley\",title=\"Proportion of male/female Lesley-like names over time\"\n",
    "table.plot(style={\"M\": \"k-\", \"F\": \"k--\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=b35c37358f2744baa362accb22c7b050\n",
    "import json\n",
    "db = json.load(open(\"datasets/usda_food/database.json\"))\n",
    "len(db)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=f04bd1e02f604ea4bd5d295855fd1289\n",
    "db[0].keys()\n",
    "db[0][\"nutrients\"][0]\n",
    "nutrients = pd.DataFrame(db[0][\"nutrients\"])\n",
    "nutrients.head(7)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=292500dd7d93484c98341fb4c74fa514\n",
    "info_keys = [\"description\", \"group\", \"id\", \"manufacturer\"]\n",
    "info = pd.DataFrame(db, columns=info_keys)\n",
    "info.head()\n",
    "info.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=7ed01e31319d4bb6af5ef058fb0c7afc\n",
    "pd.value_counts(info[\"group\"])[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython verbatim id=24c112fe1ee44d54bf747bae742861fc\n",
    "nutrients = []\n",
    "\n",
    "for rec in db:\n",
    "    fnuts = pd.DataFrame(rec[\"nutrients\"])\n",
    "    fnuts[\"id\"] = rec[\"id\"]\n",
    "    nutrients.append(fnuts)\n",
    "\n",
    "nutrients = pd.concat(nutrients, ignore_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=185aa7bd01684e93a298345bbca145e5\n",
    "nutrients"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=1c9de0a1f9734472a7ba009a638fa50e\n",
    "nutrients.duplicated().sum()  # number of duplicates\n",
    "nutrients = nutrients.drop_duplicates()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=fad42e28e4ef433db2b3ac9a1310758a\n",
    "col_mapping = {\"description\" : \"food\",\n",
    "               \"group\"       : \"fgroup\"}\n",
    "info = info.rename(columns=col_mapping, copy=False)\n",
    "info.info()\n",
    "col_mapping = {\"description\" : \"nutrient\",\n",
    "               \"group\" : \"nutgroup\"}\n",
    "nutrients = nutrients.rename(columns=col_mapping, copy=False)\n",
    "nutrients"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=3d1bb6fec99b4df09611f192fbd12462\n",
    "ndata = pd.merge(nutrients, info, on=\"id\")\n",
    "ndata.info()\n",
    "ndata.iloc[30000]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython suppress id=02da7933628d4350b5c495755caed9a4\n",
    "fig = plt.figure()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=276ba38a83974b82bbc4849eb024bcb7\n",
    "result = ndata.groupby([\"nutrient\", \"fgroup\"])[\"value\"].quantile(0.5)\n",
    "#! figure,id=fig_wrangle_zinc,width=4in,title=\"Median zinc values by food group\"\n",
    "result[\"Zinc, Zn\"].sort_values().plot(kind=\"barh\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython verbatim id=539c0a81efd34f95835032021f1e5ab6\n",
    "by_nutrient = ndata.groupby([\"nutgroup\", \"nutrient\"])\n",
    "\n",
    "def get_maximum(x):\n",
    "    return x.loc[x.value.idxmax()]\n",
    "\n",
    "max_foods = by_nutrient.apply(get_maximum)[[\"value\", \"food\"]]\n",
    "\n",
    "# make the food a little smaller\n",
    "max_foods[\"food\"] = max_foods[\"food\"].str[:50]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=bdb361d50f794410823645a33be3c365\n",
    "max_foods.loc[\"Amino Acids\"][\"food\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=1d39de88d04247a692ef7c90ac52edbc\n",
    "fec = pd.read_csv(\"datasets/fec/P00000001-ALL.csv\", low_memory=False)\n",
    "fec.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=00bb57847d384a1c868ac10bdf94db5e\n",
    "fec.iloc[123456]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=747ed603f23f442abe038ca6108295b6\n",
    "unique_cands = fec[\"cand_nm\"].unique()\n",
    "unique_cands\n",
    "unique_cands[2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython verbatim id=f3fee9f744a642bb93414dcdcc4b3eff\n",
    "parties = {\"Bachmann, Michelle\": \"Republican\",\n",
    "           \"Cain, Herman\": \"Republican\",\n",
    "           \"Gingrich, Newt\": \"Republican\",\n",
    "           \"Huntsman, Jon\": \"Republican\",\n",
    "           \"Johnson, Gary Earl\": \"Republican\",\n",
    "           \"McCotter, Thaddeus G\": \"Republican\",\n",
    "           \"Obama, Barack\": \"Democrat\",\n",
    "           \"Paul, Ron\": \"Republican\",\n",
    "           \"Pawlenty, Timothy\": \"Republican\",\n",
    "           \"Perry, Rick\": \"Republican\",\n",
    "           \"Roemer, Charles E. 'Buddy' III\": \"Republican\",\n",
    "           \"Romney, Mitt\": \"Republican\",\n",
    "           \"Santorum, Rick\": \"Republican\"}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=b659a8f33d0d48adad4fca5927da7a48\n",
    "fec[\"cand_nm\"][123456:123461]\n",
    "fec[\"cand_nm\"][123456:123461].map(parties)\n",
    "# Add it as a column\n",
    "fec[\"party\"] = fec[\"cand_nm\"].map(parties)\n",
    "fec[\"party\"].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=8d2df181b8d0474d9b3509df8c27c743\n",
    "(fec[\"contb_receipt_amt\"] > 0).value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=6fcec3476c974121b94a2147252276ca\n",
    "fec = fec[fec[\"contb_receipt_amt\"] > 0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=2f86e0fe1e334c6ab6597b4531d650a5\n",
    "fec_mrbo = fec[fec[\"cand_nm\"].isin([\"Obama, Barack\", \"Romney, Mitt\"])]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=5f3ce51804ef4ef29e4b443d69fc6e6d\n",
    "fec[\"contbr_occupation\"].value_counts()[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython verbatim id=c6c82bbba04f445282d55d072300889e\n",
    "occ_mapping = {\n",
    "   \"INFORMATION REQUESTED PER BEST EFFORTS\" : \"NOT PROVIDED\",\n",
    "   \"INFORMATION REQUESTED\" : \"NOT PROVIDED\",\n",
    "   \"INFORMATION REQUESTED (BEST EFFORTS)\" : \"NOT PROVIDED\",\n",
    "   \"C.E.O.\": \"CEO\"\n",
    "}\n",
    "\n",
    "def get_occ(x):\n",
    "    # If no mapping provided, return x\n",
    "    return occ_mapping.get(x, x)\n",
    "\n",
    "fec[\"contbr_occupation\"] = fec[\"contbr_occupation\"].map(get_occ)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython verbatim id=6e79332d5ea4476f89503731765aedc7\n",
    "emp_mapping = {\n",
    "   \"INFORMATION REQUESTED PER BEST EFFORTS\" : \"NOT PROVIDED\",\n",
    "   \"INFORMATION REQUESTED\" : \"NOT PROVIDED\",\n",
    "   \"SELF\" : \"SELF-EMPLOYED\",\n",
    "   \"SELF EMPLOYED\" : \"SELF-EMPLOYED\",\n",
    "}\n",
    "\n",
    "def get_emp(x):\n",
    "    # If no mapping provided, return x\n",
    "    return emp_mapping.get(x, x)\n",
    "\n",
    "fec[\"contbr_employer\"] = fec[\"contbr_employer\"].map(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=52c5ed9cbb6042a2a31d92b66349d3df\n",
    "by_occupation = fec.pivot_table(\"contb_receipt_amt\",\n",
    "                                index=\"contbr_occupation\",\n",
    "                                columns=\"party\", aggfunc=\"sum\")\n",
    "over_2mm = by_occupation[by_occupation.sum(axis=\"columns\") > 2000000]\n",
    "over_2mm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython suppress id=ce914115742b401dbe73c3a8574063d3\n",
    "plt.figure()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=33becfe73ad14bd584bf661685329273\n",
    "#! figure,id=groupby_fec_occ_party,width=4.5in,title=\"Total donations by party for top occupations\"\n",
    "over_2mm.plot(kind=\"barh\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython verbatim id=3dcbd1e24a774aa0a87f1459733b8ecf\n",
    "def get_top_amounts(group, key, n=5):\n",
    "    totals = group.groupby(key)[\"contb_receipt_amt\"].sum()\n",
    "    return totals.nlargest(n)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=ed90d977b2e8476ab7a241886ac725ca\n",
    "grouped = fec_mrbo.groupby(\"cand_nm\")\n",
    "grouped.apply(get_top_amounts, \"contbr_occupation\", n=7)\n",
    "grouped.apply(get_top_amounts, \"contbr_employer\", n=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=9430750304c744018ebfe44fa7825f15\n",
    "bins = np.array([0, 1, 10, 100, 1000, 10000,\n",
    "                 100_000, 1_000_000, 10_000_000])\n",
    "labels = pd.cut(fec_mrbo[\"contb_receipt_amt\"], bins)\n",
    "labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=53f9ac1f658442ab824c5684308e083d\n",
    "grouped = fec_mrbo.groupby([\"cand_nm\", labels])\n",
    "grouped.size().unstack(level=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython suppress id=6e9668803e4f4b6ea96904c4ae9d7d1c\n",
    "plt.figure()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=94fab00fdb7847b88c42b8faba89c0b2\n",
    "bucket_sums = grouped[\"contb_receipt_amt\"].sum().unstack(level=0)\n",
    "normed_sums = bucket_sums.div(bucket_sums.sum(axis=\"columns\"),\n",
    "                              axis=\"index\")\n",
    "normed_sums\n",
    "#! figure,id=fig_groupby_fec_bucket,width=4.5in,title=\"Percentage of total donations received by candidates for each donation size\"\n",
    "normed_sums[:-2].plot(kind=\"barh\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=bcffb8dcc3b741fe87058d44f39324f1\n",
    "grouped = fec_mrbo.groupby([\"cand_nm\", \"contbr_st\"])\n",
    "totals = grouped[\"contb_receipt_amt\"].sum().unstack(level=0).fillna(0)\n",
    "totals = totals[totals.sum(axis=\"columns\") > 100000]\n",
    "totals.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=eda82c69272d44b3a8eb23c0adb6ede4\n",
    "percent = totals.div(totals.sum(axis=\"columns\"), axis=\"index\")\n",
    "percent.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython suppress id=c7894724cf8b4ce087cfee42b10068b3\n",
    "%popd"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}