solutions.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Solutions"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## #1: Display the total number of watchers per language (ignore repos w/o a language)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Solution"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "C# 35\n",
      "C++ 64\n",
      "HTML 352\n",
      "JavaScript 4702\n",
      "Jupyter Notebook 5573\n",
      "PHP 129\n",
      "Python 16235\n",
      "R 18\n"
     ]
    }
   ],
   "source": [
    "from urllib.request import urlopen\n",
    "from itertools import groupby\n",
    "from operator import itemgetter\n",
    "from ijson import items\n",
    "\n",
    "url2 = 'https://api.github.com/search/repositories?q=data'\n",
    "f = urlopen(url2)\n",
    "repos = items(f, 'items.item')\n",
    "\n",
    "keyfunc = itemgetter('language')\n",
    "cleaned = filter(keyfunc, repos)\n",
    "records = sorted(cleaned, key=keyfunc)\n",
    "grouped = groupby(records, keyfunc)\n",
    "\n",
    "for key, group in grouped:\n",
    "    cnt = sum(g['watchers'] for g in group)\n",
    "    print(key, cnt)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## #2: Language with the most number of watchers, per `owner_type` per `has_pages`"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Solution"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'archive_url': 'https://api.github.com/repos/d3/d3/{archive_format}{/ref}',\n",
       " 'assignees_url': 'https://api.github.com/repos/d3/d3/assignees{/user}',\n",
       " 'blobs_url': 'https://api.github.com/repos/d3/d3/git/blobs{/sha}',\n",
       " 'branches_url': 'https://api.github.com/repos/d3/d3/branches{/branch}',\n",
       " 'clone_url': 'https://github.com/d3/d3.git',\n",
       " 'collaborators_url': 'https://api.github.com/repos/d3/d3/collaborators{/collaborator}',\n",
       " 'comments_url': 'https://api.github.com/repos/d3/d3/comments{/number}',\n",
       " 'commits_url': 'https://api.github.com/repos/d3/d3/commits{/sha}',\n",
       " 'compare_url': 'https://api.github.com/repos/d3/d3/compare/{base}...{head}',\n",
       " 'contents_url': 'https://api.github.com/repos/d3/d3/contents/{+path}',\n",
       " 'contributors_url': 'https://api.github.com/repos/d3/d3/contributors',\n",
       " 'created_at': '2010-09-27T17:22:42Z',\n",
       " 'default_branch': 'master',\n",
       " 'deployments_url': 'https://api.github.com/repos/d3/d3/deployments',\n",
       " 'description': 'Bring data to life with SVG, Canvas and HTML. :bar_chart::chart_with_upwards_trend::tada:',\n",
       " 'downloads_url': 'https://api.github.com/repos/d3/d3/downloads',\n",
       " 'events_url': 'https://api.github.com/repos/d3/d3/events',\n",
       " 'fork': False,\n",
       " 'forks': 16964,\n",
       " 'forks_count': 16964,\n",
       " 'forks_url': 'https://api.github.com/repos/d3/d3/forks',\n",
       " 'full_name': 'd3/d3',\n",
       " 'git_commits_url': 'https://api.github.com/repos/d3/d3/git/commits{/sha}',\n",
       " 'git_refs_url': 'https://api.github.com/repos/d3/d3/git/refs{/sha}',\n",
       " 'git_tags_url': 'https://api.github.com/repos/d3/d3/git/tags{/sha}',\n",
       " 'git_url': 'git://github.com/d3/d3.git',\n",
       " 'has_downloads': True,\n",
       " 'has_issues': True,\n",
       " 'has_pages': True,\n",
       " 'has_projects': False,\n",
       " 'has_wiki': True,\n",
       " 'homepage': 'https://d3js.org',\n",
       " 'hooks_url': 'https://api.github.com/repos/d3/d3/hooks',\n",
       " 'html_url': 'https://github.com/d3/d3',\n",
       " 'id': 943149,\n",
       " 'issue_comment_url': 'https://api.github.com/repos/d3/d3/issues/comments{/number}',\n",
       " 'issue_events_url': 'https://api.github.com/repos/d3/d3/issues/events{/number}',\n",
       " 'issues_url': 'https://api.github.com/repos/d3/d3/issues{/number}',\n",
       " 'keys_url': 'https://api.github.com/repos/d3/d3/keys{/key_id}',\n",
       " 'labels_url': 'https://api.github.com/repos/d3/d3/labels{/name}',\n",
       " 'language': 'JavaScript',\n",
       " 'languages_url': 'https://api.github.com/repos/d3/d3/languages',\n",
       " 'merges_url': 'https://api.github.com/repos/d3/d3/merges',\n",
       " 'milestones_url': 'https://api.github.com/repos/d3/d3/milestones{/number}',\n",
       " 'mirror_url': None,\n",
       " 'name': 'd3',\n",
       " 'notifications_url': 'https://api.github.com/repos/d3/d3/notifications{?since,all,participating}',\n",
       " 'open_issues': 0,\n",
       " 'open_issues_count': 0,\n",
       " 'owner': {'avatar_url': 'https://avatars2.githubusercontent.com/u/1562726?v=3',\n",
       "  'events_url': 'https://api.github.com/users/d3/events{/privacy}',\n",
       "  'followers_url': 'https://api.github.com/users/d3/followers',\n",
       "  'following_url': 'https://api.github.com/users/d3/following{/other_user}',\n",
       "  'gists_url': 'https://api.github.com/users/d3/gists{/gist_id}',\n",
       "  'gravatar_id': '',\n",
       "  'html_url': 'https://github.com/d3',\n",
       "  'id': 1562726,\n",
       "  'login': 'd3',\n",
       "  'organizations_url': 'https://api.github.com/users/d3/orgs',\n",
       "  'received_events_url': 'https://api.github.com/users/d3/received_events',\n",
       "  'repos_url': 'https://api.github.com/users/d3/repos',\n",
       "  'site_admin': False,\n",
       "  'starred_url': 'https://api.github.com/users/d3/starred{/owner}{/repo}',\n",
       "  'subscriptions_url': 'https://api.github.com/users/d3/subscriptions',\n",
       "  'type': 'Organization',\n",
       "  'url': 'https://api.github.com/users/d3'},\n",
       " 'private': False,\n",
       " 'pulls_url': 'https://api.github.com/repos/d3/d3/pulls{/number}',\n",
       " 'pushed_at': '2017-05-16T14:50:33Z',\n",
       " 'releases_url': 'https://api.github.com/repos/d3/d3/releases{/id}',\n",
       " 'score': Decimal('7.8259406'),\n",
       " 'size': 41570,\n",
       " 'ssh_url': 'git@github.com:d3/d3.git',\n",
       " 'stargazers_count': 64706,\n",
       " 'stargazers_url': 'https://api.github.com/repos/d3/d3/stargazers',\n",
       " 'statuses_url': 'https://api.github.com/repos/d3/d3/statuses/{sha}',\n",
       " 'subscribers_url': 'https://api.github.com/repos/d3/d3/subscribers',\n",
       " 'subscription_url': 'https://api.github.com/repos/d3/d3/subscription',\n",
       " 'svn_url': 'https://github.com/d3/d3',\n",
       " 'tags_url': 'https://api.github.com/repos/d3/d3/tags',\n",
       " 'teams_url': 'https://api.github.com/repos/d3/d3/teams',\n",
       " 'trees_url': 'https://api.github.com/repos/d3/d3/git/trees{/sha}',\n",
       " 'updated_at': '2017-05-25T21:34:57Z',\n",
       " 'url': 'https://api.github.com/repos/d3/d3',\n",
       " 'watchers': 64706,\n",
       " 'watchers_count': 64706}"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from urllib.request import urlopen\n",
    "from operator import itemgetter\n",
    "from functools import partial\n",
    "from meza import process as pr, fntools as ft\n",
    "from meza.io import read_json\n",
    "\n",
    "url4 = 'https://api.github.com/search/repositories?q=data&sort=stars&order=desc'\n",
    "f = urlopen(url4)\n",
    "records = read_json(f, path='items.item')\n",
    "\n",
    "# repos without a language have a value of None, which meza doesn't like\n",
    "filled = pr.fillempty(records, value='', fields=['language'])\n",
    "filled, preview = pr.peek(filled)\n",
    "preview[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'archive_url': 'https://api.github.com/repos/d3/d3/{archive_format}{/ref}',\n",
       " 'assignees_url': 'https://api.github.com/repos/d3/d3/assignees{/user}',\n",
       " 'blobs_url': 'https://api.github.com/repos/d3/d3/git/blobs{/sha}',\n",
       " 'branches_url': 'https://api.github.com/repos/d3/d3/branches{/branch}',\n",
       " 'clone_url': 'https://github.com/d3/d3.git',\n",
       " 'collaborators_url': 'https://api.github.com/repos/d3/d3/collaborators{/collaborator}',\n",
       " 'comments_url': 'https://api.github.com/repos/d3/d3/comments{/number}',\n",
       " 'commits_url': 'https://api.github.com/repos/d3/d3/commits{/sha}',\n",
       " 'compare_url': 'https://api.github.com/repos/d3/d3/compare/{base}...{head}',\n",
       " 'contents_url': 'https://api.github.com/repos/d3/d3/contents/{+path}',\n",
       " 'contributors_url': 'https://api.github.com/repos/d3/d3/contributors',\n",
       " 'created_at': '2010-09-27T17:22:42Z',\n",
       " 'default_branch': 'master',\n",
       " 'deployments_url': 'https://api.github.com/repos/d3/d3/deployments',\n",
       " 'description': 'Bring data to life with SVG, Canvas and HTML. :bar_chart::chart_with_upwards_trend::tada:',\n",
       " 'downloads_url': 'https://api.github.com/repos/d3/d3/downloads',\n",
       " 'events_url': 'https://api.github.com/repos/d3/d3/events',\n",
       " 'fork': False,\n",
       " 'forks': 16964,\n",
       " 'forks_count': 16964,\n",
       " 'forks_url': 'https://api.github.com/repos/d3/d3/forks',\n",
       " 'full_name': 'd3/d3',\n",
       " 'git_commits_url': 'https://api.github.com/repos/d3/d3/git/commits{/sha}',\n",
       " 'git_refs_url': 'https://api.github.com/repos/d3/d3/git/refs{/sha}',\n",
       " 'git_tags_url': 'https://api.github.com/repos/d3/d3/git/tags{/sha}',\n",
       " 'git_url': 'git://github.com/d3/d3.git',\n",
       " 'has_downloads': True,\n",
       " 'has_issues': True,\n",
       " 'has_pages': True,\n",
       " 'has_projects': False,\n",
       " 'has_wiki': True,\n",
       " 'homepage': 'https://d3js.org',\n",
       " 'hooks_url': 'https://api.github.com/repos/d3/d3/hooks',\n",
       " 'html_url': 'https://github.com/d3/d3',\n",
       " 'id': 943149,\n",
       " 'issue_comment_url': 'https://api.github.com/repos/d3/d3/issues/comments{/number}',\n",
       " 'issue_events_url': 'https://api.github.com/repos/d3/d3/issues/events{/number}',\n",
       " 'issues_url': 'https://api.github.com/repos/d3/d3/issues{/number}',\n",
       " 'keys_url': 'https://api.github.com/repos/d3/d3/keys{/key_id}',\n",
       " 'labels_url': 'https://api.github.com/repos/d3/d3/labels{/name}',\n",
       " 'language': 'JavaScript',\n",
       " 'languages_url': 'https://api.github.com/repos/d3/d3/languages',\n",
       " 'merges_url': 'https://api.github.com/repos/d3/d3/merges',\n",
       " 'milestones_url': 'https://api.github.com/repos/d3/d3/milestones{/number}',\n",
       " 'mirror_url': None,\n",
       " 'name': 'd3',\n",
       " 'notifications_url': 'https://api.github.com/repos/d3/d3/notifications{?since,all,participating}',\n",
       " 'open_issues': 0,\n",
       " 'open_issues_count': 0,\n",
       " 'owner_avatar_url': 'https://avatars2.githubusercontent.com/u/1562726?v=3',\n",
       " 'owner_events_url': 'https://api.github.com/users/d3/events{/privacy}',\n",
       " 'owner_followers_url': 'https://api.github.com/users/d3/followers',\n",
       " 'owner_following_url': 'https://api.github.com/users/d3/following{/other_user}',\n",
       " 'owner_gists_url': 'https://api.github.com/users/d3/gists{/gist_id}',\n",
       " 'owner_gravatar_id': '',\n",
       " 'owner_html_url': 'https://github.com/d3',\n",
       " 'owner_id': 1562726,\n",
       " 'owner_login': 'd3',\n",
       " 'owner_organizations_url': 'https://api.github.com/users/d3/orgs',\n",
       " 'owner_received_events_url': 'https://api.github.com/users/d3/received_events',\n",
       " 'owner_repos_url': 'https://api.github.com/users/d3/repos',\n",
       " 'owner_site_admin': False,\n",
       " 'owner_starred_url': 'https://api.github.com/users/d3/starred{/owner}{/repo}',\n",
       " 'owner_subscriptions_url': 'https://api.github.com/users/d3/subscriptions',\n",
       " 'owner_type': 'Organization',\n",
       " 'owner_url': 'https://api.github.com/users/d3',\n",
       " 'private': False,\n",
       " 'pulls_url': 'https://api.github.com/repos/d3/d3/pulls{/number}',\n",
       " 'pushed_at': '2017-05-16T14:50:33Z',\n",
       " 'releases_url': 'https://api.github.com/repos/d3/d3/releases{/id}',\n",
       " 'score': Decimal('7.8259406'),\n",
       " 'size': 41570,\n",
       " 'ssh_url': 'git@github.com:d3/d3.git',\n",
       " 'stargazers_count': 64706,\n",
       " 'stargazers_url': 'https://api.github.com/repos/d3/d3/stargazers',\n",
       " 'statuses_url': 'https://api.github.com/repos/d3/d3/statuses/{sha}',\n",
       " 'subscribers_url': 'https://api.github.com/repos/d3/d3/subscribers',\n",
       " 'subscription_url': 'https://api.github.com/repos/d3/d3/subscription',\n",
       " 'svn_url': 'https://github.com/d3/d3',\n",
       " 'tags_url': 'https://api.github.com/repos/d3/d3/tags',\n",
       " 'teams_url': 'https://api.github.com/repos/d3/d3/teams',\n",
       " 'trees_url': 'https://api.github.com/repos/d3/d3/git/trees{/sha}',\n",
       " 'updated_at': '2017-05-25T21:34:57Z',\n",
       " 'url': 'https://api.github.com/repos/d3/d3',\n",
       " 'watchers': 64706,\n",
       " 'watchers_count': 64706}"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# meza doesn't do well with nested dicts\n",
    "flat = (dict(ft.flatten(r)) for r in filled)\n",
    "flat, preview = pr.peek(flat)\n",
    "preview[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'C#': 7772,\n",
       " 'C++': 58473,\n",
       " 'Go': 13510,\n",
       " 'Objective-C': 10702,\n",
       " 'Ruby': 7504,\n",
       " 'Swift': 27142,\n",
       " 'has_pages': False,\n",
       " 'owner_type': 'Organization'}"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# `watchers` is the pivot field to aggregate by\n",
    "# `language` is the pivot field to group by\n",
    "args = ('watchers', 'language')\n",
    "\n",
    "# the pivot fields we want to include in each row\n",
    "rows = ['has_pages', 'owner_type']\n",
    "pivotted = pr.pivot(flat, *args, rows=rows, op=sum)\n",
    "pivotted, preview = pr.peek(pivotted)\n",
    "preview[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'has_pages': False,\n",
       " 'language': 'Objective-C',\n",
       " 'owner_type': 'Organization',\n",
       " 'watchers': 10702}"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# `rows` are the fields we don't want to normalize (since `invert` is true)\n",
    "kwargs = {'rows': rows, 'invert': True}\n",
    "\n",
    "# `watchers` is the field to use for the normalized values\n",
    "# `language` is the field to use for the normalized key\n",
    "normal = pr.normalize(pivotted, *args, **kwargs)\n",
    "normal, preview = pr.peek(normal)\n",
    "preview[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'has_pages': False,\n",
       " 'language': 'C++',\n",
       " 'owner_type': 'Organization',\n",
       " 'watchers': 58473}"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# aggregate by `watchers`\n",
    "agg_keyfunc = itemgetter('watchers')\n",
    "\n",
    "# group by `has_pages` and `owner_type`\n",
    "group_keyfunc = lambda x: tuple(x[r] for r in rows)\n",
    "aggregator = partial(max, key=agg_keyfunc)\n",
    "\n",
    "# Only emit the groups, not the group key (since `tupled` is False)\n",
    "kwargs = {'tupled': False, 'aggregator': aggregator}\n",
    "\n",
    "grouped = pr.group(normal, group_keyfunc, **kwargs)\n",
    "grouped, preview = pr.peek(grouped)\n",
    "preview[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'has_pages': True,\n",
      " 'language': 'JavaScript',\n",
      " 'owner_type': 'Organization',\n",
      " 'watchers': 130152}\n",
      "{'has_pages': False,\n",
      " 'language': 'C++',\n",
      " 'owner_type': 'Organization',\n",
      " 'watchers': 58473}\n",
      "{'has_pages': False,\n",
      " 'language': 'Python',\n",
      " 'owner_type': 'User',\n",
      " 'watchers': 48543}\n",
      "{'has_pages': True,\n",
      " 'language': 'JavaScript',\n",
      " 'owner_type': 'User',\n",
      " 'watchers': 10285}\n"
     ]
    }
   ],
   "source": [
    "from pprint import pprint\n",
    "\n",
    "sgrouped = sorted(grouped, key=agg_keyfunc, reverse=True)\n",
    "\n",
    "for record in sgrouped:\n",
    "    pprint(record)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}