From ff839e598b053c76cf7e55ba1c631d79c430b75e Mon Sep 17 00:00:00 2001 From: Mike Matera Date: Wed, 27 Jan 2021 20:50:28 -0800 Subject: [PATCH] Update sets to be compatible with colab. --- notebooks/DatabaseConceptsBook/pets_joins.py | 3 +- .../DatabaseConceptsBook/queen_anne_joins.py | 3 +- notebooks/Flights/intermediate_selects.py | 3 +- notebooks/Flights/simple_selects.py | 3 +- notebooks/Population/advanced_selects.py | 5 +- tools/gen_questions.py | 104 ++++++++++-------- 6 files changed, 70 insertions(+), 51 deletions(-) diff --git a/notebooks/DatabaseConceptsBook/pets_joins.py b/notebooks/DatabaseConceptsBook/pets_joins.py index 660d759..0a2ca1a 100644 --- a/notebooks/DatabaseConceptsBook/pets_joins.py +++ b/notebooks/DatabaseConceptsBook/pets_joins.py @@ -2,7 +2,8 @@ # Pet Joins """ -db_url = ('sqlite:///pets.sqlite3') +db_url = 'http://www.lifealgorithmic.com/_static/databases/pets.sqlite3' +#db_url = ('sqlite:///pets.sqlite3') class Question01: """ diff --git a/notebooks/DatabaseConceptsBook/queen_anne_joins.py b/notebooks/DatabaseConceptsBook/queen_anne_joins.py index f82c9bd..c1266b2 100644 --- a/notebooks/DatabaseConceptsBook/queen_anne_joins.py +++ b/notebooks/DatabaseConceptsBook/queen_anne_joins.py @@ -2,7 +2,8 @@ # Queen Anne Joins """ -db_url = 'sqlite:///queen_anne.sqlite3' +db_url = 'http://www.lifealgorithmic.com/_static/databases/queen_anne.sqlite3' +#db_url = 'sqlite:///queen_anne.sqlite3' class Question01: """ diff --git a/notebooks/Flights/intermediate_selects.py b/notebooks/Flights/intermediate_selects.py index 4583f92..a37b3a1 100644 --- a/notebooks/Flights/intermediate_selects.py +++ b/notebooks/Flights/intermediate_selects.py @@ -5,7 +5,8 @@ `order by` and `limit` clauses and the `avg()` function. """ -db_url = 'sqlite:///flights.sqlite3' +db_url = 'http://www.lifealgorithmic.com/_static/databases/flights.sqlite3' +#db_url = 'sqlite:///flights.sqlite3' class Question01: """ diff --git a/notebooks/Flights/simple_selects.py b/notebooks/Flights/simple_selects.py index 63a35e5..9873bbd 100644 --- a/notebooks/Flights/simple_selects.py +++ b/notebooks/Flights/simple_selects.py @@ -4,7 +4,8 @@ This assignment will get you started writing select statements in SQL. """ -db_url = "sqlite:///flights.sqlite3" +db_url = 'http://www.lifealgorithmic.com/_static/databases/flights.sqlite3' +#db_url = "sqlite:///flights.sqlite3" class Question01: """ diff --git a/notebooks/Population/advanced_selects.py b/notebooks/Population/advanced_selects.py index 0a11e70..fc54ddc 100644 --- a/notebooks/Population/advanced_selects.py +++ b/notebooks/Population/advanced_selects.py @@ -2,7 +2,8 @@ # Population Queries """ -db_url = 'sqlite:///population.sqlite3' +db_url = 'http://www.lifealgorithmic.com/_static/databases/population.sqlite3' +#db_url = 'sqlite:///population.sqlite3' class Question01: """ @@ -11,7 +12,7 @@ class Question01: Write a query that select all rows from the `population` table. """ - answer = "select * from population limit 5;" + answer = "select * from population;" class Question02: """ diff --git a/tools/gen_questions.py b/tools/gen_questions.py index a92f757..5c6f59b 100644 --- a/tools/gen_questions.py +++ b/tools/gen_questions.py @@ -32,8 +32,13 @@ import pandas as pd import sys import re +import requests +import shutil +import tempfile +import pathlib +import subprocess -query_limit = 50 +query_limit = 500 setup_md_head = f"""# Setup The code in the cell below connects your notebook to the database. Queries are limited to @@ -42,9 +47,13 @@ are also limited to {query_limit} results. """ -setup_code = f"""%load_ext sql +setup_code = f"""import pathlib +import subprocess +if not pathlib.Path('{{filename}}').exists(): + subprocess.run('wget {{url}}', shell=True) +%load_ext sql %config SqlMagic.autolimit={query_limit} -%sql {{url}}""" +%sql sqlite:///{{filename}}""" setup_md_tail = """*Run this cell before you begin.*""" @@ -79,49 +88,54 @@ def main(): mod = import_questions(args.mod) outfile = args.mod + '.ipynb' - # Get the db_url variable. + # Download the DB db_url = mod.db_url - engine = db.create_engine(db_url) - - # Search for questions - questions = [] - for key in sorted(mod.__dict__): - if key.startswith('Question') and inspect.isclass(mod.__dict__[key]): - questions.append(mod.__dict__[key]) - - # Generate the document. - nb = nbformat.v4.new_notebook() - nb["metadata"] = { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3", - }, - } - nb.cells.append(nbformat.v4.new_markdown_cell(mod.__doc__)) - nb.cells.append(nbformat.v4.new_markdown_cell(setup_md_head)) - nb.cells.append(nbformat.v4.new_code_cell(setup_code.format(url=db_url))) - nb.cells.append(nbformat.v4.new_markdown_cell(setup_md_tail)) - - for q in questions: - # Question cell - nb.cells.append(nbformat.v4.new_markdown_cell(q.__doc__)) - - # Preview cell - query = q.answer.replace(';', '') - if re.search(r'(?i)limit', query) is None: - query += f' limit {query_limit}' - - df = pd.read_sql_query(query, engine) - html = df.to_html(index=False, notebook=True, max_rows=5) - rows = df.shape[0] - nb.cells.append(nbformat.v4.new_markdown_cell( - preview_md.format( - preview_html = html, - rows = rows))) - - # Answer cell - nb.cells.append(nbformat.v4.new_code_cell("%%sql\n\n")) + filename = db_url.split('/')[-1] + with tempfile.TemporaryDirectory() as tmpdir: + subprocess.run(f"wget {db_url}", shell=True, cwd=tmpdir) + + # Get the db_url variable. + engine = db.create_engine(f"sqlite:///{tmpdir}/{filename}") + + # Search for questions + questions = [] + for key in sorted(mod.__dict__): + if key.startswith('Question') and inspect.isclass(mod.__dict__[key]): + questions.append(mod.__dict__[key]) + + # Generate the document. + nb = nbformat.v4.new_notebook() + nb["metadata"] = { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3", + }, + } + nb.cells.append(nbformat.v4.new_markdown_cell(mod.__doc__)) + nb.cells.append(nbformat.v4.new_markdown_cell(setup_md_head)) + nb.cells.append(nbformat.v4.new_code_cell(setup_code.format(url=db_url, filename=filename))) + nb.cells.append(nbformat.v4.new_markdown_cell(setup_md_tail)) + + for q in questions: + # Question cell + nb.cells.append(nbformat.v4.new_markdown_cell(q.__doc__)) + + # Preview cell + query = q.answer.replace(';', '') + if re.search(r'(?i)limit', query) is None: + query += f' limit {query_limit}' + + df = pd.read_sql_query(query, engine) + html = df.to_html(index=False, notebook=True, max_rows=10) + rows = df.shape[0] + nb.cells.append(nbformat.v4.new_markdown_cell( + preview_md.format( + preview_html = html, + rows = rows))) + + # Answer cell + nb.cells.append(nbformat.v4.new_code_cell("%%sql\n\n")) nbformat.write(nb, outfile)