Add documentation for knockoff run cli. Fix issue with --ephemeral fl…

…ag for knockoff run cli

CHANGELOG.md

@@ -4,7 +4,8 @@ All notable changes to this project will be documented in this file.
 `knockoff-factory` adheres to [Semantic Versioning](http://semver.org/).
 
 #### 4.x Releases
-- `4.1.x` Releases - [4.1.0](#410) | [4.2.0](#420)
+- `4.2.x` Releases - [4.2.0](#420) | [4.2.1](#421)
+- `4.1.x` Releases - [4.1.0](#410)
 - `4.0.x` Releases - [4.0.0](#400)
 
 #### 3.x Releases
@@ -41,8 +42,29 @@ All notable changes to this project will be documented in this file.
 
 ---
 
+## 4.2.1
+
+#### Added
+- Documentation and jupyter notebook for the `knockoff run` CLI
+    - Added by [Gregory Yu](https://github.com/gregyu) in Pull Request [#5](https://github.com/Nike-Inc/knockoff-factory/pull/5)
+- Added default configurations for `knockoff run` CLI with environment variable override options
+    - Added by [Gregory Yu](https://github.com/gregyu) in Pull Request [#5](https://github.com/Nike-Inc/knockoff-factory/pull/5)
+
+
+#### Updated
+- Moved clear_env_vars from `knockoff.orm` to `knockoff.utilities.environ` 
+    - Added by [Gregory Yu](https://github.com/gregyu) in Pull Request [#5](https://github.com/Nike-Inc/knockoff-factory/pull/5)
+
+#### Fixed
+- Fixed issue where `knockoff run` CLI was not using the temp db created with the `--ephemeral` flag
+    - Fixed by [Gregory Yu](https://github.com/gregyu) in Pull Request [#5](https://github.com/Nike-Inc/knockoff-factory/pull/5)
+
+---
+
 ## 4.2.0
 
+Note: The python package and docker image for this version was released as **4.1.1**.
+
 #### Added
 - Add --ephemeral flag for `knockoff run` CLI to create temp database for loading knockoff configuration from sdk
     - Added by [Gregory Yu](https://github.com/gregyu) in Pull Request [#4](https://github.com/Nike-Inc/knockoff-factory/pull/4)

README.md

@@ -36,7 +36,7 @@ We are working on adding more documentation and examples!
     * [KnockoffTable](notebook/KnockoffTable.ipynb)
     * [KnockoffDB](notebook/KnockoffDB.ipynb)
 * [TempDatabaseService](notebook/TempDatabaseService.ipynb)
-* Knockoff CLI
+* [Knockoff CLI](notebook/KnockoffCLI.ipynb)
 * Unit Testing Example: Sample App
 
 

knockoff/command/run.py

@@ -4,52 +4,44 @@
 # This source code is licensed under the Apache-2.0 license found in
 # the LICENSE file in the root directory of this source tree.
 
-
+import os
 import argparse
 import sys
 import logging
 
+from knockoff.utilities.environ import clear_env_vars
 from knockoff.utilities.ioc import get_container
 from knockoff.sdk.db import KnockoffDB, DefaultDatabaseService
 from knockoff.sdk.blueprint import Blueprint
-from knockoff.tempdb.db import TempDatabaseService
 
 logger = logging.getLogger(__name__)
 
 
+DEFAULT_KNOCKOFF_CONTAINER = "knockoff.sdk.container.default:KnockoffContainer"
+DEFAULT_TEMPDB_CONTAINER = "knockoff.tempdb.container:TempDBContainer"
+
+KNOCKOFF_RUN_DB_URL_ENV = "KNOCKOFF_RUN_DB_URL"
+KNOCKOFF_RUN_BLUEPRINT_PLAN_ENV = "KNOCKOFF_RUN_BLUEPRINT_PLAN"
+
+
+def clear_run_env_vars():
+    clear_env_vars([
+        KNOCKOFF_RUN_DB_URL_ENV,
+        KNOCKOFF_RUN_BLUEPRINT_PLAN_ENV
+    ])
+
+
 def testable_input(prompt=None, **kwargs):
     """pass through input so this can be unit tested"""
     input(prompt)
 
 
 def run(knockoff_db: KnockoffDB,
-        blueprint: Blueprint,
-        temp_db: TempDatabaseService = None):
-
-    if temp_db:
-        temp_url = temp_db.start()
-        logger.info("TempDatabaseService created temp database:\n"
-                    f"{temp_url}")
-
+        blueprint: Blueprint):
     dfs, knockoff_db = blueprint.construct(knockoff_db)
     knockoff_db.insert()
     logger.info("knockoff data successfully loaded into database.")
 
-    if temp_db:
-        try:
-            testable_input(
-                "Press Enter when finished to destroy temp database.",
-                # the following kwargs can be used by a mock for assertions
-                test_temp_url=temp_url,
-                test_temp_db=temp_db,
-                test_blueprint=blueprint,
-                test_knockoff_db=knockoff_db,
-            )
-        finally:
-            temp_db.stop()
-
-    logger.info("knockoff done.")
-
 
 def seed(i):
     """TODO: move this to a utility module?"""
@@ -63,20 +55,52 @@ def seed(i):
                 f"modules have been set to {i}")
 
 
+def default_config(container_package):
+    return {
+        DEFAULT_KNOCKOFF_CONTAINER: {
+            "database_service": {
+                "url": os.getenv(
+                    KNOCKOFF_RUN_DB_URL_ENV,
+                    "postgresql://postgres@localhost:5432/postgres"
+                )
+            },
+            "blueprint": {
+                "plan": {
+                    "package": os.getenv(
+                        KNOCKOFF_RUN_BLUEPRINT_PLAN_ENV,
+                        "knockoff.sdk.blueprint:noplan"
+                    )
+                }
+            }
+        },
+        DEFAULT_TEMPDB_CONTAINER: {
+            "tempdb": {
+                "url": os.getenv(
+                    KNOCKOFF_RUN_DB_URL_ENV,
+                    "postgresql://postgres@localhost:5432/postgres"
+                ),
+                "setup_teardown": {
+                    "package": "knockoff.tempdb.setup_teardown:postgres_setup_teardown"
+                }
+            }
+        }
+    }.get(container_package)
+
+
 def parse_args(argv=None):
     parser = argparse.ArgumentParser(
         usage='''knockoff run [<args>]'''
     )
     parser.add_argument("-c", "--container",
-                        default="knockoff.sdk.container.default:KnockoffContainer",
+                        default=DEFAULT_KNOCKOFF_CONTAINER,
                         help="Default KnockoffContainer")
     parser.add_argument("--yaml-config",
                         help="Container configuration")
     parser.add_argument("--ephemeral",
                         action="store_true",
                         help="flag to run interactively with an ephemeral database")
     parser.add_argument("--tempdb-container",
-                        default="knockoff.tempdb.container:TempDBContainer",
+                        default=DEFAULT_TEMPDB_CONTAINER,
                         help="Default TempDBContainer")
     parser.add_argument("-s", "--seed", type=int,
                         help="Set seed")
@@ -89,16 +113,50 @@ def main(argv=None):
     if args.seed:
         seed(args.seed)
 
-    container = get_container(args.container, args.yaml_config)
+    override_dict = None
+
+    if args.ephemeral:
+        tempdb_container = get_container(
+            args.tempdb_container,
+            config_path=args.yaml_config,
+            default_dict=default_config(args.tempdb_container)
+        )
+
+        temp_db = tempdb_container.temp_db()
+        temp_url = temp_db.start()
+        logger.info("TempDatabaseService created temp database:\n"
+                    f"{temp_url}")
+
+        # this overrides the configured url for the database service
+        # with temp_url
+        override_dict = {"database_service": {"url": temp_url}}
+
+    container = get_container(
+        args.container,
+        config_path=args.yaml_config,
+        default_dict=default_config(args.container),
+        override_dict=override_dict
+    )
+
     knockoff_db = container.knockoff_db()
     blueprint = container.blueprint()
-    temp_db = None
+
+    run(knockoff_db, blueprint)
 
     if args.ephemeral:
-        tempdb_container = get_container(args.tempdb_container, args.yaml_config)
-        temp_db = tempdb_container.temp_db()
+        try:
+            testable_input(
+                "Press Enter when finished to destroy temp database.",
+                # the following kwargs are used by a mock for assertions
+                test_temp_url=temp_url,
+                test_temp_db=temp_db,
+                test_blueprint=blueprint,
+                test_knockoff_db=knockoff_db,
+            )
+        finally:
+            temp_db.stop()
 
-    run(knockoff_db, blueprint, temp_db=temp_db)
+    logger.info("knockoff done.")
 
 
 if __name__ == "__main__":

knockoff/orm.py

@@ -5,11 +5,11 @@
 # the LICENSE file in the root directory of this source tree.
 
 
-import os
 import pandas as pd
 from sqlalchemy.pool import NullPool
 
 from .utilities.orm.sql import EngineBuilder
+from .utilities.environ import clear_env_vars
 
 
 KNOCKOFF_DB_URI = 'KNOCKOFF_DB_URI'
@@ -70,15 +70,6 @@ def clear_default_env_vars():
                     KNOCKOFF_DB_URI])
 
 
-def clear_env_vars(env_vars):
-    # TODO: move to knockoff.utilities.environ
-    for variable in env_vars:
-        try:
-            del os.environ[variable]
-        except KeyError:
-            pass
-
-
 def register_engine(name, config):
     global ENGINE_BUILDERS
     ENGINE_BUILDERS[name] = EngineBuilder.from_config(config, build='builder',

knockoff/sdk/blueprint.py

@@ -8,6 +8,7 @@
 
 from knockoff.utilities.importlib_utils import resolve_package_name
 
+
 class Blueprint(object):
 
     ConstructionResult = namedtuple('ConstructionResult', [

knockoff/utilities/environ.py

@@ -22,3 +22,11 @@ def get(self):
 
     def set(self, value):
         os.environ[self.variable] = value
+
+
+def clear_env_vars(env_vars):
+    for variable in env_vars:
+        try:
+            del os.environ[variable]
+        except KeyError:
+            pass

knockoff/utilities/ioc.py

@@ -12,14 +12,21 @@
 from knockoff.utilities.importlib_utils import resolve_package_name
 
 
-def get_container(package_name, config_path=None):
+def get_container(package_name,
+                  config_path=None,
+                  default_dict=None,
+                  override_dict=None):
     """parses declarative container, loads optional config, wires and returns"""
     Container = resolve_package_name(package_name)
     _validate_container_class(Container, package_name)
     container = Container()
     container.init_resources()
+    if default_dict:
+        container.config.from_dict(default_dict)
     if config_path:
         container.config.from_yaml(config_path)
+    if override_dict:
+        container.config.from_dict(override_dict)
     container.wire(modules=[sys.modules[__name__]])
     return container