MongoDB: Add integration test

cratedb_toolkit/io/mongodb/cli.py

@@ -17,8 +17,6 @@
 def extract_parser(subargs):
     parser = subargs.add_parser("extract", help="Extract a schema from a MongoDB database")
     parser.add_argument("--url", default="mongodb://localhost:27017", help="MongoDB URL")
-    parser.add_argument("--host", default="localhost", help="MongoDB host")
-    parser.add_argument("--port", default=27017, help="MongoDB port")
     parser.add_argument("--database", required=True, help="MongoDB database")
     parser.add_argument("--collection", help="MongoDB collection to create a schema for")
     parser.add_argument(
@@ -42,10 +40,8 @@ def translate_parser(subargs):
 def export_parser(subargs):
     parser = subargs.add_parser("export", help="Export a MongoDB collection as plain JSON")
     parser.add_argument("--url", default="mongodb://localhost:27017", help="MongoDB URL")
-    parser.add_argument("--collection", required=True)
-    parser.add_argument("--host", default="localhost", help="MongoDB host")
-    parser.add_argument("--port", default=27017, help="MongoDB port")
     parser.add_argument("--database", required=True, help="MongoDB database")
+    parser.add_argument("--collection", required=True, help="MongoDB collection to export")
     parser.add_argument("--limit", type=int, default=0, required=False, help="Limit export to N documents")
     parser.add_argument("--transformation", type=Path, required=False, help="Zyp transformation file")
 

cratedb_toolkit/io/mongodb/core.py

@@ -57,10 +57,7 @@ def gather_collections(database) -> t.List[str]:
 
 def get_mongodb_client_database(args, **kwargs) -> t.Tuple[pymongo.MongoClient, pymongo.database.Database]:
     client: pymongo.MongoClient
-    if args.url:
-        client = pymongo.MongoClient(args.url, **kwargs)
-    else:
-        client = pymongo.MongoClient(args.host, int(args.port), **kwargs)
+    client = pymongo.MongoClient(args.url, **kwargs)
     db: pymongo.database.Database = client.get_database(args.database)
     return client, db
 

tests/io/mongodb/test_cli.py

@@ -1,5 +1,9 @@
 import os
+from unittest import mock
+from uuid import UUID
 
+import bson
+import dateutil
 import pytest
 from click.testing import CliRunner
 from pueblo.testing.dataframe import DataFrameFactory
@@ -29,7 +33,26 @@ def test_version():
     assert exitcode == 0
 
 
-def test_mongodb_load_table(caplog, cratedb, mongodb):
+DOCUMENT_IN = {
+    "id": bson.Binary.from_uuid(UUID("d575540f-759d-4653-a4c4-4a9e410f1aa1")),
+    "value": {
+        "name": "foobar",
+        "active": True,
+        "created": dateutil.parser.parse("2020-06-19T15:03:53.727Z"),
+    },
+}
+DOCUMENT_OUT = {
+    "__id": mock.ANY,
+    "id": "d575540f-759d-4653-a4c4-4a9e410f1aa1",
+    "value": {
+        "name": "foobar",
+        "active": True,
+        "created": 1592579033000,
+    },
+}
+
+
+def test_mongodb_load_table_basic(caplog, cratedb, mongodb):
     """
     CLI test: Invoke `ctk load table` for MongoDB.
     """
@@ -59,3 +82,34 @@ def test_mongodb_load_table(caplog, cratedb, mongodb):
     assert cratedb.database.table_exists("testdrive.demo") is True
     assert cratedb.database.refresh_table("testdrive.demo") is True
     assert cratedb.database.count_records("testdrive.demo") == 42
+
+
+def test_mongodb_load_table_real(caplog, cratedb, mongodb):
+    """
+    CLI test: Invoke `ctk load table` for MongoDB.
+    """
+    cratedb_url = f"{cratedb.get_connection_url()}/testdrive/demo"
+    mongodb_url = f"{mongodb.get_connection_url()}/testdrive/demo"
+
+    # Populate source database.
+    client: pymongo.MongoClient = mongodb.get_connection_client()
+    testdrive = client.get_database("testdrive")
+    demo = testdrive.create_collection("demo")
+    demo.insert_many([DOCUMENT_IN])
+
+    # Run transfer command.
+    runner = CliRunner(env={"CRATEDB_SQLALCHEMY_URL": cratedb_url})
+    result = runner.invoke(
+        cli,
+        args=f"load table {mongodb_url}",
+        catch_exceptions=False,
+    )
+    assert result.exit_code == 0
+
+    # Verify data in target database.
+    assert cratedb.database.table_exists("testdrive.demo") is True
+    assert cratedb.database.refresh_table("testdrive.demo") is True
+    assert cratedb.database.count_records("testdrive.demo") == 1
+
+    results = cratedb.database.run_sql("SELECT * FROM testdrive.demo", records=True)
+    assert results[0] == DOCUMENT_OUT