handling fields >8000 char

storebrand · Jan 19, 2023 · 25d1359 · 25d1359
1 parent 1fe90c5
commit 25d1359
Show file tree

Hide file tree

Showing 3 changed files with 11 additions and 4 deletions.
diff --git a/target_mssql/connector.py b/target_mssql/connector.py
@@ -343,11 +343,15 @@ def to_sql_type(self, jsonschema_type: dict) -> sqlalchemy.types.TypeEngine:  #
                 if datelike_type == "date":
                     return cast(sqlalchemy.types.TypeEngine, sqlalchemy.types.DATE())
 
-            maxlength = jsonschema_type.get("maxLength") or 8000
-            target_length = min(maxlength, 8000)
+            maxlength = jsonschema_type.get("maxLength")
+            if maxlength is not None:
+                if maxlength > 8000:
+                    return cast(
+                        sqlalchemy.types.TypeEngine, sqlalchemy.types.TEXT()
+                    )
 
             return cast(
-                sqlalchemy.types.TypeEngine, sqlalchemy.types.VARCHAR(target_length)
+                sqlalchemy.types.TypeEngine, sqlalchemy.types.VARCHAR(maxlength)
             )
 
         if self._jsonschema_type_check(jsonschema_type, ("integer",)):

diff --git a/target_mssql/sinks.py b/target_mssql/sinks.py
@@ -237,7 +237,7 @@ def merge_upsert_from_table(
 
         self.connection.execute(merge_sql)
 
-    #        self.connection.execute("COMMIT")
+        self.connection.execute("COMMIT")
 
     def parse_full_table_name(
         self, full_table_name: str

diff --git a/target_mssql/tests/data_files/long_string.singer b/target_mssql/tests/data_files/long_string.singer
@@ -0,0 +1,3 @@
+{"type": "SCHEMA", "stream": "long_string", "schema": {"type": "object", "properties": { "CustomerID": {"type": "string", "maxlength": 12000}, "clientName": {"type": "string"} }}, "key_properties": ["CustomerID"]}
+{"type": "RECORD", "stream": "long_string", "record": {"CustomerID": "1", "clientName": "Gitter Windows Desktop App"}}
+{"type": "RECORD", "stream": "long_string", "record": {"CustomerID": "2", "clientName": "Gitter iOS App"}}