diff --git a/meltano.yml b/meltano.yml index 91235b8..81a530b 100644 --- a/meltano.yml +++ b/meltano.yml @@ -19,7 +19,6 @@ plugins: config: start_date: '2010-01-01T00:00:00Z' settings: - # TODO: To configure using Meltano, declare settings and their types here: - name: username - name: password kind: password diff --git a/target_mssql/connector.py b/target_mssql/connector.py index 662cd2d..578e2cf 100644 --- a/target_mssql/connector.py +++ b/target_mssql/connector.py @@ -62,7 +62,7 @@ def get_sqlalchemy_url(self, config: dict) -> str: connection_url = sqlalchemy.engine.url.URL.create( drivername="mssql+pymssql", - username=config["user"], + username=config["username"], password=config["password"], host=config["host"], port=config["port"], @@ -344,6 +344,10 @@ def to_sql_type(self, jsonschema_type: dict) -> sqlalchemy.types.TypeEngine: # return cast(sqlalchemy.types.TypeEngine, sqlalchemy.types.DATE()) maxlength = jsonschema_type.get("maxLength") + if maxlength is not None: + if maxlength > 8000: + return cast(sqlalchemy.types.TypeEngine, sqlalchemy.types.TEXT()) + return cast( sqlalchemy.types.TypeEngine, sqlalchemy.types.VARCHAR(maxlength) ) @@ -374,6 +378,9 @@ def create_temp_table_from_table(self, from_table_name): f"{schema_name}.#{table_name}" if schema_name else f"#{table_name}" ) + droptable = f"DROP TABLE IF EXISTS {tmp_full_table_name}" + self.connection.execute(droptable) + ddl = f""" SELECT TOP 0 * into {tmp_full_table_name} diff --git a/target_mssql/target.py b/target_mssql/target.py index a13cefe..7aca73e 100644 --- a/target_mssql/target.py +++ b/target_mssql/target.py @@ -18,6 +18,37 @@ class Targetmssql(SQLTarget): th.StringType, description="SQLAlchemy connection string", ), + th.Property( + "username", + th.StringType, + description="SQL Server username", + ), + th.Property( + "password", + th.StringType, + description="SQL Server password", + ), + th.Property( + "host", + th.StringType, + description="SQL Server host", + ), + th.Property( + "port", + th.StringType, + default="1433", + description="SQL Server port", + ), + th.Property( + "database", + th.StringType, + description="SQL Server database", + ), + th.Property( + "default_target_schema", + th.StringType, + description="Default target schema to write to", + ), ).to_dict() default_sink_class = mssqlSink diff --git a/target_mssql/tests/data_files/long_string.singer b/target_mssql/tests/data_files/long_string.singer new file mode 100644 index 0000000..ccfa532 --- /dev/null +++ b/target_mssql/tests/data_files/long_string.singer @@ -0,0 +1,3 @@ +{"type": "SCHEMA", "stream": "long_string", "schema": {"type": "object", "properties": { "CustomerID": {"type": "string", "maxlength": 12000}, "clientName": {"type": "string"} }}, "key_properties": ["CustomerID"]} +{"type": "RECORD", "stream": "long_string", "record": {"CustomerID": "1", "clientName": "Gitter Windows Desktop App"}} +{"type": "RECORD", "stream": "long_string", "record": {"CustomerID": "2", "clientName": "Gitter iOS App"}} diff --git a/target_mssql/tests/test_core.py b/target_mssql/tests/test_core.py index 1aaaeb5..9eb82c4 100644 --- a/target_mssql/tests/test_core.py +++ b/target_mssql/tests/test_core.py @@ -17,9 +17,9 @@ @pytest.fixture() def mssql_config(): return { - "sqlalchemy_url": "mssql+pymssql://sa:p@55w0rd@localhost:1433/master", + # "sqlalchemy_url": "mssql+pymssql://sa:p@55w0rd@localhost:1433/master", "schema": "dbo", - "user": "sa", + "username": "sa", "password": "P@55w0rd", "host": "localhost", "port": "1433",