diff --git a/dbtmetabase/metabase.py b/dbtmetabase/metabase.py index 32de42e7..f9ccbeb8 100644 --- a/dbtmetabase/metabase.py +++ b/dbtmetabase/metabase.py @@ -4,13 +4,13 @@ import time from typing import ( Any, + Dict, Iterable, List, Mapping, MutableMapping, Optional, Sequence, - Tuple, Union, ) @@ -33,6 +33,87 @@ class MetabaseClient: _SYNC_PERIOD_SECS = 5 + class _Metadata: + """Mutable state of metadata (tables/fields) for lookups and updates.""" + + def __init__(self, tables: Optional[Iterable[Dict]] = None): + self.tables = {} + self.updates: MutableMapping[str, MutableMapping[str, Any]] = {} + + if tables: + for table in tables: + fields = {} + for field in table.get("fields", []): + new_field = field.copy() + new_field["kind"] = "field" + + field_name = field["name"].upper() + fields[field_name] = new_field + + new_table = table.copy() + new_table["kind"] = "table" + new_table["fields"] = fields + + schema_name = table["schema"].upper() + table_name = table["name"].upper() + self.tables[f"{schema_name}.{table_name}"] = new_table + + def get_table(self, table_key: str) -> Optional[MutableMapping]: + """Looks up table by key. + + Args: + table_key (str): Table key of form SCHEMA.TABLE. + + Returns: + Optional[MutableMapping]: Table description or none. + """ + return self.tables.get(table_key) + + def get_field(self, table_key: str, field_key: str) -> Optional[MutableMapping]: + """Looks up field by table and key. + + Args: + table_key (str): Table key of form SCHEMA.TABLE. + field_key (str): Field key. + + Returns: + Optional[MutableMapping]: Field description or none. + """ + return self.tables.get(table_key, {}).get("fields", {}).get(field_key) + + def update(self, entity: MutableMapping, delta: Mapping): + """Updates entity (table or field) with arguments and stages API update. + + Args: + entity (MutableMapping): Current state of entity. + delta (Mapping): Fields that need to change. + """ + entity.update(delta) + + key = f"{entity['kind']}.{entity['id']}" + update = self.updates.get(key, {}) + update["kind"] = entity["kind"] + update["id"] = entity["id"] + + body = update.get("body", {}) + body.update(delta) + update["body"] = body + + self.updates[key] = update + + def pop_updates(self) -> Iterable[MutableMapping]: + """Clears and returns currently staged updates. + + Returns: + Iterable[MutableMapping]: List of updates. + """ + updates = self.updates.values() + self.updates = {} + return updates + + def __bool__(self) -> bool: + return bool(self.tables) + def __init__( self, host: str, @@ -73,6 +154,7 @@ def __init__( self.cte_parser = re.compile( r"[Ww][Ii][Tt][Hh]\s+\b(\w+)\b\s+as|[)]\s*[,]\s*\b(\w+)\b\s+as" ) + self.metadata = self._Metadata() logger().info(":ok_hand: Session established successfully") def get_session_id(self, user: str, password: str) -> str: @@ -125,6 +207,8 @@ def sync_and_wait( f"Timeout provided {timeout} secs, must be at least {self._SYNC_PERIOD_SECS}" ) + self.metadata = MetabaseClient._Metadata() + database_id = self.find_database_id(database) if not database_id: raise exceptions.MetabaseUnableToSync( @@ -136,7 +220,8 @@ def sync_and_wait( deadline = int(time.time()) + timeout sync_successful = False while True: - sync_successful = self.models_compatible(database_id, models) + self.metadata = self.build_metadata(database_id) + sync_successful = self.models_compatible(models) time_after_wait = int(time.time()) + self._SYNC_PERIOD_SECS if not sync_successful and time_after_wait <= deadline: time.sleep(self._SYNC_PERIOD_SECS) @@ -148,21 +233,16 @@ def sync_and_wait( ) return sync_successful - def models_compatible( - self, database_id: str, models: Sequence[MetabaseModel] - ) -> bool: + def models_compatible(self, models: Sequence[MetabaseModel]) -> bool: """Checks if models compatible with the Metabase database schema. Arguments: - database_id {str} -- Metabase database ID. models {list} -- List of dbt models read from project. Returns: bool -- True if schema compatible with models, false otherwise. """ - _, field_lookup = self.build_metadata_lookups(database_id) - are_models_compatible = True for model in models: if model.model_type == ModelType.sources and self.exclude_sources: @@ -173,20 +253,23 @@ def models_compatible( lookup_key = f"{schema_name}.{model_name}" - if lookup_key not in field_lookup: + table = self.metadata.get_table(lookup_key) + if not table: logger().warning( "Model %s not found in %s schema", lookup_key, schema_name ) are_models_compatible = False - else: - table_lookup = field_lookup[lookup_key] - for column in model.columns: - column_name = column.name.upper() - if column_name not in table_lookup: - logger().warning( - "Column %s not found in %s model", column_name, lookup_key - ) - are_models_compatible = False + continue + + for column in model.columns: + column_name = column.name.upper() + + field = self.metadata.get_field(lookup_key, column_name) + if not field: + logger().warning( + "Column %s not found in %s model", column_name, lookup_key + ) + are_models_compatible = False return are_models_compatible @@ -204,33 +287,37 @@ def export_models( aliases {dict} -- Provided by reader class. Shuttled down to column exports to resolve FK refs against relations to aliased source tables """ - database_id = self.find_database_id(database) - if not database_id: - logger().critical("Cannot find database by name %s", database) - return - - table_lookup, field_lookup = self.build_metadata_lookups(database_id) + if not self.metadata: + database_id = self.find_database_id(database) + if not database_id: + logger().critical("Cannot find database by name %s", database) + return + self.metadata = self.build_metadata(database_id) for model in models: if model.model_type == ModelType.sources and self.exclude_sources: logger().info(":fast_forward: Skipping %s source", model.unique_id) continue - self.export_model(model, table_lookup, field_lookup, aliases) + self.export_model(model, aliases) - def export_model( - self, - model: MetabaseModel, - table_lookup: dict, - field_lookup: dict, - aliases: dict, - ): + for update in self.metadata.pop_updates(): + self.api( + "put", + f"/api/{update['kind']}/{update['id']}", + json=update["body"], + ) + logger().info( + ":satellite: Update to %s %s applied successfully", + update["kind"], + update["id"], + ) + + def export_model(self, model: MetabaseModel, aliases: dict): """Exports one dbt model to Metabase database schema. Arguments: model {dict} -- One dbt model read from project. - table_lookup {dict} -- Dictionary of Metabase tables indexed by name. - field_lookup {dict} -- Dictionary of Metabase fields indexed by name, indexed by table name. aliases {dict} -- Provided by reader class. Shuttled down to column exports to resolve FK refs against relations to aliased source tables """ @@ -239,10 +326,10 @@ def export_model( lookup_key = f"{schema_name}.{aliases.get(model_name, model_name)}" - api_table = table_lookup.get(lookup_key) + api_table = self.metadata.get_table(lookup_key) if not api_table: logger().error( - "\n:cross_mark: Table %s does not exist in Metabase", lookup_key + ":cross_mark: Table %s does not exist in Metabase", lookup_key ) return @@ -270,31 +357,20 @@ def export_model( if api_table.get("visibility_type") != model_visibility: body_table["visibility_type"] = model_visibility - table_id = api_table["id"] if body_table: - # Update with new values - self.api( - "put", - f"/api/table/{table_id}", - json=body_table, - ) - logger().info("\n:raising_hands: Updated table %s successfully", lookup_key) - elif not model_description: - logger().info( - "\n:bow: No model description provided for table %s", lookup_key - ) + self.metadata.update(entity=api_table, delta=body_table) + logger().info(":raising_hands: Table %s will be updated", lookup_key) else: - logger().info("\n:thumbs_up: Table %s is up-to-date", lookup_key) + logger().info(":thumbs_up: Table %s is up-to-date", lookup_key) for column in model.columns: - self.export_column(schema_name, model_name, column, field_lookup, aliases) + self.export_column(schema_name, model_name, column, aliases) def export_column( self, schema_name: str, model_name: str, column: MetabaseColumn, - field_lookup: dict, aliases: dict, ): """Exports one dbt column to Metabase database schema. @@ -302,24 +378,19 @@ def export_column( Arguments: model_name {str} -- One dbt model name read from project. column {dict} -- One dbt column read from project. - field_lookup {dict} -- Dictionary of Metabase fields indexed by name, indexed by table name. aliases {dict} -- Provided by reader class. Used to resolve FK refs against relations to aliased source tables """ table_lookup_key = f"{schema_name}.{model_name}" column_name = column.name.upper() - field = field_lookup.get(table_lookup_key, {}).get(column_name) - if not field: + api_field = self.metadata.get_field(table_lookup_key, column_name) + if not api_field: logger().error( "Field %s.%s does not exist in Metabase", table_lookup_key, column_name ) return - field_id = field["id"] - - api_field = self.api("get", f"/api/field/{field_id}") - if "special_type" in api_field: semantic_type_key = "special_type" else: @@ -364,21 +435,27 @@ def export_column( target_table, ) - fk_target_field_id = ( - field_lookup.get(target_table, {}).get(target_field, {}).get("id") - ) - - if fk_target_field_id: - logger().info( - ":key: Setting target field %s to PK in order to facilitate FK ref for %s column", - fk_target_field_id, - column_name, - ) - self.api( - "put", - f"/api/field/{fk_target_field_id}", - json={semantic_type_key: "type/PK"}, - ) + fk_target_field = self.metadata.get_field(target_table, target_field) + if fk_target_field: + fk_target_field_id = fk_target_field.get("id") + if fk_target_field.get(semantic_type_key) != "type/PK": + logger().info( + ":key: Target field %s will be set to PK for %s column FK", + fk_target_field_id, + column_name, + ) + body_fk_target_field = { + semantic_type_key: "type/PK", + } + self.metadata.update( + entity=fk_target_field, delta=body_fk_target_field + ) + else: + logger().info( + ":thumbs_up: Target field %s is already PK, needed for %s column", + fk_target_field_id, + column_name, + ) else: logger().error( ":cross_mark: Unable to find foreign key target %s.%s", @@ -435,14 +512,9 @@ def export_column( body_field[semantic_type_key] = column.semantic_type or None if body_field: - # Update with new values - self.api( - "put", - f"/api/field/{field_id}", - json=body_field, - ) + self.metadata.update(entity=api_field, delta=body_field) logger().info( - ":sparkles: Updated field %s.%s successfully", model_name, column_name + ":sparkles: Field %s.%s will be updated", model_name, column_name ) else: logger().info( @@ -464,64 +536,35 @@ def find_database_id(self, name: str) -> Optional[str]: return database["id"] return None - def build_metadata_lookups( - self, database_id: str, schemas_to_exclude: Iterable = None - ) -> Tuple[dict, dict]: - """Builds table and field lookups. + def build_metadata(self, database_id: str) -> _Metadata: + """Builds metadata lookups. Arguments: database_id {str} -- Metabase database ID. Returns: - dict -- Dictionary of tables indexed by name. - dict -- Dictionary of fields indexed by name, indexed by table name. + _Metadata -- Metadata lookup object. """ - if schemas_to_exclude is None: - schemas_to_exclude = [] - - table_lookup = {} - field_lookup = {} + tables = [] metadata = self.api( "get", f"/api/database/{database_id}/metadata", - params=dict(include_hidden=True), + params={"include_hidden": True}, ) - for table in metadata.get("tables", []): - table_schema = table.get("schema") - # table["schema"] is null for bigquery datasets - bigquery_schema = metadata.get("details", {}).get("dataset-id") - table_schema = ( - table_schema or bigquery_schema or METABASE_MODEL_DEFAULT_SCHEMA - ).upper() - table_name = table["name"].upper() - - if schemas_to_exclude: - schemas_to_exclude = { - exclusion.upper() for exclusion in schemas_to_exclude - } - - if table_schema in schemas_to_exclude: - logger().debug( - "Ignoring Metabase table %s in schema %s. It belongs to excluded schemas %s", - table_name, - table_schema, - schemas_to_exclude, - ) - continue - lookup_key = f"{table_schema}.{table_name}" - table_lookup[lookup_key] = table - table_field_lookup = {} + bigquery_schema = metadata.get("details", {}).get("dataset-id") - for field in table.get("fields", []): - field_name = field["name"].upper() - table_field_lookup[field_name] = field + for table in metadata.get("tables", []): + # table[schema] is null for bigquery datasets + table["schema"] = ( + table.get("schema") or bigquery_schema or METABASE_MODEL_DEFAULT_SCHEMA + ).upper() - field_lookup[lookup_key] = table_field_lookup + tables.append(table) - return table_lookup, field_lookup + return MetabaseClient._Metadata(tables) def extract_exposures( self, @@ -576,7 +619,7 @@ def increase_indent(self, flow=False, indentless=False): continue # Iter through collection - logger().info("\n\n:sparkles: Exploring collection %s", collection["name"]) + logger().info(":sparkles: Exploring collection %s", collection["name"]) for item in self.api("get", f"/api/collection/{collection['id']}/items"): # Ensure collection item is of parsable type diff --git a/dbtmetabase/parsers/dbt_folder.py b/dbtmetabase/parsers/dbt_folder.py index 0d6a798c..be3aca4f 100644 --- a/dbtmetabase/parsers/dbt_folder.py +++ b/dbtmetabase/parsers/dbt_folder.py @@ -54,7 +54,7 @@ def read_models( if "alias" in model: self.alias_mapping[model_name] = model["name"].upper() - logger().info("\nProcessing model: %s", path) + logger().info("Processing model: %s", path) if not self.model_selected(model_name): logger().debug( @@ -97,7 +97,7 @@ def read_models( self.alias_mapping[model_name] = model["name"].upper() logger().info( - "\nProcessing source: %s -- table: %s", path, model_name + "Processing source: %s -- table: %s", path, model_name ) if not self.model_selected(model_name): diff --git a/tests/fixtures/lookups/field_lookups.json b/tests/fixtures/lookups/field_lookups.json deleted file mode 100644 index 507e456f..00000000 --- a/tests/fixtures/lookups/field_lookups.json +++ /dev/null @@ -1 +0,0 @@ -{"PUBLIC.CUSTOMERS": {"CUSTOMER_ID": {"description": null, "database_type": "int4", "semantic_type": null, "table_id": 7, "coercion_strategy": null, "name": "customer_id", "fingerprint_version": 5, "has_field_values": "list", "settings": null, "caveats": null, "fk_target_field_id": null, "updated_at": "2021-07-21T07:30:35.194597Z", "custom_position": 0, "effective_type": "type/Integer", "active": true, "parent_id": null, "id": 38, "last_analyzed": "2021-07-21T05:47:54.560768Z", "position": 0, "visibility_type": "normal", "target": null, "preview_display": true, "display_name": "customer_id", "database_position": 0, "fingerprint": {"global": {"distinct-count": 100, "nil%": 0.0}, "type": {"type/Number": {"min": 1.0, "q1": 25.5, "q3": 75.5, "max": 100.0, "sd": 29.008358252146028, "avg": 50.5}}}, "created_at": "2021-07-21T05:47:53.408064Z", "base_type": "type/Integer", "points_of_interest": null}, "FIRST_NAME": {"description": null, "database_type": "text", "semantic_type": "type/Name", "table_id": 7, "coercion_strategy": null, "name": "first_name", "fingerprint_version": 5, "has_field_values": "list", "settings": null, "caveats": null, "fk_target_field_id": null, "updated_at": "2021-07-21T07:30:35.242305Z", "custom_position": 0, "effective_type": "type/Text", "active": true, "parent_id": null, "id": 37, "last_analyzed": "2021-07-21T05:47:54.560768Z", "position": 1, "visibility_type": "normal", "target": null, "preview_display": true, "display_name": "first_name", "database_position": 1, "fingerprint": {"global": {"distinct-count": 79, "nil%": 0.0}, "type": {"type/Text": {"percent-json": 0.0, "percent-url": 0.0, "percent-email": 0.0, "percent-state": 0.02, "average-length": 5.86}}}, "created_at": "2021-07-21T05:47:53.404944Z", "base_type": "type/Text", "points_of_interest": null}, "LAST_NAME": {"description": null, "database_type": "text", "semantic_type": "type/Name", "table_id": 7, "coercion_strategy": null, "name": "last_name", "fingerprint_version": 5, "has_field_values": "list", "settings": null, "caveats": null, "fk_target_field_id": null, "updated_at": "2021-07-21T07:30:35.249828Z", "custom_position": 0, "effective_type": "type/Text", "active": true, "parent_id": null, "id": 43, "last_analyzed": "2021-07-21T05:47:54.560768Z", "position": 2, "visibility_type": "normal", "target": null, "preview_display": true, "display_name": "last_name", "database_position": 2, "fingerprint": {"global": {"distinct-count": 19, "nil%": 0.0}, "type": {"type/Text": {"percent-json": 0.0, "percent-url": 0.0, "percent-email": 0.0, "percent-state": 0.0, "average-length": 2.0}}}, "created_at": "2021-07-21T05:47:53.419931Z", "base_type": "type/Text", "points_of_interest": null}, "FIRST_ORDER": {"description": null, "database_type": "date", "semantic_type": null, "table_id": 7, "coercion_strategy": null, "name": "first_order", "fingerprint_version": 5, "has_field_values": "none", "settings": null, "caveats": null, "fk_target_field_id": null, "updated_at": "2021-07-21T07:30:35.232032Z", "custom_position": 0, "effective_type": "type/Date", "active": true, "parent_id": null, "id": 39, "last_analyzed": "2021-07-21T05:47:54.560768Z", "position": 3, "visibility_type": "normal", "target": null, "preview_display": true, "display_name": "first_order", "database_position": 3, "fingerprint": {"global": {"distinct-count": 47, "nil%": 0.38}, "type": {"type/DateTime": {"earliest": "2018-01-01", "latest": "2018-04-07"}}}, "created_at": "2021-07-21T05:47:53.410556Z", "base_type": "type/Date", "points_of_interest": null}, "MOST_RECENT_ORDER": {"description": null, "database_type": "date", "semantic_type": null, "table_id": 7, "coercion_strategy": null, "name": "most_recent_order", "fingerprint_version": 5, "has_field_values": "none", "settings": null, "caveats": null, "fk_target_field_id": null, "updated_at": "2021-07-21T07:30:35.186143Z", "custom_position": 0, "effective_type": "type/Date", "active": true, "parent_id": null, "id": 42, "last_analyzed": "2021-07-21T05:47:54.560768Z", "position": 4, "visibility_type": "normal", "target": null, "preview_display": true, "display_name": "most_recent_order", "database_position": 4, "fingerprint": {"global": {"distinct-count": 53, "nil%": 0.38}, "type": {"type/DateTime": {"earliest": "2018-01-09", "latest": "2018-04-09"}}}, "created_at": "2021-07-21T05:47:53.417127Z", "base_type": "type/Date", "points_of_interest": null}, "NUMBER_OF_ORDERS": {"description": null, "database_type": "int8", "semantic_type": "type/Quantity", "table_id": 7, "coercion_strategy": null, "name": "number_of_orders", "fingerprint_version": 5, "has_field_values": "list", "settings": null, "caveats": null, "fk_target_field_id": null, "updated_at": "2021-07-21T07:30:35.197576Z", "custom_position": 0, "effective_type": "type/BigInteger", "active": true, "parent_id": null, "id": 40, "last_analyzed": "2021-07-21T05:47:54.560768Z", "position": 5, "visibility_type": "normal", "target": null, "preview_display": true, "display_name": "number_of_orders", "database_position": 5, "fingerprint": {"global": {"distinct-count": 5, "nil%": 0.38}, "type": {"type/Number": {"min": 1.0, "q1": 1.0, "q3": 2.0901356485315583, "max": 5.0, "sd": 0.7779687173818424, "avg": 1.596774193548387}}}, "created_at": "2021-07-21T05:47:53.412376Z", "base_type": "type/BigInteger", "points_of_interest": null}, "CUSTOMER_LIFETIME_VALUE": {"description": null, "database_type": "int8", "semantic_type": null, "table_id": 7, "coercion_strategy": null, "name": "customer_lifetime_value", "fingerprint_version": 5, "has_field_values": "list", "settings": null, "caveats": null, "fk_target_field_id": null, "updated_at": "2021-07-21T07:30:35.288463Z", "custom_position": 0, "effective_type": "type/BigInteger", "active": true, "parent_id": null, "id": 41, "last_analyzed": "2021-07-21T05:47:54.560768Z", "position": 6, "visibility_type": "normal", "target": null, "preview_display": true, "display_name": "customer_lifetime_value", "database_position": 6, "fingerprint": {"global": {"distinct-count": 36, "nil%": 0.38}, "type": {"type/Number": {"min": 1.0, "q1": 13.464101615137753, "q3": 35.46410161513776, "max": 99.0, "sd": 18.812245525263663, "avg": 26.967741935483872}}}, "created_at": "2021-07-21T05:47:53.414671Z", "base_type": "type/BigInteger", "points_of_interest": null}}, "PUBLIC.ORDERS": {"ORDER_ID": {"description": null, "database_type": "int4", "semantic_type": null, "table_id": 6, "coercion_strategy": null, "name": "order_id", "fingerprint_version": 5, "has_field_values": "list", "settings": null, "caveats": null, "fk_target_field_id": null, "updated_at": "2021-07-21T07:30:35.255223Z", "custom_position": 0, "effective_type": "type/Integer", "active": true, "parent_id": null, "id": 47, "last_analyzed": "2021-07-21T05:47:54.560768Z", "position": 0, "visibility_type": "normal", "target": null, "preview_display": true, "display_name": "order_id", "database_position": 0, "fingerprint": {"global": {"distinct-count": 99, "nil%": 0.0}, "type": {"type/Number": {"min": 1.0, "q1": 25.25, "q3": 74.75, "max": 99.0, "sd": 28.719704534890823, "avg": 50.0}}}, "created_at": "2021-07-21T05:47:53.444318Z", "base_type": "type/Integer", "points_of_interest": null}, "CUSTOMER_ID": {"description": null, "database_type": "int4", "semantic_type": null, "table_id": 6, "coercion_strategy": null, "name": "customer_id", "fingerprint_version": 5, "has_field_values": "list", "settings": null, "caveats": null, "fk_target_field_id": null, "updated_at": "2021-07-21T07:30:35.259928Z", "custom_position": 0, "effective_type": "type/Integer", "active": true, "parent_id": null, "id": 51, "last_analyzed": "2021-07-21T05:47:54.560768Z", "position": 1, "visibility_type": "normal", "target": null, "preview_display": true, "display_name": "customer_id", "database_position": 1, "fingerprint": {"global": {"distinct-count": 62, "nil%": 0.0}, "type": {"type/Number": {"min": 1.0, "q1": 25.875, "q3": 69.625, "max": 99.0, "sd": 27.781341350472964, "avg": 48.25252525252525}}}, "created_at": "2021-07-21T05:47:53.452739Z", "base_type": "type/Integer", "points_of_interest": null}, "ORDER_DATE": {"description": null, "database_type": "date", "semantic_type": null, "table_id": 6, "coercion_strategy": null, "name": "order_date", "fingerprint_version": 5, "has_field_values": "none", "settings": null, "caveats": null, "fk_target_field_id": null, "updated_at": "2021-07-21T07:30:35.188819Z", "custom_position": 0, "effective_type": "type/Date", "active": true, "parent_id": null, "id": 46, "last_analyzed": "2021-07-21T05:47:54.560768Z", "position": 2, "visibility_type": "normal", "target": null, "preview_display": true, "display_name": "order_date", "database_position": 2, "fingerprint": {"global": {"distinct-count": 69, "nil%": 0.0}, "type": {"type/DateTime": {"earliest": "2018-01-01", "latest": "2018-04-09"}}}, "created_at": "2021-07-21T05:47:53.441254Z", "base_type": "type/Date", "points_of_interest": null}, "STATUS": {"description": null, "database_type": "text", "semantic_type": "type/Category", "table_id": 6, "coercion_strategy": null, "name": "status", "fingerprint_version": 5, "has_field_values": "list", "settings": null, "caveats": null, "fk_target_field_id": null, "updated_at": "2021-07-21T07:30:35.203655Z", "custom_position": 0, "effective_type": "type/Text", "active": true, "parent_id": null, "id": 50, "last_analyzed": "2021-07-21T05:47:54.560768Z", "position": 3, "visibility_type": "normal", "target": null, "preview_display": true, "display_name": "status", "database_position": 3, "fingerprint": {"global": {"distinct-count": 5, "nil%": 0.0}, "type": {"type/Text": {"percent-json": 0.0, "percent-url": 0.0, "percent-email": 0.0, "percent-state": 0.0, "average-length": 8.404040404040405}}}, "created_at": "2021-07-21T05:47:53.450839Z", "base_type": "type/Text", "points_of_interest": null}, "CREDIT_CARD_AMOUNT": {"description": null, "database_type": "int8", "semantic_type": "type/Category", "table_id": 6, "coercion_strategy": null, "name": "credit_card_amount", "fingerprint_version": 5, "has_field_values": "list", "settings": null, "caveats": null, "fk_target_field_id": null, "updated_at": "2021-07-21T07:30:35.24496Z", "custom_position": 0, "effective_type": "type/BigInteger", "active": true, "parent_id": null, "id": 44, "last_analyzed": "2021-07-21T05:47:54.560768Z", "position": 4, "visibility_type": "normal", "target": null, "preview_display": true, "display_name": "credit_card_amount", "database_position": 4, "fingerprint": {"global": {"distinct-count": 25, "nil%": 0.0}, "type": {"type/Number": {"min": 0.0, "q1": 0.0, "q3": 18.797054997187544, "max": 30.0, "sd": 10.959088854927673, "avg": 8.797979797979798}}}, "created_at": "2021-07-21T05:47:53.43752Z", "base_type": "type/BigInteger", "points_of_interest": null}, "COUPON_AMOUNT": {"description": null, "database_type": "int8", "semantic_type": "type/Category", "table_id": 6, "coercion_strategy": null, "name": "coupon_amount", "fingerprint_version": 5, "has_field_values": "list", "settings": null, "caveats": null, "fk_target_field_id": null, "updated_at": "2021-07-21T07:30:35.25752Z", "custom_position": 0, "effective_type": "type/BigInteger", "active": true, "parent_id": null, "id": 49, "last_analyzed": "2021-07-21T05:47:54.560768Z", "position": 5, "visibility_type": "normal", "target": null, "preview_display": true, "display_name": "coupon_amount", "database_position": 5, "fingerprint": {"global": {"distinct-count": 12, "nil%": 0.0}, "type": {"type/Number": {"min": 0.0, "q1": 0.0, "q3": 0.4747603274810728, "max": 26.0, "sd": 5.955012405351229, "avg": 1.8686868686868687}}}, "created_at": "2021-07-21T05:47:53.448941Z", "base_type": "type/BigInteger", "points_of_interest": null}, "BANK_TRANSFER_AMOUNT": {"description": null, "database_type": "int8", "semantic_type": "type/Category", "table_id": 6, "coercion_strategy": null, "name": "bank_transfer_amount", "fingerprint_version": 5, "has_field_values": "list", "settings": null, "caveats": null, "fk_target_field_id": null, "updated_at": "2021-07-21T07:30:35.25288Z", "custom_position": 0, "effective_type": "type/BigInteger", "active": true, "parent_id": null, "id": 45, "last_analyzed": "2021-07-21T05:47:54.560768Z", "position": 6, "visibility_type": "normal", "target": null, "preview_display": true, "display_name": "bank_transfer_amount", "database_position": 6, "fingerprint": {"global": {"distinct-count": 19, "nil%": 0.0}, "type": {"type/Number": {"min": 0.0, "q1": 0.0, "q3": 4.75, "max": 26.0, "sd": 7.420825132023675, "avg": 4.151515151515151}}}, "created_at": "2021-07-21T05:47:53.43953Z", "base_type": "type/BigInteger", "points_of_interest": null}, "GIFT_CARD_AMOUNT": {"description": null, "database_type": "int8", "semantic_type": "type/Category", "table_id": 6, "coercion_strategy": null, "name": "gift_card_amount", "fingerprint_version": 5, "has_field_values": "list", "settings": null, "caveats": null, "fk_target_field_id": null, "updated_at": "2021-07-21T07:30:35.201036Z", "custom_position": 0, "effective_type": "type/BigInteger", "active": true, "parent_id": null, "id": 48, "last_analyzed": "2021-07-21T05:47:54.560768Z", "position": 7, "visibility_type": "normal", "target": null, "preview_display": true, "display_name": "gift_card_amount", "database_position": 7, "fingerprint": {"global": {"distinct-count": 11, "nil%": 0.0}, "type": {"type/Number": {"min": 0.0, "q1": 0.0, "q3": 1.3692088763283736, "max": 30.0, "sd": 6.392362351566517, "avg": 2.0707070707070705}}}, "created_at": "2021-07-21T05:47:53.447026Z", "base_type": "type/BigInteger", "points_of_interest": null}, "AMOUNT": {"description": null, "database_type": "int8", "semantic_type": null, "table_id": 6, "coercion_strategy": null, "name": "amount", "fingerprint_version": 5, "has_field_values": "list", "settings": null, "caveats": null, "fk_target_field_id": null, "updated_at": "2021-07-21T07:30:35.206083Z", "custom_position": 0, "effective_type": "type/BigInteger", "active": true, "parent_id": null, "id": 52, "last_analyzed": "2021-07-21T05:47:54.560768Z", "position": 8, "visibility_type": "normal", "target": null, "preview_display": true, "display_name": "amount", "database_position": 8, "fingerprint": {"global": {"distinct-count": 32, "nil%": 0.0}, "type": {"type/Number": {"min": 0.0, "q1": 8.202945002812456, "q3": 24.26138721247417, "max": 58.0, "sd": 10.736062525374601, "avg": 16.88888888888889}}}, "created_at": "2021-07-21T05:47:53.455652Z", "base_type": "type/BigInteger", "points_of_interest": null}}, "PUBLIC.RAW_CUSTOMERS": {"ID": {"description": null, "database_type": "int4", "semantic_type": "type/PK", "table_id": 9, "coercion_strategy": null, "name": "id", "fingerprint_version": 5, "has_field_values": "none", "settings": null, "caveats": null, "fk_target_field_id": null, "updated_at": "2021-07-21T07:30:35.234376Z", "custom_position": 0, "effective_type": "type/Integer", "active": true, "parent_id": null, "id": 55, "last_analyzed": "2021-07-21T05:47:54.560768Z", "position": 0, "visibility_type": "normal", "target": null, "preview_display": true, "display_name": "id", "database_position": 0, "fingerprint": null, "created_at": "2021-07-21T05:47:53.473642Z", "base_type": "type/Integer", "points_of_interest": null}, "FIRST_NAME": {"description": null, "database_type": "text", "semantic_type": "type/Name", "table_id": 9, "coercion_strategy": null, "name": "first_name", "fingerprint_version": 5, "has_field_values": "list", "settings": null, "caveats": null, "fk_target_field_id": null, "updated_at": "2021-07-21T07:30:35.208554Z", "custom_position": 0, "effective_type": "type/Text", "active": true, "parent_id": null, "id": 53, "last_analyzed": "2021-07-21T05:47:54.560768Z", "position": 1, "visibility_type": "normal", "target": null, "preview_display": true, "display_name": "first_name", "database_position": 1, "fingerprint": {"global": {"distinct-count": 79, "nil%": 0.0}, "type": {"type/Text": {"percent-json": 0.0, "percent-url": 0.0, "percent-email": 0.0, "percent-state": 0.02, "average-length": 5.86}}}, "created_at": "2021-07-21T05:47:53.469932Z", "base_type": "type/Text", "points_of_interest": null}, "LAST_NAME": {"description": null, "database_type": "text", "semantic_type": "type/Name", "table_id": 9, "coercion_strategy": null, "name": "last_name", "fingerprint_version": 5, "has_field_values": "list", "settings": null, "caveats": null, "fk_target_field_id": null, "updated_at": "2021-07-21T07:30:35.262668Z", "custom_position": 0, "effective_type": "type/Text", "active": true, "parent_id": null, "id": 54, "last_analyzed": "2021-07-21T05:47:54.560768Z", "position": 2, "visibility_type": "normal", "target": null, "preview_display": true, "display_name": "last_name", "database_position": 2, "fingerprint": {"global": {"distinct-count": 19, "nil%": 0.0}, "type": {"type/Text": {"percent-json": 0.0, "percent-url": 0.0, "percent-email": 0.0, "percent-state": 0.0, "average-length": 2.0}}}, "created_at": "2021-07-21T05:47:53.471892Z", "base_type": "type/Text", "points_of_interest": null}}, "PUBLIC.RAW_ORDERS": {"ID": {"description": null, "database_type": "int4", "semantic_type": "type/PK", "table_id": 12, "coercion_strategy": null, "name": "id", "fingerprint_version": 5, "has_field_values": "none", "settings": null, "caveats": null, "fk_target_field_id": null, "updated_at": "2021-07-21T07:30:35.266036Z", "custom_position": 0, "effective_type": "type/Integer", "active": true, "parent_id": null, "id": 57, "last_analyzed": "2021-07-21T05:47:54.560768Z", "position": 0, "visibility_type": "normal", "target": null, "preview_display": true, "display_name": "id", "database_position": 0, "fingerprint": null, "created_at": "2021-07-21T05:47:53.48931Z", "base_type": "type/Integer", "points_of_interest": null}, "USER_ID": {"description": null, "database_type": "int4", "semantic_type": null, "table_id": 12, "coercion_strategy": null, "name": "user_id", "fingerprint_version": 5, "has_field_values": "list", "settings": null, "caveats": null, "fk_target_field_id": null, "updated_at": "2021-07-21T07:30:35.214697Z", "custom_position": 0, "effective_type": "type/Integer", "active": true, "parent_id": null, "id": 59, "last_analyzed": "2021-07-21T05:47:54.560768Z", "position": 1, "visibility_type": "normal", "target": null, "preview_display": true, "display_name": "user_id", "database_position": 1, "fingerprint": {"global": {"distinct-count": 62, "nil%": 0.0}, "type": {"type/Number": {"min": 1.0, "q1": 25.875, "q3": 69.625, "max": 99.0, "sd": 27.781341350472964, "avg": 48.25252525252525}}}, "created_at": "2021-07-21T05:47:53.494231Z", "base_type": "type/Integer", "points_of_interest": null}, "ORDER_DATE": {"description": null, "database_type": "date", "semantic_type": null, "table_id": 12, "coercion_strategy": null, "name": "order_date", "fingerprint_version": 5, "has_field_values": "none", "settings": null, "caveats": null, "fk_target_field_id": null, "updated_at": "2021-07-21T07:30:35.268564Z", "custom_position": 0, "effective_type": "type/Date", "active": true, "parent_id": null, "id": 56, "last_analyzed": "2021-07-21T05:47:54.560768Z", "position": 2, "visibility_type": "normal", "target": null, "preview_display": true, "display_name": "order_date", "database_position": 2, "fingerprint": {"global": {"distinct-count": 69, "nil%": 0.0}, "type": {"type/DateTime": {"earliest": "2018-01-01", "latest": "2018-04-09"}}}, "created_at": "2021-07-21T05:47:53.48571Z", "base_type": "type/Date", "points_of_interest": null}, "STATUS": {"description": null, "database_type": "text", "semantic_type": "type/Category", "table_id": 12, "coercion_strategy": null, "name": "status", "fingerprint_version": 5, "has_field_values": "list", "settings": null, "caveats": null, "fk_target_field_id": null, "updated_at": "2021-07-21T07:30:35.211374Z", "custom_position": 0, "effective_type": "type/Text", "active": true, "parent_id": null, "id": 58, "last_analyzed": "2021-07-21T05:47:54.560768Z", "position": 3, "visibility_type": "normal", "target": null, "preview_display": true, "display_name": "status", "database_position": 3, "fingerprint": {"global": {"distinct-count": 5, "nil%": 0.0}, "type": {"type/Text": {"percent-json": 0.0, "percent-url": 0.0, "percent-email": 0.0, "percent-state": 0.0, "average-length": 8.404040404040405}}}, "created_at": "2021-07-21T05:47:53.491699Z", "base_type": "type/Text", "points_of_interest": null}}, "PUBLIC.RAW_PAYMENTS": {"ID": {"description": null, "database_type": "int4", "semantic_type": "type/PK", "table_id": 11, "coercion_strategy": null, "name": "id", "fingerprint_version": 5, "has_field_values": "none", "settings": null, "caveats": null, "fk_target_field_id": null, "updated_at": "2021-07-21T07:30:35.271155Z", "custom_position": 0, "effective_type": "type/Integer", "active": true, "parent_id": null, "id": 62, "last_analyzed": "2021-07-21T05:47:54.560768Z", "position": 0, "visibility_type": "normal", "target": null, "preview_display": true, "display_name": "id", "database_position": 0, "fingerprint": null, "created_at": "2021-07-21T05:47:53.511724Z", "base_type": "type/Integer", "points_of_interest": null}, "ORDER_ID": {"description": null, "database_type": "int4", "semantic_type": null, "table_id": 11, "coercion_strategy": null, "name": "order_id", "fingerprint_version": 5, "has_field_values": "list", "settings": null, "caveats": null, "fk_target_field_id": null, "updated_at": "2021-07-21T07:30:35.247411Z", "custom_position": 0, "effective_type": "type/Integer", "active": true, "parent_id": null, "id": 60, "last_analyzed": "2021-07-21T05:47:54.560768Z", "position": 1, "visibility_type": "normal", "target": null, "preview_display": true, "display_name": "order_id", "database_position": 1, "fingerprint": {"global": {"distinct-count": 99, "nil%": 0.0}, "type": {"type/Number": {"min": 1.0, "q1": 24.904857366030992, "q3": 75.25, "max": 99.0, "sd": 28.540193317267853, "avg": 50.0353982300885}}}, "created_at": "2021-07-21T05:47:53.506967Z", "base_type": "type/Integer", "points_of_interest": null}, "PAYMENT_METHOD": {"description": null, "database_type": "text", "semantic_type": "type/Category", "table_id": 11, "coercion_strategy": null, "name": "payment_method", "fingerprint_version": 5, "has_field_values": "list", "settings": null, "caveats": null, "fk_target_field_id": null, "updated_at": "2021-07-21T07:30:35.217422Z", "custom_position": 0, "effective_type": "type/Text", "active": true, "parent_id": null, "id": 61, "last_analyzed": "2021-07-21T05:47:54.560768Z", "position": 2, "visibility_type": "normal", "target": null, "preview_display": true, "display_name": "payment_method", "database_position": 2, "fingerprint": {"global": {"distinct-count": 4, "nil%": 0.0}, "type": {"type/Text": {"percent-json": 0.0, "percent-url": 0.0, "percent-email": 0.0, "percent-state": 0.0, "average-length": 10.79646017699115}}}, "created_at": "2021-07-21T05:47:53.508887Z", "base_type": "type/Text", "points_of_interest": null}, "AMOUNT": {"description": null, "database_type": "int4", "semantic_type": "type/Category", "table_id": 11, "coercion_strategy": null, "name": "amount", "fingerprint_version": 5, "has_field_values": "list", "settings": null, "caveats": null, "fk_target_field_id": null, "updated_at": "2021-07-21T07:30:35.273607Z", "custom_position": 0, "effective_type": "type/Integer", "active": true, "parent_id": null, "id": 63, "last_analyzed": "2021-07-21T05:47:54.560768Z", "position": 3, "visibility_type": "normal", "target": null, "preview_display": true, "display_name": "amount", "database_position": 3, "fingerprint": {"global": {"distinct-count": 30, "nil%": 0.0}, "type": {"type/Number": {"min": 0.0, "q1": 606.4037815689348, "q3": 2278.791845139511, "max": 3000.0, "sd": 919.836873351873, "avg": 1479.646017699115}}}, "created_at": "2021-07-21T05:47:53.513727Z", "base_type": "type/Integer", "points_of_interest": null}}, "PUBLIC.STG_CUSTOMERS": {"CUSTOMER_ID": {"description": null, "database_type": "int4", "semantic_type": null, "table_id": 8, "coercion_strategy": null, "name": "customer_id", "fingerprint_version": 5, "has_field_values": "list", "settings": null, "caveats": null, "fk_target_field_id": null, "updated_at": "2021-07-21T07:30:35.276108Z", "custom_position": 0, "effective_type": "type/Integer", "active": true, "parent_id": null, "id": 65, "last_analyzed": "2021-07-21T05:47:54.560768Z", "position": 0, "visibility_type": "normal", "target": null, "preview_display": true, "display_name": "customer_id", "database_position": 0, "fingerprint": {"global": {"distinct-count": 100, "nil%": 0.0}, "type": {"type/Number": {"min": 1.0, "q1": 25.5, "q3": 75.5, "max": 100.0, "sd": 29.008358252146028, "avg": 50.5}}}, "created_at": "2021-07-21T05:47:53.528091Z", "base_type": "type/Integer", "points_of_interest": null}, "FIRST_NAME": {"description": null, "database_type": "text", "semantic_type": "type/Name", "table_id": 8, "coercion_strategy": null, "name": "first_name", "fingerprint_version": 5, "has_field_values": "list", "settings": null, "caveats": null, "fk_target_field_id": null, "updated_at": "2021-07-21T07:30:35.220226Z", "custom_position": 0, "effective_type": "type/Text", "active": true, "parent_id": null, "id": 64, "last_analyzed": "2021-07-21T05:47:54.560768Z", "position": 1, "visibility_type": "normal", "target": null, "preview_display": true, "display_name": "first_name", "database_position": 1, "fingerprint": {"global": {"distinct-count": 79, "nil%": 0.0}, "type": {"type/Text": {"percent-json": 0.0, "percent-url": 0.0, "percent-email": 0.0, "percent-state": 0.02, "average-length": 5.86}}}, "created_at": "2021-07-21T05:47:53.525589Z", "base_type": "type/Text", "points_of_interest": null}, "LAST_NAME": {"description": null, "database_type": "text", "semantic_type": "type/Name", "table_id": 8, "coercion_strategy": null, "name": "last_name", "fingerprint_version": 5, "has_field_values": "list", "settings": null, "caveats": null, "fk_target_field_id": null, "updated_at": "2021-07-21T07:30:35.222931Z", "custom_position": 0, "effective_type": "type/Text", "active": true, "parent_id": null, "id": 66, "last_analyzed": "2021-07-21T05:47:54.560768Z", "position": 2, "visibility_type": "normal", "target": null, "preview_display": true, "display_name": "last_name", "database_position": 2, "fingerprint": {"global": {"distinct-count": 19, "nil%": 0.0}, "type": {"type/Text": {"percent-json": 0.0, "percent-url": 0.0, "percent-email": 0.0, "percent-state": 0.0, "average-length": 2.0}}}, "created_at": "2021-07-21T05:47:53.530075Z", "base_type": "type/Text", "points_of_interest": null}}, "PUBLIC.STG_ORDERS": {"ORDER_ID": {"description": null, "database_type": "int4", "semantic_type": null, "table_id": 5, "coercion_strategy": null, "name": "order_id", "fingerprint_version": 5, "has_field_values": "list", "settings": null, "caveats": null, "fk_target_field_id": null, "updated_at": "2021-07-21T07:30:35.225874Z", "custom_position": 0, "effective_type": "type/Integer", "active": true, "parent_id": null, "id": 68, "last_analyzed": "2021-07-21T05:47:54.560768Z", "position": 0, "visibility_type": "normal", "target": null, "preview_display": true, "display_name": "order_id", "database_position": 0, "fingerprint": {"global": {"distinct-count": 99, "nil%": 0.0}, "type": {"type/Number": {"min": 1.0, "q1": 25.25, "q3": 74.75, "max": 99.0, "sd": 28.719704534890823, "avg": 50.0}}}, "created_at": "2021-07-21T05:47:53.545842Z", "base_type": "type/Integer", "points_of_interest": null}, "CUSTOMER_ID": {"description": null, "database_type": "int4", "semantic_type": null, "table_id": 5, "coercion_strategy": null, "name": "customer_id", "fingerprint_version": 5, "has_field_values": "list", "settings": null, "caveats": null, "fk_target_field_id": null, "updated_at": "2021-07-21T07:30:35.229545Z", "custom_position": 0, "effective_type": "type/Integer", "active": true, "parent_id": null, "id": 70, "last_analyzed": "2021-07-21T05:47:54.560768Z", "position": 1, "visibility_type": "normal", "target": null, "preview_display": true, "display_name": "customer_id", "database_position": 1, "fingerprint": {"global": {"distinct-count": 62, "nil%": 0.0}, "type": {"type/Number": {"min": 1.0, "q1": 25.875, "q3": 69.625, "max": 99.0, "sd": 27.781341350472964, "avg": 48.25252525252525}}}, "created_at": "2021-07-21T05:47:53.549796Z", "base_type": "type/Integer", "points_of_interest": null}, "ORDER_DATE": {"description": null, "database_type": "date", "semantic_type": null, "table_id": 5, "coercion_strategy": null, "name": "order_date", "fingerprint_version": 5, "has_field_values": "none", "settings": null, "caveats": null, "fk_target_field_id": null, "updated_at": "2021-07-21T07:30:35.285732Z", "custom_position": 0, "effective_type": "type/Date", "active": true, "parent_id": null, "id": 67, "last_analyzed": "2021-07-21T05:47:54.560768Z", "position": 2, "visibility_type": "normal", "target": null, "preview_display": true, "display_name": "order_date", "database_position": 2, "fingerprint": {"global": {"distinct-count": 69, "nil%": 0.0}, "type": {"type/DateTime": {"earliest": "2018-01-01", "latest": "2018-04-09"}}}, "created_at": "2021-07-21T05:47:53.543598Z", "base_type": "type/Date", "points_of_interest": null}, "STATUS": {"description": null, "database_type": "text", "semantic_type": "type/Category", "table_id": 5, "coercion_strategy": null, "name": "status", "fingerprint_version": 5, "has_field_values": "list", "settings": null, "caveats": null, "fk_target_field_id": null, "updated_at": "2021-07-21T07:30:35.27984Z", "custom_position": 0, "effective_type": "type/Text", "active": true, "parent_id": null, "id": 69, "last_analyzed": "2021-07-21T05:47:54.560768Z", "position": 3, "visibility_type": "normal", "target": null, "preview_display": true, "display_name": "status", "database_position": 3, "fingerprint": {"global": {"distinct-count": 5, "nil%": 0.0}, "type": {"type/Text": {"percent-json": 0.0, "percent-url": 0.0, "percent-email": 0.0, "percent-state": 0.0, "average-length": 8.404040404040405}}}, "created_at": "2021-07-21T05:47:53.547849Z", "base_type": "type/Text", "points_of_interest": null}}, "PUBLIC.STG_PAYMENTS": {"PAYMENT_ID": {"description": null, "database_type": "int4", "semantic_type": null, "table_id": 10, "coercion_strategy": null, "name": "payment_id", "fingerprint_version": 5, "has_field_values": "none", "settings": null, "caveats": null, "fk_target_field_id": null, "updated_at": "2021-07-21T07:30:35.191434Z", "custom_position": 0, "effective_type": "type/Integer", "active": true, "parent_id": null, "id": 74, "last_analyzed": "2021-07-21T05:47:54.560768Z", "position": 0, "visibility_type": "normal", "target": null, "preview_display": true, "display_name": "payment_id", "database_position": 0, "fingerprint": {"global": {"distinct-count": 113, "nil%": 0.0}, "type": {"type/Number": {"min": 1.0, "q1": 28.75, "q3": 85.25, "max": 113.0, "sd": 32.76097144469315, "avg": 57.0}}}, "created_at": "2021-07-21T05:47:53.570339Z", "base_type": "type/Integer", "points_of_interest": null}, "ORDER_ID": {"description": null, "database_type": "int4", "semantic_type": null, "table_id": 10, "coercion_strategy": null, "name": "order_id", "fingerprint_version": 5, "has_field_values": "list", "settings": null, "caveats": null, "fk_target_field_id": null, "updated_at": "2021-07-21T07:30:35.282867Z", "custom_position": 0, "effective_type": "type/Integer", "active": true, "parent_id": null, "id": 71, "last_analyzed": "2021-07-21T05:47:54.560768Z", "position": 1, "visibility_type": "normal", "target": null, "preview_display": true, "display_name": "order_id", "database_position": 1, "fingerprint": {"global": {"distinct-count": 99, "nil%": 0.0}, "type": {"type/Number": {"min": 1.0, "q1": 24.904857366030992, "q3": 75.25, "max": 99.0, "sd": 28.540193317267853, "avg": 50.0353982300885}}}, "created_at": "2021-07-21T05:47:53.562985Z", "base_type": "type/Integer", "points_of_interest": null}, "PAYMENT_METHOD": {"description": null, "database_type": "text", "semantic_type": "type/Category", "table_id": 10, "coercion_strategy": null, "name": "payment_method", "fingerprint_version": 5, "has_field_values": "list", "settings": null, "caveats": null, "fk_target_field_id": null, "updated_at": "2021-07-21T07:30:35.236542Z", "custom_position": 0, "effective_type": "type/Text", "active": true, "parent_id": null, "id": 72, "last_analyzed": "2021-07-21T05:47:54.560768Z", "position": 2, "visibility_type": "normal", "target": null, "preview_display": true, "display_name": "payment_method", "database_position": 2, "fingerprint": {"global": {"distinct-count": 4, "nil%": 0.0}, "type": {"type/Text": {"percent-json": 0.0, "percent-url": 0.0, "percent-email": 0.0, "percent-state": 0.0, "average-length": 10.79646017699115}}}, "created_at": "2021-07-21T05:47:53.566146Z", "base_type": "type/Text", "points_of_interest": null}, "AMOUNT": {"description": null, "database_type": "int4", "semantic_type": "type/Category", "table_id": 10, "coercion_strategy": null, "name": "amount", "fingerprint_version": 5, "has_field_values": "list", "settings": null, "caveats": null, "fk_target_field_id": null, "updated_at": "2021-07-21T07:30:35.240154Z", "custom_position": 0, "effective_type": "type/Integer", "active": true, "parent_id": null, "id": 73, "last_analyzed": "2021-07-21T05:47:54.560768Z", "position": 3, "visibility_type": "normal", "target": null, "preview_display": true, "display_name": "amount", "database_position": 3, "fingerprint": {"global": {"distinct-count": 30, "nil%": 0.0}, "type": {"type/Number": {"min": 0.0, "q1": 6.064037815689349, "q3": 22.787918451395115, "max": 30.0, "sd": 9.198368733518729, "avg": 14.79646017699115}}}, "created_at": "2021-07-21T05:47:53.568245Z", "base_type": "type/Integer", "points_of_interest": null}}} \ No newline at end of file diff --git a/tests/fixtures/lookups/table_lookups.json b/tests/fixtures/lookups/table_lookups.json deleted file mode 100644 index 03053954..00000000 --- a/tests/fixtures/lookups/table_lookups.json +++ /dev/null @@ -1 +0,0 @@ -{"PUBLIC.CUSTOMERS": {"description": null, "entity_type": "entity/GenericTable", "schema": "public", "show_in_getting_started": false, "name": "customers", "fields": [{"description": null, "database_type": "int4", "semantic_type": null, "table_id": 7, "coercion_strategy": null, "name": "customer_id", "fingerprint_version": 5, "has_field_values": "list", "settings": null, "caveats": null, "fk_target_field_id": null, "updated_at": "2021-07-21T07:30:35.194597Z", "custom_position": 0, "effective_type": "type/Integer", "active": true, "parent_id": null, "id": 38, "last_analyzed": "2021-07-21T05:47:54.560768Z", "position": 0, "visibility_type": "normal", "target": null, "preview_display": true, "display_name": "customer_id", "database_position": 0, "fingerprint": {"global": {"distinct-count": 100, "nil%": 0.0}, "type": {"type/Number": {"min": 1.0, "q1": 25.5, "q3": 75.5, "max": 100.0, "sd": 29.008358252146028, "avg": 50.5}}}, "created_at": "2021-07-21T05:47:53.408064Z", "base_type": "type/Integer", "points_of_interest": null}, {"description": null, "database_type": "text", "semantic_type": "type/Name", "table_id": 7, "coercion_strategy": null, "name": "first_name", "fingerprint_version": 5, "has_field_values": "list", "settings": null, "caveats": null, "fk_target_field_id": null, "updated_at": "2021-07-21T07:30:35.242305Z", "custom_position": 0, "effective_type": "type/Text", "active": true, "parent_id": null, "id": 37, "last_analyzed": "2021-07-21T05:47:54.560768Z", "position": 1, "visibility_type": "normal", "target": null, "preview_display": true, "display_name": "first_name", "database_position": 1, "fingerprint": {"global": {"distinct-count": 79, "nil%": 0.0}, "type": {"type/Text": {"percent-json": 0.0, "percent-url": 0.0, "percent-email": 0.0, "percent-state": 0.02, "average-length": 5.86}}}, "created_at": "2021-07-21T05:47:53.404944Z", "base_type": "type/Text", "points_of_interest": null}, {"description": null, "database_type": "text", "semantic_type": "type/Name", "table_id": 7, "coercion_strategy": null, "name": "last_name", "fingerprint_version": 5, "has_field_values": "list", "settings": null, "caveats": null, "fk_target_field_id": null, "updated_at": "2021-07-21T07:30:35.249828Z", "custom_position": 0, "effective_type": "type/Text", "active": true, "parent_id": null, "id": 43, "last_analyzed": "2021-07-21T05:47:54.560768Z", "position": 2, "visibility_type": "normal", "target": null, "preview_display": true, "display_name": "last_name", "database_position": 2, "fingerprint": {"global": {"distinct-count": 19, "nil%": 0.0}, "type": {"type/Text": {"percent-json": 0.0, "percent-url": 0.0, "percent-email": 0.0, "percent-state": 0.0, "average-length": 2.0}}}, "created_at": "2021-07-21T05:47:53.419931Z", "base_type": "type/Text", "points_of_interest": null}, {"description": null, "database_type": "date", "semantic_type": null, "table_id": 7, "coercion_strategy": null, "name": "first_order", "fingerprint_version": 5, "has_field_values": "none", "settings": null, "caveats": null, "fk_target_field_id": null, "updated_at": "2021-07-21T07:30:35.232032Z", "custom_position": 0, "effective_type": "type/Date", "active": true, "parent_id": null, "id": 39, "last_analyzed": "2021-07-21T05:47:54.560768Z", "position": 3, "visibility_type": "normal", "target": null, "preview_display": true, "display_name": "first_order", "database_position": 3, "fingerprint": {"global": {"distinct-count": 47, "nil%": 0.38}, "type": {"type/DateTime": {"earliest": "2018-01-01", "latest": "2018-04-07"}}}, "created_at": "2021-07-21T05:47:53.410556Z", "base_type": "type/Date", "points_of_interest": null}, {"description": null, "database_type": "date", "semantic_type": null, "table_id": 7, "coercion_strategy": null, "name": "most_recent_order", "fingerprint_version": 5, "has_field_values": "none", "settings": null, "caveats": null, "fk_target_field_id": null, "updated_at": "2021-07-21T07:30:35.186143Z", "custom_position": 0, "effective_type": "type/Date", "active": true, "parent_id": null, "id": 42, "last_analyzed": "2021-07-21T05:47:54.560768Z", "position": 4, "visibility_type": "normal", "target": null, "preview_display": true, "display_name": "most_recent_order", "database_position": 4, "fingerprint": {"global": {"distinct-count": 53, "nil%": 0.38}, "type": {"type/DateTime": {"earliest": "2018-01-09", "latest": "2018-04-09"}}}, "created_at": "2021-07-21T05:47:53.417127Z", "base_type": "type/Date", "points_of_interest": null}, {"description": null, "database_type": "int8", "semantic_type": "type/Quantity", "table_id": 7, "coercion_strategy": null, "name": "number_of_orders", "fingerprint_version": 5, "has_field_values": "list", "settings": null, "caveats": null, "fk_target_field_id": null, "updated_at": "2021-07-21T07:30:35.197576Z", "custom_position": 0, "effective_type": "type/BigInteger", "active": true, "parent_id": null, "id": 40, "last_analyzed": "2021-07-21T05:47:54.560768Z", "position": 5, "visibility_type": "normal", "target": null, "preview_display": true, "display_name": "number_of_orders", "database_position": 5, "fingerprint": {"global": {"distinct-count": 5, "nil%": 0.38}, "type": {"type/Number": {"min": 1.0, "q1": 1.0, "q3": 2.0901356485315583, "max": 5.0, "sd": 0.7779687173818424, "avg": 1.596774193548387}}}, "created_at": "2021-07-21T05:47:53.412376Z", "base_type": "type/BigInteger", "points_of_interest": null}, {"description": null, "database_type": "int8", "semantic_type": null, "table_id": 7, "coercion_strategy": null, "name": "customer_lifetime_value", "fingerprint_version": 5, "has_field_values": "list", "settings": null, "caveats": null, "fk_target_field_id": null, "updated_at": "2021-07-21T07:30:35.288463Z", "custom_position": 0, "effective_type": "type/BigInteger", "active": true, "parent_id": null, "id": 41, "last_analyzed": "2021-07-21T05:47:54.560768Z", "position": 6, "visibility_type": "normal", "target": null, "preview_display": true, "display_name": "customer_lifetime_value", "database_position": 6, "fingerprint": {"global": {"distinct-count": 36, "nil%": 0.38}, "type": {"type/Number": {"min": 1.0, "q1": 13.464101615137753, "q3": 35.46410161513776, "max": 99.0, "sd": 18.812245525263663, "avg": 26.967741935483872}}}, "created_at": "2021-07-21T05:47:53.414671Z", "base_type": "type/BigInteger", "points_of_interest": null}], "caveats": null, "segments": [], "updated_at": "2021-07-21T07:30:35.159586Z", "entity_name": null, "active": true, "id": 7, "db_id": 2, "visibility_type": null, "field_order": "database", "display_name": "customers", "metrics": [], "created_at": "2021-07-21T05:47:53.372467Z", "points_of_interest": null}, "PUBLIC.ORDERS": {"description": null, "entity_type": "entity/TransactionTable", "schema": "public", "show_in_getting_started": false, "name": "orders", "fields": [{"description": null, "database_type": "int4", "semantic_type": null, "table_id": 6, "coercion_strategy": null, "name": "order_id", "fingerprint_version": 5, "has_field_values": "list", "settings": null, "caveats": null, "fk_target_field_id": null, "updated_at": "2021-07-21T07:30:35.255223Z", "custom_position": 0, "effective_type": "type/Integer", "active": true, "parent_id": null, "id": 47, "last_analyzed": "2021-07-21T05:47:54.560768Z", "position": 0, "visibility_type": "normal", "target": null, "preview_display": true, "display_name": "order_id", "database_position": 0, "fingerprint": {"global": {"distinct-count": 99, "nil%": 0.0}, "type": {"type/Number": {"min": 1.0, "q1": 25.25, "q3": 74.75, "max": 99.0, "sd": 28.719704534890823, "avg": 50.0}}}, "created_at": "2021-07-21T05:47:53.444318Z", "base_type": "type/Integer", "points_of_interest": null}, {"description": null, "database_type": "int4", "semantic_type": null, "table_id": 6, "coercion_strategy": null, "name": "customer_id", "fingerprint_version": 5, "has_field_values": "list", "settings": null, "caveats": null, "fk_target_field_id": null, "updated_at": "2021-07-21T07:30:35.259928Z", "custom_position": 0, "effective_type": "type/Integer", "active": true, "parent_id": null, "id": 51, "last_analyzed": "2021-07-21T05:47:54.560768Z", "position": 1, "visibility_type": "normal", "target": null, "preview_display": true, "display_name": "customer_id", "database_position": 1, "fingerprint": {"global": {"distinct-count": 62, "nil%": 0.0}, "type": {"type/Number": {"min": 1.0, "q1": 25.875, "q3": 69.625, "max": 99.0, "sd": 27.781341350472964, "avg": 48.25252525252525}}}, "created_at": "2021-07-21T05:47:53.452739Z", "base_type": "type/Integer", "points_of_interest": null}, {"description": null, "database_type": "date", "semantic_type": null, "table_id": 6, "coercion_strategy": null, "name": "order_date", "fingerprint_version": 5, "has_field_values": "none", "settings": null, "caveats": null, "fk_target_field_id": null, "updated_at": "2021-07-21T07:30:35.188819Z", "custom_position": 0, "effective_type": "type/Date", "active": true, "parent_id": null, "id": 46, "last_analyzed": "2021-07-21T05:47:54.560768Z", "position": 2, "visibility_type": "normal", "target": null, "preview_display": true, "display_name": "order_date", "database_position": 2, "fingerprint": {"global": {"distinct-count": 69, "nil%": 0.0}, "type": {"type/DateTime": {"earliest": "2018-01-01", "latest": "2018-04-09"}}}, "created_at": "2021-07-21T05:47:53.441254Z", "base_type": "type/Date", "points_of_interest": null}, {"description": null, "database_type": "text", "semantic_type": "type/Category", "table_id": 6, "coercion_strategy": null, "name": "status", "fingerprint_version": 5, "has_field_values": "list", "settings": null, "caveats": null, "fk_target_field_id": null, "updated_at": "2021-07-21T07:30:35.203655Z", "custom_position": 0, "effective_type": "type/Text", "active": true, "parent_id": null, "id": 50, "last_analyzed": "2021-07-21T05:47:54.560768Z", "position": 3, "visibility_type": "normal", "target": null, "preview_display": true, "display_name": "status", "database_position": 3, "fingerprint": {"global": {"distinct-count": 5, "nil%": 0.0}, "type": {"type/Text": {"percent-json": 0.0, "percent-url": 0.0, "percent-email": 0.0, "percent-state": 0.0, "average-length": 8.404040404040405}}}, "created_at": "2021-07-21T05:47:53.450839Z", "base_type": "type/Text", "points_of_interest": null}, {"description": null, "database_type": "int8", "semantic_type": "type/Category", "table_id": 6, "coercion_strategy": null, "name": "credit_card_amount", "fingerprint_version": 5, "has_field_values": "list", "settings": null, "caveats": null, "fk_target_field_id": null, "updated_at": "2021-07-21T07:30:35.24496Z", "custom_position": 0, "effective_type": "type/BigInteger", "active": true, "parent_id": null, "id": 44, "last_analyzed": "2021-07-21T05:47:54.560768Z", "position": 4, "visibility_type": "normal", "target": null, "preview_display": true, "display_name": "credit_card_amount", "database_position": 4, "fingerprint": {"global": {"distinct-count": 25, "nil%": 0.0}, "type": {"type/Number": {"min": 0.0, "q1": 0.0, "q3": 18.797054997187544, "max": 30.0, "sd": 10.959088854927673, "avg": 8.797979797979798}}}, "created_at": "2021-07-21T05:47:53.43752Z", "base_type": "type/BigInteger", "points_of_interest": null}, {"description": null, "database_type": "int8", "semantic_type": "type/Category", "table_id": 6, "coercion_strategy": null, "name": "coupon_amount", "fingerprint_version": 5, "has_field_values": "list", "settings": null, "caveats": null, "fk_target_field_id": null, "updated_at": "2021-07-21T07:30:35.25752Z", "custom_position": 0, "effective_type": "type/BigInteger", "active": true, "parent_id": null, "id": 49, "last_analyzed": "2021-07-21T05:47:54.560768Z", "position": 5, "visibility_type": "normal", "target": null, "preview_display": true, "display_name": "coupon_amount", "database_position": 5, "fingerprint": {"global": {"distinct-count": 12, "nil%": 0.0}, "type": {"type/Number": {"min": 0.0, "q1": 0.0, "q3": 0.4747603274810728, "max": 26.0, "sd": 5.955012405351229, "avg": 1.8686868686868687}}}, "created_at": "2021-07-21T05:47:53.448941Z", "base_type": "type/BigInteger", "points_of_interest": null}, {"description": null, "database_type": "int8", "semantic_type": "type/Category", "table_id": 6, "coercion_strategy": null, "name": "bank_transfer_amount", "fingerprint_version": 5, "has_field_values": "list", "settings": null, "caveats": null, "fk_target_field_id": null, "updated_at": "2021-07-21T07:30:35.25288Z", "custom_position": 0, "effective_type": "type/BigInteger", "active": true, "parent_id": null, "id": 45, "last_analyzed": "2021-07-21T05:47:54.560768Z", "position": 6, "visibility_type": "normal", "target": null, "preview_display": true, "display_name": "bank_transfer_amount", "database_position": 6, "fingerprint": {"global": {"distinct-count": 19, "nil%": 0.0}, "type": {"type/Number": {"min": 0.0, "q1": 0.0, "q3": 4.75, "max": 26.0, "sd": 7.420825132023675, "avg": 4.151515151515151}}}, "created_at": "2021-07-21T05:47:53.43953Z", "base_type": "type/BigInteger", "points_of_interest": null}, {"description": null, "database_type": "int8", "semantic_type": "type/Category", "table_id": 6, "coercion_strategy": null, "name": "gift_card_amount", "fingerprint_version": 5, "has_field_values": "list", "settings": null, "caveats": null, "fk_target_field_id": null, "updated_at": "2021-07-21T07:30:35.201036Z", "custom_position": 0, "effective_type": "type/BigInteger", "active": true, "parent_id": null, "id": 48, "last_analyzed": "2021-07-21T05:47:54.560768Z", "position": 7, "visibility_type": "normal", "target": null, "preview_display": true, "display_name": "gift_card_amount", "database_position": 7, "fingerprint": {"global": {"distinct-count": 11, "nil%": 0.0}, "type": {"type/Number": {"min": 0.0, "q1": 0.0, "q3": 1.3692088763283736, "max": 30.0, "sd": 6.392362351566517, "avg": 2.0707070707070705}}}, "created_at": "2021-07-21T05:47:53.447026Z", "base_type": "type/BigInteger", "points_of_interest": null}, {"description": null, "database_type": "int8", "semantic_type": null, "table_id": 6, "coercion_strategy": null, "name": "amount", "fingerprint_version": 5, "has_field_values": "list", "settings": null, "caveats": null, "fk_target_field_id": null, "updated_at": "2021-07-21T07:30:35.206083Z", "custom_position": 0, "effective_type": "type/BigInteger", "active": true, "parent_id": null, "id": 52, "last_analyzed": "2021-07-21T05:47:54.560768Z", "position": 8, "visibility_type": "normal", "target": null, "preview_display": true, "display_name": "amount", "database_position": 8, "fingerprint": {"global": {"distinct-count": 32, "nil%": 0.0}, "type": {"type/Number": {"min": 0.0, "q1": 8.202945002812456, "q3": 24.26138721247417, "max": 58.0, "sd": 10.736062525374601, "avg": 16.88888888888889}}}, "created_at": "2021-07-21T05:47:53.455652Z", "base_type": "type/BigInteger", "points_of_interest": null}], "caveats": null, "segments": [], "updated_at": "2021-07-21T07:30:35.162732Z", "entity_name": null, "active": true, "id": 6, "db_id": 2, "visibility_type": null, "field_order": "database", "display_name": "orders", "metrics": [], "created_at": "2021-07-21T05:47:53.368244Z", "points_of_interest": null}, "PUBLIC.RAW_CUSTOMERS": {"description": null, "entity_type": "entity/GenericTable", "schema": "public", "show_in_getting_started": false, "name": "raw_customers", "fields": [{"description": null, "database_type": "int4", "semantic_type": "type/PK", "table_id": 9, "coercion_strategy": null, "name": "id", "fingerprint_version": 5, "has_field_values": "none", "settings": null, "caveats": null, "fk_target_field_id": null, "updated_at": "2021-07-21T07:30:35.234376Z", "custom_position": 0, "effective_type": "type/Integer", "active": true, "parent_id": null, "id": 55, "last_analyzed": "2021-07-21T05:47:54.560768Z", "position": 0, "visibility_type": "normal", "target": null, "preview_display": true, "display_name": "id", "database_position": 0, "fingerprint": null, "created_at": "2021-07-21T05:47:53.473642Z", "base_type": "type/Integer", "points_of_interest": null}, {"description": null, "database_type": "text", "semantic_type": "type/Name", "table_id": 9, "coercion_strategy": null, "name": "first_name", "fingerprint_version": 5, "has_field_values": "list", "settings": null, "caveats": null, "fk_target_field_id": null, "updated_at": "2021-07-21T07:30:35.208554Z", "custom_position": 0, "effective_type": "type/Text", "active": true, "parent_id": null, "id": 53, "last_analyzed": "2021-07-21T05:47:54.560768Z", "position": 1, "visibility_type": "normal", "target": null, "preview_display": true, "display_name": "first_name", "database_position": 1, "fingerprint": {"global": {"distinct-count": 79, "nil%": 0.0}, "type": {"type/Text": {"percent-json": 0.0, "percent-url": 0.0, "percent-email": 0.0, "percent-state": 0.02, "average-length": 5.86}}}, "created_at": "2021-07-21T05:47:53.469932Z", "base_type": "type/Text", "points_of_interest": null}, {"description": null, "database_type": "text", "semantic_type": "type/Name", "table_id": 9, "coercion_strategy": null, "name": "last_name", "fingerprint_version": 5, "has_field_values": "list", "settings": null, "caveats": null, "fk_target_field_id": null, "updated_at": "2021-07-21T07:30:35.262668Z", "custom_position": 0, "effective_type": "type/Text", "active": true, "parent_id": null, "id": 54, "last_analyzed": "2021-07-21T05:47:54.560768Z", "position": 2, "visibility_type": "normal", "target": null, "preview_display": true, "display_name": "last_name", "database_position": 2, "fingerprint": {"global": {"distinct-count": 19, "nil%": 0.0}, "type": {"type/Text": {"percent-json": 0.0, "percent-url": 0.0, "percent-email": 0.0, "percent-state": 0.0, "average-length": 2.0}}}, "created_at": "2021-07-21T05:47:53.471892Z", "base_type": "type/Text", "points_of_interest": null}], "caveats": null, "segments": [], "updated_at": "2021-07-21T07:30:35.166218Z", "entity_name": null, "active": true, "id": 9, "db_id": 2, "visibility_type": null, "field_order": "database", "display_name": "raw_customers", "metrics": [], "created_at": "2021-07-21T05:47:53.380782Z", "points_of_interest": null}, "PUBLIC.RAW_ORDERS": {"description": null, "entity_type": "entity/TransactionTable", "schema": "public", "show_in_getting_started": false, "name": "raw_orders", "fields": [{"description": null, "database_type": "int4", "semantic_type": "type/PK", "table_id": 12, "coercion_strategy": null, "name": "id", "fingerprint_version": 5, "has_field_values": "none", "settings": null, "caveats": null, "fk_target_field_id": null, "updated_at": "2021-07-21T07:30:35.266036Z", "custom_position": 0, "effective_type": "type/Integer", "active": true, "parent_id": null, "id": 57, "last_analyzed": "2021-07-21T05:47:54.560768Z", "position": 0, "visibility_type": "normal", "target": null, "preview_display": true, "display_name": "id", "database_position": 0, "fingerprint": null, "created_at": "2021-07-21T05:47:53.48931Z", "base_type": "type/Integer", "points_of_interest": null}, {"description": null, "database_type": "int4", "semantic_type": null, "table_id": 12, "coercion_strategy": null, "name": "user_id", "fingerprint_version": 5, "has_field_values": "list", "settings": null, "caveats": null, "fk_target_field_id": null, "updated_at": "2021-07-21T07:30:35.214697Z", "custom_position": 0, "effective_type": "type/Integer", "active": true, "parent_id": null, "id": 59, "last_analyzed": "2021-07-21T05:47:54.560768Z", "position": 1, "visibility_type": "normal", "target": null, "preview_display": true, "display_name": "user_id", "database_position": 1, "fingerprint": {"global": {"distinct-count": 62, "nil%": 0.0}, "type": {"type/Number": {"min": 1.0, "q1": 25.875, "q3": 69.625, "max": 99.0, "sd": 27.781341350472964, "avg": 48.25252525252525}}}, "created_at": "2021-07-21T05:47:53.494231Z", "base_type": "type/Integer", "points_of_interest": null}, {"description": null, "database_type": "date", "semantic_type": null, "table_id": 12, "coercion_strategy": null, "name": "order_date", "fingerprint_version": 5, "has_field_values": "none", "settings": null, "caveats": null, "fk_target_field_id": null, "updated_at": "2021-07-21T07:30:35.268564Z", "custom_position": 0, "effective_type": "type/Date", "active": true, "parent_id": null, "id": 56, "last_analyzed": "2021-07-21T05:47:54.560768Z", "position": 2, "visibility_type": "normal", "target": null, "preview_display": true, "display_name": "order_date", "database_position": 2, "fingerprint": {"global": {"distinct-count": 69, "nil%": 0.0}, "type": {"type/DateTime": {"earliest": "2018-01-01", "latest": "2018-04-09"}}}, "created_at": "2021-07-21T05:47:53.48571Z", "base_type": "type/Date", "points_of_interest": null}, {"description": null, "database_type": "text", "semantic_type": "type/Category", "table_id": 12, "coercion_strategy": null, "name": "status", "fingerprint_version": 5, "has_field_values": "list", "settings": null, "caveats": null, "fk_target_field_id": null, "updated_at": "2021-07-21T07:30:35.211374Z", "custom_position": 0, "effective_type": "type/Text", "active": true, "parent_id": null, "id": 58, "last_analyzed": "2021-07-21T05:47:54.560768Z", "position": 3, "visibility_type": "normal", "target": null, "preview_display": true, "display_name": "status", "database_position": 3, "fingerprint": {"global": {"distinct-count": 5, "nil%": 0.0}, "type": {"type/Text": {"percent-json": 0.0, "percent-url": 0.0, "percent-email": 0.0, "percent-state": 0.0, "average-length": 8.404040404040405}}}, "created_at": "2021-07-21T05:47:53.491699Z", "base_type": "type/Text", "points_of_interest": null}], "caveats": null, "segments": [], "updated_at": "2021-07-21T07:30:35.170459Z", "entity_name": null, "active": true, "id": 12, "db_id": 2, "visibility_type": null, "field_order": "database", "display_name": "raw_orders", "metrics": [], "created_at": "2021-07-21T05:47:53.391873Z", "points_of_interest": null}, "PUBLIC.RAW_PAYMENTS": {"description": null, "entity_type": "entity/GenericTable", "schema": "public", "show_in_getting_started": false, "name": "raw_payments", "fields": [{"description": null, "database_type": "int4", "semantic_type": "type/PK", "table_id": 11, "coercion_strategy": null, "name": "id", "fingerprint_version": 5, "has_field_values": "none", "settings": null, "caveats": null, "fk_target_field_id": null, "updated_at": "2021-07-21T07:30:35.271155Z", "custom_position": 0, "effective_type": "type/Integer", "active": true, "parent_id": null, "id": 62, "last_analyzed": "2021-07-21T05:47:54.560768Z", "position": 0, "visibility_type": "normal", "target": null, "preview_display": true, "display_name": "id", "database_position": 0, "fingerprint": null, "created_at": "2021-07-21T05:47:53.511724Z", "base_type": "type/Integer", "points_of_interest": null}, {"description": null, "database_type": "int4", "semantic_type": null, "table_id": 11, "coercion_strategy": null, "name": "order_id", "fingerprint_version": 5, "has_field_values": "list", "settings": null, "caveats": null, "fk_target_field_id": null, "updated_at": "2021-07-21T07:30:35.247411Z", "custom_position": 0, "effective_type": "type/Integer", "active": true, "parent_id": null, "id": 60, "last_analyzed": "2021-07-21T05:47:54.560768Z", "position": 1, "visibility_type": "normal", "target": null, "preview_display": true, "display_name": "order_id", "database_position": 1, "fingerprint": {"global": {"distinct-count": 99, "nil%": 0.0}, "type": {"type/Number": {"min": 1.0, "q1": 24.904857366030992, "q3": 75.25, "max": 99.0, "sd": 28.540193317267853, "avg": 50.0353982300885}}}, "created_at": "2021-07-21T05:47:53.506967Z", "base_type": "type/Integer", "points_of_interest": null}, {"description": null, "database_type": "text", "semantic_type": "type/Category", "table_id": 11, "coercion_strategy": null, "name": "payment_method", "fingerprint_version": 5, "has_field_values": "list", "settings": null, "caveats": null, "fk_target_field_id": null, "updated_at": "2021-07-21T07:30:35.217422Z", "custom_position": 0, "effective_type": "type/Text", "active": true, "parent_id": null, "id": 61, "last_analyzed": "2021-07-21T05:47:54.560768Z", "position": 2, "visibility_type": "normal", "target": null, "preview_display": true, "display_name": "payment_method", "database_position": 2, "fingerprint": {"global": {"distinct-count": 4, "nil%": 0.0}, "type": {"type/Text": {"percent-json": 0.0, "percent-url": 0.0, "percent-email": 0.0, "percent-state": 0.0, "average-length": 10.79646017699115}}}, "created_at": "2021-07-21T05:47:53.508887Z", "base_type": "type/Text", "points_of_interest": null}, {"description": null, "database_type": "int4", "semantic_type": "type/Category", "table_id": 11, "coercion_strategy": null, "name": "amount", "fingerprint_version": 5, "has_field_values": "list", "settings": null, "caveats": null, "fk_target_field_id": null, "updated_at": "2021-07-21T07:30:35.273607Z", "custom_position": 0, "effective_type": "type/Integer", "active": true, "parent_id": null, "id": 63, "last_analyzed": "2021-07-21T05:47:54.560768Z", "position": 3, "visibility_type": "normal", "target": null, "preview_display": true, "display_name": "amount", "database_position": 3, "fingerprint": {"global": {"distinct-count": 30, "nil%": 0.0}, "type": {"type/Number": {"min": 0.0, "q1": 606.4037815689348, "q3": 2278.791845139511, "max": 3000.0, "sd": 919.836873351873, "avg": 1479.646017699115}}}, "created_at": "2021-07-21T05:47:53.513727Z", "base_type": "type/Integer", "points_of_interest": null}], "caveats": null, "segments": [], "updated_at": "2021-07-21T07:30:35.173953Z", "entity_name": null, "active": true, "id": 11, "db_id": 2, "visibility_type": null, "field_order": "database", "display_name": "raw_payments", "metrics": [], "created_at": "2021-07-21T05:47:53.388179Z", "points_of_interest": null}, "PUBLIC.STG_CUSTOMERS": {"description": null, "entity_type": "entity/GenericTable", "schema": "public", "show_in_getting_started": false, "name": "stg_customers", "fields": [{"description": null, "database_type": "int4", "semantic_type": null, "table_id": 8, "coercion_strategy": null, "name": "customer_id", "fingerprint_version": 5, "has_field_values": "list", "settings": null, "caveats": null, "fk_target_field_id": null, "updated_at": "2021-07-21T07:30:35.276108Z", "custom_position": 0, "effective_type": "type/Integer", "active": true, "parent_id": null, "id": 65, "last_analyzed": "2021-07-21T05:47:54.560768Z", "position": 0, "visibility_type": "normal", "target": null, "preview_display": true, "display_name": "customer_id", "database_position": 0, "fingerprint": {"global": {"distinct-count": 100, "nil%": 0.0}, "type": {"type/Number": {"min": 1.0, "q1": 25.5, "q3": 75.5, "max": 100.0, "sd": 29.008358252146028, "avg": 50.5}}}, "created_at": "2021-07-21T05:47:53.528091Z", "base_type": "type/Integer", "points_of_interest": null}, {"description": null, "database_type": "text", "semantic_type": "type/Name", "table_id": 8, "coercion_strategy": null, "name": "first_name", "fingerprint_version": 5, "has_field_values": "list", "settings": null, "caveats": null, "fk_target_field_id": null, "updated_at": "2021-07-21T07:30:35.220226Z", "custom_position": 0, "effective_type": "type/Text", "active": true, "parent_id": null, "id": 64, "last_analyzed": "2021-07-21T05:47:54.560768Z", "position": 1, "visibility_type": "normal", "target": null, "preview_display": true, "display_name": "first_name", "database_position": 1, "fingerprint": {"global": {"distinct-count": 79, "nil%": 0.0}, "type": {"type/Text": {"percent-json": 0.0, "percent-url": 0.0, "percent-email": 0.0, "percent-state": 0.02, "average-length": 5.86}}}, "created_at": "2021-07-21T05:47:53.525589Z", "base_type": "type/Text", "points_of_interest": null}, {"description": null, "database_type": "text", "semantic_type": "type/Name", "table_id": 8, "coercion_strategy": null, "name": "last_name", "fingerprint_version": 5, "has_field_values": "list", "settings": null, "caveats": null, "fk_target_field_id": null, "updated_at": "2021-07-21T07:30:35.222931Z", "custom_position": 0, "effective_type": "type/Text", "active": true, "parent_id": null, "id": 66, "last_analyzed": "2021-07-21T05:47:54.560768Z", "position": 2, "visibility_type": "normal", "target": null, "preview_display": true, "display_name": "last_name", "database_position": 2, "fingerprint": {"global": {"distinct-count": 19, "nil%": 0.0}, "type": {"type/Text": {"percent-json": 0.0, "percent-url": 0.0, "percent-email": 0.0, "percent-state": 0.0, "average-length": 2.0}}}, "created_at": "2021-07-21T05:47:53.530075Z", "base_type": "type/Text", "points_of_interest": null}], "caveats": null, "segments": [], "updated_at": "2021-07-21T07:30:35.176617Z", "entity_name": null, "active": true, "id": 8, "db_id": 2, "visibility_type": null, "field_order": "database", "display_name": "stg_customers", "metrics": [], "created_at": "2021-07-21T05:47:53.376525Z", "points_of_interest": null}, "PUBLIC.STG_ORDERS": {"description": null, "entity_type": "entity/TransactionTable", "schema": "public", "show_in_getting_started": false, "name": "stg_orders", "fields": [{"description": null, "database_type": "int4", "semantic_type": null, "table_id": 5, "coercion_strategy": null, "name": "order_id", "fingerprint_version": 5, "has_field_values": "list", "settings": null, "caveats": null, "fk_target_field_id": null, "updated_at": "2021-07-21T07:30:35.225874Z", "custom_position": 0, "effective_type": "type/Integer", "active": true, "parent_id": null, "id": 68, "last_analyzed": "2021-07-21T05:47:54.560768Z", "position": 0, "visibility_type": "normal", "target": null, "preview_display": true, "display_name": "order_id", "database_position": 0, "fingerprint": {"global": {"distinct-count": 99, "nil%": 0.0}, "type": {"type/Number": {"min": 1.0, "q1": 25.25, "q3": 74.75, "max": 99.0, "sd": 28.719704534890823, "avg": 50.0}}}, "created_at": "2021-07-21T05:47:53.545842Z", "base_type": "type/Integer", "points_of_interest": null}, {"description": null, "database_type": "int4", "semantic_type": null, "table_id": 5, "coercion_strategy": null, "name": "customer_id", "fingerprint_version": 5, "has_field_values": "list", "settings": null, "caveats": null, "fk_target_field_id": null, "updated_at": "2021-07-21T07:30:35.229545Z", "custom_position": 0, "effective_type": "type/Integer", "active": true, "parent_id": null, "id": 70, "last_analyzed": "2021-07-21T05:47:54.560768Z", "position": 1, "visibility_type": "normal", "target": null, "preview_display": true, "display_name": "customer_id", "database_position": 1, "fingerprint": {"global": {"distinct-count": 62, "nil%": 0.0}, "type": {"type/Number": {"min": 1.0, "q1": 25.875, "q3": 69.625, "max": 99.0, "sd": 27.781341350472964, "avg": 48.25252525252525}}}, "created_at": "2021-07-21T05:47:53.549796Z", "base_type": "type/Integer", "points_of_interest": null}, {"description": null, "database_type": "date", "semantic_type": null, "table_id": 5, "coercion_strategy": null, "name": "order_date", "fingerprint_version": 5, "has_field_values": "none", "settings": null, "caveats": null, "fk_target_field_id": null, "updated_at": "2021-07-21T07:30:35.285732Z", "custom_position": 0, "effective_type": "type/Date", "active": true, "parent_id": null, "id": 67, "last_analyzed": "2021-07-21T05:47:54.560768Z", "position": 2, "visibility_type": "normal", "target": null, "preview_display": true, "display_name": "order_date", "database_position": 2, "fingerprint": {"global": {"distinct-count": 69, "nil%": 0.0}, "type": {"type/DateTime": {"earliest": "2018-01-01", "latest": "2018-04-09"}}}, "created_at": "2021-07-21T05:47:53.543598Z", "base_type": "type/Date", "points_of_interest": null}, {"description": null, "database_type": "text", "semantic_type": "type/Category", "table_id": 5, "coercion_strategy": null, "name": "status", "fingerprint_version": 5, "has_field_values": "list", "settings": null, "caveats": null, "fk_target_field_id": null, "updated_at": "2021-07-21T07:30:35.27984Z", "custom_position": 0, "effective_type": "type/Text", "active": true, "parent_id": null, "id": 69, "last_analyzed": "2021-07-21T05:47:54.560768Z", "position": 3, "visibility_type": "normal", "target": null, "preview_display": true, "display_name": "status", "database_position": 3, "fingerprint": {"global": {"distinct-count": 5, "nil%": 0.0}, "type": {"type/Text": {"percent-json": 0.0, "percent-url": 0.0, "percent-email": 0.0, "percent-state": 0.0, "average-length": 8.404040404040405}}}, "created_at": "2021-07-21T05:47:53.547849Z", "base_type": "type/Text", "points_of_interest": null}], "caveats": null, "segments": [], "updated_at": "2021-07-21T07:30:35.179323Z", "entity_name": null, "active": true, "id": 5, "db_id": 2, "visibility_type": null, "field_order": "database", "display_name": "stg_orders", "metrics": [], "created_at": "2021-07-21T05:47:53.363321Z", "points_of_interest": null}, "PUBLIC.STG_PAYMENTS": {"description": null, "entity_type": "entity/GenericTable", "schema": "public", "show_in_getting_started": false, "name": "stg_payments", "fields": [{"description": null, "database_type": "int4", "semantic_type": null, "table_id": 10, "coercion_strategy": null, "name": "payment_id", "fingerprint_version": 5, "has_field_values": "none", "settings": null, "caveats": null, "fk_target_field_id": null, "updated_at": "2021-07-21T07:30:35.191434Z", "custom_position": 0, "effective_type": "type/Integer", "active": true, "parent_id": null, "id": 74, "last_analyzed": "2021-07-21T05:47:54.560768Z", "position": 0, "visibility_type": "normal", "target": null, "preview_display": true, "display_name": "payment_id", "database_position": 0, "fingerprint": {"global": {"distinct-count": 113, "nil%": 0.0}, "type": {"type/Number": {"min": 1.0, "q1": 28.75, "q3": 85.25, "max": 113.0, "sd": 32.76097144469315, "avg": 57.0}}}, "created_at": "2021-07-21T05:47:53.570339Z", "base_type": "type/Integer", "points_of_interest": null}, {"description": null, "database_type": "int4", "semantic_type": null, "table_id": 10, "coercion_strategy": null, "name": "order_id", "fingerprint_version": 5, "has_field_values": "list", "settings": null, "caveats": null, "fk_target_field_id": null, "updated_at": "2021-07-21T07:30:35.282867Z", "custom_position": 0, "effective_type": "type/Integer", "active": true, "parent_id": null, "id": 71, "last_analyzed": "2021-07-21T05:47:54.560768Z", "position": 1, "visibility_type": "normal", "target": null, "preview_display": true, "display_name": "order_id", "database_position": 1, "fingerprint": {"global": {"distinct-count": 99, "nil%": 0.0}, "type": {"type/Number": {"min": 1.0, "q1": 24.904857366030992, "q3": 75.25, "max": 99.0, "sd": 28.540193317267853, "avg": 50.0353982300885}}}, "created_at": "2021-07-21T05:47:53.562985Z", "base_type": "type/Integer", "points_of_interest": null}, {"description": null, "database_type": "text", "semantic_type": "type/Category", "table_id": 10, "coercion_strategy": null, "name": "payment_method", "fingerprint_version": 5, "has_field_values": "list", "settings": null, "caveats": null, "fk_target_field_id": null, "updated_at": "2021-07-21T07:30:35.236542Z", "custom_position": 0, "effective_type": "type/Text", "active": true, "parent_id": null, "id": 72, "last_analyzed": "2021-07-21T05:47:54.560768Z", "position": 2, "visibility_type": "normal", "target": null, "preview_display": true, "display_name": "payment_method", "database_position": 2, "fingerprint": {"global": {"distinct-count": 4, "nil%": 0.0}, "type": {"type/Text": {"percent-json": 0.0, "percent-url": 0.0, "percent-email": 0.0, "percent-state": 0.0, "average-length": 10.79646017699115}}}, "created_at": "2021-07-21T05:47:53.566146Z", "base_type": "type/Text", "points_of_interest": null}, {"description": null, "database_type": "int4", "semantic_type": "type/Category", "table_id": 10, "coercion_strategy": null, "name": "amount", "fingerprint_version": 5, "has_field_values": "list", "settings": null, "caveats": null, "fk_target_field_id": null, "updated_at": "2021-07-21T07:30:35.240154Z", "custom_position": 0, "effective_type": "type/Integer", "active": true, "parent_id": null, "id": 73, "last_analyzed": "2021-07-21T05:47:54.560768Z", "position": 3, "visibility_type": "normal", "target": null, "preview_display": true, "display_name": "amount", "database_position": 3, "fingerprint": {"global": {"distinct-count": 30, "nil%": 0.0}, "type": {"type/Number": {"min": 0.0, "q1": 6.064037815689349, "q3": 22.787918451395115, "max": 30.0, "sd": 9.198368733518729, "avg": 14.79646017699115}}}, "created_at": "2021-07-21T05:47:53.568245Z", "base_type": "type/Integer", "points_of_interest": null}], "caveats": null, "segments": [], "updated_at": "2021-07-21T07:30:35.181929Z", "entity_name": null, "active": true, "id": 10, "db_id": 2, "visibility_type": null, "field_order": "database", "display_name": "stg_payments", "metrics": [], "created_at": "2021-07-21T05:47:53.384404Z", "points_of_interest": null}} \ No newline at end of file diff --git a/tests/test_metabase.py b/tests/test_metabase.py index d80c30e0..1f91db3a 100644 --- a/tests/test_metabase.py +++ b/tests/test_metabase.py @@ -277,10 +277,14 @@ def test_exposures(self): output_path="tests/fixtures/exposure/", ) # Baseline in SCM - with open("tests/fixtures/exposure/baseline_test_exposures.yml", "r") as f: + with open( + "tests/fixtures/exposure/baseline_test_exposures.yml", "r", encoding="utf-8" + ) as f: baseline = yaml.safe_load(f) # Load from YAML and tear down - with open("tests/fixtures/exposure/unittest_exposures.yml", "r") as f: + with open( + "tests/fixtures/exposure/unittest_exposures.yml", "r", encoding="utf-8" + ) as f: sample = yaml.safe_load(f) baseline_exposures = sorted(baseline["exposures"], key=lambda ele: ele["name"]) @@ -300,8 +304,8 @@ def test_build_lookups(self): "PUBLIC.STG_ORDERS", "PUBLIC.STG_PAYMENTS", ] - table_lookups, field_lookups = mbc.build_metadata_lookups(database_id=2) - self.assertEqual(baseline_tables, list(table_lookups.keys())) + metadata = mbc.build_metadata(database_id=2) + self.assertEqual(baseline_tables, list(metadata.tables.keys())) baseline_columns = [ [ "CUSTOMER_ID", @@ -331,12 +335,4 @@ def test_build_lookups(self): ["PAYMENT_ID", "ORDER_ID", "PAYMENT_METHOD", "AMOUNT"], ] for table, columns in zip(baseline_tables, baseline_columns): - self.assertEqual(columns, list(field_lookups[table].keys())) - baseline_table_lookups = json.load( - open(f"tests/fixtures/lookups/table_lookups.json") - ) - baseline_field_lookups = json.load( - open(f"tests/fixtures/lookups/field_lookups.json") - ) - self.assertEqual(baseline_table_lookups, table_lookups) - self.assertEqual(baseline_field_lookups, field_lookups) + self.assertEqual(columns, list(metadata.tables[table]["fields"].keys())) diff --git a/tests/utils_mb_test_suite.py b/tests/utils_mb_test_suite.py index e99deb1c..7b03dd69 100644 --- a/tests/utils_mb_test_suite.py +++ b/tests/utils_mb_test_suite.py @@ -24,83 +24,10 @@ def test_mock_api(method: str, path: str): BASE_PATH = "tests/fixtures/mock_api/" if method == "get": if os.path.exists(f"{BASE_PATH}/{path.lstrip('/')}.json"): - return json.load(open(f"{BASE_PATH}/{path.lstrip('/')}.json")) - else: - return {} - - -def rebuild_mock_api(): - object_models = [ - "card", - "collection", - "dashboard", - "database", - "field", - "metric", - "table", - "user", - ] - for object in object_models: - if not os.path.exists( - f"/home/alexb/dbt-metabase/tests/fixtures/mock_api/api/{object}" - ): - os.mkdir(f"/home/alexb/dbt-metabase/tests/fixtures/mock_api/api/{object}") - if not object == "field": - meta = mbc.api("get", f"/api/{object}") - with open( - f"/home/alexb/dbt-metabase/tests/fixtures/mock_api/api/{object}.json", - "w", - ) as f: - f.write(json.dumps(meta)) - for i in range(100): - meta = mbc.api("get", f"/api/{object}/{i}", critical=False) - if meta: - with open( - f"/home/alexb/dbt-metabase/tests/fixtures/mock_api/api/{object}/{i}.json", - "w", - ) as f: - f.write(json.dumps(meta)) - if object == "collection": - meta = mbc.api("get", f"/api/{object}/{i}/items", critical=False) - if meta: - if not os.path.exists( - f"/home/alexb/dbt-metabase/tests/fixtures/mock_api/api/{object}/{i}" - ): - os.mkdir( - f"/home/alexb/dbt-metabase/tests/fixtures/mock_api/api/{object}/{i}" - ) - with open( - f"/home/alexb/dbt-metabase/tests/fixtures/mock_api/api/{object}/{i}/items.json", - "w", - ) as f: - f.write(json.dumps(meta)) - meta = mbc.api("get", f"/api/{object}/root/items", critical=False) - if meta: - if not os.path.exists( - f"/home/alexb/dbt-metabase/tests/fixtures/mock_api/api/{object}/root" - ): - os.mkdir( - f"/home/alexb/dbt-metabase/tests/fixtures/mock_api/api/{object}/root" - ) - with open( - f"/home/alexb/dbt-metabase/tests/fixtures/mock_api/api/{object}/root/items.json", - "w", - ) as f: - f.write(json.dumps(meta)) - if object == "database": - meta = mbc.api("get", f"/api/{object}/{i}/metadata", critical=False) - if meta: - if not os.path.exists( - f"/home/alexb/dbt-metabase/tests/fixtures/mock_api/api/{object}/{i}" - ): - os.mkdir( - f"/home/alexb/dbt-metabase/tests/fixtures/mock_api/api/{object}/{i}" - ) - with open( - f"/home/alexb/dbt-metabase/tests/fixtures/mock_api/api/{object}/{i}/metadata.json", - "w", - ) as f: - f.write(json.dumps(meta)) + return json.load( + open(f"{BASE_PATH}/{path.lstrip('/')}.json", encoding="utf-8") + ) + return {} def rebuild_baseline_exposure_yaml(): @@ -110,7 +37,6 @@ def rebuild_baseline_exposure_yaml(): schema="PUBLIC", description="This table has basic information about a customer, as well as some derived facts based on a customer's orders", model_type=ModelType.nodes, - ref="ref('customers')", columns=[ MetabaseColumn( name="CUSTOMER_ID", @@ -182,7 +108,6 @@ def rebuild_baseline_exposure_yaml(): schema="PUBLIC", description="This table has basic information about orders, as well as some derived facts based on payments", model_type=ModelType.nodes, - ref="ref('orders')", columns=[ MetabaseColumn( name="ORDER_ID", @@ -272,7 +197,6 @@ def rebuild_baseline_exposure_yaml(): schema="PUBLIC", description="", model_type=ModelType.nodes, - ref="ref('stg_customers')", columns=[ MetabaseColumn( name="CUSTOMER_ID", @@ -290,7 +214,6 @@ def rebuild_baseline_exposure_yaml(): schema="PUBLIC", description="", model_type=ModelType.nodes, - ref="ref('stg_orders')", columns=[ MetabaseColumn( name="ORDER_ID", @@ -317,7 +240,6 @@ def rebuild_baseline_exposure_yaml(): schema="PUBLIC", description="", model_type=ModelType.nodes, - ref="ref('stg_payments')", columns=[ MetabaseColumn( name="PAYMENT_ID", @@ -345,19 +267,3 @@ def rebuild_baseline_exposure_yaml(): output_name="baseline_test_exposures", output_path="tests/fixtures/exposure", ) - - -def rebuild_lookup_artifacts(): - tables, fields = mbc.build_metadata_lookups(database_id=2) - if not os.path.exists(f"/home/alexb/dbt-metabase/tests/fixtures/lookups"): - os.mkdir(f"/home/alexb/dbt-metabase/tests/fixtures/lookups") - with open( - f"/home/alexb/dbt-metabase/tests/fixtures/lookups/table_lookups.json", - "w", - ) as f: - f.write(json.dumps(tables)) - with open( - f"/home/alexb/dbt-metabase/tests/fixtures/lookups/field_lookups.json", - "w", - ) as f: - f.write(json.dumps(fields))