Skip to content

Commit

Permalink
test: added profiling changes
Browse files Browse the repository at this point in the history
  • Loading branch information
@sagar-salvi-apptware
sagar-salvi-apptware committed Nov 12, 2024
1 parent d15e54e commit 7929bd2
Show file tree
Hide file tree
Showing 5 changed files with 385 additions and 1,609 deletions.
3 changes: 1 addition & 2 deletions metadata-ingestion/src/datahub/ingestion/source/cassandra/cassandra.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -335,9 +335,8 @@ def _extract_columns_from_table(
CassandraToSchemaFieldConverter.get_schema_fields(column_infos)
)
if not schema_fields:
logger.warning(f"Table {table_name} has no columns, skipping")
self.report.report_warning(
"table", f"Table {table_name} has no columns, skipping"
message="Table has no columns, skipping", context=table_name
)
return

Expand Down
328 changes: 328 additions & 0 deletions metadata-ingestion/tests/integration/cassandra/cassandra_mcps_golden.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1597,6 +1597,76 @@
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:cassandra,example_keyspace.shopping_cart,PROD)",
"changeType": "UPSERT",
"aspectName": "schemaMetadata",
"aspect": {
"json": {
"schemaName": "shopping_cart",
"platform": "urn:li:dataPlatform:cassandra",
"version": 0,
"created": {
"time": 0,
"actor": "urn:li:corpuser:unknown"
},
"lastModified": {
"time": 0,
"actor": "urn:li:corpuser:unknown"
},
"hash": "",
"platformSchema": {
"com.linkedin.schema.OtherSchema": {
"rawSchema": "[{\"keyspace_name\": \"example_keyspace\", \"table_name\": \"shopping_cart\", \"column_name\": \"item_count\", \"clustering_order\": \"none\", \"kind\": \"regular\", \"position\": -1, \"type\": \"int\"}, {\"keyspace_name\": \"example_keyspace\", \"table_name\": \"shopping_cart\", \"column_name\": \"last_update_timestamp\", \"clustering_order\": \"none\", \"kind\": \"regular\", \"position\": -1, \"type\": \"timestamp\"}, {\"keyspace_name\": \"example_keyspace\", \"table_name\": \"shopping_cart\", \"column_name\": \"userid\", \"clustering_order\": \"none\", \"kind\": \"partition_key\", \"position\": 0, \"type\": \"text\"}]"
}
},
"fields": [
{
"fieldPath": "[version=2.0].[type=int].item_count",
"nullable": true,
"type": {
"type": {
"com.linkedin.schema.NumberType": {}
}
},
"nativeDataType": "int",
"recursive": false,
"isPartOfKey": false
},
{
"fieldPath": "[version=2.0].[type=timestamp].last_update_timestamp",
"nullable": true,
"type": {
"type": {
"com.linkedin.schema.TimeType": {}
}
},
"nativeDataType": "timestamp",
"recursive": false,
"isPartOfKey": false
},
{
"fieldPath": "[version=2.0].[type=text].userid",
"nullable": true,
"type": {
"type": {
"com.linkedin.schema.StringType": {}
}
},
"nativeDataType": "text",
"recursive": false,
"isPartOfKey": false
}
]
}
},
"systemMetadata": {
"lastObserved": 1731410842609,
"runId": "cassandra-test",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:cassandra,example_keyspace.example_view_1,PROD)",
Expand All @@ -1622,6 +1692,40 @@
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:cassandra,example_keyspace.shopping_cart,PROD)",
"changeType": "UPSERT",
"aspectName": "subTypes",
"aspect": {
"json": {
"typeNames": [
"Table"
]
}
},
"systemMetadata": {
"lastObserved": 1731410842611,
"runId": "cassandra-test",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:cassandra,example_keyspace.shopping_cart,PROD)",
"changeType": "UPSERT",
"aspectName": "status",
"aspect": {
"json": {
"removed": false
}
},
"systemMetadata": {
"lastObserved": 1731410842610,
"runId": "cassandra-test",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:cassandra,example_keyspace.example_view_1,PROD)",
Expand All @@ -1640,6 +1744,34 @@
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:cassandra,example_keyspace.shopping_cart,PROD)",
"changeType": "UPSERT",
"aspectName": "datasetProperties",
"aspect": {
"json": {
"customProperties": {
"bloom_filter_fp_chance": "0.01",
"caching": "{'keys': 'ALL', 'rows_per_partition': 'NONE'}",
"cdc": "None",
"compaction": "{'class': 'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy', 'max_threshold': '32', 'min_threshold': '4'}",
"compression": "{'chunk_length_in_kb': '16', 'class': 'org.apache.cassandra.io.compress.LZ4Compressor'}",
"max_index_interval": "2048",
"min_index_interval": "128"
},
"name": "shopping_cart",
"qualifiedName": "example_keyspace.shopping_cart",
"description": "",
"tags": []
}
},
"systemMetadata": {
"lastObserved": 1731410842611,
"runId": "cassandra-test",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:cassandra,example_keyspace.example_view_1,PROD)",
Expand All @@ -1656,6 +1788,43 @@
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:cassandra,example_keyspace.shopping_cart,PROD)",
"changeType": "UPSERT",
"aspectName": "browsePathsV2",
"aspect": {
"json": {
"path": [
{
"id": "urn:li:container:305f73c676989511c67d97ace119138c",
"urn": "urn:li:container:305f73c676989511c67d97ace119138c"
}
]
}
},
"systemMetadata": {
"lastObserved": 1731410842612,
"runId": "cassandra-test",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:cassandra,example_keyspace.shopping_cart,PROD)",
"changeType": "UPSERT",
"aspectName": "container",
"aspect": {
"json": {
"container": "urn:li:container:305f73c676989511c67d97ace119138c"
}
},
"systemMetadata": {
"lastObserved": 1731410842611,
"runId": "cassandra-test",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:cassandra,example_keyspace.example_view_1,PROD)",
Expand Down Expand Up @@ -2022,5 +2191,164 @@
"runId": "cassandra-test",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:cassandra,example_keyspace.all_data_types,PROD)",
"changeType": "UPSERT",
"aspectName": "datasetProfile",
"aspect": {
"json": {
"timestampMillis": 1731410997521,
"partitionSpec": {
"partition": "FULL_TABLE_SNAPSHOT",
"type": "FULL_TABLE"
},
"rowCount": 0,
"columnCount": 27,
"fieldProfiles": []
}
},
"systemMetadata": {
"lastObserved": 1731410997537,
"runId": "cassandra-test",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:cassandra,cass_test_1.people,PROD)",
"changeType": "UPSERT",
"aspectName": "datasetProfile",
"aspect": {
"json": {
"timestampMillis": 1731410997579,
"partitionSpec": {
"partition": "FULL_TABLE_SNAPSHOT",
"type": "FULL_TABLE"
},
"rowCount": 0,
"columnCount": 3,
"fieldProfiles": []
}
},
"systemMetadata": {
"lastObserved": 1731410997580,
"runId": "cassandra-test",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:cassandra,example_keyspace.counter_table,PROD)",
"changeType": "UPSERT",
"aspectName": "datasetProfile",
"aspect": {
"json": {
"timestampMillis": 1731410997537,
"partitionSpec": {
"partition": "FULL_TABLE_SNAPSHOT",
"type": "FULL_TABLE"
},
"rowCount": 0,
"columnCount": 2,
"fieldProfiles": []
}
},
"systemMetadata": {
"lastObserved": 1731410997549,
"runId": "cassandra-test",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:cassandra,cass_test_2.tasks,PROD)",
"changeType": "UPSERT",
"aspectName": "datasetProfile",
"aspect": {
"json": {
"timestampMillis": 1731410997558,
"partitionSpec": {
"partition": "FULL_TABLE_SNAPSHOT",
"type": "FULL_TABLE"
},
"rowCount": 0,
"columnCount": 4,
"fieldProfiles": []
}
},
"systemMetadata": {
"lastObserved": 1731410997570,
"runId": "cassandra-test",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:cassandra,cass_test_1.information,PROD)",
"changeType": "UPSERT",
"aspectName": "datasetProfile",
"aspect": {
"json": {
"timestampMillis": 1731410997570,
"partitionSpec": {
"partition": "FULL_TABLE_SNAPSHOT",
"type": "FULL_TABLE"
},
"rowCount": 0,
"columnCount": 3,
"fieldProfiles": []
}
},
"systemMetadata": {
"lastObserved": 1731410997579,
"runId": "cassandra-test",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:cassandra,example_keyspace.shopping_cart,PROD)",
"changeType": "UPSERT",
"aspectName": "datasetProfile",
"aspect": {
"json": {
"timestampMillis": 1731410997550,
"partitionSpec": {
"partition": "FULL_TABLE_SNAPSHOT",
"type": "FULL_TABLE"
},
"rowCount": 8,
"columnCount": 3,
"fieldProfiles": [
{
"fieldPath": "item_count",
"uniqueCount": 5,
"min": "2",
"max": "100",
"mean": "46.4",
"stdev": "50.0"
},
{
"fieldPath": "last_update_timestamp",
"uniqueCount": 8,
"min": "2024-11-12 11:29:52.398000",
"max": "2024-11-12 11:29:52.411000"
},
{
"fieldPath": "userid",
"uniqueCount": 8,
"min": "1234",
"max": "9876"
}
]
}
},
"systemMetadata": {
"lastObserved": 1731410997558,
"runId": "cassandra-test",
"lastRunId": "no-run-id-provided"
}
}
]
Loading

0 comments on commit 7929bd2

Please sign in to comment.