Skip to content

Commit

Permalink
feat(ingest/snowflake): Add cluster formula to dataset properties (#1…
Browse files Browse the repository at this point in the history 
…1254)
  • Loading branch information
@alisa-aylward-toast
alisa-aylward-toast authored Aug 29, 2024
1 parent f9cd29f commit 675b225
Show file tree
Hide file tree
Showing 4 changed files with 27 additions and 22 deletions.
7 changes: 6 additions & 1 deletion metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_schema_gen.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -746,6 +746,11 @@ def get_dataset_properties(
schema_name: str,
db_name: str,
) -> DatasetProperties:
custom_properties = {}

if isinstance(table, SnowflakeTable) and table.clustering_key:
custom_properties["CLUSTERING_KEY"] = table.clustering_key

return DatasetProperties(
name=table.name,
created=(
Expand All @@ -760,7 +765,7 @@ def get_dataset_properties(
),
description=table.comment,
qualifiedName=f"{db_name}.{schema_name}.{table.name}",
customProperties={},
customProperties=custom_properties,
externalUrl=(
self.snowsight_url_builder.get_external_url_for_table(
table.name,
Expand Down
2 changes: 1 addition & 1 deletion metadata-ingestion/tests/integration/snowflake/common.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -235,7 +235,7 @@ def default_query_results( # noqa: C901
"BYTES": 1024,
"ROW_COUNT": 10000,
"COMMENT": "Comment for Table",
"CLUSTERING_KEY": None,
"CLUSTERING_KEY": "LINEAR(COL_1)",
}
for tbl_idx in range(1, num_tables + 1)
]
Expand Down
20 changes: 10 additions & 10 deletions metadata-ingestion/tests/integration/snowflake/snowflake_golden.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -488,7 +488,7 @@
"aspectName": "datasetProperties",
"aspect": {
"json": {
"customProperties": {},
"customProperties": {"CLUSTERING_KEY": "LINEAR(COL_1)"},
"externalUrl": "https://app.snowflake.com/ap-south-1.aws/abc12345/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_1/",
"name": "TABLE_1",
"qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_1",
Expand Down Expand Up @@ -787,7 +787,7 @@
"aspectName": "datasetProperties",
"aspect": {
"json": {
"customProperties": {},
"customProperties": {"CLUSTERING_KEY": "LINEAR(COL_1)"},
"externalUrl": "https://app.snowflake.com/ap-south-1.aws/abc12345/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_2/",
"name": "TABLE_2",
"qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_2",
Expand Down Expand Up @@ -1086,7 +1086,7 @@
"aspectName": "datasetProperties",
"aspect": {
"json": {
"customProperties": {},
"customProperties": {"CLUSTERING_KEY": "LINEAR(COL_1)"},
"externalUrl": "https://app.snowflake.com/ap-south-1.aws/abc12345/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_3/",
"name": "TABLE_3",
"qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_3",
Expand Down Expand Up @@ -1385,7 +1385,7 @@
"aspectName": "datasetProperties",
"aspect": {
"json": {
"customProperties": {},
"customProperties": {"CLUSTERING_KEY": "LINEAR(COL_1)"},
"externalUrl": "https://app.snowflake.com/ap-south-1.aws/abc12345/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_4/",
"name": "TABLE_4",
"qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_4",
Expand Down Expand Up @@ -1684,7 +1684,7 @@
"aspectName": "datasetProperties",
"aspect": {
"json": {
"customProperties": {},
"customProperties": {"CLUSTERING_KEY": "LINEAR(COL_1)"},
"externalUrl": "https://app.snowflake.com/ap-south-1.aws/abc12345/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_5/",
"name": "TABLE_5",
"qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_5",
Expand Down Expand Up @@ -1983,7 +1983,7 @@
"aspectName": "datasetProperties",
"aspect": {
"json": {
"customProperties": {},
"customProperties": {"CLUSTERING_KEY": "LINEAR(COL_1)"},
"externalUrl": "https://app.snowflake.com/ap-south-1.aws/abc12345/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_6/",
"name": "TABLE_6",
"qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_6",
Expand Down Expand Up @@ -2282,7 +2282,7 @@
"aspectName": "datasetProperties",
"aspect": {
"json": {
"customProperties": {},
"customProperties": {"CLUSTERING_KEY": "LINEAR(COL_1)"},
"externalUrl": "https://app.snowflake.com/ap-south-1.aws/abc12345/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_7/",
"name": "TABLE_7",
"qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_7",
Expand Down Expand Up @@ -2581,7 +2581,7 @@
"aspectName": "datasetProperties",
"aspect": {
"json": {
"customProperties": {},
"customProperties": {"CLUSTERING_KEY": "LINEAR(COL_1)"},
"externalUrl": "https://app.snowflake.com/ap-south-1.aws/abc12345/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_8/",
"name": "TABLE_8",
"qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_8",
Expand Down Expand Up @@ -2880,7 +2880,7 @@
"aspectName": "datasetProperties",
"aspect": {
"json": {
"customProperties": {},
"customProperties": {"CLUSTERING_KEY": "LINEAR(COL_1)"},
"externalUrl": "https://app.snowflake.com/ap-south-1.aws/abc12345/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_9/",
"name": "TABLE_9",
"qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_9",
Expand Down Expand Up @@ -3179,7 +3179,7 @@
"aspectName": "datasetProperties",
"aspect": {
"json": {
"customProperties": {},
"customProperties": {"CLUSTERING_KEY": "LINEAR(COL_1)"},
"externalUrl": "https://app.snowflake.com/ap-south-1.aws/abc12345/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_10/",
"name": "TABLE_10",
"qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_10",
Expand Down
20 changes: 10 additions & 10 deletions metadata-ingestion/tests/integration/snowflake/snowflake_privatelink_golden.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -141,7 +141,7 @@
"aspectName": "datasetProperties",
"aspect": {
"json": {
"customProperties": {},
"customProperties": {"CLUSTERING_KEY": "LINEAR(COL_1)"},
"externalUrl": "https://app.abc12345.ap-south-1.privatelink.snowflakecomputing.com/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_3/",
"name": "TABLE_3",
"qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_3",
Expand Down Expand Up @@ -810,7 +810,7 @@
"aspectName": "datasetProperties",
"aspect": {
"json": {
"customProperties": {},
"customProperties": {"CLUSTERING_KEY": "LINEAR(COL_1)"},
"externalUrl": "https://app.abc12345.ap-south-1.privatelink.snowflakecomputing.com/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_1/",
"name": "TABLE_1",
"qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_1",
Expand Down Expand Up @@ -1475,7 +1475,7 @@
"aspectName": "datasetProperties",
"aspect": {
"json": {
"customProperties": {},
"customProperties": {"CLUSTERING_KEY": "LINEAR(COL_1)"},
"externalUrl": "https://app.abc12345.ap-south-1.privatelink.snowflakecomputing.com/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_10/",
"name": "TABLE_10",
"qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_10",
Expand Down Expand Up @@ -1714,7 +1714,7 @@
"aspectName": "datasetProperties",
"aspect": {
"json": {
"customProperties": {},
"customProperties": {"CLUSTERING_KEY": "LINEAR(COL_1)"},
"externalUrl": "https://app.abc12345.ap-south-1.privatelink.snowflakecomputing.com/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_5/",
"name": "TABLE_5",
"qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_5",
Expand All @@ -1741,7 +1741,7 @@
"aspectName": "datasetProperties",
"aspect": {
"json": {
"customProperties": {},
"customProperties": {"CLUSTERING_KEY": "LINEAR(COL_1)"},
"externalUrl": "https://app.abc12345.ap-south-1.privatelink.snowflakecomputing.com/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_2/",
"name": "TABLE_2",
"qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_2",
Expand Down Expand Up @@ -2303,7 +2303,7 @@
"aspectName": "datasetProperties",
"aspect": {
"json": {
"customProperties": {},
"customProperties": {"CLUSTERING_KEY": "LINEAR(COL_1)"},
"externalUrl": "https://app.abc12345.ap-south-1.privatelink.snowflakecomputing.com/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_6/",
"name": "TABLE_6",
"qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_6",
Expand Down Expand Up @@ -2623,7 +2623,7 @@
"aspectName": "datasetProperties",
"aspect": {
"json": {
"customProperties": {},
"customProperties": {"CLUSTERING_KEY": "LINEAR(COL_1)"},
"externalUrl": "https://app.abc12345.ap-south-1.privatelink.snowflakecomputing.com/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_7/",
"name": "TABLE_7",
"qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_7",
Expand Down Expand Up @@ -2666,7 +2666,7 @@
"aspectName": "datasetProperties",
"aspect": {
"json": {
"customProperties": {},
"customProperties": {"CLUSTERING_KEY": "LINEAR(COL_1)"},
"externalUrl": "https://app.abc12345.ap-south-1.privatelink.snowflakecomputing.com/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_4/",
"name": "TABLE_4",
"qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_4",
Expand Down Expand Up @@ -3164,7 +3164,7 @@
"aspectName": "datasetProperties",
"aspect": {
"json": {
"customProperties": {},
"customProperties": {"CLUSTERING_KEY": "LINEAR(COL_1)"},
"externalUrl": "https://app.abc12345.ap-south-1.privatelink.snowflakecomputing.com/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_8/",
"name": "TABLE_8",
"qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_8",
Expand Down Expand Up @@ -3304,7 +3304,7 @@
"aspectName": "datasetProperties",
"aspect": {
"json": {
"customProperties": {},
"customProperties": {"CLUSTERING_KEY": "LINEAR(COL_1)"},
"externalUrl": "https://app.abc12345.ap-south-1.privatelink.snowflakecomputing.com/#/data/databases/TEST_DB/schemas/TEST_SCHEMA/table/TABLE_9/",
"name": "TABLE_9",
"qualifiedName": "TEST_DB.TEST_SCHEMA.TABLE_9",
Expand Down

0 comments on commit 675b225

Please sign in to comment.