Skip to content

Commit

Permalink
feat(ingest): emit platform for query entities (#10103)
Browse files Browse the repository at this point in the history
  • Loading branch information
@hsheth2
hsheth2 authored Mar 26, 2024
1 parent e4ebf34 commit a70e775
Show file tree
Hide file tree
Showing 13 changed files with 393 additions and 0 deletions.
1 change: 1 addition & 0 deletions docs/how/updating-datahub.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ This file documents any backwards-incompatible changes in DataHub and assists pe
- #10002 - The `DataHubGraph` client no longer makes a request to the backend during initialization. If you want to preserve the old behavior, call `graph.test_connection()` after constructing the client.
- #10026 - The dbt `use_compiled_code` option has been removed, because we now support capturing both source and compiled dbt SQL. This can be configured using `include_compiled_code`, which will be default enabled in 0.13.1.
- #10055 - Assertion entities generated by dbt are now associated with the dbt dataset entity, and not the entity in the data warehouse.
- #10090 - For Redshift ingestion, `use_lineage_v2` is now enabled by default.

### Potential Downtime

Expand Down
3 changes: 3 additions & 0 deletions metadata-ingestion/src/datahub/sql_parsing/sql_parsing_aggregator.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -995,6 +995,9 @@ def _gen_lineage_for_downstream(
)
]
),
models.DataPlatformInstanceClass(
platform=self.platform.urn(),
),
],
)

Expand Down
192 changes: 192 additions & 0 deletions metadata-ingestion/tests/integration/snowflake/snowflake_golden.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4489,6 +4489,22 @@
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "query",
"entityUrn": "urn:li:query:aec91b63e3ce03877b9d80dc77b915880fc2a5f8f3b5cdf66e9341830268776b",
"changeType": "UPSERT",
"aspectName": "dataPlatformInstance",
"aspect": {
"json": {
"platform": "urn:li:dataPlatform:snowflake"
}
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "snowflake-2024_03_22-10_26_30",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "query",
"entityUrn": "urn:li:query:aec91b63e3ce03877b9d80dc77b915880fc2a5f8f3b5cdf66e9341830268776b",
Expand Down Expand Up @@ -4849,6 +4865,22 @@
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "query",
"entityUrn": "urn:li:query:a79e59b5f5248b9247f88eaecbe3a296f788634282edc9cdca80ba1bfb504f37",
"changeType": "UPSERT",
"aspectName": "dataPlatformInstance",
"aspect": {
"json": {
"platform": "urn:li:dataPlatform:snowflake"
}
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "snowflake-2024_03_22-10_26_30",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "query",
"entityUrn": "urn:li:query:a79e59b5f5248b9247f88eaecbe3a296f788634282edc9cdca80ba1bfb504f37",
Expand Down Expand Up @@ -5080,6 +5112,22 @@
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "query",
"entityUrn": "urn:li:query:84b52a9ad9d198587fbc4e210812011da567bd505381aa7ff437f243366873e9",
"changeType": "UPSERT",
"aspectName": "dataPlatformInstance",
"aspect": {
"json": {
"platform": "urn:li:dataPlatform:snowflake"
}
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "snowflake-2024_03_22-10_26_30",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "query",
"entityUrn": "urn:li:query:a79e59b5f5248b9247f88eaecbe3a296f788634282edc9cdca80ba1bfb504f37",
Expand Down Expand Up @@ -5435,6 +5483,22 @@
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "query",
"entityUrn": "urn:li:query:d37a930ca6d2dd7100fd81bbf3d96a8cfe9f30e3469de363650d7e3146c3e4e8",
"changeType": "UPSERT",
"aspectName": "dataPlatformInstance",
"aspect": {
"json": {
"platform": "urn:li:dataPlatform:snowflake"
}
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "snowflake-2024_03_22-10_26_30",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD)",
Expand Down Expand Up @@ -5813,6 +5877,38 @@
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "query",
"entityUrn": "urn:li:query:e093b2691b3cb33b6451367d1baa472eabe3b9a38fbd30152b12f22c35acc632",
"changeType": "UPSERT",
"aspectName": "dataPlatformInstance",
"aspect": {
"json": {
"platform": "urn:li:dataPlatform:snowflake"
}
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "snowflake-2024_03_22-10_26_30",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "query",
"entityUrn": "urn:li:query:973d404f0f5a9e2d5df165087f03a01fa182a04ffbcab14228c831070ade02c2",
"changeType": "UPSERT",
"aspectName": "dataPlatformInstance",
"aspect": {
"json": {
"platform": "urn:li:dataPlatform:snowflake"
}
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "snowflake-2024_03_22-10_26_30",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD)",
Expand Down Expand Up @@ -5993,6 +6089,22 @@
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "query",
"entityUrn": "urn:li:query:bfe6b2ec9f81445aae23bc757f90defb69480689f55d7e63c11f0da6a60fff12",
"changeType": "UPSERT",
"aspectName": "dataPlatformInstance",
"aspect": {
"json": {
"platform": "urn:li:dataPlatform:snowflake"
}
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "snowflake-2024_03_22-10_26_30",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD)",
Expand Down Expand Up @@ -6378,6 +6490,22 @@
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "query",
"entityUrn": "urn:li:query:ac812b1b491a3d141f66050526c1ee2483b19083ad840ceced0ca39ace552e01",
"changeType": "UPSERT",
"aspectName": "dataPlatformInstance",
"aspect": {
"json": {
"platform": "urn:li:dataPlatform:snowflake"
}
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "snowflake-2024_03_22-10_26_30",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)",
Expand Down Expand Up @@ -6493,6 +6621,22 @@
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "query",
"entityUrn": "urn:li:query:1d69efa7115792a07468c8b3846a4c0b3ba8620c1f2263374c2efbb2e56e6200",
"changeType": "UPSERT",
"aspectName": "dataPlatformInstance",
"aspect": {
"json": {
"platform": "urn:li:dataPlatform:snowflake"
}
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "snowflake-2024_03_22-10_26_30",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD)",
Expand Down Expand Up @@ -6545,6 +6689,22 @@
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "query",
"entityUrn": "urn:li:query:62c3a7585398b147daace630104bf0827b366353152667718a84d22542f5aee4",
"changeType": "UPSERT",
"aspectName": "dataPlatformInstance",
"aspect": {
"json": {
"platform": "urn:li:dataPlatform:snowflake"
}
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "snowflake-2024_03_22-10_26_30",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD)",
Expand Down Expand Up @@ -6705,6 +6865,22 @@
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "query",
"entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.view_2%2CPROD%29",
"changeType": "UPSERT",
"aspectName": "dataPlatformInstance",
"aspect": {
"json": {
"platform": "urn:li:dataPlatform:snowflake"
}
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "snowflake-2024_03_22-10_26_30",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "query",
"entityUrn": "urn:li:query:62c3a7585398b147daace630104bf0827b366353152667718a84d22542f5aee4",
Expand Down Expand Up @@ -6762,6 +6938,22 @@
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "query",
"entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.view_1%2CPROD%29",
"changeType": "UPSERT",
"aspectName": "dataPlatformInstance",
"aspect": {
"json": {
"platform": "urn:li:dataPlatform:snowflake"
}
},
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "snowflake-2024_03_22-10_26_30",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "query",
"entityUrn": "urn:li:query:62c3a7585398b147daace630104bf0827b366353152667718a84d22542f5aee4",
Expand Down
32 changes: 32 additions & 0 deletions metadata-ingestion/tests/integration/snowflake/snowflake_privatelink_golden.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3907,6 +3907,22 @@
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "query",
"entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Cinstance1.test_db.test_schema.view_1%2CPROD%29",
"changeType": "UPSERT",
"aspectName": "dataPlatformInstance",
"aspect": {
"json": {
"platform": "urn:li:dataPlatform:snowflake"
}
},
"systemMetadata": {
"lastObserved": 1654621200000,
"runId": "snowflake-2022_06_07-17_00_00",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.view_2,PROD)",
Expand Down Expand Up @@ -4110,6 +4126,22 @@
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "query",
"entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Cinstance1.test_db.test_schema.view_2%2CPROD%29",
"changeType": "UPSERT",
"aspectName": "dataPlatformInstance",
"aspect": {
"json": {
"platform": "urn:li:dataPlatform:snowflake"
}
},
"systemMetadata": {
"lastObserved": 1654621200000,
"runId": "snowflake-2022_06_07-17_00_00",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "query",
"entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Cinstance1.test_db.test_schema.view_1%2CPROD%29",
Expand Down
11 changes: 11 additions & 0 deletions ...ata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_add_known_query_lineage.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -103,6 +103,17 @@
}
}
},
{
"entityType": "query",
"entityUrn": "urn:li:query:02e2ec36678bea2a8c4c855fed5255d087cfeb2710d326e95fd9b48a9c4fc0ae",
"changeType": "UPSERT",
"aspectName": "dataPlatformInstance",
"aspect": {
"json": {
"platform": "urn:li:dataPlatform:redshift"
}
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo,PROD)",
Expand Down
11 changes: 11 additions & 0 deletions metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_basic_lineage.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -90,5 +90,16 @@
]
}
}
},
{
"entityType": "query",
"entityUrn": "urn:li:query:cbdb3e148ea7fdae81815da4dd64f57873fb9c3d7d4bfad4e83b3d1ebd3c45c2",
"changeType": "UPSERT",
"aspectName": "dataPlatformInstance",
"aspect": {
"json": {
"platform": "urn:li:dataPlatform:redshift"
}
}
}
]
Loading

0 comments on commit a70e775

Please sign in to comment.