Skip to content

Commit 4e328c3

Browse files
mayurinehatemayurinehate
authored
feat(ingest/looker): update browse paths to align with looker UI (#10147)
1 parent 140c0f1 commit 4e328c3

26 files changed

+2463
-1073
lines changed

‎docs/how/updating-datahub.md‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,8 @@ This file documents any backwards-incompatible changes in DataHub and assists pe
2525
- #10026 - The dbt `use_compiled_code` option has been removed, because we now support capturing both source and compiled dbt SQL. This can be configured using `include_compiled_code`, which will be default enabled in 0.13.1.
2626
- #10055 - Assertion entities generated by dbt are now associated with the dbt dataset entity, and not the entity in the data warehouse.
2727
- #10090 - For Redshift ingestion, `use_lineage_v2` is now enabled by default.
28-
28+
- #10147 - For looker ingestion, the browse paths for looker Dashboard, Chart, View, Explore have been updated to align with Looker UI. This does not affect URNs or lineage but primarily affects (improves) browsing experience.
29+
-
2930
### Potential Downtime
3031

3132
### Deprecations

‎metadata-ingestion/src/datahub/ingestion/source/looker/looker_common.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1199,6 +1199,7 @@ class LookerDashboardElement:
11991199
type: Optional[str] = None
12001200
description: Optional[str] = None
12011201
input_fields: Optional[List[InputFieldElement]] = None
1202+
folder_path: Optional[str] = None # for independent looks.
12021203

12031204
def url(self, base_url: str) -> str:
12041205
# A dashboard element can use a look or just a raw query against an explore

‎metadata-ingestion/src/datahub/ingestion/source/looker/looker_config.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -108,14 +108,14 @@ class LookerCommonConfig(EnvConfigMixin, PlatformInstanceConfigMixin):
108108
)
109109
explore_browse_pattern: LookerNamingPattern = pydantic.Field(
110110
description=f"Pattern for providing browse paths to explores. {LookerNamingPattern.allowed_docstring()}",
111-
default=LookerNamingPattern(pattern="/{env}/{platform}/{project}/explores"),
111+
default=LookerNamingPattern(pattern="/Explore/{project}/{model}"),
112112
)
113113
view_naming_pattern: LookerViewNamingPattern = Field(
114114
LookerViewNamingPattern(pattern="{project}.view.{name}"),
115115
description=f"Pattern for providing dataset names to views. {LookerViewNamingPattern.allowed_docstring()}",
116116
)
117117
view_browse_pattern: LookerViewNamingPattern = Field(
118-
LookerViewNamingPattern(pattern="/{env}/{platform}/{project}/views"),
118+
LookerViewNamingPattern(pattern="/Develop/{project}/{file_path}"),
119119
description=f"Pattern for providing browse paths to views. {LookerViewNamingPattern.allowed_docstring()}",
120120
)
121121
tag_measures_and_dimensions: bool = Field(

‎metadata-ingestion/src/datahub/ingestion/source/looker/looker_source.py‎

Lines changed: 39 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -452,6 +452,12 @@ def _get_looker_dashboard_element( # noqa: C901
452452
else:
453453
slug = ""
454454

455+
look_folder_path = None
456+
if element.look.folder is not None:
457+
look_folder_path = self._get_folder_path(
458+
element.look.folder, self.looker_api
459+
)
460+
455461
return LookerDashboardElement(
456462
id=element.id,
457463
title=title,
@@ -464,6 +470,7 @@ def _get_looker_dashboard_element( # noqa: C901
464470
for exp in explores
465471
],
466472
input_fields=input_fields,
473+
folder_path=look_folder_path,
467474
)
468475

469476
# Failing the above two approaches, pick out details from result_maker
@@ -524,10 +531,12 @@ def _get_looker_dashboard_element( # noqa: C901
524531
type=element.type,
525532
description=element.subtitle_text,
526533
look_id=element.look_id,
527-
query_slug=element.result_maker.query.slug
528-
if element.result_maker.query is not None
529-
and element.result_maker.query.slug is not None
530-
else "",
534+
query_slug=(
535+
element.result_maker.query.slug
536+
if element.result_maker.query is not None
537+
and element.result_maker.query.slug is not None
538+
else ""
539+
),
531540
upstream_explores=[
532541
LookerExplore(model_name=model, name=exp) for exp in explores
533542
],
@@ -603,18 +612,29 @@ def _make_chart_metadata_events(
603612
chartUrl=dashboard_element.url(self.source_config.external_base_url or ""),
604613
inputs=dashboard_element.get_view_urns(self.source_config),
605614
customProperties={
606-
"upstream_fields": ",".join(
607-
sorted(set(field.name for field in dashboard_element.input_fields))
615+
"upstream_fields": (
616+
",".join(
617+
sorted(
618+
set(field.name for field in dashboard_element.input_fields)
619+
)
620+
)
621+
if dashboard_element.input_fields
622+
else ""
608623
)
609-
if dashboard_element.input_fields
610-
else ""
611624
},
612625
)
613626
chart_snapshot.aspects.append(chart_info)
614627

615628
if dashboard and dashboard.folder_path is not None:
616629
browse_path = BrowsePathsClass(
617-
paths=[f"/looker/{dashboard.folder_path}/{dashboard.title}"]
630+
paths=[f"/Folders/{dashboard.folder_path}/{dashboard.title}"]
631+
)
632+
chart_snapshot.aspects.append(browse_path)
633+
elif (
634+
dashboard is None and dashboard_element.folder_path is not None
635+
): # independent look
636+
browse_path = BrowsePathsClass(
637+
paths=[f"/Folders/{dashboard_element.folder_path}"]
618638
)
619639
chart_snapshot.aspects.append(browse_path)
620640

@@ -673,7 +693,7 @@ def _make_dashboard_metadata_events(
673693
dashboard_snapshot.aspects.append(dashboard_info)
674694
if looker_dashboard.folder_path is not None:
675695
browse_path = BrowsePathsClass(
676-
paths=[f"/looker/{looker_dashboard.folder_path}"]
696+
paths=[f"/Folders/{looker_dashboard.folder_path}"]
677697
)
678698
dashboard_snapshot.aspects.append(browse_path)
679699

@@ -1084,10 +1104,12 @@ def process_dashboard(
10841104
looker_dashboard = self._get_looker_dashboard(dashboard_object, self.looker_api)
10851105
mces = self._make_dashboard_and_chart_mces(looker_dashboard)
10861106
workunits = [
1087-
MetadataWorkUnit(id=f"looker-{mce.proposedSnapshot.urn}", mce=mce)
1088-
if isinstance(mce, MetadataChangeEvent)
1089-
else MetadataWorkUnit(
1090-
id=f"looker-{mce.aspectName}-{mce.entityUrn}", mcp=mce
1107+
(
1108+
MetadataWorkUnit(id=f"looker-{mce.proposedSnapshot.urn}", mce=mce)
1109+
if isinstance(mce, MetadataChangeEvent)
1110+
else MetadataWorkUnit(
1111+
id=f"looker-{mce.aspectName}-{mce.entityUrn}", mcp=mce
1112+
)
10911113
)
10921114
for mce in mces
10931115
]
@@ -1177,12 +1199,7 @@ def extract_independent_looks(self) -> Iterable[MetadataWorkUnit]:
11771199
self.reporter.report_stage_start("extract_independent_looks")
11781200

11791201
logger.debug("Extracting looks not part of Dashboard")
1180-
look_fields: List[str] = [
1181-
"id",
1182-
"title",
1183-
"description",
1184-
"query_id",
1185-
]
1202+
look_fields: List[str] = ["id", "title", "description", "query_id", "folder"]
11861203
query_fields: List[str] = [
11871204
"id",
11881205
"view",
@@ -1227,8 +1244,8 @@ def extract_independent_looks(self) -> Iterable[MetadataWorkUnit]:
12271244
subtitle_text=look.description,
12281245
look_id=look.id,
12291246
dashboard_id=None, # As this is independent look
1230-
look=LookWithQuery(query=query),
1231-
)
1247+
look=LookWithQuery(query=query, folder=look.folder),
1248+
),
12321249
)
12331250

12341251
if dashboard_element is not None:

0 commit comments

Comments
 (0)