Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix(sql-common): Sql common bug fixes #5526

Merged
merged 5 commits into from
Aug 7, 2022
+21 −35
Merged

fix(sql-common): Sql common bug fixes #5526

Changes from 3 commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
35fa127
fix(sql-common): fix bugs ralted to table comment
MugdhaHardikar-GSLab Jul 29, 2022
c42de2b
fix parameter sequence
MugdhaHardikar-GSLab Jul 29, 2022
29918f0
tweaks from code review
hsheth2 Aug 3, 2022
a5089b8
Merge branch 'master' into sql_common_bug_fix
hsheth2 Aug 5, 2022
ea36720
fix type annotations
shirshanka Aug 7, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
46 changes: 16 additions & 30 deletions metadata-ingestion/src/datahub/ingestion/source/sql/sql_common.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1018,28 +1018,34 @@ def _process_table(
def get_table_properties(
self, inspector: Inspector, schema: str, table: str
) -> Tuple[Optional[str], Optional[Dict[str, str]], Optional[str]]:
description: Optional[str] = None
properties: Dict[str, str] = {}

# The location cannot be fetched generically, but subclasses may override
# this method and provide a location.
location: Optional[str] = None

try:
location: Optional[str] = None
# SQLALchemy stubs are incomplete and missing this method.
# PR: https://github.com/dropbox/sqlalchemy-stubs/pull/223.
table_info: dict = inspector.get_table_comment(table, schema) # type: ignore
except NotImplementedError:
description: Optional[str] = None
properties: Dict[str, str] = {}
return description, properties, location
except ProgrammingError as pe:
# Snowflake needs schema names quoted when fetching table comments.
logger.debug(
f"Encountered ProgrammingError. Retrying with quoted schema name for schema {schema} and table {table}",
pe,
)
description = None
properties = {}
table_info: dict = inspector.get_table_comment(table, f'"{schema}"') # type: ignore
else:
description = table_info["text"]

# The "properties" field is a non-standard addition to SQLAlchemy's interface.
properties = table_info.get("properties", {})
description = table_info.get("text")
if type(description) is tuple:
# Handling for value type tuple which is coming for dialect 'db2+ibm_db'
description = table_info["text"][0]

# The "properties" field is a non-standard addition to SQLAlchemy's interface.
properties = table_info.get("properties", {})
return description, properties, location

def get_dataplatform_instance_aspect(
Expand Down Expand Up @@ -1208,27 +1214,7 @@ def _process_view(
columns,
canonical_schema=schema_fields,
)
try:
# SQLALchemy stubs are incomplete and missing this method.
# PR: https://github.com/dropbox/sqlalchemy-stubs/pull/223.
view_info: dict = inspector.get_table_comment(view, schema) # type: ignore
except NotImplementedError:
description: Optional[str] = None
properties: Dict[str, str] = {}
except ProgrammingError as pe:
# Snowflake needs schema names quoted when fetching table comments.
logger.debug(
f"Encountered ProgrammingError. Retrying with quoted schema name for schema {schema} and view {view}",
pe,
)
description = None
properties = {}
view_info: dict = inspector.get_table_comment(view, f'"{schema}"') # type: ignore
else:
description = view_info["text"]

# The "properties" field is a non-standard addition to SQLAlchemy's interface.
properties = view_info.get("properties", {})
description, properties, _ = self.get_table_properties(inspector, schema, view)
try:
view_definition = inspector.get_view_definition(view, schema)
if view_definition is None:
Expand Down