feat: utilize all PERF rules

hkad98 · hkad98 · commit 8311c560538f · 2024-08-19T08:40:39.000+02:00
JIRA: TRIVIAL
risk: low
diff --git a/gooddata-dbt/gooddata_dbt/dbt/metrics.py b/gooddata-dbt/gooddata_dbt/dbt/metrics.py
@@ -61,9 +61,7 @@ def __init__(self, model_ids: Optional[list[str]], ldm: CatalogDeclarativeModel)
 
     @property
     def metrics(self) -> list[DbtModelMetric]:
-        result = []
-        for metric_def in self.dbt_catalog["metrics"].values():
-            result.append(DbtModelMetric.from_dict(metric_def))
+        result = [DbtModelMetric.from_dict(metric_def) for metric_def in self.dbt_catalog["metrics"].values()]
         # Return only gooddata labelled tables marked by model_id (if requested in args)
         return [
             r
diff --git a/gooddata-dbt/gooddata_dbt/dbt/tables.py b/gooddata-dbt/gooddata_dbt/dbt/tables.py
@@ -3,7 +3,7 @@
 import json
 import re
 from pathlib import Path
-from typing import Any, Optional, Union
+from typing import Optional, Union
 
 import attrs
 from gooddata_sdk import CatalogDeclarativeColumn, CatalogDeclarativeTable, CatalogDeclarativeTables
@@ -306,10 +306,11 @@ def is_primary_key(column: DbtModelColumn) -> bool:
         return result
 
     def make_grain(self, table: DbtModelTable) -> list[dict]:
-        grain = []
-        for column in table.columns.values():
-            if self.is_primary_key(column):
-                grain.append({"id": column.ldm_id, "type": "attribute"})
+        grain = [
+            {"id": column.ldm_id, "type": "attribute"}
+            for column in table.columns.values()
+            if self.is_primary_key(column)
+        ]
         return grain
 
     # TODO - constraints are stored in special nodes
@@ -326,21 +327,22 @@ def make_grain(self, table: DbtModelTable) -> list[dict]:
 
     @staticmethod
     def find_role_playing_tables(tables: list[DbtModelTable]) -> dict:
-        result = {}
-        for table in tables:
-            references: dict[str, Any] = {}
-            for column in table.columns.values():
-                if column.is_reference():
-                    referenced_table = column.meta.gooddata.referenced_table
-                    if referenced_table is not None:
-                        if referenced_table in references:
-                            references[referenced_table].append(column.name)
-                        else:
-                            references[referenced_table] = [column.name]
-            for referenced_object_id, columns in references.items():
-                if len(columns) > 1:
-                    result[referenced_object_id] = columns
-        return result
+        return {
+            referenced_object_id: columns
+            for table in tables
+            for column in table.columns.values()
+            if column.is_reference()
+            for referenced_table in [column.meta.gooddata.referenced_table]
+            if referenced_table is not None
+            for referenced_object_id, columns in {
+                referenced_table: [
+                    col.name
+                    for col in table.columns.values()
+                    if col.is_reference() and col.meta.gooddata.referenced_table == referenced_table
+                ]
+            }.items()
+            if len(columns) > 1
+        }
 
     def make_references(self, table: DbtModelTable, role_playing_tables: dict) -> list[dict]:
         references = []
@@ -369,20 +371,19 @@ def make_references(self, table: DbtModelTable, role_playing_tables: dict) -> li
 
     @staticmethod
     def make_facts(table: DbtModelTable) -> list[dict]:
-        facts = []
-        for column in table.columns.values():
-            if column.gooddata_is_fact():
-                facts.append(
-                    {
-                        "id": column.ldm_id,
-                        # TODO - all titles filled from dbt descriptions, incorrect! No title in dbt models.
-                        "title": column.gooddata_ldm_title,
-                        "description": column.gooddata_ldm_description,
-                        "source_column": column.name,
-                        "source_column_data_type": column.data_type,
-                        "tags": [table.gooddata_ldm_title] + column.tags,
-                    }
-                )
+        facts = [
+            {
+                "id": column.ldm_id,
+                # TODO - all titles filled from dbt descriptions, incorrect! No title in dbt models.
+                "title": column.gooddata_ldm_title,
+                "description": column.gooddata_ldm_description,
+                "source_column": column.name,
+                "source_column_data_type": column.data_type,
+                "tags": [table.gooddata_ldm_title] + column.tags,
+            }
+            for column in table.columns.values()
+            if column.gooddata_is_fact()
+        ]
         return facts
 
     @staticmethod
diff --git a/gooddata-dbt/gooddata_dbt/dbt_plugin.py b/gooddata-dbt/gooddata_dbt/dbt_plugin.py
@@ -216,27 +216,26 @@ def create_localized_workspaces(
 ) -> None:
     if data_product.localization is None:
         return
+    try:
+        from deep_translator import GoogleTranslator
+    except ImportError:
+        logger.warning(
+            "create_localized_workspaces: deep_translator module not found, will not be able to localize workspaces"
+        )
     for to in data_product.localization.to:
-        try:
-            from deep_translator import GoogleTranslator
-
-            translator_func = GoogleTranslator(
-                source=data_product.localization.from_language, target=to.language
-            ).translate_batch
-            logger.info(f"create_localized_workspaces layout_root_path={GOODDATA_LAYOUTS_DIR / data_product.id}")
-            sdk_facade.generate_localized_workspaces(
-                workspace_id,
-                to=to,
-                data_product=data_product,
-                translator_func=translator_func,
-                layout_path=GOODDATA_LAYOUTS_DIR / data_product.id,
-                provision_workspace=True,
-                store_layouts=False,
-            )
-        except ImportError:
-            logger.warning(
-                "create_localized_workspaces: deep_translator module not found, will not be able to localize workspaces"
-            )
+        translator_func = GoogleTranslator(
+            source=data_product.localization.from_language, target=to.language
+        ).translate_batch
+        logger.info(f"create_localized_workspaces layout_root_path={GOODDATA_LAYOUTS_DIR / data_product.id}")
+        sdk_facade.generate_localized_workspaces(
+            workspace_id,
+            to=to,
+            data_product=data_product,
+            translator_func=translator_func,
+            layout_path=GOODDATA_LAYOUTS_DIR / data_product.id,
+            provision_workspace=True,
+            store_layouts=False,
+        )
 
 
 def get_table(data: list[list], headers: list[str], fmt: str) -> str:
@@ -296,9 +295,9 @@ def dbt_cloud_stats(
     dbt_conn = DbtConnection(credentials=DbtCredentials(account_id=args.account_id, token=args.token))
     dbt_tables = DbtModelTables.from_local(args.gooddata_upper_case, all_model_ids)
     model_executions = dbt_conn.get_last_execution(environment_id, len(dbt_tables.tables))
-    exec_list = []
-    for execution in model_executions:
-        exec_list.append([execution.unique_id, round(execution.execution_info.execution_time, 2)])
+    exec_list = [
+        [execution.unique_id, round(execution.execution_info.execution_time, 2)] for execution in model_executions
+    ]
 
     headers = ["Model ID", "Duration(s)"]
     pretty_table = get_table(exec_list, headers, "outline")
diff --git a/gooddata-flight-server/gooddata_flight_server/tasks/temporal_container.py b/gooddata-flight-server/gooddata_flight_server/tasks/temporal_container.py
@@ -127,13 +127,18 @@ def _evict_expired_items(self, now: float) -> list[T]:
         #
         # this is intentionally done outside the critical section
         for value in to_evict:
-            try:
-                self._entry_evict_fun(value)
-            except Exception:
-                pass
+            if not self._safe_evict(value):
+                self._logger.error("temporal_entries_evict_failed", value=value)
 
         return to_evict
 
+    def _safe_evict(self, value: T) -> bool:
+        try:
+            self._entry_evict_fun(value)
+            return True
+        except Exception:
+            return False
+
     @property
     def entry_timeout(self) -> float:
         """
diff --git a/gooddata-flight-server/tests/server/conftest.py b/gooddata-flight-server/tests/server/conftest.py
@@ -35,10 +35,7 @@ def _find_free_port():
 
 
 def _clean_env_vars():
-    to_drop = []
-    for key in os.environ:
-        if key.startswith("GOODDATA_FLIGHT"):
-            to_drop.append(key)
+    to_drop = [key for key in os.environ if key.startswith("GOODDATA_FLIGHT")]
 
     for key in to_drop:
         os.environ.pop(key, None)
diff --git a/gooddata-sdk/gooddata_sdk/catalog/data_source/declarative_model/data_source.py b/gooddata-sdk/gooddata_sdk/catalog/data_source/declarative_model/data_source.py
@@ -85,9 +85,10 @@ def store_to_disk(self, layout_organization_folder: Path) -> None:
     def load_from_disk(cls, layout_organization_folder: Path) -> CatalogDeclarativeDataSources:
         data_sources_folder = cls.data_sources_folder(layout_organization_folder)
         data_source_ids = sorted([p.stem for p in data_sources_folder.iterdir() if p.is_dir()])
-        data_sources = []
-        for data_source_id in data_source_ids:
-            data_sources.append(CatalogDeclarativeDataSource.load_from_disk(data_sources_folder, data_source_id))
+        data_sources = [
+            CatalogDeclarativeDataSource.load_from_disk(data_sources_folder, data_source_id)
+            for data_source_id in data_source_ids
+        ]
         return cls(data_sources=data_sources)
 
 
diff --git a/gooddata-sdk/gooddata_sdk/catalog/data_source/declarative_model/physical_model/pdm.py b/gooddata-sdk/gooddata_sdk/catalog/data_source/declarative_model/physical_model/pdm.py
@@ -35,9 +35,7 @@ def store_to_disk(self, data_source_folder: Path) -> None:
     def load_from_disk(cls, data_source_folder: Path) -> CatalogDeclarativeTables:
         pdm_folder = get_pdm_folder(data_source_folder)
         table_files = sorted([p for p in pdm_folder.glob("*.yaml")])
-        tables = []
-        for table_file in table_files:
-            tables.append(CatalogDeclarativeTable.load_from_disk(table_file))
+        tables = [CatalogDeclarativeTable.load_from_disk(table_file) for table_file in table_files]
         return cls(tables=tables)
 
 
diff --git a/gooddata-sdk/gooddata_sdk/catalog/permission/service.py b/gooddata-sdk/gooddata_sdk/catalog/permission/service.py
@@ -124,10 +124,10 @@ def get_declarative_organization_permissions(self) -> list[CatalogDeclarativeOrg
                 List of all declarative organization permissions.
         """
 
-        catalog_list = []
         organization_permissions = self._layout_api.get_organization_permissions()
-        for permission in organization_permissions:
-            catalog_list.append(CatalogDeclarativeOrganizationPermission.from_api(permission))
+        catalog_list = [
+            CatalogDeclarativeOrganizationPermission.from_api(permission) for permission in organization_permissions
+        ]
         return catalog_list
 
     def put_declarative_organization_permissions(
@@ -143,9 +143,7 @@ def put_declarative_organization_permissions(
             None
         """
 
-        declarative_organization_permissions = []
-        for catalog_permission in org_permissions:
-            declarative_organization_permissions.append(catalog_permission.to_api())
+        declarative_organization_permissions = [catalog_permission.to_api() for catalog_permission in org_permissions]
         self._layout_api.set_organization_permissions(declarative_organization_permissions)
 
     def manage_organization_permissions(
diff --git a/gooddata-sdk/gooddata_sdk/catalog/user/declarative_model/user.py b/gooddata-sdk/gooddata_sdk/catalog/user/declarative_model/user.py
@@ -31,9 +31,7 @@ def load_from_disk(cls, layout_organization_folder: Path) -> CatalogDeclarativeU
         users_directory = layout_organization_folder / LAYOUT_USERS_DIR
         users_file = users_directory / LAYOUT_USERS_FILE
         data = read_layout_from_file(users_file)
-        users = []
-        for record in data:
-            users.append(CatalogDeclarativeUser.from_dict(record, camel_case=True))
+        users = [CatalogDeclarativeUser.from_dict(record, camel_case=True) for record in data]
         return cls(users=users)
 
     def store_to_disk(self, layout_organization_folder: Path) -> None:
diff --git a/gooddata-sdk/gooddata_sdk/catalog/user/declarative_model/user_group.py b/gooddata-sdk/gooddata_sdk/catalog/user/declarative_model/user_group.py
@@ -30,9 +30,7 @@ def load_from_disk(cls, layout_organization_folder: Path) -> CatalogDeclarativeU
         user_groups_directory = layout_organization_folder / LAYOUT_USER_GROUPS_DIR
         user_groups_file = user_groups_directory / LAYOUT_USER_GROUPS_FILE
         data = read_layout_from_file(user_groups_file)
-        user_groups = []
-        for record in data:
-            user_groups.append(CatalogDeclarativeUserGroup.from_dict(record, camel_case=True))
+        user_groups = [CatalogDeclarativeUserGroup.from_dict(record, camel_case=True) for record in data]
         return cls(user_groups=user_groups)
 
     def store_to_disk(self, layout_organization_folder: Path) -> None:
diff --git a/gooddata-sdk/gooddata_sdk/catalog/workspace/declarative_model/workspace/logical_model/ldm.py b/gooddata-sdk/gooddata_sdk/catalog/workspace/declarative_model/workspace/logical_model/ldm.py
@@ -239,9 +239,10 @@ def change_tables_columns_case(self, upper_case: Optional[bool] = None) -> Catal
                             fact.source_column = self._change_case(fact.source_column, upper_case)
                 for reference in dataset.references:
                     if reference.source_columns is not None:
-                        new_columns = []
-                        for reference_column in reference.source_columns:
-                            new_columns.append(self._change_case(reference_column, upper_case))
+                        new_columns = [
+                            self._change_case(reference_column, upper_case)
+                            for reference_column in reference.source_columns
+                        ]
                         reference.source_columns = new_columns
                     elif reference.sources is not None:
                         for reference_source in reference.sources:
diff --git a/gooddata-sdk/gooddata_sdk/catalog/workspace/declarative_model/workspace/workspace.py b/gooddata-sdk/gooddata_sdk/catalog/workspace/declarative_model/workspace/workspace.py
@@ -151,11 +151,10 @@ def load_from_disk(cls, layout_organization_folder: Path) -> CatalogDeclarativeW
         workspace_data_filters_files = get_sorted_yaml_files(
             CatalogDeclarativeWorkspaces.workspace_data_filters_folder(layout_organization_folder)
         )
-        workspace_data_filters = []
-        for workspace_data_filters_file in workspace_data_filters_files:
-            workspace_data_filters.append(
-                CatalogDeclarativeWorkspaceDataFilter.load_from_disk(workspace_data_filters_file)
-            )
+        workspace_data_filters = [
+            CatalogDeclarativeWorkspaceDataFilter.load_from_disk(workspace_data_filters_file)
+            for workspace_data_filters_file in workspace_data_filters_files
+        ]
         return cls(workspace_data_filters=workspace_data_filters)
 
     def create_copy(self, source_ws_id: str, target_ws_id: str) -> tuple[CatalogDeclarativeWorkspaceDataFilters, dict]:
@@ -240,9 +239,10 @@ def load_from_disk(cls, layout_organization_folder: Path) -> CatalogDeclarativeU
         user_data_filters_files = get_sorted_yaml_files(
             CatalogDeclarativeWorkspaces.user_data_filters_folder(layout_organization_folder)
         )
-        user_data_filters = []
-        for user_data_filters_file in user_data_filters_files:
-            user_data_filters.append(CatalogDeclarativeUserDataFilter.load_from_disk(user_data_filters_file))
+        user_data_filters = [
+            CatalogDeclarativeUserDataFilter.load_from_disk(user_data_filters_file)
+            for user_data_filters_file in user_data_filters_files
+        ]
         return cls(user_data_filters=user_data_filters)
 
 
diff --git a/gooddata-sdk/gooddata_sdk/catalog/workspace/service.py b/gooddata-sdk/gooddata_sdk/catalog/workspace/service.py
@@ -602,8 +602,7 @@ def translate_in_batches(
             # Extract the translated values and add them to the list
             translated_values.extend(api_result)
         # Update the data dictionary with the translated values
-        for key, value in zip(to_translate, translated_values):
-            result[key] = value
+        result = {key: value for key, value in zip(to_translate, translated_values)}
         duration = int((time() - start) * 1000)
         logger.info(f"Translation finished duration={duration}")
         return result
diff --git a/gooddata-sdk/gooddata_sdk/utils.py b/gooddata-sdk/gooddata_sdk/utils.py
@@ -237,10 +237,11 @@ def mandatory_profile_content_check(profile: str, profile_content_keys: KeysView
         ValueError:
             Missing mandatory parameter or parameters.
     """
-    missing = []
-    for mandatory_parameter in SDK_PROFILE_MANDATORY_KEYS:
-        if mandatory_parameter not in profile_content_keys:
-            missing.append(mandatory_parameter)
+    missing = [
+        mandatory_parameter
+        for mandatory_parameter in SDK_PROFILE_MANDATORY_KEYS
+        if mandatory_parameter not in profile_content_keys
+    ]
     if missing:
         missing_str = " and ".join(missing)
         raise ValueError(f"Profile {profile} is missing mandatory parameter or parameters {missing_str}.")
@@ -250,10 +251,11 @@ def _create_profile_legacy(content: dict) -> dict:
     try:
         return structure(content, Profile).to_dict()
     except ClassValidationError as e:
-        errors = []
-        for error in e.exceptions:
-            if isinstance(error, KeyError):
-                errors.append(f"Profile file does not contain mandatory parameter: {e}")
+        errors = [
+            f"Profile file does not contain mandatory parameter: {e}"
+            for error in e.exceptions
+            if isinstance(error, KeyError)
+        ]
         msg = "\n".join(errors)
         if not msg:
             msg = "An error occurred while parsing the profile file."
diff --git a/gooddata-sdk/tests/catalog/test_catalog_data_source.py b/gooddata-sdk/tests/catalog/test_catalog_data_source.py
@@ -656,21 +656,18 @@ def test_catalog_create_data_source_greenplum_spec(test_config):
 def test_allowed_data_source_type(test_config):
     allowed_types = JsonApiDataSourceInAttributes.allowed_values.get(("type",))
     for t in allowed_types.values():
-        try:
-            CatalogDataSource(
-                id="test",
-                name="Test2",
-                type=t,
-                url="jdbc:postgresql://localhost:5432/demo",
-                schema="demo",
-                credentials=BasicCredentials(
-                    username="demouser",
-                    password="demopass",
-                ),
-                url_params=[("param", "value")],
-            )
-        except ValueError:
-            assert False, f"ValueError was raised for valid database type {t}"
+        CatalogDataSource(
+            id="test",
+            name="Test2",
+            type=t,
+            url="jdbc:postgresql://localhost:5432/demo",
+            schema="demo",
+            credentials=BasicCredentials(
+                username="demouser",
+                password="demopass",
+            ),
+            url_params=[("param", "value")],
+        )
     db_type = "nonsense"
     try:
         CatalogDataSource(
diff --git a/gooddata-sdk/tests/catalog/test_catalog_permission.py b/gooddata-sdk/tests/catalog/test_catalog_permission.py
diff --git a/pyproject.toml b/pyproject.toml
diff --git a/scripts/docs/json_builder.py b/scripts/docs/json_builder.py
