Skip to content

Commit

Permalink
Fix the integration tests.
Browse files Browse the repository at this point in the history
  • Loading branch information
@rslanka
rslanka committed Aug 8, 2022
1 parent 4b5a095 commit 837aafc
Show file tree
Hide file tree
Showing 2 changed files with 31 additions and 23 deletions.
44 changes: 22 additions & 22 deletions metadata-ingestion/tests/integration/clickhouse/clickhouse_mces_golden.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -190,11 +190,11 @@
"primary_key": "col_Int64",
"sampling_key": "",
"storage_policy": "default",
"metadata_modification_time": "2022-07-07 13:13:26",
"metadata_modification_time": "2022-08-08 19:48:20",
"total_rows": "10",
"total_bytes": "671",
"data_paths": "['/var/lib/clickhouse/store/7d2/7d234769-6d11-49c3-a147-dcb77a9e21c6/']",
"metadata_path": "/var/lib/clickhouse/store/106/1066edfa-1cc6-4629-a05a-a7d161d7ffae/mv_target_table.sql"
"data_paths": "['/var/lib/clickhouse/store/b1e/b1e5b559-2223-47ac-a34f-4db5cc18fc01/']",
"metadata_path": "/var/lib/clickhouse/store/6d5/6d529683-3153-48a7-950f-4dcecdac4c56/mv_target_table.sql"
},
"externalUrl": null,
"name": "mv_target_table",
Expand Down Expand Up @@ -424,11 +424,11 @@
"primary_key": "",
"sampling_key": "",
"storage_policy": "default",
"metadata_modification_time": "2022-07-07 13:13:26",
"metadata_modification_time": "2022-08-08 19:48:20",
"total_rows": "0",
"total_bytes": "0",
"data_paths": "['/var/lib/clickhouse/store/e36/e36e760c-8b30-453a-ba10-bd9a9c433e41/']",
"metadata_path": "/var/lib/clickhouse/store/106/1066edfa-1cc6-4629-a05a-a7d161d7ffae/test_data_types.sql"
"data_paths": "['/var/lib/clickhouse/store/7a7/7a72a81a-cba6-44fa-b0fb-a497d76dee9b/']",
"metadata_path": "/var/lib/clickhouse/store/6d5/6d529683-3153-48a7-950f-4dcecdac4c56/test_data_types.sql"
},
"externalUrl": null,
"name": "test_data_types",
Expand Down Expand Up @@ -1344,11 +1344,11 @@
"primary_key": "",
"sampling_key": "",
"storage_policy": "",
"metadata_modification_time": "2022-07-07 13:13:26",
"metadata_modification_time": "2022-08-08 19:48:20",
"total_rows": "None",
"total_bytes": "None",
"data_paths": "[]",
"metadata_path": "/var/lib/clickhouse/store/106/1066edfa-1cc6-4629-a05a-a7d161d7ffae/test_dict.sql"
"metadata_path": "/var/lib/clickhouse/store/6d5/6d529683-3153-48a7-950f-4dcecdac4c56/test_dict.sql"
},
"externalUrl": null,
"name": "test_dict",
Expand Down Expand Up @@ -1518,11 +1518,11 @@
"primary_key": "",
"sampling_key": "",
"storage_policy": "default",
"metadata_modification_time": "2022-07-07 13:13:26",
"metadata_modification_time": "2022-08-08 19:48:20",
"total_rows": "0",
"total_bytes": "0",
"data_paths": "['/var/lib/clickhouse/store/724/7243afe9-24e1-4cb4-b269-cbcb6b7a7676/']",
"metadata_path": "/var/lib/clickhouse/store/106/1066edfa-1cc6-4629-a05a-a7d161d7ffae/test_nested_data_types.sql"
"data_paths": "['/var/lib/clickhouse/store/8c6/8c60ab70-ae8a-46c9-820d-7aa9685c1aac/']",
"metadata_path": "/var/lib/clickhouse/store/6d5/6d529683-3153-48a7-950f-4dcecdac4c56/test_nested_data_types.sql"
},
"externalUrl": null,
"name": "test_nested_data_types",
Expand Down Expand Up @@ -1881,11 +1881,11 @@
"primary_key": "",
"sampling_key": "",
"storage_policy": "",
"metadata_modification_time": "2022-07-07 13:13:26",
"metadata_modification_time": "2022-08-08 19:48:20",
"total_rows": "None",
"total_bytes": "None",
"data_paths": "['/var/lib/clickhouse/store/7d2/7d234769-6d11-49c3-a147-dcb77a9e21c6/']",
"metadata_path": "/var/lib/clickhouse/store/106/1066edfa-1cc6-4629-a05a-a7d161d7ffae/mv_with_target_table.sql",
"data_paths": "['/var/lib/clickhouse/store/b1e/b1e5b559-2223-47ac-a34f-4db5cc18fc01/']",
"metadata_path": "/var/lib/clickhouse/store/6d5/6d529683-3153-48a7-950f-4dcecdac4c56/mv_with_target_table.sql",
"view_definition": "",
"is_view": "True"
},
Expand Down Expand Up @@ -2155,11 +2155,11 @@
"primary_key": "",
"sampling_key": "",
"storage_policy": "",
"metadata_modification_time": "2022-07-07 13:13:26",
"total_rows": "None",
"total_bytes": "None",
"data_paths": "['/var/lib/clickhouse/store/6ba/6bac78bd-ac3b-47fb-aaa9-702f1d2f239b/']",
"metadata_path": "/var/lib/clickhouse/store/106/1066edfa-1cc6-4629-a05a-a7d161d7ffae/mv_without_target_table.sql",
"metadata_modification_time": "2022-08-08 19:48:20",
"total_rows": "0",
"total_bytes": "0",
"data_paths": "['/var/lib/clickhouse/store/42a/42a1154e-98ea-4b06-bd9d-16837f11a181/']",
"metadata_path": "/var/lib/clickhouse/store/6d5/6d529683-3153-48a7-950f-4dcecdac4c56/mv_without_target_table.sql",
"view_definition": "",
"is_view": "True"
},
Expand Down Expand Up @@ -2433,11 +2433,11 @@
"primary_key": "",
"sampling_key": "",
"storage_policy": "",
"metadata_modification_time": "2022-07-07 13:13:26",
"metadata_modification_time": "2022-08-08 19:48:20",
"total_rows": "None",
"total_bytes": "None",
"data_paths": "[]",
"metadata_path": "/var/lib/clickhouse/store/106/1066edfa-1cc6-4629-a05a-a7d161d7ffae/test_view.sql",
"metadata_path": "/var/lib/clickhouse/store/6d5/6d529683-3153-48a7-950f-4dcecdac4c56/test_view.sql",
"view_definition": "",
"is_view": "True"
},
Expand Down Expand Up @@ -2569,4 +2569,4 @@
"properties": null
}
}
]
]
10 changes: 9 additions & 1 deletion metadata-ingestion/tests/integration/clickhouse/test_clickhouse.py
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
from typing import List

import pytest
from freezegun import freeze_time

Expand All @@ -23,10 +25,16 @@ def test_clickhouse_ingest(docker_compose_runner, pytestconfig, tmp_path, mock_t
run_datahub_cmd(
["ingest", "--strict-warnings", "-c", f"{config_file}"], tmp_path=tmp_path
)

# These paths change from one instance run of the clickhouse docker to the other, and the FROZEN_TIME does not apply to these.
ignore_paths: List[str] = [
r"root\[\d+\]\['proposedSnapshot'\].+\['aspects'\].+\['customProperties'\]\['metadata_modification_time'\]",
r"root\[\d+\]\['proposedSnapshot'\].+\['aspects'\].+\['customProperties'\]\['data_paths'\]",
r"root\[\d+\]\['proposedSnapshot'\].+\['aspects'\].+\['customProperties'\]\['metadata_path'\]",
]
# Verify the output.
mce_helpers.check_golden_file(
pytestconfig,
ignore_paths=ignore_paths,
output_path=tmp_path / "clickhouse_mces.json",
golden_path=test_resources_dir / "clickhouse_mces_golden.json",
)

0 comments on commit 837aafc

Please sign in to comment.