Skip to content

Commit

Permalink
fix(ingestion/glue): ensure date formatting works on all platforms fo…
Browse files Browse the repository at this point in the history 
…r aws glue (datahub-project#10836)
  • Loading branch information
@sagar-salvi-apptware @aviv-julienjehannet
sagar-salvi-apptware authored and aviv-julienjehannet committed Jul 17, 2024
1 parent 7b16647 commit ade8f06
Show file tree
Hide file tree
Showing 5 changed files with 115 additions and 115 deletions.
2 changes: 1 addition & 1 deletion metadata-ingestion/src/datahub/ingestion/source/aws/glue.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -903,7 +903,7 @@ def gen_database_containers(
parameters["LocationUri"] = database["LocationUri"]
if database.get("CreateTime") is not None:
create_time: datetime.datetime = database["CreateTime"]
parameters["CreateTime"] = create_time.strftime("%B %-d, %Y at %H:%M:%S")
parameters["CreateTime"] = create_time.strftime("%B %d, %Y at %H:%M:%S")
yield from gen_containers(
container_key=database_container_key,
name=database["Name"],
Expand Down
2 changes: 1 addition & 1 deletion metadata-ingestion/tests/unit/glue/glue_delta_mces_golden.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
"instance": "delta_platform_instance",
"env": "PROD",
"database": "delta-database",
"CreateTime": "June 9, 2021 at 14:14:19"
"CreateTime": "June 09, 2021 at 14:14:19"
},
"name": "delta-database",
"qualifiedName": "arn:aws:glue:us-west-2:123412341234:database/delta-database"
Expand Down
2 changes: 1 addition & 1 deletion metadata-ingestion/tests/unit/glue/glue_malformed_delta_mces_golden.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
"instance": "delta_platform_instance",
"env": "PROD",
"database": "delta-database",
"CreateTime": "June 9, 2021 at 14:14:19"
"CreateTime": "June 09, 2021 at 14:14:19"
},
"name": "delta-database",
"qualifiedName": "arn:aws:glue:us-west-2:123412341234:database/delta-database"
Expand Down
110 changes: 55 additions & 55 deletions metadata-ingestion/tests/unit/glue/glue_mces_golden.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
"param1": "value1",
"param2": "value2",
"LocationUri": "s3://test-bucket/test-prefix",
"CreateTime": "June 9, 2021 at 14:14:19"
"CreateTime": "June 09, 2021 at 14:14:19"
},
"name": "flights-database",
"qualifiedName": "arn:aws:glue:us-west-2:123412341234:database/flights-database"
Expand Down Expand Up @@ -55,6 +55,59 @@
}
}
},
{
"entityType": "container",
"entityUrn": "urn:li:container:bdf4342ea6899d162eae685bfe9074a7",
"changeType": "UPSERT",
"aspectName": "containerProperties",
"aspect": {
"json": {
"customProperties": {
"platform": "glue",
"env": "PROD",
"database": "test-database",
"CreateTime": "June 01, 2021 at 14:55:02"
},
"name": "test-database",
"qualifiedName": "arn:aws:glue:us-west-2:123412341234:database/test-database"
}
}
},
{
"entityType": "container",
"entityUrn": "urn:li:container:bdf4342ea6899d162eae685bfe9074a7",
"changeType": "UPSERT",
"aspectName": "status",
"aspect": {
"json": {
"removed": false
}
}
},
{
"entityType": "container",
"entityUrn": "urn:li:container:bdf4342ea6899d162eae685bfe9074a7",
"changeType": "UPSERT",
"aspectName": "dataPlatformInstance",
"aspect": {
"json": {
"platform": "urn:li:dataPlatform:glue"
}
}
},
{
"entityType": "container",
"entityUrn": "urn:li:container:bdf4342ea6899d162eae685bfe9074a7",
"changeType": "UPSERT",
"aspectName": "subTypes",
"aspect": {
"json": {
"typeNames": [
"Database"
]
}
}
},
{
"entityType": "container",
"entityUrn": "urn:li:container:110bc08849d1c1bde5fc345dab5c3ae7",
Expand All @@ -66,7 +119,7 @@
"platform": "glue",
"env": "PROD",
"database": "empty-database",
"CreateTime": "June 1, 2021 at 14:55:13"
"CreateTime": "June 01, 2021 at 14:55:13"
},
"name": "empty-database",
"qualifiedName": "arn:aws:glue:us-west-2:123412341234:database/empty-database"
Expand Down Expand Up @@ -336,59 +389,6 @@
}
}
},
{
"entityType": "container",
"entityUrn": "urn:li:container:bdf4342ea6899d162eae685bfe9074a7",
"changeType": "UPSERT",
"aspectName": "containerProperties",
"aspect": {
"json": {
"customProperties": {
"platform": "glue",
"env": "PROD",
"database": "test-database",
"CreateTime": "June 1, 2021 at 14:55:02"
},
"name": "test-database",
"qualifiedName": "arn:aws:glue:us-west-2:123412341234:database/test-database"
}
}
},
{
"entityType": "container",
"entityUrn": "urn:li:container:bdf4342ea6899d162eae685bfe9074a7",
"changeType": "UPSERT",
"aspectName": "status",
"aspect": {
"json": {
"removed": false
}
}
},
{
"entityType": "container",
"entityUrn": "urn:li:container:bdf4342ea6899d162eae685bfe9074a7",
"changeType": "UPSERT",
"aspectName": "dataPlatformInstance",
"aspect": {
"json": {
"platform": "urn:li:dataPlatform:glue"
}
}
},
{
"entityType": "container",
"entityUrn": "urn:li:container:bdf4342ea6899d162eae685bfe9074a7",
"changeType": "UPSERT",
"aspectName": "subTypes",
"aspect": {
"json": {
"typeNames": [
"Database"
]
}
}
},
{
"proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
Expand Down
114 changes: 57 additions & 57 deletions metadata-ingestion/tests/unit/glue/glue_mces_platform_instance_golden.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
"param1": "value1",
"param2": "value2",
"LocationUri": "s3://test-bucket/test-prefix",
"CreateTime": "June 9, 2021 at 14:14:19"
"CreateTime": "June 09, 2021 at 14:14:19"
},
"name": "flights-database",
"qualifiedName": "arn:aws:glue:us-west-2:123412341234:database/flights-database"
Expand Down Expand Up @@ -57,6 +57,61 @@
}
}
},
{
"entityType": "container",
"entityUrn": "urn:li:container:9fb26491b2c92dde9e80791dbecca9ca",
"changeType": "UPSERT",
"aspectName": "containerProperties",
"aspect": {
"json": {
"customProperties": {
"platform": "glue",
"instance": "some_instance_name",
"env": "PROD",
"database": "test-database",
"CreateTime": "June 01, 2021 at 14:55:02"
},
"name": "test-database",
"qualifiedName": "arn:aws:glue:us-west-2:123412341234:database/test-database"
}
}
},
{
"entityType": "container",
"entityUrn": "urn:li:container:9fb26491b2c92dde9e80791dbecca9ca",
"changeType": "UPSERT",
"aspectName": "status",
"aspect": {
"json": {
"removed": false
}
}
},
{
"entityType": "container",
"entityUrn": "urn:li:container:9fb26491b2c92dde9e80791dbecca9ca",
"changeType": "UPSERT",
"aspectName": "dataPlatformInstance",
"aspect": {
"json": {
"platform": "urn:li:dataPlatform:glue",
"instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:glue,some_instance_name)"
}
}
},
{
"entityType": "container",
"entityUrn": "urn:li:container:9fb26491b2c92dde9e80791dbecca9ca",
"changeType": "UPSERT",
"aspectName": "subTypes",
"aspect": {
"json": {
"typeNames": [
"Database"
]
}
}
},
{
"entityType": "container",
"entityUrn": "urn:li:container:ac4381240e82d55400c22e4392e744a4",
Expand All @@ -69,7 +124,7 @@
"instance": "some_instance_name",
"env": "PROD",
"database": "empty-database",
"CreateTime": "June 1, 2021 at 14:55:13"
"CreateTime": "June 01, 2021 at 14:55:13"
},
"name": "empty-database",
"qualifiedName": "arn:aws:glue:us-west-2:123412341234:database/empty-database"
Expand Down Expand Up @@ -341,61 +396,6 @@
}
}
},
{
"entityType": "container",
"entityUrn": "urn:li:container:9fb26491b2c92dde9e80791dbecca9ca",
"changeType": "UPSERT",
"aspectName": "containerProperties",
"aspect": {
"json": {
"customProperties": {
"platform": "glue",
"instance": "some_instance_name",
"env": "PROD",
"database": "test-database",
"CreateTime": "June 1, 2021 at 14:55:02"
},
"name": "test-database",
"qualifiedName": "arn:aws:glue:us-west-2:123412341234:database/test-database"
}
}
},
{
"entityType": "container",
"entityUrn": "urn:li:container:9fb26491b2c92dde9e80791dbecca9ca",
"changeType": "UPSERT",
"aspectName": "status",
"aspect": {
"json": {
"removed": false
}
}
},
{
"entityType": "container",
"entityUrn": "urn:li:container:9fb26491b2c92dde9e80791dbecca9ca",
"changeType": "UPSERT",
"aspectName": "dataPlatformInstance",
"aspect": {
"json": {
"platform": "urn:li:dataPlatform:glue",
"instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:glue,some_instance_name)"
}
}
},
{
"entityType": "container",
"entityUrn": "urn:li:container:9fb26491b2c92dde9e80791dbecca9ca",
"changeType": "UPSERT",
"aspectName": "subTypes",
"aspect": {
"json": {
"typeNames": [
"Database"
]
}
}
},
{
"proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
Expand Down

0 comments on commit ade8f06

Please sign in to comment.