Feature/commands for validating file #24
Conversation
Supports offline (just the format) and online validations of IDs
|
Merged with |
| @@ -242,7 +242,13 @@ def update_env_var( | |||
| return None | |||
|
|
|||
| env_var_id: int = all_env_vars[custom_env_var.name].id | |||
| url = f"{self.base_url}/api/v3/accounts/{self.account_id}/projects/{project_id}/environment-variables/{env_var_id}/" | |||
|
|
|||
| # the endpoint is different for updating an overwrite vs creating one | |||
There was a problem hiding this comment.
The trick here is update_env_var() does not create env vars, BUT, it might have to create an env var job overwrite if none exists today.
And depending on the case, the endpoint is different.
There was a problem hiding this comment.
Excellent find, @b-per!
|
|
||
| def __init__(self, **data: Any): | ||
| data["raw_value"] = data["value"] | ||
| data["raw_value"] = data["value"] if "value" in data else data["display_value"] |
There was a problem hiding this comment.
I did some tests and the API gave me back a raw_value but no value, so I added the extra logic to handle this use case.
| @click.argument("config", type=click.File("r")) | ||
| def cli(config): | ||
| def sync(config): |
There was a problem hiding this comment.
Calling this one sync and the other one validate
There was a problem hiding this comment.
@b-per Fantastic changes! I left a couple high-level architectural comments, but nothing blocking 👍🏻 ✅
| @@ -242,7 +242,13 @@ def update_env_var( | |||
| return None | |||
|
|
|||
| env_var_id: int = all_env_vars[custom_env_var.name].id | |||
| url = f"{self.base_url}/api/v3/accounts/{self.account_id}/projects/{project_id}/environment-variables/{env_var_id}/" | |||
|
|
|||
| # the endpoint is different for updating an overwrite vs creating one | |||
There was a problem hiding this comment.
Excellent find, @b-per!
| logger.info(f"Checking that Project IDs are valid") | ||
| if config_project_ids - cloud_project_ids: | ||
| logger.error( | ||
| f"❌ The following project IDs are not valid: {config_project_ids - cloud_project_ids}" | ||
| ) | ||
| online_check_issues = True | ||
|
|
||
| # Check if some Environment IDs in the config are not in Cloud | ||
| logger.info(f"Checking that Environments IDs are valid") | ||
| if config_environment_ids - cloud_environment_ids: | ||
| logger.error( | ||
| f"❌ The following environment IDs are not valid: {config_environment_ids - cloud_environment_ids}" | ||
| ) | ||
| online_check_issues = True | ||
|
|
||
| # In case deferral jobs are mentioned, check that they exist | ||
| deferral_jobs = set( | ||
| [ | ||
| job.deferring_job_definition_id | ||
| for job in defined_jobs | ||
| if job.deferring_job_definition_id | ||
| ] | ||
| ) | ||
| if deferral_jobs: | ||
| logger.info(f"Checking that Deferring Job IDs are valid") | ||
| cloud_jobs = dbt_cloud.get_jobs() | ||
| cloud_job_ids = set([job.id for job in cloud_jobs]) | ||
| if deferral_jobs - cloud_job_ids: | ||
| logger.error( | ||
| f"❌ The following deferral job IDs are not valid: {deferral_jobs - cloud_job_ids}" | ||
| ) | ||
| online_check_issues = True | ||
|
|
||
| if online_check_issues: | ||
| # return an error to handle with bash/CI | ||
| sys.exit(1) |
There was a problem hiding this comment.
More of a long-term architectural comment:
This logic is very sound, and could likely be abstracted into validation functions that can be composed easily. Something to consider for a MUCH later date 😄
Solves #12 and most of #23 (except the validating that env_var already exists)
This PR creates a new command called
validatewith a flag to check on dbt Cloud that:all exist in dbt Cloud
It also returns an exit code different from 0 if the validation fails