Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Modernize legacy insert by period materialization #410

Closed
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
29 commits
Select commit Hold shift + click to select a range
3493d3f
Update legacy macro to 0.21.latest incremental
HorvathDanielMarton Aug 17, 2021
051a2e4
Raise error if no filter specified
HorvathDanielMarton Aug 17, 2021
8536054
Create empty table first
HorvathDanielMarton Aug 17, 2021
96ff9ed
Add configurations and period filter
HorvathDanielMarton Aug 23, 2021
7ac2058
Make macro work on manually defined simple example
HorvathDanielMarton Aug 23, 2021
364772b
Implement for loop with predefined time boundaries
HorvathDanielMarton Aug 23, 2021
147494a
Move period boundary macros to helpers.sql
HorvathDanielMarton Aug 23, 2021
3bb17d5
Modify helper macros to support Snowflake
HorvathDanielMarton Aug 24, 2021
a8d3259
Use macros to automate the for loop
HorvathDanielMarton Aug 24, 2021
3e100c2
Refactor and add logging, comments
HorvathDanielMarton Aug 26, 2021
760a33d
Use incremental_upsert() in the for loop
HorvathDanielMarton Aug 26, 2021
ec73c80
Introduce check_for_period_filter macro
HorvathDanielMarton Aug 26, 2021
f876d1d
Dispatch check_for_period_filter() macro
HorvathDanielMarton Aug 26, 2021
a961622
Remove unnecessary code, improve logging
HorvathDanielMarton Aug 26, 2021
ed6d55c
Handle full refresh
HorvathDanielMarton Aug 31, 2021
e53a801
Remove verbose CLI logging
HorvathDanielMarton Aug 31, 2021
b360a91
Update CHANGELOG.md
HorvathDanielMarton Aug 31, 2021
53017a4
Update README.md
HorvathDanielMarton Aug 31, 2021
888d781
Alias table expression in a helper function
HorvathDanielMarton Aug 31, 2021
6fe4adb
Update integration tests
HorvathDanielMarton Aug 31, 2021
af2dd57
Set default period to week
HorvathDanielMarton Aug 31, 2021
cdba4b5
Use make_temp_relation() in for loop
HorvathDanielMarton Aug 31, 2021
7c81020
Update README.md
HorvathDanielMarton Sep 1, 2021
f82f2ee
Run test_insert_by_period for all targets
HorvathDanielMarton Sep 1, 2021
171e5df
Merge remote-tracking branch 'upstream/master' into fix-legacy-insert…
HorvathDanielMarton Sep 1, 2021
e1cdcee
Add aggressive CLI logging for debug purposes
HorvathDanielMarton Sep 3, 2021
713f8a6
Add logging to get_period_boundaries()
HorvathDanielMarton Sep 3, 2021
1ed1e58
Extend helper function logging
HorvathDanielMarton Sep 3, 2021
9c680f9
Remove debug logging
HorvathDanielMarton Sep 3, 2021
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Remove debug logging
  • Loading branch information
HorvathDanielMarton committed Sep 3, 2021
commit 9c680f91ad4978416481030fa1e7f1de7df1b5da
8 changes: 0 additions & 8 deletions macros/materializations/helpers.sql
Original file line number Diff line number Diff line change
Expand Up @@ -18,16 +18,8 @@
{% macro default__get_period_boundaries(target_schema, target_table, timestamp_field, start_date, stop_date, period) -%}

{% call statement('period_boundaries', fetch_result=True) -%}
{{ dbt_utils.log_info(model) }}
{%- set model_name = model['name'] -%}

{%- set target_schema_msg = " [DEBUG] (get_period_boundaries) target schema is: " ~ target_schema -%}
{%- set target_table_msg = " [DEBUG] (get_period_boundaries) target table is: " ~ target_table -%}
{%- set model_name_msg = " [DEBUG] (get_period_boundaries) model[name] is: " ~ model_name -%}
{{ dbt_utils.log_info(target_schema_msg) }}
{{ dbt_utils.log_info(target_table_msg) }}
{{ dbt_utils.log_info(model_name_msg) }}

with data as (
select
coalesce(max("{{timestamp_field}}"), '{{start_date}}')::timestamp as start_timestamp,
Expand Down
36 changes: 0 additions & 36 deletions macros/materializations/insert_by_period_materialization.sql
Original file line number Diff line number Diff line change
Expand Up @@ -12,13 +12,8 @@
{%- set stop_date = config.get('stop_date') or '' -%}}
{%- set period = config.get('period') or 'week' -%}

{{ dbt_utils.log_info("[DEBUG] Parameters are set!") }}

{% set target_relation = this.incorporate(type='table') %}

{%- set target_msg = "[DEBUG] Target is set to be " ~ target_relation -%}
{{ dbt_utils.log_info(target_msg) }}

{% set existing_relation = load_relation(this) %}
{% set tmp_relation = make_temp_relation(target_relation) %}
{%- set full_refresh_mode = (should_full_refresh()) -%}
Expand All @@ -43,13 +38,9 @@

{{ run_hooks(pre_hooks, inside_transaction=False) }}

{{ dbt_utils.log_info("[DEBUG] Prehooks (outside transaction) are done!") }}

-- `BEGIN` happens here:
{{ run_hooks(pre_hooks, inside_transaction=True) }}

{{ dbt_utils.log_info("[DEBUG] Prehooks (inside transaction) are done!") }}

{% set to_drop = [] %}

{# -- first check whether we want to full refresh for source view or config reasons #}
Expand All @@ -60,20 +51,11 @@
{%- set empty_sql = sql | replace("__PERIOD_FILTER__", 'false') -%}
{% set build_sql = create_table_as(False, target_relation, empty_sql) %}

{{ dbt_utils.log_info("[DEBUG] Within 'if existing_relation is none or trigger_full_refresh'!") }}
{%- set existing_relation_msg = "[DEBUG] Existing relation is: " ~ existing_relation -%}
{%- set trigger_full_refresh_msg = "[DEBUG] Trigger full refresh flag is: " ~ trigger_full_refresh -%}
{{ dbt_utils.log_info(existing_relation_msg) }}
{{ dbt_utils.log_info(trigger_full_refresh_msg) }}

{% call statement("main") %}
{{ dbt_utils.log_info("[DEBUG] Executing the following:") }}
{{ dbt_utils.log_info(build_sql) }}
{{ build_sql }}
{% endcall %}
{% endif %}

{{ dbt_utils.log_info("[DEBUG] We have an empty table now.") }}
-- Now that we have an empty table, let's put something in it.

{% set _ = dbt_utils.get_period_boundaries(schema,
Expand All @@ -86,32 +68,18 @@
{%- set stop_timestamp = load_result('period_boundaries')['data'][0][1] | string -%}
{%- set num_periods = load_result('period_boundaries')['data'][0][2] | int -%}


{%- set start_timestamp_msg = "[DEBUG] Start timestamp is: " ~ start_timestamp -%}
{%- set stop_timestamp_msg = "[DEBUG] End timestamp is: " ~ stop_timestamp -%}
{%- set num_periods_msg = "[DEBUG] Number of periods: " ~ num_periods ~ " " ~ period ~ "(s)" -%}
{{ dbt_utils.log_info(start_timestamp_msg) }}
{{ dbt_utils.log_info(stop_timestamp_msg) }}
{{ dbt_utils.log_info(num_periods_msg) }}

{% set target_columns = adapter.get_columns_in_relation(target_relation) %}
{%- set target_cols_csv = target_columns | map(attribute='quoted') | join(', ') -%}

{%- set loop_vars = {'sum_rows_inserted': 0} -%}

{% for i in range(num_periods) -%}
{%- set loop_enter_msg = "[DEBUG] Entering the " ~ (i + 1) ~ ". iteration of the loop." -%}
{{ dbt_utils.log_info(loop_enter_msg) }}

{%- set msg = "Running for " ~ period ~ " " ~ (i + 1) ~ " of " ~ num_periods -%}
{{ dbt_utils.log_info(msg) }}

{%- set tmp_identifier_suffix = '__dbt_incremental_period_' ~ i ~ '_tmp' -%}
{% set tmp_relation = make_temp_relation(target_relation, tmp_identifier_suffix) %}

{%- set tmp_relation_msg = "[DEBUG] (Within loop) Termporary relation is: " ~ tmp_relation -%}
{{ dbt_utils.log_info(tmp_relation_msg) }}

{% set tmp_table_sql = dbt_utils.get_period_sql(target_cols_csv,
sql,
timestamp_field,
Expand All @@ -129,8 +97,6 @@
{%- set name = 'main-' ~ i -%}
{% set build_sql = incremental_upsert(tmp_relation, target_relation, unique_key=unique_key) %}
{% call statement(name, fetch_result=True) -%}
{{ dbt_utils.log_info("[DEBUG] (Within loop) Executing the following:") }}
{{ dbt_utils.log_info(build_sql) }}
{{ build_sql }}
{%- endcall %}

Expand All @@ -148,8 +114,6 @@
{{ dbt_utils.log_info(msg) }}
{%- endfor %}

{{ dbt_utils.log_info("[DEBUG] Out of the loop.") }}

{% do persist_docs(target_relation, model) %}

{% if existing_relation is none or existing_relation.is_view or should_full_refresh() %}
Expand Down