Skip to content

Commit f4dc9e7

Browse files
JrmyyJrmyy
authored
chore: add function tests for utils macros (dbt-labs#33)
* chore: add function tests for utils macros * feat: port some utils macros to athena chore: test the new utils macros ported * chore: add tests for datediff, dateadd and array_concat
1 parent eb49b61 commit f4dc9e7

22 files changed

+325
-37
lines changed

‎.env.example

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
DBT_TEST_ATHENA_S3_STAGING_DIR=
2+
DBT_TEST_ATHENA_REGION_NAME=
3+
DBT_TEST_ATHENA_DATABASE=
4+
DBT_TEST_ATHENA_SCHEMA=
5+
DBT_TEST_ATHENA_WORK_GROUND=
6+
DBT_TEST_ATHENA_AWS_PROFILE_NAME=

‎.gitignore

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@ MANIFEST
3636
pip-log.txt
3737
pip-delete-this-directory.txt
3838

39-
# Unit test / coverage reports
39+
# Unit tests / coverage reports
4040
htmlcov/
4141
.tox/
4242
.nox/

‎Makefile

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,10 @@
1-
include dev.env
1+
include .env
22
export
33

44
CHANGED_FILES := $(shell git ls-files --modified --other --exclude-standard)
55
CHANGED_FILES_IN_BRANCH := $(shell git diff --name-only $(shell git merge-base origin/main HEAD))
66

7-
.PHONY : install_deps setup pre-commit pre-commit-in-branch pre-commit-all run_tests help
7+
.PHONY : install_deps setup pre-commit pre-commit-in-branch pre-commit-all test help
88

99
install_deps: ## Install all dependencies.
1010
pip install -r dev-requirements.txt
@@ -14,6 +14,13 @@ install_deps: ## Install all dependencies.
1414
setup: ## Install all dependencies and setup pre-commit
1515
make install_deps
1616
pre-commit install
17+
make .env
18+
19+
.env: ## Generate .env file
20+
@cp .env.example $@
21+
22+
test: .env ## Run tests.
23+
pytest tests/functional
1724

1825
pre-commit: ## check modified and added files (compared to last commit!) with pre-commit.
1926
pre-commit run --files $(CHANGED_FILES)
@@ -24,8 +31,5 @@ pre-commit-in-branch: ## check changed since origin/main files with pre-commit.
2431
pre-commit-all: ## Check all files in working directory with pre-commit.
2532
pre-commit run --all-files
2633

27-
run_tests: ## Run tests.
28-
pytest test/integration/athena.dbtspec
29-
3034
help: ## Show this help.
3135
@egrep -h '\s##\s' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m %-30s\033[0m %s\n", $$1, $$2}'

‎README.md

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -197,11 +197,12 @@ make setup
197197
It will :
198198
1. Install all dependencies.
199199
2. Install pre-commit hooks.
200+
3. Generate your `.env` file
200201

201-
Next, configure the environment variables in [dev.env](dev.env) to match your Athena development environment.
202+
Next, adjust `.env` file by configuring the environment variables to match your Athena development environment.
202203

203204
#### Running tests
204-
You can run the tests using `make`:
205+
You must have an AWS account with Athena setup in order to launch the tests. You can run the tests using `make`:
205206

206207
```bash
207208
make run_tests

‎dbt/include/athena/macros/utils/any_value.sql

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
{% macro athena__any_value(expression) -%}
2+
arbitrary({{ expression }})
3+
{%- endmacro %}

‎dbt/include/athena/macros/utils/array_append.sql

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
{% macro athena__array_append(array, new_element) -%}
2+
{{ array }} || {{ new_element }}
3+
{%- endmacro %}

‎dbt/include/athena/macros/utils/array_concat.sql

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
{% macro athena__array_concat(array_1, array_2) -%}
2+
{{ array_1 }} || {{ array_2 }}
3+
{%- endmacro %}

‎dbt/include/athena/macros/utils/array_construct.sql

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
{% macro athena__array_construct(inputs, data_type) -%}
2+
{% if inputs|length > 0 %}
3+
array[ {{ inputs|join(' , ') }} ]
4+
{% else %}
5+
{{ safe_cast('array[]', 'array(' ~ data_type ~ ')') }}
6+
{% endif %}
7+
{%- endmacro %}

‎dbt/include/athena/macros/utils/bool_or.sql

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
{% macro athena__bool_or(expression) -%}
2+
bool_or({{ expression }})
3+
{%- endmacro %}

‎dbt/include/athena/macros/utils/date_trunc.sql

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
{% macro athena__date_trunc(datepart, date) -%}
2+
date_trunc('{{datepart}}', {{date}})
3+
{%- endmacro %}

‎dbt/include/athena/macros/utils/hash.sql

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
{% macro athena__hash(field) -%}
2+
lower(to_hex(md5(to_utf8(cast({{field}} as varchar)))))
3+
{%- endmacro %}

‎dbt/include/athena/macros/utils/right.sql

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
{% macro athena__right(string_text, length_expression) %}
2+
case when {{ length_expression }} = 0
3+
then ''
4+
else
5+
substr({{ string_text }}, -1 * ({{ length_expression }}))
6+
end
7+
{%- endmacro -%}

‎dbt/include/athena/macros/utils/safe_cast.sql

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
{% macro athena__safe_cast(field, type) -%}
2+
try_cast({{field}} as {{type}})
3+
{%- endmacro %}

‎dbt/include/athena/macros/adapters/freshness.sql renamed to ‎dbt/include/athena/macros/utils/timestamps.sql

File renamed without changes.

‎dev-requirements.txt

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,9 @@
1-
# Dev requirements
21
autoflake~=1.7
32
black~=22.10
3+
dbt-tests-adapter~=1.3.0
44
flake8~=5.0
55
Flake8-pyproject~=1.1
66
isort~=5.10
77
pre-commit~=2.20
8-
9-
# Test requirements
10-
pytest-dbt-adapter==0.6.0
8+
pytest~=7.2
119
pyupgrade~=3.2

‎dev.env

Lines changed: 0 additions & 4 deletions
This file was deleted.

‎pyproject.toml

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,3 +27,9 @@ per-file-ignores = """
2727
[tool.isort]
2828
profile = 'black'
2929
src_paths = ['dbt', 'test']
30+
31+
[tool.pytest.ini_options]
32+
testpaths = [
33+
"tests/unit",
34+
"tests/functional",
35+
]

‎test/integration/athena.dbtspec

Lines changed: 0 additions & 21 deletions
This file was deleted.

‎tests/__init__.py

Whitespace-only changes.

‎tests/conftest.py

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
import os
2+
3+
import pytest
4+
5+
# Import the fuctional fixtures as a plugin
6+
# Note: fixtures with session scope need to be local
7+
8+
pytest_plugins = ["dbt.tests.fixtures.project"]
9+
10+
11+
# The profile dictionary, used to write out profiles.yml
12+
@pytest.fixture(scope="class")
13+
def dbt_profile_target():
14+
return {
15+
"type": "athena",
16+
"s3_staging_dir": os.getenv("DBT_TEST_ATHENA_S3_STAGING_DIR"),
17+
"schema": os.getenv("DBT_TEST_ATHENA_SCHEMA"),
18+
"database": os.getenv("DBT_TEST_ATHENA_DATABASE"),
19+
"region_name": os.getenv("DBT_TEST_ATHENA_REGION_NAME"),
20+
"threads": 1,
21+
"num_retries": 0,
22+
"work_group": os.getenv("DBT_TEST_ATHENA_WORK_GROUND"),
23+
"aws_profile_name": os.getenv("DBT_TEST_ATHENA_AWS_PROFILE_NAME"),
24+
}

‎tests/functional/adapter/fixture_datediff.py

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
seeds__data_datediff_csv = """first_date,second_date,datepart,result
2+
2018-01-01 01:00:00,2018-01-02 01:00:00,day,1
3+
2018-01-01 01:00:00,2018-02-01 01:00:00,month,1
4+
2018-01-01 01:00:00,2019-01-01 01:00:00,year,1
5+
2018-01-01 01:00:00,2018-01-01 02:00:00,hour,1
6+
2018-01-01 01:00:00,2018-01-01 02:01:00,minute,61
7+
2018-01-01 01:00:00,2018-01-01 02:00:01,second,3601
8+
2019-12-31 00:00:00,2019-12-27 00:00:00,week,-1
9+
2019-12-31 00:00:00,2019-12-30 00:00:00,week,0
10+
2019-12-31 00:00:00,2020-01-02 00:00:00,week,0
11+
2019-12-31 00:00:00,2020-01-06 02:00:00,week,1
12+
,2018-01-01 02:00:00,hour,
13+
2018-01-01 02:00:00,,hour,
14+
"""
15+
16+
models__test_datediff_sql = """
17+
with data as (
18+
select * from {{ ref('data_datediff') }}
19+
)
20+
select
21+
case
22+
when datepart = 'second' then {{ datediff('first_date', 'second_date', 'second') }}
23+
when datepart = 'minute' then {{ datediff('first_date', 'second_date', 'minute') }}
24+
when datepart = 'hour' then {{ datediff('first_date', 'second_date', 'hour') }}
25+
when datepart = 'day' then {{ datediff('first_date', 'second_date', 'day') }}
26+
when datepart = 'week' then {{ datediff('first_date', 'second_date', 'week') }}
27+
when datepart = 'month' then {{ datediff('first_date', 'second_date', 'month') }}
28+
when datepart = 'year' then {{ datediff('first_date', 'second_date', 'year') }}
29+
else null
30+
end as actual,
31+
result as expected
32+
from data
33+
-- Also test correct casting of literal values.
34+
union all select
35+
{{ datediff("'1999-12-31 23:59:59.999'", "'2000-01-01 00:00:00.000'", "millisecond") }} as actual, 1 as expected
36+
union all select
37+
{{ datediff("'1999-12-31 23:59:59.999'", "'2000-01-01 00:00:00.000'", "second") }} as actual, 1 as expected
38+
union all select
39+
{{ datediff("'1999-12-31 23:59:59.999'", "'2000-01-01 00:00:00.000'", "minute") }} as actual, 1 as expected
40+
union all select
41+
{{ datediff("'1999-12-31 23:59:59.999'", "'2000-01-01 00:00:00.000'", "hour") }} as actual, 1 as expected
42+
union all select
43+
{{ datediff("'1999-12-31 23:59:59.999'", "'2000-01-01 00:00:00.000'", "day") }} as actual, 1 as expected
44+
union all select
45+
{{ datediff("'1999-12-31 23:59:59.999'", "'2000-01-03 00:00:00.000'", "week") }} as actual, 1 as expected
46+
union all select
47+
{{ datediff("'1999-12-31 23:59:59.999'", "'2000-01-01 00:00:00.000'", "month") }} as actual, 1 as expected
48+
union all select
49+
{{ datediff("'1999-12-31 23:59:59.999'", "'2000-01-01 00:00:00.000'", "quarter") }} as actual, 1 as expected
50+
union all select
51+
{{ datediff("'1999-12-31 23:59:59.999'", "'2000-01-01 00:00:00.000'", "year") }} as actual, 1 as expected
52+
"""

0 commit comments

Comments
 (0)