More packaging fixes

- Add both 'dbt' and 'dbt.*' to the namespace package search path - set zip_safe=False, because I think zipping breaks macros - Make CI tests use non-editable installs - make tox use editable mode for explicit-* so development is still possible without rebuilding tox - remove {toxinidir} stuff that breaks core installation - as of tox 1.6.1, requirements executes within toxinidir. - fix paths to account for installing things
noppaz · Oct 21, 2019 · f94a7a7 · f94a7a7
1 parent 6164733
commit f94a7a7
Show file tree

Hide file tree

Showing 11 changed files with 89 additions and 77 deletions.
diff --git a/core/setup.py b/core/setup.py
@@ -22,7 +22,7 @@ def read(fname):
     author="Fishtown Analytics",
     author_email="[email protected]",
     url="https://github.com/fishtown-analytics/dbt",
-    packages=find_namespace_packages(include=['dbt.*']),
+    packages=find_namespace_packages(include=['dbt', 'dbt.*']),
     package_data={
         'dbt': [
             'include/index.html',
@@ -60,5 +60,6 @@ def read(fname):
         'logbook>=1.5,<1.6',
         'pytest-logbook>=1.2.0,<1.3',
         'typing-extensions>=3.7.4,<3.8',
-    ]
+    ],
+    zip_safe=False,
 )
diff --git a/editable_requirements.txt b/editable_requirements.txt
@@ -0,0 +1,5 @@
+-e ./core
+-e ./plugins/postgres
+-e ./plugins/redshift
+-e ./plugins/snowflake
+-e ./plugins/bigquery
diff --git a/plugins/bigquery/setup.py b/plugins/bigquery/setup.py
@@ -20,7 +20,7 @@
     author="Fishtown Analytics",
     author_email="[email protected]",
     url="https://github.com/fishtown-analytics/dbt",
-    packages=find_namespace_packages(include=['dbt.*']),
+    packages=find_namespace_packages(include=['dbt', 'dbt.*']),
     package_data={
         'dbt': [
             'include/bigquery/dbt_project.yml',
@@ -31,5 +31,6 @@
     install_requires=[
         'dbt-core=={}'.format(package_version),
         'google-cloud-bigquery>=1.0.0,<2',
-    ]
+    ],
+    zip_safe=False,
 )
diff --git a/plugins/postgres/setup.py b/plugins/postgres/setup.py
@@ -20,7 +20,7 @@
     author="Fishtown Analytics",
     author_email="[email protected]",
     url="https://github.com/fishtown-analytics/dbt",
-    packages=find_namespace_packages(include=['dbt.*']),
+    packages=find_namespace_packages(include=['dbt', 'dbt.*']),
     package_data={
         'dbt': [
             'include/postgres/dbt_project.yml',
@@ -31,5 +31,6 @@
     install_requires=[
         'dbt-core=={}'.format(package_version),
         'psycopg2>=2.7.5,<2.8',
-    ]
+    ],
+    zip_safe=False,
 )
diff --git a/plugins/redshift/setup.py b/plugins/redshift/setup.py
@@ -20,7 +20,7 @@
     author="Fishtown Analytics",
     author_email="[email protected]",
     url="https://github.com/fishtown-analytics/dbt",
-    packages=find_namespace_packages(include=['dbt.*']),
+    packages=find_namespace_packages(include=['dbt', 'dbt.*']),
     package_data={
         'dbt': [
             'include/redshift/dbt_project.yml',
@@ -34,5 +34,6 @@
         'boto3>=1.6.23,<1.10.0',
         'botocore>=1.9.23,<1.13.0',
         'psycopg2>=2.7.5,<2.8',
-    ]
+    ],
+    zip_safe=False,
 )
diff --git a/plugins/snowflake/setup.py b/plugins/snowflake/setup.py
@@ -20,7 +20,7 @@
     author="Fishtown Analytics",
     author_email="[email protected]",
     url="https://github.com/fishtown-analytics/dbt",
-    packages=find_namespace_packages(include=['dbt.*']),
+    packages=find_namespace_packages(include=['dbt', 'dbt.*']),
     package_data={
         'dbt': [
             'include/snowflake/dbt_project.yml',
@@ -31,5 +31,6 @@
     install_requires=[
         'dbt-core=={}'.format(package_version),
         'snowflake-connector-python>=1.6.12,<2.1',
-    ]
+    ],
+    zip_safe=False,
 )
diff --git a/requirements.txt b/requirements.txt
@@ -1,5 +1,5 @@
--e ./core
--e ./plugins/postgres
--e ./plugins/redshift
--e ./plugins/snowflake
--e ./plugins/bigquery
+./core
+./plugins/postgres
+./plugins/redshift
+./plugins/snowflake
+./plugins/bigquery
diff --git a/setup.py b/setup.py
@@ -32,5 +32,6 @@
         'dbt-redshift=={}'.format(package_version),
         'dbt-snowflake=={}'.format(package_version),
         'dbt-bigquery=={}'.format(package_version),
-    ]
+    ],
+    zip_safe=False,
 )
diff --git a/test/integration/029_docs_generate_tests/test_docs_generate.py b/test/integration/029_docs_generate_tests/test_docs_generate.py
@@ -793,7 +793,7 @@ def verify_manifest_macros(self, manifest):
                 'original_file_path': helpers_path,
                 'package_name': 'dbt',
                 'root_path': normalize(os.path.join(
-                    self.initial_dir, 'core', 'dbt','include', 'global_project'
+                    self.dbt_core_install_root, 'include', 'global_project'
                 )),
                 'name': 'column_list',
                 'unique_id': 'macro.dbt.column_list',

diff --git a/test/integration/base.py b/test/integration/base.py
@@ -304,6 +304,7 @@ def _symlink_test_folders(self):
         os.symlink(self._logs_dir, os.path.join(self.test_root_dir, 'logs'))
 
     def setUp(self):
+        self.dbt_core_install_root = os.path.dirname(dbt.__file__)
         log_manager.reset_handlers()
         self.initial_dir = INITIAL_ROOT
         os.chdir(self.initial_dir)

diff --git a/tox.ini b/tox.ini
@@ -7,21 +7,21 @@ envlist = unit-py36, integration-postgres-py36, integration-redshift-py36, integ
 basepython = python3.6
 commands = /bin/bash -c '$(which flake8) --select=E,W,F --ignore=W504 core/dbt plugins/*/dbt'
 deps =
-     -r{toxinidir}/dev_requirements.txt
+     -r ./dev_requirements.txt
 
 [testenv:mypy]
 basepython = python3.6
 commands = /bin/bash -c '$(which mypy) core/dbt'
 deps =
-    -r{toxinidir}/requirements.txt
-    -r{toxinidir}/dev_requirements.txt
+    -r ./requirements.txt
+    -r ./dev_requirements.txt
 
 [testenv:unit-py36]
 basepython = python3.6
 commands = /bin/bash -c '{envpython} -m pytest --durations 0 -v {posargs} -n4 test/unit'
 deps =
-    -r{toxinidir}/requirements.txt
-    -r{toxinidir}/dev_requirements.txt
+    -r ./requirements.txt
+    -r ./dev_requirements.txt
 
 
 [testenv:integration-postgres-py36]
@@ -31,9 +31,9 @@ setenv =
     HOME=/home/tox
 commands = /bin/bash -c '{envpython} -m pytest --durations 0 -v -m profile_postgres {posargs} -n4 test/integration/*'
 deps =
-    -e {toxinidir}/core
-    -e {toxinidir}/plugins/postgres
-    -r{toxinidir}/dev_requirements.txt
+    ./core
+    ./plugins/postgres
+    -r ./dev_requirements.txt
 
 [testenv:integration-snowflake-py36]
 basepython = python3.6
@@ -42,9 +42,9 @@ setenv =
     HOME=/home/tox
 commands = /bin/bash -c '{envpython} -m pytest --durations 0 -v -m profile_snowflake {posargs} -n4 test/integration/*'
 deps =
-    -e {toxinidir}/core
-    -e {toxinidir}/plugins/snowflake
-    -r{toxinidir}/dev_requirements.txt
+    ./core
+    ./plugins/snowflake
+    -r ./dev_requirements.txt
 
 [testenv:integration-bigquery-py36]
 basepython = python3.6
@@ -53,9 +53,9 @@ setenv =
     HOME=/home/tox
 commands = /bin/bash -c '{envpython} -m pytest --durations 0 -v -m profile_bigquery {posargs} -n4 test/integration/*'
 deps =
-    -e {toxinidir}/core
-    -e {toxinidir}/plugins/bigquery
-    -r{toxinidir}/dev_requirements.txt
+    ./core
+    ./plugins/bigquery
+    -r ./dev_requirements.txt
 
 [testenv:integration-redshift-py36]
 basepython = python3.6
@@ -64,10 +64,10 @@ setenv =
     HOME=/home/tox
 commands = /bin/bash -c '{envpython} -m pytest --durations 0 -v -m profile_redshift {posargs} -n4 test/integration/*'
 deps =
-    -e {toxinidir}/core
-    -e {toxinidir}/plugins/postgres
-    -e {toxinidir}/plugins/redshift
-    -r{toxinidir}/dev_requirements.txt
+    ./core
+    ./plugins/postgres
+    ./plugins/redshift
+    -r ./dev_requirements.txt
 
 [testenv:integration-presto-py36]
 basepython = python3.6
@@ -76,9 +76,9 @@ setenv =
     HOME=/home/tox
 commands = /bin/bash -c '{envpython} -m pytest --durations 0 -v -m profile_presto {posargs} -n4 test/integration/*'
 deps =
-    -e {toxinidir}/core
-    -e {toxinidir}/plugins/presto
-    -r{toxinidir}/dev_requirements.txt
+    ./core
+    ./plugins/presto
+    -r ./dev_requirements.txt
 
 [testenv:explicit-py36]
 basepython = python3.6
@@ -87,15 +87,15 @@ setenv =
     HOME=/home/tox
 commands = /bin/bash -c '{envpython} -m pytest --durations 0 -v {posargs}'
 deps =
-    -r{toxinidir}/requirements.txt
-    -r{toxinidir}/dev_requirements.txt
+    -r ./editable_requirements.txt
+    -r ./dev_requirements.txt
 
 [testenv:unit-py37]
 basepython = python3.7
 commands = /bin/bash -c '{envpython} -m pytest --durations 0 -v {posargs} -n4 test/unit'
 deps =
-    -r{toxinidir}/requirements.txt
-    -r{toxinidir}/dev_requirements.txt
+    -r ./requirements.txt
+    -r ./dev_requirements.txt
 
 [testenv:integration-postgres-py37]
 basepython = python3.7
@@ -104,9 +104,9 @@ setenv =
     HOME=/home/tox
 commands = /bin/bash -c '{envpython} -m pytest --durations 0 -v -m profile_postgres {posargs} -n4 test/integration/*'
 deps =
-    -e {toxinidir}/core
-    -e {toxinidir}/plugins/postgres
-    -r{toxinidir}/dev_requirements.txt
+    ./core
+    ./plugins/postgres
+    -r ./dev_requirements.txt
 
 [testenv:integration-snowflake-py37]
 basepython = python3.7
@@ -115,9 +115,9 @@ setenv =
     HOME=/home/tox
 commands = /bin/bash -c '{envpython} -m pytest --durations 0 -v -m profile_snowflake {posargs} -n4 test/integration/*'
 deps =
-    -e {toxinidir}/core
-    -e {toxinidir}/plugins/snowflake
-    -r{toxinidir}/dev_requirements.txt
+    ./core
+    ./plugins/snowflake
+    -r ./dev_requirements.txt
 
 [testenv:integration-bigquery-py37]
 basepython = python3.7
@@ -126,9 +126,9 @@ setenv =
     HOME=/home/tox
 commands = /bin/bash -c '{envpython} -m pytest --durations 0 -v -m profile_bigquery {posargs} -n4 test/integration/*'
 deps =
-    -e {toxinidir}/core
-    -e {toxinidir}/plugins/bigquery
-    -r{toxinidir}/dev_requirements.txt
+    ./core
+    ./plugins/bigquery
+    -r ./dev_requirements.txt
 
 [testenv:integration-redshift-py37]
 basepython = python3.7
@@ -137,10 +137,10 @@ setenv =
     HOME=/home/tox
 commands = /bin/bash -c '{envpython} -m pytest --durations 0 -v -m profile_redshift {posargs} -n4 test/integration/*'
 deps =
-    -e {toxinidir}/core
-    -e {toxinidir}/plugins/postgres
-    -e {toxinidir}/plugins/redshift
-    -r{toxinidir}/dev_requirements.txt
+    ./core
+    ./plugins/postgres
+    ./plugins/redshift
+    -r ./dev_requirements.txt
 
 [testenv:integration-presto-py37]
 basepython = python3.7
@@ -149,9 +149,9 @@ setenv =
     HOME=/home/tox
 commands = /bin/bash -c '{envpython} -m pytest --durations 0 -v -m profile_presto {posargs} -n4 test/integration/*'
 deps =
-    -e {toxinidir}/core
-    -e {toxinidir}/plugins/presto
-    -r{toxinidir}/dev_requirements.txt
+    ./core
+    ./plugins/presto
+    -r ./dev_requirements.txt
 
 
 [testenv:explicit-py37]
@@ -161,8 +161,8 @@ setenv =
     HOME=/home/tox
 commands = /bin/bash -c '{envpython} -m pytest --durations 0 -v {posargs}'
 deps =
-    -r{toxinidir}/requirements.txt
-    -r{toxinidir}/dev_requirements.txt
+    -r ./editable_requirements.txt
+    -r ./dev_requirements.txt
 
 [testenv:pywin]
 basepython = {env:PYTHON:}\python.exe
@@ -172,8 +172,8 @@ setenv =
     DBT_INVOCATION_ENV = ci-appveyor
 commands = pytest --durations 0 -v -m 'profile_postgres or profile_snowflake or profile_bigquery or profile_redshift' -n4 test/integration test/unit
 deps =
-    -r{toxinidir}/requirements.txt
-    -r{toxinidir}/dev_requirements.txt
+    -r ./requirements.txt
+    -r ./dev_requirements.txt
 
 [testenv:pywin-unit]
 basepython = python.exe
@@ -183,8 +183,8 @@ setenv =
     DBT_INVOCATION_ENV = ci-appveyor
 commands = python -m pytest --durations 0 -v {posargs} -n4 test/unit
 deps =
-    -r{toxinidir}/requirements.txt
-    -r{toxinidir}/dev_requirements.txt
+    -r ./requirements.txt
+    -r ./dev_requirements.txt
 
 
 [testenv:pywin-postgres]
@@ -195,9 +195,9 @@ setenv =
     DBT_INVOCATION_ENV = ci-appveyor
 commands = python -m pytest --durations 0 -v -m profile_postgres {posargs} -n4 test/integration
 deps =
-    -e {toxinidir}/core
-    -e {toxinidir}/plugins/postgres
-    -r{toxinidir}/dev_requirements.txt
+    ./core
+    ./plugins/postgres
+    -r ./dev_requirements.txt
 
 
 [testenv:pywin-snowflake]
@@ -208,9 +208,9 @@ setenv =
     DBT_INVOCATION_ENV = ci-appveyor
 commands = python -m pytest --durations 0 -v -m profile_snowflake {posargs} -n4 test/integration
 deps =
-    -e {toxinidir}/core
-    -e {toxinidir}/plugins/snowflake
-    -r{toxinidir}/dev_requirements.txt
+    ./core
+    ./plugins/snowflake
+    -r ./dev_requirements.txt
 
 
 [testenv:pywin-bigquery]
@@ -221,9 +221,9 @@ setenv =
     DBT_INVOCATION_ENV = ci-appveyor
 commands = python -m pytest --durations 0 -v -m profile_bigquery {posargs} -n4 test/integration
 deps =
-    -e {toxinidir}/core
-    -e {toxinidir}/plugins/bigquery
-    -r{toxinidir}/dev_requirements.txt
+    ./core
+    ./plugins/bigquery
+    -r ./dev_requirements.txt
 
 
 [testenv:pywin-redshift]
@@ -234,7 +234,7 @@ setenv =
     DBT_INVOCATION_ENV = ci-appveyor
 commands = python -m pytest --durations 0 -v -m profile_redshift {posargs} -n4 test/integration
 deps =
-    -e {toxinidir}/core
-    -e {toxinidir}/plugins/postgres
-    -e {toxinidir}/plugins/redshift
-    -r{toxinidir}/dev_requirements.txt
+    ./core
+    ./plugins/postgres
+    ./plugins/redshift
+    -r ./dev_requirements.txt