feat(ingest): presto-on-hive - Description and hive view support (#5993)

* Presto on Hive Improvements like: - Capturing table/column descriptions - Adding Hive view support - Various bug fixes
datahub-project · Sep 20, 2022 · 46494bc · 46494bc
1 parent 8e8fb51
commit 46494bc
Show file tree

Hide file tree

Showing 5 changed files with 401 additions and 111 deletions.
diff --git a/.github/workflows/metadata-ingestion.yml b/.github/workflows/metadata-ingestion.yml
@@ -20,15 +20,14 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-
   metadata-ingestion-general:
     runs-on: ubuntu-latest
     env:
       SPARK_VERSION: 3.0.3
       DATAHUB_TELEMETRY_ENABLED: false
     strategy:
       matrix:
-        python-version: ["3.7", "3.10"]
+        python-version: ["3.7", "3.10.6"]
     steps:
       - uses: actions/checkout@v2
       - uses: actions/setup-python@v2
@@ -56,7 +55,13 @@ jobs:
     strategy:
       matrix:
         python-version: ["3.7", "3.10"]
-        command: ["installAirflow1", "testIntegration", "testIntegrationBatch1", "testSlowIntegration"]
+        command:
+          [
+            "installAirflow1",
+            "testIntegration",
+            "testIntegrationBatch1",
+            "testSlowIntegration",
+          ]
       fail-fast: false
     steps:
       - uses: actions/checkout@v2
@@ -65,8 +70,8 @@ jobs:
           python-version: ${{ matrix.python-version }}
       - uses: vemonet/setup-spark@v1 # spark is required for pyspark+pydeequ data lake profiling
         with:
-          spark-version: '3.0.3'
-          hadoop-version: '3.2'
+          spark-version: "3.0.3"
+          hadoop-version: "3.2"
       - name: Install dependencies
         run: ./metadata-ingestion/scripts/install_deps.sh
       - name: Run metadata-ingestion tests

diff --git a/.gitignore b/.gitignore
@@ -72,3 +72,5 @@ temp*
 
 # frontend assets
 datahub-frontend/public/**
+
+.remote*
Original file line number	Diff line number	Diff line change
Expand Up		@@ -72,3 +72,5 @@ temp*

		# frontend assets
		datahub-frontend/public/**

		.remote*