fix(ci): simplify python release process (#10133)

datahub-project · Mar 26, 2024 · 1598070 · 1598070
1 parent 45f6c2a
commit 1598070
Show file tree

Hide file tree

Showing 8 changed files with 15 additions and 22 deletions.
diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml
@@ -47,11 +47,7 @@ jobs:
         run: ./metadata-ingestion/scripts/install_deps.sh
       - name: Build Docs
         run: |
-          ./gradlew --info docs-website:build \
-            -x :metadata-ingestion-modules:airflow-plugin:build \
-            -x :metadata-ingestion-modules:airflow-plugin:check \
-            -x :metadata-ingestion-modules:dagster-plugin:build \
-            -x :metadata-ingestion-modules:dagster-plugin:check \
+          ./gradlew --info docs-website:build
 
       - name: Deploy
         if: github.event_name == 'push' && github.repository == 'datahub-project/datahub'

diff --git a/docs-website/build.gradle b/docs-website/build.gradle
@@ -79,19 +79,14 @@ task yarnInstall(type: YarnTask) {
   )
   outputs.dir('node_modules')
 }
-task airflowPluginBuild(dependsOn: [':metadata-ingestion-modules:airflow-plugin:buildWheel']) {
-}
-
-// The Dagster plugin build and airflow plugin build can't be built at the same time; otherwise, it will raise
-// fatal: Unable to create '/home/runner/work/datahub/datahub/.git/index.lock': File exists.. on CI
-task dagsterPluginBuild(dependsOn: [':metadata-ingestion-modules:dagster-plugin:buildWheel', airflowPluginBuild]) {
-}
 
 task yarnGenerate(type: YarnTask, dependsOn: [yarnInstall,
     generateGraphQLSchema, generateJsonSchema,
     ':metadata-ingestion:modelDocGen', ':metadata-ingestion:docGen',
     ':metadata-ingestion:buildWheel',
-    airflowPluginBuild, dagsterPluginBuild] ) {
+    ':metadata-ingestion-modules:airflow-plugin:buildWheel',
+    ':metadata-ingestion-modules:dagster-plugin:buildWheel',
+    ]) {
   inputs.files(projectMdFiles)
   outputs.cacheIf { true }
   args = ['run', 'generate']

diff --git a/metadata-ingestion-modules/airflow-plugin/build.gradle b/metadata-ingestion-modules/airflow-plugin/build.gradle
@@ -106,7 +106,7 @@ task cleanPythonCache(type: Exec) {
   commandLine 'bash', '-c',
     "find src -type f -name '*.py[co]' -delete -o -type d -name __pycache__ -delete -o -type d -empty -delete"
 }
-task buildWheel(type: Exec, dependsOn: [install, cleanPythonCache]) {
+task buildWheel(type: Exec, dependsOn: [environmentSetup, cleanPythonCache]) {
   commandLine 'bash', '-c',
     "source ${venv_name}/bin/activate && set -x && " +
     'uv pip install build && RELEASE_VERSION="\${RELEASE_VERSION:-0.0.0.dev1}" RELEASE_SKIP_TEST=1 RELEASE_SKIP_INSTALL=1 RELEASE_SKIP_UPLOAD=1 ./scripts/release.sh'

diff --git a/metadata-ingestion-modules/airflow-plugin/scripts/release.sh b/metadata-ingestion-modules/airflow-plugin/scripts/release.sh
@@ -23,4 +23,4 @@ python -m build
 if [[ ! ${RELEASE_SKIP_UPLOAD:-} ]]; then
     python -m twine upload 'dist/*'
 fi
-git restore src/${MODULE}/__init__.py
+mv src/${MODULE}/__init__.py.bak src/${MODULE}/__init__.py
diff --git a/metadata-ingestion-modules/dagster-plugin/build.gradle b/metadata-ingestion-modules/dagster-plugin/build.gradle
@@ -109,7 +109,7 @@ task testFull(type: Exec, dependsOn: [testQuick, installDevTest]) {
   commandLine 'bash', '-x', '-c',
     "source ${venv_name}/bin/activate && pytest -m 'not slow_integration' -vv --continue-on-collection-errors --junit-xml=junit.full.xml"
 }
-task buildWheel(type: Exec, dependsOn: [install]) {
+task buildWheel(type: Exec, dependsOn: [environmentSetup]) {
   commandLine 'bash', '-c', "source ${venv_name}/bin/activate && " +
   'uv pip install build && RELEASE_VERSION="\${RELEASE_VERSION:-0.0.0.dev1}" RELEASE_SKIP_TEST=1 RELEASE_SKIP_UPLOAD=1 ./scripts/release.sh'
 }

diff --git a/metadata-ingestion-modules/dagster-plugin/scripts/release.sh b/metadata-ingestion-modules/dagster-plugin/scripts/release.sh
@@ -13,7 +13,7 @@ MODULE=datahub_dagster_plugin
 python -c 'import setuptools; where="./src"; assert setuptools.find_packages(where) == setuptools.find_namespace_packages(where), "you seem to be missing or have extra __init__.py files"'
 if [[ ${RELEASE_VERSION:-} ]]; then
     # Replace version with RELEASE_VERSION env variable
-    sed -i.bak "s/__version__ = \"0.0.0.dev0\"/__version__ = \"$RELEASE_VERSION\"/" src/${MODULE}/__init__.py
+    sed -i.bak "s/__version__ = \"1\!0.0.0.dev0\"/__version__ = \"$(echo $RELEASE_VERSION|sed s/-/+/)\"/" src/${MODULE}/__init__.py
 else
     vim src/${MODULE}/__init__.py
 fi
@@ -23,4 +23,4 @@ python -m build
 if [[ ! ${RELEASE_SKIP_UPLOAD:-} ]]; then
     python -m twine upload 'dist/*'
 fi
-git restore src/${MODULE}/__init__.py
+mv src/${MODULE}/__init__.py.bak src/${MODULE}/__init__.py
diff --git a/metadata-ingestion/scripts/release.sh b/metadata-ingestion/scripts/release.sh
@@ -7,18 +7,20 @@ elif [[ ! ${RELEASE_SKIP_INSTALL:-} ]]; then
 	../gradlew install
 fi
 
+MODULE=datahub
+
 # Check packaging constraint.
 python -c 'import setuptools; where="./src"; assert setuptools.find_packages(where) == setuptools.find_namespace_packages(where), "you seem to be missing or have extra __init__.py files"'
 if [[ ${RELEASE_VERSION:-} ]]; then
     # Replace version with RELEASE_VERSION env variable
-    sed -i.bak "s/__version__ = \"1\!0.0.0.dev0\"/__version__ = \"$(echo $RELEASE_VERSION|sed s/-/+/)\"/" src/datahub/__init__.py
+    sed -i.bak "s/__version__ = \"1\!0.0.0.dev0\"/__version__ = \"$(echo $RELEASE_VERSION|sed s/-/+/)\"/" src/${MODULE}/__init__.py
 else
-    vim src/datahub/__init__.py
+    vim src/${MODULE}/__init__.py
 fi
 
 rm -rf build dist || true
 python -m build
 if [[ ! ${RELEASE_SKIP_UPLOAD:-} ]]; then
     python -m twine upload 'dist/*'
 fi
-git restore src/datahub/__init__.py
+mv src/${MODULE}/__init__.py.bak src/${MODULE}/__init__.py
diff --git a/vercel.json b/vercel.json
@@ -1,5 +1,5 @@
 {
-  "buildCommand": "./gradlew -PuseSystemNode=true :docs-website:build -x :metadata-ingestion-modules:dagster-plugin:build -x :metadata-ingestion-modules:dagster-plugin:check",
+  "buildCommand": "./gradlew -PuseSystemNode=true :docs-website:build",
   "github": {
     "silent": true,
     "autoJobCancelation": true