build(coverage): enable code coverage for java and python (#11992)

Co-authored-by: david-leifker <[email protected]>
datahub-project · Dec 3, 2024 · 85b42e3 · 85b42e3
1 parent 2f20c52
commit 85b42e3
Show file tree

Hide file tree

Showing 56 changed files with 151 additions and 59 deletions.
diff --git a/datahub-graphql-core/build.gradle b/datahub-graphql-core/build.gradle
@@ -3,6 +3,7 @@ plugins {
     id "io.github.kobylynskyi.graphql.codegen" version "4.1.1"
 }
 
+apply from: '../gradle/coverage/java-coverage.gradle'
 
 dependencies {
     implementation project(':metadata-service:restli-client-api')

diff --git a/datahub-upgrade/build.gradle b/datahub-upgrade/build.gradle
@@ -5,6 +5,7 @@ plugins {
 }
 
 apply from: "../gradle/versioning/versioning.gradle"
+apply from: "../gradle/coverage/java-coverage.gradle"
 
 ext {
   docker_registry = rootProject.ext.docker_registry == 'linkedin' ? 'acryldata' : docker_registry

diff --git a/entity-registry/build.gradle b/entity-registry/build.gradle
@@ -4,6 +4,8 @@ plugins {
   id 'java-test-fixtures'
 }
 
+apply from: "../gradle/coverage/java-coverage.gradle"
+
 dependencies {
   implementation spec.product.pegasus.data
   implementation spec.product.pegasus.generator
@@ -53,3 +55,4 @@ dependencies {
   testFixturesAnnotationProcessor externalDependency.lombok
 }
 compileTestJava.dependsOn tasks.getByPath(':entity-registry:custom-test-model:modelDeploy')
+
diff --git a/...rc/test/java/com/linkedin/metadata/aspect/patch/template/UpstreamLineageTemplateTest.java b/...rc/test/java/com/linkedin/metadata/aspect/patch/template/UpstreamLineageTemplateTest.java
@@ -272,8 +272,8 @@ public void testLargePatchStandard() throws Exception {
     UpstreamLineage result = upstreamLineageTemplate.applyPatch(upstreamLineage, jsonPatch);
     long end = System.currentTimeMillis();
     assertTrue(
-        end - start < 10000,
-        String.format("Expected less then 10 seconds patch actual %s ms", end - start));
+        end - start < 20000,
+        String.format("Expected less then 20 seconds patch actual %s ms", end - start));
 
     assertEquals(result.getUpstreams().size(), 187, "Expected 1 less upstream");
     assertEquals(result.getFineGrainedLineages().size(), 607);

diff --git a/gradle/coverage/java-coverage.gradle b/gradle/coverage/java-coverage.gradle
@@ -0,0 +1,31 @@
+apply plugin: "jacoco"
+
+jacoco {
+  toolVersion = "0.8.12"
+}
+
+/*
+These need to run after evaluation since jacoco plugin alters the test task based on the included test
+lib dependencies defined in each subproject (junit or testNG)
+*/
+afterEvaluate {
+  test {
+    finalizedBy jacocoTestReport
+  }
+
+  jacocoTestReport {
+    dependsOn test
+    reports {
+      xml {
+        required = true
+        /*
+        Tools that aggregate and analyse coverage tools search for the coverage result files. Keeping them under one
+        folder will minimize the time spent searching through the full source tree.
+        */
+        outputLocation = rootProject.layout.buildDirectory.file("coverage-reports/jacoco-${project.name}.xml")
+      }
+      csv.required = false
+      html.required = false
+    }
+  }
+}
diff --git a/gradle/coverage/python-coverage.gradle b/gradle/coverage/python-coverage.gradle
@@ -0,0 +1,18 @@
+//coverage related args to be passed to pytest
+ext.get_coverage_args = { test_name = "" ->
+
+  def coverage_file_name = "pycov-${project.name}${test_name}.xml"
+
+  /*
+  Tools that aggregate and analyse coverage tools search for the coverage result files. Keeping them under one folder
+  will minimize the time spent searching through the full source tree.
+  */
+  def base_path = "${rootProject.buildDir}/coverage-reports"
+
+  /*
+  --cov=src was added via setup.cfg in many of the python projects but for some reason, was not getting picked up
+  consistently, so adding it explicitly. Centralizing these params would make it easier to adjust them for all python
+  projects (with overrides being in the sub-project build script.)
+   */
+  return "--cov-report xml:${base_path}/${coverage_file_name} --cov=src"
+}
diff --git a/ingestion-scheduler/build.gradle b/ingestion-scheduler/build.gradle
@@ -2,6 +2,8 @@ plugins {
     id 'java'
 }
 
+apply from: "../gradle/coverage/java-coverage.gradle"
+
 dependencies {
     implementation project(path: ':metadata-models')
     implementation project(path: ':metadata-io')

diff --git a/li-utils/build.gradle b/li-utils/build.gradle
@@ -1,8 +1,11 @@
+
 plugins {
     id 'java-library'
     id 'pegasus'
 }
 
+apply from: "../gradle/coverage/java-coverage.gradle"
+
 
 dependencies {
     api spec.product.pegasus.data

diff --git a/metadata-auth/auth-api/build.gradle b/metadata-auth/auth-api/build.gradle
@@ -8,6 +8,7 @@ apply plugin: 'signing'
 apply plugin: 'maven-publish'
 apply plugin: 'io.codearte.nexus-staging'
 apply from: '../../metadata-integration/java/versioning.gradle'
+apply from: '../../gradle/coverage/java-coverage.gradle'
 
 jar {
   archiveClassifier = "lib"

diff --git a/metadata-dao-impl/kafka-producer/build.gradle b/metadata-dao-impl/kafka-producer/build.gradle
@@ -1,5 +1,7 @@
 apply plugin: 'java'
 
+apply from: '../../gradle/coverage/java-coverage.gradle'
+
 dependencies {
   implementation project(':metadata-events:mxe-avro')
   implementation project(':metadata-events:mxe-registration')

diff --git a/metadata-events/mxe-utils-avro/build.gradle b/metadata-events/mxe-utils-avro/build.gradle
@@ -3,6 +3,8 @@ plugins {
   id 'pegasus'
 }
 
+apply from: "../../gradle/coverage/java-coverage.gradle"
+
 dependencies {
   api project(':metadata-events:mxe-avro')
   api project(':metadata-models')

diff --git a/metadata-ingestion-modules/airflow-plugin/build.gradle b/metadata-ingestion-modules/airflow-plugin/build.gradle
@@ -2,6 +2,8 @@ plugins {
     id 'base'
 }
 
+apply from: "../../gradle/coverage/python-coverage.gradle"
+
 ext {
   python_executable = 'python3'
   venv_name = 'venv'
@@ -97,7 +99,7 @@ task testQuick(type: Exec, dependsOn: installTest) {
   inputs.files(project.fileTree(dir: "tests/"))
   commandLine 'bash', '-c',
     "source ${venv_name}/bin/activate && set -x && " +
-    "pytest --cov-config=setup.cfg --cov-report xml:coverage_quick.xml -vv --continue-on-collection-errors --junit-xml=junit.quick.xml"
+    "pytest --cov-config=setup.cfg ${get_coverage_args('quick')} -vv --continue-on-collection-errors --junit-xml=junit.quick.xml"
 }
 
 

diff --git a/metadata-ingestion-modules/dagster-plugin/build.gradle b/metadata-ingestion-modules/dagster-plugin/build.gradle
@@ -2,6 +2,8 @@ plugins {
     id 'base'
 }
 
+apply from: "../../gradle/coverage/python-coverage.gradle"
+
 ext {
   python_executable = 'python3'
   venv_name = 'venv'
@@ -84,7 +86,7 @@ task testQuick(type: Exec, dependsOn: installDevTest) {
   outputs.dir("${venv_name}")
   commandLine 'bash', '-c',
     "source ${venv_name}/bin/activate && set -x && " +
-    "pytest -vv --continue-on-collection-errors --junit-xml=junit.quick.xml"
+    "pytest -vv ${get_coverage_args('quick')} --continue-on-collection-errors --junit-xml=junit.quick.xml"
 }
 
 task buildWheel(type: Exec, dependsOn: [environmentSetup]) {

diff --git a/metadata-ingestion-modules/gx-plugin/build.gradle b/metadata-ingestion-modules/gx-plugin/build.gradle
@@ -2,6 +2,8 @@ plugins {
     id 'base'
 }
 
+apply from: "../../gradle/coverage/python-coverage.gradle"
+
 ext {
   python_executable = 'python3'
   venv_name = 'venv'
@@ -84,7 +86,7 @@ task testQuick(type: Exec, dependsOn: installDevTest) {
   outputs.dir("${venv_name}")
   commandLine 'bash', '-c',
     "source ${venv_name}/bin/activate && set -x && " +
-    "pytest -vv --continue-on-collection-errors --junit-xml=junit.quick.xml"
+    "pytest -vv ${get_coverage_args('quick')} --continue-on-collection-errors --junit-xml=junit.quick.xml"
 }
 
 task buildWheel(type: Exec, dependsOn: [environmentSetup]) {

diff --git a/metadata-ingestion-modules/prefect-plugin/build.gradle b/metadata-ingestion-modules/prefect-plugin/build.gradle
@@ -2,6 +2,8 @@ plugins {
     id 'base'
 }
 
+apply from: "../../gradle/coverage/python-coverage.gradle"
+
 ext {
   python_executable = 'python3'
   venv_name = 'venv'
@@ -82,14 +84,14 @@ task testQuick(type: Exec, dependsOn: installDevTest) {
   outputs.dir("${venv_name}")
   commandLine 'bash', '-c',
     "source ${venv_name}/bin/activate && set -x && " +
-    "pytest --cov-config=setup.cfg --cov-report xml:coverage_quick.xml -vv --continue-on-collection-errors --junit-xml=junit.quick.xml -s"
+    "pytest --cov-config=setup.cfg ${get_coverage_args('quick')} -vv --continue-on-collection-errors --junit-xml=junit.quick.xml -s"
 }
 
 
 task testFull(type: Exec, dependsOn: [testQuick, installDevTest]) {
   commandLine 'bash', '-c',
     "source ${venv_name}/bin/activate && set -x && " +
-    "pytest -m 'not slow_integration' -vv --continue-on-collection-errors --junit-xml=junit.full.xml"
+    "pytest -m 'not slow_integration' -vv ${get_coverage_args('full')} --continue-on-collection-errors --junit-xml=junit.full.xml"
 }
 
 

diff --git a/metadata-ingestion/build.gradle b/metadata-ingestion/build.gradle
@@ -2,6 +2,8 @@ plugins {
   id 'base'
 }
 
+apply from: "../gradle/coverage/python-coverage.gradle"
+
 ext {
   python_executable = 'python3'
   venv_name = 'venv'
@@ -11,10 +13,6 @@ if (!project.hasProperty("extra_pip_requirements")) {
     ext.extra_pip_requirements = ""
 }
 
-def get_coverage_arg(test_name) {
-  return "--cov-report xml:coverage_${test_name}.xml "
-}
-
 task checkPythonVersion(type: Exec) {
   commandLine python_executable, '-c',
     'import sys; sys.version_info >= (3, 8), f"Python version {sys.version_info[:2]} not allowed"'
@@ -134,7 +132,7 @@ task testQuick(type: Exec, dependsOn: [installDev, ':metadata-models:generateJso
   inputs.files(project.fileTree(dir: "src/", include: "**/*.py"))
   inputs.files(project.fileTree(dir: "tests/"))
   outputs.dir("${venv_name}")
-  def cvg_arg = get_coverage_arg("quick")
+  def cvg_arg = get_coverage_args("quick")
   commandLine 'bash', '-c',
     "source ${venv_name}/bin/activate && set -x && " +
     "pytest ${cvg_arg} tests/unit --random-order --durations=20 -m 'not integration' -vv --continue-on-collection-errors --junit-xml=junit.quick.xml"
@@ -166,19 +164,19 @@ task testSingle(dependsOn: [installDevTest]) {
 }
 
 task testIntegrationBatch0(type: Exec, dependsOn: [installDevTest]) {
-  def cvg_arg = get_coverage_arg("intBatch0")
+  def cvg_arg = get_coverage_args("intBatch0")
   commandLine 'bash', '-c',
     "source ${venv_name}/bin/activate && set -x && " +
     "pytest ${cvg_arg} --durations=50 -m 'integration_batch_0' -vv --continue-on-collection-errors --junit-xml=junit.integrationbatch0.xml"
 }
 task testIntegrationBatch1(type: Exec, dependsOn: [installDevTest]) {
-  def cvg_arg = get_coverage_arg("intBatch1")
+  def cvg_arg = get_coverage_args("intBatch1")
   commandLine 'bash', '-c',
     "source ${venv_name}/bin/activate && set -x && " +
     "pytest ${cvg_arg} --durations=50 -m 'integration_batch_1' -vv --continue-on-collection-errors --junit-xml=junit.integrationbatch1.xml"
 }
 task testIntegrationBatch2(type: Exec, dependsOn: [installDevTest]) {
-  def cvg_arg = get_coverage_arg("intBatch2")
+  def cvg_arg = get_coverage_args("intBatch2")
   commandLine 'bash', '-c',
     "source ${venv_name}/bin/activate && set -x && " +
     "pytest ${cvg_arg} --durations=20 -m 'integration_batch_2' -vv --continue-on-collection-errors --junit-xml=junit.integrationbatch2.xml"

diff --git a/metadata-integration/java/acryl-spark-lineage/build.gradle b/metadata-integration/java/acryl-spark-lineage/build.gradle
@@ -6,7 +6,7 @@ apply plugin: 'com.github.johnrengelman.shadow'
 apply plugin: 'signing'
 apply plugin: 'io.codearte.nexus-staging'
 apply plugin: 'maven-publish'
-apply plugin: 'jacoco'
+apply from: '../../../gradle/coverage/java-coverage.gradle'
 apply from: '../versioning.gradle'
 
 jar.enabled = false // Since we only want to build shadow jars, disabling the regular jar creation
@@ -163,14 +163,10 @@ checkShadowJar {
   dependsOn shadowJar
 }
 
-jacocoTestReport {
-  dependsOn test // tests are required to run before generating the report
-}
 
 test {
   forkEvery = 1
   useJUnit()
-  finalizedBy jacocoTestReport
 }
 
 assemble {

diff --git a/metadata-integration/java/datahub-client/build.gradle b/metadata-integration/java/datahub-client/build.gradle
@@ -2,12 +2,12 @@ plugins {
   id("com.palantir.git-version") apply false
   id 'java-library'
   id 'com.github.johnrengelman.shadow'
-  id 'jacoco'
   id 'signing'
   id 'io.codearte.nexus-staging'
   id 'maven-publish'
 }
 
+apply from: "../../../gradle/coverage/java-coverage.gradle"
 apply from: "../versioning.gradle"
 import org.apache.tools.ant.filters.ReplaceTokens
 
@@ -59,15 +59,11 @@ task copyAvroSchemas {
 
 compileJava.dependsOn copyAvroSchemas
 
-jacocoTestReport {
-  dependsOn test // tests are required to run before generating the report
-}
 
 test {
   // to avoid simultaneous executions of tests when complete build is run
   mustRunAfter(":metadata-io:test")
   useJUnit()
-  finalizedBy jacocoTestReport
 }
 
 task checkShadowJar(type: Exec) {

diff --git a/metadata-integration/java/datahub-event/build.gradle b/metadata-integration/java/datahub-event/build.gradle
@@ -2,12 +2,12 @@ plugins {
   id("com.palantir.git-version") apply false
   id 'java'
   id 'com.github.johnrengelman.shadow'
-  id 'jacoco'
   id 'signing'
   id 'io.codearte.nexus-staging'
   id 'maven-publish'
 }
 
+apply from: "../../../gradle/coverage/java-coverage.gradle"
 apply from: "../versioning.gradle"
 
 dependencies {
@@ -29,10 +29,6 @@ dependencies {
   testRuntimeOnly externalDependency.logbackClassicJava8
 }
 
-jacocoTestReport {
-  dependsOn test // tests are required to run before generating the report
-}
-
 task copyAvroSchemas {
   dependsOn(':metadata-events:mxe-schemas:renameNamespace')
   copy {
@@ -47,7 +43,6 @@ test {
   // to avoid simultaneous executions of tests when complete build is run
   mustRunAfter(":metadata-io:test")
   useJUnit()
-  finalizedBy jacocoTestReport
 }
 
 // task sourcesJar(type: Jar) {

diff --git a/metadata-integration/java/datahub-protobuf/build.gradle b/metadata-integration/java/datahub-protobuf/build.gradle
@@ -3,12 +3,12 @@ plugins {
     id "application"
 }
 apply plugin: 'java'
-apply plugin: 'jacoco'
 apply plugin: 'com.github.johnrengelman.shadow'
 apply plugin: 'signing'
 apply plugin: 'io.codearte.nexus-staging'
 apply plugin: 'maven-publish'
 apply from: '../versioning.gradle'
+apply from: '../../../gradle/coverage/java-coverage.gradle'
 
 jar.enabled = false // Since we only want to build shadow jars, disabling the regular jar creation
 
@@ -57,13 +57,6 @@ task compileProtobuf {
     }
 }
 
-jacocoTestReport {
-    dependsOn test // tests are required to run before generating the report
-}
-
-test.finalizedBy jacocoTestReport
-
-
 task checkShadowJar(type: Exec) {
    commandLine 'sh', '-c', 'scripts/check_jar.sh'
 }
@@ -202,4 +195,4 @@ nexusStaging {
     password = System.getenv("NEXUS_PASSWORD")
 }
 
-startScripts.dependsOn shadowJar
+startScripts.dependsOn shadowJar
-Original file line number
+Diff line change
@@ Expand Up / @@ -5,6 +5,7 @@ plugins { @@
     }
     apply from: "../gradle/versioning/versioning.gradle"
+    apply from: "../gradle/coverage/java-coverage.gradle"
     ext {
       docker_registry = rootProject.ext.docker_registry == 'linkedin' ? 'acryldata' : docker_registry
@@ Expand Down @@