datahub-upgrade/build.gradle

plugins {
  id 'org.springframework.boot'
  id 'java'
  id 'com.palantir.docker'
}

apply from: "../gradle/versioning/versioning.gradle"
apply from: "../gradle/coverage/java-coverage.gradle"

ext {
  docker_registry = rootProject.ext.docker_registry == 'linkedin' ? 'acryldata' : docker_registry
  docker_repo = 'datahub-upgrade'
}

dependencies {
  implementation project(':metadata-io')
  implementation project(':metadata-service:factories')
  implementation project(':metadata-service:restli-client-api')
  implementation project(':metadata-service:configuration')
  implementation project(':metadata-dao-impl:kafka-producer')
  implementation externalDependency.charle

  implementation externalDependency.mustache
  implementation externalDependency.javaxInject
  implementation(externalDependency.hadoopClient) {
    exclude group: 'net.minidev', module: 'json-smart'
    exclude group: 'com.nimbusds', module: 'nimbus-jose-jwt'
    exclude group: "org.apache.htrace", module: "htrace-core4"
    exclude group: "org.eclipse.jetty"
    exclude group: "org.apache.hadoop.thirdparty", module: "hadoop-shaded-protobuf_3_7"
    exclude group: "com.charleskorn.kaml", module:"kaml"

  }

  constraints {
    implementation(externalDependency.hadoopCommon3) {
      because("previous versions are vulnerable to CVE-2021-37404")
    }
    implementation(externalDependency.snakeYaml) {
      because("previous versions are vulnerable to CVE-2022-25857")
    }
    implementation(externalDependency.woodstoxCore) {
      because("previous versions are vulnerable to CVE-2022-40151-2")
    }
    implementation(externalDependency.jettison) {
      because("previous versions are vulnerable")
    }
    implementation(externalDependency.guava) {
      because("CVE-2023-2976")
    }
    implementation('io.airlift:aircompressor:0.27') {
      because("CVE-2024-36114")
    }
    implementation('dnsjava:dnsjava:3.6.1') {
      because("CVE-2024-25638")
    }
  }


  // mock internal schema registry
  implementation externalDependency.kafkaAvroSerde
  implementation externalDependency.kafkaAvroSerializer
  implementation "org.apache.kafka:kafka_2.12:3.7.1"

  implementation externalDependency.slf4jApi
  compileOnly externalDependency.lombok
  implementation externalDependency.picocli
  implementation externalDependency.parquet
  implementation externalDependency.protobuf
  implementation externalDependency.springBeans
  implementation externalDependency.springBootAutoconfigure
  implementation externalDependency.springCore
  implementation externalDependency.springKafka

  runtimeOnly externalDependency.logbackClassic
  runtimeOnly externalDependency.mariadbConnector
  runtimeOnly externalDependency.mysqlConnector
  runtimeOnly externalDependency.postgresql

  implementation externalDependency.awsMskIamAuth

  annotationProcessor externalDependency.lombok
  annotationProcessor externalDependency.picocli

  testImplementation externalDependency.springBootTest
  testImplementation externalDependency.mockito
  testImplementation externalDependency.testng
  testImplementation 'uk.org.webcompere:system-stubs-testng:2.1.7'
  testRuntimeOnly externalDependency.logbackClassic

  constraints {
    implementation(implementation externalDependency.parquetHadoop) {
      because("CVE-2022-42003")
    }
  }
}

bootJar {
  mainClass = 'com.linkedin.datahub.upgrade.UpgradeCliApplication'
  archiveFileName = "${project.name}.jar"
}

bootRun {
  environment "ENTITY_REGISTRY_CONFIG_PATH", "../metadata-models/src/main/resources/entity-registry.yml"
  environment "ENABLE_STRUCTURED_PROPERTIES_SYSTEM_UPDATE", "true"
  environment "ELASTICSEARCH_INDEX_BUILDER_MAPPINGS_REINDEX", "true"
  environment "SERVER_PORT", "8083"
  args += ["-u", "SystemUpdate"]
}

/**
 * Runs SystemUpdate on locally running system
 */
task run(type: Exec) {
  dependsOn bootJar
  group = "Execution"
  description = "Run the datahub-upgrade SystemUpdate process locally."
  environment "ENTITY_REGISTRY_CONFIG_PATH", "../metadata-models/src/main/resources/entity-registry.yml"
  environment "ENABLE_STRUCTURED_PROPERTIES_SYSTEM_UPDATE", "true"
  environment "ELASTICSEARCH_INDEX_BUILDER_MAPPINGS_REINDEX", "true"
  commandLine "java",
          "-agentlib:jdwp=transport=dt_socket,address=5003,server=y,suspend=n",
          "-jar",
          "-Dserver.port=8083", bootJar.getArchiveFile().get(), "-u", "SystemUpdate"
}

/**
 * Runs RestoreIndices on locally running system. The batchSize are set to
 * test the process with pagination and not designed for optimal performance.
 */
task runRestoreIndices(type: Exec) {
  dependsOn bootJar
  group = "Execution"
  description = "Run the restore indices process locally."
  environment "ENTITY_REGISTRY_CONFIG_PATH", "../metadata-models/src/main/resources/entity-registry.yml"
  commandLine "java", "-agentlib:jdwp=transport=dt_socket,address=5003,server=y,suspend=n",
          "-jar",
          "-Dkafka.schemaRegistry.url=http://localhost:8080/schema-registry/api",
          "-Dserver.port=8083",
          bootJar.getArchiveFile().get(), "-u", "RestoreIndices", "-a", "batchSize=100"
}

task runRestoreIndicesUrn(type: Exec) {
  dependsOn bootJar
  group = "Execution"
  description = "Run the restore indices process locally."
  environment "ENTITY_REGISTRY_CONFIG_PATH", "../metadata-models/src/main/resources/entity-registry.yml"
  commandLine "java", "-agentlib:jdwp=transport=dt_socket,address=5003,server=y,suspend=n",
          "-jar",
          "-Dkafka.schemaRegistry.url=http://localhost:8080/schema-registry/api",
          "-Dserver.port=8083",
          bootJar.getArchiveFile().get(), "-u", "RestoreIndices", "-a", "batchSize=100", "-a", "urnBasedPagination=true"
}

docker {
  name "${docker_registry}/${docker_repo}:v${version}"
  version "v${version}"
  dockerfile file("${rootProject.projectDir}/docker/${docker_repo}/Dockerfile")
  files bootJar.outputs.files
  files fileTree(rootProject.projectDir) {
    include '.dockerignore'
    include "docker/${docker_repo}/*"
    include 'metadata-models/src/main/resources/*'
  }.exclude {
    i -> (!i.file.name.endsWith(".dockerignore") && i.file.isHidden())
  }
  tag("Debug", "${docker_registry}/${docker_repo}:debug")

  // platform('linux/arm64', 'linux/amd64')
  buildx(true)
  load(true)
  push(false)

  // Add build args if they are defined (needed for some CI or enterprise environments)
  def dockerBuildArgs = [:]
  if (project.hasProperty('alpineApkRepositoryUrl')) {
    dockerBuildArgs.ALPINE_REPO_URL = project.getProperty('alpineApkRepositoryUrl')
  }
  if (project.hasProperty('githubMirrorUrl')) {
    dockerBuildArgs.GITHUB_REPO_URL = project.getProperty('githubMirrorUrl')
  }
  if (project.hasProperty('mavenCentralRepositoryUrl')) {
    dockerBuildArgs.MAVEN_CENTRAL_REPO_URL = project.getProperty('mavenCentralRepositoryUrl')
  }

  if (dockerBuildArgs.size() > 0) {
    buildArgs(dockerBuildArgs)
  }
}
tasks.getByPath(":datahub-upgrade:docker").dependsOn([bootJar])

task cleanLocalDockerImages {
  doLast {
    rootProject.ext.cleanLocalDockerImages(docker_registry, docker_repo, "${version}")
  }
}
dockerClean.finalizedBy(cleanLocalDockerImages)