feat: add elasticsearch sanity integration tests (#2028)

These tests verify that, given an index settings and mappings, data can be written to the index, and read from it with a query_all query. These are very simple sanity tests. We can, and should, write more complex tests that specific to each index in the future.
datahub-project · Dec 3, 2020 · 838f964 · 838f964
1 parent 5f9d967
commit 838f964
Show file tree

Hide file tree

Showing 25 changed files with 1,103 additions and 896 deletions.
diff --git a/build.gradle b/build.gradle
@@ -1,6 +1,7 @@
 buildscript {
+  ext.junitJupiterVersion = '5.6.1'
+  ext.gmaVersion = '0.2.24'
   ext.pegasusVersion = '28.3.7'
-  ext.gmaVersion = '0.2.18'
 
   apply from: './repositories.gradle'
   buildscript.repositories.addAll(project.repositories)
@@ -45,6 +46,7 @@ project.ext.externalDependency = [
     'gmaDaoApiDataTemplate': "com.linkedin.datahub-gma:dao-api-data-template:$gmaVersion",
     'gmaEbeanDao': "com.linkedin.datahub-gma:ebean-dao:$gmaVersion",
     'gmaElasticsearchDao': "com.linkedin.datahub-gma:elasticsearch-dao:$gmaVersion",
+    'gmaElasticsearchIntegTest': "com.linkedin.datahub-gma:elasticsearch-dao-integ-testing-docker:$gmaVersion",
     'gmaNeo4jDao': "com.linkedin.datahub-gma:neo4j-dao:$gmaVersion",
     'gmaRestliResources': "com.linkedin.datahub-gma:restli-resources:$gmaVersion",
     'gmaRestliResourcesDataTemplate': "com.linkedin.datahub-gma:restli-resources-data-template:$gmaVersion",
@@ -64,7 +66,9 @@ project.ext.externalDependency = [
     'jerseyCore': 'org.glassfish.jersey.core:jersey-client:2.25.1',
     'jerseyGuava': 'org.glassfish.jersey.bundles.repackaged:jersey-guava:2.25.1',
     'jsonSimple': 'com.googlecode.json-simple:json-simple:1.1.1',
-    'junit': 'junit:junit:4.12',
+    'junitJupiterApi': "org.junit.jupiter:junit-jupiter-api:$junitJupiterVersion",
+    'junitJupiterParams': "org.junit.jupiter:junit-jupiter-params:$junitJupiterVersion",
+    'junitJupiterEngine': "org.junit.jupiter:junit-jupiter-engine:$junitJupiterVersion",
     // avro-serde includes dependencies for `kafka-avro-serializer` `kafka-schema-registry-client` and `avro`
     'kafkaAvroSerde': 'io.confluent:kafka-streams-avro-serde:5.5.1',
     'kafkaClients': 'org.apache.kafka:kafka-clients:2.3.0',
@@ -114,7 +118,9 @@ subprojects {
     }
 
     tasks.withType(Test) {
-      useTestNG()
+      if (!name.startsWith('integ')) {
+        useTestNG()
+      }
     }
 
     checkstyle {

diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml
@@ -118,7 +118,8 @@ services:
   # This "container" is a workaround to pre-create search indices
   elasticsearch-setup:
     build:
-      context: elasticsearch-setup
+      context: ../
+      dockerfile: docker/elasticsearch-setup/Dockerfile
     env_file: elasticsearch-setup/env/docker.env
     hostname: elasticsearch-setup
     container_name: elasticsearch-setup

diff --git a/docker/elasticsearch-setup/Dockerfile b/docker/elasticsearch-setup/Dockerfile
@@ -1,15 +1,10 @@
 # This "container" is a workaround to pre-create search indices
 FROM jwilder/dockerize:0.6.1
 
-RUN apk add --no-cache curl
+RUN apk add --no-cache curl jq
 
-COPY chart-index-config.json corpuser-index-config.json dashboard-index-config.json dataprocess-index-config.json dataset-index-config.json /
+COPY docker/elasticsearch-setup/create-indices.sh /
+RUN chmod 755 create-indices.sh
+COPY gms/impl/src/main/resources/index /
 
-CMD dockerize \
-    -wait http://$ELASTICSEARCH_HOST:$ELASTICSEARCH_PORT \
-    -timeout 120s \
-    curl -XPUT $ELASTICSEARCH_HOST:$ELASTICSEARCH_PORT/chartdocument --data @chart-index-config.json && \
-    curl -XPUT $ELASTICSEARCH_HOST:$ELASTICSEARCH_PORT/corpuserinfodocument --data @corpuser-index-config.json && \
-    curl -XPUT $ELASTICSEARCH_HOST:$ELASTICSEARCH_PORT/dashboarddocument --data @dashboard-index-config.json && \
-    curl -XPUT $ELASTICSEARCH_HOST:$ELASTICSEARCH_PORT/dataprocessdocument --data @dataprocess-index-config.json && \
-    curl -XPUT $ELASTICSEARCH_HOST:$ELASTICSEARCH_PORT/datasetdocument --data @dataset-index-config.json
+CMD dockerize -wait http://$ELASTICSEARCH_HOST:$ELASTICSEARCH_PORT -timeout 120s /create-indices.sh
diff --git a/docker/elasticsearch-setup/chart-index-config.json b/docker/elasticsearch-setup/chart-index-config.json
diff --git a/docker/elasticsearch-setup/corpuser-index-config.json b/docker/elasticsearch-setup/corpuser-index-config.json
diff --git a/docker/elasticsearch-setup/create-indices.sh b/docker/elasticsearch-setup/create-indices.sh
@@ -0,0 +1,18 @@
+#!/bin/sh
+
+set -e
+
+function create_index {
+  jq -n \
+    --slurpfile settings $2 \
+    --slurpfile mappings $3 \
+    '.settings=$settings[0] | .mappings.doc=$mappings[0]' > /tmp/data
+
+  curl -XPUT $ELASTICSEARCH_HOST:$ELASTICSEARCH_PORT/$1 --data @/tmp/data
+}
+
+create_index chartdocument chart/settings.json chart/mappings.json
+create_index corpuserinfodocument corp-user/settings.json corp-user/mappings.json
+create_index dashboarddocument dashboard/settings.json dashboard/mappings.json
+create_index dataprocessdocument data-process/settings.json data-process/mappings.json
+create_index datasetdocument dataset/settings.json dataset/mappings.json