feat(graphql): migrating GraphQL API to metadata-service (nee GMS) (#…

…3131)
datahub-project · Aug 20, 2021 · 2c5edd8 · 2c5edd8
1 parent 81eb4f1
commit 2c5edd8
Show file tree

Hide file tree

Showing 156 changed files with 715 additions and 1,641 deletions.
diff --git a/.github/workflows/docker-elasticsearch-setup.yml b/.github/workflows/docker-elasticsearch-setup.yml
@@ -12,7 +12,7 @@ on:
     paths:
       - 'docker/elasticsearch-setup/**'
       - '.github/workflows/docker-elasticsearch-setup.yml'
-      - 'gms/impl/src/main/resources/index/**'
+      - 'metadata-service/restli-impl/src/main/resources/index/**'
   release:
     types: [published, edited]
 

diff --git a/build.gradle b/build.gradle
@@ -94,6 +94,7 @@ project.ext.externalDependency = [
     'parquet': 'org.apache.parquet:parquet-avro:1.12.0',
     'picocli': 'info.picocli:picocli:4.5.0',
     'playCache': 'com.typesafe.play:play-cache_2.11:2.6.18',
+    'playWs': 'com.typesafe.play:play-ahc-ws-standalone_2.11:2.0.8',
     'playDocs': 'com.typesafe.play:play-docs_2.11:2.6.18',
     'playGuice': 'com.typesafe.play:play-guice_2.11:2.6.18',
     'playJavaJdbc': 'com.typesafe.play:play-java-jdbc_2.11:2.6.18',
@@ -110,6 +111,7 @@ project.ext.externalDependency = [
     'springCore': 'org.springframework:spring-core:5.2.3.RELEASE',
     'springJdbc': 'org.springframework:spring-jdbc:5.2.3.RELEASE',
     'springWeb': 'org.springframework:spring-web:5.2.3.RELEASE',
+    'springWebMVC': 'org.springframework:spring-webmvc:5.2.3.RELEASE',
     'springBootAutoconfigure': 'org.springframework.boot:spring-boot-autoconfigure:2.1.4.RELEASE',
     'springBootStarterWeb': 'org.springframework.boot:spring-boot-starter-web:2.1.4.RELEASE',
     'springBootStarterJetty': 'org.springframework.boot:spring-boot-starter-jetty:2.1.4.RELEASE',

diff --git a/datahub-frontend/README.md b/datahub-frontend/README.md
@@ -1,6 +1,6 @@
 # DataHub Frontend
 DataHub frontend is a [Play](https://www.playframework.com/) service written in Java. It is served as a mid-tier
-between [DataHub GMS](../gms) which is the backend service and [DataHub Web](../datahub-web-react/README.md).
+between [DataHub GMS](../metadata-service) which is the backend service and [DataHub Web](../datahub-web-react/README.md).
 
 ## Pre-requisites
 * You need to have [JDK8](https://www.oracle.com/java/technologies/jdk8-downloads.html)
@@ -19,7 +19,7 @@ However, if you only want to build `DataHub Frontend` specifically:
 ```
 
 ## Dependencies
-Before starting `DataHub Frontend`, you need to make sure that [DataHub GMS](../gms) and
+Before starting `DataHub Frontend`, you need to make sure that [DataHub GMS](../metadata-service) and
 all its dependencies have already started and running.
 
 Also, user information should already be registered into the DB,

diff --git a/...b-frontend/app/react/auth/AuthModule.java → datahub-frontend/app/auth/AuthModule.java b/...b-frontend/app/react/auth/AuthModule.java → datahub-frontend/app/auth/AuthModule.java
@@ -1,4 +1,4 @@
-package react.auth;
+package auth;
 
 import com.google.inject.AbstractModule;
 import com.google.inject.Provides;
@@ -15,14 +15,13 @@
 
 import java.util.ArrayList;
 import java.util.List;
-import react.auth.sso.oidc.OidcProvider;
-import react.auth.sso.oidc.OidcConfigs;
-import react.auth.sso.SsoConfigs;
-import react.auth.sso.SsoManager;
-import react.controllers.SsoCallbackController;
-
-import static react.auth.sso.oidc.OidcConfigs.*;
+import auth.sso.oidc.OidcProvider;
+import auth.sso.oidc.OidcConfigs;
+import auth.sso.SsoConfigs;
+import auth.sso.SsoManager;
+import controllers.SsoCallbackController;
 
+import static auth.sso.oidc.OidcConfigs.*;
 
 /**
  * Responsible for configuring, validating, and providing authentication related components.
@@ -43,7 +42,7 @@ protected void configure() {
 
         try {
             bind(SsoCallbackController.class).toConstructor(SsoCallbackController.class.getConstructor(
-                react.auth.sso.SsoManager.class));
+                SsoManager.class));
         } catch (NoSuchMethodException | SecurityException e) {
             System.out.println("Required constructor missing");
         }
@@ -54,7 +53,7 @@ protected void configure() {
     }
 
     @Provides @Singleton
-    protected Config provideConfig(react.auth.sso.SsoManager ssoManager) {
+    protected Config provideConfig(SsoManager ssoManager) {
         if (ssoManager.isSsoEnabled()) {
             final Clients clients = new Clients();
             final List<Client> clientList = new ArrayList<>();
@@ -68,11 +67,11 @@ protected Config provideConfig(react.auth.sso.SsoManager ssoManager) {
     }
 
     @Provides @Singleton
-    protected react.auth.sso.SsoManager provideSsoManager() {
-        react.auth.sso.SsoManager manager = new SsoManager();
+    protected SsoManager provideSsoManager() {
+        SsoManager manager = new SsoManager();
         // Seed the SSO manager with a default SSO provider.
         if (isSsoEnabled(_configs)) {
-            react.auth.sso.SsoConfigs ssoConfigs = new SsoConfigs(_configs);
+            SsoConfigs ssoConfigs = new SsoConfigs(_configs);
             if (ssoConfigs.isOidcEnabled()) {
                 // Register OIDC Provider, add to list of managers.
                 OidcConfigs oidcConfigs = new OidcConfigs(_configs);

diff --git a/...ub-frontend/app/react/auth/AuthUtils.java → datahub-frontend/app/auth/AuthUtils.java b/...ub-frontend/app/react/auth/AuthUtils.java → datahub-frontend/app/auth/AuthUtils.java
@@ -1,4 +1,4 @@
-package react.auth;
+package auth;
 
 import com.linkedin.common.urn.CorpuserUrn;
 import play.mvc.Http;

diff --git a/...rontend/app/react/auth/Authenticator.java → datahub-frontend/app/auth/Authenticator.java b/...rontend/app/react/auth/Authenticator.java → datahub-frontend/app/auth/Authenticator.java
@@ -1,4 +1,4 @@
-package react.auth;
+package auth;
 
 import play.mvc.Http;
 import play.mvc.Result;
@@ -7,7 +7,7 @@
 import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
 
-import static react.auth.AuthUtils.ACTOR;
+import static auth.AuthUtils.ACTOR;
 
 /**
  * Implementation of base Play Authentication used to determine if a request to a route should be

diff --git a/...b-frontend/app/react/auth/ConfigUtil.java → datahub-frontend/app/auth/ConfigUtil.java b/...b-frontend/app/react/auth/ConfigUtil.java → datahub-frontend/app/auth/ConfigUtil.java
@@ -1,4 +1,4 @@
-package react.auth;
+package auth;
 
 public class ConfigUtil {
 

diff --git a/...-frontend/app/react/auth/JAASConfigs.java → datahub-frontend/app/auth/JAASConfigs.java b/...-frontend/app/react/auth/JAASConfigs.java → datahub-frontend/app/auth/JAASConfigs.java
@@ -1,4 +1,4 @@
-package react.auth;
+package auth;
 
 /**
  * Currently, this config enables or disable custom Java Authentication and Authorization Service

diff --git a/...ontend/app/react/auth/sso/SsoConfigs.java → ...hub-frontend/app/auth/sso/SsoConfigs.java b/...ontend/app/react/auth/sso/SsoConfigs.java → ...hub-frontend/app/auth/sso/SsoConfigs.java
@@ -1,7 +1,7 @@
-package react.auth.sso;
+package auth.sso;
 
-import static react.auth.AuthUtils.*;
-import static react.auth.ConfigUtil.*;
+import static auth.AuthUtils.*;
+import static auth.ConfigUtil.*;
 
 
 /**

diff --git a/...ontend/app/react/auth/sso/SsoManager.java → ...hub-frontend/app/auth/sso/SsoManager.java b/...ontend/app/react/auth/sso/SsoManager.java → ...hub-frontend/app/auth/sso/SsoManager.java
@@ -1,4 +1,4 @@
-package react.auth.sso;
+package auth.sso;
 
 import javax.annotation.Nonnull;
 

diff --git a/...ntend/app/react/auth/sso/SsoProvider.java → ...ub-frontend/app/auth/sso/SsoProvider.java b/...ntend/app/react/auth/sso/SsoProvider.java → ...ub-frontend/app/auth/sso/SsoProvider.java
@@ -1,4 +1,4 @@
-package react.auth.sso;
+package auth.sso;
 
 import org.pac4j.core.client.Client;
 

diff --git a/...eact/auth/sso/oidc/OidcCallbackLogic.java → .../app/auth/sso/oidc/OidcCallbackLogic.java b/...eact/auth/sso/oidc/OidcCallbackLogic.java → .../app/auth/sso/oidc/OidcCallbackLogic.java
@@ -1,4 +1,4 @@
-package react.auth.sso.oidc;
+package auth.sso.oidc;
 
 import com.linkedin.common.CorpGroupUrnArray;
 import com.linkedin.common.CorpuserUrnArray;
@@ -46,10 +46,10 @@
 import org.pac4j.core.profile.ProfileManager;
 import org.pac4j.play.PlayWebContext;
 import play.mvc.Result;
-import react.auth.sso.SsoManager;
+import auth.sso.SsoManager;
 
 import static play.mvc.Results.*;
-import static react.auth.AuthUtils.*;
+import static auth.AuthUtils.*;
 
 
 /**

diff --git a/.../app/react/auth/sso/oidc/OidcConfigs.java → ...ontend/app/auth/sso/oidc/OidcConfigs.java b/.../app/react/auth/sso/oidc/OidcConfigs.java → ...ontend/app/auth/sso/oidc/OidcConfigs.java
@@ -1,8 +1,8 @@
-package react.auth.sso.oidc;
+package auth.sso.oidc;
 
-import react.auth.sso.SsoConfigs;
+import auth.sso.SsoConfigs;
 
-import static react.auth.ConfigUtil.*;
+import static auth.ConfigUtil.*;
 
 
 /**

diff --git a/...app/react/auth/sso/oidc/OidcProvider.java → ...ntend/app/auth/sso/oidc/OidcProvider.java b/...app/react/auth/sso/oidc/OidcProvider.java → ...ntend/app/auth/sso/oidc/OidcProvider.java
@@ -1,11 +1,11 @@
-package react.auth.sso.oidc;
+package auth.sso.oidc;
 
 import org.pac4j.core.client.Client;
 import org.pac4j.core.http.callback.PathParameterCallbackUrlResolver;
 import org.pac4j.oidc.config.OidcConfiguration;
 import org.pac4j.oidc.credentials.OidcCredentials;
 import org.pac4j.oidc.profile.OidcProfile;
-import react.auth.sso.SsoProvider;
+import auth.sso.SsoProvider;
 
 
 /**

diff --git a/...th/sso/oidc/OidcResponseErrorHandler.java → ...th/sso/oidc/OidcResponseErrorHandler.java b/...th/sso/oidc/OidcResponseErrorHandler.java → ...th/sso/oidc/OidcResponseErrorHandler.java
@@ -1,4 +1,4 @@
-package react.auth.sso.oidc;
+package auth.sso.oidc;
 
 import org.pac4j.play.PlayWebContext;
 import org.slf4j.Logger;

diff --git a/datahub-frontend/app/controllers/Application.java b/datahub-frontend/app/controllers/Application.java
@@ -1,44 +1,71 @@
 package controllers;
 
-import com.fasterxml.jackson.databind.JsonNode;
+import akka.actor.ActorSystem;
+import akka.stream.ActorMaterializer;
+import akka.stream.Materializer;
+import akka.util.ByteString;
+import auth.Authenticator;
 import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.linkedin.util.Configuration;
+import com.linkedin.util.Pair;
 import com.typesafe.config.Config;
-import org.apache.commons.lang3.StringUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Optional;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ExecutionException;
+import java.util.stream.Collectors;
 import play.Play;
+import play.http.HttpEntity;
+import play.libs.ws.InMemoryBodyWritable;
+import play.libs.ws.StandaloneWSClient;
 import play.libs.Json;
-import play.mvc.BodyParser;
+import play.libs.ws.ahc.StandaloneAhcWSClient;
 import play.mvc.Controller;
+import play.mvc.Http;
+import play.mvc.ResponseHeader;
 import play.mvc.Result;
-import security.AuthUtil;
-import security.AuthenticationManager;
-
 import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
 import javax.inject.Inject;
-import javax.naming.AuthenticationException;
-import javax.naming.NamingException;
-import java.io.BufferedReader;
-import java.io.FileReader;
-import java.io.IOException;
 import java.io.InputStream;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.security.InvalidKeyException;
-import java.security.NoSuchAlgorithmException;
-import java.util.stream.Stream;
+import play.mvc.Security;
+import play.shaded.ahc.org.asynchttpclient.AsyncHttpClient;
+import play.shaded.ahc.org.asynchttpclient.AsyncHttpClientConfig;
+import play.shaded.ahc.org.asynchttpclient.DefaultAsyncHttpClient;
+import play.shaded.ahc.org.asynchttpclient.DefaultAsyncHttpClientConfig;
+
+import static auth.AuthUtils.*;
 
 
 public class Application extends Controller {
 
-  private final Logger _logger = LoggerFactory.getLogger(Application.class.getName());
+  // TODO: Move to constants file.
+  private static final String GMS_HOST_ENV_VAR = "DATAHUB_GMS_HOST";
+  private static final String GMS_PORT_ENV_VAR = "DATAHUB_GMS_PORT";
+  private static final String GMS_USE_SSL_ENV_VAR = "DATAHUB_GMS_USE_SSL";
+  private static final String GMS_SSL_PROTOCOL_VAR = "DATAHUB_GMS_SSL_PROTOCOL";
+
+  private static final String GMS_HOST = Configuration.getEnvironmentVariable(GMS_HOST_ENV_VAR, "localhost");
+  private static final Integer GMS_PORT = Integer.valueOf(Configuration.getEnvironmentVariable(GMS_PORT_ENV_VAR, "8080"));
+  private static final Boolean GMS_USE_SSL = Boolean.parseBoolean(Configuration.getEnvironmentVariable(GMS_USE_SSL_ENV_VAR, "False"));
+  private static final String GMS_SSL_PROTOCOL = Configuration.getEnvironmentVariable(GMS_SSL_PROTOCOL_VAR, null);
+
+  /**
+   * Custom mappings from frontend server paths to metadata-service paths.
+   */
+  private static final Map<String, String> PATH_REMAP = new HashMap<>();
+  static {
+    PATH_REMAP.put("/api/v2/graphql", "/api/graphql");
+  }
+
   private final Config _config;
+  private final StandaloneWSClient _ws;
 
   @Inject
   public Application(@Nonnull Config config) {
     _config = config;
+    _ws = createWsClient();
   }
 
   /**
@@ -52,7 +79,6 @@ public Application(@Nonnull Config config) {
   private Result serveAsset(@Nullable String path) {
     InputStream indexHtml = Play.application().classloader().getResourceAsStream("public/index.html");
     response().setHeader("Cache-Control", "no-cache");
-
     return ok(indexHtml).as("text/html");
   }
 
@@ -73,13 +99,35 @@ public Result index(@Nullable String path) {
   }
 
   /**
-   * Generic not found response
-   * @param path
-   * @return
+   * Proxies requests to the Metadata Service
+   *
+   * TODO: Investigate using mutual SSL authentication to call Metadata Service.
    */
-  @Nonnull
-  public Result apiNotFound(@Nullable String path) {
-    return badRequest("{\"error\": \"API endpoint does not exist\"}");
+  @Security.Authenticated(Authenticator.class)
+  public CompletableFuture<Result> proxy(String path) throws ExecutionException, InterruptedException {
+    final String resolvedUri = PATH_REMAP.getOrDefault(request().uri(), request().uri());
+    return _ws.url(String.format("http://%s:%s%s", GMS_HOST, GMS_PORT, resolvedUri))
+        .setMethod(request().method())
+        .setHeaders(request()
+            .getHeaders()
+            .toMap()
+            .entrySet()
+            .stream()
+            .filter(entry -> !Http.HeaderNames.CONTENT_LENGTH.equals(entry.getKey()))
+            .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue))
+        )
+        .addHeader("X-DataHub-Principal", ctx().session().get(ACTOR)) // TODO: Replace with a token to GMS.
+        .setBody(new InMemoryBodyWritable(ByteString.fromByteBuffer(request().body().asBytes().asByteBuffer()), "application/json"))
+        .execute()
+        .thenApply(apiResponse -> {
+          final ResponseHeader header = new ResponseHeader(apiResponse.getStatus(), apiResponse.getHeaders()
+              .entrySet()
+              .stream()
+              .map(entry -> Pair.of(entry.getKey(), String.join(";", entry.getValue())))
+              .collect(Collectors.toMap(Pair::getFirst, Pair::getSecond)));
+          final HttpEntity body = new HttpEntity.Strict(apiResponse.getBodyAsBytes(), Optional.ofNullable(apiResponse.getContentType()));
+          return new Result(header, body);
+        }).toCompletableFuture();
   }
 
   /**
@@ -175,4 +223,19 @@ private ObjectNode trackingInfo() {
     trackingConfig.put("isEnabled", true);
     return trackingConfig;
   }
+
+  private StandaloneWSClient createWsClient() {
+    final String name = "proxyClient";
+    ActorSystem system = ActorSystem.create(name);
+    system.registerOnTermination(() -> System.exit(0));
+    Materializer materializer = ActorMaterializer.create(system);
+    AsyncHttpClientConfig asyncHttpClientConfig =
+        new DefaultAsyncHttpClientConfig.Builder()
+            .setMaxRequestRetry(0)
+            .setShutdownQuietPeriod(0)
+            .setShutdownTimeout(0)
+            .build();
+    AsyncHttpClient asyncHttpClient = new DefaultAsyncHttpClient(asyncHttpClientConfig);
+    return new StandaloneAhcWSClient(asyncHttpClient, materializer);
+  }
 }
diff --git a/...controllers/AuthenticationController.java → ...controllers/AuthenticationController.java b/...controllers/AuthenticationController.java → ...controllers/AuthenticationController.java
@@ -1,4 +1,4 @@
-package react.controllers;
+package controllers;
 
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.node.ObjectNode;
@@ -19,9 +19,9 @@
 import play.mvc.Controller;
 import play.mvc.Http;
 import play.mvc.Result;
-import react.auth.AuthUtils;
-import react.auth.JAASConfigs;
-import react.auth.sso.SsoManager;
+import auth.AuthUtils;
+import auth.JAASConfigs;
+import auth.sso.SsoManager;
 import security.AuthenticationManager;
 
 import javax.annotation.Nonnull;
@@ -31,7 +31,7 @@
 import java.time.Duration;
 import java.time.temporal.ChronoUnit;
 
-import static react.auth.AuthUtils.*;
+import static auth.AuthUtils.*;
 
 // TODO add logging.
 public class AuthenticationController extends Controller {