datahub-project · shirshanka · May 4, 2022 · Apr 13, 2022 · Apr 13, 2022 · Apr 13, 2022
diff --git a/.gitignore b/.gitignore
@@ -11,8 +11,11 @@
 *gma-create-all.sql
 *gma-drop-all.sql
 
-# Pegasus & Avro
+# Pegasus, Avro, & other schemas
 **/src/mainGenerated*
+**/src/generatedJsonSchema
+**/metadata-io/generated
+**/metadata-integration/java/datahub-client/generated
 **/src/testGenerated*
 metadata-events/mxe-registration/src/main/resources/**/*.avsc
 

diff --git a/build.gradle b/build.gradle
@@ -16,6 +16,7 @@ buildscript {
     }
     classpath "io.codearte.gradle.nexus:gradle-nexus-staging-plugin:0.30.0"
     classpath "com.palantir.gradle.gitversion:gradle-git-version:0.12.3"
+    classpath "gradle.plugin.org.hidetake:gradle-swagger-generator-plugin:2.18.1"
   }
 }
 
@@ -80,10 +81,12 @@ project.ext.externalDependency = [
     'jacksonDataFormatYaml': 'com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.9.10',
     'javatuples': 'org.javatuples:javatuples:1.2',
     'javaxInject' : 'javax.inject:javax.inject:1',
+    'javaxValidation' : 'javax.validation:validation-api:2.0.1.Final',
     'jerseyCore': 'org.glassfish.jersey.core:jersey-client:2.25.1',
     'jerseyGuava': 'org.glassfish.jersey.bundles.repackaged:jersey-guava:2.25.1',
     'jettyJaas': 'org.eclipse.jetty:jetty-jaas:9.4.32.v20200930',
     'jgrapht': 'org.jgrapht:jgrapht-core:1.5.1',
+    'jsonSchemaAvro': 'com.github.fge:json-schema-avro:0.1.4',
     'jsonSimple': 'com.googlecode.json-simple:json-simple:1.1.1',
     'junitJupiterApi': "org.junit.jupiter:junit-jupiter-api:$junitJupiterVersion",
     'junitJupiterParams': "org.junit.jupiter:junit-jupiter-params:$junitJupiterVersion",

diff --git a/buildSrc/build.gradle b/buildSrc/build.gradle
@@ -0,0 +1,16 @@
+apply plugin: 'java'
+
+buildscript {
+  apply from: '../repositories.gradle'
+}
+
+dependencies {
+  compile('io.acryl:json-schema-avro:0.1.5') {
+    exclude group: 'com.fasterxml.jackson.core', module: 'jackson-databind'
+    exclude group: 'com.google.guava', module: 'guava'
+  }
+  compile 'com.google.guava:guava:27.0.1-jre'
+  compile 'com.fasterxml.jackson.core:jackson-databind:2.9.10.7'
+  compile 'com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.8.11'
+  compile 'commons-io:commons-io:2.11.0'
+}
diff --git a/buildSrc/src/main/java/io/datahubproject/GenerateJsonSchemaTask.java b/buildSrc/src/main/java/io/datahubproject/GenerateJsonSchemaTask.java
@@ -0,0 +1,189 @@
+package io.datahubproject;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.node.ArrayNode;
+import com.fasterxml.jackson.databind.node.JsonNodeFactory;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.fasterxml.jackson.dataformat.yaml.YAMLMapper;
+import com.github.fge.jackson.JacksonUtils;
+import com.github.fge.jackson.JsonLoader;
+import com.github.fge.jsonschema.core.exceptions.ProcessingException;
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.FileAlreadyExistsException;
+import java.nio.file.FileSystems;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.StandardOpenOption;
+import java.util.HashSet;
+import java.util.List;
+import java.util.stream.Collectors;
+import org.gradle.api.DefaultTask;
+import org.gradle.api.tasks.CacheableTask;
+import org.gradle.api.tasks.InputDirectory;
+import org.gradle.api.tasks.OutputDirectory;
+import org.gradle.api.tasks.TaskAction;
+
+import static com.github.fge.processing.ProcessingUtil.*;
+import static org.apache.commons.io.FilenameUtils.*;
+
+
+@CacheableTask
+public class GenerateJsonSchemaTask extends DefaultTask {
+  private String inputDirectory;
+  private String outputDirectory;
+  private ArrayNode aspectType;
+  private Path combinedDirectory;
+  private Path jsonDirectory;
+  public static final String sep = FileSystems.getDefault().getSeparator();
+
+  private static final JsonNodeFactory NODE_FACTORY = JacksonUtils.nodeFactory();
+
+  public void setInputDirectory(String inputDirectory) {
+    this.inputDirectory = inputDirectory;
+  }
+
+  @InputDirectory
+  public String getInputDirectory() {
+   return inputDirectory;
+  }
+
+  @OutputDirectory
+  public String getOutputDirectory() {
+    return outputDirectory;
+  }
+
+  public void setOutputDirectory(String outputDirectory) {
+    this.outputDirectory = outputDirectory;
+  }
+
+  @TaskAction
+  public void generate() throws IOException {
+    Path baseDir = Paths.get(inputDirectory);
+    aspectType = NODE_FACTORY.arrayNode();
+
+    Path schemaDirectory = Paths.get(outputDirectory);
+    jsonDirectory = Paths.get(schemaDirectory + sep + "json");
+    try {
+      Files.createDirectory(schemaDirectory);
+    } catch (FileAlreadyExistsException fae) {
+      // No-op
+    }
+    try {
+      Files.createDirectory(jsonDirectory);
+    } catch (FileAlreadyExistsException fae) {
+      // No-op
+    }
+    Files.walk(baseDir)
+        .filter(Files::isRegularFile)
+        .map(Path::toFile)
+        .forEach(this::generateSchema);
+    List<ObjectNode> nodesList = Files.walk(jsonDirectory)
+        .filter(Files::isRegularFile)
+        .filter(path -> {
+          String fileName = path.toFile().getName();
+          return !getBaseName(fileName).contains("ChangeEvent") && !getBaseName(fileName).contains("AuditEvent");
+        })
+        .map(Path::toFile)
+        .map(file -> {
+          try {
+            return (ObjectNode) JsonLoader.fromFile(file);
+          } catch (IOException e) {
+            throw new RuntimeException(e);
+          }
+        })
+        .collect(Collectors.toList());
+
+    ObjectNode schemasNode = NODE_FACTORY.objectNode();
+    nodesList.forEach(objectNode -> {
+      ObjectNode definitions = (ObjectNode) objectNode.get("definitions");
+      // Modify GenericAspect and EnvelopedAspect to have all Aspect subtypes
+      if (definitions.has("GenericAspect")) {
+        ObjectNode genericAspect = (ObjectNode) definitions.get("GenericAspect");
+        ((ObjectNode) genericAspect.get("properties")).replace("value", NODE_FACTORY.objectNode().set("oneOf", aspectType));
+      }
+      if (definitions.has("EnvelopedAspect")) {
+        ObjectNode envelopedAspect = (ObjectNode) definitions.get("EnvelopedAspect");
+        ((ObjectNode) envelopedAspect.get("properties")).replace("value", NODE_FACTORY.objectNode().set("oneOf", aspectType));
+      }
+      schemasNode.setAll(definitions);
+    });
+    /*
+    Minimal OpenAPI header
+    openapi: 3.0.1
+    info:
+      title: OpenAPI definition
+      version: v0
+    servers:
+      - url: http://localhost:8080/openapi
+        description: Generated server url
+    paths:
+      /path:
+        get:
+          tags:
+            - path
+    */
+    ObjectNode yamlHeader = (ObjectNode) ((ObjectNode) NODE_FACTORY.objectNode()
+        .put("openapi", "3.0.1")
+        .set("info", NODE_FACTORY.objectNode()
+            .put("title", "OpenAPI Definition")
+            .put("version", "v0")))
+        .set("paths", NODE_FACTORY.objectNode()
+            .set("/path", NODE_FACTORY.objectNode()
+                .set("get", NODE_FACTORY.objectNode().set("tags", NODE_FACTORY.arrayNode().add("path")))));
+    JsonNode combinedSchemaDefinitionsYaml = ((ObjectNode) NODE_FACTORY.objectNode().set("components",
+        NODE_FACTORY.objectNode().set("schemas", schemasNode))).setAll(yamlHeader);
+
+    final String yaml = new YAMLMapper().writeValueAsString(combinedSchemaDefinitionsYaml)
+        .replaceAll("definitions", "components/schemas")
+        .replaceAll("\n\\s+- type: \"null\"", "");
+
+    combinedDirectory = Paths.get(outputDirectory + sep + "combined");
+    try {
+      Files.createDirectory(combinedDirectory);
+    } catch (FileAlreadyExistsException fae) {
+      // No-op
+    }
+    Files.write(Paths.get(combinedDirectory + sep + "open-api.yaml"),
+        yaml.getBytes(StandardCharsets.UTF_8), StandardOpenOption.WRITE, StandardOpenOption.CREATE,
+        StandardOpenOption.TRUNCATE_EXISTING);
+
+    JsonNode combinedSchemaDefinitionsJson = NODE_FACTORY.objectNode().set("definitions",schemasNode);
+    String prettySchema = JacksonUtils.prettyPrint(combinedSchemaDefinitionsJson);
+    Files.write(Paths.get(Paths.get(outputDirectory) + sep + "combined" + sep + "schema.json"),
+        prettySchema.getBytes(StandardCharsets.UTF_8), StandardOpenOption.WRITE, StandardOpenOption.CREATE,
+        StandardOpenOption.TRUNCATE_EXISTING);
+
+  }
+
+  private final HashSet<String> filenames = new HashSet<>();
+  private void generateSchema(final File file) {
+    final String fileBaseName;
+    try {
+      final JsonNode schema = JsonLoader.fromFile(file);
+      final JsonNode result = buildResult(schema.toString());
+      String prettySchema = JacksonUtils.prettyPrint(result);
+      Path absolutePath = file.getAbsoluteFile().toPath();
+      if (absolutePath.endsWith(Paths.get("com", "linkedin", "metadata", "aspect", "EnvelopedAspect.avsc"))) {
+        fileBaseName = "EnvelopedTimeseriesAspect";
+        prettySchema = prettySchema.replaceAll("EnvelopedAspect", "EnvelopedTimeseriesAspect");
+      } else if (!filenames.add(file.getName())) {
+        System.out.println("Not processing legacy schema " + absolutePath);
+        return;
+      } else {
+        fileBaseName = getBaseName(file.getName());
+      }
+      Files.write(Paths.get(jsonDirectory + sep + fileBaseName + ".json"),
+          prettySchema.getBytes(StandardCharsets.UTF_8), StandardOpenOption.WRITE, StandardOpenOption.CREATE,
+          StandardOpenOption.TRUNCATE_EXISTING);
+      if (schema.has("Aspect")) {
+        aspectType.add(NODE_FACTORY.objectNode().put("$ref", "#/definitions/" + getBaseName(file.getName())));
+      }
+    } catch (IOException | ProcessingException e) {
+      throw new RuntimeException(e);
+    }
+  }
+
+}
diff --git a/docs-website/sidebars.js b/docs-website/sidebars.js
@@ -203,6 +203,13 @@ module.exports = {
         ],
       },
     ],
+    OpenAPI: [
+      {
+        label: "Usage Guide",
+        type: "doc",
+        id: "docs/api/openapi/openapi-usage-guide",
+      },
+    ],
     "Usage Guides": [
       "docs/policies",
       "docs/domains",