feat(patch): initial support of json patch style semantics in MCPs (#…

…5901) Co-authored-by: Shirshanka Das <[email protected]>
datahub-project · Sep 23, 2022 · 0728656 · 0728656
1 parent d53cbbb
commit 0728656
Show file tree

Hide file tree

Showing 59 changed files with 2,549 additions and 92 deletions.
diff --git a/.gitignore b/.gitignore
@@ -68,7 +68,7 @@ metadata-ingestion/generated/**
 # docs
 docs/generated/
 tmp*
-temp*
+temp/**
 
 # frontend assets
 datahub-frontend/public/**

diff --git a/build.gradle b/build.gradle
@@ -90,6 +90,7 @@ project.ext.externalDependency = [
     'jerseyGuava': 'org.glassfish.jersey.bundles.repackaged:jersey-guava:2.25.1',
     'jettyJaas': 'org.eclipse.jetty:jetty-jaas:9.4.46.v20220331',
     'jgrapht': 'org.jgrapht:jgrapht-core:1.5.1',
+    'jsonPatch': 'com.github.java-json-tools:json-patch:1.13',
     'jsonSchemaAvro': 'com.github.fge:json-schema-avro:0.1.4',
     'jsonSimple': 'com.googlecode.json-simple:json-simple:1.1.1',
     'jsonSmart': 'net.minidev:json-smart:2.4.6',

diff --git a/entity-registry/build.gradle b/entity-registry/build.gradle
@@ -9,6 +9,7 @@ dependencies {
   compile externalDependency.jacksonDataBind
   compile externalDependency.jacksonDataFormatYaml
   compile externalDependency.reflections
+  compile externalDependency.jsonPatch
   dataModel project(':li-utils')
   annotationProcessor externalDependency.lombok
 

diff --git a/entity-registry/src/main/java/com/linkedin/metadata/models/AspectSpec.java b/entity-registry/src/main/java/com/linkedin/metadata/models/AspectSpec.java
@@ -9,11 +9,12 @@
 import java.util.Map;
 import java.util.stream.Collectors;
 import javax.annotation.Nonnull;
+import lombok.EqualsAndHashCode;
 import lombok.Getter;
 import lombok.Setter;
 import org.apache.maven.artifact.versioning.ComparableVersion;
 
-
+@EqualsAndHashCode
 public class AspectSpec {
 
   private final AspectAnnotation _aspectAnnotation;

diff --git a/...ty-registry/src/main/java/com/linkedin/metadata/models/registry/ConfigEntityRegistry.java b/...ty-registry/src/main/java/com/linkedin/metadata/models/registry/ConfigEntityRegistry.java
@@ -13,6 +13,7 @@
 import com.linkedin.metadata.models.registry.config.Entities;
 import com.linkedin.metadata.models.registry.config.Entity;
 import com.linkedin.metadata.models.registry.config.Event;
+import com.linkedin.metadata.models.registry.template.AspectTemplateEngine;
 import com.linkedin.util.Pair;
 import java.io.FileInputStream;
 import java.io.FileNotFoundException;
@@ -30,6 +31,8 @@
 import javax.annotation.Nonnull;
 import lombok.extern.slf4j.Slf4j;
 
+import static com.linkedin.metadata.models.registry.EntityRegistryUtils.*;
+
 
 /**
  * Implementation of {@link EntityRegistry} that builds {@link DefaultEntitySpec} objects
@@ -43,6 +46,7 @@ public class ConfigEntityRegistry implements EntityRegistry {
   private final Map<String, EventSpec> eventNameToSpec;
   private final List<EntitySpec> entitySpecs;
   private final String identifier;
+  private final Map<String, AspectSpec> _aspectNameToSpec;
 
   private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(new YAMLFactory());
 
@@ -131,6 +135,7 @@ public ConfigEntityRegistry(DataSchemaFactory dataSchemaFactory, InputStream con
       }
     }
     entitySpecs = new ArrayList<>(entityNameToSpec.values());
+    _aspectNameToSpec = populateAspectMap(entitySpecs);
   }
 
   @Override
@@ -183,9 +188,23 @@ public Map<String, EntitySpec> getEntitySpecs() {
     return entityNameToSpec;
   }
 
+  @Nonnull
+  @Override
+  public Map<String, AspectSpec> getAspectSpecs() {
+    return _aspectNameToSpec;
+  }
+
   @Nonnull
   @Override
   public Map<String, EventSpec> getEventSpecs() {
     return eventNameToSpec;
   }
+
+  @Nonnull
+  @Override
+  public AspectTemplateEngine getAspectTemplateEngine() {
+
+    //TODO: add support for config based aspect templates
+    return new AspectTemplateEngine();
+  }
 }
diff --git a/entity-registry/src/main/java/com/linkedin/metadata/models/registry/EntityRegistry.java b/entity-registry/src/main/java/com/linkedin/metadata/models/registry/EntityRegistry.java
@@ -1,8 +1,10 @@
 package com.linkedin.metadata.models.registry;
 
+import com.linkedin.metadata.models.AspectSpec;
 import com.linkedin.metadata.models.DefaultEntitySpec;
 import com.linkedin.metadata.models.EntitySpec;
 import com.linkedin.metadata.models.EventSpec;
+import com.linkedin.metadata.models.registry.template.AspectTemplateEngine;
 import java.util.Map;
 import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
@@ -37,15 +39,30 @@ default String getIdentifier() {
   EventSpec getEventSpec(@Nonnull final String eventName);
 
   /**
-   * Returns all {@link DefaultEntitySpec}s that the register is aware of.
-   * @return a list of {@link DefaultEntitySpec}s, empty list if none exists.
+   * Returns all {@link DefaultEntitySpec}s that the registry is aware of.
+   * @return a map of String to {@link DefaultEntitySpec}s, empty map if none exists.
    */
   @Nonnull
   Map<String, EntitySpec> getEntitySpecs();
 
+
+  /**
+   * Returns all {@link AspectSpec}s that the registry is aware of.
+   * @return a map of String to {@link AspectSpec}s, empty map if none exists.
+   */
+  @Nonnull
+  Map<String, AspectSpec> getAspectSpecs();
+
   /**
    * Returns all {@link EventSpec}s that the registry is aware of.
    */
   @Nonnull
   Map<String, EventSpec> getEventSpecs();
+
+  /**
+   * Returns an {@link AspectTemplateEngine} that is used for generating templates from {@link com.linkedin.metadata.models.AspectSpec}s
+   * @return a template engine instance associated with this registry
+   */
+  @Nonnull
+  AspectTemplateEngine getAspectTemplateEngine();
 }
diff --git a/entity-registry/src/main/java/com/linkedin/metadata/models/registry/EntityRegistryUtils.java b/entity-registry/src/main/java/com/linkedin/metadata/models/registry/EntityRegistryUtils.java
@@ -0,0 +1,24 @@
+package com.linkedin.metadata.models.registry;
+
+import com.linkedin.metadata.models.AspectSpec;
+import com.linkedin.metadata.models.EntitySpec;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+
+
+public class EntityRegistryUtils {
+  private EntityRegistryUtils() {
+
+  }
+
+  public static Map<String, AspectSpec> populateAspectMap(List<EntitySpec> entitySpecs) {
+    return entitySpecs.stream()
+        .map(EntitySpec::getAspectSpecs)
+        .flatMap(Collection::stream)
+        .collect(Collectors.toMap(AspectSpec::getName, Function.identity(), (aspectSpec1, aspectSpec2) -> aspectSpec1));
+  }
+
+}
diff --git a/...ty-registry/src/main/java/com/linkedin/metadata/models/registry/MergedEntityRegistry.java b/...ty-registry/src/main/java/com/linkedin/metadata/models/registry/MergedEntityRegistry.java
@@ -7,6 +7,7 @@
 import com.linkedin.metadata.models.DefaultEntitySpec;
 import com.linkedin.metadata.models.EntitySpec;
 import com.linkedin.metadata.models.EventSpec;
+import com.linkedin.metadata.models.registry.template.AspectTemplateEngine;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
@@ -26,10 +27,15 @@ public class MergedEntityRegistry implements EntityRegistry {
 
   private final Map<String, EntitySpec> entityNameToSpec;
   private final Map<String, EventSpec> eventNameToSpec;
+  private final AspectTemplateEngine _aspectTemplateEngine;
+  private final Map<String, AspectSpec> _aspectNameToSpec;
 
   public MergedEntityRegistry(EntityRegistry baseEntityRegistry) {
     entityNameToSpec = baseEntityRegistry.getEntitySpecs() != null ? baseEntityRegistry.getEntitySpecs() : new HashMap<>();
     eventNameToSpec = baseEntityRegistry.getEventSpecs() != null ? baseEntityRegistry.getEventSpecs() : new HashMap<>();
+    baseEntityRegistry.getAspectTemplateEngine();
+    _aspectTemplateEngine = baseEntityRegistry.getAspectTemplateEngine();
+    _aspectNameToSpec = baseEntityRegistry.getAspectSpecs();
   }
 
   private void validateEntitySpec(EntitySpec entitySpec, final ValidationResult validationResult) {
@@ -132,12 +138,24 @@ public Map<String, EntitySpec> getEntitySpecs() {
     return entityNameToSpec;
   }
 
+  @Nonnull
+  @Override
+  public Map<String, AspectSpec> getAspectSpecs() {
+    return _aspectNameToSpec;
+  }
+
   @Nonnull
   @Override
   public Map<String, EventSpec> getEventSpecs() {
     return eventNameToSpec;
   }
 
+  @Nonnull
+  @Override
+  public AspectTemplateEngine getAspectTemplateEngine() {
+    return _aspectTemplateEngine;
+  }
+
   @Setter
   @Getter
   private class ValidationResult {

diff --git a/entity-registry/src/main/java/com/linkedin/metadata/models/registry/PatchEntityRegistry.java b/entity-registry/src/main/java/com/linkedin/metadata/models/registry/PatchEntityRegistry.java
@@ -12,6 +12,7 @@
 import com.linkedin.metadata.models.registry.config.Entities;
 import com.linkedin.metadata.models.registry.config.Entity;
 import com.linkedin.metadata.models.registry.config.Event;
+import com.linkedin.metadata.models.registry.template.AspectTemplateEngine;
 import com.linkedin.util.Pair;
 import java.io.FileInputStream;
 import java.io.FileNotFoundException;
@@ -30,6 +31,8 @@
 import lombok.extern.slf4j.Slf4j;
 import org.apache.maven.artifact.versioning.ComparableVersion;
 
+import static com.linkedin.metadata.models.registry.EntityRegistryUtils.*;
+
 
 /**
  * Implementation of {@link EntityRegistry} that is similar to {@link ConfigEntityRegistry} but different in one important way.
@@ -41,6 +44,7 @@ public class PatchEntityRegistry implements EntityRegistry {
   private final DataSchemaFactory dataSchemaFactory;
   private final Map<String, EntitySpec> entityNameToSpec;
   private final Map<String, EventSpec> eventNameToSpec;
+  private final Map<String, AspectSpec> _aspectNameToSpec;
 
   private final String registryName;
   private final ComparableVersion registryVersion;
@@ -164,6 +168,7 @@ private PatchEntityRegistry(DataSchemaFactory dataSchemaFactory, InputStream con
         eventNameToSpec.put(event.getName().toLowerCase(), eventSpec);
       }
     }
+    _aspectNameToSpec = populateAspectMap(new ArrayList<>(entityNameToSpec.values()));
   }
 
   @Override
@@ -199,12 +204,26 @@ public Map<String, EntitySpec> getEntitySpecs() {
     return entityNameToSpec;
   }
 
+  @Nonnull
+  @Override
+  public Map<String, AspectSpec> getAspectSpecs() {
+    return _aspectNameToSpec;
+  }
+
   @Nonnull
   @Override
   public Map<String, EventSpec> getEventSpecs() {
     return eventNameToSpec;
   }
 
+  @Nonnull
+  @Override
+  public AspectTemplateEngine getAspectTemplateEngine() {
+    //TODO: support patch based templates
+
+    return new AspectTemplateEngine();
+  }
+
   private AspectSpec buildAspectSpec(String aspectName, EntitySpecBuilder entitySpecBuilder) {
     Optional<DataSchema> aspectSchema = dataSchemaFactory.getAspectSchema(aspectName);
     Optional<Class> aspectClass = dataSchemaFactory.getAspectClass(aspectName);

diff --git a/...-registry/src/main/java/com/linkedin/metadata/models/registry/SnapshotEntityRegistry.java b/...-registry/src/main/java/com/linkedin/metadata/models/registry/SnapshotEntityRegistry.java
@@ -1,10 +1,20 @@
 package com.linkedin.metadata.models.registry;
 
+import com.linkedin.data.template.RecordTemplate;
 import com.linkedin.data.template.UnionTemplate;
+import com.linkedin.metadata.models.AspectSpec;
 import com.linkedin.metadata.models.DefaultEntitySpec;
 import com.linkedin.metadata.models.EntitySpec;
 import com.linkedin.metadata.models.EntitySpecBuilder;
 import com.linkedin.metadata.models.EventSpec;
+import com.linkedin.metadata.models.registry.template.AspectTemplateEngine;
+import com.linkedin.metadata.models.registry.template.Template;
+import com.linkedin.metadata.models.registry.template.common.GlobalTagsTemplate;
+import com.linkedin.metadata.models.registry.template.common.GlossaryTermsTemplate;
+import com.linkedin.metadata.models.registry.template.common.OwnershipTemplate;
+import com.linkedin.metadata.models.registry.template.dataset.DatasetPropertiesTemplate;
+import com.linkedin.metadata.models.registry.template.dataset.EditableSchemaMetadataTemplate;
+import com.linkedin.metadata.models.registry.template.dataset.UpstreamLineageTemplate;
 import com.linkedin.metadata.snapshot.Snapshot;
 import java.util.ArrayList;
 import java.util.HashMap;
@@ -13,6 +23,9 @@
 import java.util.stream.Collectors;
 import javax.annotation.Nonnull;
 
+import static com.linkedin.metadata.Constants.*;
+import static com.linkedin.metadata.models.registry.EntityRegistryUtils.*;
+
 
 /**
  * Implementation of {@link EntityRegistry} that builds {@link DefaultEntitySpec} objects
@@ -22,6 +35,8 @@ public class SnapshotEntityRegistry implements EntityRegistry {
 
   private final Map<String, EntitySpec> entityNameToSpec;
   private final List<EntitySpec> entitySpecs;
+  private final AspectTemplateEngine _aspectTemplateEngine;
+  private final Map<String, AspectSpec> _aspectNameToSpec;
 
   private static final SnapshotEntityRegistry INSTANCE = new SnapshotEntityRegistry();
 
@@ -30,13 +45,30 @@ public SnapshotEntityRegistry() {
         .stream()
         .collect(Collectors.toMap(spec -> spec.getName().toLowerCase(), spec -> spec));
     entitySpecs = new ArrayList<>(entityNameToSpec.values());
+    _aspectNameToSpec = populateAspectMap(entitySpecs);
+    _aspectTemplateEngine = populateTemplateEngine(_aspectNameToSpec);
   }
 
   public SnapshotEntityRegistry(UnionTemplate snapshot) {
     entityNameToSpec = new EntitySpecBuilder().buildEntitySpecs(snapshot.schema())
         .stream()
         .collect(Collectors.toMap(spec -> spec.getName().toLowerCase(), spec -> spec));
     entitySpecs = new ArrayList<>(entityNameToSpec.values());
+    _aspectNameToSpec = populateAspectMap(entitySpecs);
+    _aspectTemplateEngine = populateTemplateEngine(_aspectNameToSpec);
+  }
+
+  private AspectTemplateEngine populateTemplateEngine(Map<String, AspectSpec> aspectSpecs) {
+    // TODO: This should be more dynamic ideally, "hardcoding" for now, passing in aspect spec map preemptively
+
+    Map<String, Template<? extends RecordTemplate>> aspectSpecTemplateMap = new HashMap<>();
+    aspectSpecTemplateMap.put(OWNERSHIP_ASPECT_NAME, new OwnershipTemplate());
+    aspectSpecTemplateMap.put(DATASET_PROPERTIES_ASPECT_NAME, new DatasetPropertiesTemplate());
+    aspectSpecTemplateMap.put(UPSTREAM_LINEAGE_ASPECT_NAME, new UpstreamLineageTemplate());
+    aspectSpecTemplateMap.put(GLOBAL_TAGS_ASPECT_NAME, new GlobalTagsTemplate());
+    aspectSpecTemplateMap.put(EDITABLE_SCHEMA_METADATA_ASPECT_NAME, new EditableSchemaMetadataTemplate());
+    aspectSpecTemplateMap.put(GLOSSARY_TERMS_ASPECT_NAME, new GlossaryTermsTemplate());
+    return new AspectTemplateEngine(aspectSpecTemplateMap);
   }
 
   @Nonnull
@@ -62,6 +94,12 @@ public Map<String, EventSpec> getEventSpecs() {
     return new HashMap<>();
   }
 
+  @Nonnull
+  @Override
+  public AspectTemplateEngine getAspectTemplateEngine() {
+    return _aspectTemplateEngine;
+  }
+
   @Override
   public EventSpec getEventSpec(final String ignored) {
     return null;
@@ -70,4 +108,10 @@ public EventSpec getEventSpec(final String ignored) {
   public static SnapshotEntityRegistry getInstance() {
     return INSTANCE;
   }
+
+  @Override
+  @Nonnull
+  public Map<String, AspectSpec> getAspectSpecs() {
+    return _aspectNameToSpec;
+  }
 }
-Original file line number
+Diff line change
@@ Expand Up / @@ -68,7 +68,7 @@ metadata-ingestion/generated/** @@
     # docs
     docs/generated/
     tmp*
-    temp*
+    temp/**
     # frontend assets
     datahub-frontend/public/**
@@ Expand Down @@