[scsi] preserve the order of urns

jywadhwani · John Plaisted · commit dc9c87798411 · 2020-09-24T16:02:12.000-07:00
diff --git a/metadata-dao-impl/ebean-dao/src/main/java/com/linkedin/metadata/dao/EbeanLocalDAO.java b/metadata-dao-impl/ebean-dao/src/main/java/com/linkedin/metadata/dao/EbeanLocalDAO.java
@@ -742,7 +742,7 @@ void addEntityTypeFilter(@Nonnull IndexFilter indexFilter) {
   /**
    * Returns list of urns from strongly consistent secondary index that satisfy the given filter conditions.
    *
-   * <p>Results are sorted in increasing alphabetical order of urn.
+   * <p>Results are ordered lexicographically by the string representation of the URN.
    *
    * <p>NOTE: Currently this works for upto 10 filter conditions.
    *
diff --git a/metadata-dao/src/main/java/com/linkedin/metadata/dao/BaseLocalDAO.java b/metadata-dao/src/main/java/com/linkedin/metadata/dao/BaseLocalDAO.java
@@ -29,6 +29,7 @@
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.LinkedHashMap;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
@@ -384,6 +385,8 @@ protected abstract <ASPECT extends RecordTemplate> void updateLocalIndex(@Nonnul
   /**
    * Returns list of urns from local secondary index that satisfy the given filter conditions.
    *
+   * <p>Results are ordered lexicographically by the string representation of the URN.
+   *
    * @param indexFilter {@link IndexFilter} containing filter conditions to be applied
    * @param lastUrn last urn of the previous fetched page. For the first page, this should be set as NULL
    * @param pageSize maximum number of distinct urns to return
@@ -404,13 +407,15 @@ public ListResult<URN> listUrns(@Nonnull Class<URN> urnClazz, @Nullable URN last
 
   /**
    * Retrieves {@link ListResult} of {@link UrnAspectEntry} containing latest version of aspects along with the urn for the list of urns
-   * returned from local secondary index that satisfy given filter conditions.
+   * returned from local secondary index that satisfy given filter conditions. The returned list is ordered lexicographically by the string
+   * representation of the URN.
    *
    * @param aspectClasses aspect classes whose latest versions need to be retrieved
    * @param indexFilter {@link IndexFilter} containing filter conditions to be applied
    * @param lastUrn last urn of the previous fetched page. For the first page, this should be set as NULL
    * @param pageSize maximum number of distinct urns whose aspects need to be retrieved
-   * @return {@link ListResult} containing latest versions of aspects along with urns returned from local secondary index satisfying given filter conditions
+   * @return {@link ListResult} containing ordered list of latest versions of aspects along with urns returned from local secondary index
+   *        satisfying given filter conditions
    */
   @Nonnull
   public ListResult<UrnAspectEntry<URN>> getAspects(@Nonnull Set<Class<? extends RecordTemplate>> aspectClasses,
@@ -420,17 +425,17 @@ public ListResult<UrnAspectEntry<URN>> getAspects(@Nonnull Set<Class<? extends R
     final Map<URN, Map<Class<? extends RecordTemplate>, Optional<? extends RecordTemplate>>> urnAspectMap =
         get(aspectClasses, new HashSet<>(urns.getValues()));
 
-    final Map<URN, List<RecordTemplate>> urnListAspectMap = new HashMap<>();
-    for (Map.Entry<URN, Map<Class<? extends RecordTemplate>, Optional<? extends RecordTemplate>>> entry : urnAspectMap.entrySet()) {
-      final Map<Class<? extends RecordTemplate>, Optional<? extends RecordTemplate>> aspectMap = entry.getValue();
-      urnListAspectMap.compute(entry.getKey(), (k, v) -> {
+    final Map<URN, List<RecordTemplate>> urnListAspectMap = new LinkedHashMap<>();
+    for (URN urn : urns.getValues()) {
+      final Map<Class<? extends RecordTemplate>, Optional<? extends RecordTemplate>> aspectMap = urnAspectMap.get(urn);
+      urnListAspectMap.compute(urn, (k, v) -> {
         if (v == null) {
           v = new ArrayList<>();
         }
         return v;
       });
       for (Optional<? extends RecordTemplate> aspect : aspectMap.values()) {
-        aspect.ifPresent(record -> urnListAspectMap.get(entry.getKey()).add(record));
+        aspect.ifPresent(record -> urnListAspectMap.get(urn).add(record));
       }
     }
 
diff --git a/metadata-restli-resource/src/main/java/com/linkedin/metadata/restli/BaseEntityResource.java b/metadata-restli-resource/src/main/java/com/linkedin/metadata/restli/BaseEntityResource.java
@@ -36,7 +36,7 @@
 import java.util.Collection;
 import java.util.Collections;
 import java.util.Comparator;
-import java.util.HashMap;
+import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -341,14 +341,15 @@ public Task<String[]> listUrnsFromIndex(@ActionParam(PARAM_FILTER) @Optional @Nu
   }
 
   /**
-   * Returns {@link CollectionResult} containing the values of multiple entities obtained after filtering urns from local secondary index.
-   * {@link ListResultMetadata} contains relevant list of urns.
+   * Returns {@link CollectionResult} containing ordered list of values of multiple entities obtained after filtering urns
+   * from local secondary index. The returned list is ordered lexicographically by the string representation of the URN.
+   * The list of values is in the same order as the list of urns contained in {@link ListResultMetadata}.
    *
    * @param aspectClasses set of aspect classes that needs to be populated in the values
    * @param filter {@link IndexFilter} that defines the filter conditions
    * @param lastUrn last urn of the previous fetched page. For the first page, this should be set as NULL
    * @param pagingContext {@link PagingContext} defining the paging parameters of the request
-   * @return {@link CollectionResult} containing the values of multiple entities
+   * @return {@link CollectionResult} containing ordered list of values of multiple entities
    */
   @Nonnull
   private CollectionResult<VALUE, ListResultMetadata> filterAspects(
@@ -358,7 +359,7 @@ private CollectionResult<VALUE, ListResultMetadata> filterAspects(
     final ListResult<UrnAspectEntry<URN>> urnAspectEntries =
         getLocalDAO().getAspects(aspectClasses, filter, parseUrnParam(lastUrn), pagingContext.getCount());
 
-    final Map<URN, List<UnionTemplate>> urnAspectsMap = new HashMap<>();
+    final Map<URN, List<UnionTemplate>> urnAspectsMap = new LinkedHashMap<>();
     for (UrnAspectEntry<URN> entry : urnAspectEntries.getValues()) {
       urnAspectsMap.compute(entry.getUrn(), (k, v) -> {
         if (v == null) {
@@ -383,27 +384,27 @@ private CollectionResult<VALUE, ListResultMetadata> filterAspects(
   }
 
   /**
-   * Returns {@link CollectionResult} containing the values of multiple entities obtained after filtering urns from local secondary index.
-   * Here the value does not contain any metadata aspect, only parts of the urn (if applicable). {@link ListResultMetadata} contains relevant list of urns.
+   * Returns {@link CollectionResult} containing ordered list of values of multiple entities obtained after filtering urns
+   * from local secondary index. The returned list is ordered lexicographically by the string representation of the URN.
+   * The values returned do not contain any metadata aspect, only parts of the urn (if applicable).
+   * The list of values is in the same order as the list of urns contained in {@link ListResultMetadata}.
    *
    * @param filter {@link IndexFilter} that defines the filter conditions
    * @param lastUrn last urn of the previous fetched page
    * @param pagingContext {@link PagingContext} defining the paging parameters of the request
-   * @return {@link CollectionResult} containing the values of multiple entities
+   * @return {@link CollectionResult} containing ordered list of values of multiple entities
    */
   @Nonnull
   private CollectionResult<VALUE, ListResultMetadata> filterUrns(@Nonnull IndexFilter filter, @Nullable String lastUrn,
       @Nonnull PagingContext pagingContext) {
 
     final ListResult<URN> urns = getLocalDAO().listUrns(filter, parseUrnParam(lastUrn), pagingContext.getCount());
-    final Map<URN, VALUE> urnValueMap = urns.getValues()
-        .stream()
-        .distinct()
-        .collect(Collectors.toMap(Function.identity(), urn -> toValue(newSnapshot(urn))));
-    ListResultMetadata resultMetadata = new ListResultMetadata().setExtraInfos(new ExtraInfoArray(
-        urnValueMap.keySet().stream().map(urn -> new ExtraInfo().setUrn(urn)).collect(Collectors.toList())));
+    final List<VALUE> values =
+        urns.getValues().stream().map(urn -> toValue(newSnapshot(urn))).collect(Collectors.toList());
+    final ListResultMetadata resultMetadata = new ListResultMetadata().setExtraInfos(new ExtraInfoArray(
+        urns.getValues().stream().map(urn -> new ExtraInfo().setUrn(urn)).collect(Collectors.toList())));
 
-    return new CollectionResult<>(new ArrayList<>(urnValueMap.values()), urns.getTotalCount(), resultMetadata);
+    return new CollectionResult<>(new ArrayList<>(values), urns.getTotalCount(), resultMetadata);
   }
 
   /**

Original file line number	Diff line number	Diff line change
`@@ -742,7 +742,7 @@ void addEntityTypeFilter(@Nonnull IndexFilter indexFilter) {`
`742`	`742`	`/**`
`743`	`743`	`* Returns list of urns from strongly consistent secondary index that satisfy the given filter conditions.`
`744`	`744`	`*`
`745`		`- * <p>Results are sorted in increasing alphabetical order of urn.`
	`745`	`+ * <p>Results are ordered lexicographically by the string representation of the URN.`
`746`	`746`	`*`
`747`	`747`	`* <p>NOTE: Currently this works for upto 10 filter conditions.`
`748`	`748`	`*`