Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(ui): Supporting enriched search preview + misc improvements #5419

Merged
merged 3 commits into from
Jul 18, 2022
Merged

feat(ui): Supporting enriched search preview + misc improvements #5419

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
7157cef
Supporting Enriched Search Preview
jjoyce0510 Jul 14, 2022
04be692
Reverting unnecessary changes
jjoyce0510 Jul 18, 2022
2008640
Refactoring duplicate code
jjoyce0510 Jul 18, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 20 additions & 7 deletions datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,12 +30,14 @@
import com.linkedin.datahub.graphql.generated.CorpUserInfo;
import com.linkedin.datahub.graphql.generated.Dashboard;
import com.linkedin.datahub.graphql.generated.DashboardInfo;
import com.linkedin.datahub.graphql.generated.DashboardStatsSummary;
import com.linkedin.datahub.graphql.generated.DashboardUserUsageCounts;
import com.linkedin.datahub.graphql.generated.DataFlow;
import com.linkedin.datahub.graphql.generated.DataJob;
import com.linkedin.datahub.graphql.generated.DataJobInputOutput;
import com.linkedin.datahub.graphql.generated.DataPlatformInstance;
import com.linkedin.datahub.graphql.generated.Dataset;
import com.linkedin.datahub.graphql.generated.DatasetStatsSummary;
import com.linkedin.datahub.graphql.generated.Domain;
import com.linkedin.datahub.graphql.generated.EntityRelationship;
import com.linkedin.datahub.graphql.generated.EntityRelationshipLegacy;
Expand Down Expand Up @@ -83,7 +85,10 @@
import com.linkedin.datahub.graphql.resolvers.container.ContainerEntitiesResolver;
import com.linkedin.datahub.graphql.resolvers.container.ParentContainersResolver;
import com.linkedin.datahub.graphql.resolvers.dashboard.DashboardUsageStatsResolver;
import com.linkedin.datahub.graphql.resolvers.dashboard.DashboardStatsSummaryResolver;
import com.linkedin.datahub.graphql.resolvers.dataset.DatasetHealthResolver;
import com.linkedin.datahub.graphql.resolvers.dataset.DatasetUsageStatsResolver;
import com.linkedin.datahub.graphql.resolvers.dataset.DatasetStatsSummaryResolver;
import com.linkedin.datahub.graphql.resolvers.deprecation.UpdateDeprecationResolver;
import com.linkedin.datahub.graphql.resolvers.domain.CreateDomainResolver;
import com.linkedin.datahub.graphql.resolvers.domain.DeleteDomainResolver;
Expand Down Expand Up @@ -129,7 +134,6 @@
import com.linkedin.datahub.graphql.resolvers.load.LoadableTypeResolver;
import com.linkedin.datahub.graphql.resolvers.load.OwnerTypeResolver;
import com.linkedin.datahub.graphql.resolvers.load.TimeSeriesAspectResolver;
import com.linkedin.datahub.graphql.resolvers.load.UsageTypeResolver;
import com.linkedin.datahub.graphql.resolvers.mutate.AddLinkResolver;
import com.linkedin.datahub.graphql.resolvers.mutate.AddOwnerResolver;
import com.linkedin.datahub.graphql.resolvers.mutate.AddOwnersResolver;
Expand Down Expand Up @@ -210,7 +214,6 @@
import com.linkedin.datahub.graphql.types.notebook.NotebookType;
import com.linkedin.datahub.graphql.types.tag.TagType;
import com.linkedin.datahub.graphql.types.test.TestType;
import com.linkedin.datahub.graphql.types.usage.UsageType;
import com.linkedin.entity.client.EntityClient;
import com.linkedin.metadata.config.DatahubConfiguration;
import com.linkedin.metadata.config.IngestionConfiguration;
Expand Down Expand Up @@ -304,7 +307,6 @@ public class GmsGraphQLEngine {
private final GlossaryTermType glossaryTermType;
private final GlossaryNodeType glossaryNodeType;
private final AspectType aspectType;
private final UsageType usageType;
private final ContainerType containerType;
private final DomainType domainType;
private final NotebookType notebookType;
Expand Down Expand Up @@ -406,7 +408,6 @@ public GmsGraphQLEngine(
this.glossaryTermType = new GlossaryTermType(entityClient);
this.glossaryNodeType = new GlossaryNodeType(entityClient);
this.aspectType = new AspectType(entityClient);
this.usageType = new UsageType(this.usageClient);
this.containerType = new ContainerType(entityClient);
this.domainType = new DomainType(entityClient);
this.notebookType = new NotebookType(entityClient);
Expand Down Expand Up @@ -513,7 +514,6 @@ public GraphQLEngine.Builder builder() {
.addSchema(fileBasedSchema(TESTS_SCHEMA_FILE))
.addDataLoaders(loaderSuppliers(loadableTypes))
.addDataLoader("Aspect", context -> createDataLoader(aspectType, context))
.addDataLoader("UsageQueryResult", context -> createDataLoader(usageType, context))
.configureRuntimeWiring(this::configureRuntimeWiring);
}

Expand Down Expand Up @@ -848,7 +848,8 @@ private void configureDatasetResolvers(final RuntimeWiring.Builder builder) {
OperationMapper::map
)
)
.dataFetcher("usageStats", new UsageTypeResolver())
.dataFetcher("usageStats", new DatasetUsageStatsResolver(this.usageClient))
.dataFetcher("statsSummary", new DatasetStatsSummaryResolver(this.usageClient))
.dataFetcher("health", new DatasetHealthResolver(graphClient, timeseriesAspectService))
.dataFetcher("schemaMetadata", new AspectResolver())
.dataFetcher("assertions", new EntityAssertionsResolver(entityClient, graphClient))
Expand Down Expand Up @@ -881,8 +882,13 @@ private void configureDatasetResolvers(final RuntimeWiring.Builder builder) {
.type("InstitutionalMemoryMetadata", typeWiring -> typeWiring
.dataFetcher("author", new LoadableTypeResolver<>(corpUserType,
(env) -> ((InstitutionalMemoryMetadata) env.getSource()).getAuthor().getUrn()))
)
.type("DatasetStatsSummary", typeWiring -> typeWiring
.dataFetcher("topUsersLast30Days", new LoadableTypeBatchResolver<>(corpUserType,
(env) -> ((DatasetStatsSummary) env.getSource()).getTopUsersLast30Days().stream()
.map(CorpUser::getUrn)
.collect(Collectors.toList())))
);

}

/**
Expand Down Expand Up @@ -1018,6 +1024,7 @@ private void configureDashboardResolvers(final RuntimeWiring.Builder builder) {
)
.dataFetcher("parentContainers", new ParentContainersResolver(entityClient))
.dataFetcher("usageStats", new DashboardUsageStatsResolver(timeseriesAspectService))
.dataFetcher("statsSummary", new DashboardStatsSummaryResolver(timeseriesAspectService))
);
builder.type("DashboardInfo", typeWiring -> typeWiring
.dataFetcher("charts", new LoadableTypeBatchResolver<>(chartType,
Expand All @@ -1030,6 +1037,12 @@ private void configureDashboardResolvers(final RuntimeWiring.Builder builder) {
corpUserType,
(env) -> ((DashboardUserUsageCounts) env.getSource()).getUser().getUrn()))
);
builder.type("DashboardStatsSummary", typeWiring -> typeWiring
.dataFetcher("topUsersLast30Days", new LoadableTypeBatchResolver<>(corpUserType,
(env) -> ((DashboardStatsSummary) env.getSource()).getTopUsersLast30Days().stream()
.map(CorpUser::getUrn)
.collect(Collectors.toList())))
);
}

/**
Expand Down
101 changes: 101 additions & 0 deletions .../java/com/linkedin/datahub/graphql/resolvers/dashboard/DashboardStatsSummaryResolver.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,101 @@
package com.linkedin.datahub.graphql.resolvers.dashboard;

import com.google.common.cache.Cache;
import com.google.common.cache.CacheBuilder;
import com.linkedin.common.urn.Urn;
import com.linkedin.common.urn.UrnUtils;
import com.linkedin.datahub.graphql.generated.CorpUser;
import com.linkedin.datahub.graphql.generated.DashboardUsageMetrics;
import com.linkedin.datahub.graphql.generated.DashboardStatsSummary;
import com.linkedin.datahub.graphql.generated.DashboardUserUsageCounts;
import com.linkedin.datahub.graphql.generated.Entity;
import com.linkedin.metadata.query.filter.Filter;
import com.linkedin.metadata.timeseries.TimeseriesAspectService;
import graphql.schema.DataFetcher;
import graphql.schema.DataFetchingEnvironment;
import java.util.List;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;

import static com.linkedin.datahub.graphql.resolvers.dashboard.DashboardUsageStatsUtils.*;

@Slf4j
public class DashboardStatsSummaryResolver implements DataFetcher<CompletableFuture<DashboardStatsSummary>> {

// The maximum number of top users to show in the summary stats
private static final Integer MAX_TOP_USERS = 5;

private final TimeseriesAspectService timeseriesAspectService;
private final Cache<Urn, DashboardStatsSummary> summaryCache;

public DashboardStatsSummaryResolver(final TimeseriesAspectService timeseriesAspectService) {
this.timeseriesAspectService = timeseriesAspectService;
this.summaryCache = CacheBuilder.newBuilder()
.maximumSize(10000)
.expireAfterWrite(6, TimeUnit.HOURS) // TODO: Make caching duration configurable externally.
.build();
}

@Override
public CompletableFuture<DashboardStatsSummary> get(DataFetchingEnvironment environment) throws Exception {
final Urn resourceUrn = UrnUtils.getUrn(((Entity) environment.getSource()).getUrn());

return CompletableFuture.supplyAsync(() -> {

if (this.summaryCache.getIfPresent(resourceUrn) != null) {
return this.summaryCache.getIfPresent(resourceUrn);
}

try {

final DashboardStatsSummary result = new DashboardStatsSummary();

// Obtain total dashboard view count, by viewing the latest reported dashboard metrics.
List<DashboardUsageMetrics> dashboardUsageMetrics =
getDashboardUsageMetrics(resourceUrn.toString(), null, null, 1, this.timeseriesAspectService);
if (dashboardUsageMetrics.size() > 0) {
result.setViewCount(getDashboardViewCount(resourceUrn));
}

// Obtain unique user statistics, by rolling up unique users over the past month.
List<DashboardUserUsageCounts> userUsageCounts = getDashboardUsagePerUser(resourceUrn);
result.setUniqueUserCountLast30Days(userUsageCounts.size());
result.setTopUsersLast30Days(
trimUsers(userUsageCounts.stream().map(DashboardUserUsageCounts::getUser).collect(Collectors.toList())));

this.summaryCache.put(resourceUrn, result);
return result;

} catch (Exception e) {
log.error(String.format("Failed to load dashboard usage summary for resource %s", resourceUrn.toString()), e);
return null; // Do not throw when loading usage summary fails.
}
});
}

private int getDashboardViewCount(final Urn resourceUrn) {
List<DashboardUsageMetrics> dashboardUsageMetrics = getDashboardUsageMetrics(
resourceUrn.toString(),
null,
null,
1,
this.timeseriesAspectService);
return dashboardUsageMetrics.get(0).getViewsCount();
}

private List<DashboardUserUsageCounts> getDashboardUsagePerUser(final Urn resourceUrn) {
long now = System.currentTimeMillis();
long nowMinusOneMonth = timeMinusOneMonth(now);
Filter bucketStatsFilter = createUsageFilter(resourceUrn.toString(), nowMinusOneMonth, now, true);
return getUserUsageCounts(bucketStatsFilter, this.timeseriesAspectService);
}

private List<CorpUser> trimUsers(final List<CorpUser> originalUsers) {
if (originalUsers.size() > MAX_TOP_USERS) {
return originalUsers.subList(0, MAX_TOP_USERS);
}
return originalUsers;
}
}
Loading