-
Notifications
You must be signed in to change notification settings - Fork 3k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat(bootstrap): bootstrap template mcps (#11518)
- Loading branch information
1 parent
73d8a46
commit 04349cb
Showing
58 changed files
with
1,969 additions
and
1,585 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
30 changes: 30 additions & 0 deletions
30
datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/BootstrapMCPConfig.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,30 @@ | ||
| package com.linkedin.datahub.upgrade.config; | ||
|
|
||
| import com.linkedin.datahub.upgrade.system.bootstrapmcps.BootstrapMCP; | ||
| import com.linkedin.metadata.entity.EntityService; | ||
| import io.datahubproject.metadata.context.OperationContext; | ||
| import java.io.IOException; | ||
| import javax.annotation.Nonnull; | ||
| import org.springframework.beans.factory.annotation.Value; | ||
| import org.springframework.context.annotation.Bean; | ||
| import org.springframework.context.annotation.Configuration; | ||
|
|
||
| @Configuration | ||
| public class BootstrapMCPConfig { | ||
|
|
||
| @Nonnull | ||
| @Value("${systemUpdate.bootstrap.mcpConfig}") | ||
| private String bootstrapMCPConfig; | ||
|
|
||
| @Bean(name = "bootstrapMCPNonBlocking") | ||
| public BootstrapMCP bootstrapMCPNonBlocking( | ||
| final OperationContext opContext, EntityService<?> entityService) throws IOException { | ||
| return new BootstrapMCP(opContext, bootstrapMCPConfig, entityService, false); | ||
| } | ||
|
|
||
| @Bean(name = "bootstrapMCPBlocking") | ||
| public BootstrapMCP bootstrapMCPBlocking( | ||
| final OperationContext opContext, EntityService<?> entityService) throws IOException { | ||
| return new BootstrapMCP(opContext, bootstrapMCPConfig, entityService, true); | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
8 changes: 4 additions & 4 deletions
8
datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/SystemUpdateBlocking.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,16 +1,16 @@ | ||
| package com.linkedin.datahub.upgrade.system; | ||
|
|
||
| import com.linkedin.datahub.upgrade.system.bootstrapmcps.BootstrapMCP; | ||
| import com.linkedin.datahub.upgrade.system.elasticsearch.steps.DataHubStartupStep; | ||
| import java.util.List; | ||
| import lombok.NonNull; | ||
| import org.jetbrains.annotations.Nullable; | ||
|
|
||
| public class SystemUpdateBlocking extends SystemUpdate { | ||
|
|
||
| public SystemUpdateBlocking( | ||
| @NonNull List<BlockingSystemUpgrade> blockingSystemUpgrades, | ||
| @NonNull List<NonBlockingSystemUpgrade> nonBlockingSystemUpgrades, | ||
| @Nullable DataHubStartupStep dataHubStartupStep) { | ||
| super(blockingSystemUpgrades, nonBlockingSystemUpgrades, dataHubStartupStep); | ||
| @NonNull DataHubStartupStep dataHubStartupStep, | ||
| @NonNull final BootstrapMCP bootstrapMCPBlocking) { | ||
| super(blockingSystemUpgrades, List.of(), dataHubStartupStep, bootstrapMCPBlocking, null); | ||
| } | ||
| } |
8 changes: 3 additions & 5 deletions
8
...ub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/SystemUpdateNonBlocking.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,16 +1,14 @@ | ||
| package com.linkedin.datahub.upgrade.system; | ||
|
|
||
| import com.linkedin.datahub.upgrade.system.elasticsearch.steps.DataHubStartupStep; | ||
| import com.linkedin.datahub.upgrade.system.bootstrapmcps.BootstrapMCP; | ||
| import java.util.List; | ||
| import lombok.NonNull; | ||
| import org.jetbrains.annotations.Nullable; | ||
|
|
||
| public class SystemUpdateNonBlocking extends SystemUpdate { | ||
|
|
||
| public SystemUpdateNonBlocking( | ||
| @NonNull List<BlockingSystemUpgrade> blockingSystemUpgrades, | ||
| @NonNull List<NonBlockingSystemUpgrade> nonBlockingSystemUpgrades, | ||
| @Nullable DataHubStartupStep dataHubStartupStep) { | ||
| super(blockingSystemUpgrades, nonBlockingSystemUpgrades, dataHubStartupStep); | ||
| final BootstrapMCP bootstrapMCPNonBlocking) { | ||
| super(List.of(), nonBlockingSystemUpgrades, null, null, bootstrapMCPNonBlocking); | ||
| } | ||
| } |
38 changes: 38 additions & 0 deletions
38
...upgrade/src/main/java/com/linkedin/datahub/upgrade/system/bootstrapmcps/BootstrapMCP.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,38 @@ | ||
| package com.linkedin.datahub.upgrade.system.bootstrapmcps; | ||
|
|
||
| import com.google.common.collect.ImmutableList; | ||
| import com.linkedin.datahub.upgrade.Upgrade; | ||
| import com.linkedin.datahub.upgrade.UpgradeStep; | ||
| import com.linkedin.metadata.entity.EntityService; | ||
| import io.datahubproject.metadata.context.OperationContext; | ||
| import java.io.IOException; | ||
| import java.util.List; | ||
| import javax.annotation.Nullable; | ||
|
|
||
| public class BootstrapMCP implements Upgrade { | ||
| private final List<UpgradeStep> _steps; | ||
|
|
||
| public BootstrapMCP( | ||
| OperationContext opContext, | ||
| @Nullable String bootstrapMCPConfig, | ||
| EntityService<?> entityService, | ||
| boolean isBlocking) | ||
| throws IOException { | ||
| if (bootstrapMCPConfig != null && !bootstrapMCPConfig.isEmpty()) { | ||
| _steps = | ||
| BootstrapMCPUtil.generateSteps(opContext, isBlocking, bootstrapMCPConfig, entityService); | ||
| } else { | ||
| _steps = ImmutableList.of(); | ||
| } | ||
| } | ||
|
|
||
| @Override | ||
| public String id() { | ||
| return getClass().getSimpleName(); | ||
| } | ||
|
|
||
| @Override | ||
| public List<UpgradeStep> steps() { | ||
| return _steps; | ||
| } | ||
| } |
95 changes: 95 additions & 0 deletions
95
...ade/src/main/java/com/linkedin/datahub/upgrade/system/bootstrapmcps/BootstrapMCPStep.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,95 @@ | ||
| package com.linkedin.datahub.upgrade.system.bootstrapmcps; | ||
|
|
||
| import static com.linkedin.metadata.Constants.DATA_HUB_UPGRADE_RESULT_ASPECT_NAME; | ||
|
|
||
| import com.linkedin.common.urn.Urn; | ||
| import com.linkedin.datahub.upgrade.UpgradeContext; | ||
| import com.linkedin.datahub.upgrade.UpgradeStep; | ||
| import com.linkedin.datahub.upgrade.UpgradeStepResult; | ||
| import com.linkedin.datahub.upgrade.impl.DefaultUpgradeStepResult; | ||
| import com.linkedin.datahub.upgrade.system.bootstrapmcps.model.BootstrapMCPConfigFile; | ||
| import com.linkedin.metadata.aspect.batch.AspectsBatch; | ||
| import com.linkedin.metadata.boot.BootstrapStep; | ||
| import com.linkedin.metadata.entity.EntityService; | ||
| import com.linkedin.upgrade.DataHubUpgradeState; | ||
| import io.datahubproject.metadata.context.OperationContext; | ||
| import java.io.IOException; | ||
| import java.util.List; | ||
| import java.util.function.Function; | ||
| import lombok.Getter; | ||
| import lombok.extern.slf4j.Slf4j; | ||
|
|
||
| /** | ||
| * This bootstrap step is responsible for upgrading DataHub policy documents with new searchable | ||
| * fields in ES | ||
| */ | ||
| @Slf4j | ||
| public class BootstrapMCPStep implements UpgradeStep { | ||
| private final String upgradeId; | ||
| private final Urn upgradeIdUrn; | ||
|
|
||
| private final OperationContext opContext; | ||
| private final EntityService<?> entityService; | ||
| @Getter private final BootstrapMCPConfigFile.MCPTemplate mcpTemplate; | ||
|
|
||
| public BootstrapMCPStep( | ||
| OperationContext opContext, | ||
| EntityService<?> entityService, | ||
| BootstrapMCPConfigFile.MCPTemplate mcpTemplate) { | ||
| this.opContext = opContext; | ||
| this.entityService = entityService; | ||
| this.mcpTemplate = mcpTemplate; | ||
| this.upgradeId = | ||
| String.join("-", List.of("bootstrap", mcpTemplate.getName(), mcpTemplate.getVersion())); | ||
| this.upgradeIdUrn = BootstrapStep.getUpgradeUrn(this.upgradeId); | ||
| } | ||
|
|
||
| @Override | ||
| public String id() { | ||
| return upgradeId; | ||
| } | ||
|
|
||
| @Override | ||
| public Function<UpgradeContext, UpgradeStepResult> executable() { | ||
| return (context) -> { | ||
| try { | ||
| AspectsBatch batch = BootstrapMCPUtil.generateAspectBatch(opContext, mcpTemplate); | ||
| log.info("Ingesting {} MCPs", batch.getItems().size()); | ||
| entityService.ingestProposal(opContext, batch, mcpTemplate.isAsync()); | ||
| } catch (IOException e) { | ||
| log.error("Error bootstrapping MCPs", e); | ||
| return new DefaultUpgradeStepResult(id(), DataHubUpgradeState.FAILED); | ||
| } | ||
|
|
||
| BootstrapStep.setUpgradeResult(context.opContext(), upgradeIdUrn, entityService); | ||
|
|
||
| return new DefaultUpgradeStepResult(id(), DataHubUpgradeState.SUCCEEDED); | ||
| }; | ||
| } | ||
|
|
||
| /** | ||
| * Returns whether the upgrade should proceed if the step fails after exceeding the maximum | ||
| * retries. | ||
| */ | ||
| @Override | ||
| public boolean isOptional() { | ||
| return mcpTemplate.isOptional(); | ||
| } | ||
|
|
||
| /** Returns whether the upgrade should be skipped. */ | ||
| @Override | ||
| public boolean skip(UpgradeContext context) { | ||
| if (!mcpTemplate.isForce()) { | ||
| boolean previouslyRun = | ||
| entityService.exists( | ||
| context.opContext(), upgradeIdUrn, DATA_HUB_UPGRADE_RESULT_ASPECT_NAME, true); | ||
| if (previouslyRun) { | ||
| log.info("{} was already run. Skipping.", id()); | ||
| } | ||
| return previouslyRun; | ||
| } else { | ||
| log.info("{} forced run.", id()); | ||
| return false; | ||
| } | ||
| } | ||
| } |
Oops, something went wrong.