Skip to content

Commit b3d3b63

Browse files
david-leifkerdavid-leifker
authored
feat(template-mcps): allow further control for helm (datahub-project#11816)
1 parent 33751bf commit b3d3b63

File tree

7 files changed

+75
-6
lines changed

7 files changed

+75
-6
lines changed

‎datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/bootstrapmcps/BootstrapMCPUtil.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,7 @@ static List<UpgradeStep> generateSteps(
5454
.getBootstrap()
5555
.getTemplates()
5656
.stream()
57+
.map(cfg -> cfg.withOverride(opContext.getObjectMapper()))
5758
.filter(cfg -> cfg.isBlocking() == isBlocking)
5859
.map(cfg -> new BootstrapMCPStep(opContext, entityService, cfg))
5960
.collect(Collectors.toList());

‎datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/bootstrapmcps/model/BootstrapMCPConfigFile.java‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,15 @@
11
package com.linkedin.datahub.upgrade.system.bootstrapmcps.model;
22

3+
import com.fasterxml.jackson.databind.ObjectMapper;
4+
import java.io.IOException;
35
import java.util.List;
46
import javax.annotation.Nonnull;
57
import javax.annotation.Nullable;
68
import lombok.AllArgsConstructor;
79
import lombok.Builder;
810
import lombok.Data;
911
import lombok.NoArgsConstructor;
12+
import lombok.extern.slf4j.Slf4j;
1013

1114
@AllArgsConstructor
1215
@NoArgsConstructor
@@ -23,6 +26,7 @@ public static class Bootstrap {
2326
private List<MCPTemplate> templates;
2427
}
2528

29+
@Slf4j
2630
@AllArgsConstructor
2731
@NoArgsConstructor
2832
@Data
@@ -36,5 +40,19 @@ public static class MCPTemplate {
3640
@Builder.Default private boolean optional = false;
3741
@Nonnull private String mcps_location;
3842
@Nullable private String values_env;
43+
@Nullable private String revision_env;
44+
45+
public MCPTemplate withOverride(ObjectMapper objectMapper) {
46+
if (revision_env != null) {
47+
String overrideJson = System.getenv().getOrDefault(revision_env, "{}");
48+
try {
49+
return objectMapper.readerForUpdating(this).readValue(overrideJson);
50+
} catch (IOException e) {
51+
log.error("Error applying override {} to {}", overrideJson, this);
52+
throw new RuntimeException(e);
53+
}
54+
}
55+
return this;
56+
}
3957
}
4058
}

‎datahub-upgrade/src/test/java/com/linkedin/datahub/upgrade/system/bootstrapmcps/BootstrapMCPUtilTest.java‎

Lines changed: 30 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
import static org.testng.Assert.assertFalse;
55
import static org.testng.Assert.assertTrue;
66

7+
import com.fasterxml.jackson.databind.ObjectMapper;
78
import com.fasterxml.jackson.databind.node.ObjectNode;
89
import com.linkedin.common.AuditStamp;
910
import com.linkedin.common.urn.UrnUtils;
@@ -17,6 +18,7 @@
1718
import io.datahubproject.test.metadata.context.TestOperationContexts;
1819
import java.io.IOException;
1920
import java.util.List;
21+
import org.testng.annotations.BeforeMethod;
2022
import org.testng.annotations.Listeners;
2123
import org.testng.annotations.Test;
2224
import uk.org.webcompere.systemstubs.environment.EnvironmentVariables;
@@ -28,10 +30,17 @@ public class BootstrapMCPUtilTest {
2830
static final OperationContext OP_CONTEXT =
2931
TestOperationContexts.systemContextNoSearchAuthorization();
3032
private static final String DATAHUB_TEST_VALUES_ENV = "DATAHUB_TEST_VALUES_ENV";
33+
private static final String DATAHUB_TEST_REVISION_ENV = "DATAHUB_TEST_REVISION_ENV";
3134
private static final AuditStamp TEST_AUDIT_STAMP = AuditStampUtils.createDefaultAuditStamp();
3235

3336
@SystemStub private EnvironmentVariables environmentVariables;
3437

38+
@BeforeMethod
39+
private void resetEnvironment() {
40+
environmentVariables.remove(DATAHUB_TEST_VALUES_ENV);
41+
environmentVariables.remove(DATAHUB_TEST_REVISION_ENV);
42+
}
43+
3544
@Test
3645
public void testResolveYamlConf() throws IOException {
3746
BootstrapMCPConfigFile initConfig =
@@ -51,9 +60,28 @@ public void testResolveYamlConf() throws IOException {
5160
}
5261

5362
@Test
54-
public void testResolveMCPTemplateDefaults() throws IOException {
55-
environmentVariables.remove(DATAHUB_TEST_VALUES_ENV);
63+
public void testResolveYamlConfOverride() throws IOException {
64+
environmentVariables.set(DATAHUB_TEST_REVISION_ENV, "{\"version\":\"2024110600\"}");
65+
66+
BootstrapMCPConfigFile initConfig =
67+
BootstrapMCPUtil.resolveYamlConf(
68+
OP_CONTEXT, "bootstrapmcp/test.yaml", BootstrapMCPConfigFile.class);
69+
assertEquals(initConfig.getBootstrap().getTemplates().size(), 1);
70+
71+
BootstrapMCPConfigFile.MCPTemplate template =
72+
initConfig.getBootstrap().getTemplates().get(0).withOverride(new ObjectMapper());
73+
assertEquals(template.getName(), "datahub-test");
74+
assertEquals(template.getVersion(), "2024110600");
75+
assertFalse(template.isForce());
76+
assertFalse(template.isBlocking());
77+
assertTrue(template.isAsync());
78+
assertFalse(template.isOptional());
79+
assertEquals(template.getMcps_location(), "bootstrapmcp/datahub-test-mcp.yaml");
80+
assertEquals(template.getValues_env(), "DATAHUB_TEST_VALUES_ENV");
81+
}
5682

83+
@Test
84+
public void testResolveMCPTemplateDefaults() throws IOException {
5785
BootstrapMCPConfigFile.MCPTemplate template =
5886
BootstrapMCPUtil.resolveYamlConf(
5987
OP_CONTEXT, "bootstrapmcp/test.yaml", BootstrapMCPConfigFile.class)
@@ -186,8 +214,6 @@ public void testResolveMCPTemplateOverride() throws IOException {
186214

187215
@Test
188216
public void testMCPBatch() throws IOException {
189-
environmentVariables.remove(DATAHUB_TEST_VALUES_ENV);
190-
191217
BootstrapMCPConfigFile.MCPTemplate template =
192218
BootstrapMCPUtil.resolveYamlConf(
193219
OP_CONTEXT, "bootstrapmcp/test.yaml", BootstrapMCPConfigFile.class)

‎datahub-upgrade/src/test/resources/bootstrapmcp/test.yaml‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,4 +6,5 @@ bootstrap:
66
# blocking: false
77
# async: true
88
mcps_location: "bootstrapmcp/datahub-test-mcp.yaml"
9-
values_env: "DATAHUB_TEST_VALUES_ENV"
9+
values_env: "DATAHUB_TEST_VALUES_ENV"
10+
revision_env: "DATAHUB_TEST_REVISION_ENV"

‎docs/advanced/bootstrap-mcps.md‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -149,6 +149,28 @@ to the required json structure and stored as a string.
149149
executorId: default
150150
```
151151

152+
## `bootstrap_mcps.yaml` Override
153+
154+
Additionally, the `bootstrap_mcps.yaml` can be overridden.
155+
This might be useful for applying changes to the version when using helm defined template values.
156+
157+
```yaml
158+
bootstrap:
159+
templates:
160+
- name: myMCPTemplate
161+
version: v1
162+
mcps_location: <classpath or file location>
163+
values_env: <value environment variable>
164+
revision_env: REVISION_ENV
165+
```
166+
167+
In the above example, we've added a `revision_env` which allows overriding the MCP bootstrap definition itself (excluding `revision_env`).
168+
169+
In this example we could configure `REVISION_ENV` to contain a timestamp or hash: `{"version":"2024060600"}`
170+
This value can be changed/incremented each time the helm supplied template values change. This ensures the MCP is updated
171+
with the latest values during deployment.
172+
173+
152174
## Known Limitations
153175

154176
* Supported change types:

‎metadata-service/configuration/src/main/resources/bootstrap_mcps.yaml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,3 +42,4 @@ bootstrap:
4242
optional: false
4343
mcps_location: "bootstrap_mcps/ingestion-datahub-gc.yaml"
4444
values_env: "DATAHUB_GC_BOOTSTRAP_VALUES"
45+
revision_env: "DATAHUB_GC_BOOTSTRAP_REVISION"

‎metadata-service/configuration/src/main/resources/bootstrap_mcps/ingestion-datahub-gc.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@
1212
timezone: '{{schedule.timezone}}{{^schedule.timezone}}UTC{{/schedule.timezone}}'
1313
interval: '{{schedule.interval}}{{^schedule.interval}}0 1 * * *{{/schedule.interval}}'
1414
config:
15-
version: '{{&ingestion.version}}{{^ingestion.version}}0.14.1.6{{/ingestion.version}}'
15+
version: '{{&ingestion.version}}{{^ingestion.version}}0.14.1.7rc2{{/ingestion.version}}'
1616
recipe:
1717
source:
1818
type: 'datahub-gc'

0 commit comments

Comments
 (0)