Skip to content

Commit

Permalink
Bump up kafkaAvroSerde to support SSL for Schema Registry (#1898)
Browse files Browse the repository at this point in the history 
* Bump up kafkaAvroSerde to support security config for Confluent Schema Registry

* Support certs in secrets

* Extra Spring config (e.g. security)

* Optional values

* Clarify log warnings

* Update faq.md

Co-authored-by: Lars Nielsen <[email protected]>
Co-authored-by: Mars Lan <[email protected]>
  • Loading branch information
@themightylaz @mars-lan
3 people authored Sep 29, 2020
1 parent 9bcf273 commit b26d6fe
Show file tree
Hide file tree
Showing 7 changed files with 108 additions and 2 deletions.
2 changes: 1 addition & 1 deletion build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ project.ext.externalDependency = [
'jsonSimple': 'com.googlecode.json-simple:json-simple:1.1.1',
'junit': 'junit:junit:4.12',
// avro-serde includes dependencies for `kafka-avro-serializer` `kafka-schema-registry-client` and `avro`
'kafkaAvroSerde': 'io.confluent:kafka-streams-avro-serde:5.2.2',
'kafkaAvroSerde': 'io.confluent:kafka-streams-avro-serde:5.5.1',
'kafkaClients': 'org.apache.kafka:kafka-clients:2.3.0',
'logbackClassic': 'ch.qos.logback:logback-classic:1.2.3',
'lombok': 'org.projectlombok:lombok:1.18.12',
Expand Down
7 changes: 7 additions & 0 deletions contrib/kubernetes/datahub/README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,3 +51,10 @@ Current chart version is `0.1.0`
| global.sql.datasource.username | string | `"datahub"` | |
| global.sql.datasource.password.secretRef | string | `"mysql-secrets"` | |
| global.sql.datasource.password.secretKey | string | `"mysql-password"` | |

#### Optional Chart Values

| global.credentialsAndCertsSecretPath | string | `"/mnt/certs"` | |
| global.credentialsAndCertsSecrets.name | string | `""` | |
| global.credentialsAndCertsSecrets.secureEnv | string | `""` | |
| global.springKafkaConfigurationOverrides | string | `""` | |
25 changes: 25 additions & 0 deletions contrib/kubernetes/datahub/charts/datahub-gms/templates/deployment.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,12 @@ spec:
securityContext:
{{- toYaml .Values.podSecurityContext | nindent 8 }}
volumes:
{{- if .Values.global.credentialsAndCertsSecrets }}
- name: datahub-certs-dir
secret:
defaultMode: 256
secretName: {{ .Values.global.credentialsAndCertsSecrets.name }}
{{- end }}
{{- if .Values.extraVolumes }}
{{ toYaml .Values.extraVolumes | indent 8 }}
{{- end }}
Expand Down Expand Up @@ -72,10 +78,29 @@ spec:
secretKeyRef:
name: "{{ .Values.global.neo4j.password.secretRef }}"
key: "{{ .Values.global.neo4j.password.secretKey }}"
{{- if .Values.global.springKafkaConfigurationOverrides }}
{{- range $configName, $configValue := .Values.global.springKafkaConfigurationOverrides }}
- name: SPRING_KAFKA_PROPERTIES_{{ $configName | replace "." "_" | upper }}
value: {{ $configValue }}
{{- end }}
{{- end }}
{{- if .Values.global.credentialsAndCertsSecrets }}
{{- range $envVarName, $envVarValue := .Values.global.credentialsAndCertsSecrets.secureEnv }}
- name: SPRING_KAFKA_PROPERTIES_{{ $envVarName | replace "." "_" | upper }}
valueFrom:
secretKeyRef:
name: {{ $.Values.global.credentialsAndCertsSecrets.name }}
key: {{ $envVarValue }}
{{- end }}
{{- end }}
{{- if .Values.extraEnvs }}
{{ toYaml .Values.extraEnvs | indent 10 }}
{{- end }}
volumeMounts:
{{- if .Values.global.credentialsAndCertsSecrets }}
- name: datahub-certs-dir
mountPath: {{ .Values.global.credentialsAndCertsSecretPath | default "/mnt/certs" }}
{{- end }}
{{- if .Values.extraVolumeMounts }}
{{ toYaml .Values.extraVolumeMounts | indent 10 }}
{{- end }}
Expand Down
25 changes: 25 additions & 0 deletions contrib/kubernetes/datahub/charts/datahub-mae-consumer/templates/deployment.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,12 @@ spec:
securityContext:
{{- toYaml .Values.podSecurityContext | nindent 8 }}
volumes:
{{- if .Values.global.credentialsAndCertsSecrets }}
- name: datahub-certs-dir
secret:
defaultMode: 256
secretName: {{ .Values.global.credentialsAndCertsSecrets.name }}
{{- end }}
{{- if .Values.extraVolumes }}
{{ toYaml .Values.extraVolumes | indent 8 }}
{{- end }}
Expand Down Expand Up @@ -55,10 +61,29 @@ spec:
secretKeyRef:
name: "{{ .Values.global.neo4j.password.secretRef }}"
key: "{{ .Values.global.neo4j.password.secretKey }}"
{{- if .Values.global.springKafkaConfigurationOverrides }}
{{- range $configName, $configValue := .Values.global.springKafkaConfigurationOverrides }}
- name: SPRING_KAFKA_PROPERTIES_{{ $configName | replace "." "_" | upper }}
value: {{ $configValue }}
{{- end }}
{{- end }}
{{- if .Values.global.credentialsAndCertsSecrets }}
{{- range $envVarName, $envVarValue := .Values.global.credentialsAndCertsSecrets.secureEnv }}
- name: SPRING_KAFKA_PROPERTIES_{{ $envVarName | replace "." "_" | upper }}
valueFrom:
secretKeyRef:
name: {{ $.Values.global.credentialsAndCertsSecrets.name }}
key: {{ $envVarValue }}
{{- end }}
{{- end }}
{{- if .Values.extraEnvs }}
{{ toYaml .Values.extraEnvs | indent 10 }}
{{- end }}
volumeMounts:
{{- if .Values.global.credentialsAndCertsSecrets }}
- name: datahub-certs-dir
mountPath: {{ .Values.global.credentialsAndCertsSecretPath | default "/mnt/certs" }}
{{- end }}
{{- if .Values.extraVolumeMounts }}
{{ toYaml .Values.extraVolumeMounts | indent 10 }}
{{- end }}
Expand Down
25 changes: 25 additions & 0 deletions contrib/kubernetes/datahub/charts/datahub-mce-consumer/templates/deployment.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,12 @@ spec:
securityContext:
{{- toYaml .Values.podSecurityContext | nindent 8 }}
volumes:
{{- if .Values.global.credentialsAndCertsSecrets }}
- name: datahub-certs-dir
secret:
defaultMode: 256
secretName: {{ .Values.global.credentialsAndCertsSecrets.name }}
{{- end }}
{{- if .Values.extraVolumes }}
{{ toYaml .Values.extraVolumes | indent 8 }}
{{- end }}
Expand All @@ -44,10 +50,29 @@ spec:
value: {{ printf "%s-%s" .Release.Name "datahub-gms" }}
- name: GMS_PORT
value: "{{ .Values.global.datahub.gms.port }}"
{{- if .Values.global.springKafkaConfigurationOverrides }}
{{- range $configName, $configValue := .Values.global.springKafkaConfigurationOverrides }}
- name: SPRING_KAFKA_PROPERTIES_{{ $configName | replace "." "_" | upper }}
value: {{ $configValue }}
{{- end }}
{{- end }}
{{- if .Values.global.credentialsAndCertsSecrets }}
{{- range $envVarName, $envVarValue := .Values.global.credentialsAndCertsSecrets.secureEnv }}
- name: SPRING_KAFKA_PROPERTIES_{{ $envVarName | replace "." "_" | upper }}
valueFrom:
secretKeyRef:
name: {{ $.Values.global.credentialsAndCertsSecrets.name }}
key: {{ $envVarValue }}
{{- end }}
{{- end }}
{{- if .Values.extraEnvs }}
{{ toYaml .Values.extraEnvs | indent 10 }}
{{- end }}
volumeMounts:
{{- if .Values.global.credentialsAndCertsSecrets }}
- name: datahub-certs-dir
mountPath: {{ .Values.global.credentialsAndCertsSecretPath | default "/mnt/certs" }}
{{- end }}
{{- if .Values.extraVolumeMounts }}
{{ toYaml .Values.extraVolumeMounts | indent 10 }}
{{- end }}
Expand Down
22 changes: 21 additions & 1 deletion contrib/kubernetes/datahub/values.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,4 +66,24 @@ global:
- "broker"
- "mysql"
- "elasticsearch"
- "neo4j"
- "neo4j"

# credentialsAndCertsSecretPath: /mnt/datahub/certs
# credentialsAndCertsSecrets:
# name: datahub-certs
# secureEnv:
# ssl.key.password: datahub.linkedin.com.KeyPass
# ssl.keystore.password: datahub.linkedin.com.KeyStorePass
# ssl.truststore.password: datahub.linkedin.com.TrustStorePass
# kafkastore.ssl.truststore.password: datahub.linkedin.com.TrustStorePass

# springKafkaConfigurationOverrides:
# ssl.keystore.location: /mnt/datahub/certs/datahub.linkedin.com.keystore.jks
# ssl.truststore.location: /mnt/datahub/certs/datahub.linkedin.com.truststore.jks
# kafkastore.ssl.truststore.location: /mnt/datahub/certs/datahub.linkedin.com.truststore.jks
# security.protocol: SSL
# kafkastore.security.protocol: SSL
# ssl.keystore.type: JKS
# ssl.truststore.type: JKS
# ssl.protocol: TLS
# ssl.endpoint.identification.algorithm:
4 changes: 4 additions & 0 deletions docs/faq.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -116,3 +116,7 @@ You can call the [rest.li](https://github.com/linkedin/rest.li) API to ingest me
## Does Kafka support SSL? If so, how?

Yes. We are using the Spring Boot framework to start our apps, including setting up Kafka. You can [use environment variables to set system properties](https://docs.spring.io/spring-boot/docs/current/reference/html/spring-boot-features.html#boot-features-external-config-relaxed-binding-from-environment-variables), including [Kafka properties](https://docs.spring.io/spring-boot/docs/current/reference/html/appendix-application-properties.html#integration-properties). From there you can set your SSL configuration for Kafka.

If Schema Registry is configured to use security (SSL), then you also need to set the following config: https://docs.confluent.io/current/kafka/encryption.html#encryption-ssl-schema-registry.

> **Note** In the logs you might see something like `The configuration 'kafkastore.ssl.truststore.password' was supplied but isn't a known config.` The configuration is not a configuration required for the producer. These WARN message can be safely ignored. Each of Datahub services are passed a full set of configuration but may not require all the configurations that are passed to them. These warn messages indicate that the service was passed a configuration that is not relevant to it and can be safely ignored.

0 comments on commit b26d6fe

Please sign in to comment.