feat(metadata-model): adding metadata model doc generation and upload… (

#3667)
datahub-project · Dec 5, 2021 · a16c432 · a16c432
1 parent 999e9e5
commit a16c432
Show file tree

Hide file tree

Showing 14 changed files with 2,395 additions and 1,601 deletions.
diff --git a/.gitignore b/.gitignore
@@ -40,3 +40,6 @@ MANIFEST
 **/.DS_Store
 
 .vscode
+
+# Metadata Ingestion Generated
+metadata-ingestion/generated/**
diff --git a/docs/modeling/metadata-model.md b/docs/modeling/metadata-model.md
@@ -39,6 +39,17 @@ For example, here are helpful links to the most popular entities in DataHub's me
 * Feature Table (a.k.a. MLFeatureTable): [Profile](https://demo.datahubproject.io/dataset/urn:li:dataset:(urn:li:dataPlatform:datahub,MlFeatureTable,PROD)/Schema?is_lineage_mode=false) [Documentation](https://demo.datahubproject.io/dataset/urn:li:dataset:(urn:li:dataPlatform:datahub,MlFeatureTable,PROD)/Documentation?is_lineage_mode=false)
 * For the full list of entities in the metadata model, browse them [here](https://demo.datahubproject.io/browse/dataset/prod/datahub/entities)
 
+### Generating documentation for the Metadata Model
+
+The metadata model documentation can be generated and uploaded into a running DataHub instance using the following command below.
+
+```console
+./gradlew :metadata-ingestion:modelDocUpload
+```
+
+**_NOTE_**: This will upload the model documentation to the DataHub instance running at the environment variable `$DATAHUB_HOST` (http://localhost:8080 by default)
+
+It will also generate a few files under `metadata-ingestion/generated/docs` such as a dot file called `metadata_graph.dot` that you can use to visualize the relationships among the entities.
 
 ## Querying the Metadata Graph 
 

diff --git a/entity-registry/src/main/java/com/linkedin/metadata/models/registry/config/Entity.java b/entity-registry/src/main/java/com/linkedin/metadata/models/registry/config/Entity.java
@@ -12,6 +12,7 @@
 @AllArgsConstructor
 public class Entity {
   String name;
+  String doc;
   String keyAspect;
   List<String> aspects;
 }
diff --git a/metadata-ingestion/build.gradle b/metadata-ingestion/build.gradle
@@ -42,6 +42,18 @@ task installDev(type: Exec, dependsOn: [install]) {
   commandLine 'bash', '-x', '-c',
     "${venv_name}/bin/pip install -e .[dev] && touch ${venv_name}/.build_install_dev_sentinel"
 }
+
+task modelDocGen(type: Exec, dependsOn: [codegen, installDev]) {
+    inputs.files(project.fileTree(dir: "../metadata-events/mxe-schemas/src/", include: "**/*.avsc"))
+    outputs.dir('generated/docs')
+    commandLine 'bash', '-c', "source ${venv_name}/bin/activate && ./scripts/modeldocgen.sh"
+}
+
+task modelDocUpload(type: Exec, dependsOn: [modelDocGen]) {
+    commandLine 'bash', '-c', "source ${venv_name}/bin/activate && ./scripts/modeldocupload.sh"
+}
+
+
 task lint(type: Exec, dependsOn: installDev) {
   /*
     The find/sed combo below is a temporary work-around for the following mypy issue with airflow 2.2.0:
@@ -79,7 +91,6 @@ task installDevTest(type: Exec, dependsOn: [installDev]) {
 
 def testFile = hasProperty('testFile') ? testFile : 'unknown'
 task testSingle(dependsOn: [installDevTest]) {
-  println "$testFile"
   doLast {
   if (testFile != 'unknown')  {
   exec {