feat(ingest): Support System Ingestion Sources, Show and hide system …

…ingestion sources with Command-S (datahub-project#10938) Co-authored-by: John Joyce <[email protected]>
mayurinehate · Jul 18, 2024 · 1617d62 · 1617d62
1 parent 452b94f
commit 1617d62
Show file tree

Hide file tree

Showing 3 changed files with 51 additions and 0 deletions.
diff --git a/datahub-web-react/src/app/ingest/source/IngestionSourceList.tsx b/datahub-web-react/src/app/ingest/source/IngestionSourceList.tsx
@@ -32,6 +32,7 @@ import {
     INGESTION_REFRESH_SOURCES_ID,
 } from '../../onboarding/config/IngestionOnboardingConfig';
 import { ONE_SECOND_IN_MS } from '../../entity/shared/tabs/Dataset/Queries/utils/constants';
+import { useCommandS } from './hooks';
 
 const PLACEHOLDER_URN = 'placeholder-urn';
 
@@ -51,6 +52,8 @@ const FilterWrapper = styled.div`
     display: flex;
 `;
 
+const SYSTEM_INTERNAL_SOURCE_TYPE = 'SYSTEM';
+
 export enum IngestionSourceType {
     ALL,
     UI,
@@ -102,6 +105,17 @@ export const IngestionSourceList = () => {
     // Set of removed urns used to account for eventual consistency
     const [removedUrns, setRemovedUrns] = useState<string[]>([]);
     const [sourceFilter, setSourceFilter] = useState(IngestionSourceType.ALL);
+    const [hideSystemSources, setHideSystemSources] = useState(true);
+
+    /**
+     * Show or hide system ingestion sources using a hidden command S command.
+     */
+    useCommandS(() => setHideSystemSources(!hideSystemSources));
+
+    // Ingestion Source Default Filters
+    const filters = hideSystemSources
+        ? [{ field: 'sourceType', values: [SYSTEM_INTERNAL_SOURCE_TYPE], negated: true }]
+        : undefined;
 
     // Ingestion Source Queries
     const { loading, error, data, client, refetch } = useListIngestionSourcesQuery({
@@ -110,6 +124,7 @@ export const IngestionSourceList = () => {
                 start,
                 count: pageSize,
                 query: (query?.length && query) || undefined,
+                filters,
             },
         },
         fetchPolicy: (query?.length || 0) > 0 ? 'no-cache' : 'cache-first',

diff --git a/datahub-web-react/src/app/ingest/source/hooks.ts b/datahub-web-react/src/app/ingest/source/hooks.ts
@@ -0,0 +1,16 @@
+import { useEffect } from 'react';
+
+export const useCommandS = (onPress: () => void) => {
+    useEffect(() => {
+        const handleKeyDown = (event: KeyboardEvent) => {
+            if (event.metaKey && event.key === 's') {
+                event.preventDefault();
+                onPress();
+            }
+        };
+        window.addEventListener('keydown', handleKeyDown);
+        return () => {
+            window.removeEventListener('keydown', handleKeyDown);
+        };
+    }, [onPress]);
+};
diff --git a/metadata-models/src/main/pegasus/com/linkedin/ingestion/DataHubIngestionSourceInfo.pdl b/metadata-models/src/main/pegasus/com/linkedin/ingestion/DataHubIngestionSourceInfo.pdl
@@ -62,4 +62,24 @@ record DataHubIngestionSourceInfo {
       */
       extraArgs: optional map[string, string]
   }
+
+  /**
+   * The source or origin of the Ingestion Source
+   *
+   * Currently CLI and UI do not provide an explicit source.
+   */
+   source: optional record DataHubIngestionSourceSource {
+        /**
+         * The source type of the ingestion source
+         */
+        @Searchable = {
+           "fieldName": "sourceType"
+        }
+        type: enum DataHubIngestionSourceSourceType {
+            /**
+             * A system internal source, e.g. for running search indexing operations, feature computation, etc.
+             */
+            SYSTEM
+        }
+   }
 }