Skip to content

Commit

Permalink
Merge branch 'master' into feat-sap-analytics-cloud
Browse files Browse the repository at this point in the history
  • Loading branch information
@hsheth2
hsheth2 authored Aug 9, 2024
2 parents 3dd7c06 + 840b150 commit 1db83a3
Show file tree
Hide file tree
Showing 505 changed files with 17,932 additions and 7,932 deletions.
31 changes: 23 additions & 8 deletions .github/scripts/check_python_package.py
View file
Original file line number Diff line number Diff line change
@@ -1,18 +1,33 @@
import setuptools
import os

folders = ["./smoke-test/tests"]

for folder in folders:
print(f"Checking folder {folder}")
a = [i for i in setuptools.find_packages(folder) if "cypress" not in i]
b = [i for i in setuptools.find_namespace_packages(folder) if "cypress" not in i]
packages = [i for i in setuptools.find_packages(folder) if "cypress" not in i]
namespace_packages = [
i for i in setuptools.find_namespace_packages(folder) if "cypress" not in i
]

in_a_not_b = set(a) - set(b)
in_b_not_a = set(b) - set(a)
print("Packages found:", packages)
print("Namespace packages found:", namespace_packages)

in_packages_not_namespace = set(packages) - set(namespace_packages)
in_namespace_not_packages = set(namespace_packages) - set(packages)

if in_packages_not_namespace:
print(f"Packages not in namespace packages: {in_packages_not_namespace}")
if in_namespace_not_packages:
print(f"Namespace packages not in packages: {in_namespace_not_packages}")
for pkg in in_namespace_not_packages:
pkg_path = os.path.join(folder, pkg.replace(".", os.path.sep))
print(f"Contents of {pkg_path}:")
print(os.listdir(pkg_path))

assert (
len(in_a_not_b) == 0
), f"Found packages in {folder} that are not in namespace packages: {in_a_not_b}"
len(in_packages_not_namespace) == 0
), f"Found packages in {folder} that are not in namespace packages: {in_packages_not_namespace}"
assert (
len(in_b_not_a) == 0
), f"Found namespace packages in {folder} that are not in packages: {in_b_not_a}"
len(in_namespace_not_packages) == 0
), f"Found namespace packages in {folder} that are not in packages: {in_namespace_not_packages}"
8 changes: 4 additions & 4 deletions .github/scripts/docker_helpers.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,11 +16,11 @@ function get_tag {
}

function get_tag_slim {
echo $(echo ${GITHUB_REF} | sed -e "s,refs/heads/${MAIN_BRANCH},${MAIN_BRANCH_TAG}-slim,g" -e 's,refs/tags/,,g' -e 's,refs/pull/\([0-9]*\).*,pr\1-slim,g'),${SHORT_SHA}-slim
echo $(echo ${GITHUB_REF} | sed -e "s,refs/heads/${MAIN_BRANCH},${MAIN_BRANCH_TAG}-slim,g" -e 's,refs/tags/\(.*\),\1-slim,g' -e 's,refs/pull/\([0-9]*\).*,pr\1-slim,g'),${SHORT_SHA}-slim
}

function get_tag_full {
echo $(echo ${GITHUB_REF} | sed -e "s,refs/heads/${MAIN_BRANCH},${MAIN_BRANCH_TAG}-full,g" -e 's,refs/tags/,,g' -e 's,refs/pull/\([0-9]*\).*,pr\1-full,g'),${SHORT_SHA}-full
echo $(echo ${GITHUB_REF} | sed -e "s,refs/heads/${MAIN_BRANCH},${MAIN_BRANCH_TAG}-full,g" -e 's,refs/tags/\(.*\),\1-full,g' -e 's,refs/pull/\([0-9]*\).*,pr\1-full,g'),${SHORT_SHA}-full
}

function get_python_docker_release_v {
Expand All @@ -32,9 +32,9 @@ function get_unique_tag {
}

function get_unique_tag_slim {
echo $(echo ${GITHUB_REF} | sed -e "s,refs/heads/${MAIN_BRANCH},${SHORT_SHA}-slim,g" -e 's,refs/tags/,,g' -e 's,refs/pull/\([0-9]*\).*,pr\1-slim,g')
echo $(echo ${GITHUB_REF} | sed -e "s,refs/heads/${MAIN_BRANCH},${SHORT_SHA}-slim,g" -e 's,refs/tags/\(.*\),\1-slim,g' -e 's,refs/pull/\([0-9]*\).*,pr\1-slim,g')
}

function get_unique_tag_full {
echo $(echo ${GITHUB_REF} | sed -e "s,refs/heads/${MAIN_BRANCH},${SHORT_SHA}-full,g" -e 's,refs/tags/,,g' -e 's,refs/pull/\([0-9]*\).*,pr\1-full,g')
echo $(echo ${GITHUB_REF} | sed -e "s,refs/heads/${MAIN_BRANCH},${SHORT_SHA}-full,g" -e 's,refs/tags/\(.*\),\1-full,g' -e 's,refs/pull/\([0-9]*\).*,pr\1-full,g')
}
2 changes: 1 addition & 1 deletion .github/workflows/airflow-plugin.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@ jobs:
extra_pip_requirements: 'apache-airflow~=2.8.1 -c https://raw.githubusercontent.com/apache/airflow/constraints-2.8.1/constraints-3.10.txt'
extra_pip_extras: plugin-v2
- python-version: "3.11"
extra_pip_requirements: 'apache-airflow~=2.9.3 -c https://raw.githubusercontent.com/apache/airflow/constraints-2.9.3/constraints-3.10.txt'
extra_pip_requirements: 'apache-airflow~=2.9.3 -c https://raw.githubusercontent.com/apache/airflow/constraints-2.9.3/constraints-3.11.txt'
extra_pip_extras: plugin-v2
fail-fast: false
steps:
Expand Down
6 changes: 2 additions & 4 deletions .github/workflows/pr-labeler.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,12 +30,10 @@ jobs:
"treff7es",
"yoonhyejin",
"eboneil",
"ethan-cartwright",
"gabe-lyons",
"hsheth2",
"jjoyce0510",
"maggiehays",
"mrjefflewis",
"pedro93",
"RyanHolstien",
"Kunal-kankriya",
Expand All @@ -45,7 +43,8 @@ jobs:
"kushagra-apptware",
"Salman-Apptware",
"mayurinehate",
"noggi"
"noggi",
"skrydal"
]'),
github.actor
)
Expand All @@ -60,7 +59,6 @@ jobs:
${{
contains(
fromJson('[
"skrydal",
"siladitya2",
"sgomezvillamor",
"ngamanda",
Expand Down
3 changes: 2 additions & 1 deletion datahub-frontend/app/controllers/SsoCallbackController.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,8 @@ public SsoCallbackController(

public CompletionStage<Result> handleCallback(String protocol, Http.Request request) {
if (shouldHandleCallback(protocol)) {
log.debug(String.format("Handling SSO callback. Protocol: %s", protocol));
log.debug("Handling SSO callback. Protocol: {}",
_ssoManager.getSsoProvider().protocol().getCommonName());
return callback(request)
.handle(
(res, e) -> {
Expand Down
1 change: 1 addition & 0 deletions datahub-graphql-core/build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ dependencies {
implementation externalDependency.opentelemetryAnnotations

implementation externalDependency.slf4jApi
implementation externalDependency.springContext
compileOnly externalDependency.lombok
annotationProcessor externalDependency.lombok

Expand Down
1 change: 1 addition & 0 deletions datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/Constants.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ private Constants() {}
public static final String PROPERTIES_SCHEMA_FILE = "properties.graphql";
public static final String FORMS_SCHEMA_FILE = "forms.graphql";
public static final String ASSERTIONS_SCHEMA_FILE = "assertions.graphql";
public static final String COMMON_SCHEMA_FILE = "common.graphql";
public static final String INCIDENTS_SCHEMA_FILE = "incident.graphql";
public static final String CONTRACTS_SCHEMA_FILE = "contract.graphql";
public static final String CONNECTIONS_SCHEMA_FILE = "connection.graphql";
Expand Down
43 changes: 37 additions & 6 deletions datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -96,6 +96,7 @@
import com.linkedin.datahub.graphql.generated.MLPrimaryKey;
import com.linkedin.datahub.graphql.generated.MLPrimaryKeyProperties;
import com.linkedin.datahub.graphql.generated.MatchedField;
import com.linkedin.datahub.graphql.generated.MetadataAttribution;
import com.linkedin.datahub.graphql.generated.Notebook;
import com.linkedin.datahub.graphql.generated.Owner;
import com.linkedin.datahub.graphql.generated.OwnershipTypeEntity;
Expand Down Expand Up @@ -284,6 +285,8 @@
import com.linkedin.datahub.graphql.resolvers.search.SearchAcrossEntitiesResolver;
import com.linkedin.datahub.graphql.resolvers.search.SearchAcrossLineageResolver;
import com.linkedin.datahub.graphql.resolvers.search.SearchResolver;
import com.linkedin.datahub.graphql.resolvers.settings.docPropagation.DocPropagationSettingsResolver;
import com.linkedin.datahub.graphql.resolvers.settings.docPropagation.UpdateDocPropagationSettingsResolver;
import com.linkedin.datahub.graphql.resolvers.settings.user.UpdateCorpUserViewsSettingsResolver;
import com.linkedin.datahub.graphql.resolvers.settings.view.GlobalViewsSettingsResolver;
import com.linkedin.datahub.graphql.resolvers.settings.view.UpdateGlobalViewsSettingsResolver;
Expand Down Expand Up @@ -695,7 +698,8 @@ public GmsGraphQLEngine(final GmsGraphQLEngineArgs args) {
businessAttributeType));
this.loadableTypes = new ArrayList<>(entityTypes);
// Extend loadable types with types from the plugins
// This allows us to offer search and browse capabilities out of the box for those types
// This allows us to offer search and browse capabilities out of the box for
// those types
for (GmsGraphQLPlugin plugin : this.graphQLPlugins) {
this.entityTypes.addAll(plugin.getEntityTypes());
Collection<? extends LoadableType<?, ?>> pluginLoadableTypes = plugin.getLoadableTypes();
Expand Down Expand Up @@ -790,6 +794,7 @@ public void configureRuntimeWiring(final RuntimeWiring.Builder builder) {
configureBusinessAttributeAssociationResolver(builder);
configureConnectionResolvers(builder);
configureDeprecationResolvers(builder);
configureMetadataAttributionResolver(builder);
}

private void configureOrganisationRoleResolvers(RuntimeWiring.Builder builder) {
Expand Down Expand Up @@ -843,7 +848,8 @@ public GraphQLEngine.Builder builder() {
.addSchema(fileBasedSchema(CONNECTIONS_SCHEMA_FILE))
.addSchema(fileBasedSchema(ASSERTIONS_SCHEMA_FILE))
.addSchema(fileBasedSchema(INCIDENTS_SCHEMA_FILE))
.addSchema(fileBasedSchema(CONTRACTS_SCHEMA_FILE));
.addSchema(fileBasedSchema(CONTRACTS_SCHEMA_FILE))
.addSchema(fileBasedSchema(COMMON_SCHEMA_FILE));

for (GmsGraphQLPlugin plugin : this.graphQLPlugins) {
List<String> pluginSchemaFiles = plugin.getSchemaFiles();
Expand Down Expand Up @@ -1026,6 +1032,8 @@ private void configureQueryResolvers(final RuntimeWiring.Builder builder) {
.dataFetcher("mlModel", getResolver(mlModelType))
.dataFetcher("mlModelGroup", getResolver(mlModelGroupType))
.dataFetcher("assertion", getResolver(assertionType))
.dataFetcher("form", getResolver(formType))
.dataFetcher("view", getResolver(dataHubViewType))
.dataFetcher("listPolicies", new ListPoliciesResolver(this.entityClient))
.dataFetcher("getGrantedPrivileges", new GetGrantedPrivilegesResolver())
.dataFetcher("listUsers", new ListUsersResolver(this.entityClient))
Expand Down Expand Up @@ -1087,8 +1095,10 @@ private void configureQueryResolvers(final RuntimeWiring.Builder builder) {
new BrowseV2Resolver(this.entityClient, this.viewService, this.formService))
.dataFetcher("businessAttribute", getResolver(businessAttributeType))
.dataFetcher(
"listBusinessAttributes",
new ListBusinessAttributesResolver(this.entityClient)));
"listBusinessAttributes", new ListBusinessAttributesResolver(this.entityClient))
.dataFetcher(
"docPropagationSettings",
new DocPropagationSettingsResolver(this.settingsService)));
}

private DataFetcher getEntitiesResolver() {
Expand Down Expand Up @@ -1340,7 +1350,11 @@ private void configureMutationResolvers(final RuntimeWiring.Builder builder) {
.dataFetcher(
"createForm", new CreateFormResolver(this.entityClient, this.formService))
.dataFetcher("deleteForm", new DeleteFormResolver(this.entityClient))
.dataFetcher("updateForm", new UpdateFormResolver(this.entityClient));
.dataFetcher("updateForm", new UpdateFormResolver(this.entityClient))
.dataFetcher(
"updateDocPropagationSettings",
new UpdateDocPropagationSettingsResolver(this.settingsService));

if (featureFlags.isBusinessAttributeEntityEnabled()) {
typeWiring
.dataFetcher(
Expand Down Expand Up @@ -2823,7 +2837,8 @@ private void configureContractResolvers(final RuntimeWiring.Builder builder) {
}

private void configurePolicyResolvers(final RuntimeWiring.Builder builder) {
// Register resolvers for "resolvedUsers" and "resolvedGroups" field of the Policy type.
// Register resolvers for "resolvedUsers" and "resolvedGroups" field of the
// Policy type.
builder.type(
"ActorFilter",
typeWiring ->
Expand Down Expand Up @@ -3176,4 +3191,20 @@ private void configureDeprecationResolvers(final RuntimeWiring.Builder builder)
new EntityTypeResolver(
entityTypes, (env) -> ((Deprecation) env.getSource()).getActorEntity())));
}

private void configureMetadataAttributionResolver(final RuntimeWiring.Builder builder) {
builder.type(
"MetadataAttribution",
typeWiring ->
typeWiring
.dataFetcher(
"actor",
new EntityTypeResolver(
entityTypes, (env) -> ((MetadataAttribution) env.getSource()).getActor()))
.dataFetcher(
"source",
new EntityTypeResolver(
entityTypes,
(env) -> ((MetadataAttribution) env.getSource()).getSource())));
}
}
5 changes: 5 additions & 0 deletions ...hql-core/src/main/java/com/linkedin/datahub/graphql/authorization/AuthorizationUtils.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -398,6 +398,11 @@ public static boolean canManageForms(@Nonnull QueryContext context) {
PoliciesConfig.MANAGE_DOCUMENTATION_FORMS_PRIVILEGE);
}

public static boolean canManageFeatures(@Nonnull QueryContext context) {
return AuthUtil.isAuthorized(
context.getAuthorizer(), context.getActorUrn(), PoliciesConfig.MANAGE_FEATURES_PRIVILEGE);
}

public static boolean isAuthorized(
@Nonnull Authorizer authorizer,
@Nonnull String actor,
Expand Down
2 changes: 1 addition & 1 deletion .../src/main/java/com/linkedin/datahub/graphql/resolvers/auth/RevokeAccessTokenResolver.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ public CompletableFuture<Boolean> get(DataFetchingEnvironment environment) throw
final QueryContext context = environment.getContext();
final String tokenId = bindArgument(environment.getArgument("tokenId"), String.class);

log.info("User {} revoking access token {}", context.getActorUrn(), tokenId);
log.info("User {} revoking access token", context.getActorUrn());

if (isAuthorizedToRevokeToken(context, tokenId)) {
try {
Expand Down
8 changes: 4 additions & 4 deletions .../main/java/com/linkedin/datahub/graphql/resolvers/ingest/secret/DeleteSecretResolver.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,16 +23,16 @@ public DeleteSecretResolver(final EntityClient entityClient) {
public CompletableFuture<String> get(final DataFetchingEnvironment environment) throws Exception {
final QueryContext context = environment.getContext();
if (IngestionAuthUtils.canManageSecrets(context)) {
final String secretUrn = environment.getArgument("urn");
final Urn urn = Urn.createFromString(secretUrn);
final String inputUrn = environment.getArgument("urn");
final Urn urn = Urn.createFromString(inputUrn);
return GraphQLConcurrencyUtils.supplyAsync(
() -> {
try {
_entityClient.deleteEntity(context.getOperationContext(), urn);
return secretUrn;
return inputUrn;
} catch (Exception e) {
throw new RuntimeException(
String.format("Failed to perform delete against secret with urn %s", secretUrn),
String.format("Failed to perform delete against secret with urn %s", inputUrn),
e);
}
},
Expand Down
4 changes: 2 additions & 2 deletions ...-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ingest/secret/SecretUtils.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ static String encrypt(String value, String secret) {
} catch (NoSuchAlgorithmException e) {
e.printStackTrace();
}
Cipher cipher = Cipher.getInstance("AES/ECB/PKCS5Padding");
Cipher cipher = Cipher.getInstance("AES");
cipher.init(Cipher.ENCRYPT_MODE, secretKey);
return Base64.getEncoder()
.encodeToString(cipher.doFinal(value.getBytes(StandardCharsets.UTF_8)));
Expand All @@ -48,7 +48,7 @@ static String decrypt(String encryptedValue, String secret) {
} catch (NoSuchAlgorithmException e) {
e.printStackTrace();
}
Cipher cipher = Cipher.getInstance("AES/ECB/PKCS5PADDING");
Cipher cipher = Cipher.getInstance("AES");
cipher.init(Cipher.DECRYPT_MODE, secretKey);
return new String(cipher.doFinal(Base64.getDecoder().decode(encryptedValue)));
} catch (Exception e) {
Expand Down
Loading

0 comments on commit 1db83a3

Please sign in to comment.