Skip to content

Commit

Permalink
Merge branch 'master' of https://github.com/haeniya/datahub into feat…
Browse files Browse the repository at this point in the history 
…ure/tableau-ingestion-hidden-views
  • Loading branch information
Yanik Häni authored and Yanik Häni committed Nov 14, 2024
2 parents 249ca09 + 00a2751 commit d2a25fa
Show file tree
Hide file tree
Showing 341 changed files with 54,740 additions and 2,085 deletions.
30 changes: 17 additions & 13 deletions build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,9 @@ buildscript {
ext.elasticsearchVersion = '2.11.1' // ES 7.10, Opensearch 1.x, 2.x
ext.jacksonVersion = '2.15.3'
ext.jettyVersion = '11.0.21'
// see also datahub-frontend/play.gradle
ext.playVersion = '2.8.22'
ext.playScalaVersion = '2.13'
ext.log4jVersion = '2.23.1'
ext.slf4jVersion = '1.7.36'
ext.logbackClassic = '1.4.14'
Expand Down Expand Up @@ -103,7 +105,7 @@ project.ext.spec = [
]

project.ext.externalDependency = [
'akkaHttp': 'com.typesafe.akka:akka-http-core_2.12:10.2.10',
'akkaHttp': "com.typesafe.akka:akka-http-core_$playScalaVersion:10.2.10",
'antlr4Runtime': 'org.antlr:antlr4-runtime:4.9.3',
'antlr4': 'org.antlr:antlr4:4.9.3',
'assertJ': 'org.assertj:assertj-core:3.11.1',
Expand Down Expand Up @@ -212,18 +214,18 @@ project.ext.externalDependency = [
'parquet': 'org.apache.parquet:parquet-avro:1.12.3',
'parquetHadoop': 'org.apache.parquet:parquet-hadoop:1.13.1',
'picocli': 'info.picocli:picocli:4.5.0',
'playCache': "com.typesafe.play:play-cache_2.12:$playVersion",
'playCaffeineCache': "com.typesafe.play:play-caffeine-cache_2.12:$playVersion",
'playWs': 'com.typesafe.play:play-ahc-ws-standalone_2.12:2.1.10',
'playDocs': "com.typesafe.play:play-docs_2.12:$playVersion",
'playGuice': "com.typesafe.play:play-guice_2.12:$playVersion",
'playJavaJdbc': "com.typesafe.play:play-java-jdbc_2.12:$playVersion",
'playAkkaHttpServer': "com.typesafe.play:play-akka-http-server_2.12:$playVersion",
'playServer': "com.typesafe.play:play-server_2.12:$playVersion",
'playTest': "com.typesafe.play:play-test_2.12:$playVersion",
'playFilters': "com.typesafe.play:filters-helpers_2.12:$playVersion",
'pac4j': 'org.pac4j:pac4j-oidc:4.5.8',
'playPac4j': 'org.pac4j:play-pac4j_2.12:9.0.2',
'playCache': "com.typesafe.play:play-cache_$playScalaVersion:$playVersion",
'playCaffeineCache': "com.typesafe.play:play-caffeine-cache_$playScalaVersion:$playVersion",
'playWs': "com.typesafe.play:play-ahc-ws-standalone_$playScalaVersion:2.1.10",
'playDocs': "com.typesafe.play:play-docs_$playScalaVersion:$playVersion",
'playGuice': "com.typesafe.play:play-guice_$playScalaVersion:$playVersion",
'playJavaJdbc': "com.typesafe.play:play-java-jdbc_$playScalaVersion:$playVersion",
'playAkkaHttpServer': "com.typesafe.play:play-akka-http-server_$playScalaVersion:$playVersion",
'playServer': "com.typesafe.play:play-server_$playScalaVersion:$playVersion",
'playTest': "com.typesafe.play:play-test_$playScalaVersion:$playVersion",
'playFilters': "com.typesafe.play:filters-helpers_$playScalaVersion:$playVersion",
'pac4j': 'org.pac4j:pac4j-oidc:6.0.6',
'playPac4j': "org.pac4j:play-pac4j_$playScalaVersion:12.0.0-PLAY2.8",
'postgresql': 'org.postgresql:postgresql:42.3.9',
'protobuf': 'com.google.protobuf:protobuf-java:3.25.5',
'grpcProtobuf': 'io.grpc:grpc-protobuf:1.53.0',
Expand Down Expand Up @@ -407,6 +409,8 @@ subprojects {
googleJavaFormat()
target project.fileTree(project.projectDir) {
include 'src/**/*.java'
include 'app/**/*.java'
include 'test/**/*.java'
exclude 'src/**/resources/'
exclude 'src/**/generated/'
exclude 'src/**/mainGeneratedDataTemplate/'
Expand Down
91 changes: 48 additions & 43 deletions datahub-frontend/app/auth/AuthModule.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,27 +21,28 @@
import com.linkedin.util.Configuration;
import config.ConfigurationProvider;
import controllers.SsoCallbackController;
import io.datahubproject.metadata.context.ValidationContext;
import java.nio.charset.StandardCharsets;
import java.util.Collections;

import io.datahubproject.metadata.context.ActorContext;
import io.datahubproject.metadata.context.AuthorizationContext;
import io.datahubproject.metadata.context.EntityRegistryContext;
import io.datahubproject.metadata.context.OperationContext;
import io.datahubproject.metadata.context.OperationContextConfig;
import io.datahubproject.metadata.context.SearchContext;
import io.datahubproject.metadata.context.ValidationContext;
import java.nio.charset.StandardCharsets;
import java.util.Collections;
import javax.annotation.Nonnull;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.pac4j.core.config.Config;
import org.pac4j.core.context.session.SessionStore;
import org.pac4j.core.profile.ProfileManager;
import org.pac4j.core.util.serializer.JavaSerializer;
import org.pac4j.play.LogoutController;
import org.pac4j.play.http.PlayHttpActionAdapter;
import org.pac4j.play.store.PlayCacheSessionStore;
import org.pac4j.play.store.PlayCookieSessionStore;
import org.pac4j.play.store.PlaySessionStore;
import org.pac4j.play.store.ShiroAesDataEncrypter;
import org.springframework.context.annotation.AnnotationConfigApplicationContext;
import play.Environment;
Expand All @@ -63,14 +64,16 @@ public class AuthModule extends AbstractModule {
private static final String PAC4J_SESSIONSTORE_PROVIDER_CONF = "pac4j.sessionStore.provider";
private static final String ENTITY_CLIENT_RETRY_INTERVAL = "entityClient.retryInterval";
private static final String ENTITY_CLIENT_NUM_RETRIES = "entityClient.numRetries";
private static final String ENTITY_CLIENT_RESTLI_GET_BATCH_SIZE = "entityClient.restli.get.batchSize";
private static final String ENTITY_CLIENT_RESTLI_GET_BATCH_CONCURRENCY = "entityClient.restli.get.batchConcurrency";
private static final String ENTITY_CLIENT_RESTLI_GET_BATCH_SIZE =
"entityClient.restli.get.batchSize";
private static final String ENTITY_CLIENT_RESTLI_GET_BATCH_CONCURRENCY =
"entityClient.restli.get.batchConcurrency";
private static final String GET_SSO_SETTINGS_ENDPOINT = "auth/getSsoSettings";

private final com.typesafe.config.Config _configs;
private final com.typesafe.config.Config configs;

public AuthModule(final Environment environment, final com.typesafe.config.Config configs) {
_configs = configs;
this.configs = configs;
}

@Override
Expand All @@ -84,31 +87,32 @@ protected void configure() {
* the response will be rejected by the browser. Default to PlayCacheCookieStore so that
* datahub-frontend container remains as a stateless service
*/
String sessionStoreProvider = _configs.getString(PAC4J_SESSIONSTORE_PROVIDER_CONF);
String sessionStoreProvider = configs.getString(PAC4J_SESSIONSTORE_PROVIDER_CONF);

if (sessionStoreProvider.equals("PlayCacheSessionStore")) {
final PlayCacheSessionStore playCacheSessionStore =
new PlayCacheSessionStore(getProvider(SyncCacheApi.class));
bind(SessionStore.class).toInstance(playCacheSessionStore);
bind(PlaySessionStore.class).toInstance(playCacheSessionStore);
bind(PlayCacheSessionStore.class).toInstance(playCacheSessionStore);
} else {
PlayCookieSessionStore playCacheCookieStore;
try {
// To generate a valid encryption key from an input value, we first
// hash the input to generate a fixed-length string. Then, we convert
// it to hex and slice the first 16 bytes, because AES key length must strictly
// have a specific length.
final String aesKeyBase = _configs.getString(PAC4J_AES_KEY_BASE_CONF);
final String aesKeyBase = configs.getString(PAC4J_AES_KEY_BASE_CONF);
final String aesKeyHash =
DigestUtils.sha256Hex(aesKeyBase.getBytes(StandardCharsets.UTF_8));
final String aesEncryptionKey = aesKeyHash.substring(0, 16);
playCacheCookieStore =
new PlayCookieSessionStore(new ShiroAesDataEncrypter(aesEncryptionKey.getBytes()));
playCacheCookieStore.setSerializer(new JavaSerializer());
} catch (Exception e) {
throw new RuntimeException("Failed to instantiate Pac4j cookie session store!", e);
}
bind(SessionStore.class).toInstance(playCacheCookieStore);
bind(PlaySessionStore.class).toInstance(playCacheCookieStore);
bind(PlayCookieSessionStore.class).toInstance(playCacheCookieStore);
}

try {
Expand All @@ -133,9 +137,12 @@ protected void configure() {

@Provides
@Singleton
protected Config provideConfig() {
protected Config provideConfig(@Nonnull SessionStore sessionStore) {
Config config = new Config();
config.setSessionStoreFactory(parameters -> sessionStore);
config.setHttpActionAdapter(new PlayHttpActionAdapter());
config.setProfileManagerFactory(ProfileManager::new);

return config;
}

Expand All @@ -145,7 +152,7 @@ protected SsoManager provideSsoManager(
Authentication systemAuthentication, CloseableHttpClient httpClient) {
SsoManager manager =
new SsoManager(
_configs, systemAuthentication, getSsoSettingsRequestUrl(_configs), httpClient);
configs, systemAuthentication, getSsoSettingsRequestUrl(configs), httpClient);
manager.initializeSsoProvider();
return manager;
}
Expand All @@ -155,8 +162,8 @@ protected SsoManager provideSsoManager(
protected Authentication provideSystemAuthentication() {
// Returns an instance of Authentication used to authenticate system initiated calls to Metadata
// Service.
String systemClientId = _configs.getString(SYSTEM_CLIENT_ID_CONFIG_PATH);
String systemSecret = _configs.getString(SYSTEM_CLIENT_SECRET_CONFIG_PATH);
String systemClientId = configs.getString(SYSTEM_CLIENT_ID_CONFIG_PATH);
String systemSecret = configs.getString(SYSTEM_CLIENT_SECRET_CONFIG_PATH);
final Actor systemActor =
new Actor(ActorType.USER, systemClientId); // TODO: Change to service actor once supported.
return new Authentication(
Expand All @@ -169,27 +176,25 @@ protected Authentication provideSystemAuthentication() {
@Singleton
@Named("systemOperationContext")
protected OperationContext provideOperationContext(
final Authentication systemAuthentication,
final ConfigurationProvider configurationProvider) {
final Authentication systemAuthentication,
final ConfigurationProvider configurationProvider) {
ActorContext systemActorContext =
ActorContext.builder()
.systemAuth(true)
.authentication(systemAuthentication)
.build();
OperationContextConfig systemConfig = OperationContextConfig.builder()
ActorContext.builder().systemAuth(true).authentication(systemAuthentication).build();
OperationContextConfig systemConfig =
OperationContextConfig.builder()
.viewAuthorizationConfiguration(configurationProvider.getAuthorization().getView())
.allowSystemAuthentication(true)
.build();

return OperationContext.builder()
.operationContextConfig(systemConfig)
.systemActorContext(systemActorContext)
// Authorizer.EMPTY is fine since it doesn't actually apply to system auth
.authorizationContext(AuthorizationContext.builder().authorizer(Authorizer.EMPTY).build())
.searchContext(SearchContext.EMPTY)
.entityRegistryContext(EntityRegistryContext.builder().build(EmptyEntityRegistry.EMPTY))
.validationContext(ValidationContext.builder().alternateValidation(false).build())
.build(systemAuthentication);
.operationContextConfig(systemConfig)
.systemActorContext(systemActorContext)
// Authorizer.EMPTY is fine since it doesn't actually apply to system auth
.authorizationContext(AuthorizationContext.builder().authorizer(Authorizer.EMPTY).build())
.searchContext(SearchContext.EMPTY)
.entityRegistryContext(EntityRegistryContext.builder().build(EmptyEntityRegistry.EMPTY))
.validationContext(ValidationContext.builder().alternateValidation(false).build())
.build(systemAuthentication);
}

@Provides
Expand All @@ -208,23 +213,23 @@ protected SystemEntityClient provideEntityClient(

return new SystemRestliEntityClient(
buildRestliClient(),
new ExponentialBackoff(_configs.getInt(ENTITY_CLIENT_RETRY_INTERVAL)),
_configs.getInt(ENTITY_CLIENT_NUM_RETRIES),
new ExponentialBackoff(configs.getInt(ENTITY_CLIENT_RETRY_INTERVAL)),
configs.getInt(ENTITY_CLIENT_NUM_RETRIES),
configurationProvider.getCache().getClient().getEntityClient(),
Math.max(1, _configs.getInt(ENTITY_CLIENT_RESTLI_GET_BATCH_SIZE)),
Math.max(1, _configs.getInt(ENTITY_CLIENT_RESTLI_GET_BATCH_CONCURRENCY)));
Math.max(1, configs.getInt(ENTITY_CLIENT_RESTLI_GET_BATCH_SIZE)),
Math.max(1, configs.getInt(ENTITY_CLIENT_RESTLI_GET_BATCH_CONCURRENCY)));
}

@Provides
@Singleton
protected AuthServiceClient provideAuthClient(
Authentication systemAuthentication, CloseableHttpClient httpClient) {
// Init a GMS auth client
final String metadataServiceHost = getMetadataServiceHost(_configs);
final String metadataServiceHost = getMetadataServiceHost(configs);

final int metadataServicePort = getMetadataServicePort(_configs);
final int metadataServicePort = getMetadataServicePort(configs);

final boolean metadataServiceUseSsl = doesMetadataServiceUseSsl(_configs);
final boolean metadataServiceUseSsl = doesMetadataServiceUseSsl(configs);

return new AuthServiceClient(
metadataServiceHost,
Expand All @@ -243,22 +248,22 @@ protected CloseableHttpClient provideHttpClient() {
private com.linkedin.restli.client.Client buildRestliClient() {
final String metadataServiceHost =
utils.ConfigUtil.getString(
_configs,
configs,
METADATA_SERVICE_HOST_CONFIG_PATH,
utils.ConfigUtil.DEFAULT_METADATA_SERVICE_HOST);
final int metadataServicePort =
utils.ConfigUtil.getInt(
_configs,
configs,
utils.ConfigUtil.METADATA_SERVICE_PORT_CONFIG_PATH,
utils.ConfigUtil.DEFAULT_METADATA_SERVICE_PORT);
final boolean metadataServiceUseSsl =
utils.ConfigUtil.getBoolean(
_configs,
configs,
utils.ConfigUtil.METADATA_SERVICE_USE_SSL_CONFIG_PATH,
ConfigUtil.DEFAULT_METADATA_SERVICE_USE_SSL);
final String metadataServiceSslProtocol =
utils.ConfigUtil.getString(
_configs,
configs,
utils.ConfigUtil.METADATA_SERVICE_SSL_PROTOCOL_CONFIG_PATH,
ConfigUtil.DEFAULT_METADATA_SERVICE_SSL_PROTOCOL);
return DefaultRestliClientFactory.getRestLiClient(
Expand Down
1 change: 1 addition & 0 deletions datahub-frontend/app/auth/AuthUtils.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,7 @@ public class AuthUtils {
public static final String RESPONSE_MODE = "responseMode";
public static final String USE_NONCE = "useNonce";
public static final String READ_TIMEOUT = "readTimeout";
public static final String CONNECT_TIMEOUT = "connectTimeout";
public static final String EXTRACT_JWT_ACCESS_TOKEN_CLAIMS = "extractJwtAccessTokenClaims";
// Retained for backwards compatibility
public static final String PREFERRED_JWS_ALGORITHM = "preferredJwsAlgorithm";
Expand Down
18 changes: 9 additions & 9 deletions datahub-frontend/app/auth/CookieConfigs.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,34 +10,34 @@ public class CookieConfigs {
public static final String AUTH_COOKIE_SECURE = "play.http.session.secure";
public static final boolean DEFAULT_AUTH_COOKIE_SECURE = false;

private final int _ttlInHours;
private final String _authCookieSameSite;
private final boolean _authCookieSecure;
private final int ttlInHours;
private final String authCookieSameSite;
private final boolean authCookieSecure;

public CookieConfigs(final Config configs) {
_ttlInHours =
ttlInHours =
configs.hasPath(SESSION_TTL_CONFIG_PATH)
? configs.getInt(SESSION_TTL_CONFIG_PATH)
: DEFAULT_SESSION_TTL_HOURS;
_authCookieSameSite =
authCookieSameSite =
configs.hasPath(AUTH_COOKIE_SAME_SITE)
? configs.getString(AUTH_COOKIE_SAME_SITE)
: DEFAULT_AUTH_COOKIE_SAME_SITE;
_authCookieSecure =
authCookieSecure =
configs.hasPath(AUTH_COOKIE_SECURE)
? configs.getBoolean(AUTH_COOKIE_SECURE)
: DEFAULT_AUTH_COOKIE_SECURE;
}

public int getTtlInHours() {
return _ttlInHours;
return ttlInHours;
}

public String getAuthCookieSameSite() {
return _authCookieSameSite;
return authCookieSameSite;
}

public boolean getAuthCookieSecure() {
return _authCookieSecure;
return authCookieSecure;
}
}
6 changes: 3 additions & 3 deletions datahub-frontend/app/auth/JAASConfigs.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,16 +8,16 @@ public class JAASConfigs {

public static final String JAAS_ENABLED_CONFIG_PATH = "auth.jaas.enabled";

private Boolean _isEnabled = true;
private Boolean isEnabled = true;

public JAASConfigs(final com.typesafe.config.Config configs) {
if (configs.hasPath(JAAS_ENABLED_CONFIG_PATH)
&& !configs.getBoolean(JAAS_ENABLED_CONFIG_PATH)) {
_isEnabled = false;
isEnabled = false;
}
}

public boolean isJAASEnabled() {
return _isEnabled;
return isEnabled;
}
}
12 changes: 6 additions & 6 deletions datahub-frontend/app/auth/NativeAuthenticationConfigs.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,17 +7,17 @@ public class NativeAuthenticationConfigs {
public static final String NATIVE_AUTHENTICATION_ENFORCE_VALID_EMAIL_ENABLED_CONFIG_PATH =
"auth.native.signUp.enforceValidEmail";

private Boolean _isEnabled = true;
private Boolean _isEnforceValidEmailEnabled = true;
private Boolean isEnabled = true;
private Boolean isEnforceValidEmailEnabled = true;

public NativeAuthenticationConfigs(final com.typesafe.config.Config configs) {
if (configs.hasPath(NATIVE_AUTHENTICATION_ENABLED_CONFIG_PATH)) {
_isEnabled =
isEnabled =
Boolean.parseBoolean(
configs.getValue(NATIVE_AUTHENTICATION_ENABLED_CONFIG_PATH).toString());
}
if (configs.hasPath(NATIVE_AUTHENTICATION_ENFORCE_VALID_EMAIL_ENABLED_CONFIG_PATH)) {
_isEnforceValidEmailEnabled =
isEnforceValidEmailEnabled =
Boolean.parseBoolean(
configs
.getValue(NATIVE_AUTHENTICATION_ENFORCE_VALID_EMAIL_ENABLED_CONFIG_PATH)
Expand All @@ -26,10 +26,10 @@ public NativeAuthenticationConfigs(final com.typesafe.config.Config configs) {
}

public boolean isNativeAuthenticationEnabled() {
return _isEnabled;
return isEnabled;
}

public boolean isEnforceValidEmailEnabled() {
return _isEnforceValidEmailEnabled;
return isEnforceValidEmailEnabled;
}
}
Loading

0 comments on commit d2a25fa

Please sign in to comment.