Merge pull request #16 from rtekal/rtekal-pl-join-ui-ERModelRelation

Rtekal pl join UI er model relation

.github/workflows/airflow-plugin.yml

@@ -87,8 +87,8 @@ jobs:
           token: ${{ secrets.CODECOV_TOKEN }}
           directory: .
           fail_ci_if_error: false
-          flags: airflow-${{ matrix.python-version }}-${{ matrix.extraPythonRequirement }}
-          name: pytest-airflow
+          flags: airflow,airflow-${{ matrix.extra_pip_extras }}
+          name: pytest-airflow-${{ matrix.python-version }}-${{ matrix.extra_pip_requirements }}
           verbose: true
 
   event-file:

.github/workflows/build-and-test.yml

@@ -83,6 +83,10 @@ jobs:
           ./gradlew :datahub-frontend:build :datahub-web-react:build --parallel
         env:
           NODE_OPTIONS: "--max-old-space-size=3072"
+      - name: Gradle compile (jdk8) for legacy Spark
+        if: ${{  matrix.command == 'except_metadata_ingestion' && needs.setup.outputs.backend_change == 'true' }}
+        run: |
+          ./gradlew -PjavaClassVersionDefault=8 :metadata-integration:java:spark-lineage:compileJava
       - uses: actions/upload-artifact@v3
         if: always()
         with:

.github/workflows/docker-unified.yml

@@ -64,6 +64,18 @@ jobs:
     steps:
       - name: Check out the repo
         uses: hsheth2/sane-checkout-action@v1
+      - uses: actions/setup-python@v4
+        with:
+          python-version: "3.10"
+          cache: "pip"
+      - name: Set up JDK 17
+        uses: actions/setup-java@v3
+        with:
+          distribution: "zulu"
+          java-version: 17
+      - name: Run lint on smoke test
+        run: |
+          ./gradlew :smoke-test:lint
       - name: Compute Tag
         id: tag
         run: |

.github/workflows/metadata-ingestion.yml

@@ -31,7 +31,7 @@ jobs:
       # DATAHUB_LOOKML_GIT_TEST_SSH_KEY: ${{ secrets.DATAHUB_LOOKML_GIT_TEST_SSH_KEY }}
     strategy:
       matrix:
-        python-version: ["3.7", "3.10"]
+        python-version: ["3.8", "3.10"]
         command:
           [
             "testQuick",
@@ -40,7 +40,7 @@ jobs:
             "testIntegrationBatch2",
           ]
         include:
-          - python-version: "3.7"
+          - python-version: "3.8"
           - python-version: "3.10"
       fail-fast: false
     steps:

.github/workflows/spark-smoke-test.yml

@@ -42,8 +42,12 @@ jobs:
           cache: "pip"
       - name: Install dependencies
         run: ./metadata-ingestion/scripts/install_deps.sh
+      - name: Disk Check
+        run: df -h . && docker images
       - name: Remove images
         run: docker image prune -a -f || true
+      - name: Disk Check
+        run: df -h . && docker images
       - name: Smoke test
         run: |
           ./gradlew :metadata-integration:java:spark-lineage:integrationTest \
@@ -54,12 +58,24 @@ jobs:
              -x :datahub-web-react:yarnBuild \
              -x :datahub-web-react:distZip \
              -x :datahub-web-react:jar
+      - name: store logs
+        if: failure()
+        run: |
+          docker ps -a
+          docker logs datahub-gms >& gms-${{ matrix.test_strategy }}.log || true
+          docker logs datahub-actions >& actions-${{ matrix.test_strategy }}.log || true
+          docker logs broker >& broker-${{ matrix.test_strategy }}.log || true
+          docker logs mysql >& mysql-${{ matrix.test_strategy }}.log || true
+          docker logs elasticsearch >& elasticsearch-${{ matrix.test_strategy }}.log || true
+          docker logs datahub-frontend-react >& frontend-${{ matrix.test_strategy }}.log || true
       - name: Upload logs
         uses: actions/upload-artifact@v3
         if: failure()
         with:
           name: docker logs
-          path: "docker/build/container-logs/*.log"
+          path: |
+            "**/build/container-logs/*.log"
+            "*.log"
       - uses: actions/upload-artifact@v3
         if: always()
         with:

build.gradle

@@ -1,6 +1,32 @@
 buildscript {
-  ext.jdkVersion = 17
-  ext.javaClassVersion = 11
+  ext.jdkVersionDefault = 17
+  ext.javaClassVersionDefault = 11
+
+  ext.jdkVersion = { p ->
+    // If Spring 6 is present, hard dependency on jdk17
+    if (p.configurations.any { it.getDependencies().any{
+      (it.getGroup().equals("org.springframework") && it.getVersion().startsWith("6."))
+      || (it.getGroup().equals("org.springframework.boot") && it.getVersion().startsWith("3.") && !it.getName().equals("spring-boot-starter-test"))
+    }}) {
+      return 17
+    } else {
+      // otherwise we can use the preferred default which can be overridden with a property: -PjdkVersionDefault
+      return p.hasProperty('jdkVersionDefault') ? Integer.valueOf((String) p.getProperty('jdkVersionDefault')) : ext.jdkVersionDefault
+    }
+  }
+
+  ext.javaClassVersion = { p ->
+    // If Spring 6 is present, hard dependency on jdk17
+    if (p.configurations.any { it.getDependencies().any{
+      (it.getGroup().equals("org.springframework") && it.getVersion().startsWith("6."))
+              || (it.getGroup().equals("org.springframework.boot") && it.getVersion().startsWith("3.") && !it.getName().equals("spring-boot-starter-test"))
+    }}) {
+      return 17
+    } else {
+      // otherwise we can use the preferred default which can be overridden with a property: -PjavaClassVersionDefault
+      return p.hasProperty('javaClassVersionDefault') ? Integer.valueOf((String) p.getProperty('javaClassVersionDefault')) : ext.javaClassVersionDefault
+    }
+  }
 
   ext.junitJupiterVersion = '5.6.1'
   // Releases: https://github.com/linkedin/rest.li/blob/master/CHANGELOG.md
@@ -217,6 +243,7 @@ project.ext.externalDependency = [
     'springActuator': "org.springframework.boot:spring-boot-starter-actuator:$springBootVersion",
     'swaggerAnnotations': 'io.swagger.core.v3:swagger-annotations:2.2.15',
     'swaggerCli': 'io.swagger.codegen.v3:swagger-codegen-cli:3.0.46',
+    'springBootAutoconfigureJdk11': 'org.springframework.boot:spring-boot-autoconfigure:2.7.18',
     'testng': 'org.testng:testng:7.8.0',
     'testContainers': 'org.testcontainers:testcontainers:' + testContainersVersion,
     'testContainersJunit': 'org.testcontainers:junit-jupiter:' + testContainersVersion,
@@ -252,48 +279,56 @@ allprojects {
     }
   }
 
-  if (project.plugins.hasPlugin('java')
+  /**
+   * If making changes to this section also see the sections for pegasus below
+   * which use project.plugins.hasPlugin('pegasus')
+   **/
+  if (!project.plugins.hasPlugin('pegasus') && (project.plugins.hasPlugin('java')
           || project.plugins.hasPlugin('java-library')
-          || project.plugins.hasPlugin('application')
-          || project.plugins.hasPlugin('pegasus')) {
+          || project.plugins.hasPlugin('application'))) {
 
     java {
       toolchain {
-        languageVersion = JavaLanguageVersion.of(jdkVersion)
+        languageVersion = JavaLanguageVersion.of(jdkVersion(project))
       }
     }
 
     compileJava {
-      options.release = javaClassVersion
+      options.release = javaClassVersion(project)
     }
+
     tasks.withType(JavaCompile).configureEach {
       javaCompiler = javaToolchains.compilerFor {
-        languageVersion = JavaLanguageVersion.of(jdkVersion)
+        languageVersion = JavaLanguageVersion.of(jdkVersion(project))
       }
       // Puts parameter names into compiled class files, necessary for Spring 6
       options.compilerArgs.add("-parameters")
     }
 
     tasks.withType(JavaExec).configureEach {
       javaLauncher = javaToolchains.launcherFor {
-        languageVersion = JavaLanguageVersion.of(jdkVersion)
+        languageVersion = JavaLanguageVersion.of(jdkVersion(project))
       }
     }
+  }
+
+  // not duplicated, need to set this outside and inside afterEvaluate
+  afterEvaluate {
+    /**
+     * If making changes to this section also see the sections for pegasus below
+     * which use project.plugins.hasPlugin('pegasus')
+     **/
+    if (!project.plugins.hasPlugin('pegasus') && (project.plugins.hasPlugin('java')
+            || project.plugins.hasPlugin('java-library')
+            || project.plugins.hasPlugin('application'))) {
 
-    // not duplicated, need to set this outside and inside afterEvaluate
-    afterEvaluate {
       compileJava {
-        options.release = javaClassVersion
-      }
-      tasks.withType(JavaCompile).configureEach {
-        javaCompiler = javaToolchains.compilerFor {
-          languageVersion = JavaLanguageVersion.of(jdkVersion)
-        }
+        options.release = javaClassVersion(project)
       }
 
       tasks.withType(JavaExec).configureEach {
         javaLauncher = javaToolchains.launcherFor {
-          languageVersion = JavaLanguageVersion.of(jdkVersion)
+          languageVersion = JavaLanguageVersion.of(jdkVersion(project))
         }
       }
     }
@@ -368,6 +403,30 @@ subprojects {
         dataTemplateCompile externalDependency.annotationApi // support > jdk8
         restClientCompile spec.product.pegasus.restliClient
       }
+
+      java {
+        toolchain {
+          languageVersion = JavaLanguageVersion.of(jdkVersion(project))
+        }
+      }
+
+      compileJava {
+        options.release = javaClassVersion(project)
+      }
+
+      tasks.withType(JavaCompile).configureEach {
+        javaCompiler = javaToolchains.compilerFor {
+          languageVersion = JavaLanguageVersion.of(jdkVersion(project))
+        }
+        // Puts parameter names into compiled class files, necessary for Spring 6
+        options.compilerArgs.add("-parameters")
+      }
+
+      tasks.withType(JavaExec).configureEach {
+        javaLauncher = javaToolchains.launcherFor {
+          languageVersion = JavaLanguageVersion.of(jdkVersion(project))
+        }
+      }
     }
 
     afterEvaluate {
@@ -394,6 +453,16 @@ subprojects {
         dataTemplateCompile externalDependency.annotationApi // support > jdk8
         restClientCompile spec.product.pegasus.restliClient
       }
+
+      compileJava {
+        options.release = javaClassVersion(project)
+      }
+
+      tasks.withType(JavaExec).configureEach {
+        javaLauncher = javaToolchains.launcherFor {
+          languageVersion = JavaLanguageVersion.of(jdkVersion(project))
+        }
+      }
     }
   }
 }

datahub-frontend/app/auth/AuthUtils.java

@@ -76,6 +76,9 @@ public class AuthUtils {
   public static final String USE_NONCE = "useNonce";
   public static final String READ_TIMEOUT = "readTimeout";
   public static final String EXTRACT_JWT_ACCESS_TOKEN_CLAIMS = "extractJwtAccessTokenClaims";
+  // Retained for backwards compatibility
+  public static final String PREFERRED_JWS_ALGORITHM = "preferredJwsAlgorithm";
+  public static final String PREFERRED_JWS_ALGORITHM_2 = "preferredJwsAlgorithm2";
 
   /**
    * Determines whether the inbound request should be forward to downstream Metadata Service. Today,

datahub-frontend/app/auth/sso/oidc/OidcConfigs.java

@@ -226,8 +226,8 @@ public Builder from(final com.typesafe.config.Config configs, final String ssoSe
         extractJwtAccessTokenClaims =
             Optional.of(jsonNode.get(EXTRACT_JWT_ACCESS_TOKEN_CLAIMS).asBoolean());
       }
-      if (jsonNode.has(OIDC_PREFERRED_JWS_ALGORITHM)) {
-        preferredJwsAlgorithm = Optional.of(jsonNode.get(OIDC_PREFERRED_JWS_ALGORITHM).asText());
+      if (jsonNode.has(PREFERRED_JWS_ALGORITHM_2)) {
+        preferredJwsAlgorithm = Optional.of(jsonNode.get(PREFERRED_JWS_ALGORITHM_2).asText());
       } else {
         preferredJwsAlgorithm =
             Optional.ofNullable(getOptional(configs, OIDC_PREFERRED_JWS_ALGORITHM, null));

datahub-frontend/build.gradle

@@ -87,7 +87,7 @@ docker {
   }
 }
 
-task unversionZip(type: Copy, dependsOn: [':datahub-web-react:build', dist]) {
+task unversionZip(type: Copy, dependsOn: [':datahub-web-react:distZip', dist]) {
   from ("${buildDir}/distributions")
   include "datahub-frontend-${version}.zip"
   into "${buildDir}/docker/"

datahub-frontend/play.gradle

@@ -101,6 +101,9 @@ play {
 test {
   useJUnitPlatform()
 
+  testLogging.showStandardStreams = true
+  testLogging.exceptionFormat = 'full'
+
   def playJava17CompatibleJvmArgs = [
           "--add-opens=java.base/java.lang=ALL-UNNAMED",
           //"--add-opens=java.base/java.lang.invoke=ALL-UNNAMED",

datahub-frontend/test/security/OidcConfigurationTest.java

@@ -1,5 +1,6 @@
 package security;
 
+import static auth.AuthUtils.*;
 import static auth.sso.oidc.OidcConfigs.*;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 
@@ -24,6 +25,7 @@
 import java.util.concurrent.TimeUnit;
 import org.junit.jupiter.api.Test;
 import org.pac4j.oidc.client.OidcClient;
+import org.json.JSONObject;
 
 public class OidcConfigurationTest {
 
@@ -317,4 +319,26 @@ public void readTimeoutPropagation() {
     OidcProvider oidcProvider = new OidcProvider(oidcConfigs);
     assertEquals(10000, ((OidcClient) oidcProvider.client()).getConfiguration().getReadTimeout());
   }
+
+  @Test
+  public void readPreferredJwsAlgorithmPropagationFromConfig() {
+    final String SSO_SETTINGS_JSON_STR = new JSONObject().put(PREFERRED_JWS_ALGORITHM, "HS256").toString();
+    CONFIG.withValue(OIDC_PREFERRED_JWS_ALGORITHM, ConfigValueFactory.fromAnyRef("RS256"));
+    OidcConfigs.Builder oidcConfigsBuilder = new OidcConfigs.Builder();
+    oidcConfigsBuilder.from(CONFIG, SSO_SETTINGS_JSON_STR);
+    OidcConfigs oidcConfigs = new OidcConfigs(oidcConfigsBuilder);
+    OidcProvider oidcProvider = new OidcProvider(oidcConfigs);
+    assertEquals("RS256", ((OidcClient) oidcProvider.client()).getConfiguration().getPreferredJwsAlgorithm().toString());
+  }
+
+  @Test
+  public void readPreferredJwsAlgorithmPropagationFromJSON() {
+    final String SSO_SETTINGS_JSON_STR = new JSONObject().put(PREFERRED_JWS_ALGORITHM, "Unused").put(PREFERRED_JWS_ALGORITHM_2, "HS256").toString();
+    CONFIG.withValue(OIDC_PREFERRED_JWS_ALGORITHM, ConfigValueFactory.fromAnyRef("RS256"));
+    OidcConfigs.Builder oidcConfigsBuilder = new OidcConfigs.Builder();
+    oidcConfigsBuilder.from(CONFIG, SSO_SETTINGS_JSON_STR);
+    OidcConfigs oidcConfigs = new OidcConfigs(oidcConfigsBuilder);
+    OidcProvider oidcProvider = new OidcProvider(oidcConfigs);
+    assertEquals("HS256", ((OidcClient) oidcProvider.client()).getConfiguration().getPreferredJwsAlgorithm().toString());
+  }
 }

datahub-graphql-core/build.gradle

@@ -31,30 +31,16 @@ dependencies {
 
 graphqlCodegen {
     // For options: https://github.com/kobylynskyi/graphql-java-codegen/blob/master/docs/codegen-options.md
-    graphqlSchemaPaths = [
-        "$projectDir/src/main/resources/entity.graphql".toString(),
-        "$projectDir/src/main/resources/app.graphql".toString(),
-        "$projectDir/src/main/resources/search.graphql".toString(),
-        "$projectDir/src/main/resources/analytics.graphql".toString(),
-        "$projectDir/src/main/resources/recommendation.graphql".toString(),
-        "$projectDir/src/main/resources/ingestion.graphql".toString(),
-        "$projectDir/src/main/resources/auth.graphql".toString(),
-        "$projectDir/src/main/resources/timeline.graphql".toString(),
-        "$projectDir/src/main/resources/tests.graphql".toString(),
-        "$projectDir/src/main/resources/properties.graphql".toString(),
-        "$projectDir/src/main/resources/step.graphql".toString(),
-        "$projectDir/src/main/resources/lineage.graphql".toString(),
-        "$projectDir/src/main/resources/forms.graphql".toString()
-    ]
-    outputDir = new File("$projectDir/src/mainGeneratedGraphQL/java")
+    graphqlSchemaPaths = fileTree(dir: "${projectDir}/src/main/resources", include: '**/*.graphql').collect { it.absolutePath }
+    outputDir = new File("${projectDir}/src/mainGeneratedGraphQL/java")
     packageName = "com.linkedin.datahub.graphql.generated"
     generateToString = true
     generateApis = true
     generateParameterizedFieldsResolvers = false
     modelValidationAnnotation = "@javax.annotation.Nonnull"
     customTypesMapping = [
-            Long: "Long",
-            Float: "Float"
+        Long: "Long",
+        Float: "Float"
     ]
 }