Merge branch 'master' into jj--incidents-oss-v1

datahub-web-react/src/App.tsx

@@ -1,20 +1,19 @@
-import React, { useEffect, useState } from 'react';
+import React from 'react';
 import Cookies from 'js-cookie';
 import { message } from 'antd';
 import { BrowserRouter as Router } from 'react-router-dom';
 import { ApolloClient, ApolloProvider, createHttpLink, InMemoryCache, ServerError } from '@apollo/client';
 import { onError } from '@apollo/client/link/error';
-import { ThemeProvider } from 'styled-components';
 import { Helmet, HelmetProvider } from 'react-helmet-async';
 import './App.less';
 import { Routes } from './app/Routes';
-import { Theme } from './conf/theme/types';
-import defaultThemeConfig from './conf/theme/theme_light.config.json';
 import { PageRoutes } from './conf/Global';
 import { isLoggedInVar } from './app/auth/checkAuthStatus';
 import { GlobalCfg } from './conf';
 import possibleTypesResult from './possibleTypes.generated';
 import { ErrorCodes } from './app/shared/constants';
+import CustomThemeProvider from './CustomThemeProvider';
+import { useCustomTheme } from './customThemeContext';
 
 /*
     Construct Apollo Client
@@ -76,33 +75,16 @@ const client = new ApolloClient({
 });
 
 export const InnerApp: React.VFC = () => {
-    const [dynamicThemeConfig, setDynamicThemeConfig] = useState<Theme>(defaultThemeConfig);
-
-    useEffect(() => {
-        if (import.meta.env.DEV) {
-            import(/* @vite-ignore */ `./conf/theme/${import.meta.env.REACT_APP_THEME_CONFIG}`).then((theme) => {
-                setDynamicThemeConfig(theme);
-            });
-        } else {
-            // Send a request to the server to get the theme config.
-            fetch(`/assets/conf/theme/${import.meta.env.REACT_APP_THEME_CONFIG}`)
-                .then((response) => response.json())
-                .then((theme) => {
-                    setDynamicThemeConfig(theme);
-                });
-        }
-    }, []);
-
     return (
         <HelmetProvider>
-            <Helmet>
-                <title>{dynamicThemeConfig.content.title}</title>
-            </Helmet>
-            <ThemeProvider theme={dynamicThemeConfig}>
+            <CustomThemeProvider>
+                <Helmet>
+                    <title>{useCustomTheme().theme?.content.title}</title>
+                </Helmet>
                 <Router>
                     <Routes />
                 </Router>
-            </ThemeProvider>
+            </CustomThemeProvider>
         </HelmetProvider>
     );
 };

datahub-web-react/src/CustomThemeProvider.tsx

@@ -0,0 +1,32 @@
+import React, { useEffect, useState } from 'react';
+import { ThemeProvider } from 'styled-components';
+import { Theme } from './conf/theme/types';
+import defaultThemeConfig from './conf/theme/theme_light.config.json';
+import { CustomThemeContext } from './customThemeContext';
+
+const CustomThemeProvider = ({ children }: { children: React.ReactNode }) => {
+    const [currentTheme, setTheme] = useState<Theme>(defaultThemeConfig);
+
+    useEffect(() => {
+        if (import.meta.env.DEV) {
+            import(/* @vite-ignore */ `./conf/theme/${import.meta.env.REACT_APP_THEME_CONFIG}`).then((theme) => {
+                setTheme(theme);
+            });
+        } else {
+            // Send a request to the server to get the theme config.
+            fetch(`/assets/conf/theme/${import.meta.env.REACT_APP_THEME_CONFIG}`)
+                .then((response) => response.json())
+                .then((theme) => {
+                    setTheme(theme);
+                });
+        }
+    }, []);
+
+    return (
+        <CustomThemeContext.Provider value={{ theme: currentTheme, updateTheme: setTheme }}>
+            <ThemeProvider theme={currentTheme}>{children}</ThemeProvider>
+        </CustomThemeContext.Provider>
+    );
+};
+
+export default CustomThemeProvider;

datahub-web-react/src/customThemeContext.tsx

@@ -0,0 +1,10 @@
+import React, { useContext } from 'react';
+
+export const CustomThemeContext = React.createContext<{
+    theme: any;
+    updateTheme: (theme: any) => void;
+}>({ theme: undefined, updateTheme: (_) => null });
+
+export function useCustomTheme() {
+    return useContext(CustomThemeContext);
+}

entity-registry/src/main/java/com/linkedin/metadata/models/ConfigEntitySpec.java

@@ -3,10 +3,12 @@
 import com.linkedin.data.schema.RecordDataSchema;
 import com.linkedin.data.schema.TyperefDataSchema;
 import com.linkedin.metadata.models.annotation.EntityAnnotation;
+import com.linkedin.metadata.models.annotation.SearchableAnnotation;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.function.Function;
 import java.util.stream.Collectors;
 import javax.annotation.Nonnull;
@@ -19,6 +21,7 @@ public class ConfigEntitySpec implements EntitySpec {
   private final Map<String, AspectSpec> _aspectSpecs;
 
   private List<SearchableFieldSpec> _searchableFieldSpecs;
+  private Map<String, Set<SearchableAnnotation.FieldType>> searchableFieldTypeMap;
 
   public ConfigEntitySpec(
       @Nonnull final String entityName,
@@ -89,4 +92,13 @@ public List<SearchableFieldSpec> getSearchableFieldSpecs() {
 
     return _searchableFieldSpecs;
   }
+
+  @Override
+  public Map<String, Set<SearchableAnnotation.FieldType>> getSearchableFieldTypes() {
+    if (searchableFieldTypeMap == null) {
+      searchableFieldTypeMap = EntitySpec.super.getSearchableFieldTypes();
+    }
+
+    return searchableFieldTypeMap;
+  }
 }

entity-registry/src/main/java/com/linkedin/metadata/models/DefaultEntitySpec.java

@@ -3,10 +3,12 @@
 import com.linkedin.data.schema.RecordDataSchema;
 import com.linkedin.data.schema.TyperefDataSchema;
 import com.linkedin.metadata.models.annotation.EntityAnnotation;
+import com.linkedin.metadata.models.annotation.SearchableAnnotation;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.function.Function;
 import java.util.stream.Collectors;
 import javax.annotation.Nonnull;
@@ -24,6 +26,7 @@ public class DefaultEntitySpec implements EntitySpec {
   private final TyperefDataSchema _aspectTyperefSchema;
 
   private List<SearchableFieldSpec> _searchableFieldSpecs;
+  private Map<String, Set<SearchableAnnotation.FieldType>> searchableFieldTypeMap;
 
   public DefaultEntitySpec(
       @Nonnull final Collection<AspectSpec> aspectSpecs,
@@ -102,4 +105,13 @@ public List<SearchableFieldSpec> getSearchableFieldSpecs() {
 
     return _searchableFieldSpecs;
   }
+
+  @Override
+  public Map<String, Set<SearchableAnnotation.FieldType>> getSearchableFieldTypes() {
+    if (searchableFieldTypeMap == null) {
+      searchableFieldTypeMap = EntitySpec.super.getSearchableFieldTypes();
+    }
+
+    return searchableFieldTypeMap;
+  }
 }

entity-registry/src/main/java/com/linkedin/metadata/models/EntitySpec.java

@@ -3,7 +3,9 @@
 import com.linkedin.data.schema.RecordDataSchema;
 import com.linkedin.data.schema.TyperefDataSchema;
 import com.linkedin.metadata.models.annotation.EntityAnnotation;
+import com.linkedin.metadata.models.annotation.SearchableAnnotation;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
@@ -39,16 +41,39 @@ default List<SearchableFieldSpec> getSearchableFieldSpecs() {
         .collect(Collectors.toList());
   }
 
-  default Map<String, Set<SearchableFieldSpec>> getSearchableFieldSpecMap() {
-    return getSearchableFieldSpecs().stream()
-        .collect(
-            Collectors.toMap(
-                searchableFieldSpec -> searchableFieldSpec.getSearchableAnnotation().getFieldName(),
-                searchableFieldSpec -> new HashSet<>(Collections.singleton(searchableFieldSpec)),
-                (set1, set2) -> {
-                  set1.addAll(set2);
-                  return set1;
-                }));
+  default Map<String, Set<SearchableAnnotation.FieldType>> getSearchableFieldTypes() {
+    // Get additional fields and mint SearchableFieldSpecs for them
+    Map<String, Set<SearchableAnnotation.FieldType>> fieldSpecMap = new HashMap<>();
+    for (SearchableFieldSpec fieldSpec : getSearchableFieldSpecs()) {
+      SearchableAnnotation searchableAnnotation = fieldSpec.getSearchableAnnotation();
+      if (searchableAnnotation.getNumValuesFieldName().isPresent()) {
+        String fieldName = searchableAnnotation.getNumValuesFieldName().get();
+        Set<SearchableAnnotation.FieldType> fieldSet = new HashSet<>();
+        fieldSet.add(SearchableAnnotation.FieldType.COUNT);
+        fieldSpecMap.put(fieldName, fieldSet);
+      }
+      if (searchableAnnotation.getHasValuesFieldName().isPresent()) {
+        String fieldName = searchableAnnotation.getHasValuesFieldName().get();
+        Set<SearchableAnnotation.FieldType> fieldSet = new HashSet<>();
+        fieldSet.add(SearchableAnnotation.FieldType.BOOLEAN);
+        fieldSpecMap.put(fieldName, fieldSet);
+      }
+    }
+    fieldSpecMap.putAll(
+        getSearchableFieldSpecs().stream()
+            .collect(
+                Collectors.toMap(
+                    searchableFieldSpec ->
+                        searchableFieldSpec.getSearchableAnnotation().getFieldName(),
+                    searchableFieldSpec ->
+                        new HashSet<>(
+                            Collections.singleton(
+                                searchableFieldSpec.getSearchableAnnotation().getFieldType())),
+                    (set1, set2) -> {
+                      set1.addAll(set2);
+                      return set1;
+                    })));
+    return fieldSpecMap;
   }
 
   default List<SearchScoreFieldSpec> getSearchScoreFieldSpecs() {

metadata-ingestion/developing.md

@@ -10,7 +10,7 @@ Also take a look at the guide to [adding a source](./adding-source.md).
 ### Requirements
 
 1. Python 3.7+ must be installed in your host environment.
-2. Java8 (gradle won't work with newer versions)
+2. Java 17 (gradle won't work with newer or older versions)
 4. On Debian/Ubuntu: `sudo apt install python3-dev python3-venv`
 5. On Fedora (if using LDAP source integration): `sudo yum install openldap-devel`
 

metadata-ingestion/docs/sources/metabase/metabase.md

@@ -19,4 +19,4 @@ The key in this map must be string, not integer although  Metabase API provides
 If `database_id_to_instance_map` is not specified, `platform_instance_map` is used for platform instance mapping. If none of the above are specified, platform instance is not used when constructing `urn` when searching for dataset relations.
 ## Compatibility
 
-Metabase version [v0.41.2](https://www.metabase.com/start/oss/)
+Metabase version [v0.48.3](https://www.metabase.com/start/oss/)

metadata-ingestion/src/datahub/ingestion/source/aws/glue.py

@@ -833,9 +833,8 @@ def get_profile_if_enabled(
                 **{k: v for k, v in kwargs.items() if v}
             )
 
-            partition_keys = response["Table"]["PartitionKeys"]
-
             # check if this table is partitioned
+            partition_keys = response["Table"].get("PartitionKeys")
             if partition_keys:
                 # ingest data profile with partitions
                 # for cross-account ingestion

metadata-ingestion/src/datahub/ingestion/source/metabase.py

@@ -90,10 +90,17 @@ class MetabaseSource(Source):
     """
     This plugin extracts Charts, dashboards, and associated metadata. This plugin is in beta and has only been tested
     on PostgreSQL and H2 database.
-    ### Dashboard
 
-    [/api/dashboard](https://www.metabase.com/docs/latest/api-documentation.html#dashboard) endpoint is used to
-    retrieve the following dashboard information.
+    ### Collection
+
+    [/api/collection](https://www.metabase.com/docs/latest/api/collection) endpoint is used to
+    retrieve the available collections.
+
+    [/api/collection/<COLLECTION_ID>/items?models=dashboard](https://www.metabase.com/docs/latest/api/collection#get-apicollectioniditems) endpoint is used to retrieve a given collection and list their dashboards.
+
+     ### Dashboard
+
+    [/api/dashboard/<DASHBOARD_ID>](https://www.metabase.com/docs/latest/api/dashboard) endpoint is used to retrieve a given Dashboard and grab its information.
 
     - Title and description
     - Last edited by
@@ -187,19 +194,29 @@ def close(self) -> None:
 
     def emit_dashboard_mces(self) -> Iterable[MetadataWorkUnit]:
         try:
-            dashboard_response = self.session.get(
-                f"{self.config.connect_uri}/api/dashboard"
+            collections_response = self.session.get(
+                f"{self.config.connect_uri}/api/collection/"
             )
-            dashboard_response.raise_for_status()
-            dashboards = dashboard_response.json()
+            collections_response.raise_for_status()
+            collections = collections_response.json()
 
-            for dashboard_info in dashboards:
-                dashboard_snapshot = self.construct_dashboard_from_api_data(
-                    dashboard_info
+            for collection in collections:
+                collection_dashboards_response = self.session.get(
+                    f"{self.config.connect_uri}/api/collection/{collection['id']}/items?models=dashboard"
                 )
-                if dashboard_snapshot is not None:
-                    mce = MetadataChangeEvent(proposedSnapshot=dashboard_snapshot)
-                    yield MetadataWorkUnit(id=dashboard_snapshot.urn, mce=mce)
+                collection_dashboards_response.raise_for_status()
+                collection_dashboards = collection_dashboards_response.json()
+
+                if not collection_dashboards.get("data"):
+                    continue
+
+                for dashboard_info in collection_dashboards.get("data"):
+                    dashboard_snapshot = self.construct_dashboard_from_api_data(
+                        dashboard_info
+                    )
+                    if dashboard_snapshot is not None:
+                        mce = MetadataChangeEvent(proposedSnapshot=dashboard_snapshot)
+                        yield MetadataWorkUnit(id=dashboard_snapshot.urn, mce=mce)
 
         except HTTPError as http_error:
             self.report.report_failure(
@@ -254,10 +271,10 @@ def construct_dashboard_from_api_data(
         )
 
         chart_urns = []
-        cards_data = dashboard_details.get("ordered_cards", "{}")
+        cards_data = dashboard_details.get("dashcards", {})
         for card_info in cards_data:
             chart_urn = builder.make_chart_urn(
-                self.platform, card_info.get("card_id", "")
+                self.platform, card_info.get("card").get("id", "")
             )
             chart_urns.append(chart_urn)