From cacfabd9ce6060179be2a36e0d0b23c14bd9d489 Mon Sep 17 00:00:00 2001 From: Teddy Date: Wed, 22 Jan 2025 08:08:07 +0100 Subject: [PATCH 1/3] ISSUE #1052 - Implement List entities with test suites repo logic (#19461) * fix: centralized listWithOffset logic * feat: dq app config + list entities with tests logic * fix: test case --- .../service/jdbi3/CollectionDAO.java | 31 +++++ .../service/jdbi3/EntityRepository.java | 127 +++++++++++++----- .../jdbi3/EntityTimeSeriesRepository.java | 23 +--- .../service/util/ListWithOffsetFunction.java | 14 ++ .../service/util/jdbi/JdbiUtils.java | 21 +++ .../searchIndex/PaginatedEntitiesSource.java | 25 +++- .../elasticsearch/en/table_index_mapping.json | 8 +- .../databases/TableResourceTest.java | 45 ++++++- .../configuration/applicationConfig.json | 5 +- .../collateAIQualityAgentAppConfig.json | 37 +++++ .../collateAIQualityAgentAppConfig.ts | 40 ++++++ .../internal/dataRetentionConfiguration.ts | 2 +- 12 files changed, 316 insertions(+), 62 deletions(-) create mode 100644 openmetadata-service/src/main/java/org/openmetadata/service/util/ListWithOffsetFunction.java create mode 100644 openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/internal/collateAIQualityAgentAppConfig.json create mode 100644 openmetadata-ui/src/main/resources/ui/src/generated/entity/applications/configuration/internal/collateAIQualityAgentAppConfig.ts diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java index 07c71724b755..28a6dfea4e6e 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java @@ -4650,6 +4650,37 @@ default List listAfter(ListFilter filter, int limit, String afterName, S return listAfter( getTableName(), mySqlCondition, postgresCondition, limit, afterName, afterId, groupBy); } + + @SqlQuery( + "SELECT json FROM tn\n" + + "INNER JOIN (SELECT DISTINCT fromId FROM entity_relationship er\n" + + " AND toEntity = 'testSuite' and fromEntity = :entityType) er ON fromId = tn.id\n" + + "LIMIT :limit OFFSET :offset;") + List listEntitiesWithTestSuite( + @Define("table") String table, + @BindMap Map params, + @Define("cond") String cond, + @Bind("entityType") String entityType, + @Bind("limit") int limit, + @Bind("offset") int offset); + + default List listEntitiesWithTestsuite( + ListFilter filter, String table, String entityType, int limit, int offset) { + return listEntitiesWithTestSuite( + table, filter.getQueryParams(), filter.getCondition(), entityType, limit, offset); + } + + @SqlQuery( + "SELECT COUNT(DISTINCT fromId) FROM entity_relationship er\n" + + " AND toEntity = 'testSuite' and fromEntity = :entityType;") + Integer countEntitiesWithTestSuite( + @BindMap Map params, + @Define("cond") String cond, + @Bind("entityType") String entityType); + + default Integer countEntitiesWithTestsuite(ListFilter filter, String entityType) { + return countEntitiesWithTestSuite(filter.getQueryParams(), filter.getCondition(), entityType); + } } interface TestCaseDAO extends EntityDAO { diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java index b52cfec9d0fe..5958b8eba064 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java @@ -67,6 +67,9 @@ import static org.openmetadata.service.util.EntityUtil.nextVersion; import static org.openmetadata.service.util.EntityUtil.objectMatch; import static org.openmetadata.service.util.EntityUtil.tagLabelMatch; +import static org.openmetadata.service.util.jdbi.JdbiUtils.getAfterOffset; +import static org.openmetadata.service.util.jdbi.JdbiUtils.getBeforeOffset; +import static org.openmetadata.service.util.jdbi.JdbiUtils.getOffset; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; @@ -180,11 +183,13 @@ import org.openmetadata.service.util.EntityUtil.Fields; import org.openmetadata.service.util.FullyQualifiedName; import org.openmetadata.service.util.JsonUtils; +import org.openmetadata.service.util.ListWithOffsetFunction; import org.openmetadata.service.util.RestUtil; import org.openmetadata.service.util.RestUtil.DeleteResponse; import org.openmetadata.service.util.RestUtil.PatchResponse; import org.openmetadata.service.util.RestUtil.PutResponse; import org.openmetadata.service.util.ResultList; +import software.amazon.awssdk.utils.Either; /** * This is the base class used by Entity Resources to perform READ and WRITE operations to the backend database to @@ -767,40 +772,6 @@ public ResultList listAfter( } } - public final ResultList listAfterWithSkipFailure( - UriInfo uriInfo, Fields fields, ListFilter filter, int limitParam, String after) { - List errors = new ArrayList<>(); - List entities = new ArrayList<>(); - int beforeOffset = Integer.parseInt(RestUtil.decodeCursor(after)); - int currentOffset = beforeOffset; - int total = dao.listCount(filter); - if (limitParam > 0) { - List jsons = dao.listAfter(filter, limitParam, currentOffset); - - for (String json : jsons) { - T parsedEntity = JsonUtils.readValue(json, entityClass); - try { - T entity = setFieldsInternal(parsedEntity, fields); - setInheritedFields(entity, fields); - clearFieldsInternal(entity, fields); - entities.add(withHref(uriInfo, entity)); - } catch (Exception e) { - clearFieldsInternal(parsedEntity, fields); - EntityError entityError = - new EntityError().withMessage(e.getMessage()).withEntity(parsedEntity); - errors.add(entityError); - LOG.error("[ListForIndexing] Failed for Entity : {}", entityError); - } - } - currentOffset = currentOffset + limitParam; - String newAfter = currentOffset > total ? null : String.valueOf(currentOffset); - return getResultList(entities, errors, String.valueOf(beforeOffset), newAfter, total); - } else { - // limit == 0 , return total count of entity. - return getResultList(entities, errors, null, null, total); - } - } - @SuppressWarnings("unchecked") Map parseCursorMap(String param) { Map cursorMap; @@ -897,6 +868,45 @@ public final EntityHistoryWithOffset listVersionsWithOffset(UUID id, int limit, new EntityHistory().withEntityType(entityType).withVersions(versions), offset + limit); } + public final ResultList listWithOffset( + ListWithOffsetFunction> callable, + Function countCallable, + ListFilter filter, + Integer limitParam, + String offset, + boolean skipErrors, + Fields fields, + UriInfo uriInfo) { + List entities = new ArrayList<>(); + List errors = new ArrayList<>(); + + Integer total = countCallable.apply(filter); + + int offsetInt = getOffset(offset); + String afterOffset = getAfterOffset(offsetInt, limitParam, total); + String beforeOffset = getBeforeOffset(offsetInt, limitParam); + if (limitParam > 0) { + List jsons = callable.apply(filter, limitParam, offsetInt); + Iterator> iterator = serializeJsons(jsons, fields, uriInfo); + while (iterator.hasNext()) { + Either either = iterator.next(); + if (either.right().isPresent()) { + if (!skipErrors) { + throw new RuntimeException(either.right().get().getMessage()); + } else { + errors.add(either.right().get()); + LOG.error("[List] Failed for Entity : {}", either.right().get()); + } + } else { + entities.add(either.left().get()); + } + } + return getResultList(entities, errors, beforeOffset, afterOffset, total); + } else { + return getResultList(entities, errors, null, null, total); + } + } + public final EntityHistory listVersions(UUID id) { T latest = setFieldsInternal(find(id, ALL), putFields); setInheritedFields(latest, putFields); @@ -1580,6 +1590,23 @@ public final List getResultsFromAndToTimestamps( .listBetweenTimestampsByOrder(fqn, extension, startTs, endTs, orderBy); } + @Transaction + public ResultList getEntitiesWithTestSuite( + ListFilter filter, Integer limit, String offset, EntityUtil.Fields fields) { + CollectionDAO.TestSuiteDAO testSuiteDAO = daoCollection.testSuiteDAO(); + return listWithOffset( + (filterParam, limitParam, offsetParam) -> + testSuiteDAO.listEntitiesWithTestsuite( + filterParam, dao.getTableName(), entityType, limitParam, offsetParam), + (filterParam) -> testSuiteDAO.countEntitiesWithTestsuite(filterParam, entityType), + filter, + limit, + offset, + false, + fields, + null); + } + @Transaction public final void deleteExtensionAtTimestamp(String fqn, String extension, Long timestamp) { daoCollection.entityExtensionTimeSeriesDao().deleteAtTimestamp(fqn, extension, timestamp); @@ -3925,4 +3952,36 @@ private Map batchFetchExtensions(List entities) { private List entityListToStrings(List entities) { return entities.stream().map(EntityInterface::getId).map(UUID::toString).toList(); } + + private Iterator> serializeJsons( + List jsons, Fields fields, UriInfo uriInfo) { + return new Iterator<>() { + private final Iterator iterator = jsons.iterator(); + + @Override + public boolean hasNext() { + return iterator.hasNext(); + } + + @Override + public Either next() { + String json = iterator.next(); + T entity = JsonUtils.readValue(json, entityClass); + try { + setFieldsInternal(entity, fields); + setInheritedFields(entity, fields); + clearFieldsInternal(entity, fields); + if (!nullOrEmpty(uriInfo)) { + entity = withHref(uriInfo, entity); + } + return Either.left(entity); + } catch (Exception e) { + clearFieldsInternal(entity, fields); + EntityError entityError = + new EntityError().withMessage(e.getMessage()).withEntity(entity); + return Either.right(entityError); + } + } + }; + } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityTimeSeriesRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityTimeSeriesRepository.java index 572cc8b8c517..aa2bf8cd75f6 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityTimeSeriesRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityTimeSeriesRepository.java @@ -3,6 +3,9 @@ import static org.openmetadata.schema.type.EventType.ENTITY_UPDATED; import static org.openmetadata.schema.type.Include.ALL; import static org.openmetadata.service.Entity.getEntityFields; +import static org.openmetadata.service.util.jdbi.JdbiUtils.getAfterOffset; +import static org.openmetadata.service.util.jdbi.JdbiUtils.getBeforeOffset; +import static org.openmetadata.service.util.jdbi.JdbiUtils.getOffset; import java.beans.IntrospectionException; import java.io.IOException; @@ -316,26 +319,6 @@ public void deleteById(UUID id, boolean hardDelete) { timeSeriesDao.deleteById(id); } - private String getAfterOffset(int offsetInt, int limit, int total) { - int afterOffset = offsetInt + limit; - // If afterOffset is greater than total, then set it to null to indicate end of list - return afterOffset >= total ? null : String.valueOf(afterOffset); - } - - private String getBeforeOffset(int offsetInt, int limit) { - int beforeOffsetInt = offsetInt - limit; - // If offset is negative, then set it to 0 if you pass offset 4 and limit 10, then the previous - // page will be at offset 0 - if (beforeOffsetInt < 0) beforeOffsetInt = 0; - // if offsetInt is 0 (i.e. either no offset or offset is 0), then set it to null as there is no - // previous page - return (offsetInt == 0) ? null : String.valueOf(beforeOffsetInt); - } - - private int getOffset(String offset) { - return offset != null ? Integer.parseInt(RestUtil.decodeCursor(offset)) : 0; - } - private Map> getEntityList(List jsons, boolean skipErrors) { List entityList = new ArrayList<>(); List errors = new ArrayList<>(); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/util/ListWithOffsetFunction.java b/openmetadata-service/src/main/java/org/openmetadata/service/util/ListWithOffsetFunction.java new file mode 100644 index 000000000000..bd4348c0ce4d --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/util/ListWithOffsetFunction.java @@ -0,0 +1,14 @@ +package org.openmetadata.service.util; + +import java.util.Objects; +import java.util.function.Function; + +@FunctionalInterface +public interface ListWithOffsetFunction { + R apply(A a, B b, C c); + + default ListWithOffsetFunction andThen(Function after) { + Objects.requireNonNull(after); + return (A a, B b, C c) -> after.apply(apply(a, b, c)); + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/util/jdbi/JdbiUtils.java b/openmetadata-service/src/main/java/org/openmetadata/service/util/jdbi/JdbiUtils.java index ce471b7868fc..6c5e34ba09c9 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/util/jdbi/JdbiUtils.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/util/jdbi/JdbiUtils.java @@ -9,6 +9,7 @@ import org.jdbi.v3.sqlobject.SqlObjectPlugin; import org.jdbi.v3.sqlobject.SqlObjects; import org.openmetadata.service.jdbi3.locator.ConnectionAwareAnnotationSqlLocator; +import org.openmetadata.service.util.RestUtil; public class JdbiUtils { @@ -52,4 +53,24 @@ public static Jdbi createAndSetupJDBI(DataSourceFactory dbFactory) { return jdbiInstance; } + + public static int getOffset(String offset) { + return offset != null ? Integer.parseInt(RestUtil.decodeCursor(offset)) : 0; + } + + public static String getAfterOffset(int offsetInt, int limit, int total) { + int afterOffset = offsetInt + limit; + // If afterOffset is greater than total, then set it to null to indicate end of list + return afterOffset >= total ? null : String.valueOf(afterOffset); + } + + public static String getBeforeOffset(int offsetInt, int limit) { + int beforeOffsetInt = offsetInt - limit; + // If offset is negative, then set it to 0 if you pass offset 4 and limit 10, then the previous + // page will be at offset 0 + if (beforeOffsetInt < 0) beforeOffsetInt = 0; + // if offsetInt is 0 (i.e. either no offset or offset is 0), then set it to null as there is no + // previous page + return (offsetInt == 0) ? null : String.valueOf(beforeOffsetInt); + } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/workflows/searchIndex/PaginatedEntitiesSource.java b/openmetadata-service/src/main/java/org/openmetadata/service/workflows/searchIndex/PaginatedEntitiesSource.java index 234d079ee354..fdad4e88bf80 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/workflows/searchIndex/PaginatedEntitiesSource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/workflows/searchIndex/PaginatedEntitiesSource.java @@ -29,6 +29,7 @@ import org.openmetadata.schema.type.Include; import org.openmetadata.service.Entity; import org.openmetadata.service.exception.SearchIndexException; +import org.openmetadata.service.jdbi3.EntityDAO; import org.openmetadata.service.jdbi3.EntityRepository; import org.openmetadata.service.jdbi3.ListFilter; import org.openmetadata.service.util.RestUtil; @@ -96,9 +97,17 @@ private ResultList read(String cursor) throws SearchI EntityRepository entityRepository = Entity.getEntityRepository(entityType); ResultList result; try { + EntityDAO entityDAO = entityRepository.getDao(); result = - entityRepository.listAfterWithSkipFailure( - null, Entity.getFields(entityType, fields), filter, batchSize, cursor); + entityRepository.listWithOffset( + entityDAO::listAfter, + entityDAO::listCount, + filter, + batchSize, + cursor, + true, + Entity.getFields(entityType, fields), + null); if (!result.getErrors().isEmpty()) { lastFailedCursor = this.cursor.get(); if (result.getPaging().getAfter() == null) { @@ -154,9 +163,17 @@ public ResultList readWithCursor(String currentCursor EntityRepository entityRepository = Entity.getEntityRepository(entityType); ResultList result; try { + EntityDAO entityDAO = entityRepository.getDao(); result = - entityRepository.listAfterWithSkipFailure( - null, Entity.getFields(entityType, fields), filter, batchSize, currentCursor); + entityRepository.listWithOffset( + entityDAO::listAfter, + entityDAO::listCount, + filter, + batchSize, + currentCursor, + true, + Entity.getFields(entityType, fields), + null); LOG.debug( "[PaginatedEntitiesSource] Batch Stats :- %n Submitted : {} Success: {} Failed: {}", batchSize, result.getData().size(), result.getErrors().size()); diff --git a/openmetadata-service/src/main/resources/elasticsearch/en/table_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/en/table_index_mapping.json index 6894c8b18378..18be89f5f98d 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/en/table_index_mapping.json +++ b/openmetadata-service/src/main/resources/elasticsearch/en/table_index_mapping.json @@ -48,7 +48,13 @@ "mappings": { "properties": { "id": { - "type": "text" + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } }, "name": { "type": "text", diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/databases/TableResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/databases/TableResourceTest.java index 9d12cdb0b9d0..d48e8dc340f8 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/databases/TableResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/databases/TableResourceTest.java @@ -80,6 +80,7 @@ import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.Set; import java.util.UUID; import java.util.function.Predicate; import java.util.stream.Collectors; @@ -149,6 +150,8 @@ import org.openmetadata.schema.type.csv.CsvImportResult; import org.openmetadata.service.Entity; import org.openmetadata.service.exception.CatalogExceptionMessage; +import org.openmetadata.service.jdbi3.ListFilter; +import org.openmetadata.service.jdbi3.TableRepository; import org.openmetadata.service.jdbi3.TableRepository.TableCsv; import org.openmetadata.service.resources.EntityResourceTest; import org.openmetadata.service.resources.databases.TableResource.TableList; @@ -2266,6 +2269,46 @@ void test_ownershipInheritance(TestInfo test) throws IOException { schemaTest.assertOwnershipInheritanceOverride(schema, createSchema.withOwners(null), USER2_REF); } + @Test + void test_listTablesWithTestSuite(TestInfo test) throws IOException { + CreateDatabase createDb = dbTest.createRequest(test).withOwners(Lists.newArrayList(USER1_REF)); + Database db = dbTest.createEntity(createDb, ADMIN_AUTH_HEADERS); + CreateDatabaseSchema createSchema = + schemaTest.createRequest(test).withDatabase(db.getFullyQualifiedName()); + DatabaseSchema schema = schemaTest.createEntity(createSchema, ADMIN_AUTH_HEADERS); + CreateTable createTable = + createRequest(test).withDatabaseSchema(schema.getFullyQualifiedName()); + Table table = createEntity(createTable, ADMIN_AUTH_HEADERS); + + CreateTestSuite createTestSuite = + testSuiteResourceTest.createRequest(table.getFullyQualifiedName()); + TestSuite testSuite = + testSuiteResourceTest.createBasicTestSuite(createTestSuite, ADMIN_AUTH_HEADERS); + + CreateTestCase createTestCase = + testCaseResourceTest + .createRequest(test) + .withEntityLink(String.format("<#E::table::%s>", table.getFullyQualifiedName())) + .withTestSuite(testSuite.getFullyQualifiedName()) + .withTestDefinition(TEST_DEFINITION2.getFullyQualifiedName()); + TestCase testCase = testCaseResourceTest.createEntity(createTestCase, ADMIN_AUTH_HEADERS); + + TableRepository tableRepository = (TableRepository) Entity.getEntityRepository(TABLE); + + ResultList
allTables = listEntities(null, ADMIN_AUTH_HEADERS); + ResultList
tablesWithTestSuite = + tableRepository.getEntitiesWithTestSuite( + new ListFilter(), + 10, + "MA==", + new Fields( + Set.of( + "tags", "testSuite", "columns", "table.tableProfile", "table.columnProfile"))); + + // Ensure the number of tables with test suite is less than the total number of tables + assertTrue(allTables.getData().size() > tablesWithTestSuite.getData().size()); + } + @Test void test_domainInheritance(TestInfo test) throws IOException { // Domain is inherited from databaseService > database > databaseSchema > table @@ -2441,7 +2484,7 @@ void get_tablesWithTestCases(TestInfo test) throws IOException { queryParams.put("limit", "100"); ResultList
tables = listEntities(queryParams, ADMIN_AUTH_HEADERS); - assertEquals(4, tables.getData().size()); + assertEquals(5, tables.getData().size()); assertNotNull(tables.getData().get(0).getTestSuite()); } diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/applicationConfig.json b/openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/applicationConfig.json index 947d54a309fc..9c54599ce765 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/applicationConfig.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/applicationConfig.json @@ -13,6 +13,9 @@ { "$ref": "external/automatorAppConfig.json" }, + { + "$ref": "external/slackAppTokenConfiguration.json" + }, { "$ref": "internal/dataInsightsAppConfig.json" }, @@ -23,7 +26,7 @@ "$ref": "internal/searchIndexingAppConfig.json" }, { - "$ref": "external/slackAppTokenConfiguration.json" + "$ref": "internal/collateAIQualityAgentAppConfig.json" }, { "$ref": "internal/dataRetentionConfiguration.json" diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/internal/collateAIQualityAgentAppConfig.json b/openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/internal/collateAIQualityAgentAppConfig.json new file mode 100644 index 000000000000..67f58a6c3d46 --- /dev/null +++ b/openmetadata-spec/src/main/resources/json/schema/entity/applications/configuration/internal/collateAIQualityAgentAppConfig.json @@ -0,0 +1,37 @@ +{ + "$id": "https://open-metadata.org/schema/entity/applications/configuration/external/collateAIQualityAgentAppConfig.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "CollateAIQualityAgentAppConfig.json", + "description": "Configuration for the Collate AI Quality Agent.", + "type": "object", + "javaType": "org.openmetadata.schema.entity.app.internal.CollateAIQualityAgentAppConfig", + "definitions": { + "collateAIQualityAgentAppType": { + "description": "Application type.", + "type": "string", + "enum": ["CollateAIQualityAgent"], + "default": "CollateAIQualityAgent" + } + }, + "properties": { + "type": { + "title": "Application Type", + "description": "Application Type", + "$ref": "#/definitions/collateAIQualityAgentAppType", + "default": "CollateAIQualityAgent" + }, + "filter": { + "title": "Filter", + "description": "Query filter to be passed to ES. E.g., `{\"query\":{\"bool\":{\"must\":[{\"bool\":{\"should\":[{\"term\":{\"domain.displayName.keyword\":\"DG Anim\"}}]}}]}}}`. This is the same payload as in the Explore page.", + "type": "string" + }, + "active": { + "title": "Active", + "description": "Whether the suggested tests should be active or not upon suggestion", + "type": "boolean", + "default": false + } + }, + "additionalProperties": false, + "required": ["filter"] +} diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/entity/applications/configuration/internal/collateAIQualityAgentAppConfig.ts b/openmetadata-ui/src/main/resources/ui/src/generated/entity/applications/configuration/internal/collateAIQualityAgentAppConfig.ts new file mode 100644 index 000000000000..d2e32b7ec88f --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/generated/entity/applications/configuration/internal/collateAIQualityAgentAppConfig.ts @@ -0,0 +1,40 @@ +/* + * Copyright 2025 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/** + * Configuration for the Collate AI Quality Agent. + */ +export interface CollateAIQualityAgentAppConfig { + /** + * Whether the suggested tests should be active or not upon suggestion + */ + active?: boolean; + /** + * Query filter to be passed to ES. E.g., + * `{"query":{"bool":{"must":[{"bool":{"should":[{"term":{"domain.displayName.keyword":"DG + * Anim"}}]}}]}}}`. This is the same payload as in the Explore page. + */ + filter: string; + /** + * Application Type + */ + type?: CollateAIQualityAgentAppType; +} + +/** + * Application Type + * + * Application type. + */ +export enum CollateAIQualityAgentAppType { + CollateAIQualityAgent = "CollateAIQualityAgent", +} diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/entity/applications/configuration/internal/dataRetentionConfiguration.ts b/openmetadata-ui/src/main/resources/ui/src/generated/entity/applications/configuration/internal/dataRetentionConfiguration.ts index 2e3cd4b41266..01d766f7b47a 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/entity/applications/configuration/internal/dataRetentionConfiguration.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/entity/applications/configuration/internal/dataRetentionConfiguration.ts @@ -1,5 +1,5 @@ /* - * Copyright 2024 Collate. + * Copyright 2025 Collate. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at From 51bdaace13934ab5075f1505e489493dc58ac5d8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 22 Jan 2025 12:49:07 +0530 Subject: [PATCH 2/3] Build(deps): Bump undici in /openmetadata-ui/src/main/resources/ui (#19472) Bumps [undici](https://github.com/nodejs/undici) from 5.28.4 to 5.28.5. - [Release notes](https://github.com/nodejs/undici/releases) - [Commits](https://github.com/nodejs/undici/compare/v5.28.4...v5.28.5) --- updated-dependencies: - dependency-name: undici dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- openmetadata-ui/src/main/resources/ui/yarn.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/openmetadata-ui/src/main/resources/ui/yarn.lock b/openmetadata-ui/src/main/resources/ui/yarn.lock index 40a9ae76f7e2..c97f2b9b7609 100644 --- a/openmetadata-ui/src/main/resources/ui/yarn.lock +++ b/openmetadata-ui/src/main/resources/ui/yarn.lock @@ -14413,9 +14413,9 @@ unbox-primitive@^1.0.2: which-boxed-primitive "^1.0.2" undici@^5.25.4: - version "5.28.4" - resolved "https://registry.yarnpkg.com/undici/-/undici-5.28.4.tgz#6b280408edb6a1a604a9b20340f45b422e373068" - integrity sha512-72RFADWFqKmUb2hmmvNODKL3p9hcB6Gt2DOQMis1SEBaV6a4MH8soBvzg+95CYhCKPFedut2JY9bMfrDl9D23g== + version "5.28.5" + resolved "https://registry.yarnpkg.com/undici/-/undici-5.28.5.tgz#b2b94b6bf8f1d919bc5a6f31f2c01deb02e54d4b" + integrity sha512-zICwjrDrcrUE0pyyJc1I2QzBkLM8FINsgOrt6WjA+BgajVq9Nxu2PbFFXUrAggLfDXlZGZBVZYw7WNV5KiBiBA== dependencies: "@fastify/busboy" "^2.0.0" From 68c324679ab9078849d4300043d4fc55d11657e5 Mon Sep 17 00:00:00 2001 From: Ashish Gupta Date: Wed, 22 Jan 2025 13:19:45 +0530 Subject: [PATCH 3/3] #19474: fix the async export csv not happening in lineage (#19397) * fix lineage playwrigt flaky test * increase the timer * fix the async export issue * remove the timer * minor revert --- .../EntityExportModalProvider.component.tsx | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityExportModalProvider/EntityExportModalProvider.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityExportModalProvider/EntityExportModalProvider.component.tsx index 7bae7271755f..a976e6bc483c 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityExportModalProvider/EntityExportModalProvider.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityExportModalProvider/EntityExportModalProvider.component.tsx @@ -78,6 +78,12 @@ export const EntityExportModalProvider = ({ } try { setDownloading(true); + // assigning the job data to ref here, as exportData.onExport may take time to return the data + // and websocket connection may be respond before that, so we need to keep the job data in ref + // to handle the download + csvExportJobRef.current = { + fileName: fileName, + }; const data = await exportData.onExport(exportData.name); if (isString(data)) {