From f6989400a1e24e25fda193b2129b6c1e47fc73e7 Mon Sep 17 00:00:00 2001 From: Andrea Patricelli Date: Tue, 15 Oct 2024 10:04:28 +0200 Subject: [PATCH 1/3] [SYNCOPE-1830] added support for search on membership attributes on Elasticsearch and OpenSearch (#859) * [SYNCOPE-1830] added support for search on membership attributes on Elasticsearch and Opensearch --- .../test/resources/domains/MasterContent.xml | 3 ++ .../test/resources/domains/MasterContent.xml | 3 ++ .../client/ElasticsearchUtils.java | 21 +++++++++++ .../opensearch/client/OpenSearchUtils.java | 21 +++++++++++ .../syncope/fit/core/LinkedAccountITCase.java | 8 ++-- .../syncope/fit/core/MembershipITCase.java | 2 +- .../apache/syncope/fit/core/SearchITCase.java | 37 +++++++++++++++++++ 7 files changed, 90 insertions(+), 5 deletions(-) diff --git a/core/persistence-jpa/src/test/resources/domains/MasterContent.xml b/core/persistence-jpa/src/test/resources/domains/MasterContent.xml index 4b3244d439..76c056f9b1 100644 --- a/core/persistence-jpa/src/test/resources/domains/MasterContent.xml +++ b/core/persistence-jpa/src/test/resources/domains/MasterContent.xml @@ -384,6 +384,9 @@ under the License. realm_id="e4c28e7a-9dbf-4ee7-9441-93812a0d4a28" creator="admin" lastModifier="admin" creationDate="2010-10-20 11:00:00" lastChangeDate="2010-10-20 11:00:00"/> + + + + diff --git a/ext/elasticsearch/client-elasticsearch/src/main/java/org/apache/syncope/ext/elasticsearch/client/ElasticsearchUtils.java b/ext/elasticsearch/client-elasticsearch/src/main/java/org/apache/syncope/ext/elasticsearch/client/ElasticsearchUtils.java index 31ea96b55f..4674257a5f 100644 --- a/ext/elasticsearch/client-elasticsearch/src/main/java/org/apache/syncope/ext/elasticsearch/client/ElasticsearchUtils.java +++ b/ext/elasticsearch/client-elasticsearch/src/main/java/org/apache/syncope/ext/elasticsearch/client/ElasticsearchUtils.java @@ -35,10 +35,14 @@ import org.apache.syncope.core.persistence.api.entity.Any; import org.apache.syncope.core.persistence.api.entity.AnyTypeClass; import org.apache.syncope.core.persistence.api.entity.AuditEvent; +import org.apache.syncope.core.persistence.api.entity.GroupablePlainAttr; +import org.apache.syncope.core.persistence.api.entity.GroupableRelatable; +import org.apache.syncope.core.persistence.api.entity.Membership; import org.apache.syncope.core.persistence.api.entity.PlainAttr; import org.apache.syncope.core.persistence.api.entity.PlainAttrValue; import org.apache.syncope.core.persistence.api.entity.Privilege; import org.apache.syncope.core.persistence.api.entity.Realm; +import org.apache.syncope.core.persistence.api.entity.Relationship; import org.apache.syncope.core.persistence.api.entity.anyobject.AnyObject; import org.apache.syncope.core.persistence.api.entity.group.Group; import org.apache.syncope.core.persistence.api.entity.user.User; @@ -200,6 +204,23 @@ public Map document(final Any any) { builder.put(plainAttr.getSchema().getKey(), values.size() == 1 ? values.get(0) : values); } + // add also flattened membership attributes + if (any instanceof GroupableRelatable) { + GroupableRelatable entity = GroupableRelatable.class.cast(any); + entity.getMemberships().forEach(m -> entity.getPlainAttrs(m).forEach(mAttr -> { + List values = mAttr.getValues().stream().map(PlainAttrValue::getValue) + .collect(Collectors.toList()); + + Optional.ofNullable(mAttr.getUniqueValue()).ifPresent(v -> values.add(v.getValue())); + + if (!builder.containsKey(mAttr.getSchema().getKey())) { + builder.put(mAttr.getSchema().getKey(), new HashSet<>()); + } + builder.put(mAttr.getSchema().getKey(), values.size() == 1 ? values.get(0) : values); + })); + } + return builder; } diff --git a/ext/opensearch/client-opensearch/src/main/java/org/apache/syncope/ext/opensearch/client/OpenSearchUtils.java b/ext/opensearch/client-opensearch/src/main/java/org/apache/syncope/ext/opensearch/client/OpenSearchUtils.java index 1794ada4a6..29e1945f08 100644 --- a/ext/opensearch/client-opensearch/src/main/java/org/apache/syncope/ext/opensearch/client/OpenSearchUtils.java +++ b/ext/opensearch/client-opensearch/src/main/java/org/apache/syncope/ext/opensearch/client/OpenSearchUtils.java @@ -35,10 +35,14 @@ import org.apache.syncope.core.persistence.api.entity.Any; import org.apache.syncope.core.persistence.api.entity.AnyTypeClass; import org.apache.syncope.core.persistence.api.entity.AuditEvent; +import org.apache.syncope.core.persistence.api.entity.GroupablePlainAttr; +import org.apache.syncope.core.persistence.api.entity.GroupableRelatable; +import org.apache.syncope.core.persistence.api.entity.Membership; import org.apache.syncope.core.persistence.api.entity.PlainAttr; import org.apache.syncope.core.persistence.api.entity.PlainAttrValue; import org.apache.syncope.core.persistence.api.entity.Privilege; import org.apache.syncope.core.persistence.api.entity.Realm; +import org.apache.syncope.core.persistence.api.entity.Relationship; import org.apache.syncope.core.persistence.api.entity.anyobject.AnyObject; import org.apache.syncope.core.persistence.api.entity.group.Group; import org.apache.syncope.core.persistence.api.entity.user.User; @@ -200,6 +204,23 @@ public Map document(final Any any) { builder.put(plainAttr.getSchema().getKey(), values.size() == 1 ? values.get(0) : values); } + // add also flattened membership attributes + if (any instanceof GroupableRelatable) { + GroupableRelatable entity = GroupableRelatable.class.cast(any); + entity.getMemberships().forEach(m -> entity.getPlainAttrs(m).forEach(mAttr -> { + List values = mAttr.getValues().stream().map(PlainAttrValue::getValue) + .collect(Collectors.toList()); + + Optional.ofNullable(mAttr.getUniqueValue()).ifPresent(v -> values.add(v.getValue())); + + if (!builder.containsKey(mAttr.getSchema().getKey())) { + builder.put(mAttr.getSchema().getKey(), new HashSet<>()); + } + builder.put(mAttr.getSchema().getKey(), values.size() == 1 ? values.get(0) : values); + })); + } + return builder; } diff --git a/fit/core-reference/src/test/java/org/apache/syncope/fit/core/LinkedAccountITCase.java b/fit/core-reference/src/test/java/org/apache/syncope/fit/core/LinkedAccountITCase.java index 819b74101b..61b38a3ba1 100644 --- a/fit/core-reference/src/test/java/org/apache/syncope/fit/core/LinkedAccountITCase.java +++ b/fit/core-reference/src/test/java/org/apache/syncope/fit/core/LinkedAccountITCase.java @@ -18,6 +18,7 @@ */ package org.apache.syncope.fit.core; +import static org.awaitility.Awaitility.await; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotEquals; @@ -377,10 +378,9 @@ public void push() { assertEquals(1, task.getExecutions().size()); assertEquals(ExecStatus.SUCCESS.name(), task.getExecutions().get(0).getStatus()); - tasks = TASK_SERVICE.search( - new TaskQuery.Builder(TaskType.PROPAGATION).resource(RESOURCE_NAME_REST). - anyTypeKind(AnyTypeKind.USER).entityKey(user.getKey()).build()); - assertEquals(3, tasks.getTotalCount()); + await().until(() -> TASK_SERVICE.search( + new TaskQuery.Builder(TaskType.PROPAGATION).resource(RESOURCE_NAME_REST) + .anyTypeKind(AnyTypeKind.USER).entityKey(user.getKey()).build()).getTotalCount() == 3); // 6. verify that both user and account are now found on resource response = webClient.get(); diff --git a/fit/core-reference/src/test/java/org/apache/syncope/fit/core/MembershipITCase.java b/fit/core-reference/src/test/java/org/apache/syncope/fit/core/MembershipITCase.java index 7b52447845..8575e5ebaa 100644 --- a/fit/core-reference/src/test/java/org/apache/syncope/fit/core/MembershipITCase.java +++ b/fit/core-reference/src/test/java/org/apache/syncope/fit/core/MembershipITCase.java @@ -105,7 +105,7 @@ public void misc() throws JsonProcessingException { assertEquals(1, membership.getPlainAttr("aLong").orElseThrow().getValues().size()); assertEquals("1977", membership.getPlainAttr("aLong").orElseThrow().getValues().get(0)); - // 3. verify that derived attrbutes from 'csv' and 'other' are also populated for user's membership + // 3. verify that derived attributes from 'csv' and 'other' are also populated for user's membership assertFalse(membership.getDerAttr("csvuserid").orElseThrow().getValues().isEmpty()); assertFalse(membership.getDerAttr("noschema").orElseThrow().getValues().isEmpty()); diff --git a/fit/core-reference/src/test/java/org/apache/syncope/fit/core/SearchITCase.java b/fit/core-reference/src/test/java/org/apache/syncope/fit/core/SearchITCase.java index 6c7870945d..f7d3c5b635 100644 --- a/fit/core-reference/src/test/java/org/apache/syncope/fit/core/SearchITCase.java +++ b/fit/core-reference/src/test/java/org/apache/syncope/fit/core/SearchITCase.java @@ -18,6 +18,7 @@ */ package org.apache.syncope.fit.core; +import static org.awaitility.Awaitility.await; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotEquals; @@ -1058,4 +1059,40 @@ void issueSYNCOPE1826() { } } + @Test + void userByMembershipAttribute() { + // search user by membership attribute + UserTO puccini = USER_SERVICE.read("puccini"); + GroupTO additional = GROUP_SERVICE.read("additional"); + // add a membership and its plain attribute + updateUser(new UserUR.Builder(puccini.getKey()).memberships( + new MembershipUR.Builder(additional.getKey()).plainAttrs(attr("ctype", "additionalctype")) + .build()).build()); + await().until(() -> USER_SERVICE.search(new AnyQuery.Builder().page(1).size(10) + .fiql(SyncopeClient.getUserSearchConditionBuilder().is("ctype").equalTo("additionalctype").query()) + .build()).getTotalCount() == 1); + assertTrue(USER_SERVICE.search(new AnyQuery.Builder().page(1).size(10) + .fiql(SyncopeClient.getUserSearchConditionBuilder().is("ctype").equalTo("additionalctype").query()) + .build()).getResult().stream().anyMatch(u -> "puccini".equals(u.getUsername()))); + } + + @Test + void anyObjectByMembershipAttribute() { + // search user by membership attribute + AnyObjectTO canonMf = ANY_OBJECT_SERVICE.read("8559d14d-58c2-46eb-a2d4-a7d35161e8f8"); + GroupTO otherchild = GROUP_SERVICE.read("otherchild"); + // add a membership and its plain attribute + updateAnyObject(new AnyObjectUR.Builder(canonMf.getKey()).memberships( + new MembershipUR.Builder(otherchild.getKey()).plainAttrs(attr("ctype", "otherchildctype")) + .build()).build()); + await().until(() -> ANY_OBJECT_SERVICE.search(new AnyQuery.Builder().page(1).size(10) + .fiql(SyncopeClient.getAnyObjectSearchConditionBuilder(PRINTER).is("ctype").equalTo("otherchildctype") + .query()).build()).getTotalCount() == 1); + assertTrue(ANY_OBJECT_SERVICE.search(new AnyQuery.Builder().page(1).size(10) + .fiql(SyncopeClient.getAnyObjectSearchConditionBuilder(PRINTER).is("ctype").equalTo( + "otherchildctype") + .query()).build()).getResult().stream() + .anyMatch(u -> "8559d14d-58c2-46eb-a2d4-a7d35161e8f8".equals(u.getKey()))); + } + } From 870ea02cddc3a868da5704dfc2aecee51e07ad68 Mon Sep 17 00:00:00 2001 From: Andrea Patricelli Date: Tue, 15 Oct 2024 13:01:47 +0200 Subject: [PATCH 2/3] [SYNCOPE-1830] fix membership plain attribute map (#862) --- .../client/ElasticsearchUtils.java | 11 +++++++--- .../opensearch/client/OpenSearchUtils.java | 11 +++++++--- .../apache/syncope/fit/core/SearchITCase.java | 22 +++++++++++++++++-- 3 files changed, 36 insertions(+), 8 deletions(-) diff --git a/ext/elasticsearch/client-elasticsearch/src/main/java/org/apache/syncope/ext/elasticsearch/client/ElasticsearchUtils.java b/ext/elasticsearch/client-elasticsearch/src/main/java/org/apache/syncope/ext/elasticsearch/client/ElasticsearchUtils.java index 4674257a5f..340df8cf77 100644 --- a/ext/elasticsearch/client-elasticsearch/src/main/java/org/apache/syncope/ext/elasticsearch/client/ElasticsearchUtils.java +++ b/ext/elasticsearch/client-elasticsearch/src/main/java/org/apache/syncope/ext/elasticsearch/client/ElasticsearchUtils.java @@ -102,6 +102,7 @@ public ElasticsearchUtils( * @param any user, group or any object to index * @return document specialized with content from the provided any */ + @SuppressWarnings("unchecked") @Transactional public Map document(final Any any) { Map builder = new HashMap<>(); @@ -214,10 +215,14 @@ public Map document(final Any any) { Optional.ofNullable(mAttr.getUniqueValue()).ifPresent(v -> values.add(v.getValue())); - if (!builder.containsKey(mAttr.getSchema().getKey())) { - builder.put(mAttr.getSchema().getKey(), new HashSet<>()); + Object attr = builder.computeIfAbsent(mAttr.getSchema().getKey(), k -> new HashSet<>()); + // also support case in which there is also an existing attribute set previously + if (attr instanceof Collection) { + ((Collection) attr).addAll(values); + } else { + values.add(attr); + builder.put(mAttr.getSchema().getKey(), values.size() == 1 ? values.get(0) : values); } - builder.put(mAttr.getSchema().getKey(), values.size() == 1 ? values.get(0) : values); })); } diff --git a/ext/opensearch/client-opensearch/src/main/java/org/apache/syncope/ext/opensearch/client/OpenSearchUtils.java b/ext/opensearch/client-opensearch/src/main/java/org/apache/syncope/ext/opensearch/client/OpenSearchUtils.java index 29e1945f08..be327ed146 100644 --- a/ext/opensearch/client-opensearch/src/main/java/org/apache/syncope/ext/opensearch/client/OpenSearchUtils.java +++ b/ext/opensearch/client-opensearch/src/main/java/org/apache/syncope/ext/opensearch/client/OpenSearchUtils.java @@ -102,6 +102,7 @@ public OpenSearchUtils( * @param any user, group or any object to index * @return document specialized with content from the provided any */ + @SuppressWarnings("unchecked") @Transactional public Map document(final Any any) { Map builder = new HashMap<>(); @@ -214,10 +215,14 @@ public Map document(final Any any) { Optional.ofNullable(mAttr.getUniqueValue()).ifPresent(v -> values.add(v.getValue())); - if (!builder.containsKey(mAttr.getSchema().getKey())) { - builder.put(mAttr.getSchema().getKey(), new HashSet<>()); + Object attr = builder.computeIfAbsent(mAttr.getSchema().getKey(), k -> new HashSet<>()); + // also support case in which there is also an existing attribute set previously + if (attr instanceof Collection) { + ((Collection) attr).addAll(values); + } else { + values.add(attr); + builder.put(mAttr.getSchema().getKey(), values.size() == 1 ? values.get(0) : values); } - builder.put(mAttr.getSchema().getKey(), values.size() == 1 ? values.get(0) : values); })); } diff --git a/fit/core-reference/src/test/java/org/apache/syncope/fit/core/SearchITCase.java b/fit/core-reference/src/test/java/org/apache/syncope/fit/core/SearchITCase.java index f7d3c5b635..4609efbf09 100644 --- a/fit/core-reference/src/test/java/org/apache/syncope/fit/core/SearchITCase.java +++ b/fit/core-reference/src/test/java/org/apache/syncope/fit/core/SearchITCase.java @@ -43,6 +43,7 @@ import org.apache.syncope.common.lib.request.AnyObjectUR; import org.apache.syncope.common.lib.request.AttrPatch; import org.apache.syncope.common.lib.request.GroupCR; +import org.apache.syncope.common.lib.request.GroupUR; import org.apache.syncope.common.lib.request.MembershipUR; import org.apache.syncope.common.lib.request.UserCR; import org.apache.syncope.common.lib.request.UserUR; @@ -54,6 +55,7 @@ import org.apache.syncope.common.lib.to.PagedResult; import org.apache.syncope.common.lib.to.RealmTO; import org.apache.syncope.common.lib.to.RoleTO; +import org.apache.syncope.common.lib.to.TypeExtensionTO; import org.apache.syncope.common.lib.to.UserTO; import org.apache.syncope.common.lib.types.AnyTypeKind; import org.apache.syncope.common.lib.types.ClientExceptionType; @@ -1064,16 +1066,32 @@ void userByMembershipAttribute() { // search user by membership attribute UserTO puccini = USER_SERVICE.read("puccini"); GroupTO additional = GROUP_SERVICE.read("additional"); + GroupTO employee = GROUP_SERVICE.read("employee"); + TypeExtensionTO typeExtensionTO = new TypeExtensionTO(); + typeExtensionTO.setAnyType(AnyTypeKind.USER.name()); + typeExtensionTO.getAuxClasses().add("other"); + updateGroup(new GroupUR.Builder(employee.getKey()).typeExtension(typeExtensionTO).build()); // add a membership and its plain attribute - updateUser(new UserUR.Builder(puccini.getKey()).memberships( + updateUser(new UserUR.Builder(puccini.getKey()) + .plainAttr(attrAddReplacePatch("ctype", "myownctype")) + .memberships( new MembershipUR.Builder(additional.getKey()).plainAttrs(attr("ctype", "additionalctype")) - .build()).build()); + .build(), new MembershipUR.Builder(employee.getKey()) + .plainAttrs(attr("ctype", "additionalemployeectype")) + .build()).build()); await().until(() -> USER_SERVICE.search(new AnyQuery.Builder().page(1).size(10) .fiql(SyncopeClient.getUserSearchConditionBuilder().is("ctype").equalTo("additionalctype").query()) .build()).getTotalCount() == 1); assertTrue(USER_SERVICE.search(new AnyQuery.Builder().page(1).size(10) .fiql(SyncopeClient.getUserSearchConditionBuilder().is("ctype").equalTo("additionalctype").query()) .build()).getResult().stream().anyMatch(u -> "puccini".equals(u.getUsername()))); + assertTrue(USER_SERVICE.search(new AnyQuery.Builder().page(1).size(10) + .fiql(SyncopeClient.getUserSearchConditionBuilder().is("ctype").equalTo("additionalemployeectype") + .query()).build()).getResult().stream().anyMatch(u -> "puccini".equals(u.getUsername()))); + // check also that search on user plain attribute (not in membership) works + assertTrue(USER_SERVICE.search(new AnyQuery.Builder().page(1).size(10) + .fiql(SyncopeClient.getUserSearchConditionBuilder().is("ctype").equalTo("myownctype").query()) + .build()).getResult().stream().anyMatch(u -> "puccini".equals(u.getUsername()))); } @Test From 191792126a9f8fd71e2ed551c1c223b2337ba4b3 Mon Sep 17 00:00:00 2001 From: Andrea Patricelli Date: Tue, 15 Oct 2024 14:40:13 +0200 Subject: [PATCH 3/3] restore import --- .../syncope/ext/elasticsearch/client/ElasticsearchUtils.java | 1 + .../apache/syncope/ext/opensearch/client/OpenSearchUtils.java | 1 + 2 files changed, 2 insertions(+) diff --git a/ext/elasticsearch/client-elasticsearch/src/main/java/org/apache/syncope/ext/elasticsearch/client/ElasticsearchUtils.java b/ext/elasticsearch/client-elasticsearch/src/main/java/org/apache/syncope/ext/elasticsearch/client/ElasticsearchUtils.java index 340df8cf77..621f0723ac 100644 --- a/ext/elasticsearch/client-elasticsearch/src/main/java/org/apache/syncope/ext/elasticsearch/client/ElasticsearchUtils.java +++ b/ext/elasticsearch/client-elasticsearch/src/main/java/org/apache/syncope/ext/elasticsearch/client/ElasticsearchUtils.java @@ -20,6 +20,7 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.Collection; import java.util.HashMap; import java.util.HashSet; import java.util.List; diff --git a/ext/opensearch/client-opensearch/src/main/java/org/apache/syncope/ext/opensearch/client/OpenSearchUtils.java b/ext/opensearch/client-opensearch/src/main/java/org/apache/syncope/ext/opensearch/client/OpenSearchUtils.java index be327ed146..6ef66d10ae 100644 --- a/ext/opensearch/client-opensearch/src/main/java/org/apache/syncope/ext/opensearch/client/OpenSearchUtils.java +++ b/ext/opensearch/client-opensearch/src/main/java/org/apache/syncope/ext/opensearch/client/OpenSearchUtils.java @@ -20,6 +20,7 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.Collection; import java.util.HashMap; import java.util.HashSet; import java.util.List;