Skip to content

Commit

Permalink
merge changes from develop
Browse files Browse the repository at this point in the history
  • Loading branch information
gsergiu committed Jan 30, 2025
1 parent 4b2a3eb commit 9296c75
Show file tree
Hide file tree
Showing 4 changed files with 44 additions and 51 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,13 @@ public class EntityManagementConfiguration implements InitializingBean {

@Value("${entitymanagement.solr.pr.url}")
private String prSolrUrl;

@Value("${entitymanagement.solr.pr.zookeeper.url:}")
private String prSolrZookeeperUrl;

@Value("${entitymanagement.solr.pr.collection}")
private String prSolrCollection;

@Value("${europeana.searchapi.urlPrefix}")
private String searchApiUrlPrefix;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,10 +7,7 @@
sortMissingLast="true" />
<fieldType name="json" class="solr.StrField"
sortMissingLast="true" />
<!--
<fieldType name="strings" class="solr.StrField"
sortMissingLast="true" multiValued="true" docValues="true" />
-->

<!-- boolean type: "true" or "false" -->
<fieldType name="boolean" class="solr.BoolField"
sortMissingLast="true" />
Expand Down Expand Up @@ -123,10 +120,6 @@
<!-- Entity type, in human-readable form (Concept|Agent|Place|Organization|Work) -->
<field name="type" type="string" indexed="true" stored="true"
multiValued="false" required="true" />
<!-- check if needed for aggregators
<field name="parentType" type="string" indexed="true" stored="true"
multiValued="false" required="false" />
-->

<!-- License info -->
<field name="rights" type="string" indexed="true" stored="true"
Expand Down Expand Up @@ -195,9 +188,6 @@
<copyField source="owl_sameAs" dest="coref" />
<copyField source="skos_exactMatch" dest="coref" />

<field name="edm_aggregatedVia" type="string" indexed="yes"
stored="true" multiValued="true" />

<field name="foaf_depiction" type="string" indexed="false"
stored="true" multiValued="false" />

Expand Down Expand Up @@ -273,13 +263,6 @@
stored="true" multiValued="true" />
<copyField source="dc_identifier" dest="identifier" />

<!-- not available yet -->
<!-- <field name="foaf_name" type="string" indexed="true" stored="true"
multiValued="true"/> <dynamicField name="foaf_name.*" type="proper_name"
indexed="true" stored="true" multiValued="true"/> <copyField source="foaf_name.*"
dest="foaf_name"/> -->


<!-- some Agent-related date fields. Check datatyping -->
<!-- BEGIN RELATED FIELDS -->
<field name="edm_begin" type="string" indexed="false"
Expand Down Expand Up @@ -402,9 +385,6 @@
<copyField source="rdagr2_placeOfDeath.*"
dest="placeOfDeath" />

<!-- NOT USED specs to be fixed <field name="rdagr2_placeOfDeath" type="string"
indexed="true" stored="true" multiValued="true"/> -->

<!-- HAS MET -->
<field name="edm_hasMet" type="string" indexed="false"
stored="true" multiValued="true" />
Expand Down Expand Up @@ -458,12 +438,7 @@
stored="true" multiValued="false" />
<field name="wgs84_pos_lat_long" type="string" indexed="false"
stored="true" multiValued="false" />
<!-- this fields was never used <field name="coord" type="location" indexed="true"
stored="true" multiValued="false" /> -->
<!-- required for spatial search -->
<!-- this field was never used see <dynamicField name="*_coords" type="double"
indexed="true" stored="false" /> -->
<!-- TODO: alias -->
<field name="alt" type="float" indexed="false" stored="true"
multiValued="false" />
<copyField source="wgs84_pos_alt" dest="alt" />
Expand All @@ -481,9 +456,10 @@
stored="true" multiValued="false" />
<field name="foaf_phone" type="string" indexed="false"
stored="true" multiValued="true" />
<!-- deprecated: to delete in the next version -->
<field name="foaf_mbox" type="string" indexed="false"
stored="true" multiValued="false" />

<field name="aggregatedVia" type="string" indexed="yes"
stored="true" multiValued="true" />

<!-- Aggregator Fields -->
<field name="mbox" type="string" indexed="false"
stored="true" multiValued="false" />
Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,5 @@
package eu.europeana.entitymanagement.config;

import eu.europeana.entitymanagement.common.config.EntityManagementConfiguration;
import eu.europeana.entitymanagement.common.vocabulary.AppConfigConstants;
import java.util.Arrays;
import java.util.Optional;
import org.apache.commons.lang3.StringUtils;
Expand All @@ -14,6 +12,8 @@
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import eu.europeana.entitymanagement.common.config.EntityManagementConfiguration;
import eu.europeana.entitymanagement.common.vocabulary.AppConfigConstants;

@Configuration
public class SolrConfig {
Expand All @@ -28,52 +28,57 @@ public SolrConfig(EntityManagementConfiguration emConfiguration) {

@Bean(AppConfigConstants.BEAN_PR_SOLR_CLIENT)
public SolrClient pageRankSolrClient() {
logger.info("Configuring pageRank solr client at the url: {}", emConfiguration.getPrSolrUrl());
return new HttpSolrClient.Builder(emConfiguration.getPrSolrUrl()).build();
if (StringUtils.isNotBlank(emConfiguration.getPrSolrZookeeperUrl())) {
return initSolrCloudClient(emConfiguration.getPrSolrZookeeperUrl(), emConfiguration.getIndexingSolrTimeoutMillis(),
emConfiguration.getPrSolrCollection());
} else {
return initSolrClient(emConfiguration.getPrSolrUrl(), emConfiguration.getIndexingSolrTimeoutMillis());
}
}

@Bean(AppConfigConstants.BEAN_INDEXING_SOLR_CLIENT)
public SolrClient indexingSolrClient() {
if (StringUtils.isNotBlank(emConfiguration.getIndexingSolrZookeeperUrl())) {
return initSolrCloudClient();
return initSolrCloudClient(emConfiguration.getIndexingSolrZookeeperUrl(), emConfiguration.getIndexingSolrTimeoutMillis(),
emConfiguration.getIndexingSolrCollection());
} else {
return initSolrClient();
return initSolrClient(emConfiguration.getIndexingSolrUrl(), emConfiguration.getIndexingSolrTimeoutMillis());
}
}

private SolrClient initSolrClient() {
private SolrClient initSolrClient(String solrUrl, int timeoutMillis) {
logger.info(
"Configuring indexing solr client at the url: {}", emConfiguration.getIndexingSolrUrl());
"Configuring solr client at the url: {}", solrUrl);

if (emConfiguration.getIndexingSolrUrl().contains(",")) {
if (solrUrl.contains(",")) {
LBHttpSolrClient.Builder builder = new LBHttpSolrClient.Builder();
return builder
.withBaseSolrUrls(emConfiguration.getIndexingSolrUrl().split(","))
.withConnectionTimeout(emConfiguration.getIndexingSolrTimeoutMillis())
.withBaseSolrUrls(solrUrl.split(","))
.withConnectionTimeout(timeoutMillis)
.build();
} else {
HttpSolrClient.Builder builder = new HttpSolrClient.Builder();
return builder
.withBaseSolrUrl(emConfiguration.getIndexingSolrUrl())
.withConnectionTimeout(emConfiguration.getIndexingSolrTimeoutMillis())
.withBaseSolrUrl(solrUrl)
.withConnectionTimeout(timeoutMillis)
.build();
}
}

private SolrClient initSolrCloudClient() {
private SolrClient initSolrCloudClient(String solrZookeeperUrl, int timeout, String solrCollection) {
logger.info(
"Configuring indexing solr client with the zookeperurls: {} and collection: {}",
emConfiguration.getIndexingSolrUrl(),
emConfiguration.getIndexingSolrCollection());
"Configuring solr client with the zookeperurls: {} and collection: {}",
solrZookeeperUrl,
solrCollection);

String[] solrZookeeperUrls = emConfiguration.getIndexingSolrZookeeperUrl().trim().split(",");
String[] solrZookeeperUrlsList = solrZookeeperUrl.trim().split(",");

CloudSolrClient client =
new CloudSolrClient.Builder(Arrays.asList(solrZookeeperUrls), Optional.empty())
.withConnectionTimeout(emConfiguration.getIndexingSolrTimeoutMillis())
new CloudSolrClient.Builder(Arrays.asList(solrZookeeperUrlsList), Optional.empty())
.withConnectionTimeout(timeout)
.build();

client.setDefaultCollection(emConfiguration.getIndexingSolrCollection());
client.setDefaultCollection(solrCollection);
return client;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,12 @@ datasources.config=/datasources.xml
#string value representing the key used for JWT signature verification
europeana.apikey.jwttoken.signaturekey=

#zookeeper url for the page-rank solr (if not present, normal is used)
#entitymanagement.solr.pr.zookeeper.url=localhost:9983

#the name of the page-rank solr collection (mandatory when using zookeeper url)
entitymanagement.solr.pr.collection=pagerank1

#url for the page rank solr
#entitymanagement.solr.pr.url=
entitymanagement.solr.pr.url=http://annotation-entity-test.eanadev.org:9393/solr/pagerank1
Expand Down

0 comments on commit 9296c75

Please sign in to comment.