Skip to content

Commit

Permalink
Merged in task/dspace-cris-2023_02_x/DSC-1949 (pull request DSpace#2826)
Browse files Browse the repository at this point in the history 
DSC-1949 add support to update index for a specific indexable object type, deal with inprogress submission

Approved-by: Vincenzo Mecca
  • Loading branch information
@abollini @vins01-4science
abollini authored and vins01-4science committed Oct 11, 2024
2 parents 978726f + 01b2e3a commit f21e6f5
Show file tree
Hide file tree
Showing 2 changed files with 73 additions and 41 deletions.
109 changes: 68 additions & 41 deletions dspace-api/src/main/java/org/dspace/discovery/IndexClient.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
import java.io.IOException;
import java.sql.SQLException;
import java.util.Iterator;
import java.util.List;
import java.util.Optional;
import java.util.UUID;

Expand All @@ -23,9 +24,7 @@
import org.dspace.content.service.ItemService;
import org.dspace.core.Context;
import org.dspace.discovery.indexobject.IndexableCollection;
import org.dspace.discovery.indexobject.IndexableCommunity;
import org.dspace.discovery.indexobject.IndexableItem;
import org.dspace.discovery.indexobject.factory.IndexFactory;
import org.dspace.discovery.indexobject.factory.IndexObjectFactoryFactory;
import org.dspace.handle.factory.HandleServiceFactory;
import org.dspace.metrics.UpdateCrisMetricsInSolrDocService;
Expand All @@ -43,6 +42,8 @@ public class IndexClient extends DSpaceRunnable<IndexDiscoveryScriptConfiguratio
.getServiceByName(IndexingService.class.getName(),
IndexingService.class);

private IndexObjectFactoryFactory indexObjectServiceFactory = IndexObjectFactoryFactory.getInstance();

private IndexClientOptions indexClientOptions;

private UpdateCrisMetricsInSolrDocService updateCrisMetricsInSolrDocService;
Expand All @@ -59,8 +60,24 @@ public void internalRun() throws Exception {
* new DSpace.getServiceManager().getServiceByName("org.dspace.discovery.SolrIndexer");
*/

Optional<IndexableObject> indexableObject = Optional.empty();
Optional<List<IndexableObject>> indexableObjects = Optional.empty();
// limit the index update to a specific indexable object type
String type = null;

if (indexClientOptions == IndexClientOptions.UPDATE
|| indexClientOptions == IndexClientOptions.UPDATEANDSPELLCHECK
|| indexClientOptions == IndexClientOptions.FORCEUPDATE
|| indexClientOptions == IndexClientOptions.FORCEUPDATEANDSPELLCHECK) {
final String param = commandLine.getOptionValue('t');
if (param != null) {
// check if the param is a valid indable object type
if (indexObjectServiceFactory.getIndexFactoryByType(param) != null) {
type = param;
} else {
throw new IllegalArgumentException(param + " is not a valid Indexable Object Type");
}
}
}
if (indexClientOptions == IndexClientOptions.REMOVE || indexClientOptions == IndexClientOptions.INDEX) {
final String param = indexClientOptions == IndexClientOptions.REMOVE ? commandLine.getOptionValue('r') :
commandLine.getOptionValue('i');
Expand All @@ -74,39 +91,45 @@ public void internalRun() throws Exception {
if (uuid != null) {
final Item item = ContentServiceFactory.getInstance().getItemService().find(context, uuid);
if (item != null) {
indexableObject = Optional.of(new IndexableItem(item));
indexableObjects = Optional.of(indexObjectServiceFactory.getIndexableObjects(context, item));
} else {
// it could be a community
final Community community = ContentServiceFactory.getInstance().
getCommunityService().find(context, uuid);
if (community != null) {
indexableObject = Optional.of(new IndexableCommunity(community));
indexableObjects = Optional
.of(indexObjectServiceFactory.getIndexableObjects(context, community));
} else {
// it could be a collection
final Collection collection = ContentServiceFactory.getInstance().
getCollectionService().find(context, uuid);
if (collection != null) {
indexableObject = Optional.of(new IndexableCollection(collection));
indexableObjects = Optional
.of(indexObjectServiceFactory.getIndexableObjects(context, collection));
}
}
}
} else {
final DSpaceObject dso = HandleServiceFactory.getInstance()
.getHandleService().resolveToObject(context, param);
if (dso != null) {
final IndexFactory indexableObjectService = IndexObjectFactoryFactory.getInstance().
getIndexFactoryByType(String.valueOf(dso.getType()));
indexableObject = indexableObjectService.findIndexableObject(context, dso.getID().toString());
indexableObjects = Optional.of(indexObjectServiceFactory.getIndexableObjects(context, dso));
}
}
if (!indexableObject.isPresent()) {
if (!indexableObjects.isPresent()) {
throw new IllegalArgumentException("Cannot resolve " + param + " to a DSpace object");
}
}

boolean metricUpdate = true;
if (commandLine.hasOption("m")) {
metricUpdate = false;
}
if (indexClientOptions == IndexClientOptions.REMOVE) {
handler.logInfo("Removing " + commandLine.getOptionValue("r") + " from Index");
indexer.unIndexContent(context, indexableObject.get().getUniqueIndexID());
for (IndexableObject idxObj : indexableObjects.get()) {
indexer.unIndexContent(context, idxObj.getUniqueIndexID());
}
} else if (indexClientOptions == IndexClientOptions.CLEAN) {
handler.logInfo("Cleaning Index");
indexer.cleanIndex();
Expand All @@ -130,27 +153,29 @@ public void internalRun() throws Exception {
handler.logInfo("Indexing " + commandLine.getOptionValue('i') + " force " + commandLine.hasOption("f"));
final long startTimeMillis = System.currentTimeMillis();
final long count = indexAll(indexer, ContentServiceFactory.getInstance().
getItemService(), context, indexableObject.get());
getItemService(), context, indexableObjects.get());
final long seconds = (System.currentTimeMillis() - startTimeMillis) / 1000;
handler.logInfo("Indexed " + count + " object" + (count > 1 ? "s" : "") + " in " + seconds + " seconds");
} else if (indexClientOptions == IndexClientOptions.UPDATE ||
indexClientOptions == IndexClientOptions.UPDATEANDSPELLCHECK) {
handler.logInfo("Updating Index");
indexer.updateIndex(context, false);
indexer.updateIndex(context, false, type);
if (indexClientOptions == IndexClientOptions.UPDATEANDSPELLCHECK) {
checkRebuildSpellCheck(commandLine, indexer);
}
} else if (indexClientOptions == IndexClientOptions.FORCEUPDATE ||
indexClientOptions == IndexClientOptions.FORCEUPDATEANDSPELLCHECK) {
handler.logInfo("Updating Index");
indexer.updateIndex(context, true);
indexer.updateIndex(context, true, type);
if (indexClientOptions == IndexClientOptions.FORCEUPDATEANDSPELLCHECK) {
checkRebuildSpellCheck(commandLine, indexer);
}
}

handler.logInfo("Done with indexing");
updateCrisMetricsInSolrDocService.performUpdate(context, handler, true);
if (metricUpdate) {
updateCrisMetricsInSolrDocService.performUpdate(context, handler, true);
}
}

@Override
Expand All @@ -175,43 +200,45 @@ public void setup() throws ParseException {
*
* @param indexingService
* @param itemService
* @param context The relevant DSpace Context.
* @param dso DSpace object to index recursively
* @param context The relevant DSpace Context.
* @param indexableObjects DSpace objects to index recursively
* @throws IOException A general class of exceptions produced by failed or interrupted I/O operations.
* @throws SearchServiceException in case of a solr exception
* @throws SQLException An exception that provides information on a database access error or other errors.
*/
private static long indexAll(final IndexingService indexingService,
final ItemService itemService,
final Context context,
final IndexableObject dso)
final List<IndexableObject> indexableObjects)
throws IOException, SearchServiceException, SQLException {
long count = 0;

indexingService.indexContent(context, dso, true, true);
count++;
if (dso.getIndexedObject() instanceof Community) {
final Community community = (Community) dso.getIndexedObject();
final String communityHandle = community.getHandle();
for (final Community subcommunity : community.getSubcommunities()) {
count += indexAll(indexingService, itemService, context, new IndexableCommunity(subcommunity));
//To prevent memory issues, discard an object from the cache after processing
context.uncacheEntity(subcommunity);
}
final Community reloadedCommunity = (Community) HandleServiceFactory.getInstance().getHandleService()
.resolveToObject(context,
communityHandle);
for (final Collection collection : reloadedCommunity.getCollections()) {
count++;
indexingService.indexContent(context, new IndexableCollection(collection), true, true);
count += indexItems(indexingService, itemService, context, collection);
//To prevent memory issues, discard an object from the cache after processing
context.uncacheEntity(collection);
IndexObjectFactoryFactory indexObjectServiceFactory = IndexObjectFactoryFactory.getInstance();
for (IndexableObject iObj : indexableObjects) {
indexingService.indexContent(context, iObj, true, true);
count++;
if (iObj.getIndexedObject() instanceof Community) {
final Community community = (Community) iObj.getIndexedObject();
final String communityHandle = community.getHandle();
for (final Community subcommunity : community.getSubcommunities()) {
count += indexAll(indexingService, itemService, context,
indexObjectServiceFactory.getIndexableObjects(context, subcommunity));
//To prevent memory issues, discard an object from the cache after processing
context.uncacheEntity(subcommunity);
}
final Community reloadedCommunity = (Community) HandleServiceFactory.getInstance().getHandleService()
.resolveToObject(context,
communityHandle);
for (final Collection collection : reloadedCommunity.getCollections()) {
count++;
indexingService.indexContent(context, new IndexableCollection(collection), true, true);
count += indexItems(indexingService, itemService, context, collection);
//To prevent memory issues, discard an object from the cache after processing
context.uncacheEntity(collection);
}
} else if (iObj instanceof IndexableCollection) {
count += indexItems(indexingService, itemService, context, (Collection) iObj.getIndexedObject());
}
} else if (dso instanceof IndexableCollection) {
count += indexItems(indexingService, itemService, context, (Collection) dso.getIndexedObject());
}

return count;
}

Expand Down
5 changes: 5 additions & 0 deletions dspace-api/src/main/java/org/dspace/discovery/IndexClientOptions.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -82,8 +82,13 @@ protected static Options constructOptions() {
"delete all records from existing index");
options.addOption("b", "build", false, "(re)build index, wiping out current one if it exists");
options.addOption("s", "spellchecker", false, "Rebuild the spellchecker, can be combined with -b and -f.");
options.addOption("t", "type", true,
"limit the update operation to a specific indexable object type " +
"(such as Item, ClaimedTask, PoolTask, etc.), can be combined with -f and -s.");
options.addOption("f", "force", false,
"if updating existing index, force each handle to be reindexed even if uptodate");
options.addOption("m", "nometric", false,
"DISABLE the full rebuild of cris metric in solr, can be combined with all the parameter.");
options.addOption("h", "help", false, "print this help message");
return options;
}
Expand Down

0 comments on commit f21e6f5

Please sign in to comment.