From e3121164ef37f654e17c626425188ce80218b765 Mon Sep 17 00:00:00 2001 From: agazzarini Date: Thu, 9 Apr 2015 21:50:55 +0200 Subject: [PATCH] [ issue #47 ] First draft of facet.object.queries --- .../handler/search/faceting/RDFacets.java | 140 +++++++++++ .../search/faceting/rq/FacetObjectQuery.java | 218 ++++++++++++++++++ 2 files changed, 358 insertions(+) create mode 100644 solrdf/src/main/java/org/gazzax/labs/solrdf/handler/search/faceting/rq/FacetObjectQuery.java diff --git a/solrdf/src/main/java/org/gazzax/labs/solrdf/handler/search/faceting/RDFacets.java b/solrdf/src/main/java/org/gazzax/labs/solrdf/handler/search/faceting/RDFacets.java index 3995098..b55a495 100644 --- a/solrdf/src/main/java/org/gazzax/labs/solrdf/handler/search/faceting/RDFacets.java +++ b/solrdf/src/main/java/org/gazzax/labs/solrdf/handler/search/faceting/RDFacets.java @@ -5,6 +5,16 @@ import java.util.EnumSet; import java.util.List; import java.util.Set; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Executor; +import java.util.concurrent.Future; +import java.util.concurrent.FutureTask; +import java.util.concurrent.RunnableFuture; +import java.util.concurrent.Semaphore; +import java.util.concurrent.SynchronousQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; import org.apache.lucene.search.Query; import org.apache.solr.common.SolrException; @@ -18,6 +28,7 @@ import org.apache.solr.common.params.SolrParams; import org.apache.solr.common.util.NamedList; import org.apache.solr.common.util.SimpleOrderedMap; +import org.apache.solr.common.util.StrUtils; import org.apache.solr.handler.component.ResponseBuilder; import org.apache.solr.request.SimpleFacets; import org.apache.solr.schema.FieldType; @@ -29,8 +40,10 @@ import org.apache.solr.search.DocSetCollector; import org.apache.solr.search.QParser; import org.apache.solr.search.SyntaxError; +import org.apache.solr.util.DefaultSolrThreadFactory; import org.gazzax.labs.solrdf.handler.search.faceting.rq.DateRangeEndpointCalculator; import org.gazzax.labs.solrdf.handler.search.faceting.rq.DoubleRangeEndpointCalculator; +import org.gazzax.labs.solrdf.handler.search.faceting.rq.FacetObjectQuery; import org.gazzax.labs.solrdf.handler.search.faceting.rq.FacetRangeQuery; import org.gazzax.labs.solrdf.handler.search.faceting.rq.RangeEndpointCalculator; @@ -43,6 +56,22 @@ * @since 1.0 */ public class RDFacets extends SimpleFacets { + + static final Executor directExecutor = new Executor() { + @Override + public void execute(Runnable task) { + task.run(); + } + }; + + static final Executor facetExecutor = new ThreadPoolExecutor( + 0, + Integer.MAX_VALUE, + 10, + TimeUnit.SECONDS, + new SynchronousQueue(), + new DefaultSolrThreadFactory("facetExecutor")); + /** * Builds a new {@link RDFacets} with the given data. * @@ -58,6 +87,13 @@ public RDFacets(final ResponseBuilder responseBuilder, final DocSet docs, final public NamedList getFacetCounts() { final NamedList result = super.getFacetCounts(); result.remove("facet_dates"); + + try { + result.add("facet_object_queries", getFacetObjectQueriesCounts()); + } catch (IOException | SyntaxError e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } result.add("facet_object_ranges_queries", result.remove("facet_ranges")); return result; } @@ -66,7 +102,111 @@ public NamedList getFacetCounts() { public NamedList getFacetDateCounts() { return null; } + + /** + * &facet.object.q= + * @return + * @throws IOException + * @throws SyntaxError + */ + public NamedList getFacetObjectQueriesCounts() throws IOException, SyntaxError { + final NamedList result = new SimpleOrderedMap<>(); + + final List facetObjectQueries = new ArrayList(); + final String[] anonymousQueries = params.getParams(FacetObjectQuery.QUERY); + int index = 0; + final SolrParams requiredParams = new RequiredSolrParams(params); + if (anonymousQueries != null && anonymousQueries.length > 0) { + for (final String query : anonymousQueries) { + facetObjectQueries.add( + FacetObjectQuery.newAnonymousQuery( + query, + index++ == 0 ? params.get(FacetObjectQuery.QUERY_ALIAS) : null, + params, + requiredParams)); + } + } + + index = 0; + String query = null; + while ((query = params.get(FacetObjectQuery.QUERY + "." + (++index))) != null) { + facetObjectQueries.add( + FacetObjectQuery.newQuery( + query, + index, + params, + requiredParams)); + } + + if (facetObjectQueries.isEmpty()) { + return result; + } + + // Passing a negative number for FACET_THREADS implies an unlimited number of threads is acceptable. + // Also, a subtlety of directExecutor is that no matter how many times you "submit" a job, it's really + // just a method call in that it's run by the calling thread. + int maxThreads = req.getParams().getInt(FacetParams.FACET_THREADS, 0); + final Executor executor = maxThreads == 0 ? directExecutor : facetExecutor; + final Semaphore semaphore = new Semaphore((maxThreads <= 0) ? Integer.MAX_VALUE : maxThreads); + final List>> futures = new ArrayList<>(facetObjectQueries.size()); + + try { + for (final FacetObjectQuery foq : facetObjectQueries) { + // parseParams(FacetParams.FACET_FIELD, f); + final String termList = localParams == null ? null : localParams.get(CommonParams.TERMS); + final String workerFacetValue = facetValue; + final DocSet workerBase = this.docs; + final Callable> callable = new Callable>() { + @Override + public NamedList call() throws Exception { + try { + final DocSetCollector collector = new DocSetCollector(docs.size() >> 6, docs.size()); + req.getSearcher().search( + QParser.getParser(foq.query(), null, req).getQuery(), + docs.getTopFilter(), + collector); + + final NamedList result = new SimpleOrderedMap<>(); + // TBU + if(termList != null) { + List terms = StrUtils.splitSmart(termList, ",", true); + result.add(foq.alias(), getListedTermCounts(workerFacetValue, workerBase, terms)); + } else { + result.add(foq.alias(), getTermCounts(foq.fieldName(), collector.getDocSet())); + } + return result; + } catch (SolrException se) { + throw se; + } catch (Exception e) { + throw new SolrException(ErrorCode.SERVER_ERROR, "Exception during facet.field: " + workerFacetValue, e); + } finally { + semaphore.release(); + } + } + }; + final RunnableFuture> runnableFuture = new FutureTask>(callable); + semaphore.acquire(); + executor.execute(runnableFuture); + futures.add(runnableFuture); + } + + for (final Future> future : futures) { + result.addAll(future.get()); + } + assert semaphore.availablePermits() >= maxThreads; + } catch (InterruptedException exception) { + throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Error while processing facet fields: InterruptedException", exception); + } catch (ExecutionException exception) { + final Throwable cause = exception.getCause(); + if (cause instanceof RuntimeException) { + throw (RuntimeException) cause; + } + throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Error while processing facet fields: " + cause.toString(), cause); + } + return result; + } + @Override public NamedList getFacetRangeCounts() throws IOException, SyntaxError { final NamedList result = new SimpleOrderedMap<>(); diff --git a/solrdf/src/main/java/org/gazzax/labs/solrdf/handler/search/faceting/rq/FacetObjectQuery.java b/solrdf/src/main/java/org/gazzax/labs/solrdf/handler/search/faceting/rq/FacetObjectQuery.java new file mode 100644 index 0000000..63a48d8 --- /dev/null +++ b/solrdf/src/main/java/org/gazzax/labs/solrdf/handler/search/faceting/rq/FacetObjectQuery.java @@ -0,0 +1,218 @@ +package org.gazzax.labs.solrdf.handler.search.faceting.rq; + +import static org.gazzax.labs.solrdf.Strings.isNullOrEmpty; + +import org.apache.solr.common.SolrException; +import org.apache.solr.common.params.FacetParams; +import org.apache.solr.common.params.SolrParams; +import org.gazzax.labs.solrdf.Field; + +/** + * A stupid value object for encapsulating a facet query with all related parameters. + * + * Apart from some parameter like alias, hint, all other parameters are described in the Solr Wiki or + * Reference Guide. + * + * @author Andrea Gazzarini + * @since 1.0 + * @see https://cwiki.apache.org/confluence/display/solr/Faceting + * @see https://cwiki.apache.org/confluence/display/solr/Faceting#Faceting-RangeFaceting + */ +public class FacetObjectQuery { + public static String QUERY = FacetParams.FACET + ".object.q"; + public static String QUERY_HINT = QUERY + ".hint"; + public static String QUERY_ALIAS = QUERY + ".alias"; + + private final int index; + + private final SolrParams optionals; + private final SolrParams requireds; + + String fieldName; + private final String q; + private final String alias; + + /** + * Builds a new {@link FacetObjectQuery}. + * + * @param q the query. + * @param index the query index. + * @param alias the query alias. + * @param optionals the incoming parameters. + * @param requireds the incoming required parameters. + */ + private FacetObjectQuery( + final String q, + final int index, + final String alias, + final SolrParams optionals, + final SolrParams required) { + this.q = q; + this.index = index; + this.requireds = required; + this.optionals = optionals; + this.alias = alias != null ? alias : optionals.get(fdqn(QUERY_ALIAS)); + } + + /** + * Factory method for creating a new anonymous (not indexed) query. + * + * @param q the query string. + * @param alias the query alias. + * @param optionals the incoming parameters. + * @param requireds the incoming required parameters. + * @return a new anonymous (not indexed) query. + */ + public static FacetObjectQuery newAnonymousQuery( + final String q, + final String alias, + final SolrParams optionals, + final SolrParams required) { + return new FacetObjectQuery(q, 0, alias, optionals, required); + } + + /** + * Factory method for creating a new indexed query. + * + * @param q the query string. + * @param index the index. + * @param optionals the incoming parameters. + * @param requireds the incoming required parameters. + * @return a new anonymous (not indexed) query. + */ + public static FacetObjectQuery newQuery( + final String q, + final int index, + final SolrParams optionals, + final SolrParams required) { + return new FacetObjectQuery(q, index, null, optionals, required); + } + + /** + * Returns the query string associated with this query object. + * + * @return the query string associated with this query object. + */ + public String query() { + return q; + } + + /** + * Returns the alias associated with this query object. + * + * @return the alias associated with this query object. + */ + public String alias() { + return alias; + } + + /** + * Returns the target field of this facet query. + * + * @return the target field of this facet query. + */ + public String fieldName() { + if (fieldName != null) { + return fieldName; + } + + return fieldName = + "date".equals(optionalString(QUERY_HINT)) + ? Field.DATE_OBJECT + : Field.NUMERIC_OBJECT; + } + + /** + * Returns a required int parameter value. + * + * @param name the parameter name. + * @return the value for the requested parameter. + * @throws SolrException in case a valid value cannot be found. + */ + public int requiredInt(final String name) { + return Integer.parseInt(requiredString(name)); + } + + /** + * Returns a required string parameter value. + * + * @param name the parameter name. + * @return the value for the requested parameter. + * @throws SolrException in case a valid value cannot be found. + */ + public String requiredString(final String name) { + final String result = optionals.get(fdqn(name)); + return isNullOrEmpty(result) ? requireds.get(name) : result; + } + + /** + * Returns an optional boolean parameter value. + * + * @param name the parameter name. + * @return the value for the requested parameter. + */ + public boolean optionalBoolean(final String name) { + return Boolean.parseBoolean(optionalString(name)); + } + + /** + * Returns the value of the parameter associated with a given name. + * + * @param name the parameter name. + * @return the value of the parameter associated with a given name, null if it doesn't exist. + */ + public String optionalString(final String name) { + final String result = optionals.get(fdqn(name)); + return isNullOrEmpty(result) ? optionals.get(name) : result; + } + + /** + * Returns the value of the parameter associated with a given name. + * + * @param name the parameter name. + * @return the value of the parameter associated with a given name, null if it doesn't exist. + */ + public String [] optionalStrings(final String name) { + final String [] result = optionals.getParams(fdqn(name)); + return result == null || result.length == 0 ? optionals.getParams(name) : result; + } + + /** + * Returns the scope suffix for this query. + * + * @return the scope suffix for this query. + */ + String suffix() { + return isAnonymous() ? "" : "." + index; + } + + /** + * Returns the fully qualified name of the given attribute. + * In case the query object is anonymous then the result is equal to the input parameter. + * Otherwise, a suffix is appended to the attribute name. + * + * @param unscopedName the plain attribute name, without any scope. + * @return the fully qualified name of the given attribute. + */ + String fdqn(final String unscopedName) { + return new StringBuilder(unscopedName).append(suffix()).toString(); + } + + /** + * Returns the key identifier associated with this facet range query. + * + * @return the key identifier associated with this facet range query. + */ + public String key() { + return alias != null ? alias : q; + } + + /** + * Returns true if this {@link FacetObjectQuery} is anonymous. + * + * @return true if this {@link FacetObjectQuery} is anonymous. + */ + public boolean isAnonymous() { + return index == 0; + } +} \ No newline at end of file