Skip to content

Commit

Permalink
Remove the HPPC dependency from all modules and move the HPPC fork to…
Browse files Browse the repository at this point in the history
… internal. (#13422)

* Remove hppc dependency
* Change fork version to 0.10.0
* Add @lucene.internal
* Move hppc classes to oal.internal.hppc but export it.
* Delete hppc license since it's no longer a dependency.

---------

Co-authored-by: Dawid Weiss <[email protected]>
  • Loading branch information
bruno-roustant and dweiss committed May 27, 2024
1 parent 6f77493 commit 0ba07db
Show file tree
Hide file tree
Showing 133 changed files with 5,849 additions and 575 deletions.
5 changes: 5 additions & 0 deletions lucene/CHANGES.txt
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,11 @@ API Changes

* GITHUB#13145: Deprecate ByteBufferIndexInput as it will be removed in Lucene 10.0. (Uwe Schindler)

* GITHUB#13422: an explicit dependency on the HPPC library is removed in favor of an internal repackaged copy in
oal.internal.hppc. If you relied on HPPC as a transitive dependency, you'll have to add it to your project explicitly.
The HPPC classes now bundled in Lucene core are internal and will have restricted access in future releases, please do
not use them. (Bruno Roustant, Dawid Weiss, Uwe Schindler, Chris Hegarty)

New Features
---------------------

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,11 +20,11 @@
import java.io.Reader;
import org.apache.lucene.analysis.CharFilter; // javadocs
import org.apache.lucene.analysis.util.RollingCharBuffer;
import org.apache.lucene.internal.hppc.CharObjectHashMap;
import org.apache.lucene.util.CharsRef;
import org.apache.lucene.util.fst.CharSequenceOutputs;
import org.apache.lucene.util.fst.FST;
import org.apache.lucene.util.fst.Outputs;
import org.apache.lucene.util.hppc.CharObjectHashMap;

/**
* Simplistic {@link CharFilter} that applies the mappings contained in a {@link NormalizeCharMap}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,14 +19,14 @@
import java.io.IOException;
import java.util.Map;
import java.util.TreeMap;
import org.apache.lucene.internal.hppc.CharObjectHashMap;
import org.apache.lucene.util.CharsRef;
import org.apache.lucene.util.IntsRefBuilder;
import org.apache.lucene.util.fst.CharSequenceOutputs;
import org.apache.lucene.util.fst.FST;
import org.apache.lucene.util.fst.FSTCompiler;
import org.apache.lucene.util.fst.Outputs;
import org.apache.lucene.util.fst.Util;
import org.apache.lucene.util.hppc.CharObjectHashMap;

// TODO: save/load?

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,10 +26,10 @@
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionLengthAttribute;
import org.apache.lucene.internal.hppc.IntArrayList;
import org.apache.lucene.internal.hppc.IntCursor;
import org.apache.lucene.util.AttributeSource;
import org.apache.lucene.util.RollingBuffer;
import org.apache.lucene.util.hppc.IntArrayList;
import org.apache.lucene.util.hppc.IntCursor;

/**
* Converts an incoming graph token stream, such as one from {@link SynonymGraphFilter}, into a flat
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,9 @@
import java.util.stream.Collectors;
import org.apache.lucene.analysis.hunspell.SortingStrategy.EntryAccumulator;
import org.apache.lucene.analysis.hunspell.SortingStrategy.EntrySupplier;
import org.apache.lucene.internal.hppc.CharHashSet;
import org.apache.lucene.internal.hppc.IntArrayList;
import org.apache.lucene.internal.hppc.IntCursor;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.IntsRef;
Expand All @@ -58,9 +61,6 @@
import org.apache.lucene.util.fst.FSTCompiler;
import org.apache.lucene.util.fst.IntSequenceOutputs;
import org.apache.lucene.util.fst.Util;
import org.apache.lucene.util.hppc.CharHashSet;
import org.apache.lucene.util.hppc.IntArrayList;
import org.apache.lucene.util.hppc.IntCursor;

/** In-memory structure for the dictionary (.dic) and affix (.aff) data of a hunspell dictionary. */
public class Dictionary {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -40,11 +40,11 @@
import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.apache.lucene.analysis.hunspell.AffixedWord.Affix;
import org.apache.lucene.internal.hppc.CharHashSet;
import org.apache.lucene.internal.hppc.CharObjectHashMap;
import org.apache.lucene.util.IntsRef;
import org.apache.lucene.util.fst.FST;
import org.apache.lucene.util.fst.IntsRefFSTEnum;
import org.apache.lucene.util.hppc.CharHashSet;
import org.apache.lucene.util.hppc.CharObjectHashMap;

/**
* A utility class used for generating possible word forms by adding affixes to stems ({@link
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
import java.util.List;
import java.util.function.BiConsumer;
import java.util.function.Supplier;
import org.apache.lucene.internal.hppc.IntArrayList;
import org.apache.lucene.store.ByteArrayDataInput;
import org.apache.lucene.store.ByteArrayDataOutput;
import org.apache.lucene.store.DataOutput;
Expand All @@ -29,7 +30,6 @@
import org.apache.lucene.util.IntsRef;
import org.apache.lucene.util.IntsRefBuilder;
import org.apache.lucene.util.fst.IntSequenceOutputs;
import org.apache.lucene.util.hppc.IntArrayList;

/**
* A data structure for memory-efficient word storage and fast lookup/enumeration. Each dictionary
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,9 +21,9 @@
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
import org.apache.lucene.internal.hppc.IntArrayList;
import org.apache.lucene.util.AttributeFactory;
import org.apache.lucene.util.IgnoreRandomChains;
import org.apache.lucene.util.hppc.IntArrayList;

/**
* Tokenizer for domain-like hierarchies.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,8 @@
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
import org.apache.lucene.internal.hppc.IntArrayList;
import org.apache.lucene.internal.hppc.IntHashSet;
import org.apache.lucene.store.ByteArrayDataOutput;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.BytesRefBuilder;
Expand All @@ -37,8 +39,6 @@
import org.apache.lucene.util.fst.FST;
import org.apache.lucene.util.fst.FSTCompiler;
import org.apache.lucene.util.fst.Util;
import org.apache.lucene.util.hppc.IntArrayList;
import org.apache.lucene.util.hppc.IntHashSet;

/**
* A map of synonyms, keys and values are phrases.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,11 +27,11 @@
import java.util.List;
import java.util.Map;
import org.apache.lucene.analysis.TokenizerFactory;
import org.apache.lucene.internal.hppc.IntObjectHashMap;
import org.apache.lucene.util.AttributeFactory;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.ResourceLoader;
import org.apache.lucene.util.ResourceLoaderAware;
import org.apache.lucene.util.hppc.IntObjectHashMap;

/**
* Factory for {@link ICUTokenizer}. Words are broken across script boundaries, then segmented
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
package org.apache.lucene.analysis.ja;

import java.util.Map;
import org.apache.lucene.util.hppc.CharObjectHashMap;
import org.apache.lucene.internal.hppc.CharObjectHashMap;

/** Utility methods for Japanese filters. */
class JapaneseFilterUtil {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@
import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.util.hppc.CharObjectHashMap;
import org.apache.lucene.internal.hppc.CharObjectHashMap;

/**
* A {@link TokenFilter} that normalizes small letters (捨て仮名) in hiragana into normal letters. For
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@
import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.util.hppc.CharObjectHashMap;
import org.apache.lucene.internal.hppc.CharObjectHashMap;

/**
* A {@link TokenFilter} that normalizes small letters (捨て仮名) in katakana into normal letters. For
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,9 @@
import java.util.ArrayList;
import java.util.List;
import org.apache.lucene.analysis.cn.smart.Utility;
import org.apache.lucene.util.hppc.IntArrayList;
import org.apache.lucene.util.hppc.IntObjectHashMap;
import org.apache.lucene.util.hppc.ObjectCursor;
import org.apache.lucene.internal.hppc.IntArrayList;
import org.apache.lucene.internal.hppc.IntObjectHashMap;
import org.apache.lucene.internal.hppc.ObjectCursor;

/**
* Graph representing possible token pairs (bigrams) at each start offset in the sentence.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@

import java.util.ArrayList;
import java.util.List;
import org.apache.lucene.util.hppc.IntObjectHashMap;
import org.apache.lucene.internal.hppc.IntObjectHashMap;

/**
* Graph representing possible tokens at each start offset in the sentence.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ created by Leo Galambos ([email protected]).
import java.util.Arrays;
import java.util.Iterator;
import java.util.List;
import org.apache.lucene.util.hppc.ObjectCursor;
import org.apache.lucene.internal.hppc.ObjectCursor;

/**
* The Gener object helps in the discarding of nodes which break the reduction effort and defend the
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ created by Leo Galambos ([email protected]).
import java.util.Arrays;
import java.util.Iterator;
import java.util.List;
import org.apache.lucene.util.hppc.ObjectCursor;
import org.apache.lucene.internal.hppc.ObjectCursor;

/**
* The Lift class is a data structure that is a variation of a Patricia trie.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ created by Leo Galambos ([email protected]).
import java.util.Arrays;
import java.util.Iterator;
import java.util.List;
import org.apache.lucene.util.hppc.CharCursor;
import org.apache.lucene.internal.hppc.CharCursor;

/**
* The Optimizer class is a Trie that will be reduced (have empty rows removed).
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -58,8 +58,8 @@ created by Leo Galambos ([email protected]).
import java.util.Arrays;
import java.util.Iterator;
import java.util.List;
import org.apache.lucene.util.hppc.CharCursor;
import org.apache.lucene.util.hppc.ObjectCursor;
import org.apache.lucene.internal.hppc.CharCursor;
import org.apache.lucene.internal.hppc.ObjectCursor;

/** The Reduce object is used to remove gaps in a Trie which stores a dictionary. */
public class Reduce {
Expand Down
6 changes: 3 additions & 3 deletions lucene/analysis/stempel/src/java/org/egothor/stemmer/Row.java
Original file line number Diff line number Diff line change
Expand Up @@ -59,9 +59,9 @@ created by Leo Galambos ([email protected]).
import java.io.IOException;
import java.io.PrintStream;
import java.util.Iterator;
import org.apache.lucene.util.hppc.CharCursor;
import org.apache.lucene.util.hppc.CharObjectHashMap;
import org.apache.lucene.util.hppc.ObjectCursor;
import org.apache.lucene.internal.hppc.CharCursor;
import org.apache.lucene.internal.hppc.CharObjectHashMap;
import org.apache.lucene.internal.hppc.ObjectCursor;

/** The Row class represents a row in a matrix representation of a trie. */
public class Row {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,14 +23,14 @@
import org.apache.lucene.benchmark.byTask.utils.Config;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.internal.hppc.IntObjectHashMap;
import org.apache.lucene.spatial.SpatialStrategy;
import org.apache.lucene.spatial.composite.CompositeSpatialStrategy;
import org.apache.lucene.spatial.prefix.RecursivePrefixTreeStrategy;
import org.apache.lucene.spatial.prefix.tree.PackedQuadPrefixTree;
import org.apache.lucene.spatial.prefix.tree.SpatialPrefixTree;
import org.apache.lucene.spatial.prefix.tree.SpatialPrefixTreeFactory;
import org.apache.lucene.spatial.serialized.SerializedDVStrategy;
import org.apache.lucene.util.hppc.IntObjectHashMap;
import org.locationtech.spatial4j.context.SpatialContext;
import org.locationtech.spatial4j.context.SpatialContextFactory;
import org.locationtech.spatial4j.shape.Point;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@
import java.util.Map;
import java.util.Properties;
import java.util.StringTokenizer;
import org.apache.lucene.util.hppc.IntArrayList;
import org.apache.lucene.internal.hppc.IntArrayList;

/**
* Perf run configuration properties.
Expand Down
19 changes: 11 additions & 8 deletions lucene/core/src/java/module-info.java
Original file line number Diff line number Diff line change
Expand Up @@ -24,36 +24,39 @@
requires static jdk.unsupported; // this is optional but without it MMapDirectory won't be enabled
requires static jdk.management; // this is optional but explicit declaration is recommended

exports org.apache.lucene.analysis;
exports org.apache.lucene.analysis.standard;
exports org.apache.lucene.analysis.tokenattributes;
exports org.apache.lucene.codecs;
exports org.apache.lucene.analysis;
exports org.apache.lucene.codecs.compressing;
exports org.apache.lucene.codecs.lucene90.blocktree;
exports org.apache.lucene.codecs.lucene90.compressing;
exports org.apache.lucene.codecs.lucene90;
exports org.apache.lucene.codecs.lucene94;
exports org.apache.lucene.codecs.lucene95;
exports org.apache.lucene.codecs.lucene99;
exports org.apache.lucene.codecs.lucene90.blocktree;
exports org.apache.lucene.codecs.lucene90.compressing;
exports org.apache.lucene.codecs.perfield;
exports org.apache.lucene.codecs;
exports org.apache.lucene.document;
exports org.apache.lucene.geo;
exports org.apache.lucene.index;
exports org.apache.lucene.search;
exports org.apache.lucene.search.comparators;
exports org.apache.lucene.search.similarities;
exports org.apache.lucene.search.knn;
exports org.apache.lucene.search.similarities;
exports org.apache.lucene.search;
exports org.apache.lucene.store;
exports org.apache.lucene.util;
exports org.apache.lucene.util.automaton;
exports org.apache.lucene.util.bkd;
exports org.apache.lucene.util.compress;
exports org.apache.lucene.util.fst;
exports org.apache.lucene.util.graph;
exports org.apache.lucene.util.hnsw;
exports org.apache.lucene.util.hppc;
exports org.apache.lucene.util.mutable;
exports org.apache.lucene.util.packed;
exports org.apache.lucene.util;

// Temporarily export HPPC to all modules (eventually, this
// should be restricted to only Lucene modules)
exports org.apache.lucene.internal.hppc;

// Only export internal packages to the test framework.
exports org.apache.lucene.internal.tests to
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,12 +25,12 @@
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionLengthAttribute;
import org.apache.lucene.internal.hppc.IntArrayList;
import org.apache.lucene.internal.hppc.IntCursor;
import org.apache.lucene.internal.hppc.IntIntHashMap;
import org.apache.lucene.util.automaton.Automaton;
import org.apache.lucene.util.automaton.Operations;
import org.apache.lucene.util.automaton.Transition;
import org.apache.lucene.util.hppc.IntArrayList;
import org.apache.lucene.util.hppc.IntCursor;
import org.apache.lucene.util.hppc.IntIntHashMap;

/** Converts an Automaton into a TokenStream. */
public class AutomatonToTokenStream {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,8 @@
import org.apache.lucene.index.SortedNumericDocValues;
import org.apache.lucene.index.SortedSetDocValues;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.internal.hppc.LongHashSet;
import org.apache.lucene.internal.hppc.LongIntHashMap;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.SortedSetSelector;
import org.apache.lucene.store.ByteArrayDataOutput;
Expand All @@ -50,8 +52,6 @@
import org.apache.lucene.util.MathUtil;
import org.apache.lucene.util.StringHelper;
import org.apache.lucene.util.compress.LZ4;
import org.apache.lucene.util.hppc.LongHashSet;
import org.apache.lucene.util.hppc.LongIntHashMap;
import org.apache.lucene.util.packed.DirectMonotonicWriter;
import org.apache.lucene.util.packed.DirectWriter;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -29,12 +29,12 @@
import org.apache.lucene.index.IndexFileNames;
import org.apache.lucene.index.NumericDocValues;
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.internal.hppc.IntObjectHashMap;
import org.apache.lucene.store.ChecksumIndexInput;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.RandomAccessInput;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.hppc.IntObjectHashMap;

/** Reader for {@link Lucene90NormsFormat} */
final class Lucene90NormsProducer extends NormsProducer implements Cloneable {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,12 +24,12 @@
import org.apache.lucene.index.IndexFileNames;
import org.apache.lucene.index.PointValues;
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.internal.hppc.IntObjectHashMap;
import org.apache.lucene.store.ChecksumIndexInput;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.bkd.BKDReader;
import org.apache.lucene.util.hppc.IntObjectHashMap;

/** Reads point values previously written with {@link Lucene90PointsWriter} */
public class Lucene90PointsReader extends PointsReader {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -31,15 +31,15 @@
import org.apache.lucene.index.IndexOptions;
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.Terms;
import org.apache.lucene.internal.hppc.IntCursor;
import org.apache.lucene.internal.hppc.IntObjectHashMap;
import org.apache.lucene.store.ChecksumIndexInput;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.fst.ByteSequenceOutputs;
import org.apache.lucene.util.fst.Outputs;
import org.apache.lucene.util.hppc.IntCursor;
import org.apache.lucene.util.hppc.IntObjectHashMap;

/**
* A block-based terms index and dictionary that assigns terms to variable length blocks according
Expand Down
Loading

0 comments on commit 0ba07db

Please sign in to comment.