diff --git a/engine/src/main/java/com/arcadedb/index/IndexInternal.java b/engine/src/main/java/com/arcadedb/index/IndexInternal.java index a746f20f9..85050fd36 100644 --- a/engine/src/main/java/com/arcadedb/index/IndexInternal.java +++ b/engine/src/main/java/com/arcadedb/index/IndexInternal.java @@ -59,6 +59,8 @@ public interface IndexInternal extends Index { boolean isCompacting(); + boolean isValid(); + boolean scheduleCompaction(); String getMostRecentFileName(); diff --git a/engine/src/main/java/com/arcadedb/index/TypeIndex.java b/engine/src/main/java/com/arcadedb/index/TypeIndex.java index 1add88889..01973c484 100644 --- a/engine/src/main/java/com/arcadedb/index/TypeIndex.java +++ b/engine/src/main/java/com/arcadedb/index/TypeIndex.java @@ -467,6 +467,11 @@ public List getIndexesByKeys(final Object[] keys) { return indexesOnBuckets; } + @Override + public boolean isValid() { + return valid; + } + private void checkIsValid() { if (!valid) throw new IndexException("Index '" + getName() + "' is not valid. Probably has been drop or rebuilt"); diff --git a/engine/src/main/java/com/arcadedb/index/lsm/LSMTreeFullTextIndex.java b/engine/src/main/java/com/arcadedb/index/lsm/LSMTreeFullTextIndex.java index 03876a7c9..1c0cb08fd 100644 --- a/engine/src/main/java/com/arcadedb/index/lsm/LSMTreeFullTextIndex.java +++ b/engine/src/main/java/com/arcadedb/index/lsm/LSMTreeFullTextIndex.java @@ -75,8 +75,8 @@ public IndexInternal create(final IndexBuilder builder) { if (builder.isUnique()) throw new IllegalArgumentException("Full text index cannot be unique"); - return new LSMTreeFullTextIndex(builder.getDatabase(), builder.getIndexName(), builder.getFilePath(), ComponentFile.MODE.READ_WRITE, builder.getPageSize(), - builder.getNullStrategy()); + return new LSMTreeFullTextIndex(builder.getDatabase(), builder.getIndexName(), builder.getFilePath(), ComponentFile.MODE.READ_WRITE, + builder.getPageSize(), builder.getNullStrategy()); } } @@ -332,6 +332,11 @@ public Analyzer getAnalyzer() { return analyzer; } + @Override + public boolean isValid() { + return underlyingIndex.isValid(); + } + public List analyzeText(final Analyzer analyzer, final Object[] text) { final List tokens = new ArrayList<>(); diff --git a/engine/src/main/java/com/arcadedb/index/lsm/LSMTreeIndex.java b/engine/src/main/java/com/arcadedb/index/lsm/LSMTreeIndex.java index 0015d0722..98f62357a 100644 --- a/engine/src/main/java/com/arcadedb/index/lsm/LSMTreeIndex.java +++ b/engine/src/main/java/com/arcadedb/index/lsm/LSMTreeIndex.java @@ -253,6 +253,11 @@ public boolean isCompacting() { return status.get() == INDEX_STATUS.COMPACTION_IN_PROGRESS; } + @Override + public boolean isValid() { + return valid; + } + public boolean setStatus(final INDEX_STATUS expectedStatus, final INDEX_STATUS newStatus) { return this.status.compareAndSet(expectedStatus, newStatus); } diff --git a/engine/src/main/java/com/arcadedb/index/vector/HnswVectorIndex.java b/engine/src/main/java/com/arcadedb/index/vector/HnswVectorIndex.java index 22fb4cf55..7bb73f95e 100644 --- a/engine/src/main/java/com/arcadedb/index/vector/HnswVectorIndex.java +++ b/engine/src/main/java/com/arcadedb/index/vector/HnswVectorIndex.java @@ -73,6 +73,10 @@ public interface BuildVectorIndexCallback { void onVertexIndexed(Vertex document, Item item, long totalIndexed); } + public interface IgnoreVertexCallback { + boolean ignoreVertex(Vertex v); + } + public static final String FILE_EXT = "hnswidx"; public static final int CURRENT_VERSION = 0; @@ -219,18 +223,22 @@ public String getName() { } public List> findNeighborsFromId(final TId id, final int k) { + return findNeighborsFromId(id, k, null); + } + + public List> findNeighborsFromId(final TId id, final int k, IgnoreVertexCallback ignoreVertexCallback) { final Vertex start = get(id); if (start == null) return Collections.emptyList(); - return findNeighborsFromVertex(start, k); + return findNeighborsFromVertex(start, k, ignoreVertexCallback); } - public List> findNeighborsFromVertex(final Vertex start, final int k) { + public List> findNeighborsFromVertex(final Vertex start, final int k, final IgnoreVertexCallback ignoreVertexCallback) { final RID startRID = start.getIdentity(); final TVector vector = getVectorFromVertex(start); - final List> neighbors = findNearest(vector, k + 1).stream()// + final List> neighbors = findNearest(vector, k + 1, ignoreVertexCallback).stream()// .filter(result -> !result.item().getIdentity().equals(startRID))// .limit(k)// .collect(Collectors.toList()); @@ -242,7 +250,12 @@ public String getName() { } public List> findNeighborsFromVector(final TVector vector, final int k) { - final List> neighbors = findNearest(vector, k + 1).stream().limit(k).collect(Collectors.toList()); + return findNeighborsFromVector(vector, k, null); + } + + public List> findNeighborsFromVector(final TVector vector, final int k, + final IgnoreVertexCallback ignoreVertexCallback) { + final List> neighbors = findNearest(vector, k + 1, ignoreVertexCallback).stream().limit(k).collect(Collectors.toList()); final List> result = new ArrayList<>(neighbors.size()); for (SearchResult neighbor : neighbors) @@ -339,7 +352,7 @@ public boolean add(Vertex vertex) { } for (int level = Math.min(randomLevel, entryPointCopyMaxLevel); level >= 0; level--) { - final PriorityQueue> topCandidates = searchBaseLayer(currObj, vertexVector, efConstruction, level); + final PriorityQueue> topCandidates = searchBaseLayer(currObj, vertexVector, efConstruction, level, null); final boolean entryPointDeleted = isDeletedFromVertex(entryPointCopy); if (entryPointDeleted) { @@ -453,7 +466,7 @@ public TypeIndex getUnderlyingIndex() { return underlyingIndex; } - public List> findNearest(final TVector destination, final int k) { + public List> findNearest(final TVector destination, final int k, final IgnoreVertexCallback ignoreVertexCallback) { if (entryPoint == null) return Collections.emptyList(); @@ -483,7 +496,7 @@ public List> findNearest(final TVector destinati } } - final PriorityQueue> topCandidates = searchBaseLayer(currObj, destination, Math.max(ef, k), 0); + final PriorityQueue> topCandidates = searchBaseLayer(currObj, destination, Math.max(ef, k), 0, ignoreVertexCallback); while (topCandidates.size() > k) { topCandidates.poll(); @@ -498,7 +511,8 @@ public List> findNearest(final TVector destinati return results; } - private PriorityQueue> searchBaseLayer(final Vertex entryPointNode, final TVector destination, final int k, final int layer) { + private PriorityQueue> searchBaseLayer(final Vertex entryPointNode, final TVector destination, final int k, final int layer, + final IgnoreVertexCallback ignoreVertexCallback) { final Set visitedNodes = new HashSet<>(); final PriorityQueue> topCandidates = new PriorityQueue<>(Comparator.>naturalOrder().reversed()); @@ -506,7 +520,7 @@ private PriorityQueue> searchBaseLayer(final Vertex TDistance lowerBound; - if (!isDeletedFromVertex(entryPointNode)) { + if (!ignoreVertex(entryPointNode, ignoreVertexCallback)) { final TVector entryPointVector = getVectorFromVertex(entryPointNode); final TDistance distance = distanceFunction.distance(destination, entryPointVector); final NodeIdAndDistance pair = new NodeIdAndDistance<>(entryPointNode.getIdentity(), distance, maxValueDistanceComparator); @@ -544,7 +558,7 @@ private PriorityQueue> searchBaseLayer(final Vertex candidateSet.add(candidatePair); - if (!isDeletedFromVertex(candidateNode)) + if (!ignoreVertex(candidateNode, ignoreVertexCallback)) topCandidates.add(candidatePair); if (topCandidates.size() > k) @@ -669,6 +683,14 @@ public boolean isDeletedFromVertex(final Vertex vertex) { return deleted != null && deleted; } + public boolean ignoreVertex(final Vertex vertex, final IgnoreVertexCallback ignoreVertexCallback) { + if (isDeletedFromVertex(vertex)) + return true; + if (ignoreVertexCallback != null) + return ignoreVertexCallback.ignoreVertex(vertex); + return false; + } + public int getDimensionFromVertex(final Vertex vertex) { return Array.getLength(getVectorFromVertex(vertex)); } @@ -775,7 +797,7 @@ public void setNullStrategy(final LSMTreeIndexAbstract.NULL_STRATEGY nullStrateg @Override public boolean isUnique() { - return underlyingIndex.isUnique(); + return true; } @Override @@ -1055,6 +1077,11 @@ public boolean isCompacting() { return underlyingIndex.isCompacting(); } + @Override + public boolean isValid() { + return underlyingIndex.isValid(); + } + @Override public boolean scheduleCompaction() { return underlyingIndex.scheduleCompaction(); diff --git a/engine/src/main/java/com/arcadedb/query/sql/function/vector/SQLFunctionVectorNeighbors.java b/engine/src/main/java/com/arcadedb/query/sql/function/vector/SQLFunctionVectorNeighbors.java index 1caeeed25..35d996924 100644 --- a/engine/src/main/java/com/arcadedb/query/sql/function/vector/SQLFunctionVectorNeighbors.java +++ b/engine/src/main/java/com/arcadedb/query/sql/function/vector/SQLFunctionVectorNeighbors.java @@ -61,7 +61,7 @@ public Object execute(final Object iThis, final Identifiable iCurrentRecord, fin final int limit = iParams[2] instanceof Number ? ((Number) iParams[2]).intValue() : Integer.parseInt(iParams[2].toString()); - final List> neighbors = vIndex.findNeighborsFromVector(key, limit); + final List> neighbors = vIndex.findNeighborsFromVector(key, limit, null); final ArrayList result = new ArrayList<>(neighbors.size()); for (Pair n : neighbors) diff --git a/engine/src/main/java/com/arcadedb/query/sql/parser/RebuildIndexStatement.java b/engine/src/main/java/com/arcadedb/query/sql/parser/RebuildIndexStatement.java index 92aea6ee2..f26f73341 100644 --- a/engine/src/main/java/com/arcadedb/query/sql/parser/RebuildIndexStatement.java +++ b/engine/src/main/java/com/arcadedb/query/sql/parser/RebuildIndexStatement.java @@ -126,6 +126,11 @@ private static void buildIndex(final int maxAttempts, Database database, Index.B for (int attempt = 1; attempt <= maxAttempts; attempt++) { try { + if (!((IndexInternal) idx).isValid()) { + LogManager.instance().log(RebuildIndexStatement.class, Level.SEVERE, "Error on rebuild invalid index '%s'. The index will be removed", idx.getName()); + return; + } + if (((IndexInternal) idx).isCompacting()) throw new NeedRetryException("Cannot rebuild the index '" + idx.getName() + "' while is compacting");