fix: avoid rebuilding an invalid index

engine/src/main/java/com/arcadedb/index/IndexInternal.java

@@ -59,6 +59,8 @@ public interface IndexInternal extends Index {
 
   boolean isCompacting();
 
+  boolean isValid();
+
   boolean scheduleCompaction();
 
   String getMostRecentFileName();

engine/src/main/java/com/arcadedb/index/TypeIndex.java

@@ -467,6 +467,11 @@ public List<? extends Index> getIndexesByKeys(final Object[] keys) {
     return indexesOnBuckets;
   }
 
+  @Override
+  public boolean isValid() {
+    return valid;
+  }
+
   private void checkIsValid() {
     if (!valid)
       throw new IndexException("Index '" + getName() + "' is not valid. Probably has been drop or rebuilt");

engine/src/main/java/com/arcadedb/index/lsm/LSMTreeFullTextIndex.java

@@ -75,8 +75,8 @@ public IndexInternal create(final IndexBuilder builder) {
       if (builder.isUnique())
         throw new IllegalArgumentException("Full text index cannot be unique");
 
-      return new LSMTreeFullTextIndex(builder.getDatabase(), builder.getIndexName(), builder.getFilePath(), ComponentFile.MODE.READ_WRITE, builder.getPageSize(),
-          builder.getNullStrategy());
+      return new LSMTreeFullTextIndex(builder.getDatabase(), builder.getIndexName(), builder.getFilePath(), ComponentFile.MODE.READ_WRITE,
+          builder.getPageSize(), builder.getNullStrategy());
     }
   }
 
@@ -332,6 +332,11 @@ public Analyzer getAnalyzer() {
     return analyzer;
   }
 
+  @Override
+  public boolean isValid() {
+    return underlyingIndex.isValid();
+  }
+
   public List<String> analyzeText(final Analyzer analyzer, final Object[] text) {
     final List<String> tokens = new ArrayList<>();
 

engine/src/main/java/com/arcadedb/index/lsm/LSMTreeIndex.java

@@ -253,6 +253,11 @@ public boolean isCompacting() {
     return status.get() == INDEX_STATUS.COMPACTION_IN_PROGRESS;
   }
 
+  @Override
+  public boolean isValid() {
+    return valid;
+  }
+
   public boolean setStatus(final INDEX_STATUS expectedStatus, final INDEX_STATUS newStatus) {
     return this.status.compareAndSet(expectedStatus, newStatus);
   }

engine/src/main/java/com/arcadedb/index/vector/HnswVectorIndex.java

@@ -73,6 +73,10 @@ public interface BuildVectorIndexCallback<TId, TVector> {
     void onVertexIndexed(Vertex document, Item<TId, TVector> item, long totalIndexed);
   }
 
+  public interface IgnoreVertexCallback {
+    boolean ignoreVertex(Vertex v);
+  }
+
   public static final String FILE_EXT        = "hnswidx";
   public static final int    CURRENT_VERSION = 0;
 
@@ -219,18 +223,22 @@ public String getName() {
   }
 
   public List<Pair<Identifiable, ? extends Number>> findNeighborsFromId(final TId id, final int k) {
+    return findNeighborsFromId(id, k, null);
+  }
+
+  public List<Pair<Identifiable, ? extends Number>> findNeighborsFromId(final TId id, final int k, IgnoreVertexCallback ignoreVertexCallback) {
     final Vertex start = get(id);
     if (start == null)
       return Collections.emptyList();
 
-    return findNeighborsFromVertex(start, k);
+    return findNeighborsFromVertex(start, k, ignoreVertexCallback);
   }
 
-  public List<Pair<Identifiable, ? extends Number>> findNeighborsFromVertex(final Vertex start, final int k) {
+  public List<Pair<Identifiable, ? extends Number>> findNeighborsFromVertex(final Vertex start, final int k, final IgnoreVertexCallback ignoreVertexCallback) {
     final RID startRID = start.getIdentity();
     final TVector vector = getVectorFromVertex(start);
 
-    final List<SearchResult<Vertex, TDistance>> neighbors = findNearest(vector, k + 1).stream()//
+    final List<SearchResult<Vertex, TDistance>> neighbors = findNearest(vector, k + 1, ignoreVertexCallback).stream()//
         .filter(result -> !result.item().getIdentity().equals(startRID))//
         .limit(k)//
         .collect(Collectors.toList());
@@ -242,7 +250,12 @@ public String getName() {
   }
 
   public List<Pair<Identifiable, ? extends Number>> findNeighborsFromVector(final TVector vector, final int k) {
-    final List<SearchResult<Vertex, TDistance>> neighbors = findNearest(vector, k + 1).stream().limit(k).collect(Collectors.toList());
+    return findNeighborsFromVector(vector, k, null);
+  }
+
+  public List<Pair<Identifiable, ? extends Number>> findNeighborsFromVector(final TVector vector, final int k,
+      final IgnoreVertexCallback ignoreVertexCallback) {
+    final List<SearchResult<Vertex, TDistance>> neighbors = findNearest(vector, k + 1, ignoreVertexCallback).stream().limit(k).collect(Collectors.toList());
 
     final List<Pair<Identifiable, ? extends Number>> result = new ArrayList<>(neighbors.size());
     for (SearchResult<Vertex, TDistance> neighbor : neighbors)
@@ -339,7 +352,7 @@ public boolean add(Vertex vertex) {
           }
 
           for (int level = Math.min(randomLevel, entryPointCopyMaxLevel); level >= 0; level--) {
-            final PriorityQueue<NodeIdAndDistance<TDistance>> topCandidates = searchBaseLayer(currObj, vertexVector, efConstruction, level);
+            final PriorityQueue<NodeIdAndDistance<TDistance>> topCandidates = searchBaseLayer(currObj, vertexVector, efConstruction, level, null);
 
             final boolean entryPointDeleted = isDeletedFromVertex(entryPointCopy);
             if (entryPointDeleted) {
@@ -453,7 +466,7 @@ public TypeIndex getUnderlyingIndex() {
     return underlyingIndex;
   }
 
-  public List<SearchResult<Vertex, TDistance>> findNearest(final TVector destination, final int k) {
+  public List<SearchResult<Vertex, TDistance>> findNearest(final TVector destination, final int k, final IgnoreVertexCallback ignoreVertexCallback) {
     if (entryPoint == null)
       return Collections.emptyList();
 
@@ -483,7 +496,7 @@ public List<SearchResult<Vertex, TDistance>> findNearest(final TVector destinati
       }
     }
 
-    final PriorityQueue<NodeIdAndDistance<TDistance>> topCandidates = searchBaseLayer(currObj, destination, Math.max(ef, k), 0);
+    final PriorityQueue<NodeIdAndDistance<TDistance>> topCandidates = searchBaseLayer(currObj, destination, Math.max(ef, k), 0, ignoreVertexCallback);
 
     while (topCandidates.size() > k) {
       topCandidates.poll();
@@ -498,15 +511,16 @@ public List<SearchResult<Vertex, TDistance>> findNearest(final TVector destinati
     return results;
   }
 
-  private PriorityQueue<NodeIdAndDistance<TDistance>> searchBaseLayer(final Vertex entryPointNode, final TVector destination, final int k, final int layer) {
+  private PriorityQueue<NodeIdAndDistance<TDistance>> searchBaseLayer(final Vertex entryPointNode, final TVector destination, final int k, final int layer,
+      final IgnoreVertexCallback ignoreVertexCallback) {
     final Set<RID> visitedNodes = new HashSet<>();
 
     final PriorityQueue<NodeIdAndDistance<TDistance>> topCandidates = new PriorityQueue<>(Comparator.<NodeIdAndDistance<TDistance>>naturalOrder().reversed());
     final PriorityQueue<NodeIdAndDistance<TDistance>> candidateSet = new PriorityQueue<>();
 
     TDistance lowerBound;
 
-    if (!isDeletedFromVertex(entryPointNode)) {
+    if (!ignoreVertex(entryPointNode, ignoreVertexCallback)) {
       final TVector entryPointVector = getVectorFromVertex(entryPointNode);
       final TDistance distance = distanceFunction.distance(destination, entryPointVector);
       final NodeIdAndDistance<TDistance> pair = new NodeIdAndDistance<>(entryPointNode.getIdentity(), distance, maxValueDistanceComparator);
@@ -544,7 +558,7 @@ private PriorityQueue<NodeIdAndDistance<TDistance>> searchBaseLayer(final Vertex
 
             candidateSet.add(candidatePair);
 
-            if (!isDeletedFromVertex(candidateNode))
+            if (!ignoreVertex(candidateNode, ignoreVertexCallback))
               topCandidates.add(candidatePair);
 
             if (topCandidates.size() > k)
@@ -669,6 +683,14 @@ public boolean isDeletedFromVertex(final Vertex vertex) {
     return deleted != null && deleted;
   }
 
+  public boolean ignoreVertex(final Vertex vertex, final IgnoreVertexCallback ignoreVertexCallback) {
+    if (isDeletedFromVertex(vertex))
+      return true;
+    if (ignoreVertexCallback != null)
+      return ignoreVertexCallback.ignoreVertex(vertex);
+    return false;
+  }
+
   public int getDimensionFromVertex(final Vertex vertex) {
     return Array.getLength(getVectorFromVertex(vertex));
   }
@@ -775,7 +797,7 @@ public void setNullStrategy(final LSMTreeIndexAbstract.NULL_STRATEGY nullStrateg
 
   @Override
   public boolean isUnique() {
-    return underlyingIndex.isUnique();
+    return true;
   }
 
   @Override
@@ -1055,6 +1077,11 @@ public boolean isCompacting() {
     return underlyingIndex.isCompacting();
   }
 
+  @Override
+  public boolean isValid() {
+    return underlyingIndex.isValid();
+  }
+
   @Override
   public boolean scheduleCompaction() {
     return underlyingIndex.scheduleCompaction();

engine/src/main/java/com/arcadedb/query/sql/function/vector/SQLFunctionVectorNeighbors.java

@@ -61,7 +61,7 @@ public Object execute(final Object iThis, final Identifiable iCurrentRecord, fin
 
     final int limit = iParams[2] instanceof Number ? ((Number) iParams[2]).intValue() : Integer.parseInt(iParams[2].toString());
 
-    final List<Pair<Vertex, ? extends Number>> neighbors = vIndex.findNeighborsFromVector(key, limit);
+    final List<Pair<Vertex, ? extends Number>> neighbors = vIndex.findNeighborsFromVector(key, limit, null);
 
     final ArrayList<Object> result = new ArrayList<>(neighbors.size());
     for (Pair<Vertex, ? extends Number> n : neighbors)

engine/src/main/java/com/arcadedb/query/sql/parser/RebuildIndexStatement.java

@@ -126,6 +126,11 @@ private static void buildIndex(final int maxAttempts, Database database, Index.B
 
     for (int attempt = 1; attempt <= maxAttempts; attempt++) {
       try {
+        if (!((IndexInternal) idx).isValid()) {
+          LogManager.instance().log(RebuildIndexStatement.class, Level.SEVERE, "Error on rebuild invalid index '%s'. The index will be removed", idx.getName());
+          return;
+        }
+
         if (((IndexInternal) idx).isCompacting())
           throw new NeedRetryException("Cannot rebuild the index '" + idx.getName() + "' while is compacting");