Skip to content

Commit

Permalink
Added logs to understand the time taken by different merge related items
Browse files Browse the repository at this point in the history
Signed-off-by: Navneet Verma <[email protected]>
  • Loading branch information
navneet1v committed Sep 28, 2024
1 parent 7772c96 commit 9f43164
Showing 1 changed file with 9 additions and 5 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -108,31 +108,35 @@ public void flush(int maxDoc, final Sorter.DocMap sortMap) throws IOException {
@Override
public void mergeOneField(final FieldInfo fieldInfo, final MergeState mergeState) throws IOException {
// This will ensure that we are merging the FlatIndex during force merge.
StopWatch stopwatch = new StopWatch().start();
flatVectorsWriter.mergeOneField(fieldInfo, mergeState);

log.info("Merge : Reading and writing merged FlatVector Took : {} ms", stopwatch.stop().totalTime().millis());
final VectorDataType vectorDataType = extractVectorDataType(fieldInfo);
final Supplier<KNNVectorValues<?>> knnVectorValuesSupplier = () -> getKNNVectorValuesForMerge(
vectorDataType,
fieldInfo,
mergeState
);
stopwatch = new StopWatch().start();
int totalLiveDocs = getLiveDocs(knnVectorValuesSupplier.get());
log.info("Merge : Reading live docs Took : {} ms", stopwatch.stop().totalTime().millis());
if (totalLiveDocs == 0) {
log.debug("[Merge] No live docs for field {}", fieldInfo.getName());
return;
}

final QuantizationState quantizationState = train(fieldInfo, knnVectorValuesSupplier, totalLiveDocs);
final NativeIndexWriter writer = NativeIndexWriter.getWriter(fieldInfo, segmentWriteState, quantizationState);
final KNNVectorValues<?> knnVectorValues = knnVectorValuesSupplier.get();

StopWatch stopWatch2 = new StopWatch().start();
final KNNVectorValues<?> knnVectorValues = knnVectorValuesSupplier.get();
log.info("Merge : Creating merged VectorValues Took : {} ms", stopWatch2.stop().totalTime().millis());
StopWatch stopWatch = new StopWatch().start();

final NativeIndexWriter writer = NativeIndexWriter.getWriter(fieldInfo, segmentWriteState, quantizationState);
writer.mergeIndex(knnVectorValues, totalLiveDocs);

long time_in_millis = stopWatch.stop().totalTime().millis();
KNNGraphValue.MERGE_TOTAL_TIME_IN_MILLIS.incrementBy(time_in_millis);
log.debug("Merge took {} ms for vector field [{}]", time_in_millis, fieldInfo.getName());
log.info("Merge took {} ms for vector field [{}]", time_in_millis, fieldInfo.getName());
}

/**
Expand Down

0 comments on commit 9f43164

Please sign in to comment.