Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[9.x] Ensure to use IOContext.READONCE when reading segment files #13578

Merged
merged 5 commits into from
Jul 17, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -299,7 +299,8 @@ static final SegmentInfos readCommit(

long generation = generationFromSegmentsFileName(segmentFileName);
// System.out.println(Thread.currentThread() + ": SegmentInfos.readCommit " + segmentFileName);
try (ChecksumIndexInput input = directory.openChecksumInput(segmentFileName, IOContext.READ)) {
try (ChecksumIndexInput input =
directory.openChecksumInput(segmentFileName, IOContext.READONCE)) {
try {
return readCommit(directory, input, generation, minSupportedMajorVersion);
} catch (EOFException | NoSuchFileException | FileNotFoundException e) {
Expand Down
4 changes: 2 additions & 2 deletions lucene/core/src/java/org/apache/lucene/store/Directory.java
Original file line number Diff line number Diff line change
Expand Up @@ -172,12 +172,12 @@ public ChecksumIndexInput openChecksumInput(String name, IOContext context) thro

/**
* Copies an existing {@code src} file from directory {@code from} to a non-existent file {@code
* dest} in this directory.
* dest} in this directory. The given IOContext is only used for opening the destination file.
*/
public void copyFrom(Directory from, String src, String dest, IOContext context)
throws IOException {
boolean success = false;
try (IndexInput is = from.openInput(src, context);
try (IndexInput is = from.openInput(src, IOContext.READONCE);
IndexOutput os = createOutput(dest, context)) {
os.copyBytes(is, is.length());
success = true;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -115,7 +115,7 @@ private void checkOneFile(Directory dir, String victim) throws IOException {
// time this will only require one iteration!
while (true) {
try (IndexOutput out = dirCopy.createOutput(name, IOContext.DEFAULT);
IndexInput in = dir.openInput(name, IOContext.DEFAULT)) {
IndexInput in = dir.openInput(name, IOContext.READONCE)) {
// keeps same file length, but replaces the first wrongBytes with random bytes:
byte[] bytes = new byte[wrongBytes];
random().nextBytes(bytes);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -122,7 +122,7 @@ private void corruptFile(Directory dir, String victim) throws IOException {
dirCopy.copyFrom(dir, name, name, IOContext.DEFAULT);
} else {
try (IndexOutput out = dirCopy.createOutput(name, IOContext.DEFAULT);
IndexInput in = dir.openInput(name, IOContext.DEFAULT)) {
IndexInput in = dir.openInput(name, IOContext.READONCE)) {
out.copyBytes(in, flipOffset);
out.writeByte((byte) (in.readByte() + TestUtil.nextInt(random(), 0x01, 0xFF)));
out.copyBytes(in, victimLength - flipOffset - 1);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -138,7 +138,7 @@ private void truncateOneFile(Directory dir, String victim) throws IOException {
if (name.equals(victim) == false) {
dirCopy.copyFrom(dir, name, name, IOContext.DEFAULT);
} else {
try (ChecksumIndexInput in = dir.openChecksumInput(name, IOContext.DEFAULT)) {
try (ChecksumIndexInput in = dir.openChecksumInput(name, IOContext.READONCE)) {
try {
CodecUtil.checkFooter(in);
// In some rare cases, the codec footer would still appear as correct even though the
Expand All @@ -152,7 +152,7 @@ private void truncateOneFile(Directory dir, String victim) throws IOException {
}

try (IndexOutput out = dirCopy.createOutput(name, IOContext.DEFAULT);
IndexInput in = dir.openInput(name, IOContext.DEFAULT)) {
IndexInput in = dir.openInput(name, IOContext.READONCE)) {
out.copyBytes(in, victimLength - lostBytes);
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
import java.io.IOException;
import org.apache.lucene.codecs.CodecUtil;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.tests.analysis.MockAnalyzer;
import org.apache.lucene.tests.index.RandomIndexWriter;
Expand Down Expand Up @@ -77,7 +78,7 @@ private void checkFooters(Directory dir) throws IOException {
}

private void checkFooter(Directory dir, String file) throws IOException {
try (IndexInput in = dir.openInput(file, newIOContext(random()))) {
try (IndexInput in = dir.openInput(file, IOContext.READONCE)) {
CodecUtil.checksumEntireFile(in);
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
import java.util.Map;
import org.apache.lucene.codecs.CodecUtil;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.tests.analysis.MockAnalyzer;
import org.apache.lucene.tests.index.RandomIndexWriter;
Expand Down Expand Up @@ -84,7 +85,7 @@ private void checkHeaders(Directory dir, Map<String, String> namesToExtensions)
private void checkHeader(
Directory dir, String file, Map<String, String> namesToExtensions, byte[] id)
throws IOException {
try (IndexInput in = dir.openInput(file, newIOContext(random()))) {
try (IndexInput in = dir.openInput(file, IOContext.READONCE)) {
int val = CodecUtil.readBEInt(in);
assertEquals(
file + " has no codec header, instead found: " + val, CodecUtil.CODEC_MAGIC, val);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1275,7 +1275,7 @@ public void testSegmentsChecksumError() throws IOException {
assertTrue("segment generation should be > 0 but got " + gen, gen > 0);

final String segmentsFileName = SegmentInfos.getLastCommitSegmentsFileName(dir);
IndexInput in = dir.openInput(segmentsFileName, newIOContext(random()));
IndexInput in = dir.openInput(segmentsFileName, IOContext.READONCE);
IndexOutput out =
dir.createOutput(
IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS, "", 1 + gen),
Expand Down Expand Up @@ -1320,7 +1320,7 @@ public void testSimulatedCorruptIndex1() throws IOException {
String fileNameIn = SegmentInfos.getLastCommitSegmentsFileName(dir);
String fileNameOut =
IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS, "", 1 + gen);
IndexInput in = dir.openInput(fileNameIn, newIOContext(random()));
IndexInput in = dir.openInput(fileNameIn, IOContext.READONCE);
IndexOutput out = dir.createOutput(fileNameOut, newIOContext(random()));
long length = in.length();
for (int i = 0; i < length - 1; i++) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -367,15 +367,15 @@ public void testBitFlippedTriggersCorruptIndexException() throws IOException {
boolean corrupt = false;
for (String file : dir.listAll()) {
if (file.startsWith(IndexFileNames.SEGMENTS)) {
try (IndexInput in = dir.openInput(file, IOContext.DEFAULT);
try (IndexInput in = dir.openInput(file, IOContext.READONCE);
IndexOutput out = corruptDir.createOutput(file, IOContext.DEFAULT)) {
final long corruptIndex = TestUtil.nextLong(random(), 0, in.length() - 1);
out.copyBytes(in, corruptIndex);
final int b = Byte.toUnsignedInt(in.readByte()) + TestUtil.nextInt(random(), 0x01, 0xff);
out.writeByte((byte) b);
out.copyBytes(in, in.length() - in.getFilePointer());
}
try (IndexInput in = corruptDir.openInput(file, IOContext.DEFAULT)) {
try (IndexInput in = corruptDir.openInput(file, IOContext.READONCE)) {
CodecUtil.checksumEntireFile(in);
if (VERBOSE) {
System.out.println("TEST: Altering the file did not update the checksum, aborting...");
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
import org.apache.lucene.document.FieldType;
import org.apache.lucene.document.TextField;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.tests.analysis.MockAnalyzer;
import org.apache.lucene.tests.util.LuceneTestCase;
Expand Down Expand Up @@ -226,7 +227,7 @@ private void copyFiles(Directory dir, IndexCommit cp) throws Exception {
byte[] buffer = new byte[4096];

private void readFile(Directory dir, String name) throws Exception {
IndexInput input = dir.openInput(name, newIOContext(random()));
IndexInput input = dir.openInput(name, IOContext.READONCE);
try {
long size = dir.fileLength(name);
long bytesLeft = size;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -342,7 +342,7 @@ public static String getIndexFormat(Directory dir) throws IOException {
@Override
protected String doBody(String segmentFileName) throws IOException {
String format = "unknown";
try (IndexInput in = dir.openInput(segmentFileName, IOContext.READ)) {
try (IndexInput in = dir.openInput(segmentFileName, IOContext.READONCE)) {
if (CodecUtil.CODEC_MAGIC == CodecUtil.readBEInt(in)) {
int actualVersion =
CodecUtil.checkHeaderNoMagic(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -813,7 +813,15 @@ public synchronized IndexInput openInput(String name, IOContext context) throws
}

context = LuceneTestCase.newIOContext(randomState, context);
final boolean confined = context == IOContext.READONCE;
final boolean confined = context.readOnce;
if (name.startsWith(IndexFileNames.SEGMENTS) && confined == false) {
throw new RuntimeException(
"MockDirectoryWrapper: opening segments file ["
+ name
+ "] with a non-READONCE context["
+ context
+ "]");
}
IndexInput delegateInput = in.openInput(name, context);

final IndexInput ii;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3061,7 +3061,7 @@ private static Throwable _expectThrows(
*/
public static boolean slowFileExists(Directory dir, String fileName) throws IOException {
try {
dir.openInput(fileName, IOContext.DEFAULT).close();
dir.openInput(fileName, IOContext.READONCE).close();
return true;
} catch (@SuppressWarnings("unused") NoSuchFileException | FileNotFoundException e) {
return false;
Expand Down