diff --git a/modules/packages/ParallelIO.chpl b/modules/packages/ParallelIO.chpl index 3bc3e7dd900e..2e8a68e17cbe 100644 --- a/modules/packages/ParallelIO.chpl +++ b/modules/packages/ParallelIO.chpl @@ -128,7 +128,7 @@ module ParallelIO { param delim = b"\n"; const fMeta = try! open(filePath, ioMode.r), - fileBounds = 0..<(try! fMeta.size); + fileBounds = 0..(try! fMeta.size); if tld.size == 0 || (tld.size == 1 && targetLocales.first == here) { const byteOffsets = try! findDelimChunks(fMeta, delim, nTasks, fileBounds, header); @@ -143,17 +143,21 @@ module ParallelIO { } else { const byteOffsets = try! findDelimChunks(fMeta, delim, tld.size, fileBounds, header); coforall (loc, id) in zip(targetLocales, 0..) do on loc { - const locBounds = byteOffsets[id]..byteOffsets[id+1], - locFile = try! open(filePath, ioMode.r), - locByteOffsets = try! findDelimChunks(locFile, delim, nTasks, locBounds, headerPolicy.noHeader); + const locBounds = byteOffsets[id]..byteOffsets[id+1]; - coforall tid in 0.. 1 { + const locFile = try! open(filePath, ioMode.r), + locByteOffsets = try! findDelimChunks(locFile, delim, nTasks, locBounds, headerPolicy.noHeader); - var line: lineType; - while (try! r.readLine(line, stripNewline=true)) do - yield line; + coforall tid in 0.. 1 { + const locFile = try! open(filePath, ioMode.r), + locByteOffsets = try! findDelimChunks(locFile, delim, nTasks, locBounds, headerPolicy.noHeader); + + coforall tid in 0..0 then dataParTasksPerLocale else here.maxTaskPar; try { @@ -7158,36 +7158,38 @@ iter fileReader.lines( const f = chpl_fileFromReaderOrWriter(this), myStart = qio_channel_start_offset_unlocked(this._channel_internal), myEnd = qio_channel_end_offset_unlocked(this._channel_internal), - myBounds = myStart..0 then dataParTasksPerLocale - else here.maxTaskPar; - - try { - // try to break this locale's chunk into 'nTasks' chunks and read the lines in parallel - const locByteOffsets = findFileChunks(locFile, nTasks, locBounds); - coforall tid in 0.. 0 { - var line: t; - while (try! r.readLine(line, stripNewline=stripNewline)) do - yield line; + const locBounds = byteOffsets[lid]..byteOffsets[lid+1]; + + // if byteOffsets looks like [0, 10, 10, 14, 21], then don't try to read 10..10 (locale 1) + if locBounds.size > 1 { + const locFile = try! open(fpath, ioMode.r), + nTasks = if dataParTasksPerLocale>0 then dataParTasksPerLocale + else here.maxTaskPar; + try { + // try to break this locale's chunk into 'nTasks' chunks and read the lines in parallel + const locByteOffsets = findFileChunks(locFile, nTasks, locBounds); + coforall tid in 0.. 0 { + var line: t; + while (try! r.readLine(line, stripNewline=stripNewline)) do + yield line; + } } + } catch { + // fall back to serial iteration for this locale if 'findFileChunks' fails + for line in locFile.reader(region=locBounds)._lines_serial(stripNewline, t) do yield line; } - } catch { - // fall back to serial iteration for this locale if 'findFileChunks' fails - for line in locFile.reader(region=locBounds)._lines_serial(stripNewline, t) do yield line; } } } catch { diff --git a/test/library/standard/IO/linesIterator/lines.chpl b/test/library/standard/IO/linesIterator/lines.chpl index 7daed6cfe0bc..58227cecc3ae 100644 --- a/test/library/standard/IO/linesIterator/lines.chpl +++ b/test/library/standard/IO/linesIterator/lines.chpl @@ -1,7 +1,7 @@ use IO, FileSystem; config const fileName = "lines.txt", - n = 10000; + n = 9999; const nsum = (n * (n + 1)) / 2; @@ -74,4 +74,32 @@ assert(dataFileSum(18.., false, bytes, [Locales.first,]) == nsum - 45); assert(dataFileSum(18.., false, bytes, [Locales.last,]) == nsum - 45); assert(dataFileSum(18.., false, bytes, none) == nsum - 45); + +// skipping last 5 lines +// digit-len + \n + ('region' not inclusive) +const nBytes = 5 * ((n:string).size + 1) + 1, + fileLen = try! open(fileName, mode=ioMode.r).size, + toRead = fileLen - nBytes, + nsum2 = ((n - 5) * ((n - 5) + 1)) / 2; + +assert(dataFileSum(..toRead, true, string, Locales) == nsum2); +assert(dataFileSum(..toRead, true, string, [Locales.first,]) == nsum2); +assert(dataFileSum(..toRead, true, string, [Locales.last,]) == nsum2); +assert(dataFileSum(..toRead, true, string, none) == nsum2); + +assert(dataFileSum(..toRead, false, string, Locales) == nsum2); +assert(dataFileSum(..toRead, false, string, [Locales.first,]) == nsum2); +assert(dataFileSum(..toRead, false, string, [Locales.last,]) == nsum2); +assert(dataFileSum(..toRead, false, string, none) == nsum2); + +assert(dataFileSum(..toRead, true, bytes, Locales) == nsum2); +assert(dataFileSum(..toRead, true, bytes, [Locales.first,]) == nsum2); +assert(dataFileSum(..toRead, true, bytes, [Locales.last,]) == nsum2); +assert(dataFileSum(..toRead, true, bytes, none) == nsum2); + +assert(dataFileSum(..toRead, false, bytes, Locales) == nsum2); +assert(dataFileSum(..toRead, false, bytes, [Locales.first,]) == nsum2); +assert(dataFileSum(..toRead, false, bytes, [Locales.last,]) == nsum2); +assert(dataFileSum(..toRead, false, bytes, none) == nsum2); + remove(fileName); diff --git a/test/library/standard/IO/linesIterator/lines.execopts b/test/library/standard/IO/linesIterator/lines.execopts index d11c4f499a08..a0f5f5d53e3b 100644 --- a/test/library/standard/IO/linesIterator/lines.execopts +++ b/test/library/standard/IO/linesIterator/lines.execopts @@ -1,3 +1,3 @@ --n=15 --n=85 ---n=10000 +--n=9999 diff --git a/test/library/standard/IO/linesIterator/small.chpl b/test/library/standard/IO/linesIterator/small.chpl new file mode 100644 index 000000000000..1e6293f56134 --- /dev/null +++ b/test/library/standard/IO/linesIterator/small.chpl @@ -0,0 +1,4 @@ +use IO; + +forall line in openReader("small.in").lines(targetLocales=Locales) do + write("Locale ", here.id, " read: ", line); diff --git a/test/library/standard/IO/linesIterator/small.comm-none.good b/test/library/standard/IO/linesIterator/small.comm-none.good new file mode 100644 index 000000000000..acc2c5900d0a --- /dev/null +++ b/test/library/standard/IO/linesIterator/small.comm-none.good @@ -0,0 +1,4 @@ +Locale 0 read: 1 +Locale 0 read: 2 +Locale 0 read: 3 +Locale 0 read: 4 diff --git a/test/library/standard/IO/linesIterator/small.good b/test/library/standard/IO/linesIterator/small.good new file mode 100644 index 000000000000..a807fc7b9e08 --- /dev/null +++ b/test/library/standard/IO/linesIterator/small.good @@ -0,0 +1,4 @@ +Locale 0 read: 1 +Locale 0 read: 2 +Locale 1 read: 3 +Locale 2 read: 4 diff --git a/test/library/standard/IO/linesIterator/small.in b/test/library/standard/IO/linesIterator/small.in new file mode 100644 index 000000000000..94ebaf900161 --- /dev/null +++ b/test/library/standard/IO/linesIterator/small.in @@ -0,0 +1,4 @@ +1 +2 +3 +4 diff --git a/test/library/standard/IO/linesIterator/small.numlocales b/test/library/standard/IO/linesIterator/small.numlocales new file mode 100644 index 000000000000..b8626c4cff28 --- /dev/null +++ b/test/library/standard/IO/linesIterator/small.numlocales @@ -0,0 +1 @@ +4 diff --git a/test/library/standard/IO/linesIterator/small.prediff b/test/library/standard/IO/linesIterator/small.prediff new file mode 120000 index 000000000000..6713f3252f98 --- /dev/null +++ b/test/library/standard/IO/linesIterator/small.prediff @@ -0,0 +1 @@ +../../../../../util/test/sort-lines \ No newline at end of file