diff --git a/hbase-diagnostics/src/main/java/org/apache/hadoop/hbase/PerformanceEvaluation.java b/hbase-diagnostics/src/main/java/org/apache/hadoop/hbase/PerformanceEvaluation.java index 1b397546aca6..8504d33e43ee 100644 --- a/hbase-diagnostics/src/main/java/org/apache/hadoop/hbase/PerformanceEvaluation.java +++ b/hbase-diagnostics/src/main/java/org/apache/hadoop/hbase/PerformanceEvaluation.java @@ -466,9 +466,9 @@ protected static byte[][] getSplits(TestOptions opts) { int numSplitPoints = opts.presplitRegions - 1; byte[][] splits = new byte[numSplitPoints][]; - int jump = opts.totalRows / opts.presplitRegions; + long jump = opts.totalRows / opts.presplitRegions; for (int i = 0; i < numSplitPoints; i++) { - int rowkey = jump * (1 + i); + long rowkey = jump * (1 + i); splits[i] = format(rowkey); } return splits; @@ -645,7 +645,7 @@ static Path writeInputFile(final Configuration c, final TestOptions opts, final // Make input random. Map m = new TreeMap<>(); Hash h = MurmurHash.getInstance(); - int perClientRows = (opts.totalRows / opts.numClientThreads); + long perClientRows = (opts.totalRows / opts.numClientThreads); try { for (int j = 0; j < opts.numClientThreads; j++) { TestOptions next = new TestOptions(opts); @@ -704,11 +704,11 @@ static class TestOptions { String cmdName = null; boolean nomapred = false; boolean filterAll = false; - int startRow = 0; + long startRow = 0; float size = 1.0f; - int perClientRunRows = DEFAULT_ROWS_PER_GB; + long perClientRunRows = DEFAULT_ROWS_PER_GB; int numClientThreads = 1; - int totalRows = DEFAULT_ROWS_PER_GB; + long totalRows = DEFAULT_ROWS_PER_GB; int measureAfter = 0; float sampleRate = 1.0f; /** @@ -740,7 +740,7 @@ static class TestOptions { boolean valueRandom = false; boolean valueZipf = false; int valueSize = DEFAULT_VALUE_LENGTH; - int period = (this.perClientRunRows / 10) == 0 ? perClientRunRows : perClientRunRows / 10; + long period = (this.perClientRunRows / 10) == 0 ? perClientRunRows : perClientRunRows / 10; int cycles = 1; int columns = 1; int families = 1; @@ -897,7 +897,7 @@ public void setFilterAll(boolean filterAll) { this.filterAll = filterAll; } - public void setStartRow(int startRow) { + public void setStartRow(long startRow) { this.startRow = startRow; } @@ -913,7 +913,7 @@ public void setNumClientThreads(int numClientThreads) { this.numClientThreads = numClientThreads; } - public void setTotalRows(int totalRows) { + public void setTotalRows(long totalRows) { this.totalRows = totalRows; } @@ -1025,7 +1025,7 @@ public boolean isFilterAll() { return filterAll; } - public int getStartRow() { + public long getStartRow() { return startRow; } @@ -1033,7 +1033,7 @@ public float getSize() { return size; } - public int getPerClientRunRows() { + public long getPerClientRunRows() { return perClientRunRows; } @@ -1041,7 +1041,7 @@ public int getNumClientThreads() { return numClientThreads; } - public int getTotalRows() { + public long getTotalRows() { return totalRows; } @@ -1117,7 +1117,7 @@ public int getValueSize() { return valueSize; } - public int getPeriod() { + public long getPeriod() { return period; } @@ -1174,7 +1174,7 @@ private static long nextRandomSeed() { return randomSeed.nextLong(); } - private final int everyN; + private final long everyN; protected final Random rand = new Random(nextRandomSeed()); protected final Configuration conf; @@ -1205,7 +1205,7 @@ private static long nextRandomSeed() { this.opts = options; this.status = status; this.testName = this.getClass().getSimpleName(); - everyN = (int) (opts.totalRows / (opts.totalRows * opts.sampleRate)); + everyN = (long) (opts.totalRows / (opts.totalRows * opts.sampleRate)); if (options.isValueZipf()) { this.zipf = new RandomDistribution.Zipf(this.rand, 1, options.getValueSize(), 1.2); } @@ -1286,7 +1286,7 @@ void updateScanMetrics(final ScanMetrics metrics) { } } - String generateStatus(final int sr, final int i, final int lr) { + String generateStatus(final long sr, final long i, final long lr) { return "row [start=" + sr + ", current=" + i + ", last=" + lr + "], latency [" + getShortLatencyReport() + "]" + (!isRandomValueSize() ? "" : ", value size [" + getShortValueSizeReport() + "]"); @@ -1296,7 +1296,7 @@ boolean isRandomValueSize() { return opts.valueRandom; } - protected int getReportingPeriod() { + protected long getReportingPeriod() { return opts.period; } @@ -1399,11 +1399,11 @@ long test() throws IOException, InterruptedException { return (System.nanoTime() - startTime) / 1000000; } - int getStartRow() { + long getStartRow() { return opts.startRow; } - int getLastRow() { + long getLastRow() { return getStartRow() + opts.perClientRunRows; } @@ -1411,12 +1411,12 @@ int getLastRow() { * Provides an extension point for tests that don't want a per row invocation. */ void testTimed() throws IOException, InterruptedException { - int startRow = getStartRow(); - int lastRow = getLastRow(); + long startRow = getStartRow(); + long lastRow = getLastRow(); // Report on completion of 1/10th of total. for (int ii = 0; ii < opts.cycles; ii++) { if (opts.cycles > 1) LOG.info("Cycle=" + ii + " of " + opts.cycles); - for (int i = startRow; i < lastRow; i++) { + for (long i = startRow; i < lastRow; i++) { if (i % everyN != 0) continue; long startTime = System.nanoTime(); boolean requestSent = false; @@ -1462,7 +1462,7 @@ public String getShortValueSizeReport() { * @return true if the row was sent to server and need to record metrics. False if not, multiGet * and multiPut e.g., the rows are sent to server only if enough gets/puts are gathered. */ - abstract boolean testRow(final int i, final long startTime) + abstract boolean testRow(final long i, final long startTime) throws IOException, InterruptedException; } @@ -1510,7 +1510,7 @@ static abstract class MetaTest extends TableTest { MetaTest(Connection con, TestOptions options, Status status) { super(con, options, status); - keyLength = Integer.toString(opts.perClientRunRows).length(); + keyLength = Long.toString(opts.perClientRunRows).length(); } @Override @@ -1521,7 +1521,7 @@ void onTakedown() throws IOException { /* * Generates Lexicographically ascending strings */ - protected byte[] getSplitKey(final int i) { + protected byte[] getSplitKey(final long i) { return Bytes.toBytes(String.format("%0" + keyLength + "d", i)); } @@ -1558,7 +1558,7 @@ static class AsyncRandomReadTest extends AsyncTableTest { } @Override - boolean testRow(final int i, final long startTime) throws IOException, InterruptedException { + boolean testRow(final long i, final long startTime) throws IOException, InterruptedException { if (opts.randomSleep > 0) { Thread.sleep(ThreadLocalRandom.current().nextInt(opts.randomSleep)); } @@ -1615,8 +1615,8 @@ public static V propagate(Callable callable) { } @Override - protected int getReportingPeriod() { - int period = opts.perClientRunRows / 10; + protected long getReportingPeriod() { + long period = opts.perClientRunRows / 10; return period == 0 ? opts.perClientRunRows : period; } @@ -1637,7 +1637,7 @@ static class AsyncRandomWriteTest extends AsyncSequentialWriteTest { } @Override - protected byte[] generateRow(final int i) { + protected byte[] generateRow(final long i) { return getRandomRow(this.rand, opts.totalRows); } } @@ -1666,7 +1666,7 @@ void testTakedown() throws IOException { } @Override - boolean testRow(final int i, final long startTime) throws IOException { + boolean testRow(final long i, final long startTime) throws IOException { if (this.testScanner == null) { Scan scan = new Scan().withStartRow(format(opts.startRow)).setCaching(opts.caching) .setCacheBlocks(opts.cacheBlocks).setAsyncPrefetch(opts.asyncPrefetch) @@ -1699,7 +1699,7 @@ static class AsyncSequentialReadTest extends AsyncTableTest { } @Override - boolean testRow(final int i, final long startTime) throws IOException, InterruptedException { + boolean testRow(final long i, final long startTime) throws IOException, InterruptedException { Get get = new Get(format(i)); for (int family = 0; family < opts.families; family++) { byte[] familyName = Bytes.toBytes(FAMILY_NAME_BASE + family); @@ -1735,13 +1735,13 @@ static class AsyncSequentialWriteTest extends AsyncTableTest { } } - protected byte[] generateRow(final int i) { + protected byte[] generateRow(final long i) { return format(i); } @Override @SuppressWarnings("ReturnValueIgnored") - boolean testRow(final int i, final long startTime) throws IOException, InterruptedException { + boolean testRow(final long i, final long startTime) throws IOException, InterruptedException { byte[] row = generateRow(i); Put put = new Put(row); for (int family = 0; family < opts.families; family++) { @@ -1816,7 +1816,7 @@ static class RandomSeekScanTest extends TableTest { } @Override - boolean testRow(final int i, final long startTime) throws IOException { + boolean testRow(final long i, final long startTime) throws IOException { Scan scan = new Scan().withStartRow(getRandomRow(this.rand, opts.totalRows)).setCaching(opts.caching) .setCacheBlocks(opts.cacheBlocks).setAsyncPrefetch(opts.asyncPrefetch) @@ -1851,8 +1851,8 @@ boolean testRow(final int i, final long startTime) throws IOException { } @Override - protected int getReportingPeriod() { - int period = opts.perClientRunRows / 100; + protected long getReportingPeriod() { + long period = opts.perClientRunRows / 100; return period == 0 ? opts.perClientRunRows : period; } @@ -1864,7 +1864,7 @@ static abstract class RandomScanWithRangeTest extends TableTest { } @Override - boolean testRow(final int i, final long startTime) throws IOException { + boolean testRow(final long i, final long startTime) throws IOException { Pair startAndStopRow = getStartAndStopRow(); Scan scan = new Scan().withStartRow(startAndStopRow.getFirst()) .withStopRow(startAndStopRow.getSecond()).setCaching(opts.caching) @@ -1906,15 +1906,15 @@ boolean testRow(final int i, final long startTime) throws IOException { protected abstract Pair getStartAndStopRow(); - protected Pair generateStartAndStopRows(int maxRange) { - int start = this.rand.nextInt(Integer.MAX_VALUE) % opts.totalRows; - int stop = start + maxRange; + protected Pair generateStartAndStopRows(long maxRange) { + long start = this.rand.nextLong(Long.MAX_VALUE) % opts.totalRows; + long stop = start + maxRange; return new Pair<>(format(start), format(stop)); } @Override - protected int getReportingPeriod() { - int period = opts.perClientRunRows / 100; + protected long getReportingPeriod() { + long period = opts.perClientRunRows / 100; return period == 0 ? opts.perClientRunRows : period; } } @@ -1977,7 +1977,7 @@ static class RandomReadTest extends TableTest { } @Override - boolean testRow(final int i, final long startTime) throws IOException, InterruptedException { + boolean testRow(final long i, final long startTime) throws IOException, InterruptedException { if (opts.randomSleep > 0) { Thread.sleep(ThreadLocalRandom.current().nextInt(opts.randomSleep)); } @@ -2025,8 +2025,8 @@ boolean testRow(final int i, final long startTime) throws IOException, Interrupt } @Override - protected int getReportingPeriod() { - int period = opts.perClientRunRows / 10; + protected long getReportingPeriod() { + long period = opts.perClientRunRows / 10; return period == 0 ? opts.perClientRunRows : period; } @@ -2058,19 +2058,19 @@ void onStartup() throws IOException { } @Override - boolean testRow(final int i, final long startTime) throws IOException, InterruptedException { + boolean testRow(final long i, final long startTime) throws IOException, InterruptedException { if (opts.randomSleep > 0) { Thread.sleep(rand.nextInt(opts.randomSleep)); } HRegionLocation hRegionLocation = - regionLocator.getRegionLocation(getSplitKey(rand.nextInt(opts.perClientRunRows)), true); + regionLocator.getRegionLocation(getSplitKey(rand.nextLong(opts.perClientRunRows)), true); LOG.debug("get location for region: " + hRegionLocation); return true; } @Override - protected int getReportingPeriod() { - int period = opts.perClientRunRows / 10; + protected long getReportingPeriod() { + long period = opts.perClientRunRows / 10; return period == 0 ? opts.perClientRunRows : period; } @@ -2086,7 +2086,7 @@ static class RandomWriteTest extends SequentialWriteTest { } @Override - protected byte[] generateRow(final int i) { + protected byte[] generateRow(final long i) { return getRandomRow(this.rand, opts.totalRows); } @@ -2098,7 +2098,7 @@ static class RandomDeleteTest extends SequentialDeleteTest { } @Override - protected byte[] generateRow(final int i) { + protected byte[] generateRow(final long i) { return getRandomRow(this.rand, opts.totalRows); } @@ -2120,7 +2120,7 @@ void testTakedown() throws IOException { } @Override - boolean testRow(final int i, final long startTime) throws IOException { + boolean testRow(final long i, final long startTime) throws IOException { if (this.testScanner == null) { Scan scan = new Scan().withStartRow(format(opts.startRow)).setCaching(opts.caching) .setCacheBlocks(opts.cacheBlocks).setAsyncPrefetch(opts.asyncPrefetch) @@ -2163,7 +2163,7 @@ void testTakedown() throws IOException { } @Override - boolean testRow(final int i, final long startTime) throws IOException { + boolean testRow(final long i, final long startTime) throws IOException { if (this.testScanner == null) { Scan scan = new Scan().setCaching(opts.caching).setCacheBlocks(opts.cacheBlocks) .setAsyncPrefetch(opts.asyncPrefetch).setReadType(opts.scanReadType) @@ -2211,12 +2211,12 @@ byte[] getQualifier() { } @Override - int getStartRow() { + long getStartRow() { return 0; } @Override - int getLastRow() { + long getLastRow() { return opts.perClientRunRows; } } @@ -2227,7 +2227,7 @@ static class IncrementTest extends CASTableTest { } @Override - boolean testRow(final int i, final long startTime) throws IOException { + boolean testRow(final long i, final long startTime) throws IOException { Increment increment = new Increment(format(i)); // unlike checkAndXXX tests, which make most sense to do on a single value, // if multiple families are specified for an increment test we assume it is @@ -2247,7 +2247,7 @@ static class AppendTest extends CASTableTest { } @Override - boolean testRow(final int i, final long startTime) throws IOException { + boolean testRow(final long i, final long startTime) throws IOException { byte[] bytes = format(i); Append append = new Append(bytes); // unlike checkAndXXX tests, which make most sense to do on a single value, @@ -2268,7 +2268,7 @@ static class CheckAndMutateTest extends CASTableTest { } @Override - boolean testRow(final int i, final long startTime) throws IOException { + boolean testRow(final long i, final long startTime) throws IOException { final byte[] bytes = format(i); // checkAndXXX tests operate on only a single value // Put a known value so when we go to check it, it is there. @@ -2289,7 +2289,7 @@ static class CheckAndPutTest extends CASTableTest { } @Override - boolean testRow(final int i, final long startTime) throws IOException { + boolean testRow(final long i, final long startTime) throws IOException { final byte[] bytes = format(i); // checkAndXXX tests operate on only a single value // Put a known value so when we go to check it, it is there. @@ -2308,7 +2308,7 @@ static class CheckAndDeleteTest extends CASTableTest { } @Override - boolean testRow(final int i, final long startTime) throws IOException { + boolean testRow(final long i, final long startTime) throws IOException { final byte[] bytes = format(i); // checkAndXXX tests operate on only a single value // Put a known value so when we go to check it, it is there. @@ -2332,7 +2332,7 @@ static class CleanMetaTest extends MetaTest { } @Override - boolean testRow(final int i, final long startTime) throws IOException { + boolean testRow(final long i, final long startTime) throws IOException { try { RegionInfo regionInfo = connection.getRegionLocator(table.getName()) .getRegionLocation(getSplitKey(i), false).getRegion(); @@ -2357,7 +2357,7 @@ static class SequentialReadTest extends TableTest { } @Override - boolean testRow(final int i, final long startTime) throws IOException { + boolean testRow(final long i, final long startTime) throws IOException { Get get = new Get(format(i)); for (int family = 0; family < opts.families; family++) { byte[] familyName = Bytes.toBytes(FAMILY_NAME_BASE + family); @@ -2389,12 +2389,12 @@ static class SequentialWriteTest extends BufferedMutatorTest { } } - protected byte[] generateRow(final int i) { + protected byte[] generateRow(final long i) { return format(i); } @Override - boolean testRow(final int i, final long startTime) throws IOException { + boolean testRow(final long i, final long startTime) throws IOException { byte[] row = generateRow(i); Put put = new Put(row); for (int family = 0; family < opts.families; family++) { @@ -2445,12 +2445,12 @@ static class SequentialDeleteTest extends BufferedMutatorTest { super(con, options, status); } - protected byte[] generateRow(final int i) { + protected byte[] generateRow(final long i) { return format(i); } @Override - boolean testRow(final int i, final long startTime) throws IOException { + boolean testRow(final long i, final long startTime) throws IOException { byte[] row = generateRow(i); Delete delete = new Delete(row); for (int family = 0; family < opts.families; family++) { @@ -2477,7 +2477,7 @@ static class MetaWriteTest extends MetaTest { } @Override - boolean testRow(final int i, final long startTime) throws IOException { + boolean testRow(final long i, final long startTime) throws IOException { List regionInfos = new ArrayList(); RegionInfo regionInfo = (RegionInfoBuilder.newBuilder(TableName.valueOf(TABLE_NAME)) .setStartKey(getSplitKey(i)).setEndKey(getSplitKey(i + 1)).build()); @@ -2504,7 +2504,7 @@ static class FilteredScanTest extends TableTest { } @Override - boolean testRow(int i, final long startTime) throws IOException { + boolean testRow(long i, final long startTime) throws IOException { byte[] value = generateData(this.rand, getValueLength(this.rand)); Scan scan = constructScan(value); ResultScanner scanner = null; @@ -2552,7 +2552,7 @@ protected Scan constructScan(byte[] valuePrefix) throws IOException { * @param timeMs Time taken in milliseconds. * @return String value with label, ie '123.76 MB/s' */ - private static String calculateMbps(int rows, long timeMs, final int valueSize, int families, + private static String calculateMbps(long rows, long timeMs, final int valueSize, int families, int columns) { BigDecimal rowSize = BigDecimal.valueOf(ROW_LENGTH + ((valueSize + (FAMILY_NAME_BASE.length() + 1) + COLUMN_ZERO.length) * columns) * families); @@ -2566,9 +2566,9 @@ private static String calculateMbps(int rows, long timeMs, final int valueSize, * @return Returns zero-prefixed ROW_LENGTH-byte wide decimal version of passed number (Does * absolute in case number is negative). */ - public static byte[] format(final int number) { + public static byte[] format(final long number) { byte[] b = new byte[ROW_LENGTH]; - int d = Math.abs(number); + long d = Math.abs(number); for (int i = b.length - 1; i >= 0; i--) { b[i] = (byte) ((d % 10) + '0'); d /= 10; @@ -2603,12 +2603,12 @@ public static byte[] generateData(final Random r, int length) { return b; } - static byte[] getRandomRow(final Random random, final int totalRows) { + static byte[] getRandomRow(final Random random, final long totalRows) { return format(generateRandomRow(random, totalRows)); } - static int generateRandomRow(final Random random, final int totalRows) { - return random.nextInt(Integer.MAX_VALUE) % totalRows; + static long generateRandomRow(final Random random, final long totalRows) { + return random.nextLong(Long.MAX_VALUE) % totalRows; } static RunResult runOneClient(final Class cmd, Configuration conf, @@ -2643,7 +2643,7 @@ static RunResult runOneClient(final Class cmd, Configuration status.setStatus("Finished " + cmd + " in " + totalElapsedTime + "ms at offset " + opts.startRow + " for " + opts.perClientRunRows + " rows" + " (" - + calculateMbps((int) (opts.perClientRunRows * opts.sampleRate), totalElapsedTime, + + calculateMbps((long) (opts.perClientRunRows * opts.sampleRate), totalElapsedTime, getAverageValueLength(opts), opts.families, opts.columns) + ")"); @@ -2838,7 +2838,7 @@ static TestOptions parseOpts(Queue args) { final String rows = "--rows="; if (cmd.startsWith(rows)) { - opts.perClientRunRows = Integer.parseInt(cmd.substring(rows.length())); + opts.perClientRunRows = Long.parseLong(cmd.substring(rows.length())); continue; } @@ -2862,7 +2862,7 @@ static TestOptions parseOpts(Queue args) { final String startRow = "--startRow="; if (cmd.startsWith(startRow)) { - opts.startRow = Integer.parseInt(cmd.substring(startRow.length())); + opts.startRow = Long.parseLong(cmd.substring(startRow.length())); continue; } @@ -3160,10 +3160,10 @@ static TestOptions calculateRowsAndSize(final TestOptions opts) { && (opts.getCmdName().equals(RANDOM_READ) || opts.getCmdName().equals(RANDOM_SEEK_SCAN))) && opts.size != DEFAULT_OPTS.size && opts.perClientRunRows != DEFAULT_OPTS.perClientRunRows ) { - opts.totalRows = (int) (opts.size * rowsPerGB); + opts.totalRows = (long) (opts.size * rowsPerGB); } else if (opts.size != DEFAULT_OPTS.size) { // total size in GB specified - opts.totalRows = (int) (opts.size * rowsPerGB); + opts.totalRows = (long) (opts.size * rowsPerGB); opts.perClientRunRows = opts.totalRows / opts.numClientThreads; } else { opts.totalRows = opts.perClientRunRows * opts.numClientThreads; diff --git a/hbase-diagnostics/src/test/java/org/apache/hadoop/hbase/TestPerformanceEvaluation.java b/hbase-diagnostics/src/test/java/org/apache/hadoop/hbase/TestPerformanceEvaluation.java index cf11510a897a..7fff3390e60a 100644 --- a/hbase-diagnostics/src/test/java/org/apache/hadoop/hbase/TestPerformanceEvaluation.java +++ b/hbase-diagnostics/src/test/java/org/apache/hadoop/hbase/TestPerformanceEvaluation.java @@ -124,7 +124,7 @@ public void testWriteInputFile() throws IOException { public void testSizeCalculation() { TestOptions opts = new PerformanceEvaluation.TestOptions(); opts = PerformanceEvaluation.calculateRowsAndSize(opts); - int rows = opts.getPerClientRunRows(); + long rows = opts.getPerClientRunRows(); // Default row count final int defaultPerClientRunRows = 1024 * 1024; assertEquals(defaultPerClientRunRows, rows); @@ -146,7 +146,7 @@ public void testSizeCalculation() { public void testRandomReadCalculation() { TestOptions opts = new PerformanceEvaluation.TestOptions(); opts = PerformanceEvaluation.calculateRowsAndSize(opts); - int rows = opts.getPerClientRunRows(); + long rows = opts.getPerClientRunRows(); // Default row count final int defaultPerClientRunRows = 1024 * 1024; assertEquals(defaultPerClientRunRows, rows); @@ -164,7 +164,7 @@ public void testRandomReadCalculation() { boolean foundValue = false; Random rand = ThreadLocalRandom.current(); for (int i = 0; i < 10000000; i++) { - int randomRow = PerformanceEvaluation.generateRandomRow(rand, opts.totalRows); + long randomRow = PerformanceEvaluation.generateRandomRow(rand, opts.totalRows); if (randomRow > 1000) { foundValue = true; break; @@ -402,7 +402,7 @@ void onTakedown() throws IOException { } @Override - boolean testRow(int i, long startTime) throws IOException, InterruptedException { + boolean testRow(long i, long startTime) throws IOException, InterruptedException { return false; } }