From 3e1411f3f8486e4e803b7712549501f94ffea7fc Mon Sep 17 00:00:00 2001 From: Jeff Nelson Date: Tue, 14 May 2024 10:53:57 -0400 Subject: [PATCH 1/4] Change default value of max_search_substring_length configuration option to 40 (#517) * Revert "Revert "update changelog"" This reverts commit 504d7bab6a50150a03f33034e669d96a8bbcf226. * Revert "Revert "change default value of max_search_substring_length"" This reverts commit f0a6664beae558a13c1d29c2be6bd15da6498b45. --- CHANGELOG.md | 1 + concourse-server/conf/concourse.yaml | 4 +--- .../main/java/com/cinchapi/concourse/server/GlobalState.java | 2 +- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 14759e02d..590507f8b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ We made several changes to improve search performance and accuracy: * **Previous Configuration**: In earlier versions, Concourse Server could be configured using the `conf/stopwords.txt` file to exclude common stopwords from indexing and search operations. This approach was designed to reduce storage requirements and improve search performance by removing frequently occurring, but generally less significant words. * **Rationale for Change**: Preserving stopwords is crucial for maintaining context, which can significantly enhance the accuracy of search results and the effectiveness of ranking algorithms. Since affordable storage and computational resources are more abundant, resource usage is no longer a concern and it makes more sense to prioritze better search accuracy and system robustness. Lastly, preserving stopwords eliminates corner case bugs that are inherent to the way Concourse's search algorithm interacts with the buffered storage system. * **Upgrade Implications**: Upon upgrading to this version, an automatic reindexing task will be initiated to ensure that all previously indexed data conforms to the new no-stopword-removal policy. It's important to allocate downtime for this reindexing to occur. And, it is wise to anticipate more storage spaced being used due to stopwords being included in the search corpus. +* Changed the default value of the `max_search_substring_length` configuration option to `40`. The previous default allowed unlimited substring lengths, which increased search index size and hurt performance. Existing explicit configurations for this option remain unchanged. ##### Locking Optimizations We made several changes to improve the safety, scalability and operational efficiency of the Just-in-Time (JIT) locking protocol: diff --git a/concourse-server/conf/concourse.yaml b/concourse-server/conf/concourse.yaml index 3234feb39..aa611ccc9 100644 --- a/concourse-server/conf/concourse.yaml +++ b/concourse-server/conf/concourse.yaml @@ -175,9 +175,7 @@ log_level: # length of the longest possible word in the search language. For example, the longest # possible word in English is about 40 characters long. # -# By default, search substrings are not limited. -# -# DEFAULT: no limit +# DEFAULT: 40 max_search_substring_length: # The listener port (1-65535) for shutdown commands. Choose a port between diff --git a/concourse-server/src/main/java/com/cinchapi/concourse/server/GlobalState.java b/concourse-server/src/main/java/com/cinchapi/concourse/server/GlobalState.java index 24c0734b6..a74062bbc 100644 --- a/concourse-server/src/main/java/com/cinchapi/concourse/server/GlobalState.java +++ b/concourse-server/src/main/java/com/cinchapi/concourse/server/GlobalState.java @@ -276,7 +276,7 @@ public final class GlobalState extends Constants { * is about 40 characters long. *

*/ - public static int MAX_SEARCH_SUBSTRING_LENGTH = -1; + public static int MAX_SEARCH_SUBSTRING_LENGTH = 40; /** * The password that is assigned to the root administrator account when From 5600bc32a4a91f54703a729ed1a1592853db5d9e Mon Sep 17 00:00:00 2001 From: Jeff Nelson Date: Sun, 19 May 2024 09:18:22 -0400 Subject: [PATCH 2/4] pass default value correctly when getting init_root_username and init_root_password --- .../main/java/com/cinchapi/concourse/server/GlobalState.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/concourse-server/src/main/java/com/cinchapi/concourse/server/GlobalState.java b/concourse-server/src/main/java/com/cinchapi/concourse/server/GlobalState.java index a74062bbc..42dd348e7 100644 --- a/concourse-server/src/main/java/com/cinchapi/concourse/server/GlobalState.java +++ b/concourse-server/src/main/java/com/cinchapi/concourse/server/GlobalState.java @@ -415,10 +415,10 @@ public final class GlobalState extends Constants { ENABLE_VERIFY_BY_LOOKUP); INIT_ROOT_PASSWORD = config.getOrDefault("init.root.password", - config.getOrDefault("init_root_password", "admin")); + config.getOrDefault("init_root_password", INIT_ROOT_PASSWORD)); INIT_ROOT_USERNAME = config.getOrDefault("init.root.username", - config.getOrDefault("init_root_username", "admin")); + config.getOrDefault("init_root_username", INIT_ROOT_USERNAME)); // =================== PREF READING BLOCK ==================== } From ea07fa3fef7c7eeec637c826252714de1d3ec1f7 Mon Sep 17 00:00:00 2001 From: Jeff Nelson Date: Sat, 29 Jun 2024 10:10:14 +0100 Subject: [PATCH 3/4] GH-509: Create connection pool by copying existing connection (#521) * Rely on Supplier constructor for all ConnectionPool factory methods Remove references and usage of deprecated constructors * Add factory methods to create connection pools that copy and existing connection * update changelog * doc update * update changelog --- CHANGELOG.md | 3 + .../concourse/CachedConnectionPool.java | 27 +- .../cinchapi/concourse/ConnectionPool.java | 264 ++++++++++++------ .../concourse/FixedConnectionPool.java | 27 +- .../concourse/CachedConnectionPoolTest.java | 5 + .../concourse/ConnectionPoolTest.java | 36 +++ .../concourse/CustomConnectionPoolTest.java | 15 + .../concourse/FixedConnectionPoolTest.java | 5 + 8 files changed, 247 insertions(+), 135 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 590507f8b..4dd5d2f22 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -37,6 +37,9 @@ We made several changes to improve the safety, scalability and operational effic * `ConcourseArtifacts` - Provides factory methods to retrieve local copies of Concourse artifacts for any version. Can be used to download the installer for a released version. * `ManagedConcourseServer` - Provdes the ability to control an external Concourse Server process within another application. +##### New Functionality +* Added the ability to create `ConnectionPool`s that copy the credentials and connection information from an existing handler These copying connection pools can be created by using the respective "cached" or "fixed" factory methods in the `ConnectionPool` class that take a `Concourse` parameter. + ##### Bug Fixes * [GH-454](https://github.com/cinchapi/concourse/issues/454): Fixed an issue that caused JVM startup options overriden in a ".dev" configuration file to be ignored (e.g., `heap_size`). * [GH-491](https://github.com/cinchapi/concourse/issues/491) Fixed a race condition that made it possible for a range bloked operation to spurriously be allowed to proceed if it was waiting to acquire a range lock whose intended scope of protection intersected the scope of a range lock that was concurrently released. diff --git a/concourse-driver-java/src/main/java/com/cinchapi/concourse/CachedConnectionPool.java b/concourse-driver-java/src/main/java/com/cinchapi/concourse/CachedConnectionPool.java index a66e17045..7ef83df83 100644 --- a/concourse-driver-java/src/main/java/com/cinchapi/concourse/CachedConnectionPool.java +++ b/concourse-driver-java/src/main/java/com/cinchapi/concourse/CachedConnectionPool.java @@ -16,6 +16,7 @@ package com.cinchapi.concourse; import java.util.Queue; +import java.util.function.Supplier; import com.cinchapi.concourse.util.ConcurrentLoadingQueue; @@ -31,31 +32,11 @@ class CachedConnectionPool extends ConnectionPool { /** * Construct a new instance. * - * @param host - * @param port - * @param username - * @param password + * @param supplier * @param poolSize */ - protected CachedConnectionPool(String host, int port, String username, - String password, int poolSize) { - this(host, port, username, password, "", poolSize); - } - - /** - * Construct a new instance. - * - * @param host - * @param port - * @param username - * @param password - * @param environment - * @param poolSize - */ - protected CachedConnectionPool(String host, int port, String username, - String password, String environment, int poolSize) { - super(() -> Concourse.connect(host, port, username, password, - environment), poolSize); + protected CachedConnectionPool(Supplier supplier, int poolSize) { + super(supplier, poolSize); } @Override diff --git a/concourse-driver-java/src/main/java/com/cinchapi/concourse/ConnectionPool.java b/concourse-driver-java/src/main/java/com/cinchapi/concourse/ConnectionPool.java index 2bb118ad6..6d5e73508 100644 --- a/concourse-driver-java/src/main/java/com/cinchapi/concourse/ConnectionPool.java +++ b/concourse-driver-java/src/main/java/com/cinchapi/concourse/ConnectionPool.java @@ -60,18 +60,6 @@ public abstract class ConnectionPool implements AutoCloseable { // NOTE: This class does not define #hashCode or #equals because the // defaults are the desired behaviour - /** - * The default connection pool size. - */ - protected static final int DEFAULT_POOL_SIZE = 10; - - /** - * The default configuration files to use if none are specified. - */ - private static final Path[] DEFAULT_CONFIG_FILES = new Path[] { - Paths.get("concourse_client.prefs"), - Paths.get("concourse_client.yaml") }; - /** * Return a {@link ConnectionPool} that has no limit on the number of * connections it can manage to the Concourse instance described in the @@ -87,17 +75,25 @@ public static ConnectionPool newCachedConnectionPool() { } /** - * Return a {@link ConnectionPool} that has no limit on the number of - * connections it can manage to the Concourse instance described in the - * {@code config} on behalf of the user identified in the {@code config}, - * but will try to use previously created connections before establishing - * new ones for any request. - * - * @param config - * @return the ConnectionPool - */ - public static ConnectionPool newCachedConnectionPool(String config) { - return newCachedConnectionPool(Paths.get(config)); + * Returns a {@link ConnectionPool} populated with handlers that + * {@link Concourse#copyExistingConnection(Concourse) copy} the connection + * information of the provided {@code concourse} instance. The pool has no + * limit on the number of connections it can manage, but will attempt to use + * previously created connections before establishing new ones on request. + * + *

+ * NOTE: The provided {@code concourse} connection will not + * be a member of the returned {@link ConnectionPool} and its status will + * not affect the status of any connections managed by the pool. + *

+ * + * @param concourse the {@link Concourse} connection to copy when populating + * the {@link ConnectionPool} + * @return the populated {@link ConnectionPool} + */ + public static ConnectionPool newCachedConnectionPool(Concourse concourse) { + Supplier supplier = () -> concourse.copyConnection(); + return new CachedConnectionPool(supplier, DEFAULT_POOL_SIZE); } /** @@ -113,9 +109,24 @@ public static ConnectionPool newCachedConnectionPool(String config) { public static ConnectionPool newCachedConnectionPool(Path... configFiles) { ConcourseClientConfiguration config = ConcourseClientConfiguration .from(configFiles); - return new CachedConnectionPool(config.getHost(), config.getPort(), - config.getUsername(), new String(config.getPassword()), - config.getEnvironment(), DEFAULT_POOL_SIZE); + Supplier supplier = getConcourseSupplier(config.getHost(), + config.getPort(), config.getUsername(), + new String(config.getPassword()), config.getEnvironment()); + return new CachedConnectionPool(supplier, DEFAULT_POOL_SIZE); + } + + /** + * Return a {@link ConnectionPool} that has no limit on the number of + * connections it can manage to the Concourse instance described in the + * {@code config} on behalf of the user identified in the {@code config}, + * but will try to use previously created connections before establishing + * new ones for any request. + * + * @param config + * @return the ConnectionPool + */ + public static ConnectionPool newCachedConnectionPool(String config) { + return newCachedConnectionPool(Paths.get(config)); } /** @@ -133,8 +144,9 @@ public static ConnectionPool newCachedConnectionPool(Path... configFiles) { */ public static ConnectionPool newCachedConnectionPool(String host, int port, String username, String password) { - return new CachedConnectionPool(host, port, username, password, - DEFAULT_POOL_SIZE); + Supplier supplier = getConcourseSupplier(host, port, + username, password, ""); + return new CachedConnectionPool(supplier, DEFAULT_POOL_SIZE); } /** @@ -153,8 +165,9 @@ public static ConnectionPool newCachedConnectionPool(String host, int port, */ public static ConnectionPool newCachedConnectionPool(String host, int port, String username, String password, String environment) { - return new CachedConnectionPool(host, port, username, password, - environment, DEFAULT_POOL_SIZE); + Supplier supplier = getConcourseSupplier(host, port, + username, password, environment); + return new CachedConnectionPool(supplier, DEFAULT_POOL_SIZE); } /** @@ -230,6 +243,30 @@ public static ConnectionPool newConnectionPool(String host, int port, return newFixedConnectionPool(host, port, username, password, poolSize); } + /** + * Returns a {@link ConnectionPool} populated with handlers that + * {@link Concourse#copyExistingConnection(Concourse) copy} the connection + * information of the provided {@code concourse} instance. The pool will + * contain {@code poolSize} connections. If all connections are active, + * subsequent requests will block until a connection is returned. + * + *

+ * NOTE: The provided {@code concourse} connection will not + * be a member of the returned {@link ConnectionPool} and its status will + * not affect the status of any connections managed by the pool. + *

+ * + * @param concourse the {@link Concourse} connection to copy when populating + * the {@link ConnectionPool} + * @param poolSize the number of connections in the pool + * @return the populated {@link ConnectionPool} + */ + public static ConnectionPool newFixedConnectionPool(Concourse concourse, + int poolSize) { + Supplier supplier = () -> concourse.copyConnection(); + return new FixedConnectionPool(supplier, poolSize); + } + /** * Return a new {@link ConnectionPool} with a fixed number of connections to * the Concourse instance defined in the {@code concourse_client.prefs} file @@ -256,13 +293,18 @@ public static ConnectionPool newFixedConnectionPool(int poolSize) { * attempts will block until a connection is returned. *

* - * @param config * @param poolSize + * @param config * @return the ConnectionPool */ - public static ConnectionPool newFixedConnectionPool(String config, - int poolSize) { - return newFixedConnectionPool(poolSize, Paths.get(config)); + public static ConnectionPool newFixedConnectionPool(int poolSize, + Path... configFiles) { + ConcourseClientConfiguration config = ConcourseClientConfiguration + .from(configFiles); + Supplier supplier = getConcourseSupplier(config.getHost(), + config.getPort(), config.getUsername(), + new String(config.getPassword()), config.getEnvironment()); + return new FixedConnectionPool(supplier, poolSize); } /** @@ -274,17 +316,13 @@ public static ConnectionPool newFixedConnectionPool(String config, * attempts will block until a connection is returned. *

* - * @param poolSize * @param config + * @param poolSize * @return the ConnectionPool */ - public static ConnectionPool newFixedConnectionPool(int poolSize, - Path... configFiles) { - ConcourseClientConfiguration config = ConcourseClientConfiguration - .from(configFiles); - return new FixedConnectionPool(config.getHost(), config.getPort(), - config.getUsername(), new String(config.getPassword()), - config.getEnvironment(), poolSize); + public static ConnectionPool newFixedConnectionPool(String config, + int poolSize) { + return newFixedConnectionPool(poolSize, Paths.get(config)); } /** @@ -306,8 +344,9 @@ public static ConnectionPool newFixedConnectionPool(int poolSize, */ public static ConnectionPool newFixedConnectionPool(String host, int port, String username, String password, int poolSize) { - return new FixedConnectionPool(host, port, username, password, - poolSize); + Supplier supplier = getConcourseSupplier(host, port, + username, password, ""); + return new FixedConnectionPool(supplier, poolSize); } /** @@ -331,10 +370,40 @@ public static ConnectionPool newFixedConnectionPool(String host, int port, public static ConnectionPool newFixedConnectionPool(String host, int port, String username, String password, String environment, int poolSize) { - return new FixedConnectionPool(host, port, username, password, - environment, poolSize); + Supplier supplier = getConcourseSupplier(host, port, + username, password, environment); + return new FixedConnectionPool(supplier, poolSize); } + /** + * Return a {@link Supplier} that generates a new {@link Concourse} + * connection from the provided credentials and connection information. + * + * @param host + * @param port + * @param username + * @param password + * @param environment + * @return the {@link Supplier} + */ + private static Supplier getConcourseSupplier(String host, + int port, String username, String password, String environment) { + return () -> Concourse.connect(host, port, username, password, + environment); + } + + /** + * The default connection pool size. + */ + protected static final int DEFAULT_POOL_SIZE = 10; + + /** + * The default configuration files to use if none are specified. + */ + private static final Path[] DEFAULT_CONFIG_FILES = new Path[] { + Paths.get("concourse_client.prefs"), + Paths.get("concourse_client.yaml") }; + /** * A FIFO queue of connections that are available to be leased. */ @@ -356,26 +425,25 @@ public static ConnectionPool newFixedConnectionPool(String host, int port, protected final Supplier supplier; /** - * Construct a new instance. + * Construct a new instance that provides {@link Concourse} connections that + * copy the connection information from the provided {@code concourse} + * handler. + *

+ * NOTE:This constructor is provided for subclasses to + * conveniently implement connection copying while abstracting away the + * details of how to construct an appropriate {@link Supplier}. + *

* - * @param supplier + * @param concourse * @param poolSize */ - protected ConnectionPool(Supplier supplier, int poolSize) { - this.supplier = supplier; - this.available = buildQueue(poolSize); - this.leased = Sets.newConcurrentHashSet(); - for (int i = 0; i < poolSize; ++i) { - available.offer(supplier.get()); - } - // Ensure that the client connections are forced closed when the JVM is - // shutdown in case the user does not properly close the pool - Runtime.getRuntime().addShutdownHook(new Thread(() -> forceClose())); + protected ConnectionPool(Concourse concourse, int poolSize) { + this(() -> concourse.copyConnection(), poolSize); } /** * Construct a new instance. - * + * * @param host * @param port * @param username @@ -391,7 +459,7 @@ protected ConnectionPool(String host, int port, String username, /** * Construct a new instance. - * + * * @param host * @param port * @param username @@ -406,6 +474,24 @@ protected ConnectionPool(String host, int port, String username, environment), poolSize); } + /** + * Construct a new instance. + * + * @param supplier + * @param poolSize + */ + protected ConnectionPool(Supplier supplier, int poolSize) { + this.supplier = supplier; + this.available = buildQueue(poolSize); + this.leased = Sets.newConcurrentHashSet(); + for (int i = 0; i < poolSize; ++i) { + available.offer(supplier.get()); + } + // Ensure that the client connections are forced closed when the JVM is + // shutdown in case the user does not properly close the pool + Runtime.getRuntime().addShutdownHook(new Thread(() -> forceClose())); + } + @Override public void close() throws Exception { Preconditions.checkState(isClosable(), @@ -464,6 +550,35 @@ public Concourse request() { return connection; } + /** + * Return the {@link Queue} that will hold the connections. + * + * @param size + * + * @return the connections cache + */ + protected abstract Queue buildQueue(int size); + + /** + * Force the connection pool to close regardless of whether it is or is not + * in a {@link #isClosable() closable} state. + */ + protected void forceClose() { + if(open.compareAndSet(true, false)) { + exitConnections(available); + exitConnections(leased); + } + } + + /** + * Get a connection from the queue of {@code available} ones. The subclass + * should use the correct method depending upon whether this method should + * block or not. + * + * @return the connection + */ + protected abstract Concourse getConnection(); + /** * Exit all the connections managed of the pool that has a * {@link #available}. @@ -529,33 +644,4 @@ private void verifyValidOrigin(Concourse connection) { + "was not previously requested from this pool"); } } - - /** - * Return the {@link Queue} that will hold the connections. - * - * @param size - * - * @return the connections cache - */ - protected abstract Queue buildQueue(int size); - - /** - * Force the connection pool to close regardless of whether it is or is not - * in a {@link #isClosable() closable} state. - */ - protected void forceClose() { - if(open.compareAndSet(true, false)) { - exitConnections(available); - exitConnections(leased); - } - } - - /** - * Get a connection from the queue of {@code available} ones. The subclass - * should use the correct method depending upon whether this method should - * block or not. - * - * @return the connection - */ - protected abstract Concourse getConnection(); } diff --git a/concourse-driver-java/src/main/java/com/cinchapi/concourse/FixedConnectionPool.java b/concourse-driver-java/src/main/java/com/cinchapi/concourse/FixedConnectionPool.java index 8e65323ff..75eb4c223 100644 --- a/concourse-driver-java/src/main/java/com/cinchapi/concourse/FixedConnectionPool.java +++ b/concourse-driver-java/src/main/java/com/cinchapi/concourse/FixedConnectionPool.java @@ -18,6 +18,7 @@ import java.util.Queue; import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.BlockingQueue; +import java.util.function.Supplier; import com.cinchapi.common.base.CheckedExceptions; @@ -33,31 +34,11 @@ class FixedConnectionPool extends ConnectionPool { /** * Construct a new instance. * - * @param host - * @param port - * @param username - * @param password + * @param supplier * @param poolSize */ - protected FixedConnectionPool(String host, int port, String username, - String password, int poolSize) { - this(host, port, username, password, "", poolSize); - } - - /** - * Construct a new instance. - * - * @param host - * @param port - * @param username - * @param password - * @param environment - * @param poolSize - */ - protected FixedConnectionPool(String host, int port, String username, - String password, String environment, int poolSize) { - super(() -> Concourse.connect(host, port, username, password, - environment), poolSize); + protected FixedConnectionPool(Supplier supplier, int poolSize) { + super(supplier, poolSize); } @Override diff --git a/concourse-integration-tests/src/test/java/com/cinchapi/concourse/CachedConnectionPoolTest.java b/concourse-integration-tests/src/test/java/com/cinchapi/concourse/CachedConnectionPoolTest.java index 63747b969..f4ff5c638 100644 --- a/concourse-integration-tests/src/test/java/com/cinchapi/concourse/CachedConnectionPoolTest.java +++ b/concourse-integration-tests/src/test/java/com/cinchapi/concourse/CachedConnectionPoolTest.java @@ -56,4 +56,9 @@ protected ConnectionPool getConnectionPool(String env) { USERNAME, PASSWORD, env); } + @Override + protected ConnectionPool getConnectionPool(Concourse concourse) { + return ConnectionPool.newCachedConnectionPool(concourse); + } + } diff --git a/concourse-integration-tests/src/test/java/com/cinchapi/concourse/ConnectionPoolTest.java b/concourse-integration-tests/src/test/java/com/cinchapi/concourse/ConnectionPoolTest.java index d703366b3..d014ecac9 100644 --- a/concourse-integration-tests/src/test/java/com/cinchapi/concourse/ConnectionPoolTest.java +++ b/concourse-integration-tests/src/test/java/com/cinchapi/concourse/ConnectionPoolTest.java @@ -25,6 +25,7 @@ import com.cinchapi.concourse.server.concurrent.Threads; import com.cinchapi.concourse.test.ConcourseIntegrationTest; import com.cinchapi.concourse.util.Environments; +import com.cinchapi.concourse.util.Random; import com.cinchapi.concourse.util.TestData; import com.google.common.base.Strings; import com.google.common.collect.Lists; @@ -193,6 +194,33 @@ public void testRequestReleaseRaceCondition() { }); } + @Test + public void testCopyConnectionPoolIndependentOfCopiedHandler() + throws Exception { + Concourse concourse = Concourse.connect(SERVER_HOST, SERVER_PORT, + USERNAME, PASSWORD, Random.getSimpleString()); + ConnectionPool pool = getConnectionPool(concourse); + try { + concourse.exit(); + try { + concourse.inventory(); + Assert.fail(); + } + catch (Exception e) { + Assert.assertTrue(true); + } + for (int i = 0; i < TestData.getScaleCount(); ++i) { + Concourse connection = pool.request(); + connection.inventory(); + Assert.assertTrue(true); + pool.release(connection); + } + } + finally { + pool.close(); + } + } + /** * Return a {@link com.cinchapi.concourse.ConnectionPool} to use in a unit * test. @@ -211,4 +239,12 @@ public void testRequestReleaseRaceCondition() { */ protected abstract ConnectionPool getConnectionPool(String env); + /** + * Return a {@link ConnectionPool} to use in a unit test. + * + * @param concourse + * @return the {@link ConnectionPool} + */ + protected abstract ConnectionPool getConnectionPool(Concourse concourse); + } diff --git a/concourse-integration-tests/src/test/java/com/cinchapi/concourse/CustomConnectionPoolTest.java b/concourse-integration-tests/src/test/java/com/cinchapi/concourse/CustomConnectionPoolTest.java index 63d507408..ecc7bd86d 100644 --- a/concourse-integration-tests/src/test/java/com/cinchapi/concourse/CustomConnectionPoolTest.java +++ b/concourse-integration-tests/src/test/java/com/cinchapi/concourse/CustomConnectionPoolTest.java @@ -73,6 +73,16 @@ public CustomConcourse(Concourse concourse) { static class CustomConnectionPool extends ConnectionPool { + /** + * Construct a new instance. + * + * @param concourse + * @param poolSize + */ + public CustomConnectionPool(Concourse concourse, int poolSize) { + super(concourse, poolSize); + } + /** * Construct a new instance. * @@ -114,4 +124,9 @@ protected Concourse getConnection() { } + @Override + protected ConnectionPool getConnectionPool(Concourse concourse) { + return new CustomConnectionPool(concourse, 10); + } + } diff --git a/concourse-integration-tests/src/test/java/com/cinchapi/concourse/FixedConnectionPoolTest.java b/concourse-integration-tests/src/test/java/com/cinchapi/concourse/FixedConnectionPoolTest.java index b23dd8712..534f46ca2 100644 --- a/concourse-integration-tests/src/test/java/com/cinchapi/concourse/FixedConnectionPoolTest.java +++ b/concourse-integration-tests/src/test/java/com/cinchapi/concourse/FixedConnectionPoolTest.java @@ -83,4 +83,9 @@ protected ConnectionPool getConnectionPool(String env) { USERNAME, PASSWORD, env, POOL_SIZE); } + @Override + protected ConnectionPool getConnectionPool(Concourse concourse) { + return ConnectionPool.newFixedConnectionPool(concourse, POOL_SIZE); + } + } From c2ff869c1d0f7651572cf36b5a58689a591a6e14 Mon Sep 17 00:00:00 2001 From: Jeff Nelson Date: Sun, 30 Jun 2024 04:35:34 -0700 Subject: [PATCH 4/4] Reduce Manifest Memory Consumption (#520) * Reduce the amount of memory required for Manifests by optimizing heap layout and hashmap sizing * update changelog * further optimize hashmap sizing * Use a custom hashmap from fastutil to further reduce Manifest memory by mapping byte[] to byte[] * slight code change * add feature flag * rename from enable_minimized_metadata to enable_efficient_metadata * rename HeapEntries to BinaryHashMap * rename HeapEntries to BinaryHashMap * rename HeapEntries to BinaryHashMap * rename HeapEntries to BinaryHashMap * rename HeapEntries to BinaryHashMap * no temp object creation * fix weird performance issue using getOrDefault... * clean up some things * fix up names * doc update * update changelog * doc updates * rename MutableRange to Span * formatting * fix up * Get the naming right, I think * get all the names right * add getOrDefault impl just in case anyone tries to use it in the future --- CHANGELOG.md | 2 + concourse-server/build.gradle | 1 + concourse-server/conf/concourse.yaml | 12 + .../concourse/server/GlobalState.java | 19 + .../server/storage/db/kernel/Chunk.java | 1 - .../server/storage/db/kernel/Manifest.java | 542 +++++++++++++----- .../server/storage/db/kernel/Range.java | 47 ++ .../storage/db/kernel/ManifestTest.java | 3 +- licenses/fastutil-core-8.5.13.txt | 201 +++++++ 9 files changed, 667 insertions(+), 161 deletions(-) create mode 100644 concourse-server/src/main/java/com/cinchapi/concourse/server/storage/db/kernel/Range.java create mode 100644 licenses/fastutil-core-8.5.13.txt diff --git a/CHANGELOG.md b/CHANGELOG.md index 4dd5d2f22..30784cc3b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -39,6 +39,8 @@ We made several changes to improve the safety, scalability and operational effic ##### New Functionality * Added the ability to create `ConnectionPool`s that copy the credentials and connection information from an existing handler These copying connection pools can be created by using the respective "cached" or "fixed" factory methods in the `ConnectionPool` class that take a `Concourse` parameter. +* Reduced the amount of heap space required for essential storage metadata. +* Added the `enable_efficient_metadata` configuration option to further reduce the amount of heap space required for essential storage metadata. When this option is set to `true`, metadata will occupy approximately one-third less heap space and likely improve overall system performance due to a decrease in garbage collection pauses (although per-operation performance may be slightly affected by additional overhead). ##### Bug Fixes * [GH-454](https://github.com/cinchapi/concourse/issues/454): Fixed an issue that caused JVM startup options overriden in a ".dev" configuration file to be ignored (e.g., `heap_size`). diff --git a/concourse-server/build.gradle b/concourse-server/build.gradle index 5c765e720..6be722c2e 100644 --- a/concourse-server/build.gradle +++ b/concourse-server/build.gradle @@ -43,6 +43,7 @@ dependencies { compile 'com.github.davidmoten:bplustree:0.1.3' compile group:'com.cinchapi', name: 'off-heap-memory', version: '1.1.0-SNAPSHOT', changing:true compile group:'com.cinchapi', name: 'configctl-lib', version: '1.2.0', changing:true + compile 'it.unimi.dsi:fastutil-core:8.5.13' testCompile 'com.carrotsearch:junit-benchmarks:0.7.2' testCompile 'io.takari.junit:takari-cpsuite:1.2.7' diff --git a/concourse-server/conf/concourse.yaml b/concourse-server/conf/concourse.yaml index aa611ccc9..39ed7392e 100644 --- a/concourse-server/conf/concourse.yaml +++ b/concourse-server/conf/concourse.yaml @@ -209,6 +209,18 @@ remote_debugger_port: # DEFAULT: false enable_compaction: +# Use a more memory-efficient representation for storage metadata. +# +# On average, enabling this setting will reduce the amount of heap space needed +# for essential metadata by a third. As a result, overall system performance may +# improve due to a reduction in garbage collection pauses. +# +# However, this setting may increase CPU usage and slightly reduce peak performance +# on a per-operation basis due to weaker reference locality. +# +# DEFAULT: false +enable_efficient_metadata: + # Maintain and in-memory cache of the data indexes used to respond to search commands. # Search indexes tend to be much larger than those used for primary and secondary # lookups, so enabling the search cache may cause memory issues (and overall diff --git a/concourse-server/src/main/java/com/cinchapi/concourse/server/GlobalState.java b/concourse-server/src/main/java/com/cinchapi/concourse/server/GlobalState.java index 42dd348e7..f6765f8a1 100644 --- a/concourse-server/src/main/java/com/cinchapi/concourse/server/GlobalState.java +++ b/concourse-server/src/main/java/com/cinchapi/concourse/server/GlobalState.java @@ -333,6 +333,22 @@ public final class GlobalState extends Constants { @Experimental public static boolean ENABLE_VERIFY_BY_LOOKUP = false; + /** + * Use a more memory-efficient representation for storage metadata. + *

+ * On average, enabling this setting will reduce the amount of heap space + * needed for essential metadata by 33%. As a result, overall system + * performance may improve due to a reduction in garbage collection pauses. + *

+ *

+ * However, this setting may increase CPU usage and slightly reduce + * peak performance on a per-operation basis due to weaker reference + * locality. + *

+ */ + @Experimental + public static boolean ENABLE_EFFICIENT_METADATA = false; + static { List files = ImmutableList.of( "conf" + File.separator + "concourse.prefs", @@ -419,6 +435,9 @@ public final class GlobalState extends Constants { INIT_ROOT_USERNAME = config.getOrDefault("init.root.username", config.getOrDefault("init_root_username", INIT_ROOT_USERNAME)); + + ENABLE_EFFICIENT_METADATA = config.getOrDefault( + "enable_efficient_metadata", ENABLE_EFFICIENT_METADATA); // =================== PREF READING BLOCK ==================== } diff --git a/concourse-server/src/main/java/com/cinchapi/concourse/server/storage/db/kernel/Chunk.java b/concourse-server/src/main/java/com/cinchapi/concourse/server/storage/db/kernel/Chunk.java index 2aa564401..d5ffae1d0 100644 --- a/concourse-server/src/main/java/com/cinchapi/concourse/server/storage/db/kernel/Chunk.java +++ b/concourse-server/src/main/java/com/cinchapi/concourse/server/storage/db/kernel/Chunk.java @@ -47,7 +47,6 @@ import com.cinchapi.concourse.server.storage.cache.BloomFilter; import com.cinchapi.concourse.server.storage.db.Record; import com.cinchapi.concourse.server.storage.db.Revision; -import com.cinchapi.concourse.server.storage.db.kernel.Manifest.Range; import com.cinchapi.concourse.util.Logger; import com.cinchapi.lib.offheap.collect.OffHeapSortedSet; import com.cinchapi.lib.offheap.io.Serializer; diff --git a/concourse-server/src/main/java/com/cinchapi/concourse/server/storage/db/kernel/Manifest.java b/concourse-server/src/main/java/com/cinchapi/concourse/server/storage/db/kernel/Manifest.java index d51a14f19..3383645df 100644 --- a/concourse-server/src/main/java/com/cinchapi/concourse/server/storage/db/kernel/Manifest.java +++ b/concourse-server/src/main/java/com/cinchapi/concourse/server/storage/db/kernel/Manifest.java @@ -21,11 +21,12 @@ import java.nio.file.Path; import java.util.AbstractMap; import java.util.AbstractSet; +import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.Iterator; import java.util.Map; -import java.util.Objects; +import java.util.Map.Entry; import java.util.Set; import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.BlockingQueue; @@ -35,7 +36,6 @@ import java.util.function.BiConsumer; import javax.annotation.Nullable; -import javax.annotation.concurrent.Immutable; import javax.annotation.concurrent.NotThreadSafe; import com.cinchapi.common.base.CheckedExceptions; @@ -48,9 +48,13 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Iterators; import com.google.common.util.concurrent.MoreExecutors; import com.google.common.util.concurrent.ThreadFactoryBuilder; +import it.unimi.dsi.fastutil.Hash; +import it.unimi.dsi.fastutil.objects.Object2ObjectOpenCustomHashMap; + /** * A {@link Manifest} stores and provides the efficient lookup for the start and * end position for sequences of bytes that relate to a key that is described by @@ -136,7 +140,7 @@ public static Manifest load(Segment segment, long position, long length) { * Returned from {@link #lookup(Composite)} when an associated entry does * not exist. */ - private static final Range NULL_RANGE = new Range() { + private static final com.cinchapi.concourse.server.storage.db.kernel.Range NULL_RANGE = new com.cinchapi.concourse.server.storage.db.kernel.Range() { @Override public long end() { @@ -151,8 +155,21 @@ public long start() { }; /** - * A {@link SoftReference} to the entries contained in the {@link Manifest} - * that is used to reduce memory overhead. + * The estimated number of bytes required to store an entry. This + * calculation is based on the variable range of {@link Composite} lengths. + */ + // @formatter:off + private static final int ESTIMATED_ENTRY_SIZE_IN_BYTES = + ((Range.CONSTANT_SIZE + 1) + + (Range.CONSTANT_SIZE + Composite.MAX_SIZE)) + / 2; + //@formatter:on + + /** + * A {@link SoftReference} to the entries contained in the {@link Manifest}. + * After the {@link Manifest}'s memory is {@link {@link #free() freed}, this + * reference is populated to opportunistically keep the entries in memory if + * there is room to do so. * *

* It is {@code null} until the {@link Manifest} is {@link #flush(ByteSink) @@ -160,20 +177,29 @@ public long start() { *

*/ @Nullable - private SoftReference> $entries; + private SoftReference> $entries; /** * The entries contained in the {@link Manifest}. *

+ * Entries are represented as a mapping from a lookup {@link Composite key} + * to a {@link Range} that encapsulates a start and end position of a + * specific data block in a {@link Chunk}. + *

+ *

* It is {@code null} if the - * {@link Manifest} has been {@link #freeze(Path, long) frozen}. + * {@link Manifest}'s memory has been {@link #free() freed} or the + * {@link Manifest} has been loaded from disk. The intent is to only keep + * the entries in memory for a {@link Manifest} that is being actively + * updated. *

*/ @Nullable - private Map entries; + private Map entries; /** - * The running size of the {@link Manifest} in bytes. + * The running size of the {@link Manifest} in bytes. Incremented as entries + * are {@link #putStart(long, Composite) record}. */ private long length = 0; @@ -192,7 +218,9 @@ public long start() { private Manifest(int expectedInsertions) { super(); this.length = 0; - this.entries = new HashMap<>(expectedInsertions); + this.entries = GlobalState.ENABLE_EFFICIENT_METADATA + ? new BinaryHashMap(expectedInsertions) + : new HashMap<>(expectedInsertions); this.$entries = null; } @@ -245,30 +273,36 @@ public long length() { } /** - * Return the {@link Entry} which contains the start and end positions of - * data related to the {@code byteables}, if data was recorded using + * Return the {@link com.cinchapi.concourse.server.storage.db.kernel.Range + * Range} which contains the start and end positions of data related to the + * {@code byteables}, if data was recorded using * {@link #putStart(long, Byteable...)} and * {@link #putEnd(long, Byteable...)}. * * @param composite - * @return the {@link Entry} containing the start and end positions + * @return the {@link com.cinchapi.concourse.server.storage.db.kernel.Range + * Range} containing the start and end positions */ - public Range lookup(Byteable... bytables) { + public com.cinchapi.concourse.server.storage.db.kernel.Range lookup( + Byteable... bytables) { return lookup(Composite.create(bytables)); } /** - * Return the {@link Entry} which contains the start and end positions of - * data related to the {@code composite}, if data was recorded using + * Return the {@link com.cinchapi.concourse.server.storage.db.kernel.Range + * Range} which contains the start and end positions of data related to the + * {@code composite}, if data was recorded using * {@link #putStart(long, Byteable...)} and * {@link #putEnd(long, Byteable...)}. * * @param composite - * @return the {@link Entry} containing the start and end positions + * @return the {@link com.cinchapi.concourse.server.storage.db.kernel.Range + * Range} containing the start and end positions */ - public Range lookup(Composite composite) { - Entry entry = entries(composite).get(composite); - return entry != null ? new EntryRange(entry) : NULL_RANGE; + public com.cinchapi.concourse.server.storage.db.kernel.Range lookup( + Composite composite) { + Range range = entries(composite).get(composite); + return range != null ? range : NULL_RANGE; } /** @@ -291,12 +325,12 @@ public void putEnd(long end, Composite composite) { Preconditions.checkArgument(end >= 0, "Cannot have negative index. Tried to put %s", end); Preconditions.checkState(isMutable()); - Entry entry = entries.get(composite); - Preconditions.checkState(entry != null, + Range range = entries.get(composite); + Preconditions.checkState(range != null, "Cannot set the end position before setting " + "the start position. Tried to put %s", end); - entry.setEnd(end); + range.setEnd(end); } /** @@ -319,26 +353,38 @@ public void putStart(long start, Composite composite) { Preconditions.checkArgument(start >= 0, "Cannot have negative index. Tried to put %s", start); Preconditions.checkState(isMutable()); - Entry entry = entries.get(composite); - if(entry == null) { - entry = new Entry(composite); - entries.put(composite, entry); - length += entry.size() + 4; + Range range = entries.get(composite); + if(range == null) { + // @formatter:off + range = GlobalState.ENABLE_EFFICIENT_METADATA + ? new BinaryRange() + : new LongRange(); + // @formatter:on + entries.put(composite, range); + // @formatter:off + length += composite.size() + + Range.CONSTANT_SIZE + + 4; // (each entry is preceded by 4 bytes that gives the overall length) + // @formatter:on } - entry.setStart(start); + range.setStart(start); } @Override protected void flush(ByteSink sink) { - for (Entry entry : entries().values()) { - sink.putInt(entry.size()); - entry.copyTo(sink); + for (Entry entry : entries().entrySet()) { + Composite key = entry.getKey(); + Range range = entry.getValue(); + int size = Range.CONSTANT_SIZE + key.size(); + sink.putInt(size); + sink.put(range.bytes()); + key.copyTo(sink); } } @Override protected void free() { - this.$entries = new SoftReference>(entries); + this.$entries = new SoftReference>(entries); this.entries = null; // Make eligible for GC } @@ -358,7 +404,7 @@ protected boolean isLoaded() { // visible for testing * * @return the entries */ - private synchronized Map entries() { + private synchronized Map entries() { return entries(null); } @@ -380,7 +426,7 @@ private synchronized Map entries() { * {@link Map#get(Object)} * @return the entries */ - private synchronized Map entries( + private synchronized Map entries( @Nullable Composite composite) { if(entries != null) { return entries; @@ -389,7 +435,7 @@ else if($entries != null && $entries.get() != null) { return $entries.get(); } else { - Map entries = new StreamedEntries(); + Map entries = new StreamedEntries(); // If the Manifest is small enough to fit comfortably into memory, // eagerly load all of the entries instead of streaming them from // disk one-by-one (as is done in the StreamedEntries). @@ -398,21 +444,23 @@ else if($entries != null && $entries.get() != null) { // forking the job to a background thread which listens for the // sought #composite to be found and returned immediately while // the other entries continue to be read - BlockingQueue> queue = new ArrayBlockingQueue<>( + BlockingQueue> queue = new ArrayBlockingQueue<>( 1); // @formatter:off Executor executor = composite != null ? ASYNC_BACKGROUND_LOADER : MoreExecutors.directExecutor(); // @formatter:on - Map heapEntries = new HashMap<>( - (int) length / Entry.CONSTANT_SIZE); + int capacity = (int) length + / (4 + ESTIMATED_ENTRY_SIZE_IN_BYTES); + Map heapEntries = GlobalState.ENABLE_EFFICIENT_METADATA + ? new BinaryHashMap(capacity) + : new HashMap<>(capacity); executor.execute(() -> { boolean found = false; - for (Map.Entry entry : entries - .entrySet()) { + for (Entry entry : entries.entrySet()) { Composite key = entry.getKey(); - Entry value = entry.getValue(); + Range value = entry.getValue(); heapEntries.put(key, value); if(composite != null && !found && key.equals(composite)) { @@ -422,7 +470,7 @@ else if($entries != null && $entries.get() != null) { } queue.offer(composite != null ? Collections.emptyMap() : heapEntries); - $entries = new SoftReference>( + $entries = new SoftReference>( heapEntries); }); try { @@ -439,94 +487,184 @@ else if($entries != null && $entries.get() != null) { } /** - * Contains the start and end positions for an entry in the - * {@link Manifest}. + * A {@link Map} that stores {@link Manifest} entries on heap in a + * memory-efficient manner. + *

+ * Used instead of a normal {@link HashMap} when + * {@link GlobalState#ENABLE_EFFICIENT_METADATA} is {@code true}. + *

* * @author Jeff Nelson */ - @Immutable - public static abstract class Range { + private final static class BinaryHashMap + extends AbstractMap { + + /** + * Strategy used to correctly determine hash codes and equality among + * byte arrays. + */ + private final static Hash.Strategy HASH_STRATEGY = new Hash.Strategy() { + + @Override + public boolean equals(byte[] a, byte[] b) { + return Arrays.equals(a, b); + } + + @Override + public int hashCode(byte[] o) { + return Arrays.hashCode(o); + } - // This class is returned from the #lookup methods to provide a clean - // interface to callers without exposing the totality of what is - // encapsulated in each Entry. + }; /** - * Return the end position. If it has not been recorded, return - * {@link Manifest#NO_ENTRY}. - * - * @return the end position + * The internal on-heap data structure where the entries are maintained. + *

+ * An entry is represented as the mapping between two byte arrays to + * avoid memory overhead that would accompany the storage of + * {@link Composite} and {@link Range} objects directly. This is + * necessary because the storage overhead (especially in the case of a + * {@link Range}) would equal or exceed the amount of memory + * needed for + * the essence of the data). + *

+ *

+ * Ad hoc, {@link Composite} and {@link Range} objects are read + * and + * constructed on the fly, as necessary. + *

*/ - public abstract long end(); + private final Map internal; /** - * Return the start position. If it has not been recorded, return - * {@link Manifest#NO_ENTRY}. + * Construct a new instance. * - * @return the start position + * @param initialCapacity */ - public abstract long start(); + private BinaryHashMap(int initialCapacity) { + this.internal = new Object2ObjectOpenCustomHashMap<>( + initialCapacity, HASH_STRATEGY); + } + + @Override + public Set> entrySet() { + return new AbstractSet>() { + + @Override + public Iterator> iterator() { + return Iterators.transform(internal.entrySet().iterator(), + entry -> { + Composite key = Composite + .load(ByteBuffer.wrap(entry.getKey())); + Range value = new BinaryRange(entry.getValue()); + return new SimpleImmutableEntry<>(key, value); + }); + } + + @Override + public int size() { + return internal.size(); + } + + }; + } + + @Override + public Range get(Object key) { + if(key instanceof Composite) { + byte[] value = internal.get(((Composite) key).bytes()); + if(value != null) { + return new BinaryRange(value); + } + } + return null; + } + + @Override + public boolean isEmpty() { + return internal.isEmpty(); + } + + @Override + public Range put(Composite key, Range value) { + byte[] k = key.bytes(); + byte[] v = value.bytes(); + byte[] prev = internal.put(k, v); + return prev != null ? new BinaryRange(prev) : null; + } + + @Override + public int size() { + return internal.size(); + } + } /** - * Represents a single entry in the {@link Manifest}. - * + * A {@link Range} that stores positions in a byte array. + * * @author Jeff Nelson */ - private final class Entry implements Byteable { + private static class BinaryRange implements Range { - private static final int CONSTANT_SIZE = 16; // start(8), end(8) + /** + * The bytes for each marker. + */ + private byte[] bytes; - private long end = NO_ENTRY; - private final Composite key; - private long start = NO_ENTRY; + /** + * Construct a new instance. + */ + BinaryRange() { + this.bytes = new byte[CONSTANT_SIZE]; + long value = NO_ENTRY; + for (int i = 7; i >= 0; i--) { + bytes[i] = bytes[i + 8] = (byte) (value & 0xFF); + value >>= 8; + } + } /** - * Construct an instance that represents an existing Entry from a - * ByteBuffer. This constructor is public so as to comply with the - * {@link Byteable} interface. Calling this constructor directly is not - * recommend. Use {@link #fromByteBuffer(ByteBuffer)} instead to take - * advantage of reference caching. + * Load an existing instance. * * @param bytes */ - public Entry(ByteBuffer bytes) { - this.start = bytes.getLong(); - this.end = bytes.getLong(); - this.key = Composite.load(bytes); + BinaryRange(ByteBuffer bytes) { + this.bytes = new byte[CONSTANT_SIZE]; + bytes.get(this.bytes); } /** - * Construct a new instance. + * Construct a new ad-hoc instance. * - * @param key + * @param bytes */ - public Entry(Composite key) { - this.key = key; + private BinaryRange(byte[] bytes) { + // This constructor should only be used to construct ad ad-hoc Range + // from a byte array that is already stored in a HeapEntries + // instance. + Preconditions.checkArgument(bytes.length == CONSTANT_SIZE); + this.bytes = bytes; } @Override - public void copyTo(ByteSink sink) { - sink.putLong(start); - sink.putLong(end); - key.copyTo(sink); + public byte[] bytes() { + return bytes; } - /** - * Return the end position. - * - * @return the end - */ + @Override public long end() { - return end; + return read(8); } @Override public boolean equals(Object obj) { - if(obj instanceof Entry) { - Entry other = (Entry) obj; - return start == other.start && end == other.end - && key.equals(other.key); + if(this == obj) { + return true; + } + else if(obj instanceof Range) { + Range other = (Range) obj; + return Arrays.equals(bytes, other.bytes()); } else { return false; @@ -535,46 +673,111 @@ public boolean equals(Object obj) { @Override public int hashCode() { - return Objects.hash(key, start, end); + return Arrays.hashCode(bytes); + } + + @Override + public void setEnd(long value) { + write(8, value); + } + + @Override + public void setStart(long value) { + write(0, value); + } + + @Override + public long start() { + return read(0); } /** - * Return the entry key + * Read 8 of the {@link #bytes} starting at {@code index} and return + * the corresponding long. * - * @return the key + * @param index + * @return the read value */ - public Composite key() { - return key; + private long read(int index) { + return ((long) bytes[index] << 56) + | ((long) (bytes[index + 1] & 0xff) << 48) + | ((long) (bytes[index + 2] & 0xff) << 40) + | ((long) (bytes[index + 3] & 0xff) << 32) + | ((long) (bytes[index + 4] & 0xff) << 24) + | ((long) (bytes[index + 5] & 0xff) << 16) + | ((long) (bytes[index + 6] & 0xff) << 8) + | ((long) (bytes[index + 7] & 0xff)); } /** - * Set the end position. + * Write {@code value} to {@link #bytes} starting at {@code index}. * - * @param end the end to set + * @param index + * @param value */ - public void setEnd(long end) { - this.end = end; + private void write(int index, long value) { + bytes[index] = (byte) (value >> 56); + bytes[index + 1] = (byte) (value >> 48); + bytes[index + 2] = (byte) (value >> 40); + bytes[index + 3] = (byte) (value >> 32); + bytes[index + 4] = (byte) (value >> 24); + bytes[index + 5] = (byte) (value >> 16); + bytes[index + 6] = (byte) (value >> 8); + bytes[index + 7] = (byte) value; } + } + + /** + * A {@link Range} that stores positions as 64-bit long values. + * + * @author Jeff Nelson + */ + private static class LongRange implements Range { /** - * Set the start position. - * - * @param start the start to set + * The start position. */ - public void setStart(long start) { - this.start = start; - } + private long start; - @Override - public int size() { - return CONSTANT_SIZE + key.size(); + /** + * The end position. + */ + private long end; + + /** + * Construct a new instance. + */ + LongRange() { + this.start = NO_ENTRY; + this.end = NO_ENTRY; } /** - * Return the start position. + * Construct a new instance. * - * @return the start + * @param bytes */ + LongRange(ByteBuffer bytes) { + this.start = bytes.getLong(); + this.end = bytes.getLong(); + } + + @Override + public long end() { + return end; + } + + @Override + public void setEnd(long end) { + this.end = end; + } + + @Override + public void setStart(long start) { + this.start = start; + } + + @Override public long start() { return start; } @@ -582,38 +785,51 @@ public long start() { } /** - * A {@link Range} backed by an {@link Entry}. + * A {@link com.cinchapi.concourse.server.storage.db.kernel.Range Range} for + * {@link Manifest} entries. + * + * @author Jeff Nelson */ - @Immutable - private final class EntryRange extends Range { + private static interface Range extends + com.cinchapi.concourse.server.storage.db.kernel.Range { /** - * The underlying {@link Entry}. + * The number of bytes required to record each {@link Range}. */ - private final Entry entry; - - private EntryRange() { - this.entry = null; - }; + static final int CONSTANT_SIZE = 16; // start(8), end(8) /** - * Construct a new instance. + * Return the binary representation of this {@link Range}. * - * @param entry + * @return the {@link Range} bytes */ - private EntryRange(Entry entry) { - this.entry = entry; + public default byte[] bytes() { + byte[] bytes = new byte[CONSTANT_SIZE]; + long start = start(); + long end = end(); + for (int i = 7; i >= 0; i--) { + bytes[i] = (byte) (start & 0xFF); + bytes[i + 8] = (byte) (end & 0xFF); + start >>= 8; + end >>= 8; + } + return bytes; } - @Override - public long end() { - return entry.end(); - } + /** + * Set the end position to {@code value}. + * + * @param value + */ + void setEnd(long end); + + /** + * Set the start position to {@code value}. + * + * @param value + */ + void setStart(long start); - @Override - public long start() { - return entry.start(); - } } /** @@ -640,6 +856,12 @@ public V getValue() { return value; } + public K setKey(K key) { + K old = key; + this.key = key; + return old; + } + @Override public V setValue(V value) { V old = value; @@ -647,12 +869,6 @@ public V setValue(V value) { return old; } - public K setKey(K key) { - K old = key; - this.key = key; - return old; - } - } /** @@ -662,18 +878,18 @@ public K setKey(K key) { * * @author Jeff Nelson */ - private final class StreamedEntries extends AbstractMap { + private final class StreamedEntries extends AbstractMap { @Override - public Set> entrySet() { + public Set> entrySet() { // It is assumed that the return #entrySet is only used to // facilitate streaming all the entries, so it is not appropriate to // perform query operations (e.g. get()) directly on it. - return new AbstractSet>() { + return new AbstractSet>() { @Override - public Iterator> iterator() { - return new Iterator>() { + public Iterator> iterator() { + return new Iterator>() { Iterator it = ByteableCollections.stream( channel(), position(), length, @@ -684,7 +900,7 @@ public Iterator> iterator() { * returned on each call to {@link #next()} so that we * don't create unnecessary temporary objects. */ - ReusableMapEntry reusable = new ReusableMapEntry<>(); + ReusableMapEntry reusable = new ReusableMapEntry<>(); @Override public boolean hasNext() { @@ -692,11 +908,14 @@ public boolean hasNext() { } @Override - public Entry next() { - Manifest.Entry entry = new Manifest.Entry( - it.next()); - reusable.setKey(entry.key()); - reusable.setValue(entry); + public Entry next() { + ByteBuffer next = it.next(); + Range range = GlobalState.ENABLE_EFFICIENT_METADATA + ? new BinaryRange(next) + : new LongRange(next); + Composite key = Composite.load(next); + reusable.setKey(key); + reusable.setValue(range); return reusable; } @@ -713,14 +932,14 @@ public int size() { @Override public void forEach( - BiConsumer action) { - for (Entry entry : entrySet()) { + BiConsumer action) { + for (Entry entry : entrySet()) { action.accept(entry.getKey(), entry.getValue()); } } @Override - public Manifest.Entry get(Object o) { + public Range get(Object o) { if(o instanceof Composite) { Composite key = (Composite) o; Iterator it = ByteableCollections.stream(channel(), @@ -729,7 +948,9 @@ public Manifest.Entry get(Object o) { while (it.hasNext()) { ByteBuffer next = it.next(); if(equals(keyBytes, next)) { - return new Manifest.Entry(next); + return GlobalState.ENABLE_EFFICIENT_METADATA + ? new BinaryRange(next) + : new LongRange(next); } } @@ -737,6 +958,12 @@ public Manifest.Entry get(Object o) { return null; } + @Override + public Range getOrDefault(Object key, Range defaultValue) { + Range value = get(key); + return value != null ? value : defaultValue; + } + /** * Assuming {@code key} is the {@link Composite#getBytes() byte buffer} * of a {@link Composite}, return {@code true} if the {@link Composite} @@ -747,10 +974,9 @@ public Manifest.Entry get(Object o) { * @return {@code true} if there is a match */ private boolean equals(ByteBuffer key, ByteBuffer next) { - if(key.remaining() + Manifest.Entry.CONSTANT_SIZE == next - .remaining()) { + if(key.remaining() + Range.CONSTANT_SIZE == next.remaining()) { next.mark(); - next.position(next.position() + Manifest.Entry.CONSTANT_SIZE); + next.position(next.position() + Range.CONSTANT_SIZE); if(key.equals(next)) { next.reset(); return true; diff --git a/concourse-server/src/main/java/com/cinchapi/concourse/server/storage/db/kernel/Range.java b/concourse-server/src/main/java/com/cinchapi/concourse/server/storage/db/kernel/Range.java new file mode 100644 index 000000000..5bc9a29ad --- /dev/null +++ b/concourse-server/src/main/java/com/cinchapi/concourse/server/storage/db/kernel/Range.java @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2013-2024 Cinchapi Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.cinchapi.concourse.server.storage.db.kernel; + +/** + * An interface representing a range defined by start and end positions. + * + * This interface encapsulates the concept of a range with distinct start and + * end points. It can be used to represent any interval or span that is defined + * by two positions, typically in a linear space such as indices, timestamps, or + * memory addresses. + * + * Implementations of this interface should ensure that the start position is + * always less than or equal to the end position. + * + * @author Jeff Nelson + */ +interface Range { + + /** + * Return the end position. + * + * @return the end position + */ + public long end(); + + /** + * Return the start position. + * + * @return the start position + */ + public long start(); + +} diff --git a/concourse-server/src/test/java/com/cinchapi/concourse/server/storage/db/kernel/ManifestTest.java b/concourse-server/src/test/java/com/cinchapi/concourse/server/storage/db/kernel/ManifestTest.java index c94e83585..ea04e3673 100644 --- a/concourse-server/src/test/java/com/cinchapi/concourse/server/storage/db/kernel/ManifestTest.java +++ b/concourse-server/src/test/java/com/cinchapi/concourse/server/storage/db/kernel/ManifestTest.java @@ -34,7 +34,6 @@ import com.cinchapi.concourse.server.io.FileSystem; import com.cinchapi.concourse.server.model.Identifier; import com.cinchapi.concourse.server.model.Text; -import com.cinchapi.concourse.server.storage.db.kernel.Manifest.Range; import com.cinchapi.concourse.test.ConcourseBaseTest; import com.cinchapi.concourse.time.Time; import com.cinchapi.concourse.util.Random; @@ -135,7 +134,7 @@ public void testManifestStreamedEntriesAccuracy() { Identifier record = Identifier.of(count); int $start = start; int end = start + TestData.getScaleCount(); - Range range = new Manifest.Range() { + Range range = new Range() { @Override public long start() { diff --git a/licenses/fastutil-core-8.5.13.txt b/licenses/fastutil-core-8.5.13.txt new file mode 100644 index 000000000..027d0f5d0 --- /dev/null +++ b/licenses/fastutil-core-8.5.13.txt @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License.