From 4b77d3fb3ae06f620c0cff3c47dd3643b6272f10 Mon Sep 17 00:00:00 2001 From: lakde Date: Mon, 19 Aug 2024 11:38:26 -0500 Subject: [PATCH 01/21] Initial Changes. Signed-off-by: lakde --- .../lib/core/models/ExtensionConfig.java | 12 +-- .../lib/core/models/features/Extension.java | 12 +++ .../naksha/lib/extmanager/ExtensionCache.java | 19 +++-- .../extmanager/helpers/AmazonS3Helper.java | 17 ---- .../here/naksha/lib/extmanager/BaseSetup.java | 2 +- .../com/here/naksha/lib/hub/NakshaHub.java | 81 +++++++++++++------ 6 files changed, 81 insertions(+), 62 deletions(-) diff --git a/here-naksha-lib-core/src/main/java/com/here/naksha/lib/core/models/ExtensionConfig.java b/here-naksha-lib-core/src/main/java/com/here/naksha/lib/core/models/ExtensionConfig.java index 36c38e25e..1d7a101a7 100644 --- a/here-naksha-lib-core/src/main/java/com/here/naksha/lib/core/models/ExtensionConfig.java +++ b/here-naksha-lib-core/src/main/java/com/here/naksha/lib/core/models/ExtensionConfig.java @@ -32,7 +32,6 @@ public class ExtensionConfig { public static final String EXPIRY = "expiry"; public static final String EXTENSIONS = "extensions"; public static final String WHITELIST_DELEGATE_CLASSES = "whitelistDelegateClasses"; - public static final String ENV_NAME = "env"; @JsonProperty(EXPIRY) long expiry; @@ -43,19 +42,14 @@ public class ExtensionConfig { @JsonProperty(WHITELIST_DELEGATE_CLASSES) List whitelistDelegateClasses; - @JsonProperty(ENV_NAME) - String env; - @JsonCreator public ExtensionConfig( @JsonProperty(EXPIRY) @NotNull Long expiry, @JsonProperty(EXTENSIONS) @Nullable List extensions, - @JsonProperty(WHITELIST_DELEGATE_CLASSES) @Nullable List whitelistDelegateClasses, - @JsonProperty(ENV_NAME) @NotNull String env) { + @JsonProperty(WHITELIST_DELEGATE_CLASSES) @Nullable List whitelistDelegateClasses) { this.expiry = expiry; this.extensions = extensions; this.whitelistDelegateClasses = whitelistDelegateClasses; - this.env = env; } public long getExpiry() { @@ -69,8 +63,4 @@ public List getExtensions() { public List getWhilelistDelegateClass() { return whitelistDelegateClasses; } - - public String getEnv() { - return env; - } } diff --git a/here-naksha-lib-core/src/main/java/com/here/naksha/lib/core/models/features/Extension.java b/here-naksha-lib-core/src/main/java/com/here/naksha/lib/core/models/features/Extension.java index 29aad5f86..031ed3868 100644 --- a/here-naksha-lib-core/src/main/java/com/here/naksha/lib/core/models/features/Extension.java +++ b/here-naksha-lib-core/src/main/java/com/here/naksha/lib/core/models/features/Extension.java @@ -36,6 +36,7 @@ public class Extension extends XyzFeature { public static final String URL = "url"; public static final String VERSION = "version"; public static final String INIT_CLASS_NAME = "initClassName"; + public static final String ENV = "env"; @JsonProperty(URL) String url; @@ -46,6 +47,9 @@ public class Extension extends XyzFeature { @JsonProperty(INIT_CLASS_NAME) String initClassName; + @JsonProperty(ENV) + String env; + /** * Create an extension. * @@ -78,4 +82,12 @@ public String getVersion() { public String getInitClassName() { return initClassName; } + + public void setEnv(String env) { + this.env = env; + } + + public String getEnv() { + return env; + } } diff --git a/here-naksha-lib-ext-manager/src/main/java/com/here/naksha/lib/extmanager/ExtensionCache.java b/here-naksha-lib-ext-manager/src/main/java/com/here/naksha/lib/extmanager/ExtensionCache.java index ec1fdc17b..5118cb5b3 100644 --- a/here-naksha-lib-ext-manager/src/main/java/com/here/naksha/lib/extmanager/ExtensionCache.java +++ b/here-naksha-lib-ext-manager/src/main/java/com/here/naksha/lib/extmanager/ExtensionCache.java @@ -83,9 +83,10 @@ protected void buildExtensionCache(ExtensionConfig extensionConfig) { publishIntoCache(result, extensionConfig); }); - // Removing existing extension which has been removed from the configuration - List extIds = - extensionConfig.getExtensions().stream().map(Extension::getId).toList(); + // Removing existing extension which has been removed from the configuration. + List extIds = extensionConfig.getExtensions().stream() + .map(extension -> extension.getEnv() + ":" + extension.getId()) + .toList(); for (String key : loaderCache.keySet()) { if (!extIds.contains(key)) { @@ -100,6 +101,7 @@ protected void buildExtensionCache(ExtensionConfig extensionConfig) { private void publishIntoCache(KVPair result, ExtensionConfig extensionConfig) { if (result != null && result.getValue() != null) { final Extension extension = result.getKey(); + final String extensionIdWthEnv = extension.getEnv() + ":" + extension.getId(); final File jarFile = result.getValue(); ClassLoader loader; try { @@ -121,7 +123,7 @@ private void publishIntoCache(KVPair result, ExtensionConfig ex logger.error( "Failed to instantiate class {} for extension {} ", extension.getInitClassName(), - extension.getId(), + extensionIdWthEnv, e); return; } @@ -129,21 +131,22 @@ private void publishIntoCache(KVPair result, ExtensionConfig ex if (!isNullOrEmpty(extension.getInitClassName())) logger.info( "Extension {} initialization using initClassName {} done successfully.", - extension.getId(), + extensionIdWthEnv, extension.getInitClassName()); loaderCache.put(extension.getId(), new KVPair(extension, loader)); PluginCache.removeExtensionCache(extension.getId()); logger.info( "Extension id={}, version={} is successfully loaded into the cache, using Jar at {} for env={}.", - extension.getId(), + extensionIdWthEnv, extension.getVersion(), extension.getUrl().substring(extension.getUrl().lastIndexOf("/") + 1), - extensionConfig.getEnv()); + extension.getEnv()); } } private boolean isLoaderMappingExist(Extension extension) { - KVPair existingMapping = loaderCache.get(extension.getId()); + final String extensionIdWthEnv = extension.getEnv() + ":" + extension.getId(); + KVPair existingMapping = loaderCache.get(extensionIdWthEnv); if (existingMapping == null) return false; final Extension exExtension = existingMapping.getKey(); diff --git a/here-naksha-lib-ext-manager/src/main/java/com/here/naksha/lib/extmanager/helpers/AmazonS3Helper.java b/here-naksha-lib-ext-manager/src/main/java/com/here/naksha/lib/extmanager/helpers/AmazonS3Helper.java index 5ed3364b3..c811cc035 100644 --- a/here-naksha-lib-ext-manager/src/main/java/com/here/naksha/lib/extmanager/helpers/AmazonS3Helper.java +++ b/here-naksha-lib-ext-manager/src/main/java/com/here/naksha/lib/extmanager/helpers/AmazonS3Helper.java @@ -26,14 +26,11 @@ import java.io.InputStream; import java.io.InputStreamReader; import java.net.URI; -import java.util.List; import org.jetbrains.annotations.NotNull; import software.amazon.awssdk.services.s3.S3Client; import software.amazon.awssdk.services.s3.S3Uri; import software.amazon.awssdk.services.s3.model.GetObjectRequest; import software.amazon.awssdk.services.s3.model.GetObjectRequest.Builder; -import software.amazon.awssdk.services.s3.model.ListObjectsRequest; -import software.amazon.awssdk.services.s3.model.ListObjectsResponse; public class AmazonS3Helper implements FileClient { private static S3Client s3Client; @@ -93,20 +90,6 @@ public String getFileContent(String url) throws IOException { } } - public List listKeysInBucket(String url) { - S3Uri fileUri = getS3Uri(url); - String delimiter = "/"; - - ListObjectsRequest.Builder listObjectsRequestBuilder = - ListObjectsRequest.builder().bucket(fileUri.bucket().get()).delimiter(delimiter); - - if (fileUri.key().isPresent()) - listObjectsRequestBuilder.prefix(fileUri.key().get()); - - ListObjectsResponse response = getS3Client().listObjects(listObjectsRequestBuilder.build()); - return response.commonPrefixes().stream().map(cm -> cm.prefix()).toList(); - } - public S3Uri getS3Uri(String url) { URI uri = URI.create(url); return getS3Client().utilities().parseUri(uri); diff --git a/here-naksha-lib-ext-manager/src/test/java/com/here/naksha/lib/extmanager/BaseSetup.java b/here-naksha-lib-ext-manager/src/test/java/com/here/naksha/lib/extmanager/BaseSetup.java index f997e0f67..fc5b546e1 100644 --- a/here-naksha-lib-ext-manager/src/test/java/com/here/naksha/lib/extmanager/BaseSetup.java +++ b/here-naksha-lib-ext-manager/src/test/java/com/here/naksha/lib/extmanager/BaseSetup.java @@ -25,6 +25,6 @@ public ExtensionConfig getExtensionConfig() { } catch (IOException e) { throw new RuntimeException(e); } - return new ExtensionConfig(System.currentTimeMillis() + 6000, list,whitelistUrls,"test"); + return new ExtensionConfig(System.currentTimeMillis() + 6000, list,whitelistUrls); } } diff --git a/here-naksha-lib-hub/src/main/java/com/here/naksha/lib/hub/NakshaHub.java b/here-naksha-lib-hub/src/main/java/com/here/naksha/lib/hub/NakshaHub.java index be3f1e175..06c183600 100644 --- a/here-naksha-lib-hub/src/main/java/com/here/naksha/lib/hub/NakshaHub.java +++ b/here-naksha-lib-hub/src/main/java/com/here/naksha/lib/hub/NakshaHub.java @@ -18,6 +18,7 @@ */ package com.here.naksha.lib.hub; +import static com.here.naksha.lib.core.NakshaAdminCollection.EVENT_HANDLERS; import static com.here.naksha.lib.core.exceptions.UncheckedException.unchecked; import static com.here.naksha.lib.core.models.PluginCache.getStorageConstructor; import static com.here.naksha.lib.core.util.storage.RequestHelper.createFeatureRequest; @@ -25,6 +26,7 @@ import static com.here.naksha.lib.core.util.storage.RequestHelper.readFeaturesByIdsRequest; import static com.here.naksha.lib.core.util.storage.RequestHelper.upsertFeaturesRequest; import static com.here.naksha.lib.core.util.storage.ResultHelper.readFeatureFromResult; +import static com.here.naksha.lib.core.util.storage.ResultHelper.readFeaturesFromResult; import com.fasterxml.jackson.databind.ObjectMapper; import com.here.naksha.lib.core.*; @@ -35,19 +37,16 @@ import com.here.naksha.lib.core.models.XyzError; import com.here.naksha.lib.core.models.features.Extension; import com.here.naksha.lib.core.models.geojson.implementation.XyzFeature; +import com.here.naksha.lib.core.models.naksha.EventHandler; import com.here.naksha.lib.core.models.naksha.Storage; import com.here.naksha.lib.core.models.naksha.XyzCollection; -import com.here.naksha.lib.core.models.storage.EExecutedOp; -import com.here.naksha.lib.core.models.storage.ErrorResult; -import com.here.naksha.lib.core.models.storage.ForwardCursor; -import com.here.naksha.lib.core.models.storage.Result; -import com.here.naksha.lib.core.models.storage.WriteXyzCollections; -import com.here.naksha.lib.core.models.storage.XyzCollectionCodec; +import com.here.naksha.lib.core.models.storage.*; import com.here.naksha.lib.core.storage.IReadSession; import com.here.naksha.lib.core.storage.IStorage; import com.here.naksha.lib.core.storage.IWriteSession; import com.here.naksha.lib.core.util.IoHelp; import com.here.naksha.lib.core.util.json.Json; +import com.here.naksha.lib.core.util.storage.RequestHelper; import com.here.naksha.lib.core.util.storage.ResultHelper; import com.here.naksha.lib.core.view.ViewDeserialize; import com.here.naksha.lib.extmanager.ExtensionManager; @@ -56,9 +55,9 @@ import com.here.naksha.lib.hub.storages.NHAdminStorage; import com.here.naksha.lib.hub.storages.NHSpaceStorage; import com.here.naksha.lib.psql.PsqlStorage; -import java.util.ArrayList; -import java.util.List; -import java.util.NoSuchElementException; + +import java.util.*; + import org.jetbrains.annotations.ApiStatus; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; @@ -98,6 +97,11 @@ public class NakshaHub implements INaksha { */ protected @NotNull IExtensionManager extensionManager; + /** + * The extensionId property path in handler json. + */ + protected static final String[] EXTN_ID_PROP_PATH = {"extensionId"}; + @ApiStatus.AvailableSince(NakshaVersion.v2_0_7) public NakshaHub( final @NotNull String appName, @@ -239,7 +243,7 @@ private static WriteXyzCollections createAdminCollectionsRequest() { } else { try { List nakshaHubConfigs = - ResultHelper.readFeaturesFromResult(rdResult, NakshaHubConfig.class); + readFeaturesFromResult(rdResult, NakshaHubConfig.class); for (final NakshaHubConfig cfg : nakshaHubConfigs) { if (cfg.getId().equals(configId)) { customDbCfg = cfg; @@ -292,28 +296,57 @@ private static WriteXyzCollections createAdminCollectionsRequest() { @Override public @NotNull ExtensionConfig getExtensionConfig() { + // Create ReadFeatures Request to read all handlers where extensionId not null from Admin DB + final NakshaContext nakshaContext = new NakshaContext().withAppId(NakshaHubConfig.defaultAppName()); + final ReadFeatures request = new ReadFeatures(EVENT_HANDLERS); + final PRef pref = RequestHelper.pRefFromPropPath(EXTN_ID_PROP_PATH); + POp notNullCondition = POp.exists(pref); + request.setPropertyOp(notNullCondition); + + final Result rdResult; + try (IReadSession readSession = getAdminStorage().newReadSession(nakshaContext, false)) { + rdResult = readSession.execute(request); + } catch (Exception e) { + logger.error("Failed reading from collections {}. ", request.getCollections(), e); + throw new RuntimeException("Failed to execute single read session", e); + } + final List eventHandlers; + try { + eventHandlers = readFeaturesFromResult(rdResult, EventHandler.class); + } catch (NoCursor e) { + logger.error("NoCursor exception encountered", e); + throw new RuntimeException("Failed to open cursor", e); + } + + Set extensionIds = new HashSet<>(); + for (EventHandler eventHandler : eventHandlers) { + String extensionId = eventHandler.getExtensionId(); + if (extensionId != null && extensionId.contains(":")) { + extensionIds.add(extensionId); + } else { + logger.error("Environment is missing for an extension Id"); + } + } + final ExtensionConfigParams extensionConfigParams = nakshaHubConfig.extensionConfigParams; if (!extensionConfigParams.extensionRootPath.startsWith("s3://")) throw new UnsupportedOperationException( "ExtensionRootPath must be a valid s3 bucket url which should be prefixed with s3://"); - List extList = loadExtensionConfigFromS3(extensionConfigParams.getExtensionRootPath()); + List extList = loadExtensionConfigFromS3(extensionConfigParams.getExtensionRootPath(), extensionIds); return new ExtensionConfig( System.currentTimeMillis() + extensionConfigParams.getIntervalMs(), extList, - extensionConfigParams.getWhiteListClasses(), - this.nakshaHubConfig.env.toLowerCase()); + extensionConfigParams.getWhiteListClasses()); } - private List loadExtensionConfigFromS3(String extensionRootPath) { + private List loadExtensionConfigFromS3(String extensionRootPath, Set extensionIds) { AmazonS3Helper s3Helper = new AmazonS3Helper(); - final String bucketName = s3Helper.getS3Uri(extensionRootPath).bucket().get(); - - List list = s3Helper.listKeysInBucket(extensionRootPath); List extList = new ArrayList<>(); - list.stream().forEach(extensionPath -> { - String filePath = - "s3://" + bucketName + "/" + extensionPath + "latest-" + nakshaHubConfig.env.toLowerCase() + ".txt"; + extensionIds.forEach(extensionId -> { + String env = extensionId.split(":")[0]; + String extensionIdWotEnv = extensionId.split(":")[1]; + String filePath = extensionRootPath + extensionIdWotEnv + "/" + "latest-" + env.toLowerCase() + ".txt"; String version; try { version = s3Helper.getFileContent(filePath); @@ -322,11 +355,8 @@ private List loadExtensionConfigFromS3(String extensionRootPath) { return; } - String bits[] = extensionPath.split("/"); - String extensionId = bits[bits.length - 1]; - - filePath = "s3://" + bucketName + "/" + extensionPath + extensionId + "-" + version + "." - + nakshaHubConfig.env.toLowerCase().toLowerCase() + ".json"; + filePath = extensionRootPath + extensionIdWotEnv + "/" + extensionIdWotEnv + "-" + version + "." + + env.toLowerCase() + ".json"; String exJson; try { exJson = s3Helper.getFileContent(filePath); @@ -337,6 +367,7 @@ private List loadExtensionConfigFromS3(String extensionRootPath) { Extension extension; try { extension = new ObjectMapper().readValue(exJson, Extension.class); + extension.setEnv(env); extList.add(extension); } catch (Exception e) { logger.error("Failed to convert extension meta data to Extension object. {} ", exJson, e); From ba85d9d82c36d6e4a44b2bb872253fb4bf7222c7 Mon Sep 17 00:00:00 2001 From: lakde Date: Mon, 19 Aug 2024 16:41:21 -0500 Subject: [PATCH 02/21] Fix Unit Tests Signed-off-by: lakde --- .../here/naksha/lib/extmanager/ExtensionCache.java | 14 +++++++------- .../naksha/lib/extmanager/ExtensionCacheTest.java | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/here-naksha-lib-ext-manager/src/main/java/com/here/naksha/lib/extmanager/ExtensionCache.java b/here-naksha-lib-ext-manager/src/main/java/com/here/naksha/lib/extmanager/ExtensionCache.java index 5118cb5b3..e45229fb1 100644 --- a/here-naksha-lib-ext-manager/src/main/java/com/here/naksha/lib/extmanager/ExtensionCache.java +++ b/here-naksha-lib-ext-manager/src/main/java/com/here/naksha/lib/extmanager/ExtensionCache.java @@ -85,8 +85,8 @@ protected void buildExtensionCache(ExtensionConfig extensionConfig) { // Removing existing extension which has been removed from the configuration. List extIds = extensionConfig.getExtensions().stream() - .map(extension -> extension.getEnv() + ":" + extension.getId()) - .toList(); + .map(extension -> extension.getEnv() + ":" + extension.getId()) + .toList(); for (String key : loaderCache.keySet()) { if (!extIds.contains(key)) { @@ -123,7 +123,7 @@ private void publishIntoCache(KVPair result, ExtensionConfig ex logger.error( "Failed to instantiate class {} for extension {} ", extension.getInitClassName(), - extensionIdWthEnv, + extensionIdWthEnv, e); return; } @@ -131,13 +131,13 @@ private void publishIntoCache(KVPair result, ExtensionConfig ex if (!isNullOrEmpty(extension.getInitClassName())) logger.info( "Extension {} initialization using initClassName {} done successfully.", - extensionIdWthEnv, + extensionIdWthEnv, extension.getInitClassName()); - loaderCache.put(extension.getId(), new KVPair(extension, loader)); - PluginCache.removeExtensionCache(extension.getId()); + loaderCache.put(extensionIdWthEnv, new KVPair(extension, loader)); + PluginCache.removeExtensionCache(extensionIdWthEnv); logger.info( "Extension id={}, version={} is successfully loaded into the cache, using Jar at {} for env={}.", - extensionIdWthEnv, + extensionIdWthEnv, extension.getVersion(), extension.getUrl().substring(extension.getUrl().lastIndexOf("/") + 1), extension.getEnv()); diff --git a/here-naksha-lib-ext-manager/src/test/java/com/here/naksha/lib/extmanager/ExtensionCacheTest.java b/here-naksha-lib-ext-manager/src/test/java/com/here/naksha/lib/extmanager/ExtensionCacheTest.java index 6c203456e..91796031d 100644 --- a/here-naksha-lib-ext-manager/src/test/java/com/here/naksha/lib/extmanager/ExtensionCacheTest.java +++ b/here-naksha-lib-ext-manager/src/test/java/com/here/naksha/lib/extmanager/ExtensionCacheTest.java @@ -76,7 +76,7 @@ public void testGetClassLoaderById() throws IOException { extensionCache.buildExtensionCache(extensionConfig); Assertions.assertEquals(2,extensionCache.getCacheLength()); - ClassLoader loader=extensionCache.getClassLoaderById(extensionConfig.getExtensions().get(0).getId()); + ClassLoader loader=extensionCache.getClassLoaderById(extensionConfig.getExtensions().get(0).getEnv()+":"+extensionConfig.getExtensions().get(0).getId()); Assertions.assertNotNull(loader); Assertions.assertEquals(classLoader,loader); } From 1befc37be24dd8da48ddefaf348bff937b0a168d Mon Sep 17 00:00:00 2001 From: lakde Date: Wed, 18 Sep 2024 09:32:57 -0500 Subject: [PATCH 03/21] Review Fixes Signed-off-by: lakde --- gradle.properties | 2 +- .../src/main/resources/swagger/openapi.yaml | 2 +- .../src/main/java/com/here/naksha/lib/hub/NakshaHub.java | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/gradle.properties b/gradle.properties index 98293820d..92a7e516e 100644 --- a/gradle.properties +++ b/gradle.properties @@ -11,4 +11,4 @@ mavenPassword=YourPassword # - here-naksha-lib-core/NakshaVersion (static property: latest) # - here-naksha-lib-psql/resources/naksha_plpgsql.sql (method: naksha_version) # - here-naksha-app-service/src/main/resources/swagger/openapi.yaml (info.version property) -version=2.1.0 +version=2.2.0 diff --git a/here-naksha-app-service/src/main/resources/swagger/openapi.yaml b/here-naksha-app-service/src/main/resources/swagger/openapi.yaml index 915b3ed00..5a080a407 100644 --- a/here-naksha-app-service/src/main/resources/swagger/openapi.yaml +++ b/here-naksha-app-service/src/main/resources/swagger/openapi.yaml @@ -12,7 +12,7 @@ servers: info: title: "Naskha Hub-API" description: "Naksha Hub-API is a REST API to provide simple access to geo data." - version: "2.1.0" + version: "2.2.0" security: - AccessToken: [ ] diff --git a/here-naksha-lib-hub/src/main/java/com/here/naksha/lib/hub/NakshaHub.java b/here-naksha-lib-hub/src/main/java/com/here/naksha/lib/hub/NakshaHub.java index 06c183600..a6f453bf2 100644 --- a/here-naksha-lib-hub/src/main/java/com/here/naksha/lib/hub/NakshaHub.java +++ b/here-naksha-lib-hub/src/main/java/com/here/naksha/lib/hub/NakshaHub.java @@ -307,8 +307,8 @@ private static WriteXyzCollections createAdminCollectionsRequest() { try (IReadSession readSession = getAdminStorage().newReadSession(nakshaContext, false)) { rdResult = readSession.execute(request); } catch (Exception e) { - logger.error("Failed reading from collections {}. ", request.getCollections(), e); - throw new RuntimeException("Failed to execute single read session", e); + logger.error("Failed during reading extension handler configurations from collections {}. ", request.getCollections(), e); + throw new RuntimeException("Failed reading extension handler configurations", e); } final List eventHandlers; try { From 58995d321ea67e817a1681e214a11abfa7883c38 Mon Sep 17 00:00:00 2001 From: Rohit Lakde <37445331+rlakde@users.noreply.github.com> Date: Fri, 25 Oct 2024 08:56:14 -0700 Subject: [PATCH 04/21] Update openapi.yaml --- here-naksha-app-service/src/main/resources/swagger/openapi.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/here-naksha-app-service/src/main/resources/swagger/openapi.yaml b/here-naksha-app-service/src/main/resources/swagger/openapi.yaml index 4cf34add4..55ee704f1 100644 --- a/here-naksha-app-service/src/main/resources/swagger/openapi.yaml +++ b/here-naksha-app-service/src/main/resources/swagger/openapi.yaml @@ -12,7 +12,7 @@ servers: info: title: "Naskha Hub-API" description: "Naksha Hub-API is a REST API to provide simple access to geo data." - version: "2.2.0" + version: "2.1.5" security: - AccessToken: [ ] From 7ec0f8624f9b7d58a3ba560cfe737c0c76a4612d Mon Sep 17 00:00:00 2001 From: Rohit Lakde <37445331+rlakde@users.noreply.github.com> Date: Fri, 25 Oct 2024 08:56:40 -0700 Subject: [PATCH 05/21] Update gradle.properties --- gradle.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle.properties b/gradle.properties index 20ca314ce..793f190a6 100644 --- a/gradle.properties +++ b/gradle.properties @@ -12,4 +12,4 @@ mavenPassword=YourPassword # - here-naksha-lib-psql/resources/naksha_plpgsql.sql (method: naksha_version) # - here-naksha-app-service/src/main/resources/swagger/openapi.yaml (info.version property) -version=2.2.0 +version=2.1.5 From acc577027755159a4ff7c2706190f4bc2e35959f Mon Sep 17 00:00:00 2001 From: Rohit Lakde <37445331+rlakde@users.noreply.github.com> Date: Fri, 25 Oct 2024 08:58:41 -0700 Subject: [PATCH 06/21] Update gradle.properties --- gradle.properties | 1 - 1 file changed, 1 deletion(-) diff --git a/gradle.properties b/gradle.properties index 793f190a6..9b66cc554 100644 --- a/gradle.properties +++ b/gradle.properties @@ -11,5 +11,4 @@ mavenPassword=YourPassword # - here-naksha-lib-core/NakshaVersion (static property: latest) # - here-naksha-lib-psql/resources/naksha_plpgsql.sql (method: naksha_version) # - here-naksha-app-service/src/main/resources/swagger/openapi.yaml (info.version property) - version=2.1.5 From b07480f42478b0a3c79be8c03b3409a7f0d6120f Mon Sep 17 00:00:00 2001 From: Rohit Lakde <37445331+rlakde@users.noreply.github.com> Date: Fri, 25 Oct 2024 08:59:17 -0700 Subject: [PATCH 07/21] Rebase CASL-321 (#371) * Fixed array patching (#365) * Fixed array patching * corrected patcher test expectation * Fix thread stuck issue + addtnl logs (#368) * Added logs to troubleshoot thread hanging issue * CASL-258 unreleased lock fix (#358) * Fixed array patching (#365) (#367) * corrected patcher test expectation * updated version * updated changelog --------- Co-authored-by: Pawel Mazurek <52866094+cyberhead-pl@users.noreply.github.com> --------- Co-authored-by: Hiren Patel <80465571+hirenkp2000@users.noreply.github.com> Co-authored-by: Pawel Mazurek <52866094+cyberhead-pl@users.noreply.github.com> --- CHANGELOG.md | 10 ++ .../http/tasks/WriteFeatureApiTask.java | 4 +- .../src/main/resources/swagger/openapi.yaml | 2 +- .../here/naksha/lib/core/AbstractTask.java | 17 +++ .../here/naksha/lib/core/NakshaContext.java | 1 + .../naksha/lib/core/storage/IStorage.java | 8 +- .../com/here/naksha/lib/core/util/FibMap.java | 12 ++ .../lib/core/util/diff/PatcherUtils.java | 27 +++-- .../core/util/fib/FibLinearProbeTable.java | 2 +- .../here/naksha/lib/core/util/fib/FibSet.java | 12 ++ .../lib/core/util/json/JsonFieldBool.java | 5 + .../lib/core/util/json/JsonFieldByte.java | 5 + .../lib/core/util/json/JsonFieldChar.java | 5 + .../lib/core/util/json/JsonFieldShort.java | 5 + .../naksha/lib/core/util/json/JsonMap.java | 9 ++ .../lib/core/util/diff/PatcherTest.java | 33 +++++- .../util/fib/FibLinearProbeTableTest.java | 41 +++++++ .../feature_3_patched_to_4_no_remove.json | 4 +- .../resources/patcher/topology/existing.json | 109 ++++++++++++++++++ .../resources/patcher/topology/expected.json | 109 ++++++++++++++++++ .../resources/patcher/topology/input.json | 92 +++++++++++++++ 21 files changed, 487 insertions(+), 25 deletions(-) create mode 100644 here-naksha-lib-core/src/test/resources/patcher/topology/existing.json create mode 100644 here-naksha-lib-core/src/test/resources/patcher/topology/expected.json create mode 100644 here-naksha-lib-core/src/test/resources/patcher/topology/input.json diff --git a/CHANGELOG.md b/CHANGELOG.md index c7922bd32..ec3443aaf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,13 @@ +## Naksha_2.1.5 + +- Fixed thread hanging issue fixed by avoiding indefinite locking in `FibLinearProbeTable.java` + +## Naksha_2.1.4 + +- Fixes: + - Patch API (POST /features) fixed to `replace` entire array instead of `append` nodes during patch operation, which otherwise prevents removal of the node even-though is desired + + ## Naksha_1.1.1 - Fixes: diff --git a/here-naksha-app-service/src/main/java/com/here/naksha/app/service/http/tasks/WriteFeatureApiTask.java b/here-naksha-app-service/src/main/java/com/here/naksha/app/service/http/tasks/WriteFeatureApiTask.java index 628630f59..4ab53e69e 100644 --- a/here-naksha-app-service/src/main/java/com/here/naksha/app/service/http/tasks/WriteFeatureApiTask.java +++ b/here-naksha-app-service/src/main/java/com/here/naksha/app/service/http/tasks/WriteFeatureApiTask.java @@ -20,7 +20,7 @@ import static com.here.naksha.app.service.http.apis.ApiParams.*; import static com.here.naksha.common.http.apis.ApiParamsConst.*; -import static com.here.naksha.lib.core.util.diff.PatcherUtils.removeAllRemoveOp; +import static com.here.naksha.lib.core.util.diff.PatcherUtils.removeAllRemoveOpExceptForList; import static com.here.naksha.lib.core.util.storage.ResultHelper.readFeaturesFromResult; import com.fasterxml.jackson.core.JsonProcessingException; @@ -389,7 +389,7 @@ private List performInMemoryPatching( if (inputFeature.getId().equals(storageFeature.getId())) { // we found matching feature in storage, so we take patched version of the feature final Difference difference = Patcher.getDifference(storageFeature, inputFeature); - final Difference diffNoRemoveOp = removeAllRemoveOp(difference); + final Difference diffNoRemoveOp = removeAllRemoveOpExceptForList(difference); featureToPatch = Patcher.patch(storageFeature, diffNoRemoveOp); break; } diff --git a/here-naksha-app-service/src/main/resources/swagger/openapi.yaml b/here-naksha-app-service/src/main/resources/swagger/openapi.yaml index 55ee704f1..f3b01152a 100644 --- a/here-naksha-app-service/src/main/resources/swagger/openapi.yaml +++ b/here-naksha-app-service/src/main/resources/swagger/openapi.yaml @@ -10,7 +10,7 @@ servers: - url: "https://naksha-v2.ext.mapcreator.here.com/" description: "PRD" info: - title: "Naskha Hub-API" + title: "Naksha Hub-API" description: "Naksha Hub-API is a REST API to provide simple access to geo data." version: "2.1.5" diff --git a/here-naksha-lib-core/src/main/java/com/here/naksha/lib/core/AbstractTask.java b/here-naksha-lib-core/src/main/java/com/here/naksha/lib/core/AbstractTask.java index b0c7c7503..949eb0476 100644 --- a/here-naksha-lib-core/src/main/java/com/here/naksha/lib/core/AbstractTask.java +++ b/here-naksha-lib-core/src/main/java/com/here/naksha/lib/core/AbstractTask.java @@ -613,6 +613,9 @@ private void incInstanceLevelUsage(String actorId, long limit) { break; } // Failed, conflict, repeat + log.info( + "Concurrency conflict while incrementing instance level threadCount from {}. Will retry...", + threadCount); } } @@ -649,6 +652,9 @@ private void incActorLevelUsage(String actorId, long limit) { if (counter == null) { Long existing = actorUsageMap.putIfAbsent(actorId, 1L); if (existing != null) { + log.info( + "Concurrency conflict while initializing threadCount to 1 for actorId [{}]. Will retry...", + actorId); continue; // Repeat, conflict with other thread } return; @@ -669,6 +675,10 @@ private void incActorLevelUsage(String actorId, long limit) { break; } // Failed, conflict, repeat + log.info( + "Concurrency conflict while incrementing actor level threadCount from {} for actorId [{}]. Will retry...", + counter, + actorId); } } @@ -693,6 +703,9 @@ private void decActorLevelUsage(String actorId) { log.error("Invalid actor usage value for actor: " + actorId + " value: " + current); } if (!actorUsageMap.remove(actorId, current)) { + log.info( + "Concurrency conflict while removing actor level threadCount for actorId [{}]. Will retry...", + actorId); continue; } break; @@ -700,6 +713,10 @@ private void decActorLevelUsage(String actorId) { break; } // Failed, repeat, conflict with other thread + log.info( + "Concurrency conflict while decrementing actor level threadCount from {} for actorId [{}]. Will retry...", + current, + actorId); } } } diff --git a/here-naksha-lib-core/src/main/java/com/here/naksha/lib/core/NakshaContext.java b/here-naksha-lib-core/src/main/java/com/here/naksha/lib/core/NakshaContext.java index d1302cd25..ff3062dcb 100644 --- a/here-naksha-lib-core/src/main/java/com/here/naksha/lib/core/NakshaContext.java +++ b/here-naksha-lib-core/src/main/java/com/here/naksha/lib/core/NakshaContext.java @@ -221,6 +221,7 @@ public long startNanos() { return newValue; } // Conflict, two threads seem to want to update the same key the same time! + logger.info("Concurrency conflict while updating attachment map for key {}", valueClass); } } diff --git a/here-naksha-lib-core/src/main/java/com/here/naksha/lib/core/storage/IStorage.java b/here-naksha-lib-core/src/main/java/com/here/naksha/lib/core/storage/IStorage.java index f0313a810..c0043bc84 100644 --- a/here-naksha-lib-core/src/main/java/com/here/naksha/lib/core/storage/IStorage.java +++ b/here-naksha-lib-core/src/main/java/com/here/naksha/lib/core/storage/IStorage.java @@ -32,6 +32,8 @@ import org.jetbrains.annotations.ApiStatus.AvailableSince; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * Storage API to gain access to storages. @@ -39,6 +41,8 @@ @AvailableSince(NakshaVersion.v2_0_6) public interface IStorage extends AutoCloseable { + Logger logger = LoggerFactory.getLogger(IStorage.class); + /** * Initializes the storage, create the transaction table, install needed scripts and extensions. * @@ -191,10 +195,12 @@ default void close() { try { shutdown(null).get(); return; - } catch (InterruptedException ignore) { + } catch (InterruptedException ie) { + logger.warn("Exception while shutting down IStorage. ", ie); } catch (Exception e) { throw unchecked(e); } + logger.info("Unable to shutdown IStorage. Will retry..."); } } diff --git a/here-naksha-lib-core/src/main/java/com/here/naksha/lib/core/util/FibMap.java b/here-naksha-lib-core/src/main/java/com/here/naksha/lib/core/util/FibMap.java index 904298d01..444fbbd96 100644 --- a/here-naksha-lib-core/src/main/java/com/here/naksha/lib/core/util/FibMap.java +++ b/here-naksha-lib-core/src/main/java/com/here/naksha/lib/core/util/FibMap.java @@ -29,6 +29,8 @@ import java.util.List; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * Functional implementation of a recursive, thread safe, growing map, based upon iterator = listdiff.iterator(); - while (iterator.hasNext()) { - Difference next = iterator.next(); - if (next == null) continue; - next = removeAllRemoveOp(next); - if (next == null) iterator.remove(); - } - return listdiff; } else if (difference instanceof MapDiff) { final MapDiff mapdiff = (MapDiff) difference; final Iterator> iterator = mapdiff.entrySet().iterator(); while (iterator.hasNext()) { Entry next = iterator.next(); - next.setValue(removeAllRemoveOp(next.getValue())); + next.setValue(removeAllRemoveOpExceptForList(next.getValue())); if (next.getValue() == null) iterator.remove(); } return mapdiff; } + // NOTE - we avoid removal of nodes for ListDiff, to ensure List is always replaced during patch and not + // appended + /*else if (difference instanceof ListDiff) { + final ListDiff listdiff = (ListDiff) difference; + final Iterator iterator = listdiff.iterator(); + while (iterator.hasNext()) { + Difference next = iterator.next(); + if (next == null) continue; + next = removeAllRemoveOpExceptForList(next); + if (next == null) iterator.remove(); + } + return listdiff; + }*/ return difference; } } diff --git a/here-naksha-lib-core/src/main/java/com/here/naksha/lib/core/util/fib/FibLinearProbeTable.java b/here-naksha-lib-core/src/main/java/com/here/naksha/lib/core/util/fib/FibLinearProbeTable.java index a9f39515a..d74d70706 100644 --- a/here-naksha-lib-core/src/main/java/com/here/naksha/lib/core/util/fib/FibLinearProbeTable.java +++ b/here-naksha-lib-core/src/main/java/com/here/naksha/lib/core/util/fib/FibLinearProbeTable.java @@ -82,7 +82,7 @@ ENTRY execute(final @NotNull FibSetOp op, final @NotNull KEY key, final @NotNull if (raw instanceof Reference) { Reference ref = (Reference) raw; entry = (ENTRY) ref.get(); - if (entry == null && lock.tryLock()) { + if (entry == null && (lock.isHeldByCurrentThread() || lock.tryLock())) { locked = true; array[i] = null; SIZE.getAndAdd(fibSet, -1L); diff --git a/here-naksha-lib-core/src/main/java/com/here/naksha/lib/core/util/fib/FibSet.java b/here-naksha-lib-core/src/main/java/com/here/naksha/lib/core/util/fib/FibSet.java index 7033ad47c..928e31af1 100644 --- a/here-naksha-lib-core/src/main/java/com/here/naksha/lib/core/util/fib/FibSet.java +++ b/here-naksha-lib-core/src/main/java/com/here/naksha/lib/core/util/fib/FibSet.java @@ -37,6 +37,8 @@ import org.jetbrains.annotations.ApiStatus.AvailableSince; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * A recursive, thread safe, weak/soft/strong referencing set, based upon > { + private static final Logger logger = LoggerFactory.getLogger(FibSet.class); + /** * The empty array used by default, so that empty maps do not consume memory. */ @@ -452,6 +456,9 @@ ENTRY _execute( if (raw_entry == null) { if (!ARRAY.compareAndSet(array, index, ref, null)) { // Race condition, another thread modified the array slot. + logger.info( + "Concurrency conflict while initializing array value at index {}. Will retry...", + index); continue; } SIZE.getAndAdd(this, -1L); @@ -487,6 +494,7 @@ ENTRY _execute( return (ENTRY) entry; } // Race condition, other thread updated the reference concurrently. + logger.info("Concurrency conflict while setting array value at index {}. Will retry...", index); continue; } assert op == REMOVE; @@ -495,6 +503,7 @@ ENTRY _execute( return (ENTRY) entry; } // Race condition, other thread updated the reference concurrently. + logger.info("Concurrency conflict while nullifying array value at index {}. Will retry...", index); continue; } @@ -516,6 +525,7 @@ ENTRY _execute( return _execute(op, key, key_hash, refType, sub_array, depth + 1); } // Race condition, another thread modified concurrently. + logger.info("Concurrency conflict while setting array value at index {}. Will retry...", index); continue; } @@ -529,6 +539,7 @@ ENTRY _execute( return new_entry; } // Race condition, another thread modified concurrently. + logger.info("Concurrency conflict while setting array value at index {}. Will retry...", index); continue; } @@ -546,6 +557,7 @@ ENTRY _execute( return new_entry; } // Race condition, another thread modified concurrently. + logger.info("Concurrency conflict while initializing array value at index {}. Will retry...", index); } } diff --git a/here-naksha-lib-core/src/main/java/com/here/naksha/lib/core/util/json/JsonFieldBool.java b/here-naksha-lib-core/src/main/java/com/here/naksha/lib/core/util/json/JsonFieldBool.java index f0a96c6c5..61e056c75 100644 --- a/here-naksha-lib-core/src/main/java/com/here/naksha/lib/core/util/json/JsonFieldBool.java +++ b/here-naksha-lib-core/src/main/java/com/here/naksha/lib/core/util/json/JsonFieldBool.java @@ -23,9 +23,13 @@ import java.nio.ByteOrder; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public final class JsonFieldBool extends JsonField { + private static final Logger logger = LoggerFactory.getLogger(JsonFieldBool.class); + JsonFieldBool( @NotNull JsonClass jsonClass, @NotNull Field javaField, @@ -94,6 +98,7 @@ public boolean _compareAndSwap(@NotNull OBJECT object, Boolean expected, Boolean return true; } // We need to loop, because possibly some code modified bytes we're not interested in. + logger.info("Concurrency conflict while setting value at offset {}. Will retry...", offset); } } diff --git a/here-naksha-lib-core/src/main/java/com/here/naksha/lib/core/util/json/JsonFieldByte.java b/here-naksha-lib-core/src/main/java/com/here/naksha/lib/core/util/json/JsonFieldByte.java index ce17f34b1..b84c07e0b 100644 --- a/here-naksha-lib-core/src/main/java/com/here/naksha/lib/core/util/json/JsonFieldByte.java +++ b/here-naksha-lib-core/src/main/java/com/here/naksha/lib/core/util/json/JsonFieldByte.java @@ -23,9 +23,13 @@ import java.nio.ByteOrder; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public final class JsonFieldByte extends JsonField { + private static final Logger logger = LoggerFactory.getLogger(JsonFieldByte.class); + JsonFieldByte( @NotNull JsonClass jsonClass, @NotNull Field javaField, @@ -100,6 +104,7 @@ public boolean _compareAndSwap(@NotNull OBJECT object, Byte expected, Byte value return true; } // We need to loop, because possibly some code modified bytes we're not interested in. + logger.info("Concurrency conflict while setting value at offset {}. Will retry...", offset); } } diff --git a/here-naksha-lib-core/src/main/java/com/here/naksha/lib/core/util/json/JsonFieldChar.java b/here-naksha-lib-core/src/main/java/com/here/naksha/lib/core/util/json/JsonFieldChar.java index fe42df8e9..72cb31fe2 100644 --- a/here-naksha-lib-core/src/main/java/com/here/naksha/lib/core/util/json/JsonFieldChar.java +++ b/here-naksha-lib-core/src/main/java/com/here/naksha/lib/core/util/json/JsonFieldChar.java @@ -23,9 +23,13 @@ import java.nio.ByteOrder; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public final class JsonFieldChar extends JsonField { + private static final Logger logger = LoggerFactory.getLogger(JsonFieldChar.class); + JsonFieldChar( @NotNull JsonClass jsonClass, @NotNull Field javaField, @@ -101,6 +105,7 @@ public boolean _compareAndSwap(@NotNull OBJECT object, Character expected, Chara return true; } // We need to loop, because possibly some code modified bytes we're not interested in. + logger.info("Concurrency conflict while setting value at offset {}. Will retry...", offset); } } diff --git a/here-naksha-lib-core/src/main/java/com/here/naksha/lib/core/util/json/JsonFieldShort.java b/here-naksha-lib-core/src/main/java/com/here/naksha/lib/core/util/json/JsonFieldShort.java index e47d7059b..408d78de6 100644 --- a/here-naksha-lib-core/src/main/java/com/here/naksha/lib/core/util/json/JsonFieldShort.java +++ b/here-naksha-lib-core/src/main/java/com/here/naksha/lib/core/util/json/JsonFieldShort.java @@ -23,9 +23,13 @@ import java.nio.ByteOrder; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public final class JsonFieldShort extends JsonField { + private static final Logger logger = LoggerFactory.getLogger(JsonFieldShort.class); + JsonFieldShort( @NotNull JsonClass jsonClass, @NotNull Field javaField, @@ -102,6 +106,7 @@ public boolean _compareAndSwap(@NotNull OBJECT object, Short expected, Short val return true; } // We need to loop, because possibly some code modified bytes we're not interested in. + logger.info("Concurrency conflict while setting value at offset {}. Will retry...", offset); } } diff --git a/here-naksha-lib-core/src/main/java/com/here/naksha/lib/core/util/json/JsonMap.java b/here-naksha-lib-core/src/main/java/com/here/naksha/lib/core/util/json/JsonMap.java index 9129ef13c..d689e1872 100644 --- a/here-naksha-lib-core/src/main/java/com/here/naksha/lib/core/util/json/JsonMap.java +++ b/here-naksha-lib-core/src/main/java/com/here/naksha/lib/core/util/json/JsonMap.java @@ -42,6 +42,8 @@ import org.jetbrains.annotations.ApiStatus.AvailableSince; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * A map that uses {@link String} key and arbitrary values. The map is thread safe for concurrent @@ -55,6 +57,8 @@ public class JsonMap implements Map<@NotNull String, @Nullable Object>, Iterable> { + private static final Logger logger = LoggerFactory.getLogger(JsonMap.class); + /** * Create a new empty map. * @@ -278,6 +282,7 @@ public boolean containsKey(@Nullable Object key) { final Object result = FibMap.put(key, oldValue, newValue, true, rootMutable(), this::intern, this::conflict); if (result instanceof FibMapConflict) { + logger.info("Concurrency conflict while setting value for key {}. Will retry...", key); continue; } assert result == oldValue; @@ -305,6 +310,7 @@ public boolean containsKey(@Nullable Object key) { final Object result = FibMap.put(key, original, newValue, true, rootMutable(), this::intern, this::conflict); if (result instanceof FibMapConflict) { + logger.info("Concurrency conflict while setting value for key {}. Will retry...", key); continue; } assert result == original; @@ -339,6 +345,7 @@ public boolean containsKey(@Nullable Object key) { final Object result = FibMap.put(key, original, UNDEFINED, true, rootMutable(), this::intern, this::conflict); if (result instanceof FibMapConflict) { + logger.info("Concurrency conflict while removing value for key {}. Will retry...", key); continue; } assert result == UNDEFINED; @@ -350,6 +357,7 @@ public boolean containsKey(@Nullable Object key) { final Object result = FibMap.put(key, original, newValue, true, rootMutable(), this::intern, this::conflict); if (result instanceof FibMapConflict) { + logger.info("Concurrency conflict while setting value for key {}. Will retry...", key); continue; } assert result == original; @@ -468,6 +476,7 @@ public void clear() { SIZE.getAndAdd(this, -oldSize); return; } + logger.info("Concurrency conflict while clearing map. Will retry..."); } } diff --git a/here-naksha-lib-core/src/test/java/com/here/naksha/lib/core/util/diff/PatcherTest.java b/here-naksha-lib-core/src/test/java/com/here/naksha/lib/core/util/diff/PatcherTest.java index 4b0955425..c30fc5322 100644 --- a/here-naksha-lib-core/src/test/java/com/here/naksha/lib/core/util/diff/PatcherTest.java +++ b/here-naksha-lib-core/src/test/java/com/here/naksha/lib/core/util/diff/PatcherTest.java @@ -24,15 +24,16 @@ import com.here.naksha.lib.core.util.IoHelp; import com.here.naksha.lib.core.util.json.JsonObject; import com.here.naksha.lib.core.util.json.JsonSerializable; -import java.util.ArrayList; + import java.util.Arrays; import java.util.List; import java.util.stream.Stream; + +import com.here.naksha.test.common.FileUtil; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; import org.json.JSONException; import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Named; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; @@ -42,7 +43,7 @@ import java.util.Map; -import static com.here.naksha.lib.core.util.diff.PatcherUtils.removeAllRemoveOp; +import static com.here.naksha.lib.core.util.diff.PatcherUtils.removeAllRemoveOpExceptForList; import static org.junit.jupiter.api.Assertions.*; import static org.junit.jupiter.params.provider.Arguments.arguments; @@ -113,7 +114,7 @@ void testCompareBasicNestedJson() throws JSONException { assertTrue(((MapDiff) nestedArrayDiff34.get(2)).get("willBeDeletedProperty") instanceof RemoveOp); // Modify the whole difference to get rid of all RemoveOp - Difference newDiff34 = removeAllRemoveOp(mapDiff34); + Difference newDiff34 = removeAllRemoveOpExceptForList(mapDiff34); final JsonObject patchedf3 = Patcher.patch(f3,newDiff34); assertNotNull(patchedf3); @@ -158,6 +159,28 @@ void testCompareSameArrayDifferentOrder() throws JSONException { assertNull(newDiff); } + @Test + void testCompareArrayRemovalOfNode() throws JSONException { + final XyzFeature input = + FileUtil.parseJsonFileOrFail("src/test/resources/", "patcher/topology/input.json", XyzFeature.class); + //JsonSerializable.deserialize(IoHelp.readResource("patcher/topology/input.json"), XyzFeature.class); + assertNotNull(input); + final XyzFeature existing = + FileUtil.parseJsonFileOrFail("src/test/resources/", "patcher/topology/existing.json", XyzFeature.class); + //JsonSerializable.deserialize(IoHelp.readResource("patcher/topology/existing.json"), XyzFeature.class); + assertNotNull(existing); + final XyzFeature expected = + FileUtil.parseJsonFileOrFail("src/test/resources/", "patcher/topology/expected.json", XyzFeature.class); + assertNotNull(expected); + + final Difference difference = Patcher.getDifference(existing, input); + final Difference diffNoRemoveOp = removeAllRemoveOpExceptForList(difference); + final XyzFeature patchedFeature = Patcher.patch(existing, diffNoRemoveOp); + assertNotNull(patchedFeature); + + JSONAssert.assertEquals(expected.toString(), patchedFeature.toString(), JSONCompareMode.STRICT); + } + @Test void testPatchingOnlyShuffledArrayProvided() throws JSONException { final JsonObject f3 = @@ -171,7 +194,7 @@ void testPatchingOnlyShuffledArrayProvided() throws JSONException { final Difference diff36 = Patcher.getDifference(f3, f6); assertNotNull(diff36); // Simulate REST API behaviour, ignore all RemoveOp type of Difference - final Difference diff36NoRemove = removeAllRemoveOp(diff36); + final Difference diff36NoRemove = removeAllRemoveOpExceptForList(diff36); final JsonObject patchedf3Tof6 = Patcher.patch(f3, diff36NoRemove); final JsonObject expectedPatchedf3 = diff --git a/here-naksha-lib-core/src/test/java/com/here/naksha/lib/core/util/fib/FibLinearProbeTableTest.java b/here-naksha-lib-core/src/test/java/com/here/naksha/lib/core/util/fib/FibLinearProbeTableTest.java index 1e93c2339..a78bcdf73 100644 --- a/here-naksha-lib-core/src/test/java/com/here/naksha/lib/core/util/fib/FibLinearProbeTableTest.java +++ b/here-naksha-lib-core/src/test/java/com/here/naksha/lib/core/util/fib/FibLinearProbeTableTest.java @@ -29,9 +29,16 @@ import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.fail; import java.lang.ref.SoftReference; import java.lang.ref.WeakReference; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import org.junit.jupiter.api.Test; class FibLinearProbeTableTest { @@ -151,4 +158,38 @@ void test_expansion() { assertEquals(TOTAL_SIZE - i, SET.size); } } + + @Test + void test_Lock() { + final FibSet> SET = new FibSet<>(FibMapEntry::new); + final FibLinearProbeTable> lpt = new FibLinearProbeTable<>(SET, 0); + + /** + * First we put 2 weak references to lpt, then we call gc() - in such scenario next GET call should remove + * all empty references, but to do this it has to acquire lock, and release it at the end. + */ + + lpt.execute(PUT, "foo", WEAK); + lpt.execute(PUT, "foo1", WEAK); + System.gc(); + lpt.execute(GET, "foo1", WEAK); + + // then + ExecutorService executor = Executors.newSingleThreadExecutor(); + Future future = executor.submit(() -> { + lpt.execute(PUT, "foo3", WEAK); + return "done"; + }); + + /** + * Now we try to put new value to lpt in another thread - if previous locks were not released it should + * throw timeout exception. + */ + try { + future.get(1, TimeUnit.SECONDS); + } catch (TimeoutException | InterruptedException | ExecutionException e) { + fail("lock not released! " + e); + } + executor.shutdownNow(); + } } diff --git a/here-naksha-lib-core/src/test/resources/patcher/feature_3_patched_to_4_no_remove.json b/here-naksha-lib-core/src/test/resources/patcher/feature_3_patched_to_4_no_remove.json index ef6767850..a5d12d73b 100644 --- a/here-naksha-lib-core/src/test/resources/patcher/feature_3_patched_to_4_no_remove.json +++ b/here-naksha-lib-core/src/test/resources/patcher/feature_3_patched_to_4_no_remove.json @@ -15,10 +15,8 @@ { "id": "element1", "nestedShouldBeUpdated": "updated", - "willBeDeletedProperty": "not yet", "isAddedProperty": "yes" - }, - "willBeDeletedElement" + } ], "speedLimit": [ { diff --git a/here-naksha-lib-core/src/test/resources/patcher/topology/existing.json b/here-naksha-lib-core/src/test/resources/patcher/topology/existing.json new file mode 100644 index 000000000..57f6ae503 --- /dev/null +++ b/here-naksha-lib-core/src/test/resources/patcher/topology/existing.json @@ -0,0 +1,109 @@ +{ + "id": "778407118", + "bbox": [ + 14.20594, + 50.02377, + 14.20667, + 50.02386 + ], + "type": "Feature", + "momType": "Topology", + "properties": { + "rightAdmin": [ + { + "range": { + "endOffset": 1.0, + "startOffset": 0.0 + }, + "value": { + "id": "urn:here::here:Admin:20445433", + "featureType": "Admin", + "referenceName": "adminId" + }, + "confidence": { + "score": 0.0, + "scoreType": "EXISTENCE", + "updatedOn": "2021-02-17T21:44:27.863Z", + "featureType": "ROAD_ADMIN_INFORMATION", + "updateSource": "INJECTION_TMOB" + } + } + ], + "isLongHaul": [ + { + "range": { + "endOffset": 1.0, + "startOffset": 0.0 + }, + "value": false + } + ], + "isMotorway": [ + { + "range": { + "endOffset": 1, + "startOffset": 0 + }, + "value": false, + "confidence": { + "score": 0.9, + "sources": [ + "BMW_RSD" + ], + "scoreType": "GENERIC", + "updatedOn": "2024-10-15T16:39:51.329Z", + "featureType": "UNDEFINED", + "updateSource": "MODERATED", + "triggeringReferences": [ + { + "momFeatureType": "LogicalRoadSign", + "genericFeatureId": "urn:here::here:logicalroadsign:1622677077480690247" + } + ] + } + }, + { + "range": { + "endOffset": 1.0, + "startOffset": 0.6918789974069862 + }, + "value": true, + "confidence": { + "score": 0.75, + "sources": [ + "BMW_RSD" + ], + "scoreType": "GENERIC", + "updatedOn": "2024-10-05T20:59:55.69448Z", + "featureType": "UNDEFINED", + "updateSource": "WALLE", + "triggeringReferences": [ + { + "momFeatureType": "LogicalRoadSign", + "genericFeatureId": "urn:here::here:logicalroadsign:1622677077480690247" + }, + { + "momFeatureType": "LogicalRoadSign", + "genericFeatureId": "urn:here::here:logicalroadsign:1622677077990856712" + } + ] + } + }, + { + "range": { + "endOffset": 0.6873084947544703, + "startOffset": 0.0 + }, + "value": false + } + ], + "referencePoint": { + "type": "Point", + "coordinates": [ + 14.20667, + 50.02379, + 416.38 + ] + } + } +} \ No newline at end of file diff --git a/here-naksha-lib-core/src/test/resources/patcher/topology/expected.json b/here-naksha-lib-core/src/test/resources/patcher/topology/expected.json new file mode 100644 index 000000000..390e16cb6 --- /dev/null +++ b/here-naksha-lib-core/src/test/resources/patcher/topology/expected.json @@ -0,0 +1,109 @@ +{ + "momType": "Topology", + "referencePoint": { + "type": "Point", + "coordinates": [ + 14.20667, + 50.02379, + 416.38 + ] + }, + "id": "778407118", + "type": "Feature", + "bbox": [ + 14.20594, + 50.02377, + 14.20667, + 50.02386 + ], + "properties": { + "isLongHaul": [ + { + "range": { + "endOffset": 1.0, + "startOffset": 0.0 + }, + "value": false + } + ], + "isMotorway": [ + { + "range": { + "endOffset": 1, + "startOffset": 0 + }, + "value": false, + "confidence": { + "score": 0.75, + "sources": [ + "BMW_RSD" + ], + "scoreType": "GENERIC", + "updatedOn": "2024-10-15T16:39:51.329Z", + "featureType": "UNDEFINED", + "updateSource": "MODERATED", + "triggeringReferences": [ + { + "momFeatureType": "LogicalRoadSign", + "genericFeatureId": "urn:here::here:logicalroadsign:1622677077480690247" + } + ] + } + } + ], + "rightAdmin": [ + { + "range": { + "endOffset": 1, + "startOffset": 0 + }, + "value": { + "id": "urn:here::here:Admin:20445433", + "featureType": "Admin", + "referenceName": "adminId" + }, + "confidence": { + "score": 0, + "scoreType": "EXISTENCE", + "updatedOn": "2021-02-17T21:44:27.863Z", + "featureType": "ROAD_ADMIN_INFORMATION", + "updateSource": "INJECTION_TMOB" + } + } + ], + "referencePoint": { + "type": "Point", + "coordinates": [ + 14.20667, + 50.02379, + 416.38 + ] + } + }, + "geometry": { + "type": "LineString", + "coordinates": [ + [ + 14.20667, + 50.02379, + 416.38 + ], + [ + 14.20636, + 50.02377, + 417.17 + ], + [ + 14.20617, + 50.02379, + 417.38 + ], + [ + 14.20594, + 50.02386, + 417.16 + ] + ] + }, + "class": "NAVLINK" +} \ No newline at end of file diff --git a/here-naksha-lib-core/src/test/resources/patcher/topology/input.json b/here-naksha-lib-core/src/test/resources/patcher/topology/input.json new file mode 100644 index 000000000..bfa527fa4 --- /dev/null +++ b/here-naksha-lib-core/src/test/resources/patcher/topology/input.json @@ -0,0 +1,92 @@ +{ + "momType": "Topology", + "referencePoint": { + "type": "Point", + "coordinates": [ + 14.20667, + 50.02379, + 416.38 + ] + }, + "id": "778407118", + "type": "Feature", + "bbox": [ + 14.20594, + 50.02377, + 14.20667, + 50.02386 + ], + "properties": { + "isMotorway": [ + { + "range": { + "endOffset": 1, + "startOffset": 0 + }, + "value": false, + "confidence": { + "score": 0.75, + "sources": [ + "BMW_RSD" + ], + "scoreType": "GENERIC", + "updatedOn": "2024-10-15T16:39:51.329Z", + "featureType": "UNDEFINED", + "updateSource": "MODERATED", + "triggeringReferences": [ + { + "momFeatureType": "LogicalRoadSign", + "genericFeatureId": "urn:here::here:logicalroadsign:1622677077480690247" + } + ] + } + } + ], + "rightAdmin": [ + { + "range": { + "endOffset": 1, + "startOffset": 0 + }, + "value": { + "id": "urn:here::here:Admin:20445433", + "featureType": "Admin", + "referenceName": "adminId" + }, + "confidence": { + "score": 0, + "scoreType": "EXISTENCE", + "updatedOn": "2021-02-17T21:44:27.863Z", + "featureType": "ROAD_ADMIN_INFORMATION", + "updateSource": "INJECTION_TMOB" + } + } + ] + }, + "geometry": { + "type": "LineString", + "coordinates": [ + [ + 14.20667, + 50.02379, + 416.38 + ], + [ + 14.20636, + 50.02377, + 417.17 + ], + [ + 14.20617, + 50.02379, + 417.38 + ], + [ + 14.20594, + 50.02386, + 417.16 + ] + ] + }, + "class": "NAVLINK" +} \ No newline at end of file From 3e55ff93438756b275577163e144df46ce178fde Mon Sep 17 00:00:00 2001 From: Rohit Lakde <37445331+rlakde@users.noreply.github.com> Date: Mon, 28 Oct 2024 10:07:06 -0500 Subject: [PATCH 08/21] Update reusable-build-and-publish.yml --- .github/workflows/reusable-build-and-publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/reusable-build-and-publish.yml b/.github/workflows/reusable-build-and-publish.yml index d7ec83b11..1c2c5030e 100644 --- a/.github/workflows/reusable-build-and-publish.yml +++ b/.github/workflows/reusable-build-and-publish.yml @@ -17,7 +17,7 @@ jobs: runs-on: ubuntu-latest services: postgres: - image: postgis/postgis + image: postgis/postgis:16-master env: POSTGRES_PASSWORD: password POSTGRES_USER: postgres From dbd6e7965fb75bf53f0c6fade5109c2790ace381 Mon Sep 17 00:00:00 2001 From: Rohit Lakde <37445331+rlakde@users.noreply.github.com> Date: Mon, 28 Oct 2024 10:31:10 -0500 Subject: [PATCH 09/21] Update gradle.properties --- gradle.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle.properties b/gradle.properties index 9b66cc554..92a7e516e 100644 --- a/gradle.properties +++ b/gradle.properties @@ -11,4 +11,4 @@ mavenPassword=YourPassword # - here-naksha-lib-core/NakshaVersion (static property: latest) # - here-naksha-lib-psql/resources/naksha_plpgsql.sql (method: naksha_version) # - here-naksha-app-service/src/main/resources/swagger/openapi.yaml (info.version property) -version=2.1.5 +version=2.2.0 From 9ea31d5932c0eafd165b0ebc55ff0a9ddbb4bfbe Mon Sep 17 00:00:00 2001 From: Rohit Lakde <37445331+rlakde@users.noreply.github.com> Date: Mon, 28 Oct 2024 10:31:35 -0500 Subject: [PATCH 10/21] Update openapi.yaml --- here-naksha-app-service/src/main/resources/swagger/openapi.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/here-naksha-app-service/src/main/resources/swagger/openapi.yaml b/here-naksha-app-service/src/main/resources/swagger/openapi.yaml index f3b01152a..061878e6d 100644 --- a/here-naksha-app-service/src/main/resources/swagger/openapi.yaml +++ b/here-naksha-app-service/src/main/resources/swagger/openapi.yaml @@ -12,7 +12,7 @@ servers: info: title: "Naksha Hub-API" description: "Naksha Hub-API is a REST API to provide simple access to geo data." - version: "2.1.5" + version: "2.2.0" security: - AccessToken: [ ] From 0b204a59ad93bea06fb93ecc246500b6c578a958 Mon Sep 17 00:00:00 2001 From: Rohit Lakde <37445331+rlakde@users.noreply.github.com> Date: Tue, 29 Oct 2024 15:42:44 -0500 Subject: [PATCH 11/21] Add Interface IExtensionInit --- .../here/naksha/lib/core/IExtensionInit.java | 43 +++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 here-naksha-lib-core/src/main/java/com/here/naksha/lib/core/IExtensionInit.java diff --git a/here-naksha-lib-core/src/main/java/com/here/naksha/lib/core/IExtensionInit.java b/here-naksha-lib-core/src/main/java/com/here/naksha/lib/core/IExtensionInit.java new file mode 100644 index 000000000..4e2cab31a --- /dev/null +++ b/here-naksha-lib-core/src/main/java/com/here/naksha/lib/core/IExtensionInit.java @@ -0,0 +1,43 @@ +/* + * Copyright (C) 2017-2024 HERE Europe B.V. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * License-Filename: LICENSE + */ +package com.here.naksha.lib.core; + +import com.here.naksha.lib.core.models.features.Extension; + +/** + * Naksha Extension Interface for all extensions providing initClassName. + */ +public interface IExtensionInit { + + /** + * Initializes the extension with the specified hub and extension parameters. + * This method should be called to set up any necessary resources or configurations + * required by the extension to operate correctly. + * + * @param hub The hub instance to be used by the extension. + * @param extension The extension instance being initialized. + */ + void init(INaksha hub, Extension extension); + + /** + * Closes the extension. This method should be called to ensure proper + * cleanup when the extension is no longer needed. + */ + void close(); +} From 528c9507067a2795e7f4ece77965350efad94c3a Mon Sep 17 00:00:00 2001 From: Rohit Lakde <37445331+rlakde@users.noreply.github.com> Date: Tue, 29 Oct 2024 15:50:37 -0500 Subject: [PATCH 12/21] Add instanceCache logic. --- .../naksha/lib/extmanager/ExtensionCache.java | 60 +++++++++++++++---- 1 file changed, 47 insertions(+), 13 deletions(-) diff --git a/here-naksha-lib-ext-manager/src/main/java/com/here/naksha/lib/extmanager/ExtensionCache.java b/here-naksha-lib-ext-manager/src/main/java/com/here/naksha/lib/extmanager/ExtensionCache.java index e45229fb1..b2e838d84 100644 --- a/here-naksha-lib-ext-manager/src/main/java/com/here/naksha/lib/extmanager/ExtensionCache.java +++ b/here-naksha-lib-ext-manager/src/main/java/com/here/naksha/lib/extmanager/ExtensionCache.java @@ -18,6 +18,7 @@ */ package com.here.naksha.lib.extmanager; +import com.here.naksha.lib.core.IExtensionInit; import com.here.naksha.lib.core.INaksha; import com.here.naksha.lib.core.SimpleTask; import com.here.naksha.lib.core.models.ExtensionConfig; @@ -29,7 +30,6 @@ import com.here.naksha.lib.extmanager.models.KVPair; import java.io.File; import java.io.IOException; -import java.lang.reflect.InvocationTargetException; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -50,6 +50,7 @@ public class ExtensionCache { new ConcurrentHashMap<>(); private static final Map jarClientMap = new HashMap<>(); private final @NotNull INaksha naksha; + private Map instanceCache = new ConcurrentHashMap<>(); static { jarClientMap.put(JarClientType.S3.getType(), new AmazonS3Helper()); @@ -65,6 +66,7 @@ public ExtensionCache(@NotNull INaksha naksha) { * Also it removes existing mapping from cache which is not available in config store anymore */ protected void buildExtensionCache(ExtensionConfig extensionConfig) { + IExtensionInit instance = null; List>> futures = extensionConfig.getExtensions().stream() .filter(extension -> !this.isLoaderMappingExist(extension)) .map(extension -> { @@ -90,9 +92,31 @@ protected void buildExtensionCache(ExtensionConfig extensionConfig) { for (String key : loaderCache.keySet()) { if (!extIds.contains(key)) { - loaderCache.remove(key); - PluginCache.removeExtensionCache(key); - logger.info("Extension {} removed from cache.", key); + KVPair kvPair = loaderCache.get(key); + Object instanceObj = kvPair.getValue(); + Extension extension = kvPair.getKey(); + final String extensionIdWthEnv = extension.getEnv() + ":" + extension.getId(); + if (!isNullOrEmpty(extension.getInitClassName())) { + if (instanceObj instanceof IExtensionInit initInstance) { + try { + initInstance.close(); + instance = initInstance; + logger.info("Extension {} closed successfully.", extensionIdWthEnv); + } catch (Exception e) { + logger.error("Failed to close extension {}", extensionIdWthEnv, e); + } + } else { + logger.error("Instance is not of type IExtensionInit for extension {}", extensionIdWthEnv); + } + } + synchronized (this) { + if (instance != null) { + instanceCache.remove(key); + } + loaderCache.remove(key); + PluginCache.removeExtensionCache(key); + logger.info("Extension {} removed from cache.", key); + } } } logger.info("Extension cache size " + loaderCache.size()); @@ -103,6 +127,7 @@ private void publishIntoCache(KVPair result, ExtensionConfig ex final Extension extension = result.getKey(); final String extensionIdWthEnv = extension.getEnv() + ":" + extension.getId(); final File jarFile = result.getValue(); + IExtensionInit instance = null; ClassLoader loader; try { loader = ClassLoaderHelper.getClassLoader(jarFile, extensionConfig.getWhilelistDelegateClass()); @@ -114,12 +139,16 @@ private void publishIntoCache(KVPair result, ExtensionConfig ex if (!isNullOrEmpty(extension.getInitClassName())) { try { Class clz = loader.loadClass(extension.getInitClassName()); - clz.getConstructor(INaksha.class, Extension.class).newInstance(naksha, extension); - } catch (ClassNotFoundException - | InvocationTargetException - | InstantiationException - | NoSuchMethodException - | IllegalAccessException e) { + Object obj = clz.getConstructor(INaksha.class, Extension.class).newInstance(naksha, extension); + if (obj instanceof IExtensionInit initInstance) { + initInstance.close(); + initInstance.init(naksha, extension); + instance = initInstance; + } else { + logger.error("Extension does not implement IExtensionInit for extension {}", extension.getId()); + return; + } + } catch (Exception e) { logger.error( "Failed to instantiate class {} for extension {} ", extension.getInitClassName(), @@ -133,8 +162,13 @@ private void publishIntoCache(KVPair result, ExtensionConfig ex "Extension {} initialization using initClassName {} done successfully.", extensionIdWthEnv, extension.getInitClassName()); - loaderCache.put(extensionIdWthEnv, new KVPair(extension, loader)); - PluginCache.removeExtensionCache(extensionIdWthEnv); + synchronized (this) { + if (instance != null) { + instanceCache.put(extensionIdWthEnv, instance); + } + loaderCache.put(extensionIdWthEnv, new KVPair(extension, loader)); + PluginCache.removeExtensionCache(extensionIdWthEnv); + } logger.info( "Extension id={}, version={} is successfully loaded into the cache, using Jar at {} for env={}.", extensionIdWthEnv, @@ -160,7 +194,7 @@ private boolean isLoaderMappingExist(Extension extension) { } /** - * Lamda function which will initiate the downloading for extension jar + * Lambda function which will initiate the downloading for extension jar */ private KVPair downloadJar(Extension extension) { logger.info("Downloading jar {} with version {} ", extension.getId(), extension.getVersion()); From 7325ec87bcb64b14f7d0764572e81935e04c2d70 Mon Sep 17 00:00:00 2001 From: Rohit Lakde <37445331+rlakde@users.noreply.github.com> Date: Mon, 4 Nov 2024 12:35:48 -0600 Subject: [PATCH 13/21] Create ValueTuple.java --- .../naksha/lib/extmanager/ValueTuple.java | 46 +++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 here-naksha-lib-ext-manager/src/main/java/com/here/naksha/lib/extmanager/ValueTuple.java diff --git a/here-naksha-lib-ext-manager/src/main/java/com/here/naksha/lib/extmanager/ValueTuple.java b/here-naksha-lib-ext-manager/src/main/java/com/here/naksha/lib/extmanager/ValueTuple.java new file mode 100644 index 000000000..6eca51d76 --- /dev/null +++ b/here-naksha-lib-ext-manager/src/main/java/com/here/naksha/lib/extmanager/ValueTuple.java @@ -0,0 +1,46 @@ +/* + * Copyright (C) 2017-2024 HERE Europe B.V. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * License-Filename: LICENSE + */ +package com.here.naksha.lib.extmanager; + +import com.here.naksha.lib.core.IExtensionInit; +import com.here.naksha.lib.core.models.features.Extension; + +public class ValueTuple { + private final Extension extension; + private final ClassLoader classLoader; + private final IExtensionInit instance; + + public ValueTuple(Extension extension, ClassLoader classLoader, IExtensionInit instance) { + this.extension = extension; + this.classLoader = classLoader; + this.instance = instance; + } + + public Extension getExtension() { + return extension; + } + + public ClassLoader getClassLoader() { + return classLoader; + } + + public IExtensionInit getInstance() { + return instance; + } +} From 94431001603c2fa098845dc560ca4f0f859f5373 Mon Sep 17 00:00:00 2001 From: Rohit Lakde <37445331+rlakde@users.noreply.github.com> Date: Mon, 4 Nov 2024 12:43:28 -0600 Subject: [PATCH 14/21] loaderCache Changes --- .../naksha/lib/extmanager/ExtensionCache.java | 74 +++++++++---------- 1 file changed, 33 insertions(+), 41 deletions(-) diff --git a/here-naksha-lib-ext-manager/src/main/java/com/here/naksha/lib/extmanager/ExtensionCache.java b/here-naksha-lib-ext-manager/src/main/java/com/here/naksha/lib/extmanager/ExtensionCache.java index b2e838d84..b58858119 100644 --- a/here-naksha-lib-ext-manager/src/main/java/com/here/naksha/lib/extmanager/ExtensionCache.java +++ b/here-naksha-lib-ext-manager/src/main/java/com/here/naksha/lib/extmanager/ExtensionCache.java @@ -46,11 +46,9 @@ */ public class ExtensionCache { private static final @NotNull Logger logger = LoggerFactory.getLogger(ExtensionCache.class); - private static final ConcurrentHashMap> loaderCache = - new ConcurrentHashMap<>(); + private static final ConcurrentHashMap loaderCache = new ConcurrentHashMap<>(); private static final Map jarClientMap = new HashMap<>(); private final @NotNull INaksha naksha; - private Map instanceCache = new ConcurrentHashMap<>(); static { jarClientMap.put(JarClientType.S3.getType(), new AmazonS3Helper()); @@ -66,7 +64,6 @@ public ExtensionCache(@NotNull INaksha naksha) { * Also it removes existing mapping from cache which is not available in config store anymore */ protected void buildExtensionCache(ExtensionConfig extensionConfig) { - IExtensionInit instance = null; List>> futures = extensionConfig.getExtensions().stream() .filter(extension -> !this.isLoaderMappingExist(extension)) .map(extension -> { @@ -92,32 +89,8 @@ protected void buildExtensionCache(ExtensionConfig extensionConfig) { for (String key : loaderCache.keySet()) { if (!extIds.contains(key)) { - KVPair kvPair = loaderCache.get(key); - Object instanceObj = kvPair.getValue(); - Extension extension = kvPair.getKey(); - final String extensionIdWthEnv = extension.getEnv() + ":" + extension.getId(); - if (!isNullOrEmpty(extension.getInitClassName())) { - if (instanceObj instanceof IExtensionInit initInstance) { - try { - initInstance.close(); - instance = initInstance; - logger.info("Extension {} closed successfully.", extensionIdWthEnv); - } catch (Exception e) { - logger.error("Failed to close extension {}", extensionIdWthEnv, e); - } - } else { - logger.error("Instance is not of type IExtensionInit for extension {}", extensionIdWthEnv); - } + removeExtensionFromCache(key); } - synchronized (this) { - if (instance != null) { - instanceCache.remove(key); - } - loaderCache.remove(key); - PluginCache.removeExtensionCache(key); - logger.info("Extension {} removed from cache.", key); - } - } } logger.info("Extension cache size " + loaderCache.size()); } @@ -162,13 +135,10 @@ private void publishIntoCache(KVPair result, ExtensionConfig ex "Extension {} initialization using initClassName {} done successfully.", extensionIdWthEnv, extension.getInitClassName()); - synchronized (this) { - if (instance != null) { - instanceCache.put(extensionIdWthEnv, instance); - } - loaderCache.put(extensionIdWthEnv, new KVPair(extension, loader)); - PluginCache.removeExtensionCache(extensionIdWthEnv); - } + + loaderCache.put(extensionIdWthEnv, new KVPair(extension, loader)); + PluginCache.removeExtensionCache(extensionIdWthEnv); + logger.info( "Extension id={}, version={} is successfully loaded into the cache, using Jar at {} for env={}.", extensionIdWthEnv, @@ -178,12 +148,34 @@ private void publishIntoCache(KVPair result, ExtensionConfig ex } } + private void removeExtensionFromCache(String extensionId) { + ValueTuple valueTuple = loaderCache.get(extensionId); + if (valueTuple != null) { + IExtensionInit instance = valueTuple.getInstance(); + Extension extension = valueTuple.getExtension(); + final String extensionIdWithEnv = extension.getEnv() + ":" + extension.getId(); + + if (instance != null) { + try { + instance.close(); + logger.info("Extension {} closed successfully.", extensionIdWithEnv); + } catch (Exception e) { + logger.error("Failed to close extension {}", extensionIdWithEnv, e); + } + } + + loaderCache.remove(extensionId); + PluginCache.removeExtensionCache(extensionId); + logger.info("Extension {} removed from cache.", extensionId); + } + } + private boolean isLoaderMappingExist(Extension extension) { final String extensionIdWthEnv = extension.getEnv() + ":" + extension.getId(); - KVPair existingMapping = loaderCache.get(extensionIdWthEnv); + ValueTuple existingMapping = loaderCache.get(extensionIdWthEnv); if (existingMapping == null) return false; - final Extension exExtension = existingMapping.getKey(); + final Extension exExtension = existingMapping.getExtension(); final String exInitClassName = isNullOrEmpty(exExtension.getInitClassName()) ? "" : exExtension.getInitClassName(); final String initClassName = isNullOrEmpty(extension.getInitClassName()) ? "" : extension.getInitClassName(); @@ -218,8 +210,8 @@ protected FileClient getJarClient(String url) { } protected ClassLoader getClassLoaderById(@NotNull String extensionId) { - KVPair mappedLoader = loaderCache.get(extensionId); - return mappedLoader == null ? null : mappedLoader.getValue(); + ValueTuple mappedLoader = loaderCache.get(extensionId); + return mappedLoader == null ? null : mappedLoader.getClassLoader(); } public int getCacheLength() { @@ -227,7 +219,7 @@ public int getCacheLength() { } public List getCachedExtensions() { - return loaderCache.values().stream().map(KVPair::getKey).toList(); + return loaderCache.values().stream().map(ValueTuple::getExtension).toList(); } private boolean isNullOrEmpty(String value) { From 32c1b9e36b74cdbd0bef0740a91ed92e59423e9d Mon Sep 17 00:00:00 2001 From: Rohit Lakde <37445331+rlakde@users.noreply.github.com> Date: Mon, 4 Nov 2024 12:49:00 -0600 Subject: [PATCH 15/21] Correct logger messages --- .../src/main/java/com/here/naksha/lib/hub/NakshaHub.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/here-naksha-lib-hub/src/main/java/com/here/naksha/lib/hub/NakshaHub.java b/here-naksha-lib-hub/src/main/java/com/here/naksha/lib/hub/NakshaHub.java index a6f453bf2..4fc0a3455 100644 --- a/here-naksha-lib-hub/src/main/java/com/here/naksha/lib/hub/NakshaHub.java +++ b/here-naksha-lib-hub/src/main/java/com/here/naksha/lib/hub/NakshaHub.java @@ -314,8 +314,8 @@ private static WriteXyzCollections createAdminCollectionsRequest() { try { eventHandlers = readFeaturesFromResult(rdResult, EventHandler.class); } catch (NoCursor e) { - logger.error("NoCursor exception encountered", e); - throw new RuntimeException("Failed to open cursor", e); + logger.error("NoCursor exception encountered while reading Extension based Handlers", e); + throw new RuntimeException("Failed to open Cursor while reading Extension based Handlers", e); } Set extensionIds = new HashSet<>(); From 3bd4148ec2e956e6a792b1a27714b60ca1c369db Mon Sep 17 00:00:00 2001 From: Rohit Lakde <37445331+rlakde@users.noreply.github.com> Date: Mon, 4 Nov 2024 12:50:45 -0600 Subject: [PATCH 16/21] Update IExtensionInit description --- .../main/java/com/here/naksha/lib/core/IExtensionInit.java | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/here-naksha-lib-core/src/main/java/com/here/naksha/lib/core/IExtensionInit.java b/here-naksha-lib-core/src/main/java/com/here/naksha/lib/core/IExtensionInit.java index 4e2cab31a..5b038ff6d 100644 --- a/here-naksha-lib-core/src/main/java/com/here/naksha/lib/core/IExtensionInit.java +++ b/here-naksha-lib-core/src/main/java/com/here/naksha/lib/core/IExtensionInit.java @@ -26,10 +26,8 @@ public interface IExtensionInit { /** - * Initializes the extension with the specified hub and extension parameters. - * This method should be called to set up any necessary resources or configurations - * required by the extension to operate correctly. - * + * This method should be called to set up any necessary configurations + * Extension configuration supplied as part of deployment pipeline for respective Extension and sub-env. * @param hub The hub instance to be used by the extension. * @param extension The extension instance being initialized. */ From d718694fa1c8630d26d911c4cb55d1d9ee607368 Mon Sep 17 00:00:00 2001 From: Rohit Lakde <37445331+rlakde@users.noreply.github.com> Date: Mon, 4 Nov 2024 12:55:47 -0600 Subject: [PATCH 17/21] Fix bug --- .../java/com/here/naksha/lib/extmanager/ExtensionCache.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/here-naksha-lib-ext-manager/src/main/java/com/here/naksha/lib/extmanager/ExtensionCache.java b/here-naksha-lib-ext-manager/src/main/java/com/here/naksha/lib/extmanager/ExtensionCache.java index b58858119..fa235f106 100644 --- a/here-naksha-lib-ext-manager/src/main/java/com/here/naksha/lib/extmanager/ExtensionCache.java +++ b/here-naksha-lib-ext-manager/src/main/java/com/here/naksha/lib/extmanager/ExtensionCache.java @@ -136,7 +136,7 @@ private void publishIntoCache(KVPair result, ExtensionConfig ex extensionIdWthEnv, extension.getInitClassName()); - loaderCache.put(extensionIdWthEnv, new KVPair(extension, loader)); + loaderCache.put(extensionIdWthEnv, new ValueTuple(extension, loader, instance)); PluginCache.removeExtensionCache(extensionIdWthEnv); logger.info( From f9314f63212197deba91afa963bce6215656ad06 Mon Sep 17 00:00:00 2001 From: Rohit Lakde <37445331+rlakde@users.noreply.github.com> Date: Mon, 4 Nov 2024 13:02:48 -0600 Subject: [PATCH 18/21] Correct logger in Hub. --- .../src/main/java/com/here/naksha/lib/hub/NakshaHub.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/here-naksha-lib-hub/src/main/java/com/here/naksha/lib/hub/NakshaHub.java b/here-naksha-lib-hub/src/main/java/com/here/naksha/lib/hub/NakshaHub.java index 4fc0a3455..ccb4376a0 100644 --- a/here-naksha-lib-hub/src/main/java/com/here/naksha/lib/hub/NakshaHub.java +++ b/here-naksha-lib-hub/src/main/java/com/here/naksha/lib/hub/NakshaHub.java @@ -324,7 +324,7 @@ private static WriteXyzCollections createAdminCollectionsRequest() { if (extensionId != null && extensionId.contains(":")) { extensionIds.add(extensionId); } else { - logger.error("Environment is missing for an extension Id"); + logger.error("Environment is missing for an extension Id {}", extensionId); } } From a8cfeaad5cf679eac9ceb1014deee11bac84ec68 Mon Sep 17 00:00:00 2001 From: Rohit Lakde <37445331+rlakde@users.noreply.github.com> Date: Mon, 4 Nov 2024 19:35:17 -0600 Subject: [PATCH 19/21] Update IExtensionInit Description. --- .../main/java/com/here/naksha/lib/core/IExtensionInit.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/here-naksha-lib-core/src/main/java/com/here/naksha/lib/core/IExtensionInit.java b/here-naksha-lib-core/src/main/java/com/here/naksha/lib/core/IExtensionInit.java index 5b038ff6d..33d21a305 100644 --- a/here-naksha-lib-core/src/main/java/com/here/naksha/lib/core/IExtensionInit.java +++ b/here-naksha-lib-core/src/main/java/com/here/naksha/lib/core/IExtensionInit.java @@ -26,10 +26,11 @@ public interface IExtensionInit { /** - * This method should be called to set up any necessary configurations - * Extension configuration supplied as part of deployment pipeline for respective Extension and sub-env. + * Initializes the extension with the specified hub and extension parameters. + * This method should be called to set up any necessary resources or configurations + * required by the extension to operate correctly. * @param hub The hub instance to be used by the extension. - * @param extension The extension instance being initialized. + * @param extension Extension configuration supplied as part of deployment pipeline for respective Extension and sub-env. */ void init(INaksha hub, Extension extension); From f05cd53e9d60d389af80eedd6774aa0b567dbdd5 Mon Sep 17 00:00:00 2001 From: Rohit Lakde <37445331+rlakde@users.noreply.github.com> Date: Mon, 4 Nov 2024 19:38:38 -0600 Subject: [PATCH 20/21] Correct removeExtensionFromCache. --- .../com/here/naksha/lib/extmanager/ExtensionCache.java | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/here-naksha-lib-ext-manager/src/main/java/com/here/naksha/lib/extmanager/ExtensionCache.java b/here-naksha-lib-ext-manager/src/main/java/com/here/naksha/lib/extmanager/ExtensionCache.java index fa235f106..e38b9729b 100644 --- a/here-naksha-lib-ext-manager/src/main/java/com/here/naksha/lib/extmanager/ExtensionCache.java +++ b/here-naksha-lib-ext-manager/src/main/java/com/here/naksha/lib/extmanager/ExtensionCache.java @@ -112,9 +112,8 @@ private void publishIntoCache(KVPair result, ExtensionConfig ex if (!isNullOrEmpty(extension.getInitClassName())) { try { Class clz = loader.loadClass(extension.getInitClassName()); - Object obj = clz.getConstructor(INaksha.class, Extension.class).newInstance(naksha, extension); + Object obj = clz.getConstructor().newInstance(); if (obj instanceof IExtensionInit initInstance) { - initInstance.close(); initInstance.init(naksha, extension); instance = initInstance; } else { @@ -152,15 +151,12 @@ private void removeExtensionFromCache(String extensionId) { ValueTuple valueTuple = loaderCache.get(extensionId); if (valueTuple != null) { IExtensionInit instance = valueTuple.getInstance(); - Extension extension = valueTuple.getExtension(); - final String extensionIdWithEnv = extension.getEnv() + ":" + extension.getId(); - if (instance != null) { try { instance.close(); - logger.info("Extension {} closed successfully.", extensionIdWithEnv); + logger.info("Extension {} closed successfully.", extensionId); } catch (Exception e) { - logger.error("Failed to close extension {}", extensionIdWithEnv, e); + logger.error("Failed to close extension {}", extensionId, e); } } From 5fda3754bd1f6f72632527b57f076e90062aef21 Mon Sep 17 00:00:00 2001 From: Rohit Lakde <37445331+rlakde@users.noreply.github.com> Date: Tue, 5 Nov 2024 01:55:39 -0600 Subject: [PATCH 21/21] Review Fixes for ExtensionCache --- .../naksha/lib/extmanager/ExtensionCache.java | 57 ++++++++++--------- 1 file changed, 30 insertions(+), 27 deletions(-) diff --git a/here-naksha-lib-ext-manager/src/main/java/com/here/naksha/lib/extmanager/ExtensionCache.java b/here-naksha-lib-ext-manager/src/main/java/com/here/naksha/lib/extmanager/ExtensionCache.java index e38b9729b..1c8fb6223 100644 --- a/here-naksha-lib-ext-manager/src/main/java/com/here/naksha/lib/extmanager/ExtensionCache.java +++ b/here-naksha-lib-ext-manager/src/main/java/com/here/naksha/lib/extmanager/ExtensionCache.java @@ -100,12 +100,12 @@ private void publishIntoCache(KVPair result, ExtensionConfig ex final Extension extension = result.getKey(); final String extensionIdWthEnv = extension.getEnv() + ":" + extension.getId(); final File jarFile = result.getValue(); - IExtensionInit instance = null; + IExtensionInit initObj = null; ClassLoader loader; try { loader = ClassLoaderHelper.getClassLoader(jarFile, extensionConfig.getWhilelistDelegateClass()); } catch (Exception e) { - logger.error("Failed to load extension jar " + extension.getId(), e); + logger.error("Failed to load extension jar " + extensionIdWthEnv, e); return; } @@ -115,9 +115,13 @@ private void publishIntoCache(KVPair result, ExtensionConfig ex Object obj = clz.getConstructor().newInstance(); if (obj instanceof IExtensionInit initInstance) { initInstance.init(naksha, extension); - instance = initInstance; + initObj = initInstance; + logger.info( + "Extension {} initialization using initClassName {} done successfully.", + extensionIdWthEnv, + extension.getInitClassName()); } else { - logger.error("Extension does not implement IExtensionInit for extension {}", extension.getId()); + logger.error("Extension does not implement IExtensionInit for extension {}", extensionIdWthEnv); return; } } catch (Exception e) { @@ -129,40 +133,39 @@ private void publishIntoCache(KVPair result, ExtensionConfig ex return; } } - if (!isNullOrEmpty(extension.getInitClassName())) - logger.info( - "Extension {} initialization using initClassName {} done successfully.", - extensionIdWthEnv, - extension.getInitClassName()); - loaderCache.put(extensionIdWthEnv, new ValueTuple(extension, loader, instance)); - PluginCache.removeExtensionCache(extensionIdWthEnv); + ValueTuple previousValue = loaderCache.put(extensionIdWthEnv, new ValueTuple(extension, loader, initObj)); + if (previousValue != null) { + IExtensionInit previousInitObj = previousValue.getInstance(); + closeExtensionInstance(extensionIdWthEnv, previousInitObj); + } logger.info( - "Extension id={}, version={} is successfully loaded into the cache, using Jar at {} for env={}.", + "Extension id={}, version={} is successfully loaded into the cache, using Jar at {}.", extensionIdWthEnv, extension.getVersion(), - extension.getUrl().substring(extension.getUrl().lastIndexOf("/") + 1), - extension.getEnv()); + extension.getUrl().substring(extension.getUrl().lastIndexOf("/") + 1)); } } private void removeExtensionFromCache(String extensionId) { - ValueTuple valueTuple = loaderCache.get(extensionId); + ValueTuple valueTuple = loaderCache.remove(extensionId); + PluginCache.removeExtensionCache(extensionId); + logger.info("Extension {} removed from cache.", extensionId); if (valueTuple != null) { - IExtensionInit instance = valueTuple.getInstance(); - if (instance != null) { - try { - instance.close(); - logger.info("Extension {} closed successfully.", extensionId); - } catch (Exception e) { - logger.error("Failed to close extension {}", extensionId, e); - } - } + IExtensionInit initObj = valueTuple.getInstance(); + closeExtensionInstance(extensionId, initObj); + } + } - loaderCache.remove(extensionId); - PluginCache.removeExtensionCache(extensionId); - logger.info("Extension {} removed from cache.", extensionId); + private void closeExtensionInstance(String extensionId, IExtensionInit initObj) { + if (initObj != null) { + try { + initObj.close(); + logger.info("Extension {} closed successfully.", extensionId); + } catch (Exception e) { + logger.error("Failed to close extension {}", extensionId, e); + } } }