CASL-592 Extension Cleanup Capability

.github/workflows/reusable-build-and-publish.yml

@@ -17,7 +17,7 @@ jobs:
     runs-on: ubuntu-latest
     services:
       postgres:
-        image: postgis/postgis
+        image: postgis/postgis:16-master
         env:
           POSTGRES_PASSWORD: password
           POSTGRES_USER: postgres

gradle.properties

@@ -11,4 +11,5 @@ mavenPassword=YourPassword
 # - here-naksha-lib-core/NakshaVersion (static property: latest)
 # - here-naksha-lib-psql/resources/naksha_plpgsql.sql (method: naksha_version)
 # - here-naksha-app-service/src/main/resources/swagger/openapi.yaml (info.version property)
-version=2.1.5
+version=2.2.0
+

here-naksha-app-service/src/main/resources/swagger/openapi.yaml

@@ -12,7 +12,7 @@ servers:
 info:
   title: "Naksha Hub-API"
   description: "Naksha Hub-API is a REST API to provide simple access to geo data."
-  version: "2.1.5"
+  version: "2.2.0"
 
 security:
   - AccessToken: [ ]

here-naksha-lib-core/src/main/java/com/here/naksha/lib/core/IExtensionInit.java

@@ -0,0 +1,42 @@
+/*
+ * Copyright (C) 2017-2024 HERE Europe B.V.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ * License-Filename: LICENSE
+ */
+package com.here.naksha.lib.core;
+
+import com.here.naksha.lib.core.models.features.Extension;
+
+/**
+ * Naksha Extension Interface for all extensions providing initClassName.
+ */
+public interface IExtensionInit {
+
+  /**
+   * Initializes the extension with the specified hub and extension parameters.
+   * This method should be called to set up any necessary resources or configurations
+   * required by the extension to operate correctly.
+   * @param hub The hub instance to be used by the extension.
+   * @param extension Extension configuration supplied as part of deployment pipeline for respective Extension and sub-env.
+   */
+  void init(INaksha hub, Extension extension);
+
+  /**
+   * Closes the extension. This method should be called to ensure proper
+   * cleanup when the extension is no longer needed.
+   */
+  void close();
+}

here-naksha-lib-core/src/main/java/com/here/naksha/lib/core/models/ExtensionConfig.java

@@ -32,7 +32,6 @@ public class ExtensionConfig {
   public static final String EXPIRY = "expiry";
   public static final String EXTENSIONS = "extensions";
   public static final String WHITELIST_DELEGATE_CLASSES = "whitelistDelegateClasses";
-  public static final String ENV_NAME = "env";
 
   @JsonProperty(EXPIRY)
   long expiry;
@@ -43,19 +42,14 @@ public class ExtensionConfig {
   @JsonProperty(WHITELIST_DELEGATE_CLASSES)
   List<String> whitelistDelegateClasses;
 
-  @JsonProperty(ENV_NAME)
-  String env;
-
   @JsonCreator
   public ExtensionConfig(
       @JsonProperty(EXPIRY) @NotNull Long expiry,
       @JsonProperty(EXTENSIONS) @Nullable List<Extension> extensions,
-      @JsonProperty(WHITELIST_DELEGATE_CLASSES) @Nullable List<String> whitelistDelegateClasses,
-      @JsonProperty(ENV_NAME) @NotNull String env) {
+      @JsonProperty(WHITELIST_DELEGATE_CLASSES) @Nullable List<String> whitelistDelegateClasses) {
     this.expiry = expiry;
     this.extensions = extensions;
     this.whitelistDelegateClasses = whitelistDelegateClasses;
-    this.env = env;
   }
 
   public long getExpiry() {
@@ -69,8 +63,4 @@ public List<Extension> getExtensions() {
   public List<String> getWhilelistDelegateClass() {
     return whitelistDelegateClasses;
   }
-
-  public String getEnv() {
-    return env;
-  }
 }

here-naksha-lib-core/src/main/java/com/here/naksha/lib/core/models/features/Extension.java

@@ -36,6 +36,7 @@ public class Extension extends XyzFeature {
   public static final String URL = "url";
   public static final String VERSION = "version";
   public static final String INIT_CLASS_NAME = "initClassName";
+  public static final String ENV = "env";
 
   @JsonProperty(URL)
   String url;
@@ -46,6 +47,9 @@ public class Extension extends XyzFeature {
   @JsonProperty(INIT_CLASS_NAME)
   String initClassName;
 
+  @JsonProperty(ENV)
+  String env;
+
   /**
    * Create an extension.
    *
@@ -78,4 +82,12 @@ public String getVersion() {
   public String getInitClassName() {
     return initClassName;
   }
+
+  public void setEnv(String env) {
+    this.env = env;
+  }
+
+  public String getEnv() {
+    return env;
+  }
 }

here-naksha-lib-ext-manager/src/main/java/com/here/naksha/lib/extmanager/ExtensionCache.java

@@ -18,6 +18,7 @@
  */
 package com.here.naksha.lib.extmanager;
 
+import com.here.naksha.lib.core.IExtensionInit;
 import com.here.naksha.lib.core.INaksha;
 import com.here.naksha.lib.core.SimpleTask;
 import com.here.naksha.lib.core.models.ExtensionConfig;
@@ -29,7 +30,6 @@
 import com.here.naksha.lib.extmanager.models.KVPair;
 import java.io.File;
 import java.io.IOException;
-import java.lang.reflect.InvocationTargetException;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -46,8 +46,7 @@
  */
 public class ExtensionCache {
   private static final @NotNull Logger logger = LoggerFactory.getLogger(ExtensionCache.class);
-  private static final ConcurrentHashMap<String, KVPair<Extension, ClassLoader>> loaderCache =
-      new ConcurrentHashMap<>();
+  private static final ConcurrentHashMap<String, ValueTuple> loaderCache = new ConcurrentHashMap<>();
   private static final Map<String, FileClient> jarClientMap = new HashMap<>();
   private final @NotNull INaksha naksha;
 
@@ -83,24 +82,25 @@ protected void buildExtensionCache(ExtensionConfig extensionConfig) {
       publishIntoCache(result, extensionConfig);
     });
 
-    // Removing existing extension which has been removed from the configuration
-    List<String> extIds =
-        extensionConfig.getExtensions().stream().map(Extension::getId).toList();
+    // Removing existing extension which has been removed from the configuration.
+    List<String> extIds = extensionConfig.getExtensions().stream()
+        .map(extension -> extension.getEnv() + ":" + extension.getId())
+        .toList();
 
     for (String key : loaderCache.keySet()) {
       if (!extIds.contains(key)) {
-        loaderCache.remove(key);
-        PluginCache.removeExtensionCache(key);
-        logger.info("Extension {} removed from cache.", key);
-      }
+        removeExtensionFromCache(key);
+        }
     }
     logger.info("Extension cache size " + loaderCache.size());
   }
 
   private void publishIntoCache(KVPair<Extension, File> result, ExtensionConfig extensionConfig) {
     if (result != null && result.getValue() != null) {
       final Extension extension = result.getKey();
+      final String extensionIdWthEnv = extension.getEnv() + ":" + extension.getId();
       final File jarFile = result.getValue();
+      IExtensionInit instance = null;
       ClassLoader loader;
       try {
         loader = ClassLoaderHelper.getClassLoader(jarFile, extensionConfig.getWhilelistDelegateClass());
@@ -112,41 +112,66 @@ private void publishIntoCache(KVPair<Extension, File> result, ExtensionConfig ex
       if (!isNullOrEmpty(extension.getInitClassName())) {
         try {
           Class<?> clz = loader.loadClass(extension.getInitClassName());
-          clz.getConstructor(INaksha.class, Extension.class).newInstance(naksha, extension);
-        } catch (ClassNotFoundException
-            | InvocationTargetException
-            | InstantiationException
-            | NoSuchMethodException
-            | IllegalAccessException e) {
+          Object obj = clz.getConstructor().newInstance();
+          if (obj instanceof IExtensionInit initInstance) {
+            initInstance.init(naksha, extension);
+            instance = initInstance;
+          } else {
+            logger.error("Extension does not implement IExtensionInit for extension {}", extension.getId());
+            return;
+          }
+        } catch (Exception e) {
           logger.error(
               "Failed to instantiate class {} for extension {} ",
               extension.getInitClassName(),
-              extension.getId(),
+              extensionIdWthEnv,
               e);
           return;
         }
       }
       if (!isNullOrEmpty(extension.getInitClassName()))
         logger.info(
             "Extension {} initialization using initClassName {} done successfully.",
-            extension.getId(),
+            extensionIdWthEnv,
             extension.getInitClassName());
-      loaderCache.put(extension.getId(), new KVPair<Extension, ClassLoader>(extension, loader));
-      PluginCache.removeExtensionCache(extension.getId());
+
+      loaderCache.put(extensionIdWthEnv, new ValueTuple(extension, loader, instance));
+      PluginCache.removeExtensionCache(extensionIdWthEnv);
+
       logger.info(
           "Extension id={}, version={} is successfully loaded into the cache, using Jar at {} for env={}.",
-          extension.getId(),
+          extensionIdWthEnv,
           extension.getVersion(),
           extension.getUrl().substring(extension.getUrl().lastIndexOf("/") + 1),
-          extensionConfig.getEnv());
+          extension.getEnv());
+    }
+  }
+
+  private void removeExtensionFromCache(String extensionId) {
+    ValueTuple valueTuple = loaderCache.get(extensionId);
+    if (valueTuple != null) {
+      IExtensionInit instance = valueTuple.getInstance();
+      if (instance != null) {
+        try {
+          instance.close();
+          logger.info("Extension {} closed successfully.", extensionId);
+        } catch (Exception e) {
+          logger.error("Failed to close extension {}", extensionId, e);
+        }
+      }
+
+      loaderCache.remove(extensionId);
+      PluginCache.removeExtensionCache(extensionId);
+      logger.info("Extension {} removed from cache.", extensionId);
     }
   }
 
   private boolean isLoaderMappingExist(Extension extension) {
-    KVPair<Extension, ClassLoader> existingMapping = loaderCache.get(extension.getId());
+    final String extensionIdWthEnv = extension.getEnv() + ":" + extension.getId();
+    ValueTuple existingMapping = loaderCache.get(extensionIdWthEnv);
     if (existingMapping == null) return false;
 
-    final Extension exExtension = existingMapping.getKey();
+    final Extension exExtension = existingMapping.getExtension();
     final String exInitClassName =
         isNullOrEmpty(exExtension.getInitClassName()) ? "" : exExtension.getInitClassName();
     final String initClassName = isNullOrEmpty(extension.getInitClassName()) ? "" : extension.getInitClassName();
@@ -157,7 +182,7 @@ private boolean isLoaderMappingExist(Extension extension) {
   }
 
   /**
-   * Lamda function which will initiate the downloading for extension jar
+   * Lambda function which will initiate the downloading for extension jar
    */
   private KVPair<Extension, File> downloadJar(Extension extension) {
     logger.info("Downloading jar {} with version {} ", extension.getId(), extension.getVersion());
@@ -181,16 +206,16 @@ protected FileClient getJarClient(String url) {
   }
 
   protected ClassLoader getClassLoaderById(@NotNull String extensionId) {
-    KVPair<Extension, ClassLoader> mappedLoader = loaderCache.get(extensionId);
-    return mappedLoader == null ? null : mappedLoader.getValue();
+    ValueTuple mappedLoader = loaderCache.get(extensionId);
+    return mappedLoader == null ? null : mappedLoader.getClassLoader();
   }
 
   public int getCacheLength() {
     return loaderCache.size();
   }
 
   public List<Extension> getCachedExtensions() {
-    return loaderCache.values().stream().map(KVPair::getKey).toList();
+    return loaderCache.values().stream().map(ValueTuple::getExtension).toList();
   }
 
   private boolean isNullOrEmpty(String value) {

here-naksha-lib-ext-manager/src/main/java/com/here/naksha/lib/extmanager/ValueTuple.java

@@ -0,0 +1,46 @@
+/*
+ * Copyright (C) 2017-2024 HERE Europe B.V.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ * License-Filename: LICENSE
+ */
+package com.here.naksha.lib.extmanager;
+
+import com.here.naksha.lib.core.IExtensionInit;
+import com.here.naksha.lib.core.models.features.Extension;
+
+public class ValueTuple {
+  private final Extension extension;
+  private final ClassLoader classLoader;
+  private final IExtensionInit instance;
+
+  public ValueTuple(Extension extension, ClassLoader classLoader, IExtensionInit instance) {
+    this.extension = extension;
+    this.classLoader = classLoader;
+    this.instance = instance;
+  }
+
+  public Extension getExtension() {
+    return extension;
+  }
+
+  public ClassLoader getClassLoader() {
+    return classLoader;
+  }
+
+  public IExtensionInit getInstance() {
+    return instance;
+  }
+}

here-naksha-lib-ext-manager/src/main/java/com/here/naksha/lib/extmanager/helpers/AmazonS3Helper.java

@@ -26,14 +26,11 @@
 import java.io.InputStream;
 import java.io.InputStreamReader;
 import java.net.URI;
-import java.util.List;
 import org.jetbrains.annotations.NotNull;
 import software.amazon.awssdk.services.s3.S3Client;
 import software.amazon.awssdk.services.s3.S3Uri;
 import software.amazon.awssdk.services.s3.model.GetObjectRequest;
 import software.amazon.awssdk.services.s3.model.GetObjectRequest.Builder;
-import software.amazon.awssdk.services.s3.model.ListObjectsRequest;
-import software.amazon.awssdk.services.s3.model.ListObjectsResponse;
 
 public class AmazonS3Helper implements FileClient {
   private static S3Client s3Client;
@@ -93,20 +90,6 @@ public String getFileContent(String url) throws IOException {
     }
   }
 
-  public List<String> listKeysInBucket(String url) {
-    S3Uri fileUri = getS3Uri(url);
-    String delimiter = "/";
-
-    ListObjectsRequest.Builder listObjectsRequestBuilder =
-        ListObjectsRequest.builder().bucket(fileUri.bucket().get()).delimiter(delimiter);
-
-    if (fileUri.key().isPresent())
-      listObjectsRequestBuilder.prefix(fileUri.key().get());
-
-    ListObjectsResponse response = getS3Client().listObjects(listObjectsRequestBuilder.build());
-    return response.commonPrefixes().stream().map(cm -> cm.prefix()).toList();
-  }
-
   public S3Uri getS3Uri(String url) {
     URI uri = URI.create(url);
     return getS3Client().utilities().parseUri(uri);

here-naksha-lib-ext-manager/src/test/java/com/here/naksha/lib/extmanager/BaseSetup.java

@@ -25,6 +25,6 @@ public ExtensionConfig getExtensionConfig() {
     } catch (IOException e) {
       throw new RuntimeException(e);
     }
-    return new ExtensionConfig(System.currentTimeMillis() + 6000, list,whitelistUrls,"test");
+    return new ExtensionConfig(System.currentTimeMillis() + 6000, list,whitelistUrls);
   }
 }