From 8fd163bb79579fd0262a45545f8a350e12cdc3b6 Mon Sep 17 00:00:00 2001 From: michaeloffner Date: Fri, 30 Aug 2024 17:02:23 +0200 Subject: [PATCH] LDEV-4356 - cache for a day and also cache the read method --- .../runtime/config/s3/BundleProvider.java | 124 ++++++++---------- loader/build.xml | 2 +- loader/pom.xml | 2 +- 3 files changed, 54 insertions(+), 74 deletions(-) diff --git a/core/src/main/java/lucee/runtime/config/s3/BundleProvider.java b/core/src/main/java/lucee/runtime/config/s3/BundleProvider.java index 49d75ed458..9dac5e4c61 100644 --- a/core/src/main/java/lucee/runtime/config/s3/BundleProvider.java +++ b/core/src/main/java/lucee/runtime/config/s3/BundleProvider.java @@ -68,13 +68,12 @@ import lucee.runtime.type.StructImpl; import lucee.runtime.type.dt.DateTime; import lucee.runtime.type.util.ListUtil; -import lucee.runtime.util.Pack200Util; import lucee.transformer.library.function.FunctionLibEntityResolver; import lucee.transformer.library.function.FunctionLibException; public final class BundleProvider extends DefaultHandler { public static final int CONNECTION_TIMEOUT = 1000; - private static final long MAX_AGE = 10000; + private static final long MAX_AGE = 24 * 60 * 60 * 1000; private static final int MAX_REDIRECTS = 10; private static URL DEFAULT_PROVIDER_LIST = null; @@ -149,6 +148,7 @@ public static URL getDefaultProviderDetailMvn() { private final URL url; private boolean insideContents; private Map elements = new LinkedHashMap<>(); + private List elementsSorted; private Element element; private boolean isTruncated; private String lastKey; @@ -245,7 +245,7 @@ public URL getBundleAsURL(BundleDefinition bd, boolean includeS3) throws Malform // S3: we loop through all records and S3 and pick one if (url == null) { try { - for (Element e: read()) { + for (Element e: read(false)) { if (bd.equals(e.getBundleDefinition())) { url = e.getJAR(); if (url != null) return url; @@ -370,8 +370,6 @@ private File deployBundledBundle(File bundleDirectory, String symbolicName, Stri String sub = "bundles/"; String nameAndVersion = symbolicName + "|" + symbolicVersion; String osgiFileName = symbolicName + "-" + symbolicVersion + ".jar"; - String pack20Ext = ".jar.pack.gz"; - boolean isPack200 = false; // first we look for an exact match InputStream is = getClass().getResourceAsStream("bundles/" + osgiFileName); @@ -380,30 +378,14 @@ private File deployBundledBundle(File bundleDirectory, String symbolicName, Stri if (is != null) LogUtil.log(Logger.LOG_DEBUG, "deploy", "bundle-download", "Found ]/bundles/" + osgiFileName + "] in lucee.jar"); else LogUtil.log(Logger.LOG_INFO, "deploy", "bundle-download", "Could not find [/bundles/" + osgiFileName + "] in lucee.jar"); - if (is == null) { - is = getClass().getResourceAsStream("bundles/" + osgiFileName + pack20Ext); - if (is == null) is = getClass().getResourceAsStream("/bundles/" + osgiFileName + pack20Ext); - isPack200 = true; - - if (is != null) LogUtil.log(Logger.LOG_DEBUG, "deploy", "bundle-download", "Found [/bundles/" + osgiFileName + pack20Ext + "] in lucee.jar"); - else LogUtil.log(Logger.LOG_INFO, "deploy", "bundle-download", "Could not find [/bundles/" + osgiFileName + pack20Ext + "] in lucee.jar"); - } if (is != null) { File temp = null; try { // copy to temp file temp = File.createTempFile("bundle", ".tmp"); - LogUtil.log(Logger.LOG_DEBUG, "deploy", "bundle-download", "Copying [lucee.jar!/bundles/" + osgiFileName + pack20Ext + "] to [" + temp + "]"); + LogUtil.log(Logger.LOG_DEBUG, "deploy", "bundle-download", "Copying [lucee.jar!/bundles/" + osgiFileName + "] to [" + temp + "]"); Util.copy(new BufferedInputStream(is), new FileOutputStream(temp), true, true); - if (isPack200) { - File temp2 = File.createTempFile("bundle", ".tmp2"); - Pack200Util.pack2Jar(temp, temp2); - LogUtil.log(Logger.LOG_DEBUG, "deploy", "bundle-download", "Upack [" + temp + "] to [" + temp2 + "]"); - temp.delete(); - temp = temp2; - } - // adding bundle File trg = new File(bundleDirectory, osgiFileName); FileUtil.move(temp, trg); @@ -418,8 +400,6 @@ private File deployBundledBundle(File bundleDirectory, String symbolicName, Stri } } - // now we search the current jar as an external zip what is slow (we do not support pack200 in this - // case) // this also not works with windows if (SystemUtil.isWindows()) return null; ZipEntry entry; @@ -437,8 +417,7 @@ private File deployBundledBundle(File bundleDirectory, String symbolicName, Stri temp = null; path = entry.getName().replace('\\', '/'); if (path.startsWith("/")) path = path.substring(1); // some zip path start with "/" some not - isPack200 = false; - if (path.startsWith(sub) && (path.endsWith(".jar") /* || (isPack200=path.endsWith(".jar.pack.gz")) */)) { // ignore non jar files or file from elsewhere + if (path.startsWith(sub) && (path.endsWith(".jar"))) { // ignore non jar files or file from elsewhere index = path.lastIndexOf('/') + 1; if (index == sub.length()) { // ignore sub directories name = path.substring(index); @@ -447,11 +426,6 @@ private File deployBundledBundle(File bundleDirectory, String symbolicName, Stri temp = File.createTempFile("bundle", ".tmp"); Util.copy(zis, new FileOutputStream(temp), false, true); - /* - * if(isPack200) { File temp2 = File.createTempFile("bundle", ".tmp2"); Pack200Util.pack2Jar(temp, - * temp2); temp.delete(); temp=temp2; name=name.substring(0,name.length()-".pack.gz".length()); } - */ - bundleInfo = BundleLoader.loadBundleInfo(temp); if (bundleInfo != null && nameAndVersion.equals(bundleInfo)) { File trg = new File(bundleDirectory, name); @@ -483,50 +457,56 @@ private File deployBundledBundle(File bundleDirectory, String symbolicName, Stri // 1146:size:305198;bundle:name:xmlgraphics.batik.awt.util;version:version EQ 1.8.0;;last-mod:{ts // '2024-01-14 22:32:05'}; - public List read() throws IOException, GeneralSecurityException, SAXException { - int count = 100; - URL url = null; - - if (lastKey != null) url = new URL(this.url.toExternalForm() + "?marker=" + lastKey); - - do { - if (url == null) url = isTruncated ? new URL(this.url.toExternalForm() + "?marker=" + this.lastKey) : this.url; - HTTPResponse rsp = HTTPEngine4Impl.get(url, null, null, BundleProvider.CONNECTION_TIMEOUT, true, null, null, null, null); - if (rsp != null) { - int sc = rsp.getStatusCode(); - if (sc < 200 || sc >= 300) throw new IOException("unable to invoke [" + url + "], status code [" + sc + "]"); - } - else { - throw new IOException("unable to invoke [" + url + "], no response."); - } + public List read(boolean flush) throws IOException, GeneralSecurityException, SAXException { + long now = System.currentTimeMillis(); + if (elementsSorted == null) { + synchronized (elements) { + if (elementsSorted == null) { + int count = 100; + URL url = null; + if (lastKey != null) url = new URL(this.url.toExternalForm() + "?marker=" + lastKey); + + do { + if (url == null) url = isTruncated ? new URL(this.url.toExternalForm() + "?marker=" + this.lastKey) : this.url; + HTTPResponse rsp = HTTPEngine4Impl.get(url, null, null, BundleProvider.CONNECTION_TIMEOUT, true, null, null, null, null); + if (rsp != null) { + int sc = rsp.getStatusCode(); + if (sc < 200 || sc >= 300) throw new IOException("unable to invoke [" + url + "], status code [" + sc + "]"); + } + else { + throw new IOException("unable to invoke [" + url + "], no response."); + } - Reader r = null; - try { - init(new InputSource(r = IOUtil.getReader(rsp.getContentAsStream(), (Charset) null))); - } - finally { - url = null; - IOUtil.close(r); - } + Reader r = null; + try { + init(new InputSource(r = IOUtil.getReader(rsp.getContentAsStream(), (Charset) null))); + } + finally { + url = null; + IOUtil.close(r); + } - } - while (isTruncated || --count == 0); + } + while (isTruncated || --count == 0); - List list = new ArrayList<>(); - for (Element e: elements.values()) { - list.add(e); - } + List list = new ArrayList<>(); + for (Element e: elements.values()) { + list.add(e); + } - Collections.sort(list, new Comparator() { - @Override - public int compare(Element l, Element r) { - int cmp = l.getBundleDefinition().getName().compareTo(r.getBundleDefinition().getName()); - if (cmp != 0) return cmp; - return OSGiUtil.compare(l.getBundleDefinition().getVersion(), r.getBundleDefinition().getVersion()); + Collections.sort(list, new Comparator() { + @Override + public int compare(Element l, Element r) { + int cmp = l.getBundleDefinition().getName().compareTo(r.getBundleDefinition().getName()); + if (cmp != 0) return cmp; + return OSGiUtil.compare(l.getBundleDefinition().getVersion(), r.getBundleDefinition().getVersion()); + } + }); + elementsSorted = list; + } } - }); - - return list; + } + return elementsSorted; } /** @@ -806,7 +786,7 @@ public void createOSGiMavenMapping() throws IOException, GeneralSecurityExceptio Set has = new HashSet<>(); List infos; Info info; - for (Element e: read()) { + for (Element e: read(true)) { infos = mappings.get(e.bd.getName()); if (infos != null) continue; @@ -843,7 +823,7 @@ else if (!info.isComplete()) { public void whatcanBeRemovedFromS3() throws IOException, GeneralSecurityException, SAXException { URL url; - for (Element e: read()) { + for (Element e: read(true)) { url = getBundleAsURL(e.bd, false, null); if (url != null) { diff --git a/loader/build.xml b/loader/build.xml index 1a70c7bfaf..018e0c871e 100644 --- a/loader/build.xml +++ b/loader/build.xml @@ -2,7 +2,7 @@ - + diff --git a/loader/pom.xml b/loader/pom.xml index d1e89e7524..774ac0da32 100644 --- a/loader/pom.xml +++ b/loader/pom.xml @@ -3,7 +3,7 @@ org.lucee lucee - 6.1.1.78-RC + 6.1.1.79-RC jar Lucee Loader Build