Skip to content

Commit

Permalink
LDEV-4356 - cache for a day and also cache the read method
Browse files Browse the repository at this point in the history
  • Loading branch information
@michaeloffner
michaeloffner committed Aug 30, 2024
1 parent 83b1f91 commit 8fd163b
Show file tree
Hide file tree
Showing 3 changed files with 54 additions and 74 deletions.
124 changes: 52 additions & 72 deletions core/src/main/java/lucee/runtime/config/s3/BundleProvider.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -68,13 +68,12 @@
import lucee.runtime.type.StructImpl;
import lucee.runtime.type.dt.DateTime;
import lucee.runtime.type.util.ListUtil;
import lucee.runtime.util.Pack200Util;
import lucee.transformer.library.function.FunctionLibEntityResolver;
import lucee.transformer.library.function.FunctionLibException;

public final class BundleProvider extends DefaultHandler {
public static final int CONNECTION_TIMEOUT = 1000;
private static final long MAX_AGE = 10000;
private static final long MAX_AGE = 24 * 60 * 60 * 1000;
private static final int MAX_REDIRECTS = 10;

private static URL DEFAULT_PROVIDER_LIST = null;
Expand Down Expand Up @@ -149,6 +148,7 @@ public static URL getDefaultProviderDetailMvn() {
private final URL url;
private boolean insideContents;
private Map<String, Element> elements = new LinkedHashMap<>();
private List<Element> elementsSorted;
private Element element;
private boolean isTruncated;
private String lastKey;
Expand Down Expand Up @@ -245,7 +245,7 @@ public URL getBundleAsURL(BundleDefinition bd, boolean includeS3) throws Malform
// S3: we loop through all records and S3 and pick one
if (url == null) {
try {
for (Element e: read()) {
for (Element e: read(false)) {
if (bd.equals(e.getBundleDefinition())) {
url = e.getJAR();
if (url != null) return url;
Expand Down Expand Up @@ -370,8 +370,6 @@ private File deployBundledBundle(File bundleDirectory, String symbolicName, Stri
String sub = "bundles/";
String nameAndVersion = symbolicName + "|" + symbolicVersion;
String osgiFileName = symbolicName + "-" + symbolicVersion + ".jar";
String pack20Ext = ".jar.pack.gz";
boolean isPack200 = false;

// first we look for an exact match
InputStream is = getClass().getResourceAsStream("bundles/" + osgiFileName);
Expand All @@ -380,30 +378,14 @@ private File deployBundledBundle(File bundleDirectory, String symbolicName, Stri
if (is != null) LogUtil.log(Logger.LOG_DEBUG, "deploy", "bundle-download", "Found ]/bundles/" + osgiFileName + "] in lucee.jar");
else LogUtil.log(Logger.LOG_INFO, "deploy", "bundle-download", "Could not find [/bundles/" + osgiFileName + "] in lucee.jar");

if (is == null) {
is = getClass().getResourceAsStream("bundles/" + osgiFileName + pack20Ext);
if (is == null) is = getClass().getResourceAsStream("/bundles/" + osgiFileName + pack20Ext);
isPack200 = true;

if (is != null) LogUtil.log(Logger.LOG_DEBUG, "deploy", "bundle-download", "Found [/bundles/" + osgiFileName + pack20Ext + "] in lucee.jar");
else LogUtil.log(Logger.LOG_INFO, "deploy", "bundle-download", "Could not find [/bundles/" + osgiFileName + pack20Ext + "] in lucee.jar");
}
if (is != null) {
File temp = null;
try {
// copy to temp file
temp = File.createTempFile("bundle", ".tmp");
LogUtil.log(Logger.LOG_DEBUG, "deploy", "bundle-download", "Copying [lucee.jar!/bundles/" + osgiFileName + pack20Ext + "] to [" + temp + "]");
LogUtil.log(Logger.LOG_DEBUG, "deploy", "bundle-download", "Copying [lucee.jar!/bundles/" + osgiFileName + "] to [" + temp + "]");
Util.copy(new BufferedInputStream(is), new FileOutputStream(temp), true, true);

if (isPack200) {
File temp2 = File.createTempFile("bundle", ".tmp2");
Pack200Util.pack2Jar(temp, temp2);
LogUtil.log(Logger.LOG_DEBUG, "deploy", "bundle-download", "Upack [" + temp + "] to [" + temp2 + "]");
temp.delete();
temp = temp2;
}

// adding bundle
File trg = new File(bundleDirectory, osgiFileName);
FileUtil.move(temp, trg);
Expand All @@ -418,8 +400,6 @@ private File deployBundledBundle(File bundleDirectory, String symbolicName, Stri
}
}

// now we search the current jar as an external zip what is slow (we do not support pack200 in this
// case)
// this also not works with windows
if (SystemUtil.isWindows()) return null;
ZipEntry entry;
Expand All @@ -437,8 +417,7 @@ private File deployBundledBundle(File bundleDirectory, String symbolicName, Stri
temp = null;
path = entry.getName().replace('\\', '/');
if (path.startsWith("/")) path = path.substring(1); // some zip path start with "/" some not
isPack200 = false;
if (path.startsWith(sub) && (path.endsWith(".jar") /* || (isPack200=path.endsWith(".jar.pack.gz")) */)) { // ignore non jar files or file from elsewhere
if (path.startsWith(sub) && (path.endsWith(".jar"))) { // ignore non jar files or file from elsewhere
index = path.lastIndexOf('/') + 1;
if (index == sub.length()) { // ignore sub directories
name = path.substring(index);
Expand All @@ -447,11 +426,6 @@ private File deployBundledBundle(File bundleDirectory, String symbolicName, Stri
temp = File.createTempFile("bundle", ".tmp");
Util.copy(zis, new FileOutputStream(temp), false, true);

/*
* if(isPack200) { File temp2 = File.createTempFile("bundle", ".tmp2"); Pack200Util.pack2Jar(temp,
* temp2); temp.delete(); temp=temp2; name=name.substring(0,name.length()-".pack.gz".length()); }
*/

bundleInfo = BundleLoader.loadBundleInfo(temp);
if (bundleInfo != null && nameAndVersion.equals(bundleInfo)) {
File trg = new File(bundleDirectory, name);
Expand Down Expand Up @@ -483,50 +457,56 @@ private File deployBundledBundle(File bundleDirectory, String symbolicName, Stri
// 1146:size:305198;bundle:name:xmlgraphics.batik.awt.util;version:version EQ 1.8.0;;last-mod:{ts
// '2024-01-14 22:32:05'};

public List<Element> read() throws IOException, GeneralSecurityException, SAXException {
int count = 100;
URL url = null;

if (lastKey != null) url = new URL(this.url.toExternalForm() + "?marker=" + lastKey);

do {
if (url == null) url = isTruncated ? new URL(this.url.toExternalForm() + "?marker=" + this.lastKey) : this.url;
HTTPResponse rsp = HTTPEngine4Impl.get(url, null, null, BundleProvider.CONNECTION_TIMEOUT, true, null, null, null, null);
if (rsp != null) {
int sc = rsp.getStatusCode();
if (sc < 200 || sc >= 300) throw new IOException("unable to invoke [" + url + "], status code [" + sc + "]");
}
else {
throw new IOException("unable to invoke [" + url + "], no response.");
}
public List<Element> read(boolean flush) throws IOException, GeneralSecurityException, SAXException {
long now = System.currentTimeMillis();
if (elementsSorted == null) {
synchronized (elements) {
if (elementsSorted == null) {
int count = 100;
URL url = null;
if (lastKey != null) url = new URL(this.url.toExternalForm() + "?marker=" + lastKey);

do {
if (url == null) url = isTruncated ? new URL(this.url.toExternalForm() + "?marker=" + this.lastKey) : this.url;
HTTPResponse rsp = HTTPEngine4Impl.get(url, null, null, BundleProvider.CONNECTION_TIMEOUT, true, null, null, null, null);
if (rsp != null) {
int sc = rsp.getStatusCode();
if (sc < 200 || sc >= 300) throw new IOException("unable to invoke [" + url + "], status code [" + sc + "]");
}
else {
throw new IOException("unable to invoke [" + url + "], no response.");
}

Reader r = null;
try {
init(new InputSource(r = IOUtil.getReader(rsp.getContentAsStream(), (Charset) null)));
}
finally {
url = null;
IOUtil.close(r);
}
Reader r = null;
try {
init(new InputSource(r = IOUtil.getReader(rsp.getContentAsStream(), (Charset) null)));
}
finally {
url = null;
IOUtil.close(r);
}

}
while (isTruncated || --count == 0);
}
while (isTruncated || --count == 0);

List<Element> list = new ArrayList<>();
for (Element e: elements.values()) {
list.add(e);
}
List<Element> list = new ArrayList<>();
for (Element e: elements.values()) {
list.add(e);
}

Collections.sort(list, new Comparator<Element>() {
@Override
public int compare(Element l, Element r) {
int cmp = l.getBundleDefinition().getName().compareTo(r.getBundleDefinition().getName());
if (cmp != 0) return cmp;
return OSGiUtil.compare(l.getBundleDefinition().getVersion(), r.getBundleDefinition().getVersion());
Collections.sort(list, new Comparator<Element>() {
@Override
public int compare(Element l, Element r) {
int cmp = l.getBundleDefinition().getName().compareTo(r.getBundleDefinition().getName());
if (cmp != 0) return cmp;
return OSGiUtil.compare(l.getBundleDefinition().getVersion(), r.getBundleDefinition().getVersion());
}
});
elementsSorted = list;
}
}
});

return list;
}
return elementsSorted;
}

/**
Expand Down Expand Up @@ -806,7 +786,7 @@ public void createOSGiMavenMapping() throws IOException, GeneralSecurityExceptio
Set<String> has = new HashSet<>();
List<Info> infos;
Info info;
for (Element e: read()) {
for (Element e: read(true)) {
infos = mappings.get(e.bd.getName());
if (infos != null) continue;

Expand Down Expand Up @@ -843,7 +823,7 @@ else if (!info.isComplete()) {

public void whatcanBeRemovedFromS3() throws IOException, GeneralSecurityException, SAXException {
URL url;
for (Element e: read()) {
for (Element e: read(true)) {
url = getBundleAsURL(e.bd, false, null);

if (url != null) {
Expand Down
2 changes: 1 addition & 1 deletion loader/build.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
<project default="core" basedir="." name="Lucee"
xmlns:resolver="antlib:org.apache.maven.resolver.ant">

<property name="version" value="6.1.1.78-RC"/>
<property name="version" value="6.1.1.79-RC"/>

<taskdef uri="antlib:org.apache.maven.resolver.ant" resource="org/apache/maven/resolver/ant/antlib.xml">
<classpath>
Expand Down
2 changes: 1 addition & 1 deletion loader/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@

<groupId>org.lucee</groupId>
<artifactId>lucee</artifactId>
<version>6.1.1.78-RC</version>
<version>6.1.1.79-RC</version>
<packaging>jar</packaging>

<name>Lucee Loader Build</name>
Expand Down

0 comments on commit 8fd163b

Please sign in to comment.