Skip to content

Commit

Permalink
add multi threading to pom
Browse files Browse the repository at this point in the history
  • Loading branch information
@michaeloffner
michaeloffner committed Jul 13, 2024
1 parent c63e9a6 commit 89d5684
Show file tree
Hide file tree
Showing 3 changed files with 115 additions and 53 deletions.
52 changes: 38 additions & 14 deletions core/src/main/java/lucee/runtime/mvn/MavenUtil.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,9 @@
import java.util.Map;
import java.util.Map.Entry;
import java.util.Properties;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;

import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
Expand All @@ -21,7 +24,9 @@

import lucee.print;
import lucee.commons.io.IOUtil;
import lucee.commons.io.log.Log;
import lucee.commons.io.res.Resource;
import lucee.commons.lang.ExceptionUtil;
import lucee.commons.lang.SerializableObject;
import lucee.runtime.mvn.POMReader.Dependency;
import lucee.runtime.type.util.ListUtil;
Expand Down Expand Up @@ -98,26 +103,42 @@ public static Collection<Repository> getRepositories(List<POMReader.Repository>
}

public static List<POM> getDependencies(List<POMReader.Dependency> rawDependencies, POM current, POM parent, Map<String, String> properties, Resource localDirectory,
boolean management) throws IOException {
boolean management, Log log) throws IOException {
List<POM> dependencies = new ArrayList<>();
List<POM> parentDendencyManagement = null;

ExecutorService executor = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors());

if (parent != null) {
parentDendencyManagement = current.getDependencyManagement();
for (POM pom: parent.getDependencies()) {
dependencies.add(pom); // TODO clone?
}
}
if (rawDependencies != null) {
List<Future<POM>> futures = new ArrayList<>();
for (POMReader.Dependency rd: rawDependencies) {
GAVSO gavso = getDependency(rd, parent, current, properties, parentDendencyManagement, management);
if (gavso == null) continue;
POM p = POM.getInstance(localDirectory, current.getRepositories(), gavso.g, gavso.a, gavso.v, gavso.s, gavso.o, current.getDependencyScope(),
current.getDependencyScopeManagement());
dependencies.add(p);

Future<POM> future = executor.submit(() -> {
POM p = POM.getInstance(localDirectory, current.getRepositories(), gavso.g, gavso.a, gavso.v, gavso.s, gavso.o, current.getDependencyScope(),
current.getDependencyScopeManagement(), log);
p.initXML();
return p;
});
futures.add(future);
}
try {
for (Future<POM> future: futures) {
dependencies.add(future.get()); // Wait for init to complete
}
}
catch (Exception e) {
throw ExceptionUtil.toIOException(e);
}
}
executor.shutdown();
return dependencies;
}

Expand Down Expand Up @@ -214,8 +235,8 @@ private static POM getDendency(List<POM> dependencies, String groupId, String ar
return null;
}

public static List<POM> getDependencyManagement(List<POMReader.Dependency> rawDependencies, POM current, POM parent, Map<String, String> properties, Resource localDirectory)
throws IOException {
public static List<POM> getDependencyManagement(List<POMReader.Dependency> rawDependencies, POM current, POM parent, Map<String, String> properties, Resource localDirectory,
Log log) throws IOException {

List<POM> dependencies = new ArrayList<>();

Expand All @@ -233,7 +254,7 @@ public static List<POM> getDependencyManagement(List<POMReader.Dependency> rawDe
GAVSO gavso = getDependency(rd, parent, current, properties, null, true);
if (gavso == null) continue;
POM p = POM.getInstance(localDirectory, current.getRepositories(), gavso.g, gavso.a, gavso.v, gavso.s, gavso.o, current.getDependencyScope(),
current.getDependencyScopeManagement());
current.getDependencyScopeManagement(), log);
dependencies.add(p);
}
}
Expand Down Expand Up @@ -300,13 +321,14 @@ public static boolean hasPlaceholders(String str) {
return str != null && str.indexOf("${") != -1;
}

public static void downloadPOM(POM pom, Collection<Repository> repositories) throws IOException {
Resource res = pom.getPath();
public static void download(POM pom, Collection<Repository> repositories, String type, Log log) throws IOException {
Resource res = pom.getArtifact(type);
if (!res.isFile()) {
URL url = pom.getArtifact("pom", repositories);
print.o("download:" + url);
URL url = pom.getArtifact(type, repositories);
print.e("download:" + url);
if (log != null) log.info("maven", "download [" + url + "]");
try (CloseableHttpClient httpClient = HttpClients.createDefault()) {
HttpGet request = new HttpGet(pom.getArtifact("pom", repositories).toExternalForm());
HttpGet request = new HttpGet(pom.getArtifact(type, repositories).toExternalForm());
HttpResponse response = httpClient.execute(request);
HttpEntity entity = response.getEntity();
int sc = response.getStatusLine().getStatusCode();
Expand All @@ -326,7 +348,7 @@ public static void downloadPOM(POM pom, Collection<Repository> repositories) thr
}

public static POM toPOM(Resource localDirectory, Collection<Repository> repositories, POMReader.Dependency dependency, Map<String, String> properties, int dependencyScope,
int dependencyScopeManagement) throws IOException {
int dependencyScopeManagement, Log log) throws IOException {

return POM.getInstance(localDirectory, repositories,

Expand All @@ -338,7 +360,9 @@ public static POM toPOM(Resource localDirectory, Collection<Repository> reposito

null, null,

dependencyScope, dependencyScopeManagement
dependencyScope, dependencyScopeManagement,

log

);
}
Expand Down
104 changes: 67 additions & 37 deletions core/src/main/java/lucee/runtime/mvn/POM.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,10 @@
import org.xml.sax.SAXException;

import lucee.print;
import lucee.commons.io.log.Log;
import lucee.commons.io.res.Resource;
import lucee.commons.lang.ExceptionUtil;
import lucee.commons.lang.SerializableObject;
import lucee.commons.tree.TreeNode;
import lucee.runtime.mvn.POMReader.Dependency;
import lucee.runtime.op.Caster;
Expand Down Expand Up @@ -65,41 +67,48 @@ public class POM {
private String description;
private String url;

private Object token = new SerializableObject();

private POMReader reader;

public static POM getInstance(Resource localDirectory, String groupId, String version, String artifactId) {
return getInstance(localDirectory, null, groupId, artifactId, version, null, null, SCOPE_NOT_TEST, SCOPE_ALL);
private Log log;

private String artifactExtension;

public static POM getInstance(Resource localDirectory, String groupId, String version, String artifactId, Log log) {
return getInstance(localDirectory, null, groupId, artifactId, version, null, null, SCOPE_NOT_TEST, SCOPE_ALL, log);
}

public static POM getInstance(Resource localDirectory, Collection<Repository> repositories, String groupId, String version, String artifactId) {
return getInstance(localDirectory, repositories, groupId, artifactId, version, null, null, SCOPE_NOT_TEST, SCOPE_ALL);
public static POM getInstance(Resource localDirectory, Collection<Repository> repositories, String groupId, String version, String artifactId, Log log) {
return getInstance(localDirectory, repositories, groupId, artifactId, version, null, null, SCOPE_NOT_TEST, SCOPE_ALL, log);
}

public static POM getInstance(Resource localDirectory, String groupId, String artifactId, String version, int dependencyScope) {
return getInstance(localDirectory, null, groupId, artifactId, version, null, null, dependencyScope, POM.SCOPE_ALL);
public static POM getInstance(Resource localDirectory, String groupId, String artifactId, String version, int dependencyScope, Log log) {
return getInstance(localDirectory, null, groupId, artifactId, version, null, null, dependencyScope, POM.SCOPE_ALL, log);
}

public static POM getInstance(Resource localDirectory, Collection<Repository> repositories, String groupId, String artifactId, String version, int dependencyScope) {
return getInstance(localDirectory, repositories, groupId, artifactId, version, null, null, dependencyScope, POM.SCOPE_ALL);
public static POM getInstance(Resource localDirectory, Collection<Repository> repositories, String groupId, String artifactId, String version, int dependencyScope, Log log) {
return getInstance(localDirectory, repositories, groupId, artifactId, version, null, null, dependencyScope, POM.SCOPE_ALL, log);
}

public static POM getInstance(Resource localDirectory, Collection<Repository> repositories, String groupId, String artifactId, String version, int dependencyScope,
int dependencyScopeManagement) {
return getInstance(localDirectory, repositories, groupId, artifactId, version, null, null, dependencyScope, dependencyScopeManagement);
int dependencyScopeManagement, Log log) {
return getInstance(localDirectory, repositories, groupId, artifactId, version, null, null, dependencyScope, dependencyScopeManagement, log);
}

public static POM getInstance(Resource localDirectory, String groupId, String artifactId, String version, int dependencyScope, int dependencyScopeManagement) {
return getInstance(localDirectory, null, groupId, artifactId, version, null, null, dependencyScope, dependencyScopeManagement);
public static POM getInstance(Resource localDirectory, String groupId, String artifactId, String version, int dependencyScope, int dependencyScopeManagement, Log log) {
return getInstance(localDirectory, null, groupId, artifactId, version, null, null, dependencyScope, dependencyScopeManagement, log);
}

public static POM getInstance(Resource localDirectory, Collection<Repository> repositories, String groupId, String artifactId, String version, String scope, String optional,
int dependencyScope, int dependencyScopeManagement) {
int dependencyScope, int dependencyScopeManagement, Log log) {
String id = toId(localDirectory, groupId, artifactId, version, scope, optional, dependencyScope, dependencyScopeManagement);
POM pom = cache.get(id);
if (pom != null) {
return pom;
}
pom = new POM(localDirectory, repositories, groupId, artifactId, version, scope, optional, dependencyScope, dependencyScopeManagement);

pom = new POM(localDirectory, repositories, groupId, artifactId, version, scope, optional, dependencyScope, dependencyScopeManagement, log);
cache.put(id, pom);
return pom;
}
Expand All @@ -111,7 +120,7 @@ private static String toId(Resource localDirectory, String groupId, String artif
}

private POM(Resource localDirectory, Collection<Repository> repositories, String groupId, String artifactId, String version, String scope, String optional, int dependencyScope,
int dependencyScopeManagement) {
int dependencyScopeManagement, Log log) {
if (groupId == null) throw new IllegalArgumentException("groupId cannot be null");
if (artifactId == null) throw new IllegalArgumentException("artifactId cannot be null");
if (version == null) throw new IllegalArgumentException("version cannot be null");
Expand All @@ -130,32 +139,43 @@ private POM(Resource localDirectory, Collection<Repository> repositories, String
this.optional = optional == null ? null : optional.trim();
this.dependencyScopeManagement = dependencyScopeManagement;
this.dependencyScope = dependencyScope;
this.log = log;

cache.put(id(), this);
}

private void initXML() throws IOException {
if (isInitXML) return;
isInitXML = true;
if (debug) print.e("xxxxxx initXML " + this + " xxxxxx");
void initXML() throws IOException {
if (!isInitXML) {
synchronized (token) {
if (!isInitXML) {
if (debug) print.e("xxxxxx initXML " + this + " xxxxxx");

MavenUtil.downloadPOM(this, initRepositories);
MavenUtil.download(this, initRepositories, "pom", log);

reader = new POMReader(getPath());
try {
reader.read();
}
catch (SAXException e) {
IOException cause = ExceptionUtil.toIOException(e);
IOException ioe = new IOException("failed to load pom file [" + getArtifact("pom", initRepositories) + "]");
ExceptionUtil.initCauseEL(ioe, cause);
throw ioe;
}
reader = new POMReader(getPath());
try {
reader.read();
}
catch (SAXException e) {
IOException cause = ExceptionUtil.toIOException(e);
IOException ioe = new IOException("failed to load pom file [" + getArtifact("pom", initRepositories) + "]");
ExceptionUtil.initCauseEL(ioe, cause);
throw ioe;
}
this.packaging = reader.getPackaging();
this.artifactExtension = this.packaging;
if ("bundle".equalsIgnoreCase(artifactExtension)) this.artifactExtension = "jar";

this.name = reader.getName();
this.description = reader.getDescription();
this.url = reader.getURL();

if (this.packaging != null && !"pom".equalsIgnoreCase(this.packaging)) MavenUtil.download(this, initRepositories, artifactExtension, log);

this.packaging = reader.getPackaging();
this.name = reader.getName();
this.description = reader.getDescription();
this.url = reader.getURL();
isInitXML = true;
}
}
}
}

private void initParent() throws IOException {
Expand All @@ -167,7 +187,7 @@ private void initParent() throws IOException {
Dependency p = reader.getParent();
if (p != null) {
// chicken egg, because there is no parent yet, this cannot use properties from parent
this.parent = MavenUtil.toPOM(this.localDirectory, initRepositories, p, reader.getProperties(), dependencyScope, dependencyScopeManagement);
this.parent = MavenUtil.toPOM(this.localDirectory, initRepositories, p, reader.getProperties(), dependencyScope, dependencyScopeManagement, log);
parent.init();
}
}
Expand All @@ -194,7 +214,7 @@ private void initDependencies() throws IOException {
isInitDependencies = true;
if (debug) print.e("xxxxxx initDependencies " + this + " xxxxxx");
initProperties();
if (dependencyScope > 0) dependencies = MavenUtil.getDependencies(reader.getDependencies(), this, parent, properties, localDirectory, false);
if (dependencyScope > 0) dependencies = MavenUtil.getDependencies(reader.getDependencies(), this, parent, properties, localDirectory, false, log);
}

private void initDependencyManagement() throws IOException {
Expand All @@ -203,7 +223,8 @@ private void initDependencyManagement() throws IOException {
if (debug) print.e("xxxxxx initDependencyManagement " + this + " xxxxxx");
initProperties();

if (dependencyScopeManagement > 0) dependencyManagement = MavenUtil.getDependencyManagement(reader.getDependencyManagements(), this, parent, properties, localDirectory);
if (dependencyScopeManagement > 0)
dependencyManagement = MavenUtil.getDependencyManagement(reader.getDependencyManagements(), this, parent, properties, localDirectory, log);

}

Expand Down Expand Up @@ -289,6 +310,15 @@ public Resource getPath() {
return local(localDirectory, "pom");
}

Resource getArtifact(String type) {
return local(localDirectory, type);
}

public Resource getArtifact() {
if (artifactExtension == null) return null;
return local(localDirectory, artifactExtension);
}

public boolean isInit() {
return isInit;
}
Expand Down
12 changes: 10 additions & 2 deletions core/src/main/java/lucee/runtime/mvn/Test.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -74,8 +74,9 @@ public static void main(String[] args) throws Exception {
* for (Artifact a: examples) { print.e("------------ " + a); print.e(maven.download(a.groupId,
* a.artifactId, a.version, true, false)); }
*/
long start = System.currentTimeMillis();
for (GAVSO gav: arr) {
POM pom = POM.getInstance(dir, gav.g, gav.a, gav.v, POM.SCOPE_NOT_TEST);
POM pom = POM.getInstance(dir, gav.g, gav.a, gav.v, POM.SCOPE_NOT_TEST, null);
print.e("==========================================");
print.e(pom.getName());
print.e(pom);
Expand All @@ -87,6 +88,12 @@ public static void main(String[] args) throws Exception {
print.e("--- packaging ---");
print.e(pom.getPackaging());

print.e("--- path ---");
print.e(pom.getPath());

print.e("--- artifact ---");
print.e(pom.getArtifact());

print.e("--- parents ---");
// print.e(pom.getAllParentsAsTree());
print.e(pom.getAllParents());
Expand All @@ -104,13 +111,14 @@ public static void main(String[] args) throws Exception {
print.e("--- dependencies ---");
// print.e(getDependenciesAsTrees(pom, true));
print.e(pom.getAllDependencies());
print.e(System.currentTimeMillis() - start);

// pom.getScope();
// print.e(pom.getDependencyManagement());

// print.e(maven.getDependencies(groupId, artifactId, version, true, false, true));

break;
// break;
}
}

Expand Down

0 comments on commit 89d5684

Please sign in to comment.