Skip to content

Commit

Permalink
Optionally specify limit for number of entities in a record.
Browse files Browse the repository at this point in the history
This is a brute-force approach to dealing with OOM situations when Alma records have an excessive number of items (e.g. 99374518570506441: >12000 entities = ~10 GB heap for the Record instance).

Use Metafix instance setter `setMaxEntityCount(int)` or set system property `org.metafacture.metafix.maxEntityCount=<int>`.

Alternative options:

- Increase maximum heap size for JVM.
- Significantly reduce memory requirement for Record instances.
  • Loading branch information
blackwinter committed Oct 10, 2024
1 parent 2d57726 commit f70996f
Show file tree
Hide file tree
Showing 2 changed files with 38 additions and 1 deletion.
10 changes: 10 additions & 0 deletions metafix-runner/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -47,3 +47,13 @@ application {
applicationDefaultJvmArgs = ["-agentlib:hprof=heap=sites,cpu=samples,depth=${depth},cutoff=${cutoff},file=${file}.hprof.txt"]
}
}

tasks.withType(JavaExec) {
doFirst {
def prefix = project.group + '.'

System.properties.each { k, v ->
if (k.startsWith(prefix)) systemProperties[k] = v
}
}
}
29 changes: 28 additions & 1 deletion metafix/src/main/java/org/metafacture/metafix/Metafix.java
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,7 @@ public class Metafix implements StreamPipe<StreamReceiver>, Maps { // checkstyle
private boolean repeatedFieldsToEntities;
private boolean strictnessHandlesProcessExceptions;
private int entityCount;
private int maxEntityCount = Integer.getInteger("org.metafacture.metafix.maxEntityCount", -1);

public Metafix() {
this(NO_VARS);
Expand Down Expand Up @@ -305,22 +306,36 @@ public void startEntity(final String name) {
throw new IllegalArgumentException("Entity name must not be null.");
}

++entityCount;
if (maxEntityCountExceeded()) {
LOG.debug("Maximum number of entities exceeded: {}/{}", entityCount, maxEntityCount);
return;
}

final Value value = isArrayName(name) ? Value.newArray() : Value.newHash();
addValue(name, value);
entities.add(value);

entityCountStack.push(++entityCount);
entityCountStack.push(entityCount);
flattener.startEntity(name);
}

@Override
public void endEntity() {
if (maxEntityCountExceeded()) {
return;
}

entityCountStack.pop();
flattener.endEntity();
}

@Override
public void literal(final String name, final String value) {
if (entityCountStack.size() > 1 && maxEntityCountExceeded()) {
return;
}

LOG.debug("Putting '{}': '{}'", name, value);
flattener.literal(name, value);
}
Expand Down Expand Up @@ -430,6 +445,18 @@ public String getEntityMemberName() {
return entityMemberName;
}

public void setMaxEntityCount(final int maxEntityCount) {
this.maxEntityCount = maxEntityCount;
}

public int getMaxEntityCount() {
return maxEntityCount;
}

private boolean maxEntityCountExceeded() {
return maxEntityCount >= 0 && entityCount > maxEntityCount;
}

public enum Strictness {

/**
Expand Down

0 comments on commit f70996f

Please sign in to comment.