Skip to content

Commit

Permalink
Improve a lot of minor things
Browse files Browse the repository at this point in the history
  • Loading branch information
dr0i committed Aug 18, 2023
1 parent 778b560 commit f4352f2
Show file tree
Hide file tree
Showing 18 changed files with 75 additions and 90 deletions.
6 changes: 3 additions & 3 deletions app/controllers/Accept.java
Original file line number Diff line number Diff line change
Expand Up @@ -28,10 +28,10 @@ enum Format {
N_TRIPLE("nt", "application/n-triples", "text/plain"), //
TURTLE("ttl", "text/turtle", "application/x-turtle");

String[] types;
String queryParamString;
final String[] types;
final String queryParamString;

private Format(String format, String... types) {
Format(String format, String... types) {
this.queryParamString = format;
this.types = types;
}
Expand Down
11 changes: 6 additions & 5 deletions app/controllers/Application.java
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@
import org.elasticsearch.index.query.GeoPolygonQueryBuilder;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.sort.SortOrder;
import org.elasticsearch.search.sort.SortParseElement;

Expand Down Expand Up @@ -510,7 +511,7 @@ private static String defaultFields() {

private static String searchQueryResult(String q, String location, int from,
int size, String aggregations) {
String result = null;
String result;
if (location == null || location.isEmpty()) {
result = buildSimpleQuery(q, from, size, aggregations);
} else {
Expand Down Expand Up @@ -619,8 +620,8 @@ static String[] defaultAggregations() {

private static String returnAsJson(SearchResponse queryResponse) {
List<Map<String, Object>> hits =
Arrays.asList(queryResponse.getHits().hits()).stream()
.map(hit -> hit.getSource()).collect(Collectors.toList());
Arrays.stream(queryResponse.getHits().hits())
.map(SearchHit::getSource).collect(Collectors.toList());
ObjectNode object = Json.newObject();
object.put("@context",
CONFIG.getString("host") + routes.Application.context());
Expand Down Expand Up @@ -714,8 +715,8 @@ private static Result resultFor(String id, JsonNode json, String format) {

private static Pair<String, String> contentAndType(JsonNode responseJson,
String responseFormat) {
String content = "";
String contentType = "";
String content;
String contentType;
switch (responseFormat) {
case "rdf": {
content = RdfConverter.toRdf(responseJson.toString(), RdfFormat.RDF_XML);
Expand Down
44 changes: 20 additions & 24 deletions app/controllers/Index.java
Original file line number Diff line number Diff line change
Expand Up @@ -69,14 +69,14 @@ public ConfigurableNode(Settings settings,
}
}

private static Settings clientSettings =
private final static Settings clientSettings =
Settings.settingsBuilder().put("path.home", ".")
.put("http.port", Application.CONFIG.getString("index.es.port.http"))
.put("transport.tcp.port",
Application.CONFIG.getString("index.es.port.tcp"))
.put("script.default_lang", "native").build();

private static Node node = new ConfigurableNode(
private final static Node node = new ConfigurableNode(
nodeBuilder().settings(clientSettings).local(true).getSettings().build(),
Arrays.asList(BundlePlugin.class, LocationAggregation.class, Zero.class))
.start();
Expand Down Expand Up @@ -112,8 +112,8 @@ public static void initialize(String pathToJson) throws IOException {
long minimumSize =
Long.parseLong(Application.CONFIG.getString("index.file.minsize"));
if (new File(pathToJson).length() >= minimumSize) {
createEmptyIndex(CLIENT, INDEX_NAME, "conf/index-settings.json");
indexData(CLIENT, pathToJson, INDEX_NAME);
createEmptyIndex();
indexData(pathToJson);
} else {
throw new IllegalArgumentException(
"File not large enough: " + pathToJson);
Expand Down Expand Up @@ -184,40 +184,36 @@ private static SearchRequestBuilder withAggregations(
return searchRequest;
}

static void createEmptyIndex(final Client aClient, final String aIndexName,
final String aPathToIndexSettings) throws IOException {
deleteIndex(aClient, aIndexName);
static void createEmptyIndex() throws IOException {
deleteIndex(Index.CLIENT, Index.INDEX_NAME);
CreateIndexRequestBuilder cirb =
aClient.admin().indices().prepareCreate(aIndexName);
if (aPathToIndexSettings != null) {
String settingsMappings = Files.lines(Paths.get(aPathToIndexSettings))
.collect(Collectors.joining());
cirb.setSource(settingsMappings);
}
Index.CLIENT.admin().indices().prepareCreate(Index.INDEX_NAME);
String settingsMappings = Files.lines(Paths.get("conf/index-settings.json"))
.collect(Collectors.joining());
cirb.setSource(settingsMappings);
cirb.execute().actionGet();
aClient.admin().indices().refresh(new RefreshRequest()).actionGet();
Index.CLIENT.admin().indices().refresh(new RefreshRequest()).actionGet();
}

static void indexData(final Client aClient, final String aPath,
final String aIndex) throws IOException {
final BulkRequestBuilder bulkRequest = aClient.prepareBulk();
static void indexData(final String aPath) throws IOException {
final BulkRequestBuilder bulkRequest = Index.CLIENT.prepareBulk();
try (BufferedReader br =
new BufferedReader(new InputStreamReader(new FileInputStream(aPath),
new BufferedReader(new InputStreamReader(Files.newInputStream(Paths.get(aPath)),
StandardCharsets.UTF_8))) {
readData(bulkRequest, br, aClient, aIndex);
readData(bulkRequest, br);
}
bulkRequest.execute().actionGet();
aClient.admin().indices().refresh(new RefreshRequest()).actionGet();
Index.CLIENT.admin().indices().refresh(new RefreshRequest()).actionGet();
}

private static void readData(final BulkRequestBuilder bulkRequest,
final BufferedReader br, final Client client, final String aIndex)
final BufferedReader br)
throws IOException {
final ObjectMapper mapper = new ObjectMapper();
String line;
int currentLine = 1;
String organisationData = null;
String[] idUriParts = null;
String organisationData;
String[] idUriParts;
String organisationId = null;

// First line: index with id, second line: source
Expand All @@ -229,7 +225,7 @@ private static void readData(final BulkRequestBuilder bulkRequest,
organisationId = idUriParts[idUriParts.length - 1].replace("#!", "");
} else {
organisationData = line;
bulkRequest.add(client.prepareIndex(aIndex, INDEX_TYPE, organisationId)
bulkRequest.add(Index.CLIENT.prepareIndex(Index.INDEX_NAME, INDEX_TYPE, organisationId)
.setSource(organisationData));
}
currentLine++;
Expand Down
2 changes: 1 addition & 1 deletion app/controllers/RdfConverter.java
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ public class RdfConverter {
* RDF serialization formats.
*/
@SuppressWarnings("javadoc")
public static enum RdfFormat {
public enum RdfFormat {
RDF_XML("RDF/XML"), //
N_TRIPLE("N-TRIPLE"), //
TURTLE("TURTLE");
Expand Down
5 changes: 2 additions & 3 deletions app/controllers/Reconcile.java
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,7 @@ public static Result reconcile() {

private static List<JsonNode> mapToResults(String mainQuery,
SearchHits searchHits) {
return Arrays.asList(searchHits.getHits()).stream().map(hit -> {
return Arrays.stream(searchHits.getHits()).map(hit -> {
Map<String, Object> map = hit.getSource();
ObjectNode resultForHit = Json.newObject();
resultForHit.put("id", hit.getId());
Expand All @@ -101,8 +101,7 @@ private static SearchResponse executeQuery(Entry<String, JsonNode> entry,
QueryBuilders.simpleQueryStringQuery(queryString);
BoolQueryBuilder boolQuery = QueryBuilders.boolQuery().must(stringQuery)
.must(QueryBuilders.existsQuery("type"));
SearchResponse response = Index.executeQuery(0, limit, boolQuery, "");
return response;
return Index.executeQuery(0, limit, boolQuery, "");
}

private static String buildQueryString(Entry<String, JsonNode> entry) {
Expand Down
13 changes: 7 additions & 6 deletions app/transformation/CsvExport.java
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@
*/
public class CsvExport {

private JsonNode organisations;
private final JsonNode organisations;

/**
* @param json The organisations JSON data to export
Expand All @@ -35,23 +35,24 @@ public CsvExport(String json) {
* @return The data for the given fields in CSV format
*/
public String of(String fields) {
String csv = fields + "\n";
StringBuilder csv = new StringBuilder(fields + "\n");
for (Iterator<JsonNode> iter = organisations.elements(); iter.hasNext();) {
JsonNode org = iter.next();
csv += Arrays.asList(fields.split(",")).stream().map(field -> {
csv.append(Arrays.asList(fields.split(",")).stream().map(field -> {
try {
Object value = JsonPath.read(Configuration.defaultConfiguration()
.jsonProvider().parse(org.toString()), "$." + field);
return String.format("\"%s\"",
value.toString().replaceAll("\"", "\"\""));
} catch (PathNotFoundException x) {
}
catch (PathNotFoundException x) {
Logger.trace(x.getMessage());
// https://www.w3.org/TR/2015/REC-tabular-data-model-20151217/#empty-and-quoted-cells
return "";
}
}).collect(Collectors.joining(",")) + "\n";
}).collect(Collectors.joining(","))).append("\n");
}
return csv;
return csv.toString();
}

}
4 changes: 2 additions & 2 deletions app/transformation/GeoLookupMap.java
Original file line number Diff line number Diff line change
Expand Up @@ -29,9 +29,9 @@ public class GeoLookupMap extends HashMap<String, String> {
Application.CONFIG.getString("transformation.geo.lookup.server");
private static final Double THRESHOLD =
Application.CONFIG.getDouble("transformation.geo.lookup.threshold");
private LookupType lookupType;
private final LookupType lookupType;

static enum LookupType {
enum LookupType {
LAT, LON
}

Expand Down
7 changes: 3 additions & 4 deletions app/transformation/TransformAll.java
Original file line number Diff line number Diff line change
Expand Up @@ -51,11 +51,11 @@ public static void process(String startOfUpdates, int intervalSize,
final String outputPath, String geoServer) throws IOException {
String dbsOutput = outputPath + "-dbs";
String sigelOutput = outputPath + "-sigel";
TransformSigel.processBulk(startOfUpdates, intervalSize, sigelOutput, geoServer); //Start processing Sigel pica binary bulk.
TransformSigel.processBulk(sigelOutput, geoServer); //Start processing Sigel pica binary bulk.
TransformSigel.processUpdates(startOfUpdates, intervalSize, sigelOutput, geoServer); //Start process Sigel Pica XML Updates via OAI-PMH.
TransformDbs.process(dbsOutput, geoServer); //Start process DBS data.

// DBS-Data, Sigel Bulk and Updates are joined in a single ES-Bulk-file.
// DBS-Data, Sigel Bulk and Updates are joined in a single ES-Bulk-file.
// DBS data first, so that ES prefers Sigel entries that come later and overwrite DBS entries if available.
try (FileWriter resultWriter = new FileWriter(outputPath)) {
writeAll(dbsOutput, resultWriter);
Expand All @@ -76,10 +76,9 @@ private static void writeAll(String dbsOutput, FileWriter resultWriter)
}

static JsonToElasticsearchBulk esBulk() {
final JsonToElasticsearchBulk esBulk = new JsonToElasticsearchBulk("id",
return new JsonToElasticsearchBulk("id",
Application.CONFIG.getString("index.es.type"),
Application.CONFIG.getString("index.es.name"));
return esBulk;
}

static Metafix fixEnriched(String geoLookupServer) throws FileNotFoundException {
Expand Down
9 changes: 3 additions & 6 deletions app/transformation/TransformSigel.java
Original file line number Diff line number Diff line change
Expand Up @@ -40,18 +40,15 @@
*
*/
public class TransformSigel {

static final String DUMP_TOP_LEVEL_TAG = "collection";
static final String DUMP_ENTITY = "record";
static final String UPDATE_TOP_LEVEL_TAG = "harvest";
static final String DUMP_TOP_LEVEL_TAG = "collection";
static final String UPDATE_ENTITY = "metadata";
static final String XPATH =
"/*[local-name() = 'record']/*[local-name() = 'global']/*[local-name() = 'tag'][@id='008H']/*[local-name() = 'subf'][@id='e']";
static final String DUMP_XPATH = "/" + DUMP_TOP_LEVEL_TAG + "/" + XPATH;

// This opens the pica binary bulk we have, transforms them and saves them as JSON ES Bulk.
static void processBulk(String startOfUpdates, int intervalSize,
final String outputPath, String geoLookupServer) throws IOException {
// This opens the pica binary bulk we have, transforms them and saves them as JSON ES Bulk.
static void processBulk(final String outputPath, String geoLookupServer) throws IOException {
final FileOpener dumpOpener = new FileOpener();
PicaDecoder picaDecoder = new PicaDecoder();
picaDecoder.setNormalizeUTF8(true);
Expand Down
4 changes: 2 additions & 2 deletions test/controllers/AcceptIntegrationTest.java
Original file line number Diff line number Diff line change
Expand Up @@ -83,8 +83,8 @@ public static Collection<Object[]> data() {
{ fakeRequest(GET, "/organisations/DE-38").withHeader("Accept", "application/n-triples"), /*->*/ "application/n-triples" }});
} // @formatter:on

private FakeRequest fakeRequest;
private String contentType;
private final FakeRequest fakeRequest;
private final String contentType;

public AcceptIntegrationTest(FakeRequest request, String contentType) {
this.fakeRequest = request;
Expand Down
6 changes: 3 additions & 3 deletions test/controllers/AcceptUnitTest.java
Original file line number Diff line number Diff line change
Expand Up @@ -70,9 +70,9 @@ public static Collection<Object[]> data() {
{ fakeRequest().withHeader("Accept", "text/html"), "json", /*->*/ "json" }});
} // @formatter:on

private FakeRequest fakeRequest;
private String passedFormat;
private String expectedFormat;
private final FakeRequest fakeRequest;
private final String passedFormat;
private final String expectedFormat;

public AcceptUnitTest(FakeRequest request, String givenFormat,
String expectedFormat) {
Expand Down
9 changes: 3 additions & 6 deletions test/controllers/IntegrationTest.java
Original file line number Diff line number Diff line change
Expand Up @@ -35,12 +35,9 @@ public class IntegrationTest extends ElasticsearchTest {
@Test
public void testMainPage() {
running(testServer(3333, fakeApplication(inMemoryDatabase())), HTMLUNIT,
new Callback<TestBrowser>() {
@Override
public void invoke(TestBrowser browser) {
browser.goTo("http://localhost:3333/organisations");
assertThat(browser.pageSource()).contains("lobid-organisations");
}
browser -> {
browser.goTo("http://localhost:3333/organisations");
assertThat(browser.pageSource()).contains("lobid-organisations");
});
}

Expand Down
6 changes: 3 additions & 3 deletions test/index/AggregationsTest.java
Original file line number Diff line number Diff line change
Expand Up @@ -51,9 +51,9 @@ public static Collection<Object[]> data() {
{ "&aggregations=invalid", /*->*/ 0, Status.BAD_REQUEST },});
} // @formatter:on

private FakeRequest fakeRequest;
private int expectedNumberOfAggragations;
private int expectedResponseStatus;
private final FakeRequest fakeRequest;
private final int expectedNumberOfAggragations;
private final int expectedResponseStatus;

public AggregationsTest(String param, int expectedNumberOfAggragations,
int status) {
Expand Down
12 changes: 5 additions & 7 deletions test/index/ElasticsearchTest.java
Original file line number Diff line number Diff line change
Expand Up @@ -38,13 +38,11 @@ public static void closeElasticSearch() {

public static SearchResponse search(final String aField,
final String aValue) {
SearchResponse responseOfSearch =
client.prepareSearch(Application.CONFIG.getString("index.es.name"))
.setTypes(Application.CONFIG.getString("index.es.type"))
.setSearchType(SearchType.DFS_QUERY_AND_FETCH)
.setQuery(QueryBuilders.matchQuery(aField, aValue)).execute()
.actionGet();
return responseOfSearch;
return client.prepareSearch(Application.CONFIG.getString("index.es.name"))
.setTypes(Application.CONFIG.getString("index.es.type"))
.setSearchType(SearchType.DFS_QUERY_AND_FETCH)
.setQuery(QueryBuilders.matchQuery(aField, aValue)).execute()
.actionGet();
}

}
4 changes: 2 additions & 2 deletions test/index/RdfConverterTests.java
Original file line number Diff line number Diff line change
Expand Up @@ -32,11 +32,11 @@ public class RdfConverterTests extends ElasticsearchTest {

@Parameters(name = "{0}")
public static Collection<Object[]> data() {
return Arrays.asList(RdfFormat.values()).stream()
return Arrays.stream(RdfFormat.values())
.map(format -> new Object[] { format }).collect(Collectors.toList());
}

private RdfFormat format;
private final RdfFormat format;

public RdfConverterTests(RdfFormat format) {
this.format = format;
Expand Down
10 changes: 4 additions & 6 deletions test/index/TestGeoSearch.java
Original file line number Diff line number Diff line change
Expand Up @@ -19,12 +19,10 @@ private static SearchResponse geoSearch(double lon, double lat) {
QueryBuilder geoQuery = geoPolygonQuery("location.geo")
.addPoint(lat - 1, lon - 1).addPoint(lat - 1, lon + 1)
.addPoint(lat + 1, lon - 1).addPoint(lat + 1, lon + 1);
SearchResponse responseOfSearch =
client.prepareSearch(Application.CONFIG.getString("index.es.name"))
.setTypes(Application.CONFIG.getString("index.es.type"))
.setSearchType(SearchType.DFS_QUERY_AND_FETCH).setQuery(geoQuery)
.execute().actionGet();
return responseOfSearch;
return client.prepareSearch(Application.CONFIG.getString("index.es.name"))
.setTypes(Application.CONFIG.getString("index.es.type"))
.setSearchType(SearchType.DFS_QUERY_AND_FETCH).setQuery(geoQuery)
.execute().actionGet();
}

@Test
Expand Down
7 changes: 3 additions & 4 deletions test/index/TestJsonLd.java
Original file line number Diff line number Diff line change
Expand Up @@ -28,20 +28,19 @@ private static String getSource(String id) {
.prepareGet(Application.CONFIG.getString("index.es.name"),
Application.CONFIG.getString("index.es.type"), id)
.execute().actionGet();
String source = response.getSourceAsString();
return source;
return response.getSourceAsString();
}

@Test
public void validateJsonLd()
throws JsonParseException, IOException, JsonLdError {
throws IOException, JsonLdError {
Object sourceAsJson = JsonUtils.fromString(getSource("DE-1a"));
Object sourceAsRdf = JsonLdProcessor.toRDF(sourceAsJson);
assertNotNull("Index documents should be parsable as JSON-LD", sourceAsRdf);
}

@Test
public void testOverwrite() throws JsonParseException, IOException {
public void testOverwrite() throws IOException {
@SuppressWarnings("unchecked")
Map<String, Object> entries =
((HashMap<String, Object>) JsonUtils.fromString(getSource("DE-38")));
Expand Down
Loading

0 comments on commit f4352f2

Please sign in to comment.