Split processes for bulk and updates of sigel #462

hbz · Aug 10, 2023 · bf2d271 · bf2d271
1 parent f96fa93
commit bf2d271
Show file tree

Hide file tree

Showing 4 changed files with 33 additions and 12 deletions.
diff --git a/app/transformation/TransformAll.java b/app/transformation/TransformAll.java
@@ -52,7 +52,9 @@ public static void process(String startOfUpdates, int intervalSize,
 			final String outputPath, String geoServer) throws IOException {
 		String dbsOutput = outputPath + "-dbs";
 		String sigelOutput = outputPath + "-sigel";
-		TransformSigel.process(startOfUpdates, intervalSize, sigelOutput,
+		TransformSigel.processBulk(startOfUpdates, intervalSize, sigelOutput,
+				geoServer);
+		TransformSigel.processUpdates(startOfUpdates, intervalSize, sigelOutput,
 				geoServer);
 		TransformDbs.process(dbsOutput, geoServer);
 		try (FileWriter resultWriter = new FileWriter(outputPath)) {

diff --git a/app/transformation/TransformSigel.java b/app/transformation/TransformSigel.java
@@ -49,22 +49,30 @@ public class TransformSigel {
 			"/*[local-name() = 'record']/*[local-name() = 'global']/*[local-name() = 'tag'][@id='008H']/*[local-name() = 'subf'][@id='e']";
 	static final String DUMP_XPATH = "/" + DUMP_TOP_LEVEL_TAG + "/" + XPATH;
 
-	static void process(String startOfUpdates, int intervalSize,
+	static void processBulk(String startOfUpdates, int intervalSize,
 			final String outputPath, String geoLookupServer) throws IOException {
-		final FileOpener splitFileOpener = new FileOpener();
-		final FileOpener dumpOpener = new FileOpener();
+		final FileOpener dumpOpener = new FileOpener();	
+		PicaDecoder picaDecoder = new PicaDecoder();
+		picaDecoder.setNormalizeUTF8(true);
 		JsonEncoder encodeJson = new JsonEncoder();
 		encodeJson.setPrettyPrinting(true);
 		dumpOpener//
 				.setReceiver(new LineReader())//
-				.setReceiver(new PicaDecoder())//
+				.setReceiver(picaDecoder)//
 				.setReceiver(new Metafix("conf/fix-sigel.fix"))//
 				.setReceiver(TransformAll.fixEnriched(geoLookupServer))//
 				.setReceiver(encodeJson)//
 				.setReceiver(TransformAll.esBulk())//
 				.setReceiver(new ObjectWriter<>(outputPath));
 		dumpOpener.process(TransformAll.DATA_INPUT_DIR + "sigil.dat");
+ 		dumpOpener.closeStream(); 
+	}
 
+		static void processUpdates(String startOfUpdates, int intervalSize,
+			final String outputPath, String geoLookupServer) throws IOException {
+		final FileOpener splitFileOpener = new FileOpener();		
+		JsonEncoder encodeJson = new JsonEncoder();
+		encodeJson.setPrettyPrinting(true);
 		ObjectWriter objectWriter = new ObjectWriter<>(outputPath);
 		objectWriter.setAppendIfFileExists(true);
 		splitFileOpener//
@@ -84,7 +92,8 @@ static void process(String startOfUpdates, int intervalSize,
 				.collect(Collectors.toList()).forEach(path -> {
 					splitFileOpener.process(path.toString());
 				});
-
+		splitFileOpener.closeStream();
+
 
 	}