Update filter mechanism and add comments

hbz · Aug 10, 2023 · f63f951 · f63f951
1 parent 88d290c
commit f63f951
Show file tree

Hide file tree

Showing 4 changed files with 6 additions and 7 deletions.
diff --git a/app/transformation/TransformDbs.java b/app/transformation/TransformDbs.java
@@ -28,7 +28,7 @@ static void process(final String outputPath, String geoLookupServer) throws File
 		opener//
 				.setReceiver(new LineReader())//
 				.setReceiver(decoder)//
-				.setReceiver(new Metafix("conf/fix-dbs.fix"))//
+				.setReceiver(new Metafix("conf/fix-dbs.fix"))// Fix skips all records that have no "inr"
 				.setReceiver(TransformAll.fixEnriched(geoLookupServer))//
 				.setReceiver(encodeJson)//
 				.setReceiver(TransformAll.esBulk())//

diff --git a/app/transformation/TransformSigel.java b/app/transformation/TransformSigel.java
@@ -56,7 +56,7 @@ static void process(String startOfUpdates, int intervalSize,
 		splitFileOpener//
 				.setReceiver(new XmlDecoder())//
 				.setReceiver(new PicaXmlHandler())//
-				.setReceiver(new Metafix("conf/fix-sigel.fix"))//fix also kicks out all records without _id
+				.setReceiver(new Metafix("conf/fix-sigel.fix")) // Fix skips all records that have no "inr" and "isil"
 				.setReceiver(TransformAll.fixEnriched(geoLookupServer))//
 				.setReceiver(encodeJson)//
 				.setReceiver(TransformAll.esBulk())//

diff --git a/conf/fix-dbs.fix b/conf/fix-dbs.fix
@@ -35,6 +35,7 @@ end
 
 vacuum()
 
+# Skip all records that have no "inr"
 unless exists("inr")
 	reject()
 end
diff --git a/conf/fix-sigel.fix b/conf/fix-sigel.fix
@@ -10,8 +10,6 @@ replace_all("@dbsId"," ","")
 replace_all("isil"," ","")
 if any_match("@dbsId","[A-Z]{2}\\d{3}")
 	copy_field("@dbsId","inr")
-else
-	copy_field("isil","_id")
 end
 
 do list(path: "032P","var":"$i")
@@ -67,10 +65,10 @@ do list(path: "035B","var":"$i")
 		copy_field("$i.k","email")
 	end
 end
+
+vacuum()
 
-
-	vacuum()
-
+# Skip all records that have no "inr" or "isil"
 unless exists("inr")
 	unless exists("isil")
 		reject()