WARC spout to emit captures into topology (implements apache#755)

sebastian-nagel · May 27, 2020 · 6a3f7a7 · 6a3f7a7
1 parent 14ed86d
commit 6a3f7a7
Show file tree

Hide file tree

Showing 3 changed files with 451 additions and 6 deletions.
diff --git a/core/src/main/java/com/digitalpebble/stormcrawler/spout/FileSpout.java b/core/src/main/java/com/digitalpebble/stormcrawler/spout/FileSpout.java
@@ -58,15 +58,15 @@ public class FileSpout extends BaseRichSpout {
     public static final int BATCH_SIZE = 10000;
     public static final Logger LOG = LoggerFactory.getLogger(FileSpout.class);
 
-    private SpoutOutputCollector _collector;
+    protected SpoutOutputCollector _collector;
 
     private Queue<String> _inputFiles;
     private BufferedReader currentBuffer;
 
-    private Scheme _scheme = new StringTabScheme();
+    protected Scheme _scheme = new StringTabScheme();
 
-    private LinkedList<byte[]> buffer = new LinkedList<>();
-    private boolean active;
+    protected LinkedList<byte[]> buffer = new LinkedList<>();
+    protected boolean active;
     private boolean withDiscoveredStatus = false;
 
     /**
@@ -146,7 +146,7 @@ public void setScheme(Scheme scheme) {
         _scheme = scheme;
     }
 
-    private void populateBuffer() throws IOException {
+    protected void populateBuffer() throws IOException {
         if (currentBuffer == null) {
             String file = _inputFiles.poll();
             if (file == null)

diff --git a/external/warc/pom.xml b/external/warc/pom.xml
@@ -18,6 +18,7 @@
 
 	<properties>
 		<commons-codec.version>1.10</commons-codec.version>
+		<jwarc.version>0.11.0</jwarc.version>
 		<mockito-all.version>1.10.8</mockito-all.version>
 		<wiremock.version>2.19.0</wiremock.version>
 	</properties>
@@ -34,7 +35,13 @@
 		<dependency>
 			<groupId>org.apache.storm</groupId>
 			<artifactId>storm-hdfs</artifactId>
-			<version>1.2.3</version>
+			<version>${storm-core.version}</version>
+		</dependency>
+
+		<dependency>
+			<groupId>org.netpreserve</groupId>
+			<artifactId>jwarc</artifactId>
+			<version>${jwarc.version}</version>
 		</dependency>
 
 		<dependency>