quick workaround for #20

bitextor · Jan 21, 2021 · 1b6068a · 1b6068a
1 parent fd73cc2
commit 1b6068a
Show file tree

Hide file tree

Showing 3 changed files with 11 additions and 5 deletions.
diff --git a/src/warcpreprocessor.cc b/src/warcpreprocessor.cc
@@ -48,7 +48,7 @@ namespace warc2text {
 
         while (!done) {
             done = !reader.getRecord(content);
-            if (done)
+            if (done or content.empty())
                 continue;
 
             Record record(content);
@@ -67,7 +67,7 @@ namespace warc2text {
                     // Work-around for https://github.com/bitextor/warc2text/issues/16 for ParaCrawl
                     // we do not really have a use case for massive PDFs at this moment. Skip em.
                     if (content.size() >= static_cast<std::size_t>(std::numeric_limits<uInt>::max())) {
-                        BOOST_LOG_TRIVIAL(info) << "PDF too large to compress with util::BZCompress";
+                        BOOST_LOG_TRIVIAL(info) << "PDF too large to compress with util::GZCompress";
                         continue;
                     }
 

diff --git a/src/warcreader.cc b/src/warcreader.cc
@@ -30,10 +30,11 @@ namespace warc2text {
         closeFile();
     }
 
-    bool WARCReader::getRecord(std::string& out){
+    bool WARCReader::getRecord(std::string& out, std::size_t max_size){
         int inflate_ret = 0;
         out.clear();
         std::size_t len;
+        bool skip_record = false;
         while (inflate_ret != Z_STREAM_END) {
             if (s.avail_in == 0) {
                 len = readChunk();
@@ -55,7 +56,12 @@ namespace warc2text {
                     out.clear();
                     return false;
                 }
-                out.append(scratch, scratch + (BUFFER_SIZE - s.avail_out));
+                if (not skip_record) out.append(scratch, scratch + (BUFFER_SIZE - s.avail_out));
+                if (out.size() > max_size) {
+                    BOOST_LOG_TRIVIAL(error) << "WARC " << warc_filename << ": skipping large record";
+                    out.clear();
+                    skip_record = true;
+                }
             }
             if (inflate_ret == Z_STREAM_END) {
                 assert(inflateReset(&s) == Z_OK);

diff --git a/src/warcreader.hh b/src/warcreader.hh
@@ -9,7 +9,7 @@ namespace warc2text {
         public:
             WARCReader();
             explicit WARCReader(const std::string& filename);
-            bool getRecord(std::string& out);
+            bool getRecord(std::string& out, std::size_t max_size = 1024*1024*20); //20MB
             ~WARCReader();
         private:
             std::FILE* file;