Merge pull request #34 from timothyryanwalsh/dev/unicodedecodeerror

Add UnicodeDecodeError handling
tw4l · Jan 16, 2018 · a84bbcd · a84bbcd
2 parents 40a32d9 + 0aa2777
commit a84bbcd
Show file tree

Hide file tree

Showing 3 changed files with 13 additions and 6 deletions.
diff --git a/README.md b/README.md
@@ -1,6 +1,6 @@
 ## Brunnhilde - A reporting companion to Siegfried  
 
-### Version: Brunnhilde 1.6.1
+### Version: Brunnhilde 1.6.2
 
 [![Build Status](https://travis-ci.org/timothyryanwalsh/brunnhilde.svg?branch=master)](https://travis-ci.org/timothyryanwalsh/brunnhilde)
 

diff --git a/brunnhilde.py b/brunnhilde.py
@@ -107,7 +107,11 @@ def import_csv(cursor, conn, use_hash):
         f = open(sf_file, 'r', encoding='utf8')
     else:
         f = open(sf_file, 'rb')
-    reader = csv.reader(x.replace('\0', '') for x in f) # replace null bytes with empty strings on read
+    try:
+        reader = csv.reader(x.replace('\0', '') for x in f) # replace null bytes with empty strings on read
+    except UnicodeDecodeError:
+        f = (x.encode('utf-8').strip() for x in f) # skip non-utf8 encodable characters
+        reader = csv.reader(x.replace('\0', '') for x in f) # replace null bytes with empty strings on read
     header = True
     for row in reader:
         if header:
@@ -241,8 +245,11 @@ def get_stats(args, source_dir, scan_started, cursor, html, brunnhilde_version,
         for root, dirs, files in os.walk(unicode(source_dir, 'utf-8')):
             for f in files:
                 file_path = os.path.join(root, f)
-                file_info = os.stat(file_path)
-                size_bytes += file_info.st_size
+                try:
+                    file_info = os.stat(file_path)
+                    size_bytes += file_info.st_size
+                except OSError as e: # report when Brunnhilde can't find file
+                    print("\nOSError: %s. File size of this file not included in Brunnhilde HTML report statistics." % (e))
     size = convert_size(size_bytes)
 
     # write html
@@ -572,7 +579,7 @@ def _make_parser(version):
 
 def main():
     # system info
-    brunnhilde_version = 'brunnhilde 1.6.1'
+    brunnhilde_version = 'brunnhilde 1.6.2'
     siegfried_version = subprocess.check_output(["sf", "-version"]).decode()
 
     parser = _make_parser(brunnhilde_version)

diff --git a/setup.py b/setup.py
@@ -2,7 +2,7 @@
 
 setup(
     name = 'brunnhilde',
-    version = '1.6.1',
+    version = '1.6.2',
     url = 'https://github.com/timothyryanwalsh/brunnhilde',
     author = 'Tim Walsh',
     author_email = '[email protected]',