Bugfixes for RC1 and 1.7.2rc1 release (#332)

_version.py

@@ -14,8 +14,8 @@ def raw_versions():
 {
     "codename": "Hurricane Ridge",
     "post": "0",
-    "rc": "0",
-    "version": "1.7.1"
+    "rc": "1",
+    "version": "1.7.2"
 }
 """)
 

debian/changelog

@@ -1,6 +1,6 @@
-rekall-forensic (1.7.1) RELEASED; urgency=low
+rekall-forensic (1.7.2) RELEASED; urgency=low
 
   [ Rekall Team ]
-  * Release 1.7.1 Hurricane Ridge
+  * Release 1.7.2 Hurricane Ridge
 
- -- Rekall Team <[email protected]>  Mon, 6 Nov 2017 5:20:55 +0000
+ -- Rekall Team <[email protected]>  Wed, 6 Dec 2017 6:51:02 +0000

rekall-agent/_version.py

@@ -14,8 +14,8 @@ def raw_versions():
 {
     "codename": "Hurricane Ridge",
     "post": "0",
-    "rc": "0",
-    "version": "1.7.1"
+    "rc": "1",
+    "version": "1.7.2"
 }
 """)
 

rekall-core/rekall/_version.py

@@ -14,8 +14,8 @@ def raw_versions():
 {
     "codename": "Hurricane Ridge",
     "post": "0",
-    "rc": "0",
-    "version": "1.7.1"
+    "rc": "1",
+    "version": "1.7.2"
 }
 """)
 

rekall-core/rekall/plugins/addrspaces/aff4.py

@@ -182,8 +182,7 @@ def _LocateAFF4Volume(self, filename):
                         return volume.urn, None
 
                 else:
-                    # volume_path is not valid.
-                    return None, None
+                    raise IOError("Not found: %s" % volume_urn)
 
             elif volume_urn_parts.scheme == "gs" and aff4_cloud:
                 with aff4_cloud.AFF4GStore.NewAFF4GStore(

rekall-core/rekall/plugins/addrspaces/elfcore.py

@@ -52,7 +52,7 @@ def ParseIOMap(string):
                     start=int("0x"+m.group(1), 16),
                     end=int("0x"+m.group(2), 16)))
         else:
-            import pdb; pdb.set_trace()
+            raise IOError("Unable to parse iomap")
 
     return result
 

rekall-core/rekall/plugins/addrspaces/pmem.py

@@ -30,12 +30,18 @@
 
 
 class _StreamWrapper(object):
-    def __init__(self, stream):
+    def __init__(self, session, stream):
         self.stream = stream
+        self.session = session
 
     def read(self, offset, length):
         self.stream.seek(offset)
-        return self.stream.read(length)
+        try:
+            return self.stream.read(length)
+        except IOError:
+            self.session.logging.warn(
+                "IOError reading at offset 0x%0x. Null Padding", offset)
+            return addrspace.ZEROER.GetZeros(length)
 
     def write(self, offset, length):
         self.stream.seek(offset)
@@ -83,7 +89,7 @@ def __init__(self, base=None, filename=None, **kwargs):
         # permissions may be set up such that opening for writing would be
         # disallowed.
         try:
-            self.fd = open(self.fname, "r")
+            self.fd = open(self.fname, "rb")
         except (OSError, IOError):
             raise addrspace.ASAssertionError(
                 "Filename does not exist or can not be opened.")
@@ -104,7 +110,7 @@ def _get_readable_runs(self, records):
             if record["type"] == "efi_range":
                 if efi_type_readable(record["efi_type"]):
                     yield (record["start"], record["start"], record["length"],
-                           _StreamWrapper(self.fd))
+                           _StreamWrapper(self.session, self.fd))
 
     def ConfigureSession(self, session_obj):
         session_obj.SetCache("dtb", self.pmem_metadata["meta"]["dtb_off"],

rekall-core/rekall/plugins/linux/common.py

@@ -456,4 +456,11 @@ def calculate(self):
         if io_map_vm != None:
             io_map_data = utils.SmartUnicode(io_map_vm.read(
                 0, 100000).split(b"\x00")[0])
-            return elfcore.ParseIOMap(io_map_data)
+            result = {}
+            for name, runs in elfcore.ParseIOMap(io_map_data).items():
+                for run in runs:
+                    result.setdefault(name, []).append(
+                        dict(start=run.start, end=run.end,
+                             file_offset=run.file_offset))
+
+            return result

rekall-core/rekall/plugins/overlays/basic.py

@@ -416,8 +416,8 @@ def __hash__(self):
        return hash(self.v())
 
     def __repr__(self):
-        return u"%s (%s)" % (super(Enumeration, self).__repr__(),
-                             self.__str__())
+        return u" [{0}:{1}]: 0x{2:08x} ({3})".format(
+            self.obj_type, self.obj_name, self.v(), self.__str__())
 
     _reverse_choices = None
 

rekall-core/rekall/plugins/overlays/linux/linux.py

@@ -1216,7 +1216,7 @@ def GetPageOffset(self):
             # _text symbol and the iomap's report of the kernel code
             # page start.
             result = (self.get_constant("_text", is_address=True) -
-                      iomap["Kernel code"][0].start)
+                      iomap["Kernel code"][0]["start"])
 
         elif self.metadata("arch") == "I386":
             result = (self.get_constant("_text", True) -

rekall-core/rekall/plugins/tools/aff4acquire.py

@@ -33,6 +33,7 @@
 import glob
 import os
 import re
+import sys
 import stat
 import tempfile
 
@@ -67,12 +68,22 @@ def __init__(self, session, **kwargs):
         super(AFF4ProgressReporter, self).__init__(**kwargs)
         self.session = session
 
+        # Running average over 20 values to make the rate not
+        # fluctuate so much.
+        self.avg = 0.0
+        self.avg_count = 10.0
+        self.count = 0
+
     def Report(self, readptr):
         """This will be called periodically to report the progress.
 
         Note that readptr is specified relative to the start of the range
         operation (WriteStream and CopyToStream)
         """
+        self.count += 1
+        if self.count % 1000 != 0:
+            return
+
         readptr = readptr + self.start
 
         # Rate in MB/s.
@@ -82,11 +93,14 @@ def Report(self, readptr):
         except ZeroDivisionError:
             rate = "?"
 
+        self.avg -= self.avg / self.avg_count
+        self.avg += rate / self.avg_count
+
         self.session.report_progress(
             " Reading %sMiB / %sMiB  %s MiB/s     ",
             readptr//1024//1024,
             self.length//1024//1024,
-            rate)
+            int(self.avg))
 
         self.last_time = self.now()
         self.last_offset = readptr
@@ -269,6 +283,9 @@ def __init__(self, *args, **kwargs):
             self.compression = lexicon.AFF4_IMAGE_COMPRESSION_STORED
         elif self.plugin_args.compression == "zlib":
             self.compression = lexicon.AFF4_IMAGE_COMPRESSION_ZLIB
+        else:
+            raise plugin.InvalidArgs(
+                "Unsupported compression %s " % self.plugin_args.compression)
 
         # Do not acquire memory if we are told to do something else as well,
         # unless specifically asked to.
@@ -438,10 +455,10 @@ def _copy_file_to_image(self, resolver, volume, filename,
                             resolver, image_urn, volume.urn) as out_fd:
                         out_fd.WriteStream(in_fd, progress=progress)
 
-        except IOError:
+        except (IOError, OSError):
             try:
                 # Currently we can only access NTFS filesystems.
-                if self.session.profile.metadata("os") == "windows":
+                if self._get_os() == "windows":
                     self.session.logging.debug(
                         "Unable to read %s. Attempting raw access.", filename)
 
@@ -457,6 +474,15 @@ def _copy_file_to_image(self, resolver, volume, filename,
             if out_fd:
                 resolver.Close(out_fd)
 
+    def _get_os(self):
+        # Do not attempt to get the profile in live mode.
+        if self.session.GetParameter("live_mode"):
+            return sys.platform
+
+        # This will trigger profile autodetection which should only
+        # occur on an image.
+        return self.session.profile.metadata("os")
+
     def _copy_raw_file_to_image(self, resolver, volume, filename):
         image_urn = volume.urn.Append(utils.SmartStr(filename))
 
@@ -515,10 +541,7 @@ def windows_copy_mapped_files(self, resolver, volume):
 
 
     def copy_mapped_files(self, resolver, volume):
-        # Forces profile autodetection if needed.
-        profile = self.session.profile
-
-        os_name = profile.metadata("os")
+        os_name = self._get_os()
         if os_name == "windows":
             for  x in self.windows_copy_mapped_files(resolver, volume):
                 yield x

rekall-core/rekall/plugins/windows/dns.py

@@ -379,7 +379,7 @@ def collect(self):
                             "DNS_HASHTABLE_ENTRY", "List",
                             include_current=True):
 
-                        if entries.obj_offset in buckets:
+                        if entry.obj_offset in entries:
                             continue
 
                         entries.add(bucket.obj_offset)

rekall-core/rekall/plugins/windows/gui/sessions.py

@@ -59,15 +59,14 @@ def session_spaces(self):
           _MM_SESSION_SPACE instantiated from the session space's address space.
         """
         # Dedup based on sessions.
-        for proc in utils.Deduplicate(self.filter_processes()):
+        for proc in utils.Deduplicate(self.filter_processes(),
+                                      key=lambda x: x.Session):
             ps_ad = proc.get_process_address_space()
 
-            session = proc.Session
+            session = proc.Session.deref(vm=ps_ad)
             # Session pointer is invalid (e.g. for System process).
-            if not session:
-                continue
-
-            yield proc.Session.deref(vm=ps_ad)
+            if session:
+                yield session
 
     def find_session_space(self, session_id):
         """Get a _MM_SESSION_SPACE object by its ID.
@@ -88,7 +87,6 @@ def collect(self):
         for session in self.session_spaces():
             processes = list(session.ProcessList.list_of_type(
                 "_EPROCESS", "SessionProcessLinks"))
-
             yield dict(divider=("_MM_SESSION_SPACE: {0:#x} ID: {1} "
                                 "Processes: {2}".format(
                                     session.obj_offset,
@@ -102,7 +100,6 @@ def collect(self):
             # Follow the undocumented _IMAGE_ENTRY_IN_SESSION list to find the
             # kernel modules loaded in this session.
             for image in session.ImageIterator:
-
                 yield dict(
                     session_id=session.SessionId,
                     image=image)

rekall-core/rekall/session_test.py

@@ -30,8 +30,8 @@ def testSessionCache(self):
         self.physical_AS.volatile = False
         self.session.physical_address_space = self.physical_AS
 
-        # None volatile physical address space should use the user specified
-        # cache type.
+        # Non-volatile physical address space should use the user
+        # specified cache type.
         self.assertEqual(self.session.cache.__class__.__name__, "Cache")
 
         # Assigning the physical address space causes the cache to be

rekall-core/rekall/ui/text.py

@@ -1251,7 +1251,7 @@ class TextRenderer(renderer_module.BaseRenderer):
 
     table_class = TextTable
 
-    def __init__(self, tablesep=" ", output=None, mode="a+b", fd=None,
+    def __init__(self, tablesep=" ", output=None, mode="a+t", fd=None,
                  **kwargs):
         super(TextRenderer, self).__init__(**kwargs)
 

rekall-gui/_version.py

@@ -14,8 +14,8 @@ def raw_versions():
 {
     "codename": "Hurricane Ridge",
     "post": "0",
-    "rc": "0",
-    "version": "1.7.1"
+    "rc": "1",
+    "version": "1.7.2"
 }
 """)