From c73ef5788c51155d7ee34fb47bd273b994b2e7e1 Mon Sep 17 00:00:00 2001 From: Michael Cohen Date: Tue, 5 Dec 2017 23:41:35 -0800 Subject: [PATCH] Bugfixes for RC1 and 1.7.2rc1 release (#332) --- _version.py | 4 +- debian/changelog | 6 +- rekall-agent/_version.py | 4 +- rekall-core/rekall/_version.py | 4 +- rekall-core/rekall/plugins/addrspaces/aff4.py | 3 +- .../rekall/plugins/addrspaces/elfcore.py | 2 +- rekall-core/rekall/plugins/addrspaces/pmem.py | 14 ++- rekall-core/rekall/plugins/linux/common.py | 9 +- rekall-core/rekall/plugins/overlays/basic.py | 4 +- .../rekall/plugins/overlays/linux/linux.py | 2 +- .../rekall/plugins/tools/aff4acquire.py | 37 +++++-- rekall-core/rekall/plugins/windows/dns.py | 2 +- .../rekall/plugins/windows/gui/sessions.py | 13 +-- rekall-core/rekall/session_test.py | 4 +- rekall-core/rekall/ui/text.py | 2 +- rekall-gui/_version.py | 4 +- rekall-lib/_version.py | 104 ++++++++++++++++++ tools/layout_expert/layout_expert/_version.py | 4 +- tools/pmem/linux_pmem.cc | 3 + tools/pmem/pmem.h | 6 + version.yaml | 5 +- 21 files changed, 190 insertions(+), 46 deletions(-) create mode 100644 rekall-lib/_version.py diff --git a/_version.py b/_version.py index a112d0344..53c9d1043 100644 --- a/_version.py +++ b/_version.py @@ -14,8 +14,8 @@ def raw_versions(): { "codename": "Hurricane Ridge", "post": "0", - "rc": "0", - "version": "1.7.1" + "rc": "1", + "version": "1.7.2" } """) diff --git a/debian/changelog b/debian/changelog index 5fadcecdd..91056ffd0 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,6 +1,6 @@ -rekall-forensic (1.7.1) RELEASED; urgency=low +rekall-forensic (1.7.2) RELEASED; urgency=low [ Rekall Team ] - * Release 1.7.1 Hurricane Ridge + * Release 1.7.2 Hurricane Ridge - -- Rekall Team Mon, 6 Nov 2017 5:20:55 +0000 + -- Rekall Team Wed, 6 Dec 2017 6:51:02 +0000 diff --git a/rekall-agent/_version.py b/rekall-agent/_version.py index a112d0344..53c9d1043 100644 --- a/rekall-agent/_version.py +++ b/rekall-agent/_version.py @@ -14,8 +14,8 @@ def raw_versions(): { "codename": "Hurricane Ridge", "post": "0", - "rc": "0", - "version": "1.7.1" + "rc": "1", + "version": "1.7.2" } """) diff --git a/rekall-core/rekall/_version.py b/rekall-core/rekall/_version.py index a112d0344..53c9d1043 100644 --- a/rekall-core/rekall/_version.py +++ b/rekall-core/rekall/_version.py @@ -14,8 +14,8 @@ def raw_versions(): { "codename": "Hurricane Ridge", "post": "0", - "rc": "0", - "version": "1.7.1" + "rc": "1", + "version": "1.7.2" } """) diff --git a/rekall-core/rekall/plugins/addrspaces/aff4.py b/rekall-core/rekall/plugins/addrspaces/aff4.py index b6a5012fb..d47fbaa71 100644 --- a/rekall-core/rekall/plugins/addrspaces/aff4.py +++ b/rekall-core/rekall/plugins/addrspaces/aff4.py @@ -182,8 +182,7 @@ def _LocateAFF4Volume(self, filename): return volume.urn, None else: - # volume_path is not valid. - return None, None + raise IOError("Not found: %s" % volume_urn) elif volume_urn_parts.scheme == "gs" and aff4_cloud: with aff4_cloud.AFF4GStore.NewAFF4GStore( diff --git a/rekall-core/rekall/plugins/addrspaces/elfcore.py b/rekall-core/rekall/plugins/addrspaces/elfcore.py index 2da334af4..9e668a907 100644 --- a/rekall-core/rekall/plugins/addrspaces/elfcore.py +++ b/rekall-core/rekall/plugins/addrspaces/elfcore.py @@ -52,7 +52,7 @@ def ParseIOMap(string): start=int("0x"+m.group(1), 16), end=int("0x"+m.group(2), 16))) else: - import pdb; pdb.set_trace() + raise IOError("Unable to parse iomap") return result diff --git a/rekall-core/rekall/plugins/addrspaces/pmem.py b/rekall-core/rekall/plugins/addrspaces/pmem.py index d0664b44d..f70a671c0 100644 --- a/rekall-core/rekall/plugins/addrspaces/pmem.py +++ b/rekall-core/rekall/plugins/addrspaces/pmem.py @@ -30,12 +30,18 @@ class _StreamWrapper(object): - def __init__(self, stream): + def __init__(self, session, stream): self.stream = stream + self.session = session def read(self, offset, length): self.stream.seek(offset) - return self.stream.read(length) + try: + return self.stream.read(length) + except IOError: + self.session.logging.warn( + "IOError reading at offset 0x%0x. Null Padding", offset) + return addrspace.ZEROER.GetZeros(length) def write(self, offset, length): self.stream.seek(offset) @@ -83,7 +89,7 @@ def __init__(self, base=None, filename=None, **kwargs): # permissions may be set up such that opening for writing would be # disallowed. try: - self.fd = open(self.fname, "r") + self.fd = open(self.fname, "rb") except (OSError, IOError): raise addrspace.ASAssertionError( "Filename does not exist or can not be opened.") @@ -104,7 +110,7 @@ def _get_readable_runs(self, records): if record["type"] == "efi_range": if efi_type_readable(record["efi_type"]): yield (record["start"], record["start"], record["length"], - _StreamWrapper(self.fd)) + _StreamWrapper(self.session, self.fd)) def ConfigureSession(self, session_obj): session_obj.SetCache("dtb", self.pmem_metadata["meta"]["dtb_off"], diff --git a/rekall-core/rekall/plugins/linux/common.py b/rekall-core/rekall/plugins/linux/common.py index 6f07dcdb7..9577cc6e1 100644 --- a/rekall-core/rekall/plugins/linux/common.py +++ b/rekall-core/rekall/plugins/linux/common.py @@ -456,4 +456,11 @@ def calculate(self): if io_map_vm != None: io_map_data = utils.SmartUnicode(io_map_vm.read( 0, 100000).split(b"\x00")[0]) - return elfcore.ParseIOMap(io_map_data) + result = {} + for name, runs in elfcore.ParseIOMap(io_map_data).items(): + for run in runs: + result.setdefault(name, []).append( + dict(start=run.start, end=run.end, + file_offset=run.file_offset)) + + return result diff --git a/rekall-core/rekall/plugins/overlays/basic.py b/rekall-core/rekall/plugins/overlays/basic.py index ed1f93876..bd35c234f 100644 --- a/rekall-core/rekall/plugins/overlays/basic.py +++ b/rekall-core/rekall/plugins/overlays/basic.py @@ -416,8 +416,8 @@ def __hash__(self): return hash(self.v()) def __repr__(self): - return u"%s (%s)" % (super(Enumeration, self).__repr__(), - self.__str__()) + return u" [{0}:{1}]: 0x{2:08x} ({3})".format( + self.obj_type, self.obj_name, self.v(), self.__str__()) _reverse_choices = None diff --git a/rekall-core/rekall/plugins/overlays/linux/linux.py b/rekall-core/rekall/plugins/overlays/linux/linux.py index 80bb54681..9aadd1336 100644 --- a/rekall-core/rekall/plugins/overlays/linux/linux.py +++ b/rekall-core/rekall/plugins/overlays/linux/linux.py @@ -1216,7 +1216,7 @@ def GetPageOffset(self): # _text symbol and the iomap's report of the kernel code # page start. result = (self.get_constant("_text", is_address=True) - - iomap["Kernel code"][0].start) + iomap["Kernel code"][0]["start"]) elif self.metadata("arch") == "I386": result = (self.get_constant("_text", True) - diff --git a/rekall-core/rekall/plugins/tools/aff4acquire.py b/rekall-core/rekall/plugins/tools/aff4acquire.py index 16685588d..13768f8d7 100644 --- a/rekall-core/rekall/plugins/tools/aff4acquire.py +++ b/rekall-core/rekall/plugins/tools/aff4acquire.py @@ -33,6 +33,7 @@ import glob import os import re +import sys import stat import tempfile @@ -67,12 +68,22 @@ def __init__(self, session, **kwargs): super(AFF4ProgressReporter, self).__init__(**kwargs) self.session = session + # Running average over 20 values to make the rate not + # fluctuate so much. + self.avg = 0.0 + self.avg_count = 10.0 + self.count = 0 + def Report(self, readptr): """This will be called periodically to report the progress. Note that readptr is specified relative to the start of the range operation (WriteStream and CopyToStream) """ + self.count += 1 + if self.count % 1000 != 0: + return + readptr = readptr + self.start # Rate in MB/s. @@ -82,11 +93,14 @@ def Report(self, readptr): except ZeroDivisionError: rate = "?" + self.avg -= self.avg / self.avg_count + self.avg += rate / self.avg_count + self.session.report_progress( " Reading %sMiB / %sMiB %s MiB/s ", readptr//1024//1024, self.length//1024//1024, - rate) + int(self.avg)) self.last_time = self.now() self.last_offset = readptr @@ -269,6 +283,9 @@ def __init__(self, *args, **kwargs): self.compression = lexicon.AFF4_IMAGE_COMPRESSION_STORED elif self.plugin_args.compression == "zlib": self.compression = lexicon.AFF4_IMAGE_COMPRESSION_ZLIB + else: + raise plugin.InvalidArgs( + "Unsupported compression %s " % self.plugin_args.compression) # Do not acquire memory if we are told to do something else as well, # unless specifically asked to. @@ -438,10 +455,10 @@ def _copy_file_to_image(self, resolver, volume, filename, resolver, image_urn, volume.urn) as out_fd: out_fd.WriteStream(in_fd, progress=progress) - except IOError: + except (IOError, OSError): try: # Currently we can only access NTFS filesystems. - if self.session.profile.metadata("os") == "windows": + if self._get_os() == "windows": self.session.logging.debug( "Unable to read %s. Attempting raw access.", filename) @@ -457,6 +474,15 @@ def _copy_file_to_image(self, resolver, volume, filename, if out_fd: resolver.Close(out_fd) + def _get_os(self): + # Do not attempt to get the profile in live mode. + if self.session.GetParameter("live_mode"): + return sys.platform + + # This will trigger profile autodetection which should only + # occur on an image. + return self.session.profile.metadata("os") + def _copy_raw_file_to_image(self, resolver, volume, filename): image_urn = volume.urn.Append(utils.SmartStr(filename)) @@ -515,10 +541,7 @@ def windows_copy_mapped_files(self, resolver, volume): def copy_mapped_files(self, resolver, volume): - # Forces profile autodetection if needed. - profile = self.session.profile - - os_name = profile.metadata("os") + os_name = self._get_os() if os_name == "windows": for x in self.windows_copy_mapped_files(resolver, volume): yield x diff --git a/rekall-core/rekall/plugins/windows/dns.py b/rekall-core/rekall/plugins/windows/dns.py index 46b146cab..55a581de9 100644 --- a/rekall-core/rekall/plugins/windows/dns.py +++ b/rekall-core/rekall/plugins/windows/dns.py @@ -379,7 +379,7 @@ def collect(self): "DNS_HASHTABLE_ENTRY", "List", include_current=True): - if entries.obj_offset in buckets: + if entry.obj_offset in entries: continue entries.add(bucket.obj_offset) diff --git a/rekall-core/rekall/plugins/windows/gui/sessions.py b/rekall-core/rekall/plugins/windows/gui/sessions.py index a5d2746ff..95c273e63 100644 --- a/rekall-core/rekall/plugins/windows/gui/sessions.py +++ b/rekall-core/rekall/plugins/windows/gui/sessions.py @@ -59,15 +59,14 @@ def session_spaces(self): _MM_SESSION_SPACE instantiated from the session space's address space. """ # Dedup based on sessions. - for proc in utils.Deduplicate(self.filter_processes()): + for proc in utils.Deduplicate(self.filter_processes(), + key=lambda x: x.Session): ps_ad = proc.get_process_address_space() - session = proc.Session + session = proc.Session.deref(vm=ps_ad) # Session pointer is invalid (e.g. for System process). - if not session: - continue - - yield proc.Session.deref(vm=ps_ad) + if session: + yield session def find_session_space(self, session_id): """Get a _MM_SESSION_SPACE object by its ID. @@ -88,7 +87,6 @@ def collect(self): for session in self.session_spaces(): processes = list(session.ProcessList.list_of_type( "_EPROCESS", "SessionProcessLinks")) - yield dict(divider=("_MM_SESSION_SPACE: {0:#x} ID: {1} " "Processes: {2}".format( session.obj_offset, @@ -102,7 +100,6 @@ def collect(self): # Follow the undocumented _IMAGE_ENTRY_IN_SESSION list to find the # kernel modules loaded in this session. for image in session.ImageIterator: - yield dict( session_id=session.SessionId, image=image) diff --git a/rekall-core/rekall/session_test.py b/rekall-core/rekall/session_test.py index e858310cb..fe7f3cddc 100644 --- a/rekall-core/rekall/session_test.py +++ b/rekall-core/rekall/session_test.py @@ -30,8 +30,8 @@ def testSessionCache(self): self.physical_AS.volatile = False self.session.physical_address_space = self.physical_AS - # None volatile physical address space should use the user specified - # cache type. + # Non-volatile physical address space should use the user + # specified cache type. self.assertEqual(self.session.cache.__class__.__name__, "Cache") # Assigning the physical address space causes the cache to be diff --git a/rekall-core/rekall/ui/text.py b/rekall-core/rekall/ui/text.py index 7402be150..253d8af99 100644 --- a/rekall-core/rekall/ui/text.py +++ b/rekall-core/rekall/ui/text.py @@ -1251,7 +1251,7 @@ class TextRenderer(renderer_module.BaseRenderer): table_class = TextTable - def __init__(self, tablesep=" ", output=None, mode="a+b", fd=None, + def __init__(self, tablesep=" ", output=None, mode="a+t", fd=None, **kwargs): super(TextRenderer, self).__init__(**kwargs) diff --git a/rekall-gui/_version.py b/rekall-gui/_version.py index a112d0344..53c9d1043 100644 --- a/rekall-gui/_version.py +++ b/rekall-gui/_version.py @@ -14,8 +14,8 @@ def raw_versions(): { "codename": "Hurricane Ridge", "post": "0", - "rc": "0", - "version": "1.7.1" + "rc": "1", + "version": "1.7.2" } """) diff --git a/rekall-lib/_version.py b/rekall-lib/_version.py new file mode 100644 index 000000000..53c9d1043 --- /dev/null +++ b/rekall-lib/_version.py @@ -0,0 +1,104 @@ + +# Machine Generated - do not edit! + +# This file is produced when the main "version.py update" command is run. That +# command copies this file to all sub-packages which contain +# setup.py. Configuration is maintain in version.yaml at the project's top +# level. + +def get_versions(): + return tag_version_data(raw_versions(), """version.yaml""") + +def raw_versions(): + return json.loads(""" +{ + "codename": "Hurricane Ridge", + "post": "0", + "rc": "1", + "version": "1.7.2" +} +""") + +import json +import os +import subprocess + +try: + # We are looking for the git repo which contains this file. + MY_DIR = os.path.dirname(os.path.abspath(__file__)) +except: + MY_DIR = None + +def is_tree_dirty(): + try: + return bool(subprocess.check_output( + ["git", "diff", "--name-only"], stderr=subprocess.PIPE, + cwd=MY_DIR, + ).splitlines()) + except (OSError, subprocess.CalledProcessError): + return False + +def get_version_file_path(version_file="version.yaml"): + try: + return os.path.join(subprocess.check_output( + ["git", "rev-parse", "--show-toplevel"], stderr=subprocess.PIPE, + cwd=MY_DIR, + ).decode("utf-8").strip(), version_file) + except (OSError, subprocess.CalledProcessError): + return None + +def number_of_commit_since(version_file="version.yaml"): + """Returns the number of commits since version.yaml was changed.""" + try: + last_commit_to_touch_version_file = subprocess.check_output( + ["git", "log", "--no-merges", "-n", "1", "--pretty=format:%H", + version_file], cwd=MY_DIR, stderr=subprocess.PIPE, + ).strip() + + all_commits = subprocess.check_output( + ["git", "log", "--no-merges", "-n", "1000", "--pretty=format:%H"], + stderr=subprocess.PIPE, cwd=MY_DIR, + ).splitlines() + return all_commits.index(last_commit_to_touch_version_file) + except (OSError, subprocess.CalledProcessError, ValueError): + return None + + +def get_current_git_hash(): + try: + return subprocess.check_output( + ["git", "log", "--no-merges", "-n", "1", "--pretty=format:%H"], + stderr=subprocess.PIPE, cwd=MY_DIR, + ).strip() + except (OSError, subprocess.CalledProcessError): + return None + +def tag_version_data(version_data, version_path="version.yaml"): + current_hash = get_current_git_hash() + # Not in a git repository. + if current_hash is None: + version_data["error"] = "Not in a git repository." + + else: + version_data["revisionid"] = current_hash + version_data["dirty"] = is_tree_dirty() + version_data["dev"] = number_of_commit_since( + get_version_file_path(version_path)) + + # Format the version according to pep440: + pep440 = version_data["version"] + if int(version_data.get("post", 0)) > 0: + pep440 += ".post" + version_data["post"] + + elif int(version_data.get("rc", 0)) > 0: + pep440 += ".rc" + version_data["rc"] + + if version_data.get("dev", 0): + # A Development release comes _before_ the main release. + last = version_data["version"].rsplit(".", 1) + version_data["version"] = "%s.%s" % (last[0], int(last[1]) + 1) + pep440 = version_data["version"] + ".dev" + str(version_data["dev"]) + + version_data["pep440"] = pep440 + + return version_data diff --git a/tools/layout_expert/layout_expert/_version.py b/tools/layout_expert/layout_expert/_version.py index a112d0344..53c9d1043 100644 --- a/tools/layout_expert/layout_expert/_version.py +++ b/tools/layout_expert/layout_expert/_version.py @@ -14,8 +14,8 @@ def raw_versions(): { "codename": "Hurricane Ridge", "post": "0", - "rc": "0", - "version": "1.7.1" + "rc": "1", + "version": "1.7.2" } """) diff --git a/tools/pmem/linux_pmem.cc b/tools/pmem/linux_pmem.cc index 0c406cfcf..3b18a6b05 100644 --- a/tools/pmem/linux_pmem.cc +++ b/tools/pmem/linux_pmem.cc @@ -76,6 +76,9 @@ AFF4Status LinuxPmemImager::CreateMap_(AFF4Map *map, aff4_off_t *length) { pheader.memsz, kcore_urn); *length += pheader.memsz; + } else { + LOG(INFO) << "Skipped range " << std::hex << pheader.vaddr << " " << + std::hex << pheader.memsz << " At offset " << std::hex << pheader.off; } } diff --git a/tools/pmem/pmem.h b/tools/pmem/pmem.h index d14a5926b..d5710beb3 100644 --- a/tools/pmem/pmem.h +++ b/tools/pmem/pmem.h @@ -18,9 +18,12 @@ specific language governing permissions and limitations under the License. #define PMEM_VERSION "2.1.post4"; +#include #include #include +using std::vector; +using std::string; class PmemImager: public BasicImager { protected: @@ -98,4 +101,7 @@ class PmemImager: public BasicImager { } }; + + + #endif // TOOLS_PMEM_PMEM_H_ diff --git a/version.yaml b/version.yaml index f439b34e1..1e585a1ec 100644 --- a/version.yaml +++ b/version.yaml @@ -5,9 +5,8 @@ dependent_versions: - rekall-gui/_version.py - tools/layout_expert/layout_expert/_version.py - rekall-lib/_version.py - version_data: codename: Hurricane Ridge post: '0' - rc: '0' - version: 1.7.1 + rc: '1' + version: 1.7.2