diff --git a/rekall/args.py b/rekall/args.py index 71e96d6ff..61e9e4add 100755 --- a/rekall/args.py +++ b/rekall/args.py @@ -368,6 +368,9 @@ def ConfigureCommandLineParser(command_metadata, parser, critical=False): kwargs["nargs"] = "+" if required else "*" kwargs["choices"] = list(kwargs["choices"]) + elif arg_type == "Choices": + kwargs["choices"] = list(kwargs["choices"]) + # Skip option if not critical. critical_arg = kwargs.pop("critical", False) if critical and critical_arg: diff --git a/rekall/constants.py b/rekall/constants.py index 585d6c4b8..681ac9f8b 100644 --- a/rekall/constants.py +++ b/rekall/constants.py @@ -18,7 +18,7 @@ # import time -VERSION = "1.3.0" +VERSION = "1.3.1" CODENAME = "Dammastock" SCAN_BLOCKSIZE = 1024 * 1024 * 10 diff --git a/rekall/plugins/guess_profile.py b/rekall/plugins/guess_profile.py index 8d6f82815..b1520d2c3 100644 --- a/rekall/plugins/guess_profile.py +++ b/rekall/plugins/guess_profile.py @@ -64,6 +64,21 @@ def Keywords(self): def VerifyProfile(self, profile_name): profile = self.session.LoadProfile(profile_name) + + # If the user allows it we can just try to fetch and build the profile + # locally. + if profile == None and self.session.GetParameter( + "autodetect_build_local") in ("full", "basic"): + build_local_profile = self.session.plugins.build_local_profile() + try: + logging.debug("Will build local profile %s", profile_name) + build_local_profile.fetch_and_parse(profile_name) + profile = self.session.LoadProfile( + profile_name, use_cache=False) + + except IOError: + pass + if profile != None: return self._ApplyFindDTB(self.find_dtb_impl, profile) @@ -115,6 +130,12 @@ def DetectFromHit(self, hit, file_offset, address_space): " (Default 1.0)", type="Float") +config.DeclareOption("autodetect_build_local", default="basic", + group="Autodetection Overrides", + choices=["full", "basic", "none"], + help="Attempts to fetch and build profile locally.", + type="Choices") + config.DeclareOption("autodetect_scan_length", default=1000000000, group="Autodetection Overrides", help="How much of physical memory to scan before failing") diff --git a/rekall/plugins/tools/caching_url_manager.py b/rekall/plugins/tools/caching_url_manager.py index ab28c7ae8..76e792292 100644 --- a/rekall/plugins/tools/caching_url_manager.py +++ b/rekall/plugins/tools/caching_url_manager.py @@ -41,10 +41,14 @@ help="Location of the profile cache directory.") -class CachingURLManager(io_manager.IOManager): +class CachingManager(io_manager.IOManager): + + # We wrap this io manager class + DELEGATE = io_manager.URLManager + # If the cache is available we should be selected before the regular - # URLManager. - order = io_manager.URLManager.order - 10 + # manager. + order = DELEGATE.order - 10 def __init__(self, session=None, **kwargs): cache_dir = session.GetParameter("cache_dir") @@ -67,11 +71,11 @@ def __init__(self, session=None, **kwargs): # We use an IO manager to manage the cache directory directly. self.cache_io_manager = io_manager.DirectoryIOManager(urn=cache_dir) - self.url_manager = io_manager.URLManager(session=session, **kwargs) + self.url_manager = self.DELEGATE(session=session, **kwargs) self.CheckUpstreamRepository() - super(CachingURLManager, self).__init__(session=session, **kwargs) + super(CachingManager, self).__init__(session=session, **kwargs) def __str__(self): return "Local Cache %s" % self.cache_io_manager @@ -118,3 +122,7 @@ def CheckUpstreamRepository(self): if modified: self.cache_io_manager.FlushInventory() + + +class CacheDirectoryManager(CachingManager): + DELEGATE = io_manager.DirectoryIOManager diff --git a/rekall/plugins/tools/ipython.py b/rekall/plugins/tools/ipython.py index f5f50c5c6..59c6db496 100644 --- a/rekall/plugins/tools/ipython.py +++ b/rekall/plugins/tools/ipython.py @@ -168,8 +168,8 @@ def args(cls, parser): help="The name of a program to page output " "(e.g. notepad or less).") - def __init__(self, session=None, **kwargs): - super(SessionMod, self).__init__(session=session) + def __init__(self, **kwargs): + super(SessionMod, self).__init__(session=kwargs.pop("session")) self.kwargs = kwargs def render(self, renderer): diff --git a/rekall/plugins/tools/mspdb.py b/rekall/plugins/tools/mspdb.py index 5df69bd0b..c21eba827 100644 --- a/rekall/plugins/tools/mspdb.py +++ b/rekall/plugins/tools/mspdb.py @@ -88,6 +88,7 @@ import logging import ntpath import os +import platform import subprocess import urllib2 @@ -157,32 +158,68 @@ def render(self, renderer): self.pdb_filename = self.filename self.guid = self.guid.upper() + # Write the file data to the renderer. + pdb_file_data = self.FetchPDBFile(self.pdb_filename, self.guid) + with renderer.open(filename=self.pdb_filename, + directory=self.dump_dir, + mode="wb") as fd: + fd.write(pdb_file_data) + + def FetchPDBFile(self, pdb_filename, guid): + # Ensure the pdb filename has the correct extension. + if not pdb_filename.endswith(".pdb"): + pdb_filename += ".pdb" + for url in self.SYM_URLS: - basename = ntpath.splitext(self.pdb_filename)[0] - url += "/%s/%s/%s.pd_" % (self.pdb_filename, - self.guid, basename) + basename = ntpath.splitext(pdb_filename)[0] + url += "/%s/%s/%s.pd_" % (pdb_filename, guid, basename) - renderer.format("Trying to fetch {0}\n", url) + self.session.report_progress("Trying to fetch %s\n", url) request = urllib2.Request(url, None, headers={ 'User-Agent': self.USER_AGENT}) - data = urllib2.urlopen(request).read() - renderer.format("Received {0} bytes\n", len(data)) + url_handler = urllib2.urlopen(request) + with utils.TempDirectory() as temp_dir: + compressed_output_file = os.path.join( + temp_dir, "%s.pd_" % basename) - output_file = "%s.pd_" % basename - with renderer.open(filename=output_file, - directory=self.dump_dir, - mode="wb") as fd: - fd.write(data) + output_file = os.path.join( + temp_dir, "%s.pdb" % basename) - try: - subprocess.check_call(["cabextract", - os.path.basename(output_file)], - cwd=self.dump_dir) - except subprocess.CalledProcessError: - renderer.report_error( - "Failed to decompress output file {0}. " - "Ensure cabextract is installed.\n", output_file) + # Download the compressed file to a temp file. + with open(compressed_output_file, "wb") as outfd: + while True: + data = url_handler.read(8192) + if not data: + break + + outfd.write(data) + self.session.report_progress( + "%s: Downloaded %s bytes", basename, outfd.tell()) + + # Now try to decompress it with system tools. This might fail. + try: + if platform.system() == "Windows": + # This should already be installed on windows systems. + subprocess.check_call( + ["expand", compressed_output_file, output_file], + cwd=self.dump_dir) + else: + # In Linux we just hope the cabextract program was + # installed. + subprocess.check_call( + ["cabextract", compressed_output_file], + cwd=self.dump_dir) + + except subprocess.CalledProcessError: + raise RuntimeError( + "Failed to decompress output file %s. " + "Ensure cabextract is installed.\n" % output_file) + + # We read the entire file into memory here - it should not be + # larger than approximately 10mb. + with open(output_file, "rb") as fd: + return fd.read(50 * 1024 * 1024) class TestFetchPDB(testlib.DisabledTest): @@ -1044,6 +1081,13 @@ def Resolve(self, idx): except KeyError: return obj.NoneObject("Index not known") + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, trace): + self.address_space.close() + + class ParsePDB(core.DirectoryDumperMixin, plugin.Command): """Parse the PDB streams.""" @@ -1161,49 +1205,57 @@ def PostProcessVTypes(self, vtypes): return vtypes + def parse_pdb(self): + with self.tpi: + vtypes = {} + + for i, (struct_name, definition) in enumerate(self.tpi.Structs()): + self.session.report_progress( + " Exporting %s: %s", i, struct_name) + + struct_name = str(struct_name) + existing_definition = vtypes.get(struct_name) + if existing_definition: + # Merge the old definition into the new definition. + definition[1].update(existing_definition[1]) + + vtypes[struct_name] = definition + + self.metadata.update(dict( + ProfileClass=self.profile_class, + Type="Profile", + PDBFile=os.path.basename(self.filename), + )) + + self.metadata.update(self.tpi.metadata) + + # Demangle all constants. + demangler = pe_vtypes.Demangler(self.metadata) + constants = {} + for name, value in self.tpi.constants.iteritems(): + constants[demangler.DemangleName(name)] = value + + functions = {} + for name, value in self.tpi.functions.iteritems(): + functions[demangler.DemangleName(name)] = value + + vtypes = self.PostProcessVTypes(vtypes) + + result = { + "$METADATA": self.metadata, + "$STRUCTS": vtypes, + "$ENUMS": self.tpi.enums, + } + + if not self.concise: + result["$REVENUMS"] = self.tpi.rev_enums + result["$CONSTANTS"] = constants + result["$FUNCTIONS"] = functions + + return result + def render(self, renderer): - vtypes = {} - - for i, (struct_name, definition) in enumerate(self.tpi.Structs()): - self.session.report_progress(" Exporting %s: %s", i, struct_name) - struct_name = str(struct_name) - existing_definition = vtypes.get(struct_name) - if existing_definition: - # Merge the old definition into the new definition. - definition[1].update(existing_definition[1]) - - vtypes[struct_name] = definition - - self.metadata.update(dict( - ProfileClass=self.profile_class, - Type="Profile", - PDBFile=os.path.basename(self.filename), - )) - - self.metadata.update(self.tpi.metadata) - - # Demangle all constants. - demangler = pe_vtypes.Demangler(self.metadata) - constants = {} - for name, value in self.tpi.constants.iteritems(): - constants[demangler.DemangleName(name)] = value - - functions = {} - for name, value in self.tpi.functions.iteritems(): - functions[demangler.DemangleName(name)] = value - - vtypes = self.PostProcessVTypes(vtypes) - - result = { - "$METADATA": self.metadata, - "$STRUCTS": vtypes, - "$ENUMS": self.tpi.enums, - } - - if not self.concise: - result["$REVENUMS"] = self.tpi.rev_enums - result["$CONSTANTS"] = constants - result["$FUNCTIONS"] = functions + result = self.parse_pdb() if self.output_filename: with renderer.open(filename=self.output_filename, diff --git a/rekall/plugins/tools/profile_tool.py b/rekall/plugins/tools/profile_tool.py index a01255a93..2177fa7f3 100755 --- a/rekall/plugins/tools/profile_tool.py +++ b/rekall/plugins/tools/profile_tool.py @@ -99,6 +99,7 @@ from rekall.plugins import core from rekall.plugins.overlays.linux import dwarfdump from rekall.plugins.overlays.linux import dwarfparser +from rekall.plugins.windows import common class ProfileConverter(object): @@ -261,7 +262,7 @@ def Convert(self): class OSXConverter(LinuxConverter): - """Automatic converted for Volatility OSX style profiles. + """Automatic conversion from Volatility OSX style profiles. You can generate one of those using the instructions here: http://code.google.com/p/volatility/wiki/MacMemoryForensics#Building_a_Profile @@ -316,25 +317,6 @@ def Convert(self): raise RuntimeError("Unknown profile format.") -class WindowsConverter(ProfileConverter): - """A converter from Volatility windows profiles. - - This converter must be manually specified. - """ - - def Convert(self): - if not self.profile_class: - raise RuntimeError("Profile class implementation not provided.") - - # The input file is a python file with a data structure in it. - with open(self.input, "rb") as fd: - l = {} - exec(fd.read(), {}, l) - - profile_file = self.BuildProfile({}, l["ntkrnlmp_types"]) - self.WriteProfile(profile_file) - - class ConvertProfile(core.OutputFileMixin, plugin.Command): """Convert a profile from another program to the Rekall format. @@ -708,39 +690,81 @@ def args(cls, parser): parser.add_argument( "guid", help="The guid of the module.", - required=True) + required=False) def __init__(self, module_name=None, guid=None, **kwargs): super(BuildProfileLocally, self).__init__(**kwargs) self.module_name = module_name self.guid = guid - def render(self, renderer): - profile_name = "{0}/GUID/{1}".format(self.module_name, self.guid) - renderer.format("Fetching Profile {0}", profile_name) - - dump_dir = "/tmp/" - fetch_pdb = self.session.RunPlugin( - "fetch_pdb", - pdb_filename="%s.pdb" % self.module_name, - guid=self.guid, dump_dir=dump_dir) - - if fetch_pdb.error_status: - raise RuntimeError( - "Failed fetching the pdb file: %s" % renderer.error_status) - - out_file = os.path.join(dump_dir, "%s.json" % self.guid) - parse_pdb = self.session.RunPlugin( - "parse_pdb", - pdb_filename=os.path.join(dump_dir, "%s.pdb" % self.module_name), - output_filename="%s.json" % self.guid, - dump_dir=dump_dir) - - if parse_pdb.error_status: - raise RuntimeError( - "Failed parsing pdb file: %s" % renderer.error_status) + def _fetch_and_parse(self, module_name, guid): + """Fetch the profile from the symbol server. + + Raises: + IOError if the profile is not found on the symbol server or can not be + retrieved. + + Returns: + the profile data. + """ + with utils.TempDirectory() as dump_dir: + pdb_filename = "%s.pdb" % module_name + fetch_pdb_plugin = self.session.plugins.fetch_pdb( + pdb_filename=pdb_filename, + guid=guid, dump_dir=dump_dir) + + # Store the PDB file somewhere. + pdb_pathname = os.path.join(dump_dir, pdb_filename) + with open(pdb_pathname, "wb") as outfd: + outfd.write(fetch_pdb_plugin.FetchPDBFile( + module_name, guid)) + + parse_pdb = self.session.plugins.parse_pdb( + pdb_filename=pdb_pathname, + dump_dir=dump_dir) + + return parse_pdb.parse_pdb() + + def fetch_and_parse(self, module_name=None, guid=None): + if module_name is None: + module_name = self.module_name + + if guid is None: + guid = self.guid + + # Allow the user to specify the required profile by name. + m = re.match("([^/]+)/GUID/([^/]+)$", module_name) + if m: + module_name = m.group(1) + guid = m.group(2) + + if not guid or not module_name: + raise TypeError("GUID not specified.") + + profile_name = "{0}/GUID/{1}".format(module_name, guid) # Get the first repository to write to. repository = self.session.repository_managers.values()[0] - data = json.load(open(out_file)) - repository.StoreData(profile_name, data) + if module_name != "nt": + data = self._fetch_and_parse(module_name, guid) + return repository.StoreData(profile_name, data) + + for module_name in common.KERNEL_NAMES: + if module_name.endswith(".pdb"): + module_name, _ = os.path.splitext(module_name) + try: + data = self._fetch_and_parse(module_name, guid) + logging.warning( + "Profile %s fetched and built. Please " + "consider reporting this profile to the " + "Rekall team so we may add it to the public " + "profile repository.", profile_name) + + return repository.StoreData(profile_name, data) + except IOError: + pass + + raise IOError("Profile not found") + + def render(self, renderer): + self.fetch_and_parse(self.module_name, self.guid) diff --git a/rekall/plugins/tools/webconsole/static/components/runplugin/paged-table.html b/rekall/plugins/tools/webconsole/static/components/runplugin/paged-table.html index faf7a885d..22019815a 100644 --- a/rekall/plugins/tools/webconsole/static/components/runplugin/paged-table.html +++ b/rekall/plugins/tools/webconsole/static/components/runplugin/paged-table.html @@ -31,7 +31,6 @@ -