From 2192994f9f75b878ad65cfcb098e881a1186077e Mon Sep 17 00:00:00 2001 From: Schamper <1254028+Schamper@users.noreply.github.com> Date: Thu, 25 Jul 2024 16:42:12 +0200 Subject: [PATCH] Initial commit for plugin internals refactor --- dissect/target/helpers/docs.py | 132 +- dissect/target/plugin.py | 1414 ++++++++--------- .../target/plugins/apps/webserver/apache.py | 1 - .../target/plugins/apps/webserver/caddy.py | 1 - dissect/target/plugins/apps/webserver/iis.py | 5 - .../target/plugins/apps/webserver/nginx.py | 1 - .../plugins/apps/webserver/webserver.py | 1 - dissect/target/plugins/general/osinfo.py | 2 - dissect/target/plugins/general/plugins.py | 181 +-- dissect/target/plugins/os/default/__init__.py | 0 .../{general/default.py => os/default/_os.py} | 4 +- .../plugins/os/unix/linux/debian/apt.py | 8 +- .../plugins/os/unix/linux/redhat/yum.py | 7 +- .../plugins/os/unix/linux/suse/zypper.py | 7 +- dissect/target/plugins/os/unix/log/audit.py | 3 +- .../target/plugins/os/unix/packagemanager.py | 38 +- dissect/target/plugins/os/windows/log/evt.py | 3 - dissect/target/plugins/os/windows/registry.py | 7 +- dissect/target/report.py | 38 +- dissect/target/target.py | 194 ++- dissect/target/tools/build_pluginlist.py | 9 +- dissect/target/tools/dump/run.py | 8 +- dissect/target/tools/dump/utils.py | 6 +- dissect/target/tools/query.py | 131 +- dissect/target/tools/shell.py | 8 +- dissect/target/tools/utils.py | 118 +- tests/conftest.py | 4 +- tests/helpers/test_docs.py | 2 +- tests/plugins/apps/webserver/test_apache.py | 9 +- tests/plugins/apps/webserver/test_caddy.py | 21 +- tests/plugins/apps/webserver/test_citrix.py | 3 +- tests/plugins/apps/webserver/test_nginx.py | 25 +- tests/plugins/general/test_default.py | 2 +- tests/plugins/general/test_plugins.py | 69 +- tests/plugins/os/unix/log/test_audit.py | 9 +- tests/plugins/os/unix/log/test_messages.py | 4 +- tests/plugins/os/windows/test_mru.py | 2 +- tests/plugins/os/windows/test_ual.py | 2 +- tests/test_plugin.py | 518 ++++-- tests/test_registration.py | 23 +- tests/test_report.py | 266 ++-- tests/tools/test_query.py | 47 +- tests/tools/test_utils.py | 24 +- 43 files changed, 1734 insertions(+), 1623 deletions(-) create mode 100644 dissect/target/plugins/os/default/__init__.py rename dissect/target/plugins/{general/default.py => os/default/_os.py} (93%) diff --git a/dissect/target/helpers/docs.py b/dissect/target/helpers/docs.py index f5fa71578..e505f3972 100644 --- a/dissect/target/helpers/docs.py +++ b/dissect/target/helpers/docs.py @@ -1,7 +1,9 @@ import inspect import itertools import textwrap -from typing import Any, Callable, Tuple, Type +from typing import Any, Callable + +from dissect.target.plugin import Plugin NO_DOCS = "No documentation" @@ -14,76 +16,23 @@ INDENT_STEP = " " * 4 - -def get_plugin_class_for_func(func: Callable) -> Type: - """Return pluging class for provided function instance""" - func_parent_name = func.__qualname__.rsplit(".", 1)[0] - klass = getattr(inspect.getmodule(func), func_parent_name, None) - return klass - - -def get_real_func_obj(func: Callable) -> Tuple[Type, Callable]: - """Return a tuple with plugin class and underlying func object for provided function instance""" - klass = None - - if isinstance(func, property): - # turn property into function - func = func.fget - - if inspect.ismethod(func): - for klass in inspect.getmro(func.__self__.__class__): - if func.__name__ in klass.__dict__: - break - else: - func = getattr(func, "__func__", func) - - if inspect.isfunction(func): - klass = get_plugin_class_for_func(func) - - if not klass: - raise ValueError(f"Can't find class for {func}") - - return (klass, func) +FUNC_DOC_TEMPLATE = "{func_name} - {short_description} (output: {output_type})" def get_docstring(obj: Any, placeholder=NO_DOCS) -> str: - """Get object's docstring or a placeholder if no docstring found""" + """Get object's docstring or a placeholder if no docstring found.""" # Use of `inspect.cleandoc()` is preferred to `textwrap.dedent()` here # because many multi-line docstrings in the codebase # have no indentation in the first line, which confuses `dedent()` return inspect.cleandoc(obj.__doc__) if obj.__doc__ else placeholder -def get_func_details(func: Callable) -> Tuple[str, str]: - """Return a tuple with function's name, output label and docstring""" - func_doc = get_docstring(func) - - if hasattr(func, "__output__") and func.__output__ in FUNCTION_OUTPUT_DESCRIPTION: - func_output = FUNCTION_OUTPUT_DESCRIPTION[func.__output__] - else: - func_output = "unknown" - - return (func_output, func_doc) - - -def get_full_func_name(plugin_class: Type, func: Callable) -> str: - func_name = func.__name__ - - if hasattr(plugin_class, "__namespace__") and plugin_class.__namespace__: - func_name = f"{plugin_class.__namespace__}.{func_name}" - - return func_name - - -FUNC_DOC_TEMPLATE = "{func_name} - {short_description} (output: {output_type})" - - def get_func_description(func: Callable, with_docstrings: bool = False) -> str: - klass, func = get_real_func_obj(func) - func_output, func_doc = get_func_details(func) + klass, func = _get_real_func_obj(func) + func_output, func_doc = _get_func_details(func) # get user-friendly function name - func_name = get_full_func_name(klass, func) + func_name = _get_full_func_name(klass, func) if with_docstrings: func_title = f"`{func_name}` (output: {func_output})" @@ -98,14 +47,17 @@ def get_func_description(func: Callable, with_docstrings: bool = False) -> str: return desc -def get_plugin_functions_desc(plugin_class: Type, with_docstrings: bool = False) -> str: +def get_plugin_functions_desc(plugin_class: type[Plugin], with_docstrings: bool = False) -> str: descriptions = [] for func_name in plugin_class.__exports__: func_obj = getattr(plugin_class, func_name) + if func_obj is getattr(plugin_class, "__call__", None): + continue + if getattr(func_obj, "get_func_doc_spec", None): func_desc = FUNC_DOC_TEMPLATE.format_map(func_obj.get_func_doc_spec()) else: - _, func = get_real_func_obj(func_obj) + _, func = _get_real_func_obj(func_obj) func_desc = get_func_description(func, with_docstrings=with_docstrings) descriptions.append(func_desc) @@ -120,7 +72,7 @@ def get_plugin_functions_desc(plugin_class: Type, with_docstrings: bool = False) return paragraph -def get_plugin_description(plugin_class: Type) -> str: +def get_plugin_description(plugin_class: type[Plugin]) -> str: plugin_name = plugin_class.__name__ plugin_desc_title = f"`{plugin_name}` (`{plugin_class.__module__}.{plugin_name}`)" plugin_doc = textwrap.indent(get_docstring(plugin_class), prefix=INDENT_STEP) @@ -128,7 +80,9 @@ def get_plugin_description(plugin_class: Type) -> str: return paragraph -def get_plugin_overview(plugin_class: Type, with_plugin_desc: bool = False, with_func_docstrings: bool = False) -> str: +def get_plugin_overview( + plugin_class: type[Plugin], with_plugin_desc: bool = False, with_func_docstrings: bool = False +) -> str: paragraphs = [] if with_plugin_desc: @@ -150,3 +104,55 @@ def get_plugin_overview(plugin_class: Type, with_plugin_desc: bool = False, with paragraphs.append(func_descriptions_paragraph) overview = "\n".join(paragraphs) return overview + + +def _get_plugin_class_for_func(func: Callable) -> type[Plugin]: + """Return plugin class for provided function instance.""" + func_parent_name = func.__qualname__.rsplit(".", 1)[0] + klass = getattr(inspect.getmodule(func), func_parent_name, None) + return klass + + +def _get_real_func_obj(func: Callable) -> tuple[type[Plugin], Callable]: + """Return a tuple with plugin class and underlying function object for provided function instance.""" + klass = None + + if isinstance(func, property): + # turn property into function + func = func.fget + + if inspect.ismethod(func): + for klass in inspect.getmro(func.__self__.__class__): + if func.__name__ in klass.__dict__: + break + else: + func = getattr(func, "__func__", func) + + if inspect.isfunction(func): + klass = _get_plugin_class_for_func(func) + + if not klass: + raise ValueError(f"Can't find class for {func}") + + return (klass, func) + + +def _get_func_details(func: Callable) -> tuple[str, str]: + """Return a tuple with function's name, output label and docstring""" + func_doc = get_docstring(func) + + if hasattr(func, "__output__") and func.__output__ in FUNCTION_OUTPUT_DESCRIPTION: + func_output = FUNCTION_OUTPUT_DESCRIPTION[func.__output__] + else: + func_output = "unknown" + + return (func_output, func_doc) + + +def _get_full_func_name(plugin_class: type[Plugin], func: Callable) -> str: + func_name = func.__name__ + + if hasattr(plugin_class, "__namespace__") and plugin_class.__namespace__: + func_name = f"{plugin_class.__namespace__}.{func_name}" + + return func_name diff --git a/dissect/target/plugin.py b/dissect/target/plugin.py index dfd08d1ad..fcd5dd920 100644 --- a/dissect/target/plugin.py +++ b/dissect/target/plugin.py @@ -2,9 +2,11 @@ See dissect/target/plugins/general/example.py for an example plugin. """ + from __future__ import annotations import fnmatch +import functools import importlib import importlib.util import inspect @@ -13,36 +15,32 @@ import sys import traceback from collections import defaultdict -from dataclasses import dataclass, field +from dataclasses import dataclass +from itertools import zip_longest from pathlib import Path -from typing import TYPE_CHECKING, Any, Callable, Iterator, Optional, Type +from typing import TYPE_CHECKING, Any, Callable, Iterator from flow.record import Record, RecordDescriptor -import dissect.target.plugins.general as general +import dissect.target.plugins.os.default as default from dissect.target.exceptions import PluginError, UnsupportedPluginError from dissect.target.helpers import cache +from dissect.target.helpers.fsutil import has_glob_magic from dissect.target.helpers.record import EmptyRecord from dissect.target.helpers.utils import StrEnum -try: - from dissect.target.plugins._pluginlist import PLUGINS - - GENERATED = True -except Exception: - PLUGINS = {} - GENERATED = False - if TYPE_CHECKING: from dissect.target import Target from dissect.target.filesystem import Filesystem from dissect.target.helpers.record import ChildTargetRecord -PluginDescriptor = dict[str, Any] -"""A dictionary type, for what the plugin descriptor looks like.""" +log = logging.getLogger(__name__) MODULE_PATH = "dissect.target.plugins" """The base module path to the in-tree plugins.""" + +OS_MODULE_PATH = "dissect.target.plugins.os" + OUTPUTS = ( "default", "record", @@ -51,7 +49,43 @@ ) """The different output types supported by ``@export``.""" -log = logging.getLogger(__name__) +INTERNAL_METHODS = ( + "is_compatible", + "check_compatible", +) +"""The methods that are internal to the plugin system.""" + +PLUGINS = { + # Plugin descriptor lookup + # {"": {"": PluginDescriptor}} + "__plugins__": { + # All regular plugins + # {"": PluginDescriptor} + None: {}, + # All OS plugins + # {"": PluginDescriptor} + "__os__": {}, + # All child plugins + # {"": PluginDescriptor} + "__child__": {}, + }, + # Function descriptor lookup + # {"": {"": FunctionDescriptor}} + "__functions__": {}, + # OS plugin tree + # {"": {"": PluginDescriptor}} + "__ostree__": {}, + # Failures + # [FailureDescriptor] + "__failed__": [], +} +"""The plugin registry. + +Note: It's very important that all values in this dictionary are serializable. +The plugin registry can be stored in a file and loaded later. Plain Python syntax is used to store the registry. +An exception is made for :class:`FailureDescriptor`, :class:`FunctionDescriptor` and :class:`PluginDescriptor`. +""" +GENERATED = False class OperatingSystem(StrEnum): @@ -68,6 +102,37 @@ class OperatingSystem(StrEnum): CITRIX = "citrix-netscaler" +@dataclass(frozen=True, eq=True) +class PluginDescriptor: + module: str + qualname: str + namespace: str + path: str + findable: bool + functions: list[str] + exports: list[str] + + +@dataclass(frozen=True, eq=True) +class FunctionDescriptor: + name: str + namespace: str + path: str + exported: bool + internal: bool + findable: bool + output: str | None + method_name: str + module: str + qualname: str + + +@dataclass(frozen=True, eq=True) +class FailureDescriptor: + module: str + stacktrace: list[str] + + def export(*args, **kwargs) -> Callable: """Decorator to be used on Plugin functions that should be exported. @@ -100,7 +165,7 @@ def export(*args, **kwargs) -> Callable: def decorator(obj): # Properties are implicitly cached # Important! Currently it's crucial that this is *always* called - # See the comment in Plugin.__init_subclass__ for more detail regarding Plugin.get_all_records + # See the comment in Plugin.__init_subclass__ for more detail regarding Plugin.__call__ obj = cache.wrap(obj, no_cache=not kwargs.get("cache", True), cls=kwargs.get("cls", None)) output = kwargs.get("output", "default") @@ -127,41 +192,48 @@ def decorator(obj): return decorator -def get_nonprivate_attribute_names(cls: Type[Plugin]) -> list[str]: - """Retrieve all attributes that do not start with ``_``.""" - return [attr for attr in dir(cls) if not attr.startswith("_")] +def internal(*args, **kwargs) -> Callable: + """Decorator to be used on plugin functions that should be internal only. + Making a plugin internal means that it's only callable from the Python API and not through ``target-query``. -def get_nonprivate_attributes(cls: Type[Plugin]) -> list[Any]: - """Retrieve all public attributes of a :class:`Plugin`.""" - # Note: `dir()` might return attributes from parent class - return [getattr(cls, attr) for attr in get_nonprivate_attribute_names(cls)] + This decorator adds the ``__internal__`` private attribute to a method or property. + The attribute is always set to ``True``, to tell :func:`register` that it is an internal + method or property. + """ + def decorator(obj): + obj.__internal__ = True + if kwargs.get("property", False): + obj = property(obj) + return obj -def get_nonprivate_methods(cls: Type[Plugin]) -> list[Callable]: - """Retrieve all public methods of a :class:`Plugin`.""" - return [attr for attr in get_nonprivate_attributes(cls) if not isinstance(attr, property)] + if len(args) == 1: + return decorator(args[0]) + else: + return decorator -def get_descriptors_on_nonprivate_methods(cls: Type[Plugin]) -> list[RecordDescriptor]: - """Return record descriptors set on nonprivate methods in `cls` class.""" - descriptors = set() - methods = get_nonprivate_methods(cls) +def arg(*args, **kwargs) -> Callable: + """Decorator to be used on Plugin functions that accept additional command line arguments. - for m in methods: - if not hasattr(m, "__record__"): - continue + Command line arguments can be added using the ``@arg`` decorator. + Arguments to this decorator are directly forwarded to the ``ArgumentParser.add_argument`` function of ``argparse``. + Resulting arguments are passed to the function using kwargs. + The keyword argument name must match the argparse argument name. - record = m.__record__ - if not record: - continue + This decorator adds the ``__args__`` private attribute to a method or property. + This attribute holds all the command line arguments that were added to the plugin function. + """ - try: - # check if __record__ value is iterable (for example, a list) - descriptors.update(record) - except TypeError: - descriptors.add(record) - return list(descriptors) + def decorator(obj): + if not hasattr(obj, "__args__"): + obj.__args__ = [] + arglist = getattr(obj, "__args__", []) + arglist.append((args, kwargs)) + return obj + + return decorator class Plugin: @@ -180,7 +252,6 @@ class attribute. Namespacing results in your plugin needing to be prefixed With the following three being assigned in :func:`register`: - - ``__plugin__`` - ``__functions__`` - ``__exports__`` @@ -215,23 +286,25 @@ class attribute. Namespacing results in your plugin needing to be prefixed produce redundant results when used with a wild card (browser.* -> browser.history + browser.*.history). """ + __functions__: list[str] + """Internal. A list of all method names decorated with ``@internal`` or ``@export``.""" + __exports__: list[str] + """Internal. A list of all method names decorated with ``@export``.""" def __init_subclass__(cls, **kwargs): super().__init_subclass__(**kwargs) - # Do not register the "base" subclassess `OSPlugin` and `ChildTargetPlugin` - if cls.__name__ not in ("OSPlugin", "ChildTargetPlugin") and cls.__register__: + # Do not register the "base" subclasses defined in this file + if cls.__module__ != Plugin.__module__: register(cls) - record_descriptors = get_descriptors_on_nonprivate_methods(cls) + record_descriptors = _get_descriptors_on_nonprivate_methods(cls) cls.__record_descriptors__ = record_descriptors # This is a bit tricky currently - # cls.get_all_records is the *function* Plugin.get_all_records, not from the subclass + # cls.__call__ is the *function* Plugin.__call__, not from the subclass # export() currently will _always_ return a new object because it always calls ``cache.wrap(obj)`` - # This allows this to work, otherwise the Plugin.get_all_records would get all the plugin attributes set on it - cls.get_all_records = export(output="record", record=record_descriptors, cache=False, cls=cls)( - cls.get_all_records - ) + # This allows this to work, otherwise the Plugin.__call__ would get all the plugin attributes set on it + cls.__call__ = export(output="record", record=record_descriptors, cache=False, cls=cls)(cls.__call__) def __init__(self, target: Target): self.target = target @@ -258,563 +331,513 @@ def check_compatible(self) -> None: """ raise NotImplementedError - def get_all_records(self) -> Iterator[Record]: + def __call__(self, *args, **kwargs) -> Iterator[Record]: """Return the records of all exported methods. Raises: PluginError: If the subclass is not a namespace plugin. """ if not self.__namespace__: - raise PluginError(f"Plugin {self.__class__.__name__} is not a namespace plugin") + raise PluginError(f"Plugin {self.__class__.__name__} is not a callable") for method_name in self.__exports__: + if method_name == "__call__": + continue + method = getattr(self, method_name) + if getattr(method, "__output__", None) != "record": + continue try: yield from method() except Exception: self.target.log.error("Error while executing `%s.%s`", self.__namespace__, method_name, exc_info=True) - def __call__(self, *args, **kwargs): - """A shortcut to :func:`get_all_records`. - Raises: - PluginError: If the subclass is not a namespace plugin. - """ - if not self.__namespace__: - raise PluginError(f"Plugin {self.__class__.__name__} is not a callable") - return self.get_all_records() +def register(plugincls: type[Plugin]) -> None: + """Register a plugin, and put related data inside :attr:`PLUGINS`. + This function uses the following private attributes that are set using decorators: -class OSPlugin(Plugin): - """Base class for OS plugins. + - ``__exported__``: Set in :func:`export`. + - ``__internal__``: Set in :func:`internal`. - This provides a base class for certain common functions of OS's, which each OS plugin has to implement separately. + Additionally, ``register`` sets the following private attributes on the `plugincls`: - For example, it provides an interface for retrieving the hostname and users of a target. + - ``__functions__``: A list of all the methods and properties that are ``__internal__`` or ``__exported__``. + - ``__exports__``: A list of all the methods or properties that were explicitly exported. - All derived classes MUST implement ALL the classmethods and exported - methods with the same ``@classmethod`` or ``@export(...)`` annotation. - """ + If a plugincls ``__register__`` attribute is set to ``False``, the plugin will not be registered, but the + plugin will still be processed for the private attributes mentioned above. - def __init_subclass__(cls, **kwargs): - # Note that cls is the subclass - super().__init_subclass__(**kwargs) + Args: + plugincls: A plugin class to register. - for os_method in get_nonprivate_attributes(OSPlugin): - if isinstance(os_method, property): - os_method = os_method.fget - os_docstring = os_method.__doc__ + Raises: + ValueError: If ``plugincls`` is not a subclass of :class:`Plugin`. + """ + if not issubclass(plugincls, Plugin): + raise ValueError("Not a subclass of Plugin") - method = getattr(cls, os_method.__name__, None) - if isinstance(method, property): - method = method.fget - # This works as None has a __doc__ property (which is None). - docstring = method.__doc__ + # Register the plugin in the correct tree + key = None + if issubclass(plugincls, OSPlugin): + key = "__os__" + elif issubclass(plugincls, ChildTargetPlugin): + key = "__child__" - if method and not docstring: - if hasattr(method, "__func__"): - method = method.__func__ - method.__doc__ = os_docstring + __plugins__ = PLUGINS.setdefault("__plugins__", {}) + __ostree__ = PLUGINS.setdefault("__ostree__", {}) + __functions__ = PLUGINS.setdefault("__functions__", {}) - def check_compatible(self) -> bool: - """OSPlugin's use a different compatibility check, override the one from the :class:`Plugin` class. + function_index = __functions__.setdefault(key, {}) - Returns: - This function always returns ``True``. - """ - return True + exports = [] + functions = [] + module_path = _module_path(plugincls) + module_key = f"{module_path}.{plugincls.__qualname__}" - @classmethod - def detect(cls, fs: Filesystem) -> Optional[Filesystem]: - """Provide detection of this OSPlugin on a given filesystem. + if not issubclass(plugincls, ChildTargetPlugin): + for attr in _get_nonprivate_attributes(plugincls): + if isinstance(attr, property): + attr = attr.fget - Args: - fs: :class:`~dissect.target.filesystem.Filesystem` to detect the OS on. + if getattr(attr, "__autogen__", False) and plugincls != plugincls.__nsplugin__: + continue - Returns: - The root filesystem / sysvol when found. - """ - raise NotImplementedError + exported = getattr(attr, "__exported__", False) + internal = getattr(attr, "__internal__", False) - @classmethod - def create(cls, target: Target, sysvol: Filesystem) -> OSPlugin: - """Initiate this OSPlugin with the given target and detected filesystem. + if exported or internal: + functions.append(attr.__name__) + if exported: + exports.append(attr.__name__) - Args: - target: The :class:`~dissect.target.target.Target` object. - sysvol: The filesystem that was detected in the ``detect()`` function. + if plugincls.__register__: + name = attr.__name__ + if plugincls.__namespace__: + name = f"{plugincls.__namespace__}.{name}" - Returns: - An instantiated version of the OSPlugin. - """ - raise NotImplementedError + path = f"{module_path}.{attr.__name__}" - @export(property=True) - def hostname(self) -> Optional[str]: - """Return the target's hostname. + members: dict[str, list] = function_index.setdefault(name, {}) + if module_key in members: + continue - Returns: - The hostname as string. - """ - raise NotImplementedError + descriptor = FunctionDescriptor( + name=name, + namespace=plugincls.__namespace__, + path=path, + exported=exported, + internal=internal, + findable=plugincls.__findable__, + output=getattr(attr, "__output__", None), + method_name=attr.__name__, + module=plugincls.__module__, + qualname=plugincls.__qualname__, + ) - @export(property=True) - def ips(self) -> list[str]: - """Return the IP addresses configured in the target. + # Register the functions in the lookup + members[module_key] = descriptor + + if plugincls.__namespace__: + # Namespaces are also callable, so register the namespace itself as well + if module_key not in function_index.get(plugincls.__namespace__, {}): + functions.append("__call__") + if len(exports): + exports.append("__call__") + + if plugincls.__register__: + descriptor = FunctionDescriptor( + name=plugincls.__namespace__, + namespace=plugincls.__namespace__, + path=module_path, + exported=bool(len(exports)), + internal=bool(len(functions)) and not bool(len(exports)), + findable=plugincls.__findable__, + output=getattr(plugincls.__call__, "__output__", None), + method_name="__call__", + module=plugincls.__module__, + qualname=plugincls.__qualname__, + ) - Returns: - The IPs as list. - """ - raise NotImplementedError + function_index.setdefault(plugincls.__namespace__, {})[module_key] = descriptor - @export(property=True) - def version(self) -> Optional[str]: - """Return the target's OS version. + # Update the class with the plugin attributes + plugincls.__functions__ = functions + plugincls.__exports__ = exports - Returns: - The OS version as string. - """ - raise NotImplementedError + if plugincls.__register__: + index: dict[str, list] = __plugins__.setdefault(key, {}) + if module_key in index: + return - @export(record=EmptyRecord) - def users(self) -> list[Record]: - """Return the users available in the target. + index[module_key] = PluginDescriptor( + module=plugincls.__module__, + qualname=plugincls.__qualname__, + namespace=plugincls.__namespace__, + path=module_path, + findable=plugincls.__findable__, + functions=functions, + exports=exports, + ) - Returns: - A list of user records. - """ - raise NotImplementedError + if issubclass(plugincls, OSPlugin): + # Also store the OS plugins in a tree by module path + # This is used to filter plugins based on the OSPlugin subclass + # We don't store anything at the end of the tree, as we only use the tree to check if a plugin is compatible - @export(property=True) - def os(self) -> str: - """Return a slug of the target's OS name. + # Also slightly modify the module key to allow for more efficient filtering later + # This is done by removing the last two parts of the module key, which are the file name and the class name + module_parts = module_key.split(".") + if module_parts[-2] != "_os": + log.warning("OS plugin modules should be named as /_os.py: %s", module_key) - Returns: - A slug of the OS name, e.g. 'windows' or 'linux'. - """ - raise NotImplementedError + obj = __ostree__ + for part in module_parts[:-2]: + obj = obj.setdefault(part, {}) - @export(property=True) - def architecture(self) -> Optional[str]: - """Return a slug of the target's OS architecture. + log.debug("Plugin registered: %s", module_key) - Returns: - A slug of the OS architecture, e.g. 'x86_32-unix', 'MIPS-linux' or - 'AMD64-win32', or 'unknown' if the architecture is unknown. - """ - raise NotImplementedError +def _get_plugins() -> dict[str, Any]: + """Load the plugin registry, or generate it if it doesn't exist yet.""" + global PLUGINS, GENERATED -class ChildTargetPlugin(Plugin): - """A Child target is a special plugin that can list more Targets. + if not GENERATED: + try: + from dissect.target.plugins._pluginlist import PLUGINS + except ImportError: + PLUGINS = generate() - For example, :class:`~dissect.target.plugins.child.esxi.ESXiChildTargetPlugin` can - list all of the Virtual Machines on the host. - """ + GENERATED = True - __type__ = None + return PLUGINS - def list_children(self) -> Iterator[ChildTargetRecord]: - """Yield :class:`~dissect.target.helpers.record.ChildTargetRecord` records of all - possible child targets on this target. - """ - raise NotImplementedError +def _module_path(cls: type[Plugin] | str) -> str: + """Returns the module path relative to ``dissect.target.plugins``.""" + if issubclass(cls, Plugin): + module = getattr(cls, "__module__", "") + elif isinstance(cls, str): + module = cls + else: + raise ValueError(f"Invalid argument type: {cls}") -def register(plugincls: Type[Plugin]) -> None: - """Register a plugin, and put related data inside :attr:`PLUGINS`. + return module.replace(MODULE_PATH, "").lstrip(".") - This function uses the following private attributes that are set using decorators: - - ``__exported__``: Set in :func:`export`. - - ``__internal__``: Set in :func:`internal`. +def _os_match(osfilter: type[OSPlugin], module_path: str) -> bool: + """Check if the a plugin is compatible with the given OS filter.""" + if issubclass(osfilter, default._os.DefaultPlugin): + return True - Additionally, ``register`` sets the following private attributes on the `plugincls`: + os_parts = _module_path(osfilter).split(".")[:-1] - - ``__plugin__``: Always set to ``True``. - - ``__functions__``: A list of all the methods and properties that are ``__internal__`` or ``__exported__``. - - ``__exports__``: A list of all the methods or properties that were explicitly exported. + obj = _get_plugins()["__ostree__"] + for plugin_part, os_part in zip_longest(module_path.split("."), os_parts): + if plugin_part not in obj: + break - Args: - plugincls: A plugin class to register. + if plugin_part != os_part: + return False - Raises: - ValueError: If ``plugincls`` is not a subclass of :class:`Plugin`. - """ - if not issubclass(plugincls, Plugin): - raise ValueError("Not a subclass of Plugin") + obj = obj[plugin_part] - exports = [] - functions = [] + return True - for attr in get_nonprivate_attributes(plugincls): - if isinstance(attr, property): - attr = attr.fget - if getattr(attr, "__autogen__", False) and plugincls != plugincls.__nsplugin__: - continue +def plugins(osfilter: type[OSPlugin] | None = None, *, index: str | None = None) -> Iterator[PluginDescriptor]: + """Walk the plugin registry and return plugin descriptors. - if getattr(attr, "__exported__", False): - exports.append(attr.__name__) - functions.append(attr.__name__) + If ``osfilter`` is specified, only plugins related to the provided OSPlugin, or plugins + with no OS relation are returned. If ``osfilter`` is ``None``, all plugins will be returned. - if getattr(attr, "__internal__", False): - functions.append(attr.__name__) + One exception to this is if the ``osfilter`` is a (sub-)class of DefaultPlugin, then plugins + are returned as if no ``osfilter`` was specified. - plugincls.__plugin__ = True - plugincls.__functions__ = functions - plugincls.__exports__ = exports + The ``index`` parameter can be used to specify the index to return plugins from. By default, + this is set to return regular plugins. Other possible values are ``__os__`` and ``__child__``. + These return :class:`OSPlugin` and :class:`ChildTargetPlugin` respectively. - modpath = _modulepath(plugincls) - lookup_path = modpath - if modpath.endswith("._os"): - lookup_path, _, _ = modpath.rpartition(".") + Args: + osfilter: The optional :class:`OSPlugin` to filter the returned plugins on. + index: The plugin index to return plugins from. Defaults to regular plugins. - root = _traverse(lookup_path, PLUGINS) + Yields: + Plugin descriptors in the plugin registry based on the given filter criteria. + """ - log.debug("Plugin registered: %s.%s", plugincls.__module__, plugincls.__qualname__) + yield from ( + value + for key, value in _get_plugins().get("__plugins__", {}).get(index, {}).items() + if (index != "__os__" and (osfilter is None or _os_match(osfilter, key))) + or (index == "__os__" and (osfilter is None or osfilter.__module__ == value.module)) + ) - if issubclass(plugincls, (OSPlugin, ChildTargetPlugin)): - if issubclass(plugincls, OSPlugin): - special_key = "_os" - elif issubclass(plugincls, ChildTargetPlugin): - special_key = "_child" - root[special_key] = {} - root = root[special_key] +def os_plugins() -> Iterator[PluginDescriptor]: + """Retrieve all OS plugin descriptors.""" + yield from plugins(index="__os__") - # Check if the plugin was already registered - if "class" in root and root["class"] == plugincls.__name__: - return - # Finally register the plugin - root["class"] = plugincls.__name__ - root["module"] = modpath - root["functions"] = plugincls.__functions__ - root["exports"] = plugincls.__exports__ - root["namespace"] = plugincls.__namespace__ - root["fullname"] = ".".join((plugincls.__module__, plugincls.__qualname__)) - root["is_osplugin"] = issubclass(plugincls, OSPlugin) +def child_plugins() -> Iterator[PluginDescriptor]: + """Retrieve all child plugin descriptors.""" + yield from plugins(index="__child__") -def internal(*args, **kwargs) -> Callable: - """Decorator to be used on plugin functions that should be internal only. +def functions(osfilter: type[OSPlugin] | None = None, *, index: str | None = None) -> Iterator[FunctionDescriptor]: + """Retrieve all function descriptors. - Making a plugin internal means that it's only callable from the Python API and not through ``target-query``. + Args: + osfilter: The optional :class:`OSPlugin` to filter the returned functions on. + index: The plugin index to return functions from. Defaults to regular functions. - This decorator adds the ``__internal__`` private attribute to a method or property. - The attribute is always set to ``True``, to tell :func:`register` that it is an internal - method or property. + Yields: + Function descriptors in the plugin registry based on the given filter criteria. """ + yield from ( + value + for entry in _get_plugins().get("__functions__", {}).get(index, {}).values() + for key, value in entry.items() + if osfilter is None or _os_match(osfilter, key) + ) - def decorator(obj): - obj.__internal__ = True - if kwargs.get("property", False): - obj = property(obj) - return obj - - if len(args) == 1: - return decorator(args[0]) - else: - return decorator +def lookup( + func_name: str, osfilter: type[OSPlugin] | None = None, *, index: str | None = None +) -> Iterator[FunctionDescriptor]: + """Lookup a function descriptor by function name. -def arg(*args, **kwargs) -> Callable: - """Decorator to be used on Plugin functions that accept additional command line arguments. - - Command line arguments can be added using the ``@arg`` decorator. - Arguments to this decorator are directly forwarded to the ``ArgumentParser.add_argument`` function of ``argparse``. - Resulting arguments are passed to the function using kwargs. - The keyword argument name must match the argparse argument name. + Args: + func_name: Function name to lookup. + osfilter: The optional ``OSPlugin`` to filter results with for compatibility. + index: The plugin index to return plugins from. Defaults to regular functions. - This decorator adds the ``__args__`` private attribute to a method or property. - This attribute holds all the command line arguments that were added to the plugin function. + Yields: + Function descriptors that match the given function name and filter criteria. """ - def decorator(obj): - if not hasattr(obj, "__args__"): - obj.__args__ = [] - arglist = getattr(obj, "__args__", []) - arglist.append((args, kwargs)) - return obj + entries: Iterator[FunctionDescriptor] = ( + value + for key, value in _get_plugins().get("__functions__", {}).get(index, {}).get(func_name, {}).items() + if osfilter is None or _os_match(osfilter, key) + ) - return decorator + yield from sorted(entries, key=lambda x: x.module.count("."), reverse=True) -def plugins( - osfilter: Optional[type[OSPlugin]] = None, - special_keys: set[str] = set(), - only_special_keys: bool = False, -) -> Iterator[PluginDescriptor]: - """Walk the ``PLUGINS`` tree and return plugins. +def load(desc: FunctionDescriptor | PluginDescriptor) -> type[Plugin]: + """Helper function that loads a plugin from a given function or plugin descriptor. - If ``osfilter`` is specified, only plugins related to the provided - OSPlugin, or plugins with no OS relation are returned. - If ``osfilter`` is ``None``, all plugins will be returned. + Args: + desc: Function descriptor as returned by :func:`plugin.lookup` or plugin descriptor + as returned by :func:`plugin.plugins`. - One exception to this is if the ``osfilter`` is a (sub-)class of - DefaultPlugin, then plugins are returned as if no ``osfilter`` was - specified. + Returns: + The plugin class. - Another exeption to this are plugins in the ``PLUGINS`` tree which are - under a key that starts with a '_'. Those are only returned if their exact - key is specified in ``special_keys``. + Raises: + PluginError: Raised when any other exception occurs while trying to load the plugin. + """ + module = desc.module + + try: + obj = importlib.import_module(module) + for part in desc.qualname.split("."): + obj = getattr(obj, part) + return obj + except Exception as e: + raise PluginError(f"An exception occurred while trying to load a plugin: {module}", cause=e) - An exception to these exceptions is in the case of ``OSPlugin`` (sub-)class - plugins and ``os_filter`` is not ``None``. These plugins live in the - ``PLUGINS`` tree under the ``_os`` special key. Those plugins are only - returned if they fully match the provided ``osfilter``. - The ``only_special_keys`` option returns only the plugins which are under a - special key that is defined in ``special_keys``. All filtering here will - happen as stated in the above cases. +def os_match(target: Target, descriptor: PluginDescriptor) -> bool: + """Check if a plugin descriptor is compatible with the target OS. Args: - osfilter: The optional OSPlugin to filter the returned plugins on. - special_keys: Also return plugins which are under the special ('_') keys in this set. - only_special_keys: Only return the plugins under the keys in ``special_keys`` and no others. - - Yields: - Plugins in the ``PLUGINS`` tree based on the given filter criteria. + target: The target to check compatibility with. + descriptor: The plugin descriptor to check compatibility for. """ + return _os_match(target._os_plugin, f"{descriptor.module}.{descriptor.qualname}") - if osfilter is not None: - # The PLUGINS tree does not include the hierarchy up to the plugins - # directory (dissect.target.plugins) for the built-in plugins. For the - # plugins in the directory specified in --plugin-path, the hierarchy - # starts at that directory. - # - # Built-in OSPlugins do have the dissect.target.plugins path in their - # module name, so it needs to be stripped, e.g.: - # dissect.target.plugins.general.default -> general.default - # dissect.target.plugins.windows._os -> plugins.windows._os - # - # The module name of OSPlugins from --plugin-path starts at the - # directory specified in that option, e.g.: - # --plugin-path=/some/path/, with a file foo/baros/_os.py - # will have a module name of: foo.baros._os - filter_path = _modulepath(osfilter).split(".") - - # If an OSPlugin is not defined in a file called _os.py, an extra `_os` - # part is added to the PLUGINS tree. - # For example the default OS plugin with module name general.default - # (after stripping of the build-in hierarchy) will be added at: - # general - # \- default - # \- _os - # However the `_os` part is not in the module name. Modules that are - # defined in an _os.py file have the `_os` part in their module name. - # It is stripped out, so the filter is similar for both types of - # OSPlugin files. - if filter_path[-1] == "_os": - filter_path = filter_path[:-1] - else: - filter_path = [] - - def _walk( - root: dict, - special_keys: set[str] = set(), - only_special_keys: bool = False, - prev_module_path: list[str] = [], - ): - for key, obj in root.items(): - module_path = prev_module_path.copy() - module_path.append(key) - - # A branch in the PLUGINS tree is traversed to the next level if: - # - there are no filters (which in effect means all plugins are - # returned including all _os plugins). - # - the osfilter is the default plugin (which means all normal plugins but - # only the default _os plugin is returned). - # - there is no _os plugin on the next level (we're traversing a - # "normal" plugin branch or already jumped into an OS specific - # branch because of a filter_path match) - # - the current module_path fully matches the (beginning of) the - # filter path (this allows traversing into the specific os branch - # for the given os filter and any sub branches which are not os - # branches (of a sub-os) themselves). - if ( - not filter_path - or issubclass(osfilter, general.default.DefaultPlugin) - or "_os" not in obj - or module_path == filter_path[: len(module_path)] - ): - if key.startswith("_"): - if key in special_keys: - # OSPlugins are treated special and are only returned - # if their module_path matches the full filter_path. - # - # Note that the module_path includes the `_os` part, - # which may have been explicitly added in the - # hierarchy. This part needs to be stripped out when - # matching against the filter_path, where it was either - # not present or stripped out. - if key != "_os" or ( - key == "_os" and (not filter_path or (filter_path and module_path[:-1] == filter_path)) - ): - # If the special key is a leaf-node, we just give it back. - # If it is a branch, we give back the full branch, - # not just the special_keys if only_special_keys - # was set to True. - if "functions" in obj: - yield obj - else: - yield from _walk( - obj, - special_keys=special_keys, - only_special_keys=False, - prev_module_path=module_path, - ) - else: - continue - else: - continue - else: - if "functions" in obj: - if not (special_keys and only_special_keys): - yield obj - else: - yield from _walk( - obj, - special_keys=special_keys, - only_special_keys=only_special_keys, - prev_module_path=module_path, - ) - - yield from sorted( - _walk( - _get_plugins(), - special_keys=special_keys, - only_special_keys=only_special_keys, - ), - key=lambda plugin: len(plugin["module"]), - reverse=True, - ) +def failed() -> list[FailureDescriptor]: + """Return all plugins that failed to load.""" + return _get_plugins().get("__failed__", []) + + +@functools.cache +def _generate_long_paths() -> dict[str, FunctionDescriptor]: + """Generate a dictionary of all long paths to their function descriptors.""" + paths = {} + for value in _get_plugins().get("__functions__", {}).get(None, {}).values(): + value: dict[str, FunctionDescriptor] + for descriptor in value.values(): + # Namespace plugins are callable so exclude the explicit __call__ method + if descriptor.method_name == "__call__": + continue + paths[descriptor.path] = descriptor + return paths -def os_plugins() -> Iterator[PluginDescriptor]: - """Retrieve all OS plugin descriptors.""" - yield from plugins(special_keys={"_os"}, only_special_keys=True) +def find_plugin_functions( + patterns: str, + target: Target | None = None, + compatibility: bool = False, + show_hidden: bool = False, + ignore_load_errors: bool = False, +) -> tuple[list[FunctionDescriptor], set[str]]: + """Finds exported plugin functions that match the target and the patterns. -def child_plugins() -> Iterator[PluginDescriptor]: - """Retrieve all child plugin descriptors.""" - yield from plugins(special_keys={"_child"}, only_special_keys=True) + Given a target, a comma separated list of patterns and an optional compatibility flag, + this function finds matching plugins, optionally checking compatibility and returns + a list of plugin function descriptors (including output types). + """ + found = [] + registry = _get_plugins() + __functions__: dict[str, dict[str, FunctionDescriptor]] = registry.get("__functions__", {}) -def lookup(func_name: str, osfilter: Optional[type[OSPlugin]] = None) -> Iterator[PluginDescriptor]: - """Lookup a plugin descriptor by function name. + base_functions = __functions__.get(None, {}) + os_functions = __functions__.get("__os__", {}) - Args: - func_name: Function name to lookup. - osfilter: The ``OSPlugin`` to use as template to find os specific plugins for. - """ - yield from get_plugins_by_func_name(func_name, osfilter=osfilter) - yield from get_plugins_by_namespace(func_name, osfilter=osfilter) + os_filter = target._os_plugin if target is not None else None + invalid_functions = set() -def get_plugins_by_func_name(func_name: str, osfilter: Optional[type[OSPlugin]] = None) -> Iterator[PluginDescriptor]: - """Get a plugin descriptor by function name. + for pattern in patterns.split(","): + if not pattern: + continue - Args: - func_name: Function name to lookup. - osfilter: The ``OSPlugin`` to use as template to find os specific plugins for. - """ - for plugin_desc in plugins(osfilter): - if not plugin_desc["namespace"] and func_name in plugin_desc["functions"]: - yield plugin_desc + exact_match = pattern in base_functions + exact_os_match = pattern in os_functions + if exact_match or exact_os_match: + if exact_match: + descriptors = lookup(pattern, os_filter, index=None) + elif exact_os_match: + descriptors = lookup(pattern, os_filter, index="__os__") -def get_plugins_by_namespace(namespace: str, osfilter: Optional[type[OSPlugin]] = None) -> Iterator[PluginDescriptor]: - """Get a plugin descriptor by namespace. + for descriptor in descriptors: + if not descriptor.exported: + continue - Args: - namespace: Plugin namespace to match. - osfilter: The ``OSPlugin`` to use as template to find os specific plugins for. - """ - for plugin_desc in plugins(osfilter): - if namespace == plugin_desc["namespace"]: - yield plugin_desc + found.append(descriptor) + else: + # If we don't have an exact function match, do a slower treematch + path_lookup = _generate_long_paths() + + # Change the treematch pattern into an fnmatch-able pattern to give back all functions from the sub-tree + # (if there is a subtree). + # + # Examples: + # -f apps.webservers.iis -> apps.webservers.iis* (logs etc) + # -f apps.webservers.iis.logs -> apps.webservers.iis.logs* (only the logs, there is no subtree) + # We do not include a dot because that does not work if the full path is given: + # -f apps.webservers.iis.logs != apps.webservers.iis.logs.* (does not work) + search_pattern = pattern + if not has_glob_magic(pattern): + search_pattern += "*" -def load(plugin_desc: PluginDescriptor) -> Type[Plugin]: - """Helper function that loads a plugin from a given plugin description. + matches = False + for path in fnmatch.filter(path_lookup.keys(), search_pattern): + descriptor = path_lookup[path] - Args: - plugin_desc: Plugin description as returned by plugin.lookup(). + # Skip plugins that don't want to be found by wildcards + if not descriptor or not descriptor.exported or (not show_hidden and not descriptor.findable): + continue - Returns: - The plugin class. + # Skip plugins that do not match our OS + if os_filter and not _os_match(os_filter, descriptor.path): + continue - Raises: - PluginError: Raised when any other exception occurs while trying to load the plugin. - """ - module = plugin_desc["fullname"].rsplit(".", 1)[0] + found.append(descriptor) + matches = True - try: - module = importlib.import_module(module) - return getattr(module, plugin_desc["class"]) - except Exception as e: - raise PluginError(f"An exception occurred while trying to load a plugin: {module}", cause=e) + if not matches: + invalid_functions.add(pattern) + if compatibility and target is not None: + result = filter_compatible(found, target, ignore_load_errors) + else: + result = found -def failed() -> list[dict[str, Any]]: - """Return all plugins that failed to load.""" - return _get_plugins().get("_failed", []) + return result, invalid_functions -def _get_plugins() -> dict[str, PluginDescriptor]: - """Load the plugin registry, or generate it if it doesn't exist yet.""" - global PLUGINS, GENERATED - if not GENERATED: - PLUGINS = generate() - GENERATED = True - return PLUGINS +def filter_compatible( + descriptors: list[FunctionDescriptor], target: Target, ignore_load_errors: bool = False +) -> list[FunctionDescriptor]: + """Filter a list of function descriptors based on compatibility with a target.""" + result = [] + seen = set() + for descriptor in descriptors: + print(descriptor) + try: + plugincls = load(descriptor) + except Exception: + if ignore_load_errors: + continue + raise + if plugincls not in seen: + try: + if not plugincls(target).is_compatible(): + continue + except Exception: + continue -def save_plugin_import_failure(module: str) -> None: - """Store errors that occurred during plugin import.""" - if "_failed" not in PLUGINS: - PLUGINS["_failed"] = [] + result.append(descriptor) + return result - stacktrace = traceback.format_exception(*sys.exc_info()) - PLUGINS["_failed"].append( - { - "module": module, - "stacktrace": stacktrace, - } - ) + +def generate() -> dict[str, Any]: + """Internal function to generate the list of available plugins. + + Walks the plugins directory and imports any ``.py`` files in there. + Plugins will be automatically registered. + + Returns: + The global ``PLUGINS`` dictionary. + """ + plugins_dir = Path(__file__).parent / "plugins" + for path in _find_py_files(plugins_dir): + relative_path = path.relative_to(plugins_dir) + module_tuple = (MODULE_PATH, *relative_path.parent.parts, relative_path.stem) + load_module_from_name(".".join(module_tuple)) + + return PLUGINS -def find_py_files(plugin_path: Path) -> Iterator[Path]: - """Walk all the files and directories in ``plugin_path`` and return all files ending in ``.py``. +def _find_py_files(path: Path) -> Iterator[Path]: + """Walk all the files and directories in ``path`` and return all files ending in ``.py``. Do not walk or yield paths containing the following names: - __pycache__ - __init__ - Furthermore, it logs an error if ``plugin_path`` does not exist. + Furthermore, it logs an error if ``path`` does not exist. Args: - plugin_path: The path to a directory or file to walk and filter. + path: The path to a directory or file to walk and filter. """ - if not plugin_path.exists(): - log.error("Path %s does not exist.", plugin_path) + if not path.exists(): + log.error("Path %s does not exist.", path) return - if plugin_path.is_file(): - path_iterator = [plugin_path] + if path.is_file(): + it = [path] else: - path_iterator = plugin_path.glob("**/*.py") + it = path.glob("**/*.py") - for path in path_iterator: - if not path.is_file() or str(path).endswith("__init__.py"): + for entry in it: + if not entry.is_file() or entry.name == "__init__.py": continue - yield path + yield entry def load_module_from_name(module_path: str) -> None: @@ -825,33 +848,10 @@ def load_module_from_name(module_path: str) -> None: except Exception as e: log.info("Unable to import %s", module_path) log.debug("Error while trying to import module %s", module_path, exc_info=e) - save_plugin_import_failure(module_path) - - -def generate() -> dict[str, Any]: - """Internal function to generate the list of available plugins. - - Walks the plugins directory and imports any .py files in there. - Plugins will be automatically registered due to the decorators on them. - - Returns: - The global ``PLUGINS`` dictionary. - """ - global PLUGINS + _save_plugin_import_failure(module_path) - if "_failed" not in PLUGINS: - PLUGINS["_failed"] = [] - plugins_dir = Path(__file__).parent / "plugins" - for path in find_py_files(plugins_dir): - relative_path = path.relative_to(plugins_dir) - module_tuple = (MODULE_PATH, *relative_path.parent.parts, relative_path.stem) - load_module_from_name(".".join(module_tuple)) - - return PLUGINS - - -def load_module_from_file(path: Path, base_path: Path): +def load_module_from_file(path: Path, base_path: Path) -> None: """Loads a module from a file indicated by ``path`` relative to ``base_path``. The module is added to ``sys.modules`` so it can be found everywhere. @@ -870,25 +870,26 @@ def load_module_from_file(path: Path, base_path: Path): except Exception as e: log.error("Unable to import %s", path) log.debug("Error while trying to import module %s", path, exc_info=e) - save_plugin_import_failure(str(path)) + _save_plugin_import_failure(str(path)) -def load_modules_from_paths(plugin_dirs: list[Path]) -> None: - """Iterate over the ``plugin_dirs`` and load all ``.py`` files.""" - for plugin_path in plugin_dirs: - for path in find_py_files(plugin_path): - base_path = plugin_path.parent if path == plugin_path else plugin_path +def load_modules_from_paths(paths: list[Path]) -> None: + """Iterate over the ``paths`` and load all ``.py`` files.""" + for path in paths: + for path in _find_py_files(path): + base_path = path.parent if path == path else path load_module_from_file(path, base_path) def get_external_module_paths(path_list: list[Path]) -> list[Path]: - """Create a deduplicated list of paths.""" + """Return a list of external plugin directories.""" output_list = environment_variable_paths() + path_list return list(set(output_list)) def environment_variable_paths() -> list[Path]: + """Return additional plugin directories specified by the ``DISSECT_PLUGINS`` environment variable.""" env_var = os.environ.get("DISSECT_PLUGINS") plugin_dirs = env_var.split(":") if env_var else [] @@ -896,25 +897,184 @@ def environment_variable_paths() -> list[Path]: return [Path(directory) for directory in plugin_dirs] -def _traverse(key: str, obj: dict[str, Any]) -> dict[str, Any]: - """Split a module path up in a dictionary.""" - for p in key.split("."): - if p not in obj: - obj[p] = {} +def _save_plugin_import_failure(module: str) -> None: + """Store errors that occurred during plugin import.""" + stacktrace = traceback.format_exception(*sys.exc_info()) + PLUGINS.setdefault("__failed__", []).append(FailureDescriptor(module, stacktrace)) + - obj = obj[p] +def _get_nonprivate_attribute_names(cls: type[Plugin]) -> list[str]: + """Retrieve all attributes that do not start with ``_``.""" + return [attr for attr in dir(cls) if not attr.startswith("_")] - return obj +def _get_nonprivate_attributes(cls: type[Plugin]) -> list[Any]: + """Retrieve all public attributes of a :class:`Plugin`.""" + # Note: `dir()` might return attributes from parent class + return [getattr(cls, attr) for attr in _get_nonprivate_attribute_names(cls)] -def _modulepath(cls) -> str: - """Returns the module path of a :class:`Plugin` relative to ``dissect.target.plugins``.""" - module = getattr(cls, "__module__", "") - return module.replace(MODULE_PATH, "").lstrip(".") + +def _get_nonprivate_methods(cls: type[Plugin]) -> list[Callable]: + """Retrieve all public methods of a :class:`Plugin`.""" + return [attr for attr in _get_nonprivate_attributes(cls) if not isinstance(attr, property) and callable(attr)] + + +def _get_descriptors_on_nonprivate_methods(cls: type[Plugin]) -> list[RecordDescriptor]: + """Return record descriptors set on nonprivate methods in `cls` class.""" + descriptors = set() + methods = _get_nonprivate_methods(cls) + + for m in methods: + if not (record := getattr(m, "__record__", None)): + continue + + try: + # check if __record__ value is iterable (for example, a list) + descriptors.update(record) + except TypeError: + descriptors.add(record) + return list(descriptors) +# Class for specific types of plugins # These need to be at the bottom of the module because __init_subclass__ requires everything # in the parent class Plugin to be defined and resolved. +class OSPlugin(Plugin): + """Base class for OS plugins. + + This provides a base class for certain common functions of OS's, which each OS plugin has to implement separately. + + For example, it provides an interface for retrieving the hostname and users of a target. + + All derived classes MUST implement ALL the classmethods and exported + methods with the same ``@classmethod`` or ``@export(...)`` annotation. + """ + + def __init_subclass__(cls, **kwargs): + # Note that cls is the subclass + super().__init_subclass__(**kwargs) + + for os_method in _get_nonprivate_attributes(OSPlugin): + if isinstance(os_method, property): + os_method = os_method.fget + os_docstring = os_method.__doc__ + + method = getattr(cls, os_method.__name__, None) + if isinstance(method, property): + method = method.fget + # This works as None has a __doc__ property (which is None). + docstring = method.__doc__ + + if method and not docstring: + if hasattr(method, "__func__"): + method = method.__func__ + method.__doc__ = os_docstring + + def check_compatible(self) -> bool: + """OSPlugin's use a different compatibility check, override the one from the :class:`Plugin` class. + + Returns: + This function always returns ``True``. + """ + return True + + @classmethod + def detect(cls, fs: Filesystem) -> Filesystem | None: + """Provide detection of this OSPlugin on a given filesystem. + + Args: + fs: :class:`~dissect.target.filesystem.Filesystem` to detect the OS on. + + Returns: + The root filesystem / sysvol when found. + """ + raise NotImplementedError + + @classmethod + def create(cls, target: Target, sysvol: Filesystem) -> OSPlugin: + """Initiate this OSPlugin with the given target and detected filesystem. + + Args: + target: The :class:`~dissect.target.target.Target` object. + sysvol: The filesystem that was detected in the ``detect()`` function. + + Returns: + An instantiated version of the OSPlugin. + """ + raise NotImplementedError + + @export(property=True) + def hostname(self) -> str | None: + """Return the target's hostname. + + Returns: + The hostname as string. + """ + raise NotImplementedError + + @export(property=True) + def ips(self) -> list[str]: + """Return the IP addresses configured in the target. + + Returns: + The IPs as list. + """ + raise NotImplementedError + + @export(property=True) + def version(self) -> str | None: + """Return the target's OS version. + + Returns: + The OS version as string. + """ + raise NotImplementedError + + @export(record=EmptyRecord) + def users(self) -> list[Record]: + """Return the users available in the target. + + Returns: + A list of user records. + """ + raise NotImplementedError + + @export(property=True) + def os(self) -> str: + """Return a slug of the target's OS name. + + Returns: + A slug of the OS name, e.g. 'windows' or 'linux'. + """ + raise NotImplementedError + + @export(property=True) + def architecture(self) -> str | None: + """Return a slug of the target's OS architecture. + + Returns: + A slug of the OS architecture, e.g. 'x86_32-unix', 'MIPS-linux' or + 'AMD64-win32', or 'unknown' if the architecture is unknown. + """ + raise NotImplementedError + + +class ChildTargetPlugin(Plugin): + """A Child target is a special plugin that can list more Targets. + + For example, :class:`~dissect.target.plugins.child.esxi.ESXiChildTargetPlugin` can + list all of the Virtual Machines on the host. + """ + + __type__ = None + + def list_children(self) -> Iterator[ChildTargetRecord]: + """Yield :class:`~dissect.target.helpers.record.ChildTargetRecord` records of all + possible child targets on this target. + """ + raise NotImplementedError + + class NamespacePlugin(Plugin): def __init__(self, target: Target): """A namespace plugin provides services to access functionality from a group of subplugins. @@ -932,8 +1092,6 @@ def __init__(self, target: Target): try: subplugin = getattr(self.target, entry) self._subplugins.append(subplugin) - except UnsupportedPluginError: - target.log.warning("Subplugin %s is not compatible with target.", entry) except Exception: target.log.exception("Failed to load subplugin: %s", entry) @@ -945,11 +1103,8 @@ def __init_subclass_namespace__(cls, **kwargs): # If this is a direct subclass of a Namespace plugin, create a reference to the current class for indirect # subclasses. This is necessary to autogenerate aggregate methods there cls.__nsplugin__ = cls - cls.__findable__ = False def __init_subclass_subplugin__(cls, **kwargs): - cls.__findable__ = True - if not getattr(cls.__nsplugin__, "SUBPLUGINS", None): cls.__nsplugin__.SUBPLUGINS = set() @@ -1049,10 +1204,13 @@ def documentor(): def __init_subclass__(cls, **kwargs): # Upon subclassing, decide whether this is a direct subclass of NamespacePlugin # If this is not the case, autogenerate aggregate methods for methods record output. - super().__init_subclass__(**kwargs) if cls.__bases__[0] != NamespacePlugin: + cls.__findable__ = True + super().__init_subclass__(**kwargs) cls.__init_subclass_subplugin__(cls, **kwargs) else: + cls.__findable__ = False + super().__init_subclass__(**kwargs) cls.__init_subclass_namespace__(cls, **kwargs) @@ -1064,225 +1222,9 @@ class InternalPlugin(Plugin): """ def __init_subclass__(cls, **kwargs): - for method in get_nonprivate_methods(cls): - if callable(method): + for method in _get_nonprivate_methods(cls): + if method.__name__ not in INTERNAL_METHODS and callable(method): method.__internal__ = True super().__init_subclass__(**kwargs) return cls - - -@dataclass(frozen=True, eq=True) -class PluginFunction: - name: str - path: str - output_type: str - class_object: type[Plugin] - method_name: str - plugin_desc: PluginDescriptor = field(hash=False) - - -def plugin_function_index(target: Optional[Target]) -> tuple[dict[str, PluginDescriptor], set[str]]: - """Returns an index-list for plugins. - - This list is used to match CLI expressions against to find the desired plugin. - Also returns the roots to determine whether a CLI expression has to be compared - to the plugin tree or parsed using legacy rules. - """ - - if target is None: - os_type = None - elif target._os_plugin is None: - os_type = general.default.DefaultPlugin - elif isinstance(target._os_plugin, type) and issubclass(target._os_plugin, OSPlugin): - os_type = target._os_plugin - elif isinstance(target._os_plugin, OSPlugin): - os_type = type(target._os_plugin) - else: - raise TypeError( - "target must be None or target._os_plugin must be either None, " - "a subclass of OSPlugin or an instance of OSPlugin" - ) - - index = {} - rootset = set() - - all_plugins = plugins(osfilter=os_type, special_keys={"_child", "_os"}) - - for available_original in all_plugins: - # Prevent modifying the global PLUGINS dict, otherwise -f os.windows._os.users fails for instance. - available = available_original.copy() - - modulepath = available["module"] - rootset.add(modulepath.split(".")[0]) - - if "get_all_records" in available["exports"]: - # The get_all_records does not only need to be not present in the - # index, it also needs to be removed from the exports list, else - # the 'plugins' plugin will still display them. - exports = available["exports"].copy() - exports.remove("get_all_records") - available["exports"] = exports - - if "get_all_records" not in available_original["exports"]: - raise Exception(f"get_all_records removed from {available_original}") - - for exported in available["exports"]: - if available["is_osplugin"] and os_type == general.default.DefaultPlugin: - # This makes the os plugin exports listed under the special - # "OS plugins" header by the 'plugins' plugin. - available["module"] = "" - - index[f"{modulepath}.{exported}"] = available - - return index, rootset - - -def find_plugin_functions( - target: Optional[Target], - patterns: str, - compatibility: bool = False, - **kwargs, -) -> tuple[list[PluginFunction], set[str]]: - """Finds plugins that match the target and the patterns. - - Given a target, a comma separated list of patterns and an optional compatibility flag, - this function finds matching plugins, optionally checking compatibility and returns - a list of plugin function descriptors (including output types). - """ - result = [] - - functions, rootset = plugin_function_index(target) - - invalid_funcs = set() - show_hidden = kwargs.get("show_hidden", False) - ignore_load_errors = kwargs.get("ignore_load_errors", False) - - for pattern in patterns.split(","): - # Backward compatibility fix for namespace-level plugins (i.e. chrome) - # If an exact namespace match is found, the pattern is changed to the tree to that namespace. - # Examples: - # -f browser -> apps.browser.browser - # -f iexplore -> apps.browser.iexplore - namespace_match = False - for index_name, func in functions.items(): - if func["namespace"] == pattern: - pattern = func["module"] - namespace_match = True - break - - wildcard = any(char in pattern for char in ["*", "!", "?", "[", "]"]) - treematch = pattern.split(".")[0] in rootset and pattern != "os" - exact_match = pattern in functions - - # Allow for exact and namespace matches even if the plugin does not want to be found, otherwise you cannot - # reach documented namespace plugins like apps.browser.browser.downloads. - # You can *always* run these using the namespace/classic-style like: browser.downloads (but -l lists them - # in the tree for documentation purposes so it would be misleading not to allow tree access as well). - # - # Note that these tree items will never respond to wildcards though to avoid duplicate results, e.g. when - # querying apps.browser.*, this also means apps.browser.browser.* won't work. - if exact_match or namespace_match: - show_hidden = True - - # Change the treematch pattern into an fnmatch-able pattern to give back all functions from the sub-tree - # (if there is a subtree). - # - # Examples: - # -f browser -> apps.browser.browser* (the whole package, due to a namespace match) - # -f apps.webservers.iis -> apps.webservers.iis* (logs etc) - # -f apps.webservers.iis.logs -> apps.webservers.iis.logs* (only the logs, there is no subtree) - # We do not include a dot because that does not work if the full path is given: - # -f apps.webservers.iis.logs != apps.webservers.iis.logs.* (does not work) - # - # In practice a namespace_match would almost always also be a treematch, except when the namespace plugin - # is in the root of the plugin tree. - if (treematch or namespace_match) and not wildcard and not exact_match: - pattern += "*" - - if wildcard or treematch: - matches = False - for index_name in fnmatch.filter(functions.keys(), pattern): - func = functions[index_name] - - method_name = index_name.split(".")[-1] - try: - loaded_plugin_object = load(func) - except Exception: - if ignore_load_errors: - continue - raise - - # Skip plugins that don't want to be found by wildcards - if not show_hidden and not loaded_plugin_object.__findable__: - continue - - fobject = inspect.getattr_static(loaded_plugin_object, method_name) - - if compatibility: - if target is None: - continue - try: - if not loaded_plugin_object(target).is_compatible(): - continue - except Exception: - continue - - matches = True - result.append( - PluginFunction( - name=f"{func['namespace']}.{method_name}" if func["namespace"] else method_name, - path=index_name, - class_object=loaded_plugin_object, - method_name=method_name, - output_type=getattr(fobject, "__output__", "text"), - plugin_desc=func, - ) - ) - - if not matches: - invalid_funcs.add(pattern) - - else: - # otherwise match using ~ classic style - if pattern.find(".") > -1: - namespace, funcname = pattern.split(".", 1) - else: - funcname = pattern - namespace = None - - plugin_descriptions = [] - for func_path, func in functions.items(): - nsmatch = namespace and func["namespace"] == namespace and func_path.split(".")[-1] == funcname - fmatch = not namespace and not func["namespace"] and func_path.split(".")[-1] == funcname - if nsmatch or fmatch: - plugin_descriptions.append(func) - - if not plugin_descriptions: - invalid_funcs.add(pattern) - - for description in plugin_descriptions: - try: - loaded_plugin_object = load(description) - except Exception: - if ignore_load_errors: - continue - raise - - fobject = inspect.getattr_static(loaded_plugin_object, funcname) - - if compatibility and not loaded_plugin_object(target).is_compatible(): - continue - - result.append( - PluginFunction( - name=f"{description['namespace']}.{funcname}" if description["namespace"] else funcname, - path=f"{description['module']}.{funcname}", - class_object=loaded_plugin_object, - method_name=funcname, - output_type=getattr(fobject, "__output__", "text"), - plugin_desc=description, - ) - ) - - return result, invalid_funcs diff --git a/dissect/target/plugins/apps/webserver/apache.py b/dissect/target/plugins/apps/webserver/apache.py index 94cb80b2f..88b929c7f 100644 --- a/dissect/target/plugins/apps/webserver/apache.py +++ b/dissect/target/plugins/apps/webserver/apache.py @@ -196,7 +196,6 @@ def check_compatible(self) -> None: if not len(self.access_log_paths) and not len(self.error_log_paths): raise UnsupportedPluginError("No Apache directories found") - @plugin.internal def get_log_paths(self) -> tuple[list[Path], list[Path]]: """ Discover any present Apache log paths on the target system. diff --git a/dissect/target/plugins/apps/webserver/caddy.py b/dissect/target/plugins/apps/webserver/caddy.py index cd83a1b4d..1e393dc0a 100644 --- a/dissect/target/plugins/apps/webserver/caddy.py +++ b/dissect/target/plugins/apps/webserver/caddy.py @@ -32,7 +32,6 @@ def check_compatible(self) -> None: if not len(self.log_paths): raise UnsupportedPluginError("No Caddy paths found") - @plugin.internal def get_log_paths(self) -> list[Path]: log_paths = [] diff --git a/dissect/target/plugins/apps/webserver/iis.py b/dissect/target/plugins/apps/webserver/iis.py index 5c37ffcf1..f6c49db46 100644 --- a/dissect/target/plugins/apps/webserver/iis.py +++ b/dissect/target/plugins/apps/webserver/iis.py @@ -76,7 +76,6 @@ def check_compatible(self) -> None: if not self.log_dirs: raise UnsupportedPluginError("No IIS log files found") - @plugin.internal def get_log_dirs(self) -> list[tuple[str, Path]]: log_paths = set() @@ -113,13 +112,11 @@ def get_log_dirs(self) -> list[tuple[str, Path]]: return list(log_paths) - @plugin.internal def iter_log_format_path_pairs(self) -> list[tuple[str, str]]: for log_format, log_dir_path in self.log_dirs: for log_file in log_dir_path.glob("*/*.log"): yield (log_format, log_file) - @plugin.internal def parse_autodetect_format_log(self, path: Path) -> Iterator[BasicRecordDescriptor]: first_line = path.open().readline().decode("utf-8", errors="backslashreplace").strip() if first_line.startswith("#"): @@ -127,7 +124,6 @@ def parse_autodetect_format_log(self, path: Path) -> Iterator[BasicRecordDescrip else: yield from self.parse_iis_format_log(path) - @plugin.internal def parse_iis_format_log(self, path: Path) -> Iterator[BasicRecordDescriptor]: """Parse log file in IIS format and stream log records. @@ -189,7 +185,6 @@ def parse_datetime(date_str: str, time_str: str) -> datetime: def _create_extended_descriptor(self, extra_fields: tuple[tuple[str, str]]) -> TargetRecordDescriptor: return TargetRecordDescriptor(LOG_RECORD_NAME, BASIC_RECORD_FIELDS + list(extra_fields)) - @plugin.internal def parse_w3c_format_log(self, path: Path) -> Iterator[TargetRecordDescriptor]: """Parse log file in W3C format and stream log records. diff --git a/dissect/target/plugins/apps/webserver/nginx.py b/dissect/target/plugins/apps/webserver/nginx.py index 08c5297e6..edeb9af3a 100644 --- a/dissect/target/plugins/apps/webserver/nginx.py +++ b/dissect/target/plugins/apps/webserver/nginx.py @@ -28,7 +28,6 @@ def check_compatible(self) -> None: if not len(self.log_paths): raise UnsupportedPluginError("No NGINX directories found") - @plugin.internal def get_log_paths(self) -> list[Path]: log_paths = [] diff --git a/dissect/target/plugins/apps/webserver/webserver.py b/dissect/target/plugins/apps/webserver/webserver.py index 0092c4ca0..7c274ee61 100644 --- a/dissect/target/plugins/apps/webserver/webserver.py +++ b/dissect/target/plugins/apps/webserver/webserver.py @@ -41,7 +41,6 @@ class WebserverPlugin(NamespacePlugin): __namespace__ = "webserver" - __findable__ = False @export(record=[WebserverAccessLogRecord, WebserverErrorLogRecord]) def logs(self) -> Iterator[Union[WebserverAccessLogRecord, WebserverErrorLogRecord]]: diff --git a/dissect/target/plugins/general/osinfo.py b/dissect/target/plugins/general/osinfo.py index 892b4b58e..a40130a38 100644 --- a/dissect/target/plugins/general/osinfo.py +++ b/dissect/target/plugins/general/osinfo.py @@ -23,8 +23,6 @@ def check_compatible(self) -> None: @plugin.export(record=OSInfoRecord) def osinfo(self) -> Iterator[Union[OSInfoRecord, GroupedRecord]]: for os_func in self.target._os.__functions__: - if os_func in ["is_compatible", "get_all_records"]: - continue value = getattr(self.target._os, os_func) record = OSInfoRecord(name=os_func, value=None, _target=self.target) if isinstance(value, Callable) and isinstance(subrecords := value(), Generator): diff --git a/dissect/target/plugins/general/plugins.py b/dissect/target/plugins/general/plugins.py index a3a0e65a1..22184cba1 100644 --- a/dissect/target/plugins/general/plugins.py +++ b/dissect/target/plugins/general/plugins.py @@ -1,100 +1,104 @@ +from __future__ import annotations + import textwrap -from typing import Dict, List, Type, Union from dissect.target import plugin from dissect.target.helpers.docs import INDENT_STEP, get_plugin_overview from dissect.target.plugin import Plugin, arg, export -def categorize_plugins(plugins_selection: list[dict] = None) -> dict: - """Categorize plugins based on the module it's from.""" +def generate_function_overview( + functions: list[plugin.FunctionDescriptor] | None = None, include_docs: bool = False +) -> list[str]: + """Generate a tree list of functions with optional documentation.""" - output_dict = dict() + categorized_plugins = _categorize_functions(functions) + plugin_descriptions = _generate_plugin_tree_overview(categorized_plugins, include_docs) - plugins_selection = plugins_selection or get_exported_plugins() + plugins_list = textwrap.indent( + "\n".join(plugin_descriptions) if plugin_descriptions else "None", + prefix=INDENT_STEP, + ) - for plugin_dict in plugins_selection: - tmp_dict = dictify_module_recursive( - list_of_items=plugin_dict["module"].split("."), - last_value=plugin.load(plugin_dict), + failed_descriptions = [] + failed_items = plugin.failed() + for failed_item in failed_items: + module = failed_item.module + exception = failed_item.stacktrace[-1].rstrip() + failed_descriptions.append( + textwrap.dedent( + f"""\ + Module: {module} + Reason: {exception} + """ + ) ) - update_dict_recursive(output_dict, tmp_dict) - - return output_dict - - -def get_exported_plugins(): - return [p for p in plugin.plugins() if len(p["exports"])] + failed_list = textwrap.indent( + "\n".join(failed_descriptions) if failed_descriptions else "None", + prefix=INDENT_STEP, + ) -def dictify_module_recursive(list_of_items: list, last_value: Plugin) -> dict: - """Create a dict from a list of strings. + lines = [ + "Available plugins:", + plugins_list, + "", + "Failed to load:", + failed_list, + "", + ] + return "\n".join(lines) - The last element inside the list, will point to `last_value` - """ - if len(list_of_items) == 1: - return {list_of_items[0]: last_value} - else: - return {list_of_items[0]: dictify_module_recursive(list_of_items[1:], last_value)} +def _categorize_functions(functions: list[plugin.FunctionDescriptor] | None = None) -> dict: + """Categorize functions based on its module path.""" -def update_dict_recursive(source_dict: dict, updated_dict: dict) -> dict: - """Update source dictionary with data in updated_dict.""" + functions = functions or [f for f in plugin.functions() if f.exported] + result = {} - for key, value in updated_dict.items(): - if isinstance(value, dict): - source_dict[key] = update_dict_recursive(source_dict.get(key, {}), value) - else: - source_dict[key] = value - return dict(sorted(source_dict.items())) + for desc in functions: + obj = result + parts = desc.path.split(".") + if not desc.namespace or (desc.namespace and desc.method_name != "__call__"): + parts = parts[:-1] -def output_plugin_description_recursive( - structure_dict: Union[Dict, Plugin], - print_docs: bool, - indentation_step=0, -) -> List[str]: - """Create plugin overview with identations.""" + for part in parts[:-1]: + obj = obj.setdefault(part, {}) - if isinstance(structure_dict, type) and issubclass(structure_dict, Plugin): - return [get_plugin_description(structure_dict, print_docs, indentation_step)] + if parts[-1] not in obj: + obj[parts[-1]] = plugin.load(desc) - return get_description_dict(structure_dict, print_docs, indentation_step) + return dict(sorted(result.items())) -def get_plugin_description( - plugin_class: Type[Plugin], +def _generate_plugin_tree_overview( + plugin_tree: dict | type[Plugin], print_docs: bool, - indentation_step: int, -) -> str: - """Returns plugin_overview with specific indentation.""" - - plugin_overview = get_plugin_overview( - plugin_class=plugin_class, - with_func_docstrings=print_docs, - with_plugin_desc=print_docs, - ) - return textwrap.indent(plugin_overview, prefix=" " * indentation_step) + indent: int = 0, +) -> list[str]: + """Create plugin overview with identations.""" + if isinstance(plugin_tree, type) and issubclass(plugin_tree, Plugin): + return [ + textwrap.indent( + get_plugin_overview(plugin_tree, print_docs, print_docs), + prefix=" " * indent, + ) + ] -def get_description_dict( - structure_dict: Dict, - print_docs: bool, - indentation_step: int, -) -> List[str]: - """Returns a list of indented descriptions.""" - - output_descriptions = [] - for key in structure_dict.keys(): - output_descriptions += [ - textwrap.indent(key + ":", prefix=" " * indentation_step) if key != "" else "OS plugins" - ] + output_plugin_description_recursive( - structure_dict[key], - print_docs, - indentation_step=indentation_step + 2, + result = [] + for key in plugin_tree.keys(): + result.append(textwrap.indent(key + ":", prefix=" " * indent) if key != "" else "OS plugins") + result.extend( + _generate_plugin_tree_overview( + plugin_tree[key], + print_docs, + indent=indent + 2, + ) ) - return output_descriptions + return result class PluginListPlugin(Plugin): @@ -103,39 +107,6 @@ def check_compatible(self) -> None: @export(output="none", cache=False) @arg("--docs", dest="print_docs", action="store_true") - def plugins(self, plugins: list[dict] = None, print_docs: bool = False) -> None: - categorized_plugins = dict(sorted(categorize_plugins(plugins).items())) - plugin_descriptions = output_plugin_description_recursive(categorized_plugins, print_docs) - - plugins_list = textwrap.indent( - "\n".join(plugin_descriptions) if plugin_descriptions else "None", - prefix=INDENT_STEP, - ) - - failed_descriptions = [] - failed_items = plugin.failed() - for failed_item in failed_items: - module = failed_item["module"] - exception = failed_item["stacktrace"][-1].rstrip() - failed_descriptions.append( - textwrap.dedent( - f"""\ - Module: {module} - Reason: {exception} - """ - ) - ) - - failed_list = textwrap.indent( - "\n".join(failed_descriptions) if failed_descriptions else "None", - prefix=INDENT_STEP, - ) - - output_lines = [ - "Available plugins:", - plugins_list, - "", - "Failed to load:", - failed_list, - ] - print("\n".join(output_lines)) + def plugins(self, print_docs: bool = False) -> None: + overview = generate_function_overview(include_docs=print_docs) + print(overview) diff --git a/dissect/target/plugins/os/default/__init__.py b/dissect/target/plugins/os/default/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/dissect/target/plugins/general/default.py b/dissect/target/plugins/os/default/_os.py similarity index 93% rename from dissect/target/plugins/general/default.py rename to dissect/target/plugins/os/default/_os.py index 140e9386a..bda3e1bc3 100644 --- a/dissect/target/plugins/general/default.py +++ b/dissect/target/plugins/os/default/_os.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Optional +from typing import TYPE_CHECKING, Iterator, Optional if TYPE_CHECKING: from flow.record import Record @@ -44,7 +44,7 @@ def version(self) -> Optional[str]: pass @export(record=EmptyRecord) - def users(self) -> list[Record]: + def users(self) -> Iterator[Record]: yield from () @export(property=True) diff --git a/dissect/target/plugins/os/unix/linux/debian/apt.py b/dissect/target/plugins/os/unix/linux/debian/apt.py index 09f97c8b5..fa557a107 100644 --- a/dissect/target/plugins/os/unix/linux/debian/apt.py +++ b/dissect/target/plugins/os/unix/linux/debian/apt.py @@ -4,19 +4,21 @@ from typing import Iterator from zoneinfo import ZoneInfo -from dissect.target import Target, plugin +from dissect.target import Target from dissect.target.exceptions import UnsupportedPluginError from dissect.target.helpers.fsutil import open_decompress +from dissect.target.plugin import export from dissect.target.plugins.os.unix.packagemanager import ( OperationTypes, PackageManagerLogRecord, + PackageManagerPlugin, ) APT_LOG_OPERATIONS = ["Install", "Reinstall", "Upgrade", "Downgrade", "Remove", "Purge"] REGEX_PACKAGE_NAMES = re.compile(r"(.*?\)),?") -class AptPlugin(plugin.Plugin): +class AptPlugin(PackageManagerPlugin): __namespace__ = "apt" LOG_DIR_PATH = "/var/log/apt" @@ -27,7 +29,7 @@ def check_compatible(self) -> None: if not len(log_files): raise UnsupportedPluginError("No APT files found") - @plugin.export(record=PackageManagerLogRecord) + @export(record=PackageManagerLogRecord) def logs(self) -> Iterator[PackageManagerLogRecord]: """Package manager log parser for Apt. diff --git a/dissect/target/plugins/os/unix/linux/redhat/yum.py b/dissect/target/plugins/os/unix/linux/redhat/yum.py index 9d5488625..05e9f3fb9 100644 --- a/dissect/target/plugins/os/unix/linux/redhat/yum.py +++ b/dissect/target/plugins/os/unix/linux/redhat/yum.py @@ -1,19 +1,20 @@ import re from typing import Iterator -from dissect.target import plugin from dissect.target.exceptions import UnsupportedPluginError from dissect.target.helpers.utils import year_rollover_helper +from dissect.target.plugin import export from dissect.target.plugins.os.unix.packagemanager import ( OperationTypes, PackageManagerLogRecord, + PackageManagerPlugin, ) YUM_LOG_KEYWORDS = ["Installed", "Updated", "Erased", "Obsoleted"] RE_TS = re.compile(r"(\w+\s{1,2}\d+\s\d{2}:\d{2}:\d{2})") -class YumPlugin(plugin.Plugin): +class YumPlugin(PackageManagerPlugin): __namespace__ = "yum" LOG_DIR_PATH = "/var/log" @@ -24,7 +25,7 @@ def check_compatible(self) -> None: if not len(log_files): raise UnsupportedPluginError("No Yum files found") - @plugin.export(record=PackageManagerLogRecord) + @export(record=PackageManagerLogRecord) def logs(self) -> Iterator[PackageManagerLogRecord]: """Package manager log parser for CentOS' Yellowdog Updater (Yum). diff --git a/dissect/target/plugins/os/unix/linux/suse/zypper.py b/dissect/target/plugins/os/unix/linux/suse/zypper.py index fca0dce27..91f67b2f1 100644 --- a/dissect/target/plugins/os/unix/linux/suse/zypper.py +++ b/dissect/target/plugins/os/unix/linux/suse/zypper.py @@ -1,16 +1,17 @@ from datetime import datetime from typing import Iterator -from dissect.target import plugin from dissect.target.exceptions import UnsupportedPluginError from dissect.target.helpers.fsutil import open_decompress +from dissect.target.plugin import export from dissect.target.plugins.os.unix.packagemanager import ( OperationTypes, PackageManagerLogRecord, + PackageManagerPlugin, ) -class ZypperPlugin(plugin.Plugin): +class ZypperPlugin(PackageManagerPlugin): __namespace__ = "zypper" LOG_DIR_PATH = "/var/log/zypp" @@ -21,7 +22,7 @@ def check_compatible(self) -> None: if not len(log_files): raise UnsupportedPluginError("No zypper files found") - @plugin.export(record=PackageManagerLogRecord) + @export(record=PackageManagerLogRecord) def logs(self) -> Iterator[PackageManagerLogRecord]: """Package manager log parser for SuSE's Zypper. diff --git a/dissect/target/plugins/os/unix/log/audit.py b/dissect/target/plugins/os/unix/log/audit.py index 3f3c63283..03b6835c7 100644 --- a/dissect/target/plugins/os/unix/log/audit.py +++ b/dissect/target/plugins/os/unix/log/audit.py @@ -7,7 +7,7 @@ from dissect.target.exceptions import UnsupportedPluginError from dissect.target.helpers.fsutil import basename, open_decompress from dissect.target.helpers.record import TargetRecordDescriptor -from dissect.target.plugin import Plugin, export, internal +from dissect.target.plugin import Plugin, export AuditRecord = TargetRecordDescriptor( "linux/log/audit", @@ -32,7 +32,6 @@ def check_compatible(self) -> None: if not len(self.log_paths): raise UnsupportedPluginError("No audit path found") - @internal def get_log_paths(self) -> list[Path]: log_paths = [] diff --git a/dissect/target/plugins/os/unix/packagemanager.py b/dissect/target/plugins/os/unix/packagemanager.py index 3bdf34175..370dbbe14 100644 --- a/dissect/target/plugins/os/unix/packagemanager.py +++ b/dissect/target/plugins/os/unix/packagemanager.py @@ -1,12 +1,9 @@ from __future__ import annotations from enum import Enum -from typing import Iterator -from dissect.target import Target -from dissect.target.exceptions import UnsupportedPluginError from dissect.target.helpers.record import TargetRecordDescriptor -from dissect.target.plugin import Plugin, export +from dissect.target.plugin import NamespacePlugin PackageManagerLogRecord = TargetRecordDescriptor( "unix/log/packagemanager", @@ -45,37 +42,6 @@ def infer(cls, keyword: str) -> OperationTypes: return OperationTypes.Other -class PackageManagerPlugin(Plugin): +class PackageManagerPlugin(NamespacePlugin): __namespace__ = "packagemanager" __findable__ = False - - TOOLS = [ - "apt", - "yum", - "zypper", - ] - - def __init__(self, target: Target): - super().__init__(target) - self._plugins = [] - for entry in self.TOOLS: - try: - self._plugins.append(getattr(self.target, entry)) - except Exception: - target.log.exception(f"Failed to load tool plugin: {entry}") - - def check_compatible(self) -> None: - if not len(self._plugins): - raise UnsupportedPluginError("No compatible plugins found") - - def _func(self, f: str) -> Iterator[PackageManagerLogRecord]: - for p in self._plugins: - try: - yield from getattr(p, f)() - except Exception: - self.target.log.exception("Failed to execute package manager plugin: %s.%s", p._name, f) - - @export(record=PackageManagerLogRecord) - def logs(self) -> Iterator[PackageManagerLogRecord]: - """Returns logs from all available Unix package managers.""" - yield from self._func("logs") diff --git a/dissect/target/plugins/os/windows/log/evt.py b/dissect/target/plugins/os/windows/log/evt.py index d3f4ddddc..a589fa204 100644 --- a/dissect/target/plugins/os/windows/log/evt.py +++ b/dissect/target/plugins/os/windows/log/evt.py @@ -47,7 +47,6 @@ class WindowsEventlogsMixin: EVENTLOG_REGISTRY_KEY = "HKEY_LOCAL_MACHINE\\SYSTEM\\CurrentControlSet\\Services\\Eventlog" LOGS_DIR_PATH = None - @plugin.internal def get_logs(self, filename_glob="*") -> List[Path]: file_paths = [] file_paths.extend(self.get_logs_from_dir(self.LOGS_DIR_PATH, filename_glob=filename_glob)) @@ -65,7 +64,6 @@ def get_logs(self, filename_glob="*") -> List[Path]: return file_paths - @plugin.internal def get_logs_from_dir(self, logs_dir: str, filename_glob: str = "*") -> List[Path]: file_paths = [] logs_dir = self.target.fs.path(logs_dir) @@ -75,7 +73,6 @@ def get_logs_from_dir(self, logs_dir: str, filename_glob: str = "*") -> List[Pat self.target.log.debug("Log files found in '%s': %d", self.LOGS_DIR_PATH, len(file_paths)) return file_paths - @plugin.internal def get_logs_from_registry(self, filename_glob: str = "*") -> List[Path]: # compile glob into case-insensitive regex filename_regex = re.compile(fnmatch.translate(filename_glob), re.IGNORECASE) diff --git a/dissect/target/plugins/os/windows/registry.py b/dissect/target/plugins/os/windows/registry.py index a401b2517..5a69bc94e 100644 --- a/dissect/target/plugins/os/windows/registry.py +++ b/dissect/target/plugins/os/windows/registry.py @@ -284,8 +284,7 @@ def subkey(self, key: str, subkey: str) -> KeyCollection: @internal def iterkeys(self, keys: Union[str, list[str]]) -> Iterator[KeyCollection]: warnings.warn("The iterkeys() function is deprecated, use keys() instead", DeprecationWarning) - for key in self.keys(keys): - yield key + yield from self.keys(keys) @internal def keys(self, keys: Union[str, list[str]]) -> Iterator[KeyCollection]: @@ -297,9 +296,7 @@ def keys(self, keys: Union[str, list[str]]) -> Iterator[KeyCollection]: for key in self._iter_controlset_keypaths(keys): try: - res = self.key(key) - for r in res: - yield r + yield from self.key(key) except RegistryKeyNotFoundError: pass except HiveUnavailableError: diff --git a/dissect/target/report.py b/dissect/target/report.py index 82afe04a3..70d09fc79 100644 --- a/dissect/target/report.py +++ b/dissect/target/report.py @@ -1,11 +1,13 @@ +from __future__ import annotations + import argparse import dataclasses import textwrap from dataclasses import dataclass -from typing import Any, Dict, List, Optional, Set, Type +from typing import Any from dissect.target import Target -from dissect.target.plugin import Plugin +from dissect.target.plugin import FunctionDescriptor, Plugin from dissect.target.target import Event BLOCK_INDENT = 4 * " " @@ -15,11 +17,11 @@ class TargetExecutionReport: target: Target - incompatible_plugins: Set[str] = dataclasses.field(default_factory=set) - registered_plugins: Set[str] = dataclasses.field(default_factory=set) + incompatible_plugins: set[str] = dataclasses.field(default_factory=set) + registered_plugins: set[str] = dataclasses.field(default_factory=set) - func_errors: Dict[str, str] = dataclasses.field(default_factory=dict) - func_execs: Set[str] = dataclasses.field(default_factory=set) + func_errors: dict[str, str] = dataclasses.field(default_factory=dict) + func_execs: set[str] = dataclasses.field(default_factory=set) def add_incompatible_plugin(self, plugin_name: str) -> None: self.incompatible_plugins.add(plugin_name) @@ -30,7 +32,7 @@ def add_registered_plugin(self, plugin_name: str) -> None: def add_func_error(self, func, stacktrace: str) -> None: self.func_errors[func] = stacktrace - def as_dict(self) -> Dict[str, Any]: + def as_dict(self) -> dict[str, Any]: return { "target": str(self.target), "incompatible_plugins": sorted(self.incompatible_plugins), @@ -42,19 +44,19 @@ def as_dict(self) -> Dict[str, Any]: @dataclass class ExecutionReport: - plugin_import_errors: Dict[str, str] = dataclasses.field(default_factory=dict) + plugin_import_errors: dict[str, str] = dataclasses.field(default_factory=dict) - target_reports: List[TargetExecutionReport] = dataclasses.field(default_factory=list) + target_reports: list[TargetExecutionReport] = dataclasses.field(default_factory=list) - cli_args: Dict[str, Any] = dataclasses.field(default_factory=dict) + cli_args: dict[str, Any] = dataclasses.field(default_factory=dict) def set_cli_args(self, args: argparse.Namespace) -> None: args = ((key, str(value)) for (key, value) in vars(args).items()) self.cli_args.update(args) - def set_plugin_stats(self, plugins: Dict[str, Any]) -> None: - for details in plugins.get("_failed", []): - self.plugin_import_errors[details["module"]] = "".join(details["stacktrace"]) + def set_plugin_stats(self, plugins: dict[str, Any]) -> None: + for details in plugins.get("__failed__", []): + self.plugin_import_errors[details.module] = "".join(details.stacktrace) def get_formatted_report(self) -> str: blocks = [ @@ -83,8 +85,8 @@ def log_incompatible_plugin( self, target: Target, _, - plugin_cls: Optional[Type[Plugin]] = None, - plugin_desc: Optional[Dict[str, Any]] = None, + plugin_cls: type[Plugin] | None = None, + plugin_desc: FunctionDescriptor | None = None, ) -> None: if not plugin_cls and not plugin_desc: raise ValueError("Either `plugin_cls` or `plugin_desc` must be set") @@ -94,7 +96,7 @@ def log_incompatible_plugin( if plugin_cls: plugin_name = self._get_plugin_name(plugin_cls) elif plugin_desc: - plugin_name = plugin_desc["fullname"] + plugin_name = f"{plugin_desc.module}.{plugin_desc.qualname}" target_report.add_incompatible_plugin(plugin_name) @@ -112,7 +114,7 @@ def log_func_execution(self, target: Target, _, func: str) -> None: target_report = self.get_target_report(target, create=True) target_report.func_execs.add(func) - def set_event_callbacks(self, target_cls: Type[Target]) -> None: + def set_event_callbacks(self, target_cls: type[Target]) -> None: target_cls.set_event_callback( event_type=Event.INCOMPATIBLE_PLUGIN, event_callback=self.log_incompatible_plugin, @@ -130,7 +132,7 @@ def set_event_callbacks(self, target_cls: Type[Target]) -> None: event_callback=self.log_func_error, ) - def as_dict(self) -> Dict[str, Any]: + def as_dict(self) -> dict[str, Any]: return { "plugin_import_errors": self.plugin_import_errors, "target_reports": [report.as_dict() for report in self.target_reports], diff --git a/dissect/target/target.py b/dissect/target/target.py index 1b732a322..58aa4ba41 100644 --- a/dissect/target/target.py +++ b/dissect/target/target.py @@ -22,7 +22,7 @@ from dissect.target.helpers.loaderutil import extract_path_info from dissect.target.helpers.record import ChildTargetRecord from dissect.target.helpers.utils import StrEnum, parse_path_uri, slugify -from dissect.target.plugins.general import default +from dissect.target.plugins.os.default._os import DefaultPlugin log = logging.getLogger(__name__) @@ -112,6 +112,46 @@ def __init__(self, path: Union[str, Path] = None): self.fs = filesystem.RootFilesystem(self) + def __repr__(self) -> str: + return f"" + + def __getattr__(self, attr: str) -> Union[plugin.Plugin, Any]: + """Override of the default ``__getattr__`` so plugins and functions can be called from a ``Target`` object.""" + p, func = self.get_function(attr) + + if isinstance(func, property): + # If it's a property, execute it and return the result + try: + result = func.__get__(p) + self.send_event(Event.FUNC_EXEC, func=attr) + return result + except Exception: + if not attr.startswith("__"): + self.send_event( + Event.FUNC_EXEC_ERROR, + func=attr, + stacktrace=traceback.format_exc(), + ) + raise + + return func + + def __dir__(self) -> list[str]: + """Override the default ``__dir__`` to provide autocomplete for things like IPython.""" + funcs = [] + if self._os_plugin: + funcs = list(self._os_plugin.__functions__) + + for plugin_desc in plugin.plugins(self._os_plugin): + funcs.extend(plugin_desc.functions) + + result = set(self.__dict__.keys()) + result.update(self.__class__.__dict__.keys()) + result.update(object.__dict__.keys()) + result.update(funcs) + + return list(result) + @classmethod def set_event_callback(cls, *, event_type: Optional[Event] = None, event_callback: Callable) -> None: """Sets ``event_callbacks`` on a Target class. @@ -332,23 +372,23 @@ def _load_child_plugins(self) -> None: for plugin_desc in plugin.child_plugins(): try: - plugin_cls = plugin.load(plugin_desc) + plugin_cls: type[plugin.ChildTargetPlugin] = plugin.load(plugin_desc) child_plugin = plugin_cls(self) except PluginError: - self.log.exception("Failed to load child plugin: %s", plugin_desc["class"]) + self.log.exception("Failed to load child plugin: %s", plugin_desc.qualname) continue except Exception: - self.log.exception("Broken child plugin: %s", plugin_desc["class"]) + self.log.exception("Broken child plugin: %s", plugin_desc.qualname) continue try: child_plugin.check_compatible() self._child_plugins[child_plugin.__type__] = child_plugin except PluginError as e: - self.log.debug("Child plugin reported itself as incompatible: %s (%s)", plugin_desc["class"], e) + self.log.debug("Child plugin reported itself as incompatible: %s (%s)", plugin_desc.qualname, e) except Exception: self.log.exception( - "An exception occurred while checking for child plugin compatibility: %s", plugin_desc["class"] + "An exception occurred while checking for child plugin compatibility: %s", plugin_desc.qualname ) def open_child(self, child: Union[str, Path]) -> Target: @@ -445,7 +485,7 @@ def _init_os(self) -> None: if not len(self.disks) and not len(self.volumes) and not len(self.filesystems): raise TargetError(f"Failed to load target. No disks, volumes or filesystems: {self.path}") - candidates = [] + candidates: list[tuple[plugin.PluginDescriptor, type[plugin.OSPlugin], filesystem.Filesystem]] = [] for plugin_desc in plugin.os_plugins(): # Subclassed OS Plugins used to also subclass the detection of the @@ -459,25 +499,26 @@ def _init_os(self) -> None: # # Now subclassed OS Plugins are on the same detection "layer" as # regular OS Plugins, but can still inherit functions. - self.log.debug("Loading OS plugin: %s", plugin_desc["class"]) + qualname = plugin_desc.qualname + self.log.debug("Loading OS plugin: %s", qualname) try: - os_plugin = plugin.load(plugin_desc) + os_plugin: type[plugin.OSPlugin] = plugin.load(plugin_desc) fs = os_plugin.detect(self) except PluginError: - self.log.exception("Failed to load OS plugin: %s", plugin_desc["class"]) + self.log.exception("Failed to load OS plugin: %s", qualname) continue except Exception: - self.log.exception("Broken OS plugin: %s", plugin_desc["class"]) + self.log.exception("Broken OS plugin: %s", qualname) continue if not fs: continue - self.log.info("Found compatible OS plugin: %s", plugin_desc["class"]) + self.log.info("Found compatible OS plugin: %s", qualname) candidates.append((plugin_desc, os_plugin, fs)) fs = None - os_plugin = default.DefaultPlugin + os_plugin = DefaultPlugin if candidates: plugin_desc, os_plugin, fs = candidates[0] @@ -486,7 +527,7 @@ def _init_os(self) -> None: if len(candidate_plugin.mro()) > len(os_plugin.mro()): plugin_desc, os_plugin, fs = candidate_plugin_desc, candidate_plugin, candidate_fs - self.log.debug("Selected OS plugin: %s", plugin_desc["class"]) + self.log.debug("Selected OS plugin: %s", plugin_desc.qualname) else: # No OS detected self.log.warning("Failed to find OS plugin, falling back to default") @@ -510,7 +551,7 @@ def _mount_others(self) -> None: def add_plugin( self, - plugin_cls: Union[plugin.Plugin, type[plugin.Plugin]], + plugin_cls: plugin.Plugin | type[plugin.Plugin], check_compatible: bool = True, ) -> plugin.Plugin: """Add and register a plugin by class. @@ -557,6 +598,20 @@ def add_plugin( return p + def load_plugin(self, descriptor: plugin.PluginDescriptor | plugin.FunctionDescriptor) -> plugin.Plugin: + """Load a plugin by descriptor. + + Args: + plugin_desc: The descriptor of the plugin to load. + + Returns: + The loaded plugin instance. + + Raises: + PluginError: Raised when any exception occurs while trying to load the plugin. + """ + return self.add_plugin(plugin.load(descriptor)) + def _register_plugin_functions(self, plugin_inst: plugin.Plugin) -> None: """Internal function that registers all the exported functions from a given plugin. @@ -569,12 +624,15 @@ def _register_plugin_functions(self, plugin_inst: plugin.Plugin) -> None: if plugin_inst.__namespace__: self._functions[plugin_inst.__namespace__] = (plugin_inst, plugin_inst) + + for func in plugin_inst.__functions__: + self._functions[f"{plugin_inst.__namespace__}.{func}"] = (plugin_inst, None) else: for func in plugin_inst.__functions__: # If we getattr here, property members will be executed, so we do that in __getattr__ self._functions[func] = (plugin_inst, None) - def get_function(self, function: str) -> FunctionTuple: + def get_function(self, function: str | plugin.FunctionDescriptor) -> FunctionTuple: """Attempt to get a given function. If the function is not already registered, look for plugins that export the function and register them. @@ -589,43 +647,59 @@ def get_function(self, function: str) -> FunctionTuple: UnsupportedPluginError: Raised when plugins were found, but they were incompatible PluginError: Raised when any other exception occurs while trying to load the plugin. """ - if function not in self._functions: - causes = [] + if isinstance(function, plugin.FunctionDescriptor): + function_name = function.name - plugin_desc = None - for plugin_desc in plugin.lookup(function, self._os_plugin): + if function_name not in self._functions: try: - plugin_cls = plugin.load(plugin_desc) - self.add_plugin(plugin_cls) - self.log.debug("Found compatible plugin '%s' for function '%s'", plugin_desc["class"], function) - break - except UnsupportedPluginError as e: - self.send_event(Event.INCOMPATIBLE_PLUGIN, plugin_desc=plugin_desc) - causes.append(e) - else: - if plugin_desc: - # In this case we made at least one iteration but it was skipped due incompatibility. - # Just take the last known cause for now + self.load_plugin(function) + except UnsupportedPluginError: + self.send_event(Event.INCOMPATIBLE_PLUGIN, plugin_desc=function) raise UnsupportedPluginError( - f"Unsupported function `{function}` for target with OS plugin {self._os_plugin}", - cause=causes[0] if causes else None, - extra=causes[1:] if len(causes) > 1 else None, + f"Unsupported function `{function.name}` (`{function.module}`)", ) + else: + function_name = function + + if function not in self._functions: + causes = [] + + descriptor = None + for descriptor in plugin.lookup(function, self._os_plugin): + try: + self.load_plugin(descriptor) + self.log.debug("Found compatible plugin '%s' for function '%s'", descriptor.qualname, function) + break + except UnsupportedPluginError as e: + self.send_event(Event.INCOMPATIBLE_PLUGIN, plugin_desc=descriptor) + causes.append(e) + else: + if descriptor: + # In this case we made at least one iteration but it was skipped due incompatibility. + # Just take the last known cause for now + raise UnsupportedPluginError( + f"Unsupported function `{function}` for target with OS plugin {self._os_plugin}", + cause=causes[0] if causes else None, + extra=causes[1:] if len(causes) > 1 else None, + ) + # We still ended up with no compatible plugins - if function not in self._functions: - raise PluginNotFoundError(f"Can't find plugin with function `{function}`") + if function_name not in self._functions: + raise PluginNotFoundError(f"Can't find plugin with function `{function_name}`") - p, func = self._functions[function] + p, func = self._functions[function_name] if func is None: - func = getattr(p.__class__, function) + method_attr = function_name.rpartition(".")[2] if p.__namespace__ else function_name + func = getattr(p.__class__, method_attr) + if not isinstance(func, property) or ( isinstance(func, property) and getattr(func.fget, "__persist__", False) ): # If the persist flag is set on a property, store the property result in the function cache # This is so we don't have to evaluate the property again - func = getattr(p, function) - self._functions[function] = (p, func) + func = getattr(p, method_attr) + self._functions[function_name] = (p, func) return p, func @@ -644,46 +718,6 @@ def has_function(self, function: str) -> bool: except PluginError: return False - def __getattr__(self, attr: str) -> Union[plugin.Plugin, Any]: - """Override of the default __getattr__ so plugins and functions can be called from a ``Target`` object.""" - p, func = self.get_function(attr) - - if isinstance(func, property): - # If it's a property, execute it and return the result - try: - result = func.__get__(p) - self.send_event(Event.FUNC_EXEC, func=attr) - return result - except Exception: - if not attr.startswith("__"): - self.send_event( - Event.FUNC_EXEC_ERROR, - func=attr, - stacktrace=traceback.format_exc(), - ) - raise - - return func - - def __dir__(self): - """Override the default __dir__ to provide autocomplete for things like IPython.""" - funcs = [] - if self._os_plugin: - funcs = list(self._os_plugin.__functions__) - - for plugin_desc in plugin.plugins(self._os_plugin): - funcs.extend(plugin_desc["functions"]) - - result = set(self.__dict__.keys()) - result.update(self.__class__.__dict__.keys()) - result.update(object.__dict__.keys()) - result.update(funcs) - - return list(result) - - def __repr__(self): - return f"" - T = TypeVar("T") diff --git a/dissect/target/tools/build_pluginlist.py b/dissect/target/tools/build_pluginlist.py index 87b9fc18f..7b14908bb 100644 --- a/dissect/target/tools/build_pluginlist.py +++ b/dissect/target/tools/build_pluginlist.py @@ -3,7 +3,7 @@ import argparse import logging -import pprint +import textwrap from dissect.target import plugin @@ -25,7 +25,12 @@ def main(): logging.basicConfig(level=logging.CRITICAL) pluginlist = plugin.generate() - print(f"PLUGINS = \\\n{pprint.pformat(pluginlist)}") + template = """ + from dissect.target.plugin import FailureDescriptor, FunctionDescriptor, PluginDescriptor + + PLUGINS = {} + """ + print(textwrap.dedent(template).format(pluginlist)) if __name__ == "__main__": diff --git a/dissect/target/tools/dump/run.py b/dissect/target/tools/dump/run.py index a4c9245ac..aedce369d 100644 --- a/dissect/target/tools/dump/run.py +++ b/dissect/target/tools/dump/run.py @@ -27,7 +27,7 @@ cached_sink_writers, ) from dissect.target.tools.utils import ( - PluginFunction, + FunctionDescriptor, configure_generic_arguments, execute_function_on_target, find_and_filter_plugins, @@ -52,7 +52,7 @@ def get_targets(targets: list[str]) -> Iterator[Target]: yield target -def execute_function(target: Target, function: PluginFunction) -> TargetRecordDescriptor: +def execute_function(target: Target, function: FunctionDescriptor) -> TargetRecordDescriptor: """ Execute function `function` on provided target `target` and return a generator with the records produced. @@ -91,7 +91,7 @@ def produce_target_func_pairs( targets: Iterable[Target], functions: str, state: DumpState, -) -> Iterator[tuple[Target, PluginFunction]]: +) -> Iterator[tuple[Target, FunctionDescriptor]]: """ Return a generator with target and function pairs for execution. @@ -102,7 +102,7 @@ def produce_target_func_pairs( pairs_to_skip.update((str(sink.target_path), sink.func) for sink in state.finished_sinks) for target in targets: - for func_def in find_and_filter_plugins(target, functions): + for func_def in find_and_filter_plugins(functions, target): if state and (target.path, func_def.name) in pairs_to_skip: log.info( "Skipping target/func pair since its marked as done in provided state", diff --git a/dissect/target/tools/dump/utils.py b/dissect/target/tools/dump/utils.py index 3b513ee3e..1a1066d9c 100644 --- a/dissect/target/tools/dump/utils.py +++ b/dissect/target/tools/dump/utils.py @@ -32,7 +32,7 @@ from flow.record.jsonpacker import JsonRecordPacker from dissect.target import Target -from dissect.target.plugin import PluginFunction +from dissect.target.plugin import FunctionDescriptor log = structlog.get_logger(__name__) @@ -70,13 +70,13 @@ def get_nested_attr(obj: Any, nested_attr: str) -> Any: @lru_cache(maxsize=DEST_DIR_CACHE_SIZE) -def get_sink_dir_by_target(target: Target, function: PluginFunction) -> Path: +def get_sink_dir_by_target(target: Target, function: FunctionDescriptor) -> Path: func_first_name, _, _ = function.name.partition(".") return Path(target.name) / func_first_name @functools.lru_cache(maxsize=DEST_DIR_CACHE_SIZE) -def get_sink_dir_by_func(target: Target, function: PluginFunction) -> Path: +def get_sink_dir_by_func(target: Target, function: FunctionDescriptor) -> Path: func_first_name, _, _ = function.name.partition(".") return Path(func_first_name) / target.name diff --git a/dissect/target/tools/query.py b/dissect/target/tools/query.py index f861b9a26..d67fd54a1 100644 --- a/dissect/target/tools/query.py +++ b/dissect/target/tools/query.py @@ -6,11 +6,12 @@ import pathlib import sys from datetime import datetime -from typing import Callable +from typing import Callable, Optional from flow.record import RecordPrinter, RecordStreamWriter, RecordWriter +from flow.record.base import AbstractWriter -from dissect.target import Target +from dissect.target import Target, plugin from dissect.target.exceptions import ( FatalError, PluginNotFoundError, @@ -20,6 +21,7 @@ from dissect.target.helpers import cache, record_modifier from dissect.target.loaders.targetd import ProxyLoader from dissect.target.plugin import PLUGINS, OSPlugin, Plugin, find_plugin_functions +from dissect.target.plugins.general.plugins import generate_function_overview from dissect.target.report import ExecutionReport from dissect.target.tools.utils import ( args_to_uri, @@ -42,7 +44,7 @@ USAGE_FORMAT_TMPL = "{prog} -f {name}{usage}" -def record_output(strings=False, json=False): +def record_output(strings: bool = False, json: bool = False) -> AbstractWriter: if json: return RecordWriter("jsonfile://-") @@ -54,8 +56,41 @@ def record_output(strings=False, json=False): return RecordStreamWriter(fp) +def list_plugins( + targets: Optional[list[str]] = None, + patterns: str = "", + include_children: bool = False, + argv: Optional[list[str]] = None, +) -> None: + collected = set() + if targets: + for target in Target.open_all(targets, include_children): + if isinstance(target._loader, ProxyLoader): + raise TargetError("can't list compatible plugins for remote targets") + + funcs, _ = find_plugin_functions(patterns, target, compatibility=True, show_hidden=True) + collected.update(funcs) + else: + funcs, _ = find_plugin_functions(patterns, Target(), show_hidden=True) + collected.update(funcs) + + # Display in a user friendly manner + target = Target() + fparser = generate_argparse_for_bound_method(target.plugins, usage_tmpl=USAGE_FORMAT_TMPL) + fargs, rest = fparser.parse_known_args(argv or []) + + if collected: + print(generate_function_overview(collected, include_docs=fargs.print_docs)) + + # No real targets specified, show the available loaders + if not targets: + fparser = generate_argparse_for_bound_method(target.loaders, usage_tmpl=USAGE_FORMAT_TMPL) + fargs, rest = fparser.parse_known_args(rest) + target.loaders(**vars(fargs)) + + @catch_sigpipe -def main(): +def main() -> None: help_formatter = argparse.ArgumentDefaultsHelpFormatter parser = argparse.ArgumentParser( description="dissect.target", @@ -150,14 +185,17 @@ def main(): # Show help for a function or in general if "-h" in rest or "--help" in rest: - found_functions, _ = find_plugin_functions(None, args.function, compatibility=False) + found_functions, _ = find_plugin_functions(args.function) if not len(found_functions): parser.error("function(s) not found, see -l for available plugins") + func = found_functions[0] - if issubclass(func.class_object, OSPlugin): + plugin_class = plugin.load(func) + if issubclass(plugin_class, OSPlugin): obj = getattr(OSPlugin, func.method_name) else: - obj = getattr(func.class_object, func.method_name) + obj = getattr(plugin_class, func.method_name) + if isinstance(obj, type) and issubclass(obj, Plugin): parser = generate_argparse_for_plugin_class(obj, usage_tmpl=USAGE_FORMAT_TMPL) elif isinstance(obj, Callable) or isinstance(obj, property): @@ -170,33 +208,7 @@ def main(): # Show the list of available plugins for the given optional target and optional # search pattern, only display plugins that can be applied to ANY targets if args.list: - collected_plugins = {} - - if targets: - for plugin_target in Target.open_all(targets, args.children): - if isinstance(plugin_target._loader, ProxyLoader): - parser.error("can't list compatible plugins for remote targets.") - funcs, _ = find_plugin_functions(plugin_target, args.list, compatibility=True, show_hidden=True) - for func in funcs: - collected_plugins[func.path] = func.plugin_desc - else: - funcs, _ = find_plugin_functions(Target(), args.list, compatibility=False, show_hidden=True) - for func in funcs: - collected_plugins[func.path] = func.plugin_desc - - # Display in a user friendly manner - target = Target() - fparser = generate_argparse_for_bound_method(target.plugins, usage_tmpl=USAGE_FORMAT_TMPL) - fargs, rest = fparser.parse_known_args(rest) - - if collected_plugins: - target.plugins(list(collected_plugins.values())) - - # No real targets specified, show the available loaders - if not targets: - fparser = generate_argparse_for_bound_method(target.loaders, usage_tmpl=USAGE_FORMAT_TMPL) - fargs, rest = fparser.parse_known_args(rest) - target.loaders(**vars(fargs)) + list_plugins(targets, args.list, args.children, rest) parser.exit() if not targets: @@ -205,6 +217,19 @@ def main(): if not args.function: parser.error("argument -f/--function is required") + if args.report_dir and not args.report_dir.is_dir(): + parser.error(f"--report-dir {args.report_dir} is not a valid directory") + + funcs, invalid_funcs = find_plugin_functions(args.function) + if any(invalid_funcs): + parser.error(f"argument -f/--function contains invalid plugin(s): {', '.join(invalid_funcs)}") + + excluded_funcs, invalid_excluded_funcs = find_plugin_functions(args.excluded_functions) + if any(invalid_excluded_funcs): + parser.error( + f"argument -xf/--excluded-functions contains invalid plugin(s): {', '.join(invalid_excluded_funcs)}", + ) + # Verify uniformity of output types, otherwise default to records. # Note that this is a heuristic, the targets are not opened yet because of # performance, so it might generate a false positive @@ -218,28 +243,12 @@ def main(): # The only scenario that might cause this is with # custom plugins with idiosyncratic output across OS-versions/branches. output_types = set() - funcs, invalid_funcs = find_plugin_functions(None, args.function, compatibility=False) - - if any(invalid_funcs): - parser.error(f"argument -f/--function contains invalid plugin(s): {', '.join(invalid_funcs)}") - - excluded_funcs, invalid_excluded_funcs = find_plugin_functions( - None, - args.excluded_functions, - compatibility=False, - ) - - if any(invalid_excluded_funcs): - parser.error( - f"argument -xf/--excluded-functions contains invalid plugin(s): {', '.join(invalid_excluded_funcs)}", - ) - excluded_func_paths = {excluded_func.path for excluded_func in excluded_funcs} for func in funcs: if func.path in excluded_func_paths: continue - output_types.add(func.output_type) + output_types.add(func.output) default_output_type = None @@ -248,9 +257,6 @@ def main(): log.warning("Mixed output types detected: %s. Only outputting records.", ",".join(output_types)) default_output_type = "record" - if args.report_dir and not args.report_dir.is_dir(): - parser.error(f"--report-dir {args.report_dir} is not a valid directory") - execution_report = ExecutionReport() execution_report.set_cli_args(args) execution_report.set_event_callbacks(Target) @@ -271,18 +277,14 @@ def main(): yield_entries = [] first_seen_output_type = default_output_type - cli_params_unparsed = rest - - excluded_funcs, _ = find_plugin_functions(target, args.excluded_functions, compatibility=False) - excluded_func_paths = {excluded_func.path for excluded_func in excluded_funcs} - for func_def in find_and_filter_plugins(target, args.function, excluded_func_paths): + for func_def in find_and_filter_plugins(args.function, target, excluded_func_paths): # If the default type is record (meaning we skip everything else) # and actual output type is not record, continue. # We perform this check here because plugins that require output files/dirs # will exit if we attempt to exec them without (because they are implied by the wildcard). # Also this saves cycles of course. - if default_output_type == "record" and func_def.output_type != "record": + if default_output_type == "record" and func_def.output != "record": continue if args.dry_run: @@ -290,9 +292,7 @@ def main(): continue try: - output_type, result, cli_params_unparsed = execute_function_on_target( - target, func_def, cli_params_unparsed - ) + output_type, result, rest = execute_function_on_target(target, func_def, rest) except UnsupportedPluginError as e: target.log.error( "Unsupported plugin for %s: %s", @@ -309,7 +309,10 @@ def main(): fatal.emit_last_message(target.log.error) parser.exit(1) except Exception: - target.log.error("Exception while executing function `%s`", func_def, exc_info=True) + target.log.error( + "Exception while executing function `%s` (`%s`)", func_def.name, func_def.path, exc_info=True + ) + target.log.debug("Function info: %s", func_def) continue if first_seen_output_type and output_type != first_seen_output_type: diff --git a/dissect/target/tools/shell.py b/dissect/target/tools/shell.py index 29b06781d..f7e1d7dba 100644 --- a/dissect/target/tools/shell.py +++ b/dissect/target/tools/shell.py @@ -32,7 +32,7 @@ ) from dissect.target.filesystem import FilesystemEntry, LayerFilesystemEntry from dissect.target.helpers import cyber, fsutil, regutil -from dissect.target.plugin import PluginFunction, arg +from dissect.target.plugin import FunctionDescriptor, arg from dissect.target.target import Target from dissect.target.tools.info import print_target_info from dissect.target.tools.utils import ( @@ -188,8 +188,8 @@ def default(self, line: str) -> Optional[bool]: except AttributeError: pass - if plugins := list(find_and_filter_plugins(self.target, command, [])): - return self._exec_target(plugins, command_args_str) + if functions := list(find_and_filter_plugins(command, self.target)): + return self._exec_target(functions, command_args_str) return cmd.Cmd.default(self, line) @@ -247,7 +247,7 @@ def _exec_(argparts: list[str], stdout: TextIO) -> bool: no_cyber = cmdfunc.__func__ in (TargetCli.cmd_registry, TargetCli.cmd_enter) return self._exec(_exec_, command_args_str, no_cyber) - def _exec_target(self, funcs: list[PluginFunction], command_args_str: str) -> Optional[bool]: + def _exec_target(self, funcs: list[FunctionDescriptor], command_args_str: str) -> Optional[bool]: """Command exection helper for target plugins.""" def _exec_(argparts: list[str], stdout: TextIO) -> Optional[bool]: diff --git a/dissect/target/tools/utils.py b/dissect/target/tools/utils.py index 04730b01e..3500f8975 100644 --- a/dissect/target/tools/utils.py +++ b/dissect/target/tools/utils.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import argparse import errno import inspect @@ -9,18 +11,16 @@ from datetime import datetime from functools import wraps from pathlib import Path -from typing import Any, Callable, Dict, Iterator, List, Optional, Tuple, Type, Union +from typing import Any, Callable, Iterator from dissect.target import Target -from dissect.target.exceptions import UnsupportedPluginError from dissect.target.helpers import docs, keychain from dissect.target.helpers.docs import get_docstring from dissect.target.helpers.targetd import CommandProxy from dissect.target.loader import LOADERS_BY_SCHEME from dissect.target.plugin import ( - OSPlugin, + FunctionDescriptor, Plugin, - PluginFunction, find_plugin_functions, get_external_module_paths, load_modules_from_paths, @@ -57,9 +57,9 @@ def process_generic_arguments(args: argparse.Namespace) -> None: def generate_argparse_for_bound_method( method: Callable, - usage_tmpl: Optional[str] = None, + usage_tmpl: str | None = None, ) -> argparse.ArgumentParser: - """Generate an `argparse.ArgumentParser` for a bound `Plugin` class method""" + """Generate an ``argparse.ArgumentParser`` for a bound ``Plugin` class method.""" # allow functools.partial wrapped method while hasattr(method, "func"): @@ -74,9 +74,9 @@ def generate_argparse_for_bound_method( def generate_argparse_for_unbound_method( method: Callable, - usage_tmpl: Optional[str] = None, + usage_tmpl: str | None = None, ) -> argparse.ArgumentParser: - """Generate an `argparse.ArgumentParser` for an unbound `Plugin` class method""" + """Generate an ``argparse.ArgumentParser`` for an unbound ``Plugin`` class method.""" if not inspect.isfunction(method): raise ValueError(f"Value `{method}` is not an unbound plugin method") @@ -100,10 +100,10 @@ def generate_argparse_for_unbound_method( def generate_argparse_for_plugin_class( - plugin_cls: Type[Plugin], - usage_tmpl: Optional[str] = None, + plugin_cls: type[Plugin], + usage_tmpl: str | None = None, ) -> argparse.ArgumentParser: - """Generate an `argparse.ArgumentParser` for a `Plugin` class""" + """Generate an ``argparse.ArgumentParser`` for a ``Plugin`` class.""" if not isinstance(plugin_cls, type) or not issubclass(plugin_cls, Plugin): raise ValueError(f"`plugin_cls` must be a valid plugin class, not `{plugin_cls}`") @@ -125,9 +125,9 @@ def generate_argparse_for_plugin_class( def generate_argparse_for_plugin( plugin_instance: Plugin, - usage_tmpl: Optional[str] = None, + usage_tmpl: str | None = None, ) -> argparse.ArgumentParser: - """Generate an `argparse.ArgumentParser` for a `Plugin` instance""" + """Generate an ``argparse.ArgumentParser`` for a ``Plugin`` instance.""" if not isinstance(plugin_instance, Plugin): raise ValueError(f"`plugin_instance` must be a valid plugin instance, not `{plugin_instance}`") @@ -138,84 +138,32 @@ def generate_argparse_for_plugin( return generate_argparse_for_plugin_class(plugin_instance.__class__, usage_tmpl=usage_tmpl) -def plugin_factory( - target: Target, plugin: Union[type, object], funcname: str, namespace: Optional[str] -) -> tuple[Plugin, str]: - if hasattr(target._loader, "instance"): - return target.get_function(funcname, namespace=namespace) - - if isinstance(plugin, type): - plugin_obj = plugin(target) - target_attr = getattr(plugin_obj, funcname) - return plugin_obj, target_attr - else: - return plugin, getattr(plugin, funcname) - - def execute_function_on_target( target: Target, - func: PluginFunction, - cli_params: Optional[List[str]] = None, -) -> Tuple[str, Any, List[str]]: - """ - Execute function `func` on provided target `target` with provided `cli_params` list. - """ + func: FunctionDescriptor, + arguments: list[str] | None = None, +) -> tuple[str, Any, list[str]]: + """Execute function on provided target with provided arguments.""" - cli_params = cli_params or [] + arguments = arguments or [] - target_attr = get_target_attribute(target, func) - plugin_method, parser = plugin_function_with_argparser(target_attr) + _, func_obj = target.get_function(func.name) + plugin_method, parser = plugin_function_with_argparser(func_obj) if parser: - parsed_params, cli_params = parser.parse_known_args(cli_params) - method_kwargs = vars(parsed_params) - value = plugin_method(**method_kwargs) + known_args, rest = parser.parse_known_args(arguments) + value = plugin_method(**vars(known_args)) else: - value = target_attr + rest = arguments + value = func_obj output_type = getattr(plugin_method, "__output__", "default") if plugin_method else "default" - return (output_type, value, cli_params) - - -def get_target_attribute(target: Target, func: PluginFunction) -> Union[Plugin, Callable]: - """Retrieves the function attribute from the target. - - If the function does not exist yet, it will attempt to load it into the target. - - Args: - target: The target we wish to run the function on. - func: The function to run on the target. - - Returns: - The function, either plugin or a callable to execute. - - Raises: - UnsupportedPluginError: When the function was incompatible with the target. - """ - plugin_class = func.class_object - if ns := getattr(func, "plugin_desc", {}).get("namespace", None): - plugin_class = getattr(target, ns) - elif target.has_function(func.method_name): - # If the function is already attached, use the one inside the target. - plugin_class, _ = target.get_function(func.method_name) - elif issubclass(plugin_class, OSPlugin): - # OS plugin does not need to be added - plugin_class = target._os_plugin - else: - try: - target.add_plugin(plugin_class) - except UnsupportedPluginError as e: - raise UnsupportedPluginError( - f"Unsupported function `{func.method_name}` for target with plugin {func.class_object}", cause=e - ) - - _, target_attr = plugin_factory(target, plugin_class, func.method_name, func.plugin_desc["namespace"]) - return target_attr + return (output_type, value, rest) def plugin_function_with_argparser( - target_attr: Union[Plugin, Callable] -) -> tuple[Optional[Iterator], Optional[argparse.ArgumentParser]]: + target_attr: Plugin | Callable, +) -> tuple[Callable | None, argparse.ArgumentParser | None]: """Resolves which plugin function to execute, and creates the argument parser for said plugin.""" plugin_method = None parser = None @@ -227,7 +175,7 @@ def plugin_function_with_argparser( if not plugin_obj.__namespace__: raise ValueError(f"Plugin {plugin_obj} is not callable") - plugin_method = plugin_obj.get_all_records + plugin_method = plugin_obj.__call__ parser = generate_argparse_for_plugin(plugin_obj) elif isinstance(target_attr, CommandProxy): plugin_method = target_attr.command() @@ -238,7 +186,7 @@ def plugin_function_with_argparser( return plugin_method, parser -def persist_execution_report(output_dir: Path, report_data: Dict, timestamp: datetime) -> Path: +def persist_execution_report(output_dir: Path, report_data: dict, timestamp: datetime) -> Path: timestamp = timestamp.strftime("%Y-%m-%d-%H%M%S") report_filename = f"target-report-{timestamp}.json" report_full_path = output_dir / report_filename @@ -247,7 +195,7 @@ def persist_execution_report(output_dir: Path, report_data: Dict, timestamp: dat def catch_sigpipe(func: Callable) -> Callable: - """Catches KeyboardInterrupt and BrokenPipeError (OSError 22 on Windows).""" + """Catches ``KeyboardInterrupt`` and ``BrokenPipeError`` (``OSError 22`` on Windows).""" @wraps(func) def wrapper(*args, **kwargs): @@ -293,13 +241,13 @@ def args_to_uri(targets: list[str], loader_name: str, rest: list[str]) -> list[s def find_and_filter_plugins( - target: Target, functions: str, excluded_func_paths: set[str] = None -) -> Iterator[PluginFunction]: + functions: str, target: Target, excluded_func_paths: set[str] | None = None +) -> Iterator[FunctionDescriptor]: # Keep a set of plugins that were already executed on the target. executed_plugins = set() excluded_func_paths = excluded_func_paths or set() - func_defs, _ = find_plugin_functions(target, functions, compatibility=False) + func_defs, _ = find_plugin_functions(functions, target) for func_def in func_defs: if func_def.path in excluded_func_paths: diff --git a/tests/conftest.py b/tests/conftest.py index c31f29479..60e979b0b 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -11,7 +11,7 @@ from dissect.target.helpers.fsutil import TargetPath from dissect.target.helpers.regutil import VirtualHive, VirtualKey, VirtualValue from dissect.target.plugin import OSPlugin -from dissect.target.plugins.general import default +from dissect.target.plugins.os.default._os import DefaultPlugin from dissect.target.plugins.os.unix._os import UnixPlugin from dissect.target.plugins.os.unix.bsd.citrix._os import CitrixPlugin from dissect.target.plugins.os.unix.bsd.osx._os import MacPlugin @@ -244,7 +244,7 @@ def target_bare(tmp_path: pathlib.Path) -> Iterator[Target]: @pytest.fixture def target_default(tmp_path: pathlib.Path) -> Iterator[Target]: - yield make_os_target(tmp_path, default.DefaultPlugin) + yield make_os_target(tmp_path, DefaultPlugin) @pytest.fixture diff --git a/tests/helpers/test_docs.py b/tests/helpers/test_docs.py index 4b50afe8f..ba104a024 100644 --- a/tests/helpers/test_docs.py +++ b/tests/helpers/test_docs.py @@ -21,7 +21,7 @@ def test_docs_plugin_functions_desc() -> None: assert functions_short_desc desc_lines = functions_short_desc.splitlines() - assert len(desc_lines) == 3 + assert len(desc_lines) == 2 assert "iis.logs" in functions_short_desc assert "Return contents of IIS (v7 and above) log files." in functions_short_desc assert "output: records" in functions_short_desc diff --git a/tests/plugins/apps/webserver/test_apache.py b/tests/plugins/apps/webserver/test_apache.py index 152743832..825c2c539 100644 --- a/tests/plugins/apps/webserver/test_apache.py +++ b/tests/plugins/apps/webserver/test_apache.py @@ -158,8 +158,7 @@ def test_logrotate(target_unix: Target, fs_unix: VirtualFilesystem) -> None: fs_unix.map_file("var/log/apache2/access.log.2", data_file) fs_unix.map_file("var/log/apache2/access.log.3", data_file) - target_unix.add_plugin(ApachePlugin) - access_log_paths, error_log_paths = target_unix.apache.get_log_paths() + access_log_paths, error_log_paths = ApachePlugin(target_unix).get_log_paths() assert len(access_log_paths) == 4 assert len(error_log_paths) == 0 @@ -172,8 +171,7 @@ def test_custom_config(target_unix: Target, fs_unix: VirtualFilesystem) -> None: fs_unix.map_file_fh("custom/log/location/access.log.2", BytesIO(b"Foo2")) fs_unix.map_file_fh("custom/log/location/access.log.3", BytesIO(b"Foo3")) - target_unix.add_plugin(ApachePlugin) - access_log_paths, error_log_paths = target_unix.apache.get_log_paths() + access_log_paths, error_log_paths = ApachePlugin(target_unix).get_log_paths() assert len(access_log_paths) == 4 assert len(error_log_paths) == 0 @@ -192,9 +190,8 @@ def test_config_commented_logs(target_unix: Target, fs_unix: VirtualFilesystem) fs_unix.map_file_fh("custom/log/location/old.log", BytesIO(b"Old")) fs_unix.map_file_fh("custom/log/location/old_error.log", BytesIO(b"Old")) fs_unix.map_file_fh("custom/log/location/new_error.log", BytesIO(b"New")) - target_unix.add_plugin(ApachePlugin) - access_log_paths, error_log_paths = target_unix.apache.get_log_paths() + access_log_paths, error_log_paths = ApachePlugin(target_unix).get_log_paths() # Log paths are returned in alphabetical order assert str(access_log_paths[0]) == "/custom/log/location/new.log" diff --git a/tests/plugins/apps/webserver/test_caddy.py b/tests/plugins/apps/webserver/test_caddy.py index 02f3b7439..91e6f04b8 100644 --- a/tests/plugins/apps/webserver/test_caddy.py +++ b/tests/plugins/apps/webserver/test_caddy.py @@ -4,11 +4,13 @@ from flow.record.fieldtypes import datetime as dt +from dissect.target.filesystem import VirtualFilesystem from dissect.target.plugins.apps.webserver.caddy import CaddyPlugin +from dissect.target.target import Target from tests._utils import absolute_path -def test_plugins_apps_webservers_caddy_txt(target_unix, fs_unix): +def test_plugins_apps_webservers_caddy_txt(target_unix: Target, fs_unix: VirtualFilesystem) -> None: tz = timezone(timedelta(hours=-7)) fs_unix.map_file_fh( "var/log/caddy_access.log", @@ -30,7 +32,7 @@ def test_plugins_apps_webservers_caddy_txt(target_unix, fs_unix): assert record.bytes_sent == 2326 -def test_plugins_apps_webservers_caddy_json(target_unix, fs_unix): +def test_plugins_apps_webservers_caddy_json(target_unix: Target, fs_unix: VirtualFilesystem) -> None: fs_unix.map_file( "var/log/caddy_access.log", absolute_path("_data/plugins/apps/webserver/caddy/access.log"), @@ -51,7 +53,7 @@ def test_plugins_apps_webservers_caddy_json(target_unix, fs_unix): assert record.bytes_sent == 12 -def test_plugins_apps_webservers_caddy_config(target_unix, fs_unix): +def test_plugins_apps_webservers_caddy_config(target_unix: Target, fs_unix: VirtualFilesystem) -> None: config_file = absolute_path("_data/plugins/apps/webserver/caddy/Caddyfile") fs_unix.map_file("etc/caddy/Caddyfile", config_file) @@ -59,15 +61,14 @@ def test_plugins_apps_webservers_caddy_config(target_unix, fs_unix): fs_unix.map_file_fh("var/www/log/access.log", BytesIO(b"Foo")) fs_unix.map_file_fh("var/log/caddy/access.log", BytesIO(b"Foo")) - target_unix.add_plugin(CaddyPlugin) - log_paths = target_unix.caddy.get_log_paths() + log_paths = CaddyPlugin(target_unix).get_log_paths() assert len(log_paths) == 2 assert str(log_paths[0]) == "/var/log/caddy/access.log" assert str(log_paths[1]) == "/var/www/log/access.log" -def test_plugins_apps_webservers_caddy_config_logs_logrotated(target_unix, fs_unix): +def test_plugins_apps_webservers_caddy_config_logs_logrotated(target_unix: Target, fs_unix: VirtualFilesystem) -> None: config_file = absolute_path("_data/plugins/apps/webserver/caddy/Caddyfile") fs_unix.map_file("etc/caddy/Caddyfile", config_file) @@ -76,13 +77,12 @@ def test_plugins_apps_webservers_caddy_config_logs_logrotated(target_unix, fs_un fs_unix.map_file_fh("var/www/log/access.log.2", BytesIO(b"Foo2")) fs_unix.map_file_fh("var/www/log/access.log.3", BytesIO(b"Foo3")) - target_unix.add_plugin(CaddyPlugin) - log_paths = target_unix.caddy.get_log_paths() + log_paths = CaddyPlugin(target_unix).get_log_paths() assert len(log_paths) == 4 -def test_plugins_apps_webservers_caddy_config_commented(target_unix, fs_unix): +def test_plugins_apps_webservers_caddy_config_commented(target_unix: Target, fs_unix: VirtualFilesystem) -> None: config = """ root /var/www/html 1.example.com { @@ -102,8 +102,7 @@ def test_plugins_apps_webservers_caddy_config_commented(target_unix, fs_unix): fs_unix.map_file_fh("var/www/log/new.log", BytesIO(b"Foo")) fs_unix.map_file_fh("completely/disabled/access.log", BytesIO(b"Foo")) - target_unix.add_plugin(CaddyPlugin) - log_paths = target_unix.caddy.get_log_paths() + log_paths = CaddyPlugin(target_unix).get_log_paths() assert len(log_paths) == 3 assert str(log_paths[0]) == "/var/www/log/old.log" diff --git a/tests/plugins/apps/webserver/test_citrix.py b/tests/plugins/apps/webserver/test_citrix.py index 8c115f8dd..32f856742 100644 --- a/tests/plugins/apps/webserver/test_citrix.py +++ b/tests/plugins/apps/webserver/test_citrix.py @@ -75,8 +75,7 @@ def test_error_logs(target_citrix: Target, fs_bsd: VirtualFilesystem) -> None: fs_bsd.map_file("var/log/httperror-vpn.log", BytesIO(b"Foo")) fs_bsd.map_file("var/log/httperror.log", BytesIO(b"Bar")) - target_citrix.add_plugin(CitrixWebserverPlugin) - access_log_paths, error_log_paths = target_citrix.citrix.get_log_paths() + access_log_paths, error_log_paths = CitrixWebserverPlugin(target_citrix).get_log_paths() assert len(error_log_paths) == 2 diff --git a/tests/plugins/apps/webserver/test_nginx.py b/tests/plugins/apps/webserver/test_nginx.py index 77337e15e..646fb2648 100644 --- a/tests/plugins/apps/webserver/test_nginx.py +++ b/tests/plugins/apps/webserver/test_nginx.py @@ -2,11 +2,13 @@ from datetime import datetime, timezone from io import BytesIO +from dissect.target.filesystem import VirtualFilesystem from dissect.target.plugins.apps.webserver.nginx import NginxPlugin +from dissect.target.target import Target from tests._utils import absolute_path -def test_plugins_apps_webservers_nginx_txt(target_unix, fs_unix): +def test_plugins_apps_webservers_nginx_txt(target_unix: Target, fs_unix: VirtualFilesystem) -> None: data_file = absolute_path("_data/plugins/apps/webserver/nginx/access.log") fs_unix.map_file("var/log/nginx/access.log", data_file) @@ -26,7 +28,7 @@ def test_plugins_apps_webservers_nginx_txt(target_unix, fs_unix): assert record.bytes_sent == 123 -def test_plugins_apps_webservers_nginx_ipv6(target_unix, fs_unix): +def test_plugins_apps_webservers_nginx_ipv6(target_unix: Target, fs_unix: VirtualFilesystem) -> None: data_file = absolute_path("_data/plugins/apps/webserver/nginx/access.log") fs_unix.map_file("var/log/nginx/access.log", data_file) @@ -45,7 +47,7 @@ def test_plugins_apps_webservers_nginx_ipv6(target_unix, fs_unix): assert record.bytes_sent == 123 -def test_plugins_apps_webservers_nginx_gz(target_unix, fs_unix): +def test_plugins_apps_webservers_nginx_gz(target_unix: Target, fs_unix: VirtualFilesystem) -> None: data_file = absolute_path("_data/plugins/apps/webserver/nginx/access.log.gz") fs_unix.map_file("var/log/nginx/access.log.1.gz", data_file) @@ -64,7 +66,7 @@ def test_plugins_apps_webservers_nginx_gz(target_unix, fs_unix): assert record.bytes_sent == 123 -def test_plugins_apps_webservers_nginx_bz2(target_unix, fs_unix): +def test_plugins_apps_webservers_nginx_bz2(target_unix: Target, fs_unix: VirtualFilesystem) -> None: data_file = absolute_path("_data/plugins/apps/webserver/nginx/access.log.bz2") fs_unix.map_file("var/log/nginx/access.log.1.bz2", data_file) @@ -83,20 +85,19 @@ def test_plugins_apps_webservers_nginx_bz2(target_unix, fs_unix): assert record.bytes_sent == 123 -def test_plugins_apps_webservers_nginx_config(target_unix, fs_unix): +def test_plugins_apps_webservers_nginx_config(target_unix: Target, fs_unix: VirtualFilesystem) -> None: config_file = absolute_path("_data/plugins/apps/webserver/nginx/nginx.conf") fs_unix.map_file("etc/nginx/nginx.conf", config_file) for i, log in enumerate(["access.log", "domain1.access.log", "domain2.access.log", "big.server.access.log"]): fs_unix.map_file_fh(f"opt/logs/{i}/{log}", BytesIO(b"Foo")) - target_unix.add_plugin(NginxPlugin) - log_paths = target_unix.nginx.get_log_paths() + log_paths = NginxPlugin(target_unix).get_log_paths() assert len(log_paths) == 4 -def test_plugins_apps_webservers_nginx_config_logs_logrotated(target_unix, fs_unix): +def test_plugins_apps_webservers_nginx_config_logs_logrotated(target_unix: Target, fs_unix: VirtualFilesystem) -> None: config_file = absolute_path("_data/plugins/apps/webserver/nginx/nginx.conf") fs_unix.map_file("etc/nginx/nginx.conf", config_file) fs_unix.map_file_fh("opt/logs/0/access.log", BytesIO(b"Foo1")) @@ -105,13 +106,12 @@ def test_plugins_apps_webservers_nginx_config_logs_logrotated(target_unix, fs_un fs_unix.map_file_fh("opt/logs/1/domain1.access.log", BytesIO(b"Foo4")) fs_unix.map_file_fh("var/log/nginx/access.log", BytesIO(b"Foo5")) - target_unix.add_plugin(NginxPlugin) - log_paths = target_unix.nginx.get_log_paths() + log_paths = NginxPlugin(target_unix).get_log_paths() assert len(log_paths) == 5 -def test_plugins_apps_webservers_nginx_config_commented_logs(target_unix, fs_unix): +def test_plugins_apps_webservers_nginx_config_commented_logs(target_unix: Target, fs_unix: VirtualFilesystem) -> None: config = """ # access_log /foo/bar/old.log main; access_log /foo/bar/new.log main; @@ -119,8 +119,7 @@ def test_plugins_apps_webservers_nginx_config_commented_logs(target_unix, fs_uni fs_unix.map_file_fh("etc/nginx/nginx.conf", BytesIO(textwrap.dedent(config).encode())) fs_unix.map_file_fh("foo/bar/new.log", BytesIO(b"New")) fs_unix.map_file_fh("foo/bar/old.log", BytesIO(b"Old")) - target_unix.add_plugin(NginxPlugin) - log_paths = target_unix.nginx.get_log_paths() + log_paths = NginxPlugin(target_unix).get_log_paths() assert str(log_paths[0]) == "/foo/bar/old.log" assert str(log_paths[1]) == "/foo/bar/new.log" diff --git a/tests/plugins/general/test_default.py b/tests/plugins/general/test_default.py index 6977192b2..179b84d8d 100644 --- a/tests/plugins/general/test_default.py +++ b/tests/plugins/general/test_default.py @@ -2,7 +2,7 @@ import pytest -from dissect.target.plugins.general.default import DefaultPlugin +from dissect.target.plugins.os.default._os import DefaultPlugin from dissect.target.target import Target diff --git a/tests/plugins/general/test_plugins.py b/tests/plugins/general/test_plugins.py index 0e2acc543..ebbae8448 100644 --- a/tests/plugins/general/test_plugins.py +++ b/tests/plugins/general/test_plugins.py @@ -1,41 +1,22 @@ -from unittest.mock import Mock, patch +from unittest.mock import MagicMock, patch -import dissect.target.plugins.general.plugins as plugin +from dissect.target import plugin from dissect.target.plugins.general.plugins import ( PluginListPlugin, - categorize_plugins, - dictify_module_recursive, - output_plugin_description_recursive, - update_dict_recursive, + _categorize_functions, + _generate_plugin_tree_overview, ) -def test_dictify_module(): - last_value = Mock() - - output_dict = dictify_module_recursive(["hello", "world"], last_value) - - assert output_dict == {"hello": {"world": last_value}} - - -def test_update_dict(): - tmp_dictionary = dict() - - update_dict_recursive(tmp_dictionary, dictify_module_recursive(["hello", "world"], None)) - update_dict_recursive(tmp_dictionary, dictify_module_recursive(["hello", "lawrence"], None)) - - assert tmp_dictionary == {"hello": {"world": None, "lawrence": None}} - - -def test_plugin_description(): - description = [x for x in output_plugin_description_recursive(PluginListPlugin, False)] +def test_plugin_description() -> None: + description = [x for x in _generate_plugin_tree_overview(PluginListPlugin, False)] assert description == ["plugins - No documentation (output: no output)"] -def test_plugin_description_compacting(): - module = dictify_module_recursive(["hello", "world"], PluginListPlugin) +def test_plugin_description_compacting() -> None: + module = {"hello": {"world": PluginListPlugin}} - description = [x for x in output_plugin_description_recursive(module, False)] + description = [x for x in _generate_plugin_tree_overview(module, False)] assert description == [ "hello:", " world:", @@ -43,13 +24,10 @@ def test_plugin_description_compacting(): ] -def test_plugin_description_in_dict_multiple(): - tmp_dictionary = dict() +def test_plugin_description_in_dict_multiple() -> None: + module = {"hello": {"world": {"data": PluginListPlugin, "data2": PluginListPlugin}}} - update_dict_recursive(tmp_dictionary, dictify_module_recursive(["hello", "world", "data"], PluginListPlugin)) - update_dict_recursive(tmp_dictionary, dictify_module_recursive(["hello", "world", "data2"], PluginListPlugin)) - - description = [x for x in output_plugin_description_recursive(tmp_dictionary, False)] + description = [x for x in _generate_plugin_tree_overview(module, False)] assert description == [ "hello:", " world:", @@ -60,8 +38,21 @@ def test_plugin_description_in_dict_multiple(): ] -@patch.object(plugin.plugin, "load") -@patch.object(plugin, "get_exported_plugins") -def test_categorize_plugins(mocked_export, mocked_load): - mocked_export.return_value = [{"module": "something.data"}] - assert categorize_plugins() == {"something": {"data": mocked_load.return_value}} +@patch("dissect.target.plugins.general.plugins.plugin.load") +@patch("dissect.target.plugins.general.plugins.plugin.functions") +def test_categorize_plugins(mocked_plugins: MagicMock, mocked_load: MagicMock) -> None: + mocked_plugins.return_value = [ + plugin.FunctionDescriptor( + name="data", + namespace=None, + path="something.data", + exported=True, + internal=False, + findable=True, + output=None, + method_name="data", + module="other.root.something.data", + qualname="DataClass", + ), + ] + assert _categorize_functions() == {"something": mocked_load.return_value} diff --git a/tests/plugins/os/unix/log/test_audit.py b/tests/plugins/os/unix/log/test_audit.py index 941d7ee46..5eb205150 100644 --- a/tests/plugins/os/unix/log/test_audit.py +++ b/tests/plugins/os/unix/log/test_audit.py @@ -3,11 +3,13 @@ from dissect.util.ts import from_unix +from dissect.target.filesystem import VirtualFilesystem from dissect.target.plugins.os.unix.log.audit import AuditPlugin +from dissect.target.target import Target from tests._utils import absolute_path -def test_audit_plugin(target_unix, fs_unix): +def test_audit_plugin(target_unix: Target, fs_unix: VirtualFilesystem) -> None: data_file = absolute_path("_data/plugins/os/unix/log/audit/audit.log") fs_unix.map_file("var/log/audit/audit.log", data_file) @@ -32,7 +34,7 @@ def test_audit_plugin(target_unix, fs_unix): assert result.message == 'cwd="/home/shadowman"' -def test_audit_plugin_config(target_unix, fs_unix): +def test_audit_plugin_config(target_unix: Target, fs_unix: VirtualFilesystem) -> None: config = """ log_file = /foo/bar/audit/audit.log # log_file=/tmp/disabled/audit/audit.log @@ -41,8 +43,7 @@ def test_audit_plugin_config(target_unix, fs_unix): fs_unix.map_file_fh("tmp/disabled/audit/audit.log", BytesIO(b"Foo")) fs_unix.map_file_fh("foo/bar/audit/audit.log", BytesIO(b"Foo")) - audit = AuditPlugin(target_unix) - log_paths = audit.get_log_paths() + log_paths = AuditPlugin(target_unix).get_log_paths() assert len(log_paths) == 2 assert str(log_paths[0]) == "/foo/bar/audit/audit.log" assert str(log_paths[1]) == "/tmp/disabled/audit/audit.log" diff --git a/tests/plugins/os/unix/log/test_messages.py b/tests/plugins/os/unix/log/test_messages.py index 810d1fd30..eee625bea 100644 --- a/tests/plugins/os/unix/log/test_messages.py +++ b/tests/plugins/os/unix/log/test_messages.py @@ -10,7 +10,7 @@ from dissect.target import Target from dissect.target.filesystem import VirtualFilesystem from dissect.target.filesystems.tar import TarFilesystem -from dissect.target.plugins.general import default +from dissect.target.plugins.os.default._os import DefaultPlugin from dissect.target.plugins.os.unix.log.messages import MessagesPlugin, MessagesRecord from tests._utils import absolute_path @@ -73,7 +73,7 @@ def test_unix_log_messages_compressed_timezone_year_rollover() -> None: fs = TarFilesystem(bio) target.filesystems.add(fs) target.fs.mount("/", fs) - target._os_plugin = default.DefaultPlugin + target._os_plugin = DefaultPlugin target.apply() target.add_plugin(MessagesPlugin) diff --git a/tests/plugins/os/windows/test_mru.py b/tests/plugins/os/windows/test_mru.py index 80c65a2c2..b2df63e47 100644 --- a/tests/plugins/os/windows/test_mru.py +++ b/tests/plugins/os/windows/test_mru.py @@ -141,4 +141,4 @@ def test_mru_plugin(target_win_mru): assert len(mstsc) == 3 assert len(msoffice) == 6 - assert len(list(target_win_mru.mru.get_all_records())) == 23 + assert len(list(target_win_mru.mru())) == 23 diff --git a/tests/plugins/os/windows/test_ual.py b/tests/plugins/os/windows/test_ual.py index 983cb94a6..f45c13761 100644 --- a/tests/plugins/os/windows/test_ual.py +++ b/tests/plugins/os/windows/test_ual.py @@ -24,5 +24,5 @@ def test_ual_plugin(target_win, fs_win): domains_seen_records = list(target_win.ual.domains_seen()) assert len(domains_seen_records) == 12 - ual_all_records = list(target_win.ual.get_all_records()) + ual_all_records = list(target_win.ual()) assert len(ual_all_records) == 123 diff --git a/tests/test_plugin.py b/tests/test_plugin.py index e109d9ddf..d70bf9b06 100644 --- a/tests/test_plugin.py +++ b/tests/test_plugin.py @@ -1,8 +1,8 @@ import os from functools import reduce from pathlib import Path -from typing import Optional -from unittest.mock import Mock, patch +from typing import Iterator, Optional +from unittest.mock import MagicMock, Mock, patch import pytest from flow.record import Record @@ -11,32 +11,41 @@ from dissect.target.helpers.descriptor_extensions import UserRecordDescriptorExtension from dissect.target.helpers.record import EmptyRecord, create_extended_descriptor from dissect.target.plugin import ( - PLUGINS, + FunctionDescriptor, NamespacePlugin, OSPlugin, Plugin, + PluginDescriptor, + _generate_long_paths, + _save_plugin_import_failure, environment_variable_paths, export, find_plugin_functions, get_external_module_paths, plugins, - save_plugin_import_failure, ) from dissect.target.target import Target -def test_save_plugin_import_failure(): +@pytest.fixture(autouse=True) +def clear_caches() -> Iterator[None]: + _generate_long_paths.cache_clear() + yield + _generate_long_paths.cache_clear() + + +def test_save_plugin_import_failure() -> None: test_trace = ["test-trace"] test_module_name = "test-module" with patch("traceback.format_exception", Mock(return_value=test_trace)): - with patch("dissect.target.plugin.PLUGINS", new_callable=dict) as MOCK_PLUGINS: - MOCK_PLUGINS["_failed"] = [] - save_plugin_import_failure(test_module_name) + with patch("dissect.target.plugin.PLUGINS", new_callable=dict) as mock_plugins: + mock_plugins["__failed__"] = [] + _save_plugin_import_failure(test_module_name) - assert len(MOCK_PLUGINS["_failed"]) == 1 - assert MOCK_PLUGINS["_failed"][0].get("module") == test_module_name - assert MOCK_PLUGINS["_failed"][0].get("stacktrace") == test_trace + assert len(mock_plugins["__failed__"]) == 1 + assert mock_plugins["__failed__"][0].module == test_module_name + assert mock_plugins["__failed__"][0].stacktrace == test_trace @pytest.mark.parametrize( @@ -47,16 +56,16 @@ def test_save_plugin_import_failure(): (":", [Path(""), Path("")]), ], ) -def test_load_environment_variable(env_value, expected_output): +def test_load_environment_variable(env_value: Optional[str], expected_output: list[Path]) -> None: with patch.object(os, "environ", {"DISSECT_PLUGINS": env_value}): assert environment_variable_paths() == expected_output -def test_load_module_paths(): +def test_load_module_paths() -> None: assert get_external_module_paths([Path(""), Path("")]) == [Path("")] -def test_load_paths_with_env(): +def test_load_paths_with_env() -> None: with patch.object(os, "environ", {"DISSECT_PLUGINS": ":"}): assert get_external_module_paths([Path(""), Path("")]) == [Path("")] @@ -69,62 +78,114 @@ class MockOSWarpPlugin(OSPlugin): def __init__(self): pass - def get_all_records(): - return [] - - def f3(self): + def f3(self) -> str: return "F3" - def f6(self): + def f6(self) -> str: return "F6" @patch( - "dissect.target.plugin.plugins", - return_value=[ - {"module": "test.x13", "exports": ["f3"], "namespace": "Warp", "class": "x13", "is_osplugin": False}, - {"module": "os", "exports": ["f3"], "namespace": None, "class": "f3", "is_osplugin": False}, - {"module": "os.warp._os", "exports": ["f6"], "namespace": None, "class": "warp", "is_osplugin": True}, - ], + "dissect.target.plugin._get_plugins", + return_value={ + "__functions__": { + None: { + "Warp.f3": { + "test.x13.x13": FunctionDescriptor( + name="Warp.f3", + namespace="Warp", + path="test.x13.f3", + exported=True, + internal=False, + findable=True, + output="record", + method_name="f3", + module="test.x13", + qualname="x13", + ) + }, + "f3": { + "os.f3": FunctionDescriptor( + name="f3", + namespace=None, + path="os.f3", + exported=True, + internal=False, + findable=True, + output="record", + method_name="f3", + module="os", + qualname="f3", + ) + }, + "f22": { + "test.x69.x69": FunctionDescriptor( + name="f22", + namespace=None, + path="test.x69.f22", + exported=True, + internal=False, + findable=False, + output="record", + method_name="f22", + module="test.x69", + qualname="x69", + ) + }, + }, + "__os__": { + "f6": { + "os.warp._os.warp": FunctionDescriptor( + name="f6", + namespace=None, + path="os.warp._os.f6", + exported=True, + internal=False, + findable=True, + output="record", + method_name="f6", + module="os.warp._os", + qualname="warp", + ) + } + }, + }, + "__ostree__": {"os": {"warp": {}}}, + }, ) @patch("dissect.target.Target", create=True) -@patch("dissect.target.plugin.load") @pytest.mark.parametrize( - "search, findable, assert_num_found", + "search, assert_num_found", [ - ("*", True, 3), # Found with tree search using wildcard - ("*", False, 0), # Unfindable plugins are not found... - ("test.x13.*", True, 1), # Found with tree search using wildcard, expands to test.x13.f3 - ("test.x13.*", False, 0), # Unfindable plugins are not found... - ("test.x13", True, 1), # Found with tree search, same as above, because users expect +* - ("test.*", True, 1), # Found with tree search - ("test.[!x]*", True, 0), # Not Found with tree search, all in test not starting with x (no x13) - ("test.[!y]*", True, 1), # Found with tree search, all in test not starting with y (so x13 is ok) - ("test.???.??", True, 1), # Found with tree search, using question marks - ("x13", True, 0), # Not Found: Part of namespace but no match - ("Warp.*", True, 0), # Not Found: Namespace != Module so 0 - ("os.warp._os.f6", True, 1), # Found, OS-plugins also available under verbose name - ("f6", True, 1), # Found with classic search - ("f6", False, 1), # Backward compatible: unfindable has no effect on classic search - ("Warp.f3", True, 1), # Found with classic style search using namespace + function - ("Warp.f3", False, 1), # Backward compatible: unfindable has no effect on classic search - ("f3", True, 1), # Found with classic style search using only function - ("os.*", True, 2), # Found matching os.f3, os.warp._os.f6 - ("os", True, 0), # Exception for os, because it can be a 'special' plugin (tree match ignored) + ("*", 2), # Found with tree search using wildcard, excluding OS plugins and unfindable + ("test.x13.*", 1), # Found with tree search using wildcard, expands to test.x13.f3 + ("test.x13", 1), # Found with tree search, same as above, because users expect +* + ("test.x13.f3", 1), + ("test.*", 1), # Found with tree search + ("test.[!x]*", 0), # Not Found with tree search, all in test not starting with x (no x13) + ("test.[!y]*", 1), # Found with tree search, all in test not starting with y (so x13 is ok) + ("test.???.??", 1), # Found with tree search, using question marks + ("x13", 0), # Not Found: Part of namespace but no match + ("Warp.*", 0), # Not Found: Namespace != Module so 0 + ("os.warp._os.f6", 0), # OS plugins are excluded from tree search + ("f6", 1), # Found with direct match + ("f22", 1), # Unfindable has no effect on direct match + ("Warp.f3", 1), # Found with namespace + function + ("f3", 1), # Found direct match + ("os.*", 1), # Found matching os.f3 + ("os", 1), # No tree search for "os" because it's a direct match ], ) -def test_find_plugin_functions(plugin_loader, target, plugins, search, findable, assert_num_found): - os_plugin = MockOSWarpPlugin - os_plugin.__findable__ = findable - target._os_plugin = os_plugin - plugin_loader.return_value = os_plugin() +def test_find_plugin_functions(target: MagicMock, plugins: dict, search: str, assert_num_found: int) -> None: + target._os_plugin = MockOSWarpPlugin + target._os_plugin.__module__ = "dissect.target.plugins.os.warp._os" - found, _ = find_plugin_functions(target, search) + found, _ = find_plugin_functions(search, target) assert len(found) == assert_num_found def test_find_plugin_function_windows(target_win: Target) -> None: - found, _ = find_plugin_functions(target_win, "services") + found, _ = find_plugin_functions("services", target_win) assert len(found) == 1 assert found[0].name == "services" @@ -132,7 +193,7 @@ def test_find_plugin_function_windows(target_win: Target) -> None: def test_find_plugin_function_linux(target_linux: Target) -> None: - found, _ = find_plugin_functions(target_linux, "services") + found, _ = find_plugin_functions("services", target_linux) assert len(found) == 1 assert found[0].name == "services" @@ -149,6 +210,7 @@ def test_find_plugin_function_linux(target_linux: Target) -> None: class _TestNSPlugin(NamespacePlugin): __namespace__ = "NS" + __register__ = False @export(record=TestRecord) def test_all(self): @@ -158,6 +220,7 @@ def test_all(self): class _TestSubPlugin1(_TestNSPlugin): __namespace__ = "t1" + __register__ = False @export(record=TestRecord) def test(self): @@ -166,6 +229,7 @@ def test(self): class _TestSubPlugin2(_TestNSPlugin): __namespace__ = "t2" + __register__ = False @export(record=TestRecord) def test(self): @@ -174,6 +238,7 @@ def test(self): class _TestSubPlugin3(_TestSubPlugin2): __namespace__ = "t3" + __register__ = False # Override the test() function of t2 @export(record=TestRecord) @@ -186,6 +251,7 @@ def _value(self): class _TestSubPlugin4(_TestSubPlugin3): __namespace__ = "t4" + __register__ = False # Do not override the test() function of t3, but change the _value function instead. def _value(self): @@ -220,12 +286,9 @@ def test_namespace_plugin(target_win: Target) -> None: # Check whether we can access the overridden function when explicitly accessing the subplugin assert next(target_win.t4.test_all()).value == "overridden" - # Remove test plugin from list afterwards to avoid order effects - del PLUGINS["tests"] - def test_find_plugin_function_default(target_default: Target) -> None: - found, _ = find_plugin_functions(target_default, "services") + found, _ = find_plugin_functions("services", target_default) assert len(found) == 2 names = [item.name for item in found] @@ -235,7 +298,7 @@ def test_find_plugin_function_default(target_default: Target) -> None: assert "os.unix.linux.services.services" in paths assert "os.windows.services.services" in paths - found, _ = find_plugin_functions(target_default, "mcafee.msc") + found, _ = find_plugin_functions("mcafee.msc", target_default) assert found[0].path == "apps.av.mcafee.msc" @@ -250,7 +313,7 @@ def test_find_plugin_function_default(target_default: Target) -> None: ], ) def test_find_plugin_function_order(target_win: Target, pattern: str) -> None: - found = ",".join(reduce(lambda rs, el: rs + [el.method_name], find_plugin_functions(target_win, pattern)[0], [])) + found = ",".join(reduce(lambda rs, el: rs + [el.method_name], find_plugin_functions(pattern, target_win)[0], [])) assert found == pattern @@ -264,7 +327,7 @@ def test_incompatible_plugin(target_bare: Target) -> None: target_bare.add_plugin(_TestIncompatiblePlugin) -MOCK_PLUGINS = { +MOCK_PLUGINS_OLD = { "apps": { # Plugin descriptors in this branch should be returned for any osfilter "mail": {"module": "apps.mail", "functions": "mail"}, }, @@ -298,108 +361,301 @@ def test_incompatible_plugin(target_bare: Target) -> None: }, } +MOCK_PLUGINS = { + "__functions__": { + None: { + "mail": { + "apps.mail.MailPlugin": FunctionDescriptor( + name="mail", + namespace=None, + path="apps.mail.mail", + exported=True, + internal=False, + findable=True, + output="record", + method_name="mail", + module="apps.mail", + qualname="MailPlugin", + ) + }, + "app1": { + "os.apps.app1.App1Plugin": FunctionDescriptor( + name="app1", + namespace=None, + path="os.apps.app1.app1", + exported=True, + internal=False, + findable=True, + output="record", + method_name="app1", + module="os.apps.app1", + qualname="App1Plugin", + ) + }, + "app2": { + "os.apps.app2.App2Plugin": FunctionDescriptor( + name="app2", + namespace=None, + path="os.apps.app2.app2", + exported=True, + internal=False, + findable=True, + output="record", + method_name="app2", + module="os.apps.app2", + qualname="App2Plugin", + ), + "os.fooos.apps.app2.App2Plugin": FunctionDescriptor( + name="app2", + namespace=None, + path="os.fooos.apps.app2.app2", + exported=True, + internal=False, + findable=True, + output="record", + method_name="app2", + module="os.fooos.apps.app2", + qualname="App2Plugin", + ), + }, + "foo_app": { + "os.fooos.apps.foo_app.FooAppPlugin": FunctionDescriptor( + name="foo_app", + namespace=None, + path="os.fooos.apps.foo_app.foo_app", + exported=True, + internal=False, + findable=True, + output="record", + method_name="foo_app", + module="os.foos.apps.foo_app", + qualname="FooAppPlugin", + ) + }, + "bar_app": { + "os.fooos.apps.bar_app.BarAppPlugin": FunctionDescriptor( + name="bar_app", + namespace=None, + path="os.fooos.apps.bar_app.bar_app", + exported=True, + internal=False, + findable=True, + output="record", + method_name="bar_app", + module="os.foos.apps.bar_app", + qualname="BarAppPlugin", + ) + }, + "foobar": { + "os.fooos.foobar.FooBarPlugin": FunctionDescriptor( + name="foobar", + namespace=None, + path="os.fooos.foobar.foobar", + exported=True, + internal=False, + findable=True, + output="record", + method_name="foobar", + module="os.foos.foobar", + qualname="FooBarPlugin", + ) + }, + }, + "__os__": { + "generic_os": { + "os._os.GenericOS": FunctionDescriptor( + name="generic_os", + namespace=None, + path="os._os.generic_os", + exported=True, + internal=False, + findable=True, + output="record", + method_name="generic_os", + module="os._os", + qualname="GenericOS", + ) + }, + "foo_os": { + "os.fooos._os.FooOS": FunctionDescriptor( + name="foo_os", + namespace=None, + path="os.fooos._os.foo_os", + exported=True, + internal=False, + findable=True, + output="record", + method_name="foo_os", + module="os.fooos._os", + qualname="FooOS", + ) + }, + }, + }, + "__plugins__": { + None: { + "apps.mail.MailPlugin": PluginDescriptor( + module="apps.mail", + qualname="MailPlugin", + namespace=None, + path="apps.mail", + findable=True, + functions=["mail"], + exports=["mail"], + ), + "os.apps.app1.App1Plugin": PluginDescriptor( + module="os.apps.app1", + qualname="App1Plugin", + namespace=None, + path="os.apps.app1", + findable=True, + functions=["app1"], + exports=["app1"], + ), + "os.apps.app2.App2Plugin": PluginDescriptor( + module="os.apps.app2", + qualname="App2Plugin", + namespace=None, + path="os.apps.app2", + findable=True, + functions=["app2"], + exports=["app2"], + ), + "os.fooos.apps.app2.App2Plugin": PluginDescriptor( + module="os.fooos.apps.app2", + qualname="App2Plugin", + namespace=None, + path="os.fooos.apps.app2", + findable=True, + functions=["app2"], + exports=["app2"], + ), + "os.fooos.apps.foo_app.FooAppPlugin": PluginDescriptor( + module="os.fooos.apps.foo_app", + qualname="FooAppPlugin", + namespace=None, + path="os.fooos.apps.foo_app", + findable=True, + functions=["foo_app"], + exports=["foo_app"], + ), + "os.fooos.apps.bar_app.BarAppPlugin": PluginDescriptor( + module="os.fooos.apps.bar_app", + qualname="BarAppPlugin", + namespace=None, + path="os.fooos.apps.bar_app", + findable=True, + functions=["bar_app"], + exports=["bar_app"], + ), + "os.fooos.foobar.FooBarPlugin": PluginDescriptor( + module="os.fooos.foobar", + qualname="FooBarPlugin", + namespace=None, + path="os.fooos.foobar", + findable=True, + functions=["foobar"], + exports=["foobar"], + ), + }, + "__os__": { + "os._os.GenericOS": PluginDescriptor( + module="os._os", + qualname="GenericOS", + namespace=None, + path="os._os", + findable=True, + functions=["generic_os"], + exports=["generic_os"], + ), + "os.fooos._os.FooOS": PluginDescriptor( + module="os.fooos._os", + qualname="FooOS", + namespace=None, + path="os.fooos._os", + findable=True, + functions=["foo_os"], + exports=["foo_os"], + ), + }, + }, + "__ostree__": { + "os": { + "fooos": {}, + } + }, +} + @pytest.mark.parametrize( - "osfilter, special_keys, only_special_keys, expected_plugin_functions", + "osfilter, index, expected_plugins", [ ( None, - set(["_os", "_misc"]), - False, - [ - "mail", - "GenericOS", - "app1", - "app2", - "FooOS", - "foobar", - "bar", - "tender", - "foo_app", - "bar_app", - ], - ), - ( - "os._os", - set(["_os"]), - False, + None, [ - "mail", - "GenericOS", - "app1", - "app2", + "apps.mail", + "os.apps.app1", + "os.apps.app2", + "os.fooos.apps.app2", + "os.fooos.apps.bar_app", + "os.fooos.apps.foo_app", + "os.fooos.foobar", ], ), ( - "os.fooos._os", - set(), - False, + None, + "__os__", [ - "mail", - "app1", - "app2", - "foobar", - "foo_app", - "bar_app", + "os._os", + "os.fooos._os", ], ), ( - "os.fooos", - set(["_os"]), - False, + "os._os", + None, [ - "mail", - "app1", - "app2", - "FooOS", - "foobar", - "foo_app", - "bar_app", + "apps.mail", + "os.apps.app1", + "os.apps.app2", ], ), ( "os.fooos._os", - set(["_os", "_misc"]), - True, + None, [ - "FooOS", - "bar", - "tender", + "apps.mail", + "os.apps.app1", + "os.apps.app2", + "os.fooos.apps.app2", + "os.fooos.apps.bar_app", + "os.fooos.apps.foo_app", + "os.fooos.foobar", ], ), ( "bar", - set(["_os"]), - False, - [ - "mail", - ], + None, + ["apps.mail"], ), ], ) def test_plugins( osfilter: str, - special_keys: set[str], - only_special_keys: bool, - expected_plugin_functions: list[str], + index: str, + expected_plugins: list[str], ) -> None: with ( - patch("dissect.target.plugin.PLUGINS", MOCK_PLUGINS), - patch("dissect.target.plugin._modulepath", return_value=osfilter), + patch("dissect.target.plugin._get_plugins", return_value=MOCK_PLUGINS), + patch("dissect.target.plugin._module_path", return_value=osfilter), ): if osfilter is not None: # osfilter must be a class or None osfilter = Mock - plugin_descriptors = plugins( - osfilter=osfilter, - special_keys=special_keys, - only_special_keys=only_special_keys, - ) - - plugin_functions = [descriptor["functions"] for descriptor in plugin_descriptors] + plugin_descriptors = plugins(osfilter=osfilter, index=index) - assert sorted(plugin_functions) == sorted(expected_plugin_functions) + assert sorted([desc.module for desc in plugin_descriptors]) == sorted(expected_plugins) def test_plugins_default_plugin(target_default: Target) -> None: @@ -413,17 +669,13 @@ def test_plugins_default_plugin(target_default: Target) -> None: sentinel_function = "all_with_home" has_sentinel_function = False for plugin in default_plugin_plugins: - if sentinel_function in plugin.get("functions", []): + if sentinel_function in plugin.functions: has_sentinel_function = True break assert has_sentinel_function - default_os_plugin_desc = plugins( - osfilter=target_default._os_plugin, - special_keys=set(["_os"]), - only_special_keys=True, - ) + default_os_plugin_desc = plugins(osfilter=target_default._os_plugin, index="__os__") assert len(list(default_os_plugin_desc)) == 1 diff --git a/tests/test_registration.py b/tests/test_registration.py index d47bb4e0f..8d0decda5 100644 --- a/tests/test_registration.py +++ b/tests/test_registration.py @@ -5,7 +5,7 @@ import pytest -from dissect.target.plugin import PLUGINS, find_py_files, load_modules_from_paths +from dissect.target.plugin import PLUGINS, _find_py_files, load_modules_from_paths @pytest.fixture @@ -26,13 +26,13 @@ def copy_different_plugin_files(path: Path, file_name: str) -> None: def test_load_environment_variable_empty_string() -> None: - with patch("dissect.target.plugin.find_py_files") as mocked_find_py_files: + with patch("dissect.target.plugin._find_py_files") as mocked_find_py_files: load_modules_from_paths([]) mocked_find_py_files.assert_not_called() def test_load_environment_variable_comma_seperated_string() -> None: - with patch("dissect.target.plugin.find_py_files") as mocked_find_py_files: + with patch("dissect.target.plugin._find_py_files") as mocked_find_py_files: load_modules_from_paths([Path(""), Path("")]) mocked_find_py_files.assert_has_calls(calls=[call(Path(""))]) @@ -41,14 +41,14 @@ def test_filter_file(tmp_path: Path) -> None: file = tmp_path / "hello.py" file.touch() - assert list(find_py_files(file)) == [file] + assert list(_find_py_files(file)) == [file] test_file = tmp_path / "non_existent_file" - assert list(find_py_files(test_file)) == [] + assert list(_find_py_files(test_file)) == [] test_file = tmp_path / "__init__.py" test_file.touch() - assert list(find_py_files(test_file)) == [] + assert list(_find_py_files(test_file)) == [] @pytest.mark.parametrize( @@ -65,21 +65,22 @@ def test_filter_directory(tmp_path: Path, filename: str, empty_list: bool) -> No file.touch() if empty_list: - assert list(find_py_files(tmp_path)) == [] + assert list(_find_py_files(tmp_path)) == [] else: - assert file in list(find_py_files(tmp_path)) + assert file in list(_find_py_files(tmp_path)) def test_new_plugin_registration(environment_path: Path) -> None: copy_different_plugin_files(environment_path, "plugin.py") load_modules_from_paths([environment_path]) - assert "plugin" in PLUGINS + assert "plugin.TestPlugin" in PLUGINS["__plugins__"][None] def test_loader_registration(environment_path: Path) -> None: - with patch("dissect.target.loader.LOADERS", []) as mocked_loaders, patch( - "dissect.target.loader.LOADERS_BY_SCHEME", {} + with ( + patch("dissect.target.loader.LOADERS", []) as mocked_loaders, + patch("dissect.target.loader.LOADERS_BY_SCHEME", {}), ): copy_different_plugin_files(environment_path, "loader.py") load_modules_from_paths([environment_path]) diff --git a/tests/test_report.py b/tests/test_report.py index 831d75889..539dbd7e6 100644 --- a/tests/test_report.py +++ b/tests/test_report.py @@ -4,6 +4,7 @@ import pytest from dissect.target import Target +from dissect.target.plugin import FailureDescriptor, FunctionDescriptor from dissect.target.report import ( ExecutionReport, TargetExecutionReport, @@ -15,17 +16,17 @@ @pytest.fixture -def test_target(): +def test_target() -> Target: return Target("test_target") @pytest.fixture -def func_execs(): +def func_execs() -> set[str]: return {"exec2", "exec1"} @pytest.fixture -def target_execution_report(test_target, func_execs): +def target_execution_report(test_target: Target, func_execs: set[str]) -> TargetExecutionReport: return TargetExecutionReport( target=test_target, func_execs=func_execs, @@ -33,29 +34,29 @@ def target_execution_report(test_target, func_execs): @pytest.fixture -def incompatible_plugins(): +def incompatible_plugins() -> set[str]: return {"incomp_plugin2", "incomp_plugin1"} @pytest.fixture -def add_incompatible_plugins(target_execution_report, incompatible_plugins): +def add_incompatible_plugins(target_execution_report: TargetExecutionReport, incompatible_plugins: set[str]) -> None: for plugin in incompatible_plugins: target_execution_report.add_incompatible_plugin(plugin) @pytest.fixture -def registered_plugins(): +def registered_plugins() -> set[str]: return {"regist_plugin2", "regist_plugin1"} @pytest.fixture -def add_registered_plugins(target_execution_report, registered_plugins): +def add_registered_plugins(target_execution_report: TargetExecutionReport, registered_plugins: set[str]) -> None: for plugin in registered_plugins: target_execution_report.add_registered_plugin(plugin) @pytest.fixture -def func_errors(): +def func_errors() -> dict[str, str]: return { "func1": "trace1", "func2": "trace2", @@ -63,49 +64,49 @@ def func_errors(): @pytest.fixture -def add_func_errors(target_execution_report, func_errors): +def add_func_errors(target_execution_report: TargetExecutionReport, func_errors: dict[str, str]) -> None: for func, trace in func_errors.items(): target_execution_report.add_func_error(func, trace) def test_target_execution_report_add_incompatible_plugin( - target_execution_report, - add_incompatible_plugins, - incompatible_plugins, -): + target_execution_report: TargetExecutionReport, + add_incompatible_plugins: None, + incompatible_plugins: set[str], +) -> None: for plugin in target_execution_report.incompatible_plugins: assert plugin in incompatible_plugins def test_target_execution_report_add_registered_plugin( - target_execution_report, - add_registered_plugins, - registered_plugins, -): + target_execution_report: TargetExecutionReport, + add_registered_plugins: None, + registered_plugins: set[str], +) -> None: for plugin in target_execution_report.registered_plugins: assert plugin in registered_plugins def test_target_execution_report_add_func_error( - target_execution_report, - add_func_errors, - func_errors, -): + target_execution_report: TargetExecutionReport, + add_func_errors: None, + func_errors: dict[str, str], +) -> None: for func, trace in target_execution_report.func_errors.items(): assert func_errors.get(func) == trace def test_target_execution_report_as_dict( - test_target, - target_execution_report, - add_incompatible_plugins, - add_registered_plugins, - add_func_errors, - incompatible_plugins, - registered_plugins, - func_errors, - func_execs, -): + test_target: Target, + target_execution_report: TargetExecutionReport, + add_incompatible_plugins: None, + add_registered_plugins: None, + add_func_errors: None, + incompatible_plugins: set[str], + registered_plugins: set[str], + func_errors: dict[str, str], + func_execs: set[str], +) -> None: report_dict = target_execution_report.as_dict() assert report_dict.get("target") == str(test_target) assert report_dict.get("incompatible_plugins") == sorted(incompatible_plugins) @@ -115,63 +116,63 @@ def test_target_execution_report_as_dict( @pytest.fixture -def execution_report(): +def execution_report() -> ExecutionReport: return ExecutionReport() @pytest.fixture -def cli_args(): +def cli_args() -> argparse.Namespace: return argparse.Namespace(foo="bar", baz="bla") @pytest.fixture -def set_cli_args(execution_report, cli_args): +def set_cli_args(execution_report: ExecutionReport, cli_args: argparse.Namespace) -> None: execution_report.set_cli_args(cli_args) @pytest.fixture -def plugin_stats(): +def plugin_stats() -> dict: return { - "_failed": [ - { - "module": "plugin1", - "stacktrace": "trace1", - }, - { - "module": "plugin2", - "stacktrace": "trace2", - }, + "__failed__": [ + FailureDescriptor( + module="plugin1", + stacktrace="trace1", + ), + FailureDescriptor( + module="plugin2", + stacktrace="trace2", + ), ] } @pytest.fixture -def set_plugin_stats(execution_report, plugin_stats): +def set_plugin_stats(execution_report: ExecutionReport, plugin_stats: dict) -> None: execution_report.set_plugin_stats(plugin_stats) @pytest.fixture -def target1(): +def target1() -> Target: return Target("test1") @pytest.fixture -def target2(): +def target2() -> Target: return Target("test2") @pytest.fixture -def target_report1(execution_report, target1): +def target_report1(execution_report: ExecutionReport, target1: Target) -> TargetExecutionReport: return execution_report.add_target_report(target1) @pytest.fixture -def target_report2(execution_report, target2): +def target_report2(execution_report: ExecutionReport, target2: Target) -> TargetExecutionReport: return execution_report.add_target_report(target2) @pytest.fixture -def plugin1(): +def plugin1() -> MagicMock: plugin1 = MagicMock() plugin1.__module__ = "test_module" plugin1.__qualname__ = "plugin1" @@ -179,32 +180,32 @@ def plugin1(): def test_execution_report_set_cli_args( - execution_report, - set_cli_args, - cli_args, -): + execution_report: ExecutionReport, + set_cli_args: None, + cli_args: argparse.Namespace, +) -> None: assert execution_report.cli_args == vars(cli_args) def test_execution_report_set_plugin_stats( - execution_report, - set_plugin_stats, - plugin_stats, -): - failed_plugins = plugin_stats["_failed"] + execution_report: ExecutionReport, + set_plugin_stats: None, + plugin_stats: dict, +) -> None: + failed_plugins = plugin_stats["__failed__"] assert len(execution_report.plugin_import_errors) == len(failed_plugins) for failed_plugin in failed_plugins: - module = failed_plugin["module"] - stacktrace = failed_plugin["stacktrace"] + module = failed_plugin.module + stacktrace = failed_plugin.stacktrace assert execution_report.plugin_import_errors.get(module) == stacktrace def test_execution_report_get_formatted_report( - execution_report, - target_report1, - target_report2, -): + execution_report: ExecutionReport, + target_report1: TargetExecutionReport, + target_report2: TargetExecutionReport, +) -> None: with patch("dissect.target.report.make_cli_args_overview", return_value="line_1"): with patch("dissect.target.report.make_plugin_import_errors_overview", return_value="line_2"): with patch("dissect.target.report.format_target_report", return_value="line_x"): @@ -212,22 +213,22 @@ def test_execution_report_get_formatted_report( def test_execution_report_add_target_report( - execution_report, - target_report1, - target_report2, -): + execution_report: ExecutionReport, + target_report1: TargetExecutionReport, + target_report2: TargetExecutionReport, +) -> None: assert len(execution_report.target_reports) == 2 assert target_report1 in execution_report.target_reports assert target_report2 in execution_report.target_reports def test_execution_report_get_target_report( - execution_report, - target_report1, - target_report2, - target1, - target2, -): + execution_report: ExecutionReport, + target_report1: TargetExecutionReport, + target_report2: TargetExecutionReport, + target1: Target, + target2: Target, +) -> None: assert target_report1 == execution_report.get_target_report(target1) assert target_report2 == execution_report.get_target_report(target2) target3 = Target("nope") @@ -236,62 +237,73 @@ def test_execution_report_get_target_report( assert target_report3.target == target3 -def test_execution_report__get_plugin_name(execution_report, plugin1): +def test_execution_report__get_plugin_name(execution_report: ExecutionReport, plugin1: MagicMock) -> None: assert execution_report._get_plugin_name(plugin1) == "test_module.plugin1" def test_execution_report_log_incompatible_plugin_plugin_cls( - execution_report, - target_report1, - target1, - plugin1, -): + execution_report: ExecutionReport, + target_report1: TargetExecutionReport, + target1: Target, + plugin1: MagicMock, +) -> None: execution_report.log_incompatible_plugin(target1, None, plugin_cls=plugin1) assert "test_module.plugin1" in target_report1.incompatible_plugins def test_execution_report_log_incompatible_plugin_plugin_desc( - execution_report, - target_report1, - target1, -): - plugin_desc = {"fullname": "test_module.plugin1"} + execution_report: ExecutionReport, + target_report1: TargetExecutionReport, + target1: Target, +) -> None: + plugin_desc = FunctionDescriptor( + name="plugin1", + namespace=None, + path="", + exported=True, + internal=False, + findable=True, + output=None, + method_name="plugin1", + module="test_module", + qualname="plugin1", + ) execution_report.log_incompatible_plugin(target1, None, plugin_desc=plugin_desc) assert "test_module.plugin1" in target_report1.incompatible_plugins def test_execution_report_log_registered_plugin( - execution_report, - target_report1, - target1, -): + execution_report: ExecutionReport, + target_report1: TargetExecutionReport, + target1: Target, +) -> None: execution_report.log_registered_plugin(target1, None, plugin_inst=MagicMock()) assert "unittest.mock.MagicMock" in target_report1.registered_plugins def test_execution_report_log_func_error( - execution_report, - target_report1, - target1, - func_errors, -): + execution_report: ExecutionReport, + target_report1: TargetExecutionReport, + target1: Target, + func_errors: dict[str, str], +) -> None: func, trace = next(iter(func_errors.items())) execution_report.log_func_error(target1, None, func, trace) assert target_report1.func_errors.get(func) == trace def test_execution_report_log_func_execution( - execution_report, - target_report1, - target1, - func_execs, -): + execution_report: ExecutionReport, + target_report1: TargetExecutionReport, + target1: Target, + func_execs: set[str], +) -> None: func = next(iter(func_execs)) execution_report.log_func_execution(target1, None, func) assert func in target_report1.func_execs -def test_execution_report_set_event_callbacks(execution_report): +def test_execution_report_set_event_callbacks(execution_report: ExecutionReport) -> None: mock_target = MagicMock() event_callbacks = ( (Event.INCOMPATIBLE_PLUGIN, execution_report.log_incompatible_plugin), @@ -309,14 +321,14 @@ def test_execution_report_set_event_callbacks(execution_report): def test_execution_report_as_dict( - execution_report, - set_plugin_stats, - plugin_stats, - target_report1, - target_report2, - set_cli_args, - cli_args, -): + execution_report: ExecutionReport, + set_plugin_stats: None, + plugin_stats: dict, + target_report1: TargetExecutionReport, + target_report2: TargetExecutionReport, + set_cli_args: None, + cli_args: argparse.Namespace, +) -> None: expected_dict = { "plugin_import_errors": { "plugin1": "trace1", @@ -336,36 +348,36 @@ def test_execution_report_as_dict( def test_report_make_cli_args_overview( - execution_report, - set_cli_args, - cli_args, -): + execution_report: ExecutionReport, + set_cli_args: None, + cli_args: argparse.Namespace, +) -> None: cli_args_overview = make_cli_args_overview(execution_report) assert "foo: bar" in cli_args_overview assert "baz: bla" in cli_args_overview def test_report_make_plugin_import_errors_overview( - execution_report, - set_plugin_stats, - plugin_stats, -): + execution_report: ExecutionReport, + set_plugin_stats: None, + plugin_stats: dict, +) -> None: plugin_import_errors_overview = make_plugin_import_errors_overview(execution_report) assert "plugin1:\n trace1" in plugin_import_errors_overview assert "plugin2:\n trace2" in plugin_import_errors_overview def test_report_format_target_report( - test_target, - target_execution_report, - add_incompatible_plugins, - add_registered_plugins, - add_func_errors, - incompatible_plugins, - registered_plugins, - func_errors, - func_execs, -): + test_target: Target, + target_execution_report: TargetExecutionReport, + add_incompatible_plugins: None, + add_registered_plugins: None, + add_func_errors: None, + incompatible_plugins: set[str], + registered_plugins: set[str], + func_errors: dict[str, str], + func_execs: set[str], +) -> None: target_report = format_target_report(target_execution_report) assert str(test_target) in target_report diff --git a/tests/tools/test_query.py b/tests/tools/test_query.py index bb7db6ace..499de490b 100644 --- a/tests/tools/test_query.py +++ b/tests/tools/test_query.py @@ -1,11 +1,11 @@ import os import re from typing import Any, Optional -from unittest.mock import MagicMock, patch +from unittest.mock import patch import pytest -from dissect.target.plugin import PluginFunction +from dissect.target.plugin import FunctionDescriptor from dissect.target.target import Target from dissect.target.tools.query import main as target_query @@ -19,7 +19,7 @@ def test_target_query_list(capsys: pytest.CaptureFixture, monkeypatch: pytest.Mo out, _ = capsys.readouterr() assert out.startswith("Available plugins:") - assert "Failed to load:\n None\nAvailable loaders:\n" in out + assert "Failed to load:\n None\n\nAvailable loaders:\n" in out @pytest.mark.parametrize( @@ -43,7 +43,7 @@ def test_target_query_list(capsys: pytest.CaptureFixture, monkeypatch: pytest.Mo ), ( ["apps.webserver.iis.doesnt.exist", "apps.webserver.apache.access"], - ["apps.webserver.iis.doesnt.exist*"], + ["apps.webserver.iis.doesnt.exist"], ), ], ) @@ -96,11 +96,11 @@ def test_target_query_invalid_functions( ), ( ["apps.webserver.iis.doesnt.exist", "apps.webserver.apache.access"], - ["apps.webserver.iis.doesnt.exist*"], + ["apps.webserver.iis.doesnt.exist"], ), ], ) -def test_target_query_invalid_excluded_functions( +def test_invalid_excluded_functions( capsys: pytest.CaptureFixture, monkeypatch: pytest.MonkeyPatch, given_funcs: list[str], @@ -148,22 +148,21 @@ def test_target_query_unsupported_plugin_log(capsys: pytest.CaptureFixture, monk assert "Unsupported plugin for regf: Registry plugin not loaded" in err -def mock_find_plugin_function( - target: Target, - patterns: str, - compatibility: bool = False, - **kwargs, -) -> tuple[list[PluginFunction], set[str]]: +def mock_find_plugin_functions(patterns: str, *args, **kwargs) -> tuple[list[FunctionDescriptor], set[str]]: plugins = [] for pattern in patterns.split(","): plugins.append( - PluginFunction( + FunctionDescriptor( name=pattern, - output_type="record", + namespace=None, path=pattern, - class_object=MagicMock(), + exported=True, + internal=False, + findable=True, + output="record", method_name=pattern, - plugin_desc={}, + module=pattern, + qualname=pattern.capitalize(), ), ) @@ -172,10 +171,10 @@ def mock_find_plugin_function( def mock_execute_function( target: Target, - func: PluginFunction, - cli_params: Optional[list[str]] = None, + func: FunctionDescriptor, + arguments: Optional[list[str]] = None, ) -> tuple[str, Any, list[str]]: - return (func.output_type, func.name, "") + return (func.output, func.name, "") def test_target_query_filtered_functions(monkeypatch: pytest.MonkeyPatch) -> None: @@ -196,12 +195,12 @@ def test_target_query_filtered_functions(monkeypatch: pytest.MonkeyPatch) -> Non patch( "dissect.target.tools.query.find_plugin_functions", autospec=True, - side_effect=mock_find_plugin_function, + side_effect=mock_find_plugin_functions, ), patch( "dissect.target.tools.utils.find_plugin_functions", autospec=True, - side_effect=mock_find_plugin_function, + side_effect=mock_find_plugin_functions, ), patch( "dissect.target.tools.query.execute_function_on_target", @@ -238,8 +237,4 @@ def test_target_query_dry_run(capsys: pytest.CaptureFixture, monkeypatch: pytest target_query() out, _ = capsys.readouterr() - assert out == ( - f"Dry run on: \n" - " execute: users (general.default.users)\n" - " execute: osinfo (general.osinfo.osinfo)\n" - ) + assert out == (f"Dry run on: \n execute: osinfo (general.osinfo.osinfo)\n") diff --git a/tests/tools/test_utils.py b/tests/tools/test_utils.py index d7066dbc0..c82e38dcb 100644 --- a/tests/tools/test_utils.py +++ b/tests/tools/test_utils.py @@ -1,19 +1,21 @@ +from __future__ import annotations + from datetime import datetime from pathlib import Path +from typing import TYPE_CHECKING from unittest.mock import patch import pytest from dissect.target.exceptions import UnsupportedPluginError from dissect.target.plugin import arg, find_plugin_functions -from dissect.target.tools.utils import ( - args_to_uri, - get_target_attribute, - persist_execution_report, -) +from dissect.target.tools.utils import args_to_uri, persist_execution_report + +if TYPE_CHECKING: + from dissect.target.target import Target -def test_persist_execution_report(): +def test_persist_execution_report() -> None: output_path = Path("/tmp/test/path") report_data = { "item1": { @@ -61,12 +63,12 @@ class FakeLoader: @pytest.mark.parametrize( "pattern, expected_function", [ - ("passwords", "dissect.target.plugins.os.unix.shadow.ShadowPlugin"), - ("firefox.passwords", "Unsupported function `firefox` for target"), + ("passwords", "dissect.target.plugins.os.unix.shadow"), + ("firefox.passwords", "Unsupported function `firefox.passwords`"), ], ) -def test_plugin_name_confusion_regression(target_unix_users, pattern, expected_function): - plugins, _ = find_plugin_functions(target_unix_users, pattern) +def test_plugin_name_confusion_regression(target_unix_users: Target, pattern: str, expected_function: str) -> None: + plugins, _ = find_plugin_functions(pattern, target_unix_users) assert len(plugins) == 1 # We don't expect these functions to work since our target_unix_users fixture @@ -74,6 +76,6 @@ def test_plugin_name_confusion_regression(target_unix_users, pattern, expected_f # only interested in the plugin or namespace that was called so we check # the exception stack trace. with pytest.raises(UnsupportedPluginError) as exc_info: - get_target_attribute(target_unix_users, plugins[0]) + target_unix_users.get_function(plugins[0]) assert expected_function in str(exc_info.value)