diff --git a/misp_stix_converter/__init__.py b/misp_stix_converter/__init__.py index 08838e8..02ac123 100644 --- a/misp_stix_converter/__init__.py +++ b/misp_stix_converter/__init__.py @@ -10,13 +10,14 @@ from .misp2stix import stix20_framing, stix21_framing # noqa # Helpers from .misp_stix_converter import ( # noqa - _from_misp, misp_attribute_collection_to_stix1, misp_collection_to_stix2, + _is_stix1_from_misp, _is_stix2_from_misp, + misp_attribute_collection_to_stix1, misp_collection_to_stix2, misp_event_collection_to_stix1, misp_to_stix1, misp_to_stix2, stix_1_to_misp, stix_2_to_misp, stix2_to_misp_instance) # STIX 1 special helpers from .misp_stix_converter import ( # noqa _get_campaigns, _get_courses_of_action, _get_events, _get_indicators, - _get_observables, _get_threat_actors, _get_ttps, _from_misp) + _get_observables, _get_threat_actors, _get_ttps) # STIX 1 footers from .misp_stix_converter import ( # noqa _get_campaigns_footer, _get_courses_of_action_footer, _get_indicators_footer, @@ -110,8 +111,7 @@ def main(): # IMPORT SUBPARSER import_parser = subparsers.add_parser( - 'import', help='Import STIX to MISP - try ' - '`misp_stix_converter import -h` for more help.' + 'import', help='Import STIX to MISP - try `misp_stix_converter import -h` for more help.' ) import_parser.add_argument( '-f', '--file', nargs='+', type=Path, required=True, @@ -138,7 +138,14 @@ def main(): ) import_parser.add_argument( '-d', '--distribution', type=int, default=0, choices=[0, 1, 2, 3, 4], - help='Distribution level for the imported MISP content - default is 0' + help=''' + Distribution level for the imported MISP content (default is 0) + - 0: Your organisation only + - 1: This community only + - 2: Connected communities + - 3: All communities + - 4: Sharing Group + ''' ) import_parser.add_argument( '-sg', '--sharing_group', type=int, default=None, @@ -153,7 +160,15 @@ def main(): ) import_parser.add_argument( '-cd', '--cluster_distribution', type=int, default=0, choices=[0, 1, 2, 3, 4], - help='Galaxy Clusters distribution level in case of External STIX 2 content - default id 0' + help=''' + Galaxy Clusters distribution level + in case of External STIX 2 content (default id 0) + - 0: Your organisation only + - 1: This community only + - 2: Connected communities + - 3: All communities + - 4: Sharing Group + ''' ) import_parser.add_argument( '-cg', '--cluster_sharing_group', type=int, default=None, @@ -161,7 +176,7 @@ def main(): ) import_parser.add_argument( '-t', '--title', type=str, default=None, - help='Title prefix to add to the MISP Event `info` field.' + help='Title used to set the MISP Event `info` field.' ) import_parser.add_argument( '-p', '--producer', diff --git a/misp_stix_converter/misp2stix/misp_to_stix2.py b/misp_stix_converter/misp2stix/misp_to_stix2.py index 7d29570..eedbb38 100644 --- a/misp_stix_converter/misp2stix/misp_to_stix2.py +++ b/misp_stix_converter/misp2stix/misp_to_stix2.py @@ -12,13 +12,29 @@ from datetime import datetime from pathlib import Path from pymisp import ( - MISPAttribute, MISPEvent, MISPGalaxy, MISPGalaxyCluster, MISPObject) + MISPAttribute, MISPEvent, MISPEventReport, MISPGalaxy, MISPGalaxyCluster, + MISPNote, MISPObject, MISPOpinion) from stix2.hashes import check_hash, Hash from stix2.properties import ListProperty, StringProperty from stix2.v20.bundle import Bundle as Bundle_v20 from stix2.v21.bundle import Bundle as Bundle_v21 +from stix2.v20.sdo import ( + AttackPattern as AttackPattern_v20, Campaign as Campaign_v20, + CourseOfAction as CourseOfAction_v20, CustomObject as CustomObject_v20, + Identity as Identity_v20, Indicator as Indicator_v20, + IntrusionSet as IntrusionSet_v20, Malware as Malware_v20, + ObservedData as ObservedData_v20, Tool as Tool_v20, + Vulnerability as Vulnerability_v20) +from stix2.v21.sdo import ( + AttackPattern as AttackPattern_v21, Campaign as Campaign_v21, + CourseOfAction as CourseOfAction_v21, CustomObject as CustomObject_v21, + Identity as Identity_v21, Indicator as Indicator_v21, + IntrusionSet as IntrusionSet_v21, Location, Malware as Malware_v21, Note, + ObservedData as ObservedData_v21, Tool as Tool_v21, + Vulnerability as Vulnerability_v21) from typing import Generator, Optional, Tuple, Union +_event_report_regex = r'@[!]?\[%s\]\([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\)' _label_fields = ('type', 'category', 'to_ids') _labelled_object_types = ('malware', 'threat-actor', 'tool') _misp_time_fields = ('first_seen', 'last_seen') @@ -30,6 +46,18 @@ 'observed-data': ('first_observed', 'last_observed') } +_MISP_DATA_LAYER = Union[ + dict, MISPAttribute, MISPEventReport, MISPObject +] +_STIX_OBJECT_TYPING = Union[ + AttackPattern_v20, AttackPattern_v21, Campaign_v20, Campaign_v21, + CourseOfAction_v20, CourseOfAction_v21, CustomObject_v20, CustomObject_v21, + Identity_v20, Identity_v21, Indicator_v20, Indicator_v21, + IntrusionSet_v20, IntrusionSet_v21, Location, Malware_v20, Malware_v21, + Note, ObservedData_v20, ObservedData_v21, Tool_v20, Tool_v21, + Vulnerability_v20, Vulnerability_v21, dict +] + class InvalidHashValueError(Exception): pass @@ -117,14 +145,72 @@ def _parse_misp_event(self, misp_event: Union[MISPEvent, dict]): self.__object_refs = [] self.__relationships = [] self._handle_identity_from_event() - self._parse_event_data() - report = self._generate_event_report() + if self._misp_event.get('EventReport'): + self._id_parsing_function = { + 'attribute': '_define_stix_object_id_from_attribute', + 'object': '_define_stix_object_id_from_object' + } + self._event_report_matching = defaultdict(list) + self._handle_attributes_and_objects() + for event_report in self._misp_event['EventReport']: + note = self._parse_event_report(event_report) + self._append_SDO(note) + self._handle_analyst_data(note, event_report) + else: + self._id_parsing_function = { + 'attribute': '_define_stix_object_id', + 'object': '_define_stix_object_id' + } + self._handle_attributes_and_objects() + report = self._generate_report_from_event() self.__objects.insert(self.__index, report) def _define_stix_object_id( self, feature: str, misp_object: Union[MISPObject, dict]) -> str: return f"{feature}--{misp_object['uuid']}" + def _define_stix_object_id_from_attribute( + self, feature: str, attribute: Union[MISPAttribute, dict]) -> str: + attribute_uuid = attribute['uuid'] + stix_id = f'{feature}--{attribute_uuid}' + self._event_report_matching[attribute_uuid].append(stix_id) + return stix_id + + def _define_stix_object_id_from_object( + self, feature: str, misp_object: Union[MISPObject, dict]) -> str: + object_uuid = misp_object['uuid'] + stix_id = f'{feature}--{object_uuid}' + self._event_report_matching[object_uuid].append(stix_id) + for attribute in misp_object['Attribute']: + self._event_report_matching[attribute['uuid']].append(stix_id) + return stix_id + + def _handle_attributes_and_objects(self): + if self._misp_event.get('Attribute'): + for attribute in self._misp_event['Attribute']: + self._resolve_attribute(attribute) + if self._misp_event.get('Object'): + self._objects_to_parse = defaultdict(dict) + self._resolve_objects() + if self._objects_to_parse: + self._resolve_objects_to_parse() + if self._objects_to_parse.get('annotation'): + objects_to_parse = self._objects_to_parse['annotation'] + for misp_object in objects_to_parse.values(): + to_ids, annotation_object = misp_object + custom = ( + annotation_object.get('ObjectReference') is None or + not self._annotates( + annotation_object['ObjectReference'] + ) + ) + if custom: + self._parse_custom_object(annotation_object) + else: + self._parse_annotation_object( + to_ids, annotation_object + ) + def _handle_default_identity(self): misp_identity_args = self._mapping.misp_identity_args() self.__identity_id = misp_identity_args['id'] @@ -253,12 +339,12 @@ def unique_ids(self) -> dict: def _append_SDO(self, stix_object): self.__objects.append(stix_object) - self.__object_refs.append(stix_object.id) + self.object_refs.append(stix_object.id) def _append_SDO_without_refs(self, stix_object): self.__objects.append(stix_object) - def _generate_event_report(self): + def _generate_report_from_event(self): report_args = { 'name': self._misp_event.get( 'info', @@ -280,20 +366,21 @@ def _generate_event_report(self): marking['used'] = True if self._is_published(): report_id = f"report--{self._misp_event['uuid']}" - if not self.__object_refs: + if not self.object_refs: self._handle_empty_object_refs(report_id, self.event_timestamp) published = self._datetime_from_timestamp( self._misp_event['publish_timestamp'] ) report_args.update( { - 'id': report_id, 'type': 'report', 'published': published, - 'object_refs': self.__object_refs, 'allow_custom': True + 'id': report_id, 'type': 'report', + 'published': published, 'allow_custom': True } ) - if self._version == '2.1': - self._handle_analyst_data(report_id) - return self._create_report(report_args) + self._handle_analyst_data(report_args) + report_args['object_refs'] = self.object_refs + report = self._create_report(report_args) + return report return self._handle_unpublished_report(report_args) def _generate_galaxies_catalog(self): @@ -343,6 +430,21 @@ def _get_object_ids( in self._galaxies_catalog[name][object_type] ) + def _handle_analyst_data(self, stix_object: _STIX_OBJECT_TYPING, + data_layer: _MISP_DATA_LAYER = None): + if data_layer is None: + data_layer = self._misp_event + for note in data_layer.get('Note', []): + self._handle_note_data(stix_object, note) + for opinion in data_layer.get('Opinion', []): + self._handle_opinion_data(stix_object, opinion) + + def _handle_object_analyst_data(self, stix_object: _STIX_OBJECT_TYPING, + misp_object: Union[MISPObject, dict]): + self._handle_analyst_data(stix_object, misp_object) + for attribute in misp_object['Attribute']: + self._handle_analyst_data(stix_object, attribute) + def _handle_relationships(self): for relationship in self.__relationships: if relationship.get('undefined_target_ref'): @@ -392,6 +494,19 @@ def _handle_sighting_identity(self, uuid: str, name: str) -> str: self._handle_identity(identity_id, name) return identity_id + def _parse_event_report_references( + self, event_report: Union[MISPEventReport, dict]): + references = { + reference.split('(')[1][:-1] + for feature in ('attribute', 'object') + for reference in re.findall( + _event_report_regex % feature, event_report['content'] + ) + } + for reference in references: + if reference in self._event_report_matching: + yield from self._event_report_matching[reference] + ############################################################################ # ATTRIBUTES PARSING FUNCTIONS # ############################################################################ @@ -436,11 +551,9 @@ def _handle_attribute_indicator( ) if markings: self._handle_markings(indicator_arguments, markings) - getattr(self, self._results_handling_function)( - self._create_indicator(indicator_arguments) - ) - if self._version == '2.1': - self._handle_analyst_data(indicator_id, attribute) + indicator = self._create_indicator(indicator_arguments) + getattr(self, self._results_handling_function)(indicator) + self._handle_analyst_data(indicator, attribute) if attribute.get('Sighting'): self._handle_sightings(attribute['Sighting'], indicator_id) @@ -462,7 +575,8 @@ def _handle_attribute_observable( ) if markings: self._handle_markings(observable_args, markings) - self._create_observed_data(observable_args, observable) + observed_data = self._create_observed_data(observable_args, observable) + self._handle_analyst_data(observed_data, attribute) if attribute.get('Sighting'): self._handle_sightings(attribute['Sighting'], observable_id) @@ -541,11 +655,9 @@ def _parse_campaign_name_attribute( ) if markings: self._handle_markings(campaign_args, markings) - getattr(self, self._results_handling_function)( - self._create_campaign(campaign_args) - ) - if self._version == '2.1': - self._handle_analyst_data(campaign_id, attribute) + campaign = self._create_campaign(campaign_args) + getattr(self, self._results_handling_function)(campaign) + self._handle_analyst_data(campaign, attribute) if attribute.get('Sighting'): self._handle_sightings(attribute['Sighting'], campaign_id) @@ -570,11 +682,9 @@ def _parse_custom_attribute(self, attribute: Union[MISPAttribute, dict]): ) if markings: self._handle_markings(custom_args, markings) - getattr(self, self._results_handling_function)( - self._create_custom_attribute(custom_args) - ) - if self._version == '2.1': - self._handle_analyst_data(custom_id, attribute) + custom_attribute = self._create_custom_attribute(custom_args) + getattr(self, self._results_handling_function)(custom_attribute) + self._handle_analyst_data(custom_attribute, attribute) if attribute.get('Sighting'): self._handle_sightings(attribute['Sighting'], custom_id) @@ -967,11 +1077,9 @@ def _parse_vulnerability_attribute( ) if markings: self._handle_markings(vulnerability_args, markings) - getattr(self, self._results_handling_function)( - self._create_vulnerability(vulnerability_args) - ) - if self._version == '2.1': - self._handle_analyst_data(vulnerability_id, attribute) + vulnerability = self._create_vulnerability(vulnerability_args) + getattr(self, self._results_handling_function)(vulnerability) + self._handle_analyst_data(vulnerability, attribute) if attribute.get('Sighting'): self._handle_sightings(attribute['Sighting'], vulnerability_id) @@ -1080,13 +1188,10 @@ def _handle_non_indicator_object( misp_object['ObjectReference'], object_id, object_args['modified'] ) - self._append_SDO( - getattr(self, f"_create_{object_type.replace('-', '_')}")( - object_args - ) - ) - if self._version == '2.1': - self._handle_object_analyst_data(misp_object, object_id) + feature = f"_create_{object_type.replace('-', '_')}" + stix_object = getattr(self, feature)(object_args) + getattr(self, self._results_handling_function)(stix_object) + self._handle_object_analyst_data(stix_object, misp_object) def _handle_object_indicator( self, misp_object: Union[MISPObject, dict], pattern: list): @@ -1115,9 +1220,9 @@ def _handle_object_indicator( misp_object['ObjectReference'], indicator_id, indicator_args['modified'] ) - self._append_SDO(self._create_indicator(indicator_args)) - if self._version == '2.1': - self._handle_object_analyst_data(misp_object, indicator_id) + indicator = self._create_indicator(indicator_args) + getattr(self, self._results_handling_function)(indicator) + self._handle_object_analyst_data(indicator, misp_object) def _handle_object_observable( self, misp_object: Union[MISPObject, dict], @@ -1142,7 +1247,8 @@ def _handle_object_observable( misp_object['ObjectReference'], observable_id, observable_args['modified'] ) - self._create_observed_data(observable_args, observable) + observed_data = self._create_observed_data(observable_args, observable) + self._handle_object_analyst_data(observed_data, misp_object) def _handle_object_tags_and_galaxies( self, misp_object: Union[MISPObject, dict], @@ -1463,9 +1569,9 @@ def _parse_custom_object(self, misp_object: Union[MISPObject, dict]): self._parse_object_relationships( misp_object['ObjectReference'], custom_id, timestamp ) - self._append_SDO(self._create_custom_object(custom_args)) - if self._version == '2.1': - self._handle_object_analyst_data(misp_object, custom_id) + custom_object = self._create_custom_object(custom_args) + getattr(self, self._results_handling_function)(custom_object) + self._handle_object_analyst_data(custom_object, misp_object) @staticmethod def _parse_custom_object_attribute( @@ -1610,9 +1716,9 @@ def _parse_employee_object(self, misp_object: Union[MISPObject, dict]): misp_object['ObjectReference'], identity_args['id'], identity_args['modified'] ) - self._append_SDO(self._create_identity(identity_args)) - if self._version == '2.1': - self._handle_object_analyst_data(misp_object, identity_args['id']) + identity = self._create_identity(identity_args) + getattr(self, self._results_handling_function)(identity) + self._handle_object_analyst_data(identity, misp_object) def _parse_file_object(self, misp_object: Union[MISPObject, dict]): to_ids = self._fetch_ids_flag(misp_object['Attribute']) @@ -1879,9 +1985,9 @@ def _parse_legal_entity_object(self, misp_object: Union[MISPObject, dict]): misp_object['ObjectReference'], identity_args['id'], identity_args['modified'] ) - self._append_SDO(self._create_identity(identity_args)) - if self._version == '2.1': - self._handle_object_analyst_data(misp_object, identity_args['id']) + identity = self._create_identity(identity_args) + getattr(self, self._results_handling_function)(identity) + self._handle_object_analyst_data(identity, misp_object) def _parse_lnk_object(self, misp_object: Union[MISPObject, dict]): if self._fetch_ids_flag(misp_object['Attribute']): @@ -2158,9 +2264,9 @@ def _parse_news_agency_object(self, misp_object: Union[MISPObject, dict]): misp_object['ObjectReference'], identity_args['id'], identity_args['modified'] ) - self._append_SDO(self._create_identity(identity_args)) - if self._version == '2.1': - self._handle_object_analyst_data(misp_object, identity_args['id']) + identity = self._create_identity(identity_args) + getattr(self, self._results_handling_function)(identity) + self._handle_object_analyst_data(identity, misp_object) def _parse_person_object(self, misp_object: Union[MISPObject, dict]): identity_args = self._parse_identity_args(misp_object, 'individual') @@ -2196,9 +2302,9 @@ def _parse_person_object(self, misp_object: Union[MISPObject, dict]): misp_object['ObjectReference'], identity_args['id'], identity_args['modified'] ) - self._append_SDO(self._create_identity(identity_args)) - if self._version == '2.1': - self._handle_object_analyst_data(misp_object, identity_args['id']) + identity = self._create_identity(identity_args) + getattr(self, self._results_handling_function)(identity) + self._handle_object_analyst_data(identity, misp_object) def _parse_organization_object(self, misp_object: Union[MISPObject, dict]): identity_args = self._parse_identity_args(misp_object, 'organization') @@ -2224,9 +2330,9 @@ def _parse_organization_object(self, misp_object: Union[MISPObject, dict]): misp_object['ObjectReference'], identity_args['id'], identity_args['modified'] ) - self._append_SDO(self._create_identity(identity_args)) - if self._version == '2.1': - self._handle_object_analyst_data(misp_object, identity_args['id']) + identity = self._create_identity(identity_args) + getattr(self, self._results_handling_function)(identity) + self._handle_object_analyst_data(identity, misp_object) def _parse_pe_extensions_observable( self, pe_object: dict, uuids: Optional[list] = None) -> dict: @@ -4027,9 +4133,9 @@ def _handle_patterning_object_indicator( misp_object['ObjectReference'], indicator_id, indicator_args['modified'] ) - self._append_SDO(self._create_indicator(indicator_args)) - if self._version == '2.1': - self._handle_object_analyst_data(misp_object, indicator_id) + indicator = self._create_indicator(indicator_args) + getattr(self, self._results_handling_function)(indicator) + self._handle_object_analyst_data(indicator, misp_object) ############################################################################ # UTILITY FUNCTIONS. # @@ -4098,7 +4204,7 @@ def _fetch_included_reference_uuids( return uuids def _find_target_uuid(self, reference: str) -> Union[str, None]: - for object_ref in self.__object_refs: + for object_ref in self.object_refs: if reference in object_ref: return object_ref @@ -4125,6 +4231,14 @@ def _get_matching_email_display_name( def _get_vulnerability_references(vulnerability: str) -> dict: return {'source_name': 'cve', 'external_id': vulnerability} + def _handle_analyst_time_fields(self, stix_object: _STIX_OBJECT_TYPING, + misp_object: Union[MISPNote, MISPOpinion]): + for feature in ('created', 'modified'): + if misp_object.get(feature): + yield feature, self._datetime_from_str(misp_object[feature]) + continue + yield feature, stix_object[feature] + def _handle_custom_data_field( self, values: Union[list, str, tuple]) -> Union[dict, list, str]: if isinstance(values, list): diff --git a/misp_stix_converter/misp2stix/misp_to_stix20.py b/misp_stix_converter/misp2stix/misp_to_stix20.py index 972cb3d..0cc4944 100644 --- a/misp_stix_converter/misp2stix/misp_to_stix20.py +++ b/misp_stix_converter/misp2stix/misp_to_stix20.py @@ -6,10 +6,11 @@ from base64 import b64encode from collections import defaultdict from datetime import datetime -from pymisp import MISPAttribute, MISPObject +from pymisp import ( + MISPAttribute, MISPEventReport, MISPNote, MISPObject, MISPOpinion) from stix2.properties import ( - DictionaryProperty, IDProperty, ListProperty, ReferenceProperty, - StringProperty, TimestampProperty) + DictionaryProperty, IDProperty, IntegerProperty, ListProperty, + ReferenceProperty, StringProperty, TimestampProperty) from stix2.v20.bundle import Bundle from stix2.v20.observables import ( Artifact, AutonomousSystem, Directory, DomainName, EmailAddress, @@ -25,6 +26,60 @@ from stix2.v20.vocab import HASHING_ALGORITHM from typing import Optional, Union +_ANALYST_DATA_REFERENCE_TYPES = [ + 'attack-pattern', 'campaign', 'course-of-action', 'identity', 'indicator', + 'intrusion-set', 'malware', 'observed-data', 'report', 'threat-actor', + 'tool', 'vulnerability', 'x-misp-analyst-note', 'x-misp-analyst-opinion', + 'x-misp-attribute', 'x-misp-event-note', 'x-misp-event-report', + 'x-misp-galaxy-cluster', 'x-misp-object' +] +_STIX_OBJECT_TYPING = Union[ + AttackPattern, Campaign, CourseOfAction, CustomObject, Identity, Indicator, + IntrusionSet, Malware, ObservedData, Tool, Vulnerability, dict +] + + +@CustomObject( + 'x-misp-analyst-note', + [ + ('id', IDProperty('x-misp-analyst-note')), + ('created', TimestampProperty(precision='millisecond')), + ('modified', TimestampProperty(precision='millisecond')), + ('x_misp_note', StringProperty(required=True)), + ('x_misp_author', StringProperty()), + ('x_misp_language', StringProperty()), + ( + 'object_ref', + ReferenceProperty( + valid_types=_ANALYST_DATA_REFERENCE_TYPES, spec_version='2.0' + ) + ) + ] +) +class CustomAnalystNote: + pass + + +@CustomObject( + 'x-misp-analyst-opinion', + [ + ('id', IDProperty('x-misp-analyst-opinion')), + ('created', TimestampProperty(precision='millisecond')), + ('modified', TimestampProperty(precision='millisecond')), + ('x_misp_opinion', IntegerProperty(required=True)), + ('x_misp_author', StringProperty()), + ('x_misp_comment', StringProperty()), + ( + 'object_ref', + ReferenceProperty( + valid_types=_ANALYST_DATA_REFERENCE_TYPES, spec_version='2.0' + ) + ) + ] +) +class CustomAnalystOpinion: + pass + @CustomObject( 'x-misp-attribute', @@ -56,10 +111,9 @@ class CustomAttribute: @CustomObject( - 'x-misp-object', + 'x-misp-event-report', [ - ('id', IDProperty('x-misp-object')), - ('labels', ListProperty(StringProperty, required=True)), + ('id', IDProperty('x-misp-event-report')), ('created', TimestampProperty(required=True, precision='millisecond')), ('modified', TimestampProperty(required=True, precision='millisecond')), ( @@ -67,20 +121,20 @@ class CustomAttribute: ReferenceProperty(valid_types='identity', spec_version='2.0') ), ( - 'object_marking_refs', + 'object_refs', ListProperty( ReferenceProperty( - valid_types='marking-definition', spec_version='2.0' - ) + valid_types=_ANALYST_DATA_REFERENCE_TYPES, + spec_version='2.0' + ), + required=True ) ), - ('x_misp_name', StringProperty(required=True)), - ('x_misp_attributes', ListProperty(DictionaryProperty())), - ('x_misp_comment', StringProperty()), - ('x_misp_meta_category', StringProperty()) + ('x_misp_content', StringProperty(required=True)), + ('x_misp_name', StringProperty()), ] ) -class CustomMispObject: +class CustomEventReport: pass @@ -106,6 +160,35 @@ class CustomGalaxyCluster: pass +@CustomObject( + 'x-misp-object', + [ + ('id', IDProperty('x-misp-object')), + ('labels', ListProperty(StringProperty, required=True)), + ('created', TimestampProperty(required=True, precision='millisecond')), + ('modified', TimestampProperty(required=True, precision='millisecond')), + ( + 'created_by_ref', + ReferenceProperty(valid_types='identity', spec_version='2.0') + ), + ( + 'object_marking_refs', + ListProperty( + ReferenceProperty( + valid_types='marking-definition', spec_version='2.0' + ) + ) + ), + ('x_misp_name', StringProperty(required=True)), + ('x_misp_attributes', ListProperty(DictionaryProperty())), + ('x_misp_comment', StringProperty()), + ('x_misp_meta_category', StringProperty()) + ] +) +class CustomMispObject: + pass + + @CustomObject( 'x-misp-event-note', [ @@ -119,7 +202,7 @@ class CustomGalaxyCluster: ('x_misp_event_note', StringProperty(required=True)), ( 'object_ref', - ReferenceProperty(valid_types=['report'], spec_version='2.0') + ReferenceProperty(valid_types='report', spec_version='2.0') ) ] ) @@ -161,15 +244,21 @@ def __init__(self, interoperability=False): self._version = '2.0' self._mapping = MISPtoSTIX20Mapping - def _parse_event_data(self): - if self._misp_event.get('Attribute'): - for attribute in self._misp_event['Attribute']: - self._resolve_attribute(attribute) - if self._misp_event.get('Object'): - self._objects_to_parse = defaultdict(dict) - self._resolve_objects() - if self._objects_to_parse: - self._resolve_objects_to_parse() + def _parse_event_report( + self, event_report: Union[MISPEventReport, dict]) -> CustomEventReport: + timestamp = self._datetime_from_timestamp(event_report['timestamp']) + note_args = { + 'id': f"x-misp-event-report--{event_report['uuid']}", + 'created': timestamp, 'modified': timestamp, + 'created_by_ref': self.identity_id, + 'x_misp_content': event_report['content'] + } + if event_report.get('name'): + note_args['x_misp_name'] = event_report['name'] + references = set(self._parse_event_report_references(event_report)) + if references: + note_args['object_refs'] = list(references) + return CustomEventReport(**note_args) def _handle_empty_object_refs(self, object_id: str, timestamp: datetime): object_type = 'x-misp-event-note' @@ -202,6 +291,41 @@ def _handle_markings(self, object_args: dict, markings: tuple): if marking_ids: object_args['object_marking_refs'] = marking_ids + def _handle_note_data(self, stix_object: _STIX_OBJECT_TYPING, + note: Union[MISPNote, dict]): + note_args = { + 'id': f"x-misp-analyst-note--{note['uuid']}", + 'object_ref': stix_object['id'], 'x_misp_note': note['note'], + **dict(self._handle_analyst_time_fields(stix_object, note)) + } + if note.get('authors'): + note_args['x_misp_author'] = note['authors'] + if note.get('language'): + note_args['x_misp_language'] = note['language'] + if stix_object['id'].startswith('x-misp-'): + note_args['allow_custom'] = True + getattr(self, self._results_handling_function)( + CustomAnalystNote(**note_args) + ) + + def _handle_opinion_data(self, stix_object: _STIX_OBJECT_TYPING, + opinion: Union[MISPOpinion, dict]): + opinion_args = { + 'id': f"x-misp-analyst-opinion--{opinion['uuid']}", + 'object_ref': stix_object['id'], + 'x_misp_opinion': opinion['opinion'], + **dict(self._handle_analyst_time_fields(stix_object, opinion)) + } + if opinion.get('authors'): + opinion_args['x_misp_author'] = opinion['authors'] + if opinion.get('comment'): + opinion_args['x_misp_comment'] = opinion['comment'] + if stix_object['id'].startswith('x-misp-'): + opinion_args['allow_custom'] = True + getattr(self, self._results_handling_function)( + CustomAnalystOpinion(**opinion_args) + ) + def _handle_opinion_object(self, sighting: dict, reference_id: str): opinion_args = { 'id': f"x-misp-opinion--{sighting['uuid']}", @@ -239,15 +363,17 @@ def _handle_unpublished_report(self, report_args: dict) -> Report: report_args.update( { 'id': report_id, 'type': 'report', - 'published': report_args['modified'], - 'object_refs': self.object_refs, 'allow_custom': True + 'published': report_args['modified'], 'allow_custom': True } ) - return Report(**report_args) + self._handle_analyst_data(report_args) + report_args['object_refs'] = self.object_refs + report = self._create_report(report_args) + return report - ################################################################################ - # ATTRIBUTES PARSING FUNCTIONS # - ################################################################################ + ############################################################################ + # ATTRIBUTES PARSING FUNCTIONS # + ############################################################################ def _parse_attachment_attribute_observable( self, attribute: Union[MISPAttribute, dict]): @@ -1288,9 +1414,12 @@ def _create_intrusion_set(intrusion_set_args: dict) -> IntrusionSet: def _create_malware(malware_args: dict) -> Malware: return Malware(**malware_args) - def _create_observed_data(self, args: dict, observable: dict): + def _create_observed_data( + self, args: dict, observable: dict) -> ObservedData: args['objects'] = observable - getattr(self, self._results_handling_function)(ObservedData(**args)) + observed_data = ObservedData(**args) + getattr(self, self._results_handling_function)(observed_data) + return observed_data @staticmethod def _create_PE_extension(extension_args: dict) -> WindowsPEBinaryExt: diff --git a/misp_stix_converter/misp2stix/misp_to_stix21.py b/misp_stix_converter/misp2stix/misp_to_stix21.py index 5c66ef5..4e87f4b 100644 --- a/misp_stix_converter/misp2stix/misp_to_stix21.py +++ b/misp_stix_converter/misp2stix/misp_to_stix21.py @@ -8,7 +8,8 @@ from collections import defaultdict from datetime import datetime from pymisp import ( - MISPAttribute, MISPEventReport, MISPGalaxy, MISPGalaxyCluster, MISPObject) + MISPAttribute, MISPEventReport, MISPGalaxy, MISPGalaxyCluster, MISPNote, + MISPObject, MISPOpinion) from stix2.properties import ( DictionaryProperty, IDProperty, ListProperty, ReferenceProperty, StringProperty, TimestampProperty) @@ -27,8 +28,10 @@ from stix2.v21.vocab import HASHING_ALGORITHM from typing import Optional, Union -_MISP_DATA_LAYER = Union[ - dict, MISPAttribute, MISPEventReport +_STIX_OBJECT_TYPING = Union[ + AttackPattern, Campaign, CourseOfAction, CustomObject, Identity, + Indicator, IntrusionSet, Location, Malware, Note, ObservedData, Tool, + Vulnerability, dict ] @@ -95,109 +98,32 @@ def __init__(self, interoperability=False): self._version = '2.1' self._mapping = MISPtoSTIX21Mapping - def _parse_event_data(self): - if self._misp_event.get('EventReport'): - self._id_parsing_function = { - 'attribute': '_define_stix_object_id_from_attribute', - 'object': '_define_stix_object_id_from_object' - } - self._event_report_matching = defaultdict(list) - self._handle_attributes_and_objects() - regex = r'@[!]?\[%s\]\([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\)' - for event_report in self._misp_event['EventReport']: - timestamp = self._datetime_from_timestamp( - event_report['timestamp'] - ) - note_args = { - 'id': f"note--{event_report['uuid']}", - 'created': timestamp, 'modified': timestamp, - 'created_by_ref': self.identity_id, - 'content': event_report['content'], - 'abstract': event_report['name'] - } - references = { - reference.split('(')[1][:-1] - for feature in ('attribute', 'object') - for reference in re.findall( - regex % feature, event_report['content'] - ) - } - object_refs = set() - for reference in references: - if reference in self._event_report_matching: - object_refs.update( - self._event_report_matching[reference] - ) - note_args['object_refs'] = ( - list(object_refs) if object_refs - else self._handle_empty_note_refs() - ) - self._append_SDO(self._create_note(note_args)) - self._handle_analyst_data(note_args['id'], event_report) - else: - self._id_parsing_function = { - 'attribute': '_define_stix_object_id', - 'object': '_define_stix_object_id' - } - self._handle_attributes_and_objects() - - def _define_stix_object_id_from_attribute( - self, feature: str, attribute: Union[MISPAttribute, dict]) -> str: - attribute_uuid = attribute['uuid'] - stix_id = f'{feature}--{attribute_uuid}' - self._event_report_matching[attribute_uuid].append(stix_id) - return stix_id - - def _define_stix_object_id_from_object( - self, feature: str, misp_object: Union[MISPObject, dict]) -> str: - object_uuid = misp_object['uuid'] - stix_id = f'{feature}--{object_uuid}' - self._event_report_matching[object_uuid].append(stix_id) - for attribute in misp_object['Attribute']: - self._event_report_matching[attribute['uuid']].append(stix_id) - return stix_id - - def _handle_analyst_data( - self, object_id: str, data_layer: _MISP_DATA_LAYER = None): - if data_layer is None: - data_layer = self._misp_event - for note in data_layer.get('Note', []): - self._handle_note_data(note, object_id) - for opinion in data_layer.get('Opinion', []): - self._handle_opinion_data(opinion, object_id) - - def _handle_attributes_and_objects(self): - if self._misp_event.get('Attribute'): - for attribute in self._misp_event['Attribute']: - self._resolve_attribute(attribute) - if self._misp_event.get('Object'): - self._objects_to_parse = defaultdict(dict) - self._resolve_objects() - if self._objects_to_parse: - self._resolve_objects_to_parse() - if self._objects_to_parse.get('annotation'): - objects_to_parse = self._objects_to_parse['annotation'] - for misp_object in objects_to_parse.values(): - to_ids, annotation_object = misp_object - custom = ( - annotation_object.get('ObjectReference') is None or - not self._annotates( - annotation_object['ObjectReference'] - ) - ) - if custom: - self._parse_custom_object(annotation_object) - else: - self._parse_annotation_object( - to_ids, annotation_object - ) + def _parse_event_report( + self, event_report: Union[MISPEventReport, dict]) -> Note: + timestamp = self._datetime_from_timestamp(event_report['timestamp']) + note_args = { + 'id': f"note--{event_report['uuid']}", + 'created': timestamp, 'modified': timestamp, + 'created_by_ref': self.identity_id, + 'content': event_report['content'], + 'abstract': event_report['name'], + 'labels': ['misp:data-layer="Event Report"'] + } + references = set(self._parse_event_report_references(event_report)) + note_args['object_refs'] = ( + list(references) if references else self._handle_empty_note_refs() + ) + return self._create_note(note_args) def _handle_empty_object_refs(self, object_id: str, timestamp: datetime): note_args = { 'id': f"note--{self._misp_event['uuid']}", 'created': timestamp, 'modified': timestamp, 'created_by_ref': self.identity_id, 'object_refs': [object_id], - 'content': 'This MISP Event is empty and contains no attribute, object, galaxy or tag.' + 'content': ( + 'This MISP Event is empty and contains ' + 'no attribute, object, galaxy or tag.' + ) } self._append_SDO(self._create_note(note_args)) @@ -227,37 +153,43 @@ def _handle_markings(self, object_args: dict, markings: tuple): if marking_ids: object_args['object_marking_refs'] = marking_ids - def _handle_note_data(self, note, object_id: str): + def _handle_note_data(self, stix_object: _STIX_OBJECT_TYPING, + note: Union[MISPNote, dict]): note_args = { - 'authors': [note['authors']], 'content': note['note'], - 'created': self._datetime_from_str(note['created']), - 'modified': self._datetime_from_str(note['modified']), - 'id': f"note--{note['uuid']}", 'object_refs': [object_id] + 'content': note['note'], 'id': f"note--{note['uuid']}", + 'labels': ['misp:context-layer="Analyst Note"'], + 'object_refs': [stix_object['id']], + **dict(self._handle_analyst_time_fields(stix_object, note)) } + if note.get('authors'): + note_args['authors'] = [note['authors']] if note.get('language'): note_args['lang'] = note['language'] + if stix_object['id'].startswith('x-misp--'): + note_args['allow_custom'] = True getattr(self, self._results_handling_function)( self._create_note(note_args) ) - def _handle_object_analyst_data( - self, misp_object: Union[MISPObject, dict], object_id: str): - for attribute in misp_object['Attribute']: - self._handle_analyst_data(object_id, attribute) - - def _handle_opinion_data(self, opinion, object_id: str): + def _handle_opinion_data(self, stix_object: _STIX_OBJECT_TYPING, + opinion: Union[MISPOpinion, dict]): opinion_value = int(opinion['opinion']) opinion_args = { - 'allow_custom': True, 'authors': [opinion['authors']], - 'created': self._datetime_from_str(opinion['created']), - 'modified': self._datetime_from_str(opinion['modified']), - 'id': f"opinion--{opinion['uuid']}", 'object_refs': [object_id], + 'allow_custom': True, 'id': f"opinion--{opinion['uuid']}", + 'labels': ['misp:context-layer="Analyst Opinion"'], 'opinion': self._parse_opinion_level(opinion_value), - 'x_misp_opinion': opinion_value + 'object_refs': [stix_object['id']], 'x_misp_opinion': opinion_value, + **dict(self._handle_analyst_time_fields(stix_object, opinion)) } + if opinion.get('authors'): + opinion_args['authors'] = [opinion['authors']] if opinion.get('comment'): opinion_args['explanation'] = opinion['comment'] - getattr(self, self._results_handling_function)(Opinion(**opinion_args)) + if stix_object['id'].startswith('x-misp--'): + opinion_args['allow_custom'] = True + getattr(self, self._results_handling_function)( + self._create_opinion(opinion_args) + ) def _handle_opinion_object(self, sighting: dict, reference_id: str): opinion_args = { @@ -284,7 +216,9 @@ def _handle_opinion_object(self, sighting: dict, reference_id: str): ) } ) - getattr(self, self._results_handling_function)(Opinion(**opinion_args)) + getattr(self, self._results_handling_function)( + self._create_opinion(opinion_args) + ) def _handle_unpublished_report(self, report_args: dict) -> Grouping: grouping_id = f"grouping--{self._misp_event['uuid']}" @@ -293,12 +227,13 @@ def _handle_unpublished_report(self, report_args: dict) -> Grouping: report_args.update( { 'id': grouping_id, 'type': 'grouping', - 'context': 'suspicious-activity', - 'object_refs': self.object_refs, 'allow_custom': True + 'context': 'suspicious-activity', 'allow_custom': True } ) - self._handle_analyst_data(grouping_id) - return Grouping(**report_args) + self._handle_analyst_data(report_args) + report_args['object_refs'] = self.object_refs + grouping = Grouping(**report_args) + return grouping ############################################################################ # ATTRIBUTES PARSING FUNCTIONS # @@ -801,8 +736,9 @@ def _parse_annotation_object( values[0] if isinstance(values, list) and len(values) == 1 else values ) - self._append_SDO(self._create_note(note_args)) - self._handle_object_analyst_data(misp_object, note_id) + note = self._create_note(note_args) + getattr(self, self._results_handling_function)(note) + self._handle_object_analyst_data(note, misp_object) def _parse_asn_object_observable( self, misp_object: Union[MISPObject, dict]): @@ -1067,8 +1003,9 @@ def _parse_geolocation_object(self, misp_object: Union[MISPObject, dict]): location_args[feature] = attributes.pop(key) if attributes: location_args.update(self._handle_observable_properties(attributes)) - self._append_SDO(self._create_location(location_args)) - self._handle_object_analyst_data(misp_object, location_id) + location = self._create_location(location_args) + getattr(self, self._results_handling_function)(location) + self._handle_object_analyst_data(location, misp_object) def _parse_http_request_object_observable( self, misp_object: Union[MISPObject, dict]): @@ -1758,11 +1695,18 @@ def _create_note(note_args: dict) -> Note: note_args['allow_custom'] = True return Note(**note_args) - def _create_observed_data(self, args: dict, observables: list): + def _create_observed_data( + self, args: dict, observables: list) -> ObservedData: args['object_refs'] = [observable.id for observable in observables] - getattr(self, self._results_handling_function)(ObservedData(**args)) + observed_data = ObservedData(**args) + getattr(self, self._results_handling_function)(observed_data) for observable in observables: getattr(self, self._results_handling_function)(observable) + return observed_data + + @staticmethod + def _create_opinion(opinion_args: dict) -> Opinion: + return Opinion(**opinion_args) @staticmethod def _create_PE_extension(extension_args: dict) -> WindowsPEBinaryExt: diff --git a/misp_stix_converter/misp_stix_converter.py b/misp_stix_converter/misp_stix_converter.py index 08918c7..bca92e1 100644 --- a/misp_stix_converter/misp_stix_converter.py +++ b/misp_stix_converter/misp_stix_converter.py @@ -14,9 +14,9 @@ from .misp2stix.misp_to_stix21 import MISPtoSTIX21Parser from .misp2stix.stix1_mapping import NS_DICT, SCHEMALOC_DICT from .stix2misp.external_stix1_to_misp import ExternalSTIX1toMISPParser -from .stix2misp.external_stix2_to_misp import ( - ExternalSTIX2toMISPParser, MISP_org_uuid) -from .stix2misp.importparser import _load_stix2_content +from .stix2misp.external_stix2_to_misp import ExternalSTIX2toMISPParser +from .stix2misp.importparser import ( + _load_stix1_package, _load_stix2_content, MISP_org_uuid) from .stix2misp.internal_stix1_to_misp import InternalSTIX1toMISPParser from .stix2misp.internal_stix2_to_misp import InternalSTIX2toMISPParser from collections import defaultdict @@ -639,31 +639,96 @@ def misp_to_stix2(filename: _files_type, debug: Optional[bool] = False, # STIX to MISP MAIN FUNCTIONS. # ################################################################################ -def stix_1_to_misp( - filename: _files_type, output_filename: Optional[_files_type]=None): - event = _load_stix_event(filename) - if isinstance(event, str): - return event - title = event.stix_header.title - from_misp = ( - title is not None and - all(feature in title for feature in ('Export from ', 'MISP')) - ) - stix_parser = ( - InternalSTIX1toMISPParser() if from_misp - else ExternalSTIX1toMISPParser() +def stix_1_to_misp(filename: _files_type, + cluster_distribution: Optional[int] = 0, + cluster_sharing_group_id: Optional[int] = None, + debug: Optional[bool] = False, + distribution: Optional[int] = 0, + galaxies_as_tags: Optional[bool] = False, + organisation_uuid: Optional[str] = MISP_org_uuid, + output_dir: Optional[_files_type]=None, + output_name: Optional[_files_type]=None, + producer: Optional[str] = None, + sharing_group_id: Optional[int] = None, + single_event: Optional[bool] = False, + title: Optional[str] = None) -> dict: + if isinstance(filename, str): + filename = Path(filename).resolve() + try: + stix_package = _load_stix1_package(filename) + except Exception as error: + return {'errors': [f'{filename} - {error.__str__()}']} + parser, args = _get_stix1_parser( + _is_stix1_from_misp(stix_package), distribution, sharing_group_id, + title, producer, galaxies_as_tags, single_event, + organisation_uuid, cluster_distribution, cluster_sharing_group_id ) - stix_parser.load_event() - stix_parser.build_misp_event(event) - if output_filename is None: - output_filename = f'{filename}.out' - with open(output_filename, 'wt', encoding='utf-8') as f: - f.write(stix_parser.misp_event.to_json(indent=4)) - return 1 + stix_parser = parser() + stix_parser.load_stix_package(stix_package) + stix_parser.parse_stix_package(**args) + if output_dir is None: + output_dir = filename.parent + if stix_parser.single_event: + name = _check_filename( + filename.parent, f'{filename.name}.out', output_dir, output_name + ) + with open(name, 'wt', encoding='utf-8') as f: + f.write(stix_parser.misp_event.to_json(indent=4)) + return _generate_traceback(debug, stix_parser, name) + output_names = [] + for misp_event in stix_parser.misp_events: + output = output_dir / f'{filename.name}.{misp_event.uuid}.misp.out' + with open(output, 'wt', encoding='utf-8') as f: + f.write(misp_event.to_json(indent=4)) + output_names.append(output) + return _generate_traceback(debug, stix_parser, *output_names) -def stix1_to_misp_instance(): - return +def stix1_to_misp_instance(misp: PyMISP, filename: _files_type, + cluster_distribution: Optional[int] = 0, + cluster_sharing_group_id: Optional[int] = None, + debug: Optional[bool] = False, + distribution: Optional[int] = 0, + galaxies_as_tags: Optional[bool] = False, + organisation_uuid: Optional[str] = MISP_org_uuid, + producer: Optional[str] = None, + sharing_group_id: Optional[int] = None, + single_event: Optional[bool] = False, + title: Optional[str] = None) -> dict: + if isinstance(filename, str): + filename = Path(filename).resolve() + try: + stix_package = _load_stix1_package(filename) + except Exception as error: + return {'errors': [f'{filename} - {error.__str__()}']} + parser, args = _get_stix1_parser( + _is_stix1_from_misp(stix_package), distribution, sharing_group_id, + title, producer, galaxies_as_tags, single_event, + organisation_uuid, cluster_distribution, cluster_sharing_group_id + ) + stix_parser = parser() + stix_parser.load_stix_package(stix_package) + stix_parser.parse_stix_package(**args) + if stix_parser.single_event: + misp_event = misp.add_event(stix_parser.misp_event, pythonify=True) + if not isinstance(misp_event, MISPEvent): + return _generate_traceback( + debug, stix_parser, errors={ + stix_parser.misp_event.uuid: misp_event['errors'][1]['message'] + } + ) + return _generate_traceback(debug, stix_parser, misp_event.id) + event_ids = [] + errors = {} + for event in stix_parser.misp_events: + misp_event = misp.add_event(event, pythonify=True) + if not isinstance(misp_event, MISPEvent): + errors[event.uuid] = misp_event['errors'][1]['message'] + continue + event_ids.append(misp_event.id) + return _generate_traceback( + debug, stix_parser, *event_ids, errors=list(errors) + ) def stix_2_to_misp(filename: _files_type, @@ -686,13 +751,13 @@ def stix_2_to_misp(filename: _files_type, except Exception as error: return {'errors': [f'{filename} - {error.__str__()}']} parser, args = _get_stix2_parser( - _from_misp(bundle.objects), distribution, sharing_group_id, - title, producer, galaxies_as_tags, organisation_uuid, - cluster_distribution, cluster_sharing_group_id + _is_stix2_from_misp(bundle.objects), distribution, sharing_group_id, + title, producer, galaxies_as_tags, single_event, + organisation_uuid, cluster_distribution, cluster_sharing_group_id ) - stix_parser = parser(*args) + stix_parser = parser() stix_parser.load_stix_bundle(bundle) - stix_parser.parse_stix_bundle(single_event) + stix_parser.parse_stix_bundle(**args) if output_dir is None: output_dir = filename.parent if stix_parser.single_event: @@ -711,18 +776,17 @@ def stix_2_to_misp(filename: _files_type, return _generate_traceback(debug, stix_parser, *output_names) -def stix2_to_misp_instance( - misp: PyMISP, filename: _files_type, - cluster_distribution: Optional[int] = 0, - cluster_sharing_group_id: Optional[int] = None, - debug: Optional[bool] = False, - distribution: Optional[int] = 0, - galaxies_as_tags: Optional[bool] = False, - organisation_uuid: Optional[str] = MISP_org_uuid, - producer: Optional[str] = None, - sharing_group_id: Optional[int] = None, - single_event: Optional[bool] = False, - title: Optional[str] = None) -> dict: +def stix2_to_misp_instance(misp: PyMISP, filename: _files_type, + cluster_distribution: Optional[int] = 0, + cluster_sharing_group_id: Optional[int] = None, + debug: Optional[bool] = False, + distribution: Optional[int] = 0, + galaxies_as_tags: Optional[bool] = False, + organisation_uuid: Optional[str] = MISP_org_uuid, + producer: Optional[str] = None, + sharing_group_id: Optional[int] = None, + single_event: Optional[bool] = False, + title: Optional[str] = None) -> dict: if isinstance(filename, str): filename = Path(filename).resolve() try: @@ -730,13 +794,13 @@ def stix2_to_misp_instance( except Exception as error: return {'errors': [f'{filename} - {error.__str__()}']} parser, args = _get_stix2_parser( - _from_misp(bundle.objects), distribution, sharing_group_id, - title, producer, galaxies_as_tags, organisation_uuid, - cluster_distribution, cluster_sharing_group_id + _is_stix2_from_misp(bundle.objects), distribution, sharing_group_id, + title, producer, galaxies_as_tags, single_event, + organisation_uuid, cluster_distribution, cluster_sharing_group_id ) - stix_parser = parser(*args) + stix_parser = parser() stix_parser.load_stix_bundle(bundle) - stix_parser.parse_stix_bundle(single_event) + stix_parser.parse_stix_bundle(**args) if stix_parser.single_event: misp_event = misp.add_event(stix_parser.misp_event, pythonify=True) if not isinstance(misp_event, MISPEvent): @@ -763,7 +827,67 @@ def stix2_to_misp_instance( # STIX CONTENT LOADING FUNCTIONS # ################################################################################ -def _from_misp(stix_objects): +def _get_stix1_parser(from_misp: bool, distribution: int, + sharing_group_id: Union[int, None], + title: Union[str, None], producer: Union[str, None], + galaxies_as_tags: bool, single_event: bool, + organisation_uuid: str, cluster_distribution: int, + cluster_sharing_group_id: Union[int, None]) -> tuple: + args = { + 'distribution': distribution, + 'galaxies_as_tags': galaxies_as_tags, + 'producer': producer, + 'sharing_group_id': sharing_group_id, + 'single_event': single_event, + 'title': title + } + if from_misp: + return InternalSTIX1toMISPParser, args + args.update( + { + 'cluster_distribution': cluster_distribution, + 'cluster_sharing_group_id': cluster_sharing_group_id, + 'organisation_uuid': organisation_uuid + } + ) + return ExternalSTIX1toMISPParser, args + + +def _get_stix2_parser(from_misp: bool, distribution: int, + sharing_group_id: Union[int, None], + title: Union[str, None], producer: Union[str, None], + galaxies_as_tags: bool, single_event: bool, + organisation_uuid: str, cluster_distribution: int, + cluster_sharing_group_id: Union[int, None]) -> tuple: + args = { + 'distribution': distribution, + 'galaxies_as_tags': galaxies_as_tags, + 'producer': producer, + 'sharing_group_id': sharing_group_id, + 'single_event': single_event, + 'title': title + } + if from_misp: + return InternalSTIX2toMISPParser, args + args.update( + { + 'cluster_distribution': cluster_distribution, + 'cluster_sharing_group_id': cluster_sharing_group_id, + 'organisation_uuid': organisation_uuid + } + ) + return ExternalSTIX2toMISPParser, args + + +def _is_stix1_from_misp(stix_package: STIXPackage) -> bool: + try: + title = stix_package.stix_header.title + except AttributeError: + return False + return 'Export from ' in title and 'MISP' in title + + +def _is_stix2_from_misp(stix_objects: list): for stix_object in stix_objects: labels = stix_object.get('labels', []) if stix_object['type'] not in _STIX2_event_types or not labels: @@ -773,12 +897,6 @@ def _from_misp(stix_objects): return False -def _get_stix2_parser(from_misp: bool, *args: tuple) -> tuple: - if from_misp: - return InternalSTIX2toMISPParser, args[:-3] - return ExternalSTIX2toMISPParser, args - - def _load_stix_event(filename, tries=0): try: return STIXPackage.from_xml(filename) diff --git a/misp_stix_converter/stix2misp/__init__.py b/misp_stix_converter/stix2misp/__init__.py index b4f7633..f9f09f3 100644 --- a/misp_stix_converter/stix2misp/__init__.py +++ b/misp_stix_converter/stix2misp/__init__.py @@ -1,7 +1,7 @@ from .external_stix1_to_misp import ExternalSTIX1toMISPParser # noqa from .external_stix2_mapping import ExternalSTIX2toMISPMapping # noqa -from .external_stix2_to_misp import ExternalSTIX2toMISPParser, MISP_org_uuid # noqa -from .importparser import _load_stix2_content # noqa +from .external_stix2_to_misp import ExternalSTIX2toMISPParser # noqa +from .importparser import _load_stix2_content, MISP_org_uuid # noqa from .internal_stix1_to_misp import InternalSTIX1toMISPParser # noqa from .internal_stix2_mapping import InternalSTIX2toMISPMapping # noqa from .internal_stix2_to_misp import InternalSTIX2toMISPParser # noqa diff --git a/misp_stix_converter/stix2misp/converters/stix2_attack_pattern_converter.py b/misp_stix_converter/stix2misp/converters/stix2_attack_pattern_converter.py index bb0b79c..45cbaa5 100644 --- a/misp_stix_converter/stix2misp/converters/stix2_attack_pattern_converter.py +++ b/misp_stix_converter/stix2misp/converters/stix2_attack_pattern_converter.py @@ -116,7 +116,11 @@ def parse(self, attack_pattern_ref: str): try: parser(attack_pattern) except Exception as exception: - self.main_parser._attack_pattern_error(attack_pattern.id, exception) + _traceback = self.main_parser._parse_traceback(exception) + self.main_parser._add_error( + 'Error parsing the Attack Pattern object with id ' + f'{attack_pattern.id}: {_traceback}' + ) def _create_cluster(self, attack_pattern: _ATTACK_PATTERN_TYPING, description: Optional[str] = None, diff --git a/misp_stix_converter/stix2misp/converters/stix2_course_of_action_converter.py b/misp_stix_converter/stix2misp/converters/stix2_course_of_action_converter.py index 458e145..32072c3 100644 --- a/misp_stix_converter/stix2misp/converters/stix2_course_of_action_converter.py +++ b/misp_stix_converter/stix2misp/converters/stix2_course_of_action_converter.py @@ -87,8 +87,10 @@ def parse(self, course_of_action_ref: str): try: parser(course_of_action) except Exception as exception: - self.main_parser._course_of_action_error( - course_of_action.id, exception + _traceback = self.main_parser._parse_traceback(exception) + self.main_parser._add_error( + 'Error parsing the Course of Action object with id ' + f'{course_of_action.id}: {_traceback}' ) def _create_cluster(self, course_of_action: _COURSE_OF_ACTION_TYPING, @@ -121,4 +123,4 @@ def _parse_course_of_action_object( ) for attribute in self._generic_parser(course_of_action): misp_object.add_attribute(**attribute) - self.main_parser._add_misp_object(misp_object, course_of_action) \ No newline at end of file + self.main_parser._add_misp_object(misp_object, course_of_action) diff --git a/misp_stix_converter/stix2misp/converters/stix2_custom_object_converter.py b/misp_stix_converter/stix2misp/converters/stix2_custom_object_converter.py index 24c964e..47abe9a 100644 --- a/misp_stix_converter/stix2misp/converters/stix2_custom_object_converter.py +++ b/misp_stix_converter/stix2misp/converters/stix2_custom_object_converter.py @@ -53,7 +53,11 @@ def parse(self, custom_ref: str): try: parser(custom_object) except Exception as exception: - self.main_parser._custom_object_error(custom_object.id, exception) + _traceback = self.main_parser._parse_traceback(exception) + self.main_parser._add_error( + 'Error parsing the Custom object with id ' + f'{custom_object.id}: {_traceback}' + ) def _parse_custom_attribute(self, custom_attribute: _CUSTOM_OBJECT_TYPING): attribute = { diff --git a/misp_stix_converter/stix2misp/converters/stix2_identity_converter.py b/misp_stix_converter/stix2misp/converters/stix2_identity_converter.py index 1ee282f..48439bc 100644 --- a/misp_stix_converter/stix2misp/converters/stix2_identity_converter.py +++ b/misp_stix_converter/stix2misp/converters/stix2_identity_converter.py @@ -322,7 +322,11 @@ def parse(self, identity_ref: str): try: parser(identity) except Exception as exception: - self.main_parser._identity_error(identity.id, exception) + _traceback = self.main_parser._parse_traceback(exception) + self.main_parser._add_error( + 'Error while parsing the Identity object with id ' + f'{identity.id}: {_traceback}' + ) def _parse_employee_object(self, identity: _IDENTITY_TYPING): misp_object = self._create_misp_object('employee', identity) diff --git a/misp_stix_converter/stix2misp/converters/stix2_indicator_converter.py b/misp_stix_converter/stix2misp/converters/stix2_indicator_converter.py index da10f64..94a597f 100644 --- a/misp_stix_converter/stix2misp/converters/stix2_indicator_converter.py +++ b/misp_stix_converter/stix2misp/converters/stix2_indicator_converter.py @@ -19,14 +19,18 @@ from abc import ABCMeta from collections import defaultdict from pymisp import MISPObject -from stix2.v21.sdo import Indicator +from stix2.v20.sdo import Indicator as Indicator_v20 +from stix2.v21.sdo import Indicator as Indicator_v21 from stix2patterns.inspector import _PatternData as PatternData +from types import GeneratorType from typing import TYPE_CHECKING, Tuple, Union if TYPE_CHECKING: from ..external_stix2_to_misp import ExternalSTIX2toMISPParser from ..internal_stix2_to_misp import InternalSTIX2toMISPParser +_INDICATOR_TYPING = Union[Indicator_v20, Indicator_v21] + class STIX2IndicatorMapping(STIX2Mapping, metaclass=ABCMeta): # SINGLE ATTRIBUTES MAPPING @@ -308,8 +312,8 @@ def parse(self, indicator_ref: str): try: parser(indicator) except UnknownPatternMappingError as error: - self.main_parser._unknown_pattern_mapping_warning( - indicator.id, error.__str__() + self._unknown_pattern_mapping_warning( + indicator.id, error.__str__().split('_') ) self._create_stix_pattern_object(indicator) except InvalidSTIXPatternError as error: @@ -352,7 +356,7 @@ def _compile_stix_pattern( return self._pattern_parser.pattern def _handle_pattern_mapping(self, indicator: _INDICATOR_TYPING) -> str: - if isinstance(indicator, Indicator): + if isinstance(indicator, Indicator_v21): pattern_type = indicator.pattern_type if pattern_type != 'stix': try: @@ -367,6 +371,14 @@ def _handle_pattern_mapping(self, indicator: _INDICATOR_TYPING) -> str: return '_create_stix_pattern_object' return '_parse_stix_pattern' + # Errors handlin + def _no_converted_content_from_pattern_warning( + self, indicator: _INDICATOR_TYPING): + self.main_parser._add_warning( + "No content extracted from the following Indicator's (id: " + f'{indicator.id}) pattern: {indicator.pattern}' + ) + ############################################################################ # INDICATORS PARSING METHODS # ############################################################################ @@ -380,7 +392,7 @@ def _parse_asn_pattern( field = keys[0] mapping = self._mapping.asn_pattern_mapping(field) if mapping is None: - self.main_parser._unmapped_pattern_warning(indicator.id, field) + self._unmapped_pattern_warning(indicator.id, field) continue if not isinstance(values, tuple): attributes.append( @@ -405,9 +417,7 @@ def _parse_asn_pattern( if assertion not in self._mapping.valid_pattern_assertions(): continue if keys[0] != 'value': - self.main_parser._unmapped_pattern_warning( - indicator.id, '.'.join(keys) - ) + self._unmapped_pattern_warning(indicator.id, '.'.join(keys)) continue if isinstance(values, tuple): for value in values: @@ -417,9 +427,7 @@ def _parse_asn_pattern( if 'asn' in (attr['object_relation'] for attr in attributes): self._handle_import_case(indicator, attributes, 'asn') else: - self.main_parser._no_converted_content_from_pattern_warning( - indicator - ) + self._no_converted_content_from_pattern_warning(indicator) self._create_stix_pattern_object(indicator) def _parse_directory_pattern( @@ -430,9 +438,7 @@ def _parse_directory_pattern( continue mapping = self._mapping.directory_pattern_mapping(keys[0]) if mapping is None: - self.main_parser._unmapped_pattern_warning( - indicator.id, '.'.join(keys) - ) + self._unmapped_pattern_warning(indicator.id, '.'.join(keys)) continue if isinstance(values, tuple): for value in values: @@ -442,9 +448,7 @@ def _parse_directory_pattern( if misp_object.attributes: self.main_parser._add_misp_object(misp_object, indicator) else: - self.main_parser._no_converted_content_from_pattern_warning( - indicator - ) + self._no_converted_content_from_pattern_warning(indicator) self._create_stix_pattern_object(indicator) def _parse_domain_ip_port_pattern( @@ -459,9 +463,7 @@ def _parse_domain_ip_port_pattern( if assertion not in self._mapping.valid_pattern_assertions(): continue if keys[0] != 'value': - self.main_parser._unmapped_pattern_warning( - indicator.id, '.'.join(keys) - ) + self._unmapped_pattern_warning(indicator.id, '.'.join(keys)) continue if isinstance(values, tuple): for value in values: @@ -469,7 +471,7 @@ def _parse_domain_ip_port_pattern( else: attributes.append({'value': values, **mapping}) if any(key not in features for key in pattern.comparisons.keys()): - self.main_parser._unknown_pattern_mapping_warning( + self._unknown_pattern_mapping_warning( indicator.id, ( key for key in pattern.comparisons.keys() @@ -482,9 +484,7 @@ def _parse_domain_ip_port_pattern( 'first-seen', 'last-seen' ) else: - self.main_parser._no_converted_content_from_pattern_warning( - indicator - ) + self._no_converted_content_from_pattern_warning(indicator) self._create_stix_pattern_object(indicator) def _parse_email_address_pattern( @@ -495,9 +495,7 @@ def _parse_email_address_pattern( continue mapping = self._mapping.email_address_pattern_mapping(keys[0]) if mapping is None: - self.main_parser._unmapped_pattern_warning( - indicator.id, '.'.join(keys) - ) + self._unmapped_pattern_warning(indicator.id, '.'.join(keys)) continue if isinstance(values, tuple): for value in values: @@ -507,9 +505,7 @@ def _parse_email_address_pattern( if attributes: self._handle_import_case(indicator, attributes, 'email') else: - self.main_parser._no_converted_content_from_pattern_warning( - indicator - ) + self._no_converted_content_from_pattern_warning(indicator) self._create_stix_pattern_object(indicator) def _parse_email_message_pattern( @@ -521,7 +517,7 @@ def _parse_email_message_pattern( field = '.'.join(keys) if len(keys) > 1 else keys[0] mapping = self._mapping.email_message_mapping(field) if mapping is None: - self.main_parser._unmapped_pattern_warning(indicator.id, field) + self._unmapped_pattern_warning(indicator.id, field) continue if isinstance(values, tuple): for value in values: @@ -534,9 +530,7 @@ def _parse_email_message_pattern( 'bcc', 'cc', 'to' ) else: - self.main_parser._no_converted_content_from_pattern_warning( - indicator - ) + self._no_converted_content_from_pattern_warning(indicator) self._create_stix_pattern_object(indicator) def _parse_file_and_pe_pattern( @@ -554,7 +548,7 @@ def _parse_file_and_pe_pattern( _, _, _, index, _, hash_type = keys mapping = self._mapping.file_hashes_mapping(hash_type) if mapping is None: - self.main_parser._unmapped_pattern_warning( + self._unmapped_pattern_warning( indicator.id, '.'.join(keys) ) continue @@ -571,7 +565,7 @@ def _parse_file_and_pe_pattern( _, _, _, index, feature = keys mapping = self._mapping.pe_section_pattern_mapping(feature) if mapping is None: - self.main_parser._unmapped_pattern_warning( + self._unmapped_pattern_warning( indicator.id, '.'.join(keys) ) continue @@ -605,9 +599,7 @@ def _parse_file_and_pe_pattern( **{'value': values, **attribute} ) continue - self.main_parser._unmapped_pattern_warning( - indicator.id, '.'.join(keys) - ) + self._unmapped_pattern_warning(indicator.id, '.'.join(keys)) continue file_attributes = self._parse_file_attribute( keys, values, indicator.id @@ -635,9 +627,7 @@ def _parse_file_and_pe_pattern( if file_object.attributes: self.main_parser._add_misp_object(file_object, indicator) else: - self.main_parser._no_converted_content_from_pattern_warning( - indicator - ) + self._no_converted_content_from_pattern_warning(indicator) self._create_stix_pattern_object(indicator) def _parse_file_attribute(self, keys: list, values: Union[str, tuple], @@ -653,9 +643,7 @@ def _parse_file_attribute(self, keys: list, values: Union[str, tuple], else: yield {'value': values, **mapping} else: - self.main_parser._unmapped_pattern_warning( - indicator_id, '.'.join(keys) - ) + self._unmapped_pattern_warning(indicator_id, '.'.join(keys)) def _parse_file_pattern( self, pattern: PatternData, indicator: _INDICATOR_TYPING): @@ -678,9 +666,7 @@ def _parse_file_pattern( 'file-encoding', 'fullpath', 'modification-time', 'path' ) else: - self.main_parser._no_converted_content_from_pattern_warning( - indicator - ) + self._no_converted_content_from_pattern_warning(indicator) self._create_stix_pattern_object(indicator) def _parse_ip_address_pattern( @@ -692,7 +678,7 @@ def _parse_ip_address_pattern( if assertion not in self._mapping.valid_pattern_assertions(): continue if keys[0] != 'value': - self.main_parser._unmapped_pattern_warning( + self._unmapped_pattern_warning( indicator.id, '.'.join(keys) ) continue @@ -708,9 +694,7 @@ def _parse_ip_address_pattern( if attributes: self._handle_import_case(indicator, attributes, 'ip-port') else: - self.main_parser._no_converted_content_from_pattern_warning( - indicator - ) + self._no_converted_content_from_pattern_warning(indicator) self._create_stix_pattern_object(indicator) def _parse_mutex_pattern( @@ -733,9 +717,7 @@ def _parse_mutex_pattern( if attributes: self._handle_import_case(indicator, attributes, 'mutex', 'name') else: - self.main_parser._no_converted_content_from_pattern_warning( - indicator - ) + self._no_converted_content_from_pattern_warning(indicator) self._create_stix_pattern_object(indicator) def _parse_network_connection_pattern( @@ -765,9 +747,7 @@ def _parse_network_connection_pattern( if misp_object.attributes: self.main_parser._add_misp_object(misp_object, indicator) else: - self.main_parser._no_converted_content_from_pattern_warning( - indicator - ) + self._no_converted_content_from_pattern_warning(indicator) self._create_stix_pattern_object(indicator) def _parse_network_socket_pattern( @@ -781,9 +761,7 @@ def _parse_network_socket_pattern( keys[-1] ) if mapping is None: - self.main_parser._unmapped_pattern_warning( - indicator.id, '.'.join(keys) - ) + self._unmapped_pattern_warning(indicator.id, '.'.join(keys)) continue if isinstance(values, tuple): for value in values: @@ -824,9 +802,7 @@ def _parse_network_traffic_attribute( return mapping = getattr(self._mapping, f'{name}_pattern_mapping')(field) if mapping is None: - self.main_parser._unmapped_pattern_warning( - indicator_id, '.'.join(keys) - ) + self._unmapped_pattern_warning(indicator_id, '.'.join(keys)) return if isinstance(values, tuple): for value in values: @@ -861,9 +837,7 @@ def _parse_process_pattern( continue mapping = self._mapping.process_pattern_mapping(keys[0]) if mapping is None: - self.main_parser._unmapped_pattern_warning( - indicator.id, '.'.join(keys) - ) + self._unmapped_pattern_warning(indicator.id, '.'.join(keys)) continue if isinstance(values, tuple): for value in values: @@ -876,9 +850,7 @@ def _parse_process_pattern( 'args', 'command-line', 'current-directory', 'name', 'pid' ) else: - self.main_parser._no_converted_content_from_pattern_warning( - indicator - ) + self._no_converted_content_from_pattern_warning(indicator) self._create_stix_pattern_object(indicator) def _parse_registry_key_pattern( @@ -891,9 +863,7 @@ def _parse_registry_key_pattern( keys[-1 if 'values' in keys else 0] ) if mapping is None: - self.main_parser._unmapped_pattern_warning( - indicator.id, '.'.join(keys) - ) + self._unmapped_pattern_warning(indicator.id, '.'.join(keys)) continue if isinstance(values, tuple): for value in values: @@ -906,12 +876,10 @@ def _parse_registry_key_pattern( 'data', 'data-type', 'name' ) else: - self.main_parser._no_converted_content_from_pattern_warning( - indicator - ) + self._no_converted_content_from_pattern_warning(indicator) self._create_stix_pattern_object(indicator) - def _parse_sigma_pattern(self, indicator: Indicator): + def _parse_sigma_pattern(self, indicator: _INDICATOR_TYPING): if hasattr(indicator, 'name') or hasattr(indicator, 'external_references'): attributes = [] for field, mapping in self._mapping.sigma_object_mapping().items(): @@ -963,7 +931,7 @@ def _parse_sigma_pattern(self, indicator: Indicator): indicator ) - def _parse_snort_pattern(self, indicator: Indicator): + def _parse_snort_pattern(self, indicator: _INDICATOR_TYPING): self.main_parser._add_misp_attribute( { 'value': indicator.pattern, @@ -981,9 +949,7 @@ def _parse_software_pattern( continue mapping = self._mapping.software_pattern_mapping(keys[0]) if mapping is None: - self.main_parser._unmapped_pattern_warning( - indicator.id, '.'.join(keys) - ) + self._unmapped_pattern_warning(indicator.id, '.'.join(keys)) continue if isinstance(values, tuple): for value in values: @@ -993,9 +959,7 @@ def _parse_software_pattern( if attributes: self._handle_object_case(indicator, attributes, 'software') else: - self.main_parser._no_converted_content_from_pattern_warning( - indicator - ) + self._no_converted_content_from_pattern_warning(indicator) self._create_stix_pattern_object(indicator) def _parse_stix_pattern(self, indicator: _INDICATOR_TYPING): @@ -1011,7 +975,7 @@ def _parse_stix_pattern(self, indicator: _INDICATOR_TYPING): raise UnknownParsingFunctionError(feature) parser(compiled_pattern, indicator) - def _parse_suricata_pattern(self, indicator: Indicator): + def _parse_suricata_pattern(self, indicator: _INDICATOR_TYPING): misp_object = self._create_misp_object('suricata', indicator) for feature, mapping in self._mapping.suricata_object_mapping().items(): if hasattr(indicator, feature): @@ -1032,9 +996,7 @@ def _parse_url_pattern( if assertion not in self._mapping.valid_pattern_assertions(): continue if keys[0] != 'value': - self.main_parser._unmapped_pattern_warning( - indicator.id, '.'.join(keys) - ) + self._unmapped_pattern_warning(indicator.id, '.'.join(keys)) continue if isinstance(values, tuple): for value in values: @@ -1046,16 +1008,14 @@ def _parse_url_pattern( {'value': values, **self._mapping.url_attribute()} ) if any(key != 'url' for key in pattern.comparisons.keys()): - self.main_parser._unknown_pattern_mapping_warning( + self._unknown_pattern_mapping_warning( indicator.id, (key for key in pattern.comparisons.keys() if key != 'url') ) if attributes: self._handle_import_case(indicator, attributes, 'url') else: - self.main_parser._no_converted_content_from_pattern_warning( - indicator - ) + self._no_converted_content_from_pattern_warning(indicator) self._create_stix_pattern_object(indicator) def _parse_user_account_pattern( @@ -1068,9 +1028,7 @@ def _parse_user_account_pattern( keys[-1 if 'unix-account-ext' in keys else 0] ) if mapping is None: - self.main_parser._unmapped_pattern_warning( - indicator.id, '.'.join(keys) - ) + self._unmapped_pattern_warning(indicator.id, '.'.join(keys)) continue if isinstance(values, tuple): for value in values: @@ -1080,9 +1038,7 @@ def _parse_user_account_pattern( if attributes: self._handle_object_case(indicator, attributes, 'user-account') else: - self.main_parser._no_converted_content_from_pattern_warning( - indicator - ) + self._no_converted_content_from_pattern_warning(indicator) self._create_stix_pattern_object(indicator) def _parse_x509_pattern( @@ -1095,9 +1051,7 @@ def _parse_x509_pattern( keys[1 if 'hashes' in keys else 0] ) if mapping is None: - self.main_parser._unmapped_pattern_warning( - indicator.id, '.'.join(keys) - ) + self._unmapped_pattern_warning(indicator.id, '.'.join(keys)) continue if isinstance(values, tuple): for value in values: @@ -1113,10 +1067,10 @@ def _parse_x509_pattern( 'validity-not-before', 'version' ) else: - self.main_parser._no_converted_content_from_pattern_warning(indicator) + self._no_converted_content_from_pattern_warning(indicator) self._create_stix_pattern_object(indicator) - def _parse_yara_pattern(self, indicator: Indicator): + def _parse_yara_pattern(self, indicator: _INDICATOR_TYPING): if hasattr(indicator, 'pattern_version'): misp_object = self._create_misp_object('yara', indicator) for feature, mapping in self._mapping.yara_object_mapping().items(): @@ -1146,6 +1100,23 @@ def _parse_yara_pattern(self, indicator: Indicator): indicator ) + ############################################################################ + # ERRORS AND WARNINGS HANDLING METHODS # + ############################################################################ + + def _unknown_pattern_mapping_warning( + self, indicator_id: str, pattern_types: GeneratorType): + self._add_warning( + f'Unable to map pattern from the Indicator with id {indicator_id}, ' + f"containing the following types: {', '.join(pattern_types)}" + ) + + def _unmapped_pattern_warning(self, indicator_id: str, feature: str): + self._add_warning( + 'Unmapped pattern part in indicator with id ' + f'{indicator_id}: {feature}' + ) + class InternalSTIX2IndicatorMapping( STIX2IndicatorMapping, InternalSTIX2Mapping): @@ -1476,10 +1447,17 @@ def parse(self, indicator_ref: str): raise UnknownParsingFunctionError(f"{feature}_indicator") try: parser(indicator) - except AttributeFromPatternParsingError as error: - self.main_parser._attribute_from_pattern_parsing_error(error) + except AttributeFromPatternParsingError as indicator_id: + self.main_parser._add_error( + 'Error while parsing pattern from ' + f'indicator with id {indicator_id}' + ) except Exception as exception: - self.main_parser._indicator_error(indicator.id, exception) + _traceback = self.main_parser._parse_traceback(exception) + self.main_parser._add_error( + 'Error while parsing the Indicator object with id ' + f'{indicator.id}: {_traceback}' + ) ############################################################################ # ATTRIBUTES PARSING METHODS # @@ -1562,7 +1540,7 @@ def _attribute_from_malware_sample_indicator( self.main_parser._add_misp_attribute(attribute, indicator) def _attribute_from_patterning_language_indicator( - self, indicator: Indicator): + self, indicator: _INDICATOR_TYPING): attribute = self._create_attribute_dict(indicator) attribute['value'] = indicator.pattern self.main_parser._add_misp_attribute(attribute, indicator) @@ -1981,7 +1959,8 @@ def _object_from_parler_account_indicator( indicator, 'parler-account' ) - def _object_from_patterning_language_indicator(self, indicator: Indicator): + def _object_from_patterning_language_indicator( + self, indicator: _INDICATOR_TYPING): name = ( 'suricata' if indicator.pattern_type == 'snort' else indicator.pattern_type diff --git a/misp_stix_converter/stix2misp/converters/stix2_location_converter.py b/misp_stix_converter/stix2misp/converters/stix2_location_converter.py index ce8ca17..ba4fbee 100644 --- a/misp_stix_converter/stix2misp/converters/stix2_location_converter.py +++ b/misp_stix_converter/stix2misp/converters/stix2_location_converter.py @@ -189,4 +189,8 @@ def parse(self, location_ref: str): try: parser(location) except Exception as exception: - self.main_parser._location_error(location.id, exception) + _traceback = self.main_parser._parse_traceback(exception) + self.main_parser._add_error( + 'Error while parsing the Location object with id ' + f'{location.id}: {_traceback}' + ) diff --git a/misp_stix_converter/stix2misp/converters/stix2_malware_converter.py b/misp_stix_converter/stix2misp/converters/stix2_malware_converter.py index 625217f..1150845 100644 --- a/misp_stix_converter/stix2misp/converters/stix2_malware_converter.py +++ b/misp_stix_converter/stix2misp/converters/stix2_malware_converter.py @@ -94,6 +94,14 @@ def _parse_malware_object(self, malware: _MALWARE_TYPING): sample = feature(sample_ref, malware) sample.add_reference(malware_object.uuid, 'sample-of') + # Error handling + def _malware_error(self, malware_id: str, exception: Exception): + _traceback = self.main_parser._parse_traceback(exception) + self.main_parser._add_error( + 'Error while parsing the Malware object with id ' + f'{malware_id}: {_traceback}' + ) + class ExternalSTIX2MalwareMapping(STIX2MalwareMapping, ExternalSTIX2Mapping): pass @@ -119,7 +127,7 @@ def parse(self, malware_ref: str): else: self._parse_malware_object(malware) except Exception as exception: - self.main_parser._malware_error(malware.id, exception) + self._malware_error(malware.id, exception) def _convert_malware_objects( self, malware: _MALWARE_TYPING) -> Iterator[MISPObject]: @@ -207,7 +215,7 @@ def parse(self, malware_ref: str): try: parser(malware) except Exception as exception: - self.main_parser._malware_error(malware.id, exception) + self._malware_error(malware.id, exception) def _create_cluster(self, malware: _MALWARE_TYPING, description: Optional[str] = None, diff --git a/misp_stix_converter/stix2misp/converters/stix2_observable_converter.py b/misp_stix_converter/stix2misp/converters/stix2_observable_converter.py index 2c73d58..f961e6b 100644 --- a/misp_stix_converter/stix2misp/converters/stix2_observable_converter.py +++ b/misp_stix_converter/stix2misp/converters/stix2_observable_converter.py @@ -275,7 +275,9 @@ def _parse_file_observable( for hash_type, value in observable.hashes.items(): attribute = self._mapping.file_hashes_mapping(hash_type) if attribute is None: - self.main_parser._hash_type_error(hash_type) + self.main_parser._add_error( + f'Wrong hash_type: {hash_type}' + ) continue yield from self._populate_object_attributes( attribute, value, object_id diff --git a/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py b/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py index 31b7fb8..9f9a7af 100644 --- a/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py +++ b/misp_stix_converter/stix2misp/converters/stix2_observed_data_converter.py @@ -65,6 +65,15 @@ def _fetch_observables(self, object_refs: Union[tuple, str]) -> Generator: for object_ref in object_refs: yield self.main_parser._observable[object_ref] + # Errors handling + def _observable_mapping_error( + self, observed_data_id: str, observable_types: Exception): + self.main_parser._add_error( + 'Unable to map observable objects related to the Observed Data ' + f'object with id {observed_data_id} containing the folowing types' + f": {observable_types.__str__().replace('_', ', ')}" + ) + class ExternalSTIX2ObservedDataConverter( STIX2ObservedDataConverter, ExternalSTIX2ObservableConverter): @@ -95,9 +104,7 @@ def parse(self, observed_data_ref: str): else: self._parse_observable_objects(observed_data) except UnknownObservableMappingError as observable_types: - self.main_parser._observable_mapping_error( - observed_data.id, observable_types - ) + self._observable_mapping_error(observed_data.id, observable_types) def parse_relationships(self): for misp_object in self.main_parser.misp_event.objects: @@ -204,9 +211,7 @@ def _parse_multiple_observable_object_refs( object_type = object_ref.split('--')[0] mapping = self._mapping.observable_mapping(object_type) if mapping is None: - self.main_parser._observable_mapping_error( - observed_data.id, object_type - ) + self._observable_mapping_error(observed_data.id, object_type) continue feature = f'_parse_{mapping}_observable_object_refs' try: @@ -255,17 +260,13 @@ def _parse_multiple_observable_objects( observable_objects.update(observables) continue if len(observable_types) == 1: - self.main_parser._observable_mapping_error( - observed_data.id, object_type - ) + self._observable_mapping_error(observed_data.id, object_type) continue mapping = self._mapping.observable_mapping( observed_data.objects[object_id]['type'] ) if mapping is None: - self.main_parser._observable_mapping_error( - observed_data.id, object_type - ) + self._observable_mapping_error(observed_data.id, object_type) continue feature = f'_parse_{mapping}_observable_objects' try: @@ -2256,9 +2257,7 @@ def parse(self, observed_data_ref: str): try: parser(observed_data) except UnknownObservableMappingError as observable_types: - self.main_parser._observable_mapping_error( - observed_data.id, observable_types - ) + self._observable_mapping_error(observed_data.id, observable_types) ############################################################################ # ATTRIBUTES PARSING METHODS # diff --git a/misp_stix_converter/stix2misp/converters/stix2_threat_actor_converter.py b/misp_stix_converter/stix2misp/converters/stix2_threat_actor_converter.py index c026768..fec3e40 100644 --- a/misp_stix_converter/stix2misp/converters/stix2_threat_actor_converter.py +++ b/misp_stix_converter/stix2misp/converters/stix2_threat_actor_converter.py @@ -104,4 +104,8 @@ def parse(self, threat_actor_ref: str): try: parser(threat_actor) except Exception as exception: - self.main_parser._threat_actor_error(threat_actor.id, exception) + _traceback = self.main_parser._parse_traceback(exception) + self.main_parser._add_error( + 'Error while parsing the Threat Actor object with id ' + f'{threat_actor.id}: {_traceback}' + ) diff --git a/misp_stix_converter/stix2misp/converters/stix2_tool_converter.py b/misp_stix_converter/stix2misp/converters/stix2_tool_converter.py index 756c755..f996fad 100644 --- a/misp_stix_converter/stix2misp/converters/stix2_tool_converter.py +++ b/misp_stix_converter/stix2misp/converters/stix2_tool_converter.py @@ -97,7 +97,11 @@ def parse(self, tool_ref: str): try: parser(tool) except Exception as exception: - self.main_parser._tool_error(tool.id, exception) + _traceback = self.main_parser._parse_traceback(exception) + self.main_parser._add_error( + 'Error while parsing the Tool object with id ' + f'{tool.id}: {_traceback}' + ) def _create_cluster(self, tool: _TOOL_TYPING, description: Optional[str] = None, diff --git a/misp_stix_converter/stix2misp/converters/stix2_vulnerability_converter.py b/misp_stix_converter/stix2misp/converters/stix2_vulnerability_converter.py index 2bf01fb..8791cc2 100644 --- a/misp_stix_converter/stix2misp/converters/stix2_vulnerability_converter.py +++ b/misp_stix_converter/stix2misp/converters/stix2_vulnerability_converter.py @@ -108,7 +108,11 @@ def parse(self, vulnerability_ref: str): try: parser(vulnerability) except Exception as exception: - self.main_parser._vulnerability_error(vulnerability.id, exception) + _traceback = self.main_parser._parse_traceback(exception) + self.main_parser._add_error( + 'Error while parsing the Vulnerability object with id ' + f'{vulnerability.id}: {_traceback}' + ) def _parse_vulnerability_attribute( self, vulnerability: _VULNERABILITY_TYPING): diff --git a/misp_stix_converter/stix2misp/converters/stix2converter.py b/misp_stix_converter/stix2misp/converters/stix2converter.py index f288e0b..c830ab8 100644 --- a/misp_stix_converter/stix2misp/converters/stix2converter.py +++ b/misp_stix_converter/stix2misp/converters/stix2converter.py @@ -212,9 +212,9 @@ def parse(self, stix_object_ref: str): ############################################################################ def _check_existing_galaxy_name(self, stix_object_name: str) -> Union[list, None]: - if stix_object_name in self.synonyms_mapping: - return self.synonyms_mapping[stix_object_name] - for name, tag_names in self.synonyms_mapping.items(): + if stix_object_name in self.main_parser.synonyms_mapping: + return self.main_parser.synonyms_mapping[stix_object_name] + for name, tag_names in self.main_parser.synonyms_mapping.items(): if stix_object_name in name: return tag_names diff --git a/misp_stix_converter/stix2misp/external_stix1_to_misp.py b/misp_stix_converter/stix2misp/external_stix1_to_misp.py index 129e4ad..e848ab7 100644 --- a/misp_stix_converter/stix2misp/external_stix1_to_misp.py +++ b/misp_stix_converter/stix2misp/external_stix1_to_misp.py @@ -1,9 +1,431 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -from .stix1_to_misp import STIX1toMISPParser +from .importparser import ExternalSTIXtoMISPParser +from .stix1_mapping import ExternalSTIX1toMISPMapping +from .stix1_to_misp import StixObjectTypeError, STIX1toMISPParser +from collections import defaultdict +from cybox.core import Observable, Observables +from pymisp.abstract import misp_objects_path +from pymisp import MISPAttribute, MISPEvent, MISPObject +from stix.data_marking import MarkingSpecification +from stix.extensions.marking.ais import AISMarkingStructure +from stix.extensions.marking.tlp import TLPMarkingStructure +from stix.indicator import Indicator +from stix.threat_actor import ThreatActor +from stix.ttp import TTP +from typing import Optional - -class ExternalSTIX1toMISPParser(STIX1toMISPParser): +class ExternalSTIX1toMISPParser(STIX1toMISPParser, ExternalSTIXtoMISPParser): def __init__(self): - super().__init__() \ No newline at end of file + super().__init__() + self._mapping = ExternalSTIX1toMISPMapping + self.__dns_objects = defaultdict(dict) + self.__dns_ips = [] + + def parse_stix_package(self, cluster_distribution: Optional[int] = 0, + cluster_sharing_group_id: Optional[int] = None, + organisation_uuid: Optional[str] = None, **kwargs): + self._set_parameters(**kwargs) + self._set_single_event(True) + self._set_cluster_distribution( + cluster_distribution, cluster_sharing_group_id + ) + self._set_organisation_uuid(organisation_uuid) + self._set_misp_event(MISPEvent()) + if self.stix_package.timestamp: + stix_date = self.stix_package.timestamp + try: + self.misp_event.date = stix_date.date() + except AttributeError: + self.misp_event.date = stix_date + self.misp_event.timestamp = self._timestamp_from_date(stix_date) + self.misp_event.info = self._get_event_info() + header = self.stix_package.stix_header + if getattr(getattr(header, 'description', None), 'value', None): + self.misp_event.add_attribute( + **{ + 'type': 'text', 'value': header.description.value, + 'comment': 'STIX Header Description' + } + ) + if getattr(header, 'handling', None): + for handling in header.handling: + for tag in self._parse_marking(handling): + self.misp_event.add_tag(tag) + if self.stix_package.indicators: + for indicator in self.stix_package.indicators: + if indicator.related_indicators: + for related_indicator in indicator.related_indicators: + self._parse_indicator(related_indicator) + else: + self._parse_indicator(indicator) + if self.stix_package.observables: + self._parse_observables() + if self.stix_package.ttps: + for ttp in self.stix_package.ttps.ttp: + self._parse_ttp(ttp) + if self.stix_package.courses_of_action: + for course_of_action in self.stix_package.courses_of_action: + self._parse_course_of_action(course_of_action) + if self.stix_package.threat_actors: + for threat_actor in self.stix_package.threat_actors: + self._parse_threat_actor(threat_actor) + if self.dns_objects: + for domain in self.dns_objects['domain'].values(): + domain_attribute = domain['data'] + ip_reference = domain['related'] + if ip_reference in self.dns_objects['ip']: + misp_object = MISPObject( + 'passive-dns', misp_objects_path_custom=misp_objects_path + ) + domain_attribute['object_relation'] = "rrname" + misp_object.add_attribute(**domain_attribute) + ip_address = self.dns_objects['ip'][ip_reference]['value'] + misp_object.add_attribute( + **{ + "type": "text", "object_relation": "rdata", + "value": ip_address + } + ) + misp_object.add_attribute( + **{ + 'type': 'text', 'object_relation': 'rrtype', + 'value': "AAAA" if ":" in ip_address else "A" + } + ) + self.misp_event.add_object(misp_object) + else: + self.misp_event.add_attribute(**domain_attribute) + for ip, ip_attribute in self.dns_objects['ip'].items(): + if ip not in self.dns_ips: + self.misp_event.add_attribute(**ip_attribute) + + ############################################################################ + # PROPERTIES # + ############################################################################ + + @property + def dns_ips(self) -> list: + return self.__dns_ips + + @property + def dns_objects(self) -> dict: + return self.__dns_objects + + ############################################################################ + # STIX OBJECTS PARSING METHODS # + ############################################################################ + + def _parse_attributes_from_ttp(self, ttp: TTP, galaxies: set): + attributes = [] + if ttp.resources and getattr(ttp.resources, 'infrastructure', None).observable_characterization: + observables = ttp.resources.infrastructure.observable_characterization + if observables.observables: + for observable in observables.observables: + if not self._has_properties(observable): + continue + properties = observable.object_.properties + try: + attribute_type, attribute_value, _ = self._handle_attribute_type(properties) + except StixObjectTypeError as xsi_type: + self._stix_object_type_error(xsi_type, ttp.id_) + continue + if isinstance(attribute_value, list): + attributes.extend( + {'type': attribute_type, 'value': value, 'to_ids': False} + for value in attribute_value + ) + else: + attributes.append( + { + 'type': attribute_type, + 'value': attribute_value, + 'to_ids': False + } + ) + if ttp.exploit_targets and ttp.exploit_targets.exploit_target: + for exploit_target in ttp.exploit_targets.exploit_target: + if exploit_target.item.vulnerabilities: + for vulnerability in exploit_target.item.vulnerabilities: + if vulnerability.cve_id: + attributes.append( + { + 'type': 'vulnerability', + 'value': vulnerability.cve_id + } + ) + elif vulnerability.title: + title = vulnerability.title + if title in self.synonyms_mapping: + galaxies.update(self.synonyms_mapping[title]) + else: + galaxies.add(f'misp-galaxy:branded-vulnerability="{title}"') + if len(attributes) == 1: + attributes[0].update(self._sanitise_attribute_uuid(ttp.id_)) + return attributes + + def _parse_description(self, stix_object: Indicator | Observable): + if stix_object.description: + misp_attribute = { + 'type': 'text', 'value': stix_object.description.value + } + if stix_object.timestamp: + misp_attribute['timestamp'] = self._timestamp_from_date( + stix_object.timestamp + ) + self.misp_event.add_attribute(**misp_attribute) + + def _parse_galaxies_from_ttp(self, ttp: TTP): + if ttp.behavior: + if ttp.behavior.attack_patterns: + for attack_pattern in ttp.behavior.attack_patterns: + yield from self._parse_galaxy(attack_pattern, 'title', 'misp-attack-pattern') + if ttp.behavior.malware_instances: + for malware_instance in ttp.behavior.malware_instances: + yield from self._parse_galaxy(malware_instance, 'title', 'ransomware') + if ttp.resources and ttp.resources.tools: + for tool in ttp.resources.tools: + yield from self._parse_galaxy(tool, 'name', 'tool') + + def _parse_indicator(self, indicator: Indicator): + if hasattr(indicator, 'observable') and indicator.observable: + observable = indicator.observable + if self._has_properties(observable): + properties = observable.object_.properties + uuid = self._sanitise_uuid(observable.object_.id_) + try: + attribute_type, attribute_value, compl_data = self._handle_attribute_type(properties) + except StixObjectTypeError as xsi_type: + self._stix_object_type_error(xsi_type, indicator.id_) + return + if isinstance(attribute_value, (str, int)): + if observable.object_.related_objects: + related_objects = observable.object_.related_objects + resolving = ( + attribute_type == "url" and len(related_objects) == 1 and + related_objects[0].relationship.value == "Resolved_To" + ) + if resolving: + related_ip = self._sanitise_uuid(related_objects[0].idref) + self.dns_objects['domain'][uuid] = { + "related": related_ip, "data": { + "type": "text", "value": attribute_value + } + } + if related_ip not in self.dns_ips: + self.dns_ips.append(related_ip) + return + # if the returned value is a simple value, we build an attribute + attribute = {'to_ids': True, 'uuid': uuid} + if indicator.timestamp: + attribute['timestamp'] = self._timestamp_from_date(indicator.timestamp) + if hasattr(observable, 'handling') and observable.handling: + attribute['Tag'] = [] + for handling in observable.handling: + attribute['Tag'].extend(self._parse_marking(handling)) + if attribute_type in ('ip-src', 'ip-dst'): + attribute.update( + {'type': attribute_type, 'value': attribute_value} + ) + self.dns_objects['ip'][uuid] = attribute + return + self._handle_attribute_case(attribute_type, attribute_value, compl_data, attribute) + elif attribute_value: + if all(isinstance(value, dict) for value in attribute_value): + # it is a list of attributes, so we build an object + test_mechanisms = [] + if hasattr(indicator, 'test_mechanisms') and indicator.test_mechanisms: + for test_mechanism in indicator.test_mechanisms: + attribute_type = self._mapping.test_mechanisms_mapping(test_mechanism._XSI_TYPE) + if attribute_type is None: + self._add_error( + 'Unknown Test Mechanism type' + f': {test_mechanism._XSI_TYPE}' + ) + continue + if test_mechanism.rule.value is None: + continue + self.misp_event.add_attribute( + **{ + 'type': attribute_type, + 'value': test_mechanism.rule.value + } + ) + test_mechanisms.append(attribute.uuid) + self._handle_object_case( + attribute_type, attribute_value, compl_data, + to_ids=True, object_uuid=uuid, + test_mechanisms=test_mechanisms + ) + else: + # it is a list of attribute values, so we add single attributes + for value in attribute_value: + self.misp_event.add_attribute(**{'type': attribute_type, 'value': value, 'to_ids': True}) + elif hasattr(observable, 'observable_composition') and observable.observable_composition: + self._parse_observables(observable.observable_composition.observables, to_ids=True) + else: + self._parse_description(indicator) + + def _parse_marking(self, handling: MarkingSpecification): + if getattr(handling, 'marking_structures', None): + for marking in handling.marking_structures: + parser = self._mapping.marking_mapping(marking._XSI_TYPE) + if parser is not None: + yield from getattr(self, parser)(marking) + + def _parse_observables(self, observables: Optional[Observables] = None, to_ids: bool = False): + for observable in observables or self.stix_package.observables: + if self._has_properties(observable): + observable_object = observable.object_ + properties = observable_object.properties + try: + attribute_type, attribute_value, compl_data = self._handle_attribute_type(properties, title=observable.title) + except StixObjectTypeError as xsi_type: + self._stix_object_type_error(xsi_type, observable.id_) + continue + uuid = self._sanitise_uuid(observable_object.id_) + if isinstance(attribute_value, (str, int)): + if observable.object_.related_objects: + related_objects = observable.object_.related_objects + resolving = ( + attribute_type == "url" and len(related_objects) == 1 and + related_objects[0].relationship.value == "Resolved_To" + ) + if resolving: + related_ip = self._sanitise_uuid(related_objects[0].idref) + self.dns_objects['domain'][uuid] = { + "related": related_ip, "data": { + "type": "text", "value": attribute_value + } + } + if related_ip not in self.dns_ips: + self.dns_ips.append(related_ip) + continue + # if the returned value is a simple value, we build an attribute + attribute = {'to_ids': to_ids, 'uuid': uuid} + if hasattr(observable, 'handling') and observable.handling: + attribute['Tag'] = [] + for handling in observable.handling: + attribute['Tag'].extend(self._parse_marking(handling)) + if attribute_type in ('ip-src', 'ip-dst'): + attribute.update( + {'type': attribute_type, 'value': attribute_value} + ) + self.dns_objects['ip'][uuid] = attribute + continue + elif attribute_value: + if all(isinstance(value, dict) for value in attribute_value): + # it is a list of attributes, so we build an object + self._handle_object_case( + attribute_type, attribute_value, compl_data, + to_ids=to_ids, object_uuid=uuid + ) + else: + # it is a list of attribute values, so we add single attributes + for value in attribute_value: + self.misp_event.add_attribute( + **{'type': attribute_type, 'value': value, 'to_ids': to_ids} + ) + elif observable_object.related_objects: + for related_object in observable_object.related_objects: + relationship = related_object.relationship.value.lower().replace('_', '-') + self.references[uuid].append( + { + "idref": self.fetch_uuid(related_object.idref), + "relationship": relationship + } + ) + else: + self._parse_description(observable) + + def _parse_threat_actor(self, threat_actor: ThreatActor): + if getattr(threat_actor, 'title', None) is not None: + self.galaxies.update(self._parse_galaxy(threat_actor, 'title', 'threat-actor')) + elif getattr(threat_actor, 'identity', None) is not None: + identity = threat_actor.identity + if getattr(identity, 'name', None) is not None: + self.galaxies.update(self._resolve_galaxy(identity.name, 'threat-actor')) + elif hasattr(identity, 'specification') and getattr(identity.specification, 'party_name', None) is not None: + party_name = identity.specification.party_name + if getattr(party_name, 'person_names', None) is not None: + for person_name in party_name.person_names: + self.galaxies.update( + self._resolve_galaxy(person_name.name_elements[0].value, 'threat-actor') + ) + elif getattr(party_name, 'organisation_names', None) is not None: + for organisation_name in party_name.organisation_names: + self.galaxies.update( + self._resolve_galaxy(organisation_name.name_elements[0].value, 'threat-actor') + ) + + def _parse_ttp(self, ttp: TTP): + galaxies = set(self._parse_galaxies_from_ttp(ttp)) + if self._has_ttp_content(ttp): + attributes = self._parse_attributes_from_ttp(ttp, galaxies) + if attributes: + for attribute in attributes: + misp_attribute = MISPAttribute() + misp_attribute.from_dict(**attribute) + for galaxy in galaxies: + misp_attribute.add_tag(galaxy) + self.misp_event.add_attribute(**misp_attribute) + return + self.galaxies.update(galaxies) + + ############################################################################ + # MARKING DEFINITIONS PARSING METHODS. # + ############################################################################ + + @staticmethod + def _parse_AIS_marking(marking: AISMarkingStructure): + for feature in ('is_proprietary', 'not_proprietary'): + proprietary = getattr(marking, feature) + if proprietary is None: + continue + yield f'ais-marking:AISMarking="{feature.title()}"' + if hasattr(proprietary, 'cisa_proprietary'): + cisa_proprietary = ( + 'true' if proprietary.cisa_proprietary.numerator == 1 + else 'false' + ) + yield f'ais-marking:CISA_Proprietary="{cisa_proprietary}"' + if hasattr(proprietary, 'ais_consent'): + consent = proprietary.ais_consent.consent + yield f'ais-marking:AISConsent="{consent}"' + if hasattr(proprietary, 'tlp_marking'): + color = proprietary.tlp_marking.color + yield f'ais-marking:TLPMarking="{color}"' + + @staticmethod + def _parse_TLP_marking(marking: TLPMarkingStructure): + yield f'tlp:{marking.color.lower()}' + + ############################################################################ + # UTILITY METHODS. # + ############################################################################ + + def _get_event_info(self): + if hasattr(self.stix_package, 'title'): + return self.stix_package.title + if hasattr(getattr(self.stix_package, 'stix_header', None), 'title'): + return self.stix_package.stix_header.title + return f"Imported from external STIX {self.stix_version} Package" + + @staticmethod + def _has_properties(observable): + if not hasattr(observable, 'object_') or not observable.object_: + return False + if hasattr(observable.object_, 'properties') and observable.object_.properties: + return True + return False + + def _has_ttp_content(self, ttp: TTP) -> bool: + if ttp.resources is not None and ttp.resources.infrastructure is not None: + return True + if ttp.exploit_targets is None or ttp.exploit_targets.exploit_target is None: + return False + return any( + exploit_target.item.vulnerability is not None + for exploit_target in ttp.exploit_targets.exploit_target + ) diff --git a/misp_stix_converter/stix2misp/external_stix2_mapping.py b/misp_stix_converter/stix2misp/external_stix2_mapping.py index 53455d8..2410eba 100644 --- a/misp_stix_converter/stix2misp/external_stix2_mapping.py +++ b/misp_stix_converter/stix2misp/external_stix2_mapping.py @@ -472,7 +472,7 @@ class ExternalSTIX2toMISPMapping(STIX2toMISPMapping): ) @classmethod - def asn_pattern_mapping(cls, field) -> Union[dict, None]: + def asn_pattern_mapping(cls, field: str) -> Union[dict, None]: return cls.__asn_pattern_mapping.get(field) @classmethod @@ -480,7 +480,7 @@ def course_of_action_object_mapping(cls) -> dict: return cls.__course_of_action_object_mapping @classmethod - def directory_pattern_mapping(cls, field) -> Union[dict, None]: + def directory_pattern_mapping(cls, field: str) -> Union[dict, None]: return cls.__directory_object_mapping.get(field) @classmethod @@ -488,15 +488,15 @@ def directory_object_mapping(cls) -> dict: return cls.__directory_object_mapping @classmethod - def domain_ip_pattern_mapping(cls, field) -> Union[dict, None]: + def domain_ip_pattern_mapping(cls, field: str) -> Union[dict, None]: return cls.__domain_ip_pattern_mapping.get(field) @classmethod - def email_address_pattern_mapping(cls, field) -> Union[dict, None]: + def email_address_pattern_mapping(cls, field: str) -> Union[dict, None]: return cls.__email_address_pattern_mapping.get(field) @classmethod - def email_message_mapping(cls, field) -> Union[dict, None]: + def email_message_mapping(cls, field: str) -> Union[dict, None]: return cls.__email_object_mapping.get(field) @classmethod @@ -512,7 +512,7 @@ def file_hashes_object_mapping(cls) -> dict: return cls.__file_hashes_mapping @classmethod - def file_hashes_pattern_mapping(cls, field) -> Union[dict, None]: + def file_hashes_pattern_mapping(cls, field: str) -> Union[dict, None]: return cls.__file_hashes_mapping.get(field) @classmethod @@ -524,23 +524,23 @@ def file_object_mapping(cls) -> dict: return cls.__file_object_mapping @classmethod - def file_pattern_mapping(cls, field) -> Union[dict, None]: + def file_pattern_mapping(cls, field: str) -> Union[dict, None]: return cls.__file_object_mapping.get(field) @classmethod - def galaxy_name_mapping(cls, field) -> Union[dict, None]: + def galaxy_name_mapping(cls, field: str) -> Union[dict, None]: return cls.__galaxy_name_mapping.get(field) @classmethod - def http_request_extension_mapping(cls, field) -> Union[dict, None]: + def http_request_extension_mapping(cls, field: str) -> Union[dict, None]: return cls.__http_request_extension_mapping.get(field) @classmethod - def network_connection_object_reference_mapping(cls, field) -> Union[str, None]: + def network_connection_object_reference_mapping(cls, field: str) -> Union[str, None]: return cls.__network_connection_object_reference_mapping.get(field) @classmethod - def network_socket_object_reference_mapping(cls, field) -> Union[str, None]: + def network_socket_object_reference_mapping(cls, field: str) -> Union[str, None]: return cls.__network_socket_object_reference_mapping.get(field) @classmethod @@ -548,11 +548,11 @@ def network_traffic_object_mapping(cls) -> dict: return cls.__network_traffic_object_mapping @classmethod - def network_traffic_pattern_mapping(cls, field) -> Union[dict, None]: + def network_traffic_pattern_mapping(cls, field: str) -> Union[dict, None]: return cls.__network_traffic_object_mapping.get(field) @classmethod - def observable_mapping(cls, field) -> Union[str, None]: + def observable_mapping(cls, field: str) -> Union[str, None]: return cls.__observable_mapping.get(field) @classmethod @@ -560,7 +560,7 @@ def pattern_forbidden_relations(cls) -> tuple: return cls.__pattern_forbidden_relations @classmethod - def pattern_mapping(cls, field) -> Union[str, None]: + def pattern_mapping(cls, field: str) -> Union[str, None]: return cls.__pattern_mapping.get(field) @classmethod @@ -568,7 +568,7 @@ def pe_object_mapping(cls) -> dict: return cls.__pe_object_mapping @classmethod - def pe_pattern_mapping(cls, field) -> Union[dict, None]: + def pe_pattern_mapping(cls, field: str) -> Union[dict, None]: return cls.__pe_object_mapping.get(field) @classmethod @@ -576,7 +576,7 @@ def pe_optional_header_object_mapping(cls) -> dict: return cls.__pe_optional_header_mapping @classmethod - def pe_optional_header_pattern_mapping(cls, field) -> Union[dict, None]: + def pe_optional_header_pattern_mapping(cls, field: str) -> Union[dict, None]: return cls.__pe_optional_header_mapping.get(field) @classmethod @@ -584,7 +584,7 @@ def pe_section_object_mapping(cls) -> dict: return cls.__pe_section_object_mapping @classmethod - def pe_section_pattern_mapping(cls, field) -> Union[dict, None]: + def pe_section_pattern_mapping(cls, field: str) -> Union[dict, None]: return cls.__pe_section_object_mapping.get(field) @classmethod @@ -592,7 +592,7 @@ def process_object_mapping(cls) -> dict: return cls.__process_object_mapping @classmethod - def process_pattern_mapping(cls, field) -> Union[dict, None]: + def process_pattern_mapping(cls, field: str) -> Union[dict, None]: return cls.__process_pattern_mapping.get(field) @classmethod @@ -604,7 +604,7 @@ def registry_key_object_mapping(cls) -> dict: return cls.__registry_key_object_mapping @classmethod - def registry_key_pattern_mapping(cls, field) -> Union[dict, None]: + def registry_key_pattern_mapping(cls, field: str) -> Union[dict, None]: return cls.__registry_key_pattern_mapping.get(field) @classmethod @@ -616,7 +616,7 @@ def software_object_mapping(cls) -> dict: return cls.__software_object_mapping @classmethod - def software_pattern_mapping(cls, field) -> Union[dict, None]: + def software_pattern_mapping(cls, field: str) -> Union[dict, None]: return cls.__software_pattern_mapping.get(field) @classmethod @@ -624,7 +624,7 @@ def user_account_object_mapping(cls) -> dict: return cls.__user_account_object_mapping @classmethod - def user_account_pattern_mapping(cls, field) -> Union[dict, None]: + def user_account_pattern_mapping(cls, field: str) -> Union[dict, None]: return cls.__user_account_pattern_mapping.get(field) @classmethod @@ -632,7 +632,7 @@ def x509_hashes_object_mapping(cls) -> dict: return cls.__x509_hashes_mapping @classmethod - def x509_hashes_pattern_mapping(cls, field) -> Union[dict, None]: + def x509_hashes_pattern_mapping(cls, field: str) -> Union[dict, None]: return cls.__x509_hashes_mapping.get(field) @classmethod @@ -640,7 +640,7 @@ def x509_object_mapping(cls) -> dict: return cls.__x509_object_mapping @classmethod - def x509_pattern_mapping(cls, field) -> Union[dict, None]: + def x509_pattern_mapping(cls, field: str) -> Union[dict, None]: return cls.__x509_object_mapping.get(field) @classmethod diff --git a/misp_stix_converter/stix2misp/external_stix2_to_misp.py b/misp_stix_converter/stix2misp/external_stix2_to_misp.py index 7d238d8..d39a68c 100644 --- a/misp_stix_converter/stix2misp/external_stix2_to_misp.py +++ b/misp_stix_converter/stix2misp/external_stix2_to_misp.py @@ -11,31 +11,15 @@ ExternalSTIX2MalwareConverter, ExternalSTIX2ObservedDataConverter, ExternalSTIX2ThreatActorConverter, ExternalSTIX2ToolConverter, ExternalSTIX2VulnerabilityConverter, STIX2ObservableObjectConverter) +from .importparser import ExternalSTIXtoMISPParser from .stix2_to_misp import STIX2toMISPParser, _OBSERVABLE_TYPING from collections import defaultdict from typing import Optional, Union -MISP_org_uuid = '55f6ea65-aa10-4c5a-bf01-4f84950d210f' - - -class ExternalSTIX2toMISPParser(STIX2toMISPParser): - def __init__(self, distribution: Optional[int] = 0, - sharing_group_id: Optional[int] = None, - title: Optional[str] = None, - producer: Optional[str] = None, - galaxies_as_tags: Optional[bool] = False, - organisation_uuid: Optional[str] = MISP_org_uuid, - cluster_distribution: Optional[int] = 0, - cluster_sharing_group_id: Optional[int] = None): - super().__init__( - distribution, sharing_group_id, title, producer, galaxies_as_tags - ) - self._set_cluster_distribution( - self._sanitise_distribution(cluster_distribution), - self._sanitise_sharing_group_id(cluster_sharing_group_id) - ) - self.__organisation_uuid = organisation_uuid +class ExternalSTIX2toMISPParser(STIX2toMISPParser, ExternalSTIXtoMISPParser): + def __init__(self): + super().__init__() self._mapping = ExternalSTIX2toMISPMapping # parsers self._attack_pattern_parser: ExternalSTIX2AttackPatternConverter @@ -53,9 +37,15 @@ def __init__(self, distribution: Optional[int] = 0, self._tool_parser: ExternalSTIX2ToolConverter self._vulnerability_parser: ExternalSTIX2VulnerabilityConverter - @property - def cluster_distribution(self) -> dict: - return self.__cluster_distribution + def parse_stix_bundle(self, cluster_distribution: Optional[int] = 0, + cluster_sharing_group_id: Optional[int] = None, + organisation_uuid: Optional[str] = None, **kwargs): + self._set_parameters(**kwargs) + self._set_cluster_distribution( + cluster_distribution, cluster_sharing_group_id + ) + self._set_organisation_uuid(organisation_uuid) + self._parse_stix_bundle() @property def observable_object_parser(self) -> STIX2ObservableObjectConverter: @@ -63,9 +53,9 @@ def observable_object_parser(self) -> STIX2ObservableObjectConverter: self._set_observable_object_parser() return self._observable_object_parser - @property - def organisation_uuid(self) -> str: - return self.__organisation_uuid + ############################################################################ + # PARSER SETTERS # + ############################################################################ def _set_attack_pattern_parser(self): self._attack_pattern_parser = ExternalSTIX2AttackPatternConverter(self) @@ -73,13 +63,6 @@ def _set_attack_pattern_parser(self): def _set_campaign_parser(self): self._campaign_parser = ExternalSTIX2CampaignConverter(self) - def _set_cluster_distribution( - self, distribution: int, sharing_group_id: Union[int, None]): - cluster_distribution = {'distribution': distribution} - if distribution == 4 and sharing_group_id is not None: - cluster_distribution['sharing_group_id'] = sharing_group_id - self.__cluster_distribution = cluster_distribution - def _set_course_of_action_parser(self): self._course_of_action_parser = ExternalSTIX2CourseOfActionConverter(self) @@ -163,18 +146,24 @@ def _handle_unparsed_content(self): continue feature = self._mapping.observable_mapping(observable_type) if feature is None: - self._observable_object_mapping_error( - unparsed_content[observable_type][0] + observable_id = unparsed_content[observable_type][0] + self._add_error( + f'Unable to map observable object with id {observable_id}' ) continue to_call = f'_parse_{feature}_observable_object' for object_id in unparsed_content[observable_type]: if self._observable[object_id]['used'][self.misp_event.uuid]: + # if object_id.split('--')[0] not in _force_observables_list: continue try: getattr(self.observable_object_parser, to_call)(object_id) except Exception as exception: - self._observable_object_error(object_id, exception) + _traceback = self._parse_traceback(exception) + self._add_error( + 'Error parsing the Observable object with id ' + f'{object_id}: {_traceback}' + ) super()._handle_unparsed_content() def _parse_loaded_features(self): diff --git a/misp_stix_converter/stix2misp/importparser.py b/misp_stix_converter/stix2misp/importparser.py index 31ddaf6..cfd49aa 100644 --- a/misp_stix_converter/stix2misp/importparser.py +++ b/misp_stix_converter/stix2misp/importparser.py @@ -1,12 +1,17 @@ #!/usr/bin/env python3 import json +import sys import traceback from .exceptions import UnavailableGalaxyResourcesError from abc import ABCMeta from collections import defaultdict +from datetime import datetime +from mixbox.namespaces import NamespaceNotFoundError from pathlib import Path -from pymisp import AbstractMISP, MISPEvent, MISPObject +from pymisp import MISPEvent, MISPObject +from pymisp.abstract import resources_path +from stix.core import STIXPackage from stix2.exceptions import InvalidValueError from stix2.parsing import dict_to_stix2, parse as stix2_parser, ParseError from stix2.v20.bundle import Bundle as Bundle_v20 @@ -23,6 +28,10 @@ ] _DATA_PATH = Path(__file__).parents[1].resolve() / 'data' +MISP_org_uuid = '55f6ea65-aa10-4c5a-bf01-4f84950d210f' + +_DEFAULT_DISTRIBUTION = 0 + _VALID_DISTRIBUTIONS = (0, 1, 2, 3, 4) _RFC_VERSIONS = (1, 3, 4, 5) _UUIDv4 = UUID('76beed5f-7251-457e-8c2a-b45f7b589d3d') @@ -54,6 +63,26 @@ def _handle_stix2_loading_error(stix2_content: dict): return bundle(*stix2_content, allow_custom=True, interoperability=True) +def _load_stix1_package(filename, tries=0): + try: + return STIXPackage.from_xml(filename) + except NamespaceNotFoundError: + if tries > 0: + sys.exit('Cannot handle STIX namespace') + _update_namespaces() + return _load_stix1_package(filename, tries + 1) + except NotImplementedError: + sys.exit('Missing python library: stix_edh') + except Exception: + try: + import maec + return STIXPackage.from_xml(filename) + except ImportError: + sys.exit('Missing python library: maec') + except Exception as error: + sys.exit(f'Error while loading STIX1 package: {error.__str__()}') + + def _load_stix2_content(filename): with open(filename, 'rt', encoding='utf-8') as f: stix2_content = f.read() @@ -70,32 +99,66 @@ def _load_json_file(path): return json.load(f) +def _update_namespaces(): + from mixbox.namespaces import Namespace, register_namespace + # LIST OF ADDITIONAL NAMESPACES + # can add additional ones whenever it is needed + ADDITIONAL_NAMESPACES = [ + Namespace('http://us-cert.gov/ciscp', 'CISCP', + 'http://www.us-cert.gov/sites/default/files/STIX_Namespace/ciscp_vocab_v1.1.1.xsd'), + Namespace('http://taxii.mitre.org/messages/taxii_xml_binding-1.1', 'TAXII', + 'http://docs.oasis-open.org/cti/taxii/v1.1.1/cs01/schemas/TAXII-XMLMessageBinding-Schema.xsd') + ] + for namespace in ADDITIONAL_NAMESPACES: + register_namespace(namespace) + + +class ExternalSTIXtoMISPParser(metaclass=ABCMeta): + def _set_cluster_distribution( + self, distribution: int, sharing_group_id: Union[int, None]): + cl_dis = {'distribution': self._sanitise_distribution(distribution)} + if distribution == 4: + if sharing_group_id is not None: + cl_dis['sharing_group_id'] = self._sanitise_sharing_group_id( + sharing_group_id + ) + else: + cl_dis['distribution'] = 0 + self._cluster_distribution_and_sharing_group_id_error() + self.__cluster_distribution = cl_dis + + def _set_organisation_uuid(self, organisation_uuid: Union[str, None]): + self.__organisation_uuid = organisation_uuid or MISP_org_uuid + + @property + def cluster_distribution(self) -> dict: + return self.__cluster_distribution + + @property + def organisation_uuid(self) -> str: + return self.__organisation_uuid + + class STIXtoMISPParser(metaclass=ABCMeta): - def __init__(self, distribution: int, sharing_group_id: Union[int, None], - title: Union[str, None], producer: Union[str, None], - galaxies_as_tags: bool): + def __init__(self): self._identifier: str + self.__distribution: int + self.__galaxies_as_tags: bool + self.__galaxy_feature: str + self.__producer: Union[str, None] self.__relationship_types: dict + self.__sharing_group_id: Union[int, None] + self.__title: Union[str, None] self._clusters: dict = {} + self._galaxies: dict = {} self.__errors: defaultdict = defaultdict(set) self.__warnings: defaultdict = defaultdict(set) - self.__distribution = self._sanitise_distribution(distribution) - self.__sharing_group_id = self._sanitise_sharing_group_id( - sharing_group_id - ) - self.__title = title - self.__producer = producer - self.__galaxies_as_tags = self._sanitise_galaxies_as_tags( - galaxies_as_tags - ) - if self.galaxies_as_tags: - self.__galaxy_feature = 'as_tag_names' - else: - self._galaxies: dict = {} - self.__galaxy_feature = 'as_container' self.__replacement_uuids: dict = {} + def _populate_misp_event(self): + self.misp_events.append(self.misp_event) + def _sanitise_distribution(self, distribution: int) -> int: try: sanitised = int(distribution) @@ -107,7 +170,8 @@ def _sanitise_distribution(self, distribution: int) -> int: self._distribution_value_error(sanitised) return 0 - def _sanitise_galaxies_as_tags(self, galaxies_as_tags: bool): + def _sanitise_galaxies_as_tags( + self, galaxies_as_tags: Union[bool, str, int]) -> bool: if isinstance(galaxies_as_tags, bool): return galaxies_as_tags if galaxies_as_tags in ('true', 'True', '1', 1): @@ -127,6 +191,38 @@ def _sanitise_sharing_group_id( self._sharing_group_id_error(error) return None + def _set_misp_event(self, misp_event: MISPEvent): + self.__misp_event = misp_event + + def _set_misp_events(self): + self.__misp_events = [] + + def _set_parameters(self, distribution: int = _DEFAULT_DISTRIBUTION, + sharing_group_id: Optional[int] = None, + galaxies_as_tags: Optional[bool] = False, + single_event: Optional[bool] = False, + producer: Optional[str] = None, + title: Optional[str] = None): + self.__distribution = self._sanitise_distribution(distribution) + self.__sharing_group_id = self._sanitise_sharing_group_id( + sharing_group_id + ) + if self.sharing_group_id is None and self.distribution == 4: + self.__distribution = 0 + self._distribution_and_sharing_group_id_error() + self.__galaxies_as_tags = self._sanitise_galaxies_as_tags( + galaxies_as_tags + ) + self.__galaxy_feature = ( + 'as_tag_names' if self.galaxies_as_tags else 'as_container' + ) + self.__single_event = single_event + self.__producer = producer + self.__title = title + + def _set_single_event(self, single_event: bool): + self.__single_event = single_event + ############################################################################ # PROPERTIES # ############################################################################ @@ -159,6 +255,16 @@ def galaxy_definitions(self) -> Path: def galaxy_feature(self) -> str: return self.__galaxy_feature + @property + def misp_event(self) -> MISPEvent: + return self.__misp_event + + @property + def misp_events(self) -> Union[list, MISPEvent]: + return getattr( + self, '_STIXtoMISPParser__misp_events', self.__misp_event + ) + @property def producer(self) -> Union[str, None]: return self.__producer @@ -179,6 +285,10 @@ def replacement_uuids(self) -> dict: def sharing_group_id(self) -> Union[int, None]: return self.__sharing_group_id + @property + def single_event(self) -> bool: + return self.__single_event + @property def synonyms_mapping(self) -> dict: try: @@ -195,35 +305,21 @@ def warnings(self) -> defaultdict: # ERRORS AND WARNINGS HANDLING METHODS # ############################################################################ - def _attack_pattern_error( - self, attack_pattern_id: str, exception: Exception): - tb = self._parse_traceback(exception) - self.__errors[self._identifier].add( - 'Error parsing the Attack Pattern object with id ' - f'{attack_pattern_id}: {tb}' - ) + def _add_error(self, error: str): + self.__errors[self._identifier].add(error) - def _attribute_from_pattern_parsing_error(self, indicator_id: str): - self.__errors[self._identifier].add( - f'Error while parsing pattern from indicator with id {indicator_id}' - ) + def _add_warning(self, warning: str): + self.__warnings[self._identifier].add(warning) - def _course_of_action_error( - self, course_of_action_id: str, exception: Exception): - self.__errors[self._identifier].add( - 'Error parsing the Course of Action object with id' - f'{course_of_action_id}: {self._parse_traceback(exception)}' - ) - - def _critical_error(self, exception: Exception): - self.__errors[self._identifier].add( - f'The following exception was raised: {exception}' + def _cluster_distribution_and_sharing_group_id_error(self): + self.__errors['init'].add( + 'Invalid Cluster Sharing Group ID - ' + 'cannot be None when distribution is 4' ) - def _custom_object_error(self, custom_object_id: str, exception: Exception): - self.__errors[self._identifier].add( - 'Error parsing the Custom object with id' - f'{custom_object_id}: {self._parse_traceback(exception)}' + def _distribution_and_sharing_group_id_error(self): + self.__errors['init'].add( + 'Invalid Sharing Group ID - cannot be None when distribution is 4' ) def _distribution_error(self, exception: Exception): @@ -241,88 +337,6 @@ def _galaxies_as_tags_error(self, galaxies_as_tags): f'Invalid galaxies_as_tags flag: {galaxies_as_tags} (bool expected)' ) - def _hash_type_error(self, hash_type: str): - self.__errors[self._identifier].add(f'Wrong hash type: {hash_type}') - - def _identity_error(self, identity_id: str, exception: Exception): - tb = self._parse_traceback(exception) - self.__errors[self._identifier].add( - f'Error parsing the Identity object with id {identity_id}: {tb}' - ) - - def _indicator_error(self, indicator_id: str, exception: Exception): - tb = self._parse_traceback(exception) - self.__errors[self._identifier].add( - f'Error parsing the Indicator object with id {indicator_id}: {tb}' - ) - - def _intrusion_set_error(self, intrusion_set_id: str, exception: Exception): - self.__errors[self._identifier].add( - f'Error parsing the Intrusion Set object with id {intrusion_set_id}' - f': {self._parse_traceback(exception)}' - ) - - def _location_error(self, location_id: str, exception: Exception): - tb = self._parse_traceback(exception) - self.__errors[self._identifier].add( - f'Error parsing the Location object with id {location_id}: {tb}' - ) - - def _malware_error(self, malware_id: str, exception: Exception): - tb = self._parse_traceback(exception) - self.__errors[self._identifier].add( - f'Error parsing the Malware object with id {malware_id}: {tb}' - ) - - def _marking_definition_error(self, marking_definition_id: str): - self.__errors[self._identifier].add( - f'Error parsing the Marking Definition object with id ' - f'{marking_definition_id}' - ) - - def _no_converted_content_from_pattern_warning( - self, indicator: _INDICATOR_TYPING): - self.__warnings[self._identifier].add( - "No content to extract from the following Indicator's (id: " - f'{indicator.id}) pattern: {indicator.pattern}' - ) - - def _object_ref_loading_error(self, object_ref: str): - self.__errors[self._identifier].add( - f'Error loading the STIX object with id {object_ref}' - ) - - def _object_type_loading_error(self, object_type: str): - self.__errors[self._identifier].add( - f'Error loading the STIX object of type {object_type}' - ) - - def _observable_mapping_error( - self, observed_data_id: str, observable_types: str): - self.__errors[self._identifier].add( - 'Unable to map observable objects related to the Observed Data ' - f'object with id {observed_data_id} containing the folowing types' - f": {observable_types.__str__().replace('_', ', ')}" - ) - - def _observable_object_error( - self, observable_id: str, exception: Exception): - self.__errors[self._identifier].add( - f'Error parsing the Observable object with id {observable_id}' - f': {self._parse_traceback(exception)}' - ) - - def _observable_object_mapping_error(self, observable_id: str): - self.__errors[self._identifier].add( - f'Unable to map observable object with id {observable_id}.' - ) - - def _observed_data_error(self, observed_data_id: str, exception: Exception): - self.__errors[self._identifier].add( - f'Error parsing the Observed Data object with id {observed_data_id}' - f': {self._parse_traceback(exception)}' - ) - @staticmethod def _parse_traceback(exception: Exception) -> str: tb = ''.join(traceback.format_tb(exception.__traceback__)) @@ -333,105 +347,12 @@ def _sharing_group_id_error(self, exception: Exception): f'Wrong sharing group id format: {exception}' ) - def _threat_actor_error(self, threat_actor_id: str, exception: Exception): - self.__errors[self._identifier].add( - f'Error parsing the Threat Actor object with id {threat_actor_id}' - f': {self._parse_traceback(exception)}' - ) - - def _tool_error(self, tool_id: str, exception: Exception): - tb = self._parse_traceback(exception) - self.__errors[self._identifier].add( - f'Error parsing the Tool object with id {tool_id}: {tb}' - ) - - def _unable_to_load_stix_object_type_error(self, object_type: str): - self.__errors[self._identifier].add( - f'Unable to load STIX object type: {object_type}' - ) - - def _undefined_object_error(self, object_id: str): - self.__errors[self._identifier].add( - f'Unable to define the object identified with the id: {object_id}' - ) - - def _unknown_attribute_type_warning(self, attribute_type: str): - self.__warnings[self._identifier].add( - f'MISP attribute type not mapped: {attribute_type}' - ) - - def _unknown_marking_object_warning(self, marking_ref: str): - self.__warnings[self._identifier].add( - f'Unknown marking definition object referenced by id {marking_ref}' - ) - - def _unknown_marking_ref_warning(self, marking_ref: str): - self.__warnings[self._identifier].add( - f'Unknown marking ref: {marking_ref}' - ) - - def _unknown_network_protocol_warning( - self, protocol: str, object_id: str, - object_type: Optional[str] = 'indicator'): - message = ( - 'in patterning expression within the indicator with id' - if object_type == 'indicator' else - f'within the {object_type} object with id' - ) - self.__warnings[self._identifier].add( - f'Unknown network protocol: {protocol}, {message} {object_id}' - ) - - def _unknown_object_name_warning(self, name: str): - self.__warnings[self._identifier].add( - f'MISP object name not mapped: {name}' - ) - - def _unknown_parsing_function_error(self, feature: str): - self.__errors[self._identifier].add( - f'Unknown STIX parsing function name: {feature}' - ) - - def _unknown_pattern_mapping_warning( - self, indicator_id: str, pattern_types: Union[GeneratorType, str]): - if not isinstance(pattern_types, GeneratorType): - pattern_types = pattern_types.split('_') - self.__warnings[self._identifier].add( - f'Unable to map pattern from the Indicator with id {indicator_id}, ' - f"containing the following types: {', '.join(pattern_types)}" - ) - - def _unknown_pattern_type_error(self, indicator_id: str, pattern_type: str): - self.__errors[self._identifier].add( - f'Unknown pattern type in indicator with id {indicator_id}' - f': {pattern_type}' - ) - - def _unknown_stix_object_type_error(self, object_type: str): - self.__errors[self._identifier].add( - f'Unknown STIX object type: {object_type}' - ) - - def _unmapped_pattern_warning(self, indicator_id: str, feature: str): - self.__warnings[self._identifier].add( - f'Unmapped pattern part in indicator with id {indicator_id}' - f': {feature}' - ) - - def _vulnerability_error(self, vulnerability_id: str, exception: Exception): - self.__errors[self._identifier].add( - f'Error parsing the Vulnerability object with id {vulnerability_id}' - f': {self._parse_traceback(exception)}' - ) - ############################################################################ # MISP OBJECT RELATIONSHIPS MAPPING CREATION METHODS # ############################################################################ def __get_relationship_types(self): - relationships_path = Path( - AbstractMISP().resources_path / 'misp-objects' / 'relationships' - ) + relationships_path = resources_path / 'misp-objects' / 'relationships' relationships = _load_json_file(relationships_path / 'definition.json') self.__relationship_types = { relationship['name']: relationship['opposite'] for relationship @@ -586,3 +507,11 @@ def _sanitise_uuid(self, object_id: str) -> str: self.replacement_uuids[object_uuid] = sanitised_uuid return sanitised_uuid return object_uuid + + ############################################################################ + # UTILITY METHODS. # + ############################################################################ + + @staticmethod + def _timestamp_from_date(date: datetime) -> int: + return int(date.timestamp()) diff --git a/misp_stix_converter/stix2misp/internal_stix1_to_misp.py b/misp_stix_converter/stix2misp/internal_stix1_to_misp.py index ef2c411..9fa58cd 100644 --- a/misp_stix_converter/stix2misp/internal_stix1_to_misp.py +++ b/misp_stix_converter/stix2misp/internal_stix1_to_misp.py @@ -1,9 +1,435 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -from .stix1_to_misp import STIX1toMISPParser +from .stix1_mapping import InternalSTIX1toMISPMapping +from .stix1_to_misp import StixObjectTypeError, STIX1toMISPParser +from pymisp import MISPAttribute, MISPEvent, MISPObject +from pymisp.abstract import resources_path +from pymisp.api import describe_types +from stix.exploit_target import Vulnerability, Weakness +from stix.indicator import Indicator, Observable +from stix.ttp import TTP +from stix.ttp.attack_pattern import AttackPattern +from typing import Optional + +_MISP_categories = describe_types.get('categories') +_MISP_objects_path = resources_path / 'objects' class InternalSTIX1toMISPParser(STIX1toMISPParser): def __init__(self): - super().__init__() \ No newline at end of file + super().__init__() + self._mapping = InternalSTIX1toMISPMapping + self.__dates = set() + self.__timestamps = set() + self.__titles = set() + + def parse_stix_package(self, **kwargs): + self._set_parameters(**kwargs) + self._set_misp_event(MISPEvent()) + for item in self.stix_package.related_packages.related_package: + package = item.item + self._event = package.incidents[0] + object_references = [] + for coa_taken in self._event.coa_taken: + self._parse_course_of_action(coa_taken.course_of_action) + if self._event.attributed_threat_actors: + object_references.extend( + threat_actor.item.idref for threat_actor + in self._event.attributed_threat_actors.threat_actor + ) + if self._event.leveraged_ttps and self._event.leveraged_ttps.ttp: + object_references.extend( + ttp.item.idref for ttp in self._event.leveraged_ttps.ttp + ) + object_references = tuple( + '-'.join(part for part in reference.split('-')[-5:]) + for reference in object_references if reference is not None + ) + if self._event.timestamp: + stix_date = self._event.timestamp + try: + self.dates.add(stix_date.date()) + except AttributeError: + self.dates.add(stix_date) + self.timestamps.add(self._timestamp_from_date(stix_date)) + self.titles.add(self._get_event_info()) + if self._event.related_indicators: + for indicator in self._event.related_indicators.indicator: + self._parse_indicator(indicator) + if self._event.related_observables: + for observable in self._event.related_observables.observable: + self._parse_observable(observable) + if self._event.history: + for entry in self.event.history.history_items: + journal_entry = entry.journal_entry.value + try: + entry_type, entry_value = journal_entry.split(': ') + if entry_type == "MISP Tag": + self.misp_event.add_tag(entry_value) + elif entry_type.startswith('attribute['): + _, category, attribute_type = entry_type.split('[') + self.misp_event.add_attribute( + **{ + 'type': attribute_type[:-1], + 'category': category[:-1], + 'value': entry_value + } + ) + elif entry_type == "Event Threat Level": + threat_level = self._mapping.threat_level_mapping( + entry_value + ) + if threat_level is not None: + self.misp_event.threat_level_id = threat_level + except ValueError: + continue + if self._event.information_source and self._event.information_source.references: + for reference in self._event.information_source.references: + self.misp_event.add_attribute(**{'type': 'link', 'value': reference}) + if package.courses_of_action: + for course_of_action in package.courses_of_action: + self.galaxies.update( + self._parse_galaxy(course_of_action, 'title', 'course-of-action') + ) + if package.threat_actors: + for threat_actor in package.threat_actors: + self.galaxies.update( + self._parse_galaxy(threat_actor, 'title', 'threat-actor') + ) + if package.ttps: + for ttp in package.ttps.ttp: + ttp_id = '-'.join((part for part in ttp.id_.split('-')[-5:])) + if ttp_id not in object_references: + self._parse_ttp(ttp) + continue + if ttp.behavior: + if ttp.behavior.attack_patterns: + for attack_pattern in ttp.behavior.attack_patterns: + self._parse_attack_pattern_object(attack_pattern, ttp_id) + continue + if ttp.exploit_targets and ttp.exploit_targets.exploit_target: + for exploit_target in ttp.exploit_targets.exploit_target: + if exploit_target.item.vulnerabilities: + for vulnerability in exploit_target.item.vulnerabilities: + self._parse_vulnerability_object(vulnerability, ttp_id) + if exploit_target.item.weaknesses: + for weakness in exploit_target.item.weaknesses: + self._parse_weakness_object(weakness, ttp_id) + # if ttp.handling: + # self.parse_tlp_marking(ttp.handling) + self._set_distribution() + self.misp_event.info = ' - '.join(self.titles) + self.misp_event.date = max(self.dates) + self.misp_event.timestamp = max(self.timestamps) + + ############################################################################ + # PROPERTIES # + ############################################################################ + + @property + def dates(self) -> set: + return self.__dates + + @property + def timestamps(self) -> set: + return self.__timestamps + + @property + def titles(self) -> set: + return self.__titles + + ############################################################################ + # STIX OBJECTS PARSING METHODS # + ############################################################################ + + def _parse_attack_pattern_object(self, attack_pattern: AttackPattern, ttp_id: str): + attributes = [] + for key, relation in self._mapping.attack_pattern_object_mapping().items(): + value = getattr(attack_pattern, key) + if value: + attributes.append( + (relation, value if isinstance(value, str) else value.value) + ) + if attributes: + attack_pattern_object = MISPObject('attack-pattern') + attack_pattern_object.uuid = ttp_id + for attribute in attributes: + attack_pattern_object.add_attribute(*attribute) + self.misp_event.add_object(attack_pattern_object) + + # Parse indicators of a STIX document coming from our exporter + def _parse_indicator(self, indicator: Indicator): + # define is an indicator will be imported as attribute or object + if indicator.relationship in _MISP_categories: + self._parse_misp_attribute_indicator(indicator) + else: + self._parse_misp_object_indicator(indicator) + + def _parse_observable(self, observable: Observable): + if observable.relationship in _MISP_categories: + self.parse_misp_attribute_observable(observable) + else: + self.parse_misp_object_observable(observable) + + def _parse_ttp(self, ttp: TTP): + if ttp.behavior: + if ttp.behavior.attack_patterns: + for attack_pattern in ttp.behavior.attack_patterns: + self.galaxies.update(self._parse_galaxy(attack_pattern, 'title', 'misp-attack-pattern')) + if ttp.behavior.malware_instances: + for malware_instance in ttp.behavior.malware_instances: + if not malware_instance._XSI_TYPE or 'stix-maec' not in malware_instance._XSI_TYPE: + self.galaxies.update(self._parse_galaxy(malware_instance, 'title', 'ransomware')) + elif ttp.exploit_targets: + if ttp.exploit_targets.exploit_target: + for exploit_target in ttp.exploit_targets.exploit_target: + if exploit_target.item.vulnerabilities: + for vulnerability in exploit_target.item.vulnerabilities: + self.galaxies.update( + self._parse_galaxy(vulnerability, 'title', 'branded-vulnerability') + ) + elif ttp.resources: + if ttp.resources.tools: + for tool in ttp.resources.tools: + self.galaxies.update(self._parse_galaxy(tool, 'name', 'tool')) + + def _parse_vulnerability_object(self, vulnerability: Vulnerability, ttp_id: str): + attributes = [] + for key, mapping in self._mapping.vulnerability_object_mapping().items(): + value = getattr(vulnerability, key) + if value: + attribute_type, relation = mapping + attributes.append( + { + 'type': attribute_type, 'object_relation': relation, + 'value': value if isinstance(value, str) else value.value + } + ) + if attributes: + if len(attributes) == 1 and attributes[0]['object_relation'] == 'id': + attributes = attributes[0] + attributes['uuid'] = ttp_id + self.misp_event.add_attribute(**attributes) + else: + vulnerability_object = MISPObject('vulnerability') + vulnerability_object.uuid = ttp_id + for attribute in attributes: + vulnerability_object.add_attribute(*attribute) + self.misp_event.add_object(vulnerability_object) + + def _parse_weakness_object(self, weakness: Weakness, ttp_id: str): + attributes = [] + for key, relation in self._mapping.weakness_object_mapping().items(): + value = getattr(weakness, key) + if value: + attributes.append( + (relation, value if isinstance(value, str) else value.value) + ) + if attributes: + weakness_object = MISPObject('weakness') + weakness_object.uuid = ttp_id + for attribute in attributes: + weakness_object.add_attribute(*attribute) + self.misp_event.add_object(weakness_object) + + ############################################################################ + # MISP PARSING METHODS # + ############################################################################ + + # Parse STIX objects that we know will give MISP attributes + def _parse_misp_attribute_indicator(self, indicator: Indicator): + item = indicator.item + if item.observable: + misp_attribute = { + 'to_ids': True, 'category': str(indicator.relationship), + 'timestamp': self._timestamp_from_date(item.timestamp) + } + misp_attribute.update(self._sanitise_attribute_uuid(indicator.id_)) + observable = item.observable + self._parse_misp_attribute(observable, misp_attribute, indicator.id_, to_ids=True) + + def _parse_misp_attribute_observable(self, observable): + if observable.item: + misp_attribute = { + 'to_ids': False, 'category': str(observable.relationship) + } + misp_attribute.update( + self._sanitise_attribute_uuid(observable.item.id_) + ) + self._parse_misp_attribute(observable.item, misp_attribute, observable.id_) + + def _parse_misp_attribute( + self, observable: Observable, misp_attribute: dict, + stix_object_id: str, to_ids: Optional[bool] = False): + if getattr(observable.object_, 'properties', None) is not None: + properties = observable.object_.properties + try: + attribute_type, attribute_value, compl_data = self._handle_attribute_type( + properties, title=observable.title + ) + if isinstance(attribute_value, (str, int)): + self._handle_attribute_case(attribute_type, attribute_value, compl_data, misp_attribute) + else: + self._handle_object_case(attribute_type, attribute_value, compl_data, to_ids=to_ids) + except StixObjectTypeError as xsi_type: + self._stix_object_type_error(xsi_type, stix_object_id) + elif getattr(observable.observable_composition, 'observables', None) is not None: + attribute_dict = {} + for observables in observable.observable_composition.observables: + properties = observables.object_.properties + try: + attribute_type, attribute_value, _ = self._handle_attribute_type( + properties, observable_id=observable.id_ + ) + attribute_dict[attribute_type] = attribute_value + except StixObjectTypeError as xsi_type: + self._stix_object_type_error(xsi_type, stix_object_id) + if attribute_dict: + attribute_type, attribute_value = self._composite_type(attribute_dict) + self.misp_event.add_attribute(attribute_type, attribute_value, **misp_attribute) + + # Parse STIX object that we know will give MISP objects + def _parse_misp_object_indicator(self, indicator: Indicator): + name = self._define_name(indicator.item.observable, indicator.relationship) + if name == 'passive-dns' and str(indicator.relationship) != "misc": + self._add_error( + f'Unable to parse the Indicator object with id {indicator.id_}' + ) + else: + self._fill_misp_object(indicator.item, name, to_ids=True) + + def _parse_misp_object_observable(self, observable: Observable): + name = self._define_name(observable.item, observable.relationship) + try: + self._fill_misp_object(observable, name) + except Exception: + self._add_error( + 'Unable to parse the Observable ' + f'object with id {observable.id_}' + ) + + ############################################################################ + # MISP OBJECTS PARSING METHODS # + ############################################################################ + + # Create a MISP object, its attributes, and add it in the MISP event + def _fill_misp_object(self, item, name, to_ids=False): + composition = any( + ( + ( + hasattr(item, 'observable') and + hasattr(item.observable, 'observable_composition') and + item.observable.observable_composition + ), + ( + hasattr(item, 'observable_composition') and + item.observable_composition + ) + ) + ) + if composition: + misp_object = MISPObject(name, misp_objects_path_custom=_MISP_objects_path) + self._sanitise_object_uuid(misp_object, item.id_) + if to_ids: + observables = item.observable.observable_composition.observables + misp_object.timestamp = self._get_imestamp_from_date(item.timestamp) + else: + observables = item.observable_composition.observables + args = (misp_object, observables, to_ids) + self._handle_file_composition(*args) if name == 'file' else self._handle_composition(*args) + self.misp_event.add_object(**misp_object) + else: + properties = item.observable.object_.properties if to_ids else item.object_.properties + self._parse_observable_object(properties, to_ids, self._sanitise_uuid(item.id_)) + + def _handle_composition(self, misp_object, observables, to_ids): + for observable in observables: + properties = observable.object_.properties + try: + attribute = self._handle_attribute_type(properties) + except StixObjectTypeError as xsi_type: + self._stix_object_type_error(xsi_type, observable.id_) + continue + misp_attribute = MISPAttribute() + misp_attribute.type, misp_attribute.value, misp_attribute.object_relation = attribute + if 'Port' in observable.id_: + misp_attribute.object_relation = '-'.join( + ( + observable.id_.split('-')[0].split(':')[1][:3], + misp_attribute.object_relation + ) + ) + misp_attribute.to_ids = to_ids + misp_object.add_attribute(**misp_attribute) + return misp_object + + def _handle_file_composition(self, misp_object, observables, to_ids): + for observable in observables: + try: + attribute_type, attribute_value, compl_data = self._handle_attribute_type( + observable.object_.properties, title=observable.title + ) + except StixObjectTypeError as xsi_type: + self._stix_object_type_error(xsi_type, observable.id_) + continue + if isinstance(attribute_value, str): + misp_object.add_attribute( + **{ + 'type': attribute_type, 'value': attribute_value, + 'object_relation': attribute_type, 'to_ids': to_ids, + 'data': compl_data + } + ) + else: + for attribute in attribute_value: + attribute['to_ids'] = to_ids + misp_object.add_attribute(**attribute) + return misp_object + + # Create a MISP attribute and add it in its MISP object + def _parse_observable_object(self, properties, to_ids, uuid): + attribute_type, attribute_value, compl_data = self._handle_attribute_type(properties) + if isinstance(attribute_value, (str, int)): + attribute = {'to_ids': to_ids, 'uuid': uuid} + self._handle_attribute_case(attribute_type, attribute_value, compl_data, attribute) + else: + self._handle_object_case(attribute_type, attribute_value, compl_data, to_ids=to_ids, object_uuid=uuid) + + ############################################################################ + # UTILITY METHODS. # + ############################################################################ + + # Return type & value of a composite attribute in MISP + @staticmethod + def _composite_type(attributes: dict): + if "port" in attributes: + if "ip-src" in attributes: + return "ip-src|port", f"{attributes['ip-src']}|{attributes['port']}" + elif "ip-dst" in attributes: + return "ip-dst|port", f"{attributes['ip-dst']}|{attributes['port']}" + elif "hostname" in attributes: + return "hostname|port", f"{attributes['hostname']}|{attributes['port']}" + elif "domain" in attributes: + if "ip-src" in attributes: + ip_value = attributes["ip-src"] + elif "ip-dst" in attributes: + ip_value = attributes["ip-dst"] + return "domain|ip", f"{attributes['domain']}|{ip_value}" + + def _define_name(self, observable: Observable, relationship): + observable_id = observable.id_ + if relationship == "file": + return "registry-key" if "WinRegistryKey" in observable_id else "file" + if "Custom" in observable_id: + return observable_id.split("Custom")[0].split(":")[1] + if relationship == "network" and "ObservableComposition" in observable_id: + return observable_id.split("_")[0].split(":")[1] + return self._mapping.cybox_to_misp_object()[observable_id.split('-')[0].split(':')[1]] + + def _get_event_info(self): + if hasattr(self._event, 'title'): + return self._event.title + if hasattr(getattr(self._event, 'stix_header', None), 'title'): + return self.event.stix_header.title + return f"Imported from STIX {self.stix_version} Package generated with MISP" diff --git a/misp_stix_converter/stix2misp/internal_stix2_to_misp.py b/misp_stix_converter/stix2misp/internal_stix2_to_misp.py index d4247c2..9d5bba8 100644 --- a/misp_stix_converter/stix2misp/internal_stix2_to_misp.py +++ b/misp_stix_converter/stix2misp/internal_stix2_to_misp.py @@ -17,7 +17,7 @@ from pymisp import MISPSighting from stix2.v20.sdo import CustomObject as CustomObject_v20 from stix2.v21.sdo import CustomObject as CustomObject_v21 -from typing import Optional, Union +from typing import Union _CUSTOM_TYPING = Union[ CustomObject_v20, @@ -26,14 +26,8 @@ class InternalSTIX2toMISPParser(STIX2toMISPParser): - def __init__(self, distribution: Optional[int] = 0, - sharing_group_id: Optional[int] = None, - title: Optional[str] = None, - producer: Optional[str] = None, - galaxies_as_tags: Optional[bool] = False): - super().__init__( - distribution, sharing_group_id, title, producer, galaxies_as_tags - ) + def __init__(self): + super().__init__() self._mapping = InternalSTIX2toMISPMapping # parsers self._attack_pattern_parser: InternalSTIX2AttackPatternConverter @@ -52,6 +46,14 @@ def __init__(self, distribution: Optional[int] = 0, self._tool_parser: InternalSTIX2ToolConverter self._vulnerability_parser: InternalSTIX2VulnerabilityConverter + def parse_stix_bundle(self, **kwargs): + self._set_parameters(**kwargs) + self._parse_stix_bundle() + + ############################################################################ + # PROPERTIES # + ############################################################################ + @property def custom_object_parser(self) -> STIX2CustomObjectConverter: if not hasattr(self, '_custom_object_parser'): @@ -70,6 +72,10 @@ def observed_data_parser(self) -> InternalSTIX2ObservedDataConverter: self, '_observed_data_parser', self._set_observed_data_parser() ) + ############################################################################ + # PARSER SETTERS # + ############################################################################ + def _set_attack_pattern_parser(self) -> InternalSTIX2AttackPatternConverter: self._attack_pattern_parser = InternalSTIX2AttackPatternConverter(self) diff --git a/misp_stix_converter/stix2misp/stix1_mapping.py b/misp_stix_converter/stix2misp/stix1_mapping.py index 8d9a75b..08ed1a2 100644 --- a/misp_stix_converter/stix2misp/stix1_mapping.py +++ b/misp_stix_converter/stix2misp/stix1_mapping.py @@ -2,17 +2,302 @@ # -*- coding: utf-8 -*- from .. import Mapping +from typing import Union -class STIX1Mapping: - def __init__(self): - self.__threat_level_mapping = Mapping( - High = '1', - Medium = '2', - Low = '3', - Undefined = '4' - ) +class STIX1toMISPMapping: + __attribute_types_mapping = Mapping( + AccountObjectType = '_handle_credential', + AddressObjectType = '_handle_address', + ArtifactObjectType = '_handle_attachment', + ASObjectType = '_handle_as', + CustomObjectType = '_handle_custom', + DNSRecordObjectType = '_handle_dns', + DomainNameObjectType = '_handle_domain_or_url', + EmailMessageObjectType = '_handle_email', + FileObjectType = '_handle_file', + HostnameObjectType = '_handle_hostname', + HTTPSessionObjectType = '_handle_http', + LinkObjectType = '_handle_link', + MutexObjectType = '_handle_mutex', + NetworkConnectionObjectType = '_handle_network_connection', + NetworkSocketObjectType = '_handle_network_socket', + PDFFileObjectType = '_handle_file', + PipeObjectType = '_handle_pipe', + PortObjectType = '_handle_port', + ProcessObjectType = '_handle_process', + SocketAddressObjectType = '_handle_socket_address', + SystemObjectType = '_handle_system', + UnixUserAccountObjectType = '_handle_unix_user', + URIObjectType = '_handle_domain_or_url', + UserAccountObjectType = '_handle_user', + WhoisObjectType = '_handle_whois', + WindowsExecutableFileObjectType = '_handle_pe', + WindowsFileObjectType = '_handle_file', + WindowsRegistryKeyObjectType = '_handle_regkey', + WindowsServiceObjectType = '_handle_windows_service', + WindowsUserAccountObjectType = '_handle_windows_user', + X509CertificateObjectType = '_handle_x509' + ) + _file_attribute_type = ('filename', 'filename') + __event_types = Mapping( + ArtifactObjectType = {"type": "attachment", "relation": "attachment"}, + DomainNameObjectType = {"type": "domain", "relation": "domain"}, + FileObjectType = _file_attribute_type, + HostnameObjectType = {"type": "hostname", "relation": "host"}, + MutexObjectType = {"type": "mutex", "relation": "mutex"}, + PDFFileObjectType = _file_attribute_type, + PortObjectType = {"type": "port", "relation": "port"}, + URIObjectType = {"type": "url", "relation": "url"}, + WindowsFileObjectType = _file_attribute_type, + WindowsExecutableFileObjectType = _file_attribute_type, + WindowsRegistryKeyObjectType = {"type": "regkey", "relation": ""} + ) - @property - def threat_level_mapping(self) -> dict: - return self.__threat_level_mapping \ No newline at end of file + # Objects mappings + _AS_attribute = ('AS', 'asn') + __as_mapping = Mapping( + number = _AS_attribute, + handle = _AS_attribute, + name = ('text', 'description') + ) + __credential_authentication_mapping = Mapping( + authentication_type = ('text', 'value', 'type'), + authentication_data = ('text', 'value', 'password'), + structured_authentication_mechanism = ('text', 'description.value', 'format') + ) + __credential_custom_types = ("username", "origin", "notification") + __email_mapping = Mapping( + boundary = ("email-mime-boundary", 'value', "mime-boundary"), + from_ = ("email-src", "address_value.value", "from"), + message_id = ("email-message-id", "value", "message-id"), + reply_to = ("email-reply-to", 'address_value.value', "reply-to"), + subject = ("email-subject", 'value', "subject"), + user_agent = ("text", 'value', "user-agent"), + x_mailer = ("email-x-mailer", 'value', "x-mailer") + ) + _file_mapping = Mapping( + file_path = ('text', 'file_path.value', 'path'), + full_path = ('text', 'full_path.value', 'fullpath'), + file_format = ('mime-type', 'file_format.value', 'mimetype'), + byte_runs = ('pattern-in-file', 'byte_runs[0].byte_run_data', 'pattern-in-file'), + size_in_bytes = ('size-in-bytes', 'size_in_bytes.value', 'size-in-bytes'), + peak_entropy = ('float', 'peak_entropy.value', 'entropy') + ) + __network_connection_fields = ('source_socket_address', 'destination_socket_address') + __network_fields = ('src', 'dst') + __network_reference_mapping = Mapping( + ip_address = ('ip-{}', 'address_value', 'ip-{}'), + port = ('port', 'port_value', '{}-port'), + hostname = ('hostname', 'hostname_value', 'hostname-{}') + ) + __network_socket_fields = ('local_address', 'remote_address') + __network_socket_mapping = Mapping( + protocol = ('text', 'protocol.value', 'protocol'), + address_family = ('text', 'address_family.value', 'address-family'), + domain = ('text', 'domain.value', 'domain-family') + ) + __pe_header_mapping = Mapping( + characteristics = ('hex', 'characteristics-hex'), + machine = ('hex', 'machine-hex'), + number_of_sections = ('counter', 'number-of-sections'), + pointer_to_symbol_table = ('hex', 'pointer-to-symbol-table'), + size_of_optional_header = ('counter', 'size-of-optional-header') + ) + __pe_mapping = Mapping( + **{ + 'file_name': ('filename', 'original-filename'), + 'type': ('text', 'type') + } + ) + __process_mapping = Mapping( + creation_time = ('datetime', 'creation-time'), + start_time = ('datetime', 'start-time'), + name = ('text', 'name'), + pid = ('text', 'pid'), + parent_pid = ('text', 'parent-pid') + ) + __regkey_mapping = Mapping( + **{'hive': ('text', 'hive'), 'key': ('regkey', 'key')} + ) + __regkey_value_mapping = Mapping( + data = ('text', 'data'), + datatype = ('text', 'data-type'), + name = ('text', 'name') + ) + __user_account_object_mapping = Mapping( + username = ('text', 'username'), + full_name = ('text', 'display-name'), + disabled = ('boolean', 'disabled'), + creation_date = ('datetime', 'created'), + last_login = ('datetime', 'last_login'), + home_directory = ('text', 'home_dir'), + script_path = ('text', 'shell') + ) + __whois_mapping = Mapping( + registrar_info = ('whois-registrar', 'value', 'whois-registrar'), + ip_address = ('ip-src', 'address_value.value', 'ip-address'), + domain_name = ('domain', 'value.value', 'domain') + ) + __whois_registrant_mapping = Mapping( + email_address = ('whois-registrant-email', 'address_value.value', 'registrant-email'), + name = ('whois-registrant-name', 'value', 'registrant-name'), + phone_number = ('whois-registrant-phone', 'value', 'registrant-phone'), + organization = ('whois-registrant-org', 'value', 'registrant-org') + ) + __x509_certificate_types = ('version', 'serial_number', 'issuer', 'subject') + __x509_datetime_types = ('not_before', 'not_after') + __x509_pubkey_types = ('exponent', 'modulus') + + @classmethod + def as_mapping(cls) -> dict: + return cls.__as_mapping + + @classmethod + def attribute_types_mapping(cls, object_type: str) -> Union[str, None]: + return cls.__attribute_types_mapping.get(object_type) + + @classmethod + def credential_authentication_mapping(cls) -> dict: + return cls.__credential_authentication_mapping + + @classmethod + def credential_custom_types(cls) -> tuple: + return cls.__credential_custom_types + + @classmethod + def email_mapping(cls) -> dict: + return cls.__email_mapping + + @classmethod + def event_types(cls, object_type: str) -> Union[dict, None]: + return cls.__event_types.get(object_type) + + @classmethod + def file_mapping(cls) -> dict: + return cls._file_mapping + + @classmethod + def network_fields(cls) -> tuple: + return cls.__network_fields + + @classmethod + def network_connection_fields(cls) -> tuple: + return cls.__network_connection_fields + + @classmethod + def network_reference_mapping(cls) -> dict: + return cls.__network_reference_mapping + + @classmethod + def network_socket_fields(cls) -> tuple: + return cls.__network_socket_fields + + @classmethod + def network_socket_mapping(cls) -> dict: + return cls.__network_socket_mapping + + @classmethod + def pe_header_mapping(cls) -> dict: + return cls.__pe_header_mapping + + @classmethod + def pe_mapping(cls) -> dict: + return cls.__pe_mapping + + @classmethod + def process_mapping(cls) -> dict: + return cls.__process_mapping + + @classmethod + def regkey_mapping(cls) -> dict: + return cls.__regkey_mapping + + @classmethod + def regkey_value_mapping(cls) -> dict: + return cls.__regkey_value_mapping + + @classmethod + def user_account_object_mapping(cls) -> dict: + return cls.__user_account_object_mapping + + @classmethod + def whois_mapping(cls) -> dict: + return cls.__whois_mapping + + @classmethod + def whois_registrant_mapping(cls) -> dict: + return cls.__whois_registrant_mapping + + @classmethod + def x509_certificate_types(cls) -> tuple: + return cls.__x509_certificate_types + + @classmethod + def x509_datetime_types(cls) -> tuple: + return cls.__x509_datetime_types + + @classmethod + def x509_pubkey_types(cls) -> tuple: + return cls.__x509_pubkey_types + + +class ExternalSTIX1toMISPMapping(STIX1toMISPMapping): + __marking_mapping = Mapping( + **{ + 'AIS:AISMarkingStructure': '_parse_AIS_marking', + 'tlpMarking:TLPMarkingStructureType': '_parse_TLP_marking' + } + ) + __test_mechanism_mapping = Mapping( + **{ + 'yaraTM:YaraTestMechanismType': 'yara' + } + ) + + @classmethod + def marking_mapping(cls, marking_type: str) -> Union[str, None]: + return cls.__marking_mapping.get(marking_type) + + @classmethod + def test_mechanism_mapping(cls, test_mechanism_type: str) -> Union[str, None]: + return cls.__test_mechanism_mapping.get(test_mechanism_type) + + +class InternalSTIX1toMISPMapping(STIX1toMISPMapping): + __attack_pattern_object_mapping = Mapping( + capec_id = 'id', + title = 'name', + description = 'summary' + ) + __threat_level_mapping = Mapping( + High = '1', + Medium = '2', + Low = '3', + Undefined = '4' + ) + __vulnerability_object_mapping = Mapping( + cve_id = ('vulnerability', 'id'), + description = ('text', 'summary'), + published_datetime = ('datetime', 'published') + ) + __weakness_object_mapping = Mapping( + cwe_id = 'id', + description = 'description' + ) + + @classmethod + def attack_pattern_object_mappin(cls) -> dict: + return cls.__attack_pattern_object_mapping + + @classmethod + def threat_level_mapping(cls, threat_level: str) -> Union[str, None]: + return cls.__threat_level_mapping.get(threat_level) + + @classmethod + def vulnerability_object_mapping(cls) -> dict: + return cls.__vulnerability_object_mapping + + @classmethod + def weakness_object_mapping(cls) -> dict: + return cls.__weakness_object_mapping diff --git a/misp_stix_converter/stix2misp/stix1_to_misp.py b/misp_stix_converter/stix2misp/stix1_to_misp.py index 4e02a58..21bd213 100644 --- a/misp_stix_converter/stix2misp/stix1_to_misp.py +++ b/misp_stix_converter/stix2misp/stix1_to_misp.py @@ -1,9 +1,693 @@ # -*- coding: utf-8 -*- #!/usr/bin/env python3 -from .importparser import STIXtoMISPParser +from .importparser import STIXtoMISPParser, _load_stix1_package +from .stix1_mapping import STIX1toMISPMapping +from abc import ABCMeta +from base64 import b64decode, b64encode +from collections import defaultdict +from cybox.common import Hash +from cybox.objects import ( + account_object, address_object, artifact_object, as_object, + email_message_object, dns_record_object, domain_name_object, file_object, + hostname_object, http_session_object, link_object, mutex_object, + network_connection_object, network_socket_object, pipe_object, + process_object, socket_address_object, system_object, uri_object, + unix_user_account_object, user_account_object, whois_object, + win_executable_file_object, win_registry_key_object, win_service_object, + win_user_account_object, x509_certificate_object) +from operator import attrgetter +from pathlib import Path +from pymisp.abstract import misp_objects_path +from pymisp import MISPAttribute, MISPEvent, MISPObject +from stix.coa import CourseOfAction +from stix.core import STIXPackage +from stix.threat_actor import ThreatActor +from typing import Union +from uuid import uuid4 +_ADDRESS_TYPING = Union[address_object.Address, address_object.EmailAddress] +_NETWORK_PROPERTIES_TYPING = Union[ + network_connection_object.NetworkConnection, + network_socket_object.NetworkSocket +] +_PROPERTIES_TYPING = Union[ + account_object.Authentication, email_message_object.EmailHeader, + whois_object.WhoisEntry, whois_object.WhoisRegistrant +] +_PARTIAL_PROPERTIES_TYPING = Union[ + as_object.AS, process_object.Process, user_account_object.UserAccount, + win_executable_file_object.WinExecutableFile, win_registry_key_object.WinRegistryKey, + win_registry_key_object.RegistryValue +] +_SIMPLE_PROPERTIES_TYPING = Union[ + file_object.File, network_socket_object.NetworkSocket +] +_STIX_OBJECT_TYPING = Union[CourseOfAction, ThreatActor] -class STIX1toMISPParser(STIXtoMISPParser): + +class StixObjectTypeError(Exception): + pass + + +class STIX1toMISPParser(STIXtoMISPParser, metaclass=ABCMeta): def __init__(self): - super.__init__() \ No newline at end of file + super().__init__() + self.__galaxies = set() + self.__references = defaultdict(list) + + def load_stix_package(self, stix_package: STIXPackage): + self.__stix_package = stix_package + + def parse_stix_content(self, filename: Union[Path, str], **kwargs): + self.__stix_package = _load_stix1_package(filename) + self.parse_stix_package(**kwargs) + + ############################################################################ + # PROPERTIES # + ############################################################################ + + @property + def galaxies(self) -> set: + return self.__galaxies + + @property + def references(self) -> dict: + return self.__references + + @property + def stix_package(self) -> STIXPackage: + return self.__stix_package + + @property + def stix_version(self) -> str: + return getattr(self.__stix_package, 'stix_version', '1.1.1') + + ############################################################################ + # PARSING METHODS USED BY BOTH CHILD CLASSES # + ############################################################################ + + # Define type & value of an attribute or object in MISP + def _handle_attribute_type(self, properties, is_object=False, title=None): + xsi_type = properties._XSI_TYPE + args = [properties] + if xsi_type in ("FileObjectType", "PDFFileObjectType", "WindowsFileObjectType"): + args.append(is_object) + elif xsi_type == "ArtifactObjectType": + args.append(title) + parser = self._mapping.attribute_types_mapping(xsi_type) + if parser is None: + raise StixObjectTypeError(xsi_type) + return getattr(self, parser)(*args) + + def _handle_attribute_case(self, attribute_type, attribute_value, data, attribute): + if attribute_type in ('attachment', 'malware-sample'): + attribute['data'] = data + elif attribute_type == 'text': + attribute['comment'] = data + self.misp_event.add_attribute(attribute_type, attribute_value, **attribute) + + # The value returned by the indicators or observables parser is a list of dictionaries + # These dictionaries are the attributes we add in an object, itself added in the MISP event + def _handle_object_case(self, name, attribute_value, compl_data, to_ids=False, object_uuid=None, test_mechanisms=[]): + misp_object = MISPObject(name, misp_objects_path_custom=misp_objects_path) + if object_uuid: + misp_object.uuid = object_uuid + for attribute in attribute_value: + attribute['to_ids'] = to_ids + misp_object.add_attribute(**attribute) + if isinstance(compl_data, dict): + # if some complementary data is a dictionary containing an uuid, + # it means we are using it to add an object reference + if "pe_uuid" in compl_data: + misp_object.add_reference(compl_data['pe_uuid'], 'includes') + if "process_uuid" in compl_data: + for uuid in compl_data["process_uuid"]: + misp_object.add_reference(uuid, 'connected-to') + if test_mechanisms: + for test_mechanism in test_mechanisms: + misp_object.add_reference(test_mechanism, 'detected-with') + self.misp_event.add_object(misp_object) + + # Parse a course of action and add a MISP object to the event + def parse_course_of_action(self, course_of_action): + misp_object = MISPObject('course-of-action', misp_objects_path_custom=misp_objects_path) + misp_object.uuid = self.fetch_uuid(course_of_action.id_) + if course_of_action.title: + attribute = {'type': 'text', 'object_relation': 'name', + 'value': course_of_action.title} + misp_object.add_attribute(**attribute) + for prop, properties_key in self._mapping._coa_mapping().items(): + if getattr(course_of_action, prop): + attribute = { + 'type': 'text', 'object_relation': prop.replace('_', ''), + 'value': attrgetter('{}.{}'.format(prop, properties_key))(course_of_action) + } + misp_object.add_attribute(**attribute) + if course_of_action.parameter_observables: + for observable in course_of_action.parameter_observables.observables: + properties = observable.object_.properties + attribute = MISPAttribute() + attribute.type, attribute.value, _ = self.handle_attribute_type(properties) + referenced_uuid = str(uuid4()) + attribute.uuid = referenced_uuid + self.misp_event.add_attribute(**attribute) + misp_object.add_reference(referenced_uuid, 'observable', None, **attribute) + self.misp_event.add_object(misp_object) + + ############################################################################ + # OBSERVABLE OBJECTS PARSING METHODS # + ############################################################################ + + @staticmethod + def _handle_address(properties: _ADDRESS_TYPING) -> tuple: + if properties.category == 'e-mail': + return 'email-src', properties.address_value.value, 'from' + return "ip-src" if properties.is_source else "ip-dst", properties.address_value.value, 'ip' + + def _handle_as(self, properties: as_object.AS) -> tuple: + attributes = tuple( + self._fetch_attributes_with_partial_key_parsing(properties, 'as_mapping') + ) + return attributes[0] if len(attributes) == 1 else ('asn', self._return_object_attributes(attributes), '') + + # Return type & value of an attachment attribute + def _handle_attachment(self, properties: artifact_object.Artifact, title: str) -> tuple: + if properties.hashes: + return "malware-sample", f"{title}|{properties.hashes[0]}", properties.raw_artifact.value + return self._mapping.event_types(properties._XSI_TYPE)['type'], title, properties.raw_artifact.value + + # Return type & attributes of a credential object + def _handle_credential(self, properties: account_object.Account) -> tuple: + attributes = [] + if properties.description: + attributes.append(["text", properties.description.value, "text"]) + if properties.authentication: + for authentication in properties.authentication: + attributes.extend( + self._fetch_attributes_with_key_parsing(authentication, 'credential_authentication_mapping') + ) + if properties.custom_properties: + for prop in properties.custom_properties: + if prop.name in self._mapping.credential_custom_types: + attributes.append(['text', prop.value, prop.name]) + return attributes[0] if len(attributes) == 1 else ("credential", self._return_object_attributes(attributes), "") + + # Return type & attributes of a dns object + def _handle_dns(self, properties: dns_record_object.DNSRecord) -> tuple: + relation = [] + if properties.domain_name: + relation.append(["domain", str(properties.domain_name.value), ""]) + if properties.ip_address: + relation.append(["ip-dst", str(properties.ip_address.value), ""]) + if relation: + if len(relation) == '2': + domain = relation[0][1] + ip = relation[1][1] + attributes = [["text", domain, "rrname"], ["text", ip, "rdata"]] + rrtype = "AAAA" if ":" in ip else "A" + attributes.append(["text", rrtype, "rrtype"]) + return "passive-dns", self._return_object_attributes(attributes), "" + return relation[0] + + # Return type & value of a domain or url attribute + def _handle_domain_or_url(self, properties: Union[domain_name_object.DomainName, uri_object.URI]) -> tuple: + event_types = self._mapping.event_types(properties._XSI_TYPE) + return event_types['type'], properties.value.value, event_types['relation'] + + # Return type & value of an email attribute + def _handle_email(self, properties: email_message_object.EmailMessage) -> tuple: + if properties.header: + header = properties.header + attributes = list(self._fetch_attributes_with_key_parsing(header, 'email_mapping')) + if header.to: + for to in header.to: + attributes.append(["email-dst", to.address_value.value, "to"]) + if header.cc: + for cc in header.cc: + attributes.append(["email-dst", cc.address_value.value, "cc"]) + else: + attributes = [] + if properties.attachments: + attributes.extend(self._handle_email_attachment(properties)) + return attributes[0] if len(attributes) == 1 else ("email", self._return_object_attributes(attributes), "") + + # Return type & value of an email attachment + def _handle_email_attachment(self, properties: email_message_object.EmailMessage): + related_objects = ( + {related.id_: related.properties for related in properties.parent.related_objects} + if properties.parent.related_objects else {} + ) + for attachment in (attachment.object_reference for attachment in properties.attachments): + if attachment in related_objects: + yield ("email-attachment", related_objects[attachment].file_name.value, "attachment") + else: + parent_id = self._sanitise_uuid(properties.parent.id_) + referenced_id = self._sanitise_uuid(attachment) + self.references[parent_id].append( + {'idref': referenced_id, 'relationship': 'attachment'} + ) + + # Return type & attributes of a file object + def _handle_file(self, properties: file_object.File, is_object: bool) -> tuple: + b_hash, b_file = False, False + attributes = list(self._fetch_attributes_with_keys(properties, 'file_mapping')) + if properties.hashes: + b_hash = True + for hash_property in properties.hashes: + attributes.append(self._handle_hashes_attribute(hash_property)) + if properties.file_name: + value = properties.file_name.value + if value: + b_file = True + attribute_type, relation = self._mapping.event_types(properties._XSI_TYPE) + attributes.append([attribute_type, value, relation]) + if len(attributes) == 1: + attribute = attributes[0] + return attribute[0] if attribute[2] != "fullpath" else "filename", attribute[1], "" + if len(attributes) == 2: + if b_hash and b_file: + return self._handle_filename_object(attributes, is_object) + path, filename = self._handle_filename_path_case(attributes) + if path and filename: + attribute_value = f"{path}\\{filename}" + if '\\' in filename and path == filename: + attribute_value = filename + return "filename", attribute_value, "" + return "file", self._return_object_attributes(attributes), "" + + # Determine path & filename from a complete path or filename attribute + @staticmethod + def _handle_filename_path_case(attributes: list) -> tuple: + path, filename = [""] * 2 + if attributes[0][2] == 'filename' and attributes[1][2] == 'path': + path = attributes[1][1] + filename = attributes[0][1] + elif attributes[0][2] == 'path' and attributes[1][2] == 'filename': + path = attributes[0][1] + filename = attributes[1][1] + return path, filename + + # Return the appropriate type & value when we have 1 filename & 1 hash value + @staticmethod + def _handle_filename_object(attributes: list, is_object: bool) -> tuple: + for attribute in attributes: + attribute_type, attribute_value, _ = attribute + if attribute_type == "filename": + filename_value = attribute_value + else: + hash_type, hash_value = attribute_type, attribute_value + value = f"{filename_value}|{hash_value}" + if is_object: + # file object attributes cannot be filename|hash, so it is malware-sample + attr_type = "malware-sample" + return attr_type, value, attr_type + # it could be malware-sample as well, but STIX is losing this information + return f"filename|{hash_type}", value, "" + + # Return type & value of a hash attribute + @staticmethod + def _handle_hashes_attribute(hash_property: Hash) -> tuple: + hash_type = hash_property.type_.value.lower() + try: + hash_value = hash_property.simple_hash_value.value + except AttributeError: + hash_value = hash_property.fuzzy_hash_value.value + return hash_type, hash_value, hash_type + + # Return type & value of a hostname attribute + def _handle_hostname(self, properties: hostname_object.Hostname) -> tuple: + event_types = self._mapping.event_types(properties._XSI_TYPE) + return event_types['type'], properties.hostname_value.value, event_types['relation'] + + # Return type & value of a http request attribute + @staticmethod + def _handle_http(properties: http_session_object.HTTPSession) -> tuple: + client_request = properties.http_request_response[0].http_client_request + if client_request.http_request_header: + request_header = client_request.http_request_header + if request_header.parsed_header: + value = request_header.parsed_header.user_agent.value + return "user-agent", value, "user-agent" + elif request_header.raw_header: + value = request_header.raw_header.value + return "http-method", value, "method" + elif client_request.http_request_line: + value = client_request.http_request_line.http_method.value + return "http-method", value, "method" + + # Return type & value of a link attribute + @staticmethod + def _handle_link(properties: link_object.Link) -> tuple: + return "link", properties.value.value, "link" + + # Return type & value of a mutex attribute + def _handle_mutex(self, properties: mutex_object.Mutex) -> tuple: + event_types = self._mapping.event_types(properties._XSI_TYPE) + return event_types['type'], properties.name.value, event_types['relation'] + + def _handle_network(self, properties: _NETWORK_PROPERTIES_TYPING, mapping: str): + for feature, field in zip(self._mapping.network_fields(), getattr(self._mapping, mapping)()): + address_property = getattr(properties, field) + if address_property is None: + continue + for prop, attribute in self._mapping.network_reference_mapping().items(): + if getattr(address_property, prop): + attribute_type, key, relation = attribute + yield ( + attribute_type.format(feature), + attrgetter(f'{prop}.{key}.value')(address_property), + relation.format(feature) + ) + + # Return type & attributes of a network connection object + def _handle_network_connection(self, properties: network_connection_object.NetworkConnection) -> tuple: + attributes = list(self._handle_network(properties, 'network_connection_addresses')) + for feature in ('layer3_protocol', 'layer4_protocol', 'layer7_protocol'): + if getattr(properties, feature): + attributes.append( + ('text', attrgetter(f"{feature}.value")(properties), feature.replace('_', '-')) + ) + if attributes: + return "network-connection", self._return_object_attributes(attributes), "" + + # Return type & attributes of a network socket objet + def _handle_network_socket(self, properties: network_socket_object.NetworkSocket) -> tuple: + attributes = list(self._handle_network(properties, 'network_socket_addresses')) + attributes.extend(self._fetch_attributes_with_keys(properties, 'network_socket_mapping')) + for prop in ('is_listening', 'is_blocking'): + if getattr(properties, prop): + attributes.append(("text", prop.split('_')[1], "state")) + if attributes: + return "network-socket", self._return_object_attributes(attributes), "" + + # Return type & attributes of the file defining a portable executable object + def _handle_pe(self, properties: win_executable_file_object.WinExecutableFile) -> tuple: + pe_object = MISPObject('pe', misp_objects_path_custom=misp_objects_path) + for attribute in self._fetch_attributes_with_partial_key_parsing(properties, 'pe_mapping'): + attribute_type, value, relation = attribute + pe_object.add_attribute(relation, value, type=attribute_type) + if getattr(properties.headers, 'file_header', None) is not None: + header = properties.headers.file_header + for attribute in self._fetch_attributes_with_partial_key_parsing(header, 'pe_header_mapping'): + attribute_type, value, relation = attribute + pe_object.add_attribute(relation, value, type=attribute_type) + misp_object = self.misp_event.add_object(pe_object) + if properties.sections: + for section in properties.sections: + section_uuid = self._handle_pe_section(section) + misp_object.add_reference(section_uuid, 'includes') + file_type, file_value, _ = self._handle_file(properties, False) + return file_type, file_value, {'pe_uuid': misp_object.uuid} + + def _handle_pe_section(self, section: win_executable_file_object.PESection) -> str: + section_object = MISPObject('pe-section', misp_objects_path_custom=misp_objects_path) + header_hashes = section.header_hashes + if header_hashes is None: + header_hashes = section.data_hashes + for _hash in header_hashes: + hash_type, hash_value, _ = self._handle_hashes_attribute(_hash) + section_object.add_attribute(hash_type, hash_value) + if section.entropy: + section_object.add_attribute("entropy", section.entropy.value.value) + if section.section_header: + section_header = section.section_header + section_object.add_attribute("name", section_header.name.value) + section_object.add_attribute("size-in-bytes", section_header.size_of_raw_data.value) + return self.misp_event.add_object(section_object).uuid + + # Return type & value of a names pipe attribute + @staticmethod + def _handle_pipe(properties: pipe_object.Pipe) -> tuple: + return "named pipe", properties.name.value, "" + + # Return type & value of a port attribute + def _handle_port(self, *args): + properties = args[0] + event_types = self._mapping.event_types(properties._XSI_TYPE) + relation = event_types['relation'] + if len(args) > 1: + observable_id = args[1] + if "srcPort" in observable_id: + return event_types['type'], properties.port_value.value, f"src-{relation}" + if "dstPort" in observable_id: + return event_types['type'], properties.port_value.value, f"dst-{relation}" + return event_types['type'], properties.port_value.value, relation + + # Return type & attributes of a process object + def _handle_process(self, properties: process_object.Process): + attributes = list( + self._fetch_attributes_with_partial_key_parsing( + properties, '_process_mapping' + ) + ) + if properties.child_pid_list: + for child in properties.child_pid_list: + attributes.append(["text", child.value, "child-pid"]) + if properties.port_list: + for port in properties.port_list: + attributes.append(["port", port.port_value.value, "port"]) + if properties.image_info: + if properties.image_info.file_name: + attributes.append(["filename", properties.image_info.file_name.value, "image"]) + if properties.image_info.command_line: + attributes.append(["text", properties.image_info.command_line.value, "command-line"]) + if properties.network_connection_list: + references = [] + for connection in properties.network_connection_list: + object_name, object_attributes, _ = self._handle_network_connection(connection) + misp_object = MISPObject(object_name, misp_objects_path_custom=misp_objects_path) + for attribute in object_attributes: + misp_object.add_attribute(**attribute) + self.misp_event.add_object(**misp_object) + references.append(misp_object.uuid) + return "process", self._return_object_attributes(attributes), {"process_uuid": references} + return "process", self._return_object_attributes(attributes), "" + + # Return type & value of a regkey attribute + def _handle_regkey(self, properties: win_registry_key_object.WinRegistryKey): + attributes = list( + self._fetch_attributes_with_partial_key_parsing( + properties, '_regkey_mapping' + ) + ) + if properties.values: + value = properties.values[0] + attributes.extend( + self._fetch_attributes_with_partial_key_parsing( + value, '_regkey_value_mapping' + ) + ) + if len(attributes) in (2,3): + d_regkey = {key: value for (_, value, key) in attributes} + if 'hive' in d_regkey and 'key' in d_regkey: + regkey = f"{d_regkey['hive']}\\{d_regkey['key']}" + if 'data' in d_regkey: + return "regkey|value", f"{regkey} | {d_regkey['data']}", "" + return "regkey", regkey, "" + return "registry-key", self._return_object_attributes(attributes), "" + + # Parse a socket address object in order to return type & value + # of a composite attribute ip|port or hostname|port + def _handle_socket_address(self, properties: socket_address_object.SocketAddress) -> tuple: + if properties.ip_address: + type1, value1, _ = self._handle_address(properties.ip_address) + elif properties.hostname: + type1 = "hostname" + value1 = properties.hostname.hostname_value.value + if properties.port: + return f"{type1}|port", f"{value1}|{properties.port.port_value.value}", "" + return type1, value1, '' + + # Parse a system object to extract a mac-address attribute + @staticmethod + def _handle_system(properties: system_object.System) -> tuple: + if properties.network_interface_list: + return "mac-address", str(properties.network_interface_list[0].mac), "" + + # Parse a UNIX user account object + def _handle_unix_user(self, properties: unix_user_account_object.UnixUserAccount) -> tuple: + attributes = list( + self._fetch_attributes_with_partial_key_parsing( + properties, 'user_account_object_mapping' + ) + ) + if properties.user_id: + attributes.append(['text', properties.user_id.value, 'user-id']) + if properties.group_id: + attributes.append(['text', properties.group_id.value, 'group-id']) + return 'user-account', self._return_object_attributes(attributes), '' + + # Parse a user account object + def _handle_user(self, properties: user_account_object.UserAccount) -> tuple: + attributes = tuple( + self._fetch_attributes_with_partial_key_parsing( + properties, 'user_account_object_mapping' + ) + ) + return 'user-account', self.return_attributes(attributes), '' + + # Parse a whois object: + # Return type & attributes of a whois object if we have the required fields + # Otherwise create attributes and return type & value of the last attribute to avoid crashing the parent function + def _handle_whois(self, properties: whois_object.WhoisEntry): + attributes = list(self._fetch_attributes_with_key_parsing(properties, '_whois_mapping')) + required_one_of = True if attributes else False + if properties.registrants: + registrant = properties.registrants[0] + attributes.append(self._fetch_attributes_with_key_parsing(registrant, '_whois_registrant_mapping')) + if properties.creation_date: + attributes.append(("datetime", properties.creation_date.value.strftime('%Y-%m-%d'), "creation-date")) + required_one_of = True + if properties.updated_date: + attributes.append(("datetime", properties.updated_date.value.strftime('%Y-%m-%d'), "modification-date")) + if properties.expiration_date: + attributes.append(("datetime", properties.expiration_date.value.strftime('%Y-%m-%d'), "expiration-date")) + if properties.nameservers: + for nameserver in properties.nameservers: + attributes.append(("hostname", nameserver.value.value, "nameserver")) + if properties.remarks: + attribute_type = "text" + relation = "comment" if attributes else attribute_type + attributes.append([attribute_type, properties.remarks.value, relation]) + required_one_of = True + # Testing if we have the required attribute types for Object whois + if required_one_of: + # if yes, we return the object type and the attributes + return "whois", self._return_object_attributes(attributes), "" + # otherwise, attributes are added in the event, and one attribute is returned to not make the function crash + if len(attributes) == 1: + return attributes[0] + last_attribute = attributes.pop(-1) + for attribute in attributes: + attribute_type, attribute_value, attribute_relation = attribute + misp_attributes = {"comment": f"Whois {attribute_relation}"} + self.misp_event.add_attribute(attribute_type, attribute_value, **misp_attributes) + return last_attribute + + # Return type & value of a windows service object + @staticmethod + def _handle_windows_service(properties: win_service_object.WinService) -> tuple: + if properties.name: + return "windows-service-name", properties.name.value, "" + + # Parse a windows user account object + def _handle_windows_user(self, properties: win_user_account_object.WinUser) -> tuple: + attributes = list( + self._fetch_attributes_with_partial_key_parsing( + properties, 'user_account_object_mapping' + ) + ) + if properties.security_id: + attributes.append(['text', properties.security_id.value, 'user-id']) + return 'user-account', self._return_object_attributes(attributes), '' + + def _handle_x509(self, properties: x509_certificate_object.X509Certificate) -> tuple: + attributes = list(self.handle_x509_certificate(properties)) + if properties.raw_certificate: + raw = properties.raw_certificate.value + try: + relation = "raw-base64" if raw == b64encode(b64decode(raw)).strip() else "pem" + except Exception: + relation = "pem" + attributes.append(["text", raw, relation]) + if properties.certificate_signature: + signature = properties.certificate_signature + attribute_type = f"x509-fingerprint-{signature.signature_algorithm.value.lower()}" + attributes.append([attribute_type, signature.signature.value, attribute_type]) + return "x509", self._return_object_attributes(attributes), "" + + def _handle_x509_certificate(self, properties: x509_certificate_object.X509Certificate): + if properties.certificate is None: + return [] + certificate = properties.certificate + if certificate.validity: + validity = certificate.validity + for prop in self._mapping._x509_datetime_types(): + if getattr(validity, prop): + yield ['datetime', getattr(validity, prop).value, f"validity-{prop.replace('_', '-')}"] + if certificate.subject_public_key: + subject_pubkey = certificate.subject_public_key + if subject_pubkey.rsa_public_key: + rsa_pubkey = subject_pubkey.rsa_public_key + for prop in self._mapping._x509_pubkey_types(): + if getattr(rsa_pubkey, prop): + yield ['text', getattr(rsa_pubkey, prop).value, f'pubkey-info-{prop}'] + if subject_pubkey.public_key_algorithm: + yield ["text", subject_pubkey.public_key_algorithm.value, "pubkey-info-algorithm"] + for prop in self._mapping._x509_certificate_types(): + if getattr(certificate, prop): + yield ['text', getattr(certificate, prop).value, prop.replace('_', '-')] + + ############################################################################ + # GALAXIES PARSING SPECIFIC METHODS USED BY BOTH SUBCLASSES. # + ############################################################################ + + @staticmethod + def _get_galaxy_name(stix_object: _STIX_OBJECT_TYPING, + feature: str) -> Union[str, list, None]: + if getattr(stix_object, feature, None) is not None: + return getattr(stix_object, feature) + for feature in ('name', 'names'): + if getattr(stix_object, feature, None) is not None: + return [value.value for value in getattr(stix_object, feature)] + + def _parse_galaxy(self, stix_object: _STIX_OBJECT_TYPING, + feature: str, default_value: str): + names = self._get_galaxy_name(stix_object, feature) + if names: + if isinstance(names, list): + for name in names: + yield from self._resolve_galaxy(name, default_value) + else: + yield from self._resolve_galaxy(names, default_value) + + def _resolve_galaxy(self, galaxy_name: str, default_value: str) -> list: + if galaxy_name in self.synonyms_mapping: + return self.synonyms_mapping[galaxy_name] + for identifier in galaxy_name.split(' - '): + if identifier[0].isalpha() and any(character.isdecimal() for character in identifier[1:]): + for name, tag_names in self.synonyms_mapping.items(): + if identifier in name: + return tag_names + return [f'misp-galaxy:{default_value}="{galaxy_name}"'] + + ############################################################################ + # UTILITY METHODS. # + ############################################################################ + + @staticmethod + def _extract_uuid(object_id: str) -> str: + return '-'.join(object_id.split('-')[1:]) + + def _fetch_attributes_with_keys(self, properties: _SIMPLE_PROPERTIES_TYPING, mapping: str): + for field, attribute in getattr(self._mapping, mapping)().items(): + if getattr(properties, field): + attribute_type, feature, relation = attribute + yield (attribute_type, attrgetter(feature)(properties), relation) + + def _fetch_attributes_with_key_parsing(self, properties: _PROPERTIES_TYPING, mapping: str): + for field, attribute in getattr(self._mapping, mapping)().items(): + if getattr(properties, field): + attribute_type, feature, relation = attribute + yield (attribute_type, attrgetter(f'{field}.{feature}')(properties), relation) + + def _fetch_attributes_with_partial_key_parsing(self, properties: _PARTIAL_PROPERTIES_TYPING, mapping: str): + for field, attribute in getattr(self._mapping, mapping)().items(): + if getattr(properties, field): + attribute_type, relation = attribute + yield (attribute_type, getattr(properties, field).value, relation) + + @staticmethod + def _return_object_attributes(attributes: Union[list, tuple]) -> tuple: + return tuple( + dict(zip(('type', 'value', 'object_relation'), attribute)) + for attribute in attributes + ) + + ############################################################################ + # ERRORS AND WARNINGS HANDLING METHODS # + ############################################################################ + + def _stix_object_type_error(self, xsi_type: str, object_id: str): + self._add_error(f"Unknown Observable type within STIX object with id {object_id}: {xsi_type}") \ No newline at end of file diff --git a/misp_stix_converter/stix2misp/stix2_to_misp.py b/misp_stix_converter/stix2misp/stix2_to_misp.py index 2d7f497..c95658d 100644 --- a/misp_stix_converter/stix2misp/stix2_to_misp.py +++ b/misp_stix_converter/stix2misp/stix2_to_misp.py @@ -2,7 +2,6 @@ # -*- coding: utf-8 -*- import sys -import time from .converters import ( ExternalSTIX2AttackPatternConverter, ExternalSTIX2MalwareAnalysisConverter, ExternalSTIX2CampaignConverter, InternalSTIX2CampaignConverter, @@ -31,7 +30,6 @@ from .internal_stix2_mapping import InternalSTIX2toMISPMapping from abc import ABCMeta from collections import defaultdict -from datetime import datetime from pymisp import ( MISPEvent, MISPAttribute, MISPGalaxy, MISPGalaxyCluster, MISPObject, MISPSighting) @@ -202,12 +200,8 @@ class STIX2toMISPParser(STIXtoMISPParser, metaclass=ABCMeta): - def __init__(self, distribution: int, sharing_group_id: Union[int, None], - title: Union[str, None], producer: Union[str, None], - galaxies_as_tags: bool): - super().__init__( - distribution, sharing_group_id, title, producer, galaxies_as_tags - ) + def __init__(self): + super().__init__() self._creators: set = set() self._mapping: Union[ ExternalSTIX2toMISPMapping, InternalSTIX2toMISPMapping @@ -247,20 +241,33 @@ def load_stix_bundle(self, bundle: Union[Bundle_v20, Bundle_v21]): n_report += 1 feature = self._mapping.stix_object_loading_mapping(object_type) if feature is None: - self._unable_to_load_stix_object_type_error(object_type) + self._add_error( + f'Unable to load STIX object type: {object_type}' + ) continue if hasattr(stix_object, 'created_by_ref'): self._creators.add(stix_object.created_by_ref) try: getattr(self, feature)(stix_object) - except MarkingDefinitionLoadingError as error: - self._marking_definition_error(error) + except MarkingDefinitionLoadingError as marking_definition_id: + self._add_error( + 'Error whil parsing the Marking Definition ' + f'object with id {marking_definition_id}' + ) except AttributeError as exception: self._critical_error(exception) self.__n_report = 2 if n_report >= 2 else n_report - def parse_stix_bundle(self, single_event: Optional[bool] = False): - self.__single_event = single_event + def parse_stix_content(self, filename: str, **kwargs): + try: + bundle = _load_stix2_content(filename) + except Exception as exception: + sys.exit(exception) + self.load_stix_bundle(bundle) + del bundle + self.parse_stix_bundle(**kwargs) + + def _parse_stix_bundle(self): try: feature = self._mapping.bundle_to_misp_mapping(str(self.__n_report)) except AttributeError: @@ -279,16 +286,6 @@ def parse_stix_bundle(self, single_event: Optional[bool] = False): if hasattr(self, feature): setattr(self, feature, {}) - def parse_stix_content( - self, filename: str, single_event: Optional[bool] = False): - try: - bundle = _load_stix2_content(filename) - except Exception as exception: - sys.exit(exception) - self.load_stix_bundle(bundle) - del bundle - self.parse_stix_bundle(single_event) - ############################################################################ # PROPERTIES # ############################################################################ @@ -317,11 +314,9 @@ def event_tags(self) -> list: @property def generic_info_field(self) -> str: - message = f'STIX {self.stix_version} Bundle ({self._identifier})' if self.event_title is not None: - message = f'{self.event_title} {message}' - if self.producer is not None: - message += f' produced by {self.producer}' + return self.event_title + message = f'STIX {self.stix_version} Bundle ({self._identifier})' return f'{message} and converted with the MISP-STIX import feature.' @property @@ -376,10 +371,6 @@ def observed_data_parser(self) -> _OBSERVED_DATA_PARSER_TYPING: self._set_observed_data_parser() return self._observed_data_parser - @property - def single_event(self) -> bool: - return self.__single_event - @property def stix_version(self) -> str: return self.__stix_version @@ -601,18 +592,26 @@ def _handle_object(self, object_type: str, object_ref: str): self._object_type_loading_error(error) except UndefinedIndicatorError as error: self._undefined_indicator_error(error) - except UndefinedSTIXObjectError as error: - self._undefined_object_error(error) + except UndefinedSTIXObjectError as object_id: + self._add_error( + 'Unable to define the object identified ' + f'with the id {object_id}' + ) except UndefinedObservableError as error: self._undefined_observable_error(error) - except UnknownAttributeTypeError as error: - self._unknown_attribute_type_warning(error) - except UnknownObjectNameError as error: - self._unknown_object_name_warning(error) + except UnknownAttributeTypeError as attribute_type: + self._add_warning( + f'MISP attribute type not mapped: {attribute_type}' + ) + except UnknownObjectNameError as name: + self._add_warning(f'MISP object name not mapped: {name}') except UnknownParsingFunctionError as error: self._unknown_parsing_function_error(error) - except UnknownPatternTypeError as error: - self._unknown_pattern_type_error(object_ref, error) + except UnknownPatternTypeError as pattern_type: + self._add_error( + 'Unknown pattern type in Indicator object with id ' + f'{object_ref}: {pattern_type}' + ) def _handle_misp_event_tags( self, misp_event: MISPEvent, stix_object: _GROUPING_REPORT_TYPING): @@ -660,22 +659,22 @@ def _misp_event_from_report(self, report: _REPORT_TYPING) -> MISPEvent: def _parse_bundle_with_multiple_reports(self): if self.single_event: self.__misp_event = self._create_generic_event() - if hasattr(self, '_report') and self._report is not None: + if getattr(self, '_report', None): for report in self._report.values(): self._handle_object_refs(report.object_refs) - if hasattr(self, '_grouping') and self._grouping is not None: + if getattr(self, '_grouping', None): for grouping in self._grouping.values(): self._handle_object_refs(grouping.object_refs) self._handle_unparsed_content() else: self.__misp_events = [] - if hasattr(self, '_report') and self._report is not None: + if getattr(self, '_report', None): for report in self._report.values(): self.__misp_event = self._misp_event_from_report(report) self._handle_object_refs(report.object_refs) self._handle_unparsed_content() self.__misp_events.append(self.misp_event) - if hasattr(self, '_grouping') and self._grouping is not None: + if getattr(self, '_grouping', None): for grouping in self._grouping.values(): self.__misp_event = self._misp_event_from_grouping(grouping) self._handle_object_refs(grouping.object_refs) @@ -683,18 +682,18 @@ def _parse_bundle_with_multiple_reports(self): self.__misp_events.append(self.misp_event) def _parse_bundle_with_no_report(self): - self.__single_event = True + self._set_single_event(True) self.__misp_event = self._create_generic_event() self._parse_loaded_features() self._handle_unparsed_content() def _parse_bundle_with_single_report(self): - self.__single_event = True - if hasattr(self, '_report') and self._report is not None: + self._set_single_event(True) + if getattr(self, '_report', None): for report in self._report.values(): self.__misp_event = self._misp_event_from_report(report) self._handle_object_refs(report.object_refs) - elif hasattr(self, '_grouping') and self._grouping is not None: + elif getattr(self, '_grouping', None): for grouping in self._grouping.values(): self.__misp_event = self._misp_event_from_grouping(grouping) self._handle_object_refs(grouping.object_refs) @@ -1234,14 +1233,33 @@ def _parse_confidence_level(confidence_level: int) -> str: return 'misp:confidence-level="rarely-confident"' return 'misp:confidence-level="unconfident"' - @staticmethod - def _timestamp_from_date(date: datetime) -> int: - return int(date.timestamp()) - try: - return int(date.timestamp()) - except AttributeError: - return int( - time.mktime( - time.strptime(date.split('+')[0], "%Y-%m-%dT%H:%M:%S.%fZ") - ) - ) + ############################################################################ + # ERRORS AND WARNINGS HANDLING METHODS # + ############################################################################ + + def _critical_error(self, exception: Exception): + self._add_error(f'The following exception was raised: {exception}') + + def _object_ref_loading_error(self, object_ref: str): + self._add_error(f'Error loading the STIX object with id {object_ref}') + + def _object_type_loading_error(self, object_type: str): + self._add_error(f'Error loading the STIX object of type {object_type}') + + def _unknown_network_protocol_warning( + self, protocol: str, object_id: str, + object_type: Optional[str] = 'indicator'): + message = ( + 'in patterning expression within the indicator with id' + if object_type == 'indicator' else + f'within the {object_type} object with id' + ) + self._add_warning( + f'Unknown network protocol: {protocol}, {message} {object_id}' + ) + + def _unknown_parsing_function_error(self, feature: Exception): + self._add_error(f'Unknown STIX parsing function name: {feature}') + + def _unknown_stix_object_type_error(self, object_type: Exception): + self._add_error(f'Unknown STIX object type: {object_type}') diff --git a/tests/_test_stix.py b/tests/_test_stix.py index 006b8aa..bfe3476 100644 --- a/tests/_test_stix.py +++ b/tests/_test_stix.py @@ -13,6 +13,8 @@ def _assert_multiple_equal(self, reference, *elements): @staticmethod def _datetime_from_str(timestamp): + if isinstance(timestamp, datetime): + return timestamp regex = f"%Y-%m-%d{'T' if 'T' in timestamp else ' '}%H:%M:%S" if '.' in timestamp: regex = f'{regex}.%f' diff --git a/tests/test_events.py b/tests/test_events.py index 5738296..30828ed 100644 --- a/tests/test_events.py +++ b/tests/test_events.py @@ -770,6 +770,24 @@ "comment": "Fully agree with the malicious nature of the IP" } ] + }, + { + "type": "ip-dst", + "category": "Network activity", + "to_ids": False, + "uuid": "76fd763a-45fb-49a6-a732-64aeedbfd7d4", + "timestamp": "1603642920", + "value": "8.8.8.8", + "Note": [ + { + "uuid": "31fc7048-9ede-4db9-a423-ef97670ed4c6", + "authors": "opinion@foo.bar", + "created": "2024-06-12 12:52:45", + "modified": "2024-06-12 12:52:45", + "note": "DNS Resolver used to resolve the malicious domain", + "language": "en" + } + ] } ], "Object": [ @@ -823,6 +841,16 @@ "referenced_uuid": "f7ef1b4a-964a-4a69-9e21-808f85c56238", "relationship_type": "downloaded-from" } + ], + "Opinion": [ + { + "uuid": "74258748-78f2-4b19-bedc-27ec61b1c5df", + "authors": "john.doe@foo.bar", + "created": "2024-06-12 12:52:48", + "modified": "2024-06-12 12:53:58", + "opinion": "50", + "comment": "No warning from my antivirus" + } ] } ], @@ -3185,7 +3213,7 @@ def get_base_event(): def get_event_with_analyst_data(): event = deepcopy(_BASE_EVENT) - event['Event'].update(_TEST_EVENT_WITH_ANALYST_DATA) + event['Event'].update(deepcopy(_TEST_EVENT_WITH_ANALYST_DATA)) return event diff --git a/tests/test_external_stix20_bundles.py b/tests/test_external_stix20_bundles.py index 9e0489f..557731e 100644 --- a/tests/test_external_stix20_bundles.py +++ b/tests/test_external_stix20_bundles.py @@ -1536,6 +1536,18 @@ def __assemble_galaxy_bundle(cls, event_galaxy, attribute_galaxy): ] return dict_to_stix2(bundle, allow_custom=True) + ############################################################################ + # EVENTS SAMPLES # + ############################################################################ + + @classmethod + def get_bundle_without_report(cls): + bundle = deepcopy(cls.__bundle) + bundle['objects'] = [ + deepcopy(cls.__identity), *_IP_ADDRESS_ATTRIBUTES + ] + return dict_to_stix2(bundle, allow_custom=True) + ############################################################################ # GALAXIES SAMPLES # ############################################################################ diff --git a/tests/test_external_stix20_import.py b/tests/test_external_stix20_import.py index 3b40295..06d9d94 100644 --- a/tests/test_external_stix20_import.py +++ b/tests/test_external_stix20_import.py @@ -2,12 +2,30 @@ # -*- coding: utf-8 -*- from .test_external_stix20_bundles import TestExternalSTIX20Bundles -from ._test_stix import TestSTIX21 -from ._test_stix_import import TestExternalSTIX2Import, TestSTIX21Import +from ._test_stix import TestSTIX20 +from ._test_stix_import import TestExternalSTIX2Import, TestSTIX20Import from uuid import uuid5 -class TestExternalSTIX21Import(TestExternalSTIX2Import, TestSTIX21, TestSTIX21Import): +class TestExternalSTIX20Import(TestExternalSTIX2Import, TestSTIX20, TestSTIX20Import): + + ############################################################################ + # MISP EVENT IMPORT TESTS. # + ############################################################################ + + def test_stix20_bundle_with_event_title_and_producer(self): + bundle = TestExternalSTIX20Bundles.get_bundle_without_report() + self.parser.load_stix_bundle(bundle) + self.parser.parse_stix_bundle( + title='Malicious IP addresses report', + producer='MISP Project' + ) + event = self.parser.misp_event + self.assertEqual(event.info, self.parser.event_title) + self.assertEqual( + event.tags[0]['name'], + f'misp-galaxy:producer="{self.parser.producer}"' + ) ############################################################################ # MISP GALAXIES IMPORT TESTS # diff --git a/tests/test_external_stix21_bundles.py b/tests/test_external_stix21_bundles.py index 71c7063..352c5e6 100644 --- a/tests/test_external_stix21_bundles.py +++ b/tests/test_external_stix21_bundles.py @@ -1831,6 +1831,18 @@ def __assemble_galaxy_bundle(cls, event_galaxy, attribute_galaxy): ] return dict_to_stix2(bundle, allow_custom=True) + ############################################################################ + # EVENTS SAMPLES # + ############################################################################ + + @classmethod + def get_bundle_without_grouping(cls): + bundle = deepcopy(cls.__bundle) + bundle['objects'] = [ + deepcopy(cls.__identity), *_IP_ADDRESS_ATTRIBUTES + ] + return dict_to_stix2(bundle, allow_custom=True) + ############################################################################ # GALAXIES SAMPLES # ############################################################################ diff --git a/tests/test_external_stix21_import.py b/tests/test_external_stix21_import.py index 5f3d6fe..d2ada9b 100644 --- a/tests/test_external_stix21_import.py +++ b/tests/test_external_stix21_import.py @@ -10,11 +10,30 @@ class TestExternalSTIX21Import(TestExternalSTIX2Import, TestSTIX21, TestSTIX21Import): - ################################################################################ - # MISP GALAXIES IMPORT TESTS # - ################################################################################ + ############################################################################ + # MISP EVENT IMPORT TESTS. # + ############################################################################ + + def test_stix21_bundle_with_event_title_and_producer(self): + bundle = TestExternalSTIX21Bundles.get_bundle_without_grouping() + self.parser.load_stix_bundle(bundle) + self.parser.parse_stix_bundle( + title='Malicious IP addresses report', + producer='MISP Project' + ) + event = self.parser.misp_event + self.assertEqual(event.info, f'{self.parser.event_title}') + self.assertEqual( + event.tags[0]['name'], + f'misp-galaxy:producer="{self.parser.producer}"' + ) + + ############################################################################ + # MISP GALAXIES IMPORT TESTS # + ############################################################################ - def _check_location_galaxy_features(self, galaxies, stix_object, galaxy_type, cluster_value=None): + def _check_location_galaxy_features( + self, galaxies, stix_object, galaxy_type, cluster_value=None): self.assertEqual(len(galaxies), 1) galaxy = galaxies[0] self.assertEqual(len(galaxy.clusters), 1) diff --git a/tests/test_stix20_export.py b/tests/test_stix20_export.py index 6f877e5..ac8dd02 100644 --- a/tests/test_stix20_export.py +++ b/tests/test_stix20_export.py @@ -27,6 +27,36 @@ def _check_bundle_features(self, length): class TestSTIX20EventExport(TestSTIX20GenericExport): + def _check_analyst_note(self, stix_object, misp_layer): + self.assertEqual( + stix_object.id, f"x-misp-analyst-note--{misp_layer['uuid']}" + ) + self.assertEqual(stix_object.x_misp_note, misp_layer['note']) + self.assertEqual(stix_object.x_misp_author, misp_layer['authors']) + self.assertEqual(stix_object.x_misp_language, misp_layer['language']) + self.assertEqual( + stix_object.created, self._datetime_from_str(misp_layer['created']) + ) + self.assertEqual( + stix_object.modified, + self._datetime_from_str(misp_layer['modified']) + ) + + def _check_analyst_opinion(self, stix_object, misp_layer): + self.assertEqual( + stix_object.id, f"x-misp-analyst-opinion--{misp_layer['uuid']}" + ) + self.assertEqual(stix_object.x_misp_opinion, int(misp_layer['opinion'])) + self.assertEqual(stix_object.x_misp_author, misp_layer['authors']) + self.assertEqual(stix_object.x_misp_comment, misp_layer['comment']) + self.assertEqual( + stix_object.created, self._datetime_from_str(misp_layer['created']) + ) + self.assertEqual( + stix_object.modified, + self._datetime_from_str(misp_layer['modified']) + ) + def _check_opinion_features(self, opinion, sighting, object_id): self.assertEqual(opinion.type, 'x-misp-opinion') self.assertEqual(opinion.id, f"x-misp-opinion--{sighting['uuid']}") @@ -70,6 +100,75 @@ def _test_base_event(self, event): "This MISP Event is empty and contains no attribute, object, galaxy or tag." ) + def _test_event_with_analyst_data(self, event): + orgc = event['Orgc'] + event_report = event['EventReport'][0] + note = event['Note'][0] + src_attribute, dst_attribute = event['Attribute'] + misp_object = event['Object'][0] + self.parser.parse_misp_event(event) + bundle = self._check_bundle_features(13) + identity, report, *stix_objects = bundle.objects + timestamp = event['timestamp'] + if not isinstance(timestamp, datetime): + timestamp = self._datetime_from_timestamp(timestamp) + identity_id = self._check_identity_features(identity, orgc, timestamp) + object_refs = self._check_report_features(report, event, identity_id, timestamp) + self.assertEqual(report.published, timestamp) + for stix_object, object_ref in zip(stix_objects, object_refs): + self.assertEqual(stix_object.id, object_ref) + (attr_indicator, attr_opinion, observed_data, observed_data_note, + obj_indicator, obj_opinion, obj_attr_note, report, report_opinion, + relationship, event_note) = stix_objects + self._assert_multiple_equal( + attr_indicator.id, + relationship.target_ref, + attr_opinion.object_ref, + f"indicator--{src_attribute['uuid']}" + ) + self._assert_multiple_equal( + attr_opinion.type, + obj_opinion.type, + report_opinion.type, + 'x-misp-analyst-opinion' + ) + attribute_opinion = src_attribute['Opinion'][0] + self._check_analyst_opinion(attr_opinion, attribute_opinion) + self._assert_multiple_equal( + observed_data.id, + observed_data_note.object_ref, + f"observed-data--{dst_attribute['uuid']}" + ) + self._assert_multiple_equal( + observed_data_note.type, + obj_attr_note.type, + event_note.type, + 'x-misp-analyst-note' + ) + attribute_note = dst_attribute['Note'][0] + self._check_analyst_note(observed_data_note, attribute_note) + self._assert_multiple_equal( + obj_indicator.id, + relationship.source_ref, + obj_opinion.object_ref, + obj_attr_note.object_ref, + f"indicator--{misp_object['uuid']}" + ) + object_opinion = misp_object['Opinion'][0] + self._check_analyst_opinion(obj_opinion, object_opinion) + object_attribute_note = misp_object['Attribute'][0]['Note'][0] + self._check_analyst_note(obj_attr_note, object_attribute_note) + self._assert_multiple_equal( + report.id, + report_opinion.object_ref, + f"x-misp-event-report--{event_report['uuid']}" + ) + self.assertEqual(report.type, 'x-misp-event-report') + event_report_opinion = event_report['Opinion'][0] + self._check_analyst_opinion(report_opinion, event_report_opinion) + self.assertEqual(relationship.relationship_type, 'downloaded-from') + self._check_analyst_note(event_note, note) + def _test_event_with_escaped_characters(self, event): attributes = deepcopy(event['Attribute']) self.parser.parse_misp_event(event) @@ -89,6 +188,33 @@ def _test_event_with_escaped_characters(self, event): data = b64encode(data.getvalue()).decode() self.assertIn(self._sanitize_pattern_value(data), indicator.pattern) + def _test_event_with_event_report(self, event): + orgc = event['Orgc'] + event_report = event['EventReport'][0] + self.parser.parse_misp_event(event) + bundle = self._check_bundle_features(7) + identity, report, *stix_objects = bundle.objects + timestamp = event['timestamp'] + if not isinstance(timestamp, datetime): + timestamp = self._datetime_from_timestamp(timestamp) + identity_id = self._check_identity_features(identity, orgc, timestamp) + object_refs = self._check_report_features(report, event, identity_id, timestamp) + self.assertEqual(report.published, timestamp) + for stix_object, object_ref in zip(stix_objects, object_refs): + self.assertEqual(stix_object.id, object_ref) + ip_src, observed_data, domain_ip, note, _ = stix_objects + self.assertEqual(note.id, f"x-misp-event-report--{event_report['uuid']}") + timestamp = event_report['timestamp'] + if not isinstance(timestamp, datetime): + timestamp = self._datetime_from_timestamp(timestamp) + self._assert_multiple_equal(note.created, note.modified, timestamp) + self.assertEqual(note.x_misp_content, event_report['content']) + self.assertEqual(note.x_misp_name, event_report['name']) + object_refs = note.object_refs + self.assertEqual(len(object_refs), 3) + object_ids = {ip_src.id, observed_data.id, domain_ip.id} + self.assertEqual(set(object_refs), object_ids) + def _test_event_with_sightings(self, event): orgc = event['Orgc'] attribute1, attribute2 = event['Attribute'] @@ -185,10 +311,18 @@ def test_base_event(self): event = get_base_event() self._test_base_event(event['Event']) + def test_event_with_analyst_data(self): + event = get_event_with_analyst_data() + self._test_event_with_analyst_data(event['Event']) + def test_event_with_escaped_characters(self): event = get_event_with_escaped_values_v20() self._test_event_with_escaped_characters(event['Event']) + def test_event_with_event_report(self): + event = get_event_with_event_report() + self._test_event_with_event_report(event['Event']) + def test_event_with_sightings(self): event = get_event_with_sightings() self._test_event_with_sightings(event['Event']) @@ -209,12 +343,24 @@ def test_base_event(self): misp_event.from_dict(**event) self._test_base_event(misp_event) + def test_event_with_analyst_data(self): + event = get_event_with_analyst_data() + misp_event = MISPEvent() + misp_event.from_dict(**event) + self._test_event_with_analyst_data(misp_event) + def test_event_with_escaped_characters(self): event = get_event_with_escaped_values_v20() misp_event = MISPEvent() misp_event.from_dict(**event) self._test_event_with_escaped_characters(misp_event) + def test_event_with_event_report(self): + event = get_event_with_event_report() + misp_event = MISPEvent() + misp_event.from_dict(**event) + self._test_event_with_event_report(misp_event) + def test_event_with_sightings(self): event = get_event_with_sightings() misp_event = MISPEvent() diff --git a/tests/test_stix21_export.py b/tests/test_stix21_export.py index bc5cf47..9f9743f 100644 --- a/tests/test_stix21_export.py +++ b/tests/test_stix21_export.py @@ -34,6 +34,49 @@ def _check_spec_versions(self, stix_objects): class TestSTIX21EventExport(TestSTIX21GenericExport): + def _check_analyst_note(self, stix_object, misp_layer): + self.assertEqual( + stix_object.id, f"note--{misp_layer['uuid']}" + ) + self.assertEqual(stix_object.content, misp_layer['note']) + self.assertEqual(stix_object.lang, misp_layer['language']) + self.assertEqual( + stix_object.authors, [misp_layer['authors']] + ) + self.assertEqual( + stix_object.created, + self._datetime_from_str(misp_layer['created']) + ) + self.assertEqual( + stix_object.modified, + self._datetime_from_str(misp_layer['modified']) + ) + self.assertEqual( + stix_object.labels, ['misp:context-layer="Analyst Note"'] + ) + + def _check_analyst_opinion(self, stix_object, misp_layer, opinion): + self.assertEqual( + stix_object.id, f"opinion--{misp_layer['uuid']}" + ) + self.assertEqual(stix_object.opinion, opinion) + self.assertEqual( + stix_object.x_misp_opinion, int(misp_layer['opinion']) + ) + self.assertEqual(stix_object.explanation, misp_layer['comment']) + self.assertEqual(stix_object.authors, [misp_layer['authors']]) + self.assertEqual( + stix_object.created, + self._datetime_from_str(misp_layer['created']) + ) + self.assertEqual( + stix_object.modified, + self._datetime_from_str(misp_layer['modified']) + ) + self.assertEqual( + stix_object.labels, ['misp:context-layer="Analyst Opinion"'] + ) + def _check_attribute_confidence_tags(self, stix_object, attribute): self.assertEqual( stix_object.confidence, @@ -97,10 +140,10 @@ def _test_event_with_analyst_data(self, event): orgc = event['Orgc'] event_report = event['EventReport'][0] note = event['Note'][0] - attribute = event['Attribute'][0] + src_attribute, dst_attribute = event['Attribute'] misp_object = event['Object'][0] self.parser.parse_misp_event(event) - stix_objects = self._check_bundle_features(10) + stix_objects = self._check_bundle_features(15) self._check_spec_versions(stix_objects) identity, grouping, *stix_objects = stix_objects timestamp = event['timestamp'] @@ -110,87 +153,45 @@ def _test_event_with_analyst_data(self, event): object_refs = self._check_grouping_features(grouping, event, identity_id) for stix_object, object_ref in zip(stix_objects, object_refs): self.assertEqual(stix_object.id, object_ref) - (attr_indicator, attr_opinion, obj_indicator, obj_note, report, - report_opinion, relationship, event_note) = stix_objects + (attr_indicator, attr_indicator_opinion, observed_data, _, + _, obs_data_note, obj_indicator, obj_opinion, obj_attr_note, + report, report_opinion, relationship, event_note) = stix_objects self._assert_multiple_equal( attr_indicator.id, relationship.target_ref, - attr_opinion['object_refs'][0], - f"indicator--{attribute['uuid']}" - ) - attribute_opinion = attribute['Opinion'][0] - self.assertEqual( - attr_opinion.id, f"opinion--{attribute_opinion['uuid']}" - ) - self.assertEqual(attr_opinion.opinion, 'strongly-agree') - self.assertEqual( - attr_opinion.x_misp_opinion, int(attribute_opinion['opinion']) - ) - self.assertEqual(attr_opinion.explanation, attribute_opinion['comment']) - self.assertEqual(attr_opinion.authors, [attribute_opinion['authors']]) - self.assertEqual( - attr_opinion.created, - self._datetime_from_str(attribute_opinion['created']) + attr_indicator_opinion.object_refs[0], + f"indicator--{src_attribute['uuid']}" ) - self.assertEqual( - attr_opinion.modified, - self._datetime_from_str(attribute_opinion['modified']) + attribute_opinion = src_attribute['Opinion'][0] + self._check_analyst_opinion(attr_indicator_opinion, attribute_opinion, 'strongly-agree') + self._assert_multiple_equal( + observed_data.id, + obs_data_note.object_refs[0], + f"observed-data--{dst_attribute['uuid']}" ) + attribute_note = dst_attribute['Note'][0] + self._check_analyst_note(obs_data_note, attribute_note) self._assert_multiple_equal( obj_indicator.id, relationship.source_ref, - obj_note.object_refs[0], + obj_opinion.object_refs[0], + obj_attr_note.object_refs[0], f"indicator--{misp_object['uuid']}" ) - object_note = misp_object['Attribute'][0]['Note'][0] - self.assertEqual(obj_note.id, f"note--{object_note['uuid']}") - self.assertEqual(obj_note.content, object_note['note']) - self.assertEqual(obj_note.lang, object_note['language']) - self.assertEqual(obj_note.authors, [object_note['authors']]) - self.assertEqual( - obj_note.created, self._datetime_from_str(object_note['created']) - ) - self.assertEqual( - obj_note.modified, self._datetime_from_str(object_note['modified']) - ) + object_opinion = misp_object['Opinion'][0] + self._check_analyst_opinion(obj_opinion, object_opinion, 'neutral') + object_attribute_note = misp_object['Attribute'][0]['Note'][0] + self._check_analyst_note(obj_attr_note, object_attribute_note) self._assert_multiple_equal( report.id, report_opinion.object_refs[0], f"note--{event_report['uuid']}" ) + self.assertEqual(report.labels, ['misp:data-layer="Event Report"']) event_report_opinion = event_report['Opinion'][0] - self.assertEqual( - report_opinion.id, f"opinion--{event_report_opinion['uuid']}" - ) - self.assertEqual(report_opinion.opinion, 'agree') - self.assertEqual( - report_opinion.x_misp_opinion, int(event_report_opinion['opinion']) - ) - self.assertEqual( - report_opinion.explanation, event_report_opinion['comment'] - ) - self.assertEqual( - report_opinion.authors, [event_report_opinion['authors']] - ) - self.assertEqual( - report_opinion.created, - self._datetime_from_str(event_report_opinion['created']) - ) - self.assertEqual( - report_opinion.modified, - self._datetime_from_str(event_report_opinion['modified']) - ) + self._check_analyst_opinion(report_opinion, event_report_opinion, 'agree') self.assertEqual(relationship.relationship_type, 'downloaded-from') - self.assertEqual(event_note.id, f"note--{note['uuid']}") - self.assertEqual(event_note.content, note['note']) - self.assertEqual(event_note.lang, note['language']) - self.assertEqual(event_note.authors, [note['authors']]) - self.assertEqual( - event_note.created, self._datetime_from_str(note['created']) - ) - self.assertEqual( - event_note.modified, self._datetime_from_str(note['modified']) - ) + self._check_analyst_note(event_note, note) def _test_event_with_attribute_confidence_tags(self, event): tlp_tag, *confidence_tags = event['Tag'] @@ -261,7 +262,7 @@ def _test_event_with_event_report(self, event): timestamp = event_report['timestamp'] if not isinstance(timestamp, datetime): timestamp = self._datetime_from_timestamp(timestamp) - self.assertEqual(note.created, timestamp) + self._assert_multiple_equal(note.created, note.modified, timestamp) self.assertEqual(note.content, event_report['content']) object_refs = note.object_refs self.assertEqual(len(object_refs), 3) @@ -309,12 +310,8 @@ def _test_event_with_sightings(self, event): if not isinstance(timestamp, datetime): timestamp = self._datetime_from_timestamp(timestamp) identity_id = self._check_identity_features(identity, orgc, timestamp) - args = ( - grouping, - event, - identity_id - ) - for stix_object, object_ref in zip(stix_objects, self._check_grouping_features(*args)): + object_refs = self._check_grouping_features(grouping, event, identity_id) + for stix_object, object_ref in zip(stix_objects, object_refs): self.assertEqual(stix_object.id, object_ref) self._check_identities_from_sighting( identities,