diff --git a/spidermon/contrib/scrapy/pipelines.py b/spidermon/contrib/scrapy/pipelines.py index 5611c375..9d97fccc 100644 --- a/spidermon/contrib/scrapy/pipelines.py +++ b/spidermon/contrib/scrapy/pipelines.py @@ -109,8 +109,7 @@ def process_item(self, item, _): # No validators match this specific item type return item - item_adapter = ItemAdapter(item) - item_dict = item_adapter.asdict() + item_dict = self._convert_item_to_dict(item) self.stats.add_item() self.stats.add_fields(len(item_dict.keys())) for validator in validators: @@ -158,3 +157,15 @@ def _add_error_stats(self, errors): for message in messages: self.stats.add_field_error(field_name, message) self.stats.add_item_with_errors() + + @staticmethod + def _convert_item_to_dict(item: Item) -> dict: + """ + Convert a Scrapy item to a dict. + + :param item: Scrapy item + :returns: The item as a dict + """ + item_adapter = ItemAdapter(item) + item_dict = item_adapter.asdict() + return item_dict diff --git a/tests/contrib/scrapy/test_pipelines.py b/tests/contrib/scrapy/test_pipelines.py index b4e55f51..13108350 100644 --- a/tests/contrib/scrapy/test_pipelines.py +++ b/tests/contrib/scrapy/test_pipelines.py @@ -252,3 +252,11 @@ def test_add_errors_to_item_prefilled(self): "prefilled", "some_message", ] + + @staticmethod + def test_convert_item_to_dict(self): + test_item = TestItem( + {"url": "http://example.com", "error_test": "error_message", "title": "test title"} + ) + item_dict = ItemValidationPipeline._convert_item_to_dict(test_item) + assert item_dict == {"url": "http://example.com", "error_test": "error_message", "title": "test title"}