diff --git a/grimoire_elk/elk.py b/grimoire_elk/elk.py index 478ec1be0..028c96741 100755 --- a/grimoire_elk/elk.py +++ b/grimoire_elk/elk.py @@ -203,8 +203,18 @@ def feed_backend(url, clean, fetch_archive, backend_name, backend_params, else: error_msg = "Error feeding raw from {}".format(ex) logger.error(error_msg, exc_info=True) + except SystemExit: + anonymized_params = anonymize_params(backend_params) + msg = "Wrong {} arguments: {}".format(backend_name, anonymized_params) + error_msg = "Error feeding raw. {}".format(msg) + logger.error(error_msg, exc_info=True) + + try: + msg = "[{}] Done collection for {}".format(backend_name, anonymize_url(backend.origin)) + except AttributeError: + msg = "[{}] Done collection for {}".format(backend_name, anonymize_url(projects_json_repo)) + logger.info(msg) - logger.info("[{}] Done collection for {}".format(backend_name, anonymize_url(backend.origin))) return error_msg @@ -603,8 +613,18 @@ def enrich_backend(url, clean, backend_name, backend_params, cfg_section_name, backend_name, anonymize_url(backend.origin), ex), exc_info=True) else: logger.error("Error enriching raw {}".format(ex), exc_info=True) + except SystemExit: + anonymized_params = anonymize_params(backend_params) + msg = "Wrong {} arguments: {}".format(backend_name, anonymized_params) + error_msg = "Error enriching raw. {}".format(msg) + logger.error(error_msg, exc_info=True) + + try: + msg = "[{}] Done enrichment for {}".format(backend_name, anonymize_url(backend.origin)) + except AttributeError: + msg = "[{}] Done enrichment for {}".format(backend_name, anonymize_url(projects_json_repo)) - logger.info("[{}] Done enrichment for {}".format(backend_name, anonymize_url(backend.origin))) + logger.info(msg) def delete_orphan_unique_identities(es, sortinghat_db, current_data_source, active_data_sources): diff --git a/releases/unreleased/skip-sources-with-wrong-arguments.yml b/releases/unreleased/skip-sources-with-wrong-arguments.yml new file mode 100644 index 000000000..ff0a6572a --- /dev/null +++ b/releases/unreleased/skip-sources-with-wrong-arguments.yml @@ -0,0 +1,8 @@ +--- +title: Skip sources with wrong arguments +category: fixed +author: Quan Zhou +issue: null +notes: > + Sources won't interrupt collection or enrichment processes when any + of their arguments are invalid. Instead, they will be skipped. diff --git a/tests/test_elk.py b/tests/test_elk.py index b858bd67a..1f39cd04f 100644 --- a/tests/test_elk.py +++ b/tests/test_elk.py @@ -21,9 +21,10 @@ # import configparser +import logging import unittest -from grimoire_elk.elk import anonymize_params +from grimoire_elk.elk import anonymize_params, enrich_backend, feed_backend, logger CONFIG_FILE = 'tests.conf' @@ -50,3 +51,51 @@ def test_anonymize_params(self): params = ("--backend-password", "mypassword", "--no-archive", "--api-token", "token") anonymized_params = anonymize_params(params) self.assertTupleEqual(anonymized_params, expected_params) + + def test_feed_backend_wrong_params(self): + """Test feed_backend with wrong parameters.""" + + # Jira has not --api-token argument + backend_params = ['https://jira.example.org', '--api-token', 'mypersonaltoken'] + projects_json_repo = 'https://jira.example.org' + expected_msg = "Error feeding raw. Wrong jira arguments: ('https://jira.example.org', " \ + "'--api-token', 'xxxxx')" + error_msg = feed_backend(self.es_con, False, False, "jira", backend_params, + "jira_raw", "jira_enriched", projects_json_repo=projects_json_repo) + self.assertEqual(error_msg, expected_msg) + + # Github the repository name cannot start with '-' + backend_params = ['chaoss', '-grimoirelab-elk', '--api-token', 'mypersonaltoken'] + projects_json_repo = 'https://github.com/chaoss/-grimoirelab-elk' + expected_msg = "Error feeding raw. Wrong github arguments: ('chaoss', " \ + "'-grimoirelab-elk', '--api-token', 'xxxxx')" + error_msg = feed_backend(self.es_con, False, False, "github", backend_params, + "github_raw", "github_enriched", projects_json_repo=projects_json_repo) + self.assertEqual(error_msg, expected_msg) + + def test_enrich_backend_wrong_params(self): + """Test enrich_backend with wrong parameters.""" + + # Jira has not --api-token argument + backend_params = ['https://jira.example.org', '--api-token', 'mypersonaltoken'] + projects_json_repo = 'https://jira.example.org' + expected_msg = "Error enriching raw. Wrong jira arguments: ('https://jira.example.org', '--api-token', 'xxxxx')" + with self.assertLogs(logger, level='ERROR') as cm: + enrich_backend(self.es_con, False, "jira", backend_params, "jira", + "jira_raw", "jira_enriched", projects_json_repo=projects_json_repo) + self.assertEqual(cm.records[0].msg, expected_msg) + + # Github the repository name cannot start with '-' + backend_params = ['chaoss', '-grimoirelab-elk', '--api-token', 'mypersonaltoken'] + projects_json_repo = 'https://github.com/chaoss/-grimoirelab-elk' + expected_msg = "Error enriching raw. Wrong github arguments: ('chaoss', " \ + "'-grimoirelab-elk', '--api-token', 'xxxxx')" + with self.assertLogs(logger, level='ERROR') as cm: + enrich_backend(self.es_con, False, "github", backend_params, "github", + "github_raw", "github_enriched", projects_json_repo=projects_json_repo) + self.assertEqual(cm.records[0].msg, expected_msg) + + +if __name__ == "__main__": + logging.basicConfig(level=logging.INFO, format='%(asctime)s %(message)s') + unittest.main()