benchalerts: allow for custom alerts to not post comments (conbench#1588

)

benchalerts/benchalerts/pipeline_steps/github.py

@@ -175,15 +175,19 @@ def __init__(
         self.check_step_name = check_step_name
         self.alerter = alerter or Alerter()
 
-    def run_step(self, previous_outputs: Dict[str, Any]) -> dict:
+    def run_step(self, previous_outputs: Dict[str, Any]) -> Optional[dict]:
         check_details, full_comparison = previous_outputs[self.check_step_name]
 
+        comment = self.alerter.github_pr_comment(
+            full_comparison=full_comparison,
+            check_link=check_details["html_url"],
+        )
+        if not comment:
+            log.info("No comment; not posting to GitHub.")
+            return None
+
         res = self.github_client.create_pull_request_comment(
-            comment=self.alerter.github_pr_comment(
-                full_comparison=full_comparison,
-                check_link=check_details["html_url"],
-            ),
-            pull_number=self.pr_number,
+            comment=comment, pull_number=self.pr_number
         )
         return res
 

benchalerts/benchalerts/pipeline_steps/slack.py

@@ -78,13 +78,17 @@ def run_step(self, previous_outputs: Dict[str, Any]) -> Optional[dict]:
             log.info("GitHub Check was successful; not posting to Slack.")
             return None
 
+        message = self.alerter.slack_message(
+            full_comparison=full_comparison,
+            check_details=check_details,
+            comment_details=comment_details,
+        )
+        if not message:
+            log.info("No message; not posting to Slack.")
+            return None
+
         res = self.slack_client.post_message(
-            message=self.alerter.slack_message(
-                full_comparison=full_comparison,
-                check_details=check_details,
-                comment_details=comment_details,
-            ),
-            channel_id=self.channel_id,
+            message=message, channel_id=self.channel_id
         )
         return res
 

benchalerts/tests/unit_tests/test_pipeline_steps/test_github.py

@@ -121,6 +121,36 @@ def test_GitHubPRCommentAboutCheckStep(
     check_posted_comment(caplog, [expected_comment])
 
 
+@pytest.mark.parametrize("mock_comparison_info", ["regressions"], indirect=True)
+@pytest.mark.parametrize("github_auth", ["pat", "app"], indirect=True)
+def test_GitHubPRCommentAboutCheckStep_no_comment(
+    mock_comparison_info: FullComparisonInfo,
+    caplog: pytest.LogCaptureFixture,
+    github_auth: str,
+):
+    class NoneAlerter(Alerter):
+        def github_pr_comment(
+            self, full_comparison: FullComparisonInfo, check_link: str
+        ) -> str:
+            return ""
+
+    mock_check_response = MockResponse.from_file(
+        response_dir / "POST_github_check-runs.json"
+    ).json()
+
+    step = GitHubPRCommentAboutCheckStep(
+        pr_number=1,
+        github_client=GitHubRepoClient(repo="some/repo", adapter=MockAdapter()),
+        check_step_name="check_step",
+        alerter=NoneAlerter(),
+    )
+    res = step.run_step(
+        previous_outputs={"check_step": (mock_check_response, mock_comparison_info)}
+    )
+    assert res is None
+    check_posted_comment(caplog, [])
+
+
 @pytest.mark.parametrize("github_auth", ["pat", "app"], indirect=True)
 def test_GitHubCheckErrorHandler(caplog: pytest.LogCaptureFixture, github_auth: str):
     try:

benchalerts/tests/unit_tests/test_pipeline_steps/test_slack.py

@@ -94,6 +94,42 @@ def test_SlackMessageAboutBadCheckStep(
     check_posted_slack_message(caplog, expected_message)
 
 
+@pytest.mark.parametrize("mock_comparison_info", ["regressions"], indirect=True)
+def test_SlackMessageAboutBadCheckStep_no_comment(
+    mock_comparison_info: FullComparisonInfo,
+    caplog: pytest.LogCaptureFixture,
+    slack_env,
+):
+    """Test that SlackMessageAboutBadCheckStep posts the right message to Slack."""
+    mock_check_response = MockResponse.from_file(
+        response_dir / "POST_github_check-runs.json"
+    ).json()
+    mock_comment_response = MockResponse.from_file(
+        response_dir / "POST_github_issues_1_comments.json"
+    ).json()
+
+    class NoneAlerter(Alerter):
+        def slack_message(self, **kwargs) -> str:
+            return ""
+
+    step = SlackMessageAboutBadCheckStep(
+        channel_id="123",
+        slack_client=MockSlackClient(),
+        check_step_name="check_step",
+        pr_comment_step_name="pr_comment_step",
+        alerter=NoneAlerter(),
+    )
+    res = step.run_step(
+        {
+            "check_step": (mock_check_response, mock_comparison_info),
+            "pr_comment_step": mock_comment_response,
+        }
+    )
+    assert not res
+    expected_message = None
+    check_posted_slack_message(caplog, expected_message)
+
+
 def test_SlackErrorHandler(caplog: pytest.LogCaptureFixture, slack_env):
     """Test that SlackErrorHandler posts the right message to Slack."""
     try: