From 38a06c7ab9e6e43bed8b11b5f48a4199e640fdbf Mon Sep 17 00:00:00 2001 From: Eric Radman Date: Tue, 2 Apr 2024 21:31:55 -0400 Subject: [PATCH 001/122] Autoformat hyperlinks in Slack alerts (#6845) Format the Slack message using the "mrkdwn" type, which will make hyperlinks clickable. New test for Slack destination. Co-authored-by: github-actions --- redash/destinations/slack.py | 4 +-- tests/handlers/test_destinations.py | 54 +++++++++++++++++++++++++++++ 2 files changed, 56 insertions(+), 2 deletions(-) diff --git a/redash/destinations/slack.py b/redash/destinations/slack.py index e4722fd446..a7e44b6a7c 100644 --- a/redash/destinations/slack.py +++ b/redash/destinations/slack.py @@ -26,13 +26,13 @@ def notify(self, alert, query, user, new_state, app, host, metadata, options): fields = [ { "title": "Query", + "type": "mrkdwn", "value": "{host}/queries/{query_id}".format(host=host, query_id=query.id), - "short": True, }, { "title": "Alert", + "type": "mrkdwn", "value": "{host}/alerts/{alert_id}".format(host=host, alert_id=alert.id), - "short": True, }, ] if alert.custom_body: diff --git a/tests/handlers/test_destinations.py b/tests/handlers/test_destinations.py index 9a7778829f..95fcffad09 100644 --- a/tests/handlers/test_destinations.py +++ b/tests/handlers/test_destinations.py @@ -5,6 +5,7 @@ from redash.destinations.asana import Asana from redash.destinations.datadog import Datadog from redash.destinations.discord import Discord +from redash.destinations.slack import Slack from redash.destinations.webex import Webex from redash.models import Alert, NotificationDestination from tests import BaseTestCase @@ -201,6 +202,59 @@ def test_asana_notify_calls_requests_post(): assert mock_response.status_code == 204 +def test_slack_notify_calls_requests_post(): + alert = mock.Mock(spec_set=["id", "name", "custom_subject", "custom_body", "render_template"]) + alert.id = 1 + alert.name = "Test Alert" + alert.custom_subject = "Test custom subject" + alert.custom_body = "Test custom body" + + alert.render_template = mock.Mock(return_value={"Rendered": "template"}) + query = mock.Mock() + query.id = 1 + + user = mock.Mock() + app = mock.Mock() + host = "https://localhost:5000" + options = {"url": "https://slack.com/api/api.test"} + metadata = {"Scheduled": False} + + new_state = Alert.TRIGGERED_STATE + destination = Slack(options) + + with mock.patch("redash.destinations.slack.requests.post") as mock_post: + mock_response = mock.Mock() + mock_response.status_code = 204 + mock_post.return_value = mock_response + + destination.notify(alert, query, user, new_state, app, host, metadata, options) + + query_link = f"{host}/queries/{query.id}" + alert_link = f"{host}/alerts/{alert.id}" + + expected_payload = { + "attachments": [ + { + "text": "Test custom subject", + "color": "#c0392b", + "fields": [ + {"title": "Query", "type": "mrkdwn", "value": query_link}, + {"title": "Alert", "type": "mrkdwn", "value": alert_link}, + {"title": "Description", "value": "Test custom body"}, + ], + } + ] + } + + mock_post.assert_called_once_with( + "https://slack.com/api/api.test", + data=json.dumps(expected_payload).encode(), + timeout=5.0, + ) + + assert mock_response.status_code == 204 + + def test_webex_notify_calls_requests_post(): alert = mock.Mock(spec_set=["id", "name", "custom_subject", "custom_body", "render_template"]) alert.id = 1 From 702a5506590f4af2776f5bafe5b25cd5d63a5430 Mon Sep 17 00:00:00 2001 From: Will Lachance Date: Wed, 3 Apr 2024 11:44:08 -0400 Subject: [PATCH 002/122] Handle timedelta in query results (#6846) --- redash/query_runner/query_results.py | 3 +++ tests/query_runner/test_query_results.py | 7 ++++--- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/redash/query_runner/query_results.py b/redash/query_runner/query_results.py index b22bac556f..5bc0eed29a 100644 --- a/redash/query_runner/query_results.py +++ b/redash/query_runner/query_results.py @@ -1,3 +1,4 @@ +import datetime import decimal import hashlib import logging @@ -108,6 +109,8 @@ def flatten(value): return json_dumps(value) elif isinstance(value, decimal.Decimal): return float(value) + elif isinstance(value, datetime.timedelta): + return str(value) else: return value diff --git a/tests/query_runner/test_query_results.py b/tests/query_runner/test_query_results.py index 756797492c..1c7ff70cc8 100644 --- a/tests/query_runner/test_query_results.py +++ b/tests/query_runner/test_query_results.py @@ -1,3 +1,4 @@ +import datetime import decimal import sqlite3 from unittest import TestCase @@ -108,11 +109,11 @@ def test_creates_table_with_non_ascii_in_column_name(self): create_table(connection, table_name, results) connection.execute("SELECT 1 FROM query_123") - def test_creates_table_with_decimal_in_column_value(self): + def test_creates_table_with_decimal_and_timedelta_in_column_value(self): connection = sqlite3.connect(":memory:") results = { - "columns": [{"name": "test1"}, {"name": "test2"}], - "rows": [{"test1": 1, "test2": decimal.Decimal(2)}], + "columns": [{"name": "test1"}, {"name": "test2"}, {"name": "test3"}], + "rows": [{"test1": 1, "test2": decimal.Decimal(2), "test3": datetime.timedelta(seconds=3)}], } table_name = "query_123" create_table(connection, table_name, results) From a0f5c706ffa1825045cbc2ed0d014895cd93c4a4 Mon Sep 17 00:00:00 2001 From: Eric Radman Date: Wed, 3 Apr 2024 17:52:18 -0400 Subject: [PATCH 003/122] Remove Qubole query runner (#6848) The qds-sdk-py package along with the rest of the Qubole project is no longer maintained: https://github.com/qubole/qds-sdk-py/commit/3c6a34ce336561dffe8ab5ed44ddbc151d3f81bf Removing this eliminates these warnings when running Redash management commands: ./qds_sdk/commands.py:1124: SyntaxWarning: "is" with a literal. Did you mean "=="? if options.mode is "1": ./qds_sdk/commands.py:1137: SyntaxWarning: "is" with a literal. Did you mean "=="? if options.db_update_mode is "updateonly": ./qds_sdk/commands.py:1424: SyntaxWarning: "is" with a literal. Did you mean "=="? if (total is 0) or (downloaded == total): Co-authored-by: github-actions --- client/app/assets/images/db-logos/qubole.png | Bin 2428 -> 0 bytes poetry.lock | 40 +---- pyproject.toml | 1 - redash/query_runner/qubole.py | 169 ------------------- redash/settings/__init__.py | 1 - 5 files changed, 1 insertion(+), 210 deletions(-) delete mode 100644 client/app/assets/images/db-logos/qubole.png delete mode 100644 redash/query_runner/qubole.py diff --git a/client/app/assets/images/db-logos/qubole.png b/client/app/assets/images/db-logos/qubole.png deleted file mode 100644 index dfdc2fa2e20f2ce14da46f0f71d406c70c1b46a1..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2428 zcmcJRhf@>E7Kb+kDbjn9>a)tUc zhA09T9_=bMK;RM(5a}feA>;+`%=-)8+nL=xyEA9^%$(nMc7JztxFB#?_AmfIz{dKl z695pl2?0D@?8zhibpQaoRyJqPINu{JPYMM;nUm*UgVHH`?_Z0_Ajfk58X#?B59Rtt zgaiah`n&a950K2$z!*Kw26GvrkrD3i8ejqL&lWr$AR`d-3u(?c70dd^vI}Q|Fb?_5OAv2X0q8Qsmn@+ z6P^u(0S`FN9&LdVcmjwiXj@PQb>sl@Xi=~LIMD#kt@NvxJu^Y;8fpuNcg42V>84qJi$3=JQ?uxgNcIr}WTHQ5)PZ<|6D$VO~aj|E)^ELdc;C8EraX!?? zuSQwoRARE*SqV`P;qJxj>G{%qFU73oR-h53!#`0Ed|ljnvAerl>)gH7<)`PJX7I^U zZ4=NweH|Vk|0T0)C93B>6%8wjZ2Njf3Lw|x13xG6t_}_krdKB|>ZA5(N3JhdVX=Kz z?fhlI{NzWqW;%I)pfpSPYW>dRFw%s7q8gaFo^OVn`66sJ?Zcn&`y9$BlM^0?EyfLx z3=Q!ng!7|~aoaG3J3q%@N-&{?i{O|9i(ssk+V5-$j`icq%YCEb2=iB+CB#ATdU4}` z+6Y)!@V>?1CLu~)XeLAv(Q7o~_ z^6*IVe26)MU-!CypK0GNh?dsyeoP&6A)@L!5fi@aKY-d%5OacQRaaN{N{;f%*o+){ z7uSg?4|l_HC|RUrNa;$i;?Pk*LIjvkeP~kOo?;>L5xy?%xKgd5QJh$QQFnLTuCYXk=?Y_gW$ci&8wZ*)DAH2E| z2UQ~9LFTq)Cq_4)2n}3JCj=h&Z@HmoudI5gf9!eeMjm3w0i>^o40S%sY^lJB8@@+X zUvDTXq%4IefB}f}7b=+fQsV3b;`F9&iFA5S2tn2|X4j7AG1+3ZaQO!`qrGUaKxIAv zDT5M~(QLOes4y@Akq4PPFrhjil_p^GYx{stHnjf*RPA4YaGM<#Cvn=8Z!PFVc7qNb z6>pKVN1b_*r4N;SCkO}5#9Q!Rhbe)582AzDImTq{MW{lQfQk!)8vhgbYs=h)c)s{iVje$ zj6u+#Dt(1gUR6~kdrLV_2b`bWnt$88{bpcjC|)^}v(H5c%bc>b(;70M?@L);xDaU5 zi5MSIz{dC>F4qc3SDQQRx6WQey5Hub7o*qLHF1)&Ztje7=%@>EKgxBYBay8%#k$Ok z(!i$pG!^%@DNm@^*0=jvl<&=4X8y)NV;07AwYJt@9$yP_^!8?r8L3jAtl>99YAP$~ z*4`XtoKh+c8dKmAwMm7%AHut{sF0_;lme`Oh`~^1HL=g%cymnuEj7h+BgA6kL8)RR z>K3glRp6XrDC-j@Y)R}0AHnQiKaP=ubRu}CwwJ~sPmJexEhBaJnv&AuQMuDy+l@e< zvdl=O#WWYN(F+xk5(k=_;l>E+%kfg)+U1EW8(<+l<;dL{lAJ@q=BNlG(?gY(+sSep zYOBpHoqYj?~w2p0ZaqMYtGC1vO(6Qw9j6WcYH@%Ayvcq~L%U@_X$ zByBAw(tpWX#g{wmHP(~nG&=mT_GTRwzM3K#@pL2*(Um>E;>C=)+~G}bH0LiIqs~vI z0HZjz!X$QSts$1UR0T zhdZ5Q7g+^2yyhTtrt3yYh7@oNkSF@6Kn~dA0Bn`R*|I+{InnGN$UjTz`h$we3385r zUcB3&F&F&CW<+Hu59t21^*yv*tE}zw%2HGO#lWCz--5+b^cF#G@gG6KjRu5j1fIwc z2Hy%5?CqnEA!Zyc*iZAlpzdc+2L~@`W5o7Ehc4QC#`*prF&qqi!T+=Q=+J}=@ZUqD TBKuz+++%aj;VjnD=fQsfma;V1 diff --git a/poetry.lock b/poetry.lock index 300e5d79b7..7ac8188042 100644 --- a/poetry.lock +++ b/poetry.lock @@ -419,17 +419,6 @@ files = [ {file = "blinker-1.6.2.tar.gz", hash = "sha256:4afd3de66ef3a9f8067559fb7a1cbe555c17dcbe15971b05d1b625c3e7abe213"}, ] -[[package]] -name = "boto" -version = "2.49.0" -description = "Amazon Web Services Library" -optional = false -python-versions = "*" -files = [ - {file = "boto-2.49.0-py2.py3-none-any.whl", hash = "sha256:147758d41ae7240dc989f0039f27da8ca0d53734be0eb869ef16e3adcfa462e8"}, - {file = "boto-2.49.0.tar.gz", hash = "sha256:ea0d3b40a2d852767be77ca343b58a9e3a4b00d9db440efb8da74b4e58025e5a"}, -] - [[package]] name = "boto3" version = "1.28.8" @@ -1897,16 +1886,6 @@ thriftpy2 = {version = ">=0.4.0,<0.5.0", markers = "python_version >= \"3.0\""} [package.extras] kerberos = ["thrift_sasl (==0.2.1)"] -[[package]] -name = "inflection" -version = "0.3.1" -description = "A port of Ruby on Rails inflector to Python" -optional = false -python-versions = "*" -files = [ - {file = "inflection-0.3.1.tar.gz", hash = "sha256:18ea7fb7a7d152853386523def08736aa8c32636b047ade55f7578c4edeb16ca"}, -] - [[package]] name = "influxdb" version = "5.2.3" @@ -3875,23 +3854,6 @@ files = [ {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, ] -[[package]] -name = "qds-sdk" -version = "1.16.1" -description = "Python SDK for coding to the Qubole Data Service API" -optional = false -python-versions = "*" -files = [ - {file = "qds_sdk-1.16.1.tar.gz", hash = "sha256:28850682afcf3ab0f2a74a9fd442715519db3ee2ba91c7ecb0b1a56773748ffd"}, -] - -[package.dependencies] -boto = ">=2.45.0" -inflection = "0.3.1" -requests = ">=1.0.3" -six = ">=1.2.0" -urllib3 = ">=1.0.2" - [[package]] name = "rdflib" version = "6.3.2" @@ -5333,4 +5295,4 @@ testing = ["coverage (>=5.0.3)", "zope.event", "zope.testing"] [metadata] lock-version = "2.0" python-versions = ">=3.8,<3.11" -content-hash = "db3de74fd61befac80d3a2d3351ebffb719aae21686e2f22a5c07b01fea590ce" +content-hash = "c12875e9f500f6563ea6ccdfc7f816de4bb742fb5faa1cd375a972ef6b64deb8" diff --git a/pyproject.toml b/pyproject.toml index cbeb9ae919..bdb0b47bd3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -126,7 +126,6 @@ pymssql = "2.2.8" pyodbc = "4.0.28" python-arango = "6.1.0" python-rapidjson = "1.1.0" -qds-sdk = ">=1.9.6" requests-aws-sign = "0.1.5" sasl = ">=0.1.3" simple-salesforce = "0.74.3" diff --git a/redash/query_runner/qubole.py b/redash/query_runner/qubole.py deleted file mode 100644 index 717b2cdc48..0000000000 --- a/redash/query_runner/qubole.py +++ /dev/null @@ -1,169 +0,0 @@ -import logging -import time -from io import StringIO - -import requests - -from redash.query_runner import ( - TYPE_STRING, - BaseQueryRunner, - JobTimeoutException, - register, -) - -try: - import qds_sdk # noqa: F401 - from qds_sdk.commands import ( - Command, - HiveCommand, - PrestoCommand, - SqlCommand, - ) - from qds_sdk.qubole import Qubole as qbol - - enabled = True -except ImportError: - enabled = False - - -class Qubole(BaseQueryRunner): - should_annotate_query = False - - @classmethod - def configuration_schema(cls): - return { - "type": "object", - "properties": { - "query_type": { - "type": "string", - "title": "Query Type (quantum / presto / hive)", - "default": "hive", - }, - "endpoint": { - "type": "string", - "title": "API Endpoint", - "default": "https://api.qubole.com", - }, - "token": {"type": "string", "title": "Auth Token"}, - "cluster": { - "type": "string", - "title": "Cluster Label", - "default": "default", - }, - }, - "order": ["query_type", "endpoint", "token", "cluster"], - "required": ["endpoint", "token"], - "secret": ["token"], - } - - @classmethod - def type(cls): - return "qubole" - - @classmethod - def name(cls): - return "Qubole" - - @classmethod - def enabled(cls): - return enabled - - def test_connection(self): - headers = self._get_header() - r = requests.head("%s/api/latest/users" % self.configuration.get("endpoint"), headers=headers) - r.status_code == 200 - - def run_query(self, query, user): - qbol.configure( - api_token=self.configuration.get("token"), - api_url="%s/api" % self.configuration.get("endpoint"), - ) - - try: - query_type = self.configuration.get("query_type", "hive") - - if query_type == "quantum": - cmd = SqlCommand.create(query=query) - elif query_type == "hive": - cmd = HiveCommand.create(query=query, label=self.configuration.get("cluster")) - elif query_type == "presto": - cmd = PrestoCommand.create(query=query, label=self.configuration.get("cluster")) - else: - raise Exception( - "Invalid Query Type:%s.\ - It must be : hive / presto / quantum." - % self.configuration.get("query_type") - ) - - logging.info("Qubole command created with Id: %s and Status: %s", cmd.id, cmd.status) - - while not Command.is_done(cmd.status): - time.sleep(qbol.poll_interval) - cmd = Command.find(cmd.id) - logging.info("Qubole command Id: %s and Status: %s", cmd.id, cmd.status) - - rows = [] - columns = [] - error = None - - if cmd.status == "done": - fp = StringIO() - cmd.get_results( - fp=fp, - inline=True, - delim="\t", - fetch=False, - qlog=None, - arguments=["true"], - ) - - results = fp.getvalue() - fp.close() - - data = results.split("\r\n") - columns = self.fetch_columns([(i, TYPE_STRING) for i in data.pop(0).split("\t")]) - rows = [dict(zip((column["name"] for column in columns), row.split("\t"))) for row in data] - - data = {"columns": columns, "rows": rows} - except (KeyboardInterrupt, JobTimeoutException): - logging.info("Sending KILL signal to Qubole Command Id: %s", cmd.id) - cmd.cancel() - raise - - return data, error - - def get_schema(self, get_stats=False): - schemas = {} - try: - headers = self._get_header() - content = requests.get( - "%s/api/latest/hive?describe=true&per_page=10000" % self.configuration.get("endpoint"), - headers=headers, - ) - data = content.json() - - for schema in data["schemas"]: - tables = data["schemas"][schema] - for table in tables: - table_name = list(table.keys())[0] - columns = [f["name"] for f in table[table_name]["columns"]] - - if schema != "default": - table_name = "{}.{}".format(schema, table_name) - - schemas[table_name] = {"name": table_name, "columns": columns} - - except Exception as e: - logging.error("Failed to get schema information from Qubole. Error {}".format(str(e))) - - return list(schemas.values()) - - def _get_header(self): - return { - "Content-type": "application/json", - "Accept": "application/json", - "X-AUTH-TOKEN": self.configuration.get("token"), - } - - -register(Qubole) diff --git a/redash/settings/__init__.py b/redash/settings/__init__.py index f62ba611e0..e43a952c59 100644 --- a/redash/settings/__init__.py +++ b/redash/settings/__init__.py @@ -312,7 +312,6 @@ def email_server_is_configured(): "redash.query_runner.salesforce", "redash.query_runner.query_results", "redash.query_runner.prometheus", - "redash.query_runner.qubole", "redash.query_runner.db2", "redash.query_runner.druid", "redash.query_runner.kylin", From 4eb5f4e47feab1eff84ea83b7f9d9c658065151b Mon Sep 17 00:00:00 2001 From: Justin Clift Date: Sat, 6 Apr 2024 00:02:31 +1000 Subject: [PATCH 004/122] Remove version check and all of the data sharing (#6852) --- .../ApplicationLayout/VersionInfo.jsx | 12 +- client/app/components/BeaconConsent.jsx | 79 -------------- client/app/components/HelpTrigger.jsx | 1 - client/app/pages/home/Home.jsx | 2 - .../GeneralSettings/BeaconConsentSettings.jsx | 38 ------- .../components/GeneralSettings/index.jsx | 2 - redash/app.py | 4 - redash/handlers/authentication.py | 19 ++-- redash/handlers/setup.py | 11 +- redash/settings/__init__.py | 1 - redash/settings/organization.py | 1 - redash/tasks/__init__.py | 1 - redash/tasks/general.py | 22 ---- redash/tasks/schedule.py | 5 +- redash/templates/setup.html | 14 --- redash/version_check.py | 103 ------------------ 16 files changed, 10 insertions(+), 305 deletions(-) delete mode 100644 client/app/components/BeaconConsent.jsx delete mode 100644 client/app/pages/settings/components/GeneralSettings/BeaconConsentSettings.jsx delete mode 100644 redash/version_check.py diff --git a/client/app/components/ApplicationArea/ApplicationLayout/VersionInfo.jsx b/client/app/components/ApplicationArea/ApplicationLayout/VersionInfo.jsx index e655e7f9fb..2ed1236e93 100644 --- a/client/app/components/ApplicationArea/ApplicationLayout/VersionInfo.jsx +++ b/client/app/components/ApplicationArea/ApplicationLayout/VersionInfo.jsx @@ -1,6 +1,5 @@ import React from "react"; -import Link from "@/components/Link"; -import { clientConfig, currentUser } from "@/services/auth"; +import { clientConfig } from "@/services/auth"; import frontendVersion from "@/version.json"; export default function VersionInfo() { @@ -10,15 +9,6 @@ export default function VersionInfo() { Version: {clientConfig.version} {frontendVersion !== clientConfig.version && ` (${frontendVersion.substring(0, 8)})`} - {clientConfig.newVersionAvailable && currentUser.hasPermission("super_admin") && ( -
- {/* eslint-disable react/jsx-no-target-blank */} - - Update Available