fix(ingest/dremio): Dremio software jobs retrieval SQL query fix query error

metadata-ingestion/src/datahub/ingestion/source/dremio/dremio_api.py

@@ -249,7 +249,7 @@ def post(self, url: str, data: str) -> Dict:
         )
         return response.json()
 
-    def execute_query(self, query: str, timeout: int = 300) -> List[Dict[str, Any]]:
+    def execute_query(self, query: str, timeout: int = 3600) -> List[Dict[str, Any]]:
         """Execute SQL query with timeout and error handling"""
         try:
             response = self.post(url="/sql", data=json.dumps({"sql": query}))

metadata-ingestion/src/datahub/ingestion/source/dremio/dremio_sql_queries.py

@@ -237,19 +237,27 @@ class DremioSQLQueries:
 
     QUERY_ALL_JOBS = """
     SELECT
-        *
+        job_id,
+        user_name,
+        submitted_ts,
+        query,
+        queried_datasets
     FROM
         SYS.JOBS_RECENT
     WHERE
         STATUS = 'COMPLETED'
-        AND ARRAY_SIZE(queried_datasets)>0
+        AND LENGTH(queried_datasets)>0
         AND user_name != '$dremio$'
         AND query_type not like '%INTERNAL%'
     """
 
     QUERY_ALL_JOBS_CLOUD = """
         SELECT
-            *
+            job_id,
+            user_name,
+            submitted_ts,
+            query,
+            CONCAT('[', ARRAY_TO_STRING(queried_datasets, ','), ']') as queried_datasets
         FROM
             sys.project.history.jobs
         WHERE