feat:job logs instead of wf-run logs

CHANGELOG.md

@@ -4,7 +4,11 @@
 
 ## [Unreleased]
 
-## [1.15.3]
+## [1.16.0]
+
+### 🚀 Features
+
+- Downloading job logs instead of entire run logs #108
 
 ## [1.15.2]
 
@@ -367,4 +371,4 @@ Full Changelog: https://github.com/cunla/github-actions-jetbrains-plugin/commits
 ### Removed
 ### Fixed
 ### Security
--->
+-->

gradle.properties

@@ -3,7 +3,7 @@ pluginGroup=com.dsoftware.ghmanager
 pluginName=github-actions-manager
 
 # SemVer format -> https://semver.org
-pluginVersion=1.15.3
+pluginVersion=1.16.0
 
 # Supported build number ranges and IntelliJ Platform versions -> https://plugins.jetbrains.com/docs/intellij/build-number-ranges.html
 pluginSinceBuild=233
@@ -12,7 +12,7 @@ pluginUntilBuild=235.*
 # IntelliJ Platform Properties -> https://plugins.jetbrains.com/docs/intellij/tools-gradle-intellij-plugin.html#configuration-intellij-extension
 platformType=IC
 #platformVersion=LATEST-EAP-SNAPSHOT
-platformVersion=2023.3.3
+platformVersion=2023.3.4
 
 # Plugin Dependencies -> https://plugins.jetbrains.com/docs/intellij/plugin-dependencies.html
 # Example: platformPlugins = com.intellij.java, com.jetbrains.php:203.4449.22

src/main/kotlin/com/dsoftware/ghmanager/Constants.kt

@@ -9,14 +9,6 @@ object Constants {
     const val LOG_MSG_MISSING = "Job logs missing for: "
     const val LOG_MSG_JOB_IN_PROGRESS = "Job is still in progress, can't view logs."
 
-    fun emptyTextMessage(logValue: String): Boolean {
-        return (logValue.startsWith(LOG_MSG_MISSING)
-            || logValue.startsWith(LOG_MSG_PICK_JOB)
-            || logValue.startsWith(
-            LOG_MSG_JOB_IN_PROGRESS
-        ))
-    }
-
     fun updateEmptyText(logValue: String, emptyText: StatusText): Boolean {
         if (logValue.startsWith(LOG_MSG_MISSING)
             || logValue.startsWith(LOG_MSG_PICK_JOB)

src/main/kotlin/com/dsoftware/ghmanager/actions/ReloadJobsAction.kt

@@ -5,7 +5,7 @@ import com.intellij.ide.actions.RefreshAction
 import com.intellij.openapi.actionSystem.AnActionEvent
 import com.intellij.openapi.diagnostic.logger
 
-class ReloadJobsAction : RefreshAction("Refresh Workflow Jobs", null, AllIcons.Actions.Refresh) {
+class ReloadJobsAction : RefreshAction("Refresh Workflow Run Jobs List", null, AllIcons.Actions.Refresh) {
     override fun update(e: AnActionEvent) {
         val selection = e.getData(ActionKeys.ACTION_DATA_CONTEXT)?.jobsDataProvider
         e.presentation.isEnabled = selection != null

src/main/kotlin/com/dsoftware/ghmanager/actions/ReloadLogAction.kt

@@ -5,18 +5,18 @@ import com.intellij.ide.actions.RefreshAction
 import com.intellij.openapi.actionSystem.AnActionEvent
 import com.intellij.openapi.diagnostic.logger
 
-class ReloadLogAction : RefreshAction("Refresh Workflow Log", null, AllIcons.Actions.Refresh) {
+class ReloadJobLogAction : RefreshAction("Refresh Job Log", null, AllIcons.Actions.Refresh) {
     override fun update(e: AnActionEvent) {
-        val selection = e.getData(ActionKeys.ACTION_DATA_CONTEXT)?.logsDataProvider
+        val selection = e.getData(ActionKeys.ACTION_DATA_CONTEXT)?.logDataProvider
         e.presentation.isEnabled = selection != null
     }
 
     override fun actionPerformed(e: AnActionEvent) {
         LOG.debug("GitHubWorkflowLogReloadAction action performed")
-        e.getRequiredData(ActionKeys.ACTION_DATA_CONTEXT).logsDataProvider?.reload()
+        e.getRequiredData(ActionKeys.ACTION_DATA_CONTEXT).logDataProvider?.reload()
     }
 
     companion object {
-        private val LOG = logger<ReloadLogAction>()
+        private val LOG = logger<ReloadJobLogAction>()
     }
 }

src/main/kotlin/com/dsoftware/ghmanager/api/GetJobLogRequest.kt

@@ -0,0 +1,128 @@
+package com.dsoftware.ghmanager.api
+
+import com.dsoftware.ghmanager.api.model.Job
+import com.dsoftware.ghmanager.api.model.JobStep
+import com.intellij.openapi.diagnostic.logger
+import org.jetbrains.plugins.github.api.GithubApiRequest
+import org.jetbrains.plugins.github.api.GithubApiResponse
+import java.io.BufferedReader
+import java.io.IOException
+import java.io.InputStream
+import java.io.InputStreamReader
+import java.text.ParseException
+import java.text.SimpleDateFormat
+import java.util.Date
+import java.util.TimeZone
+
+typealias JobLog = Map<Int, StringBuilder>
+
+class GetJobLogRequest(private val job: Job) : GithubApiRequest.Get<String>(job.url + "/logs") {
+    private val stepsPeriodMap = job.steps?.associate { step ->
+        step.number to (step.startedAt to step.completedAt)
+    } ?: emptyMap()
+    private val lastStepNumber: Int = stepsPeriodMap.keys.maxOrNull() ?: 0
+
+    override fun extractResult(response: GithubApiResponse): String {
+        LOG.debug("extracting result for $url")
+        return response.handleBody {
+            extractJobLogFromStream(it)
+        }
+    }
+
+    fun extractJobLogFromStream(inputStream: InputStream): String {
+        val stepLogs = extractLogByStep(inputStream)
+        return stepsAsLog(stepLogs)
+    }
+
+    fun extractLogByStep(inputStream: InputStream): Map<Int, StringBuilder> {
+        val dateTimePattern = "yyyy-MM-dd'T'HH:mm:ss.SSS"
+        val formatter = SimpleDateFormat(dateTimePattern)
+        val contentBuilders = HashMap<Int, StringBuilder>()
+        formatter.timeZone = TimeZone.getTimeZone("UTC")
+        var lineNum = 0
+        var currStep = 1
+        try {
+            val reader = BufferedReader(InputStreamReader(inputStream))
+            val lines = reader.lines()
+            for (line in lines) {
+                ++lineNum
+                if (line.length < 29) {
+                    contentBuilders.getOrDefault(currStep, StringBuilder()).append(line + "\n")
+                    continue
+                }
+                val datetimeStr = line.substring(0, 23)
+                try {
+                    val time = formatter.parse(datetimeStr)
+                    currStep = findStep(currStep, time)
+                } catch (e: ParseException) {
+                    LOG.warn("Failed to parse date \"$datetimeStr\" from log line $lineNum: $line, $e")
+                }
+                contentBuilders.getOrPut(currStep) { StringBuilder(400_000) }.append(line + "\n")
+            }
+        } catch (e: IOException) {
+            LOG.warn("Could not read log from input stream", e)
+        }
+        return contentBuilders
+    }
+
+    private fun findStep(initialStep: Int, time: Date): Int {
+        var currStep = initialStep
+        while (currStep < lastStepNumber) {
+            if (!stepsPeriodMap.containsKey(currStep)) {
+                currStep += 1
+                continue
+            }
+            val currStart = stepsPeriodMap[currStep]?.first
+            val currEnd = stepsPeriodMap[currStep]?.second
+            if (currStart != null && currStart.after(time)) {
+                return currStep
+            }
+            if ((currStart == null || currStart.before(time) || currStart == time)
+                && (currEnd == null || currEnd.after(time) || currEnd == time)
+            ) {
+                return currStep
+            }
+            currStep += 1
+        }
+        return currStep
+    }
+
+    private fun stepsAsLog(stepLogs: JobLog): String {
+        val stepsResult: Map<Int, JobStep> = if (job.steps == null) {
+            emptyMap()
+        } else {
+            job.steps.associateBy { it.number }
+        }
+        val stepNumbers = stepsResult.keys.sorted()
+        if (!stepNumbers.containsAll(stepLogs.keys)) {
+            LOG.warn(
+                "Some logs do not have a step-result associated " +
+                    "[steps in results=$stepNumbers, step with logs=${stepLogs.keys}] "
+            )
+        }
+        val res = StringBuilder(1_000_000)
+        for (index in stepNumbers) {
+            val stepInfo = stepsResult[index]!!
+            val indexStr = "%3d".format(index)
+            res.append(
+                when (stepInfo.conclusion) {
+                    "skipped" -> "\u001B[0m\u001B[37m---- Step ${indexStr}: ${stepInfo.name} (skipped) ----\u001b[0m\n"
+                    "failure" -> "\u001B[0m\u001B[31m---- Step ${indexStr}: ${stepInfo.name} (failed) ----\u001b[0m\n"
+                    else -> "\u001B[0m\u001B[32m---- Step ${indexStr}: ${stepInfo.name} ----\u001b[0m\n"
+                }
+            )
+            if (stepInfo.conclusion != "skipped" && stepLogs.containsKey(index) && (res.length < 950_000)) {
+                if (res.length + (stepLogs[index]?.length ?: 0) < 990_000) {
+                    res.append(stepLogs[index])
+                } else {
+                    res.append("Log is too big to display, showing only first 1mb")
+                }
+            }
+        }
+        return res.toString()
+    }
+
+    companion object {
+        private val LOG = logger<GetJobLogRequest>()
+    }
+}

src/main/kotlin/com/dsoftware/ghmanager/api/GithubApi.kt

@@ -1,5 +1,6 @@
 package com.dsoftware.ghmanager.api
 
+import com.dsoftware.ghmanager.api.model.Job
 import com.dsoftware.ghmanager.api.model.WorkflowRunJobs
 import com.dsoftware.ghmanager.api.model.WorkflowRuns
 import com.dsoftware.ghmanager.api.model.WorkflowTypes
@@ -19,12 +20,9 @@ data class WorkflowRunFilter(
     val workflowId: Long? = null,
 )
 
-typealias JobLog = Map<Int, String>
-typealias WorkflowRunLog = Map<String, JobLog>
-
 object GithubApi : GithubApiRequests.Entity("/repos") {
     private val LOG = logger<GithubApi>()
-    fun getWorkflowRunLogs(url: String) = GetRunLogRequest(url).withOperationName("Download Workflow log")
+    fun getJobLog(job: Job) = GetJobLogRequest(job).withOperationName("Get Job log ${job.id}")
 
     fun postUrl(name: String, url: String, data: Any = Object()) =
         GithubApiRequest.Post.Json(url, data, Object::class.java, null).withOperationName(name)
@@ -71,7 +69,6 @@ object GithubApi : GithubApiRequests.Entity("/repos") {
     )
 
 
-
     private inline fun <reified T> get(
         url: String, opName: String,
         pagination: GithubRequestPagination? = null

src/main/kotlin/com/dsoftware/ghmanager/api/model/JobModel.kt

@@ -35,6 +35,8 @@ data class Job(
 
     /* The id of the job. */
     val id: Long,
+    val workflowName: String?,
+    val headBranch: String?,
     /* The id of the associated workflow run. */
     val runId: Long,
     val runUrl: String,
@@ -49,6 +51,8 @@ data class Job(
     val status: String,
     /* The outcome of the job. */
     val conclusion: String?,
+    @JsonFormat(pattern = "yyyy-MM-dd'T'HH:mm:ss'Z'", timezone = "UTC")
+    val createdAt: Date?,
     /* The time that the job started, in ISO 8601 format. */
     @JsonFormat(pattern = "yyyy-MM-dd'T'HH:mm:ss'Z'", timezone = "UTC")
     val startedAt: Date?,
@@ -102,17 +106,12 @@ data class Job(
  * @param completedAt The time that the job finished, in ISO 8601 format.
  */
 data class JobStep(
-    /* The phase of the lifecycle that the job is currently in. */
     val status: String,
-    /* The outcome of the job. */
     val conclusion: String?,
-    /* The name of the job. */
     val name: String,
     val number: Int,
-    /* The time that the step started, in ISO 8601 format. */
     @JsonFormat(pattern = "yyyy-MM-dd'T'HH:mm:ss.SSSX", timezone = "UTC")
     val startedAt: Date? = null,
-    /* The time that the job finished, in ISO 8601 format. */
     @JsonFormat(pattern = "yyyy-MM-dd'T'HH:mm:ss.SSSX", timezone = "UTC")
     val completedAt: Date? = null
 )

src/main/kotlin/com/dsoftware/ghmanager/api/model/RunModel.kt

@@ -1,23 +1,9 @@
 package com.dsoftware.ghmanager.api.model
 
 import com.fasterxml.jackson.annotation.JsonFormat
-import kotlinx.serialization.Serializable
 import java.util.Date
 
 
-data class WorkflowTypes(
-    val totalCount: Int,
-    val workflows: List<WorkflowType> = emptyList()
-)
-
-@Serializable
-data class WorkflowType(
-    val id: Long,
-    val name: String,
-    val path: String,
-    val state: String,
-)
-
 data class PullRequest(
     val id: Int,
     val number: Int,

src/main/kotlin/com/dsoftware/ghmanager/api/model/WorkflowTypes.kt

@@ -0,0 +1,16 @@
+package com.dsoftware.ghmanager.api.model
+
+import kotlinx.serialization.Serializable
+
+data class WorkflowTypes(
+    val totalCount: Int,
+    val workflows: List<WorkflowType> = emptyList()
+)
+
+@Serializable
+data class WorkflowType(
+    val id: Long,
+    val name: String,
+    val path: String,
+    val state: String,
+)