Skip to content

Commit b5e67af

Browse files
committed
feat(observer): add detailed GitHub workflow failure analysis
- Add comprehensive failure details extraction from workflow runs and jobs - Include failed steps, error logs, and key error patterns in notifications - Support both job-specific and workflow-level log analysis - Implement fallback mechanisms for log retrieval via direct API calls - Reduce monitoring interval from 5 to 1 minute for faster feedback
1 parent 95688f1 commit b5e67af

File tree

1 file changed

+274
-6
lines changed

1 file changed

+274
-6
lines changed

core/src/233/main/kotlin/cc/unitmesh/devti/observer/PipelineStatusProcessor.kt

Lines changed: 274 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,6 @@ import cc.unitmesh.devti.provider.observer.AgentObserver
55
import cc.unitmesh.devti.settings.coder.coderSetting
66
import cc.unitmesh.devti.settings.devops.devopsPromptsSettings
77
import com.intellij.notification.NotificationType
8-
import com.intellij.openapi.Disposable
98
import com.intellij.openapi.diagnostic.Logger
109
import com.intellij.openapi.project.Project
1110
import com.intellij.openapi.project.ProjectManager
@@ -16,9 +15,15 @@ import git4idea.push.GitPushRepoResult
1615
import git4idea.repo.GitRepository
1716
import org.kohsuke.github.GHRepository
1817
import org.kohsuke.github.GHWorkflowRun
18+
import org.kohsuke.github.GHWorkflowJob
1919
import org.kohsuke.github.GitHub
20+
import java.io.BufferedReader
21+
import java.io.InputStreamReader
22+
import java.net.HttpURLConnection
23+
import java.net.URL
2024
import java.util.concurrent.ScheduledFuture
2125
import java.util.concurrent.TimeUnit
26+
import java.util.zip.ZipInputStream
2227

2328
class PipelineStatusProcessor : AgentObserver, GitPushListener {
2429
private val log = Logger.getInstance(PipelineStatusProcessor::class.java)
@@ -123,7 +128,7 @@ class PipelineStatusProcessor : AgentObserver, GitPushListener {
123128
)
124129
stopMonitoring()
125130
}
126-
}, 1, 5, TimeUnit.MINUTES) // 1分钟后开始第一次检查,然后每5分钟检查一次
131+
}, 0, 1, TimeUnit.MINUTES) // 1分钟后开始第一次检查,然后每5分钟检查一次
127132
}
128133

129134
private fun findWorkflowRunForCommit(remoteUrl: String, commitSha: String): GHWorkflowRun? {
@@ -132,12 +137,11 @@ class PipelineStatusProcessor : AgentObserver, GitPushListener {
132137
val ghRepository = getGitHubRepository(github, remoteUrl) ?: return null
133138

134139
// 使用 queryWorkflowRuns 查询workflow runs
135-
// 这样可以减少API调用次数
136140
val allRuns = ghRepository.queryWorkflowRuns()
137141
.list()
138142
.iterator()
139143
.asSequence()
140-
.take(50)
144+
.take(50) // 检查最近50个runs
141145
.toList()
142146

143147
// 查找匹配commit SHA的workflow run
@@ -168,12 +172,15 @@ class PipelineStatusProcessor : AgentObserver, GitPushListener {
168172
)
169173
true
170174
}
171-
GHWorkflowRun.Conclusion.FAILURE,
175+
GHWorkflowRun.Conclusion.FAILURE -> {
176+
handleWorkflowFailure(workflowRun, commitSha)
177+
true
178+
}
172179
GHWorkflowRun.Conclusion.CANCELLED,
173180
GHWorkflowRun.Conclusion.TIMED_OUT -> {
174181
AutoDevNotifications.notify(
175182
project!!,
176-
"❌ GitHub Action failed for commit: ${commitSha.take(7)} - ${workflowRun.conclusion}",
183+
"❌ GitHub Action ${workflowRun.conclusion.toString().lowercase()} for commit: ${commitSha.take(7)}",
177184
NotificationType.ERROR
178185
)
179186
true
@@ -195,6 +202,253 @@ class PipelineStatusProcessor : AgentObserver, GitPushListener {
195202
}
196203
}
197204

205+
private fun handleWorkflowFailure(workflowRun: GHWorkflowRun, commitSha: String) {
206+
try {
207+
log.info("Analyzing workflow failure for commit: $commitSha")
208+
209+
// 获取失败的构建详情
210+
val failureDetails = getWorkflowFailureDetails(workflowRun)
211+
212+
// 构建详细的错误通知
213+
val detailedMessage = buildDetailedFailureMessage(workflowRun, commitSha, failureDetails)
214+
215+
AutoDevNotifications.notify(
216+
project!!,
217+
detailedMessage,
218+
NotificationType.ERROR
219+
)
220+
221+
// 记录详细日志
222+
log.info("Workflow failure details for commit $commitSha: $failureDetails")
223+
224+
} catch (e: Exception) {
225+
log.error("Error analyzing workflow failure for commit: $commitSha", e)
226+
// 回退到简单通知
227+
AutoDevNotifications.notify(
228+
project!!,
229+
"❌ GitHub Action failed for commit: ${commitSha.take(7)} - ${workflowRun.conclusion}\nURL: ${workflowRun.htmlUrl}",
230+
NotificationType.ERROR
231+
)
232+
}
233+
}
234+
235+
private fun getWorkflowFailureDetails(workflowRun: GHWorkflowRun): WorkflowFailureDetails {
236+
val failureDetails = WorkflowFailureDetails()
237+
238+
try {
239+
// 获取所有jobs
240+
val jobs = workflowRun.listJobs().toList()
241+
242+
for (job in jobs) {
243+
if (job.conclusion == GHWorkflowRun.Conclusion.FAILURE) {
244+
val jobFailure = JobFailure(
245+
jobName = job.name,
246+
errorSteps = extractFailedSteps(job),
247+
logs = getJobLogsFromAPI(workflowRun, job),
248+
jobUrl = job.htmlUrl?.toString()
249+
)
250+
failureDetails.failedJobs.add(jobFailure)
251+
}
252+
}
253+
254+
// 如果没有具体的job失败信息,尝试获取整个workflow的日志
255+
if (failureDetails.failedJobs.isEmpty()) {
256+
failureDetails.workflowLogs = getWorkflowLogsFromAPI(workflowRun)
257+
}
258+
259+
} catch (e: Exception) {
260+
log.error("Error getting workflow failure details", e)
261+
failureDetails.error = e.message
262+
}
263+
264+
return failureDetails
265+
}
266+
267+
private fun extractFailedSteps(job: GHWorkflowJob): List<String> {
268+
val failedSteps = mutableListOf<String>()
269+
270+
try {
271+
val steps = job.steps
272+
for (step in steps) {
273+
if (step.conclusion == GHWorkflowRun.Conclusion.FAILURE) {
274+
failedSteps.add("${step.name}: ${step.conclusion} (${step.number})")
275+
}
276+
}
277+
} catch (e: Exception) {
278+
log.error("Error extracting failed steps", e)
279+
}
280+
281+
return failedSteps
282+
}
283+
284+
private fun getJobLogsFromAPI(workflowRun: GHWorkflowRun, job: GHWorkflowJob): String? {
285+
return try {
286+
// 尝试使用 GitHub Java API 获取日志
287+
job.downloadLogs { logStream ->
288+
logStream.use { stream ->
289+
BufferedReader(InputStreamReader(stream)).use { reader ->
290+
val logs = reader.readText()
291+
// 提取关键错误信息(最后2000个字符,通常包含错误信息)
292+
if (logs.length > 2000) {
293+
"...\n" + logs.takeLast(2000)
294+
} else logs
295+
}
296+
}
297+
}
298+
} catch (e: Exception) {
299+
log.error("Error downloading job logs for job: ${job.name}", e)
300+
// 如果GitHub Java API失败,尝试直接API调用
301+
getLogsViaDirectAPI(workflowRun, job.id)
302+
}
303+
}
304+
305+
private fun getWorkflowLogsFromAPI(workflowRun: GHWorkflowRun): String? {
306+
return try {
307+
// 使用GitHub Java API获取整个workflow的日志
308+
workflowRun.downloadLogs { logStream ->
309+
// 处理ZIP格式的日志文件
310+
ZipInputStream(logStream).use { zipStream ->
311+
val logContents = StringBuilder()
312+
var entry = zipStream.nextEntry
313+
while (entry != null && logContents.length < 5000) { // 限制日志长度
314+
if (entry.name.endsWith(".txt")) {
315+
val content = zipStream.readBytes().toString(Charsets.UTF_8)
316+
logContents.append("=== ${entry.name} ===\n")
317+
logContents.append(content)
318+
logContents.append("\n\n")
319+
}
320+
entry = zipStream.nextEntry
321+
}
322+
logContents.toString()
323+
}
324+
}
325+
} catch (e: Exception) {
326+
log.error("Error downloading workflow logs", e)
327+
null
328+
}
329+
}
330+
331+
private fun getLogsViaDirectAPI(workflowRun: GHWorkflowRun, jobId: Long): String? {
332+
return try {
333+
val token = project?.devopsPromptsSettings?.githubToken
334+
val repoPath = extractRepositoryPath(workflowRun.repository.htmlUrl.toString())
335+
val apiUrl = "https://api.github.com/repos/$repoPath/actions/jobs/$jobId/logs"
336+
337+
val connection = URL(apiUrl).openConnection() as HttpURLConnection
338+
connection.requestMethod = "GET"
339+
connection.setRequestProperty("Accept", "application/vnd.github+json")
340+
connection.setRequestProperty("X-GitHub-Api-Version", "2022-11-28")
341+
342+
if (!token.isNullOrBlank()) {
343+
connection.setRequestProperty("Authorization", "Bearer $token")
344+
}
345+
346+
if (connection.responseCode == 302) {
347+
// 处理重定向到日志下载链接
348+
val redirectUrl = connection.getHeaderField("Location")
349+
if (redirectUrl != null) {
350+
val logConnection = URL(redirectUrl).openConnection() as HttpURLConnection
351+
logConnection.inputStream.use { stream ->
352+
BufferedReader(InputStreamReader(stream)).use { reader ->
353+
val logs = reader.readText()
354+
if (logs.length > 2000) {
355+
"...\n" + logs.takeLast(2000)
356+
} else {
357+
logs
358+
}
359+
}
360+
}
361+
} else null
362+
} else null
363+
} catch (e: Exception) {
364+
log.error("Error getting logs via direct API", e)
365+
null
366+
}
367+
}
368+
369+
private fun buildDetailedFailureMessage(
370+
workflowRun: GHWorkflowRun,
371+
commitSha: String,
372+
failureDetails: WorkflowFailureDetails
373+
): String {
374+
val message = StringBuilder()
375+
message.append("❌ GitHub Action failed for commit: ${commitSha.take(7)}\n")
376+
message.append("Workflow: ${workflowRun.name}\n")
377+
message.append("URL: ${workflowRun.htmlUrl}\n\n")
378+
379+
if (failureDetails.error != null) {
380+
message.append("Error getting details: ${failureDetails.error}\n")
381+
} else if (failureDetails.failedJobs.isNotEmpty()) {
382+
message.append("Failed Jobs:\n")
383+
384+
for (jobFailure in failureDetails.failedJobs.take(2)) { // 最多显示2个失败的job
385+
message.append("${jobFailure.jobName}\n")
386+
if (jobFailure.jobUrl != null) {
387+
message.append(" URL: ${jobFailure.jobUrl}\n")
388+
}
389+
390+
if (jobFailure.errorSteps.isNotEmpty()) {
391+
message.append(" Failed steps:\n")
392+
for (step in jobFailure.errorSteps.take(3)) { // 最多显示3个失败步骤
393+
message.append(" - $step\n")
394+
}
395+
}
396+
397+
// 添加关键错误信息
398+
jobFailure.logs?.let { logs ->
399+
val errorLines = extractKeyErrorLines(logs)
400+
if (errorLines.isNotEmpty()) {
401+
message.append(" Key errors:\n")
402+
for (errorLine in errorLines.take(3)) { // 最多显示3行关键错误
403+
message.append(" ${errorLine.take(100)}\n") // 限制每行长度
404+
}
405+
}
406+
}
407+
message.append("\n")
408+
}
409+
} else if (failureDetails.workflowLogs != null) {
410+
val errorLines = extractKeyErrorLines(failureDetails.workflowLogs!!)
411+
if (errorLines.isNotEmpty()) {
412+
message.append("Key errors:\n")
413+
for (errorLine in errorLines.take(5)) {
414+
message.append(" ${errorLine.take(100)}\n")
415+
}
416+
}
417+
}
418+
419+
return message.toString()
420+
}
421+
422+
private fun extractKeyErrorLines(logs: String): List<String> {
423+
val errorPatterns = listOf(
424+
"Error:",
425+
"Exception:",
426+
"Failed:",
427+
"FAILED:",
428+
"BUILD FAILED",
429+
"npm ERR!",
430+
"fatal:",
431+
"Traceback",
432+
"AssertionError",
433+
"SyntaxError",
434+
"CompileError",
435+
"",
436+
"FAIL:",
437+
"stderr:"
438+
)
439+
440+
return logs.lines()
441+
.filter { line ->
442+
errorPatterns.any { pattern ->
443+
line.contains(pattern, ignoreCase = true)
444+
}
445+
}
446+
.map { it.trim() }
447+
.filter { it.isNotBlank() && it.length > 10 } // 过滤太短的行
448+
.distinct()
449+
.take(10) // 最多返回10行
450+
}
451+
198452
private fun createGitHubConnection(): GitHub {
199453
val token = project?.devopsPromptsSettings?.githubToken
200454
return if (token.isNullOrBlank()) {
@@ -227,4 +481,18 @@ class PipelineStatusProcessor : AgentObserver, GitPushListener {
227481
monitoringJob = null
228482
log.info("Pipeline monitoring stopped")
229483
}
484+
485+
// 数据类用于存储失败详情
486+
private data class WorkflowFailureDetails(
487+
val failedJobs: MutableList<JobFailure> = mutableListOf(),
488+
var workflowLogs: String? = null,
489+
var error: String? = null
490+
)
491+
492+
private data class JobFailure(
493+
val jobName: String,
494+
val errorSteps: List<String>,
495+
val logs: String?,
496+
val jobUrl: String? = null
497+
)
230498
}

0 commit comments

Comments
 (0)