Skip to content

Commit 0db8390

Browse files
phodalphodal
committed
refactor(agenttool): unify search tool usage and optimize RipgrepSearcher
1 parent 39ac1ac commit 0db8390

File tree

16 files changed

+146
-143
lines changed

16 files changed

+146
-143
lines changed

‎exts/devins-lang/src/main/kotlin/cc/unitmesh/devti/language/agenttool/AgentTool.kt renamed to ‎core/src/main/kotlin/cc/unitmesh/devti/agenttool/AgentTool.kt

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
package cc.unitmesh.devti.language.agenttool
1+
package cc.unitmesh.devti.agenttool
22

33
import com.intellij.openapi.extensions.ExtensionPointName
44

‎exts/devins-lang/src/main/kotlin/cc/unitmesh/devti/language/agenttool/AgentToolContext.kt renamed to ‎core/src/main/kotlin/cc/unitmesh/devti/agenttool/AgentToolContext.kt

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
package cc.unitmesh.devti.language.agenttool
1+
package cc.unitmesh.devti.agenttool
22

33
import com.intellij.openapi.project.Project
44

‎exts/devins-lang/src/main/kotlin/cc/unitmesh/devti/language/agenttool/AgentToolResult.kt renamed to ‎core/src/main/kotlin/cc/unitmesh/devti/agenttool/AgentToolResult.kt

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
package cc.unitmesh.devti.language.agenttool
1+
package cc.unitmesh.devti.agenttool
22

33
data class AgentToolResult(
44
val isSuccess: Boolean,

‎exts/devins-lang/src/main/kotlin/cc/unitmesh/devti/language/agenttool/browse/BrowseTool.kt renamed to ‎core/src/main/kotlin/cc/unitmesh/devti/agenttool/browse/BrowseTool.kt

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
1-
package cc.unitmesh.devti.language.agenttool.browse
1+
package cc.unitmesh.devti.agenttool.browse
22

3-
import cc.unitmesh.devti.language.agenttool.AgentToolContext
4-
import cc.unitmesh.devti.language.agenttool.AgentTool
5-
import cc.unitmesh.devti.language.agenttool.AgentToolResult
3+
import cc.unitmesh.devti.agenttool.AgentToolContext
4+
import cc.unitmesh.devti.agenttool.AgentTool
5+
import cc.unitmesh.devti.agenttool.AgentToolResult
66
import org.jsoup.Jsoup
77
import org.jsoup.nodes.Document
88

‎exts/devins-lang/src/main/kotlin/cc/unitmesh/devti/language/agenttool/browse/DocumentCleaner.kt renamed to ‎core/src/main/kotlin/cc/unitmesh/devti/agenttool/browse/DocumentCleaner.kt

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
package cc.unitmesh.devti.language.agenttool.browse
1+
package cc.unitmesh.devti.agenttool.browse
22

33
import org.jsoup.Jsoup
44
import org.jsoup.nodes.Document

‎exts/devins-lang/src/main/kotlin/cc/unitmesh/devti/language/agenttool/browse/DocumentContent.kt renamed to ‎core/src/main/kotlin/cc/unitmesh/devti/agenttool/browse/DocumentContent.kt

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
package cc.unitmesh.devti.language.agenttool.browse
1+
package cc.unitmesh.devti.agenttool.browse
22

33
data class DocumentContent(
44
val title: String?,

‎core/src/main/kotlin/cc/unitmesh/devti/agenttool/search/RipgrepOutputProcessor.kt

Lines changed: 104 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,104 @@
1+
package cc.unitmesh.devti.agenttool.search
2+
3+
import com.google.gson.JsonParser
4+
import com.intellij.execution.process.ProcessAdapter
5+
import com.intellij.execution.process.ProcessEvent
6+
import com.intellij.execution.process.ProcessOutputTypes
7+
import com.intellij.openapi.util.Key
8+
import kotlinx.serialization.Serializable
9+
10+
@Serializable
11+
public data class RipgrepSearchResult(
12+
var filePath: String? = null,
13+
var line: Int = 0,
14+
var column: Int = 0,
15+
var match: String? = null,
16+
var beforeContext: MutableList<String?> = ArrayList<String?>(),
17+
var afterContext: MutableList<String?> = ArrayList<String?>()
18+
)
19+
20+
class RipgrepOutputProcessor : ProcessAdapter() {
21+
private val results: MutableList<RipgrepSearchResult> = ArrayList<RipgrepSearchResult>()
22+
private var currentResult: RipgrepSearchResult? = null
23+
24+
override fun onTextAvailable(event: ProcessEvent, outputType: Key<*>) {
25+
if (outputType === ProcessOutputTypes.STDOUT) {
26+
parseJsonLine(event.text)
27+
}
28+
}
29+
30+
private val jsonBuffer = StringBuilder()
31+
32+
fun parseJsonLine(line: String) {
33+
if (line.isBlank()) {
34+
return
35+
}
36+
37+
jsonBuffer.append(line)
38+
39+
// Try to parse the buffer as JSON
40+
val json = try {
41+
JsonParser.parseString(jsonBuffer.toString())
42+
} catch (e: Exception) {
43+
// If parsing fails, it might be because the JSON is incomplete
44+
// So we just return and wait for more lines
45+
return
46+
}
47+
48+
// If parsing succeeds, clear the buffer and process the JSON
49+
jsonBuffer.clear()
50+
51+
if (json.isJsonObject) {
52+
val jsonObject = json.asJsonObject
53+
val type = jsonObject.get("type").asString
54+
55+
when (type) {
56+
"match" -> {
57+
val data = jsonObject.getAsJsonObject("data")
58+
val path = data.getAsJsonObject("path").get("text").asString
59+
val lines = data.getAsJsonObject("lines").get("text").asString
60+
val lineNumber = data.get("line_number").asInt
61+
val absoluteOffset = data.get("absolute_offset").asInt
62+
val submatches = data.getAsJsonArray("submatches")
63+
64+
currentResult = RipgrepSearchResult(
65+
filePath = path,
66+
line = lineNumber,
67+
column = absoluteOffset,
68+
match = lines.trim()
69+
)
70+
71+
submatches.forEach { submatch ->
72+
val submatchObj = submatch.asJsonObject
73+
val matchText = submatchObj.get("match").asJsonObject.get("text").asString
74+
currentResult?.match = matchText
75+
}
76+
77+
results.add(currentResult!!)
78+
}
79+
80+
"context" -> {
81+
val data = jsonObject.getAsJsonObject("data")
82+
val lines = data.getAsJsonObject("lines").get("text").asString
83+
val lineNumber = data.get("line_number").asInt
84+
85+
if (currentResult != null) {
86+
if (lineNumber < currentResult!!.line) {
87+
currentResult!!.beforeContext.add(lines.trim())
88+
} else {
89+
currentResult!!.afterContext.add(lines.trim())
90+
}
91+
}
92+
}
93+
}
94+
}
95+
}
96+
97+
fun getResults(): MutableList<RipgrepSearchResult> {
98+
if (currentResult != null) {
99+
results.add(currentResult!!)
100+
}
101+
102+
return results
103+
}
104+
}

‎exts/devins-lang/src/main/kotlin/cc/unitmesh/devti/language/agenttool/RipgrepSearcher.kt renamed to ‎core/src/main/kotlin/cc/unitmesh/devti/agenttool/search/RipgrepSearcher.kt

Lines changed: 6 additions & 109 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,10 @@
11
// Copyright 2024 Cline Bot Inc. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
2-
package cc.unitmesh.devti.language.agenttool
2+
package cc.unitmesh.devti.agenttool.search
33

4-
import com.google.gson.JsonParser
54
import com.intellij.execution.configurations.GeneralCommandLine
65
import com.intellij.execution.process.*
76
import com.intellij.openapi.diagnostic.Logger
8-
import com.intellij.openapi.diagnostic.logger
97
import com.intellij.openapi.project.Project
10-
import com.intellij.openapi.util.Key
11-
import kotlinx.serialization.Serializable
128
import org.jetbrains.annotations.NonNls
139
import org.jetbrains.annotations.SystemIndependent
1410
import java.io.IOException
@@ -18,105 +14,6 @@ import java.nio.file.Paths
1814
import java.util.*
1915
import java.util.concurrent.CompletableFuture
2016
import java.util.concurrent.TimeUnit
21-
import kotlin.text.compareTo
22-
import kotlin.text.get
23-
24-
25-
@Serializable
26-
public data class SearchResult(
27-
var filePath: String? = null,
28-
var line: Int = 0,
29-
var column: Int = 0,
30-
var match: String? = null,
31-
var beforeContext: MutableList<String?> = ArrayList<String?>(),
32-
var afterContext: MutableList<String?> = ArrayList<String?>()
33-
)
34-
35-
public class RipgrepOutputProcessor : ProcessAdapter() {
36-
private val results: MutableList<SearchResult> = ArrayList<SearchResult>()
37-
private var currentResult: SearchResult? = null
38-
39-
override fun onTextAvailable(event: ProcessEvent, outputType: Key<*>) {
40-
if (outputType === ProcessOutputTypes.STDOUT) {
41-
parseJsonLine(event.text)
42-
}
43-
}
44-
45-
private val jsonBuffer = StringBuilder()
46-
47-
fun parseJsonLine(line: String) {
48-
if (line.isBlank()) {
49-
return
50-
}
51-
52-
jsonBuffer.append(line)
53-
54-
// Try to parse the buffer as JSON
55-
val json = try {
56-
JsonParser.parseString(jsonBuffer.toString())
57-
} catch (e: Exception) {
58-
// If parsing fails, it might be because the JSON is incomplete
59-
// So we just return and wait for more lines
60-
return
61-
}
62-
63-
// If parsing succeeds, clear the buffer and process the JSON
64-
jsonBuffer.clear()
65-
66-
if (json.isJsonObject) {
67-
val jsonObject = json.asJsonObject
68-
val type = jsonObject.get("type").asString
69-
70-
when (type) {
71-
"match" -> {
72-
val data = jsonObject.getAsJsonObject("data")
73-
val path = data.getAsJsonObject("path").get("text").asString
74-
val lines = data.getAsJsonObject("lines").get("text").asString
75-
val lineNumber = data.get("line_number").asInt
76-
val absoluteOffset = data.get("absolute_offset").asInt
77-
val submatches = data.getAsJsonArray("submatches")
78-
79-
currentResult = SearchResult(
80-
filePath = path,
81-
line = lineNumber,
82-
column = absoluteOffset,
83-
match = lines.trim()
84-
)
85-
86-
submatches.forEach { submatch ->
87-
val submatchObj = submatch.asJsonObject
88-
val matchText = submatchObj.get("match").asJsonObject.get("text").asString
89-
currentResult?.match = matchText
90-
}
91-
92-
results.add(currentResult!!)
93-
}
94-
95-
"context" -> {
96-
val data = jsonObject.getAsJsonObject("data")
97-
val lines = data.getAsJsonObject("lines").get("text").asString
98-
val lineNumber = data.get("line_number").asInt
99-
100-
if (currentResult != null) {
101-
if (lineNumber < currentResult!!.line) {
102-
currentResult!!.beforeContext.add(lines.trim())
103-
} else {
104-
currentResult!!.afterContext.add(lines.trim())
105-
}
106-
}
107-
}
108-
}
109-
}
110-
}
111-
112-
fun getResults(): MutableList<SearchResult> {
113-
if (currentResult != null) {
114-
results.add(currentResult!!)
115-
}
116-
117-
return results
118-
}
119-
}
12017

12118
/**
12219
* 使用Ripgrep进行文件搜索
@@ -170,7 +67,7 @@ object RipgrepSearcher {
17067

17168
@Throws(IOException::class)
17269
private fun executeRipgrep(project: Project, rgPath: Path, directory: String, regex: String, filePattern: String?):
173-
MutableList<SearchResult> {
70+
MutableList<RipgrepSearchResult> {
17471
val cmd = getCommandLine(rgPath, regex, filePattern, directory, project.basePath)
17572

17673
val handler: OSProcessHandler = ColoredProcessHandler(cmd)
@@ -213,14 +110,14 @@ object RipgrepSearcher {
213110
return cmd
214111
}
215112

216-
private fun formatResults(results: MutableList<SearchResult>, basePath: String): String {
113+
private fun formatResults(results: MutableList<RipgrepSearchResult>, basePath: String): String {
217114
val output = StringBuilder()
218-
val grouped: MutableMap<String?, MutableList<SearchResult?>?> =
219-
LinkedHashMap<String?, MutableList<SearchResult?>?>()
115+
val grouped: MutableMap<String?, MutableList<RipgrepSearchResult?>?> =
116+
LinkedHashMap<String?, MutableList<RipgrepSearchResult?>?>()
220117

221118
for (result in results) {
222119
val relPath = getRelativePath(basePath, result.filePath!!)
223-
grouped.computeIfAbsent(relPath) { k: String? -> ArrayList<SearchResult?>() }!!.add(result)
120+
grouped.computeIfAbsent(relPath) { k: String? -> ArrayList<RipgrepSearchResult?>() }!!.add(result)
224121
}
225122

226123
for (entry in grouped.entries) {

‎core/src/main/kotlin/cc/unitmesh/devti/sketch/SketchRunContext.kt

Lines changed: 18 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
package cc.unitmesh.devti.sketch
22

3+
import cc.unitmesh.devti.agenttool.search.RipgrepSearcher
34
import cc.unitmesh.devti.gui.chat.message.ChatActionType
45
import cc.unitmesh.devti.gui.chat.ui.relativePath
56
import cc.unitmesh.devti.provider.BuildSystemProvider
@@ -33,7 +34,8 @@ data class SketchRunContext(
3334
val toolList: String,
3435
val shell: String = System.getenv("SHELL") ?: "/bin/bash",
3536
val frameworkContext: String = "",
36-
val buildTool: String = ""
37+
val buildTool: String = "",
38+
val searchTool: String = "localSearch",
3739
) : TemplateContext {
3840
companion object {
3941
fun create(project: Project, myEditor: Editor?, input: String): SketchRunContext {
@@ -43,21 +45,8 @@ data class SketchRunContext(
4345
} else {
4446
FileEditorManager.getInstance(project).selectedFiles.firstOrNull()
4547
}
46-
47-
val otherFiles = FileEditorManager.getInstance(project).openFiles.filter { it != currentFile }
48-
49-
val psi = if (currentFile != null) {
50-
PsiManager.getInstance(project).findFile(currentFile)
51-
} else {
52-
null
53-
}
54-
55-
val currentElement = if (editor != null) {
56-
psi?.findElementAt(editor.caretModel.offset)
57-
} else {
58-
null
59-
}
60-
48+
val psi = currentFile?.let { PsiManager.getInstance(project).findFile(it) }
49+
val currentElement = editor?.let { psi?.findElementAt(it.caretModel.offset) }
6150
val creationContext =
6251
ChatCreationContext(ChatOrigin.Intention, ChatActionType.CHAT, psi, listOf(), element = psi)
6352

@@ -68,6 +57,8 @@ data class SketchRunContext(
6857
""
6958
}
7059

60+
val otherFiles = FileEditorManager.getInstance(project).openFiles.filter { it != currentFile }
61+
7162
return SketchRunContext(
7263
currentFile = currentFile?.relativePath(project),
7364
currentElement = currentElement,
@@ -81,11 +72,22 @@ data class SketchRunContext(
8172
return@runBlocking ChatContextProvider.collectChatContextList(project, creationContext)
8273
}.joinToString(",", transform = ChatContextItem::text),
8374
buildTool = buildTool,
75+
searchTool = getSearchTool()
8476
)
8577
}
8678
}
8779
}
8880

81+
fun getSearchTool(): String {
82+
val findRipgrepBinary = try {
83+
RipgrepSearcher.findRipgrepBinary()
84+
} catch (e: Exception) {
85+
null
86+
}
87+
88+
return if (findRipgrepBinary != null) "ripgrepSearch" else "localSearch"
89+
}
90+
8991
private fun osInfo() =
9092
System.getProperty("os.name") + " " + System.getProperty("os.version") + " " + System.getProperty("os.arch")
9193

‎core/src/main/resources/genius/en/code/sketch.vm

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,7 @@ have created a routes.py and main.js file, and updated the main.html file.
5757
为了帮助您创建基于 Python 的照片存储应用程序，我需要了解更多关于您的代码库的信息。请允许我调用工具来获取上下文信息。请执行以下 DevIn 代码：
5858
<devin>
5959
/dir:src
60-
/localSearch:.*photo.*
60+
/${context.searchTool}:.*photo.*
6161
</devin>
6262
// If the context information is still not enough, you should ask the user to provide more information
6363
</you.answer.step1>
@@ -127,7 +127,7 @@ feat: add delete blog functionality
127127
<devin> // 请尽可能只用一个 DevIn 代码块来获取上下文信息
128128
/dir:src/main/java
129129
/file:SketchRunContext.java // 使用绝对路径时，一定要从用户那里得到正确的路径
130-
/localSearch:SketchRunContext // 如果用户的问题是中文的，需要转换为英文的搜索关键词
130+
/${context.searchTool}:SketchRunContext // 如果用户的问题是中文的，需要转换为英文的搜索关键词
131131
</devin>
132132
</you.answer.step1>
133133
<user.answer.step1>

0 commit comments

Comments
 (0)