feat(agent): add token-based message compression

- Change `content` in `Message` to mutable (`var`) for in-place updates.
- Add `TokenizerFactory` and `MarkdownCodeHelper` to compress messages based on token count.
- Compress messages by removing markdown code if token count exceeds `maxToken`.

Author: phodal

core/src/main/kotlin/cc/unitmesh/devti/llms/custom/CustomSSEProcessor.kt

@@ -183,7 +183,7 @@ open class CustomSSEProcessor(private val project: Project) {
 }
 
 @Serializable
-data class Message(val role: String, val content: String)
+data class Message(val role: String, var content: String)
 
 @Serializable
 data class CustomRequest(val messages: List<Message>)

core/src/main/kotlin/cc/unitmesh/devti/observer/agent/AgentStateService.kt

@@ -3,11 +3,13 @@ package cc.unitmesh.devti.observer.agent
 import cc.unitmesh.devti.agent.tool.AgentTool
 import cc.unitmesh.devti.devin.dataprovider.BuiltinCommand
 import cc.unitmesh.devti.llms.custom.Message
+import cc.unitmesh.devti.llms.tokenizer.TokenizerFactory
 import cc.unitmesh.devti.observer.plan.AgentTaskEntry
 import cc.unitmesh.devti.observer.plan.MarkdownPlanParser
 import cc.unitmesh.devti.observer.plan.PlanUpdateListener
 import cc.unitmesh.devti.settings.AutoDevSettingsState
 import cc.unitmesh.devti.settings.customize.customizeSetting
+import cc.unitmesh.devti.util.parser.MarkdownCodeHelper
 import com.intellij.openapi.application.ApplicationManager
 import com.intellij.openapi.components.Service
 import com.intellij.openapi.diagnostic.logger
@@ -18,6 +20,9 @@ import com.intellij.openapi.vcs.changes.Change
 class AgentStateService(val project: Project) {
     val maxToken = AutoDevSettingsState.maxTokenLength
     var state: AgentState = AgentState()
+    var tokenizer = lazy {
+        TokenizerFactory.createTokenizer()
+    }
 
     fun addTools(tools: List<BuiltinCommand>) {
         state.usedTools = tools.map {
@@ -52,8 +57,17 @@ class AgentStateService(val project: Project) {
      * Call some LLM to compress it or use some other method to compress the history
      */
     fun processMessages(messages: List<Message>): List<Message> {
-        state.messages = messages
-        return messages
+        val countLength = tokenizer.value.count(messages.joinToString("\n") { it.content })
+        if (countLength < maxToken) {
+            state.messages = messages
+            return messages
+        }
+
+        state.messages = messages.map {
+            it.copy(content = MarkdownCodeHelper.removeAllMarkdownCode(it.content))
+        }
+
+        return state.messages
     }
 
     fun updatePlan(items: MutableList<AgentTaskEntry>) {