refactor(tokenizer): use TokenizerFactory for tokenizer creation

This change replaces the direct use of the `TokenizerImpl.INSTANCE` with the `TokenizerFactory.createTokenizer()` method to ensure consistent tokenizer creation across the application. It simplifies the process of obtaining a tokenizer instance and enhances the modularity of the codebase. Additionally, the `LLMCoroutineScope` now has a dedicated `coroutineExceptionHandler` for improved error handling.

Author: phodal

src/main/kotlin/cc/unitmesh/devti/gui/chat/AutoDevInputSection.kt

@@ -6,7 +6,7 @@ import cc.unitmesh.devti.agent.configurable.customAgentSetting
 import cc.unitmesh.devti.agent.model.CustomAgentConfig
 import cc.unitmesh.devti.agent.model.CustomAgentState
 import cc.unitmesh.devti.llms.tokenizer.Tokenizer
-import cc.unitmesh.devti.llms.tokenizer.TokenizerImpl
+import cc.unitmesh.devti.llms.tokenizer.TokenizerFactory
 import cc.unitmesh.devti.settings.AutoDevSettingsState
 import com.intellij.openapi.Disposable
 import com.intellij.openapi.actionSystem.AnActionEvent
@@ -32,7 +32,6 @@ import com.intellij.util.ui.JBEmptyBorder
 import com.intellij.util.ui.JBUI
 import com.intellij.util.ui.UIUtil
 import com.intellij.util.ui.components.BorderLayoutPanel
-import kotlinx.serialization.json.Json
 import java.awt.CardLayout
 import java.awt.Color
 import java.awt.Dimension
@@ -44,7 +43,6 @@ import javax.swing.JComponent
 import javax.swing.JPanel
 import kotlin.math.max
 import kotlin.math.min
-import kotlinx.serialization.decodeFromString
 
 /**
  *
@@ -65,7 +63,7 @@ class AutoDevInputSection(private val project: Project, val disposable: Disposab
 
     val editorListeners = EventDispatcher.create(AutoDevInputListener::class.java)
     private var tokenizer: Tokenizer? = try {
-        lazy { TokenizerImpl.INSTANCE }.value
+        lazy { TokenizerFactory.createTokenizer() }.value
     } catch (e: Exception) {
         logger.error("TokenizerImpl.INSTANCE is not available", e)
         null

src/main/kotlin/cc/unitmesh/devti/llms/tokenizer/TokenizerImpl.kt

@@ -21,8 +21,8 @@ class TokenizerImpl : Tokenizer {
     override fun tokenize(chunk: String): IntArrayList? {
         return encoding.encode(chunk, maxTokenLength).tokens
     }
+}
 
-    companion object {
-        val INSTANCE = ApplicationManager.getApplication().getService(TokenizerImpl::class.java)
-    }
+object TokenizerFactory {
+    fun createTokenizer(): Tokenizer =  ApplicationManager.getApplication().getService(TokenizerImpl::class.java)
 }

src/main/kotlin/cc/unitmesh/devti/provider/PromptStrategy.kt

@@ -1,7 +1,7 @@
 package cc.unitmesh.devti.provider
 
 import cc.unitmesh.devti.llms.tokenizer.Tokenizer
-import cc.unitmesh.devti.llms.tokenizer.TokenizerImpl
+import cc.unitmesh.devti.llms.tokenizer.TokenizerFactory
 import cc.unitmesh.devti.prompting.CodePromptText
 import cc.unitmesh.devti.settings.AutoDevSettingsState
 import com.intellij.openapi.extensions.ExtensionPointName
@@ -26,7 +26,7 @@ abstract class PromptStrategy : LazyExtensionInstance<PromptStrategy>() {
     @Attribute("implementationClass")
     var implementationClass: String? = null
 
-    private val tokenizer: Tokenizer = TokenizerImpl.INSTANCE
+    private val tokenizer: Tokenizer = TokenizerFactory.createTokenizer()
 
     override fun getImplementationClassName(): String? = implementationClass
 

src/main/kotlin/cc/unitmesh/devti/util/LLMCoroutineScope.kt

@@ -10,13 +10,13 @@ import kotlinx.coroutines.SupervisorJob
 
 @Service(Service.Level.PROJECT)
 class LLMCoroutineScope {
+    private val coroutineExceptionHandler = CoroutineExceptionHandler { _, throwable ->
+        Logger.getInstance(LLMCoroutineScope::class.java).error(throwable)
+    }
+
     val coroutineScope: CoroutineScope = CoroutineScope(SupervisorJob() + coroutineExceptionHandler)
 
     companion object {
-        val coroutineExceptionHandler = CoroutineExceptionHandler { _, throwable ->
-            Logger.getInstance(LLMCoroutineScope::class.java).error(throwable)
-        }
-
         fun scope(project: Project): CoroutineScope = project.service<LLMCoroutineScope>().coroutineScope
     }
 }

src/main/kotlin/com/intellij/temporary/error/ErrorMessageProcessor.kt

@@ -1,8 +1,8 @@
 // Copyright 2000-2021 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
 package com.intellij.temporary.error
 
+import cc.unitmesh.devti.llms.tokenizer.TokenizerFactory
 import cc.unitmesh.devti.util.isInProject
-import cc.unitmesh.devti.llms.tokenizer.TokenizerImpl
 import cc.unitmesh.devti.prompting.TextTemplatePrompt
 import cc.unitmesh.devti.settings.AutoDevSettingsState
 import com.intellij.execution.filters.FileHyperlinkInfo
@@ -80,7 +80,7 @@ object ErrorMessageProcessor {
             extractErrorPlaces(project, consoleLineFrom, consoleLineTo, consoleEditor)
 
         val errorPromptBuilder =
-            ErrorPromptBuilder(AutoDevSettingsState.maxTokenLength, TokenizerImpl.INSTANCE)
+            ErrorPromptBuilder(AutoDevSettingsState.maxTokenLength, TokenizerFactory.createTokenizer())
 
         return errorPromptBuilder.buildPrompt(extractedText, extractedErrorPlaces)
     }