feat(custom): update key names in CustomSSEProcessor and add token count support #51

- Update key names in CustomSSEProcessor to use "content" instead of "message" and "role" instead of "user" for consistency.
- Add support for token count in Tokenizer interface and implementation. Use the `IntArrayList` type for improved performance.
- Upgrade the markdown library to version 0.6.1.
- Add template engine and http request/response dependencies.
- Update `jtokkit` library version to 1.0.0 for token count functionality.
- Add dependency for GitLab API.
- Add plugin for IntelliJ IDEA.

Total characters: 270

Author: phodal

build.gradle.kts

@@ -390,21 +390,14 @@ project(":") {
         plugins.set(ideaPlugins)
     }
 
-//    sourceSets {
-//        main {
-//            resources.srcDirs("src/main/resources")
-//            resources.srcDirs("src/$platformVersion/main/resources")
-//        }
-//    }
-
     dependencies {
         implementation(libs.bundles.openai)
         implementation(libs.bundles.markdown)
         implementation(libs.yaml)
 
         implementation(libs.json.pathkt)
 
-        implementation("org.jetbrains:markdown:0.5.1")
+        implementation("org.jetbrains:markdown:0.6.1")
         implementation(libs.kotlinx.serialization.json)
 
         implementation("cc.unitmesh:cocoa-core:0.4.5")
@@ -414,11 +407,14 @@ project(":") {
         implementation(libs.github.api)
         implementation("org.gitlab4j:gitlab4j-api:5.3.0")
 
+        // template engine
         implementation("org.apache.velocity:velocity-engine-core:2.3")
 
+        // http request/response
         implementation(libs.jackson.module.kotlin)
 
-        implementation("com.knuddels:jtokkit:0.6.1")
+        // token count
+        implementation("com.knuddels:jtokkit:1.0.0")
 
         // junit
         testImplementation("io.kotest:kotest-assertions-core:5.7.2")

src/main/kotlin/cc/unitmesh/devti/llms/custom/CustomSSEProcessor.kt

@@ -149,17 +149,17 @@ fun JsonObject.updateCustomBody(customRequest: String): JsonObject {
 
             // TODO clean code with magic literals
             var roleKey = "role"
-            var contentKey = "message"
+            var contentKey = "content"
             customRequestJson.jsonObject["messageKeys"]?.let {
                 roleKey = it.jsonObject["role"]?.jsonPrimitive?.content ?: "role"
-                contentKey = it.jsonObject["content"]?.jsonPrimitive?.content ?: "message"
+                contentKey = it.jsonObject["content"]?.jsonPrimitive?.content ?: "content"
             }
 
             val messages: JsonArray = this@updateCustomBody["messages"]?.jsonArray ?: buildJsonArray { }
             this.put("messages", buildJsonArray {
                 messages.forEach { message ->
                     val role: String = message.jsonObject["role"]?.jsonPrimitive?.content ?: "user"
-                    val content: String = message.jsonObject["message"]?.jsonPrimitive?.content ?: ""
+                    val content: String = message.jsonObject["content"]?.jsonPrimitive?.content ?: ""
                     add(buildJsonObject {
                         put(roleKey, role)
                         put(contentKey, content)

src/main/kotlin/cc/unitmesh/devti/llms/tokenizer/Tokenizer.kt

@@ -1,7 +1,9 @@
 package cc.unitmesh.devti.llms.tokenizer
 
+import com.knuddels.jtokkit.api.IntArrayList
+
 interface Tokenizer {
     fun getMaxLength(): Int
     fun count(string: String): Int
-    fun tokenize(chunk: String): List<Int>
+    fun tokenize(chunk: String): IntArrayList?
 }

src/main/kotlin/cc/unitmesh/devti/llms/tokenizer/TokenizerImpl.kt

@@ -6,6 +6,7 @@ import com.knuddels.jtokkit.Encodings
 import com.knuddels.jtokkit.api.Encoding
 import com.knuddels.jtokkit.api.EncodingRegistry
 import com.knuddels.jtokkit.api.EncodingType
+import com.knuddels.jtokkit.api.IntArrayList
 
 @Service(Service.Level.APP)
 class TokenizerImpl(private val maxTokenLength: Int = 8192) : Tokenizer {
@@ -15,7 +16,8 @@ class TokenizerImpl(private val maxTokenLength: Int = 8192) : Tokenizer {
     override fun getMaxLength(): Int = maxTokenLength
 
     override fun count(string: String): Int = encoding.countTokens(string)
-    override fun tokenize(chunk: String): List<Int> {
+
+    override fun tokenize(chunk: String): IntArrayList? {
         return encoding.encode(chunk, maxTokenLength).tokens
     }