feat(shire): merge local agent language shire into AutoDev #379

Author: phodal

exts/devins-lang/src/main/kotlin/cc/unitmesh/devti/envior/PsiJsonUtil.kt

@@ -0,0 +1,23 @@
+package cc.unitmesh.devti.envior
+
+import com.intellij.json.JsonUtil
+import com.intellij.json.psi.*
+
+fun JsonProperty.valueAsString(obj: JsonObject): String? {
+    val value = JsonUtil.getPropertyValueOfType(obj, name, JsonLiteral::class.java)
+    return when (value) {
+        is JsonStringLiteral -> value.value
+        is JsonBooleanLiteral -> value.value.toString()
+        else -> value?.text
+    }
+}
+
+fun JsonObject.findString(name: String): String? {
+    val property = findProperty(name) ?: return null
+    return property.valueAsString(this)
+}
+
+fun JsonObject.findNumber(name: String): Number? {
+    val property = findProperty(name) ?: return null
+    return JsonUtil.getPropertyValueOfType(this, name, JsonNumberLiteral::class.java)?.value
+}

exts/devins-lang/src/main/kotlin/cc/unitmesh/devti/envior/ShireEnvReader.kt

@@ -0,0 +1,82 @@
+package cc.unitmesh.devti.envior
+
+import com.intellij.json.psi.JsonFile
+import com.intellij.json.psi.JsonObject
+import com.intellij.openapi.project.DumbService
+import com.intellij.openapi.project.Project
+import com.intellij.psi.PsiFile
+import com.intellij.psi.PsiManager
+import com.intellij.psi.search.GlobalSearchScope
+import com.intellij.util.indexing.FileBasedIndex
+
+object ShireEnvReader {
+    const val DEFAULT_ENV_NAME = "development"
+    /**
+     * This function attempts to retrieve a JSON file associated with a given environment name within the specified scope and project.
+     *
+     * @param envName The name of the environment for which to find the associated JSON file.
+     * @param scope The GlobalSearchScope to limit the search for the JSON file.
+     * @param project The Project within which to search for the JSON file.
+     *
+     * @return A JsonFile object if a file with the environment name is found, or null if no such file exists within the given scope and project.
+     */
+    private fun getEnvJsonFile(
+        envName: String,
+        scope: GlobalSearchScope,
+        project: Project,
+    ): JsonFile? {
+        return DumbService.getInstance(project).runReadActionInSmartMode<JsonFile?> {
+            FileBasedIndex.getInstance().getContainingFiles(ShireEnvironmentIndex.id(), envName, scope)
+                .firstOrNull()
+                ?.let {
+                    (PsiManager.getInstance(project).findFile(it) as? JsonFile)
+                }
+        }
+    }
+
+    fun getEnvObject(
+        envName: String,
+        scope: GlobalSearchScope,
+        project: Project,
+    ): JsonObject? {
+        val psiFile = getEnvJsonFile(envName, scope, project)
+        val envObject = getEnvObject(envName, psiFile)
+        return envObject
+    }
+
+    /**
+     * Read Shire env file object
+     */
+    fun getEnvObject(envName: String, psiFile: PsiFile?): JsonObject? {
+        val rootObject = (psiFile as? JsonFile)?.topLevelValue as? JsonObject ?: return null
+        return rootObject.propertyList.firstOrNull { it.name == envName }?.value as? JsonObject
+    }
+
+    fun fetchEnvironmentVariables(envName: String, scope: GlobalSearchScope): List<Set<String>> {
+        return FileBasedIndex.getInstance().getValues(
+            ShireEnvironmentIndex.id(),
+            envName,
+            scope
+        )
+    }
+
+    fun getAllEnvironments(project: Project, scope: GlobalSearchScope): Collection<String> {
+        try {
+            return DumbService.getInstance(project).runReadActionInSmartMode<Collection<String>> {
+                val index = FileBasedIndex.getInstance()
+                index.getAllKeys(ShireEnvironmentIndex.id(), project).stream()
+                    .filter {
+                        it != ShireEnvironmentIndex.MODEL_LIST && index.getContainingFiles(
+                            ShireEnvironmentIndex.id(),
+                            it,
+                            scope
+                        ).isNotEmpty()
+                    }
+                    .toList()
+            }
+        } catch (e: Exception) {
+            return emptyList()
+        }
+    }
+
+}

exts/devins-lang/src/main/kotlin/cc/unitmesh/devti/envior/ShireEnvVariableFiller.kt

@@ -0,0 +1,80 @@
+package cc.unitmesh.devti.envior
+
+import com.intellij.json.JsonUtil
+import com.intellij.json.psi.*
+import com.intellij.openapi.util.TextRange
+
+object ShireEnvVariableFiller {
+    private fun getVariableValue(jsonObject: JsonObject, name: String, processVars: Map<String, String>): String? {
+        val value = JsonUtil.getPropertyValueOfType(jsonObject, name, JsonLiteral::class.java)
+        val jsonResult = getValueAsString(value)
+        if (jsonResult != null) {
+            return jsonResult
+        }
+
+        return processVars[name]
+    }
+
+    private fun getValueAsString(value: JsonLiteral?): String? {
+        return when (value) {
+            is JsonStringLiteral -> value.value
+            is JsonBooleanLiteral -> value.value.toString()
+            else -> value?.text
+        }
+    }
+
+    fun fillVariables(
+        messageBody: String,
+        variables: List<Set<String>>,
+        obj: JsonObject?,
+        processVars: Map<String, String>
+    ): String {
+        if (obj == null) return messageBody
+        if (variables.isEmpty()) return messageBody
+
+        val envRanges = collectVariablesRangesInMessageBody(messageBody)
+
+        val result = StringBuilder(messageBody.length)
+        var lastVariableRangeEndOffset = 0
+
+        for (variableRange in envRanges) {
+            result.append(messageBody as CharSequence, lastVariableRangeEndOffset, variableRange.startOffset)
+            val variableValue = getVariableValue(obj, getVariableKey(variableRange, messageBody), processVars)
+
+            result.append(variableValue)
+            lastVariableRangeEndOffset = variableRange.endOffset
+        }
+
+        result.append(messageBody as CharSequence, lastVariableRangeEndOffset, messageBody.length)
+        val sb = result.toString()
+        return sb
+    }
+
+    private fun getVariableKey(variableRange: TextRange, messageBody: String) =
+        variableRange.substring(messageBody).removePrefix("\${").removeSuffix("}")
+
+    private fun collectVariablesRangesInMessageBody(body: String): List<TextRange> {
+        val ranges = mutableListOf<TextRange>()
+        var startIndex = 0
+
+        while (startIndex < body.length) {
+            val openBraceIndex = body.indexOf("\${", startIndex)
+            val closeBraceIndex = body.indexOf("}", openBraceIndex)
+
+            if (openBraceIndex == -1 || closeBraceIndex == -1) {
+                break
+            }
+
+            val range = TextRange(openBraceIndex, closeBraceIndex + 1)
+            val contentInsideBraces = body.substring(openBraceIndex + 2, closeBraceIndex)
+
+            if (contentInsideBraces.isNotBlank()) {
+                ranges.add(range)
+            }
+
+            startIndex = closeBraceIndex + 1
+        }
+
+        return ranges
+    }
+}

exts/devins-lang/src/main/kotlin/cc/unitmesh/devti/envior/ShireEnvironmentIndex.kt

@@ -0,0 +1,90 @@
+package cc.unitmesh.devti.envior
+
+import com.intellij.json.psi.*
+import com.intellij.openapi.util.text.StringUtil
+import com.intellij.util.indexing.*
+import com.intellij.util.io.DataExternalizer
+import com.intellij.util.io.EnumeratorStringDescriptor
+import com.intellij.util.io.KeyDescriptor
+
+
+class ShireEnvironmentIndex : FileBasedIndexExtension<String, Set<String>>() {
+    companion object {
+        val SHIRE_ENV_ID: ID<String, Set<String>> = ID.create("shire.environment")
+        const val MODEL_TITLE = "title"
+        const val MODEL_LIST = "models"
+
+        fun id(): ID<String, Set<String>> {
+            return SHIRE_ENV_ID
+        }
+    }
+
+    override fun getValueExternalizer(): DataExternalizer<Set<String>> = ShireStringsExternalizer()
+    override fun getVersion(): Int = 2
+    override fun getInputFilter(): FileBasedIndex.InputFilter = ShireEnvironmentInputFilter()
+    override fun dependsOnFileContent(): Boolean = true
+    override fun getName(): ID<String, Set<String>> = SHIRE_ENV_ID
+    override fun getKeyDescriptor(): KeyDescriptor<String> = EnumeratorStringDescriptor.INSTANCE
+
+    override fun getIndexer(): DataIndexer<String, Set<String>, FileContent> {
+        return DataIndexer { inputData: FileContent ->
+            val file = inputData.psiFile
+            require(file is JsonFile) { AssertionError() }
+
+            val variablesFromFile = getVariablesFromFile(file)
+            variablesFromFile
+        }
+    }
+
+    private fun getVariablesFromFile(file: JsonFile): Map<String, Set<String>> {
+        val result: MutableMap<String, Set<String>> = HashMap()
+        when (val topLevelValue = file.topLevelValue) {
+            is JsonObject -> {
+                for (property in topLevelValue.propertyList) {
+                    when (val value = property.value) {
+                        is JsonObject -> {
+                            result[property.name] = readEnvVariables(value, file.name)
+                        }
+
+                        is JsonArray -> {
+                            // the prop key should be "models"
+                            if (property.name != MODEL_LIST) {
+                                continue
+                            }
+
+                            // the child elements of the array are objects, which should have prop call "name"
+                            val envVariables = value.children
+                                .filterIsInstance<JsonObject>()
+                                .mapNotNull { obj ->
+                                    val name = obj.findProperty(MODEL_TITLE)?.value?.text
+                                    name?.let { StringUtil.unquoteString(it) }
+                                }
+                                .toSet()
+
+                            result[property.name] = envVariables
+                        }
+                    }
+                }
+            }
+        }
+
+        return result
+    }
+
+    private fun readEnvVariables(obj: JsonObject, fileName: String): Set<String> {
+        val properties = obj.propertyList
+        return if (properties.isEmpty()) {
+            emptySet()
+        } else {
+            val set = properties.stream()
+                .map { property ->
+                    StringUtil.nullize(property.name)
+                }
+                .toList()
+                .mapNotNull { it }
+                .toSet()
+
+            set
+        }
+    }
+}

exts/devins-lang/src/main/kotlin/cc/unitmesh/devti/envior/ShireEnvironmentInputFilter.kt

@@ -0,0 +1,16 @@
+package cc.unitmesh.devti.envior
+
+import com.intellij.json.JsonFileType
+import com.intellij.openapi.fileTypes.FileType
+import com.intellij.openapi.vfs.VirtualFile
+import com.intellij.util.indexing.DefaultFileTypeSpecificInputFilter
+
+class ShireEnvironmentInputFilter : DefaultFileTypeSpecificInputFilter(*arrayOf<FileType>(JsonFileType.INSTANCE)) {
+    override fun acceptInput(file: VirtualFile): Boolean {
+        return super.acceptInput(file) && isShireEnvFile(file)
+    }
+
+    private fun isShireEnvFile(file: VirtualFile?): Boolean {
+        return file?.name?.endsWith(".shireEnv.json") ?: false
+    }
+}

exts/devins-lang/src/main/kotlin/cc/unitmesh/devti/envior/ShireStringsExternalizer.kt

@@ -0,0 +1,25 @@
+package cc.unitmesh.devti.envior
+
+import com.intellij.util.io.DataExternalizer
+import java.io.DataInput
+import java.io.DataOutput
+
+class ShireStringsExternalizer : DataExternalizer<Set<String>> {
+    override fun save(out: DataOutput, value: Set<String>) {
+        out.writeInt(value.size)
+        for (s in value) {
+            out.writeUTF(s)
+        }
+    }
+
+    override fun read(input: DataInput): Set<String> {
+        val size = input.readInt()
+        val result: MutableSet<String> = HashSet(size)
+        for (i in 0 until size) {
+            result.add(input.readUTF())
+        }
+
+        return result
+    }
+
+}

exts/devins-lang/src/main/kotlin/cc/unitmesh/devti/envior/llm/LlmEnv.kt

@@ -0,0 +1,36 @@
+package cc.unitmesh.devti.envior.llm
+
+import com.intellij.json.psi.JsonArray
+import com.intellij.json.psi.JsonFile
+import com.intellij.json.psi.JsonObject
+import com.intellij.openapi.application.runReadAction
+import com.intellij.openapi.project.Project
+import com.intellij.psi.PsiManager
+import com.intellij.psi.search.GlobalSearchScope
+import com.intellij.util.indexing.FileBasedIndex
+import cc.unitmesh.devti.envior.ShireEnvironmentIndex
+import cc.unitmesh.devti.envior.valueAsString
+
+class LlmEnv {
+    companion object {
+        private fun configFromFile(modelName: String, psiFile: JsonFile?): JsonObject? {
+            val rootObject = psiFile?.topLevelValue as? JsonObject ?: return null
+            val envObject = rootObject.propertyList.firstOrNull { it.name == ShireEnvironmentIndex.MODEL_LIST }?.value as? JsonArray
+            return envObject?.children?.firstOrNull {
+                it is JsonObject && it.findProperty(ShireEnvironmentIndex.MODEL_TITLE)?.valueAsString(it) == modelName
+            } as? JsonObject
+        }
+
+        fun configFromFile(modelName: String, scope: GlobalSearchScope, project: Project): JsonObject? {
+            val jsonFile = runReadAction {
+                FileBasedIndex.getInstance().getContainingFiles(ShireEnvironmentIndex.id(), ShireEnvironmentIndex.MODEL_LIST, scope)
+                    .firstOrNull()
+                    ?.let {
+                        (PsiManager.getInstance(project).findFile(it) as? JsonFile)
+                    }
+            }
+
+            return configFromFile(modelName, jsonFile)
+        }
+    }
+}