Merge pull request #36 from Jibola/PYTHON-4680-local-llama

PYTHON-4680 Make llama-index use local atlas

Author: Jibola

.evergreen/scaffold_atlas.py

@@ -103,7 +103,8 @@ def generate_collections(database: Database, collection_jsons: list[Path]) -> No
 
 
 def generate_indexes(client: MongoClient, index_jsons: list[Path]) -> None:
-    """_summary_
+    """Generate Search or VectorSearch indexes based on the index_json provided
+    TODO: **Improve Documentation to include Local Atlas JSON configuration requirements*
 
     Args:
         client (MongoClient): MongoClient

llama-index-python-kvstore/run.sh

@@ -17,6 +17,8 @@ $PYTHON_BINARY -m pip install -U pip poetry
 $PYTHON_BINARY -m poetry env use $PYTHON_BINARY
 # Activate the poetry env, which itself does not include poetry
 source $($PYTHON_BINARY -m poetry env info --path)/bin/activate
+# PYTHON-4522: Will fix requirement in llama-index repo
+$PYTHON_BINARY -m poetry add motor
 # Recreate the poetry lock file
 $PYTHON_BINARY -m poetry lock --no-update
 # Install from pyproject.toml into package specific environment

llama-index-python-vectorstore/indexes/fulltext_index.json

@@ -0,0 +1,16 @@
+{
+    "mappings": {
+        "dynamic": false,
+        "fields": {
+            "text": [
+                {
+                    "type": "string"
+                }
+            ]
+        }
+    },
+    "name": "fulltext_index",
+    "type": "search",
+    "database": "llama_index_test_db",
+    "collectionName": "llama_index_test_vectorstore"
+}

llama-index-python-vectorstore/indexes/vector_index.json

@@ -5,6 +5,10 @@
       "path": "embedding",
       "similarity": "cosine",
       "type": "vector"
+    },
+    {
+      "path": "metadata.text",
+      "type": "filter"
     }
   ],
   "name": "vector_index",

llama-index-python-vectorstore/run.sh

@@ -22,8 +22,8 @@ $PYTHON_BINARY -m poetry lock --no-update
 $PYTHON_BINARY -m poetry install --with dev
 
 # Run tests. Sensitive variables in Evergreen come from Evergreen project: ai-ml-pipeline-testing/
+MONGODB_URI=$(fetch_local_atlas_uri) \
 OPENAI_API_KEY=$openai_api_key \
-MONGODB_URI=$llama_index_mongodb_uri \
 MONGODB_DATABASE="llama_index_test_db" \
 MONGODB_COLLECTION="llama_index_test_vectorstore" \
 MONGODB_INDEX="vector_index" \