PYTHON-4381 Add DocArray to AI Integration Pipeline Testing

.evergreen/config.yml

@@ -100,6 +100,11 @@ tasks:
       - func: "fetch repo"
       - func: "execute tests"
 
+  - name: test-docarray
+    commands:
+      - func: "fetch repo"
+      - func: "execute tests"
+
 buildvariants:
   - name: test-semantic-kernel-python-rhel
     display_name: Semantic-Kernel RHEL Python
@@ -161,3 +166,16 @@ buildvariants:
       - rhel87-small
     tasks:
       - name: test-llama-index
+
+  - name: test-docarray-rhel
+    display_name: DocArray RHEL
+    expansions:
+      DIR: docarray
+      REPO_NAME: docarray
+      # TODO - Update CLONE_URL once pull-request is merged
+      CLONE_URL: -b mongodb-atlas-backend --single-branch https://github.com/caseyclements/docarray.git
+      DATABASE: docarray_test_db
+    run_on:
+      - rhel87-small
+    tasks:
+      - name: test-docarray

docarray/database/flatschema.json

@@ -0,0 +1 @@
+[]

docarray/database/mydoc__docs.json

@@ -0,0 +1 @@
+[]

docarray/database/mydoc__list_docs__docs.json

@@ -0,0 +1 @@
+[]

docarray/database/nesteddoc.json

@@ -0,0 +1 @@
+[]

docarray/database/simpleschema.json

@@ -0,0 +1 @@
+[]

docarray/indexes/text_index_simpleschema.json

@@ -0,0 +1,16 @@
+{
+  "mappings": {
+    "dynamic": false,
+    "fields": {
+      "text": [
+        {
+          "type": "string"
+        }
+      ]
+    }
+  },
+  "name": "text_index",
+  "type": "search",
+  "database": "docarray_test_db",
+  "collectionName": "simpleschema"
+}

docarray/indexes/vector_index_flatschema_1.json

@@ -0,0 +1,14 @@
+{
+  "fields": [
+    {
+      "numDimensions": 10,
+      "path": "embedding1",
+      "similarity": "cosine",
+      "type": "vector"
+    }
+  ],
+  "name": "vector_index_1",
+  "type": "vectorSearch",
+  "database": "docarray_test_db",
+  "collectionName": "flatschema"
+}

docarray/indexes/vector_index_flatschema_2.json

@@ -0,0 +1,14 @@
+{
+  "fields": [
+    {
+      "numDimensions": 50,
+      "path": "embedding2",
+      "similarity": "cosine",
+      "type": "vector"
+    }
+  ],
+  "name": "vector_index_2",
+  "type": "vectorSearch",
+  "database": "docarray_test_db",
+  "collectionName": "flatschema"
+}

docarray/indexes/vector_index_mydoc__docs.json

@@ -0,0 +1,14 @@
+{
+  "fields": [
+    {
+      "numDimensions": 10,
+      "path": "simple_tens",
+      "similarity": "euclidean",
+      "type": "vector"
+    }
+  ],
+  "name": "vector_index",
+  "type": "vectorSearch",
+  "database": "docarray_test_db",
+  "collectionName": "mydoc__docs"
+}

docarray/indexes/vector_index_mydoc__list_docs__docs.json

@@ -0,0 +1,16 @@
+{
+  "fields": [
+    {
+      "numDimensions": 10,
+      "path": "simple_tens",
+      "similarity": "euclidean",
+      "type": "vector"
+    }
+  ],
+  "name": "vector_index",
+  "type": "vectorSearch",
+  "database": "docarray_test_db",
+  "collectionName": "mydoc__list_docs__docs"
+}
+
+

docarray/indexes/vector_index_nesteddoc_1.json

@@ -0,0 +1,14 @@
+{
+  "fields": [
+    {
+      "numDimensions": 10,
+      "path": "d__embedding",
+      "similarity": "cosine",
+      "type": "vector"
+    }
+  ],
+  "name": "vector_index_1",
+  "type": "vectorSearch",
+  "database": "docarray_test_db",
+  "collectionName": "nesteddoc"
+}

docarray/indexes/vector_index_nesteddoc_2.json

@@ -0,0 +1,14 @@
+{
+  "fields": [
+    {
+      "numDimensions": 10,
+      "path": "embedding",
+      "similarity": "cosine",
+      "type": "vector"
+    }
+  ],
+  "name": "vector_index",
+  "type": "vectorSearch",
+  "database": "docarray_test_db",
+  "collectionName": "nesteddoc"
+}

docarray/indexes/vector_index_simpleschema.json

@@ -0,0 +1,22 @@
+{
+  "fields": [
+    {
+      "numDimensions": 10,
+      "path": "embedding",
+      "similarity": "cosine",
+      "type": "vector"
+    },
+    {
+      "path": "number",
+      "type": "filter"
+    },
+    {
+      "path": "text",
+      "type": "filter"
+    }
+  ],
+  "name": "vector_index",
+  "type": "vectorSearch",
+  "database": "docarray_test_db",
+  "collectionName": "simpleschema"
+}

docarray/run.sh

@@ -0,0 +1,26 @@
+#!/bin/sh
+
+#  Sets up a virtual environment (poetry)
+#  Runs the mongodb tests of the upstream repo
+
+set -x
+
+. $workdir/src/.evergreen/utils.sh
+PYTHON_BINARY=$(find_python3)
+$PYTHON_BINARY -c "import sys; print(f'Python version found: {sys.version_info}')"
+
+# Create and activate an isolated python venv environment
+$PYTHON_BINARY -m venv venv
+source venv/bin/activate
+# Install Poetry
+pip install -U pip poetry
+# Recreate the poetry lock file
+poetry lock --no-update
+# Install from pyproject.toml into package specific environment
+poetry install --with dev --extras mongo
+
+
+# Run tests. Sensitive variables in Evergreen come from Evergeen project: ai-ml-pipeline-testing/
+MONGODB_URI=$docarray_mongodb_uri \
+MONGODB_DATABASE="docarray_test_db" \
+pytest -v tests/index/mongo_atlas