fix/Fix python/js client sending unrecognized list params (Unstructured-IO#412)

# Changes
## FormData fix
FastAPI expects list params to look a certain way in form data. The
Speakeasy clients use a different, more explicit format that isn't
getting parsed by the api. Therefore, when client users send
`skip_infer_table_types` or `extract_image_block_types`, the server just
sets them to `None`.

The fix is to transform the formdata params before FastAPI parses them
into a list type. I tried adding middleware for this, but it turns out
`Request._form` isn't loaded at this point. Instead, we can just
monkeypatch the `Request` class to return the right thing when it's
asked for.

I updated an old parallel mode unit test to more generally assert that
all params received in the endpoint make it down to partition. By adding
square brackets to the list params, we can see the test pass once the
fix is applied.

# Testing

Run the server at port 8000 with `make run-web-app`. Load a pyenv that
has the latest python client, and in `ipython`, run the following
snippet. In the server log, verify that we see a warning about
`extract_image_block_types` having an invalid value: `foo`.

```
from unstructured_client import UnstructuredClient
from unstructured_client.models import shared
from unstructured_client.models.errors import SDKError

s = UnstructuredClient(
    api_key_auth=None,
    server_url="http://localhost:8000",
)

filename = "/path/to/any/file"

with open(filename, "rb") as f:
    files=shared.Files(
        content=f.read(),
        file_name=filename,
)

req = shared.PartitionParameters(
    files=files,
    # Other partition params
    strategy='fast',
    extract_image_block_types=["Foo", "Foo"],
)

try:
    resp = s.general.partition(req)
    print(resp.elements[0])
except SDKError as e:
    print(e)

```

Author: awalker4

CHANGELOG.md

@@ -1,3 +1,7 @@
+## 0.0.68
+
+* Fix list params such as `extract_image_block_types` not working via the python/js clients
+
 ## 0.0.67
 
 * Allow for a different server port with the PORT variable

prepline_general/api/app.py

@@ -1,4 +1,5 @@
 from fastapi import FastAPI, Request, status, HTTPException
+from fastapi.datastructures import FormData
 from fastapi.responses import JSONResponse
 from fastapi.security import APIKeyHeader
 import logging
@@ -12,7 +13,7 @@
 app = FastAPI(
     title="Unstructured Pipeline API",
     summary="Partition documents with the Unstructured library",
-    version="0.0.67",
+    version="0.0.68",
     docs_url="/general/docs",
     openapi_url="/general/openapi.json",
     servers=[
@@ -66,6 +67,47 @@ async def error_handler(request: Request, e: Exception):
 set_custom_openapi(app)
 
 
+# Note(austin) - When FastAPI parses our FormData params,
+# it builds lists out of duplicate keys, like so:
+# FormData([('key', 'value1'), ('key', 'value2')])
+#
+# The Speakeasy clients send a more explicit form:
+# FormData([('key[]', 'value1'), ('key[]', 'value2')])
+#
+# FastAPI doesn't understand these, so we need to transform them.
+# Can't do this in middleware before the data stream is read, nor in the endpoint
+# after the fields are parsed. Thus, we have to patch it into Request.form() on startup.
+get_form = Request._get_form
+
+
+async def patched_get_form(
+    self,
+    *,
+    max_files: int | float = 1000,
+    max_fields: int | float = 1000,
+) -> FormData:
+    """
+    Call the original get_form, and iterate the results
+    If a key has brackets at the end, remove them before returning the final FormData
+    Note the extra params here are unused, but needed to match the signature
+    """
+    form_params = await get_form(self)
+
+    fixed_params = []
+    for key, value in form_params.multi_items():
+        # Transform key[] into key
+        if key and key.endswith("[]"):
+            key = key[:-2]
+
+        fixed_params.append((key, value))
+
+    return FormData(fixed_params)
+
+
+# Replace the private method with our wrapper
+Request._get_form = patched_get_form  # type: ignore[assignment]
+
+
 # Filter out /healthcheck noise
 class HealthCheckFilter(logging.Filter):
     def filter(self, record: logging.LogRecord) -> bool:

prepline_general/api/general.py

@@ -704,7 +704,7 @@ def return_content_type(filename: str):
 
 
 @router.get("/general/v0/general", include_in_schema=False)
-@router.get("/general/v0.0.67/general", include_in_schema=False)
+@router.get("/general/v0.0.68/general", include_in_schema=False)
 async def handle_invalid_get_request():
     raise HTTPException(
         status_code=status.HTTP_405_METHOD_NOT_ALLOWED, detail="Only POST requests are supported."
@@ -719,7 +719,7 @@ async def handle_invalid_get_request():
     description="Description",
     operation_id="partition_parameters",
 )
-@router.post("/general/v0.0.67/general", include_in_schema=False)
+@router.post("/general/v0.0.68/general", include_in_schema=False)
 def general_partition(
     request: Request,
     # cannot use annotated type here because of a bug described here:

preprocessing-pipeline-family.yaml

@@ -1,2 +1,2 @@
 name: general
-version: 0.0.67
+version: 0.0.68

scripts/docker-smoke-test.sh

@@ -78,8 +78,8 @@ await_server_ready 8000
 #######################
 # Smoke Tests
 #######################
-echo Running smoke tests
-PYTHONPATH=. SKIP_INFERENCE_TESTS=$SKIP_INFERENCE_TESTS pytest scripts/smoketest.py
+echo Running smoke tests with SKIP_INFERENCE_TESTS: "$SKIP_INFERENCE_TESTS"
+PYTHONPATH=. SKIP_INFERENCE_TESTS=$SKIP_INFERENCE_TESTS pytest -vv scripts/smoketest.py
 
 #######################
 # Test parallel vs single mode

test_general/api/test_app.py

@@ -664,18 +664,21 @@ def test_parallel_mode_passes_params(monkeypatch):
     client = TestClient(app)
     test_file = Path("sample-docs") / "layout-parser-paper.pdf"
 
+    # For list params, send the formdata keys with brackets
+    # This is how Speakeasy sends them
     response = client.post(
         MAIN_API_ROUTE,
         files=[("files", (str(test_file), open(test_file, "rb"), "application/pdf"))],
         data={
             "encoding": "foo",
             "hi_res_model_name": "yolox",
             "include_page_breaks": "True",
-            "languages": "foo",
+            "languages": "eng",
             "pdf_infer_table_structure": "True",
             "strategy": "hi_res",
             "xml_keep_tags": "True",
-            "skip_infer_table_types": "foo",
+            "skip_infer_table_types[]": ["pdf"],
+            "extract_image_block_types[]": ["Image", "Table"],
             "unique_element_ids": "True",
             "starting_page_number": 1,
             # -- chunking options --
@@ -699,13 +702,13 @@ def test_parallel_mode_passes_params(monkeypatch):
         encoding="foo",
         include_page_breaks=True,
         ocr_languages=None,
-        languages=["foo"],
+        languages=["eng"],
         pdf_infer_table_structure=True,
         strategy="hi_res",
         xml_keep_tags=True,
-        skip_infer_table_types=["foo"],
-        extract_image_block_types=None,
-        extract_image_block_to_payload=False,
+        skip_infer_table_types=["pdf"],
+        extract_image_block_types=["Image", "Table"],
+        extract_image_block_to_payload=True,  # Set to true because block_types is non empty
         unique_element_ids=True,
         starting_page_number=1,
         # -- chunking options --