chatbot-rag-app: use Elastic Distribution of OpenTelemetry (EDOT)

example-apps/chatbot-rag-app/Dockerfile

@@ -28,4 +28,20 @@ COPY api ./api
 COPY data ./data
 
 EXPOSE 4000
+
+# Default to disabling instrumentation, can be overridden to false in
+# docker invocations to reenable.
+ENV OTEL_SDK_DISABLED=true
+
+# https://github.com/elastic/genai-instrumentation/issues/255
+# Currently Python SDK has a bug that spams logs when opentelemetry-instrument is used
+# with SDK being disabled. Until it is fixed, we handle it in our own entrypoint by
+# avoiding opentelemetry-instrument when SDK is disabled.
+RUN echo 'if [ "${OTEL_SDK_DISABLED:-true}" == "false" ]; \
+  then \
+    opentelemetry-instrument $@; \
+  else \
+    exec $@; \
+  fi' > entrypoint.sh
+ENTRYPOINT [ "bash", "-eu", "./entrypoint.sh" ]
 CMD [ "python", "api/app.py"]

example-apps/chatbot-rag-app/README.md

@@ -141,6 +141,23 @@ dotenv run -- python api/app.py
 
 ## Advanced
 
+### OpenTelemetry
+
+If you set `OTEL_SDK_DISABLED=false` in your `.env` file, the app will send
+logs, metrics and traces to an OpenTelemetry compatible endpoint.
+
+[env.example](env.example) defaults to use Elastic APM server, started by
+[docker-compose-elastic.yml](docker-compose-elastic.yml). If you start your
+Elastic stack this way, you can access Kibana like this, authenticating with
+the username "elastic" and password "elastic":
+
+http://localhost:5601/app/apm/traces?rangeFrom=now-15m&rangeTo=now
+
+Under the scenes, chatbot-rag-app is automatically instrumented by the Elastic
+Distribution of OpenTelemetry (EDOT) Python. While this supports OpenAI, it may
+not yet support all LLM providers. You can see more details about EDOT Python
+[here](https://github.com/elastic/elastic-otel-python).
+
 ### Updating package versions
 
 To update package versions, recreate [requirements.txt](requirements.txt) and
@@ -150,12 +167,16 @@ reinstall like this. Once checked in, any commands above will use updates.
 rm -rf .venv
 python3 -m venv .venv
 source .venv/bin/activate
-# Install dev requirements for pip-compile
-pip install pip-tools
+# Install dev requirements for pip-compile and edot-bootstrap
+pip install pip-tools elastic-opentelemetry
 # Recreate requirements.txt
 pip-compile
 # Install main dependencies
 pip install -r requirements.txt
+# Add opentelemetry instrumentation for these dependencies
+edot-bootstrap >> requirements.txt
+# Install opentelemetry dependencies
+pip install -r requirements.txt
 ```
 
 ### Elasticsearch index and chat_history index

example-apps/chatbot-rag-app/api/llm_integrations.py

@@ -1,6 +1,5 @@
 import os
 
-import vertexai
 from langchain_aws import ChatBedrock
 from langchain_cohere import ChatCohere
 from langchain_google_vertexai import ChatVertexAI

example-apps/chatbot-rag-app/docker-compose.yml

@@ -9,6 +9,7 @@ services:
     environment:
       # host.docker.internal means connect to the host machine, e.g. your laptop
       ELASTICSEARCH_URL: "http://host.docker.internal:9200"
+      OTEL_EXPORTER_OTLP_ENDPOINT: "http://host.docker.internal:8200"
       FLASK_APP: api/app.py
     env_file:
       - .env
@@ -26,6 +27,7 @@ services:
     environment:
       # host.docker.internal means connect to the host machine, e.g. your laptop
       ELASTICSEARCH_URL: "http://host.docker.internal:9200"
+      OTEL_EXPORTER_OTLP_ENDPOINT: "http://host.docker.internal:8200"
     env_file:
       - .env
     ports:

example-apps/chatbot-rag-app/env.example

@@ -31,7 +31,7 @@ ES_INDEX_CHAT_HISTORY=workplace-app-docs-chat-history
 # AWS_ACCESS_KEY_ID=
 # AWS_SECRET_ACCESS_KEY=
 # AWS_DEFAULT_REGION=
-# CHAT_MODEL=anthropic.claude-3-5-sonnet-20240620-v1:0
+# CHAT_MODEL=amazon.titan-text-lite-v1
 
 # Uncomment and complete if you want to use Vertex AI
 # LLM_TYPE=vertex
@@ -56,3 +56,24 @@ ES_INDEX_CHAT_HISTORY=workplace-app-docs-chat-history
 ## Key in https://dashboard.cohere.com/api-keys
 # COHERE_API_KEY=
 # CHAT_MODEL=command-r7b-12-2024
+
+# Set to false, if you want to record logs, traces and metrics.
+OTEL_SDK_DISABLED=true
+
+# Assign the service name that shows up in Kibana
+OTEL_SERVICE_NAME=chatbot-rag-app
+
+# Default to send traces to the Elastic APM server
+OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:8200
+OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf
+
+# Change to 'false' to hide prompt and completion content
+OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT=true
+
+# Export metrics every 3 seconds instead of every minute
+OTEL_METRIC_EXPORT_INTERVAL=3000
+# Export traces every 3 seconds instead of every 5 seconds
+OTEL_BSP_SCHEDULE_DELAY=3000
+# Change to affect behavior of which resources are detected. Note: these
+# choices are specific to the language, in this case Python.
+OTEL_EXPERIMENTAL_RESOURCE_DETECTORS=process_runtime,os,otel,telemetry_distro

example-apps/chatbot-rag-app/requirements.in

@@ -12,3 +12,6 @@ langchain-cohere
 langchain-google-vertexai
 langchain-aws
 langchain-mistralai
+
+# otel dependencies
+elastic-opentelemetry

example-apps/chatbot-rag-app/requirements.txt

@@ -1,5 +1,5 @@
 #
-# This file is autogenerated by pip-compile with Python 3.13
+# This file is autogenerated by pip-compile with Python 3.12
 # by the following command:
 #
 #    pip-compile
@@ -44,10 +44,18 @@ cohere==5.13.3
     # via langchain-cohere
 dataclasses-json==0.6.7
     # via langchain-community
+deprecated==1.2.15
+    # via
+    #   opentelemetry-api
+    #   opentelemetry-exporter-otlp-proto-grpc
+    #   opentelemetry-exporter-otlp-proto-http
+    #   opentelemetry-semantic-conventions
 distro==1.9.0
     # via openai
 docstring-parser==0.16
     # via google-cloud-aiplatform
+elastic-opentelemetry==0.6.0
+    # via -r requirements.in
 elastic-transport==8.15.1
     # via elasticsearch
 elasticsearch[vectorstore-mmr]==8.17.0
@@ -112,6 +120,10 @@ googleapis-common-protos[grpc]==1.66.0
     #   google-api-core
     #   grpc-google-iam-v1
     #   grpcio-status
+    #   opentelemetry-exporter-otlp-proto-grpc
+    #   opentelemetry-exporter-otlp-proto-http
+greenlet==3.1.1
+    # via sqlalchemy
 grpc-google-iam-v1==0.13.1
     # via google-cloud-resource-manager
 grpcio==1.68.1
@@ -120,6 +132,7 @@ grpcio==1.68.1
     #   googleapis-common-protos
     #   grpc-google-iam-v1
     #   grpcio-status
+    #   opentelemetry-exporter-otlp-proto-grpc
 grpcio-status==1.68.1
     # via google-api-core
 h11==0.14.0
@@ -147,6 +160,8 @@ idna==3.10
     #   httpx
     #   requests
     #   yarl
+importlib-metadata==8.5.0
+    # via opentelemetry-api
 itsdangerous==2.2.0
     # via flask
 jinja2==3.1.4
@@ -223,15 +238,67 @@ numpy==1.26.4
     #   shapely
 openai==1.58.1
     # via langchain-openai
+opentelemetry-api==1.29.0
+    # via
+    #   elastic-opentelemetry
+    #   opentelemetry-exporter-otlp-proto-grpc
+    #   opentelemetry-exporter-otlp-proto-http
+    #   opentelemetry-instrumentation
+    #   opentelemetry-instrumentation-system-metrics
+    #   opentelemetry-resourcedetector-gcp
+    #   opentelemetry-sdk
+    #   opentelemetry-semantic-conventions
+opentelemetry-exporter-otlp==1.29.0
+    # via elastic-opentelemetry
+opentelemetry-exporter-otlp-proto-common==1.29.0
+    # via
+    #   opentelemetry-exporter-otlp-proto-grpc
+    #   opentelemetry-exporter-otlp-proto-http
+opentelemetry-exporter-otlp-proto-grpc==1.29.0
+    # via opentelemetry-exporter-otlp
+opentelemetry-exporter-otlp-proto-http==1.29.0
+    # via opentelemetry-exporter-otlp
+opentelemetry-instrumentation==0.50b0
+    # via
+    #   elastic-opentelemetry
+    #   opentelemetry-instrumentation-system-metrics
+opentelemetry-instrumentation-system-metrics==0.50b0
+    # via elastic-opentelemetry
+opentelemetry-proto==1.29.0
+    # via
+    #   opentelemetry-exporter-otlp-proto-common
+    #   opentelemetry-exporter-otlp-proto-grpc
+    #   opentelemetry-exporter-otlp-proto-http
+opentelemetry-resource-detector-azure==0.1.5
+    # via elastic-opentelemetry
+opentelemetry-resourcedetector-gcp==1.7.0a0
+    # via elastic-opentelemetry
+opentelemetry-sdk==1.29.0
+    # via
+    #   elastic-opentelemetry
+    #   opentelemetry-exporter-otlp-proto-grpc
+    #   opentelemetry-exporter-otlp-proto-http
+    #   opentelemetry-resource-detector-azure
+    #   opentelemetry-resourcedetector-gcp
+    #   opentelemetry-sdk-extension-aws
+opentelemetry-sdk-extension-aws==2.0.2
+    # via elastic-opentelemetry
+opentelemetry-semantic-conventions==0.50b0
+    # via
+    #   elastic-opentelemetry
+    #   opentelemetry-instrumentation
+    #   opentelemetry-sdk
 orjson==3.10.12
     # via langsmith
 packaging==24.2
     # via
+    #   elastic-opentelemetry
     #   google-cloud-aiplatform
     #   google-cloud-bigquery
     #   huggingface-hub
     #   langchain-core
     #   marshmallow
+    #   opentelemetry-instrumentation
 pandas==2.2.3
     # via langchain-cohere
 parameterized==0.9.0
@@ -253,7 +320,10 @@ protobuf==5.29.2
     #   googleapis-common-protos
     #   grpc-google-iam-v1
     #   grpcio-status
+    #   opentelemetry-proto
     #   proto-plus
+psutil==6.1.1
+    # via opentelemetry-instrumentation-system-metrics
 pyasn1==0.6.1
     # via
     #   pyasn1-modules
@@ -306,6 +376,8 @@ requests==2.32.3
     #   langchain
     #   langchain-community
     #   langsmith
+    #   opentelemetry-exporter-otlp-proto-http
+    #   opentelemetry-resourcedetector-gcp
     #   requests-toolbelt
     #   tiktoken
 requests-toolbelt==1.0.0
@@ -352,10 +424,13 @@ types-requests==2.32.0.20241016
     # via cohere
 typing-extensions==4.12.2
     # via
+    #   anyio
     #   cohere
     #   huggingface-hub
     #   langchain-core
     #   openai
+    #   opentelemetry-resourcedetector-gcp
+    #   opentelemetry-sdk
     #   pydantic
     #   pydantic-core
     #   sqlalchemy
@@ -372,5 +447,34 @@ urllib3==2.2.3
     #   types-requests
 werkzeug==3.1.3
     # via flask
+wrapt==1.17.0
+    # via
+    #   deprecated
+    #   opentelemetry-instrumentation
 yarl==1.18.3
     # via aiohttp
+zipp==3.21.0
+    # via importlib-metadata
+opentelemetry-instrumentation-asyncio==0.50b0
+opentelemetry-instrumentation-dbapi==0.50b0
+opentelemetry-instrumentation-logging==0.50b0
+opentelemetry-instrumentation-sqlite3==0.50b0
+opentelemetry-instrumentation-threading==0.50b0
+opentelemetry-instrumentation-urllib==0.50b0
+opentelemetry-instrumentation-wsgi==0.50b0
+opentelemetry-instrumentation-aiohttp-client==0.50b0
+opentelemetry-instrumentation-aiohttp-server==0.50b0
+opentelemetry-instrumentation-boto3sqs==0.50b0
+opentelemetry-instrumentation-botocore==0.50b0
+opentelemetry-instrumentation-click==0.50b0
+opentelemetry-instrumentation-elasticsearch==0.50b0
+opentelemetry-instrumentation-flask==0.50b0
+opentelemetry-instrumentation-grpc==0.50b0
+opentelemetry-instrumentation-httpx==0.50b0
+opentelemetry-instrumentation-jinja2==0.50b0
+opentelemetry-instrumentation-requests==0.50b0
+opentelemetry-instrumentation-sqlalchemy==0.50b0
+opentelemetry-instrumentation-system-metrics==0.50b0
+opentelemetry-instrumentation-tortoiseorm==0.50b0
+opentelemetry-instrumentation-urllib3==0.50b0
+elastic-opentelemetry-instrumentation-openai==0.6.0