Fix UnknownRemoteOperation for Database instrumentation. (#50)

Description of changes:
When `"db.operation"` is not provided in span attributes, the
`"aws.remote.operation"` will be `UnknowRemoteOperation`.
This PR updated the `_set_remote_service_and_operation` function to
parsing the `"db.statement"` attribute when `"db.operation"` is missing,
and extract the valid `"aws.remote.operation"` (We setup a set of valid
database related `"aws.remote.operation"` for security concern):
1. Add a json file `configuration/dialect_keywords.json` saving all the
valid keywords. In this file, the sequence of key words matter -> The
keyword with longer word length are placed towards the front of the list
so it will be matched first.
2. We retrieve the first 27 characters from `"db.statement"` to avoid
the case where very large statements, and use regular expression to
match the keyword, it will match the beginning of the string. If the
string's start does not conform to the regular expression, the match
fails.
3. Add unit test for _set_remote_service_and_operation performance where
tested _DB_STATEMENT is/is not present and is/is not valid.
4. Add unit test covering different `"db.statement"` cases:
a. Only 1 valid keywords match
b. More than 1 valid keywords match, we want to pick the longest match
c. More than 1 valid keywords match, but the other keywords is not at
the start of the SpanAttributes.DB_STATEMENT. We want to only pick start
match
d. No valid match
e. Have valid but it is not at the start of SpanAttributes.DB_STATEMENT
f. Have valid keywords, match the longest word
g. Have valid keywords, match with first word
h.Have valid keywords, match with upper case
5.Add unit test for testing keywords sequence in json file:
a.Confirm the keywords are sorted based on descending order of keywords
character length
b.Confirm maximum length of keywords is not longer than
MAX_KEYWORD_LENGTH
6. exclude `configuration/dialect_keywords.json` from codespell check
because the Keywords are fixed for the query, and codespell does not
recognize the comments in json.

Testing
Tested by deploying the code changes to EKS and nodes, where we can see
the actual operation instead of `UnknowRemoteOperation`:

![Screenshot 2024-02-13 at 6 21
34 PM](https://github.com/aws-observability/aws-otel-python-instrumentation/assets/146124015/cd4cb2d5-f069-45c9-8487-45a709ef6e5a)


By submitting this pull request, I confirm that you can use, modify,
copy, and redistribute this contribution, under the terms of your
choice.

Author: zzhlogin

.codespellrc

@@ -1,4 +1,4 @@
 [codespell]
 # skipping auto generated folders
-skip = ./.tox,./.mypy_cache,./target,*/LICENSE,./venv
+skip = ./.tox,./.mypy_cache,./target,*/LICENSE,./venv,*/sql_dialect_keywords.json
 ignore-words-list = ot

aws-opentelemetry-distro/src/amazon/opentelemetry/distro/_aws_metric_attribute_generator.py

@@ -1,7 +1,8 @@
 # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
 # SPDX-License-Identifier: Apache-2.0
+import re
 from logging import DEBUG, Logger, getLogger
-from typing import Optional
+from typing import Match, Optional
 from urllib.parse import ParseResult, urlparse
 
 from amazon.opentelemetry.distro._aws_attribute_keys import (
@@ -14,6 +15,8 @@
 )
 from amazon.opentelemetry.distro._aws_span_processing_util import (
     LOCAL_ROOT,
+    MAX_KEYWORD_LENGTH,
+    SQL_KEYWORD_PATTERN,
     UNKNOWN_OPERATION,
     UNKNOWN_REMOTE_OPERATION,
     UNKNOWN_REMOTE_SERVICE,
@@ -38,6 +41,7 @@
 # Pertinent OTEL attribute keys
 _SERVICE_NAME: str = ResourceAttributes.SERVICE_NAME
 _DB_OPERATION: str = SpanAttributes.DB_OPERATION
+_DB_STATEMENT: str = SpanAttributes.DB_STATEMENT
 _DB_SYSTEM: str = SpanAttributes.DB_SYSTEM
 _FAAS_INVOKED_NAME: str = SpanAttributes.FAAS_INVOKED_NAME
 _FAAS_TRIGGER: str = SpanAttributes.FAAS_TRIGGER
@@ -189,9 +193,12 @@ def _set_remote_service_and_operation(span: ReadableSpan, attributes: BoundedAtt
     elif is_key_present(span, _RPC_SERVICE) or is_key_present(span, _RPC_METHOD):
         remote_service = _normalize_service_name(span, _get_remote_service(span, _RPC_SERVICE))
         remote_operation = _get_remote_operation(span, _RPC_METHOD)
-    elif is_key_present(span, _DB_SYSTEM) or is_key_present(span, _DB_OPERATION):
+    elif is_key_present(span, _DB_SYSTEM) or is_key_present(span, _DB_OPERATION) or is_key_present(span, _DB_STATEMENT):
         remote_service = _get_remote_service(span, _DB_SYSTEM)
-        remote_operation = _get_remote_operation(span, _DB_OPERATION)
+        if is_key_present(span, _DB_OPERATION):
+            remote_operation = _get_remote_operation(span, _DB_OPERATION)
+        else:
+            remote_operation = _get_db_statement_remote_operation(span, _DB_STATEMENT)
     elif is_key_present(span, _FAAS_INVOKED_NAME) or is_key_present(span, _FAAS_TRIGGER):
         remote_service = _get_remote_service(span, _FAAS_INVOKED_NAME)
         remote_operation = _get_remote_operation(span, _FAAS_TRIGGER)
@@ -232,6 +239,28 @@ def _get_remote_operation(span: ReadableSpan, remote_operation_key: str) -> str:
     return remote_operation
 
 
+def _get_db_statement_remote_operation(span: ReadableSpan, statement_key: str) -> str:
+    """
+    If no db.operation attribute provided in the span,
+    we use db.statement to compute a valid remote operation in a best-effort manner.
+    To do this, we take the first substring of the statement
+    and compare to a regex list of known SQL keywords.
+    The substring length is determined by the longest known SQL keywords.
+    """
+    remote_operation: str = span.attributes.get(statement_key)
+
+    if remote_operation is None:
+        return UNKNOWN_REMOTE_OPERATION
+
+    # Remove all whitespace and newline characters from the beginning of remote_operation
+    # and retrieve the first MAX_KEYWORD_LENGTH characters
+    remote_operation = remote_operation.lstrip()[:MAX_KEYWORD_LENGTH]
+    match: Optional[Match[str]] = re.match(SQL_KEYWORD_PATTERN, remote_operation.upper())
+    remote_operation = match.group(0) if match else UNKNOWN_REMOTE_OPERATION
+
+    return remote_operation
+
+
 def _normalize_service_name(span: ReadableSpan, service_name: str) -> str:
     """
     TODO: Determine if problems in Java instrumentation are relevant here. Do we need normalization? If so, probably we

aws-opentelemetry-distro/src/amazon/opentelemetry/distro/_aws_span_processing_util.py

@@ -1,6 +1,10 @@
 # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
 # SPDX-License-Identifier: Apache-2.0
 """Utility module designed to support shared logic across AWS Span Processors."""
+import json
+import os
+from typing import Dict, List
+
 from amazon.opentelemetry.distro._aws_attribute_keys import AWS_CONSUMER_PARENT_SPAN_KIND, AWS_LOCAL_OPERATION
 from opentelemetry.sdk.trace import InstrumentationScope, ReadableSpan
 from opentelemetry.semconv.trace import MessagingOperationValues, SpanAttributes
@@ -18,6 +22,23 @@
 _SQS_RECEIVE_MESSAGE_SPAN_NAME: str = "Sqs.ReceiveMessage"
 _AWS_SDK_INSTRUMENTATION_SCOPE_PREFIX: str = "io.opentelemetry.aws-sdk-"
 
+# Max keyword length supported by parsing into remote_operation from DB_STATEMENT
+MAX_KEYWORD_LENGTH = 27
+
+
+# Get dialect keywords retrieved from dialect_keywords.json file.
+# Only meant to be invoked by SQL_KEYWORD_PATTERN and unit tests
+def _get_dialect_keywords() -> List[str]:
+    current_dir: str = os.path.dirname(__file__)
+    file_path: str = os.path.join(current_dir, "configuration/sql_dialect_keywords.json")
+    with open(file_path, "r", encoding="utf-8") as json_file:
+        keywords_data: Dict[str, str] = json.load(json_file)
+    return keywords_data["keywords"]
+
+
+# A regular expression pattern to match SQL keywords.
+SQL_KEYWORD_PATTERN = r"^(?:" + "|".join(_get_dialect_keywords()) + r")\b"
+
 
 def get_ingress_operation(__, span: ReadableSpan) -> str:
     """