feat: add tracing for batchUpdate, executeUpdate, and connections (#3097)

* feat: add OpenTelemetry tracing to Connection API

* chore: add more traces + run formatter

* test: add tracing tests

* chore: make db.statement optional

* feat: add tags to traces

* fix: declare test dependency

* 🦉 Updates from OwlBot post-processor

See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md

* chore: create one option for extended tracing

Create one combined option for extended tracing, and disable this
option by default for both the Connection API and the standard
client.

* 🦉 Updates from OwlBot post-processor

See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md

* chore: add ignored diff for clirr

* chore: address review comments

* chore: add documentation for env var

* 🦉 Updates from OwlBot post-processor

See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md

* 🦉 Updates from OwlBot post-processor

See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md

* 🦉 Updates from OwlBot post-processor

See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md

---------

Co-authored-by: Owl Bot <gcf-owl-bot[bot]@users.noreply.github.com>

Author: olavloite

.readme-partials.yaml

@@ -144,12 +144,27 @@ custom_content: |
           .build()
 
   SpannerOptions options = SpannerOptions.newBuilder()
-  // Inject OpenTelemetry object via Spanner Options or register OpenTelmetry object as Global
+  // Inject OpenTelemetry object via Spanner Options or register OpenTelemetry object as Global
     .setOpenTelemetry(openTelemetry)
     .build();
 
   Spanner spanner = options.getService();
   ```
+  
+  #### OpenTelemetry SQL Statement Tracing
+  The OpenTelemetry traces that are generated by the Java client include any request and transaction
+  tags that have been set. The traces can also include the SQL statements that are executed. Enable
+  this with the `enableExtendedTracing` option:
+  
+  ```
+  SpannerOptions options = SpannerOptions.newBuilder()
+    .setOpenTelemetry(openTelemetry)
+    .setEnableExtendedTracing(true)
+    .build();
+  ```
+  
+  This option can also be enabled by setting the environment variable
+  `SPANNER_ENABLE_EXTENDED_TRACING=true`.
 
   ### Instrument with OpenCensus
 

README.md

@@ -50,7 +50,7 @@ If you are using Maven without the BOM, add this to your dependencies:
 If you are using Gradle 5.x or later, add this to your dependencies:
 
 ```Groovy
-implementation platform('com.google.cloud:libraries-bom:26.38.0')
+implementation platform('com.google.cloud:libraries-bom:26.39.0')
 
 implementation 'com.google.cloud:google-cloud-spanner'
 ```
@@ -250,13 +250,28 @@ OpenTelemetry openTelemetry = OpenTelemetrySdk.builder()
         .build()
 
 SpannerOptions options = SpannerOptions.newBuilder()
-// Inject OpenTelemetry object via Spanner Options or register OpenTelmetry object as Global
+// Inject OpenTelemetry object via Spanner Options or register OpenTelemetry object as Global
   .setOpenTelemetry(openTelemetry)
   .build();
 
 Spanner spanner = options.getService();
 ```
 
+#### OpenTelemetry SQL Statement Tracing
+The OpenTelemetry traces that are generated by the Java client include any request and transaction
+tags that have been set. The traces can also include the SQL statements that are executed. Enable
+this with the `enableExtendedTracing` option:
+
+```
+SpannerOptions options = SpannerOptions.newBuilder()
+  .setOpenTelemetry(openTelemetry)
+  .setEnableExtendedTracing(true)
+  .build();
+```
+
+This option can also be enabled by setting the environment variable
+`SPANNER_ENABLE_EXTENDED_TRACING=true`.
+
 ### Instrument with OpenCensus
 
 > Note: OpenCensus project is deprecated. See [Sunsetting OpenCensus](https://opentelemetry.io/blog/2023/sunsetting-opencensus/).

google-cloud-spanner/clirr-ignored-differences.xml

@@ -668,5 +668,12 @@
     <className>com/google/cloud/spanner/connection/Connection</className>
     <method>com.google.cloud.spanner.connection.DdlInTransactionMode getDdlInTransactionMode()</method>
   </difference>
+  
+  <!-- Added extended tracing option -->
+  <difference>
+    <differenceType>7012</differenceType>
+    <className>com/google/cloud/spanner/SpannerOptions$SpannerEnvironment</className>
+    <method>boolean isEnableExtendedTracing()</method>
+  </difference>
 
 </differences>

google-cloud-spanner/pom.xml

@@ -417,6 +417,12 @@
       <version>${opentelemetry.version}</version>
       <scope>test</scope>
     </dependency>
+    <dependency>
+      <groupId>io.opentelemetry</groupId>
+      <artifactId>opentelemetry-sdk-common</artifactId>
+      <version>${opentelemetry.version}</version>
+      <scope>test</scope>
+    </dependency>
     <dependency>
       <groupId>io.opentelemetry</groupId>
       <artifactId>opentelemetry-sdk-metrics</artifactId>

google-cloud-spanner/src/main/java/com/google/cloud/spanner/AbstractReadContext.java

@@ -735,6 +735,7 @@ ResultSet executeQueryInternalWithOptions(
             SpannerImpl.QUERY,
             span,
             tracer,
+            tracer.createStatementAttributes(statement, options),
             rpc.getExecuteQueryRetrySettings(),
             rpc.getExecuteQueryRetryableCodes()) {
           @Override

google-cloud-spanner/src/main/java/com/google/cloud/spanner/DatabaseClientImpl.java

@@ -92,7 +92,7 @@ public Timestamp write(final Iterable<Mutation> mutations) throws SpannerExcepti
   public CommitResponse writeWithOptions(
       final Iterable<Mutation> mutations, final TransactionOption... options)
       throws SpannerException {
-    ISpan span = tracer.spanBuilder(READ_WRITE_TRANSACTION);
+    ISpan span = tracer.spanBuilder(READ_WRITE_TRANSACTION, options);
     try (IScope s = tracer.withSpan(span)) {
       return runWithSessionRetry(session -> session.writeWithOptions(mutations, options));
     } catch (RuntimeException e) {
@@ -112,7 +112,7 @@ public Timestamp writeAtLeastOnce(final Iterable<Mutation> mutations) throws Spa
   public CommitResponse writeAtLeastOnceWithOptions(
       final Iterable<Mutation> mutations, final TransactionOption... options)
       throws SpannerException {
-    ISpan span = tracer.spanBuilder(READ_WRITE_TRANSACTION);
+    ISpan span = tracer.spanBuilder(READ_WRITE_TRANSACTION, options);
     try (IScope s = tracer.withSpan(span)) {
       return runWithSessionRetry(
           session -> session.writeAtLeastOnceWithOptions(mutations, options));
@@ -128,7 +128,7 @@ public CommitResponse writeAtLeastOnceWithOptions(
   public ServerStream<BatchWriteResponse> batchWriteAtLeastOnce(
       final Iterable<MutationGroup> mutationGroups, final TransactionOption... options)
       throws SpannerException {
-    ISpan span = tracer.spanBuilder(READ_WRITE_TRANSACTION);
+    ISpan span = tracer.spanBuilder(READ_WRITE_TRANSACTION, options);
     try (IScope s = tracer.withSpan(span)) {
       return runWithSessionRetry(session -> session.batchWriteAtLeastOnce(mutationGroups, options));
     } catch (RuntimeException e) {
@@ -213,7 +213,7 @@ public ReadOnlyTransaction readOnlyTransaction(TimestampBound bound) {
 
   @Override
   public TransactionRunner readWriteTransaction(TransactionOption... options) {
-    ISpan span = tracer.spanBuilder(READ_WRITE_TRANSACTION);
+    ISpan span = tracer.spanBuilder(READ_WRITE_TRANSACTION, options);
     try (IScope s = tracer.withSpan(span)) {
       return getSession().readWriteTransaction(options);
     } catch (RuntimeException e) {
@@ -225,7 +225,7 @@ public TransactionRunner readWriteTransaction(TransactionOption... options) {
 
   @Override
   public TransactionManager transactionManager(TransactionOption... options) {
-    ISpan span = tracer.spanBuilder(READ_WRITE_TRANSACTION);
+    ISpan span = tracer.spanBuilder(READ_WRITE_TRANSACTION, options);
     try (IScope s = tracer.withSpan(span)) {
       return getSession().transactionManager(options);
     } catch (RuntimeException e) {
@@ -237,7 +237,7 @@ public TransactionManager transactionManager(TransactionOption... options) {
 
   @Override
   public AsyncRunner runAsync(TransactionOption... options) {
-    ISpan span = tracer.spanBuilder(READ_WRITE_TRANSACTION);
+    ISpan span = tracer.spanBuilder(READ_WRITE_TRANSACTION, options);
     try (IScope s = tracer.withSpan(span)) {
       return getSession().runAsync(options);
     } catch (RuntimeException e) {
@@ -249,7 +249,7 @@ public AsyncRunner runAsync(TransactionOption... options) {
 
   @Override
   public AsyncTransactionManager transactionManagerAsync(TransactionOption... options) {
-    ISpan span = tracer.spanBuilder(READ_WRITE_TRANSACTION);
+    ISpan span = tracer.spanBuilder(READ_WRITE_TRANSACTION, options);
     try (IScope s = tracer.withSpan(span)) {
       return getSession().transactionManagerAsync(options);
     } catch (RuntimeException e) {

google-cloud-spanner/src/main/java/com/google/cloud/spanner/Options.java

@@ -355,6 +355,10 @@ static final class TagOption extends InternalOption implements ReadQueryUpdateTr
       this.tag = tag;
     }
 
+    String getTag() {
+      return tag;
+    }
+
     @Override
     void appendToOptions(Options options) {
       options.tag = tag;

google-cloud-spanner/src/main/java/com/google/cloud/spanner/ResumableStreamIterator.java

@@ -34,6 +34,7 @@
 import com.google.protobuf.ByteString;
 import com.google.spanner.v1.PartialResultSet;
 import io.grpc.Context;
+import io.opentelemetry.api.common.Attributes;
 import java.io.IOException;
 import java.util.LinkedList;
 import java.util.Objects;
@@ -81,10 +82,28 @@ protected ResumableStreamIterator(
       TraceWrapper tracer,
       RetrySettings streamingRetrySettings,
       Set<Code> retryableCodes) {
+    this(
+        maxBufferSize,
+        streamName,
+        parent,
+        tracer,
+        Attributes.empty(),
+        streamingRetrySettings,
+        retryableCodes);
+  }
+
+  protected ResumableStreamIterator(
+      int maxBufferSize,
+      String streamName,
+      ISpan parent,
+      TraceWrapper tracer,
+      Attributes attributes,
+      RetrySettings streamingRetrySettings,
+      Set<Code> retryableCodes) {
     checkArgument(maxBufferSize >= 0);
     this.maxBufferSize = maxBufferSize;
     this.tracer = tracer;
-    this.span = tracer.spanBuilderWithExplicitParent(streamName, parent);
+    this.span = tracer.spanBuilderWithExplicitParent(streamName, parent, attributes);
     this.streamingRetrySettings = Preconditions.checkNotNull(streamingRetrySettings);
     this.retryableCodes = Preconditions.checkNotNull(retryableCodes);
     this.backOff = newBackOff();

google-cloud-spanner/src/main/java/com/google/cloud/spanner/SpannerImpl.java

@@ -63,11 +63,12 @@ class SpannerImpl extends BaseService<SpannerOptions> implements Spanner {
   final TraceWrapper tracer =
       new TraceWrapper(
           Tracing.getTracer(),
-          this.getOptions()
+          getOptions()
               .getOpenTelemetry()
               .getTracer(
                   MetricRegistryConstants.INSTRUMENTATION_SCOPE,
-                  GaxProperties.getLibraryVersion(this.getOptions().getClass())));
+                  GaxProperties.getLibraryVersion(this.getOptions().getClass())),
+          getOptions().isEnableExtendedTracing());
 
   static final String CREATE_MULTIPLEXED_SESSION = "CloudSpannerOperation.CreateMultiplexedSession";
   static final String CREATE_SESSION = "CloudSpannerOperation.CreateSession";
@@ -80,6 +81,8 @@ class SpannerImpl extends BaseService<SpannerOptions> implements Spanner {
   static final String QUERY = "CloudSpannerOperation.ExecuteStreamingQuery";
   static final String READ = "CloudSpannerOperation.ExecuteStreamingRead";
   static final String BATCH_WRITE = "CloudSpannerOperation.BatchWrite";
+  static final String UPDATE = "CloudSpannerOperation.ExecuteUpdate";
+  static final String BATCH_UPDATE = "CloudSpannerOperation.BatchUpdate";
 
   private static final Object CLIENT_ID_LOCK = new Object();
 

google-cloud-spanner/src/main/java/com/google/cloud/spanner/SpannerOptions.java

@@ -149,6 +149,7 @@ public class SpannerOptions extends ServiceOptions<Spanner, SpannerOptions> {
   private final DirectedReadOptions directedReadOptions;
   private final boolean useVirtualThreads;
   private final OpenTelemetry openTelemetry;
+  private final boolean enableExtendedTracing;
 
   enum TracingFramework {
     OPEN_CENSUS,
@@ -653,6 +654,7 @@ protected SpannerOptions(Builder builder) {
     directedReadOptions = builder.directedReadOptions;
     useVirtualThreads = builder.useVirtualThreads;
     openTelemetry = builder.openTelemetry;
+    enableExtendedTracing = builder.enableExtendedTracing;
   }
 
   /**
@@ -665,15 +667,21 @@ public interface SpannerEnvironment {
      * set.
      */
     @Nonnull
-    String getOptimizerVersion();
+    default String getOptimizerVersion() {
+      return "";
+    }
 
     /**
      * The optimizer statistics package to use. Must return an empty string to indicate that no
      * value has been set.
      */
     @Nonnull
     default String getOptimizerStatisticsPackage() {
-      throw new UnsupportedOperationException("Unimplemented");
+      return "";
+    }
+
+    default boolean isEnableExtendedTracing() {
+      return false;
     }
   }
 
@@ -686,19 +694,27 @@ private static class SpannerEnvironmentImpl implements SpannerEnvironment {
     private static final String SPANNER_OPTIMIZER_VERSION_ENV_VAR = "SPANNER_OPTIMIZER_VERSION";
     private static final String SPANNER_OPTIMIZER_STATISTICS_PACKAGE_ENV_VAR =
         "SPANNER_OPTIMIZER_STATISTICS_PACKAGE";
+    private static final String SPANNER_ENABLE_EXTENDED_TRACING = "SPANNER_ENABLE_EXTENDED_TRACING";
 
     private SpannerEnvironmentImpl() {}
 
+    @Nonnull
     @Override
     public String getOptimizerVersion() {
       return MoreObjects.firstNonNull(System.getenv(SPANNER_OPTIMIZER_VERSION_ENV_VAR), "");
     }
 
+    @Nonnull
     @Override
     public String getOptimizerStatisticsPackage() {
       return MoreObjects.firstNonNull(
           System.getenv(SPANNER_OPTIMIZER_STATISTICS_PACKAGE_ENV_VAR), "");
     }
+
+    @Override
+    public boolean isEnableExtendedTracing() {
+      return Boolean.parseBoolean(System.getenv(SPANNER_ENABLE_EXTENDED_TRACING));
+    }
   }
 
   /** Builder for {@link SpannerOptions} instances. */
@@ -762,6 +778,7 @@ public static class Builder
     private DirectedReadOptions directedReadOptions;
     private boolean useVirtualThreads = false;
     private OpenTelemetry openTelemetry;
+    private boolean enableExtendedTracing = SpannerOptions.environment.isEnableExtendedTracing();
 
     private static String createCustomClientLibToken(String token) {
       return token + " " + ServiceOptions.getGoogApiClientLibName();
@@ -825,6 +842,7 @@ protected Builder() {
       this.attemptDirectPath = options.attemptDirectPath;
       this.directedReadOptions = options.directedReadOptions;
       this.useVirtualThreads = options.useVirtualThreads;
+      this.enableExtendedTracing = options.enableExtendedTracing;
     }
 
     @Override
@@ -1321,6 +1339,19 @@ protected Builder setUseVirtualThreads(boolean useVirtualThreads) {
       return this;
     }
 
+    /**
+     * Sets whether to enable extended OpenTelemetry tracing. Enabling this option will add the
+     * following additional attributes to the traces that are generated by the client:
+     *
+     * <ul>
+     *   <li>db.statement: Contains the SQL statement that is being executed.
+     * </ul>
+     */
+    public Builder setEnableExtendedTracing(boolean enableExtendedTracing) {
+      this.enableExtendedTracing = enableExtendedTracing;
+      return this;
+    }
+
     @SuppressWarnings("rawtypes")
     @Override
     public SpannerOptions build() {
@@ -1563,6 +1594,18 @@ public boolean isUseVirtualThreads() {
     return useVirtualThreads;
   }
 
+  /**
+   * Returns whether extended OpenTelemetry tracing is enabled. Enabling this option will add the
+   * following additional attributes to the traces that are generated by the client:
+   *
+   * <ul>
+   *   <li>db.statement: Contains the SQL statement that is being executed.
+   * </ul>
+   */
+  public boolean isEnableExtendedTracing() {
+    return enableExtendedTracing;
+  }
+
   /** Returns the default query options to use for the specific database. */
   public QueryOptions getDefaultQueryOptions(DatabaseId databaseId) {
     // Use the specific query options for the database if any have been specified. These have