chore: configure backups test to run in nightly build (#1409)

* test: remove unused junit categories

* test: add slow test category

Add a new JUnit category: SlowTest. It will be used to execute tests
that are part of the nightly build.
Also refactors the categories hierarchy to be easier to setup.

* test: add maven profile to run slow tests

Configures a profile to execute only the slow tests. It will be invoked
in the nightly build.

* chore: configure nightly build for slow tests

Sets up the kokoro configuration for running slow tests nightly

* chore: ignore templated build.sh

* chore: fix linting violations

* test: reverts tracertest category removal

* chore: address PR comments

Author: thiagotnunes

.kokoro/build.sh

@@ -69,6 +69,18 @@ integration)
       verify
     RETURN_CODE=$?
     ;;
+slowtests)
+  mvn -B ${INTEGRATION_TEST_ARGS} \
+    -ntp \
+    -Pslow-tests \
+    -DskipITs=false \
+    -DtrimStackTrace=false \
+    -Dclirr.skip=true \
+    -Denforcer.skip=true \
+    -fae \
+    verify
+  RETURN_CODE=$?
+  ;;
 samples)
     SAMPLES_DIR=samples
     # only run ITs in snapshot/ on presubmit PRs. run ITs in all 3 samples/ subdirectories otherwise.

.kokoro/nightly/slow-tests.cfg

@@ -0,0 +1,38 @@
+# Format: //devtools/kokoro/config/proto/build.proto
+# Runs the slow tests only in the nightly build
+
+# Configure the docker image for kokoro-trampoline.
+env_vars: {
+  key: "TRAMPOLINE_IMAGE"
+  value: "gcr.io/cloud-devrel-kokoro-resources/java8"
+}
+
+env_vars: {
+  key: "JOB_TYPE"
+  value: "slowtests"
+}
+# TODO: remove this after we've migrated all tests and scripts
+env_vars: {
+  key: "GCLOUD_PROJECT"
+  value: "gcloud-devel"
+}
+
+env_vars: {
+  key: "GOOGLE_CLOUD_PROJECT"
+  value: "gcloud-devel"
+}
+
+env_vars: {
+  key: "ENABLE_FLAKYBOT"
+  value: "true"
+}
+
+env_vars: {
+  key: "GOOGLE_APPLICATION_CREDENTIALS"
+  value: "secret_manager/java-it-service-account"
+}
+
+env_vars: {
+  key: "SECRET_MANAGER_KEYS"
+  value: "java-it-service-account"
+}

google-cloud-spanner/pom.xml

@@ -47,9 +47,10 @@
         </configuration>
         <executions>
           <execution>
+            <!-- Executes unit tests -->
             <id>default-test</id>
             <configuration>
-              <excludedGroups>com.google.cloud.spanner.TracerTest,com.google.cloud.spanner.IntegrationTest</excludedGroups>
+              <excludedGroups>com.google.cloud.spanner.categories.TracerTest,com.google.cloud.spanner.categories.IntegrationTest</excludedGroups>
             </configuration>
           </execution>
           <execution>
@@ -58,7 +59,7 @@
               <goal>test</goal>
             </goals>
             <configuration>
-              <groups>com.google.cloud.spanner.TracerTest</groups>
+              <groups>com.google.cloud.spanner.categories.TracerTest</groups>
             </configuration>
           </execution>
         </executions>
@@ -76,21 +77,21 @@
           <forkedProcessTimeoutInSeconds>3000</forkedProcessTimeoutInSeconds>
         </configuration>
         <executions>
+          <!-- Executes serial integration tests -->
           <execution>
             <id>default</id>
             <configuration>
-              <groups>com.google.cloud.spanner.IntegrationTest</groups>
-              <excludedGroups>com.google.cloud.spanner.FlakyTest,com.google.cloud.spanner.TracerTest,com.google.cloud.spanner.ParallelIntegrationTest</excludedGroups>
+              <groups>com.google.cloud.spanner.categories.SerialIntegrationTest</groups>
             </configuration>
           </execution>
+          <!-- Executes parallel integration tests -->
           <execution>
             <id>parallel-integration-test</id>
             <goals>
               <goal>integration-test</goal>
             </goals>
             <configuration>
-              <groups>com.google.cloud.spanner.ParallelIntegrationTest</groups>
-              <excludedGroups>com.google.cloud.spanner.FlakyTest,com.google.cloud.spanner.TracerTest,com.google.cloud.spanner.IntegrationTest</excludedGroups>
+              <groups>com.google.cloud.spanner.categories.ParallelIntegrationTest</groups>
               <forkCount>8</forkCount>
               <reuseForks>true</reuseForks>
             </configuration>
@@ -365,6 +366,47 @@
         </plugins>
       </build>
     </profile>
+    <profile>
+      <id>slow-tests</id>
+      <build>
+          <plugins>
+            <plugin>
+              <groupId>org.apache.maven.plugins</groupId>
+              <artifactId>maven-surefire-plugin</artifactId>
+              <executions>
+                <execution>
+                  <id>default-test</id>
+                  <configuration>
+                    <groups>com.google.cloud.spanner.categories.SlowTest</groups>
+                  </configuration>
+                </execution>
+              </executions>
+            </plugin>
+            <plugin>
+              <groupId>org.apache.maven.plugins</groupId>
+              <artifactId>maven-failsafe-plugin</artifactId>
+              <executions>
+                <execution>
+                  <id>default</id>
+                  <configuration>
+                    <groups>com.google.cloud.spanner.categories.SlowTest</groups>
+                  </configuration>
+                </execution>
+                <!-- Overrides default configuration to skip this step -->
+                <execution>
+                  <id>parallel-integration-test</id>
+                  <goals>
+                    <goal>integration-test</goal>
+                  </goals>
+                  <configuration>
+                    <skipTests>true</skipTests>
+                  </configuration>
+                </execution>
+              </executions>
+            </plugin>
+          </plugins>
+      </build>
+    </profile>
     <profile>
       <id>spanner-directpath-it</id>
       <build>

google-cloud-spanner/src/test/java/com/google/cloud/spanner/ITSessionPoolIntegrationTest.java

@@ -20,6 +20,7 @@
 
 import com.google.cloud.grpc.GrpcTransportOptions.ExecutorFactory;
 import com.google.cloud.spanner.SessionPool.PooledSessionFuture;
+import com.google.cloud.spanner.categories.SerialIntegrationTest;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.concurrent.CountDownLatch;
@@ -40,7 +41,7 @@
  * <p>See also {@code it/WriteIntegrationTest}, which provides coverage of writing and reading back
  * all Cloud Spanner types.
  */
-@Category(IntegrationTest.class)
+@Category(SerialIntegrationTest.class)
 @RunWith(JUnit4.class)
 public class ITSessionPoolIntegrationTest {
   @ClassRule public static IntegrationTestEnv env = new IntegrationTestEnv();

google-cloud-spanner/src/test/java/com/google/cloud/spanner/SpanTest.java

@@ -25,6 +25,7 @@
 import com.google.cloud.NoCredentials;
 import com.google.cloud.spanner.MockSpannerServiceImpl.SimulatedExecutionTime;
 import com.google.cloud.spanner.MockSpannerServiceImpl.StatementResult;
+import com.google.cloud.spanner.categories.TracerTest;
 import com.google.protobuf.ListValue;
 import com.google.spanner.v1.ResultSetMetadata;
 import com.google.spanner.v1.StructType;
@@ -48,8 +49,8 @@
 import org.junit.runners.JUnit4;
 import org.threeten.bp.Duration;
 
-@RunWith(JUnit4.class)
 @Category(TracerTest.class)
+@RunWith(JUnit4.class)
 public class SpanTest {
   private static final String TEST_PROJECT = "my-project";
   private static final String TEST_INSTANCE = "my-instance";

google-cloud-spanner/src/test/java/com/google/cloud/spanner/IntegrationTest.java renamed to google-cloud-spanner/src/test/java/com/google/cloud/spanner/categories/IntegrationTest.java

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package com.google.cloud.spanner;
+package com.google.cloud.spanner.categories;
 
 /** Integration Test interface. */
 public interface IntegrationTest {}

google-cloud-spanner/src/test/java/com/google/cloud/spanner/ParallelIntegrationTest.java renamed to google-cloud-spanner/src/test/java/com/google/cloud/spanner/categories/ParallelIntegrationTest.java

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package com.google.cloud.spanner;
+package com.google.cloud.spanner.categories;
 
 /** Parallel Integration Test interface. */
-public interface ParallelIntegrationTest {}
+public interface ParallelIntegrationTest extends IntegrationTest {}

google-cloud-spanner/src/test/java/com/google/cloud/spanner/FlakyTest.java renamed to google-cloud-spanner/src/test/java/com/google/cloud/spanner/categories/SerialIntegrationTest.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2017 Google LLC
+ * Copyright 2021 Google LLC
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -14,12 +14,7 @@
  * limitations under the License.
  */
 
-package com.google.cloud.spanner;
+package com.google.cloud.spanner.categories;
 
-/**
- * Annotation for JUnit {@link org.junit.experimental.categories.Category} that indicates a test is
- * flaky. These will be excluded from integration tests. Use this annotation sparingly: typically it
- * should only be used for a test where the flakiness is dependent on a fix in a module dependency
- * (for example, grpc-java) and cannot be addressed locally.
- */
-public interface FlakyTest {}
+/** Serial Integration Test interface. */
+public interface SerialIntegrationTest {}

google-cloud-spanner/src/test/java/com/google/cloud/spanner/categories/SlowTest.java

@@ -0,0 +1,20 @@
+/*
+ * Copyright 2021 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.cloud.spanner.categories;
+
+/** Category of slow tests, to be run on the nightly build * */
+public interface SlowTest extends IntegrationTest {}

google-cloud-spanner/src/test/java/com/google/cloud/spanner/TracerTest.java renamed to google-cloud-spanner/src/test/java/com/google/cloud/spanner/categories/TracerTest.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2020 Google LLC
+ * Copyright 2021 Google LLC
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -14,13 +14,12 @@
  * limitations under the License.
  */
 
-package com.google.cloud.spanner;
-
-import org.junit.experimental.categories.Category;
+package com.google.cloud.spanner.categories;
 
 /**
- * Tests marked with this {@link Category} will be executed in a separate execution with the
- * maven-surefire plugin. The tests will be excluded from execution with the maven-failsafe plugin.
+ * Tests marked with this {@link org.junit.experimental.categories.Category} will be executed in a
+ * separate execution with the maven-surefire plugin. The tests will be excluded from execution with
+ * the maven-failsafe plugin.
  *
  * <p>Separate execution prevents the injection of any custom tracing configuration from interfering
  * with other tests, as most tracing configuration is stored in static final variables.

google-cloud-spanner/src/test/java/com/google/cloud/spanner/connection/it/ITAsyncTransactionRetryTest.java

@@ -33,11 +33,11 @@
 import com.google.cloud.spanner.KeySet;
 import com.google.cloud.spanner.Mutation;
 import com.google.cloud.spanner.Options;
-import com.google.cloud.spanner.ParallelIntegrationTest;
 import com.google.cloud.spanner.ResultSet;
 import com.google.cloud.spanner.SpannerExceptionFactory;
 import com.google.cloud.spanner.Statement;
 import com.google.cloud.spanner.Struct;
+import com.google.cloud.spanner.categories.ParallelIntegrationTest;
 import com.google.cloud.spanner.connection.Connection;
 import com.google.cloud.spanner.connection.ITAbstractSpannerTest;
 import com.google.cloud.spanner.connection.TransactionRetryListener;

google-cloud-spanner/src/test/java/com/google/cloud/spanner/connection/it/ITBulkConnectionTest.java

@@ -20,9 +20,9 @@
 import static org.hamcrest.CoreMatchers.notNullValue;
 import static org.hamcrest.MatcherAssert.assertThat;
 
-import com.google.cloud.spanner.IntegrationTest;
 import com.google.cloud.spanner.ResultSet;
 import com.google.cloud.spanner.Statement;
+import com.google.cloud.spanner.categories.SerialIntegrationTest;
 import com.google.cloud.spanner.connection.ITAbstractSpannerTest;
 import java.util.ArrayList;
 import java.util.List;
@@ -39,7 +39,7 @@
  * parallel with other tests, as it tries to close all active connections, and should not try to
  * close connections of other integration tests.
  */
-@Category(IntegrationTest.class)
+@Category(SerialIntegrationTest.class)
 @RunWith(JUnit4.class)
 public class ITBulkConnectionTest extends ITAbstractSpannerTest {
   private static final int NUMBER_OF_TEST_CONNECTIONS = 250;

google-cloud-spanner/src/test/java/com/google/cloud/spanner/connection/it/ITCommitResponseTest.java

@@ -25,9 +25,9 @@
 
 import com.google.cloud.spanner.KeySet;
 import com.google.cloud.spanner.Mutation;
-import com.google.cloud.spanner.ParallelIntegrationTest;
 import com.google.cloud.spanner.ResultSet;
 import com.google.cloud.spanner.Statement;
+import com.google.cloud.spanner.categories.ParallelIntegrationTest;
 import com.google.cloud.spanner.connection.ITAbstractSpannerTest;
 import org.junit.Before;
 import org.junit.Test;

google-cloud-spanner/src/test/java/com/google/cloud/spanner/connection/it/ITDdlTest.java

@@ -16,7 +16,7 @@
 
 package com.google.cloud.spanner.connection.it;
 
-import com.google.cloud.spanner.ParallelIntegrationTest;
+import com.google.cloud.spanner.categories.ParallelIntegrationTest;
 import com.google.cloud.spanner.connection.ITAbstractSpannerTest;
 import com.google.cloud.spanner.connection.SqlScriptVerifier;
 import org.junit.Test;

google-cloud-spanner/src/test/java/com/google/cloud/spanner/connection/it/ITReadOnlySpannerTest.java

@@ -28,11 +28,11 @@
 import com.google.cloud.spanner.ErrorCode;
 import com.google.cloud.spanner.Mutation;
 import com.google.cloud.spanner.Options;
-import com.google.cloud.spanner.ParallelIntegrationTest;
 import com.google.cloud.spanner.ReadContext.QueryAnalyzeMode;
 import com.google.cloud.spanner.ResultSet;
 import com.google.cloud.spanner.SpannerException;
 import com.google.cloud.spanner.Statement;
+import com.google.cloud.spanner.categories.ParallelIntegrationTest;
 import com.google.cloud.spanner.connection.ITAbstractSpannerTest;
 import com.google.cloud.spanner.connection.SqlScriptVerifier;
 import java.math.BigInteger;

google-cloud-spanner/src/test/java/com/google/cloud/spanner/connection/it/ITReadWriteAutocommitSpannerTest.java

@@ -24,11 +24,11 @@
 
 import com.google.cloud.spanner.ErrorCode;
 import com.google.cloud.spanner.Mutation;
-import com.google.cloud.spanner.ParallelIntegrationTest;
 import com.google.cloud.spanner.ResultSet;
 import com.google.cloud.spanner.SpannerBatchUpdateException;
 import com.google.cloud.spanner.SpannerException;
 import com.google.cloud.spanner.Statement;
+import com.google.cloud.spanner.categories.ParallelIntegrationTest;
 import com.google.cloud.spanner.connection.ITAbstractSpannerTest;
 import com.google.cloud.spanner.connection.SqlScriptVerifier;
 import java.util.Arrays;

google-cloud-spanner/src/test/java/com/google/cloud/spanner/connection/it/ITSqlMusicScriptTest.java

@@ -24,9 +24,9 @@
 
 import com.google.cloud.spanner.AbortedDueToConcurrentModificationException;
 import com.google.cloud.spanner.Mutation;
-import com.google.cloud.spanner.ParallelIntegrationTest;
 import com.google.cloud.spanner.ResultSet;
 import com.google.cloud.spanner.Statement;
+import com.google.cloud.spanner.categories.ParallelIntegrationTest;
 import com.google.cloud.spanner.connection.AbstractSqlScriptVerifier.GenericConnection;
 import com.google.cloud.spanner.connection.ITAbstractSpannerTest;
 import com.google.cloud.spanner.connection.SqlScriptVerifier;

google-cloud-spanner/src/test/java/com/google/cloud/spanner/connection/it/ITSqlScriptTest.java

@@ -19,8 +19,8 @@
 import static com.google.cloud.spanner.testing.EmulatorSpannerHelper.isUsingEmulator;
 
 import com.google.cloud.spanner.ErrorCode;
-import com.google.cloud.spanner.ParallelIntegrationTest;
 import com.google.cloud.spanner.SpannerException;
+import com.google.cloud.spanner.categories.ParallelIntegrationTest;
 import com.google.cloud.spanner.connection.ITAbstractSpannerTest;
 import com.google.cloud.spanner.connection.SqlScriptVerifier;
 import com.google.cloud.spanner.connection.SqlScriptVerifier.SpannerGenericConnection;

google-cloud-spanner/src/test/java/com/google/cloud/spanner/connection/it/ITTransactionModeTest.java

@@ -25,10 +25,10 @@
 import com.google.cloud.spanner.ErrorCode;
 import com.google.cloud.spanner.Key;
 import com.google.cloud.spanner.Mutation;
-import com.google.cloud.spanner.ParallelIntegrationTest;
 import com.google.cloud.spanner.ResultSet;
 import com.google.cloud.spanner.SpannerException;
 import com.google.cloud.spanner.Statement;
+import com.google.cloud.spanner.categories.ParallelIntegrationTest;
 import com.google.cloud.spanner.connection.ITAbstractSpannerTest;
 import com.google.cloud.spanner.connection.SqlScriptVerifier;
 import java.util.Arrays;

google-cloud-spanner/src/test/java/com/google/cloud/spanner/connection/it/ITTransactionRetryTest.java

@@ -27,10 +27,10 @@
 import com.google.cloud.spanner.AbortedException;
 import com.google.cloud.spanner.KeySet;
 import com.google.cloud.spanner.Mutation;
-import com.google.cloud.spanner.ParallelIntegrationTest;
 import com.google.cloud.spanner.ResultSet;
 import com.google.cloud.spanner.SpannerException;
 import com.google.cloud.spanner.Statement;
+import com.google.cloud.spanner.categories.ParallelIntegrationTest;
 import com.google.cloud.spanner.connection.ITAbstractSpannerTest;
 import com.google.cloud.spanner.connection.TransactionRetryListener;
 import java.sql.Connection;