test: enable S.7 retry conformance tests (#1839)

Update method mappings to properly exclude retry conformance tests for Bucket#create.

Update TestRetryConformance to lazily allocate and fill a temporary file when necessary.

Author: BenWhitehead

google-cloud-storage/src/test/java/com/google/cloud/storage/conformance/retry/ITRetryConformanceTest.java

@@ -30,17 +30,16 @@
 import com.google.cloud.conformance.storage.v1.RetryTests;
 import com.google.cloud.storage.CIUtils;
 import com.google.cloud.storage.Storage;
-import com.google.cloud.storage.TransportCompatibility.Transport;
 import com.google.cloud.storage.conformance.retry.Functions.CtxFunction;
 import com.google.cloud.storage.conformance.retry.ITRetryConformanceTest.RetryConformanceParameterProvider;
 import com.google.cloud.storage.it.runner.StorageITRunner;
 import com.google.cloud.storage.it.runner.annotations.Backend;
-import com.google.cloud.storage.it.runner.annotations.CrossRun;
 import com.google.cloud.storage.it.runner.annotations.Inject;
 import com.google.cloud.storage.it.runner.annotations.ParallelFriendly;
 import com.google.cloud.storage.it.runner.annotations.Parameterized;
 import com.google.cloud.storage.it.runner.annotations.Parameterized.Parameter;
 import com.google.cloud.storage.it.runner.annotations.Parameterized.ParametersProvider;
+import com.google.cloud.storage.it.runner.annotations.SingleBackend;
 import com.google.cloud.storage.it.runner.registry.TestBench;
 import com.google.common.base.Charsets;
 import com.google.common.base.Joiner;
@@ -78,7 +77,8 @@
  * RpcMethodMappings}.
  */
 @RunWith(StorageITRunner.class)
-@CrossRun(transports = Transport.HTTP, backends = Backend.TEST_BENCH)
+// @CrossRun(transports = Transport.HTTP, backends = Backend.TEST_BENCH)
+@SingleBackend(Backend.TEST_BENCH)
 @Parameterized(RetryConformanceParameterProvider.class)
 @ParallelFriendly
 public class ITRetryConformanceTest {
@@ -127,11 +127,17 @@ public void tearDown() throws Throwable {
   @Test
   public void test() throws Throwable {
     LOGGER.fine("Running test...");
-    ctx =
-        getReplaceStorageInObjectsFromCtx()
-            .andThen(mapping.getTest())
-            .apply(ctx, testRetryConformance)
-            .leftMap(s -> nonTestStorage);
+    try {
+      ctx =
+          getReplaceStorageInObjectsFromCtx()
+              .andThen(mapping.getTest())
+              .apply(ctx, testRetryConformance)
+              .leftMap(s -> nonTestStorage);
+      retryTestFixture.succeeded(null);
+    } catch (Throwable e) {
+      retryTestFixture.failed(e, null);
+      throw e;
+    }
     LOGGER.fine("Running test complete");
   }
 
@@ -154,14 +160,7 @@ public ImmutableList<?> parameters() {
               .setMappings(new RpcMethodMappings())
               .setProjectId("conformance-tests")
               .setHost(testBench.getBaseUri().replaceAll("https?://", ""))
-              .setTestAllowFilter(
-                  RetryTestCaseResolver.includeAll()
-                      // .and(RetryTestCaseResolver.specificMappings(44, 45))
-                      .and(
-                          (m, trc) ->
-                              trc.getScenarioId()
-                                  != 7) // Temporarily exclude resumable upload scenarios
-                  )
+              .setTestAllowFilter(RetryTestCaseResolver.includeAll())
               .build();
 
       List<RetryTestCase> retryTestCases;
@@ -384,6 +383,15 @@ static BiPredicate<RpcMethod, TestRetryConformance> specificMappings(int... mapp
       return (m, c) -> set.contains(c.getMappingId());
     }
 
+    static BiPredicate<RpcMethod, TestRetryConformance> instructionsAre(String... instructions) {
+      return (m, trc) ->
+          trc.getInstruction().getInstructionsList().equals(ImmutableList.copyOf(instructions));
+    }
+
+    static BiPredicate<RpcMethod, TestRetryConformance> scenarioIdIs(int scenarioId) {
+      return (m, trc) -> trc.getScenarioId() == scenarioId;
+    }
+
     static final class Builder {
       private String retryTestsJsonResourcePath;
       private RpcMethodMappings mappings;

google-cloud-storage/src/test/java/com/google/cloud/storage/conformance/retry/RetryTestFixture.java

@@ -120,6 +120,11 @@ protected void succeeded(Description description) {
     testSuccess = true;
   }
 
+  @Override
+  protected void failed(Throwable e, Description description) {
+    super.failed(e, description);
+  }
+
   @Override
   protected void skipped(AssumptionViolatedException e, Description description) {
     testSkipped = true;

google-cloud-storage/src/test/java/com/google/cloud/storage/conformance/retry/RpcMethodMappings.java

@@ -1780,7 +1780,8 @@ private static void insert(ArrayList<RpcMethodMapping> a) {
                 .build());
         a.add(
             RpcMethodMapping.newBuilder(118, objects.insert)
-                .withApplicable(TestRetryConformance::isPreconditionsProvided)
+                .withApplicable(
+                    and(TestRetryConformance::isPreconditionsProvided, not(groupIsResumableUpload)))
                 .withTest(
                     (ctx, c) ->
                         ctx.map(
@@ -1795,7 +1796,8 @@ private static void insert(ArrayList<RpcMethodMapping> a) {
                 .build());
         a.add(
             RpcMethodMapping.newBuilder(119, objects.insert)
-                .withApplicable(TestRetryConformance::isPreconditionsProvided)
+                .withApplicable(
+                    and(TestRetryConformance::isPreconditionsProvided, not(groupIsResumableUpload)))
                 .withTest(
                     (ctx, c) ->
                         ctx.map(
@@ -1806,7 +1808,7 @@ private static void insert(ArrayList<RpcMethodMapping> a) {
                                         .create(
                                             c.getObjectName(),
                                             c.getHelloWorldUtf8Bytes(),
-                                            "text/plain);charset=utf-8",
+                                            "text/plain;charset=utf-8",
                                             Bucket.BlobTargetOption.doesNotExist()))))
                 .build());
         a.add(

google-cloud-storage/src/test/java/com/google/cloud/storage/conformance/retry/TestRetryConformance.java

@@ -16,31 +16,33 @@
 
 package com.google.cloud.storage.conformance.retry;
 
+import static com.google.common.truth.Truth.assertThat;
 import static java.util.Objects.requireNonNull;
 import static org.junit.Assert.assertNotNull;
 
 import com.google.auth.ServiceAccountSigner;
 import com.google.auth.oauth2.ServiceAccountCredentials;
 import com.google.cloud.conformance.storage.v1.InstructionList;
 import com.google.cloud.conformance.storage.v1.Method;
+import com.google.cloud.storage.DataGenerator;
 import com.google.common.base.Joiner;
 import com.google.common.base.Suppliers;
+import com.google.common.io.ByteStreams;
 import com.google.errorprone.annotations.Immutable;
+import java.io.ByteArrayInputStream;
+import java.io.File;
+import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
-import java.net.URISyntaxException;
-import java.net.URL;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Path;
-import java.nio.file.Paths;
 import java.time.Clock;
 import java.time.Instant;
 import java.time.ZoneId;
 import java.time.ZoneOffset;
 import java.time.format.DateTimeFormatter;
 import java.util.function.Supplier;
 import java.util.stream.Collectors;
-import java.util.stream.IntStream;
 
 /**
  * An individual resolved test case correlating config from {@link
@@ -72,9 +74,8 @@ final class TestRetryConformance {
   private final String topicName;
 
   private final Supplier<byte[]> lazyHelloWorldUtf8Bytes;
-  private final Path helloWorldFilePath = resolvePathForResource();
-  private final ServiceAccountCredentials serviceAccountCredentials =
-      resolveServiceAccountCredentials();
+  private final Supplier<Path> helloWorldFilePath;
+  private final ServiceAccountCredentials serviceAccountCredentials;
 
   private final String host;
 
@@ -137,33 +138,14 @@ final class TestRetryConformance {
         String.format(
             "%s_s%03d-%s-m%03d_top1",
             BASE_ID, scenarioId, instructionsString.toLowerCase(), mappingId);
-    lazyHelloWorldUtf8Bytes =
+    this.lazyHelloWorldUtf8Bytes =
         Suppliers.memoize(
             () -> {
               // define a lazy supplier for bytes.
-              // Not all tests need data for an object, though some tests - resumable upload - needs
-              // more than 8MiB.
-              // We want to avoid allocating 8.1MiB for each test unnecessarily, especially since we
-              // instantiate all permuted test cases. ~1000 * 8.1MiB ~~ > 8GiB.
-              String helloWorld = "Hello, World!";
-              int baseDataSize;
-              switch (method.getName()) {
-                case "storage.objects.insert":
-                  baseDataSize = _8MiB + 1;
-                  break;
-                case "storage.objects.get":
-                  baseDataSize = _512KiB;
-                  break;
-                default:
-                  baseDataSize = helloWorld.length();
-                  break;
-              }
-              int endInclusive = (baseDataSize / helloWorld.length());
-              return IntStream.rangeClosed(1, endInclusive)
-                  .mapToObj(i -> helloWorld)
-                  .collect(Collectors.joining())
-                  .getBytes(StandardCharsets.UTF_8);
+              return genBytes(method);
             });
+    this.helloWorldFilePath = resolvePathForResource(objectName, method);
+    this.serviceAccountCredentials = resolveServiceAccountCredentials();
   }
 
   public String getProjectId() {
@@ -195,7 +177,7 @@ public byte[] getHelloWorldUtf8Bytes() {
   }
 
   public Path getHelloWorldFilePath() {
-    return helloWorldFilePath;
+    return helloWorldFilePath.get();
   }
 
   public int getScenarioId() {
@@ -238,15 +220,24 @@ public String toString() {
     return getTestName();
   }
 
-  private static Path resolvePathForResource() {
-    ClassLoader cl = Thread.currentThread().getContextClassLoader();
-    URL url = cl.getResource("com/google/cloud/storage/conformance/retry/hello-world.txt");
-    assertNotNull(url);
-    try {
-      return Paths.get(url.toURI());
-    } catch (URISyntaxException e) {
-      throw new RuntimeException(e);
-    }
+  private static Supplier<Path> resolvePathForResource(String objectName, Method method) {
+    return () -> {
+      try {
+        File tempFile = File.createTempFile(objectName, "");
+        tempFile.deleteOnExit();
+
+        byte[] bytes = genBytes(method);
+        try (ByteArrayInputStream in = new ByteArrayInputStream(bytes);
+            FileOutputStream out = new FileOutputStream(tempFile)) {
+          long copy = ByteStreams.copy(in, out);
+          assertThat(copy).isEqualTo(bytes.length);
+        }
+
+        return tempFile.toPath();
+      } catch (IOException e) {
+        throw new RuntimeException(e);
+      }
+    };
   }
 
   private static ServiceAccountCredentials resolveServiceAccountCredentials() {
@@ -265,4 +256,19 @@ private static ServiceAccountCredentials resolveServiceAccountCredentials() {
   public String getTopicName() {
     return topicName;
   }
+
+  private static byte[] genBytes(Method method) {
+    // Not all tests need data for an object, though some tests - resumable upload - needs
+    // more than 8MiB.
+    // We want to avoid allocating 8.1MiB for each test unnecessarily, especially since we
+    // instantiate all permuted test cases. ~1000 * 8.1MiB ~~ > 8GiB.
+    switch (method.getName()) {
+      case "storage.objects.insert":
+        return DataGenerator.base64Characters().genBytes(_8MiB * 2 + _512KiB);
+      case "storage.objects.get":
+        return DataGenerator.base64Characters().genBytes(_512KiB);
+      default:
+        return "Hello, World!".getBytes(StandardCharsets.UTF_8);
+    }
+  }
 }