Skip to content

Commit 9aeaa9c

Browse files
author
Bennett Lynch
authored
[Hackathon] Add method to S3Client: deleteBucketAndAllContents (#3070)
* [Hackathon] Add method to S3Client: deleteBucketAndAllContents
1 parent 8364df3 commit 9aeaa9c

File tree

10 files changed

+253
-197
lines changed

10 files changed

+253
-197
lines changed

‎core/annotations/src/main/java/software/amazon/awssdk/annotations/SdkExtensionMethod.java

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -20,9 +20,9 @@
2020

2121
/**
2222
* {@link SdkExtensionMethod} indicates a method that has been explicitly added to a service client interface on behalf of the
23-
* SDK. While most service clients are automatically and fully generated based upon the corresponding service's API model, the SDK
24-
* may sometimes choose to extend a service's logical API in order to provide improved abstractions or convenience functions. As
25-
* such, {@link SdkExtensionMethod} implementations may invoke one or more service requests to fulfil their behavior.
23+
* SDK. While most service clients are automatically and fully generated from the corresponding service's API model, the SDK may
24+
* sometimes choose to extend a service's logical API in order to provide improved abstractions or convenience functions. {@link
25+
* SdkExtensionMethod} implementations may invoke one or more service requests to fulfil their behavior.
2626
*/
2727
@Target(ElementType.METHOD)
2828
@SdkProtectedApi

‎docs/LaunchChangelog.md

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -374,8 +374,8 @@ The S3 client in 2.0 is drastically different from the client in 1.11, because i
374374
| `deleteObjects` | `deleteObjects` |
375375
| `deleteVersion` | `deleteObject` |
376376
| `disableRequesterPays` | `putBucketRequestPayment` |
377-
| `doesBucketExist` | `headBucket` |
378-
| `doesBucketExistV2` | `headBucket` |
377+
| `doesBucketExist` | `doesBucketExist` or `headBucket` |
378+
| `doesBucketExistV2` | `doesBucketExist` or `headBucket` |
379379
| `doesObjectExist` | `headObject` |
380380
| `enableRequesterPays` | `putBucketRequestPayment` |
381381
| `generatePresignedUrl` | ~~Not Supported~~ [S3Presigner](https://docs.aws.amazon.com/sdk-for-java/latest/developer-guide/examples-s3-presign.html) |

‎services-custom/s3-transfer-manager/src/it/java/software/amazon/awssdk/transfer/s3/S3IntegrationTestBase.java

Lines changed: 1 addition & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -27,15 +27,7 @@
2727
import software.amazon.awssdk.services.s3.model.BucketLocationConstraint;
2828
import software.amazon.awssdk.services.s3.model.CreateBucketConfiguration;
2929
import software.amazon.awssdk.services.s3.model.CreateBucketRequest;
30-
import software.amazon.awssdk.services.s3.model.DeleteBucketRequest;
31-
import software.amazon.awssdk.services.s3.model.DeleteObjectRequest;
32-
import software.amazon.awssdk.services.s3.model.ListObjectVersionsRequest;
33-
import software.amazon.awssdk.services.s3.model.ListObjectVersionsResponse;
34-
import software.amazon.awssdk.services.s3.model.ListObjectsRequest;
35-
import software.amazon.awssdk.services.s3.model.ListObjectsResponse;
36-
import software.amazon.awssdk.services.s3.model.ObjectVersion;
3730
import software.amazon.awssdk.services.s3.model.S3Exception;
38-
import software.amazon.awssdk.services.s3.model.S3Object;
3931
import software.amazon.awssdk.testutils.service.AwsTestBase;
4032

4133
/**
@@ -116,36 +108,7 @@ private static void createBucket(String bucketName, int retryCount) {
116108

117109
protected static void deleteBucketAndAllContents(String bucketName) {
118110
System.out.println("Deleting S3 bucket: " + bucketName);
119-
ListObjectsResponse response = s3.listObjects(ListObjectsRequest.builder().bucket(bucketName).build());
120-
121-
while (true) {
122-
if (response.contents() == null) {
123-
break;
124-
}
125-
for (S3Object objectSummary : response.contents()) {
126-
s3.deleteObject(DeleteObjectRequest.builder().bucket(bucketName).key(objectSummary.key()).build());
127-
}
128-
129-
if (response.isTruncated()) {
130-
response = s3.listObjects(ListObjectsRequest.builder().marker(response.nextMarker()).build());
131-
} else {
132-
break;
133-
}
134-
}
135-
136-
ListObjectVersionsResponse versionsResponse = s3
137-
.listObjectVersions(ListObjectVersionsRequest.builder().bucket(bucketName).build());
138-
if (versionsResponse.versions() != null) {
139-
for (ObjectVersion s : versionsResponse.versions()) {
140-
s3.deleteObject(DeleteObjectRequest.builder()
141-
.bucket(bucketName)
142-
.key(s.key())
143-
.versionId(s.versionId())
144-
.build());
145-
}
146-
}
147-
148-
s3.deleteBucket(DeleteBucketRequest.builder().bucket(bucketName).build());
111+
s3.deleteBucketAndAllContents(bucketName);
149112
}
150113

151114
}

‎services/cloudtrail/src/it/java/software/amazon/awssdk/services/cloudtrail/CloudTrailIntegrationTest.java

Lines changed: 1 addition & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,6 @@
2222
import static software.amazon.awssdk.testutils.service.S3BucketUtils.temporaryBucketName;
2323

2424
import java.io.IOException;
25-
import java.util.Iterator;
2625
import org.apache.commons.io.IOUtils;
2726
import org.junit.AfterClass;
2827
import org.junit.BeforeClass;
@@ -39,15 +38,7 @@
3938
import software.amazon.awssdk.services.cloudtrail.model.UpdateTrailResponse;
4039
import software.amazon.awssdk.services.s3.model.CreateBucketConfiguration;
4140
import software.amazon.awssdk.services.s3.model.CreateBucketRequest;
42-
import software.amazon.awssdk.services.s3.model.DeleteBucketRequest;
43-
import software.amazon.awssdk.services.s3.model.DeleteObjectRequest;
44-
import software.amazon.awssdk.services.s3.model.ListObjectVersionsRequest;
45-
import software.amazon.awssdk.services.s3.model.ListObjectVersionsResponse;
46-
import software.amazon.awssdk.services.s3.model.ListObjectsRequest;
47-
import software.amazon.awssdk.services.s3.model.ListObjectsResponse;
48-
import software.amazon.awssdk.services.s3.model.ObjectVersion;
4941
import software.amazon.awssdk.services.s3.model.PutBucketPolicyRequest;
50-
import software.amazon.awssdk.services.s3.model.S3Object;
5142

5243
public class CloudTrailIntegrationTest extends IntegrationTestBase {
5344
private static final String BUCKET_NAME = temporaryBucketName("aws-java-cloudtrail-integ");
@@ -84,38 +75,7 @@ public static void tearDown() {
8475

8576
public static void deleteBucketAndAllContents(String bucketName) {
8677
System.out.println("Deleting S3 bucket: " + bucketName);
87-
ListObjectsResponse response = s3.listObjects(ListObjectsRequest.builder().bucket(bucketName).build());
88-
89-
while (true) {
90-
if (response.contents() == null) {
91-
break;
92-
}
93-
for (Iterator<?> iterator = response.contents().iterator(); iterator
94-
.hasNext(); ) {
95-
S3Object objectSummary = (S3Object) iterator.next();
96-
s3.deleteObject(DeleteObjectRequest.builder().bucket(bucketName).key(objectSummary.key()).build());
97-
}
98-
99-
if (response.isTruncated()) {
100-
response = s3.listObjects(ListObjectsRequest.builder().marker(response.nextMarker()).build());
101-
} else {
102-
break;
103-
}
104-
}
105-
106-
ListObjectVersionsResponse versionsResponse = s3
107-
.listObjectVersions(ListObjectVersionsRequest.builder().bucket(bucketName).build());
108-
if (versionsResponse.versions() != null) {
109-
for (ObjectVersion s : versionsResponse.versions()) {
110-
s3.deleteObject(DeleteObjectRequest.builder()
111-
.bucket(bucketName)
112-
.key(s.key())
113-
.versionId(s.versionId())
114-
.build());
115-
}
116-
}
117-
118-
s3.deleteBucket(DeleteBucketRequest.builder().bucket(bucketName).build());
78+
s3.deleteBucketAndAllContents(bucketName);
11979
}
12080

12181
@Test

‎services/s3/src/it/java/software/amazon/awssdk/services/s3/DeleteBucketAndAllContentsIntegrationTest.java

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
/*
2+
* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License").
5+
* You may not use this file except in compliance with the License.
6+
* A copy of the License is located at
7+
*
8+
* http://aws.amazon.com/apache2.0
9+
*
10+
* or in the "license" file accompanying this file. This file is distributed
11+
* on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
12+
* express or implied. See the License for the specific language governing
13+
* permissions and limitations under the License.
14+
*/
15+
16+
package software.amazon.awssdk.services.s3;
17+
18+
import static org.hamcrest.MatcherAssert.assertThat;
19+
import static org.hamcrest.Matchers.is;
20+
import static software.amazon.awssdk.testutils.service.S3BucketUtils.temporaryBucketName;
21+
22+
import java.nio.charset.StandardCharsets;
23+
import java.util.UUID;
24+
import java.util.stream.IntStream;
25+
import org.junit.AfterClass;
26+
import org.junit.BeforeClass;
27+
import org.junit.Test;
28+
import software.amazon.awssdk.core.sync.RequestBody;
29+
import software.amazon.awssdk.services.s3.internal.extensions.DeleteBucketAndAllContents;
30+
import software.amazon.awssdk.services.s3.model.BucketVersioningStatus;
31+
32+
/**
33+
* Tests the {@link S3Client#deleteBucketAndAllContents(String)} extension method.
34+
*
35+
* @see DeleteBucketAndAllContents
36+
*/
37+
public class DeleteBucketAndAllContentsIntegrationTest extends S3IntegrationTestBase {
38+
39+
private static final String BUCKET = temporaryBucketName(DeleteBucketAndAllContentsIntegrationTest.class);
40+
41+
@BeforeClass
42+
public static void initializeTestData() {
43+
createBucket(BUCKET);
44+
s3.putBucketVersioning(r -> r
45+
.bucket(BUCKET)
46+
.versioningConfiguration(v -> v.status(BucketVersioningStatus.ENABLED)));
47+
}
48+
49+
@AfterClass
50+
public static void tearDown() {
51+
if (s3.doesBucketExist(BUCKET)) {
52+
deleteBucketAndAllContents(BUCKET);
53+
}
54+
}
55+
56+
@Test
57+
public void deleteBucketAndAllContents_WithVersioning_DeletesBucket() {
58+
// Populate the bucket with >1000 objects in order to exercise pagination behavior.
59+
int maxDeleteObjectsSize = 1_000;
60+
int numObjectsToCreate = maxDeleteObjectsSize + 50;
61+
IntStream.range(0, numObjectsToCreate).parallel().forEach(this::putObject);
62+
// Overwrite some keys to create multiple versions of objects
63+
int numKeysToOverwrite = 50;
64+
IntStream.range(0, numKeysToOverwrite).parallel().forEach(this::putObject);
65+
// Test deleting the bucket
66+
s3.deleteBucketAndAllContents(BUCKET);
67+
assertThat(s3.doesBucketExist(BUCKET), is(false));
68+
}
69+
70+
private void putObject(int i) {
71+
s3.putObject(r -> r.bucket(BUCKET)
72+
.key(String.valueOf(i)),
73+
RequestBody.fromString(UUID.randomUUID().toString(), StandardCharsets.UTF_8));
74+
}
75+
}

‎services/s3/src/it/java/software/amazon/awssdk/services/s3/utils/S3TestUtils.java

Lines changed: 1 addition & 53 deletions
Original file line numberDiff line numberDiff line change
@@ -17,23 +17,15 @@
1717

1818
import java.rmi.NoSuchObjectException;
1919
import java.util.ArrayList;
20-
import java.util.Iterator;
2120
import java.util.List;
2221
import java.util.Map;
2322
import java.util.UUID;
2423
import java.util.concurrent.ConcurrentHashMap;
2524
import software.amazon.awssdk.core.sync.RequestBody;
2625
import software.amazon.awssdk.services.s3.S3Client;
2726
import software.amazon.awssdk.services.s3.model.Bucket;
28-
import software.amazon.awssdk.services.s3.model.DeleteBucketRequest;
29-
import software.amazon.awssdk.services.s3.model.DeleteObjectRequest;
3027
import software.amazon.awssdk.services.s3.model.ExpirationStatus;
31-
import software.amazon.awssdk.services.s3.model.ListObjectVersionsRequest;
32-
import software.amazon.awssdk.services.s3.model.ListObjectVersionsResponse;
33-
import software.amazon.awssdk.services.s3.model.ListObjectsRequest;
34-
import software.amazon.awssdk.services.s3.model.ListObjectsResponse;
3528
import software.amazon.awssdk.services.s3.model.NoSuchBucketException;
36-
import software.amazon.awssdk.services.s3.model.S3Object;
3729
import software.amazon.awssdk.testutils.Waiter;
3830
import software.amazon.awssdk.utils.Logger;
3931

@@ -117,51 +109,7 @@ public static void runCleanupTasks(Class<?> testClass) {
117109
public static void deleteBucketAndAllContents(S3Client s3, String bucketName) {
118110
try {
119111
System.out.println("Deleting S3 bucket: " + bucketName);
120-
ListObjectsResponse response = Waiter.run(() -> s3.listObjects(r -> r.bucket(bucketName)))
121-
.ignoringException(NoSuchBucketException.class)
122-
.orFail();
123-
List<S3Object> objectListing = response.contents();
124-
125-
if (objectListing != null) {
126-
while (true) {
127-
for (Iterator<?> iterator = objectListing.iterator(); iterator.hasNext(); ) {
128-
S3Object objectSummary = (S3Object) iterator.next();
129-
s3.deleteObject(DeleteObjectRequest.builder().bucket(bucketName).key(objectSummary.key()).build());
130-
}
131-
132-
if (response.isTruncated()) {
133-
objectListing = s3.listObjects(ListObjectsRequest.builder()
134-
.bucket(bucketName)
135-
.marker(response.marker())
136-
.build())
137-
.contents();
138-
} else {
139-
break;
140-
}
141-
}
142-
}
143-
144-
145-
ListObjectVersionsResponse versions = s3
146-
.listObjectVersions(ListObjectVersionsRequest.builder().bucket(bucketName).build());
147-
148-
if (versions.deleteMarkers() != null) {
149-
versions.deleteMarkers().forEach(v -> s3.deleteObject(DeleteObjectRequest.builder()
150-
.versionId(v.versionId())
151-
.bucket(bucketName)
152-
.key(v.key())
153-
.build()));
154-
}
155-
156-
if (versions.versions() != null) {
157-
versions.versions().forEach(v -> s3.deleteObject(DeleteObjectRequest.builder()
158-
.versionId(v.versionId())
159-
.bucket(bucketName)
160-
.key(v.key())
161-
.build()));
162-
}
163-
164-
s3.deleteBucket(DeleteBucketRequest.builder().bucket(bucketName).build());
112+
s3.deleteBucketAndAllContents(bucketName);
165113
} catch (Exception e) {
166114
System.err.println("Failed to delete bucket: " + bucketName);
167115
e.printStackTrace();

‎services/s3/src/main/java/software/amazon/awssdk/services/s3/extensions/S3ClientSdkExtension.java

Lines changed: 33 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -17,8 +17,14 @@
1717

1818
import software.amazon.awssdk.annotations.SdkExtensionMethod;
1919
import software.amazon.awssdk.annotations.SdkPublicApi;
20+
import software.amazon.awssdk.core.exception.SdkException;
2021
import software.amazon.awssdk.services.s3.S3Client;
2122
import software.amazon.awssdk.services.s3.internal.extensions.DefaultS3ClientSdkExtension;
23+
import software.amazon.awssdk.services.s3.internal.extensions.DeleteBucketAndAllContents;
24+
import software.amazon.awssdk.services.s3.model.DeleteBucketRequest;
25+
import software.amazon.awssdk.services.s3.model.DeleteObjectsRequest;
26+
import software.amazon.awssdk.services.s3.model.ListObjectVersionsRequest;
27+
import software.amazon.awssdk.services.s3.model.ListObjectsV2Request;
2228
import software.amazon.awssdk.services.s3.model.S3Exception;
2329

2430
/**
@@ -32,12 +38,36 @@ public interface S3ClientSdkExtension {
3238
* not accessible (e.g., due to access being denied or the bucket existing in another region), an {@link S3Exception} will be
3339
* thrown.
3440
*
35-
* @param bucketName the bucket to check
41+
* @param bucket the bucket to check
3642
* @return true if the bucket exists and you have permission to access it; false if the bucket does not exist
3743
* @throws S3Exception if the bucket exists but is not accessible
3844
*/
3945
@SdkExtensionMethod
40-
default boolean doesBucketExist(String bucketName) {
41-
return new DefaultS3ClientSdkExtension((S3Client) this).doesBucketExist(bucketName);
46+
default boolean doesBucketExist(String bucket) {
47+
return new DefaultS3ClientSdkExtension((S3Client) this).doesBucketExist(bucket);
48+
}
49+
50+
/**
51+
* Permanently delete a bucket and all of its content, including any versioned objects and delete markers.
52+
* <p>
53+
* Internally this method will use the {@link S3Client#listObjectsV2(ListObjectsV2Request)} and {@link
54+
* S3Client#listObjectVersions(ListObjectVersionsRequest)} APIs to list a bucket's content, buffer keys that are eligible for
55+
* deletion into batches of 1000, and delete them in bulk with the {@link S3Client#deleteObjects(DeleteObjectsRequest)} API.
56+
* <p>
57+
* While this method is optimized to use batch APIs for both listing and deleting, it may not be suitable for buckets
58+
* containing a very large number of objects (i.e., hundreds of thousands). For such use cases, it is usually preferable to
59+
* either create an <i>S3 Lifecycle configuration</i> to delete the objects, or to leverage <i>S3 Batch Operations</i> to
60+
* perform large-scale deletes.
61+
* <p>
62+
* Note that this method does not attempt to protect against concurrent writes or modifications to a bucket. It will iterate
63+
* and delete the entire contents of the bucket once. If a new key is created during or after the iteration, then the final
64+
* call to {@link S3Client#deleteBucket(DeleteBucketRequest)} may fail.
65+
*
66+
* @param bucket the bucket to delete
67+
* @throws SdkException if an error occurs
68+
*/
69+
@SdkExtensionMethod
70+
default void deleteBucketAndAllContents(String bucket) {
71+
new DeleteBucketAndAllContents((S3Client) this).deleteBucketAndAllContents(bucket);
4272
}
4373
}

‎services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/extensions/DefaultS3ClientSdkExtension.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@ public DefaultS3ClientSdkExtension(S3Client s3) {
3232

3333
@Override
3434
public boolean doesBucketExist(String bucket) {
35-
Validate.notNull(bucket, "bucket");
35+
Validate.notEmpty(bucket, "bucket");
3636
try {
3737
s3.headBucket(r -> r.bucket(bucket));
3838
return true;

0 commit comments

Comments
 (0)