-
Notifications
You must be signed in to change notification settings - Fork 916
[Hackathon] Add method to S3Client: deleteBucketAndAllContents #3070
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,75 @@ | ||
/* | ||
* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. | ||
* | ||
* Licensed under the Apache License, Version 2.0 (the "License"). | ||
* You may not use this file except in compliance with the License. | ||
* A copy of the License is located at | ||
* | ||
* http://aws.amazon.com/apache2.0 | ||
* | ||
* or in the "license" file accompanying this file. This file is distributed | ||
* on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either | ||
* express or implied. See the License for the specific language governing | ||
* permissions and limitations under the License. | ||
*/ | ||
|
||
package software.amazon.awssdk.services.s3; | ||
|
||
import static org.hamcrest.MatcherAssert.assertThat; | ||
import static org.hamcrest.Matchers.is; | ||
import static software.amazon.awssdk.testutils.service.S3BucketUtils.temporaryBucketName; | ||
|
||
import java.nio.charset.StandardCharsets; | ||
import java.util.UUID; | ||
import java.util.stream.IntStream; | ||
import org.junit.AfterClass; | ||
import org.junit.BeforeClass; | ||
import org.junit.Test; | ||
import software.amazon.awssdk.core.sync.RequestBody; | ||
import software.amazon.awssdk.services.s3.internal.extensions.DeleteBucketAndAllContents; | ||
import software.amazon.awssdk.services.s3.model.BucketVersioningStatus; | ||
|
||
/** | ||
* Tests the {@link S3Client#deleteBucketAndAllContents(String)} extension method. | ||
* | ||
* @see DeleteBucketAndAllContents | ||
*/ | ||
public class DeleteBucketAndAllContentsIntegrationTest extends S3IntegrationTestBase { | ||
|
||
private static final String BUCKET = temporaryBucketName(DeleteBucketAndAllContentsIntegrationTest.class); | ||
|
||
@BeforeClass | ||
public static void initializeTestData() { | ||
createBucket(BUCKET); | ||
s3.putBucketVersioning(r -> r | ||
.bucket(BUCKET) | ||
.versioningConfiguration(v -> v.status(BucketVersioningStatus.ENABLED))); | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Should we have a test that sets up a more complicated bucket where objects have different versions? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. That's a good point. I could rewrite the same first ~50 key names. Would that be sufficient? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. yes |
||
} | ||
|
||
@AfterClass | ||
public static void tearDown() { | ||
if (s3.doesBucketExist(BUCKET)) { | ||
deleteBucketAndAllContents(BUCKET); | ||
} | ||
} | ||
|
||
@Test | ||
public void deleteBucketAndAllContents_WithVersioning_DeletesBucket() { | ||
// Populate the bucket with >1000 objects in order to exercise pagination behavior. | ||
int maxDeleteObjectsSize = 1_000; | ||
int numObjectsToCreate = maxDeleteObjectsSize + 50; | ||
IntStream.range(0, numObjectsToCreate).parallel().forEach(this::putObject); | ||
// Overwrite some keys to create multiple versions of objects | ||
int numKeysToOverwrite = 50; | ||
IntStream.range(0, numKeysToOverwrite).parallel().forEach(this::putObject); | ||
// Test deleting the bucket | ||
s3.deleteBucketAndAllContents(BUCKET); | ||
assertThat(s3.doesBucketExist(BUCKET), is(false)); | ||
} | ||
|
||
private void putObject(int i) { | ||
s3.putObject(r -> r.bucket(BUCKET) | ||
.key(String.valueOf(i)), | ||
RequestBody.fromString(UUID.randomUUID().toString(), StandardCharsets.UTF_8)); | ||
} | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -17,8 +17,14 @@ | |
|
||
import software.amazon.awssdk.annotations.SdkExtensionMethod; | ||
import software.amazon.awssdk.annotations.SdkPublicApi; | ||
import software.amazon.awssdk.core.exception.SdkException; | ||
import software.amazon.awssdk.services.s3.S3Client; | ||
import software.amazon.awssdk.services.s3.internal.extensions.DefaultS3ClientSdkExtension; | ||
import software.amazon.awssdk.services.s3.internal.extensions.DeleteBucketAndAllContents; | ||
import software.amazon.awssdk.services.s3.model.DeleteBucketRequest; | ||
import software.amazon.awssdk.services.s3.model.DeleteObjectsRequest; | ||
import software.amazon.awssdk.services.s3.model.ListObjectVersionsRequest; | ||
import software.amazon.awssdk.services.s3.model.ListObjectsV2Request; | ||
import software.amazon.awssdk.services.s3.model.S3Exception; | ||
|
||
/** | ||
|
@@ -32,12 +38,36 @@ public interface S3ClientSdkExtension { | |
* not accessible (e.g., due to access being denied or the bucket existing in another region), an {@link S3Exception} will be | ||
* thrown. | ||
* | ||
* @param bucketName the bucket to check | ||
* @param bucket the bucket to check | ||
* @return true if the bucket exists and you have permission to access it; false if the bucket does not exist | ||
* @throws S3Exception if the bucket exists but is not accessible | ||
*/ | ||
@SdkExtensionMethod | ||
default boolean doesBucketExist(String bucketName) { | ||
return new DefaultS3ClientSdkExtension((S3Client) this).doesBucketExist(bucketName); | ||
default boolean doesBucketExist(String bucket) { | ||
return new DefaultS3ClientSdkExtension((S3Client) this).doesBucketExist(bucket); | ||
} | ||
|
||
/** | ||
* Permanently delete a bucket and all of its content, including any versioned objects and delete markers. | ||
* <p> | ||
* Internally this method will use the {@link S3Client#listObjectsV2(ListObjectsV2Request)} and {@link | ||
* S3Client#listObjectVersions(ListObjectVersionsRequest)} APIs to list a bucket's content, buffer keys that are eligible for | ||
* deletion into batches of 1000, and delete them in bulk with the {@link S3Client#deleteObjects(DeleteObjectsRequest)} API. | ||
* <p> | ||
* While this method is optimized to use batch APIs for both listing and deleting, it may not be suitable for buckets | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. It may also be useful to direct people to TransferManager for their more complicated needs. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Good point, but I don't think we currently plan to support "delete directory" in TM, but it makes sense to support it there. |
||
* containing a very large number of objects (i.e., hundreds of thousands). For such use cases, it is usually preferable to | ||
* either create an <i>S3 Lifecycle configuration</i> to delete the objects, or to leverage <i>S3 Batch Operations</i> to | ||
* perform large-scale deletes. | ||
* <p> | ||
* Note that this method does not attempt to protect against concurrent writes or modifications to a bucket. It will iterate | ||
* and delete the entire contents of the bucket once. If a new key is created during or after the iteration, then the final | ||
* call to {@link S3Client#deleteBucket(DeleteBucketRequest)} may fail. | ||
* | ||
* @param bucket the bucket to delete | ||
* @throws SdkException if an error occurs | ||
*/ | ||
@SdkExtensionMethod | ||
default void deleteBucketAndAllContents(String bucket) { | ||
new DeleteBucketAndAllContents((S3Client) this).deleteBucketAndAllContents(bucket); | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Delegation has again proven to be more flexible. Good choice using a separate class. |
||
} | ||
} |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Love it!
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Red code is the best code!