Skip to content

Commit 0dac89e

Browse files
author
awstools
committed
feat(client-glue): AWS Glue is introducing two new optimizers for Apache Iceberg tables: snapshot retention and orphan file deletion. Customers can enable these optimizers and customize their configurations to perform daily maintenance tasks on their Iceberg tables based on their specific requirements.
1 parent ba923e6 commit 0dac89e

22 files changed

+1752
-869
lines changed

‎clients/client-glue/src/commands/BatchGetTableOptimizerCommand.ts

Lines changed: 52 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@ export interface BatchGetTableOptimizerCommandOutput extends BatchGetTableOptimi
4141
* catalogId: "STRING_VALUE",
4242
* databaseName: "STRING_VALUE",
4343
* tableName: "STRING_VALUE",
44-
* type: "compaction",
44+
* type: "compaction" || "retention" || "orphan_file_deletion",
4545
* },
4646
* ],
4747
* };
@@ -54,10 +54,23 @@ export interface BatchGetTableOptimizerCommandOutput extends BatchGetTableOptimi
5454
* // databaseName: "STRING_VALUE",
5555
* // tableName: "STRING_VALUE",
5656
* // tableOptimizer: { // TableOptimizer
57-
* // type: "compaction",
57+
* // type: "compaction" || "retention" || "orphan_file_deletion",
5858
* // configuration: { // TableOptimizerConfiguration
5959
* // roleArn: "STRING_VALUE",
6060
* // enabled: true || false,
61+
* // retentionConfiguration: { // RetentionConfiguration
62+
* // icebergConfiguration: { // IcebergRetentionConfiguration
63+
* // snapshotRetentionPeriodInDays: Number("int"),
64+
* // numberOfSnapshotsToRetain: Number("int"),
65+
* // cleanExpiredFiles: true || false,
66+
* // },
67+
* // },
68+
* // orphanFileDeletionConfiguration: { // OrphanFileDeletionConfiguration
69+
* // icebergConfiguration: { // IcebergOrphanFileDeletionConfiguration
70+
* // orphanFileRetentionPeriodInDays: Number("int"),
71+
* // location: "STRING_VALUE",
72+
* // },
73+
* // },
6174
* // },
6275
* // lastRun: { // TableOptimizerRun
6376
* // eventType: "starting" || "completed" || "failed" || "in_progress",
@@ -70,6 +83,30 @@ export interface BatchGetTableOptimizerCommandOutput extends BatchGetTableOptimi
7083
* // JobDurationInHour: "STRING_VALUE",
7184
* // },
7285
* // error: "STRING_VALUE",
86+
* // compactionMetrics: { // CompactionMetrics
87+
* // IcebergMetrics: { // IcebergCompactionMetrics
88+
* // NumberOfBytesCompacted: Number("long"),
89+
* // NumberOfFilesCompacted: Number("long"),
90+
* // NumberOfDpus: Number("int"),
91+
* // JobDurationInHour: Number("double"),
92+
* // },
93+
* // },
94+
* // retentionMetrics: { // RetentionMetrics
95+
* // IcebergMetrics: { // IcebergRetentionMetrics
96+
* // NumberOfDataFilesDeleted: Number("long"),
97+
* // NumberOfManifestFilesDeleted: Number("long"),
98+
* // NumberOfManifestListsDeleted: Number("long"),
99+
* // NumberOfDpus: Number("int"),
100+
* // JobDurationInHour: Number("double"),
101+
* // },
102+
* // },
103+
* // orphanFileDeletionMetrics: { // OrphanFileDeletionMetrics
104+
* // IcebergMetrics: { // IcebergOrphanFileDeletionMetrics
105+
* // NumberOfOrphanFilesDeleted: Number("long"),
106+
* // NumberOfDpus: Number("int"),
107+
* // JobDurationInHour: Number("double"),
108+
* // },
109+
* // },
73110
* // },
74111
* // },
75112
* // },
@@ -83,7 +120,7 @@ export interface BatchGetTableOptimizerCommandOutput extends BatchGetTableOptimi
83120
* // catalogId: "STRING_VALUE",
84121
* // databaseName: "STRING_VALUE",
85122
* // tableName: "STRING_VALUE",
86-
* // type: "compaction",
123+
* // type: "compaction" || "retention" || "orphan_file_deletion",
87124
* // },
88125
* // ],
89126
* // };
@@ -96,9 +133,21 @@ export interface BatchGetTableOptimizerCommandOutput extends BatchGetTableOptimi
96133
* @see {@link BatchGetTableOptimizerCommandOutput} for command's `response` shape.
97134
* @see {@link GlueClientResolvedConfig | config} for GlueClient's `config` shape.
98135
*
136+
* @throws {@link AccessDeniedException} (client fault)
137+
* <p>Access to a resource was denied.</p>
138+
*
139+
* @throws {@link EntityNotFoundException} (client fault)
140+
* <p>A specified entity does not exist</p>
141+
*
99142
* @throws {@link InternalServiceException} (server fault)
100143
* <p>An internal service error occurred.</p>
101144
*
145+
* @throws {@link InvalidInputException} (client fault)
146+
* <p>The input provided was not valid.</p>
147+
*
148+
* @throws {@link ThrottlingException} (client fault)
149+
* <p>The throttling threshhold was exceeded.</p>
150+
*
102151
* @throws {@link GlueServiceException}
103152
* <p>Base exception class for all service exceptions from Glue service.</p>
104153
*

‎clients/client-glue/src/commands/CancelDataQualityRulesetEvaluationRunCommand.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ import { GlueClientResolvedConfig, ServiceInputTypes, ServiceOutputTypes } from
99
import {
1010
CancelDataQualityRulesetEvaluationRunRequest,
1111
CancelDataQualityRulesetEvaluationRunResponse,
12-
} from "../models/models_0";
12+
} from "../models/models_1";
1313
import {
1414
de_CancelDataQualityRulesetEvaluationRunCommand,
1515
se_CancelDataQualityRulesetEvaluationRunCommand,

‎clients/client-glue/src/commands/CancelMLTaskRunCommand.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ import { MetadataBearer as __MetadataBearer } from "@smithy/types";
66

77
import { commonParams } from "../endpoint/EndpointParameters";
88
import { GlueClientResolvedConfig, ServiceInputTypes, ServiceOutputTypes } from "../GlueClient";
9-
import { CancelMLTaskRunRequest, CancelMLTaskRunResponse } from "../models/models_0";
9+
import { CancelMLTaskRunRequest, CancelMLTaskRunResponse } from "../models/models_1";
1010
import { de_CancelMLTaskRunCommand, se_CancelMLTaskRunCommand } from "../protocols/Aws_json1_1";
1111

1212
/**

‎clients/client-glue/src/commands/CancelStatementCommand.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ import { MetadataBearer as __MetadataBearer } from "@smithy/types";
66

77
import { commonParams } from "../endpoint/EndpointParameters";
88
import { GlueClientResolvedConfig, ServiceInputTypes, ServiceOutputTypes } from "../GlueClient";
9-
import { CancelStatementRequest, CancelStatementResponse } from "../models/models_0";
9+
import { CancelStatementRequest, CancelStatementResponse } from "../models/models_1";
1010
import { de_CancelStatementCommand, se_CancelStatementCommand } from "../protocols/Aws_json1_1";
1111

1212
/**

‎clients/client-glue/src/commands/CheckSchemaVersionValidityCommand.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ import { MetadataBearer as __MetadataBearer } from "@smithy/types";
66

77
import { commonParams } from "../endpoint/EndpointParameters";
88
import { GlueClientResolvedConfig, ServiceInputTypes, ServiceOutputTypes } from "../GlueClient";
9-
import { CheckSchemaVersionValidityInput, CheckSchemaVersionValidityResponse } from "../models/models_0";
9+
import { CheckSchemaVersionValidityInput, CheckSchemaVersionValidityResponse } from "../models/models_1";
1010
import { de_CheckSchemaVersionValidityCommand, se_CheckSchemaVersionValidityCommand } from "../protocols/Aws_json1_1";
1111

1212
/**

‎clients/client-glue/src/commands/CreateTableOptimizerCommand.ts

Lines changed: 20 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -39,10 +39,23 @@ export interface CreateTableOptimizerCommandOutput extends CreateTableOptimizerR
3939
* CatalogId: "STRING_VALUE", // required
4040
* DatabaseName: "STRING_VALUE", // required
4141
* TableName: "STRING_VALUE", // required
42-
* Type: "compaction", // required
42+
* Type: "compaction" || "retention" || "orphan_file_deletion", // required
4343
* TableOptimizerConfiguration: { // TableOptimizerConfiguration
4444
* roleArn: "STRING_VALUE",
4545
* enabled: true || false,
46+
* retentionConfiguration: { // RetentionConfiguration
47+
* icebergConfiguration: { // IcebergRetentionConfiguration
48+
* snapshotRetentionPeriodInDays: Number("int"),
49+
* numberOfSnapshotsToRetain: Number("int"),
50+
* cleanExpiredFiles: true || false,
51+
* },
52+
* },
53+
* orphanFileDeletionConfiguration: { // OrphanFileDeletionConfiguration
54+
* icebergConfiguration: { // IcebergOrphanFileDeletionConfiguration
55+
* orphanFileRetentionPeriodInDays: Number("int"),
56+
* location: "STRING_VALUE",
57+
* },
58+
* },
4659
* },
4760
* };
4861
* const command = new CreateTableOptimizerCommand(input);
@@ -72,6 +85,12 @@ export interface CreateTableOptimizerCommandOutput extends CreateTableOptimizerR
7285
* @throws {@link InvalidInputException} (client fault)
7386
* <p>The input provided was not valid.</p>
7487
*
88+
* @throws {@link ThrottlingException} (client fault)
89+
* <p>The throttling threshhold was exceeded.</p>
90+
*
91+
* @throws {@link ValidationException} (client fault)
92+
* <p>A value could not be validated.</p>
93+
*
7594
* @throws {@link GlueServiceException}
7695
* <p>Base exception class for all service exceptions from Glue service.</p>
7796
*

‎clients/client-glue/src/commands/DeleteTableOptimizerCommand.ts

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,7 @@ export interface DeleteTableOptimizerCommandOutput extends DeleteTableOptimizerR
3939
* CatalogId: "STRING_VALUE", // required
4040
* DatabaseName: "STRING_VALUE", // required
4141
* TableName: "STRING_VALUE", // required
42-
* Type: "compaction", // required
42+
* Type: "compaction" || "retention" || "orphan_file_deletion", // required
4343
* };
4444
* const command = new DeleteTableOptimizerCommand(input);
4545
* const response = await client.send(command);
@@ -65,6 +65,9 @@ export interface DeleteTableOptimizerCommandOutput extends DeleteTableOptimizerR
6565
* @throws {@link InvalidInputException} (client fault)
6666
* <p>The input provided was not valid.</p>
6767
*
68+
* @throws {@link ThrottlingException} (client fault)
69+
* <p>The throttling threshhold was exceeded.</p>
70+
*
6871
* @throws {@link GlueServiceException}
6972
* <p>Base exception class for all service exceptions from Glue service.</p>
7073
*

‎clients/client-glue/src/commands/GetMLTransformCommand.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ import { MetadataBearer as __MetadataBearer } from "@smithy/types";
66

77
import { commonParams } from "../endpoint/EndpointParameters";
88
import { GlueClientResolvedConfig, ServiceInputTypes, ServiceOutputTypes } from "../GlueClient";
9-
import { GetMLTransformRequest, GetMLTransformResponse } from "../models/models_1";
9+
import { GetMLTransformRequest, GetMLTransformResponse } from "../models/models_2";
1010
import { de_GetMLTransformCommand, se_GetMLTransformCommand } from "../protocols/Aws_json1_1";
1111

1212
/**

‎clients/client-glue/src/commands/GetTableOptimizerCommand.ts

Lines changed: 42 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,7 @@ export interface GetTableOptimizerCommandOutput extends GetTableOptimizerRespons
3939
* CatalogId: "STRING_VALUE", // required
4040
* DatabaseName: "STRING_VALUE", // required
4141
* TableName: "STRING_VALUE", // required
42-
* Type: "compaction", // required
42+
* Type: "compaction" || "retention" || "orphan_file_deletion", // required
4343
* };
4444
* const command = new GetTableOptimizerCommand(input);
4545
* const response = await client.send(command);
@@ -48,10 +48,23 @@ export interface GetTableOptimizerCommandOutput extends GetTableOptimizerRespons
4848
* // DatabaseName: "STRING_VALUE",
4949
* // TableName: "STRING_VALUE",
5050
* // TableOptimizer: { // TableOptimizer
51-
* // type: "compaction",
51+
* // type: "compaction" || "retention" || "orphan_file_deletion",
5252
* // configuration: { // TableOptimizerConfiguration
5353
* // roleArn: "STRING_VALUE",
5454
* // enabled: true || false,
55+
* // retentionConfiguration: { // RetentionConfiguration
56+
* // icebergConfiguration: { // IcebergRetentionConfiguration
57+
* // snapshotRetentionPeriodInDays: Number("int"),
58+
* // numberOfSnapshotsToRetain: Number("int"),
59+
* // cleanExpiredFiles: true || false,
60+
* // },
61+
* // },
62+
* // orphanFileDeletionConfiguration: { // OrphanFileDeletionConfiguration
63+
* // icebergConfiguration: { // IcebergOrphanFileDeletionConfiguration
64+
* // orphanFileRetentionPeriodInDays: Number("int"),
65+
* // location: "STRING_VALUE",
66+
* // },
67+
* // },
5568
* // },
5669
* // lastRun: { // TableOptimizerRun
5770
* // eventType: "starting" || "completed" || "failed" || "in_progress",
@@ -64,6 +77,30 @@ export interface GetTableOptimizerCommandOutput extends GetTableOptimizerRespons
6477
* // JobDurationInHour: "STRING_VALUE",
6578
* // },
6679
* // error: "STRING_VALUE",
80+
* // compactionMetrics: { // CompactionMetrics
81+
* // IcebergMetrics: { // IcebergCompactionMetrics
82+
* // NumberOfBytesCompacted: Number("long"),
83+
* // NumberOfFilesCompacted: Number("long"),
84+
* // NumberOfDpus: Number("int"),
85+
* // JobDurationInHour: Number("double"),
86+
* // },
87+
* // },
88+
* // retentionMetrics: { // RetentionMetrics
89+
* // IcebergMetrics: { // IcebergRetentionMetrics
90+
* // NumberOfDataFilesDeleted: Number("long"),
91+
* // NumberOfManifestFilesDeleted: Number("long"),
92+
* // NumberOfManifestListsDeleted: Number("long"),
93+
* // NumberOfDpus: Number("int"),
94+
* // JobDurationInHour: Number("double"),
95+
* // },
96+
* // },
97+
* // orphanFileDeletionMetrics: { // OrphanFileDeletionMetrics
98+
* // IcebergMetrics: { // IcebergOrphanFileDeletionMetrics
99+
* // NumberOfOrphanFilesDeleted: Number("long"),
100+
* // NumberOfDpus: Number("int"),
101+
* // JobDurationInHour: Number("double"),
102+
* // },
103+
* // },
67104
* // },
68105
* // },
69106
* // };
@@ -88,6 +125,9 @@ export interface GetTableOptimizerCommandOutput extends GetTableOptimizerRespons
88125
* @throws {@link InvalidInputException} (client fault)
89126
* <p>The input provided was not valid.</p>
90127
*
128+
* @throws {@link ThrottlingException} (client fault)
129+
* <p>The throttling threshhold was exceeded.</p>
130+
*
91131
* @throws {@link GlueServiceException}
92132
* <p>Base exception class for all service exceptions from Glue service.</p>
93133
*

‎clients/client-glue/src/commands/ListTableOptimizerRunsCommand.ts

Lines changed: 31 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,7 @@ export interface ListTableOptimizerRunsCommandOutput extends ListTableOptimizerR
3939
* CatalogId: "STRING_VALUE", // required
4040
* DatabaseName: "STRING_VALUE", // required
4141
* TableName: "STRING_VALUE", // required
42-
* Type: "compaction", // required
42+
* Type: "compaction" || "retention" || "orphan_file_deletion", // required
4343
* MaxResults: Number("int"),
4444
* NextToken: "STRING_VALUE",
4545
* };
@@ -62,6 +62,30 @@ export interface ListTableOptimizerRunsCommandOutput extends ListTableOptimizerR
6262
* // JobDurationInHour: "STRING_VALUE",
6363
* // },
6464
* // error: "STRING_VALUE",
65+
* // compactionMetrics: { // CompactionMetrics
66+
* // IcebergMetrics: { // IcebergCompactionMetrics
67+
* // NumberOfBytesCompacted: Number("long"),
68+
* // NumberOfFilesCompacted: Number("long"),
69+
* // NumberOfDpus: Number("int"),
70+
* // JobDurationInHour: Number("double"),
71+
* // },
72+
* // },
73+
* // retentionMetrics: { // RetentionMetrics
74+
* // IcebergMetrics: { // IcebergRetentionMetrics
75+
* // NumberOfDataFilesDeleted: Number("long"),
76+
* // NumberOfManifestFilesDeleted: Number("long"),
77+
* // NumberOfManifestListsDeleted: Number("long"),
78+
* // NumberOfDpus: Number("int"),
79+
* // JobDurationInHour: Number("double"),
80+
* // },
81+
* // },
82+
* // orphanFileDeletionMetrics: { // OrphanFileDeletionMetrics
83+
* // IcebergMetrics: { // IcebergOrphanFileDeletionMetrics
84+
* // NumberOfOrphanFilesDeleted: Number("long"),
85+
* // NumberOfDpus: Number("int"),
86+
* // JobDurationInHour: Number("double"),
87+
* // },
88+
* // },
6589
* // },
6690
* // ],
6791
* // };
@@ -86,6 +110,12 @@ export interface ListTableOptimizerRunsCommandOutput extends ListTableOptimizerR
86110
* @throws {@link InvalidInputException} (client fault)
87111
* <p>The input provided was not valid.</p>
88112
*
113+
* @throws {@link ThrottlingException} (client fault)
114+
* <p>The throttling threshhold was exceeded.</p>
115+
*
116+
* @throws {@link ValidationException} (client fault)
117+
* <p>A value could not be validated.</p>
118+
*
89119
* @throws {@link GlueServiceException}
90120
* <p>Base exception class for all service exceptions from Glue service.</p>
91121
*

‎clients/client-glue/src/commands/UpdateTableCommand.ts

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,8 @@ import { MetadataBearer as __MetadataBearer } from "@smithy/types";
66

77
import { commonParams } from "../endpoint/EndpointParameters";
88
import { GlueClientResolvedConfig, ServiceInputTypes, ServiceOutputTypes } from "../GlueClient";
9-
import { UpdateTableRequest, UpdateTableResponse } from "../models/models_2";
9+
import { UpdateTableRequest } from "../models/models_2";
10+
import { UpdateTableResponse } from "../models/models_3";
1011
import { de_UpdateTableCommand, se_UpdateTableCommand } from "../protocols/Aws_json1_1";
1112

1213
/**

‎clients/client-glue/src/commands/UpdateTableOptimizerCommand.ts

Lines changed: 24 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ import { MetadataBearer as __MetadataBearer } from "@smithy/types";
66

77
import { commonParams } from "../endpoint/EndpointParameters";
88
import { GlueClientResolvedConfig, ServiceInputTypes, ServiceOutputTypes } from "../GlueClient";
9-
import { UpdateTableOptimizerRequest, UpdateTableOptimizerResponse } from "../models/models_2";
9+
import { UpdateTableOptimizerRequest, UpdateTableOptimizerResponse } from "../models/models_3";
1010
import { de_UpdateTableOptimizerCommand, se_UpdateTableOptimizerCommand } from "../protocols/Aws_json1_1";
1111

1212
/**
@@ -39,10 +39,23 @@ export interface UpdateTableOptimizerCommandOutput extends UpdateTableOptimizerR
3939
* CatalogId: "STRING_VALUE", // required
4040
* DatabaseName: "STRING_VALUE", // required
4141
* TableName: "STRING_VALUE", // required
42-
* Type: "compaction", // required
42+
* Type: "compaction" || "retention" || "orphan_file_deletion", // required
4343
* TableOptimizerConfiguration: { // TableOptimizerConfiguration
4444
* roleArn: "STRING_VALUE",
4545
* enabled: true || false,
46+
* retentionConfiguration: { // RetentionConfiguration
47+
* icebergConfiguration: { // IcebergRetentionConfiguration
48+
* snapshotRetentionPeriodInDays: Number("int"),
49+
* numberOfSnapshotsToRetain: Number("int"),
50+
* cleanExpiredFiles: true || false,
51+
* },
52+
* },
53+
* orphanFileDeletionConfiguration: { // OrphanFileDeletionConfiguration
54+
* icebergConfiguration: { // IcebergOrphanFileDeletionConfiguration
55+
* orphanFileRetentionPeriodInDays: Number("int"),
56+
* location: "STRING_VALUE",
57+
* },
58+
* },
4659
* },
4760
* };
4861
* const command = new UpdateTableOptimizerCommand(input);
@@ -60,6 +73,9 @@ export interface UpdateTableOptimizerCommandOutput extends UpdateTableOptimizerR
6073
* @throws {@link AccessDeniedException} (client fault)
6174
* <p>Access to a resource was denied.</p>
6275
*
76+
* @throws {@link ConcurrentModificationException} (client fault)
77+
* <p>Two processes are trying to modify a resource simultaneously.</p>
78+
*
6379
* @throws {@link EntityNotFoundException} (client fault)
6480
* <p>A specified entity does not exist</p>
6581
*
@@ -69,6 +85,12 @@ export interface UpdateTableOptimizerCommandOutput extends UpdateTableOptimizerR
6985
* @throws {@link InvalidInputException} (client fault)
7086
* <p>The input provided was not valid.</p>
7187
*
88+
* @throws {@link ThrottlingException} (client fault)
89+
* <p>The throttling threshhold was exceeded.</p>
90+
*
91+
* @throws {@link ValidationException} (client fault)
92+
* <p>A value could not be validated.</p>
93+
*
7294
* @throws {@link GlueServiceException}
7395
* <p>Base exception class for all service exceptions from Glue service.</p>
7496
*

0 commit comments

Comments
 (0)