Skip to content

Commit 10916f3

Browse files
author
AWS
committed
AWS Glue Update: Changes include (1) Excel as S3 Source type and XML and Tableau's Hyper as S3 Sink types, (2) targeted number of partitions parameter in S3 sinks and (3) new compression types in CSV/JSON and Parquet S3 sinks.
1 parent 5f0db4d commit 10916f3

File tree

2 files changed

+200
-1
lines changed

2 files changed

+200
-1
lines changed

‎.changes/next-release/feature-AWSGlue-423c58e.json

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
{
2+
"type": "feature",
3+
"category": "AWS Glue",
4+
"contributor": "",
5+
"description": "Changes include (1) Excel as S3 Source type and XML and Tableau's Hyper as S3 Sink types, (2) targeted number of partitions parameter in S3 sinks and (3) new compression types in CSV/JSON and Parquet S3 sinks."
6+
}

‎services/glue/src/main/resources/codegen-resources/service-2.json

Lines changed: 194 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6589,6 +6589,10 @@
65896589
"shape":"S3CsvSource",
65906590
"documentation":"<p>Specifies a command-separated value (CSV) data store stored in Amazon S3.</p>"
65916591
},
6592+
"S3ExcelSource":{
6593+
"shape":"S3ExcelSource",
6594+
"documentation":"<p>Defines configuration parameters for reading Excel files from Amazon S3.</p>"
6595+
},
65926596
"S3JsonSource":{
65936597
"shape":"S3JsonSource",
65946598
"documentation":"<p>Specifies a JSON data store stored in Amazon S3.</p>"
@@ -6629,10 +6633,18 @@
66296633
"shape":"S3GlueParquetTarget",
66306634
"documentation":"<p>Specifies a data target that writes to Amazon S3 in Apache Parquet columnar storage.</p>"
66316635
},
6636+
"S3HyperDirectTarget":{
6637+
"shape":"S3HyperDirectTarget",
6638+
"documentation":"<p>Defines configuration parameters for writing data to Amazon S3 using HyperDirect optimization.</p>"
6639+
},
66326640
"S3DirectTarget":{
66336641
"shape":"S3DirectTarget",
66346642
"documentation":"<p>Specifies a data target that writes to Amazon S3.</p>"
66356643
},
6644+
"S3IcebergDirectTarget":{
6645+
"shape":"S3IcebergDirectTarget",
6646+
"documentation":"<p>Defines configuration parameters for writing data to Amazon S3 as an Apache Iceberg table.</p>"
6647+
},
66366648
"ApplyMapping":{
66376649
"shape":"ApplyMapping",
66386650
"documentation":"<p>Specifies a transform that maps data property keys in the data source to data property keys in the data target. You can rename keys, modify the data types for keys, and choose which keys to drop from the dataset.</p>"
@@ -16428,6 +16440,10 @@
1642816440
"type":"list",
1642916441
"member":{"shape":"HudiTarget"}
1643016442
},
16443+
"HyperTargetCompressionType":{
16444+
"type":"string",
16445+
"enum":["uncompressed"]
16446+
},
1643116447
"IAMRoleArn":{
1643216448
"type":"string",
1643316449
"pattern":"^arn:aws(-(cn|us-gov|iso(-[bef])?))?:iam::[0-9]{12}:role/.+"
@@ -16579,6 +16595,15 @@
1657916595
},
1658016596
"documentation":"<p>Specifies an Apache Iceberg data source where Iceberg tables are stored in Amazon S3.</p>"
1658116597
},
16598+
"IcebergTargetCompressionType":{
16599+
"type":"string",
16600+
"enum":[
16601+
"gzip",
16602+
"lzo",
16603+
"uncompressed",
16604+
"snappy"
16605+
]
16606+
},
1658216607
"IcebergTargetList":{
1658316608
"type":"list",
1658416609
"member":{"shape":"IcebergTarget"}
@@ -19755,6 +19780,7 @@
1975519780
"type":"string",
1975619781
"box":true
1975719782
},
19783+
"NumberTargetPartitionsString":{"type":"string"},
1975819784
"OAuth2ClientApplication":{
1975919785
"type":"structure",
1976019786
"members":{
@@ -20199,6 +20225,8 @@
2019920225
"snappy",
2020020226
"lzo",
2020120227
"gzip",
20228+
"brotli",
20229+
"lz4",
2020220230
"uncompressed",
2020320231
"none"
2020420232
]
@@ -21914,6 +21942,10 @@
2191421942
"shape":"DeltaTargetCompressionType",
2191521943
"documentation":"<p>Specifies how the data is compressed. This is generally not necessary if the data has a standard file extension. Possible values are <code>\"gzip\"</code> and <code>\"bzip\"</code>).</p>"
2191621944
},
21945+
"NumberTargetPartitions":{
21946+
"shape":"NumberTargetPartitionsString",
21947+
"documentation":"<p>Specifies the number of target partitions for distributing Delta Lake dataset files across Amazon S3.</p>"
21948+
},
2191721949
"Format":{
2191821950
"shape":"TargetFormat",
2191921951
"documentation":"<p>Specifies the data output format for the target.</p>"
@@ -22010,6 +22042,10 @@
2201022042
"shape":"EnclosedInStringProperty",
2201122043
"documentation":"<p>Specifies how the data is compressed. This is generally not necessary if the data has a standard file extension. Possible values are <code>\"gzip\"</code> and <code>\"bzip\"</code>).</p>"
2201222044
},
22045+
"NumberTargetPartitions":{
22046+
"shape":"NumberTargetPartitionsString",
22047+
"documentation":"<p>Specifies the number of target partitions when writing data directly to Amazon S3.</p>"
22048+
},
2201322049
"Format":{
2201422050
"shape":"TargetFormat",
2201522051
"documentation":"<p>Specifies the data output format for the target.</p>"
@@ -22047,6 +22083,68 @@
2204722083
"SSE-S3"
2204822084
]
2204922085
},
22086+
"S3ExcelSource":{
22087+
"type":"structure",
22088+
"required":[
22089+
"Name",
22090+
"Paths"
22091+
],
22092+
"members":{
22093+
"Name":{
22094+
"shape":"NodeName",
22095+
"documentation":"<p>The name of the S3 Excel data source.</p>"
22096+
},
22097+
"Paths":{
22098+
"shape":"EnclosedInStringProperties",
22099+
"documentation":"<p>The S3 paths where the Excel files are located.</p>"
22100+
},
22101+
"CompressionType":{
22102+
"shape":"ParquetCompressionType",
22103+
"documentation":"<p>The compression format used for the Excel files.</p>"
22104+
},
22105+
"Exclusions":{
22106+
"shape":"EnclosedInStringProperties",
22107+
"documentation":"<p>Patterns to exclude specific files or paths from processing.</p>"
22108+
},
22109+
"GroupSize":{
22110+
"shape":"EnclosedInStringProperty",
22111+
"documentation":"<p>Defines the size of file groups for batch processing.</p>"
22112+
},
22113+
"GroupFiles":{
22114+
"shape":"EnclosedInStringProperty",
22115+
"documentation":"<p>Specifies how files should be grouped for processing.</p>"
22116+
},
22117+
"Recurse":{
22118+
"shape":"BoxedBoolean",
22119+
"documentation":"<p>Indicates whether to recursively process subdirectories.</p>"
22120+
},
22121+
"MaxBand":{
22122+
"shape":"BoxedNonNegativeInt",
22123+
"documentation":"<p>The maximum number of processing bands to use.</p>"
22124+
},
22125+
"MaxFilesInBand":{
22126+
"shape":"BoxedNonNegativeInt",
22127+
"documentation":"<p>The maximum number of files to process in each band.</p>"
22128+
},
22129+
"AdditionalOptions":{
22130+
"shape":"S3DirectSourceAdditionalOptions",
22131+
"documentation":"<p>Additional configuration options for S3 direct source processing.</p>"
22132+
},
22133+
"NumberRows":{
22134+
"shape":"BoxedLong",
22135+
"documentation":"<p>The number of rows to process from each Excel file.</p>"
22136+
},
22137+
"SkipFooter":{
22138+
"shape":"BoxedNonNegativeInt",
22139+
"documentation":"<p>The number of rows to skip at the end of each Excel file.</p>"
22140+
},
22141+
"OutputSchemas":{
22142+
"shape":"GlueSchemas",
22143+
"documentation":"<p>The AWS Glue schemas to apply to the processed data.</p>"
22144+
}
22145+
},
22146+
"documentation":"<p>Specifies an S3 Excel data source.</p>"
22147+
},
2205022148
"S3GlueParquetTarget":{
2205122149
"type":"structure",
2205222150
"required":[
@@ -22075,6 +22173,10 @@
2207522173
"shape":"ParquetCompressionType",
2207622174
"documentation":"<p>Specifies how the data is compressed. This is generally not necessary if the data has a standard file extension. Possible values are <code>\"gzip\"</code> and <code>\"bzip\"</code>).</p>"
2207722175
},
22176+
"NumberTargetPartitions":{
22177+
"shape":"NumberTargetPartitionsString",
22178+
"documentation":"<p>Specifies the number of target partitions for Parquet files when writing to Amazon S3 using AWS Glue.</p>"
22179+
},
2207822180
"SchemaChangePolicy":{
2207922181
"shape":"DirectSchemaChangePolicy",
2208022182
"documentation":"<p>A policy that specifies update behavior for the crawler.</p>"
@@ -22150,6 +22252,10 @@
2215022252
"shape":"HudiTargetCompressionType",
2215122253
"documentation":"<p>Specifies how the data is compressed. This is generally not necessary if the data has a standard file extension. Possible values are <code>\"gzip\"</code> and <code>\"bzip\"</code>).</p>"
2215222254
},
22255+
"NumberTargetPartitions":{
22256+
"shape":"NumberTargetPartitionsString",
22257+
"documentation":"<p>Specifies the number of target partitions for distributing Hudi dataset files across Amazon S3.</p>"
22258+
},
2215322259
"PartitionKeys":{
2215422260
"shape":"GlueStudioPathList",
2215522261
"documentation":"<p>Specifies native partitioning using a sequence of keys.</p>"
@@ -22199,6 +22305,90 @@
2219922305
},
2220022306
"documentation":"<p>Specifies a Hudi data source stored in Amazon S3.</p>"
2220122307
},
22308+
"S3HyperDirectTarget":{
22309+
"type":"structure",
22310+
"required":[
22311+
"Name",
22312+
"Inputs",
22313+
"Path"
22314+
],
22315+
"members":{
22316+
"Name":{
22317+
"shape":"NodeName",
22318+
"documentation":"<p>The unique identifier for the HyperDirect target node.</p>"
22319+
},
22320+
"Inputs":{
22321+
"shape":"OneInput",
22322+
"documentation":"<p>Specifies the input source for the HyperDirect target.</p>"
22323+
},
22324+
"PartitionKeys":{
22325+
"shape":"GlueStudioPathList",
22326+
"documentation":"<p>Defines the partitioning strategy for the output data.</p>"
22327+
},
22328+
"Path":{
22329+
"shape":"EnclosedInStringProperty",
22330+
"documentation":"<p>The S3 location where the output data will be written.</p>"
22331+
},
22332+
"Compression":{
22333+
"shape":"HyperTargetCompressionType",
22334+
"documentation":"<p>The compression type to apply to the output data.</p>"
22335+
},
22336+
"SchemaChangePolicy":{
22337+
"shape":"DirectSchemaChangePolicy",
22338+
"documentation":"<p>Defines how schema changes are handled during write operations.</p>"
22339+
}
22340+
},
22341+
"documentation":"<p>Specifies a HyperDirect data target that writes to Amazon S3.</p>"
22342+
},
22343+
"S3IcebergDirectTarget":{
22344+
"type":"structure",
22345+
"required":[
22346+
"Name",
22347+
"Inputs",
22348+
"Path",
22349+
"Format",
22350+
"Compression"
22351+
],
22352+
"members":{
22353+
"Name":{
22354+
"shape":"NodeName",
22355+
"documentation":"<p>Specifies the unique identifier for the Iceberg target node in your data pipeline.</p>"
22356+
},
22357+
"Inputs":{
22358+
"shape":"OneInput",
22359+
"documentation":"<p>Defines the single input source that provides data to this Iceberg target.</p>"
22360+
},
22361+
"PartitionKeys":{
22362+
"shape":"GlueStudioPathList",
22363+
"documentation":"<p>Specifies the columns used to partition the Iceberg table data in S3.</p>"
22364+
},
22365+
"Path":{
22366+
"shape":"EnclosedInStringProperty",
22367+
"documentation":"<p>Defines the S3 location where the Iceberg table data will be stored.</p>"
22368+
},
22369+
"Format":{
22370+
"shape":"TargetFormat",
22371+
"documentation":"<p>Specifies the file format used for storing Iceberg table data (e.g., Parquet, ORC).</p>"
22372+
},
22373+
"AdditionalOptions":{
22374+
"shape":"AdditionalOptions",
22375+
"documentation":"<p>Provides additional configuration options for customizing the Iceberg table behavior.</p>"
22376+
},
22377+
"SchemaChangePolicy":{
22378+
"shape":"DirectSchemaChangePolicy",
22379+
"documentation":"<p>Defines how schema changes are handled when writing data to the Iceberg table.</p>"
22380+
},
22381+
"Compression":{
22382+
"shape":"IcebergTargetCompressionType",
22383+
"documentation":"<p>Specifies the compression codec used for Iceberg table files in S3.</p>"
22384+
},
22385+
"NumberTargetPartitions":{
22386+
"shape":"NumberTargetPartitionsString",
22387+
"documentation":"<p>Sets the number of target partitions for distributing Iceberg table files across S3.</p>"
22388+
}
22389+
},
22390+
"documentation":"<p>Specifies a target that writes to an Iceberg data source in Amazon S3.</p>"
22391+
},
2220222392
"S3JsonSource":{
2220322393
"type":"structure",
2220422394
"required":[
@@ -24877,7 +25067,10 @@
2487725067
"orc",
2487825068
"parquet",
2487925069
"hudi",
24880-
"delta"
25070+
"delta",
25071+
"iceberg",
25072+
"hyper",
25073+
"xml"
2488125074
]
2488225075
},
2488325076
"TargetProcessingProperties":{

0 commit comments

Comments
 (0)