Skip to content

Commit 8996b42

Browse files
Will Johnsonengelke
Will Johnson
authored and
engelke
committed
Update Transfer Service examples (#1858)
* Correct TimeZone from US Pacific to UTC Google Cloud Storage Transfer Service API requests require the |Schedule| to be in UTC [1]. This |Schedule| also supports specifying the |startTimeOfDay| with seconds granularity. [1] https://cloud.google.com/storage-transfer/docs/reference/rest/v1/transferJobs#Schedule * Correct TimeZone from US Pacific to UTC Google Cloud Storage Transfer Service API requests require the |Schedule| to be in UTC [1]. This |Schedule| also supports specifying the |startTimeOfDay| with seconds granularity. [1] https://cloud.google.com/storage-transfer/docs/reference/rest/v1/transferJobs#Schedule
1 parent df2a433 commit 8996b42

File tree

3 files changed

+59
-65
lines changed

3 files changed

+59
-65
lines changed

‎storage/transfer_service/README.rst

Lines changed: 16 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -78,11 +78,11 @@ To run this sample:
7878
$ python nearline_request.py
7979
8080
usage: nearline_request.py [-h]
81-
description project_id date time source_bucket
82-
sink_bucket
81+
description project_id start_date start_time
82+
source_bucket sink_bucket
8383
84-
Command-line sample that creates a one-time transfer from Google Cloud
85-
Storage standard class to the Nearline storage class."
84+
Command-line sample that creates a daily transfer from a standard
85+
GCS bucket to a Nearline GCS bucket for objects untouched for 30 days.
8686
8787
This sample is used on this page:
8888
@@ -93,10 +93,10 @@ To run this sample:
9393
positional arguments:
9494
description Transfer description.
9595
project_id Your Google Cloud project ID.
96-
date Date YYYY/MM/DD.
97-
time Time (24hr) HH:MM.
98-
source_bucket Source bucket name.
99-
sink_bucket Sink bucket name.
96+
start_date Date YYYY/MM/DD.
97+
start_time UTC Time (24hr) HH:MM:SS.
98+
source_bucket Standard GCS bucket name.
99+
sink_bucket Nearline GCS bucket name.
100100
101101
optional arguments:
102102
-h, --help show this help message and exit
@@ -119,8 +119,9 @@ To run this sample:
119119
$ python aws_request.py
120120
121121
usage: aws_request.py [-h]
122-
description project_id date time source_bucket
123-
access_key secret_access_key sink_bucket
122+
description project_id start_date start_time
123+
source_bucket access_key_id secret_access_key
124+
sink_bucket
124125
125126
Command-line sample that creates a one-time transfer from Amazon S3 to
126127
Google Cloud Storage.
@@ -134,12 +135,12 @@ To run this sample:
134135
positional arguments:
135136
description Transfer description.
136137
project_id Your Google Cloud project ID.
137-
date Date YYYY/MM/DD.
138-
time Time (24hr) HH:MM.
139-
source_bucket Source bucket name.
140-
access_key Your AWS access key id.
138+
start_date Date YYYY/MM/DD.
139+
start_time UTC Time (24hr) HH:MM:SS.
140+
source_bucket AWS source bucket name.
141+
access_key_id Your AWS access key id.
141142
secret_access_key Your AWS secret access key.
142-
sink_bucket Sink bucket name.
143+
sink_bucket GCS sink bucket name.
143144
144145
optional arguments:
145146
-h, --help show this help message and exit

‎storage/transfer_service/aws_request.py

Lines changed: 23 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -32,38 +32,38 @@
3232

3333

3434
# [START main]
35-
def main(description, project_id, year, month, day, hours, minutes,
36-
source_bucket, access_key, secret_access_key, sink_bucket):
37-
"""Create a one-off transfer from Amazon S3 to Google Cloud Storage."""
35+
def main(description, project_id, start_date, start_time, source_bucket,
36+
access_key_id, secret_access_key, sink_bucket):
37+
"""Create a one-time transfer from Amazon S3 to Google Cloud Storage."""
3838
storagetransfer = googleapiclient.discovery.build('storagetransfer', 'v1')
3939

4040
# Edit this template with desired parameters.
41-
# Specify times below using US Pacific Time Zone.
4241
transfer_job = {
4342
'description': description,
4443
'status': 'ENABLED',
4544
'projectId': project_id,
4645
'schedule': {
4746
'scheduleStartDate': {
48-
'day': day,
49-
'month': month,
50-
'year': year
47+
'day': start_date.day,
48+
'month': start_date.month,
49+
'year': start_date.year
5150
},
5251
'scheduleEndDate': {
53-
'day': day,
54-
'month': month,
55-
'year': year
52+
'day': start_date.day,
53+
'month': start_date.month,
54+
'year': start_date.year
5655
},
5756
'startTimeOfDay': {
58-
'hours': hours,
59-
'minutes': minutes
57+
'hours': start_time.hour,
58+
'minutes': start_time.minute,
59+
'seconds': start_time.second
6060
}
6161
},
6262
'transferSpec': {
6363
'awsS3DataSource': {
6464
'bucketName': source_bucket,
6565
'awsAccessKey': {
66-
'accessKeyId': access_key,
66+
'accessKeyId': access_key_id,
6767
'secretAccessKey': secret_access_key
6868
}
6969
},
@@ -85,28 +85,25 @@ def main(description, project_id, year, month, day, hours, minutes,
8585
formatter_class=argparse.RawDescriptionHelpFormatter)
8686
parser.add_argument('description', help='Transfer description.')
8787
parser.add_argument('project_id', help='Your Google Cloud project ID.')
88-
parser.add_argument('date', help='Date YYYY/MM/DD.')
89-
parser.add_argument('time', help='Time (24hr) HH:MM.')
90-
parser.add_argument('source_bucket', help='Source bucket name.')
91-
parser.add_argument('access_key', help='Your AWS access key id.')
88+
parser.add_argument('start_date', help='Date YYYY/MM/DD.')
89+
parser.add_argument('start_time', help='UTC Time (24hr) HH:MM:SS.')
90+
parser.add_argument('source_bucket', help='AWS source bucket name.')
91+
parser.add_argument('access_key_id', help='Your AWS access key id.')
9292
parser.add_argument('secret_access_key', help='Your AWS secret access '
9393
'key.')
94-
parser.add_argument('sink_bucket', help='Sink bucket name.')
94+
parser.add_argument('sink_bucket', help='GCS sink bucket name.')
9595

9696
args = parser.parse_args()
97-
date = datetime.datetime.strptime(args.date, '%Y/%m/%d')
98-
time = datetime.datetime.strptime(args.time, '%H:%M')
97+
start_date = datetime.datetime.strptime(args.start_date, '%Y/%m/%d')
98+
start_time = datetime.datetime.strptime(args.start_time, '%H:%M:%S')
9999

100100
main(
101101
args.description,
102102
args.project_id,
103-
date.year,
104-
date.month,
105-
date.day,
106-
time.hour,
107-
time.minute,
103+
start_date,
104+
start_time,
108105
args.source_bucket,
109-
args.access_key,
106+
args.access_key_id,
110107
args.secret_access_key,
111108
args.sink_bucket)
112109
# [END all]

‎storage/transfer_service/nearline_request.py

Lines changed: 20 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -13,8 +13,8 @@
1313

1414
# [START all]
1515

16-
"""Command-line sample that creates a one-time transfer from Google Cloud
17-
Storage standard class to the Nearline storage class."
16+
"""Command-line sample that creates a daily transfer from a standard
17+
GCS bucket to a Nearline GCS bucket for objects untouched for 30 days.
1818
1919
This sample is used on this page:
2020
@@ -31,27 +31,26 @@
3131

3232

3333
# [START main]
34-
def main(description, project_id, year, month, day, hours, minutes,
35-
source_bucket, sink_bucket):
36-
"""Create a transfer from the Google Cloud Storage Standard class to the
37-
Nearline Storage class."""
34+
def main(description, project_id, start_date, start_time, source_bucket,
35+
sink_bucket):
36+
"""Create a daily transfer from Standard to Nearline Storage class."""
3837
storagetransfer = googleapiclient.discovery.build('storagetransfer', 'v1')
3938

4039
# Edit this template with desired parameters.
41-
# Specify times below using US Pacific Time Zone.
4240
transfer_job = {
4341
'description': description,
4442
'status': 'ENABLED',
4543
'projectId': project_id,
4644
'schedule': {
4745
'scheduleStartDate': {
48-
'day': day,
49-
'month': month,
50-
'year': year
46+
'day': start_date.day,
47+
'month': start_date.month,
48+
'year': start_date.year
5149
},
5250
'startTimeOfDay': {
53-
'hours': hours,
54-
'minutes': minutes
51+
'hours': start_time.hour,
52+
'minutes': start_time.minute,
53+
'seconds': start_time.second
5554
}
5655
},
5756
'transferSpec': {
@@ -62,7 +61,7 @@ def main(description, project_id, year, month, day, hours, minutes,
6261
'bucketName': sink_bucket
6362
},
6463
'objectConditions': {
65-
'minTimeElapsedSinceLastModification': '2592000s'
64+
'minTimeElapsedSinceLastModification': '2592000s' # 30 days
6665
},
6766
'transferOptions': {
6867
'deleteObjectsFromSourceAfterTransfer': 'true'
@@ -82,23 +81,20 @@ def main(description, project_id, year, month, day, hours, minutes,
8281
formatter_class=argparse.RawDescriptionHelpFormatter)
8382
parser.add_argument('description', help='Transfer description.')
8483
parser.add_argument('project_id', help='Your Google Cloud project ID.')
85-
parser.add_argument('date', help='Date YYYY/MM/DD.')
86-
parser.add_argument('time', help='Time (24hr) HH:MM.')
87-
parser.add_argument('source_bucket', help='Source bucket name.')
88-
parser.add_argument('sink_bucket', help='Sink bucket name.')
84+
parser.add_argument('start_date', help='Date YYYY/MM/DD.')
85+
parser.add_argument('start_time', help='UTC Time (24hr) HH:MM:SS.')
86+
parser.add_argument('source_bucket', help='Standard GCS bucket name.')
87+
parser.add_argument('sink_bucket', help='Nearline GCS bucket name.')
8988

9089
args = parser.parse_args()
91-
date = datetime.datetime.strptime(args.date, '%Y/%m/%d')
92-
time = datetime.datetime.strptime(args.time, '%H:%M')
90+
start_date = datetime.datetime.strptime(args.start_date, '%Y/%m/%d')
91+
start_time = datetime.datetime.strptime(args.start_time, '%H:%M:%S')
9392

9493
main(
9594
args.description,
9695
args.project_id,
97-
date.year,
98-
date.month,
99-
date.day,
100-
time.hour,
101-
time.minute,
96+
start_date,
97+
start_time,
10298
args.source_bucket,
10399
args.sink_bucket)
104100
# [END all]

0 commit comments

Comments
 (0)