File tree Expand file tree Collapse file tree 2 files changed +9
-7
lines changed Expand file tree Collapse file tree 2 files changed +9
-7
lines changed Original file line number Diff line number Diff line change 12
12
# See the License for the specific language governing permissions and
13
13
# limitations under the License.
14
14
15
+ # [START composer_dataanalyticstutorial_azure_dag]
15
16
import datetime
16
17
17
18
from airflow import models
53
54
f"{ BQ_DESTINATION_DATASET_NAME } .{ BQ_DESTINATION_TABLE_NAME } " ,
54
55
f"{ BQ_DESTINATION_DATASET_NAME } .{ BQ_NORMALIZED_TABLE_NAME } " ,
55
56
],
56
-
57
57
},
58
58
"environment_config" : {
59
59
"execution_config" : {
60
60
"service_account" : "{{var.value.dataproc_service_account}}"
61
61
}
62
- }
62
+ },
63
63
}
64
64
65
65
yesterday = datetime .datetime .combine (
73
73
# To email on failure or retry set 'email' arg to your email and enable
74
74
# emailing here.
75
75
"email_on_failure" : False ,
76
- "email_on_retry" : False
76
+ "email_on_retry" : False ,
77
77
}
78
78
79
79
with models .DAG (
161
161
)
162
162
163
163
azure_blob_to_gcs >> load_external_dataset >> bq_join_group >> create_batch
164
+ # [END composer_dataanalyticstutorial_azure_dag]
Original file line number Diff line number Diff line change 12
12
# See the License for the specific language governing permissions and
13
13
# limitations under the License.
14
14
15
+ # [START composer_dataanalyticstutorial_aws_dag]
15
16
import datetime
16
17
17
18
from airflow import models
49
50
f"{ BQ_DESTINATION_DATASET_NAME } .{ BQ_DESTINATION_TABLE_NAME } " ,
50
51
f"{ BQ_DESTINATION_DATASET_NAME } .{ BQ_NORMALIZED_TABLE_NAME } " ,
51
52
],
52
-
53
53
},
54
54
"environment_config" : {
55
55
"execution_config" : {
56
56
"service_account" : "{{var.value.dataproc_service_account}}"
57
57
}
58
- }
58
+ },
59
59
}
60
60
61
61
yesterday = datetime .datetime .combine (
69
69
# To email on failure or retry set 'email' arg to your email and enable
70
70
# emailing here.
71
71
"email_on_failure" : False ,
72
- "email_on_retry" : False
72
+ "email_on_retry" : False ,
73
73
}
74
74
75
75
with models .DAG (
106
106
{"name" : "Holiday" , "type" : "STRING" },
107
107
],
108
108
skip_leading_rows = 1 ,
109
- write_disposition = "WRITE_TRUNCATE"
109
+ write_disposition = "WRITE_TRUNCATE" ,
110
110
)
111
111
112
112
with TaskGroup ("join_bq_datasets" ) as bq_join_group :
148
148
)
149
149
150
150
s3_to_gcs_op >> load_external_dataset >> bq_join_group >> create_batch
151
+ # [END composer_dataanalyticstutorial_aws_dag]
You can’t perform that action at this time.
0 commit comments