3
3
import os
4
4
import sys
5
5
from http import HTTPStatus
6
- from typing import Any , Callable , Dict , Optional
6
+ from typing import Any , Callable , Dict , Optional , Tuple
7
7
8
8
import httpx
9
9
10
10
from ._errors import ApifyApiError , InvalidResponseBodyError , _is_retryable_error
11
11
from ._types import JSONSerializable
12
- from ._utils import _is_content_type_json , _is_content_type_text , _is_content_type_xml , _retry_with_exp_backoff
12
+ from ._utils import _is_content_type_json , _is_content_type_text , _is_content_type_xml , _retry_with_exp_backoff , _retry_with_exp_backoff_async
13
13
from ._version import __version__
14
14
15
15
DEFAULT_BACKOFF_EXPONENTIAL_FACTOR = 2
16
16
DEFAULT_BACKOFF_RANDOM_FACTOR = 1
17
17
18
18
19
- class _HTTPClient :
19
+ class _BaseHTTPClient :
20
20
def __init__ (
21
21
self ,
22
22
* ,
@@ -41,28 +41,52 @@ def __init__(
41
41
headers ['Authorization' ] = f'Bearer { token } '
42
42
43
43
self .httpx_client = httpx .Client (headers = headers , follow_redirects = True , timeout = timeout_secs )
44
+ self .httpx_async_client = httpx .AsyncClient (headers = headers , follow_redirects = True , timeout = timeout_secs )
44
45
45
- def call (
46
+ @staticmethod
47
+ def _maybe_parse_response (response : httpx .Response ) -> Any :
48
+ if response .status_code == HTTPStatus .NO_CONTENT :
49
+ return None
50
+
51
+ content_type = ''
52
+ if 'content-type' in response .headers :
53
+ content_type = response .headers ['content-type' ].split (';' )[0 ].strip ()
54
+
55
+ try :
56
+ if _is_content_type_json (content_type ):
57
+ return response .json ()
58
+ elif _is_content_type_xml (content_type ) or _is_content_type_text (content_type ):
59
+ return response .text
60
+ else :
61
+ return response .content
62
+ except ValueError as err :
63
+ raise InvalidResponseBodyError (response ) from err
64
+
65
+ @staticmethod
66
+ def _parse_params (params : Optional [Dict ]) -> Optional [Dict ]:
67
+ if params is None :
68
+ return None
69
+
70
+ parsed_params = {}
71
+ for key , value in params .items ():
72
+ # Our API needs to have boolean parameters passed as 0 or 1, therefore we have to replace them
73
+ if isinstance (value , bool ):
74
+ parsed_params [key ] = int (value )
75
+ elif value is not None :
76
+ parsed_params [key ] = value
77
+
78
+ return parsed_params
79
+
80
+ def _prepare_request_call (
46
81
self ,
47
- * ,
48
- method : str ,
49
- url : str ,
50
82
headers : Optional [Dict ] = None ,
51
83
params : Optional [Dict ] = None ,
52
84
data : Optional [Any ] = None ,
53
85
json : Optional [JSONSerializable ] = None ,
54
- stream : Optional [bool ] = None ,
55
- parse_response : Optional [bool ] = True ,
56
- ) -> httpx .Response :
57
- if stream and parse_response :
58
- raise ValueError ('Cannot stream response and parse it at the same time!' )
59
-
86
+ ) -> Tuple [Dict , Optional [Dict ], Any ]:
60
87
if json and data :
61
88
raise ValueError ('Cannot pass both "json" and "data" parameters at the same time!' )
62
89
63
- request_params = self ._parse_params (params )
64
- httpx_client = self .httpx_client
65
-
66
90
if not headers :
67
91
headers = {}
68
92
@@ -77,23 +101,48 @@ def call(
77
101
data = gzip .compress (data )
78
102
headers ['Content-Encoding' ] = 'gzip'
79
103
80
- # httpx uses `content` instead of `data` for binary content, let's rename it here to be clear about it
81
- content = data
104
+ return (
105
+ headers ,
106
+ self ._parse_params (params ),
107
+ data ,
108
+ )
109
+
110
+
111
+ class _HTTPClient (_BaseHTTPClient ):
112
+ def call (
113
+ self ,
114
+ * ,
115
+ method : str ,
116
+ url : str ,
117
+ headers : Optional [Dict ] = None ,
118
+ params : Optional [Dict ] = None ,
119
+ data : Optional [Any ] = None ,
120
+ json : Optional [JSONSerializable ] = None ,
121
+ stream : Optional [bool ] = None ,
122
+ parse_response : Optional [bool ] = True ,
123
+ ) -> httpx .Response :
124
+ if stream and parse_response :
125
+ raise ValueError ('Cannot stream response and parse it at the same time!' )
126
+
127
+ headers , params , content = self ._prepare_request_call (headers , params , data , json )
128
+
129
+ httpx_client = self .httpx_client
82
130
83
131
def _make_request (stop_retrying : Callable , attempt : int ) -> httpx .Response :
84
132
try :
85
133
request = httpx_client .build_request (
86
134
method = method ,
87
135
url = url ,
88
136
headers = headers ,
89
- params = request_params ,
137
+ params = params ,
90
138
content = content ,
91
139
)
92
140
response = httpx_client .send (
93
141
request = request ,
94
142
stream = stream or False ,
95
143
)
96
144
145
+ # If response status is < 300, the request was successful, and we can return the result
97
146
if response .status_code < 300 :
98
147
if not stream :
99
148
if parse_response :
@@ -109,6 +158,8 @@ def _make_request(stop_retrying: Callable, attempt: int) -> httpx.Response:
109
158
stop_retrying ()
110
159
raise e
111
160
161
+ # We want to retry only requests which are server errors (status >= 500) and could resolve on their own,
162
+ # and also retry rate limited requests that throw 429 Too Many Requests errors
112
163
if response .status_code < 500 and response .status_code != HTTPStatus .TOO_MANY_REQUESTS :
113
164
stop_retrying ()
114
165
raise ApifyApiError (response , attempt )
@@ -121,36 +172,67 @@ def _make_request(stop_retrying: Callable, attempt: int) -> httpx.Response:
121
172
random_factor = DEFAULT_BACKOFF_RANDOM_FACTOR ,
122
173
)
123
174
124
- @staticmethod
125
- def _maybe_parse_response (response : httpx .Response ) -> Any :
126
- if response .status_code == HTTPStatus .NO_CONTENT :
127
- return None
128
175
129
- content_type = ''
130
- if 'content-type' in response .headers :
131
- content_type = response .headers ['content-type' ].split (';' )[0 ].strip ()
176
+ class _HTTPClientAsync (_BaseHTTPClient ):
177
+ async def call (
178
+ self ,
179
+ * ,
180
+ method : str ,
181
+ url : str ,
182
+ headers : Optional [Dict ] = None ,
183
+ params : Optional [Dict ] = None ,
184
+ data : Optional [Any ] = None ,
185
+ json : Optional [JSONSerializable ] = None ,
186
+ stream : Optional [bool ] = None ,
187
+ parse_response : Optional [bool ] = True ,
188
+ ) -> httpx .Response :
189
+ if stream and parse_response :
190
+ raise ValueError ('Cannot stream response and parse it at the same time!' )
132
191
133
- try :
134
- if _is_content_type_json (content_type ):
135
- return response .json ()
136
- elif _is_content_type_xml (content_type ) or _is_content_type_text (content_type ):
137
- return response .text
138
- else :
139
- return response .content
140
- except ValueError as err :
141
- raise InvalidResponseBodyError (response ) from err
192
+ headers , params , content = self ._prepare_request_call (headers , params , data , json )
142
193
143
- @staticmethod
144
- def _parse_params (params : Optional [Dict ]) -> Optional [Dict ]:
145
- if params is None :
146
- return None
194
+ httpx_async_client = self .httpx_async_client
147
195
148
- parsed_params = {}
149
- for key , value in params .items ():
150
- # Our API needs to have boolean parameters passed as 0 or 1, therefore we have to replace them
151
- if isinstance (value , bool ):
152
- parsed_params [key ] = int (value )
153
- elif value is not None :
154
- parsed_params [key ] = value
196
+ async def _make_request (stop_retrying : Callable , attempt : int ) -> httpx .Response :
197
+ try :
198
+ request = httpx_async_client .build_request (
199
+ method = method ,
200
+ url = url ,
201
+ headers = headers ,
202
+ params = params ,
203
+ content = content ,
204
+ )
205
+ response = await httpx_async_client .send (
206
+ request = request ,
207
+ stream = stream or False ,
208
+ )
155
209
156
- return parsed_params
210
+ # If response status is < 300, the request was successful, and we can return the result
211
+ if response .status_code < 300 :
212
+ if not stream :
213
+ if parse_response :
214
+ _maybe_parsed_body = self ._maybe_parse_response (response )
215
+ else :
216
+ _maybe_parsed_body = response .content
217
+ setattr (response , '_maybe_parsed_body' , _maybe_parsed_body ) # noqa: B010
218
+
219
+ return response
220
+
221
+ except Exception as e :
222
+ if not _is_retryable_error (e ):
223
+ stop_retrying ()
224
+ raise e
225
+
226
+ # We want to retry only requests which are server errors (status >= 500) and could resolve on their own,
227
+ # and also retry rate limited requests that throw 429 Too Many Requests errors
228
+ if response .status_code < 500 and response .status_code != HTTPStatus .TOO_MANY_REQUESTS :
229
+ stop_retrying ()
230
+ raise ApifyApiError (response , attempt )
231
+
232
+ return await _retry_with_exp_backoff_async (
233
+ _make_request ,
234
+ max_retries = self .max_retries ,
235
+ backoff_base_millis = self .min_delay_between_retries_millis ,
236
+ backoff_factor = DEFAULT_BACKOFF_EXPONENTIAL_FACTOR ,
237
+ random_factor = DEFAULT_BACKOFF_RANDOM_FACTOR ,
238
+ )
0 commit comments