Skip to content

Commit 5d6d996

Browse files
committed
fix(proxy-rotation): removed max_shape duplicate
1 parent 1e9a564 commit 5d6d996

File tree

3 files changed

+10
-15
lines changed

3 files changed

+10
-15
lines changed

examples/openai/proxy.py

Lines changed: 0 additions & 11 deletions
This file was deleted.

examples/openai/smart_scraper_openai.py

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -28,10 +28,10 @@
2828
"server": "broker",
2929
"criteria": {
3030
"anonymous": True,
31-
# "secure": True,
31+
"secure": True,
3232
"countryset": {"IT"},
33-
"timeout": 5.0,
34-
"max_shape": 2
33+
"timeout": 10.0,
34+
"max_shape": 3
3535
},
3636
},
3737
}

scrapegraphai/utils/proxy_rotation.py

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -161,7 +161,13 @@ def _search_proxy(proxy: Proxy) -> ProxySettings:
161161
Returns:
162162
A 'playwright' compliant proxy configuration.
163163
"""
164-
server = search_proxy_servers(**proxy.get("criteria", {}))[0]
164+
165+
166+
# remove max_shape from criteria
167+
criteria = proxy.get("criteria", {}).copy()
168+
criteria.pop("max_shape", None)
169+
170+
server = search_proxy_servers(max_shape=1, **criteria)[0]
165171

166172
return {"server": server}
167173

0 commit comments

Comments
 (0)