Skip to content

Commit e828c70

Browse files
perf: Proxy integration in googlesearch
1 parent 6fd9f14 commit e828c70

File tree

4 files changed

+2
-285
lines changed

4 files changed

+2
-285
lines changed

=1.2.5

Lines changed: 0 additions & 8 deletions
This file was deleted.

notebook.ipynb

Lines changed: 0 additions & 274 deletions
This file was deleted.

scrapegraphai/nodes/search_internet_node.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -94,7 +94,7 @@ def execute(self, state: dict) -> dict:
9494

9595
self.logger.info(f"Search Query: {search_query}")
9696

97-
answer = search_on_web(query=search_query, max_results=self.max_results,
97+
answer = search_on_web(query=search_query, num_results=self.max_results,
9898
search_engine=self.search_engine, proxy=self.proxy)
9999

100100
if len(answer) == 0:

scrapegraphai/utils/research_web.py

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,7 @@ def format_proxy(proxy):
4444
password = proxy.get('password')
4545

4646
if all([username, password, server]):
47-
proxy_url = f"https://{username}:{password}@{server}"
47+
proxy_url = f"http://{username}:{password}@{server}"
4848
return proxy_url
4949
else:
5050
raise ValueError("Proxy dictionary is missing required fields.")
@@ -69,7 +69,6 @@ def filter_pdf_links(links: List[str]) -> List[str]:
6969
proxy = format_proxy(proxy)
7070

7171
if search_engine.lower() == "google":
72-
print("PROXY: ", proxy)
7372
res = []
7473
for url in google_search(query, num_results=max_results, proxy=proxy):
7574
res.append(url)

0 commit comments

Comments
 (0)