Skip to content

Commit b84883b

Browse files
VinciGit00VinciGit00
committed
add smartscraper lite
1 parent 2991ca8 commit b84883b

18 files changed

+544
-0
lines changed

‎examples/anthropic/smart_scraper_lite_anthropic.py

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
"""
2+
Basic example of scraping pipeline using SmartScraper
3+
"""
4+
import os
5+
import json
6+
from dotenv import load_dotenv
7+
from scrapegraphai.graphs import SmartScraperLiteGraph
8+
from scrapegraphai.utils import prettify_exec_info
9+
10+
load_dotenv()
11+
12+
graph_config = {
13+
"llm": {
14+
"api_key": os.getenv("ANTHROPIC_API_KEY"),
15+
"model": "anthropic/claude-3-haiku-20240307",
16+
},
17+
"verbose": True,
18+
"headless": False,
19+
}
20+
21+
smart_scraper_lite_graph = SmartScraperLiteGraph(
22+
prompt="Who is Marco Perini?",
23+
source="https://perinim.github.io/",
24+
config=graph_config
25+
)
26+
27+
result = smart_scraper_lite_graph.run()
28+
print(json.dumps(result, indent=4))
29+
30+
graph_exec_info = smart_scraper_lite_graph.get_execution_info()
31+
print(prettify_exec_info(graph_exec_info))
32+

‎examples/azure/smart_scraper_lite_azure.py

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
"""
2+
Basic example of scraping pipeline using SmartScraper
3+
"""
4+
import os
5+
import json
6+
from dotenv import load_dotenv
7+
from scrapegraphai.graphs import SmartScraperLiteGraph
8+
from scrapegraphai.utils import prettify_exec_info
9+
10+
load_dotenv()
11+
12+
graph_config = {
13+
"llm": {
14+
"api_key": os.environ["AZURE_OPENAI_KEY"],
15+
"model": "azure_openai/gpt-4o"
16+
},
17+
"verbose": True,
18+
"headless": False
19+
}
20+
21+
smart_scraper_lite_graph = SmartScraperLiteGraph(
22+
prompt="Who is Marco Perini?",
23+
source="https://perinim.github.io/",
24+
config=graph_config
25+
)
26+
27+
result = smart_scraper_lite_graph.run()
28+
print(json.dumps(result, indent=4))
29+
30+
graph_exec_info = smart_scraper_lite_graph.get_execution_info()
31+
print(prettify_exec_info(graph_exec_info))

‎examples/bedrock/smart_scraper_lite_bedrock.py

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
"""
2+
Basic example of scraping pipeline using SmartScraper
3+
"""
4+
import json
5+
from scrapegraphai.graphs import SmartScraperLiteGraph
6+
from scrapegraphai.utils import prettify_exec_info
7+
8+
graph_config = {
9+
"llm": {
10+
"client": "client_name",
11+
"model": "bedrock/anthropic.claude-3-sonnet-20240229-v1:0",
12+
"temperature": 0.0
13+
}
14+
}
15+
16+
smart_scraper_lite_graph = SmartScraperLiteGraph(
17+
prompt="Who is Marco Perini?",
18+
source="https://perinim.github.io/",
19+
config=graph_config
20+
)
21+
22+
result = smart_scraper_lite_graph.run()
23+
print(json.dumps(result, indent=4))
24+
25+
graph_exec_info = smart_scraper_lite_graph.get_execution_info()
26+
print(prettify_exec_info(graph_exec_info))

‎examples/deepseek/smart_scraper_lite_deepseek.py

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
"""
2+
Basic example of scraping pipeline using SmartScraper
3+
"""
4+
import os
5+
import json
6+
from dotenv import load_dotenv
7+
from scrapegraphai.graphs import SmartScraperLiteGraph
8+
from scrapegraphai.utils import prettify_exec_info
9+
10+
load_dotenv()
11+
12+
graph_config = {
13+
"llm": {
14+
"api_key": os.getenv("DEEPSEEK_API_KEY"),
15+
"model": "deepseek/deepseek-coder-33b-instruct",
16+
},
17+
"verbose": True,
18+
"headless": False,
19+
}
20+
21+
smart_scraper_lite_graph = SmartScraperLiteGraph(
22+
prompt="Who is Marco Perini?",
23+
source="https://perinim.github.io/",
24+
config=graph_config
25+
)
26+
27+
result = smart_scraper_lite_graph.run()
28+
print(json.dumps(result, indent=4))
29+
30+
graph_exec_info = smart_scraper_lite_graph.get_execution_info()
31+
print(prettify_exec_info(graph_exec_info))

‎examples/ernie/smart_scraper_lite_ernie.py

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
"""
2+
Basic example of scraping pipeline using SmartScraper
3+
"""
4+
import os
5+
import json
6+
from dotenv import load_dotenv
7+
from scrapegraphai.graphs import SmartScraperLiteGraph
8+
from scrapegraphai.utils import prettify_exec_info
9+
10+
load_dotenv()
11+
12+
graph_config = {
13+
"llm": {
14+
"api_key": os.getenv("ERNIE_API_KEY"),
15+
"model": "ernie/ernie-bot-4",
16+
},
17+
"verbose": True,
18+
"headless": False,
19+
}
20+
21+
smart_scraper_lite_graph = SmartScraperLiteGraph(
22+
prompt="Who is Marco Perini?",
23+
source="https://perinim.github.io/",
24+
config=graph_config
25+
)
26+
27+
result = smart_scraper_lite_graph.run()
28+
print(json.dumps(result, indent=4))
29+
30+
graph_exec_info = smart_scraper_lite_graph.get_execution_info()
31+
print(prettify_exec_info(graph_exec_info))

‎examples/fireworks/smart_scraper_lite_fireworks.py

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
"""
2+
Basic example of scraping pipeline using SmartScraper
3+
"""
4+
import os
5+
import json
6+
from dotenv import load_dotenv
7+
from scrapegraphai.graphs import SmartScraperLiteGraph
8+
from scrapegraphai.utils import prettify_exec_info
9+
10+
load_dotenv()
11+
12+
graph_config = {
13+
"llm": {
14+
"api_key": os.getenv("FIREWORKS_API_KEY"),
15+
"model": "fireworks/llama-v2-70b-chat",
16+
},
17+
"verbose": True,
18+
"headless": False,
19+
}
20+
21+
smart_scraper_lite_graph = SmartScraperLiteGraph(
22+
prompt="Who is Marco Perini?",
23+
source="https://perinim.github.io/",
24+
config=graph_config
25+
)
26+
27+
result = smart_scraper_lite_graph.run()
28+
print(json.dumps(result, indent=4))
29+
30+
graph_exec_info = smart_scraper_lite_graph.get_execution_info()
31+
print(prettify_exec_info(graph_exec_info))

‎examples/google_genai/smart_scraper_lite_google_genai.py

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
"""
2+
Basic example of scraping pipeline using SmartScraper
3+
"""
4+
import os
5+
import json
6+
from dotenv import load_dotenv
7+
from scrapegraphai.graphs import SmartScraperLiteGraph
8+
from scrapegraphai.utils import prettify_exec_info
9+
10+
load_dotenv()
11+
12+
graph_config = {
13+
"llm": {
14+
"api_key": os.getenv("GOOGLE_API_KEY"),
15+
"model": "gemini-pro",
16+
},
17+
"verbose": True,
18+
"headless": False,
19+
}
20+
21+
smart_scraper_lite_graph = SmartScraperLiteGraph(
22+
prompt="Who is Marco Perini?",
23+
source="https://perinim.github.io/",
24+
config=graph_config
25+
)
26+
27+
result = smart_scraper_lite_graph.run()
28+
print(json.dumps(result, indent=4))
29+
30+
graph_exec_info = smart_scraper_lite_graph.get_execution_info()
31+
print(prettify_exec_info(graph_exec_info))

‎examples/google_vertexai/smart_scraper_lite_google_vertexai.py

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
"""
2+
Basic example of scraping pipeline using SmartScraper
3+
"""
4+
import os
5+
import json
6+
from dotenv import load_dotenv
7+
from scrapegraphai.graphs import SmartScraperLiteGraph
8+
from scrapegraphai.utils import prettify_exec_info
9+
10+
load_dotenv()
11+
12+
graph_config = {
13+
"llm": {
14+
"project": os.getenv("GOOGLE_CLOUD_PROJECT"),
15+
"location": "us-central1",
16+
"model": "text-bison@001",
17+
},
18+
"verbose": True,
19+
"headless": False,
20+
}
21+
22+
smart_scraper_lite_graph = SmartScraperLiteGraph(
23+
prompt="Who is Marco Perini?",
24+
source="https://perinim.github.io/",
25+
config=graph_config
26+
)
27+
28+
result = smart_scraper_lite_graph.run()
29+
print(json.dumps(result, indent=4))
30+
31+
graph_exec_info = smart_scraper_lite_graph.get_execution_info()
32+
print(prettify_exec_info(graph_exec_info))
33+

‎examples/google_vertexai/smart_scraper_multi_lite_vertex.py

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
"""
2+
Basic example of scraping pipeline using SmartScraper
3+
"""
4+
import os
5+
import json
6+
from dotenv import load_dotenv
7+
from scrapegraphai.graphs import SmartScraperMultiLiteGraph
8+
from scrapegraphai.utils import prettify_exec_info
9+
10+
load_dotenv()
11+
12+
# ************************************************
13+
# Define the configuration for the graph
14+
# ************************************************
15+
16+
graph_config = {
17+
"llm": {
18+
"project": os.getenv("GOOGLE_CLOUD_PROJECT"),
19+
"location": "us-central1",
20+
"model": "text-bison@001",
21+
},
22+
"verbose": True,
23+
"headless": False,
24+
}
25+
26+
# ************************************************
27+
# Create the SmartScraperGraph instance and run it
28+
# ************************************************
29+
30+
smart_scraper_multi_lite_graph = SmartScraperMultiLiteGraph(
31+
prompt="Who is Marco Perini?",
32+
source= [
33+
"https://perinim.github.io/",
34+
"https://perinim.github.io/cv/"
35+
],
36+
config=graph_config
37+
)
38+
39+
result = smart_scraper_multi_lite_graph.run()
40+
print(json.dumps(result, indent=4))
41+
42+
# ************************************************
43+
# Get graph execution info
44+
# ************************************************
45+
46+
graph_exec_info = smart_scraper_multi_lite_graph.get_execution_info()
47+
print(prettify_exec_info(graph_exec_info))

‎examples/groq/smart_scraper_lite_groq.py

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
"""
2+
Basic example of scraping pipeline using SmartScraper
3+
"""
4+
import os
5+
import json
6+
from dotenv import load_dotenv
7+
from scrapegraphai.graphs import SmartScraperLiteGraph
8+
from scrapegraphai.utils import prettify_exec_info
9+
10+
load_dotenv()
11+
12+
graph_config = {
13+
"llm": {
14+
"api_key": os.getenv("GROQ_API_KEY"),
15+
"model": "mixtral-8x7b-32768",
16+
},
17+
"verbose": True,
18+
"headless": False,
19+
}
20+
21+
smart_scraper_lite_graph = SmartScraperLiteGraph(
22+
prompt="Who is Marco Perini?",
23+
source="https://perinim.github.io/",
24+
config=graph_config
25+
)
26+
27+
result = smart_scraper_lite_graph.run()
28+
print(json.dumps(result, indent=4))
29+
30+
graph_exec_info = smart_scraper_lite_graph.get_execution_info()
31+
print(prettify_exec_info(graph_exec_info))

‎examples/huggingfacehub/smart_scraper_lite_huggingfacehub.py

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
"""
2+
Basic example of scraping pipeline using SmartScraper
3+
"""
4+
import os
5+
import json
6+
from dotenv import load_dotenv
7+
from scrapegraphai.graphs import SmartScraperLiteGraph
8+
from scrapegraphai.utils import prettify_exec_info
9+
10+
load_dotenv()
11+
12+
graph_config = {
13+
"llm": {
14+
"api_key": os.getenv("HUGGINGFACEHUB_API_TOKEN"),
15+
"model": "huggingfacehub/meta-llama/Llama-2-70b-chat-hf",
16+
},
17+
"verbose": True,
18+
"headless": False,
19+
}
20+
21+
smart_scraper_lite_graph = SmartScraperLiteGraph(
22+
prompt="Who is Marco Perini?",
23+
source="https://perinim.github.io/",
24+
config=graph_config
25+
)
26+
27+
result = smart_scraper_lite_graph.run()
28+
print(json.dumps(result, indent=4))
29+
30+
graph_exec_info = smart_scraper_lite_graph.get_execution_info()
31+
print(prettify_exec_info(graph_exec_info))

‎examples/local_models/smart_scraper_lite_ollama.py

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
"""
2+
Basic example of scraping pipeline using SmartScraper
3+
4+
"""
5+
import json
6+
from scrapegraphai.graphs import SmartScraperLiteGraph
7+
from scrapegraphai.utils import prettify_exec_info
8+
9+
graph_config = {
10+
"llm": {
11+
"model": "ollama/llama3.1",
12+
"temperature": 0,
13+
"format": "json",
14+
"base_url": "http://localhost:11434",
15+
},
16+
"verbose": True,
17+
"headless": False
18+
}
19+
20+
smart_scraper_lite_graph = SmartScraperLiteGraph(
21+
prompt="Who is Marco Perini?",
22+
source="https://perinim.github.io/",
23+
config=graph_config
24+
)
25+
26+
result = smart_scraper_lite_graph.run()
27+
print(json.dumps(result, indent=4))
28+
29+
graph_exec_info = smart_scraper_lite_graph.get_execution_info()
30+
print(prettify_exec_info(graph_exec_info))

0 commit comments

Comments
 (0)