|
1 | 1 | """
|
2 |
| -Basic example of scraping pipeline using ScriptCreatorGraph |
| 2 | +Basic example of scraping pipeline using SmartScraper |
3 | 3 | """
|
4 | 4 |
|
5 | 5 | import os
|
| 6 | +import json |
6 | 7 | from dotenv import load_dotenv
|
7 |
| -from scrapegraphai.graphs import ScriptCreatorGraph |
| 8 | +from scrapegraphai.graphs import SmartScraperGraph |
8 | 9 | from scrapegraphai.utils import prettify_exec_info
|
9 | 10 |
|
10 | 11 | load_dotenv()
|
|
13 | 14 | # Define the configuration for the graph
|
14 | 15 | # ************************************************
|
15 | 16 |
|
16 |
| -openai_key = os.getenv("OPENAI_APIKEY") |
17 | 17 |
|
18 | 18 | graph_config = {
|
19 | 19 | "llm": {
|
20 |
| - "api_key": openai_key, |
21 |
| - "model": "openai/gpt-4o", |
| 20 | + "api_key": os.getenv("OPENAI_API_KEY"), |
| 21 | + "model": "gpt-4o", |
22 | 22 | },
|
23 |
| - "library": "beautifulsoup" |
| 23 | + "verbose": True, |
| 24 | + "headless": False, |
24 | 25 | }
|
25 | 26 |
|
26 | 27 | # ************************************************
|
27 |
| -# Create the ScriptCreatorGraph instance and run it |
| 28 | +# Create the SmartScraperGraph instance and run it |
28 | 29 | # ************************************************
|
29 | 30 |
|
30 |
| -script_creator_graph = ScriptCreatorGraph( |
31 |
| - prompt="List me all the projects with their description.", |
32 |
| - # also accepts a string with the already downloaded HTML code |
33 |
| - source="https://perinim.github.io/projects", |
| 31 | +smart_scraper_graph = SmartScraperGraph( |
| 32 | + prompt="List me what does the company do, the name and a contact email.", |
| 33 | + source="https://scrapegraphai.com/", |
34 | 34 | config=graph_config
|
35 | 35 | )
|
36 | 36 |
|
37 |
| -result = script_creator_graph.run() |
38 |
| -print(result) |
| 37 | +result = smart_scraper_graph.run() |
| 38 | +print(json.dumps(result, indent=4)) |
39 | 39 |
|
40 | 40 | # ************************************************
|
41 | 41 | # Get graph execution info
|
42 | 42 | # ************************************************
|
43 | 43 |
|
44 |
| -graph_exec_info = script_creator_graph.get_execution_info() |
| 44 | +graph_exec_info = smart_scraper_graph.get_execution_info() |
45 | 45 | print(prettify_exec_info(graph_exec_info))
|
46 |
| - |
0 commit comments