File tree Expand file tree Collapse file tree 2 files changed +40
-1
lines changed Expand file tree Collapse file tree 2 files changed +40
-1
lines changed Original file line number Diff line number Diff line change
1
+ """
2
+ Basic example of scraping pipeline using SmartScraper
3
+ """
4
+
5
+ import json
6
+ from scrapegraphai .graphs import SmartScraperMultiGraph
7
+
8
+ # ************************************************
9
+ # Define the configuration for the graph
10
+ # ************************************************
11
+ graph_config = {
12
+ "llm" : {
13
+ "model" : "ollama/llama3.1" ,
14
+ "temperature" : 0 ,
15
+ "format" : "json" , # Ollama needs the format to be specified explicitly
16
+ # "base_url": "http://localhost:11434", # set ollama URL arbitrarily
17
+ },
18
+
19
+ "verbose" : True ,
20
+ "headless" : False
21
+ }
22
+
23
+
24
+ # *******************************************************
25
+ # Create the SmartScraperMultiGraph instance and run it
26
+ # *******************************************************
27
+
28
+ multiple_search_graph = SmartScraperMultiGraph (
29
+ prompt = "Who is Marco Perini?" ,
30
+ source = [
31
+ "https://perinim.github.io/" ,
32
+ "https://perinim.github.io/cv/"
33
+ ],
34
+ schema = None ,
35
+ config = graph_config
36
+ )
37
+
38
+ result = multiple_search_graph .run ()
39
+ print (json .dumps (result , indent = 4 ))
Original file line number Diff line number Diff line change @@ -24,7 +24,7 @@ class Projects(BaseModel):
24
24
"format" : "json" , # Ollama needs the format to be specified explicitly
25
25
# "base_url": "http://localhost:11434", # set ollama URL arbitrarily
26
26
},
27
-
27
+
28
28
"verbose" : True ,
29
29
"headless" : False
30
30
}
You can’t perform that action at this time.
0 commit comments