File tree Expand file tree Collapse file tree 1 file changed +40
-0
lines changed Expand file tree Collapse file tree 1 file changed +40
-0
lines changed Original file line number Diff line number Diff line change
1
+ """
2
+ Basic example of scraping pipeline using SmartScraper
3
+ """
4
+
5
+ import os
6
+ import json
7
+ from dotenv import load_dotenv
8
+ from scrapegraphai .graphs import SmartScraperMultiConcatGraph
9
+
10
+ load_dotenv ()
11
+
12
+ # ************************************************
13
+ # Define the configuration for the graph
14
+ # ************************************************
15
+
16
+ gemini_key = os .getenv ("GOOGLE_APIKEY" )
17
+
18
+ graph_config = {
19
+ "llm" : {
20
+ "api_key" : gemini_key ,
21
+ "model" : "google_genai/gemini-pro" ,
22
+ },
23
+ }
24
+
25
+ # *******************************************************
26
+ # Create the SmartScraperMultiGraph instance and run it
27
+ # *******************************************************
28
+
29
+ multiple_search_graph = SmartScraperMultiConcatGraph (
30
+ prompt = "Who is Marco Perini?" ,
31
+ source = [
32
+ "https://perinim.github.io/" ,
33
+ "https://perinim.github.io/cv/"
34
+ ],
35
+ schema = None ,
36
+ config = graph_config
37
+ )
38
+
39
+ result = multiple_search_graph .run ()
40
+ print (json .dumps (result , indent = 4 ))
You can’t perform that action at this time.
0 commit comments