feat(smart-scraper-multi): add schema to graphs and created SmartScraperMultiGraph

Author: PeriniM

examples/knowledge_graph/kg_custom_graph.py

@@ -0,0 +1,134 @@
+"""
+Example of custom graph for creating a knowledge graph
+"""
+
+import os, json
+from dotenv import load_dotenv
+
+from langchain_openai import OpenAIEmbeddings
+from scrapegraphai.models import OpenAI
+from scrapegraphai.graphs import BaseGraph, SmartScraperGraph
+from scrapegraphai.nodes import GraphIteratorNode, MergeAnswersNode, KnowledgeGraphNode
+
+load_dotenv()
+
+# ************************************************
+# Define the output schema
+# ************************************************
+
+schema= """{ 
+    "Job Postings": { 
+        "Company x": [ 
+            { 
+                "title": "...", 
+                "description": "...", 
+                "location": "...", 
+                "date_posted": "..", 
+                "requirements": ["...", "...", "..."] 
+            }, 
+            { 
+                "title": "...", 
+                "description": "...", 
+                "location": "...", 
+                "date_posted": "..", 
+                "requirements": ["...", "...", "..."] 
+            } 
+        ], 
+        "Company y": [ 
+            { 
+                "title": "...", 
+                "description": "...", 
+                "location": "...", 
+                "date_posted": "..", 
+                "requirements": ["...", "...", "..."] 
+            } 
+        ] 
+    } 
+}"""
+
+# ************************************************
+# Define the configuration for the graph
+# ************************************************
+
+openai_key = os.getenv("OPENAI_APIKEY")
+
+graph_config = {
+    "llm": {
+        "api_key": openai_key,
+        "model": "gpt-4o",
+    },
+    "verbose": True,
+    "headless": False,
+}
+
+# ************************************************
+# Define the graph nodes
+# ************************************************
+
+llm_model = OpenAI(graph_config["llm"])
+embedder = OpenAIEmbeddings(api_key=llm_model.openai_api_key)
+
+smart_scraper_instance = SmartScraperGraph(
+    prompt="",
+    source="",
+    config=graph_config,
+)
+
+# ************************************************
+# Define the graph nodes
+# ************************************************
+
+graph_iterator_node = GraphIteratorNode(
+    input="user_prompt & urls",
+    output=["results"],
+    node_config={
+        "graph_instance": smart_scraper_instance,
+    }
+)
+
+merge_answers_node = MergeAnswersNode(
+    input="user_prompt & results",
+    output=["answer"],
+    node_config={
+        "llm_model": llm_model,
+        "schema": schema
+    }
+)
+
+knowledge_graph_node = KnowledgeGraphNode(
+    input="user_prompt & answer",
+    output=["kg"],
+    node_config={
+        "llm_model": llm_model,
+    }
+)
+
+graph = BaseGraph(
+    nodes=[
+        graph_iterator_node,
+        merge_answers_node,
+        knowledge_graph_node
+    ],
+    edges=[
+        (graph_iterator_node, merge_answers_node),
+        (merge_answers_node, knowledge_graph_node)
+    ],
+    entry_point=graph_iterator_node
+)
+
+# ************************************************
+# Execute the graph
+# ************************************************
+
+result, execution_info = graph.execute({
+    "user_prompt": "List me all the Machine Learning Engineer job postings",
+    "urls": [
+        "https://www.linkedin.com/jobs/machine-learning-engineer-offerte-di-lavoro/?currentJobId=3889037104&originalSubdomain=it",
+        "https://www.glassdoor.com/Job/italy-machine-learning-engineer-jobs-SRCH_IL.0,5_IN120_KO6,31.html",
+        "https://it.indeed.com/jobs?q=ML+engineer&vjk=3c2e6d27601ffaaa"
+        ],
+})
+
+# get the answer from the result
+result = result.get("answer", "No answer found.")
+print(json.dumps(result, indent=4))

examples/openai/custom_graph_openai.py

@@ -46,7 +46,7 @@
 
 fetch_node = FetchNode(
     input="url | local_dir",
-    output=["doc"],
+    output=["doc", "link_urls", "img_urls"],
     node_config={
         "verbose": True,
         "headless": True,

examples/openai/multiple_search_openai.py

@@ -19,7 +19,7 @@
 graph_config = {
     "llm": {
         "api_key": openai_key,
-        "model": "gpt-4-turbo",
+        "model": "gpt-4o",
     },
     "verbose": True,
     "headless": True,

examples/openai/omni_scraper_openai.py

@@ -20,7 +20,7 @@
         "model": "gpt-4o",
     },
     "max_results": 2,
-    "max_images": 5,
+    "max_images": 1,
     "verbose": True,
 }
 

examples/openai/omni_search_graph_openai.py

@@ -0,0 +1,41 @@
+""" 
+Basic example of scraping pipeline using SmartScraper
+"""
+
+import os, json
+from dotenv import load_dotenv
+from scrapegraphai.graphs import SmartScraperMultiGraph
+
+load_dotenv()
+
+# ************************************************
+# Define the configuration for the graph
+# ************************************************
+
+openai_key = os.getenv("OPENAI_APIKEY")
+
+graph_config = {
+    "llm": {
+        "api_key": openai_key,
+        "model": "gpt-4o",
+    },
+    "verbose": True,
+    "headless": False,
+}
+
+# *******************************************************
+# Create the SmartScraperMultiGraph instance and run it
+# *******************************************************
+
+multiple_search_graph = SmartScraperMultiGraph(
+    prompt="Who is Marco Perini?",
+    source= [
+        "https://perinim.github.io/",
+        "https://perinim.github.io/cv/"
+        ],
+    schema=None,
+    config=graph_config
+)
+
+result = multiple_search_graph.run()
+print(json.dumps(result, indent=4))

examples/openai/smart_scraper_multi_openai.py

@@ -0,0 +1,59 @@
+""" 
+Basic example of scraping pipeline using SmartScraper
+"""
+
+import os, json
+from dotenv import load_dotenv
+from scrapegraphai.graphs import SmartScraperGraph
+
+load_dotenv()
+
+# ************************************************
+# Define the output schema for the graph
+# ************************************************
+
+schema= """
+    { 
+    "Projects": [
+        "Project #": 
+            { 
+                "title": "...", 
+                "description": "...", 
+            }, 
+        "Project #": 
+            { 
+                "title": "...", 
+                "description": "...", 
+            } 
+        ] 
+    } 
+"""
+
+# ************************************************
+# Define the configuration for the graph
+# ************************************************
+
+openai_key = os.getenv("OPENAI_APIKEY")
+
+graph_config = {
+    "llm": {
+        "api_key":openai_key,
+        "model": "gpt-3.5-turbo",
+    },
+    "verbose": True,
+    "headless": False,
+}
+
+# ************************************************
+# Create the SmartScraperGraph instance and run it
+# ************************************************
+
+smart_scraper_graph = SmartScraperGraph(
+    prompt="List me all the projects with their description",
+    source="https://perinim.github.io/projects/",
+    schema=schema,
+    config=graph_config
+)
+
+result = smart_scraper_graph.run()
+print(json.dumps(result, indent=4))

examples/openai/smart_scraper_schema_openai.py

@@ -45,6 +45,10 @@ certifi==2024.2.2
     # via requests
 charset-normalizer==3.3.2
     # via requests
+colorama==0.4.6
+    # via ipython
+    # via pytest
+    # via tqdm
 dataclasses-json==0.6.6
     # via langchain
     # via langchain-community
@@ -100,6 +104,7 @@ graphviz==0.20.3
     # via scrapegraphai
 greenlet==3.0.3
     # via playwright
+    # via sqlalchemy
 groq==0.5.0
     # via langchain-groq
 grpcio==1.63.0
@@ -212,8 +217,6 @@ pandas==2.2.2
     # via scrapegraphai
 parso==0.8.4
     # via jedi
-pexpect==4.9.0
-    # via ipython
 playwright==1.43.0
     # via scrapegraphai
 pluggy==1.5.0
@@ -230,8 +233,6 @@ protobuf==4.25.3
     # via googleapis-common-protos
     # via grpcio-status
     # via proto-plus
-ptyprocess==0.7.0
-    # via pexpect
 pure-eval==0.2.2
     # via stack-data
 pyasn1==0.6.0