fix: update all nodes that were using MergeNode or IteratorNode

LorenzoPaleari · LorenzoPaleari · commit a92dddb3e025 · 2024-09-13T04:19:40.000+02:00
diff --git a/scrapegraphai/graphs/csv_scraper_multi_graph.py b/scrapegraphai/graphs/csv_scraper_multi_graph.py
@@ -2,6 +2,7 @@
 CSVScraperMultiGraph Module
 """
 
+from copy import deepcopy
 from typing import List, Optional
 from pydantic import BaseModel
 from .base_graph import BaseGraph
@@ -48,6 +49,8 @@ def __init__(self, prompt: str, source: List[str],
 
         self.copy_config = safe_deepcopy(config)
 
+        self.copy_schema = deepcopy(schema)
+
         super().__init__(prompt, config, source, schema)
 
     def _create_graph(self) -> BaseGraph:
@@ -58,17 +61,18 @@ def _create_graph(self) -> BaseGraph:
             BaseGraph: A graph instance representing the web scraping and searching workflow.
         """
 
-        smart_scraper_instance = CSVScraperGraph(
-            prompt="",
-            source="",
-            config=self.copy_config,
-        )
+        # smart_scraper_instance = CSVScraperGraph(
+        #     prompt="",
+        #     source="",
+        #     config=self.copy_config,
+        # )
 
         graph_iterator_node = GraphIteratorNode(
             input="user_prompt & jsons",
             output=["results"],
             node_config={
-                "graph_instance": smart_scraper_instance,
+                "graph_instance": CSVScraperGraph,
+                "scraper_config": self.copy_config,
             }
         )
 
@@ -77,7 +81,7 @@ def _create_graph(self) -> BaseGraph:
             output=["answer"],
             node_config={
                 "llm_model": self.llm_model,
-                "schema": self.schema
+                "schema": self.copy_schema
             }
         )
 
diff --git a/scrapegraphai/graphs/json_scraper_multi_graph.py b/scrapegraphai/graphs/json_scraper_multi_graph.py
@@ -61,27 +61,29 @@ def _create_graph(self) -> BaseGraph:
             BaseGraph: A graph instance representing the web scraping and searching workflow.
         """
 
-        smart_scraper_instance = JSONScraperGraph(
-            prompt="",
-            source="",
-            config=self.copy_config,
-            schema=self.copy_schema
-        )
+        # smart_scraper_instance = JSONScraperGraph(
+        #     prompt="",
+        #     source="",
+        #     config=self.copy_config,
+        #     schema=self.copy_schema
+        # )
 
         graph_iterator_node = GraphIteratorNode(
             input="user_prompt & jsons",
             output=["results"],
             node_config={
-                "graph_instance": smart_scraper_instance,
-            }
+                "graph_instance": JSONScraperGraph,
+                "scraper_config": self.copy_config,
+            },
+            schema=self.copy_schema
         )
 
         merge_answers_node = MergeAnswersNode(
             input="user_prompt & results",
             output=["answer"],
             node_config={
                 "llm_model": self.llm_model,
-                "schema": self.schema
+                "schema": self.copy_schema
             }
         )
 
diff --git a/scrapegraphai/graphs/markdown_scraper_multi_graph.py b/scrapegraphai/graphs/markdown_scraper_multi_graph.py
@@ -55,28 +55,30 @@ def _create_graph(self) -> BaseGraph:
         Returns:
             BaseGraph: A graph instance representing the web scraping and searching workflow.
         """
-        smart_scraper_instance = MDScraperGraph(
-            prompt="",
-            source="",
-            config=self.copy_config,
-            schema=self.copy_schema
-        )
+        # smart_scraper_instance = MDScraperGraph(
+        #     prompt="",
+        #     source="",
+        #     config=self.copy_config,
+        #     schema=self.copy_schema
+        # )
 
         # Define the graph nodes
         graph_iterator_node = GraphIteratorNode(
             input="user_prompt & jsons",
             output=["results"],
             node_config={
-                "graph_instance": smart_scraper_instance,
-            }
+                "graph_instance": MDScraperGraph,
+                "scraper_config": self.copy_config,
+            },
+            schema=self.copy_schema
         )
 
         merge_answers_node = MergeAnswersNode(
             input="user_prompt & results",
             output=["answer"],
             node_config={
                 "llm_model": self.llm_model,
-                "schema": self.schema
+                "schema": self.copy_schema
             }
         )
 
diff --git a/scrapegraphai/graphs/omni_search_graph.py b/scrapegraphai/graphs/omni_search_graph.py
@@ -92,7 +92,7 @@ def _create_graph(self) -> BaseGraph:
             output=["answer"],
             node_config={
                 "llm_model": self.llm_model,
-                "schema": self.schema
+                "schema": self.copy_schema
             }
         )
 
diff --git a/scrapegraphai/graphs/pdf_scraper_multi_graph.py b/scrapegraphai/graphs/pdf_scraper_multi_graph.py
@@ -59,27 +59,29 @@ def _create_graph(self) -> BaseGraph:
             BaseGraph: A graph instance representing the web scraping and searching workflow.
         """
 
-        pdf_scraper_instance = PDFScraperGraph(
-            prompt="",
-            source="",
-            config=self.copy_config,
-            schema=self.copy_schema
-        )
+        # pdf_scraper_instance = PDFScraperGraph(
+        #     prompt="",
+        #     source="",
+        #     config=self.copy_config,
+        #     schema=self.copy_schema
+        # )
 
         graph_iterator_node = GraphIteratorNode(
             input="user_prompt & pdfs",
             output=["results"],
             node_config={
-                "graph_instance": pdf_scraper_instance,
-            }
+                "graph_instance": PDFScraperGraph,
+                "scraper_config": self.copy_config,
+            },
+            schema=self.copy_schema
         )
 
         merge_answers_node = MergeAnswersNode(
             input="user_prompt & results",
             output=["answer"],
             node_config={
                 "llm_model": self.llm_model,
-                "schema": self.schema
+                "schema": self.copy_schema
             }
         )
 
diff --git a/scrapegraphai/graphs/search_graph.py b/scrapegraphai/graphs/search_graph.py
@@ -93,7 +93,7 @@ def _create_graph(self) -> BaseGraph:
             output=["answer"],
             node_config={
                 "llm_model": self.llm_model,
-                "schema": self.schema
+                "schema": self.copy_schema
             }
         )
 
diff --git a/scrapegraphai/graphs/smart_scraper_multi_graph.py b/scrapegraphai/graphs/smart_scraper_multi_graph.py
@@ -61,27 +61,29 @@ def _create_graph(self) -> BaseGraph:
             BaseGraph: A graph instance representing the web scraping and searching workflow.
         """
 
-        smart_scraper_instance = SmartScraperGraph(
-            prompt="",
-            source="",
-            config=self.copy_config,
-            schema=self.copy_schema
-        )
+        # smart_scraper_instance = SmartScraperGraph(
+        #     prompt="",
+        #     source="",
+        #     config=self.copy_config,
+        #     schema=self.copy_schema
+        # )
 
         graph_iterator_node = GraphIteratorNode(
             input="user_prompt & urls",
             output=["results"],
             node_config={
-                "graph_instance": smart_scraper_instance,
-            }
+                "graph_instance": SmartScraperGraph,
+                "scraper_config": self.copy_config,
+            },
+            schema=self.copy_schema
         )
 
         merge_answers_node = MergeAnswersNode(
             input="user_prompt & results",
             output=["answer"],
             node_config={
                 "llm_model": self.llm_model,
-                "schema": self.schema
+                "schema": self.copy_schema
             }
         )
 
diff --git a/scrapegraphai/graphs/xml_scraper_multi_graph.py b/scrapegraphai/graphs/xml_scraper_multi_graph.py
@@ -59,27 +59,29 @@ def _create_graph(self) -> BaseGraph:
             BaseGraph: A graph instance representing the web scraping and searching workflow.
         """
 
-        smart_scraper_instance = XMLScraperGraph(
-            prompt="",
-            source="",
-            config=self.copy_config,
-            schema=self.copy_schema
-        )
+        # smart_scraper_instance = XMLScraperGraph(
+        #     prompt="",
+        #     source="",
+        #     config=self.copy_config,
+        #     schema=self.copy_schema
+        # )
 
         graph_iterator_node = GraphIteratorNode(
             input="user_prompt & jsons",
             output=["results"],
             node_config={
-                "graph_instance": smart_scraper_instance,
-            }
+                "graph_instance": XMLScraperGraph,
+                "scaper_config": self.copy_config,
+            },
+            schema=self.copy_schema
         )
 
         merge_answers_node = MergeAnswersNode(
             input="user_prompt & results",
             output=["answer"],
             node_config={
                 "llm_model": self.llm_model,
-                "schema": self.schema
+                "schema": self.copy_schema
             }
         )
 

Original file line number	Diff line number	Diff line change
`@@ -92,7 +92,7 @@ def _create_graph(self) -> BaseGraph:`
`92`	`92`	`output=["answer"],`
`93`	`93`	`node_config={`
`94`	`94`	`"llm_model": self.llm_model,`
`95`		`- "schema": self.schema`
	`95`	`+ "schema": self.copy_schema`
`96`	`96`	`}`
`97`	`97`	`)`
`98`	`98`
Original file line number	Diff line number	Diff line change
`@@ -93,7 +93,7 @@ def _create_graph(self) -> BaseGraph:`
`93`	`93`	`output=["answer"],`
`94`	`94`	`node_config={`
`95`	`95`	`"llm_model": self.llm_model,`
`96`		`- "schema": self.schema`
	`96`	`+ "schema": self.copy_schema`
`97`	`97`	`}`
`98`	`98`	`)`
`99`	`99`