Merge pull request #664 from LorenzoPaleari/598-fix-pydantic-validation-error

598 - Fix pydantic validation error

Author: VinciGit00

scrapegraphai/graphs/csv_scraper_multi_graph.py

@@ -2,6 +2,7 @@
 CSVScraperMultiGraph Module
 """
 
+from copy import deepcopy
 from typing import List, Optional
 from pydantic import BaseModel
 from .base_graph import BaseGraph
@@ -48,6 +49,8 @@ def __init__(self, prompt: str, source: List[str],
 
         self.copy_config = safe_deepcopy(config)
 
+        self.copy_schema = deepcopy(schema)
+
         super().__init__(prompt, config, source, schema)
 
     def _create_graph(self) -> BaseGraph:
@@ -58,17 +61,18 @@ def _create_graph(self) -> BaseGraph:
             BaseGraph: A graph instance representing the web scraping and searching workflow.
         """
 
-        smart_scraper_instance = CSVScraperGraph(
-            prompt="",
-            source="",
-            config=self.copy_config,
-        )
+        # smart_scraper_instance = CSVScraperGraph(
+        #     prompt="",
+        #     source="",
+        #     config=self.copy_config,
+        # )
 
         graph_iterator_node = GraphIteratorNode(
             input="user_prompt & jsons",
             output=["results"],
             node_config={
-                "graph_instance": smart_scraper_instance,
+                "graph_instance": CSVScraperGraph,
+                "scraper_config": self.copy_config,
             }
         )
 
@@ -77,7 +81,7 @@ def _create_graph(self) -> BaseGraph:
             output=["answer"],
             node_config={
                 "llm_model": self.llm_model,
-                "schema": self.schema
+                "schema": self.copy_schema
             }
         )
 

scrapegraphai/graphs/json_scraper_multi_graph.py

@@ -61,27 +61,29 @@ def _create_graph(self) -> BaseGraph:
             BaseGraph: A graph instance representing the web scraping and searching workflow.
         """
 
-        smart_scraper_instance = JSONScraperGraph(
-            prompt="",
-            source="",
-            config=self.copy_config,
-            schema=self.copy_schema
-        )
+        # smart_scraper_instance = JSONScraperGraph(
+        #     prompt="",
+        #     source="",
+        #     config=self.copy_config,
+        #     schema=self.copy_schema
+        # )
 
         graph_iterator_node = GraphIteratorNode(
             input="user_prompt & jsons",
             output=["results"],
             node_config={
-                "graph_instance": smart_scraper_instance,
-            }
+                "graph_instance": JSONScraperGraph,
+                "scraper_config": self.copy_config,
+            },
+            schema=self.copy_schema
         )
 
         merge_answers_node = MergeAnswersNode(
             input="user_prompt & results",
             output=["answer"],
             node_config={
                 "llm_model": self.llm_model,
-                "schema": self.schema
+                "schema": self.copy_schema
             }
         )
 

scrapegraphai/graphs/markdown_scraper_multi_graph.py

@@ -55,28 +55,30 @@ def _create_graph(self) -> BaseGraph:
         Returns:
             BaseGraph: A graph instance representing the web scraping and searching workflow.
         """
-        smart_scraper_instance = MDScraperGraph(
-            prompt="",
-            source="",
-            config=self.copy_config,
-            schema=self.copy_schema
-        )
+        # smart_scraper_instance = MDScraperGraph(
+        #     prompt="",
+        #     source="",
+        #     config=self.copy_config,
+        #     schema=self.copy_schema
+        # )
 
         # Define the graph nodes
         graph_iterator_node = GraphIteratorNode(
             input="user_prompt & jsons",
             output=["results"],
             node_config={
-                "graph_instance": smart_scraper_instance,
-            }
+                "graph_instance": MDScraperGraph,
+                "scraper_config": self.copy_config,
+            },
+            schema=self.copy_schema
         )
 
         merge_answers_node = MergeAnswersNode(
             input="user_prompt & results",
             output=["answer"],
             node_config={
                 "llm_model": self.llm_model,
-                "schema": self.schema
+                "schema": self.copy_schema
             }
         )
 

scrapegraphai/graphs/omni_search_graph.py

@@ -61,12 +61,12 @@ def _create_graph(self) -> BaseGraph:
             BaseGraph: A graph instance representing the web scraping and searching workflow.
         """
 
-        omni_scraper_instance = OmniScraperGraph(
-            prompt="",
-            source="",
-            config=self.copy_config,
-            schema=self.copy_schema
-        )
+        # omni_scraper_instance = OmniScraperGraph(
+        #     prompt="",
+        #     source="",
+        #     config=self.copy_config,
+        #     schema=self.copy_schema
+        # )
 
         search_internet_node = SearchInternetNode(
             input="user_prompt",
@@ -81,16 +81,18 @@ def _create_graph(self) -> BaseGraph:
             input="user_prompt & urls",
             output=["results"],
             node_config={
-                "graph_instance": omni_scraper_instance,
-            }
+                "graph_instance": OmniScraperGraph,
+                "scraper_config": self.copy_config,
+            },
+            schema=self.copy_schema
         )
 
         merge_answers_node = MergeAnswersNode(
             input="user_prompt & results",
             output=["answer"],
             node_config={
                 "llm_model": self.llm_model,
-                "schema": self.schema
+                "schema": self.copy_schema
             }
         )
 

scrapegraphai/graphs/pdf_scraper_multi_graph.py

@@ -59,27 +59,29 @@ def _create_graph(self) -> BaseGraph:
             BaseGraph: A graph instance representing the web scraping and searching workflow.
         """
 
-        pdf_scraper_instance = PDFScraperGraph(
-            prompt="",
-            source="",
-            config=self.copy_config,
-            schema=self.copy_schema
-        )
+        # pdf_scraper_instance = PDFScraperGraph(
+        #     prompt="",
+        #     source="",
+        #     config=self.copy_config,
+        #     schema=self.copy_schema
+        # )
 
         graph_iterator_node = GraphIteratorNode(
             input="user_prompt & pdfs",
             output=["results"],
             node_config={
-                "graph_instance": pdf_scraper_instance,
-            }
+                "graph_instance": PDFScraperGraph,
+                "scraper_config": self.copy_config,
+            },
+            schema=self.copy_schema
         )
 
         merge_answers_node = MergeAnswersNode(
             input="user_prompt & results",
             output=["answer"],
             node_config={
                 "llm_model": self.llm_model,
-                "schema": self.schema
+                "schema": self.copy_schema
             }
         )
 

scrapegraphai/graphs/search_graph.py

@@ -62,12 +62,12 @@ def _create_graph(self) -> BaseGraph:
             BaseGraph: A graph instance representing the web scraping and searching workflow.
         """
 
-        smart_scraper_instance = SmartScraperGraph(
-            prompt="",
-            source="",
-            config=self.copy_config,
-            schema=self.copy_schema
-        )
+        # smart_scraper_instance = SmartScraperGraph(
+        #     prompt="",
+        #     source="",
+        #     config=self.copy_config,
+        #     schema=self.copy_schema
+        # )
 
         search_internet_node = SearchInternetNode(
             input="user_prompt",
@@ -82,16 +82,18 @@ def _create_graph(self) -> BaseGraph:
             input="user_prompt & urls",
             output=["results"],
             node_config={
-                "graph_instance": smart_scraper_instance,
-            }
+                "graph_instance": SmartScraperGraph,
+                "scraper_config": self.copy_config
+            },
+            schema=self.copy_schema
         )
 
         merge_answers_node = MergeAnswersNode(
             input="user_prompt & results",
             output=["answer"],
             node_config={
                 "llm_model": self.llm_model,
-                "schema": self.schema
+                "schema": self.copy_schema
             }
         )
 

scrapegraphai/graphs/smart_scraper_multi_graph.py

@@ -61,27 +61,29 @@ def _create_graph(self) -> BaseGraph:
             BaseGraph: A graph instance representing the web scraping and searching workflow.
         """
 
-        smart_scraper_instance = SmartScraperGraph(
-            prompt="",
-            source="",
-            config=self.copy_config,
-            schema=self.copy_schema
-        )
+        # smart_scraper_instance = SmartScraperGraph(
+        #     prompt="",
+        #     source="",
+        #     config=self.copy_config,
+        #     schema=self.copy_schema
+        # )
 
         graph_iterator_node = GraphIteratorNode(
             input="user_prompt & urls",
             output=["results"],
             node_config={
-                "graph_instance": smart_scraper_instance,
-            }
+                "graph_instance": SmartScraperGraph,
+                "scraper_config": self.copy_config,
+            },
+            schema=self.copy_schema
         )
 
         merge_answers_node = MergeAnswersNode(
             input="user_prompt & results",
             output=["answer"],
             node_config={
                 "llm_model": self.llm_model,
-                "schema": self.schema
+                "schema": self.copy_schema
             }
         )
 

scrapegraphai/graphs/xml_scraper_multi_graph.py

@@ -59,27 +59,29 @@ def _create_graph(self) -> BaseGraph:
             BaseGraph: A graph instance representing the web scraping and searching workflow.
         """
 
-        smart_scraper_instance = XMLScraperGraph(
-            prompt="",
-            source="",
-            config=self.copy_config,
-            schema=self.copy_schema
-        )
+        # smart_scraper_instance = XMLScraperGraph(
+        #     prompt="",
+        #     source="",
+        #     config=self.copy_config,
+        #     schema=self.copy_schema
+        # )
 
         graph_iterator_node = GraphIteratorNode(
             input="user_prompt & jsons",
             output=["results"],
             node_config={
-                "graph_instance": smart_scraper_instance,
-            }
+                "graph_instance": XMLScraperGraph,
+                "scaper_config": self.copy_config,
+            },
+            schema=self.copy_schema
         )
 
         merge_answers_node = MergeAnswersNode(
             input="user_prompt & results",
             output=["answer"],
             node_config={
                 "llm_model": self.llm_model,
-                "schema": self.schema
+                "schema": self.copy_schema
             }
         )
 

scrapegraphai/nodes/generate_answer_csv_node.py

@@ -3,6 +3,7 @@
 """
 
 from typing import List, Optional
+from pydantic.v1 import BaseModel as BaseModelV1
 from langchain.prompts import PromptTemplate
 from langchain_core.output_parsers import JsonOutputParser
 from langchain_core.runnables import RunnableParallel
@@ -12,6 +13,7 @@
 from tqdm import tqdm
 from ..utils.logging import get_logger
 from .base_node import BaseNode
+from ..utils.llm_output_parser import typed_dict_output_parser, base_model_v2_output_parser, base_model_v1_output_parser
 from ..prompts import TEMPLATE_CHUKS_CSV, TEMPLATE_NO_CHUKS_CSV, TEMPLATE_MERGE_CSV
 
 class GenerateAnswerCSVNode(BaseNode):
@@ -97,13 +99,13 @@ def execute(self, state):
 
             if isinstance(self.llm_model, (ChatOpenAI, ChatMistralAI)):
                 self.llm_model = self.llm_model.with_structured_output(
-                    schema = self.node_config["schema"],
-                    method="function_calling") # json schema works only on specific models
-                
-                # default parser to empty lambda function
-                output_parser = lambda x: x
+                    schema = self.node_config["schema"]) # json schema works only on specific models
+
+                output_parser = typed_dict_output_parser
                 if is_basemodel_subclass(self.node_config["schema"]):
-                    output_parser = dict
+                    output_parser = base_model_v2_output_parser
+                    if issubclass(self.node_config["schema"], BaseModelV1):
+                        output_parser = base_model_v1_output_parser
                 format_instructions = "NA"
             else:
                 output_parser = JsonOutputParser(pydantic_object=self.node_config["schema"])