Merge branch 'pre/beta' into try

Author: VinciGit00

CHANGELOG.md

@@ -1,10 +1,12 @@
 ## [1.3.0](https://github.com/VinciGit00/Scrapegraph-ai/compare/v1.2.4...v1.3.0) (2024-05-19)
 
 
+
 ### Features
 
 * add new model ([8c7afa7](https://github.com/VinciGit00/Scrapegraph-ai/commit/8c7afa7570f0a104578deb35658168435cfe5ae1))
 
+
 ## [1.2.4](https://github.com/VinciGit00/Scrapegraph-ai/compare/v1.2.3...v1.2.4) (2024-05-17)
 
 

examples/openai/deep_scraper_openai.py

@@ -22,6 +22,7 @@
         "model": "gpt-4",
     },
     "verbose": True,
+    "max_depth": 1
 }
 
 # ************************************************

scrapegraphai/graphs/base_graph.py

@@ -145,4 +145,4 @@ def execute(self, initial_state: dict) -> Tuple[dict, list]:
             "exec_time": total_exec_time,
         })
 
-        return state, exec_info
+        return state, exec_info

scrapegraphai/graphs/deep_scraper_graph.py

@@ -8,7 +8,9 @@
     SearchLinkNode,
     ParseNode,
     RAGNode,
-    GenerateAnswerNode
+    GenerateAnswerNode,
+    GraphIteratorNode,
+    MergeAnswersNode
 )
 from .abstract_graph import AbstractGraph
 
@@ -18,12 +20,11 @@ class DeepScraperGraph(AbstractGraph):
     [WIP]
 
     DeepScraper is a scraping pipeline that automates the process of 
-    extracting information from web pages
-    using a natural language model to interpret and answer prompts.
-
-    Unlike SmartScraper, DeepScraper can navigate to the links within the input webpage,
-    to fuflfil the task within the prompt.
+    extracting information from web pages using a natural language model 
+    to interpret and answer prompts.
 
+    Unlike SmartScraper, DeepScraper can navigate to the links within,
+    the input webpage to fuflfil the task within the prompt.
     
     Attributes:
         prompt (str): The prompt for the graph.
@@ -50,12 +51,13 @@ class DeepScraperGraph(AbstractGraph):
 
     def __init__(self, prompt: str, source: str, config: dict):
         super().__init__(prompt, config, source)
-
         self.input_key = "url" if source.startswith("http") else "local_dir"
 
-    def _create_graph(self) -> BaseGraph:
+    def _create_repeated_graph(self) -> BaseGraph:
         """
-        Creates the graph of nodes representing the workflow for web scraping.
+        Creates the graph that can be repeatedly executed to conduct search on
+        hyperlinks within the webpage.
+
         Returns:
             BaseGraph: A graph instance representing the web scraping workflow.
         """
@@ -78,6 +80,13 @@ def _create_graph(self) -> BaseGraph:
                 "embedder_model": self.embedder_model
             }
         )
+        generate_answer_node = GenerateAnswerNode(
+            input="user_prompt & (relevant_chunks | parsed_doc | doc)",
+            output=["answer"],
+            node_config={
+                "llm_model": self.llm_model
+            }
+        )
         search_node = SearchLinkNode(
             input="user_prompt & relevant_chunks",
             output=["relevant_links"],
@@ -86,23 +95,60 @@ def _create_graph(self) -> BaseGraph:
                 "embedder_model": self.embedder_model
             }
         )
+        graph_iterator_node = GraphIteratorNode(
+            input="user_prompt & relevant_links",
+            output=["results"],
+            node_config={
+                "graph_instance": None,
+                "batchsize": 1
+            }
+        )
+        merge_answers_node = MergeAnswersNode(
+            input="user_prompt & results",
+            output=["answer"],
+            node_config={
+                "llm_model": self.llm_model,
+            }
+        )
 
         return BaseGraph(
             nodes=[
                 fetch_node,
                 parse_node,
                 rag_node,
-                search_node
+                generate_answer_node,
+                search_node,
+                graph_iterator_node,
+                merge_answers_node
             ],
             edges=[
                 (fetch_node, parse_node),
                 (parse_node, rag_node),
-                (rag_node, search_node)
-
+                (rag_node, generate_answer_node),
+                (rag_node, search_node),
+                (search_node, graph_iterator_node),
+                (graph_iterator_node, merge_answers_node)
             ],
             entry_point=fetch_node
         )
 
+
+
+    def _create_graph(self) -> BaseGraph:
+        """
+        Creates the graph of nodes representing the workflow for web scraping
+        n-levels deep.
+
+        Returns:
+            BaseGraph: A graph instance representing the web scraping workflow.
+        """
+
+        base_graph = self._create_repeated_graph()
+        graph_iterator_node = list(filter(lambda x: x.node_name == "GraphIterator", base_graph.nodes))[0]
+        # Graph iterator will repeat the same graph for multiple hyperlinks found within input webpage
+        graph_iterator_node.node_config["graph_instance"] = self
+        return base_graph
+
     def run(self) -> str:
         """
         Executes the scraping process and returns the answer to the prompt.

scrapegraphai/nodes/conditional_node.py

@@ -13,46 +13,33 @@ class ConditionalNode(BaseNode):
     This node type is used to implement branching logic within the graph, allowing 
     for dynamic paths based on the data available in the current state.
 
+    It is expected thar exactly two edges are created out of this node.
+    The first node is chosen for execution if the key exists and has a non-empty value,
+    and the second node is chosen if the key does not exist or is empty.
+
     Attributes:
         key_name (str): The name of the key in the state to check for its presence.
-        next_nodes (list): A list of two node instances. The first node is chosen 
-                           for execution if the key exists and has a non-empty value, 
-                           and the second node is chosen if the key does not exist or 
-                           is empty.
 
     Args:
         key_name (str): The name of the key to check in the graph's state. This is 
                         used to determine the path the graph's execution should take.
-        next_nodes (list): A list containing exactly two node instances, specifying 
-                           the next nodes to execute based on the condition's outcome.
         node_name (str, optional): The unique identifier name for the node. Defaults 
                                    to "ConditionalNode".
 
-    Raises:
-        ValueError: If next_nodes does not contain exactly two elements, indicating 
-                    a misconfiguration in specifying the conditional paths.
     """
 
-    def __init__(self, key_name: str, next_nodes: list, node_name="ConditionalNode"):
+    def __init__(self, key_name: str, node_name="ConditionalNode"):
         """
         Initializes the node with the key to check and the next node names based on the condition.
 
         Args:
             key_name (str): The name of the key to check in the state.
-            next_nodes (list): A list containing exactly two names of the next nodes.
-                               The first is used if the key exists, the second if it does not.
-
-        Raises:
-            ValueError: If next_nodes does not contain exactly two elements.
         """
 
         super().__init__(node_name, "conditional_node")
         self.key_name = key_name
-        if len(next_nodes) != 2:
-            raise ValueError("next_nodes must contain exactly two elements.")
-        self.next_nodes = next_nodes
 
-    def execute(self, state: dict) -> str:
+    def execute(self, state: dict) -> dict:
         """
         Checks if the specified key is present in the state and decides the next node accordingly.
 
@@ -64,5 +51,7 @@ def execute(self, state: dict) -> str:
         """
 
         if self.key_name in state and len(state[self.key_name]) > 0:
-            return self.next_nodes[0].node_name
-        return self.next_nodes[1].node_name
+            state["next_node"] = 0
+        else:
+            state["next_node"] = 1
+        return state

scrapegraphai/nodes/graph_iterator_node.py

@@ -104,7 +104,12 @@ async def _async_execute(self, state: dict, batchsize: int) -> dict:
         if graph_instance is None:
             raise ValueError("graph instance is required for concurrent execution")
 
-        # sets the prompt for the graph instance
+        # Assign depth level to the graph
+        if "graph_depth" in graph_instance.config:
+            graph_instance.config["graph_depth"] += 1
+        else:
+            graph_instance.config["graph_depth"] = 1
+
         graph_instance.prompt = user_prompt
 
         participants = []