removed rag node

VinciGit00 · VinciGit00 · commit a7249685cb2b · 2024-07-14T10:24:38.000+02:00
diff --git a/scrapegraphai/graphs/csv_scraper_graph.py b/scrapegraphai/graphs/csv_scraper_graph.py
@@ -10,7 +10,6 @@
 
 from ..nodes import (
     FetchNode,
-    RAGNode,
     GenerateAnswerCSVNode
 )
 
@@ -37,14 +36,7 @@ def _create_graph(self):
             input="csv | csv_dir",
             output=["doc"],
         )
-        rag_node = RAGNode(
-            input="user_prompt & doc",
-            output=["relevant_chunks"],
-            node_config={
-                "llm_model": self.llm_model,
-                "embedder_model": self.embedder_model,
-            }
-        )
+     
         generate_answer_node = GenerateAnswerCSVNode(
             input="user_prompt & (relevant_chunks | doc)",
             output=["answer"],
@@ -58,12 +50,10 @@ def _create_graph(self):
         return BaseGraph(
             nodes=[
                 fetch_node,
-                rag_node,
                 generate_answer_node,
             ],
             edges=[
-                (fetch_node, rag_node),
-                (rag_node, generate_answer_node)
+                (fetch_node, generate_answer_node)
             ],
             entry_point=fetch_node,
             graph_name=self.__class__.__name__
diff --git a/scrapegraphai/graphs/json_scraper_graph.py b/scrapegraphai/graphs/json_scraper_graph.py
@@ -10,7 +10,6 @@
 
 from ..nodes import (
     FetchNode,
-    RAGNode,
     GenerateAnswerNode
 )
 
@@ -62,14 +61,7 @@ def _create_graph(self) -> BaseGraph:
             input="json | json_dir",
             output=["doc", "link_urls", "img_urls"],
         )
-        rag_node = RAGNode(
-            input="user_prompt & (parsed_doc | doc)",
-            output=["relevant_chunks"],
-            node_config={
-                "llm_model": self.llm_model,
-                "embedder_model": self.embedder_model
-            }
-        )
+     
         generate_answer_node = GenerateAnswerNode(
             input="user_prompt & (relevant_chunks | parsed_doc | doc)",
             output=["answer"],
@@ -83,12 +75,10 @@ def _create_graph(self) -> BaseGraph:
         return BaseGraph(
             nodes=[
                 fetch_node,
-                rag_node,
                 generate_answer_node,
             ],
             edges=[
-                (fetch_node, rag_node),
-                (rag_node, generate_answer_node)
+                (fetch_node, generate_answer_node)
             ],
             entry_point=fetch_node,
             graph_name=self.__class__.__name__
diff --git a/scrapegraphai/graphs/markdown_scraper_graph.py b/scrapegraphai/graphs/markdown_scraper_graph.py
@@ -3,7 +3,7 @@
 from pydantic import BaseModel
 from .base_graph import BaseGraph
 from .abstract_graph import AbstractGraph
-from ..nodes import FetchNode, ParseNode, RAGNode, GenerateAnswerNode
+from ..nodes import FetchNode, ParseNode, GenerateAnswerNode
 
 class MDScraperGraph(AbstractGraph):
     """
@@ -63,14 +63,6 @@ def _create_graph(self) -> BaseGraph:
                 "chunk_size": self.model_token
             }
         )
-        rag_node = RAGNode(
-            input="user_prompt & (parsed_doc | doc)",
-            output=["relevant_chunks"],
-            node_config={
-                "llm_model": self.llm_model,
-                "embedder_model": self.embedder_model
-            }
-        )
         generate_answer_node = GenerateAnswerNode(
             input="user_prompt & (relevant_chunks | parsed_doc | doc)",
             output=["answer"],
@@ -86,13 +78,11 @@ def _create_graph(self) -> BaseGraph:
             nodes=[
                 fetch_node,
                 parse_node,
-                rag_node,
                 generate_answer_node,
             ],
             edges=[
                 (fetch_node, parse_node),
-                (parse_node, rag_node),
-                (rag_node, generate_answer_node)
+                (parse_node, generate_answer_node)
             ],
             entry_point=fetch_node,
             graph_name=self.__class__.__name__
diff --git a/scrapegraphai/graphs/omni_scraper_graph.py b/scrapegraphai/graphs/omni_scraper_graph.py
@@ -12,7 +12,6 @@
     FetchNode,
     ParseNode,
     ImageToTextNode,
-    RAGNode,
     GenerateAnswerOmniNode
 )
 
@@ -89,14 +88,7 @@ def _create_graph(self) -> BaseGraph:
                 "max_images": self.max_images
             }
         )
-        rag_node = RAGNode(
-            input="user_prompt & (parsed_doc | doc)",
-            output=["relevant_chunks"],
-            node_config={
-                "llm_model": self.llm_model,
-                "embedder_model": self.embedder_model
-            }
-        )
+      
         generate_answer_omni_node = GenerateAnswerOmniNode(
             input="user_prompt & (relevant_chunks | parsed_doc | doc) & img_desc",
             output=["answer"],
@@ -112,14 +104,12 @@ def _create_graph(self) -> BaseGraph:
                 fetch_node,
                 parse_node,
                 image_to_text_node,
-                rag_node,
                 generate_answer_omni_node,
             ],
             edges=[
                 (fetch_node, parse_node),
                 (parse_node, image_to_text_node),
-                (image_to_text_node, rag_node),
-                (rag_node, generate_answer_omni_node)
+                (image_to_text_node, generate_answer_omni_node)
             ],
             entry_point=fetch_node,
             graph_name=self.__class__.__name__
@@ -136,4 +126,4 @@ def run(self) -> str:
         inputs = {"user_prompt": self.prompt, self.input_key: self.source}
         self.final_state, self.execution_info = self.graph.execute(inputs)
 
-        return self.final_state.get("answer", "No answer found.")
+        return self.final_state.get("answer", "No answer found.")
diff --git a/scrapegraphai/graphs/pdf_scraper_graph.py b/scrapegraphai/graphs/pdf_scraper_graph.py
@@ -12,7 +12,6 @@
 from ..nodes import (
     FetchNode,
     ParseNode,
-    RAGNode,
     GenerateAnswerPDFNode
 )
 
@@ -76,14 +75,6 @@ def _create_graph(self) -> BaseGraph:
             }
         )
 
-        rag_node = RAGNode(
-            input="user_prompt & (parsed_doc | doc)",
-            output=["relevant_chunks"],
-            node_config={
-                "llm_model": self.llm_model,
-                "embedder_model": self.embedder_model
-            }
-        )
         generate_answer_node_pdf = GenerateAnswerPDFNode(
             input="user_prompt & (relevant_chunks | doc)",
             output=["answer"],
@@ -98,13 +89,11 @@ def _create_graph(self) -> BaseGraph:
             nodes=[
                 fetch_node,
                 parse_node,
-                rag_node,
                 generate_answer_node_pdf,
             ],
             edges=[
                 (fetch_node, parse_node),
-                (parse_node, rag_node),
-                (rag_node, generate_answer_node_pdf)
+                (parse_node, generate_answer_node_pdf)
             ],
             entry_point=fetch_node,
             graph_name=self.__class__.__name__
diff --git a/scrapegraphai/graphs/smart_scraper_graph.py b/scrapegraphai/graphs/smart_scraper_graph.py
@@ -78,14 +78,7 @@ def _create_graph(self) -> BaseGraph:
                 "chunk_size": self.model_token
             }
         )
-        rag_node = RAGNode(
-            input="user_prompt & (parsed_doc | doc)",
-            output=["relevant_chunks"],
-            node_config={
-                "llm_model": self.llm_model,
-                "embedder_model": self.embedder_model
-            }
-        )
+
         generate_answer_node = GenerateAnswerNode(
             input="user_prompt & (relevant_chunks | parsed_doc | doc)",
             output=["answer"],
@@ -100,13 +93,11 @@ def _create_graph(self) -> BaseGraph:
             nodes=[
                 fetch_node,
                 parse_node,
-                rag_node,
                 generate_answer_node,
             ],
             edges=[
                 (fetch_node, parse_node),
-                (parse_node, rag_node),
-                (rag_node, generate_answer_node)
+                (parse_node, generate_answer_node)
             ],
             entry_point=fetch_node,
             graph_name=self.__class__.__name__
diff --git a/scrapegraphai/graphs/xml_scraper_graph.py b/scrapegraphai/graphs/xml_scraper_graph.py
@@ -10,7 +10,6 @@
 
 from ..nodes import (
     FetchNode,
-    RAGNode,
     GenerateAnswerNode
 )
 
@@ -64,14 +63,7 @@ def _create_graph(self) -> BaseGraph:
             input="xml | xml_dir",
             output=["doc", "link_urls", "img_urls"]
         )
-        rag_node = RAGNode(
-            input="user_prompt & doc",
-            output=["relevant_chunks"],
-            node_config={
-                "llm_model": self.llm_model,
-                "embedder_model": self.embedder_model
-            }
-        )
+     
         generate_answer_node = GenerateAnswerNode(
             input="user_prompt & (relevant_chunks | doc)",
             output=["answer"],
@@ -85,12 +77,10 @@ def _create_graph(self) -> BaseGraph:
         return BaseGraph(
             nodes=[
                 fetch_node,
-                rag_node,
                 generate_answer_node,
             ],
             edges=[
-                (fetch_node, rag_node),
-                (rag_node, generate_answer_node)
+                (fetch_node, generate_answer_node)
             ],
             entry_point=fetch_node,
             graph_name=self.__class__.__name__
diff --git a/scrapegraphai/nodes/generate_answer_csv_node.py b/scrapegraphai/nodes/generate_answer_csv_node.py
@@ -125,7 +125,7 @@ def execute(self, state):
                     template=template_no_chunks_csv_prompt,
                     input_variables=["question"],
                     partial_variables={
-                        "context": chunk.page_content,
+                        "context": chunk,
                         "format_instructions": format_instructions,
                     },
                 )
@@ -137,7 +137,7 @@ def execute(self, state):
                     template=template_chunks_csv_prompt,
                     input_variables=["question"],
                     partial_variables={
-                        "context": chunk.page_content,
+                        "context": chunk,
                         "chunk_id": i + 1,
                         "format_instructions": format_instructions,
                     },
diff --git a/scrapegraphai/nodes/generate_answer_node.py b/scrapegraphai/nodes/generate_answer_node.py
@@ -115,7 +115,7 @@ def execute(self, state: dict) -> dict:
                 prompt = PromptTemplate(
                     template=template_no_chunks_prompt,
                     input_variables=["question"],
-                    partial_variables={"context": chunk.page_content,
+                    partial_variables={"context": chunk,
                                        "format_instructions": format_instructions})
                 chain =  prompt | self.llm_model | output_parser
                 answer = chain.invoke({"question": user_prompt})
@@ -124,7 +124,7 @@ def execute(self, state: dict) -> dict:
                 prompt = PromptTemplate(
                     template=template_chunks_prompt,
                     input_variables=["question"],
-                    partial_variables={"context": chunk.page_content,
+                    partial_variables={"context": chunk,
                                         "chunk_id": i + 1,
                                         "format_instructions": format_instructions})
             # Dynamically name the chains based on their index
diff --git a/scrapegraphai/nodes/generate_answer_omni_node.py b/scrapegraphai/nodes/generate_answer_omni_node.py
@@ -110,7 +110,7 @@ def execute(self, state: dict) -> dict:
                     template=template_no_chunk_omni_prompt,
                     input_variables=["question"],
                     partial_variables={
-                        "context": chunk.page_content,
+                        "context": chunk,
                         "format_instructions": format_instructions,
                         "img_desc": imag_desc,
                     },
@@ -123,7 +123,7 @@ def execute(self, state: dict) -> dict:
                     template=template_chunks_omni_prompt,
                     input_variables=["question"],
                     partial_variables={
-                        "context": chunk.page_content,
+                        "context": chunk,
                         "chunk_id": i + 1,
                         "format_instructions": format_instructions,
                     },
diff --git a/scrapegraphai/nodes/generate_answer_pdf_node.py b/scrapegraphai/nodes/generate_answer_pdf_node.py
@@ -124,7 +124,7 @@ def execute(self, state):
                     template=template_no_chunks_pdf_prompt,
                     input_variables=["question"],
                     partial_variables={
-                        "context":chunk.page_content,
+                        "context":chunk,
                         "format_instructions": format_instructions,
                     },
                 )

Original file line number	Diff line number	Diff line change
`@@ -124,7 +124,7 @@ def execute(self, state):`
`124`	`124`	`template=template_no_chunks_pdf_prompt,`
`125`	`125`	`input_variables=["question"],`
`126`	`126`	`partial_variables={`
`127`		`- "context":chunk.page_content,`
	`127`	`+ "context":chunk,`
`128`	`128`	`"format_instructions": format_instructions,`
`129`	`129`	`},`
`130`	`130`	`)`