refactoring of the code

Co-Authored-By: Matteo Vedovati <[email protected]>

Author: VinciGit00

examples/local_models/smart_scraper_ollama.py

@@ -22,8 +22,13 @@
 # Create the SmartScraperGraph instance and run it
 # ************************************************
 smart_scraper_graph = SmartScraperGraph(
+<<<<<<< Updated upstream
     prompt="Find some information about what does the company do, the name and a contact email.",
     source="https://scrapegraphai.com/",
+=======
+    prompt="List all the projects with their descriptions",
+    source="https://perinim.github.io/projects/",
+>>>>>>> Stashed changes
     config=graph_config
 )
 

scrapegraphai/graphs/abstract_graph.py

@@ -7,20 +7,16 @@
 import uuid
 import warnings
 from pydantic import BaseModel
-
 from langchain_community.chat_models import ErnieBotChat
 from langchain_nvidia_ai_endpoints import ChatNVIDIA
 from langchain.chat_models import init_chat_model
-
 from ..helpers import models_tokens
 from ..models import (
     OneApi,
     DeepSeek
 )
 from ..utils.logging import set_verbosity_warning, set_verbosity_info
 
-
-
 class AbstractGraph(ABC):
     """
     Scaffolding class for creating a graph representation and executing it.

scrapegraphai/graphs/base_graph.py

@@ -6,9 +6,7 @@
 from typing import Tuple
 from langchain_community.callbacks import get_openai_callback
 from ..integrations import BurrBridge
-
-# Import telemetry functions
-from ..telemetry import log_graph_execution, log_event
+from ..telemetry import log_graph_execution
 
 class BaseGraph:
     """

scrapegraphai/graphs/csv_scraper_graph.py

@@ -4,16 +4,13 @@
 
 from typing import Optional
 from pydantic import BaseModel
-
 from .base_graph import BaseGraph
 from .abstract_graph import AbstractGraph
-
 from ..nodes import (
     FetchNode,
     GenerateAnswerCSVNode
 )
 
-
 class CSVScraperGraph(AbstractGraph):
     """
     SmartScraper is a comprehensive web scraping tool that automates the process of extracting

scrapegraphai/graphs/csv_scraper_multi_graph.py

@@ -4,22 +4,19 @@
 
 from copy import copy, deepcopy
 from typing import List, Optional
-
 from pydantic import BaseModel
-
 from .base_graph import BaseGraph
 from .abstract_graph import AbstractGraph
 from .csv_scraper_graph import CSVScraperGraph
-
 from ..nodes import (
     GraphIteratorNode,
     MergeAnswersNode
 )
 
-
 class CSVScraperMultiGraph(AbstractGraph):
     """ 
-    CSVScraperMultiGraph is a scraping pipeline that scrapes a list of URLs and generates answers to a given prompt.
+    CSVScraperMultiGraph is a scraping pipeline that 
+    scrapes a list of URLs and generates answers to a given prompt.
     It only requires a user prompt and a list of URLs.
 
     Attributes:
@@ -44,7 +41,8 @@ class CSVScraperMultiGraph(AbstractGraph):
         >>> result = search_graph.run()
     """
 
-    def __init__(self, prompt: str, source: List[str], config: dict, schema: Optional[BaseModel] = None):
+    def __init__(self, prompt: str, source: List[str], 
+                 config: dict, schema: Optional[BaseModel] = None):
 
         self.max_results = config.get("max_results", 3)
 

scrapegraphai/graphs/deep_scraper_graph.py

@@ -4,10 +4,8 @@
 
 from typing import Optional
 from pydantic import BaseModel
-
 from .base_graph import BaseGraph
 from .abstract_graph import AbstractGraph
-
 from ..nodes import (
     FetchNode,
     SearchLinkNode,
@@ -18,7 +16,6 @@
     MergeAnswersNode
 )
 
-
 class DeepScraperGraph(AbstractGraph):
     """
     [WIP]
@@ -87,7 +84,6 @@ def _create_repeated_graph(self) -> BaseGraph:
             output=["relevant_chunks"],
             node_config={
                 "llm_model": self.llm_model,
-                "embedder_model": self.embedder_model
             }
         )
         generate_answer_node = GenerateAnswerNode(
@@ -104,7 +100,6 @@ def _create_repeated_graph(self) -> BaseGraph:
             output=["relevant_links"],
             node_config={
                 "llm_model": self.llm_model,
-                "embedder_model": self.embedder_model
             }
         )
         graph_iterator_node = GraphIteratorNode(

scrapegraphai/graphs/json_scraper_graph.py

@@ -4,16 +4,13 @@
 
 from typing import Optional
 from pydantic import BaseModel
-
 from .base_graph import BaseGraph
 from .abstract_graph import AbstractGraph
-
 from ..nodes import (
     FetchNode,
     GenerateAnswerNode
 )
 
-
 class JSONScraperGraph(AbstractGraph):
     """
     JSONScraperGraph defines a scraping pipeline for JSON files.

scrapegraphai/graphs/json_scraper_multi_graph.py

@@ -5,20 +5,18 @@
 from copy import copy, deepcopy
 from typing import List, Optional
 from pydantic import BaseModel
-
 from .base_graph import BaseGraph
 from .abstract_graph import AbstractGraph
 from .json_scraper_graph import JSONScraperGraph
-
 from ..nodes import (
     GraphIteratorNode,
     MergeAnswersNode
 )
 
-
 class JSONScraperMultiGraph(AbstractGraph):
     """ 
-    JSONScraperMultiGraph is a scraping pipeline that scrapes a list of URLs and generates answers to a given prompt.
+    JSONScraperMultiGraph is a scraping pipeline that scrapes a 
+    list of URLs and generates answers to a given prompt.
     It only requires a user prompt and a list of URLs.
 
     Attributes:

scrapegraphai/graphs/markdown_scraper_multi_graph.py

@@ -5,17 +5,14 @@
 from copy import copy, deepcopy
 from typing import List, Optional
 from pydantic import BaseModel
-
 from .base_graph import BaseGraph
 from .abstract_graph import AbstractGraph
 from .markdown_scraper_graph import MDScraperGraph
-
 from ..nodes import (
     GraphIteratorNode,
     MergeAnswersNode
 )
 
-
 class MDScraperMultiGraph(AbstractGraph):
     """
     MDScraperMultiGraph is a scraping pipeline that scrapes a list of URLs and 

scrapegraphai/graphs/omni_scraper_graph.py

@@ -4,17 +4,14 @@
 
 from typing import Optional
 from pydantic import BaseModel
-
 from .base_graph import BaseGraph
 from .abstract_graph import AbstractGraph
-
 from ..nodes import (
     FetchNode,
     ParseNode,
     ImageToTextNode,
     GenerateAnswerOmniNode
 )
-
 from ..models import OpenAIImageToText
 
 class OmniScraperGraph(AbstractGraph):

scrapegraphai/graphs/pdf_scraper_graph.py

@@ -5,17 +5,14 @@
 
 from typing import Optional
 from pydantic import BaseModel
-
 from .base_graph import BaseGraph
 from .abstract_graph import AbstractGraph
-
 from ..nodes import (
     FetchNode,
     ParseNode,
     GenerateAnswerPDFNode
 )
 
-
 class PDFScraperGraph(AbstractGraph):
     """
     PDFScraperGraph is a scraping pipeline that extracts information from pdf files using a natural

scrapegraphai/graphs/pdf_scraper_multi_graph.py

@@ -5,17 +5,14 @@
 from copy import copy, deepcopy
 from typing import List, Optional
 from pydantic import BaseModel
-
 from .base_graph import BaseGraph
 from .abstract_graph import AbstractGraph
 from .pdf_scraper_graph import PDFScraperGraph
-
 from ..nodes import (
     GraphIteratorNode,
     MergeAnswersNode
 )
 
-
 class PdfScraperMultiGraph(AbstractGraph):
     """ 
     PdfScraperMultiGraph is a scraping pipeline that scrapes a 

scrapegraphai/graphs/script_creator_graph.py

@@ -4,17 +4,14 @@
 
 from typing import Optional
 from pydantic import BaseModel
-
 from .base_graph import BaseGraph
 from .abstract_graph import AbstractGraph
-
 from ..nodes import (
     FetchNode,
     ParseNode,
     GenerateScraperNode
 )
 
-
 class ScriptCreatorGraph(AbstractGraph):
     """
     ScriptCreatorGraph defines a scraping pipeline for generating web scraping scripts.

scrapegraphai/graphs/script_creator_multi_graph.py

@@ -16,10 +16,10 @@
     MergeGeneratedScriptsNode
 )
 
-
 class ScriptCreatorMultiGraph(AbstractGraph):
     """ 
-    ScriptCreatorMultiGraph is a scraping pipeline that scrapes a list of URLs generating web scraping scripts.
+    ScriptCreatorMultiGraph is a scraping pipeline that scrapes a list 
+    of URLs generating web scraping scripts.
     It only requires a user prompt and a list of URLs.
     Attributes:
         prompt (str): The user prompt to search the internet.

scrapegraphai/graphs/search_graph.py

@@ -16,8 +16,6 @@
     MergeAnswersNode
 )
 
-
-
 class SearchGraph(AbstractGraph):
     """ 
     SearchGraph is a scraping pipeline that searches the internet for answers to a given prompt.

scrapegraphai/graphs/search_link_graph.py

@@ -4,13 +4,13 @@
 from pydantic import BaseModel
 from .base_graph import BaseGraph
 from .abstract_graph import AbstractGraph
-
-
 from ..nodes import ( FetchNode, ParseNode, SearchLinkNode )
 
 class SearchLinkGraph(AbstractGraph): 
     """ 
-    SearchLinkGraph is a scraping pipeline that automates the process of extracting information from web pages using a natural language model to interpret and answer prompts.
+    SearchLinkGraph is a scraping pipeline that automates the process of 
+    extracting information from web pages using a natural language model
+    to interpret and answer prompts.
 
     Attributes:
         prompt (str): The prompt for the graph.

scrapegraphai/graphs/smart_scraper_graph.py

@@ -14,7 +14,6 @@
     GenerateAnswerNode
 )
 
-
 class SmartScraperGraph(AbstractGraph):
     """
     SmartScraper is a scraping pipeline that automates the process of 

scrapegraphai/graphs/smart_scraper_multi_graph.py

@@ -15,10 +15,10 @@
     MergeAnswersNode
 )
 
-
 class SmartScraperMultiGraph(AbstractGraph):
     """ 
-    SmartScraperMultiGraph is a scraping pipeline that scrapes a list of URLs and generates answers to a given prompt.
+    SmartScraperMultiGraph is a scraping pipeline that scrapes a 
+    list of URLs and generates answers to a given prompt.
     It only requires a user prompt and a list of URLs.
 
     Attributes:
@@ -43,15 +43,16 @@ class SmartScraperMultiGraph(AbstractGraph):
         >>> result = search_graph.run()
     """
 
-    def __init__(self, prompt: str, source: List[str], config: dict, schema: Optional[BaseModel] = None):
+    def __init__(self, prompt: str, source: List[str], 
+                 config: dict, schema: Optional[BaseModel] = None):
 
         self.max_results = config.get("max_results", 3)
 
         if all(isinstance(value, str) for value in config.values()):
             self.copy_config = copy(config)
         else:
             self.copy_config = deepcopy(config)
-        
+
         self.copy_schema = deepcopy(schema)
 
         super().__init__(prompt, config, source, schema)

scrapegraphai/graphs/speech_graph.py

@@ -18,10 +18,10 @@
 from ..utils.save_audio_from_bytes import save_audio_from_bytes
 from ..models import OpenAITextToSpeech
 
-
 class SpeechGraph(AbstractGraph):
     """
-    SpeechyGraph is a scraping pipeline that scrapes the web, provide an answer to a given prompt, and generate an audio file.
+    SpeechyGraph is a scraping pipeline that scrapes the web, provide an answer 
+    to a given prompt, and generate an audio file.
 
     Attributes:
         prompt (str): The prompt for the graph.

scrapegraphai/graphs/xml_scraper_graph.py

@@ -13,7 +13,6 @@
     GenerateAnswerNode
 )
 
-
 class XMLScraperGraph(AbstractGraph):
     """
     XMLScraperGraph is a scraping pipeline that extracts information from XML files using a natural

scrapegraphai/graphs/xml_scraper_multi_graph.py

@@ -15,7 +15,6 @@
     MergeAnswersNode
 )
 
-
 class XMLScraperMultiGraph(AbstractGraph):
     """ 
     XMLScraperMultiGraph is a scraping pipeline that scrapes a list of URLs and 

scrapegraphai/nodes/conditional_node.py

@@ -42,7 +42,7 @@ def __init__(self,
         """
 
        #super().__init__(node_name, "node", input, output, 2, node_config)
-       
+       pass
 
 
     def execute(self, state: dict) -> dict:
@@ -56,8 +56,4 @@ def execute(self, state: dict) -> dict:
             str: The name of the next node to execute based on the presence of the key.
         """
 
-        if self.key_name in state and len(state[self.key_name]) > 0:
-            state["next_node"] = 0
-        else:
-            state["next_node"] = 1
-        return state
+        pass