feat: added verbose flag to suppress print statements

examples/groq/smart_scraper_groq.py

@@ -18,9 +18,16 @@
 
 graph_config = {
     "llm": {
-        "api_key": groq_key,
         "model": "groq/gemma-7b-it",
+        "api_key": groq_key,
+        "temperature": 0
+    },
+    "embeddings": {
+        "model": "ollama/nomic-embed-text",
+        "temperature": 0,
+        "base_url": "http://localhost:11434",  # set ollama URL arbitrarily
     },
+    "headless": False
 }
 
 # ************************************************

examples/openai/smart_scraper_openai.py

@@ -21,6 +21,7 @@
         "api_key": openai_key,
         "model": "gpt-3.5-turbo",
     },
+    "verbose":False,
 }
 
 # ************************************************

scrapegraphai/graphs/abstract_graph.py

@@ -22,6 +22,12 @@ def __init__(self, prompt: str, config: dict, source: Optional[str] = None):
         self.llm_model = self._create_llm(config["llm"])
         self.embedder_model = self.llm_model if "embeddings" not in config else self._create_llm(
             config["embeddings"])
+
+        # Set common configuration parameters
+        self.verbose = True if config is None else config.get("verbose", False)
+        self.headless = True if config is None else config.get("headless", True)
+
+        # Create the graph
         self.graph = self._create_graph()
         self.final_state = None
         self.execution_info = None

scrapegraphai/graphs/base_graph.py

@@ -4,7 +4,7 @@
 import time
 import warnings
 from langchain_community.callbacks import get_openai_callback
-
+from typing import Tuple
 
 class BaseGraph:
     """
@@ -56,7 +56,7 @@ def _create_edges(self, edges: list) -> dict:
             edge_dict[from_node.node_name] = to_node.node_name
         return edge_dict
 
-    def execute(self, initial_state: dict) -> (dict, list):
+    def execute(self, initial_state: dict) -> Tuple[dict, list]:
         """
         Executes the graph by traversing nodes starting from the entry point. The execution
         follows the edges based on the result of each node's execution and continues until

scrapegraphai/graphs/json_scraper_graph.py

@@ -32,24 +32,35 @@ def _create_graph(self):
         fetch_node = FetchNode(
             input="json_dir",
             output=["doc"],
+            node_config={
+                "headless": self.headless,
+                "verbose": self.verbose
+            }
         )
         parse_node = ParseNode(
             input="doc",
             output=["parsed_doc"],
-            node_config={"chunk_size": self.model_token}
+            node_config={
+                "chunk_size": self.model_token,
+                "verbose": self.verbose
+            }
         )
         rag_node = RAGNode(
             input="user_prompt & (parsed_doc | doc)",
             output=["relevant_chunks"],
             node_config={
                 "llm": self.llm_model,
-                "embedder_model": self.embedder_model
+                "embedder_model": self.embedder_model,
+                "verbose": self.verbose
             }
         )
         generate_answer_node = GenerateAnswerNode(
             input="user_prompt & (relevant_chunks | parsed_doc | doc)",
             output=["answer"],
-            node_config={"llm": self.llm_model},
+            node_config={
+                "llm": self.llm_model,
+                "verbose": self.verbose
+            }
         )
 
         return BaseGraph(

scrapegraphai/graphs/search_graph.py

@@ -24,30 +24,43 @@ def _create_graph(self):
         search_internet_node = SearchInternetNode(
             input="user_prompt",
             output=["url"],
-            node_config={"llm": self.llm_model}
+            node_config={
+                "llm": self.llm_model,
+                "verbose": self.verbose
+            }
         )
         fetch_node = FetchNode(
             input="url | local_dir",
             output=["doc"],
-            node_config={"headless": True if self.config is None else self.config.get("headless", True)}
+            node_config={
+                "headless": self.headless,
+                "verbose": self.verbose
+            }
         )
         parse_node = ParseNode(
             input="doc",
             output=["parsed_doc"],
-            node_config={"chunk_size": self.model_token}
+            node_config={
+                "chunk_size": self.model_token,
+                "verbose": self.verbose
+            }
         )
         rag_node = RAGNode(
             input="user_prompt & (parsed_doc | doc)",
             output=["relevant_chunks"],
             node_config={
                 "llm": self.llm_model,
-                "embedder_model": self.embedder_model
+                "embedder_model": self.embedder_model,
+                "verbose": self.verbose
             }
         )
         generate_answer_node = GenerateAnswerNode(
             input="user_prompt & (relevant_chunks | parsed_doc | doc)",
             output=["answer"],
-            node_config={"llm": self.llm_model},
+            node_config={
+                "llm": self.llm_model,
+                "verbose": self.verbose
+            }
         )
 
         return BaseGraph(

scrapegraphai/graphs/smart_scraper_graph.py

@@ -24,7 +24,7 @@ def __init__(self, prompt: str, source: str, config: dict):
         super().__init__(prompt, config, source)
 
         self.input_key = "url" if source.startswith("http") else "local_dir"
-
+        
 
     def _create_graph(self):
         """
@@ -33,25 +33,35 @@ def _create_graph(self):
         fetch_node = FetchNode(
             input="url | local_dir",
             output=["doc"],
-            node_config={"headless": True if self.config is None else self.config.get("headless", True)}
+            node_config={
+                "headless": self.headless,
+                "verbose": self.verbose
+            }
         )
         parse_node = ParseNode(
             input="doc",
             output=["parsed_doc"],
-            node_config={"chunk_size": self.model_token}
+            node_config={
+                "chunk_size": self.model_token,
+                "verbose": self.verbose
+            }
         )
         rag_node = RAGNode(
             input="user_prompt & (parsed_doc | doc)",
             output=["relevant_chunks"],
             node_config={
                 "llm": self.llm_model,
-                "embedder_model": self.embedder_model
+                "embedder_model": self.embedder_model,
+                "verbose": self.verbose
             }
         )
         generate_answer_node = GenerateAnswerNode(
             input="user_prompt & (relevant_chunks | parsed_doc | doc)",
             output=["answer"],
-            node_config={"llm": self.llm_model},
+            node_config={
+                "llm": self.llm_model,
+                "verbose": self.verbose
+            }
         )
 
         return BaseGraph(

scrapegraphai/graphs/speech_graph.py

@@ -35,31 +35,43 @@ def _create_graph(self):
         fetch_node = FetchNode(
             input="url | local_dir",
             output=["doc"],
-            node_config={"headless": True if self.config is None else self.config.get("headless", True)}
+            node_config={
+                "headless": self.headless,
+                "verbose": self.verbose
+            }
         )
         parse_node = ParseNode(
             input="doc",
             output=["parsed_doc"],
-            node_config={"chunk_size": self.model_token}
+            node_config={
+                "chunk_size": self.model_token,
+                "verbose": self.verbose
+            }
         )
         rag_node = RAGNode(
             input="user_prompt & (parsed_doc | doc)",
             output=["relevant_chunks"],
             node_config={
                 "llm": self.llm_model,
-                "embedder_model": self.embedder_model
+                "embedder_model": self.embedder_model,
+                "verbose": self.verbose
             }
         )
         generate_answer_node = GenerateAnswerNode(
             input="user_prompt & (relevant_chunks | parsed_doc | doc)",
             output=["answer"],
-            node_config={"llm": self.llm_model},
+            node_config={
+                "llm": self.llm_model,
+                "verbose": self.verbose
+            }
         )
         text_to_speech_node = TextToSpeechNode(
             input="answer",
             output=["audio"],
-            node_config={"tts_model": OpenAITextToSpeech(
-                self.config["tts_model"])},
+            node_config={
+                "tts_model": OpenAITextToSpeech(self.config["tts_model"]),
+                "verbose": self.verbose
+            }
         )
 
         return BaseGraph(

scrapegraphai/graphs/xml_scraper_graph.py

@@ -32,24 +32,35 @@ def _create_graph(self):
         fetch_node = FetchNode(
             input="xml_dir",
             output=["doc"],
+            node_config={
+                "headless": self.headless,
+                "verbose": self.verbose
+            }
         )
         parse_node = ParseNode(
             input="doc",
             output=["parsed_doc"],
-            node_config={"chunk_size": self.model_token}
+            node_config={
+                "chunk_size": self.model_token,
+                "verbose": self.verbose
+            }
         )
         rag_node = RAGNode(
             input="user_prompt & (parsed_doc | doc)",
             output=["relevant_chunks"],
             node_config={
                 "llm": self.llm_model,
-                "embedder_model": self.embedder_model
+                "embedder_model": self.embedder_model,
+                "verbose": self.verbose
             }
         )
         generate_answer_node = GenerateAnswerNode(
             input="user_prompt & (relevant_chunks | parsed_doc | doc)",
             output=["answer"],
-            node_config={"llm": self.llm_model},
+            node_config={
+                "llm": self.llm_model,
+                "verbose": self.verbose
+            }
         )
 
         return BaseGraph(

scrapegraphai/nodes/fetch_node.py

@@ -47,6 +47,7 @@ def __init__(self, input: str, output: List[str], node_config: Optional[dict], n
         super().__init__(node_name, "node", input, output, 1)
 
         self.headless = True if node_config is None else node_config.get("headless", True)
+        self.verbose = True if node_config is None else node_config.get("verbose", False)
 
     def execute(self, state):
         """
@@ -63,7 +64,8 @@ def execute(self, state):
             KeyError: If the 'url' key is not found in the state, indicating that the
                     necessary information to perform the operation is missing.
         """
-        print(f"--- Executing {self.node_name} Node ---")
+        if self.verbose:
+            print(f"--- Executing {self.node_name} Node ---")
 
         # Interpret input keys based on the provided input expression
         input_keys = self.get_input_keys(state)

scrapegraphai/nodes/generate_answer_node.py

@@ -49,6 +49,7 @@ def __init__(self, input: str, output: List[str], node_config: dict,
         """
         super().__init__(node_name, "node", input, output, 2, node_config)
         self.llm_model = node_config["llm"]
+        self.verbose = True if node_config is None else node_config.get("verbose", False)
 
     def execute(self, state):
         """
@@ -69,7 +70,8 @@ def execute(self, state):
                       that the necessary information for generating an answer is missing.
         """
 
-        print(f"--- Executing {self.node_name} Node ---")
+        if self.verbose:
+            print(f"--- Executing {self.node_name} Node ---")
 
         # Interpret input keys based on the provided input expression
         input_keys = self.get_input_keys(state)
@@ -116,7 +118,7 @@ def execute(self, state):
         chains_dict = {}
 
         # Use tqdm to add progress bar
-        for i, chunk in enumerate(tqdm(doc, desc="Processing chunks")):
+        for i, chunk in enumerate(tqdm(doc, desc="Processing chunks", disable=not self.verbose)):
             if len(doc) == 1:
                 prompt = PromptTemplate(
                     template=template_no_chunks,

scrapegraphai/nodes/image_to_text_node.py

@@ -29,6 +29,7 @@ def __init__(self, input: str, output: List[str], node_config: dict,
         """
         super().__init__(node_name, "node", input, output, 1, node_config)
         self.llm_model = node_config["llm_model"]
+        self.verbose = True if node_config is None else node_config.get("verbose", False)
 
     def execute(self, state: dict) -> dict:
         """
@@ -40,9 +41,11 @@ def execute(self, state: dict) -> dict:
         Returns:
             dict: The updated state after executing this node.
         """
-        print("---GENERATING TEXT FROM IMAGE---")
-        input_keys = self.get_input_keys(state)
 
+        if self.verbose:
+            print("---GENERATING TEXT FROM IMAGE---")
+
+        input_keys = self.get_input_keys(state)
         input_data = [state[key] for key in input_keys]
         url = input_data[0]
 

scrapegraphai/nodes/parse_node.py

@@ -40,6 +40,8 @@ def __init__(self, input: str, output: List[str], node_config: dict, node_name:
         """
         super().__init__(node_name, "node", input, output, 1, node_config)
 
+        self.verbose = True if node_config is None else node_config.get("verbose", False)
+
     def execute(self,  state):
         """
         Executes the node's logic to parse the HTML document based on specified tags. 
@@ -60,7 +62,8 @@ def execute(self,  state):
                       information for parsing is missing.
         """
 
-        print(f"--- Executing {self.node_name} Node ---")
+        if self.verbose:
+            print(f"--- Executing {self.node_name} Node ---")
 
         # Interpret input keys based on the provided input expression
         input_keys = self.get_input_keys(state)