Skip to content

Commit 0589083

Browse files
VinciGit00VinciGit00
committed
refactoring of loggers
1 parent e53766b commit 0589083

16 files changed

+26
-36
lines changed

‎scrapegraphai/graphs/abstract_graph.py

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@
1010
from ..helpers import models_tokens
1111
from ..models import AzureOpenAI, Bedrock, Gemini, Groq, HuggingFace, Ollama, OpenAI, Anthropic
1212
from langchain_google_genai.embeddings import GoogleGenerativeAIEmbeddings
13-
13+
from ..utils.logging import get_logger
1414

1515
class AbstractGraph(ABC):
1616
"""
@@ -61,6 +61,7 @@ def __init__(self, prompt: str, config: dict, source: Optional[str] = None):
6161
self.headless = True if config is None else config.get(
6262
"headless", True)
6363
self.loader_kwargs = config.get("loader_kwargs", {})
64+
self.logger = get_logger("graph")
6465

6566
common_params = {"headless": self.headless,
6667
"verbose": self.verbose,
@@ -79,7 +80,7 @@ def set_common_params(self, params: dict, overwrite=False):
7980

8081
for node in self.graph.nodes:
8182
node.update_config(params, overwrite)
82-
83+
8384
def _set_model_token(self, llm):
8485

8586
if 'Azure' in str(type(llm)):

‎scrapegraphai/nodes/fetch_node.py

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -74,8 +74,7 @@ def execute(self, state):
7474
necessary information to perform the operation is missing.
7575
"""
7676
if self.verbose:
77-
logger = get_logger("fetch node")
78-
logger.info(f"--- Executing {self.node_name} Node ---")
77+
self.logger.info(f"--- Executing {self.node_name} Node ---")
7978

8079
# Interpret input keys based on the provided input expression
8180
input_keys = self.get_input_keys(state)
@@ -129,7 +128,7 @@ def execute(self, state):
129128
cleanedup_html = cleanup_html(response.text, source)
130129
compressed_document = [Document(page_content=cleanedup_html)]
131130
else:
132-
logger.warning(f"Failed to retrieve contents from the webpage at url: {source}")
131+
self.logger.warning(f"Failed to retrieve contents from the webpage at url: {source}")
133132

134133
else:
135134
loader_kwargs = {}

‎scrapegraphai/nodes/generate_answer_csv_node.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
"""
2+
gg
23
Module for generating the answer node
34
"""
45
# Imports from standard library
@@ -73,8 +74,7 @@ def execute(self, state):
7374
"""
7475

7576
if self.verbose:
76-
logger = get_logger("generate_answer csv node")
77-
logger.info(f"--- Executing {self.node_name} Node ---")
77+
self.logger.info(f"--- Executing {self.node_name} Node ---")
7878

7979
# Interpret input keys based on the provided input expression
8080
input_keys = self.get_input_keys(state)

‎scrapegraphai/nodes/generate_answer_node.py

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -60,8 +60,7 @@ def execute(self, state: dict) -> dict:
6060
"""
6161

6262
if self.verbose:
63-
logger = get_logger("generate answer node")
64-
logger.info(f"--- Executing {self.node_name} Node ---")
63+
self.logger.info(f"--- Executing {self.node_name} Node ---")
6564

6665
# Interpret input keys based on the provided input expression
6766
input_keys = self.get_input_keys(state)

‎scrapegraphai/nodes/generate_answer_pdf_node.py

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -73,8 +73,7 @@ def execute(self, state):
7373
"""
7474

7575
if self.verbose:
76-
logger = get_logger("generate answer pdf node")
77-
logger.info(f"--- Executing {self.node_name} Node ---")
76+
self.logger.info(f"--- Executing {self.node_name} Node ---")
7877

7978
# Interpret input keys based on the provided input expression
8079
input_keys = self.get_input_keys(state)

‎scrapegraphai/nodes/generate_scraper_node.py

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -64,8 +64,7 @@ def execute(self, state: dict) -> dict:
6464
"""
6565

6666
if self.verbose:
67-
logger = get_logger("generate scraper node")
68-
logger.info(f"--- Executing {self.node_name} Node ---")
67+
self.logger.info(f"--- Executing {self.node_name} Node ---")
6968

7069
# Interpret input keys based on the provided input expression
7170
input_keys = self.get_input_keys(state)

‎scrapegraphai/nodes/get_probable_tags_node.py

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -51,8 +51,7 @@ def execute(self, state: dict) -> dict:
5151
"""
5252

5353
if self.verbose:
54-
logger = get_logger("get probable tags node")
55-
logger.info(f"--- Executing {self.node_name} Node ---")
54+
self.logger.info(f"--- Executing {self.node_name} Node ---")
5655

5756
# Interpret input keys based on the provided input expression
5857
input_keys = self.get_input_keys(state)

‎scrapegraphai/nodes/graph_iterator_node.py

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -60,7 +60,8 @@ def execute(self, state: dict) -> dict:
6060
batchsize = self.node_config.get("batchsize", _default_batchsize)
6161

6262
if self.verbose:
63-
print(f"--- Executing {self.node_name} Node with batchsize {batchsize} ---")
63+
self.logger.info(f"--- Executing {self.node_name} Node with batchsize {batchsize} ---")
64+
6465

6566
try:
6667
eventloop = asyncio.get_event_loop()

‎scrapegraphai/nodes/image_to_text_node.py

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -43,8 +43,7 @@ def execute(self, state: dict) -> dict:
4343
"""
4444

4545
if self.verbose:
46-
logger = get_logger("image to text node")
47-
logger.info(f"--- Executing {self.node_name} Node ---")
46+
self.logger.info(f"--- Executing {self.node_name} Node ---")
4847

4948
input_keys = self.get_input_keys(state)
5049
input_data = [state[key] for key in input_keys]

‎scrapegraphai/nodes/merge_answers_node.py

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -54,8 +54,7 @@ def execute(self, state: dict) -> dict:
5454
"""
5555

5656
if self.verbose:
57-
logger = get_logger("fetch node")
58-
logger.info(f"--- Executing {self.node_name} Node ---")
57+
self.ogger.info(f"--- Executing {self.node_name} Node ---")
5958

6059
# Interpret input keys based on the provided input expression
6160
input_keys = self.get_input_keys(state)

‎scrapegraphai/nodes/parse_node.py

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -49,8 +49,7 @@ def execute(self, state: dict) -> dict:
4949
"""
5050

5151
if self.verbose:
52-
logger = get_logger("fetch node")
53-
logger.info(f"--- Executing {self.node_name} Node ---")
52+
self.logger.info(f"--- Executing {self.node_name} Node ---")
5453

5554
# Interpret input keys based on the provided input expression
5655
input_keys = self.get_input_keys(state)

‎scrapegraphai/nodes/rag_node.py

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -56,10 +56,9 @@ def execute(self, state: dict) -> dict:
5656
KeyError: If the input keys are not found in the state, indicating that the
5757
necessary information for compressing the content is missing.
5858
"""
59-
logger = get_logger("rag node")
6059

6160
if self.verbose:
62-
logger.info(f"--- Executing {self.node_name} Node ---")
61+
self.logger.info(f"--- Executing {self.node_name} Node ---")
6362

6463
# Interpret input keys based on the provided input expression
6564
input_keys = self.get_input_keys(state)
@@ -82,7 +81,7 @@ def execute(self, state: dict) -> dict:
8281
chunked_docs.append(doc)
8382

8483
if self.verbose:
85-
logger.info("--- (updated chunks metadata) ---")
84+
self.logger.info("--- (updated chunks metadata) ---")
8685

8786
# check if embedder_model is provided, if not use llm_model
8887
self.embedder_model = self.embedder_model if self.embedder_model else self.llm_model
@@ -110,7 +109,7 @@ def execute(self, state: dict) -> dict:
110109
compressed_docs = compression_retriever.invoke(user_prompt)
111110

112111
if self.verbose:
113-
print("--- (tokens compressed and vector stored) ---")
112+
self.logger.info("--- (tokens compressed and vector stored) ---")
114113

115114
state.update({self.output[0]: compressed_docs})
116115
return state

‎scrapegraphai/nodes/robots_node.py

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -122,17 +122,17 @@ def execute(self, state: dict) -> dict:
122122

123123
if "no" in is_scrapable:
124124
if self.verbose:
125-
logger.warning("\033[31m(Scraping this website is not allowed)\033[0m")
125+
self.logger.warning("\033[31m(Scraping this website is not allowed)\033[0m")
126126

127127
if not self.force_scraping:
128128
raise ValueError(
129129
'The website you selected is not scrapable')
130130
else:
131131
if self.verbose:
132-
logger.warning("\033[33m(WARNING: Scraping this website is not allowed but you decided to force it)\033[0m")
132+
self.logger.warning("\033[33m(WARNING: Scraping this website is not allowed but you decided to force it)\033[0m")
133133
else:
134134
if self.verbose:
135-
logger.warning("\033[32m(Scraping this website is allowed)\033[0m")
135+
self.logger.warning("\033[32m(Scraping this website is allowed)\033[0m")
136136

137137
state.update({self.output[0]: is_scrapable})
138138
return state

‎scrapegraphai/nodes/search_internet_node.py

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -54,10 +54,9 @@ def execute(self, state: dict) -> dict:
5454
KeyError: If the input keys are not found in the state, indicating that the
5555
necessary information for generating the answer is missing.
5656
"""
57-
logger = get_logger("search interne node")
5857

5958
if self.verbose:
60-
logger.info(f"--- Executing {self.node_name} Node ---")
59+
self.logger.info(f"--- Executing {self.node_name} Node ---")
6160

6261
input_keys = self.get_input_keys(state)
6362

@@ -89,7 +88,7 @@ def execute(self, state: dict) -> dict:
8988
search_query = search_answer.invoke({"user_prompt": user_prompt})[0]
9089

9190
if self.verbose:
92-
logger.info(f"Search Query: {search_query}")
91+
self.logger.info(f"Search Query: {search_query}")
9392

9493

9594
answer = search_on_web(

‎scrapegraphai/nodes/search_link_node.py

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -59,8 +59,7 @@ def execute(self, state: dict) -> dict:
5959
"""
6060

6161
if self.verbose:
62-
logger = get_logger("search link node")
63-
logger.info(f"--- Executing {self.node_name} Node ---")
62+
self.logger.info(f"--- Executing {self.node_name} Node ---")
6463

6564
# Interpret input keys based on the provided input expression
6665
input_keys = self.get_input_keys(state)

‎scrapegraphai/nodes/text_to_speech_node.py

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -45,8 +45,7 @@ def execute(self, state: dict) -> dict:
4545
"""
4646

4747
if self.verbose:
48-
logger = get_logger("text to speach node")
49-
logger.info(f"--- Executing {self.node_name} Node ---")
48+
self.logger.info(f"--- Executing {self.node_name} Node ---")
5049

5150
# Interpret input keys based on the provided input expression
5251
input_keys = self.get_input_keys(state)

0 commit comments

Comments
 (0)