Merge branch 'pre/beta' into temp

Author: VinciGit00

CHANGELOG.md

@@ -25,6 +25,8 @@
 * conditional node ([ce00345](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/ce003454953e5785d4746223c252de38cd5d07ea))
 
 ## [1.13.0](https://github.com/ScrapeGraphAI/Scrapegraph-ai/compare/v1.12.2...v1.13.0) (2024-08-09)
+## [1.13.0-beta.9](https://github.com/ScrapeGraphAI/Scrapegraph-ai/compare/v1.13.0-beta.8...v1.13.0-beta.9) (2024-08-10)
+
 
 
 ### Features
@@ -65,6 +67,15 @@
 * **release:** 1.13.0-beta.5 [skip ci] ([2eba73b](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/2eba73b784ee443260117e98ab7c943934b3018d)), closes [#513](https://github.com/ScrapeGraphAI/Scrapegraph-ai/issues/513)
 * **release:** 1.13.0-beta.6 [skip ci] ([e75b574](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/e75b574b67040e127599da9ee1b0eee13d234cb9))
 * **release:** 1.13.0-beta.7 [skip ci] ([6e56925](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/6e56925355c424edae290c70fd98646ab5f420ee))
+* add refactoring of default temperature ([6c3b37a](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/6c3b37ab001b80c09ea9ffb56d4c3df338e33a7a))
+
+## [1.13.0-beta.8](https://github.com/ScrapeGraphAI/Scrapegraph-ai/compare/v1.13.0-beta.7...v1.13.0-beta.8) (2024-08-09)
+
+
+### Bug Fixes
+
+* broken node ([1272273](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/127227349915deeb0dede34aa575ad269ed7cbe3))
+
 
 ## [1.13.0-beta.7](https://github.com/ScrapeGraphAI/Scrapegraph-ai/compare/v1.13.0-beta.6...v1.13.0-beta.7) (2024-08-09)
 

examples/local_models/smart_scraper_ollama.py

@@ -14,7 +14,6 @@
         "format": "json",  # Ollama needs the format to be specified explicitly
         # "base_url": "http://localhost:11434", # set ollama URL arbitrarily
     },
-  
     "verbose": True,
     "headless": False
 }
@@ -23,8 +22,13 @@
 # Create the SmartScraperGraph instance and run it
 # ************************************************
 smart_scraper_graph = SmartScraperGraph(
+<<<<<<< Updated upstream
     prompt="Find some information about what does the company do, the name and a contact email.",
     source="https://scrapegraphai.com/",
+=======
+    prompt="List all the projects with their descriptions",
+    source="https://perinim.github.io/projects/",
+>>>>>>> Stashed changes
     config=graph_config
 )
 

pyproject.toml

@@ -3,6 +3,8 @@ name = "scrapegraphai"
 
 
 version = "1.13.3"
+version = "1.13.0b9"
+
 
 
 description = "A web scraping library based on LangChain which uses LLM and direct graph logic to create scraping pipelines."

scrapegraphai/graphs/abstract_graph.py

@@ -7,20 +7,16 @@
 import uuid
 import warnings
 from pydantic import BaseModel
-
 from langchain_community.chat_models import ErnieBotChat
 from langchain_nvidia_ai_endpoints import ChatNVIDIA
 from langchain.chat_models import init_chat_model
-
 from ..helpers import models_tokens
 from ..models import (
     OneApi,
     DeepSeek
 )
 from ..utils.logging import set_verbosity_warning, set_verbosity_info
 
-
-
 class AbstractGraph(ABC):
     """
     Scaffolding class for creating a graph representation and executing it.
@@ -53,6 +49,9 @@ class AbstractGraph(ABC):
     def __init__(self, prompt: str, config: dict,
                  source: Optional[str] = None, schema: Optional[BaseModel] = None):
 
+        if config.get("llm").get("temperature") is None:
+            config["llm"]["temperature"] = 0
+
         self.prompt = prompt
         self.source = source
         self.config = config
@@ -212,7 +211,7 @@ def handle_model(model_name, provider, token_key, default_token=8192):
                 print("model not found, using default token size (8192)")
                 self.model_token = 8192
             return ErnieBotChat(llm_params)
-        
+
         if "oneapi" in llm_params["model"]:
             # take the model after the last dash
             llm_params["model"] = llm_params["model"].split("/")[-1]
@@ -221,7 +220,7 @@ def handle_model(model_name, provider, token_key, default_token=8192):
             except KeyError as exc:
                 raise KeyError("Model not supported") from exc
             return OneApi(llm_params)
-        
+
         if "nvidia" in llm_params["model"]:
             try:
                 self.model_token = models_tokens["nvidia"][llm_params["model"].split("/")[-1]]

scrapegraphai/graphs/base_graph.py

@@ -6,9 +6,7 @@
 from typing import Tuple
 from langchain_community.callbacks import get_openai_callback
 from ..integrations import BurrBridge
-
-# Import telemetry functions
-from ..telemetry import log_graph_execution, log_event
+from ..telemetry import log_graph_execution
 
 class BaseGraph:
     """

scrapegraphai/graphs/csv_scraper_graph.py

@@ -4,16 +4,13 @@
 
 from typing import Optional
 from pydantic import BaseModel
-
 from .base_graph import BaseGraph
 from .abstract_graph import AbstractGraph
-
 from ..nodes import (
     FetchNode,
     GenerateAnswerCSVNode
 )
 
-
 class CSVScraperGraph(AbstractGraph):
     """
     SmartScraper is a comprehensive web scraping tool that automates the process of extracting

scrapegraphai/graphs/csv_scraper_multi_graph.py

@@ -4,22 +4,19 @@
 
 from copy import copy, deepcopy
 from typing import List, Optional
-
 from pydantic import BaseModel
-
 from .base_graph import BaseGraph
 from .abstract_graph import AbstractGraph
 from .csv_scraper_graph import CSVScraperGraph
-
 from ..nodes import (
     GraphIteratorNode,
     MergeAnswersNode
 )
 
-
 class CSVScraperMultiGraph(AbstractGraph):
     """ 
-    CSVScraperMultiGraph is a scraping pipeline that scrapes a list of URLs and generates answers to a given prompt.
+    CSVScraperMultiGraph is a scraping pipeline that 
+    scrapes a list of URLs and generates answers to a given prompt.
     It only requires a user prompt and a list of URLs.
 
     Attributes:
@@ -44,7 +41,8 @@ class CSVScraperMultiGraph(AbstractGraph):
         >>> result = search_graph.run()
     """
 
-    def __init__(self, prompt: str, source: List[str], config: dict, schema: Optional[BaseModel] = None):
+    def __init__(self, prompt: str, source: List[str], 
+                 config: dict, schema: Optional[BaseModel] = None):
 
         self.max_results = config.get("max_results", 3)
 

scrapegraphai/graphs/deep_scraper_graph.py

@@ -4,10 +4,8 @@
 
 from typing import Optional
 from pydantic import BaseModel
-
 from .base_graph import BaseGraph
 from .abstract_graph import AbstractGraph
-
 from ..nodes import (
     FetchNode,
     SearchLinkNode,
@@ -18,7 +16,6 @@
     MergeAnswersNode
 )
 
-
 class DeepScraperGraph(AbstractGraph):
     """
     [WIP]
@@ -87,7 +84,6 @@ def _create_repeated_graph(self) -> BaseGraph:
             output=["relevant_chunks"],
             node_config={
                 "llm_model": self.llm_model,
-                "embedder_model": self.embedder_model
             }
         )
         generate_answer_node = GenerateAnswerNode(
@@ -104,7 +100,6 @@ def _create_repeated_graph(self) -> BaseGraph:
             output=["relevant_links"],
             node_config={
                 "llm_model": self.llm_model,
-                "embedder_model": self.embedder_model
             }
         )
         graph_iterator_node = GraphIteratorNode(

scrapegraphai/graphs/json_scraper_graph.py

@@ -4,16 +4,13 @@
 
 from typing import Optional
 from pydantic import BaseModel
-
 from .base_graph import BaseGraph
 from .abstract_graph import AbstractGraph
-
 from ..nodes import (
     FetchNode,
     GenerateAnswerNode
 )
 
-
 class JSONScraperGraph(AbstractGraph):
     """
     JSONScraperGraph defines a scraping pipeline for JSON files.

scrapegraphai/graphs/json_scraper_multi_graph.py

@@ -5,20 +5,18 @@
 from copy import copy, deepcopy
 from typing import List, Optional
 from pydantic import BaseModel
-
 from .base_graph import BaseGraph
 from .abstract_graph import AbstractGraph
 from .json_scraper_graph import JSONScraperGraph
-
 from ..nodes import (
     GraphIteratorNode,
     MergeAnswersNode
 )
 
-
 class JSONScraperMultiGraph(AbstractGraph):
     """ 
-    JSONScraperMultiGraph is a scraping pipeline that scrapes a list of URLs and generates answers to a given prompt.
+    JSONScraperMultiGraph is a scraping pipeline that scrapes a 
+    list of URLs and generates answers to a given prompt.
     It only requires a user prompt and a list of URLs.
 
     Attributes:

scrapegraphai/graphs/markdown_scraper_multi_graph.py

@@ -5,17 +5,14 @@
 from copy import copy, deepcopy
 from typing import List, Optional
 from pydantic import BaseModel
-
 from .base_graph import BaseGraph
 from .abstract_graph import AbstractGraph
 from .markdown_scraper_graph import MDScraperGraph
-
 from ..nodes import (
     GraphIteratorNode,
     MergeAnswersNode
 )
 
-
 class MDScraperMultiGraph(AbstractGraph):
     """
     MDScraperMultiGraph is a scraping pipeline that scrapes a list of URLs and 

scrapegraphai/graphs/omni_scraper_graph.py

@@ -4,17 +4,14 @@
 
 from typing import Optional
 from pydantic import BaseModel
-
 from .base_graph import BaseGraph
 from .abstract_graph import AbstractGraph
-
 from ..nodes import (
     FetchNode,
     ParseNode,
     ImageToTextNode,
     GenerateAnswerOmniNode
 )
-
 from ..models import OpenAIImageToText
 
 class OmniScraperGraph(AbstractGraph):

scrapegraphai/graphs/pdf_scraper_graph.py

@@ -5,17 +5,14 @@
 
 from typing import Optional
 from pydantic import BaseModel
-
 from .base_graph import BaseGraph
 from .abstract_graph import AbstractGraph
-
 from ..nodes import (
     FetchNode,
     ParseNode,
     GenerateAnswerPDFNode
 )
 
-
 class PDFScraperGraph(AbstractGraph):
     """
     PDFScraperGraph is a scraping pipeline that extracts information from pdf files using a natural

scrapegraphai/graphs/pdf_scraper_multi_graph.py

@@ -5,17 +5,14 @@
 from copy import copy, deepcopy
 from typing import List, Optional
 from pydantic import BaseModel
-
 from .base_graph import BaseGraph
 from .abstract_graph import AbstractGraph
 from .pdf_scraper_graph import PDFScraperGraph
-
 from ..nodes import (
     GraphIteratorNode,
     MergeAnswersNode
 )
 
-
 class PdfScraperMultiGraph(AbstractGraph):
     """ 
     PdfScraperMultiGraph is a scraping pipeline that scrapes a 

scrapegraphai/graphs/script_creator_graph.py

@@ -4,17 +4,14 @@
 
 from typing import Optional
 from pydantic import BaseModel
-
 from .base_graph import BaseGraph
 from .abstract_graph import AbstractGraph
-
 from ..nodes import (
     FetchNode,
     ParseNode,
     GenerateScraperNode
 )
 
-
 class ScriptCreatorGraph(AbstractGraph):
     """
     ScriptCreatorGraph defines a scraping pipeline for generating web scraping scripts.

scrapegraphai/graphs/script_creator_multi_graph.py

@@ -16,10 +16,10 @@
     MergeGeneratedScriptsNode
 )
 
-
 class ScriptCreatorMultiGraph(AbstractGraph):
     """ 
-    ScriptCreatorMultiGraph is a scraping pipeline that scrapes a list of URLs generating web scraping scripts.
+    ScriptCreatorMultiGraph is a scraping pipeline that scrapes a list 
+    of URLs generating web scraping scripts.
     It only requires a user prompt and a list of URLs.
     Attributes:
         prompt (str): The user prompt to search the internet.

scrapegraphai/graphs/search_graph.py

@@ -16,8 +16,6 @@
     MergeAnswersNode
 )
 
-
-
 class SearchGraph(AbstractGraph):
     """ 
     SearchGraph is a scraping pipeline that searches the internet for answers to a given prompt.

scrapegraphai/graphs/search_link_graph.py

@@ -4,13 +4,13 @@
 from pydantic import BaseModel
 from .base_graph import BaseGraph
 from .abstract_graph import AbstractGraph
-
-
 from ..nodes import ( FetchNode, ParseNode, SearchLinkNode )
 
 class SearchLinkGraph(AbstractGraph): 
     """ 
-    SearchLinkGraph is a scraping pipeline that automates the process of extracting information from web pages using a natural language model to interpret and answer prompts.
+    SearchLinkGraph is a scraping pipeline that automates the process of 
+    extracting information from web pages using a natural language model
+    to interpret and answer prompts.
 
     Attributes:
         prompt (str): The prompt for the graph.

scrapegraphai/graphs/smart_scraper_graph.py

@@ -14,7 +14,6 @@
     GenerateAnswerNode
 )
 
-
 class SmartScraperGraph(AbstractGraph):
     """
     SmartScraper is a scraping pipeline that automates the process of