Merge pull request #149 from VinciGit00/pre/beta

new version

Author: VinciGit00

CHANGELOG.md

@@ -1,6 +1,26 @@
+## [0.9.0-beta.1](https://github.com/VinciGit00/Scrapegraph-ai/compare/v0.8.0...v0.9.0-beta.1) (2024-05-04)
+
+
+### Features
+
+* Enable end users to pass model instances of HuggingFaceHub ([7599234](https://github.com/VinciGit00/Scrapegraph-ai/commit/7599234ab9563ca4ee9b7f5b2d0267daac621ecf))
+
+
+### Build
+
+* **deps:** bump tqdm from 4.66.1 to 4.66.3 ([0a17c74](https://github.com/VinciGit00/Scrapegraph-ai/commit/0a17c74e50d0457aec289e81183e9c779c735842))
+* **deps:** bump tqdm from 4.66.1 to 4.66.3 ([aff6f98](https://github.com/VinciGit00/Scrapegraph-ai/commit/aff6f983b02a37ced21826847a6ace5fb15ecf3d))
+
+
+### CI
+
+* **release:** 0.8.0-beta.1 [skip ci] ([d277b34](https://github.com/VinciGit00/Scrapegraph-ai/commit/d277b349a98848749a7e38ea3c511271bced3b71))
+* **release:** 0.8.0-beta.2 [skip ci] ([892500a](https://github.com/VinciGit00/Scrapegraph-ai/commit/892500afe93c4d96dcffe897b382977a22079b83))
+
 ## [0.8.0](https://github.com/VinciGit00/Scrapegraph-ai/compare/v0.7.0...v0.8.0) (2024-05-03)
 
 
+
 ### Features
 
 * add pdf scraper ([10a9453](https://github.com/VinciGit00/Scrapegraph-ai/commit/10a94530e3fd4dfde933ecfa96cb3e21df72e606))

examples/huggingfacehub/smart_scraper_huggingfacehub.py

@@ -0,0 +1,63 @@
+""" 
+Basic example of scraping pipeline using SmartScraper using Azure OpenAI Key
+"""
+
+import os
+from dotenv import load_dotenv
+from scrapegraphai.graphs import SmartScraperGraph
+from scrapegraphai.utils import prettify_exec_info
+from langchain_community.llms import HuggingFaceEndpoint
+from langchain_community.embeddings import HuggingFaceInferenceAPIEmbeddings
+
+
+
+
+## required environment variable in .env
+#HUGGINGFACEHUB_API_TOKEN
+load_dotenv()
+
+HUGGINGFACEHUB_API_TOKEN = os.getenv('HUGGINGFACEHUB_API_TOKEN')
+# ************************************************
+# Initialize the model instances
+# ************************************************
+
+repo_id = "mistralai/Mistral-7B-Instruct-v0.2"
+
+llm_model_instance = HuggingFaceEndpoint(
+    repo_id=repo_id, max_length=128, temperature=0.5, token=HUGGINGFACEHUB_API_TOKEN
+)
+
+
+
+
+embedder_model_instance = HuggingFaceInferenceAPIEmbeddings(
+    api_key=HUGGINGFACEHUB_API_TOKEN, model_name="sentence-transformers/all-MiniLM-l6-v2"
+)
+
+# ************************************************
+# Create the SmartScraperGraph instance and run it
+# ************************************************
+
+graph_config = {
+    "llm": {"model_instance": llm_model_instance},
+    "embeddings": {"model_instance": embedder_model_instance}
+}
+
+smart_scraper_graph = SmartScraperGraph(
+    prompt="List me all the events, with the following fields: company_name, event_name, event_start_date, event_start_time, event_end_date, event_end_time, location, event_mode, event_category, third_party_redirect, no_of_days, time_in_hours, hosted_or_attending, refreshments_type,  registration_available, registration_link",
+    # also accepts a string with the already downloaded HTML code
+    source="https://www.hmhco.com/event",
+    config=graph_config
+)
+
+result = smart_scraper_graph.run()
+print(result)
+
+# ************************************************
+# Get graph execution info
+# ************************************************
+
+graph_exec_info = smart_scraper_graph.get_execution_info()
+print(prettify_exec_info(graph_exec_info))
+
+

pyproject.toml

@@ -1,7 +1,7 @@
 [tool.poetry]
 name = "scrapegraphai"
 
-version = "0.8.0"
+version = "0.9.0b1"
 
 description = "A web scraping library based on LangChain which uses LLM and direct graph logic to create scraping pipelines."
 authors = [

scrapegraphai/graphs/abstract_graph.py

@@ -69,6 +69,13 @@ def _set_model_token(self, llm):
                 self.model_token = models_tokens["azure"][llm.model_name]
             except KeyError:
                 raise KeyError("Model not supported")
+            
+        elif 'HuggingFaceEndpoint' in str(type(llm)):
+            if 'mistral' in llm.repo_id:
+                try:
+                    self.model_token = models_tokens['mistral'][llm.repo_id]
+                except KeyError:
+                    raise KeyError("Model not supported")
 
 
     def _create_llm(self, llm_config: dict, chat=False) -> object:
@@ -181,7 +188,6 @@ def _create_default_embedder(self) -> object:
         Raises:
             ValueError: If the model is not supported.
         """
-
         if isinstance(self.llm_model, OpenAI):
             return OpenAIEmbeddings(api_key=self.llm_model.openai_api_key)
         elif isinstance(self.llm_model, AzureOpenAIEmbeddings):
@@ -216,6 +222,9 @@ def _create_embedder(self, embedder_config: dict) -> object:
         Raises:
             KeyError: If the model is not supported.
         """
+
+        if 'model_instance' in embedder_config:
+            return embedder_config['model_instance']
 
         # Instantiate the embedding model based on the model name
         if "openai" in embedder_config["model"]:

scrapegraphai/helpers/models_tokens.py

@@ -66,5 +66,8 @@
         "mistral.mistral-large-2402-v1:0": 32768,
         "cohere.embed-english-v3": 512,
         "cohere.embed-multilingual-v3": 512
+    },
+    "mistral": {
+        "mistralai/Mistral-7B-Instruct-v0.2": 32000
     }
 }

scrapegraphai/nodes/rag_node.py

@@ -82,6 +82,8 @@ def execute(self, state: dict) -> dict:
         if self.verbose:
             print("--- (updated chunks metadata) ---")
 
+        # check if embedder_model is provided, if not use llm_model
+        self.embedder_model = self.embedder_model if self.embedder_model else self.llm_model
         embeddings = self.embedder_model
 
         retriever = FAISS.from_documents(