Merge pull request #567 from goasleep/feature/add_model_instance_info

Author: VinciGit00

docs/source/scrapers/llm.rst

@@ -194,3 +194,35 @@ We can also pass a model instance for the chat model and the embedding model. Fo
             "model_instance": embedder_model_instance
         }
     }
+
+Other LLM models
+^^^^^^^^^^^^^^^^
+
+We can also pass a model instance for the chat model and the embedding model through the **model_instance** parameter. 
+This feature enables you to utilize a Langchain model instance.
+You will discover the model you require within the provided list:
+
+- `chat model list <https://python.langchain.com/v0.2/docs/integrations/chat/#all-chat-models>`_
+- `embedding model list <https://python.langchain.com/v0.2/docs/integrations/text_embedding/#all-embedding-models>`_.
+
+For instance, consider **chat model** Moonshot. We can integrate it in the following manner:
+
+.. code-block:: python
+    
+    from langchain_community.chat_models.moonshot import MoonshotChat
+
+    # The configuration parameters are contingent upon the specific model you select
+    llm_instance_config = {
+        "model": "moonshot-v1-8k",
+        "base_url": "https://api.moonshot.cn/v1",
+        "moonshot_api_key": "MOONSHOT_API_KEY",
+    }
+
+    llm_model_instance = MoonshotChat(**llm_instance_config)
+    graph_config = {
+        "llm": {
+            "model_instance": llm_model_instance, 
+            "model_tokens": 5000
+        },
+    }
+    

examples/model_instance/.env.example

@@ -0,0 +1 @@
+MOONLIGHT_API_KEY="YOUR MOONLIGHT API KEY"

examples/model_instance/smart_scraper_with_model_instace.py

@@ -0,0 +1,53 @@
+""" 
+Basic example of scraping pipeline using SmartScraper and model_instace
+"""
+
+import os, json
+from scrapegraphai.graphs import SmartScraperGraph
+from scrapegraphai.utils import prettify_exec_info
+from langchain_community.chat_models.moonshot import MoonshotChat
+from dotenv import load_dotenv
+load_dotenv()
+
+# ************************************************
+# Define the configuration for the graph
+# ************************************************
+
+
+llm_instance_config = {
+    "model": "moonshot-v1-8k",
+    "base_url": "https://api.moonshot.cn/v1",
+    "moonshot_api_key": os.getenv("MOONLIGHT_API_KEY"),
+}
+
+
+llm_model_instance = MoonshotChat(**llm_instance_config)
+
+graph_config = {
+    "llm": {
+        "model_instance": llm_model_instance, 
+        "model_tokens": 10000
+    },
+    "verbose": True,
+    "headless": True,
+}
+
+# ************************************************
+# Create the SmartScraperGraph instance and run it
+# ************************************************
+
+smart_scraper_graph = SmartScraperGraph(
+    prompt="List me what does the company do, the name and a contact email.",
+    source="https://scrapegraphai.com/",
+    config=graph_config
+)
+
+result = smart_scraper_graph.run()
+print(json.dumps(result, indent=4))
+
+# ************************************************
+# Get graph execution info
+# ************************************************
+
+graph_exec_info = smart_scraper_graph.get_execution_info()
+print(prettify_exec_info(graph_exec_info))

examples/moonshot/.env.example

@@ -0,0 +1 @@
+MOONLIGHT_API_KEY="YOUR MOONLIGHT API KEY"

examples/moonshot/readme.md

@@ -0,0 +1 @@
+This folder offer an example of how to use ScrapeGraph-AI with Moonshot and SmartScraperGraph. More usage examples can refer to openai exapmles.

examples/moonshot/smart_scraper_with_moonshot.py

@@ -0,0 +1,53 @@
+""" 
+Basic example of scraping pipeline using SmartScraper and model_instace
+"""
+
+import os, json
+from scrapegraphai.graphs import SmartScraperGraph
+from scrapegraphai.utils import prettify_exec_info
+from langchain_community.chat_models.moonshot import MoonshotChat
+from dotenv import load_dotenv
+load_dotenv()
+
+# ************************************************
+# Define the configuration for the graph
+# ************************************************
+
+
+llm_instance_config = {
+    "model": "moonshot-v1-8k",
+    "base_url": "https://api.moonshot.cn/v1",
+    "moonshot_api_key": os.getenv("MOONLIGHT_API_KEY"),
+}
+
+
+llm_model_instance = MoonshotChat(**llm_instance_config)
+
+graph_config = {
+    "llm": {
+        "model_instance": llm_model_instance, 
+        "model_tokens": 10000
+    },
+    "verbose": True,
+    "headless": True,
+}
+
+# ************************************************
+# Create the SmartScraperGraph instance and run it
+# ************************************************
+
+smart_scraper_graph = SmartScraperGraph(
+    prompt="List me what does the company do, the name and a contact email.",
+    source="https://scrapegraphai.com/",
+    config=graph_config
+)
+
+result = smart_scraper_graph.run()
+print(json.dumps(result, indent=4))
+
+# ************************************************
+# Get graph execution info
+# ************************************************
+
+graph_exec_info = smart_scraper_graph.get_execution_info()
+print(prettify_exec_info(graph_exec_info))