Multi scraper implementation

examples/anthropic/script_multi_generator_haiku.py

@@ -0,0 +1,53 @@
+""" 
+Basic example of scraping pipeline using ScriptCreatorGraph
+"""
+
+import os
+from dotenv import load_dotenv
+from scrapegraphai.graphs import ScriptCreatorMultiGraph
+from scrapegraphai.utils import prettify_exec_info
+
+load_dotenv()
+
+# ************************************************
+# Define the configuration for the graph
+# ************************************************
+
+graph_config = {
+    "llm": {
+        "api_key": os.getenv("ANTHROPIC_API_KEY"),
+        "model": "claude-3-haiku-20240307",
+        "max_tokens": 4000
+        },
+        "library": "beautifulsoup"
+}
+
+# ************************************************
+# Create the ScriptCreatorGraph instance and run it
+# ************************************************
+
+urls=[
+    "https://schultzbergagency.com/emil-raste-karlsen/",
+    "https://schultzbergagency.com/johanna-hedberg/",
+]
+
+# ************************************************
+# Create the ScriptCreatorGraph instance and run it
+# ************************************************
+
+script_creator_graph = ScriptCreatorMultiGraph(
+    prompt="Find information about actors",
+    # also accepts a string with the already downloaded HTML code
+    source=urls,
+    config=graph_config
+)
+
+result = script_creator_graph.run()
+print(result)
+
+# ************************************************
+# Get graph execution info
+# ************************************************
+
+graph_exec_info = script_creator_graph.get_execution_info()
+print(prettify_exec_info(graph_exec_info))

examples/anthropic/smart_scraper_multi_haiku.py

@@ -12,31 +12,14 @@
 # Define the configuration for the graph
 # ************************************************
 
-openai_key = os.getenv("OPENAI_APIKEY")
-
-""" 
-Basic example of scraping pipeline using SmartScraper
-"""
-
-import os, json
-from dotenv import load_dotenv
-from scrapegraphai.graphs import SmartScraperMultiGraph
-
 load_dotenv()
 
-# ************************************************
-# Define the configuration for the graph
-# ************************************************
-
-openai_key = os.getenv("OPENAI_APIKEY")
-
 graph_config = {
     "llm": {
-        "api_key": openai_key,
-        "model": "gpt-4o",
-    },
-    "verbose": True,
-    "headless": False,
+        "api_key": os.getenv("ANTHROPIC_API_KEY"),
+        "model": "claude-3-haiku-20240307",
+        "max_tokens": 4000
+        },
 }
 
 # *******************************************************

examples/azure/script_generator_azure.py

@@ -25,7 +25,8 @@
 )
 graph_config = {
     "llm": {"model_instance": llm_model_instance},
-    "embeddings": {"model_instance": embedder_model_instance}
+    "embeddings": {"model_instance": embedder_model_instance},
+    "library": "beautifulsoup"
 }
 
 # ************************************************

examples/azure/script_multi_generator_azure.py

@@ -0,0 +1,61 @@
+""" 
+Basic example of scraping pipeline using ScriptCreatorGraph
+"""
+
+import os
+from dotenv import load_dotenv
+from scrapegraphai.graphs import ScriptCreatorMultiGraph
+from scrapegraphai.utils import prettify_exec_info
+from langchain_openai import AzureChatOpenAI
+from langchain_openai import AzureOpenAIEmbeddings
+
+load_dotenv()
+
+# ************************************************
+# Define the configuration for the graph
+# ************************************************
+llm_model_instance = AzureChatOpenAI(
+    openai_api_version=os.environ["AZURE_OPENAI_API_VERSION"],
+    azure_deployment=os.environ["AZURE_OPENAI_CHAT_DEPLOYMENT_NAME"]
+)
+
+embedder_model_instance = AzureOpenAIEmbeddings(
+    azure_deployment=os.environ["AZURE_OPENAI_EMBEDDINGS_DEPLOYMENT_NAME"],
+    openai_api_version=os.environ["AZURE_OPENAI_API_VERSION"],
+)
+graph_config = {
+    "llm": {"model_instance": llm_model_instance},
+    "embeddings": {"model_instance": embedder_model_instance},
+    "library": "beautifulsoup"
+}
+
+
+# ************************************************
+# Create the ScriptCreatorGraph instance and run it
+# ************************************************
+
+urls=[
+    "https://schultzbergagency.com/emil-raste-karlsen/",
+    "https://schultzbergagency.com/johanna-hedberg/",
+]
+
+# ************************************************
+# Create the ScriptCreatorGraph instance and run it
+# ************************************************
+
+script_creator_graph = ScriptCreatorMultiGraph(
+    prompt="Find information about actors",
+    # also accepts a string with the already downloaded HTML code
+    source=urls,
+    config=graph_config
+)
+
+result = script_creator_graph.run()
+print(result)
+
+# ************************************************
+# Get graph execution info
+# ************************************************
+
+graph_exec_info = script_creator_graph.get_execution_info()
+print(prettify_exec_info(graph_exec_info))

examples/bedrock/script_multi_generator_bedrock.py

@@ -0,0 +1,52 @@
+""" 
+Basic example of scraping pipeline using ScriptCreatorGraph
+"""
+
+from scrapegraphai.graphs import ScriptCreatorMultiGraph
+from scrapegraphai.utils import prettify_exec_info
+
+# ************************************************
+# Define the configuration for the graph
+# ************************************************
+
+graph_config = {
+    "llm": {
+        "client": "client_name",
+        "model": "bedrock/anthropic.claude-3-sonnet-20240229-v1:0",
+        "temperature": 0.0
+    },
+    "embeddings": {
+        "model": "bedrock/cohere.embed-multilingual-v3"
+    },
+        "library": "beautifulsoup"
+}
+
+# ************************************************
+# Create the ScriptCreatorGraph instance and run it
+# ************************************************
+
+urls=[
+    "https://schultzbergagency.com/emil-raste-karlsen/",
+    "https://schultzbergagency.com/johanna-hedberg/",
+]
+
+# ************************************************
+# Create the ScriptCreatorGraph instance and run it
+# ************************************************
+
+script_creator_graph = ScriptCreatorMultiGraph(
+    prompt="Find information about actors",
+    # also accepts a string with the already downloaded HTML code
+    source=urls,
+    config=graph_config
+)
+
+result = script_creator_graph.run()
+print(result)
+
+# ************************************************
+# Get graph execution info
+# ************************************************
+
+graph_exec_info = script_creator_graph.get_execution_info()
+print(prettify_exec_info(graph_exec_info))

examples/deepseek/script_multi_generator_deepseek.py

@@ -0,0 +1,60 @@
+""" 
+Basic example of scraping pipeline using ScriptCreatorGraph
+"""
+
+import os
+from dotenv import load_dotenv
+from scrapegraphai.graphs import ScriptCreatorMultiGraph
+from scrapegraphai.utils import prettify_exec_info
+
+load_dotenv()
+
+# ************************************************
+# Define the configuration for the graph
+# ************************************************
+
+deepseek_key = os.getenv("DEEPSEEK_APIKEY")
+
+graph_config = {
+    "llm": {
+        "model": "deepseek-chat",
+        "openai_api_key": deepseek_key,
+        "openai_api_base": 'https://api.deepseek.com/v1',
+    },
+     "embeddings": {
+        "model": "ollama/nomic-embed-text",
+        "temperature": 0,
+        # "base_url": "http://localhost:11434",  # set ollama URL arbitrarily
+    },
+    "library": "beautifulsoup"
+}
+
+# ************************************************
+# Create the ScriptCreatorGraph instance and run it
+# ************************************************
+
+urls=[
+    "https://schultzbergagency.com/emil-raste-karlsen/",
+    "https://schultzbergagency.com/johanna-hedberg/",
+]
+
+# ************************************************
+# Create the ScriptCreatorGraph instance and run it
+# ************************************************
+
+script_creator_graph = ScriptCreatorMultiGraph(
+    prompt="Find information about actors",
+    # also accepts a string with the already downloaded HTML code
+    source=urls,
+    config=graph_config
+)
+
+result = script_creator_graph.run()
+print(result)
+
+# ************************************************
+# Get graph execution info
+# ************************************************
+
+graph_exec_info = script_creator_graph.get_execution_info()
+print(prettify_exec_info(graph_exec_info))

examples/ernie/script_multi_generator_ernie.py

@@ -0,0 +1,54 @@
+""" 
+Basic example of scraping pipeline using ScriptCreatorGraph
+"""
+
+from scrapegraphai.graphs import ScriptCreatorMultiGraph
+from scrapegraphai.utils import prettify_exec_info
+
+# ************************************************
+# Define the configuration for the graph
+# ************************************************
+
+graph_config = {
+    "llm": {
+            "model": "ernie-bot-turbo",
+            "ernie_client_id": "<ernie_client_id>",
+            "ernie_client_secret": "<ernie_client_secret>",
+            "temperature": 0.1
+        },
+        "embeddings": {
+            "model": "ollama/nomic-embed-text",
+            "temperature": 0,
+            "base_url": "http://localhost:11434"},
+    "library": "beautifulsoup"
+}
+
+# ************************************************
+# Create the ScriptCreatorGraph instance and run it
+# ************************************************
+
+urls=[
+    "https://schultzbergagency.com/emil-raste-karlsen/",
+    "https://schultzbergagency.com/johanna-hedberg/",
+]
+
+# ************************************************
+# Create the ScriptCreatorGraph instance and run it
+# ************************************************
+
+script_creator_graph = ScriptCreatorMultiGraph(
+    prompt="Find information about actors",
+    # also accepts a string with the already downloaded HTML code
+    source=urls,
+    config=graph_config
+)
+
+result = script_creator_graph.run()
+print(result)
+
+# ************************************************
+# Get graph execution info
+# ************************************************
+
+graph_exec_info = script_creator_graph.get_execution_info()
+print(prettify_exec_info(graph_exec_info))

examples/gemini/script_multi_generator_gemini.py

@@ -0,0 +1,54 @@
+""" 
+Basic example of scraping pipeline using ScriptCreatorGraph
+"""
+
+import os
+from dotenv import load_dotenv
+from scrapegraphai.graphs import ScriptCreatorMultiGraph
+from scrapegraphai.utils import prettify_exec_info
+
+load_dotenv()
+
+# ************************************************
+# Define the configuration for the graph
+# ************************************************
+
+gemini_key = os.getenv("GOOGLE_APIKEY")
+
+graph_config = {
+    "llm": {
+        "api_key": gemini_key,
+        "model": "gemini-pro",
+    },
+    "library": "beautifoulsoup"
+}
+
+# ************************************************
+# Create the ScriptCreatorGraph instance and run it
+# ************************************************
+
+urls=[
+    "https://schultzbergagency.com/emil-raste-karlsen/",
+    "https://schultzbergagency.com/johanna-hedberg/",
+]
+
+# ************************************************
+# Create the ScriptCreatorGraph instance and run it
+# ************************************************
+
+script_creator_graph = ScriptCreatorMultiGraph(
+    prompt="Find information about actors",
+    # also accepts a string with the already downloaded HTML code
+    source=urls,
+    config=graph_config
+)
+
+result = script_creator_graph.run()
+print(result)
+
+# ************************************************
+# Get graph execution info
+# ************************************************
+
+graph_exec_info = script_creator_graph.get_execution_info()
+print(prettify_exec_info(graph_exec_info))