Merge pull request #720 from ScrapeGraphAI/deep_scraper_integration

Deep scraper integration

Author: VinciGit00

README.md

@@ -0,0 +1,28 @@
+"""
+depth_search_graph_opeani example
+"""
+import os
+from dotenv import load_dotenv
+from scrapegraphai.graphs import DepthSearchGraph
+
+load_dotenv()
+
+graph_config = {
+    "llm": {
+        "api_key": os.getenv("ANTHROPIC_API_KEY"),
+        "model": "openai/gpt-4o-mini",
+    },
+    "verbose": True,
+    "headless": False,
+    "depth": 2,
+    "only_inside_links": False,
+}
+
+search_graph = DepthSearchGraph(
+    prompt="List me all the projects with their description",
+    source="https://perinim.github.io",
+    config=graph_config
+)
+
+result = search_graph.run()
+print(result)

examples/anthropic/csv_scraper_haiku.py renamed to examples/anthropic/csv_scraper_anthropic.py

@@ -28,7 +28,7 @@ class Projects(BaseModel):
 graph_config = {
     "llm": {
         "api_key": os.environ["AZURE_OPENAI_KEY"],
-        "model": "azure_openai/gpt-3.5-turbo",
+        "model": "azure_openai/gpt-4o"
     },
     "verbose": True,
     "headless": False,

examples/anthropic/csv_scraper_graph_multi_haiku.py renamed to examples/anthropic/csv_scraper_graph_multi_anthropic.py

@@ -25,7 +25,7 @@
 graph_config = {
     "llm": {
         "api_key": os.environ["AZURE_OPENAI_KEY"],
-        "model": "azure_openai/gpt-3.5-turbo",
+        "model": "azure_openai/gpt-4o"
     },
     "verbose": True,
     "headless": False

examples/anthropic/custom_graph_haiku.py renamed to examples/anthropic/custom_graph_anthropic.py

@@ -25,7 +25,7 @@
 graph_config = {
     "llm": {
         "api_key": os.environ["AZURE_OPENAI_KEY"],
-        "model": "azure_openai/gpt-3.5-turbo",
+        "model": "azure_openai/gpt-4o"
     },
     "verbose": True,
     "headless": False

examples/anthropic/depth_search_graph_anthropic.py

@@ -0,0 +1,30 @@
+"""
+depth_search_graph_opeani example
+"""
+import os
+from dotenv import load_dotenv
+from scrapegraphai.graphs import DepthSearchGraph
+
+load_dotenv()
+
+openai_key = os.getenv("OPENAI_APIKEY")
+
+graph_config = {
+   "llm": {
+        "api_key": os.environ["AZURE_OPENAI_KEY"],
+        "model": "azure_openai/gpt-4o",
+    },
+    "verbose": True,
+    "headless": False,
+    "depth": 2,
+    "only_inside_links": False,
+}
+
+search_graph = DepthSearchGraph(
+    prompt="List me all the projects with their description",
+    source="https://perinim.github.io",
+    config=graph_config
+)
+
+result = search_graph.run()
+print(result)

examples/anthropic/json_scraper_haiku.py renamed to examples/anthropic/json_scraper_anthropic.py

@@ -23,7 +23,7 @@
 graph_config = {
     "llm": {
         "api_key": os.environ["AZURE_OPENAI_KEY"],
-        "model": "azure_openai/gpt-3.5-turbo",
+        "model": "azure_openai/gpt-4o"
     },
     "verbose": True,
     "headless": False

examples/anthropic/json_scraper_multi_haiku.py renamed to examples/anthropic/json_scraper_multi_anthropic.py

@@ -12,7 +12,7 @@
 graph_config = {
     "llm": {
         "api_key": os.environ["AZURE_OPENAI_KEY"],
-        "model": "azure_openai/gpt-3.5-turbo",
+        "model": "azure_openai/gpt-4o"
     },
     "verbose": True,
     "headless": False

examples/anthropic/pdf_scraper_graph_haiku.py renamed to examples/anthropic/pdf_scraper_graph_anthropic.py

@@ -10,7 +10,7 @@
 graph_config = {
     "llm": {
         "api_key": os.environ["AZURE_OPENAI_KEY"],
-        "model": "azure_openai/gpt-3.5-turbo",
+        "model": "azure_openai/gpt-4o"
     },
     "verbose": True,
     "headless": False

examples/anthropic/pdf_scraper_multi_haiku.py renamed to examples/anthropic/pdf_scraper_multi_anthropic.py

@@ -26,7 +26,7 @@
 graph_config = {
     "llm": {
         "api_key": os.environ["AZURE_OPENAI_KEY"],
-        "model": "azure_openai/gpt-3.5-turbo",
+        "model": "azure_openai/gpt-4o",
         "rate_limit": {
             "requests_per_second": 1
         },

examples/anthropic/rate_limit_haiku.py renamed to examples/anthropic/rate_limit_anthropic.py

@@ -28,7 +28,7 @@
 graph_config = {
     "llm": {
         "api_key": os.environ["AZURE_OPENAI_KEY"],
-        "model": "azure_openai/gpt-3.5-turbo",
+        "model": "azure_openai/gpt-4o"
     },
     "verbose": True,
     "headless": False

examples/anthropic/scrape_plain_text_haiku.py renamed to examples/anthropic/scrape_plain_text_anthropic.py

@@ -15,7 +15,7 @@
 graph_config = {
     "llm": {
         "api_key": os.environ["AZURE_OPENAI_KEY"],
-        "model": "azure_openai/gpt-3.5-turbo",
+        "model": "azure_openai/gpt-4o"
     },
     "verbose": True,
     "headless": False

examples/anthropic/script_generator_haiku.py renamed to examples/anthropic/script_generator_anthropic.py

@@ -16,7 +16,7 @@
 graph_config = {
     "llm": {
         "api_key": os.environ["AZURE_OPENAI_KEY"],
-        "model": "azure_openai/gpt-3.5-turbo",
+        "model": "azure_openai/gpt-4o"
     },
     "verbose": True,
     "headless": False

examples/anthropic/script_multi_generator_haiku.py renamed to examples/anthropic/script_multi_generator_anthropic.py

@@ -22,7 +22,7 @@
 graph_config = {
     "llm": {
         "api_key": os.environ["AZURE_OPENAI_KEY"],
-        "model": "azure_openai/gpt-3.5-turbo",
+        "model": "azure_openai/gpt-4o"
     },
     "verbose": True,
     "headless": False

examples/anthropic/search_graph_haiku.py renamed to examples/anthropic/search_graph_anthropic.py

@@ -30,7 +30,7 @@ class Dishes(BaseModel):
 graph_config = {
     "llm": {
         "api_key": os.environ["AZURE_OPENAI_KEY"],
-        "model": "azure_openai/gpt-3.5-turbo",
+        "model": "azure_openai/gpt-4o"
     },
     "verbose": True,
     "headless": False

examples/anthropic/search_graph_schema_haiku.py renamed to examples/anthropic/search_graph_schema_anthropic.py

@@ -15,7 +15,7 @@
 graph_config = {
     "llm": {
         "api_key": os.environ["AZURE_OPENAI_KEY"],
-        "model": "azure_openai/gpt-3.5-turbo",
+        "model": "azure_openai/gpt-4o"
     },
     "verbose": True,
     "headless": False

examples/anthropic/search_link_graph_haiku.py renamed to examples/anthropic/search_link_graph_anthropic.py

@@ -26,7 +26,7 @@
 graph_config = {
     "llm": {
         "api_key": os.environ["AZURE_OPENAI_KEY"],
-        "model": "azure_openai/gpt-3.5-turbo",
+        "model": "azure_openai/gpt-4o"
     },
     "verbose": True,
     "headless": False

examples/anthropic/smart_scraper_haiku.py renamed to examples/anthropic/smart_scraper_anthropic.py

@@ -14,7 +14,7 @@
 graph_config = {
     "llm": {
         "api_key": os.environ["AZURE_OPENAI_KEY"],
-        "model": "azure_openai/gpt-3.5-turbo",
+        "model": "azure_openai/gpt-4o"
     },
     "verbose": True,
     "headless": False

examples/anthropic/smart_scraper_multi_haiku.py renamed to examples/anthropic/smart_scraper_multi_anthropic.py

@@ -15,7 +15,7 @@
 graph_config = {
     "llm": {
         "api_key": os.environ["AZURE_OPENAI_KEY"],
-        "model": "azure_openai/gpt-3.5-turbo",
+        "model": "azure_openai/gpt-4o"
     },
     "verbose": True,
     "headless": False

examples/anthropic/smart_scraper_multi_concat_haiku.py renamed to examples/anthropic/smart_scraper_multi_concat_anthropic.py

@@ -29,7 +29,7 @@ class Projects(BaseModel):
 graph_config = {
     "llm": {
         "api_key": os.environ["AZURE_OPENAI_KEY"],
-        "model": "azure_openai/gpt-3.5-turbo",
+        "model": "azure_openai/gpt-4o"
     },
     "verbose": True,
     "headless": False

examples/anthropic/smart_scraper_schema_haiku.py renamed to examples/anthropic/smart_scraper_schema_anthropic.py

@@ -24,7 +24,7 @@
 graph_config = {
     "llm": {
         "api_key": os.environ["AZURE_OPENAI_KEY"],
-        "model": "azure_openai/gpt-3.5-turbo",
+        "model": "azure_openai/gpt-4o"
     },
     "verbose": True,
     "headless": False

examples/anthropic/xml_scraper_haiku.py renamed to examples/anthropic/xml_scraper_anthropic.py

@@ -25,7 +25,7 @@
 graph_config = {
     "llm": {
         "api_key": os.environ["AZURE_OPENAI_KEY"],
-        "model": "azure_openai/gpt-3.5-turbo",
+        "model": "azure_openai/gpt-4o",
     },
     "verbose": True,
     "headless": False

examples/anthropic/xml_scraper_graph_multi_haiku.py renamed to examples/anthropic/xml_scraper_graph_multi_anthropic.py

@@ -0,0 +1,31 @@
+"""
+depth_search_graph_opeani example
+"""
+import os
+from dotenv import load_dotenv
+from scrapegraphai.graphs import DepthSearchGraph
+
+load_dotenv()
+
+openai_key = os.getenv("OPENAI_APIKEY")
+
+graph_config = {
+    "llm": {
+        "client": "client_name",
+        "model": "bedrock/anthropic.claude-3-sonnet-20240229-v1:0",
+        "temperature": 0.0
+    },
+    "verbose": True,
+    "headless": False,
+    "depth": 2,
+    "only_inside_links": False,
+}
+
+search_graph = DepthSearchGraph(
+    prompt="List me all the projects with their description",
+    source="https://perinim.github.io",
+    config=graph_config
+)
+
+result = search_graph.run()
+print(result)

examples/azure/code_generator_graph_azure.py

@@ -0,0 +1,30 @@
+"""
+depth_search_graph_opeani example
+"""
+import os
+from dotenv import load_dotenv
+from scrapegraphai.graphs import DepthSearchGraph
+
+load_dotenv()
+
+deepseek_key = os.getenv("DEEPSEEK_APIKEY")
+
+graph_config = {
+    "llm": {
+        "model": "deepseek/deepseek-chat",
+        "api_key": deepseek_key,
+    },
+    "verbose": True,
+    "headless": False,
+    "depth": 2,
+    "only_inside_links": False,
+}
+
+search_graph = DepthSearchGraph(
+    prompt="List me all the projects with their description",
+    source="https://perinim.github.io",
+    config=graph_config
+)
+
+result = search_graph.run()
+print(result)

examples/azure/csv_scraper_azure.py

@@ -14,7 +14,7 @@
 # Define the configuration for the graph
 # ************************************************
 
-graph_config = {  
+graph_config = {
     "llm": {
         "model": "ernie/ernie-bot-turbo",
         "ernie_client_id": "<ernie_client_id>",

examples/azure/csv_scraper_graph_multi_azure.py

@@ -0,0 +1,26 @@
+"""
+depth_search_graph_opeani example
+"""
+from scrapegraphai.graphs import DepthSearchGraph
+
+graph_config = {
+    "llm": {
+        "model": "ernie/ernie-bot-turbo",
+        "ernie_client_id": "<ernie_client_id>",
+        "ernie_client_secret": "<ernie_client_secret>",
+        "temperature": 0.1
+    },
+    "verbose": True,
+    "headless": False,
+    "depth": 2,
+    "only_inside_links": False,
+}
+
+search_graph = DepthSearchGraph(
+    prompt="List me all the projects with their description",
+    source="https://perinim.github.io",
+    config=graph_config
+)
+
+result = search_graph.run()
+print(result)

examples/azure/depth_search_graph_azure.py

@@ -0,0 +1,30 @@
+"""
+depth_search_graph_opeani example
+"""
+import os
+from dotenv import load_dotenv
+from scrapegraphai.graphs import DepthSearchGraph
+
+load_dotenv()
+
+fireworks_api_key = os.getenv("FIREWORKS_APIKEY")
+
+graph_config = {
+    "llm": {
+        "api_key": fireworks_api_key,
+        "model": "fireworks/accounts/fireworks/models/mixtral-8x7b-instruct"
+    },
+    "verbose": True,
+    "headless": False,
+    "depth": 2,
+    "only_inside_links": False,
+}
+
+search_graph = DepthSearchGraph(
+    prompt="List me all the projects with their description",
+    source="https://perinim.github.io",
+    config=graph_config
+)
+
+result = search_graph.run()
+print(result)