ScrapeGraphAI · VinciGit00 · Oct 6, 2024 · Sep 26, 2024 · Sep 26, 2024 · Sep 26, 2024
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -22,11 +22,15 @@
 ## [1.24.1](https://github.com/ScrapeGraphAI/Scrapegraph-ai/compare/v1.24.0...v1.24.1) (2024-09-26)
 
 
+
 ### Bug Fixes
 
 * script creator multi ([9905be8](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/9905be8a37dc1ff4b90fe9b8be987887253be8bd))
 
 ## [1.24.0](https://github.com/ScrapeGraphAI/Scrapegraph-ai/compare/v1.23.1...v1.24.0) (2024-09-26)
+* integration with html_mode ([f87ffa1](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/f87ffa1d8db32b38c47d9f5aa2ae88f1d7978a04))
+
+## [1.22.0-beta.5](https://github.com/ScrapeGraphAI/Scrapegraph-ai/compare/v1.22.0-beta.4...v1.22.0-beta.5) (2024-09-27)
 
 
 ### Features
@@ -51,6 +55,14 @@
 * **release:** 1.22.0-beta.1 [skip ci] ([f42a95f](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/f42a95faa05de39bd9cfc05e377d4b3da372e482))
 * **release:** 1.22.0-beta.2 [skip ci] ([431c09f](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/431c09f551ac28581674c6061f055fde0350ed4c))
 * **release:** 1.22.0-beta.3 [skip ci] ([e5ac020](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/e5ac0205d1e04a8b31e86166c3673915b70fd1e3))
+* add reasoning integration ([b2822f6](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/b2822f620a610e61d295cbf4b670aa08fde9de24))
+
+## [1.22.0-beta.4](https://github.com/ScrapeGraphAI/Scrapegraph-ai/compare/v1.22.0-beta.3...v1.22.0-beta.4) (2024-09-27)
+
+
+### Features
+
+* add html_mode to smart_scraper ([bdcffd6](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/bdcffd6360237b27797546a198ceece55ce4bc81))
 
 ## [1.22.0-beta.3](https://github.com/ScrapeGraphAI/Scrapegraph-ai/compare/v1.22.0-beta.2...v1.22.0-beta.3) (2024-09-25)
 

diff --git a/README.md b/README.md
@@ -38,10 +38,9 @@ Additional dependecies can be added while installing the library:
 
 - <b>More Language Models</b>: additional language models are installed, such as Fireworks, Groq, Anthropic, Hugging Face, and Nvidia AI Endpoints.
 
-
-This group allows you to use additional language models like Fireworks, Groq, Anthropic, Together AI, Hugging Face, and Nvidia AI Endpoints.
-```bash
-pip install scrapegraphai[other-language-models]
+  This group allows you to use additional language models like Fireworks, Groq, Anthropic, Together AI, Hugging Face, and Nvidia AI Endpoints.
+  ```bash
+  pip install scrapegraphai[other-language-models]
   ```
 - <b>Semantic Options</b>: this group includes tools for advanced semantic processing, such as Graphviz.
 
@@ -55,23 +54,9 @@ pip install scrapegraphai[other-language-models]
   pip install scrapegraphai[more-browser-options]
   ```
 
-- <b>faiss Options</b>: this group includes faiss integration
-
-  ```bash
-  pip install scrapegraphai[faiss-cpu]
-  ```
-
 </details>
 
 
-
-### Installing "More Browser Options"
-
-This group includes an ocr scraper for websites
-```bash
-pip install scrapegraphai[screenshot_scraper]
-```
-
 ## 💻 Usage
 There are multiple standard scraping pipelines that can be used to extract information from a website (or local file).
 

diff --git a/examples/anthropic/csv_scraper_haiku.py → examples/anthropic/csv_scraper_anthropic.py b/examples/anthropic/csv_scraper_haiku.py → examples/anthropic/csv_scraper_anthropic.py
diff --git a/...nthropic/csv_scraper_graph_multi_haiku.py → ...opic/csv_scraper_graph_multi_anthropic.py b/...nthropic/csv_scraper_graph_multi_haiku.py → ...opic/csv_scraper_graph_multi_anthropic.py
diff --git a/examples/anthropic/custom_graph_haiku.py → examples/anthropic/custom_graph_anthropic.py b/examples/anthropic/custom_graph_haiku.py → examples/anthropic/custom_graph_anthropic.py
diff --git a/examples/anthropic/depth_search_graph_anthropic.py b/examples/anthropic/depth_search_graph_anthropic.py
@@ -0,0 +1,28 @@
+"""
+depth_search_graph_opeani example
+"""
+import os
+from dotenv import load_dotenv
+from scrapegraphai.graphs import DepthSearchGraph
+
+load_dotenv()
+
+graph_config = {
+    "llm": {
+        "api_key": os.getenv("ANTHROPIC_API_KEY"),
+        "model": "openai/gpt-4o-mini",
+    },
+    "verbose": True,
+    "headless": False,
+    "depth": 2,
+    "only_inside_links": False,
+}
+
+search_graph = DepthSearchGraph(
+    prompt="List me all the projects with their description",
+    source="https://perinim.github.io",
+    config=graph_config
+)
+
+result = search_graph.run()
+print(result)
diff --git a/examples/anthropic/json_scraper_haiku.py → examples/anthropic/json_scraper_anthropic.py b/examples/anthropic/json_scraper_haiku.py → examples/anthropic/json_scraper_anthropic.py
diff --git a/...les/anthropic/json_scraper_multi_haiku.py → ...anthropic/json_scraper_multi_anthropic.py b/...les/anthropic/json_scraper_multi_haiku.py → ...anthropic/json_scraper_multi_anthropic.py
diff --git a/...ples/anthropic/pdf_scraper_graph_haiku.py → .../anthropic/pdf_scraper_graph_anthropic.py b/...ples/anthropic/pdf_scraper_graph_haiku.py → .../anthropic/pdf_scraper_graph_anthropic.py
diff --git a/...ples/anthropic/pdf_scraper_multi_haiku.py → .../anthropic/pdf_scraper_multi_anthropic.py b/...ples/anthropic/pdf_scraper_multi_haiku.py → .../anthropic/pdf_scraper_multi_anthropic.py
diff --git a/examples/anthropic/rate_limit_haiku.py → examples/anthropic/rate_limit_anthropic.py b/examples/anthropic/rate_limit_haiku.py → examples/anthropic/rate_limit_anthropic.py
diff --git a/...ples/anthropic/scrape_plain_text_haiku.py → .../anthropic/scrape_plain_text_anthropic.py b/...ples/anthropic/scrape_plain_text_haiku.py → .../anthropic/scrape_plain_text_anthropic.py
diff --git a/examples/anthropic/script_generator_haiku.py → ...s/anthropic/script_generator_anthropic.py b/examples/anthropic/script_generator_haiku.py → ...s/anthropic/script_generator_anthropic.py
diff --git a/...anthropic/script_multi_generator_haiku.py → ...ropic/script_multi_generator_anthropic.py b/...anthropic/script_multi_generator_haiku.py → ...ropic/script_multi_generator_anthropic.py
diff --git a/examples/anthropic/search_graph_haiku.py → examples/anthropic/search_graph_anthropic.py b/examples/anthropic/search_graph_haiku.py → examples/anthropic/search_graph_anthropic.py
diff --git a/...es/anthropic/search_graph_schema_haiku.py → ...nthropic/search_graph_schema_anthropic.py b/...es/anthropic/search_graph_schema_haiku.py → ...nthropic/search_graph_schema_anthropic.py
diff --git a/...ples/anthropic/search_link_graph_haiku.py → .../anthropic/search_link_graph_anthropic.py b/...ples/anthropic/search_link_graph_haiku.py → .../anthropic/search_link_graph_anthropic.py
diff --git a/examples/anthropic/smart_scraper_haiku.py → ...ples/anthropic/smart_scraper_anthropic.py b/examples/anthropic/smart_scraper_haiku.py → ...ples/anthropic/smart_scraper_anthropic.py
diff --git a/...es/anthropic/smart_scraper_multi_haiku.py → ...nthropic/smart_scraper_multi_anthropic.py b/...es/anthropic/smart_scraper_multi_haiku.py → ...nthropic/smart_scraper_multi_anthropic.py
diff --git a/...ropic/smart_scraper_multi_concat_haiku.py → ...c/smart_scraper_multi_concat_anthropic.py b/...ropic/smart_scraper_multi_concat_haiku.py → ...c/smart_scraper_multi_concat_anthropic.py
diff --git a/...s/anthropic/smart_scraper_schema_haiku.py → ...thropic/smart_scraper_schema_anthropic.py b/...s/anthropic/smart_scraper_schema_haiku.py → ...thropic/smart_scraper_schema_anthropic.py
diff --git a/examples/anthropic/xml_scraper_haiku.py → examples/anthropic/xml_scraper_anthropic.py b/examples/anthropic/xml_scraper_haiku.py → examples/anthropic/xml_scraper_anthropic.py
diff --git a/...nthropic/xml_scraper_graph_multi_haiku.py → ...opic/xml_scraper_graph_multi_anthropic.py b/...nthropic/xml_scraper_graph_multi_haiku.py → ...opic/xml_scraper_graph_multi_anthropic.py
diff --git a/examples/azure/code_generator_graph_azure.py b/examples/azure/code_generator_graph_azure.py
@@ -28,7 +28,7 @@ class Projects(BaseModel):
 graph_config = {
     "llm": {
         "api_key": os.environ["AZURE_OPENAI_KEY"],
-        "model": "azure_openai/gpt-3.5-turbo",
+        "model": "azure_openai/gpt-4o"
     },
     "verbose": True,
     "headless": False,

diff --git a/examples/azure/csv_scraper_azure.py b/examples/azure/csv_scraper_azure.py
@@ -25,7 +25,7 @@
 graph_config = {
     "llm": {
         "api_key": os.environ["AZURE_OPENAI_KEY"],
-        "model": "azure_openai/gpt-3.5-turbo",
+        "model": "azure_openai/gpt-4o"
     },
     "verbose": True,
     "headless": False

diff --git a/examples/azure/csv_scraper_graph_multi_azure.py b/examples/azure/csv_scraper_graph_multi_azure.py
@@ -25,7 +25,7 @@
 graph_config = {
     "llm": {
         "api_key": os.environ["AZURE_OPENAI_KEY"],
-        "model": "azure_openai/gpt-3.5-turbo",
+        "model": "azure_openai/gpt-4o"
     },
     "verbose": True,
     "headless": False

diff --git a/examples/azure/depth_search_graph_azure.py b/examples/azure/depth_search_graph_azure.py
@@ -0,0 +1,30 @@
+"""
+depth_search_graph_opeani example
+"""
+import os
+from dotenv import load_dotenv
+from scrapegraphai.graphs import DepthSearchGraph
+
+load_dotenv()
+
+openai_key = os.getenv("OPENAI_APIKEY")
+
+graph_config = {
+   "llm": {
+        "api_key": os.environ["AZURE_OPENAI_KEY"],
+        "model": "azure_openai/gpt-4o",
+    },
+    "verbose": True,
+    "headless": False,
+    "depth": 2,
+    "only_inside_links": False,
+}
+
+search_graph = DepthSearchGraph(
+    prompt="List me all the projects with their description",
+    source="https://perinim.github.io",
+    config=graph_config
+)
+
+result = search_graph.run()
+print(result)
diff --git a/examples/azure/json_scraper_azure.py b/examples/azure/json_scraper_azure.py
@@ -23,7 +23,7 @@
 graph_config = {
     "llm": {
         "api_key": os.environ["AZURE_OPENAI_KEY"],
-        "model": "azure_openai/gpt-3.5-turbo",
+        "model": "azure_openai/gpt-4o"
     },
     "verbose": True,
     "headless": False

diff --git a/examples/azure/json_scraper_multi_azure.py b/examples/azure/json_scraper_multi_azure.py
@@ -12,7 +12,7 @@
 graph_config = {
     "llm": {
         "api_key": os.environ["AZURE_OPENAI_KEY"],
-        "model": "azure_openai/gpt-3.5-turbo",
+        "model": "azure_openai/gpt-4o"
     },
     "verbose": True,
     "headless": False

diff --git a/examples/azure/pdf_scraper_azure.py b/examples/azure/pdf_scraper_azure.py
@@ -10,7 +10,7 @@
 graph_config = {
     "llm": {
         "api_key": os.environ["AZURE_OPENAI_KEY"],
-        "model": "azure_openai/gpt-3.5-turbo",
+        "model": "azure_openai/gpt-4o"
     },
     "verbose": True,
     "headless": False

diff --git a/examples/azure/rate_limit_azure.py b/examples/azure/rate_limit_azure.py
@@ -26,7 +26,7 @@
 graph_config = {
     "llm": {
         "api_key": os.environ["AZURE_OPENAI_KEY"],
-        "model": "azure_openai/gpt-3.5-turbo",
+        "model": "azure_openai/gpt-4o",
         "rate_limit": {
             "requests_per_second": 1
         },

diff --git a/examples/azure/scrape_plain_text_azure.py b/examples/azure/scrape_plain_text_azure.py
@@ -28,7 +28,7 @@
 graph_config = {
     "llm": {
         "api_key": os.environ["AZURE_OPENAI_KEY"],
-        "model": "azure_openai/gpt-3.5-turbo",
+        "model": "azure_openai/gpt-4o"
     },
     "verbose": True,
     "headless": False

diff --git a/examples/azure/script_generator_azure.py b/examples/azure/script_generator_azure.py
@@ -15,7 +15,7 @@
 graph_config = {
     "llm": {
         "api_key": os.environ["AZURE_OPENAI_KEY"],
-        "model": "azure_openai/gpt-3.5-turbo",
+        "model": "azure_openai/gpt-4o"
     },
     "verbose": True,
     "headless": False

diff --git a/examples/azure/script_multi_generator_azure.py b/examples/azure/script_multi_generator_azure.py
@@ -16,7 +16,7 @@
 graph_config = {
     "llm": {
         "api_key": os.environ["AZURE_OPENAI_KEY"],
-        "model": "azure_openai/gpt-3.5-turbo",
+        "model": "azure_openai/gpt-4o"
     },
     "verbose": True,
     "headless": False

diff --git a/examples/azure/search_graph_azure.py b/examples/azure/search_graph_azure.py
@@ -22,7 +22,7 @@
 graph_config = {
     "llm": {
         "api_key": os.environ["AZURE_OPENAI_KEY"],
-        "model": "azure_openai/gpt-3.5-turbo",
+        "model": "azure_openai/gpt-4o"
     },
     "verbose": True,
     "headless": False

diff --git a/examples/azure/search_graph_schema_azure.py b/examples/azure/search_graph_schema_azure.py
@@ -30,7 +30,7 @@ class Dishes(BaseModel):
 graph_config = {
     "llm": {
         "api_key": os.environ["AZURE_OPENAI_KEY"],
-        "model": "azure_openai/gpt-3.5-turbo",
+        "model": "azure_openai/gpt-4o"
     },
     "verbose": True,
     "headless": False

diff --git a/examples/azure/search_link_graph_azure.py b/examples/azure/search_link_graph_azure.py
@@ -15,7 +15,7 @@
 graph_config = {
     "llm": {
         "api_key": os.environ["AZURE_OPENAI_KEY"],
-        "model": "azure_openai/gpt-3.5-turbo",
+        "model": "azure_openai/gpt-4o"
     },
     "verbose": True,
     "headless": False

diff --git a/examples/azure/smart_scraper_azure.py b/examples/azure/smart_scraper_azure.py
@@ -26,7 +26,7 @@
 graph_config = {
     "llm": {
         "api_key": os.environ["AZURE_OPENAI_KEY"],
-        "model": "azure_openai/gpt-3.5-turbo",
+        "model": "azure_openai/gpt-4o"
     },
     "verbose": True,
     "headless": False

diff --git a/examples/azure/smart_scraper_multi_azure.py b/examples/azure/smart_scraper_multi_azure.py
@@ -14,7 +14,7 @@
 graph_config = {
     "llm": {
         "api_key": os.environ["AZURE_OPENAI_KEY"],
-        "model": "azure_openai/gpt-3.5-turbo",
+        "model": "azure_openai/gpt-4o"
     },
     "verbose": True,
     "headless": False

diff --git a/examples/azure/smart_scraper_multi_concat_azure.py b/examples/azure/smart_scraper_multi_concat_azure.py
@@ -15,7 +15,7 @@
 graph_config = {
     "llm": {
         "api_key": os.environ["AZURE_OPENAI_KEY"],
-        "model": "azure_openai/gpt-3.5-turbo",
+        "model": "azure_openai/gpt-4o"
     },
     "verbose": True,
     "headless": False

diff --git a/examples/azure/smart_scraper_schema_azure.py b/examples/azure/smart_scraper_schema_azure.py
@@ -29,7 +29,7 @@ class Projects(BaseModel):
 graph_config = {
     "llm": {
         "api_key": os.environ["AZURE_OPENAI_KEY"],
-        "model": "azure_openai/gpt-3.5-turbo",
+        "model": "azure_openai/gpt-4o"
     },
     "verbose": True,
     "headless": False

diff --git a/examples/azure/xml_scraper_azure.py b/examples/azure/xml_scraper_azure.py
@@ -24,7 +24,7 @@
 graph_config = {
     "llm": {
         "api_key": os.environ["AZURE_OPENAI_KEY"],
-        "model": "azure_openai/gpt-3.5-turbo",
+        "model": "azure_openai/gpt-4o"
     },
     "verbose": True,
     "headless": False

diff --git a/examples/azure/xml_scraper_graph_multi_azure.py b/examples/azure/xml_scraper_graph_multi_azure.py
@@ -25,7 +25,7 @@
 graph_config = {
     "llm": {
         "api_key": os.environ["AZURE_OPENAI_KEY"],
-        "model": "azure_openai/gpt-3.5-turbo",
+        "model": "azure_openai/gpt-4o",
     },
     "verbose": True,
     "headless": False

diff --git a/examples/bedrock/depth_search_graph_bedrock.py b/examples/bedrock/depth_search_graph_bedrock.py
@@ -0,0 +1,31 @@
+"""
+depth_search_graph_opeani example
+"""
+import os
+from dotenv import load_dotenv
+from scrapegraphai.graphs import DepthSearchGraph
+
+load_dotenv()
+
+openai_key = os.getenv("OPENAI_APIKEY")
+
+graph_config = {
+    "llm": {
+        "client": "client_name",
+        "model": "bedrock/anthropic.claude-3-sonnet-20240229-v1:0",
+        "temperature": 0.0
+    },
+    "verbose": True,
+    "headless": False,
+    "depth": 2,
+    "only_inside_links": False,
+}
+
+search_graph = DepthSearchGraph(
+    prompt="List me all the projects with their description",
+    source="https://perinim.github.io",
+    config=graph_config
+)
+
+result = search_graph.run()
+print(result)
diff --git a/examples/deepseek/depth_search_graph_deepseek.py b/examples/deepseek/depth_search_graph_deepseek.py
@@ -0,0 +1,30 @@
+"""
+depth_search_graph_opeani example
+"""
+import os
+from dotenv import load_dotenv
+from scrapegraphai.graphs import DepthSearchGraph
+
+load_dotenv()
+
+deepseek_key = os.getenv("DEEPSEEK_APIKEY")
+
+graph_config = {
+    "llm": {
+        "model": "deepseek/deepseek-chat",
+        "api_key": deepseek_key,
+    },
+    "verbose": True,
+    "headless": False,
+    "depth": 2,
+    "only_inside_links": False,
+}
+
+search_graph = DepthSearchGraph(
+    prompt="List me all the projects with their description",
+    source="https://perinim.github.io",
+    config=graph_config
+)
+
+result = search_graph.run()
+print(result)
diff --git a/examples/ernie/custom_graph_ernie.py b/examples/ernie/custom_graph_ernie.py
@@ -14,7 +14,7 @@
 # Define the configuration for the graph
 # ************************************************
 
-graph_config = {  
+graph_config = {
     "llm": {
         "model": "ernie/ernie-bot-turbo",
         "ernie_client_id": "<ernie_client_id>",

diff --git a/examples/ernie/depth_search_graph_ernie.py b/examples/ernie/depth_search_graph_ernie.py
@@ -0,0 +1,26 @@
+"""
+depth_search_graph_opeani example
+"""
+from scrapegraphai.graphs import DepthSearchGraph
+
+graph_config = {
+    "llm": {
+        "model": "ernie/ernie-bot-turbo",
+        "ernie_client_id": "<ernie_client_id>",
+        "ernie_client_secret": "<ernie_client_secret>",
+        "temperature": 0.1
+    },
+    "verbose": True,
+    "headless": False,
+    "depth": 2,
+    "only_inside_links": False,
+}
+
+search_graph = DepthSearchGraph(
+    prompt="List me all the projects with their description",
+    source="https://perinim.github.io",
+    config=graph_config
+)
+
+result = search_graph.run()
+print(result)