Pre/beta

CONTRIBUTING.md

@@ -74,4 +74,10 @@ If you encounter any issues or have suggestions for improvements, please open an
 ScrapeGraphAI is licensed under the **MIT License**. See the [LICENSE](LICENSE) file for more information.
 By contributing to this project, you agree to license your contributions under the same license.
 
+ScrapeGraphAI uses code from the Langchain
+frameworks. You find their original licenses below.
+
+LANGCHAIN LICENSE
+https://github.com/langchain-ai/langchain/blob/master/LICENSE
+
 Can't wait to see your contributions! :smile:

README.md

@@ -38,9 +38,10 @@ Additional dependecies can be added while installing the library:
 
 - <b>More Language Models</b>: additional language models are installed, such as Fireworks, Groq, Anthropic, Hugging Face, and Nvidia AI Endpoints.
 
-  ```bash
-  pip install scrapegraphai[other-language-models]
-  ```
+
+This group allows you to use additional language models like Fireworks, Groq, Anthropic, Together AI, Hugging Face, and Nvidia AI Endpoints.
+```bash
+pip install scrapegraphai[other-language-models]
 
 - <b>Semantic Options</b>: this group includes tools for advanced semantic processing, such as Graphviz.
 
@@ -58,6 +59,13 @@ Additional dependecies can be added while installing the library:
 
 
 
+### Installing "More Browser Options"
+
+This group includes an ocr scraper for websites
+```bash
+pip install scrapegraphai[screenshot_scraper]
+```
+
 ## 💻 Usage
 There are multiple standard scraping pipelines that can be used to extract information from a website (or local file).
 

docs/chinese.md

@@ -133,7 +133,7 @@ from scrapegraphai.graphs import SpeechGraph
 graph_config = {
     "llm": {
         "api_key": "OPENAI_API_KEY",
-        "model": "gpt-3.5-turbo",
+        "model": "openai/gpt-3.5-turbo",
     },
     "tts_model": {
         "api_key": "OPENAI_API_KEY",

docs/japanese.md

@@ -133,7 +133,7 @@ from scrapegraphai.graphs import SpeechGraph
 graph_config = {
     "llm": {
         "api_key": "OPENAI_API_KEY",
-        "model": "gpt-3.5-turbo",
+        "model": "openai/gpt-3.5-turbo",
     },
     "tts_model": {
         "api_key": "OPENAI_API_KEY",

docs/korean.md

@@ -132,7 +132,7 @@ from scrapegraphai.graphs import SpeechGraph
 graph_config = {
     "llm": {
         "api_key": "OPENAI_API_KEY",
-        "model": "gpt-3.5-turbo",
+        "model": "openai/gpt-3.5-turbo",
     },
     "tts_model": {
         "api_key": "OPENAI_API_KEY",

docs/russian.md

@@ -138,7 +138,7 @@ from scrapegraphai.graphs import SpeechGraph
 graph_config = {
     "llm": {
         "api_key": "OPENAI_API_KEY",
-        "model": "gpt-3.5-turbo",
+        "model": "openai/gpt-3.5-turbo",
     },
     "tts_model": {
         "api_key": "OPENAI_API_KEY",

docs/source/getting_started/examples.rst

@@ -22,7 +22,7 @@ OpenAI models
    graph_config = {
       "llm": {
          "api_key": openai_key,
-         "model": "gpt-3.5-turbo",
+         "model": "openai/gpt-3.5-turbo",
       },
    }
 

examples/anthropic/custom_graph_haiku.py

@@ -40,7 +40,7 @@
 
 fetch_node = FetchNode(
     input="url | local_dir",
-    output=["doc", "link_urls", "img_urls"],
+    output=["doc"],
     node_config={
         "verbose": True,
         "headless": True,

examples/anthropic/rate_limit_haiku.py

@@ -0,0 +1,48 @@
+""" 
+Basic example of scraping pipeline using SmartScraper while setting an API rate limit.
+"""
+
+import os
+from dotenv import load_dotenv
+from scrapegraphai.graphs import SmartScraperGraph
+from scrapegraphai.utils import prettify_exec_info
+
+
+# required environment variables in .env
+# ANTHROPIC_API_KEY
+load_dotenv()
+
+# ************************************************
+# Create the SmartScraperGraph instance and run it
+# ************************************************
+
+graph_config = {
+    "llm": {
+        "api_key": os.getenv("ANTHROPIC_API_KEY"),
+        "model": "anthropic/claude-3-haiku-20240307",
+        "rate_limit": {
+            "requests_per_second": 1
+        }
+    },
+}
+
+smart_scraper_graph = SmartScraperGraph(
+    prompt="""Don't say anything else. Output JSON only. List me all the events, with the following fields: company_name, event_name, event_start_date, event_start_time, 
+    event_end_date, event_end_time, location, event_mode, event_category, 
+    third_party_redirect, no_of_days, 
+    time_in_hours, hosted_or_attending, refreshments_type, 
+    registration_available, registration_link""",
+    # also accepts a string with the already downloaded HTML code
+    source="https://www.hmhco.com/event",
+    config=graph_config
+)
+
+result = smart_scraper_graph.run()
+print(result)
+
+# ************************************************
+# Get graph execution info
+# ************************************************
+
+graph_exec_info = smart_scraper_graph.get_execution_info()
+print(prettify_exec_info(graph_exec_info))

examples/anthropic/search_graph_schema_haiku.py

@@ -5,7 +5,7 @@
 import os
 from typing import List
 from dotenv import load_dotenv
-from langchain_core.pydantic_v1 import BaseModel, Field
+from pydantic import BaseModel, Field
 from scrapegraphai.graphs import SearchGraph
 
 load_dotenv()

examples/anthropic/smart_scraper_multi_concat_haiku.py

@@ -0,0 +1,39 @@
+""" 
+Basic example of scraping pipeline using SmartScraper
+"""
+
+import os
+import json
+from dotenv import load_dotenv
+from scrapegraphai.graphs import SmartScraperMultiConcatGraph
+
+load_dotenv()
+
+# ************************************************
+# Create the SmartScraperGraph instance and run it
+# ************************************************
+
+graph_config = {
+    "llm": {
+        "api_key": os.getenv("ANTHROPIC_API_KEY"),
+        "model": "anthropic/claude-3-haiku-20240307",
+    },
+}
+
+
+# *******************************************************
+# Create the SmartScraperMultiGraph instance and run it
+# *******************************************************
+
+multiple_search_graph = SmartScraperMultiConcatGraph(
+    prompt="Who is Marco Perini?",
+    source= [
+        "https://perinim.github.io/",
+        "https://perinim.github.io/cv/"
+        ],
+    schema=None,
+    config=graph_config
+)
+
+result = multiple_search_graph.run()
+print(json.dumps(result, indent=4))

examples/anthropic/smart_scraper_schema_haiku.py

@@ -4,7 +4,7 @@
 
 import os
 from typing import List
-from langchain_core.pydantic_v1 import BaseModel, Field
+from pydantic import BaseModel, Field
 from dotenv import load_dotenv
 from scrapegraphai.graphs import SmartScraperGraph
 from scrapegraphai.utils import prettify_exec_info

examples/azure/rate_limit_azure.py

@@ -0,0 +1,57 @@
+""" 
+Basic example of scraping pipeline using SmartScraper with a custom rate limit
+"""
+
+import os
+from dotenv import load_dotenv
+from scrapegraphai.graphs import SmartScraperGraph
+from scrapegraphai.utils import prettify_exec_info
+
+
+# required environment variable in .env
+# AZURE_OPENAI_ENDPOINT
+# AZURE_OPENAI_CHAT_DEPLOYMENT_NAME
+# MODEL_NAME
+# AZURE_OPENAI_API_KEY
+# OPENAI_API_TYPE
+# AZURE_OPENAI_API_VERSION
+# AZURE_OPENAI_EMBEDDINGS_DEPLOYMENT_NAME
+load_dotenv()
+
+
+# ************************************************
+# Initialize the model instances
+# ************************************************
+
+graph_config = {
+    "llm": {
+        "api_key": os.environ["AZURE_OPENAI_KEY"],
+        "model": "azure_openai/gpt-3.5-turbo",
+        "rate_limit": {
+            "requests_per_second": 1
+        },
+    },
+    "verbose": True,
+    "headless": False
+}
+
+smart_scraper_graph = SmartScraperGraph(
+    prompt="""List me all the events, with the following fields: company_name, event_name, event_start_date, event_start_time, 
+    event_end_date, event_end_time, location, event_mode, event_category, 
+    third_party_redirect, no_of_days, 
+    time_in_hours, hosted_or_attending, refreshments_type, 
+    registration_available, registration_link""",
+    # also accepts a string with the already downloaded HTML code
+    source="https://www.hmhco.com/event",
+    config=graph_config
+)
+
+result = smart_scraper_graph.run()
+print(result)
+
+# ************************************************
+# Get graph execution info
+# ************************************************
+
+graph_exec_info = smart_scraper_graph.get_execution_info()
+print(prettify_exec_info(graph_exec_info))

examples/azure/search_graph_schema_azure.py

@@ -9,7 +9,7 @@
 from scrapegraphai.graphs import SearchGraph
 from scrapegraphai.utils import convert_to_csv, convert_to_json, prettify_exec_info
 
-from langchain_core.pydantic_v1 import BaseModel, Field
+from pydantic import BaseModel, Field
 from typing import List
 
 # ************************************************

examples/azure/smart_scraper_multi_azure.py

@@ -1,8 +1,8 @@
 """ 
 Basic example of scraping pipeline using SmartScraper
 """
-
-import os, json
+import os
+import json
 from dotenv import load_dotenv
 from scrapegraphai.graphs import SmartScraperMultiGraph
 

examples/azure/smart_scraper_multi_concat_azure.py

@@ -0,0 +1,39 @@
+""" 
+Basic example of scraping pipeline using SmartScraper
+"""
+
+import os
+import json
+from dotenv import load_dotenv
+from scrapegraphai.graphs import SmartScraperMultiConcatGraph
+
+load_dotenv()
+
+# ************************************************
+# Define the configuration for the graph
+# ************************************************
+graph_config = {
+    "llm": {
+        "api_key": os.environ["AZURE_OPENAI_KEY"],
+        "model": "azure_openai/gpt-3.5-turbo",
+    },
+    "verbose": True,
+    "headless": False
+}
+
+# *******************************************************
+# Create the SmartScraperMultiGraph instance and run it
+# *******************************************************
+
+multiple_search_graph = SmartScraperMultiConcatGraph(
+    prompt="Who is Marco Perini?",
+    source= [
+        "https://perinim.github.io/",
+        "https://perinim.github.io/cv/"
+        ],
+    schema=None,
+    config=graph_config
+)
+
+result = multiple_search_graph.run()
+print(json.dumps(result, indent=4))

examples/azure/smart_scraper_schema_azure.py

@@ -5,7 +5,7 @@
 import os
 import json
 from typing import List
-from langchain_core.pydantic_v1 import BaseModel, Field
+from pydantic import BaseModel, Field
 from dotenv import load_dotenv
 from scrapegraphai.graphs import SmartScraperGraph
 

examples/bedrock/custom_graph_bedrock.py

@@ -55,7 +55,7 @@
 
 fetch_node = FetchNode(
     input="url | local_dir",
-    output=["doc", "link_urls", "img_urls"],
+    output=["doc"],
     node_config={
         "verbose": True,
         "headless": True,

examples/bedrock/rate_limit_bedrock.py

@@ -0,0 +1,47 @@
+""" 
+Basic example of scraping pipeline using SmartScraper with a custom rate limit
+"""
+
+import os
+from dotenv import load_dotenv
+from scrapegraphai.graphs import SmartScraperGraph
+from scrapegraphai.utils import prettify_exec_info
+
+load_dotenv()
+
+
+# ************************************************
+# Define the configuration for the graph
+# ************************************************
+
+graph_config = {
+    "llm": {
+        "client": "client_name",
+        "model": "bedrock/anthropic.claude-3-sonnet-20240229-v1:0",
+        "temperature": 0.0,
+        "rate_limit": {
+            "requests_per_second": 1
+        },
+    }
+}
+
+# ************************************************
+# Create the SmartScraperGraph instance and run it
+# ************************************************
+
+smart_scraper_graph = SmartScraperGraph(
+    prompt="List me all the projects with their description",
+    # also accepts a string with the already downloaded HTML code
+    source="https://perinim.github.io/projects/",
+    config=graph_config
+)
+
+result = smart_scraper_graph.run()
+print(result)
+
+# ************************************************
+# Get graph execution info
+# ************************************************
+
+graph_exec_info = smart_scraper_graph.get_execution_info()
+print(prettify_exec_info(graph_exec_info))

examples/bedrock/search_graph_schema_bedrock.py

@@ -4,7 +4,7 @@
 from scrapegraphai.graphs import SearchGraph
 from scrapegraphai.utils import convert_to_csv, convert_to_json, prettify_exec_info
 
-from langchain_core.pydantic_v1 import BaseModel, Field
+from pydantic import BaseModel, Field
 from typing import List
 
 # ************************************************