Skip to content

Commit c5aebd7

Browse files
VinciGit00VinciGit00
authored
Merge pull request #541 from ScrapeGraphAI/496-not-match-gemini-embeddings-model
2 parents bfe1f3b + 1aa9c6e commit c5aebd7

File tree

67 files changed

+1475
-39
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

67 files changed

+1475
-39
lines changed

‎examples/azure/csv_scraper_azure.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@
2525
graph_config = {
2626
"llm": {
2727
"api_key": os.environ["AZURE_OPENAI_KEY"],
28-
"model": "azure/gpt-3.5-turbo",
28+
"model": "azure_openai/gpt-3.5-turbo",
2929
},
3030
"verbose": True,
3131
"headless": False

‎examples/azure/csv_scraper_graph_multi_azure.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@
2525
graph_config = {
2626
"llm": {
2727
"api_key": os.environ["AZURE_OPENAI_KEY"],
28-
"model": "azure/gpt-3.5-turbo",
28+
"model": "azure_openai/gpt-3.5-turbo",
2929
},
3030
"verbose": True,
3131
"headless": False

‎examples/azure/json_scraper_azure.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@
2323
graph_config = {
2424
"llm": {
2525
"api_key": os.environ["AZURE_OPENAI_KEY"],
26-
"model": "azure/gpt-3.5-turbo",
26+
"model": "azure_openai/gpt-3.5-turbo",
2727
},
2828
"verbose": True,
2929
"headless": False

‎examples/azure/json_scraper_multi_azure.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@
1212
graph_config = {
1313
"llm": {
1414
"api_key": os.environ["AZURE_OPENAI_KEY"],
15-
"model": "azure/gpt-3.5-turbo",
15+
"model": "azure_openai/gpt-3.5-turbo",
1616
},
1717
"verbose": True,
1818
"headless": False

‎examples/azure/pdf_scraper_azure.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@
1010
graph_config = {
1111
"llm": {
1212
"api_key": os.environ["AZURE_OPENAI_KEY"],
13-
"model": "azure/gpt-3.5-turbo",
13+
"model": "azure_openai/gpt-3.5-turbo",
1414
},
1515
"verbose": True,
1616
"headless": False

‎examples/azure/scrape_plain_text_azure.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@
2828
graph_config = {
2929
"llm": {
3030
"api_key": os.environ["AZURE_OPENAI_KEY"],
31-
"model": "azure/gpt-3.5-turbo",
31+
"model": "azure_openai/gpt-3.5-turbo",
3232
},
3333
"verbose": True,
3434
"headless": False

‎examples/azure/script_generator_azure.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@
1515
graph_config = {
1616
"llm": {
1717
"api_key": os.environ["AZURE_OPENAI_KEY"],
18-
"model": "azure/gpt-3.5-turbo",
18+
"model": "azure_openai/gpt-3.5-turbo",
1919
},
2020
"verbose": True,
2121
"headless": False

‎examples/azure/script_multi_generator_azure.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@
1616
graph_config = {
1717
"llm": {
1818
"api_key": os.environ["AZURE_OPENAI_KEY"],
19-
"model": "azure/gpt-3.5-turbo",
19+
"model": "azure_openai/gpt-3.5-turbo",
2020
},
2121
"verbose": True,
2222
"headless": False

‎examples/azure/search_graph_azure.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@
2222
graph_config = {
2323
"llm": {
2424
"api_key": os.environ["AZURE_OPENAI_KEY"],
25-
"model": "azure/gpt-3.5-turbo",
25+
"model": "azure_openai/gpt-3.5-turbo",
2626
},
2727
"verbose": True,
2828
"headless": False

‎examples/azure/search_graph_schema_azure.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ class Dishes(BaseModel):
3030
graph_config = {
3131
"llm": {
3232
"api_key": os.environ["AZURE_OPENAI_KEY"],
33-
"model": "azure/gpt-3.5-turbo",
33+
"model": "azure_openai/gpt-3.5-turbo",
3434
},
3535
"verbose": True,
3636
"headless": False

‎examples/azure/search_link_graph_azure.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@
1515
graph_config = {
1616
"llm": {
1717
"api_key": os.environ["AZURE_OPENAI_KEY"],
18-
"model": "azure/gpt-3.5-turbo",
18+
"model": "azure_openai/gpt-3.5-turbo",
1919
},
2020
"verbose": True,
2121
"headless": False

‎examples/azure/smart_scraper_azure.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@
2626
graph_config = {
2727
"llm": {
2828
"api_key": os.environ["AZURE_OPENAI_KEY"],
29-
"model": "azure/gpt-3.5-turbo",
29+
"model": "azure_openai/gpt-3.5-turbo",
3030
},
3131
"verbose": True,
3232
"headless": False

‎examples/azure/smart_scraper_multi_azure.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@
1414
graph_config = {
1515
"llm": {
1616
"api_key": os.environ["AZURE_OPENAI_KEY"],
17-
"model": "azure/gpt-3.5-turbo",
17+
"model": "azure_openai/gpt-3.5-turbo",
1818
},
1919
"verbose": True,
2020
"headless": False

‎examples/azure/smart_scraper_schema_azure.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ class Projects(BaseModel):
2828
graph_config = {
2929
"llm": {
3030
"api_key": os.environ["AZURE_OPENAI_KEY"],
31-
"model": "azure/gpt-3.5-turbo",
31+
"model": "azure_openai/gpt-3.5-turbo",
3232
},
3333
"verbose": True,
3434
"headless": False

‎examples/azure/xml_scraper_azure.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@
2424
graph_config = {
2525
"llm": {
2626
"api_key": os.environ["AZURE_OPENAI_KEY"],
27-
"model": "azure/gpt-3.5-turbo",
27+
"model": "azure_openai/gpt-3.5-turbo",
2828
},
2929
"verbose": True,
3030
"headless": False

‎examples/azure/xml_scraper_graph_multi_azure.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@
2525
graph_config = {
2626
"llm": {
2727
"api_key": os.environ["AZURE_OPENAI_KEY"],
28-
"model": "azure/gpt-3.5-turbo",
28+
"model": "azure_openai/gpt-3.5-turbo",
2929
},
3030
"verbose": True,
3131
"headless": False

‎examples/gemini/.env.example renamed to ‎examples/google_genai/.env.example

File renamed without changes.

‎examples/gemini/csv_scraper_gemini.py renamed to ‎examples/google_genai/csv_scraper_gemini.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@
2424
graph_config = {
2525
"llm": {
2626
"api_key": gemini_key,
27-
"model": "gemini-pro",
27+
"model": "google_genai/gemini-pro",
2828
},
2929
}
3030

‎examples/gemini/csv_scraper_graph_multi_gemini.py renamed to ‎examples/google_genai/csv_scraper_graph_multi_gemini.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@
2828
graph_config = {
2929
"llm": {
3030
"api_key": gemini_key,
31-
"model": "gemini-pro",
31+
"model": "google_genai/gemini-pro",
3232
},
3333
}
3434

‎examples/gemini/custom_graph_gemini.py renamed to ‎examples/google_genai/custom_graph_gemini.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@
1818
graph_config = {
1919
"llm": {
2020
"api_key": gemini_key,
21-
"model": "gemini-pro",
21+
"model": "google_genai/gemini-pro",
2222
"temperature": 0,
2323
"streaming": True
2424
},

‎examples/gemini/inputs/books.xml renamed to ‎examples/google_genai/inputs/books.xml

File renamed without changes.

‎examples/gemini/inputs/example.json renamed to ‎examples/google_genai/inputs/example.json

File renamed without changes.

‎examples/gemini/inputs/plain_html_example.txt renamed to ‎examples/google_genai/inputs/plain_html_example.txt

File renamed without changes.

‎examples/gemini/inputs/username.csv renamed to ‎examples/google_genai/inputs/username.csv

File renamed without changes.

‎examples/gemini/json_scraper_gemini.py renamed to ‎examples/google_genai/json_scraper_gemini.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@
2828
graph_config = {
2929
"llm": {
3030
"api_key": gemini_key,
31-
"model": "gemini-pro",
31+
"model": "google_genai/gemini-pro",
3232
},
3333
}
3434

‎examples/gemini/json_scraper_multi_gemini.py renamed to ‎examples/google_genai/json_scraper_multi_gemini.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@
1313
graph_config = {
1414
"llm": {
1515
"api_key": gemini_key,
16-
"model": "gemini-pro",
16+
"model": "google_genai/gemini-pro",
1717
},
1818
"library": "beautifulsoup"
1919
}

‎examples/gemini/pdf_scraper_graph_gemini.py renamed to ‎examples/google_genai/pdf_scraper_graph_gemini.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@
1919
graph_config = {
2020
"llm": {
2121
"api_key": gemini_key,
22-
"model": "gemini-pro",
22+
"model": "google_genai/gemini-pro",
2323
},
2424
}
2525

‎examples/gemini/pdf_scraper_multi_gemini.py renamed to ‎examples/google_genai/pdf_scraper_multi_gemini.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@
1313
graph_config = {
1414
"llm": {
1515
"api_key": gemini_key,
16-
"model": "gemini-pro",
16+
"model": "google_genai/gemini-pro",
1717
},
1818
"library": "beautifulsoup"
1919
}

‎examples/gemini/readme.md renamed to ‎examples/google_genai/readme.md

‎examples/gemini/scrape_plain_text_gemini.py renamed to ‎examples/google_genai/scrape_plain_text_gemini.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@
2929
graph_config = {
3030
"llm": {
3131
"api_key": gemini_key,
32-
"model": "gemini-pro",
32+
"model": "google_genai/gemini-pro",
3333
"temperature": 0,
3434
"streaming": True
3535
},

‎examples/gemini/scrape_xml_gemini.py renamed to ‎examples/google_genai/scrape_xml_gemini.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@
2929
graph_config = {
3030
"llm": {
3131
"api_key": gemini_key,
32-
"model": "gemini-pro",
32+
"model": "google_genai/gemini-pro",
3333
"temperature": 0,
3434
"streaming": True
3535
},

‎examples/gemini/script_generator_gemini.py renamed to ‎examples/google_genai/script_generator_gemini.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@
1919
graph_config = {
2020
"llm": {
2121
"api_key": gemini_key,
22-
"model": "gemini-pro",
22+
"model": "google_genai/gemini-pro",
2323
},
2424
"library": "beautifoulsoup"
2525
}

‎examples/gemini/script_multi_generator_gemini.py renamed to ‎examples/google_genai/script_multi_generator_gemini.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@
1818
graph_config = {
1919
"llm": {
2020
"api_key": gemini_key,
21-
"model": "gemini-pro",
21+
"model": "google_genai/gemini-pro",
2222
},
2323
"library": "beautifoulsoup"
2424
}

‎examples/gemini/search_graph_gemini.py renamed to ‎examples/google_genai/search_graph_gemini.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@
1717
graph_config = {
1818
"llm": {
1919
"api_key": gemini_key,
20-
"model": "gemini-pro",
20+
"model": "google_genai/gemini-pro",
2121
"temperature": 0,
2222
"streaming": True
2323
},

‎examples/gemini/search_graph_schema_gemini.py renamed to ‎examples/google_genai/search_graph_schema_gemini.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@ class Dishes(BaseModel):
3232
graph_config = {
3333
"llm": {
3434
"api_key": gemini_key,
35-
"model": "gemini-pro",
35+
"model": "google_genai/gemini-pro",
3636
},
3737
}
3838

‎examples/gemini/search_link_graph_gemini.py renamed to ‎examples/google_genai/search_link_graph_gemini.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@
1717
graph_config = {
1818
"llm": {
1919
"api_key": gemini_key,
20-
"model": "gemini-pro",
20+
"model": "google_genai/gemini-pro",
2121
},
2222
}
2323
# ************************************************

‎examples/gemini/smart_scraper_gemini.py renamed to ‎examples/google_genai/smart_scraper_gemini.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@
1818
graph_config = {
1919
"llm": {
2020
"api_key": gemini_key,
21-
"model": "gemini-pro",
21+
"model": "google_genai/gemini-pro",
2222
},
2323
}
2424

‎examples/gemini/smart_scraper_multi_gemini.py renamed to ‎examples/google_genai/smart_scraper_multi_gemini.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@
1717
graph_config = {
1818
"llm": {
1919
"api_key": gemini_key,
20-
"model": "gemini-pro",
20+
"model": "google_genai/gemini-pro",
2121
},
2222
}
2323

‎examples/gemini/smart_scraper_schema_gemini.py renamed to ‎examples/google_genai/smart_scraper_schema_gemini.py

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ class Projects(BaseModel):
2929
graph_config = {
3030
"llm": {
3131
"api_key": gemini_key,
32-
"model": "gemini-pro",
32+
"model": "google_genai/gemini-pro",
3333
},
3434
}
3535

@@ -54,4 +54,3 @@ class Projects(BaseModel):
5454

5555
graph_exec_info = smart_scraper_graph.get_execution_info()
5656
print(prettify_exec_info(graph_exec_info))
57-
```

‎examples/gemini/xml_scraper_gemini.py renamed to ‎examples/google_genai/xml_scraper_gemini.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@
2828
graph_config = {
2929
"llm": {
3030
"api_key": gemini_key,
31-
"model": "gemini-pro",
31+
"model": "google_genai/gemini-pro",
3232
},
3333
}
3434
# ************************************************

‎examples/gemini/xml_scraper_graph_multi_gemini.py renamed to ‎examples/google_genai/xml_scraper_graph_multi_gemini.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@
2828
graph_config = {
2929
"llm": {
3030
"api_key": gemini_key,
31-
"model": "gemini-pro",
31+
"model": "google_genai/gemini-pro",
3232
},
3333
}
3434

‎examples/google_vertexai/.env.example

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
GOOGLE_APIKEY="your google api key"

‎examples/google_vertexai/csv_scraper_gemini.py

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
"""
2+
Basic example of scraping pipeline using CSVScraperGraph from CSV documents
3+
"""
4+
5+
import os
6+
from dotenv import load_dotenv
7+
import pandas as pd
8+
from scrapegraphai.graphs import CSVScraperGraph
9+
from scrapegraphai.utils import convert_to_csv, convert_to_json, prettify_exec_info
10+
11+
load_dotenv()
12+
13+
# ************************************************
14+
# Read the csv file
15+
# ************************************************
16+
17+
text = pd.read_csv("inputs/username.csv")
18+
19+
# ************************************************
20+
# Define the configuration for the graph
21+
# ************************************************
22+
gemini_key = os.getenv("GOOGLE_APIKEY")
23+
24+
graph_config = {
25+
"llm": {
26+
"api_key": gemini_key,
27+
"model": "google_vertexai/gemini-1.5-pro",
28+
},
29+
}
30+
31+
# ************************************************
32+
# Create the CSVScraperGraph instance and run it
33+
# ************************************************
34+
35+
csv_scraper_graph = CSVScraperGraph(
36+
prompt="List me all the last names",
37+
source=str(text), # Pass the content of the file, not the file object
38+
config=graph_config
39+
)
40+
41+
result = csv_scraper_graph.run()
42+
print(result)
43+
44+
# ************************************************
45+
# Get graph execution info
46+
# ************************************************
47+
48+
graph_exec_info = csv_scraper_graph.get_execution_info()
49+
print(prettify_exec_info(graph_exec_info))
50+
51+
# Save to json or csv
52+
convert_to_csv(result, "result")
53+
convert_to_json(result, "result")

0 commit comments

Comments
 (0)