Merge branch 'pre/beta' into 479-using-nemotron-from-nvidia

Author: f-aguzzi

.github/update-requirements.yml

@@ -0,0 +1,26 @@
+name: Update requirements
+on:
+  push:
+    paths:
+      - 'pyproject.toml'
+      - '.github/workflows/update-requirements.yml'
+
+jobs:
+  update:
+    name: Update requirements
+    runs-on: ubuntu-latest
+    steps:
+      - name: Install the latest version of rye
+        uses: eifinger/setup-rye@v3
+      - name: Build app
+        run: rye run update-requirements
+  commit:
+    name: Commit changes
+    run: |
+      git config --global user.name 'github-actions'
+      git config --global user.email 'github-actions[bot]@users.noreply.github.com'
+      git add .
+      git commit -m "ci: update requirements.txt [skip ci]"
+      git push
+    env:
+      GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

CHANGELOG.md

@@ -1,3 +1,20 @@
+## [1.10.0-beta.6](https://github.com/ScrapeGraphAI/Scrapegraph-ai/compare/v1.10.0-beta.5...v1.10.0-beta.6) (2024-07-22)
+
+
+### Features
+
+* add new toml ([fcb3220](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/fcb3220868e7ef1127a7a47f40d0379be282e6eb))
+
+
+### chore
+
+* **pyproject:** upgrade dependencies ([0425124](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/0425124c570f765b98fcf67ba6649f4f9fe76b15))
+
+
+### Docs
+
+* **gpt-4o-mini:** added new gpt, fixed chromium lazy loading, ([99dc849](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/99dc8497d85289759286a973e4aecc3f924d3ada))
+
 ## [1.10.0-beta.5](https://github.com/ScrapeGraphAI/Scrapegraph-ai/compare/v1.10.0-beta.4...v1.10.0-beta.5) (2024-07-20)
 
 

manual deployment/autorequirements.py

@@ -0,0 +1,30 @@
+import toml
+
+# Load the TOML file
+data = toml.load('pyproject.toml')
+
+# Get the dependencies
+dependencies = data['project']['dependencies']
+
+# Write the dependencies to a requirements.txt file
+with open('requirements.txt', 'w') as f:
+    for dependency in dependencies:
+        f.write(dependency + '\n')
+
+# Get the dev dependencies
+dev_dependencies = data['tool']['rye']['dev-dependencies']
+
+# Expand the optional dependencies
+optional_dependencies = data['project']['optional-dependencies']
+expanded_dev_dependencies = []
+for dependency in dev_dependencies:
+    if dependency.startswith('-e file:.'):
+        optional_dependency_name = dependency.split('.')[1][1:-1]
+        expanded_dev_dependencies.extend(optional_dependencies[optional_dependency_name])
+    else:
+        expanded_dev_dependencies.append(dependency)
+
+# Write the expanded dev dependencies to a requirements-dev.txt file
+with open('requirements-dev.txt', 'w') as f:
+    for dependency in expanded_dev_dependencies:
+        f.write(dependency + '\n')

pyproject.toml

@@ -2,7 +2,7 @@
 name = "scrapegraphai"
 
 
-version = "1.10.0b5"
+version = "1.10.0b6"
 
 
 
@@ -13,30 +13,30 @@ authors = [
     { name = "Lorenzo Padoan", email = "[email protected]" }
 ]
 dependencies = [
-    "langchain", 
-    "langchain-google-genai",
+    "langchain>=0.2.10",
+    "langchain-google-genai>=1.0.7",
     "langchain-google-vertexai",
-    "langchain-openai",
-    "langchain-groq==0.1.3",
-    "langchain-aws==0.1.3",
-    "langchain-anthropic==0.1.11",
+    "langchain-openai>=0.1.17",
+    "langchain-groq>=0.1.3",
+    "langchain-aws>=0.1.3",
+    "langchain-anthropic>=0.1.11",
     "langchain-nvidia-ai-endpoints==0.1.6",
-    "html2text==2024.2.26",
-    "faiss-cpu==1.8.0",
-    "beautifulsoup4==4.12.3",
-    "pandas==2.2.2",
-    "python-dotenv==1.0.1",
-    "tiktoken==0.7",
-    "tqdm==4.66.4",
-    "graphviz==0.20.3",
-    "minify-html==0.15.0",
-    "free-proxy==1.1.1",
-    "playwright==1.43.0",
-    "google==3.0.0",
-    "undetected-playwright==0.3.0",
-    "semchunk==1.0.1",
-    "html2text==2024.2.26",
-    "langchain-fireworks==0.1.3"
+    "html2text>=2024.2.26",
+    "faiss-cpu>=1.8.0",
+    "beautifulsoup4>=4.12.3",
+    "pandas>=2.2.2",
+    "python-dotenv>=1.0.1",
+    "tiktoken>=0.7",
+    "tqdm>=4.66.4",
+    "graphviz>=0.20.3",
+    "minify-html>=0.15.0",
+    "free-proxy>=1.1.1",
+    "playwright>=1.43.0",
+    "google>=3.0.0",
+    "undetected-playwright>=0.3.0",
+    "semchunk>=1.0.1",
+    "html2text>=2024.2.26",
+    "langchain-fireworks>=0.1.3",
 ]
 
 license = "MIT"
@@ -91,4 +91,5 @@ dev-dependencies = [
 
 [tool.rye.scripts]
 pylint-local = "pylint scrapegraphai/**/*.py"
-pylint-ci = "pylint --disable=C0114,C0115,C0116 --exit-zero scrapegraphai/**/*.py"
+pylint-ci = "pylint --disable=C0114,C0115,C0116 --exit-zero scrapegraphai/**/*.py"
+update-requirements = "python 'manual deployment/autorequirements.py'"