chore: upgrade dependencies and scripts

Author: f-aguzzi

manual deployment/autorequirements.py

@@ -0,0 +1,30 @@
+import toml
+
+# Load the TOML file
+data = toml.load('pyproject.toml')
+
+# Get the dependencies
+dependencies = data['project']['dependencies']
+
+# Write the dependencies to a requirements.txt file
+with open('requirements.txt', 'w') as f:
+    for dependency in dependencies:
+        f.write(dependency + '\n')
+
+# Get the dev dependencies
+dev_dependencies = data['tool']['rye']['dev-dependencies']
+
+# Expand the optional dependencies
+optional_dependencies = data['project']['optional-dependencies']
+expanded_dev_dependencies = []
+for dependency in dev_dependencies:
+    if dependency.startswith('-e file:.'):
+        optional_dependency_name = dependency.split('.')[1][1:-1]
+        expanded_dev_dependencies.extend(optional_dependencies[optional_dependency_name])
+    else:
+        expanded_dev_dependencies.append(dependency)
+
+# Write the expanded dev dependencies to a requirements-dev.txt file
+with open('requirements-dev.txt', 'w') as f:
+    for dependency in expanded_dev_dependencies:
+        f.write(dependency + '\n')

pyproject.toml

@@ -13,29 +13,29 @@ authors = [
     { name = "Lorenzo Padoan", email = "[email protected]" }
 ]
 dependencies = [
-    "langchain", 
-    "langchain-google-genai",
+    "langchain>=0.2.10",
+    "langchain-google-genai>=1.0.7",
     "langchain-google-vertexai",
-    "langchain-openai",
-    "langchain-groq==0.1.3",
-    "langchain-aws==0.1.3",
-    "langchain-anthropic==0.1.11",
-    "html2text==2024.2.26",
-    "faiss-cpu==1.8.0",
-    "beautifulsoup4==4.12.3",
-    "pandas==2.2.2",
-    "python-dotenv==1.0.1",
-    "tiktoken==0.7",
-    "tqdm==4.66.4",
-    "graphviz==0.20.3",
-    "minify-html==0.15.0",
-    "free-proxy==1.1.1",
-    "playwright==1.43.0",
-    "google==3.0.0",
-    "undetected-playwright==0.3.0",
-    "semchunk==1.0.1",
-    "html2text==2024.2.26",
-    "langchain-fireworks==0.1.3"
+    "langchain-openai>=0.1.17",
+    "langchain-groq>=0.1.3",
+    "langchain-aws>=0.1.3",
+    "langchain-anthropic>=0.1.11",
+    "html2text>=2024.2.26",
+    "faiss-cpu>=1.8.0",
+    "beautifulsoup4>=4.12.3",
+    "pandas>=2.2.2",
+    "python-dotenv>=1.0.1",
+    "tiktoken>=0.7",
+    "tqdm>=4.66.4",
+    "graphviz>=0.20.3",
+    "minify-html>=0.15.0",
+    "free-proxy>=1.1.1",
+    "playwright>=1.43.0",
+    "google>=3.0.0",
+    "undetected-playwright>=0.3.0",
+    "semchunk>=1.0.1",
+    "html2text>=2024.2.26",
+    "langchain-fireworks>=0.1.3",
 ]
 
 license = "MIT"
@@ -91,3 +91,4 @@ dev-dependencies = [
 [tool.rye.scripts]
 pylint-local = "pylint scrapegraphai/**/*.py"
 pylint-ci = "pylint --disable=C0114,C0115,C0116 --exit-zero scrapegraphai/**/*.py"
+update-requirements = "python 'manual deployment/autorequirements.py'"

requirements-dev.lock

@@ -12,7 +12,6 @@ aiofiles==23.2.1
     # via burr
 aiohttp==3.9.5
     # via langchain
-    # via langchain-community
     # via langchain-fireworks
 aiosignal==1.3.1
     # via aiohttp
@@ -22,7 +21,7 @@ altair==5.3.0
     # via streamlit
 annotated-types==0.7.0
     # via pydantic
-anthropic==0.26.1
+anthropic==0.31.2
     # via langchain-anthropic
 anyio==4.3.0
     # via anthropic
@@ -48,9 +47,9 @@ beautifulsoup4==4.12.3
     # via scrapegraphai
 blinker==1.8.2
     # via streamlit
-boto3==1.34.113
+boto3==1.34.145
     # via langchain-aws
-botocore==1.34.113
+botocore==1.34.145
     # via boto3
     # via s3transfer
 burr==0.22.1
@@ -73,9 +72,6 @@ contourpy==1.2.1
     # via matplotlib
 cycler==0.12.1
     # via matplotlib
-dataclasses-json==0.6.6
-    # via langchain
-    # via langchain-community
 defusedxml==0.7.1
     # via langchain-anthropic
 dill==0.3.8
@@ -125,7 +121,7 @@ gitpython==3.1.43
     # via streamlit
 google==3.0.0
     # via scrapegraphai
-google-ai-generativelanguage==0.6.4
+google-ai-generativelanguage==0.6.6
     # via google-generativeai
 google-api-core==2.19.0
     # via google-ai-generativelanguage
@@ -166,7 +162,7 @@ google-cloud-storage==2.17.0
 google-crc32c==1.5.0
     # via google-cloud-storage
     # via google-resumable-media
-google-generativeai==0.5.4
+google-generativeai==0.7.2
     # via langchain-google-genai
 google-resumable-media==2.7.1
     # via google-cloud-bigquery
@@ -180,6 +176,7 @@ graphviz==0.20.3
     # via scrapegraphai
 greenlet==3.0.3
     # via playwright
+    # via sqlalchemy
 groq==0.8.0
     # via langchain-groq
 grpc-google-iam-v1==0.13.1
@@ -241,7 +238,6 @@ jmespath==1.0.1
     # via boto3
     # via botocore
 jsonpatch==1.33
-    # via langchain
     # via langchain-core
 jsonpointer==2.4
     # via jsonpatch
@@ -251,19 +247,16 @@ jsonschema-specifications==2023.12.1
     # via jsonschema
 kiwisolver==1.4.5
     # via matplotlib
-langchain==0.1.15
+langchain==0.2.10
     # via scrapegraphai
-langchain-anthropic==0.1.11
+langchain-anthropic==0.1.20
     # via scrapegraphai
-langchain-aws==0.1.3
+langchain-aws==0.1.11
     # via scrapegraphai
-langchain-community==0.0.38
-    # via langchain
-langchain-core==0.1.52
+langchain-core==0.2.22
     # via langchain
     # via langchain-anthropic
     # via langchain-aws
-    # via langchain-community
     # via langchain-fireworks
     # via langchain-google-genai
     # via langchain-google-vertexai
@@ -272,19 +265,18 @@ langchain-core==0.1.52
     # via langchain-text-splitters
 langchain-fireworks==0.1.3
     # via scrapegraphai
-langchain-google-genai==1.0.3
+langchain-google-genai==1.0.7
     # via scrapegraphai
 langchain-google-vertexai==1.0.4
     # via scrapegraphai
-langchain-groq==0.1.3
+langchain-groq==0.1.6
     # via scrapegraphai
-langchain-openai==0.1.6
+langchain-openai==0.1.17
     # via scrapegraphai
-langchain-text-splitters==0.0.2
+langchain-text-splitters==0.2.2
     # via langchain
-langsmith==0.1.63
+langsmith==0.1.93
     # via langchain
-    # via langchain-community
     # via langchain-core
 loguru==0.7.2
     # via burr
@@ -294,8 +286,6 @@ markdown-it-py==3.0.0
     # via rich
 markupsafe==2.1.5
     # via jinja2
-marshmallow==3.21.2
-    # via dataclasses-json
 matplotlib==3.9.0
     # via burr
 mccabe==0.7.0
@@ -315,15 +305,14 @@ numpy==1.26.4
     # via faiss-cpu
     # via langchain
     # via langchain-aws
-    # via langchain-community
     # via matplotlib
     # via pandas
     # via pyarrow
     # via pydeck
     # via sf-hamilton
     # via shapely
     # via streamlit
-openai==1.30.3
+openai==1.37.0
     # via burr
     # via langchain-fireworks
     # via langchain-openai
@@ -336,7 +325,6 @@ packaging==23.2
     # via google-cloud-bigquery
     # via huggingface-hub
     # via langchain-core
-    # via marshmallow
     # via matplotlib
     # via pytest
     # via sphinx
@@ -425,7 +413,6 @@ pytz==2024.1
 pyyaml==6.0.1
     # via huggingface-hub
     # via langchain
-    # via langchain-community
     # via langchain-core
     # via uvicorn
 referencing==0.35.1
@@ -441,7 +428,6 @@ requests==2.32.2
     # via google-cloud-storage
     # via huggingface-hub
     # via langchain
-    # via langchain-community
     # via langchain-fireworks
     # via langsmith
     # via sphinx
@@ -499,14 +485,12 @@ sphinxcontrib-serializinghtml==1.1.10
     # via sphinx
 sqlalchemy==2.0.30
     # via langchain
-    # via langchain-community
 starlette==0.37.2
     # via fastapi
 streamlit==1.35.0
     # via burr
 tenacity==8.3.0
     # via langchain
-    # via langchain-community
     # via langchain-core
     # via streamlit
 tiktoken==0.7.0
@@ -556,7 +540,6 @@ typing-extensions==4.12.0
     # via typing-inspect
     # via uvicorn
 typing-inspect==0.9.0
-    # via dataclasses-json
     # via sf-hamilton
 tzdata==2024.1
     # via pandas

requirements-dev.txt

@@ -1,4 +1,6 @@
-sphinx==7.1.2
-furo==2024.5.6
 pytest==8.0.0
-burr[start]==0.22.1
+pytest-mock==3.14.0
+burr[start]==0.22.1
+sphinx==6.0
+furo==2024.5.6
+pylint>=3.2.5