fix: bug with fetch node

examples/single_node/fetch_node.py

@@ -0,0 +1,27 @@
+"""
+Example of custom graph using existing nodes
+"""
+
+from scrapegraphai.nodes import FetchNode
+
+# ************************************************
+# Define the node
+# ************************************************
+
+
+robots_node = FetchNode(
+    input="url | local_dir",
+    output=["doc"],
+)
+
+# ************************************************
+# Test the node
+# ************************************************
+
+state = {
+    "url": "https://twitter.com/home"
+}
+
+result = robots_node.execute(state)
+
+print(result)

scrapegraphai/nodes/fetch_node.py

@@ -59,7 +59,7 @@ def execute(self, state):
 
         Raises:
             KeyError: If the 'url' key is not found in the state, indicating that the
-                      necessary information to perform the operation is missing.
+                    necessary information to perform the operation is missing.
         """
         print(f"--- Executing {self.node_name} Node ---")
 
@@ -78,7 +78,7 @@ def execute(self, state):
             })]
 
         else:
-            if self.node_config.get("endpoint") is not None:
+            if self.node_config is not None and self.node_config.get("endpoint") is not None:
                 loader = AsyncHtmlLoader(
                     source, proxies={"http": self.node_config["endpoint"]})
             else:

tests/nodes/fetch_node_test.py

@@ -0,0 +1,44 @@
+"""
+Module for testinh robot_node
+"""
+import pytest
+from scrapegraphai.nodes import FetchNode
+
+
+@pytest.fixture
+def setup():
+    """
+    setup
+    """
+    # ************************************************
+    # Define the node
+    # ************************************************
+
+    robots_node = FetchNode(
+        input="url | local_dir",
+        output=["doc"],
+    )
+
+    return robots_node
+
+# ************************************************
+# Test the node
+# ************************************************
+
+
+def test_robots_node(setup):
+    """
+    Run the tests
+    """
+    state = {
+        "url": "https://twitter.com/home"
+    }
+
+    result = setup.execute(state)
+
+    assert result is not None
+
+
+# If you need to run this script directly
+if __name__ == "__main__":
+    pytest.main()

tests/nodes/robot_node_test.py

@@ -1,15 +1,10 @@
 """
 Module for testinh robot_node
 """
-import os
-from dotenv import load_dotenv
 import pytest
-from scrapegraphai.models import OpenAI
+from scrapegraphai.models import Ollama
 from scrapegraphai.nodes import RobotsNode
 
-# Load environment variables from .env file
-load_dotenv()
-
 
 @pytest.fixture
 def setup():
@@ -20,12 +15,9 @@ def setup():
     # Define the configuration for the graph
     # ************************************************
 
-    openai_key = os.getenv("OPENAI_APIKEY")
-
     graph_config = {
         "llm": {
-            "api_key": openai_key,
-            "model": "gpt-3.5-turbo",
+            "model": "ollama/llama3",
             "temperature": 0,
             "streaming": True
         },
@@ -35,7 +27,7 @@ def setup():
     # Define the node
     # ************************************************
 
-    llm_model = OpenAI(graph_config["llm"])
+    llm_model = Ollama(graph_config["llm"])
 
     robots_node = RobotsNode(
         input="url",