Merge pull request #620 from goasleep/feature/export_search_engine

feat:expose the search engine params to user

Author: VinciGit00

scrapegraphai/graphs/csv_scraper_multi_graph.py

@@ -2,16 +2,18 @@
 CSVScraperMultiGraph Module
 """
 
-from copy import copy, deepcopy
 from typing import List, Optional
 from pydantic import BaseModel
+
+
 from .base_graph import BaseGraph
 from .abstract_graph import AbstractGraph
 from .csv_scraper_graph import CSVScraperGraph
 from ..nodes import (
     GraphIteratorNode,
     MergeAnswersNode
 )
+from ..utils.copy import safe_deepcopy
 
 class CSVScraperMultiGraph(AbstractGraph):
     """ 
@@ -46,10 +48,7 @@ def __init__(self, prompt: str, source: List[str],
 
         self.max_results = config.get("max_results", 3)
 
-        if all(isinstance(value, str) for value in config.values()):
-            self.copy_config = copy(config)
-        else:
-            self.copy_config = deepcopy(config)
+        self.copy_config = safe_deepcopy(config)
 
         super().__init__(prompt, config, source, schema)
 

scrapegraphai/graphs/json_scraper_multi_graph.py

@@ -2,16 +2,18 @@
 JSONScraperMultiGraph Module
 """
 
-from copy import copy, deepcopy
+from copy import deepcopy
 from typing import List, Optional
 from pydantic import BaseModel
+
 from .base_graph import BaseGraph
 from .abstract_graph import AbstractGraph
 from .json_scraper_graph import JSONScraperGraph
 from ..nodes import (
     GraphIteratorNode,
     MergeAnswersNode
 )
+from ..utils.copy import safe_deepcopy
 
 class JSONScraperMultiGraph(AbstractGraph):
     """ 
@@ -45,10 +47,7 @@ def __init__(self, prompt: str, source: List[str], config: dict, schema: Optiona
 
         self.max_results = config.get("max_results", 3)
 
-        if all(isinstance(value, str) for value in config.values()):
-            self.copy_config = copy(config)
-        else:
-            self.copy_config = deepcopy(config)
+        self.copy_config = safe_deepcopy(config)
 
         self.copy_schema = deepcopy(schema)
 

scrapegraphai/graphs/markdown_scraper_multi_graph.py

@@ -12,6 +12,7 @@
     GraphIteratorNode,
     MergeAnswersNode
 )
+from ..utils.copy import safe_deepcopy
 
 class MDScraperMultiGraph(AbstractGraph):
     """
@@ -42,11 +43,7 @@ class MDScraperMultiGraph(AbstractGraph):
     """
 
     def __init__(self, prompt: str, source: List[str], config: dict, schema: Optional[BaseModel] = None):
-        if all(isinstance(value, str) for value in config.values()):
-            self.copy_config = copy(config)
-        else:
-            self.copy_config = deepcopy(config)
-
+        self.copy_config = safe_deepcopy(config)
         self.copy_schema = deepcopy(schema)
 
         super().__init__(prompt, config, source, schema)

scrapegraphai/graphs/omni_search_graph.py

@@ -2,7 +2,7 @@
 OmniSearchGraph Module
 """
 
-from copy import copy, deepcopy
+from copy import deepcopy
 from typing import Optional
 from pydantic import BaseModel
 
@@ -15,6 +15,7 @@
     GraphIteratorNode,
     MergeAnswersNode
 )
+from ..utils.copy import safe_deepcopy
 
 
 class OmniSearchGraph(AbstractGraph):
@@ -48,10 +49,7 @@ def __init__(self, prompt: str, config: dict, schema: Optional[BaseModel] = None
 
         self.max_results = config.get("max_results", 3)
 
-        if all(isinstance(value, str) for value in config.values()):
-            self.copy_config = copy(config)
-        else:
-            self.copy_config = deepcopy(config)
+        self.copy_config = safe_deepcopy(config)
 
         self.copy_schema = deepcopy(schema)
 
@@ -85,7 +83,8 @@ def _create_graph(self) -> BaseGraph:
             output=["urls"],
             node_config={
                 "llm_model": self.llm_model,
-                "max_results": self.max_results
+                "max_results": self.max_results,
+                "search_engine": self.copy_config.get("search_engine")
             }
         )
         graph_iterator_node = GraphIteratorNode(

scrapegraphai/graphs/pdf_scraper_multi_graph.py

@@ -2,7 +2,7 @@
 PdfScraperMultiGraph Module
 """
 
-from copy import copy, deepcopy
+from copy import deepcopy
 from typing import List, Optional
 from pydantic import BaseModel
 from .base_graph import BaseGraph
@@ -12,6 +12,7 @@
     GraphIteratorNode,
     MergeAnswersNode
 )
+from ..utils.copy import safe_deepcopy
 
 class PdfScraperMultiGraph(AbstractGraph):
     """ 
@@ -44,10 +45,7 @@ class PdfScraperMultiGraph(AbstractGraph):
     def __init__(self, prompt: str, source: List[str], 
                  config: dict, schema: Optional[BaseModel] = None):
 
-        if all(isinstance(value, str) for value in config.values()):
-            self.copy_config = copy(config)
-        else:
-            self.copy_config = deepcopy(config)
+        self.copy_config = safe_deepcopy(config)
 
         self.copy_schema = deepcopy(schema)
 

scrapegraphai/graphs/script_creator_multi_graph.py

@@ -2,7 +2,6 @@
 ScriptCreatorMultiGraph Module
 """
 
-from copy import copy, deepcopy
 from typing import List, Optional
 
 from pydantic import BaseModel
@@ -15,6 +14,7 @@
     GraphIteratorNode,
     MergeGeneratedScriptsNode
 )
+from ..utils.copy import safe_deepcopy
 
 class ScriptCreatorMultiGraph(AbstractGraph):
     """ 
@@ -47,10 +47,7 @@ def __init__(self, prompt: str, source: List[str], config: dict, schema: Optiona
 
         self.max_results = config.get("max_results", 3)
 
-        if all(isinstance(value, str) for value in config.values()):
-            self.copy_config = copy(config)
-        else:
-            self.copy_config = deepcopy(config)
+        self.copy_config = safe_deepcopy(config)
 
         super().__init__(prompt, config, source, schema)
 

scrapegraphai/graphs/search_graph.py

@@ -2,7 +2,7 @@
 SearchGraph Module
 """
 
-from copy import copy, deepcopy
+from copy import deepcopy
 from typing import Optional, List
 from pydantic import BaseModel
 
@@ -15,6 +15,7 @@
     GraphIteratorNode,
     MergeAnswersNode
 )
+from ..utils.copy import safe_deepcopy
 
 class SearchGraph(AbstractGraph):
     """ 
@@ -47,10 +48,7 @@ class SearchGraph(AbstractGraph):
     def __init__(self, prompt: str, config: dict, schema: Optional[BaseModel] = None):
         self.max_results = config.get("max_results", 3)
 
-        if all(isinstance(value, str) for value in config.values()):
-            self.copy_config = copy(config)
-        else:
-            self.copy_config = deepcopy(config)
+        self.copy_config = safe_deepcopy(config)
         self.copy_schema = deepcopy(schema)
         self.considered_urls = []  # New attribute to store URLs
 
@@ -78,7 +76,8 @@ def _create_graph(self) -> BaseGraph:
             output=["urls"],
             node_config={
                 "llm_model": self.llm_model,
-                "max_results": self.max_results
+                "max_results": self.max_results,
+                "search_engine": self.copy_config.get("search_engine")
             }
         )
         graph_iterator_node = GraphIteratorNode(

scrapegraphai/graphs/smart_scraper_multi_graph.py

@@ -2,7 +2,7 @@
 SmartScraperMultiGraph Module
 """
 
-from copy import copy, deepcopy
+from copy import deepcopy
 from typing import List, Optional
 from pydantic import BaseModel
 
@@ -14,6 +14,7 @@
     GraphIteratorNode,
     MergeAnswersNode
 )
+from ..utils.copy import safe_deepcopy
 
 class SmartScraperMultiGraph(AbstractGraph):
     """ 
@@ -48,10 +49,7 @@ def __init__(self, prompt: str, source: List[str],
 
         self.max_results = config.get("max_results", 3)
 
-        if all(isinstance(value, str) for value in config.values()):
-            self.copy_config = copy(config)
-        else:
-            self.copy_config = deepcopy(config)
+        self.copy_config = safe_deepcopy(config)
 
         self.copy_schema = deepcopy(schema)
 

scrapegraphai/graphs/xml_scraper_multi_graph.py

@@ -2,7 +2,7 @@
 XMLScraperMultiGraph Module
 """
 
-from copy import copy, deepcopy
+from copy import deepcopy
 from typing import List, Optional
 from pydantic import BaseModel
 
@@ -14,6 +14,7 @@
     GraphIteratorNode,
     MergeAnswersNode
 )
+from ..utils.copy import safe_deepcopy
 
 class XMLScraperMultiGraph(AbstractGraph):
     """ 
@@ -46,10 +47,7 @@ class XMLScraperMultiGraph(AbstractGraph):
     def __init__(self, prompt: str, source: List[str], 
                  config: dict, schema: Optional[BaseModel] = None):
 
-        if all(isinstance(value, str) for value in config.values()):
-            self.copy_config = copy(config)
-        else:
-            self.copy_config = deepcopy(config)
+        self.copy_config = safe_deepcopy(config)
 
         self.copy_schema = deepcopy(schema)
 

scrapegraphai/nodes/search_internet_node.py

@@ -41,7 +41,11 @@ def __init__(
         self.verbose = (
             False if node_config is None else node_config.get("verbose", False)
         )
-        self.search_engine = node_config.get("search_engine", "google")
+        self.search_engine = (
+            node_config["search_engine"]
+            if node_config.get("search_engine")
+            else "google"
+        )
         self.max_results = node_config.get("max_results", 3)
 
     def execute(self, state: dict) -> dict:

scrapegraphai/utils/copy.py

@@ -0,0 +1,75 @@
+import copy
+from typing import Any, Dict, Optional
+from pydantic.v1 import BaseModel
+
+class DeepCopyError(Exception):
+    """Custom exception raised when an object cannot be deep-copied."""
+    pass
+
+def safe_deepcopy(obj: Any) -> Any:
+    """
+    Attempts to create a deep copy of the object using `copy.deepcopy`
+    whenever possible. If that fails, it falls back to custom deep copy
+    logic. If that also fails, it raises a `DeepCopyError`.
+    
+    Args:
+        obj (Any): The object to be copied, which can be of any type.
+
+    Returns:
+        Any: A deep copy of the object if possible; otherwise, a shallow
+        copy if deep copying fails; if neither is possible, the original
+        object is returned.
+    Raises:
+        DeepCopyError: If the object cannot be deep-copied or shallow-copied.
+    """
+
+    try:
+        
+        # Try to use copy.deepcopy first
+        return copy.deepcopy(obj)
+    except (TypeError, AttributeError) as e:
+        # If deepcopy fails, handle specific types manually
+
+        # Handle dictionaries
+        if isinstance(obj, dict):
+            new_obj = {}
+            
+            for k, v in obj.items():
+                new_obj[k] = safe_deepcopy(v)
+            return new_obj
+
+        # Handle lists
+        elif isinstance(obj, list):
+            new_obj = []
+            
+            for v in obj:
+                new_obj.append(safe_deepcopy(v))
+            return new_obj
+
+        # Handle tuples (immutable, but might contain mutable objects)
+        elif isinstance(obj, tuple):
+            new_obj = tuple(safe_deepcopy(v) for v in obj)
+            
+            return new_obj
+
+        # Handle frozensets (immutable, but might contain mutable objects)
+        elif isinstance(obj, frozenset):
+            new_obj = frozenset(safe_deepcopy(v) for v in obj)
+            return new_obj
+
+        # Handle objects with attributes
+        elif hasattr(obj, "__dict__"):
+            # If an object cannot be deep copied, then the sub-properties of \
+            # the object will not be analyzed and shallow copy will be used directly.
+            try:
+                return copy.copy(obj)
+            except (TypeError, AttributeError):
+                raise DeepCopyError(f"Cannot deep copy the object of type {type(obj)}") from e
+
+
+        # Attempt shallow copy as a fallback
+        try:
+            return copy.copy(obj)
+        except (TypeError, AttributeError):
+            raise DeepCopyError(f"Cannot deep copy the object of type {type(obj)}") from e
+