fix(chunking): count tokens from words instead of characters

closes #513

Author: f-aguzzi

scrapegraphai/nodes/parse_node.py

@@ -74,22 +74,22 @@ def execute(self, state: dict) -> dict:
             docs_transformed = docs_transformed[0]
 
             chunks = chunk(text=docs_transformed.page_content,
-                            chunk_size= self.node_config.get("chunk_size", 4096)-250,
-                            token_counter= lambda x: len(x),
+                            chunk_size=self.node_config.get("chunk_size", 4096)-250,
+                            token_counter=lambda text: len(text.split()),
                             memoize=False)
         else:
             docs_transformed = docs_transformed[0]
 
             if isinstance(docs_transformed, Document):
                 chunks = chunk(text=docs_transformed.page_content,
-                            chunk_size= self.node_config.get("chunk_size", 4096)-250,
-                            token_counter= lambda x: len(x),
+                            chunk_size=self.node_config.get("chunk_size", 4096)-250,
+                            token_counter=lambda text: len(text.split()),
                             memoize=False)
             else:
 
                 chunks = chunk(text=docs_transformed,
-                                chunk_size= self.node_config.get("chunk_size", 4096)-250,
-                                token_counter= lambda x: len(x),
+                                chunk_size=self.node_config.get("chunk_size", 4096)-250,
+                                token_counter=lambda text: len(text.split()),
                                 memoize=False)
 
         state.update({self.output[0]: chunks})