Merge pull request #479 from microsoftgraph/shem/add-page-iterator

Add Page Iterator  support

Author: shemogumbe

.gitignore

@@ -109,6 +109,7 @@ venv/
 ENV/
 env.bak/
 venv.bak/
+*env/
 
 # Spyder project settings
 .spyderproject
@@ -131,4 +132,4 @@ dmypy.json
 # Pycharm
 .idea/
 
-app.py
+app*.py

.vscode/settings.json

@@ -0,0 +1,3 @@
+{
+    "editor.formatOnSave": true
+}

requirements-dev.txt

@@ -86,6 +86,8 @@ pytest-cov==4.1.0
 
 pytest-mock==3.12.0
 
+python-dotenv==1.0.1
+
 pytest-trio==0.8.0
 
 pywin32==306 ; platform_system == 'Windows'
@@ -110,9 +112,10 @@ trio==0.25.0
 
 types-python-dateutil==2.9.0.20240316
 
+types-requests==2.31.0.20240125; python_version >= '3.7'
+urllib3==2.1.0 ; python_version >= '3.7'
 typing-extensions==4.10.0 ; python_version >= '3.7'
 
-urllib3==2.2.1 ; python_version >= '3.7'
 
 wrapt==1.15.0 ; python_version < '3.11'
 

src/msgraph_core/__init__.py

@@ -13,5 +13,7 @@
 from .authentication import AzureIdentityAuthenticationProvider
 from .base_graph_request_adapter import BaseGraphRequestAdapter
 from .graph_client_factory import GraphClientFactory
+from .tasks import PageIterator
+from .models import PageResult
 
 __version__ = SDK_VERSION

src/msgraph_core/models/__init__.py

@@ -0,0 +1,3 @@
+from .page_result import PageResult
+
+__all__ = ['PageResult']

src/msgraph_core/models/page_result.py

@@ -0,0 +1,63 @@
+"""
+This module defines the PageResult class which represents a page of
+items in a paged response.
+
+The PageResult class provides methods to get and set the next link and
+the items in the page, create a PageResult from a discriminator value, set
+the value, get the field deserializers, and serialize the PageResult.
+
+Classes:
+    PageResult: Represents a page of items in a paged response.
+"""
+from __future__ import annotations
+from typing import List, Optional, Dict, Callable
+from dataclasses import dataclass
+
+from kiota_abstractions.serialization.parsable import Parsable
+from kiota_abstractions.serialization.serialization_writer \
+     import SerializationWriter
+from kiota_abstractions.serialization.parse_node import ParseNode
+from typing import TypeVar, List, Optional
+
+T = TypeVar('T')
+
+
+@dataclass
+class PageResult(Parsable):
+    odata_next_link: Optional[str] = None
+    value: Optional[List[Parsable]] = None
+
+    @staticmethod
+    def create_from_discriminator_value(parse_node: Optional[ParseNode] = None) -> PageResult:
+        """
+        Creates a new instance of the appropriate class based on discriminator value
+        Args:
+            parseNode: The parse node to use to read the discriminator value and create the object
+        Returns: Attachment
+        """
+        if not parse_node:
+            raise TypeError("parse_node cannot be null")
+        return PageResult()
+
+    def get_field_deserializers(self) -> Dict[str, Callable[[ParseNode], None]]:
+        """Gets the deserialization information for this object.
+
+        Returns:
+            Dict[str, Callable[[ParseNode], None]]: The deserialization information for this
+            object where each entry is a property key with its deserialization callback.
+        """
+        return {
+            "@odata.nextLink": lambda x: setattr(self, "odata_next_link", x.get_str_value()),
+            "value": lambda x: setattr(self, "value", x.get_collection_of_object_values(Parsable))
+        }
+
+    def serialize(self, writer: SerializationWriter) -> None:
+        """Writes the objects properties to the current writer.
+
+        Args:
+            writer (SerializationWriter): The writer to write to.
+        """
+        if not writer:
+            raise TypeError("Writer cannot be null")
+        writer.write_str_value("@odata.nextLink", self.odata_next_link)
+        writer.write_collection_of_object_values("value", self.value)

src/msgraph_core/tasks/__init__.py

@@ -0,0 +1 @@
+from .page_iterator import PageIterator

src/msgraph_core/tasks/page_iterator.py

@@ -0,0 +1,228 @@
+"""
+This module contains the PageIterator class which is used to 
+iterate over paged responses from a server.
+
+The PageIterator class provides methods to iterate over the items 
+in the pages, fetch the next page, convert a response to a page, and
+ fetch the next page from the server.
+
+The PageIterator class uses the Parsable interface to parse the responses
+ from the server, the HttpxRequestAdapter class to send requests to the 
+ server, and the PageResult class to represent the pages.
+
+This module also imports the necessary types and exceptions from the 
+typing, requests.exceptions, kiota_http.httpx_request_adapter,
+ kiota_abstractions.method, kiota_abstractions.headers_collection, 
+kiota_abstractions.request_information, kiota_abstractions.serialization.parsable,
+and models modules.
+"""
+
+from typing import Callable, Optional, Union, Dict, List
+
+from typing import TypeVar
+from requests.exceptions import InvalidURL
+
+from kiota_http.httpx_request_adapter import HttpxRequestAdapter
+from kiota_abstractions.method import Method
+from kiota_abstractions.headers_collection import HeadersCollection
+from kiota_abstractions.request_information import RequestInformation
+from kiota_abstractions.serialization.parsable import Parsable
+
+from msgraph_core.models.page_result import PageResult  # pylint: disable=no-name-in-module, import-error
+
+T = TypeVar('T', bound=Parsable)
+
+
+class PageIterator:
+    """
+This class is used to iterate over paged responses from a server.
+
+The PageIterator class provides methods to iterate over the items in the pages,
+fetch the next page, and convert a response to a page.
+
+Attributes:
+    request_adapter (HttpxRequestAdapter): The adapter used to send HTTP requests.
+    pause_index (int): The index at which to pause iteration.
+    headers (HeadersCollection): The headers to include in the HTTP requests.
+    request_options (list): The options for the HTTP requests.
+    current_page (PageResult): The current page of items.
+    object_type (str): The type of the items in the pages.
+    has_next (bool): Whether there are more pages to fetch.
+
+Methods:
+    __init__(response: Union[T, list, object], request_adapter: HttpxRequestAdapter,
+     constructor_callable: Optional[Callable] = None): Initializes a new instance of 
+     the PageIterator class.
+    """
+
+    def __init__(
+        self,
+        response: Union[T, list, object],
+        request_adapter: HttpxRequestAdapter,
+        constructor_callable: Optional[Callable] = None
+    ):
+        self.request_adapter = request_adapter
+
+        if isinstance(response, Parsable) and not constructor_callable:
+            parsable_factory = type(response)
+        elif constructor_callable is None:
+            parsable_factory = PageResult
+        self.parsable_factory = parsable_factory
+        self.pause_index = 0
+        self.headers: HeadersCollection = HeadersCollection()
+        self.request_options = []  # type: ignore
+        self.current_page = self.convert_to_page(response)
+        self.object_type = self.current_page.value[
+            0].__class__.__name__ if self.current_page.value else None
+        page = self.current_page
+        self._next_link = response.get('odata_next_link', '') if isinstance(
+            response, dict
+        ) else getattr(response, 'odata_next_link', '')
+        self._delta_link = response.get('@odata.deltaLink', '') if isinstance(
+            response, dict
+        ) else getattr(response, '@odata.deltaLink', '')
+
+        if page is not None:
+            self.current_page = page
+            self.has_next = bool(page.odata_next_link)
+
+    def set_headers(self, headers: dict) -> HeadersCollection:
+        """
+        Sets the headers for the HTTP requests.
+        This method takes a dictionary of headers and adds them to the
+        existing headers.
+        Args:
+            headers (dict): A dictionary of headers to add. The keys are the 
+            header names and the values are the header values.
+        """
+        self.headers.add_all(**headers)
+
+    @property
+    def delta_link(self):
+        return self._delta_link
+
+    @property
+    def next_link(self):
+        return self._next_link
+
+    def set_request_options(self, request_options: list) -> None:
+        """
+        Sets the request options for the HTTP requests.
+        Args:
+            request_options (list): The request options to set.
+        """
+        self.request_options = request_options
+
+    async def iterate(self, callback: Callable) -> None:
+        """
+        Iterates over the pages and applies a callback function to each item.
+        The iteration stops when there are no more pages or the callback
+         function returns False.
+        Args:
+            callback (Callable): The function to apply to each item. 
+            It should take one argument (the item) and return a boolean.
+        """
+        while True:
+            keep_iterating = self.enumerate(callback)
+            if not keep_iterating:
+                return
+            next_page = await self.next()
+            if not next_page:
+                return
+            self.current_page = next_page
+            self.pause_index = 0
+
+    async def next(self) -> Optional[PageResult]:
+        """
+        Fetches the next page of items.
+        Returns:
+            dict: The next page of items, or None if there are no more pages.
+        """
+        if self.current_page is not None and not self.current_page.odata_next_link:
+            return None
+        response = await self.fetch_next_page()
+        print(f"Response - {type(response)}")
+        page: PageResult = PageResult(response.odata_next_link, response.value)  # type: ignore
+        return page
+
+    @staticmethod
+    def convert_to_page(response: Union[T, list, object]) -> PageResult:
+        """
+        Converts a response to a PageResult.
+        This method extracts the 'value' and 'odata_next_link' from the 
+        response and uses them to create a PageResult.
+        Args:
+            response (Union[T, list, object]): The response to convert. It can
+            be a list, an object, or any other type.
+        Returns:
+            PageResult: The PageResult created from the response.
+        Raises:
+            ValueError: If the response is None or does not contain a 'value'.
+        """
+        if not response:
+            raise ValueError('Response cannot be null.')
+        value = None
+        if isinstance(response, list):
+            value = response.value  # type: ignore
+        elif hasattr(response, 'value'):
+            value = getattr(response, 'value')
+        elif isinstance(response, object):
+            value = getattr(response, 'value', [])
+        if value is None:
+            raise ValueError('The response does not contain a value.')
+        parsable_page = response if isinstance(response, dict) else vars(response)
+        next_link = parsable_page.get('odata_next_link', '') if isinstance(
+            parsable_page, dict
+        ) else getattr(parsable_page, 'odata_next_link', '')
+
+        page: PageResult = PageResult(next_link, value)
+        return page
+
+    async def fetch_next_page(self) -> List[Parsable]:
+        """
+        Fetches the next page of items from the server.
+        Returns:
+            dict: The response from the server.
+        Raises:
+            ValueError: If the current page does not contain a next link.
+            InvalidURL: If the next link URL could not be parsed.
+        """
+
+        next_link = self.current_page.odata_next_link
+        if not next_link:
+            raise ValueError('The response does not contain a nextLink.')
+        if not next_link.startswith('http'):
+            raise InvalidURL('Could not parse nextLink URL.')
+        request_info = RequestInformation()
+        request_info.http_method = Method.GET
+        request_info.url = next_link
+        request_info.headers = self.headers
+        if self.request_options:
+            request_info.add_request_options(*self.request_options)
+        error_map: Dict[str, int] = {}
+        response = await self.request_adapter.send_async(
+            request_info, self.parsable_factory, error_map
+        )
+        return response
+
+    def enumerate(self, callback: Optional[Callable] = None) -> bool:
+        """
+        Enumerates over the items in the current page and applies a
+        callback function to each item.
+        Args:
+            callback (Callable, optional): The function to apply to each item.
+            It should take one argument (the item) and return a boolean.
+        Returns:
+            bool: False if there are no items in the current page or the
+            callback function returns False, True otherwise.
+        """
+        keep_iterating = True
+        page_items = self.current_page.value
+        if not page_items:
+            return False
+        for i in range(self.pause_index, len(page_items)):
+            keep_iterating = callback(page_items[i]) if callback is not None else True
+            if not keep_iterating:
+                self.pause_index = i + 1
+                break
+        return keep_iterating