Source code for boto3.resources.collection

# Copyright 2014 Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"). You
# may not use this file except in compliance with the License. A copy of
# the License is located at
#
# https://aws.amazon.com/apache2.0/
#
# or in the "license" file accompanying this file. This file is
# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
# ANY KIND, either express or implied. See the License for the specific
# language governing permissions and limitations under the License.

import copy
import logging

from botocore import xform_name
from botocore.utils import merge_dicts

from ..docs import docstring
from .action import BatchAction
from .params import create_request_parameters
from .response import ResourceHandler

logger = logging.getLogger(__name__)



[docs]class ResourceCollection:
    """
    Represents a collection of resources, which can be iterated through,
    optionally with filtering. Collections automatically handle pagination
    for you.

    See :ref:`guide_collections` for a high-level overview of collections,
    including when remote service requests are performed.

    :type model: :py:class:`~boto3.resources.model.Collection`
    :param model: Collection model
    :type parent: :py:class:`~boto3.resources.base.ServiceResource`
    :param parent: The collection's parent resource
    :type handler: :py:class:`~boto3.resources.response.ResourceHandler`
    :param handler: The resource response handler used to create resource
                    instances
    """

    def __init__(self, model, parent, handler, **kwargs):
        self._model = model
        self._parent = parent
        self._py_operation_name = xform_name(model.request.operation)
        self._handler = handler
        self._params = copy.deepcopy(kwargs)

    def __repr__(self):
        return '{}({}, {})'.format(
            self.__class__.__name__,
            self._parent,
            f'{self._parent.meta.service_name}.{self._model.resource.type}',
        )

    def __iter__(self):
        """
        A generator which yields resource instances after doing the
        appropriate service operation calls and handling any pagination
        on your behalf.

        Page size, item limit, and filter parameters are applied
        if they have previously been set.

            >>> bucket = s3.Bucket('boto3')
            >>> for obj in bucket.objects.all():
            ...     print(obj.key)
            'key1'
            'key2'

        """
        limit = self._params.get('limit', None)

        count = 0
        for page in self.pages():
            for item in page:
                yield item

                # If the limit is set and has been reached, then
                # we stop processing items here.
                count += 1
                if limit is not None and count >= limit:
                    return

    def _clone(self, **kwargs):
        """
        Create a clone of this collection. This is used by the methods
        below to provide a chainable interface that returns copies
        rather than the original. This allows things like:

            >>> base = collection.filter(Param1=1)
            >>> query1 = base.filter(Param2=2)
            >>> query2 = base.filter(Param3=3)
            >>> query1.params
            {'Param1': 1, 'Param2': 2}
            >>> query2.params
            {'Param1': 1, 'Param3': 3}

        :rtype: :py:class:`ResourceCollection`
        :return: A clone of this resource collection
        """
        params = copy.deepcopy(self._params)
        merge_dicts(params, kwargs, append_lists=True)
        clone = self.__class__(
            self._model, self._parent, self._handler, **params
        )
        return clone


[docs]    def pages(self):
        """
        A generator which yields pages of resource instances after
        doing the appropriate service operation calls and handling
        any pagination on your behalf. Non-paginated calls will
        return a single page of items.

        Page size, item limit, and filter parameters are applied
        if they have previously been set.

            >>> bucket = s3.Bucket('boto3')
            >>> for page in bucket.objects.pages():
            ...     for obj in page:
            ...         print(obj.key)
            'key1'
            'key2'

        :rtype: list(:py:class:`~boto3.resources.base.ServiceResource`)
        :return: List of resource instances
        """
        client = self._parent.meta.client
        cleaned_params = self._params.copy()
        limit = cleaned_params.pop('limit', None)
        page_size = cleaned_params.pop('page_size', None)
        params = create_request_parameters(self._parent, self._model.request)
        merge_dicts(params, cleaned_params, append_lists=True)

        # Is this a paginated operation? If so, we need to get an
        # iterator for the various pages. If not, then we simply
        # call the operation and return the result as a single
        # page in a list. For non-paginated results, we just ignore
        # the page size parameter.
        if client.can_paginate(self._py_operation_name):
            logger.debug(
                'Calling paginated %s:%s with %r',
                self._parent.meta.service_name,
                self._py_operation_name,
                params,
            )
            paginator = client.get_paginator(self._py_operation_name)
            pages = paginator.paginate(
                PaginationConfig={'MaxItems': limit, 'PageSize': page_size},
                **params,
            )
        else:
            logger.debug(
                'Calling %s:%s with %r',
                self._parent.meta.service_name,
                self._py_operation_name,
                params,
            )
            pages = [getattr(client, self._py_operation_name)(**params)]

        # Now that we have a page iterator or single page of results
        # we start processing and yielding individual items.
        count = 0
        for page in pages:
            page_items = []
            for item in self._handler(self._parent, params, page):
                page_items.append(item)

                # If the limit is set and has been reached, then
                # we stop processing items here.
                count += 1
                if limit is not None and count >= limit:
                    break

            yield page_items

            # Stop reading pages if we've reached out limit
            if limit is not None and count >= limit:
                break



[docs]    def all(self):
        """
        Get all items from the collection, optionally with a custom
        page size and item count limit.

        This method returns an iterable generator which yields
        individual resource instances. Example use::

            # Iterate through items
            >>> for queue in sqs.queues.all():
            ...     print(queue.url)
            'https://url1'
            'https://url2'

            # Convert to list
            >>> queues = list(sqs.queues.all())
            >>> len(queues)
            2
        """
        return self._clone()



[docs]    def filter(self, **kwargs):
        """
        Get items from the collection, passing keyword arguments along
        as parameters to the underlying service operation, which are
        typically used to filter the results.

        This method returns an iterable generator which yields
        individual resource instances. Example use::

            # Iterate through items
            >>> for queue in sqs.queues.filter(Param='foo'):
            ...     print(queue.url)
            'https://url1'
            'https://url2'

            # Convert to list
            >>> queues = list(sqs.queues.filter(Param='foo'))
            >>> len(queues)
            2

        :rtype: :py:class:`ResourceCollection`
        """
        return self._clone(**kwargs)



[docs]    def limit(self, count):
        """
        Return at most this many resources.

            >>> for bucket in s3.buckets.limit(5):
            ...     print(bucket.name)
            'bucket1'
            'bucket2'
            'bucket3'
            'bucket4'
            'bucket5'

        :type count: int
        :param count: Return no more than this many items
        :rtype: :py:class:`ResourceCollection`
        """
        return self._clone(limit=count)



[docs]    def page_size(self, count):
        """
        Fetch at most this many resources per service request.

            >>> for obj in s3.Bucket('boto3').objects.page_size(100):
            ...     print(obj.key)

        :type count: int
        :param count: Fetch this many items per request
        :rtype: :py:class:`ResourceCollection`
        """
        return self._clone(page_size=count)




[docs]class CollectionManager:
    """
    A collection manager provides access to resource collection instances,
    which can be iterated and filtered. The manager exposes some
    convenience functions that are also found on resource collections,
    such as :py:meth:`~ResourceCollection.all` and
    :py:meth:`~ResourceCollection.filter`.

    Get all items::

        >>> for bucket in s3.buckets.all():
        ...     print(bucket.name)

    Get only some items via filtering::

        >>> for queue in sqs.queues.filter(QueueNamePrefix='AWS'):
        ...     print(queue.url)

    Get whole pages of items:

        >>> for page in s3.Bucket('boto3').objects.pages():
        ...     for obj in page:
        ...         print(obj.key)

    A collection manager is not iterable. You **must** call one of the
    methods that return a :py:class:`ResourceCollection` before trying
    to iterate, slice, or convert to a list.

    See the :ref:`guide_collections` guide for a high-level overview
    of collections, including when remote service requests are performed.

    :type collection_model: :py:class:`~boto3.resources.model.Collection`
    :param model: Collection model

    :type parent: :py:class:`~boto3.resources.base.ServiceResource`
    :param parent: The collection's parent resource

    :type factory: :py:class:`~boto3.resources.factory.ResourceFactory`
    :param factory: The resource factory to create new resources

    :type service_context: :py:class:`~boto3.utils.ServiceContext`
    :param service_context: Context about the AWS service
    """

    # The class to use when creating an iterator
    _collection_cls = ResourceCollection

    def __init__(self, collection_model, parent, factory, service_context):
        self._model = collection_model
        operation_name = self._model.request.operation
        self._parent = parent

        search_path = collection_model.resource.path
        self._handler = ResourceHandler(
            search_path=search_path,
            factory=factory,
            resource_model=collection_model.resource,
            service_context=service_context,
            operation_name=operation_name,
        )

    def __repr__(self):
        return '{}({}, {})'.format(
            self.__class__.__name__,
            self._parent,
            f'{self._parent.meta.service_name}.{self._model.resource.type}',
        )


[docs]    def iterator(self, **kwargs):
        """
        Get a resource collection iterator from this manager.

        :rtype: :py:class:`ResourceCollection`
        :return: An iterable representing the collection of resources
        """
        return self._collection_cls(
            self._model, self._parent, self._handler, **kwargs
        )


    # Set up some methods to proxy ResourceCollection methods

[docs]    def all(self):
        return self.iterator()


    all.__doc__ = ResourceCollection.all.__doc__


[docs]    def filter(self, **kwargs):
        return self.iterator(**kwargs)


    filter.__doc__ = ResourceCollection.filter.__doc__


[docs]    def limit(self, count):
        return self.iterator(limit=count)


    limit.__doc__ = ResourceCollection.limit.__doc__


[docs]    def page_size(self, count):
        return self.iterator(page_size=count)


    page_size.__doc__ = ResourceCollection.page_size.__doc__


[docs]    def pages(self):
        return self.iterator().pages()


    pages.__doc__ = ResourceCollection.pages.__doc__




[docs]class CollectionFactory:
    """
    A factory to create new
    :py:class:`CollectionManager` and :py:class:`ResourceCollection`
    subclasses from a :py:class:`~boto3.resources.model.Collection`
    model. These subclasses include methods to perform batch operations.
    """


[docs]    def load_from_definition(
        self, resource_name, collection_model, service_context, event_emitter
    ):
        """
        Loads a collection from a model, creating a new
        :py:class:`CollectionManager` subclass
        with the correct properties and methods, named based on the service
        and resource name, e.g. ec2.InstanceCollectionManager. It also
        creates a new :py:class:`ResourceCollection` subclass which is used
        by the new manager class.

        :type resource_name: string
        :param resource_name: Name of the resource to look up. For services,
                              this should match the ``service_name``.

        :type service_context: :py:class:`~boto3.utils.ServiceContext`
        :param service_context: Context about the AWS service

        :type event_emitter: :py:class:`~botocore.hooks.HierarchialEmitter`
        :param event_emitter: An event emitter

        :rtype: Subclass of :py:class:`CollectionManager`
        :return: The collection class.
        """
        attrs = {}
        collection_name = collection_model.name

        # Create the batch actions for a collection
        self._load_batch_actions(
            attrs,
            resource_name,
            collection_model,
            service_context.service_model,
            event_emitter,
        )
        # Add the documentation to the collection class's methods
        self._load_documented_collection_methods(
            attrs=attrs,
            resource_name=resource_name,
            collection_model=collection_model,
            service_model=service_context.service_model,
            event_emitter=event_emitter,
            base_class=ResourceCollection,
        )

        if service_context.service_name == resource_name:
            cls_name = (
                f'{service_context.service_name}.{collection_name}Collection'
            )
        else:
            cls_name = f'{service_context.service_name}.{resource_name}.{collection_name}Collection'

        collection_cls = type(str(cls_name), (ResourceCollection,), attrs)

        # Add the documentation to the collection manager's methods
        self._load_documented_collection_methods(
            attrs=attrs,
            resource_name=resource_name,
            collection_model=collection_model,
            service_model=service_context.service_model,
            event_emitter=event_emitter,
            base_class=CollectionManager,
        )
        attrs['_collection_cls'] = collection_cls
        cls_name += 'Manager'

        return type(str(cls_name), (CollectionManager,), attrs)


    def _load_batch_actions(
        self,
        attrs,
        resource_name,
        collection_model,
        service_model,
        event_emitter,
    ):
        """
        Batch actions on the collection become methods on both
        the collection manager and iterators.
        """
        for action_model in collection_model.batch_actions:
            snake_cased = xform_name(action_model.name)
            attrs[snake_cased] = self._create_batch_action(
                resource_name,
                snake_cased,
                action_model,
                collection_model,
                service_model,
                event_emitter,
            )

    def _load_documented_collection_methods(
        factory_self,
        attrs,
        resource_name,
        collection_model,
        service_model,
        event_emitter,
        base_class,
    ):
        # The base class already has these methods defined. However
        # the docstrings are generic and not based for a particular service
        # or resource. So we override these methods by proxying to the
        # base class's builtin method and adding a docstring
        # that pertains to the resource.

        # A collection's all() method.
        def all(self):
            return base_class.all(self)

        all.__doc__ = docstring.CollectionMethodDocstring(
            resource_name=resource_name,
            action_name='all',
            event_emitter=event_emitter,
            collection_model=collection_model,
            service_model=service_model,
            include_signature=False,
        )
        attrs['all'] = all

        # The collection's filter() method.
        def filter(self, **kwargs):
            return base_class.filter(self, **kwargs)

        filter.__doc__ = docstring.CollectionMethodDocstring(
            resource_name=resource_name,
            action_name='filter',
            event_emitter=event_emitter,
            collection_model=collection_model,
            service_model=service_model,
            include_signature=False,
        )
        attrs['filter'] = filter

        # The collection's limit method.
        def limit(self, count):
            return base_class.limit(self, count)

        limit.__doc__ = docstring.CollectionMethodDocstring(
            resource_name=resource_name,
            action_name='limit',
            event_emitter=event_emitter,
            collection_model=collection_model,
            service_model=service_model,
            include_signature=False,
        )
        attrs['limit'] = limit

        # The collection's page_size method.
        def page_size(self, count):
            return base_class.page_size(self, count)

        page_size.__doc__ = docstring.CollectionMethodDocstring(
            resource_name=resource_name,
            action_name='page_size',
            event_emitter=event_emitter,
            collection_model=collection_model,
            service_model=service_model,
            include_signature=False,
        )
        attrs['page_size'] = page_size

    def _create_batch_action(
        factory_self,
        resource_name,
        snake_cased,
        action_model,
        collection_model,
        service_model,
        event_emitter,
    ):
        """
        Creates a new method which makes a batch operation request
        to the underlying service API.
        """
        action = BatchAction(action_model)

        def batch_action(self, *args, **kwargs):
            return action(self, *args, **kwargs)

        batch_action.__name__ = str(snake_cased)
        batch_action.__doc__ = docstring.BatchActionDocstring(
            resource_name=resource_name,
            event_emitter=event_emitter,
            batch_action_model=action_model,
            service_model=service_model,
            collection_model=collection_model,
            include_signature=False,
        )
        return batch_action