# Copyright 2014 Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"). You
# may not use this file except in compliance with the License. A copy of
# the License is located at
#
# https://aws.amazon.com/apache2.0/
#
# or in the "license" file accompanying this file. This file is
# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
# ANY KIND, either express or implied. See the License for the specific
# language governing permissions and limitations under the License.
import copy
import logging
from botocore import xform_name
from botocore.utils import merge_dicts
from ..docs import docstring
from .action import BatchAction
from .params import create_request_parameters
from .response import ResourceHandler
logger = logging.getLogger(__name__)
[docs]class ResourceCollection:
"""
Represents a collection of resources, which can be iterated through,
optionally with filtering. Collections automatically handle pagination
for you.
See :ref:`guide_collections` for a high-level overview of collections,
including when remote service requests are performed.
:type model: :py:class:`~boto3.resources.model.Collection`
:param model: Collection model
:type parent: :py:class:`~boto3.resources.base.ServiceResource`
:param parent: The collection's parent resource
:type handler: :py:class:`~boto3.resources.response.ResourceHandler`
:param handler: The resource response handler used to create resource
instances
"""
def __init__(self, model, parent, handler, **kwargs):
self._model = model
self._parent = parent
self._py_operation_name = xform_name(model.request.operation)
self._handler = handler
self._params = copy.deepcopy(kwargs)
def __repr__(self):
return '{}({}, {})'.format(
self.__class__.__name__,
self._parent,
f'{self._parent.meta.service_name}.{self._model.resource.type}',
)
def __iter__(self):
"""
A generator which yields resource instances after doing the
appropriate service operation calls and handling any pagination
on your behalf.
Page size, item limit, and filter parameters are applied
if they have previously been set.
>>> bucket = s3.Bucket('boto3')
>>> for obj in bucket.objects.all():
... print(obj.key)
'key1'
'key2'
"""
limit = self._params.get('limit', None)
count = 0
for page in self.pages():
for item in page:
yield item
# If the limit is set and has been reached, then
# we stop processing items here.
count += 1
if limit is not None and count >= limit:
return
def _clone(self, **kwargs):
"""
Create a clone of this collection. This is used by the methods
below to provide a chainable interface that returns copies
rather than the original. This allows things like:
>>> base = collection.filter(Param1=1)
>>> query1 = base.filter(Param2=2)
>>> query2 = base.filter(Param3=3)
>>> query1.params
{'Param1': 1, 'Param2': 2}
>>> query2.params
{'Param1': 1, 'Param3': 3}
:rtype: :py:class:`ResourceCollection`
:return: A clone of this resource collection
"""
params = copy.deepcopy(self._params)
merge_dicts(params, kwargs, append_lists=True)
clone = self.__class__(
self._model, self._parent, self._handler, **params
)
return clone
[docs] def pages(self):
"""
A generator which yields pages of resource instances after
doing the appropriate service operation calls and handling
any pagination on your behalf. Non-paginated calls will
return a single page of items.
Page size, item limit, and filter parameters are applied
if they have previously been set.
>>> bucket = s3.Bucket('boto3')
>>> for page in bucket.objects.pages():
... for obj in page:
... print(obj.key)
'key1'
'key2'
:rtype: list(:py:class:`~boto3.resources.base.ServiceResource`)
:return: List of resource instances
"""
client = self._parent.meta.client
cleaned_params = self._params.copy()
limit = cleaned_params.pop('limit', None)
page_size = cleaned_params.pop('page_size', None)
params = create_request_parameters(self._parent, self._model.request)
merge_dicts(params, cleaned_params, append_lists=True)
# Is this a paginated operation? If so, we need to get an
# iterator for the various pages. If not, then we simply
# call the operation and return the result as a single
# page in a list. For non-paginated results, we just ignore
# the page size parameter.
if client.can_paginate(self._py_operation_name):
logger.debug(
'Calling paginated %s:%s with %r',
self._parent.meta.service_name,
self._py_operation_name,
params,
)
paginator = client.get_paginator(self._py_operation_name)
pages = paginator.paginate(
PaginationConfig={'MaxItems': limit, 'PageSize': page_size},
**params,
)
else:
logger.debug(
'Calling %s:%s with %r',
self._parent.meta.service_name,
self._py_operation_name,
params,
)
pages = [getattr(client, self._py_operation_name)(**params)]
# Now that we have a page iterator or single page of results
# we start processing and yielding individual items.
count = 0
for page in pages:
page_items = []
for item in self._handler(self._parent, params, page):
page_items.append(item)
# If the limit is set and has been reached, then
# we stop processing items here.
count += 1
if limit is not None and count >= limit:
break
yield page_items
# Stop reading pages if we've reached out limit
if limit is not None and count >= limit:
break
[docs] def all(self):
"""
Get all items from the collection, optionally with a custom
page size and item count limit.
This method returns an iterable generator which yields
individual resource instances. Example use::
# Iterate through items
>>> for queue in sqs.queues.all():
... print(queue.url)
'https://url1'
'https://url2'
# Convert to list
>>> queues = list(sqs.queues.all())
>>> len(queues)
2
"""
return self._clone()
[docs] def filter(self, **kwargs):
"""
Get items from the collection, passing keyword arguments along
as parameters to the underlying service operation, which are
typically used to filter the results.
This method returns an iterable generator which yields
individual resource instances. Example use::
# Iterate through items
>>> for queue in sqs.queues.filter(Param='foo'):
... print(queue.url)
'https://url1'
'https://url2'
# Convert to list
>>> queues = list(sqs.queues.filter(Param='foo'))
>>> len(queues)
2
:rtype: :py:class:`ResourceCollection`
"""
return self._clone(**kwargs)
[docs] def limit(self, count):
"""
Return at most this many resources.
>>> for bucket in s3.buckets.limit(5):
... print(bucket.name)
'bucket1'
'bucket2'
'bucket3'
'bucket4'
'bucket5'
:type count: int
:param count: Return no more than this many items
:rtype: :py:class:`ResourceCollection`
"""
return self._clone(limit=count)
[docs] def page_size(self, count):
"""
Fetch at most this many resources per service request.
>>> for obj in s3.Bucket('boto3').objects.page_size(100):
... print(obj.key)
:type count: int
:param count: Fetch this many items per request
:rtype: :py:class:`ResourceCollection`
"""
return self._clone(page_size=count)
[docs]class CollectionManager:
"""
A collection manager provides access to resource collection instances,
which can be iterated and filtered. The manager exposes some
convenience functions that are also found on resource collections,
such as :py:meth:`~ResourceCollection.all` and
:py:meth:`~ResourceCollection.filter`.
Get all items::
>>> for bucket in s3.buckets.all():
... print(bucket.name)
Get only some items via filtering::
>>> for queue in sqs.queues.filter(QueueNamePrefix='AWS'):
... print(queue.url)
Get whole pages of items:
>>> for page in s3.Bucket('boto3').objects.pages():
... for obj in page:
... print(obj.key)
A collection manager is not iterable. You **must** call one of the
methods that return a :py:class:`ResourceCollection` before trying
to iterate, slice, or convert to a list.
See the :ref:`guide_collections` guide for a high-level overview
of collections, including when remote service requests are performed.
:type collection_model: :py:class:`~boto3.resources.model.Collection`
:param model: Collection model
:type parent: :py:class:`~boto3.resources.base.ServiceResource`
:param parent: The collection's parent resource
:type factory: :py:class:`~boto3.resources.factory.ResourceFactory`
:param factory: The resource factory to create new resources
:type service_context: :py:class:`~boto3.utils.ServiceContext`
:param service_context: Context about the AWS service
"""
# The class to use when creating an iterator
_collection_cls = ResourceCollection
def __init__(self, collection_model, parent, factory, service_context):
self._model = collection_model
operation_name = self._model.request.operation
self._parent = parent
search_path = collection_model.resource.path
self._handler = ResourceHandler(
search_path=search_path,
factory=factory,
resource_model=collection_model.resource,
service_context=service_context,
operation_name=operation_name,
)
def __repr__(self):
return '{}({}, {})'.format(
self.__class__.__name__,
self._parent,
f'{self._parent.meta.service_name}.{self._model.resource.type}',
)
[docs] def iterator(self, **kwargs):
"""
Get a resource collection iterator from this manager.
:rtype: :py:class:`ResourceCollection`
:return: An iterable representing the collection of resources
"""
return self._collection_cls(
self._model, self._parent, self._handler, **kwargs
)
# Set up some methods to proxy ResourceCollection methods
[docs] def all(self):
return self.iterator()
all.__doc__ = ResourceCollection.all.__doc__
[docs] def filter(self, **kwargs):
return self.iterator(**kwargs)
filter.__doc__ = ResourceCollection.filter.__doc__
[docs] def limit(self, count):
return self.iterator(limit=count)
limit.__doc__ = ResourceCollection.limit.__doc__
[docs] def page_size(self, count):
return self.iterator(page_size=count)
page_size.__doc__ = ResourceCollection.page_size.__doc__
[docs] def pages(self):
return self.iterator().pages()
pages.__doc__ = ResourceCollection.pages.__doc__
[docs]class CollectionFactory:
"""
A factory to create new
:py:class:`CollectionManager` and :py:class:`ResourceCollection`
subclasses from a :py:class:`~boto3.resources.model.Collection`
model. These subclasses include methods to perform batch operations.
"""
[docs] def load_from_definition(
self, resource_name, collection_model, service_context, event_emitter
):
"""
Loads a collection from a model, creating a new
:py:class:`CollectionManager` subclass
with the correct properties and methods, named based on the service
and resource name, e.g. ec2.InstanceCollectionManager. It also
creates a new :py:class:`ResourceCollection` subclass which is used
by the new manager class.
:type resource_name: string
:param resource_name: Name of the resource to look up. For services,
this should match the ``service_name``.
:type service_context: :py:class:`~boto3.utils.ServiceContext`
:param service_context: Context about the AWS service
:type event_emitter: :py:class:`~botocore.hooks.HierarchialEmitter`
:param event_emitter: An event emitter
:rtype: Subclass of :py:class:`CollectionManager`
:return: The collection class.
"""
attrs = {}
collection_name = collection_model.name
# Create the batch actions for a collection
self._load_batch_actions(
attrs,
resource_name,
collection_model,
service_context.service_model,
event_emitter,
)
# Add the documentation to the collection class's methods
self._load_documented_collection_methods(
attrs=attrs,
resource_name=resource_name,
collection_model=collection_model,
service_model=service_context.service_model,
event_emitter=event_emitter,
base_class=ResourceCollection,
)
if service_context.service_name == resource_name:
cls_name = (
f'{service_context.service_name}.{collection_name}Collection'
)
else:
cls_name = f'{service_context.service_name}.{resource_name}.{collection_name}Collection'
collection_cls = type(str(cls_name), (ResourceCollection,), attrs)
# Add the documentation to the collection manager's methods
self._load_documented_collection_methods(
attrs=attrs,
resource_name=resource_name,
collection_model=collection_model,
service_model=service_context.service_model,
event_emitter=event_emitter,
base_class=CollectionManager,
)
attrs['_collection_cls'] = collection_cls
cls_name += 'Manager'
return type(str(cls_name), (CollectionManager,), attrs)
def _load_batch_actions(
self,
attrs,
resource_name,
collection_model,
service_model,
event_emitter,
):
"""
Batch actions on the collection become methods on both
the collection manager and iterators.
"""
for action_model in collection_model.batch_actions:
snake_cased = xform_name(action_model.name)
attrs[snake_cased] = self._create_batch_action(
resource_name,
snake_cased,
action_model,
collection_model,
service_model,
event_emitter,
)
def _load_documented_collection_methods(
factory_self,
attrs,
resource_name,
collection_model,
service_model,
event_emitter,
base_class,
):
# The base class already has these methods defined. However
# the docstrings are generic and not based for a particular service
# or resource. So we override these methods by proxying to the
# base class's builtin method and adding a docstring
# that pertains to the resource.
# A collection's all() method.
def all(self):
return base_class.all(self)
all.__doc__ = docstring.CollectionMethodDocstring(
resource_name=resource_name,
action_name='all',
event_emitter=event_emitter,
collection_model=collection_model,
service_model=service_model,
include_signature=False,
)
attrs['all'] = all
# The collection's filter() method.
def filter(self, **kwargs):
return base_class.filter(self, **kwargs)
filter.__doc__ = docstring.CollectionMethodDocstring(
resource_name=resource_name,
action_name='filter',
event_emitter=event_emitter,
collection_model=collection_model,
service_model=service_model,
include_signature=False,
)
attrs['filter'] = filter
# The collection's limit method.
def limit(self, count):
return base_class.limit(self, count)
limit.__doc__ = docstring.CollectionMethodDocstring(
resource_name=resource_name,
action_name='limit',
event_emitter=event_emitter,
collection_model=collection_model,
service_model=service_model,
include_signature=False,
)
attrs['limit'] = limit
# The collection's page_size method.
def page_size(self, count):
return base_class.page_size(self, count)
page_size.__doc__ = docstring.CollectionMethodDocstring(
resource_name=resource_name,
action_name='page_size',
event_emitter=event_emitter,
collection_model=collection_model,
service_model=service_model,
include_signature=False,
)
attrs['page_size'] = page_size
def _create_batch_action(
factory_self,
resource_name,
snake_cased,
action_model,
collection_model,
service_model,
event_emitter,
):
"""
Creates a new method which makes a batch operation request
to the underlying service API.
"""
action = BatchAction(action_model)
def batch_action(self, *args, **kwargs):
return action(self, *args, **kwargs)
batch_action.__name__ = str(snake_cased)
batch_action.__doc__ = docstring.BatchActionDocstring(
resource_name=resource_name,
event_emitter=event_emitter,
batch_action_model=action_model,
service_model=service_model,
collection_model=collection_model,
include_signature=False,
)
return batch_action