diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 06c3cdc2..ca92d494 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,25 +1,25 @@ repos: - repo: 'https://github.com/pre-commit/pre-commit-hooks' - rev: v4.4.0 + rev: v4.5.0 hooks: - id: check-yaml - id: end-of-file-fixer - id: trailing-whitespace - - repo: 'https://github.com/PyCQA/isort' - rev: 5.12.0 - hooks: - - id: isort - - repo: 'https://github.com/pycqa/flake8' - rev: 6.0.0 - hooks: - - id: flake8 - repo: 'https://github.com/asottile/pyupgrade' - rev: v3.3.2 + rev: v3.15.0 hooks: - id: pyupgrade args: - '--py36-plus' + - repo: 'https://github.com/PyCQA/isort' + rev: 5.12.0 + hooks: + - id: isort - repo: 'https://github.com/psf/black' - rev: 23.3.0 + rev: 23.11.0 hooks: - id: black + - repo: 'https://github.com/pycqa/flake8' + rev: 6.1.0 + hooks: + - id: flake8 diff --git a/CHANGES.rst b/CHANGES.rst index 81f1347a..b5e9c4f9 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,10 @@ Changes ------- +2.9.0 (2023-12-12) +^^^^^^^^^^^^^^^^^^ +* bump botocore dependency specification + 2.8.0 (2023-11-28) ^^^^^^^^^^^^^^^^^^ * add AioStubber that returns AioAWSResponse() diff --git a/aiobotocore/__init__.py b/aiobotocore/__init__.py index f2df444a..387cfacc 100644 --- a/aiobotocore/__init__.py +++ b/aiobotocore/__init__.py @@ -1 +1 @@ -__version__ = '2.8.0' +__version__ = '2.9.0' diff --git a/aiobotocore/client.py b/aiobotocore/client.py index cb47e319..f284e55c 100644 --- a/aiobotocore/client.py +++ b/aiobotocore/client.py @@ -17,11 +17,12 @@ from . import waiter from .args import AioClientArgsCreator +from .credentials import AioRefreshableCredentials from .discovery import AioEndpointDiscoveryHandler, AioEndpointDiscoveryManager from .httpchecksum import apply_request_checksum from .paginate import AioPaginator from .retries import adaptive, standard -from .utils import AioS3RegionRedirectorv2 +from .utils import AioS3ExpressIdentityResolver, AioS3RegionRedirectorv2 history_recorder = get_global_history_recorder() @@ -96,6 +97,7 @@ async def create_client( client_config=client_config, scoped_config=scoped_config, ) + self._register_s3express_events(client=service_client) self._register_s3_control_events(client=service_client) self._register_endpoint_discovery( service_client, endpoint_url, client_config @@ -223,6 +225,20 @@ def _register_endpoint_discovery(self, client, endpoint_url, config): block_endpoint_discovery_required_operations, ) + def _register_s3express_events( + self, + client, + endpoint_bridge=None, + endpoint_url=None, + client_config=None, + scoped_config=None, + ): + if client.meta.service_model.service_name != 's3': + return + AioS3ExpressIdentityResolver( + client, AioRefreshableCredentials + ).register() + def _register_s3_events( self, client, @@ -331,11 +347,17 @@ async def _make_api_call(self, operation_name, api_params): operation_model=operation_model, context=request_context, ) - # fmt: off - endpoint_url, additional_headers = await self._resolve_endpoint_ruleset( + ( + endpoint_url, + additional_headers, + properties, + ) = await self._resolve_endpoint_ruleset( operation_model, api_params, request_context ) - # fmt: on + if properties: + # Pass arbitrary endpoint info with the Request + # for use during construction. + request_context['endpoint_properties'] = properties request_dict = await self._convert_to_request_dict( api_params=api_params, operation_model=operation_model, @@ -482,6 +504,7 @@ async def _resolve_endpoint_ruleset( if self._ruleset_resolver is None: endpoint_url = self.meta.endpoint_url additional_headers = {} + endpoint_properties = {} else: endpoint_info = await self._ruleset_resolver.construct_endpoint( operation_model=operation_model, @@ -490,6 +513,7 @@ async def _resolve_endpoint_ruleset( ) endpoint_url = endpoint_info.url additional_headers = endpoint_info.headers + endpoint_properties = endpoint_info.properties # If authSchemes is present, overwrite default auth type and # signing context derived from service model. auth_schemes = endpoint_info.properties.get('authSchemes') @@ -506,7 +530,7 @@ async def _resolve_endpoint_ruleset( else: request_context['signing'] = signing_context - return endpoint_url, additional_headers + return endpoint_url, additional_headers, endpoint_properties def get_paginator(self, operation_name): """Create a paginator for an operation. diff --git a/aiobotocore/signers.py b/aiobotocore/signers.py index bebbf2b8..ac2570bd 100644 --- a/aiobotocore/signers.py +++ b/aiobotocore/signers.py @@ -69,6 +69,12 @@ async def sign( kwargs['region_name'] = signing_context['region'] if signing_context.get('signing_name'): kwargs['signing_name'] = signing_context['signing_name'] + if signing_context.get('identity_cache') is not None: + self._resolve_identity_cache( + kwargs, + signing_context['identity_cache'], + signing_context['cache_key'], + ) try: auth = await self.get_auth_instance(**kwargs) except UnknownSignatureVersionError as e: @@ -141,11 +147,16 @@ async def get_auth_instance( auth = cls(frozen_token) return auth + credentials = self._credentials + if getattr(cls, "REQUIRES_IDENTITY_CACHE", None) is True: + cache = kwargs["identity_cache"] + key = kwargs["cache_key"] + credentials = await cache.get_credentials(key) + del kwargs["cache_key"] + frozen_credentials = None - if self._credentials is not None: - frozen_credentials = ( - await self._credentials.get_frozen_credentials() - ) + if credentials is not None: + frozen_credentials = await credentials.get_frozen_credentials() kwargs['credentials'] = frozen_credentials if cls.REQUIRES_REGION: if self._region_name is None: @@ -331,7 +342,11 @@ async def generate_presigned_url( context=context, ) bucket_is_arn = ArnParser.is_arn(params.get('Bucket', '')) - endpoint_url, additional_headers = await self._resolve_endpoint_ruleset( + ( + endpoint_url, + additional_headers, + properties, + ) = await self._resolve_endpoint_ruleset( operation_model, params, context, @@ -396,7 +411,11 @@ async def generate_presigned_post( context=context, ) bucket_is_arn = ArnParser.is_arn(params.get('Bucket', '')) - endpoint_url, additional_headers = await self._resolve_endpoint_ruleset( + ( + endpoint_url, + additional_headers, + properties, + ) = await self._resolve_endpoint_ruleset( operation_model, params, context, diff --git a/aiobotocore/utils.py b/aiobotocore/utils.py index 4be32445..3163bafd 100644 --- a/aiobotocore/utils.py +++ b/aiobotocore/utils.py @@ -1,4 +1,5 @@ import asyncio +import functools import inspect import json import logging @@ -17,11 +18,14 @@ ClientError, ContainerMetadataFetcher, HTTPClientError, + IdentityCache, IMDSFetcher, IMDSRegionProvider, InstanceMetadataFetcher, InstanceMetadataRegionFetcher, ReadTimeoutError, + S3ExpressIdentityCache, + S3ExpressIdentityResolver, S3RegionRedirector, S3RegionRedirectorv2, get_environ_proxies, @@ -348,6 +352,72 @@ async def _get_region(self): return region +class AioIdentityCache(IdentityCache): + async def get_credentials(self, **kwargs): + callback = self.build_refresh_callback(**kwargs) + metadata = await callback() + credential_entry = self._credential_cls.create_from_metadata( + metadata=metadata, + refresh_using=callback, + method=self.METHOD, + advisory_timeout=45, + mandatory_timeout=10, + ) + return credential_entry + + +class AioS3ExpressIdentityCache(AioIdentityCache, S3ExpressIdentityCache): + @functools.cached_property + def _aio_credential_cache(self): + """Substitutes upstream credential cache.""" + return {} + + async def get_credentials(self, bucket): + # upstream uses `@functools.lru_cache(maxsize=100)` to cache credentials. + # This is incompatible with async code. + # We need to implement custom caching logic. + + if (credentials := self._aio_credential_cache.get(bucket)) is None: + # cache miss -> get credentials asynchronously + credentials = await super().get_credentials(bucket=bucket) + + # upstream cache is bounded at 100 entries + if len(self._aio_credential_cache) >= 100: + # drop oldest entry from cache (deviates from lru_cache logic) + self._aio_credential_cache.pop( + next(iter(self._aio_credential_cache)), + ) + + self._aio_credential_cache[bucket] = credentials + + return credentials + + def build_refresh_callback(self, bucket): + async def refresher(): + response = await self._client.create_session(Bucket=bucket) + creds = response['Credentials'] + expiration = self._serialize_if_needed( + creds['Expiration'], iso=True + ) + return { + "access_key": creds['AccessKeyId'], + "secret_key": creds['SecretAccessKey'], + "token": creds['SessionToken'], + "expiry_time": expiration, + } + + return refresher + + +class AioS3ExpressIdentityResolver(S3ExpressIdentityResolver): + def __init__(self, client, credential_cls, cache=None): + super().__init__(client, credential_cls, cache) + + if cache is None: + cache = AioS3ExpressIdentityCache(self._client, credential_cls) + self._cache = cache + + class AioS3RegionRedirectorv2(S3RegionRedirectorv2): async def redirect_from_error( self, diff --git a/setup.py b/setup.py index a275c8cf..ac9e9595 100644 --- a/setup.py +++ b/setup.py @@ -7,15 +7,15 @@ # NOTE: When updating botocore make sure to update awscli/boto3 versions below install_requires = [ # pegged to also match items in `extras_require` - 'botocore>=1.32.4,<1.33.2', + 'botocore>=1.33.2,<1.33.14', 'aiohttp>=3.7.4.post0,<4.0.0', 'wrapt>=1.10.10, <2.0.0', 'aioitertools>=0.5.1,<1.0.0', ] extras_require = { - 'awscli': ['awscli>=1.30.4,<1.31.2'], - 'boto3': ['boto3>=1.29.4,<1.33.2'], + 'awscli': ['awscli>=1.31.2,<1.31.14'], + 'boto3': ['boto3>=1.33.2,<1.33.14'], } diff --git a/tests/test_patches.py b/tests/test_patches.py index bc626096..4d6e6943 100644 --- a/tests/test_patches.py +++ b/tests/test_patches.py @@ -91,10 +91,13 @@ ) from botocore.utils import ( ContainerMetadataFetcher, + IdentityCache, IMDSFetcher, IMDSRegionProvider, InstanceMetadataFetcher, InstanceMetadataRegionFetcher, + S3ExpressIdentityCache, + S3ExpressIdentityResolver, S3RegionRedirector, S3RegionRedirectorv2, ) @@ -140,7 +143,9 @@ '0f80192233321ae4a55d95b68f5b8a68f3ad18e6', }, # client.py - ClientCreator.create_client: {'ef5bef8f4b2887143165e72554fd85c36af7e822'}, + ClientCreator.create_client: { + 'eeb7c4730ac86aec37de53b2be0779490b05f50b', + }, ClientCreator._create_client_class: { 'fcecaf8d4f2c1ac3c5d0eb50c573233ef86d641d' }, @@ -150,6 +155,9 @@ ClientCreator._get_client_args: { 'd5e19b1e62f64a745de842963c2472825a66e854' }, + ClientCreator._register_s3express_events: { + '716c1549989eef6bbd048bf4f134c1b4659e124a', + }, ClientCreator._register_s3_events: { '5659a5312caeb3ea97d663d837d6d201f08824f2' }, @@ -166,7 +174,7 @@ '000b2f2a122602e2e741ec2e89308dc2e2b67329' }, BaseClient._make_api_call: { - '1ac2e166cc8e5020224a808d2ccdfda18e6bdbf2', + '2cb11088d36a89cf9f5c41508bce908acbde24c4', }, BaseClient._make_request: {'cfd8bbf19ea132134717cdf9c460694ddacdbf58'}, BaseClient._convert_to_request_dict: { @@ -174,7 +182,7 @@ }, BaseClient._emit_api_params: {'abd67874dae8d5cd2788412e4699398cb915a119'}, BaseClient._resolve_endpoint_ruleset: { - '3206a73ae79601c42f8a5ae1d7e0e903a2495acb', + 'f09731451ff6ba0645dc82e5c7948dfbf781e025', }, BaseClient.get_paginator: {'3531d5988aaaf0fbb3885044ccee1a693ec2608b'}, BaseClient.get_waiter: {'44f0473d993d49ac7502984a7ccee3240b088404'}, @@ -190,7 +198,7 @@ 'eb247f2884aee311bdabba3435e749c3b8589100' }, RefreshableCredentials.__init__: { - '1a6b83fc845f05feab117ce4fab73b13baed6e3b' + '25ee814f47e5ce617f57e893ae158e5fd6d358ea', }, # We've overridden some properties RefreshableCredentials.__dict__['access_key'].fset: { @@ -433,22 +441,28 @@ }, # signers.py RequestSigner.handler: {'371909df136a0964ef7469a63d25149176c2b442'}, - RequestSigner.sign: {'d90346d5e066e89cd902c5c936f59b644ecde275'}, + RequestSigner.sign: { + '8b6ca96055e5546a6572ad790d5af74a23bc0b52', + }, RequestSigner.get_auth: {'4f8099bef30f9a72fa3bcaa1bd3d22c4fbd224a8'}, RequestSigner.get_auth_instance: { - '4f9be5feafd6c08ffd7bb8de3c9bc36bc02cbfc8' + 'dcd41ea686506dcf056d8252ccf73acd501efd2b', }, RequestSigner._choose_signer: {'bd0e9784029b8aa182b5aec73910d94cb67c36b0'}, RequestSigner.generate_presigned_url: { '417682868eacc10bf4c65f3dfbdba7d20d9250db' }, add_generate_presigned_url: {'5820f74ac46b004eb79e00eea1adc467bcf4defe'}, - generate_presigned_url: {'48f6745f8a37cfba04b3b2f6fb3910210b4a7201'}, + generate_presigned_url: { + 'd03631d6810e2453b8874bc76619927b694a4207', + }, S3PostPresigner.generate_presigned_post: { '269efc9af054a2fd2728d5b0a27db82c48053d7f' }, add_generate_presigned_post: {'e30360f2bd893fabf47f5cdb04b0de420ccd414d'}, - generate_presigned_post: {'eedf40b48c63f6772ed05e3f335c8193d187f503'}, + generate_presigned_post: { + 'a3a834a08be2cf76c20ea137ba6b28e7a12f58ed', + }, add_generate_db_auth_token: {'f61014e6fac4b5c7ee7ac2d2bec15fb16fa9fbe5'}, generate_db_auth_token: {'1f37e1e5982d8528841ce6b79f229b3e23a18959'}, # tokens.py @@ -549,6 +563,18 @@ IMDSRegionProvider._create_fetcher: { '18da52c786a20d91615258a8127b566688ecbb39', }, + IdentityCache.get_credentials: { + 'baf98c4caaddfa0594745eb490c327c65cff8920', + }, + S3ExpressIdentityCache.get_credentials.__wrapped__: { + '71f2ae5e0ea32e9bbac6f318cba963700e23b9a0', + }, + S3ExpressIdentityCache.build_refresh_callback: { + '0e833cc5e30b76fa13e8caf5c024fe2a21c10f22', + }, + S3ExpressIdentityResolver.__init__: { + '148a10274d3268dd42df05d3bcfb98c668f01086', + }, # waiter.py NormalizedOperationMethod.__call__: { '79723632d023739aa19c8a899bc2b814b8ab12ff'