diff --git a/.changes/next-release/enhancement-endpoints-85370.json b/.changes/next-release/enhancement-endpoints-85370.json new file mode 100644 index 000000000000..27757334b857 --- /dev/null +++ b/.changes/next-release/enhancement-endpoints-85370.json @@ -0,0 +1,5 @@ +{ + "type": "enhancement", + "category": "endpoints", + "description": "Add support for ``stringArray`` parameters and the ``operationContextParams`` trait when resolving service endpoints." +} diff --git a/awscli/botocore/endpoint_provider.py b/awscli/botocore/endpoint_provider.py index 83a7b1291b20..cf1d8d7288ac 100644 --- a/awscli/botocore/endpoint_provider.py +++ b/awscli/botocore/endpoint_provider.py @@ -24,7 +24,6 @@ import logging import re from enum import Enum -from functools import lru_cache from string import Formatter from typing import NamedTuple @@ -36,6 +35,7 @@ InvalidArnException, is_valid_ipv4_endpoint_url, is_valid_ipv6_endpoint_url, + lru_cache_weakref, normalize_url_path, percent_encode, ) @@ -43,7 +43,7 @@ logger = logging.getLogger(__name__) TEMPLATE_STRING_RE = re.compile(r"\{[a-zA-Z#]+\}") -GET_ATTR_RE = re.compile(r"(\w+)\[(\d+)\]") +GET_ATTR_RE = re.compile(r"(\w*)\[(\d+)\]") VALID_HOST_LABEL_RE = re.compile( r"^(?!-)[a-zA-Z\d-]{1,63}(?= len(value): return None return value[index] @@ -577,6 +578,7 @@ class ParameterType(Enum): string = str boolean = bool + stringarray = tuple class ParameterDefinition: @@ -600,7 +602,7 @@ def __init__( except AttributeError: raise EndpointResolutionError( msg=f"Unknown parameter type: {parameter_type}. " - "A parameter must be of type string or boolean." + "A parameter must be of type string, boolean, or stringarray." ) self.documentation = documentation self.builtin = builtIn @@ -703,7 +705,7 @@ class EndpointProvider: def __init__(self, ruleset_data, partition_data): self.ruleset = RuleSet(**ruleset_data, partitions=partition_data) - @lru_cache(maxsize=CACHE_SIZE) + @lru_cache_weakref(maxsize=CACHE_SIZE) def resolve_endpoint(self, **input_parameters): """Match input parameters to a rule. diff --git a/awscli/botocore/model.py b/awscli/botocore/model.py index 0a90ca2d80ff..29e434a253fd 100644 --- a/awscli/botocore/model.py +++ b/awscli/botocore/model.py @@ -588,6 +588,10 @@ def context_parameters(self): and 'name' in shape.metadata['contextParam'] ] + @CachedProperty + def operation_context_parameters(self): + return self._operation_model.get('operationContextParams', []) + @CachedProperty def request_compression(self): return self._operation_model.get('requestcompression') diff --git a/awscli/botocore/regions.py b/awscli/botocore/regions.py index ead1c3ce8418..1e17bdeddf91 100644 --- a/awscli/botocore/regions.py +++ b/awscli/botocore/regions.py @@ -21,6 +21,8 @@ import re from enum import Enum +import jmespath + from botocore import UNSIGNED, xform_name from botocore.auth import AUTH_TYPE_MAPS from botocore.endpoint_provider import EndpointProvider @@ -530,6 +532,13 @@ def _resolve_param_from_context( ) if dynamic is not None: return dynamic + operation_context_params = ( + self._resolve_param_as_operation_context_param( + param_name, operation_model, call_args + ) + ) + if operation_context_params is not None: + return operation_context_params return self._resolve_param_as_client_context_param(param_name) def _resolve_param_as_static_context_param( @@ -552,6 +561,14 @@ def _resolve_param_as_client_context_param(self, param_name): client_ctx_varname = client_ctx_params[param_name] return self._client_context.get(client_ctx_varname) + def _resolve_param_as_operation_context_param( + self, param_name, operation_model, call_args + ): + operation_ctx_params = operation_model.operation_context_parameters + if param_name in operation_ctx_params: + path = operation_ctx_params[param_name]['path'] + return jmespath.search(path, call_args) + def _resolve_param_as_builtin(self, builtin_name, builtins): if builtin_name not in EndpointResolverBuiltins.__members__.values(): raise UnknownEndpointResolutionBuiltInName(name=builtin_name) diff --git a/awscli/botocore/utils.py b/awscli/botocore/utils.py index 13eb55b4d4d3..1d13a7d98b53 100644 --- a/awscli/botocore/utils.py +++ b/awscli/botocore/utils.py @@ -1263,6 +1263,35 @@ def _cache_guard(self, *args, **kwargs): return _cache_guard +def lru_cache_weakref(*cache_args, **cache_kwargs): + """ + Version of functools.lru_cache that stores a weak reference to ``self``. + Serves the same purpose as :py:func:`instance_cache` but uses Python's + functools implementation which offers ``max_size`` and ``typed`` properties. + lru_cache is a global cache even when used on a method. The cache's + reference to ``self`` will prevent garbage collection of the object. This + wrapper around functools.lru_cache replaces the reference to ``self`` with + a weak reference to not interfere with garbage collection. + """ + + def wrapper(func): + @functools.lru_cache(*cache_args, **cache_kwargs) + def func_with_weakref(weakref_to_self, *args, **kwargs): + return func(weakref_to_self(), *args, **kwargs) + + @functools.wraps(func) + def inner(self, *args, **kwargs): + for kwarg_key, kwarg_value in kwargs.items(): + if isinstance(kwarg_value, list): + kwargs[kwarg_key] = tuple(kwarg_value) + return func_with_weakref(weakref.ref(self), *args, **kwargs) + + inner.cache_info = func_with_weakref.cache_info + return inner + + return wrapper + + def switch_host_s3_accelerate(request, operation_name, **kwargs): """Switches the current s3 endpoint with an S3 Accelerate endpoint""" diff --git a/tests/unit/botocore/data/endpoints/test-cases/array-index.json b/tests/unit/botocore/data/endpoints/test-cases/array-index.json new file mode 100644 index 000000000000..aa73d460d33e --- /dev/null +++ b/tests/unit/botocore/data/endpoints/test-cases/array-index.json @@ -0,0 +1,37 @@ +{ + "version": "1.0", + "testCases": [ + { + "documentation": "Access an array index at index 0", + "params": { + "ResourceList": ["resource"] + }, + "expect": { + "endpoint": { + "url": "https://www.resource.example.com" + } + } + }, + { + "documentation": "Resolved value when array is explictly set to empty", + "params": { + "ResourceList": [] + }, + "expect": { + "endpoint": { + "url": "https://www.example.com" + } + } + }, + { + "documentation": "Resolved value to default if array is unset", + "params": { + }, + "expect": { + "endpoint": { + "url": "https://www.default1.example.com" + } + } + } + ] +} diff --git a/tests/unit/botocore/data/endpoints/valid-rules/array-index.json b/tests/unit/botocore/data/endpoints/valid-rules/array-index.json new file mode 100644 index 000000000000..3d426b09aa64 --- /dev/null +++ b/tests/unit/botocore/data/endpoints/valid-rules/array-index.json @@ -0,0 +1,47 @@ +{ + "version": "1.3", + "parameters": { + "ResourceList": { + "required": true, + "default": ["default1", "default2"], + "type": "stringArray" + } + }, + "rules": [ + { + "documentation": "Array is set, retrieve index 0", + "conditions": [ + { + "fn": "isSet", + "argv": [ + { + "ref": "ResourceList" + } + ] + }, + { + "fn": "getAttr", + "argv": [ + { + "ref": "ResourceList" + }, + "[0]" + ], + "assign": "resourceid" + } + ], + "endpoint": { + "url": "https://www.{resourceid}.example.com" + }, + "type": "endpoint" + }, + { + "documentation": "Fallback when array is unset", + "conditions": [], + "endpoint": { + "url": "https://www.example.com" + }, + "type": "endpoint" + } + ] +} diff --git a/tests/unit/botocore/test_endpoint_provider.py b/tests/unit/botocore/test_endpoint_provider.py index 2c1cdbb88a77..4137e86ead12 100644 --- a/tests/unit/botocore/test_endpoint_provider.py +++ b/tests/unit/botocore/test_endpoint_provider.py @@ -134,6 +134,7 @@ def endpoint_rule(): def ruleset_testcases(): filenames = [ + "array-index", "aws-region", "default-values", "eventbridge", @@ -457,3 +458,33 @@ def test_auth_schemes_conversion_first_authtype_unknown( at, sc = empty_resolver.auth_schemes_to_signing_ctx(auth_schemes) assert at == 'bar' assert sc == {'region': 'ap-south-2', 'signing_name': 's3'} + + +@pytest.mark.parametrize( + "value, path, expected_value", + [ + ({"foo": ['bar']}, 'baz[0]', None), # Missing index + ({"foo": ['bar']}, 'foo[1]', None), # Out of range index + ({"foo": ['bar']}, 'foo[0]', "bar"), # Named index + (("foo",), '[0]', "foo"), # Bare index + ({"foo": {}}, 'foo.bar[0]', None), # Missing index from split path + ( + {"foo": {'bar': []}}, + 'foo.bar[0]', + None, + ), # Out of range from split path + ( + {"foo": {"bar": "baz"}}, + 'foo.bar', + "baz", + ), # Split path with named index + ( + {"foo": {"bar": ["baz"]}}, + 'foo.bar[0]', + "baz", + ), # Split path with numeric index + ], +) +def test_get_attr(rule_lib, value, path, expected_value): + result = rule_lib.get_attr(value, path) + assert result == expected_value diff --git a/tests/unit/botocore/test_utils.py b/tests/unit/botocore/test_utils.py index 7c16e520b7e1..e457ea01f20d 100644 --- a/tests/unit/botocore/test_utils.py +++ b/tests/unit/botocore/test_utils.py @@ -24,6 +24,7 @@ import datetime import copy import operator +from sys import getrefcount import botocore from botocore import xform_name @@ -70,6 +71,7 @@ from botocore.utils import instance_cache from botocore.utils import merge_dicts from botocore.utils import lowercase_dict +from botocore.utils import lru_cache_weakref from botocore.utils import get_service_module_name from botocore.utils import get_encoding_from_headers from botocore.utils import percent_encode_sequence @@ -3619,3 +3621,40 @@ def test_is_s3_accelerate_url(url, expected): def test_get_encoding_from_headers(headers, default, expected): charset = get_encoding_from_headers(HeadersDict(headers), default=default) assert charset == expected + + +def test_lru_cache_weakref(): + class ClassWithCachedMethod: + @lru_cache_weakref(maxsize=10) + def cached_fn(self, a, b): + return a + b + + cls1 = ClassWithCachedMethod() + cls2 = ClassWithCachedMethod() + + assert cls1.cached_fn.cache_info().currsize == 0 + assert getrefcount(cls1) == 2 + assert getrefcount(cls2) == 2 + # "The count returned is generally one higher than you might expect, because + # it includes the (temporary) reference as an argument to getrefcount()." + # https://docs.python.org/3.8/library/sys.html#getrefcount + + cls1.cached_fn(1, 1) + cls2.cached_fn(1, 1) + + # The cache now has two entries, but the reference count remains the same as + # before. + assert cls1.cached_fn.cache_info().currsize == 2 + assert getrefcount(cls1) == 2 + assert getrefcount(cls2) == 2 + + # Deleting one of the objects does not interfere with the cache entries + # related to the other object. + del cls1 + assert cls2.cached_fn.cache_info().currsize == 2 + assert cls2.cached_fn.cache_info().hits == 0 + assert cls2.cached_fn.cache_info().misses == 2 + cls2.cached_fn(1, 1) + assert cls2.cached_fn.cache_info().currsize == 2 + assert cls2.cached_fn.cache_info().hits == 1 # the call was a cache hit + assert cls2.cached_fn.cache_info().misses == 2