Add support for stringArray and operationContextParams (#9153)

This allows the CLI's endpoint resolver to consider collections of strings as an input
to endpoint resolution, and use operationContextParams to resolve parameters from the
operation's input.

.changes/next-release/enhancement-endpoints-85370.json

@@ -0,0 +1,5 @@
+{
+  "type": "enhancement",
+  "category": "endpoints",
+  "description": "Add support for ``stringArray`` parameters and the ``operationContextParams`` trait when resolving service endpoints."
+}

awscli/botocore/endpoint_provider.py

@@ -24,7 +24,6 @@
 import logging
 import re
 from enum import Enum
-from functools import lru_cache
 from string import Formatter
 from typing import NamedTuple
 
@@ -36,14 +35,15 @@
     InvalidArnException,
     is_valid_ipv4_endpoint_url,
     is_valid_ipv6_endpoint_url,
+    lru_cache_weakref,
     normalize_url_path,
     percent_encode,
 )
 
 logger = logging.getLogger(__name__)
 
 TEMPLATE_STRING_RE = re.compile(r"\{[a-zA-Z#]+\}")
-GET_ATTR_RE = re.compile(r"(\w+)\[(\d+)\]")
+GET_ATTR_RE = re.compile(r"(\w*)\[(\d+)\]")
 VALID_HOST_LABEL_RE = re.compile(
     r"^(?!-)[a-zA-Z\d-]{1,63}(?<!-)$",
 )
@@ -170,7 +170,7 @@ def get_attr(self, value, path):
         names indicates the one to the right is nested. The index will always occur at
         the end of the path.
 
-        :type value: dict or list
+        :type value: dict or tuple
         :type path: str
         :rtype: Any
         """
@@ -179,7 +179,8 @@ def get_attr(self, value, path):
             if match is not None:
                 name, index = match.groups()
                 index = int(index)
-                value = value.get(name)
+                if name:
+                    value = value.get(name)
                 if value is None or index >= len(value):
                     return None
                 return value[index]
@@ -577,6 +578,7 @@ class ParameterType(Enum):
 
     string = str
     boolean = bool
+    stringarray = tuple
 
 
 class ParameterDefinition:
@@ -600,7 +602,7 @@ def __init__(
         except AttributeError:
             raise EndpointResolutionError(
                 msg=f"Unknown parameter type: {parameter_type}. "
-                "A parameter must be of type string or boolean."
+                "A parameter must be of type string, boolean, or stringarray."
             )
         self.documentation = documentation
         self.builtin = builtIn
@@ -703,7 +705,7 @@ class EndpointProvider:
     def __init__(self, ruleset_data, partition_data):
         self.ruleset = RuleSet(**ruleset_data, partitions=partition_data)
 
-    @lru_cache(maxsize=CACHE_SIZE)
+    @lru_cache_weakref(maxsize=CACHE_SIZE)
     def resolve_endpoint(self, **input_parameters):
         """Match input parameters to a rule.
 

awscli/botocore/model.py

@@ -588,6 +588,10 @@ def context_parameters(self):
             and 'name' in shape.metadata['contextParam']
         ]
 
+    @CachedProperty
+    def operation_context_parameters(self):
+        return self._operation_model.get('operationContextParams', [])
+
     @CachedProperty
     def request_compression(self):
         return self._operation_model.get('requestcompression')

awscli/botocore/regions.py

@@ -21,6 +21,8 @@
 import re
 from enum import Enum
 
+import jmespath
+
 from botocore import UNSIGNED, xform_name
 from botocore.auth import AUTH_TYPE_MAPS
 from botocore.endpoint_provider import EndpointProvider
@@ -530,6 +532,13 @@ def _resolve_param_from_context(
         )
         if dynamic is not None:
             return dynamic
+        operation_context_params = (
+            self._resolve_param_as_operation_context_param(
+                param_name, operation_model, call_args
+            )
+        )
+        if operation_context_params is not None:
+            return operation_context_params
         return self._resolve_param_as_client_context_param(param_name)
 
     def _resolve_param_as_static_context_param(
@@ -552,6 +561,14 @@ def _resolve_param_as_client_context_param(self, param_name):
             client_ctx_varname = client_ctx_params[param_name]
             return self._client_context.get(client_ctx_varname)
 
+    def _resolve_param_as_operation_context_param(
+        self, param_name, operation_model, call_args
+    ):
+        operation_ctx_params = operation_model.operation_context_parameters
+        if param_name in operation_ctx_params:
+            path = operation_ctx_params[param_name]['path']
+            return jmespath.search(path, call_args)
+
     def _resolve_param_as_builtin(self, builtin_name, builtins):
         if builtin_name not in EndpointResolverBuiltins.__members__.values():
             raise UnknownEndpointResolutionBuiltInName(name=builtin_name)

awscli/botocore/utils.py

@@ -1263,6 +1263,35 @@ def _cache_guard(self, *args, **kwargs):
     return _cache_guard
 
 
+def lru_cache_weakref(*cache_args, **cache_kwargs):
+    """
+    Version of functools.lru_cache that stores a weak reference to ``self``.
+    Serves the same purpose as :py:func:`instance_cache` but uses Python's
+    functools implementation which offers ``max_size`` and ``typed`` properties.
+    lru_cache is a global cache even when used on a method. The cache's
+    reference to ``self`` will prevent garbage collection of the object. This
+    wrapper around functools.lru_cache replaces the reference to ``self`` with
+    a weak reference to not interfere with garbage collection.
+    """
+
+    def wrapper(func):
+        @functools.lru_cache(*cache_args, **cache_kwargs)
+        def func_with_weakref(weakref_to_self, *args, **kwargs):
+            return func(weakref_to_self(), *args, **kwargs)
+
+        @functools.wraps(func)
+        def inner(self, *args, **kwargs):
+            for kwarg_key, kwarg_value in kwargs.items():
+                if isinstance(kwarg_value, list):
+                    kwargs[kwarg_key] = tuple(kwarg_value)
+            return func_with_weakref(weakref.ref(self), *args, **kwargs)
+
+        inner.cache_info = func_with_weakref.cache_info
+        return inner
+
+    return wrapper
+
+
 def switch_host_s3_accelerate(request, operation_name, **kwargs):
     """Switches the current s3 endpoint with an S3 Accelerate endpoint"""
 

tests/unit/botocore/data/endpoints/test-cases/array-index.json

@@ -0,0 +1,37 @@
+{
+  "version": "1.0",
+  "testCases": [
+    {
+      "documentation": "Access an array index at index 0",
+      "params": {
+        "ResourceList": ["resource"]
+      },
+      "expect": {
+        "endpoint": {
+          "url": "https://www.resource.example.com"
+        }
+      }
+    },
+    {
+      "documentation": "Resolved value when array is explictly set to empty",
+      "params": {
+        "ResourceList": []
+      },
+      "expect": {
+        "endpoint": {
+          "url": "https://www.example.com"
+        }
+      }
+    },
+    {
+      "documentation": "Resolved value to default if array is unset",
+      "params": {
+      },
+      "expect": {
+        "endpoint": {
+          "url": "https://www.default1.example.com"
+        }
+      }
+    }
+  ]
+}

tests/unit/botocore/data/endpoints/valid-rules/array-index.json

@@ -0,0 +1,47 @@
+{
+  "version": "1.3",
+  "parameters": {
+    "ResourceList": {
+      "required": true,
+      "default": ["default1", "default2"],
+      "type": "stringArray"
+    }
+  },
+  "rules": [
+    {
+      "documentation": "Array is set, retrieve index 0",
+      "conditions": [
+        {
+          "fn": "isSet",
+          "argv": [
+            {
+              "ref": "ResourceList"
+            }
+          ]
+        },
+        {
+          "fn": "getAttr",
+          "argv": [
+            {
+              "ref": "ResourceList"
+            },
+            "[0]"
+          ],
+          "assign": "resourceid"
+        }
+      ],
+      "endpoint": {
+        "url": "https://www.{resourceid}.example.com"
+      },
+      "type": "endpoint"
+    },
+    {
+      "documentation": "Fallback when array is unset",
+      "conditions": [],
+      "endpoint": {
+        "url": "https://www.example.com"
+      },
+      "type": "endpoint"
+    }
+  ]
+}

tests/unit/botocore/test_endpoint_provider.py

@@ -134,6 +134,7 @@ def endpoint_rule():
 
 def ruleset_testcases():
     filenames = [
+        "array-index",
         "aws-region",
         "default-values",
         "eventbridge",
@@ -457,3 +458,33 @@ def test_auth_schemes_conversion_first_authtype_unknown(
     at, sc = empty_resolver.auth_schemes_to_signing_ctx(auth_schemes)
     assert at == 'bar'
     assert sc == {'region': 'ap-south-2', 'signing_name': 's3'}
+
+
+@pytest.mark.parametrize(
+    "value, path, expected_value",
+    [
+        ({"foo": ['bar']}, 'baz[0]', None),  # Missing index
+        ({"foo": ['bar']}, 'foo[1]', None),  # Out of range index
+        ({"foo": ['bar']}, 'foo[0]', "bar"),  # Named index
+        (("foo",), '[0]', "foo"),  # Bare index
+        ({"foo": {}}, 'foo.bar[0]', None),  # Missing index from split path
+        (
+            {"foo": {'bar': []}},
+            'foo.bar[0]',
+            None,
+        ),  # Out of range from split path
+        (
+            {"foo": {"bar": "baz"}},
+            'foo.bar',
+            "baz",
+        ),  # Split path with named index
+        (
+            {"foo": {"bar": ["baz"]}},
+            'foo.bar[0]',
+            "baz",
+        ),  # Split path with numeric index
+    ],
+)
+def test_get_attr(rule_lib, value, path, expected_value):
+    result = rule_lib.get_attr(value, path)
+    assert result == expected_value

tests/unit/botocore/test_utils.py

@@ -24,6 +24,7 @@
 import datetime
 import copy
 import operator
+from sys import getrefcount
 
 import botocore
 from botocore import xform_name
@@ -70,6 +71,7 @@
 from botocore.utils import instance_cache
 from botocore.utils import merge_dicts
 from botocore.utils import lowercase_dict
+from botocore.utils import lru_cache_weakref
 from botocore.utils import get_service_module_name
 from botocore.utils import get_encoding_from_headers
 from botocore.utils import percent_encode_sequence
@@ -3619,3 +3621,40 @@ def test_is_s3_accelerate_url(url, expected):
 def test_get_encoding_from_headers(headers, default, expected):
     charset = get_encoding_from_headers(HeadersDict(headers), default=default)
     assert charset == expected
+
+
+def test_lru_cache_weakref():
+    class ClassWithCachedMethod:
+        @lru_cache_weakref(maxsize=10)
+        def cached_fn(self, a, b):
+            return a + b
+
+    cls1 = ClassWithCachedMethod()
+    cls2 = ClassWithCachedMethod()
+
+    assert cls1.cached_fn.cache_info().currsize == 0
+    assert getrefcount(cls1) == 2
+    assert getrefcount(cls2) == 2
+    # "The count returned is generally one higher than you might expect, because
+    # it includes the (temporary) reference as an argument to getrefcount()."
+    # https://docs.python.org/3.8/library/sys.html#getrefcount
+
+    cls1.cached_fn(1, 1)
+    cls2.cached_fn(1, 1)
+
+    # The cache now has two entries, but the reference count remains the same as
+    # before.
+    assert cls1.cached_fn.cache_info().currsize == 2
+    assert getrefcount(cls1) == 2
+    assert getrefcount(cls2) == 2
+
+    # Deleting one of the objects does not interfere with the cache entries
+    # related to the other object.
+    del cls1
+    assert cls2.cached_fn.cache_info().currsize == 2
+    assert cls2.cached_fn.cache_info().hits == 0
+    assert cls2.cached_fn.cache_info().misses == 2
+    cls2.cached_fn(1, 1)
+    assert cls2.cached_fn.cache_info().currsize == 2
+    assert cls2.cached_fn.cache_info().hits == 1  # the call was a cache hit
+    assert cls2.cached_fn.cache_info().misses == 2