From d4619a326382e52c9f0faeb29648df2b9a1489ed Mon Sep 17 00:00:00 2001 From: Steve Yoo <106777148+hssyoo@users.noreply.github.com> Date: Mon, 6 Jan 2025 16:47:28 -0500 Subject: [PATCH] [v2] Expose `--bucket-name-prefix` and `--bucket-region` to `s3 ls` (#9163) --- .../next-release/enhancement-s3ls-20704.json | 5 ++ awscli/customizations/s3/subcommands.py | 40 ++++++++- tests/functional/s3/test_ls_command.py | 20 +++++ .../customizations/s3/test_subcommands.py | 89 ++++++++++++++++--- 4 files changed, 138 insertions(+), 16 deletions(-) create mode 100644 .changes/next-release/enhancement-s3ls-20704.json diff --git a/.changes/next-release/enhancement-s3ls-20704.json b/.changes/next-release/enhancement-s3ls-20704.json new file mode 100644 index 000000000000..a98c6c6bb8a8 --- /dev/null +++ b/.changes/next-release/enhancement-s3ls-20704.json @@ -0,0 +1,5 @@ +{ + "type": "enhancement", + "category": "``s3 ls``", + "description": "Expose low-level ``ListBuckets` parameters ``Prefix`` and ``BucketRegion`` to high-level ``s3 ls`` command as ``--bucket-name-prefix`` and ``--bucket-region``." +} diff --git a/awscli/customizations/s3/subcommands.py b/awscli/customizations/s3/subcommands.py index 1c51238c26f6..5b3c53ca9945 100644 --- a/awscli/customizations/s3/subcommands.py +++ b/awscli/customizations/s3/subcommands.py @@ -492,6 +492,26 @@ 'help_text': 'Indicates the algorithm used to create the checksum for the object.' } +BUCKET_NAME_PREFIX = { + 'name': 'bucket-name-prefix', + 'help_text': ( + 'Limits the response to bucket names that begin with the specified ' + 'bucket name prefix.' + ) +} + +BUCKET_REGION = { + 'name': 'bucket-region', + 'help_text': ( + 'Limits the response to buckets that are located in the specified ' + 'Amazon Web Services Region. The Amazon Web Services Region must be ' + 'expressed according to the Amazon Web Services Region code, such as ' + 'us-west-2 for the US West (Oregon) Region. For a list of the valid ' + 'values for all of the Amazon Web Services Regions, see ' + 'https://docs.aws.amazon.com/general/latest/gr/rande.html#s3_region' + ) +} + TRANSFER_ARGS = [DRYRUN, QUIET, INCLUDE, EXCLUDE, ACL, FOLLOW_SYMLINKS, NO_FOLLOW_SYMLINKS, NO_GUESS_MIME_TYPE, SSE, SSE_C, SSE_C_KEY, SSE_KMS_KEY_ID, SSE_C_COPY_SOURCE, @@ -521,7 +541,8 @@ class ListCommand(S3Command): USAGE = " or NONE" ARG_TABLE = [{'name': 'paths', 'nargs': '?', 'default': 's3://', 'positional_arg': True, 'synopsis': USAGE}, RECURSIVE, - PAGE_SIZE, HUMAN_READABLE, SUMMARIZE, REQUEST_PAYER] + PAGE_SIZE, HUMAN_READABLE, SUMMARIZE, REQUEST_PAYER, + BUCKET_NAME_PREFIX, BUCKET_REGION] def _run_main(self, parsed_args, parsed_globals): super(ListCommand, self)._run_main(parsed_args, parsed_globals) @@ -535,7 +556,11 @@ def _run_main(self, parsed_args, parsed_globals): path = path[5:] bucket, key = find_bucket_key(path) if not bucket: - self._list_all_buckets(parsed_args.page_size) + self._list_all_buckets( + parsed_args.page_size, + parsed_args.bucket_name_prefix, + parsed_args.bucket_region, + ) elif parsed_args.dir_op: # Then --recursive was specified. self._list_all_objects_recursive( @@ -599,11 +624,20 @@ def _display_page(self, response_data, use_basename=True): uni_print(print_str) self._at_first_page = False - def _list_all_buckets(self, page_size=None): + def _list_all_buckets( + self, + page_size=None, + prefix=None, + bucket_region=None, + ): paginator = self.client.get_paginator('list_buckets') paging_args = { 'PaginationConfig': {'PageSize': page_size} } + if prefix: + paging_args['Prefix'] = prefix + if bucket_region: + paging_args['BucketRegion'] = bucket_region iterator = paginator.paginate(**paging_args) diff --git a/tests/functional/s3/test_ls_command.py b/tests/functional/s3/test_ls_command.py index 3fdeffb1cbf4..7bcbaef9607f 100644 --- a/tests/functional/s3/test_ls_command.py +++ b/tests/functional/s3/test_ls_command.py @@ -215,3 +215,23 @@ def test_accesspoint_arn(self): self.run_cmd('s3 ls s3://%s' % arn, expected_rc=0) call_args = self.operations_called[0][1] self.assertEqual(call_args['Bucket'], arn) + + def test_list_buckets_uses_bucket_name_prefix(self): + stdout, _, _ = self.run_cmd('s3 ls --bucket-name-prefix myprefix', expected_rc=0) + call_args = self.operations_called[0][1] + self.assertEqual(call_args['Prefix'], 'myprefix') + + def test_list_buckets_uses_bucket_region(self): + stdout, _, _ = self.run_cmd('s3 ls --bucket-region us-west-1', expected_rc=0) + call_args = self.operations_called[0][1] + self.assertEqual(call_args['BucketRegion'], 'us-west-1') + + def test_list_objects_ignores_bucket_name_prefix(self): + stdout, _, _ = self.run_cmd('s3 ls s3://mybucket --bucket-name-prefix myprefix', expected_rc=0) + call_args = self.operations_called[0][1] + self.assertEqual(call_args['Prefix'], '') + + def test_list_objects_ignores_bucket_region(self): + stdout, _, _ = self.run_cmd('s3 ls s3://mybucket --bucket-region us-west-1', expected_rc=0) + call_args = self.operations_called[0][1] + self.assertNotIn('BucketRegion', call_args) diff --git a/tests/unit/customizations/s3/test_subcommands.py b/tests/unit/customizations/s3/test_subcommands.py index fda78f5d26c6..7c36fef78f3b 100644 --- a/tests/unit/customizations/s3/test_subcommands.py +++ b/tests/unit/customizations/s3/test_subcommands.py @@ -79,11 +79,27 @@ def setUp(self): self.session.create_client.return_value.get_paginator.return_value\ .paginate.return_value = [{'Contents': [], 'CommonPrefixes': []}] + def _get_fake_kwargs(self, override=None): + fake_kwargs = { + 'paths': 's3://', + 'dir_op': False, + 'human_readable': False, + 'summarize': False, + 'page_size': None, + 'request_payer': None, + 'bucket_name_prefix': None, + 'bucket_region': None, + } + fake_kwargs.update(override or {}) + + return fake_kwargs + def test_ls_command_for_bucket(self): ls_command = ListCommand(self.session) - parsed_args = FakeArgs(paths='s3://mybucket/', dir_op=False, - page_size='5', human_readable=False, - summarize=False, request_payer=None) + parsed_args = FakeArgs(**self._get_fake_kwargs({ + 'paths': 's3://mybucket/', + 'page_size': '5', + })) parsed_globals = mock.Mock() ls_command._run_main(parsed_args, parsed_globals) call = self.session.create_client.return_value.list_objects_v2 @@ -104,9 +120,7 @@ def test_ls_command_with_no_args(self): ls_command = ListCommand(self.session) parsed_global = FakeArgs(region=None, endpoint_url=None, verify_ssl=None) - parsed_args = FakeArgs(dir_op=False, paths='s3://', - human_readable=False, summarize=False, - request_payer=None, page_size=None) + parsed_args = FakeArgs(**self._get_fake_kwargs()) ls_command._run_main(parsed_args, parsed_global) call = self.session.create_client.return_value.list_buckets paginate = self.session.create_client.return_value.get_paginator\ @@ -129,14 +143,61 @@ def test_ls_command_with_no_args(self): mock.call('s3', region_name=None, verify=None, endpoint_url=None) ) + def test_ls_with_bucket_name_prefix(self): + ls_command = ListCommand(self.session) + parsed_args = FakeArgs(**self._get_fake_kwargs({ + 'bucket_name_prefix': 'myprefix', + })) + parsed_globals = FakeArgs( + region=None, + endpoint_url=None, + verify_ssl=None, + ) + ls_command._run_main(parsed_args, parsed_globals) + call = self.session.create_client.return_value.list_objects + paginate = self.session.create_client.return_value.get_paginator\ + .return_value.paginate + # We should make no operation calls. + self.assertEqual(call.call_count, 0) + self.session.create_client.return_value.get_paginator.\ + assert_called_with('list_buckets') + ref_call_args = { + 'PaginationConfig': {'PageSize': None}, + 'Prefix': 'myprefix', + } + + paginate.assert_called_with(**ref_call_args) + + def test_ls_with_bucket_region(self): + ls_command = ListCommand(self.session) + parsed_args = FakeArgs(**self._get_fake_kwargs({ + 'bucket_region': 'us-west-1', + })) + parsed_globals = FakeArgs( + region=None, + endpoint_url=None, + verify_ssl=None, + ) + ls_command._run_main(parsed_args, parsed_globals) + call = self.session.create_client.return_value.list_objects + paginate = self.session.create_client.return_value.get_paginator\ + .return_value.paginate + # We should make no operation calls. + self.assertEqual(call.call_count, 0) + self.session.create_client.return_value.get_paginator.\ + assert_called_with('list_buckets') + ref_call_args = { + 'PaginationConfig': {'PageSize': None}, + 'BucketRegion': 'us-west-1', + } + + paginate.assert_called_with(**ref_call_args) + def test_ls_with_verify_argument(self): - options = {'default': 's3://', 'nargs': '?'} ls_command = ListCommand(self.session) parsed_global = FakeArgs(region='us-west-2', endpoint_url=None, verify_ssl=False) - parsed_args = FakeArgs(paths='s3://', dir_op=False, - human_readable=False, summarize=False, - request_payer=None, page_size=None) + parsed_args = FakeArgs(**self._get_fake_kwargs({})) ls_command._run_main(parsed_args, parsed_global) # Verify get_client get_client = self.session.create_client @@ -150,9 +211,11 @@ def test_ls_with_verify_argument(self): def test_ls_with_requester_pays(self): ls_command = ListCommand(self.session) - parsed_args = FakeArgs(paths='s3://mybucket/', dir_op=False, - human_readable=False, summarize=False, - request_payer='requester', page_size='5') + parsed_args = FakeArgs(**self._get_fake_kwargs({ + 'paths': 's3://mybucket/', + 'page_size': '5', + 'request_payer': 'requester', + })) parsed_globals = mock.Mock() ls_command._run_main(parsed_args, parsed_globals) call = self.session.create_client.return_value.list_objects