Skip to content

Commit

Permalink
[v2] Expose --bucket-name-prefix and --bucket-region to s3 ls (#…
Browse files Browse the repository at this point in the history 
…9163)
  • Loading branch information
@hssyoo
hssyoo authored Jan 6, 2025
1 parent 14a874c commit d4619a3
Show file tree
Hide file tree
Showing 4 changed files with 138 additions and 16 deletions.
5 changes: 5 additions & 0 deletions .changes/next-release/enhancement-s3ls-20704.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
{
"type": "enhancement",
"category": "``s3 ls``",
"description": "Expose low-level ``ListBuckets` parameters ``Prefix`` and ``BucketRegion`` to high-level ``s3 ls`` command as ``--bucket-name-prefix`` and ``--bucket-region``."
}
40 changes: 37 additions & 3 deletions awscli/customizations/s3/subcommands.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -492,6 +492,26 @@
'help_text': 'Indicates the algorithm used to create the checksum for the object.'
}

BUCKET_NAME_PREFIX = {
'name': 'bucket-name-prefix',
'help_text': (
'Limits the response to bucket names that begin with the specified '
'bucket name prefix.'
)
}

BUCKET_REGION = {
'name': 'bucket-region',
'help_text': (
'Limits the response to buckets that are located in the specified '
'Amazon Web Services Region. The Amazon Web Services Region must be '
'expressed according to the Amazon Web Services Region code, such as '
'us-west-2 for the US West (Oregon) Region. For a list of the valid '
'values for all of the Amazon Web Services Regions, see '
'https://docs.aws.amazon.com/general/latest/gr/rande.html#s3_region'
)
}

TRANSFER_ARGS = [DRYRUN, QUIET, INCLUDE, EXCLUDE, ACL,
FOLLOW_SYMLINKS, NO_FOLLOW_SYMLINKS, NO_GUESS_MIME_TYPE,
SSE, SSE_C, SSE_C_KEY, SSE_KMS_KEY_ID, SSE_C_COPY_SOURCE,
Expand Down Expand Up @@ -521,7 +541,8 @@ class ListCommand(S3Command):
USAGE = "<S3Uri> or NONE"
ARG_TABLE = [{'name': 'paths', 'nargs': '?', 'default': 's3://',
'positional_arg': True, 'synopsis': USAGE}, RECURSIVE,
PAGE_SIZE, HUMAN_READABLE, SUMMARIZE, REQUEST_PAYER]
PAGE_SIZE, HUMAN_READABLE, SUMMARIZE, REQUEST_PAYER,
BUCKET_NAME_PREFIX, BUCKET_REGION]

def _run_main(self, parsed_args, parsed_globals):
super(ListCommand, self)._run_main(parsed_args, parsed_globals)
Expand All @@ -535,7 +556,11 @@ def _run_main(self, parsed_args, parsed_globals):
path = path[5:]
bucket, key = find_bucket_key(path)
if not bucket:
self._list_all_buckets(parsed_args.page_size)
self._list_all_buckets(
parsed_args.page_size,
parsed_args.bucket_name_prefix,
parsed_args.bucket_region,
)
elif parsed_args.dir_op:
# Then --recursive was specified.
self._list_all_objects_recursive(
Expand Down Expand Up @@ -599,11 +624,20 @@ def _display_page(self, response_data, use_basename=True):
uni_print(print_str)
self._at_first_page = False

def _list_all_buckets(self, page_size=None):
def _list_all_buckets(
self,
page_size=None,
prefix=None,
bucket_region=None,
):
paginator = self.client.get_paginator('list_buckets')
paging_args = {
'PaginationConfig': {'PageSize': page_size}
}
if prefix:
paging_args['Prefix'] = prefix
if bucket_region:
paging_args['BucketRegion'] = bucket_region

iterator = paginator.paginate(**paging_args)

Expand Down
20 changes: 20 additions & 0 deletions tests/functional/s3/test_ls_command.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -215,3 +215,23 @@ def test_accesspoint_arn(self):
self.run_cmd('s3 ls s3://%s' % arn, expected_rc=0)
call_args = self.operations_called[0][1]
self.assertEqual(call_args['Bucket'], arn)

def test_list_buckets_uses_bucket_name_prefix(self):
stdout, _, _ = self.run_cmd('s3 ls --bucket-name-prefix myprefix', expected_rc=0)
call_args = self.operations_called[0][1]
self.assertEqual(call_args['Prefix'], 'myprefix')

def test_list_buckets_uses_bucket_region(self):
stdout, _, _ = self.run_cmd('s3 ls --bucket-region us-west-1', expected_rc=0)
call_args = self.operations_called[0][1]
self.assertEqual(call_args['BucketRegion'], 'us-west-1')

def test_list_objects_ignores_bucket_name_prefix(self):
stdout, _, _ = self.run_cmd('s3 ls s3://mybucket --bucket-name-prefix myprefix', expected_rc=0)
call_args = self.operations_called[0][1]
self.assertEqual(call_args['Prefix'], '')

def test_list_objects_ignores_bucket_region(self):
stdout, _, _ = self.run_cmd('s3 ls s3://mybucket --bucket-region us-west-1', expected_rc=0)
call_args = self.operations_called[0][1]
self.assertNotIn('BucketRegion', call_args)
89 changes: 76 additions & 13 deletions tests/unit/customizations/s3/test_subcommands.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -79,11 +79,27 @@ def setUp(self):
self.session.create_client.return_value.get_paginator.return_value\
.paginate.return_value = [{'Contents': [], 'CommonPrefixes': []}]

def _get_fake_kwargs(self, override=None):
fake_kwargs = {
'paths': 's3://',
'dir_op': False,
'human_readable': False,
'summarize': False,
'page_size': None,
'request_payer': None,
'bucket_name_prefix': None,
'bucket_region': None,
}
fake_kwargs.update(override or {})

return fake_kwargs

def test_ls_command_for_bucket(self):
ls_command = ListCommand(self.session)
parsed_args = FakeArgs(paths='s3://mybucket/', dir_op=False,
page_size='5', human_readable=False,
summarize=False, request_payer=None)
parsed_args = FakeArgs(**self._get_fake_kwargs({
'paths': 's3://mybucket/',
'page_size': '5',
}))
parsed_globals = mock.Mock()
ls_command._run_main(parsed_args, parsed_globals)
call = self.session.create_client.return_value.list_objects_v2
Expand All @@ -104,9 +120,7 @@ def test_ls_command_with_no_args(self):
ls_command = ListCommand(self.session)
parsed_global = FakeArgs(region=None, endpoint_url=None,
verify_ssl=None)
parsed_args = FakeArgs(dir_op=False, paths='s3://',
human_readable=False, summarize=False,
request_payer=None, page_size=None)
parsed_args = FakeArgs(**self._get_fake_kwargs())
ls_command._run_main(parsed_args, parsed_global)
call = self.session.create_client.return_value.list_buckets
paginate = self.session.create_client.return_value.get_paginator\
Expand All @@ -129,14 +143,61 @@ def test_ls_command_with_no_args(self):
mock.call('s3', region_name=None, verify=None, endpoint_url=None)
)

def test_ls_with_bucket_name_prefix(self):
ls_command = ListCommand(self.session)
parsed_args = FakeArgs(**self._get_fake_kwargs({
'bucket_name_prefix': 'myprefix',
}))
parsed_globals = FakeArgs(
region=None,
endpoint_url=None,
verify_ssl=None,
)
ls_command._run_main(parsed_args, parsed_globals)
call = self.session.create_client.return_value.list_objects
paginate = self.session.create_client.return_value.get_paginator\
.return_value.paginate
# We should make no operation calls.
self.assertEqual(call.call_count, 0)
self.session.create_client.return_value.get_paginator.\
assert_called_with('list_buckets')
ref_call_args = {
'PaginationConfig': {'PageSize': None},
'Prefix': 'myprefix',
}

paginate.assert_called_with(**ref_call_args)

def test_ls_with_bucket_region(self):
ls_command = ListCommand(self.session)
parsed_args = FakeArgs(**self._get_fake_kwargs({
'bucket_region': 'us-west-1',
}))
parsed_globals = FakeArgs(
region=None,
endpoint_url=None,
verify_ssl=None,
)
ls_command._run_main(parsed_args, parsed_globals)
call = self.session.create_client.return_value.list_objects
paginate = self.session.create_client.return_value.get_paginator\
.return_value.paginate
# We should make no operation calls.
self.assertEqual(call.call_count, 0)
self.session.create_client.return_value.get_paginator.\
assert_called_with('list_buckets')
ref_call_args = {
'PaginationConfig': {'PageSize': None},
'BucketRegion': 'us-west-1',
}

paginate.assert_called_with(**ref_call_args)

def test_ls_with_verify_argument(self):
options = {'default': 's3://', 'nargs': '?'}
ls_command = ListCommand(self.session)
parsed_global = FakeArgs(region='us-west-2', endpoint_url=None,
verify_ssl=False)
parsed_args = FakeArgs(paths='s3://', dir_op=False,
human_readable=False, summarize=False,
request_payer=None, page_size=None)
parsed_args = FakeArgs(**self._get_fake_kwargs({}))
ls_command._run_main(parsed_args, parsed_global)
# Verify get_client
get_client = self.session.create_client
Expand All @@ -150,9 +211,11 @@ def test_ls_with_verify_argument(self):

def test_ls_with_requester_pays(self):
ls_command = ListCommand(self.session)
parsed_args = FakeArgs(paths='s3://mybucket/', dir_op=False,
human_readable=False, summarize=False,
request_payer='requester', page_size='5')
parsed_args = FakeArgs(**self._get_fake_kwargs({
'paths': 's3://mybucket/',
'page_size': '5',
'request_payer': 'requester',
}))
parsed_globals = mock.Mock()
ls_command._run_main(parsed_args, parsed_globals)
call = self.session.create_client.return_value.list_objects
Expand Down

0 comments on commit d4619a3

Please sign in to comment.