From 822ddb1d9b79e2a266169393c59c4d9223993a86 Mon Sep 17 00:00:00 2001 From: Patryk Orwat Date: Sun, 5 Jul 2020 14:42:48 +0700 Subject: [PATCH 1/3] added service narrowing for all command, improved its reporting --- README.md | 8 +- cloudiscovery/__init__.py | 19 +++-- cloudiscovery/provider/all/command.py | 7 +- cloudiscovery/provider/all/resource/all.py | 97 ++++++++++++++++------ cloudiscovery/shared/common.py | 5 +- cloudiscovery/shared/report.py | 54 ++++++++---- cloudiscovery/templates/report_html.html | 9 +- 7 files changed, 145 insertions(+), 54 deletions(-) diff --git a/README.md b/README.md index c11117c..26ac53e 100644 --- a/README.md +++ b/README.md @@ -51,7 +51,7 @@ cloudiscovery aws-iot [--thing-name thing-xxxx] --region-name xx-xxxx-xxx [--pro 1.4 To detect all AWS resources (more on [AWS All](#aws-all)): ```sh -cloudiscovery aws-all --region-name xx-xxxx-xxx [--profile-name profile] [--filter xxx] [--verbose] +cloudiscovery aws-all --region-name xx-xxxx-xxx [--profile-name profile] [--services xxx,xxx] [--filter xxx] [--verbose] ``` 1.5 To check AWS limits per resource (more on [AWS Limit](#aws-limit)): @@ -272,13 +272,13 @@ Following resources are checked in IoT command: ### AWS All -List all AWS resources (preview) +A command to list **ALL** AWS resources. -The command tries to call all AWS services (200+) and operations with name `Describe`, `Get...` and `List...` (500+). +The command calls all AWS services (200+) and operations with name `Describe`, `Get...` and `List...` (500+). The operations must be allowed to be called by permissions described in [AWS Permissions](#aws-permissions). -Types of resources mostly cover Terraform types. +Types of resources mostly cover Terraform types. It is possible to narrow down scope of the resources to ones related with a given service with parameter `-s` e.g. `-s ec2,ecs,cloudfront,rds`. ### AWS Limit diff --git a/cloudiscovery/__init__.py b/cloudiscovery/__init__.py index 372a8e5..b9ac67b 100644 --- a/cloudiscovery/__init__.py +++ b/cloudiscovery/__init__.py @@ -92,18 +92,13 @@ def generate_parser(): all_parser = subparsers.add_parser("aws-all", help="Analyze all resources") add_default_arguments(all_parser, diagram_enabled=False) + add_services_argument(all_parser) limit_parser = subparsers.add_parser( "aws-limit", help="Analyze aws limit resources." ) add_default_arguments(limit_parser, diagram_enabled=False, filters_enabled=False) - limit_parser.add_argument( - "-s", - "--services", - required=False, - help='Inform services that you want to check, use "," (comma) to split them. \ - If not informed, script will check all services.', - ) + add_services_argument(limit_parser) limit_parser.add_argument( "-t", "--threshold", @@ -115,6 +110,16 @@ def generate_parser(): return parser +def add_services_argument(limit_parser): + limit_parser.add_argument( + "-s", + "--services", + required=False, + help='Define services that you want to check, use "," (comma) to separate multiple names. \ + If not passed, command will check all services.', + ) + + def add_default_arguments( parser, is_global=False, diagram_enabled=True, filters_enabled=True ): diff --git a/cloudiscovery/provider/all/command.py b/cloudiscovery/provider/all/command.py index b819d74..72c0f25 100644 --- a/cloudiscovery/provider/all/command.py +++ b/cloudiscovery/provider/all/command.py @@ -6,9 +6,13 @@ class AllOptions(BaseAwsOptions, BaseOptions): - def __init__(self, verbose, filters, session, region_name): + services: List[str] + + # pylint: disable=too-many-arguments + def __init__(self, verbose, filters, session, region_name, services: List[str]): BaseAwsOptions.__init__(self, session, region_name) BaseOptions.__init__(self, verbose, filters) + self.services = services class All(BaseAwsCommand): @@ -26,6 +30,7 @@ def run( filters=filters, session=self.session, region_name=region, + services=services, ) command_runner = AwsCommandRunner(filters) diff --git a/cloudiscovery/provider/all/resource/all.py b/cloudiscovery/provider/all/resource/all.py index ff1affe..070921b 100644 --- a/cloudiscovery/provider/all/resource/all.py +++ b/cloudiscovery/provider/all/resource/all.py @@ -1,3 +1,4 @@ +import collections import functools import re from concurrent.futures.thread import ThreadPoolExecutor @@ -15,7 +16,7 @@ ResourceAvailable, log_critical, ) -from shared.common_aws import get_paginator +from shared.common_aws import get_paginator, resource_tags OMITTED_RESOURCES = [ "aws_cloudhsm_available_zone", @@ -309,7 +310,9 @@ def operation_allowed( return evaluation_result -def build_resource(base_resource, operation_name, resource_type) -> Optional[Resource]: +def build_resource( + base_resource, operation_name, resource_type, group +) -> Optional[Resource]: if isinstance(base_resource, str): return None resource_name = retrieve_resource_name(base_resource, operation_name) @@ -317,8 +320,13 @@ def build_resource(base_resource, operation_name, resource_type) -> Optional[Res if resource_id is None or resource_name is None: return None + attributes = flatten(base_resource) return Resource( - digest=ResourceDigest(id=resource_id, type=resource_type), name=resource_name, + digest=ResourceDigest(id=resource_id, type=resource_type), + group=group, + name=resource_name, + attributes=attributes, + tags=resource_tags(base_resource), ) @@ -413,6 +421,17 @@ def build_resource_type(aws_service, name): ) +def flatten(d, parent_key="", sep="."): + items = [] + for k, v in d.items(): + new_key = parent_key + sep + k if parent_key else k + if isinstance(v, collections.MutableMapping): + items.extend(flatten(v, new_key, sep=sep).items()) + else: + items.append((new_key, v)) + return dict(items) + + class AllResources(ResourceProvider): def __init__(self, options: AllOptions): """ @@ -427,16 +446,20 @@ def __init__(self, options: AllOptions): @all_exception def get_resources(self) -> List[Resource]: boto_loader = Loader() - aws_services = boto_loader.list_available_services(type_name="service-2") + if self.options.services: + aws_services = self.options.services + else: + aws_services = boto_loader.list_available_services(type_name="service-2") resources = [] allowed_actions = self.get_policies_allowed_actions() - message_handler( - "Analyzing listing operations across {} service...".format( - len(aws_services) - ), - "HEADER", - ) + if self.options.verbose: + message_handler( + "Analyzing listing operations across {} service...".format( + len(aws_services) + ), + "HEADER", + ) with ThreadPoolExecutor(PARALLEL_SERVICE_CALLS) as executor: results = executor.map( lambda aws_service: self.analyze_service( @@ -501,7 +524,12 @@ def analyze_service(self, aws_service, boto_loader, allowed_actions): if not operation_allowed(allowed_actions, aws_service, name): continue analyze_operation = self.analyze_operation( - resource_type, name, has_paginator, client, service_full_name + resource_type, + name, + has_paginator, + client, + service_full_name, + aws_service, ) if analyze_operation is not None: resources.extend(analyze_operation) @@ -510,10 +538,17 @@ def analyze_service(self, aws_service, boto_loader, allowed_actions): @all_exception # pylint: disable=too-many-locals,too-many-arguments def analyze_operation( - self, resource_type, operation_name, has_paginator, client, service_full_name + self, + resource_type, + operation_name, + has_paginator, + client, + service_full_name, + aws_service, ) -> List[Resource]: resources = [] snake_operation_name = _to_snake_case(operation_name) + # pylint: disable=too-many-nested-blocks if has_paginator: pages = get_paginator( client=client, @@ -530,14 +565,23 @@ def analyze_operation( result_parent = list_metadata["children"][0]["value"] result_child = list_metadata["children"][1]["value"] else: - message_handler( - "Operation {} has unsupported pagination definition... Skipping".format( - snake_operation_name - ), - "WARNING", - ) + if self.options.verbose: + message_handler( + "Operation {} has unsupported pagination definition... Skipping".format( + snake_operation_name + ), + "WARNING", + ) return [] for page in pages: + if result_key == "Reservations": # hack for EC2 instances + for page_reservation in page["Reservations"]: + for instance in page_reservation["Instances"]: + resource = build_resource( + instance, operation_name, resource_type, aws_service + ) + if resource is not None: + resources.append(resource) if result_key is not None: page_resources = page[result_key] elif result_child in page[result_parent]: @@ -546,7 +590,7 @@ def analyze_operation( page_resources = [] for page_resource in page_resources: resource = build_resource( - page_resource, operation_name, resource_type + page_resource, operation_name, resource_type, aws_service ) if resource is not None: resources.append(resource) @@ -557,14 +601,18 @@ def analyze_operation( if isinstance(response_elem, list): for response_resource in response_elem: resource = build_resource( - response_resource, operation_name, resource_type + response_resource, + operation_name, + resource_type, + aws_service, ) if resource is not None: resources.append(resource) return resources def get_policies_allowed_actions(self): - message_handler("Fetching allowed actions...", "HEADER") + if self.options.verbose: + message_handler("Fetching allowed actions...", "HEADER") iam_client = self.options.client("iam") view_only_document = self.get_policy_allowed_calls( iam_client, "arn:aws:iam::aws:policy/job-function/ViewOnlyAccess" @@ -580,9 +628,10 @@ def get_policies_allowed_actions(self): allowed_actions[action] = True for action in ON_TOP_POLICIES: allowed_actions[action] = True - message_handler( - "Found {} allowed actions".format(len(allowed_actions)), "HEADER" - ) + if self.options.verbose: + message_handler( + "Found {} allowed actions".format(len(allowed_actions)), "HEADER" + ) return allowed_actions.keys() diff --git a/cloudiscovery/shared/common.py b/cloudiscovery/shared/common.py index 0b0332d..3975a7d 100644 --- a/cloudiscovery/shared/common.py +++ b/cloudiscovery/shared/common.py @@ -4,7 +4,7 @@ import functools import threading from abc import ABC -from typing import NamedTuple, List +from typing import NamedTuple, List, Dict from diskcache import Cache @@ -73,7 +73,8 @@ class Resource(NamedTuple): details: str = "" group: str = "" tags: List[ResourceTag] = [] - limits: List[LimitsValues] = [] + limits: LimitsValues = None + attributes: Dict[str, object] = {} class ResourceCache: diff --git a/cloudiscovery/shared/report.py b/cloudiscovery/shared/report.py index 9030ab9..c3c040d 100644 --- a/cloudiscovery/shared/report.py +++ b/cloudiscovery/shared/report.py @@ -35,24 +35,46 @@ def general_report( + "%" ) # pylint: disable=line-too-long - message = "service: {} - quota code: {} - quota name: {} - aws default quota: {} - applied quota: {} - usage: {}".format( # noqa: E501 - resource.limits.service, - resource.limits.quota_code, - resource.limits.quota_name, - resource.limits.aws_limit, - resource.limits.local_limit, - usage, + message_handler( + "service: {} - quota code: {} - quota name: {} - aws default quota: {} - applied quota: {} - usage: {}".format( # noqa: E501 + resource.limits.service, + resource.limits.quota_code, + resource.limits.quota_name, + resource.limits.aws_limit, + resource.limits.local_limit, + usage, + ), + "OKBLUE", ) + elif resource.attributes: + message_handler( + "resource type: {} - resource id: {} - resource name: {} - resource details: {}".format( + resource.digest.type, + resource.digest.id, + resource.name, + resource.details, + ), + "OKBLUE", + ) + for ( + resource_attr_key, + resource_attr_value, + ) in resource.attributes.items(): + message_handler( + "{}: {}".format(resource_attr_key, resource_attr_value,), + "OKBLUE", + ) else: - message = "resource type: {} - resource id: {} - resource name: {} - resource details: {}".format( - resource.digest.type, - resource.digest.id, - resource.name, - resource.details, + message_handler( + "resource type: {} - resource id: {} - resource name: {} - resource details: {}".format( + resource.digest.type, + resource.digest.id, + resource.name, + resource.details, + ), + "OKBLUE", ) - message_handler(message, "OKBLUE") - if resource_relations: message_handler("\n\nFound relations", "HEADER") for resource_relation in resource_relations: @@ -88,17 +110,21 @@ def html_report( with open(image_name, "rb") as image_file: diagram_image = base64.b64encode(image_file.read()).decode("utf-8") + group_title = "Group" if resources: if resources[0].limits: html_output = dir_template.get_template("report_limits.html").render( default_name=title, resources_found=resources ) else: + if resources[0].attributes: + group_title = "Service" html_output = dir_template.get_template("report_html.html").render( default_name=title, resources_found=resources, resources_relations=resource_relations, diagram_image=diagram_image, + group_title=group_title, ) self.make_directories() diff --git a/cloudiscovery/templates/report_html.html b/cloudiscovery/templates/report_html.html index ed2cdec..2945f76 100644 --- a/cloudiscovery/templates/report_html.html +++ b/cloudiscovery/templates/report_html.html @@ -5,7 +5,7 @@ Type -Group +{{ group_title }} Id Name Details @@ -17,7 +17,12 @@ {{ resource_found.group}} {{ resource_found.digest.id}} {{ resource_found.name}} - {{ resource_found.details}} + + {{ resource_found.details}} + {% for attribute_key, attribute_value in resource_found.attributes.items() %} + {{ attribute_key}}: {{ attribute_value}}
+ {%- endfor %} + {% for tag in resource_found.tags %} {{ tag.key}}: {{ tag.value}}
From c74c512bf3952efefdcefa342796f616863677fe Mon Sep 17 00:00:00 2001 From: Patryk Orwat Date: Sun, 5 Jul 2020 14:58:03 +0700 Subject: [PATCH 2/3] simplified console reporting --- cloudiscovery/shared/report.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/cloudiscovery/shared/report.py b/cloudiscovery/shared/report.py index c3c040d..977517e 100644 --- a/cloudiscovery/shared/report.py +++ b/cloudiscovery/shared/report.py @@ -47,8 +47,10 @@ def general_report( "OKBLUE", ) elif resource.attributes: + # pylint: disable=too-many-format-args message_handler( - "resource type: {} - resource id: {} - resource name: {} - resource details: {}".format( + "\nservice: {} - type: {} - id: {} - resource name: {}".format( + resource.group, resource.digest.type, resource.digest.id, resource.name, @@ -61,12 +63,18 @@ def general_report( resource_attr_value, ) in resource.attributes.items(): message_handler( - "{}: {}".format(resource_attr_key, resource_attr_value,), + "service: {} - type: {} - id: {} -> {}: {}".format( + resource.group, + resource.digest.type, + resource.digest.id, + resource_attr_key, + resource_attr_value, + ), "OKBLUE", ) else: message_handler( - "resource type: {} - resource id: {} - resource name: {} - resource details: {}".format( + "type: {} - id: {} - name: {} - details: {}".format( resource.digest.type, resource.digest.id, resource.name, @@ -78,7 +86,7 @@ def general_report( if resource_relations: message_handler("\n\nFound relations", "HEADER") for resource_relation in resource_relations: - message = "resource type: {} - resource id: {} -> resource type: {} - resource id: {}".format( + message = "type: {} - id: {} -> type: {} - id: {}".format( resource_relation.from_node.type, resource_relation.from_node.id, resource_relation.to_node.type, From 78dcd53bb2464d7455ec018872d8f91954e6cc05 Mon Sep 17 00:00:00 2001 From: Patryk Orwat Date: Sun, 5 Jul 2020 15:33:45 +0700 Subject: [PATCH 3/3] silence aws_ec2_prefix_list --- cloudiscovery/provider/all/resource/all.py | 1 + 1 file changed, 1 insertion(+) diff --git a/cloudiscovery/provider/all/resource/all.py b/cloudiscovery/provider/all/resource/all.py index 070921b..e412fe3 100644 --- a/cloudiscovery/provider/all/resource/all.py +++ b/cloudiscovery/provider/all/resource/all.py @@ -106,6 +106,7 @@ "aws_organizations_account", "aws_config_organization_config_rule_status", "aws_dynamodb_backup", + "aws_ec2_prefix_list", ] # Trying to fix documentation errors or its lack made by "happy pirates" at AWS