Merge pull request #124 from Cloud-Architects/aws-all-adjustments

Aws all adjustments

README.md

@@ -51,7 +51,7 @@ cloudiscovery aws-iot [--thing-name thing-xxxx] --region-name xx-xxxx-xxx [--pro
 1.4 To detect all AWS resources (more on [AWS All](#aws-all)):
 
 ```sh
-cloudiscovery aws-all --region-name xx-xxxx-xxx [--profile-name profile] [--filter xxx] [--verbose]
+cloudiscovery aws-all --region-name xx-xxxx-xxx [--profile-name profile] [--services xxx,xxx] [--filter xxx] [--verbose]
 ```
 
 1.5 To check AWS limits per resource (more on [AWS Limit](#aws-limit)):
@@ -272,13 +272,13 @@ Following resources are checked in IoT command:
 
 ### AWS All
 
-List all AWS resources (preview)
+A command to list **ALL** AWS resources. 
 
-The command tries to call all AWS services (200+) and operations with name `Describe`, `Get...` and `List...` (500+).
+The command calls all AWS services (200+) and operations with name `Describe`, `Get...` and `List...` (500+).
 
 The operations must be allowed to be called by permissions described in [AWS Permissions](#aws-permissions).
 
-Types of resources mostly cover Terraform types.
+Types of resources mostly cover Terraform types. It is possible to narrow down scope of the resources to ones related with a given service with parameter `-s` e.g. `-s ec2,ecs,cloudfront,rds`.
 
 ### AWS Limit
 

cloudiscovery/__init__.py

@@ -92,18 +92,13 @@ def generate_parser():
 
     all_parser = subparsers.add_parser("aws-all", help="Analyze all resources")
     add_default_arguments(all_parser, diagram_enabled=False)
+    add_services_argument(all_parser)
 
     limit_parser = subparsers.add_parser(
         "aws-limit", help="Analyze aws limit resources."
     )
     add_default_arguments(limit_parser, diagram_enabled=False, filters_enabled=False)
-    limit_parser.add_argument(
-        "-s",
-        "--services",
-        required=False,
-        help='Inform services that you want to check, use "," (comma) to split them.  \
-              If not informed, script will check all services.',
-    )
+    add_services_argument(limit_parser)
     limit_parser.add_argument(
         "-t",
         "--threshold",
@@ -115,6 +110,16 @@ def generate_parser():
     return parser
 
 
+def add_services_argument(limit_parser):
+    limit_parser.add_argument(
+        "-s",
+        "--services",
+        required=False,
+        help='Define services that you want to check, use "," (comma) to separate multiple names. \
+              If not passed, command will check all services.',
+    )
+
+
 def add_default_arguments(
     parser, is_global=False, diagram_enabled=True, filters_enabled=True
 ):

cloudiscovery/provider/all/command.py

@@ -6,9 +6,13 @@
 
 
 class AllOptions(BaseAwsOptions, BaseOptions):
-    def __init__(self, verbose, filters, session, region_name):
+    services: List[str]
+
+    # pylint: disable=too-many-arguments
+    def __init__(self, verbose, filters, session, region_name, services: List[str]):
         BaseAwsOptions.__init__(self, session, region_name)
         BaseOptions.__init__(self, verbose, filters)
+        self.services = services
 
 
 class All(BaseAwsCommand):
@@ -26,6 +30,7 @@ def run(
                 filters=filters,
                 session=self.session,
                 region_name=region,
+                services=services,
             )
 
             command_runner = AwsCommandRunner(filters)

cloudiscovery/provider/all/resource/all.py

@@ -1,3 +1,4 @@
+import collections
 import functools
 import re
 from concurrent.futures.thread import ThreadPoolExecutor
@@ -15,7 +16,7 @@
     ResourceAvailable,
     log_critical,
 )
-from shared.common_aws import get_paginator
+from shared.common_aws import get_paginator, resource_tags
 
 OMITTED_RESOURCES = [
     "aws_cloudhsm_available_zone",
@@ -105,6 +106,7 @@
     "aws_organizations_account",
     "aws_config_organization_config_rule_status",
     "aws_dynamodb_backup",
+    "aws_ec2_prefix_list",
 ]
 
 # Trying to fix documentation errors or its lack made by "happy pirates" at AWS
@@ -309,16 +311,23 @@ def operation_allowed(
     return evaluation_result
 
 
-def build_resource(base_resource, operation_name, resource_type) -> Optional[Resource]:
+def build_resource(
+    base_resource, operation_name, resource_type, group
+) -> Optional[Resource]:
     if isinstance(base_resource, str):
         return None
     resource_name = retrieve_resource_name(base_resource, operation_name)
     resource_id = retrieve_resource_id(base_resource, operation_name, resource_name)
 
     if resource_id is None or resource_name is None:
         return None
+    attributes = flatten(base_resource)
     return Resource(
-        digest=ResourceDigest(id=resource_id, type=resource_type), name=resource_name,
+        digest=ResourceDigest(id=resource_id, type=resource_type),
+        group=group,
+        name=resource_name,
+        attributes=attributes,
+        tags=resource_tags(base_resource),
     )
 
 
@@ -413,6 +422,17 @@ def build_resource_type(aws_service, name):
     )
 
 
+def flatten(d, parent_key="", sep="."):
+    items = []
+    for k, v in d.items():
+        new_key = parent_key + sep + k if parent_key else k
+        if isinstance(v, collections.MutableMapping):
+            items.extend(flatten(v, new_key, sep=sep).items())
+        else:
+            items.append((new_key, v))
+    return dict(items)
+
+
 class AllResources(ResourceProvider):
     def __init__(self, options: AllOptions):
         """
@@ -427,16 +447,20 @@ def __init__(self, options: AllOptions):
     @all_exception
     def get_resources(self) -> List[Resource]:
         boto_loader = Loader()
-        aws_services = boto_loader.list_available_services(type_name="service-2")
+        if self.options.services:
+            aws_services = self.options.services
+        else:
+            aws_services = boto_loader.list_available_services(type_name="service-2")
         resources = []
         allowed_actions = self.get_policies_allowed_actions()
 
-        message_handler(
-            "Analyzing listing operations across {} service...".format(
-                len(aws_services)
-            ),
-            "HEADER",
-        )
+        if self.options.verbose:
+            message_handler(
+                "Analyzing listing operations across {} service...".format(
+                    len(aws_services)
+                ),
+                "HEADER",
+            )
         with ThreadPoolExecutor(PARALLEL_SERVICE_CALLS) as executor:
             results = executor.map(
                 lambda aws_service: self.analyze_service(
@@ -501,7 +525,12 @@ def analyze_service(self, aws_service, boto_loader, allowed_actions):
                 if not operation_allowed(allowed_actions, aws_service, name):
                     continue
                 analyze_operation = self.analyze_operation(
-                    resource_type, name, has_paginator, client, service_full_name
+                    resource_type,
+                    name,
+                    has_paginator,
+                    client,
+                    service_full_name,
+                    aws_service,
                 )
                 if analyze_operation is not None:
                     resources.extend(analyze_operation)
@@ -510,10 +539,17 @@ def analyze_service(self, aws_service, boto_loader, allowed_actions):
     @all_exception
     # pylint: disable=too-many-locals,too-many-arguments
     def analyze_operation(
-        self, resource_type, operation_name, has_paginator, client, service_full_name
+        self,
+        resource_type,
+        operation_name,
+        has_paginator,
+        client,
+        service_full_name,
+        aws_service,
     ) -> List[Resource]:
         resources = []
         snake_operation_name = _to_snake_case(operation_name)
+        # pylint: disable=too-many-nested-blocks
         if has_paginator:
             pages = get_paginator(
                 client=client,
@@ -530,14 +566,23 @@ def analyze_operation(
                 result_parent = list_metadata["children"][0]["value"]
                 result_child = list_metadata["children"][1]["value"]
             else:
-                message_handler(
-                    "Operation {} has unsupported pagination definition... Skipping".format(
-                        snake_operation_name
-                    ),
-                    "WARNING",
-                )
+                if self.options.verbose:
+                    message_handler(
+                        "Operation {} has unsupported pagination definition... Skipping".format(
+                            snake_operation_name
+                        ),
+                        "WARNING",
+                    )
                 return []
             for page in pages:
+                if result_key == "Reservations":  # hack for EC2 instances
+                    for page_reservation in page["Reservations"]:
+                        for instance in page_reservation["Instances"]:
+                            resource = build_resource(
+                                instance, operation_name, resource_type, aws_service
+                            )
+                            if resource is not None:
+                                resources.append(resource)
                 if result_key is not None:
                     page_resources = page[result_key]
                 elif result_child in page[result_parent]:
@@ -546,7 +591,7 @@ def analyze_operation(
                     page_resources = []
                 for page_resource in page_resources:
                     resource = build_resource(
-                        page_resource, operation_name, resource_type
+                        page_resource, operation_name, resource_type, aws_service
                     )
                     if resource is not None:
                         resources.append(resource)
@@ -557,14 +602,18 @@ def analyze_operation(
                 if isinstance(response_elem, list):
                     for response_resource in response_elem:
                         resource = build_resource(
-                            response_resource, operation_name, resource_type
+                            response_resource,
+                            operation_name,
+                            resource_type,
+                            aws_service,
                         )
                         if resource is not None:
                             resources.append(resource)
         return resources
 
     def get_policies_allowed_actions(self):
-        message_handler("Fetching allowed actions...", "HEADER")
+        if self.options.verbose:
+            message_handler("Fetching allowed actions...", "HEADER")
         iam_client = self.options.client("iam")
         view_only_document = self.get_policy_allowed_calls(
             iam_client, "arn:aws:iam::aws:policy/job-function/ViewOnlyAccess"
@@ -580,9 +629,10 @@ def get_policies_allowed_actions(self):
             allowed_actions[action] = True
         for action in ON_TOP_POLICIES:
             allowed_actions[action] = True
-        message_handler(
-            "Found {} allowed actions".format(len(allowed_actions)), "HEADER"
-        )
+        if self.options.verbose:
+            message_handler(
+                "Found {} allowed actions".format(len(allowed_actions)), "HEADER"
+            )
 
         return allowed_actions.keys()
 

cloudiscovery/shared/common.py

@@ -4,7 +4,7 @@
 import functools
 import threading
 from abc import ABC
-from typing import NamedTuple, List
+from typing import NamedTuple, List, Dict
 
 from diskcache import Cache
 
@@ -73,7 +73,8 @@ class Resource(NamedTuple):
     details: str = ""
     group: str = ""
     tags: List[ResourceTag] = []
-    limits: List[LimitsValues] = []
+    limits: LimitsValues = None
+    attributes: Dict[str, object] = {}
 
 
 class ResourceCache: